Add AUTOUPDATE-SPLIT to help with format tests. (#4384)

Since formatting covers comments, and the CHECK lines are in comments,
it can create recursive behaviors. This introduces AUTOUPDATE-SPLIT as a
way of formally designating a split to exclusively be used for
autoupdate output.

testing/file_test/README.md

@@ -109,8 +109,8 @@ Supported comment markers are:
     ```
 
     Controls whether the checks in the file will be autoupdated if --autoupdate
-    is passed. Exactly one of these two markers must be present. If the file
-    uses splits, AUTOUPDATE must currently be before any splits.
+    is passed. Exactly one of these markers must be present. If the file uses
+    splits, the marker must currently be before any splits.
 
     When autoupdating, CHECKs will be inserted starting below AUTOUPDATE. When a
     CHECK has line information, autoupdate will try to insert the CHECK
@@ -121,6 +121,10 @@ Supported comment markers are:
     check line refers to any line in the test, all STDOUT check lines are placed
     at the end of the file instead of immediately after AUTOUPDATE.
 
+    When using split files, if the last split file is named
+    `// --- AUTOUPDATE-SPLIT`, all CHECKs will be added there; no line
+    associations occur.
+
 -   ```
     // ARGS: <arguments>
     ```

testing/file_test/autoupdate.cpp

@@ -235,7 +235,7 @@ auto FileTestAutoupdater::AddTips() -> void {
 
 auto FileTestAutoupdater::ShouldAddCheckLine(const CheckLines& check_lines,
                                              bool to_file_end) const -> bool {
-  return check_lines.cursor != check_lines.lines.end() &&
+  return !autoupdate_split_ && check_lines.cursor != check_lines.lines.end() &&
          (check_lines.cursor->file_number() < output_file_number_ ||
           (check_lines.cursor->file_number() == output_file_number_ &&
            (to_file_end || check_lines.cursor->line_number() <=
@@ -312,9 +312,11 @@ auto FileTestAutoupdater::Run(bool dry_run) -> bool {
   // lines after AUTOUPDATE.
   AddRemappedNonCheckLine();
   AddTips();
-  AddCheckLines(stderr_, /*to_file_end=*/false);
-  if (any_attached_stdout_lines_) {
-    AddCheckLines(stdout_, /*to_file_end=*/false);
+  if (!autoupdate_split_) {
+    AddCheckLines(stderr_, /*to_file_end=*/false);
+    if (any_attached_stdout_lines_) {
+      AddCheckLines(stdout_, /*to_file_end=*/false);
+    }
   }
   ++non_check_line_;
 
@@ -323,6 +325,10 @@ auto FileTestAutoupdater::Run(bool dry_run) -> bool {
     if (output_file_number_ < non_check_line_->file_number()) {
       FinishFile(/*is_last_file=*/false);
       StartSplitFile();
+      if (autoupdate_split_ &&
+          output_file_number_ == static_cast<int>(filenames_.size())) {
+        break;
+      }
       continue;
     }
 
@@ -341,6 +347,11 @@ auto FileTestAutoupdater::Run(bool dry_run) -> bool {
     ++non_check_line_;
   }
 
+  // When autoupdate_split_ was true, this will result in all check lines (and
+  // only check lines) being added to the split by FinishFile. We don't use
+  // autoupdate_split_ past this point.
+  autoupdate_split_ = false;
+
   FinishFile(/*is_last_file=*/true);
 
   for (auto& check_line : stdout_.lines) {

testing/file_test/autoupdate.h

@@ -37,7 +37,7 @@ class FileTestAutoupdater {
       const std::filesystem::path& file_test_path, std::string test_command,
       std::string dump_command, llvm::StringRef input_content,
       const llvm::SmallVector<llvm::StringRef>& filenames,
-      int autoupdate_line_number,
+      int autoupdate_line_number, bool autoupdate_split,
       const llvm::SmallVector<FileTestLine>& non_check_lines,
       llvm::StringRef stdout, llvm::StringRef stderr,
       const std::optional<RE2>& default_file_re,
@@ -49,6 +49,7 @@ class FileTestAutoupdater {
         input_content_(input_content),
         filenames_(filenames),
         autoupdate_line_number_(autoupdate_line_number),
+        autoupdate_split_(autoupdate_split),
         non_check_lines_(non_check_lines),
         default_file_re_(default_file_re),
         line_number_replacements_(line_number_replacements),
@@ -196,6 +197,7 @@ class FileTestAutoupdater {
   llvm::StringRef input_content_;
   const llvm::SmallVector<llvm::StringRef>& filenames_;
   int autoupdate_line_number_;
+  bool autoupdate_split_;
   const llvm::SmallVector<FileTestLine>& non_check_lines_;
   const std::optional<RE2>& default_file_re_;
   const llvm::SmallVector<LineNumberReplacement>& line_number_replacements_;

testing/file_test/file_test_base.cpp

@@ -232,8 +232,8 @@ auto FileTestBase::RunAutoupdater(const TestContext& context, bool dry_run)
              GetBazelCommand(BazelMode::Test, test_name_),
              GetBazelCommand(BazelMode::Dump, test_name_),
              context.input_content, filenames, *context.autoupdate_line_number,
-             context.non_check_lines, context.stdout, context.stderr,
-             GetDefaultFileRE(expected_filenames),
+             context.autoupdate_split, context.non_check_lines, context.stdout,
+             context.stderr, GetDefaultFileRE(expected_filenames),
              GetLineNumberReplacements(expected_filenames),
              [&](std::string& line) { DoExtraCheckReplacements(line); })
       .Run(dry_run);
@@ -907,12 +907,30 @@ auto FileTestBase::ProcessTestFile(TestContext& context) -> ErrorOr<Success> {
   }
 
   if (!found_autoupdate) {
-    return ErrorBuilder() << "Missing AUTOUPDATE/NOAUTOUPDATE setting";
+    return Error("Missing AUTOUPDATE/NOAUTOUPDATE setting");
   }
 
   context.has_splits = split.has_splits();
   CARBON_RETURN_IF_ERROR(FinishSplit(test_name_, &split, &context.test_files));
 
+  // Validate AUTOUPDATE-SPLIT use, and remove it from test files if present.
+  if (context.has_splits) {
+    constexpr llvm::StringLiteral AutoupdateSplit = "AUTOUPDATE-SPLIT";
+    for (const auto& test_file :
+         llvm::ArrayRef(context.test_files).drop_back()) {
+      if (test_file.filename == AutoupdateSplit) {
+        return Error("AUTOUPDATE-SPLIT must be the last split");
+      }
+    }
+    if (context.test_files.back().filename == AutoupdateSplit) {
+      if (!context.autoupdate_line_number) {
+        return Error("AUTOUPDATE-SPLIT requires AUTOUPDATE");
+      }
+      context.autoupdate_split = true;
+      context.test_files.pop_back();
+    }
+  }
+
   // Assume there is always a suffix `\n` in output.
   if (!context.expected_stdout.empty()) {
     context.expected_stdout.push_back(StrEq(""));

testing/file_test/file_test_base.h

@@ -151,6 +151,9 @@ class FileTestBase : public testing::Test {
     // The location of the autoupdate marker, for autoupdated files.
     std::optional<int> autoupdate_line_number;
 
+    // Whether there should be an AUTOUPDATE-SPLIT.
+    bool autoupdate_split = false;
+
     // Whether to capture stderr and stdout that would head to console,
     // generated from SET-CAPTURE-CONSOLE-OUTPUT.
     bool capture_console_output = false;

testing/file_test/testdata/autoupdate_split.carbon

@@ -0,0 +1,21 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //testing/file_test:file_test_base_test --test_arg=--file_tests=testing/file_test/testdata/autoupdate_split.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/autoupdate_split.carbon
+
+// --- a.carbon
+aaa
+
+// --- b.carbon
+bbb
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon`
+// CHECK:STDOUT: a.carbon:1: aaa
+// CHECK:STDOUT: b.carbon:1: bbb

testing/file_test/testdata/autoupdate_split_standalone.carbon

@@ -0,0 +1,13 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //testing/file_test:file_test_base_test --test_arg=--file_tests=testing/file_test/testdata/autoupdate_split_standalone.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/autoupdate_split_standalone.carbon
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDOUT: 1 args: `default_args`

toolchain/format/testdata/basics/empty.carbon

@@ -9,5 +9,6 @@
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/format/testdata/basics/empty.carbon
 
 // --- test.carbon
+// --- AUTOUPDATE-SPLIT
 
 // CHECK:STDOUT:

toolchain/format/testdata/basics/fail_invalid_comment.carbon

@@ -1,20 +1,23 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-// TODO: This can't autoupdate because the STDERR is retained in test input.
-// NOAUTOUPDATE
+//
+// AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/format/testdata/basics/fail_invalid_comment.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/format/testdata/basics/fail_invalid_comment.carbon
 
+
+// --- fail_test.carbon
+
+//f
+
+// --- AUTOUPDATE-SPLIT
+
 // CHECK:STDERR: fail_test.carbon:2:3: error: whitespace is required after '//'
 // CHECK:STDERR: //f
 // CHECK:STDERR:   ^
 // CHECK:STDOUT:
 // CHECK:STDOUT: //f
-
-// --- fail_test.carbon
-
-//f
+// CHECK:STDOUT:

toolchain/format/testdata/basics/simple.carbon

@@ -12,4 +12,6 @@
 
 fn F(x: i32) -> i32 { return x; }
 
+// --- AUTOUPDATE-SPLIT
+
 // CHECK:STDOUT: fn F ( x : i32 ) -> i32 { return x ; }