Implement autoupdate for file_test. (#3043)

I've migrated the toolchain autoupdate scripts here, I just need a
little more time to do the explorer side (which I need to check
performance, that may require multithreading as we do in the current
script). However, this felt substantial enough to share and it means I
can handle autoupdate in more of the toolchain, including preparatory
work for autoupdate on multi-file tests.

Once explorer is done I'll remove the old script.

explorer/file_test.cpp

@@ -19,17 +19,16 @@ class ExplorerFileTest : public FileTestBase {
  public:
   using FileTestBase::FileTestBase;
 
-  auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
-                    const llvm::SmallVector<TestFile>& test_files,
-                    llvm::raw_pwrite_stream& stdout,
-                    llvm::raw_pwrite_stream& stderr) -> bool override {
+  auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
+           const llvm::SmallVector<TestFile>& test_files,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<bool> override {
     // Create the files in-memory.
     llvm::vfs::InMemoryFileSystem fs(new llvm::vfs::InMemoryFileSystem());
     for (const auto& test_file : test_files) {
       if (!fs.addFile(test_file.filename, /*ModificationTime=*/0,
                       llvm::MemoryBuffer::getMemBuffer(test_file.content))) {
-        ADD_FAILURE() << "File is repeated: " << test_file.filename;
-        return false;
+        return ErrorBuilder() << "File is repeated: " << test_file.filename;
       }
     }
 
@@ -37,8 +36,7 @@ class ExplorerFileTest : public FileTestBase {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
         llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon");
     if (prelude.getError()) {
-      ADD_FAILURE() << prelude.getError().message();
-      return false;
+      return ErrorBuilder() << prelude.getError().message();
     }
     // TODO: This path is long with a prefix / because of the path expectations
     // in tests. Change those to allow a shorter path (e.g., `prelude.carbon`)
@@ -46,15 +44,13 @@ class ExplorerFileTest : public FileTestBase {
     static constexpr llvm::StringLiteral PreludePath =
         "/explorer/data/prelude.carbon";
     if (!fs.addFile(PreludePath, /*ModificationTime=*/0, std::move(*prelude))) {
-      ADD_FAILURE() << "Duplicate prelude.carbon";
-      return false;
+      return ErrorBuilder() << "Duplicate prelude.carbon";
     }
 
     llvm::SmallVector<const char*> args = {"explorer"};
     for (auto arg : test_args) {
       args.push_back(arg.data());
     }
-    TestRawOstream trace_stream;
 
     // Trace output is only checked for a few tests.
     bool check_trace_output =
@@ -62,16 +58,19 @@ class ExplorerFileTest : public FileTestBase {
 
     int exit_code = ExplorerMain(
         args.size(), args.data(), /*install_path=*/"", PreludePath, stdout,
-        stderr, check_trace_output ? stdout : trace_stream, fs);
+        stderr, check_trace_output ? stdout : trace_stream_, fs);
 
+    return exit_code == EXIT_SUCCESS;
+  }
+
+  auto ValidateRun(const llvm::SmallVector<TestFile>& /*test_files*/)
+      -> void override {
     // Skip trace test check as they use stdout stream instead of
     // trace_stream_ostream
     if (absl::GetFlag(FLAGS_trace)) {
-      EXPECT_FALSE(trace_stream.TakeStr().empty())
+      EXPECT_FALSE(trace_stream_.TakeStr().empty())
           << "Tracing should always do something";
     }
-
-    return exit_code == EXIT_SUCCESS;
   }
 
   auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
@@ -83,6 +82,8 @@ class ExplorerFileTest : public FileTestBase {
     args.push_back("%s");
     return args;
   }
+
+  TestRawOstream trace_stream_;
 };
 
 }  // namespace

scripts/fix_cc_deps.py

@@ -58,6 +58,10 @@ EXTERNAL_REPOS: Dict[str, ExternalRepo] = {
     "@com_google_absl": ExternalRepo(
         lambda x: re.sub(":", "/", x), "...", None
     ),
+    # :re2/re2.h -> re2/re2.h
+    "@com_googlesource_code_re2": ExternalRepo(
+        lambda x: re.sub(":", "", x), ":re2", None
+    ),
 }
 
 # TODO: proto rules are aspect-based and their generated files don't show up in

testing/file_test/BUILD

@@ -6,14 +6,33 @@ load("rules.bzl", "file_test")
 
 package(default_visibility = ["//visibility:public"])
 
+cc_library(
+    name = "autoupdate",
+    testonly = 1,
+    srcs = ["autoupdate.cpp"],
+    hdrs = [
+        "autoupdate.h",
+        "line.h",
+    ],
+    deps = [
+        "//common:check",
+        "//common:ostream",
+        "@com_google_absl//absl/strings:string_view",
+        "@com_googlesource_code_re2//:re2",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "file_test_base",
     testonly = 1,
     srcs = ["file_test_base.cpp"],
     hdrs = ["file_test_base.h"],
     deps = [
+        ":autoupdate",
         "//common:check",
-        "//testing/util:test_raw_ostream",
+        "//common:error",
+        "//common:ostream",
         "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/flags:parse",
         "@com_google_googletest//:gtest",

testing/file_test/README.md

@@ -0,0 +1,137 @@
+# file_test
+
+<!--
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+-->
+
+## BUILD
+
+A typical BUILD target will look like:
+
+```
+load("rules.bzl", "file_test")
+
+file_test(
+    name = "my_file_test",
+    srcs = ["my_file_test.cpp"],
+    tests = glob(["testdata/**"]),
+    deps = [
+        ":my_lib",
+        "//testing/file_test:file_test_base",
+        "@com_google_googletest//:gtest",
+        "@llvm-project//llvm:Support",
+    ],
+)
+```
+
+## Implementation
+
+A typical implementation will look like:
+
+```
+#include "my_library.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "testing/file_test/file_test_base.h"
+
+namespace Carbon::Testing {
+namespace {
+
+class MyFileTest : public FileTestBase {
+ public:
+  using FileTestBase::FileTestBase;
+
+  // Called as part of individual test executions.
+  auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
+           const llvm::SmallVector<TestFile>& test_files,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<bool> override {
+    MyFunctionality(test_args, stdout, stderr);
+  }
+
+  // Provides arguments which are used in tests that don't provide ARGS.
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {"default_args", "%s"};
+  }
+};
+
+}  // namespace
+
+// Registers for the framework to construct the tests.
+CARBON_FILE_TEST_FACTORY(MyFileTest);
+
+}  // namespace Carbon::Testing
+```
+
+## Comment markers
+
+Settings in files are provided in comments, similar to `FileCheck` syntax.
+`bazel run :file_test -- --autoupdate` automatically constructs compatible
+CHECK:STDOUT: and CHECK:STDERR: lines.
+
+Supported comment markers are:
+
+-   ```
+    // AUTOUDPATE
+    // NOAUTOUPDATE
+    ```
+
+    Controls whether the checks in the file will be autoupdated if --autoupdate
+    is passed. Exactly one of these two markers must be present. If the file
+    uses splits, AUTOUPDATE must currently be before any splits.
+
+    When autoupdating, CHECKs will be inserted starting below AUTOUPDATE. When a
+    CHECK has line information, autoupdate will try to insert the CHECK
+    immediately above the line it's associated with. When that happens, any
+    following CHECK lines without line information will immediately follow,
+    between the CHECK with line information and the associated line.
+
+-   `// ARGS: <arguments>`
+
+    Provides a space-separated list of arguments, which will be passed to
+    RunWithFiles as test_args. These are intended for use by the command as
+    arguments.
+
+    Supported replacements within arguments are:
+
+    -   `%s`
+
+        Replaced with the list of files. Currently only allowed as a standalone
+        argument, not a substring.
+
+    -   `%t`
+
+        Replaced with `${TEST_TMPDIR}/temp_file`.
+
+    ARGS can be specified at most once. If not provided, the FileTestBase child
+    is responsible for providing default arguments.
+
+-   `// SET-CHECK-SUBSET`
+
+    By default, all lines of output must have a CHECK match. Adding this as a
+    option sets it so that non-matching lines are ignored. All provided
+    CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output.
+
+    SET-CHECK-SUBSET can be specified at most once.
+
+-   `// --- <filename>`
+
+    By default, all file content is provided to the test as a single file in
+    test_files. Using this marker allows the file to be split into multiple
+    files which will all be passed to test_files.
+
+    Files are not created on disk; it's expected the child will create an
+    InMemoryFilesystem if needed.
+
+-   ```
+    // CHECK:STDOUT: <output line>
+    // CHECK:STDERR: <output line>
+    ```
+
+    These provide a match for output from the command. See `SET-CHECK-SUBSET`
+    for how to change from full to subset matching of output.
+
+    Output line matchers may contain `[[@LINE+offset]` and `{{regex}}` syntaxes,
+    similar to `FileCheck`.

testing/file_test/autoupdate.cpp

@@ -0,0 +1,241 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "testing/file_test/autoupdate.h"
+
+#include <fstream>
+
+#include "absl/strings/string_view.h"
+#include "common/check.h"
+#include "common/ostream.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "re2/re2.h"
+
+namespace Carbon::Testing {
+
+// Put helper classes in an anonymous namespace.
+namespace {
+
+class CheckLine : public FileTestLineBase {
+ public:
+  // RE2 is passed by a pointer because it doesn't support std::optional.
+  explicit CheckLine(int line_number, const RE2* line_number_re,
+                     std::string line)
+      : FileTestLineBase(line_number),
+        line_number_re_(line_number_re),
+        line_(std::move(line)) {}
+
+  auto Print(llvm::raw_ostream& out) const -> void override {
+    out << indent_ << line_;
+  }
+
+  // When the location of the CHECK in output is known, we can set the indent
+  // and its line.
+  auto SetOutputLine(llvm::StringRef indent, int output_line_number) -> void {
+    indent_ = indent;
+    output_line_number_ = output_line_number;
+  }
+
+  // When the location of all lines in a file are known, we can set the line
+  // offset based on the target line.
+  auto SetRemappedLine(const std::string& sub_for_formatv,
+                       int target_line_number) -> void {
+    // Should only be called when we have a regex.
+    CARBON_CHECK(line_number_re_);
+
+    int offset = target_line_number - output_line_number_;
+    const char* offset_prefix = offset < 0 ? "" : "+";
+    std::string replacement =
+        llvm::formatv(sub_for_formatv.data(),
+                      llvm::formatv("[[@LINE{0}{1}]]", offset_prefix, offset));
+    RE2::Replace(&line_, *line_number_re_, replacement);
+  }
+
+ private:
+  const RE2* line_number_re_;
+  std::string line_;
+  llvm::StringRef indent_;
+  int output_line_number_ = -1;
+};
+
+}  // namespace
+
+// Adds output lines for autoupdate.
+static auto AddCheckLines(
+    llvm::StringRef output, const char* label,
+    const llvm::SmallVector<llvm::StringRef>& filenames,
+    bool line_number_re_has_file, const RE2& line_number_re,
+    std::function<void(std::string&)> do_extra_check_replacements,
+    llvm::SmallVector<llvm::SmallVector<CheckLine>>& check_lines) -> void {
+  if (output.empty()) {
+    return;
+  }
+
+  // Prepare to look for filenames in lines.
+  llvm::StringRef current_filename = filenames[0];
+  const auto* remaining_filenames = filenames.begin() + 1;
+
+  // %t substitution means we may see TEST_TMPDIR in output.
+  char* tmpdir_env = getenv("TEST_TMPDIR");
+  CARBON_CHECK(tmpdir_env != nullptr);
+  llvm::StringRef tmpdir = tmpdir_env;
+
+  llvm::SmallVector<llvm::StringRef> lines(llvm::split(output, '\n'));
+  // It's typical that output ends with a newline, but we don't want to add a
+  // blank CHECK for it.
+  if (lines.back().empty()) {
+    lines.pop_back();
+  }
+
+  int append_to = 0;
+  for (const auto& line : lines) {
+    std::string check_line = llvm::formatv("// CHECK:{0}:{1}{2}", label,
+                                           line.empty() ? "" : " ", line);
+
+    // Ignore TEST_TMPDIR in output.
+    if (auto pos = check_line.find(tmpdir); pos != std::string::npos) {
+      check_line.replace(pos, tmpdir.size(), "{{.+}}");
+    }
+
+    do_extra_check_replacements(check_line);
+
+    // Look for line information in the output. line_number is only set if the
+    // match is correct.
+    int line_number = -1;
+    int match_line_number;
+    if (line_number_re_has_file) {
+      absl::string_view match_filename;
+      if (RE2::PartialMatch(check_line, line_number_re, &match_filename,
+                            &match_line_number)) {
+        llvm::StringRef match_filename_ref = match_filename;
+        if (match_filename_ref != current_filename) {
+          // If the filename doesn't match, it may be still usable if it refers
+          // to a later file.
+          const auto* pos = std::find(remaining_filenames, filenames.end(),
+                                      match_filename_ref);
+          if (pos != filenames.end()) {
+            remaining_filenames = pos + 1;
+            append_to = pos - filenames.begin();
+            line_number = match_line_number;
+          }
+        } else {
+          // The line applies to the current file.
+          line_number = match_line_number;
+        }
+      }
+    } else {
+      // There's no file association, so we only look at the line.
+      if (RE2::PartialMatch(check_line, line_number_re, &match_line_number)) {
+        line_number = match_line_number;
+      }
+    }
+    check_lines[append_to].push_back(
+        CheckLine(line_number, line_number == -1 ? nullptr : &line_number_re,
+                  check_line));
+  }
+}
+
+auto AutoupdateFileTest(
+    const std::filesystem::path& file_test_path, llvm::StringRef input_content,
+    const llvm::SmallVector<llvm::StringRef>& filenames,
+    int autoupdate_line_number,
+    llvm::SmallVector<llvm::SmallVector<FileTestLine>>& non_check_lines,
+    llvm::StringRef stdout, llvm::StringRef stderr,
+    FileTestLineNumberReplacement line_number_replacement,
+    std::function<void(std::string&)> do_extra_check_replacements) -> bool {
+  // Prepare CHECK lines.
+  llvm::SmallVector<llvm::SmallVector<CheckLine>> check_lines;
+  check_lines.resize(filenames.size());
+  RE2 line_number_re(line_number_replacement.pattern);
+  CARBON_CHECK(line_number_re.ok()) << "Invalid line replacement RE2: `"
+                                    << line_number_replacement.pattern << "`";
+
+  AddCheckLines(stdout, "STDOUT", filenames, line_number_replacement.has_file,
+                line_number_re, do_extra_check_replacements, check_lines);
+  AddCheckLines(stderr, "STDERR", filenames, line_number_replacement.has_file,
+                line_number_re, do_extra_check_replacements, check_lines);
+
+  // All CHECK lines are suppressed until we reach AUTOUPDATE.
+  bool reached_autoupdate = false;
+
+  // Stitch together content.
+  llvm::SmallVector<const FileTestLineBase*> new_lines;
+  for (auto [filename, non_check_file, check_file] :
+       llvm::zip(filenames, non_check_lines, check_lines)) {
+    llvm::DenseMap<int, int> output_line_remap;
+    int output_line_number = 0;
+    auto* check_line = check_file.begin();
+
+    // Looping through the original file, print check lines preceding each
+    // original line.
+    for (const auto& non_check_line : non_check_file) {
+      // If there are any non-check lines with an invalid line_number, it's
+      // something like a split directive which shouldn't increment
+      // output_line_number.
+      if (non_check_line.line_number() < 1) {
+        new_lines.push_back(&non_check_line);
+        continue;
+      }
+
+      if (reached_autoupdate) {
+        for (; check_line != check_file.end() &&
+               check_line->line_number() <= non_check_line.line_number();
+             ++check_line) {
+          new_lines.push_back(check_line);
+          check_line->SetOutputLine(non_check_line.indent(),
+                                    ++output_line_number);
+        }
+      } else if (autoupdate_line_number == non_check_line.line_number()) {
+        // This is the AUTOUPDATE line, so we'll print it, then start printing
+        // CHECK lines.
+        reached_autoupdate = true;
+      }
+      new_lines.push_back(&non_check_line);
+      CARBON_CHECK(
+          output_line_remap
+              .insert({non_check_line.line_number(), ++output_line_number})
+              .second);
+    }
+
+    // This should always be true after the first file is processed.
+    CARBON_CHECK(reached_autoupdate);
+
+    // Print remaining check lines which -- for whatever reason -- come after
+    // all original lines.
+    for (; check_line != check_file.end(); ++check_line) {
+      new_lines.push_back(check_line);
+      check_line->SetOutputLine("", ++output_line_number);
+    }
+
+    // Update all remapped lines in CHECK output.
+    for (auto& offset_check_line : check_file) {
+      if (offset_check_line.line_number() >= 1) {
+        auto new_line = output_line_remap.find(offset_check_line.line_number());
+        CARBON_CHECK(new_line != output_line_remap.end());
+        offset_check_line.SetRemappedLine(
+            line_number_replacement.sub_for_formatv, new_line->second);
+      }
+    }
+  }
+
+  // Generate the autoupdated file.
+  std::string new_content;
+  llvm::raw_string_ostream new_content_stream(new_content);
+  for (const auto& line : new_lines) {
+    line->Print(new_content_stream);
+    new_content_stream << '\n';
+  }
+
+  // Update the file on disk if needed.
+  if (new_content == input_content) {
+    return false;
+  }
+  std::ofstream out(file_test_path);
+  out << new_content;
+  return true;
+}
+
+}  // namespace Carbon::Testing

testing/file_test/autoupdate.h

@@ -0,0 +1,39 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TESTING_FILE_TEST_AUTOUPDATE_H_
+#define CARBON_TESTING_FILE_TEST_AUTOUPDATE_H_
+
+#include <filesystem>
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "testing/file_test/line.h"
+
+namespace Carbon::Testing {
+
+struct FileTestLineNumberReplacement {
+  bool has_file;
+
+  // The line replacement. The pattern should match lines. If has_file, pattern
+  // should have a file and line group; otherwise, only a line group.
+  std::string pattern;
+
+  // sub_for_formatv should provide {0} to substitute with [[@LINE...]] deltas.
+  std::string sub_for_formatv;
+};
+
+// Automatically updates CHECKs in the provided file. Returns true if updated.
+auto AutoupdateFileTest(
+    const std::filesystem::path& file_test_path, llvm::StringRef input_content,
+    const llvm::SmallVector<llvm::StringRef>& filenames,
+    int autoupdate_line_number,
+    llvm::SmallVector<llvm::SmallVector<FileTestLine>>& non_check_lines,
+    llvm::StringRef stdout, llvm::StringRef stderr,
+    FileTestLineNumberReplacement line_number_replacement,
+    std::function<void(std::string&)> do_extra_check_replacements) -> bool;
+
+}  // namespace Carbon::Testing
+
+#endif  // CARBON_TESTING_FILE_TEST_AUTOUPDATE_H_

testing/file_test/file_test_base.cpp

@@ -6,6 +6,7 @@
 
 #include <filesystem>
 #include <fstream>
+#include <utility>
 
 #include "absl/flags/flag.h"
 #include "absl/flags/parse.h"
@@ -14,10 +15,12 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/InitLLVM.h"
-#include "testing/util/test_raw_ostream.h"
 
 ABSL_FLAG(std::vector<std::string>, file_tests, {},
           "A comma-separated list of tests for file_test infrastructure.");
+ABSL_FLAG(bool, autoupdate, false,
+          "Instead of verifying files match test output, autoupdate files "
+          "based on test output.");
 
 namespace Carbon::Testing {
 
@@ -59,65 +62,102 @@ auto FileTestBase::TestBody() -> void {
   llvm::errs() << "\nTo test this file alone, run:\n  bazel test " << target
                << " --test_arg=--file_tests=" << test_file << "\n\n";
 
+  TestContext context;
+  auto run_result = ProcessTestFileAndRun(context);
+  ASSERT_TRUE(run_result.ok()) << run_result.error();
+  ValidateRun(context.test_files);
+  EXPECT_THAT(!llvm::StringRef(path().filename()).starts_with("fail_"),
+              Eq(context.exit_with_success))
+      << "Tests should be prefixed with `fail_` if and only if running them "
+         "is expected to fail.";
+
+  // Check results.
+  if (context.check_subset) {
+    EXPECT_THAT(SplitOutput(context.stdout),
+                IsSupersetOf(context.expected_stdout));
+    EXPECT_THAT(SplitOutput(context.stderr),
+                IsSupersetOf(context.expected_stderr));
+
+  } else {
+    EXPECT_THAT(SplitOutput(context.stdout),
+                ElementsAreArray(context.expected_stdout));
+    EXPECT_THAT(SplitOutput(context.stderr),
+                ElementsAreArray(context.expected_stderr));
+  }
+}
+
+auto FileTestBase::Autoupdate() -> bool {
+  TestContext context;
+  auto run_result = ProcessTestFileAndRun(context);
+  CARBON_CHECK(run_result.ok()) << run_result.error();
+  if (!context.autoupdate_line_number) {
+    return false;
+  }
+
+  llvm::SmallVector<llvm::StringRef> filenames;
+  filenames.reserve(context.non_check_lines.size());
+  if (context.non_check_lines.size() > 1) {
+    // There are splits, so we provide an empty name for the first file.
+    filenames.push_back({});
+  }
+  for (const auto& file : context.test_files) {
+    filenames.push_back(file.filename);
+  }
+
+  llvm::ArrayRef filenames_for_line_number = filenames;
+  if (filenames.size() > 1) {
+    filenames_for_line_number = filenames_for_line_number.drop_front();
+  }
+
+  return AutoupdateFileTest(
+      path(), context.input_content, filenames, *context.autoupdate_line_number,
+      context.non_check_lines, context.stdout, context.stderr,
+      GetLineNumberReplacement(filenames_for_line_number),
+      [&](std::string& line) { DoExtraCheckReplacements(line); });
+}
+
+auto FileTestBase::GetLineNumberReplacement(
+    llvm::ArrayRef<llvm::StringRef> filenames) -> LineNumberReplacement {
+  return {
+      .has_file = true,
+      .pattern = llvm::formatv(R"(({0}):(\d+):)", llvm::join(filenames, "|")),
+      .sub_for_formatv = R"(\1:{0}:)"};
+}
+
+auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
+    -> ErrorOr<Success> {
   // Store the file so that test_files can use references to content.
-  std::string test_content = ReadFile(path());
+  context.input_content = ReadFile(path());
 
   // Load expected output.
-  llvm::SmallVector<std::string> test_args;
-  llvm::SmallVector<TestFile> test_files;
-  llvm::SmallVector<Matcher<std::string>> expected_stdout;
-  llvm::SmallVector<Matcher<std::string>> expected_stderr;
-  bool check_subset = false;
-  ProcessTestFile(test_content, test_args, test_files, expected_stdout,
-                  expected_stderr, check_subset);
-  if (HasFailure()) {
-    return;
-  }
+  CARBON_RETURN_IF_ERROR(ProcessTestFile(context));
 
   // Process arguments.
-  if (test_args.empty()) {
-    test_args = GetDefaultArgs();
-  }
-  DoArgReplacements(test_args, test_files);
-  if (HasFailure()) {
-    return;
+  if (context.test_args.empty()) {
+    context.test_args = GetDefaultArgs();
   }
+  CARBON_RETURN_IF_ERROR(
+      DoArgReplacements(context.test_args, context.test_files));
 
   // Pass arguments as StringRef.
   llvm::SmallVector<llvm::StringRef> test_args_ref;
-  test_args_ref.reserve(test_args.size());
-  for (const auto& arg : test_args) {
+  test_args_ref.reserve(context.test_args.size());
+  for (const auto& arg : context.test_args) {
     test_args_ref.push_back(arg);
   }
 
   // Capture trace streaming, but only when in debug mode.
-  TestRawOstream stdout;
-  TestRawOstream stderr;
-  bool run_succeeded = RunWithFiles(test_args_ref, test_files, stdout, stderr);
-  if (HasFailure()) {
-    return;
-  }
-  EXPECT_THAT(!llvm::StringRef(path().filename()).starts_with("fail_"),
-              Eq(run_succeeded))
-      << "Tests should be prefixed with `fail_` if and only if running them "
-         "is expected to fail.";
-
-  // Check results.
-  if (check_subset) {
-    EXPECT_THAT(SplitOutput(stdout.TakeStr()), IsSupersetOf(expected_stdout));
-    EXPECT_THAT(SplitOutput(stderr.TakeStr()), IsSupersetOf(expected_stderr));
-
-  } else {
-    EXPECT_THAT(SplitOutput(stdout.TakeStr()),
-                ElementsAreArray(expected_stdout));
-    EXPECT_THAT(SplitOutput(stderr.TakeStr()),
-                ElementsAreArray(expected_stderr));
-  }
+  llvm::raw_svector_ostream stdout(context.stdout);
+  llvm::raw_svector_ostream stderr(context.stderr);
+  CARBON_ASSIGN_OR_RETURN(
+      context.exit_with_success,
+      Run(test_args_ref, context.test_files, stdout, stderr));
+  return Success();
 }
 
 auto FileTestBase::DoArgReplacements(
     llvm::SmallVector<std::string>& test_args,
-    const llvm::SmallVector<TestFile>& test_files) -> void {
+    const llvm::SmallVector<TestFile>& test_files) -> ErrorOr<Success> {
   for (auto* it = test_args.begin(); it != test_args.end(); ++it) {
     auto percent = it->find("%");
     if (percent == std::string::npos) {
@@ -125,13 +165,13 @@ auto FileTestBase::DoArgReplacements(
     }
 
     if (percent + 1 >= it->size()) {
-      FAIL() << "% is not allowed on its own: " << *it;
+      return ErrorBuilder() << "% is not allowed on its own: " << *it;
     }
     char c = (*it)[percent + 1];
     switch (c) {
       case 's': {
         if (*it != "%s") {
-          FAIL() << "%s must be the full argument: " << *it;
+          return ErrorBuilder() << "%s must be the full argument: " << *it;
         }
         it = test_args.erase(it);
         for (const auto& file : test_files) {
@@ -143,107 +183,150 @@ auto FileTestBase::DoArgReplacements(
         break;
       }
       case 't': {
-        char* temp = getenv("TEST_TMPDIR");
-        CARBON_CHECK(temp != nullptr);
-        it->replace(percent, 2, llvm::formatv("{0}/temp_file", temp));
+        char* tmpdir = getenv("TEST_TMPDIR");
+        CARBON_CHECK(tmpdir != nullptr);
+        it->replace(percent, 2, llvm::formatv("{0}/temp_file", tmpdir));
         break;
       }
       default:
-        FAIL() << "%" << c << " is not supported: " << *it;
+        return ErrorBuilder() << "%" << c << " is not supported: " << *it;
     }
   }
+  return Success();
 }
 
-auto FileTestBase::ProcessTestFile(
-    llvm::StringRef file_content, llvm::SmallVector<std::string>& test_args,
-    llvm::SmallVector<TestFile>& test_files,
-    llvm::SmallVector<Matcher<std::string>>& expected_stdout,
-    llvm::SmallVector<Matcher<std::string>>& expected_stderr,
-    bool& check_subset) -> void {
+auto FileTestBase::ProcessTestFile(TestContext& context) -> ErrorOr<Success> {
+  // Original file content, and a cursor for walking through it.
+  llvm::StringRef file_content = context.input_content;
   llvm::StringRef cursor = file_content;
+
+  // Whether content has been found, only updated before a file split is found
+  // (which may be never).
   bool found_content_pre_split = false;
+
+  // Whether either AUTOUDPATE or NOAUTOUPDATE was found.
+  bool found_autoupdate = false;
+
+  // The index in the current test file. Will be reset on splits.
   int line_index = 0;
+
+  // The current file name, considering splits. Not set for the default file.
   llvm::StringRef current_file_name;
+
+  // The current file's start.
   const char* current_file_start = nullptr;
+
+  context.non_check_lines.resize(1);
   while (!cursor.empty()) {
     auto [line, next_cursor] = cursor.split("\n");
     cursor = next_cursor;
+    auto line_trimmed = line.ltrim();
 
     static constexpr llvm::StringLiteral SplitPrefix = "// ---";
-    if (line.consume_front(SplitPrefix)) {
+    if (line_trimmed.consume_front(SplitPrefix)) {
+      if (!found_autoupdate) {
+        // If there's a split, all output is appended at the end of each file
+        // before AUTOUPDATE. We may want to change that, but it's not necessary
+        // to handle right now.
+        return ErrorBuilder()
+               << "AUTOUPDATE/NOAUTOUPDATE setting must be in the first file.";
+      }
+
+      context.non_check_lines.push_back({FileTestLine(0, line)});
       // On a file split, add the previous file, then start a new one.
       if (current_file_start) {
-        test_files.push_back(TestFile(
+        context.test_files.push_back(TestFile(
             current_file_name.str(),
-            llvm::StringRef(
-                current_file_start,
-                line.begin() - current_file_start - SplitPrefix.size())));
-      } else {
+            llvm::StringRef(current_file_start, line_trimmed.begin() -
+                                                    current_file_start -
+                                                    SplitPrefix.size())));
+      } else if (found_content_pre_split) {
         // For the first split, we make sure there was no content prior.
-        ASSERT_FALSE(found_content_pre_split)
-            << "When using split files, there must be no content before the "
-               "first split file.";
+        return ErrorBuilder()
+               << "When using split files, there must be no content before the "
+                  "first split file.";
       }
-      current_file_name = line.trim();
+      current_file_name = line_trimmed.trim();
       current_file_start = cursor.begin();
       line_index = 0;
       continue;
-    } else if (!current_file_start && !line.starts_with("//") &&
-               !line.trim().empty()) {
+    } else if (!current_file_start && !line_trimmed.starts_with("//") &&
+               !line_trimmed.empty()) {
       found_content_pre_split = true;
     }
     ++line_index;
 
     // Process expectations when found.
-    auto line_trimmed = line.ltrim();
-    if (line_trimmed.consume_front("// ARGS: ")) {
-      if (test_args.empty()) {
-        // Split the line into arguments.
-        std::pair<llvm::StringRef, llvm::StringRef> cursor =
-            llvm::getToken(line_trimmed);
-        while (!cursor.first.empty()) {
-          test_args.push_back(std::string(cursor.first));
-          cursor = llvm::getToken(cursor.second);
-        }
-      } else {
-        FAIL() << "ARGS was specified multiple times: " << line.str();
-      }
-    } else if (line_trimmed == "// SET-CHECK-SUBSET") {
-      if (!check_subset) {
-        check_subset = true;
-      } else {
-        FAIL() << "SET-CHECK-SUBSET was specified multiple times";
-      }
-    } else if (line_trimmed.consume_front("// CHECK")) {
+    if (line_trimmed.consume_front("// CHECK")) {
+      llvm::SmallVector<Matcher<std::string>>* expected = nullptr;
       if (line_trimmed.consume_front(":STDOUT:")) {
-        expected_stdout.push_back(
-            TransformExpectation(line_index, line_trimmed));
+        expected = &context.expected_stdout;
       } else if (line_trimmed.consume_front(":STDERR:")) {
-        expected_stderr.push_back(
-            TransformExpectation(line_index, line_trimmed));
+        expected = &context.expected_stderr;
       } else {
-        FAIL() << "Unexpected CHECK in input: " << line.str();
+        return ErrorBuilder() << "Unexpected CHECK in input: " << line.str();
+      }
+      expected->push_back(TransformExpectation(line_index, line_trimmed));
+    } else {
+      context.non_check_lines.back().push_back(FileTestLine(line_index, line));
+      if (line_trimmed.consume_front("// ARGS: ")) {
+        if (context.test_args.empty()) {
+          // Split the line into arguments.
+          std::pair<llvm::StringRef, llvm::StringRef> cursor =
+              llvm::getToken(line_trimmed);
+          while (!cursor.first.empty()) {
+            context.test_args.push_back(std::string(cursor.first));
+            cursor = llvm::getToken(cursor.second);
+          }
+        } else {
+          return ErrorBuilder()
+                 << "ARGS was specified multiple times: " << line.str();
+        }
+      } else if (line_trimmed == "// AUTOUPDATE" ||
+                 line_trimmed == "// NOAUTOUPDATE") {
+        if (found_autoupdate) {
+          return ErrorBuilder()
+                 << "Multiple AUTOUPDATE/NOAUTOUPDATE settings found";
+        }
+        found_autoupdate = true;
+        if (line_trimmed == "// AUTOUPDATE") {
+          context.autoupdate_line_number = line_index;
+        }
+      } else if (line_trimmed == "// SET-CHECK-SUBSET") {
+        if (!context.check_subset) {
+          context.check_subset = true;
+        } else {
+          return ErrorBuilder()
+                 << "SET-CHECK-SUBSET was specified multiple times";
+        }
       }
     }
   }
 
+  if (!found_autoupdate) {
+    return ErrorBuilder() << "Missing AUTOUPDATE/NOAUTOUPDATE setting";
+  }
+
   if (current_file_start) {
-    test_files.push_back(
+    context.test_files.push_back(
         TestFile(current_file_name.str(),
                  llvm::StringRef(current_file_start,
                                  file_content.end() - current_file_start)));
   } else {
     // If no file splitting happened, use the main file as the test file.
-    test_files.push_back(TestFile(path().filename().string(), file_content));
+    context.test_files.push_back(
+        TestFile(path().filename().string(), file_content));
   }
 
   // Assume there is always a suffix `\n` in output.
-  if (!expected_stdout.empty()) {
-    expected_stdout.push_back(StrEq(""));
+  if (!context.expected_stdout.empty()) {
+    context.expected_stdout.push_back(StrEq(""));
   }
-  if (!expected_stderr.empty()) {
-    expected_stderr.push_back(StrEq(""));
+  if (!context.expected_stderr.empty()) {
+    context.expected_stderr.push_back(StrEq(""));
   }
+
+  return Success();
 }
 
 auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in)
@@ -358,17 +441,30 @@ auto main(int argc, char** argv) -> int {
 
   auto test_factory = Carbon::Testing::GetFileTestFactory();
 
-  // Register tests based on their absolute path.
   for (const auto& file_test : absl::GetFlag(FLAGS_file_tests)) {
+    // Pass the absolute path to the factory function.
     auto path = std::filesystem::absolute(file_test, ec);
     CARBON_CHECK(!ec) << file_test << ": " << ec.message();
     CARBON_CHECK(llvm::StringRef(path.string()).starts_with(base_dir))
         << "\n  " << path << "\n  should start with\n  " << base_dir;
-    std::string test_name = path.string().substr(base_dir.size());
-    testing::RegisterTest(test_factory.name, test_name.c_str(), nullptr,
-                          test_name.c_str(), __FILE__, __LINE__,
-                          [=]() { return test_factory.factory_fn(path); });
+    if (absl::GetFlag(FLAGS_autoupdate)) {
+      std::unique_ptr<Carbon::Testing::FileTestBase> test(
+          test_factory.factory_fn(path));
+      llvm::errs() << (test->Autoupdate() ? "!" : ".");
+    } else {
+      std::string test_name = path.string().substr(base_dir.size());
+      testing::RegisterTest(test_factory.name, test_name.c_str(), nullptr,
+                            test_name.c_str(), __FILE__, __LINE__,
+                            [=]() { return test_factory.factory_fn(path); });
+    }
+  }
+  if (absl::GetFlag(FLAGS_autoupdate)) {
+    llvm::errs() << "\nDone!\n";
   }
 
-  return RUN_ALL_TESTS();
+  if (absl::GetFlag(FLAGS_autoupdate)) {
+    return EXIT_SUCCESS;
+  } else {
+    return RUN_ALL_TESTS();
+  }
 }

testing/file_test/file_test_base.h

@@ -10,72 +10,17 @@
 
 #include <filesystem>
 #include <functional>
-#include <vector>
 
+#include "common/error.h"
+#include "common/ostream.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/raw_os_ostream.h"
-#include "llvm/Support/raw_ostream.h"
+#include "testing/file_test/autoupdate.h"
 
 namespace Carbon::Testing {
 
-// A framework for testing files. Children write
-// `CARBON_FILE_TEST_FACTORY(MyTest)` which is used to construct the tests.
-// `RunWithFiles` must also be implemented and will be called as part of
-// individual test executions. This framework includes a `main` implementation,
-// so users must not provide one.
-//
-// Settings in files are provided in comments, similar to `FileCheck` syntax.
-// `autoupdate_testdata.py` automatically constructs compatible CHECK:STDOUT:
-// and CHECK:STDERR: lines.
-//
-// Supported comment markers are:
-//
-// - // ARGS: <arguments>
-//
-//   Provides a space-separated list of arguments, which will be passed to
-//   RunWithFiles as test_args. These are intended for use by the command as
-//   arguments.
-//
-//   Supported replacements within arguments are:
-//
-//   - %s
-//
-//     Replaced with the list of files. Currently only allowed as a standalone
-//     argument, not a substring.
-//
-//   - %t
-//
-//     Replaced with `${TEST_TMPDIR}/temp_file`.
-//
-//   ARGS can be specified at most once. If not provided, the FileTestBase child
-//   is responsible for providing default arguments.
-//
-// - // SET-CHECK-SUBSET
-//
-//   By default, all lines of output must have a CHECK match. Adding this as a
-//   flag sets it so that non-matching lines are ignored. All provided
-//   CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output.
-//
-//   SET-CHECK-SUBSET can be specified at most once.
-//
-// - // --- <filename>
-//
-//   By default, all file content is provided to the test as a single file in
-//   test_files. Using this marker allows the file to be split into multiple
-//   files which will all be passed to test_files.
-//
-//   Files are not created on disk; it's expected the child will create an
-//   InMemoryFilesystem if needed.
-//
-// - // CHECK:STDOUT: <output line>
-//   // CHECK:STDERR: <output line>
-//
-//   These provides a match for output from the command. See SET-CHECK-SUBSET
-//   for how to change from full to subset matching of output.
-//
-//   Output line matchers may contain `[[@LINE+offset]` and
-//   `{{regex}}` syntaxes, similar to `FileCheck`.
+// A framework for testing files. See README.md for documentation.
 class FileTestBase : public testing::Test {
  public:
   struct TestFile {
@@ -94,38 +39,100 @@ class FileTestBase : public testing::Test {
     llvm::StringRef content;
   };
 
+  // Provided for child class convenience.
+  using LineNumberReplacement = FileTestLineNumberReplacement;
+
   explicit FileTestBase(std::filesystem::path path) : path_(std::move(path)) {}
 
-  // Implemented by children to run the test. Called by the TestBody
-  // implementation, which will validate stdout and stderr. The return value
-  // should be false when "fail_" is in the filename.
-  virtual auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
-                            const llvm::SmallVector<TestFile>& test_files,
-                            llvm::raw_pwrite_stream& stdout,
-                            llvm::raw_pwrite_stream& stderr) -> bool = 0;
+  // Implemented by children to run the test. For example, TestBody validates
+  // stdout and stderr.
+  //
+  // Any test expectations should be called from ValidateRun, not Run.
+  //
+  // The return value should be an error if there was an abnormal error. It
+  // should be true if a binary would return EXIT_SUCCESS, and false for
+  // EXIT_FAILURE (which is a test success for `fail_*` tests).
+  virtual auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
+                   const llvm::SmallVector<TestFile>& test_files,
+                   llvm::raw_pwrite_stream& stdout,
+                   llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> = 0;
+
+  // Implemented by children to do post-Run test expectations. Only called when
+  // testing. Does not need to be provided if only CHECK test expectations are
+  // used.
+  virtual auto ValidateRun(const llvm::SmallVector<TestFile>& /*test_files*/)
+      -> void {}
 
   // Returns default arguments. Only called when a file doesn't set ARGS.
   virtual auto GetDefaultArgs() -> llvm::SmallVector<std::string> = 0;
 
+  // Returns replacement information for line numbers. See LineReplacement for
+  // construction.
+  virtual auto GetLineNumberReplacement(
+      llvm::ArrayRef<llvm::StringRef> filenames) -> LineNumberReplacement;
+
+  // Optionally allows children to provide extra replacements for autoupdate.
+  virtual auto DoExtraCheckReplacements(std::string& /*check_line*/) -> void {}
+
   // Runs a test and compares output. This keeps output split by line so that
   // issues are a little easier to identify by the different line.
   auto TestBody() -> void final;
 
+  // Runs the test and autoupdates checks. Returns true if updated.
+  auto Autoupdate() -> bool;
+
   // Returns the full path of the file being tested.
   auto path() -> const std::filesystem::path& { return path_; };
 
  private:
+  // Encapsulates test context generated by processing and running.
+  struct TestContext {
+    // The input test file content. Other parts may reference this.
+    std::string input_content;
+
+    // Lines which don't contain CHECKs, and thus need to be retained by
+    // autoupdate. Their line number in the file is attached.
+    //
+    // If there are splits, then the line is in the respective file. For N
+    // splits, there will be one vector for the parts of the input file which
+    // are not in any split, plus one vector per split file.
+    llvm::SmallVector<llvm::SmallVector<FileTestLine>> non_check_lines;
+
+    // Arguments for the test, generated from ARGS.
+    llvm::SmallVector<std::string> test_args;
+
+    // Files in the test, generated by content and splits.
+    llvm::SmallVector<TestFile> test_files;
+
+    // The location of the autoupdate marker, for autoupdated files.
+    std::optional<int> autoupdate_line_number;
+
+    // Whether checks are a subset, generated from SET-CHECK-SUBSET.
+    bool check_subset = false;
+
+    // stdout and stderr based on CHECK lines in the file.
+    llvm::SmallVector<testing::Matcher<std::string>> expected_stdout;
+    llvm::SmallVector<testing::Matcher<std::string>> expected_stderr;
+
+    // stdout and stderr from Run. 16 is arbitrary but a required value.
+    llvm::SmallString<16> stdout;
+    llvm::SmallString<16> stderr;
+
+    // Whether Run exited with success.
+    bool exit_with_success = false;
+  };
+
+  // Processes the test file and runs the test. Returns an error if something
+  // went wrong.
+  auto ProcessTestFileAndRun(TestContext& context) -> ErrorOr<Success>;
+
   // Does replacements in ARGS for %s and %t.
   auto DoArgReplacements(llvm::SmallVector<std::string>& test_args,
-                         const llvm::SmallVector<TestFile>& test_files) -> void;
+                         const llvm::SmallVector<TestFile>& test_files)
+      -> ErrorOr<Success>;
 
   // Processes the test input, producing test files and expected output.
-  auto ProcessTestFile(
-      llvm::StringRef file_content, llvm::SmallVector<std::string>& test_args,
-      llvm::SmallVector<TestFile>& test_files,
-      llvm::SmallVector<testing::Matcher<std::string>>& expected_stdout,
-      llvm::SmallVector<testing::Matcher<std::string>>& expected_stderr,
-      bool& check_subset) -> void;
+  auto ProcessTestFile(TestContext& context) -> ErrorOr<Success>;
 
   // Transforms an expectation on a given line from `FileCheck` syntax into a
   // standard regex matcher.

testing/file_test/file_test_base_test.cpp

@@ -7,11 +7,7 @@
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
-#include <fstream>
-#include <vector>
-
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace Carbon::Testing {
 namespace {
@@ -34,10 +30,10 @@ class FileTestBaseTest : public FileTestBase {
     return Field("content", &TestFile::content, Eq(content));
   }
 
-  auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
-                    const llvm::SmallVector<TestFile>& test_files,
-                    llvm::raw_pwrite_stream& stdout,
-                    llvm::raw_pwrite_stream& stderr) -> bool override {
+  auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
+           const llvm::SmallVector<TestFile>& test_files,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<bool> override {
     if (!test_args.empty()) {
       llvm::ListSeparator sep;
       stdout << test_args.size() << " args: ";
@@ -49,38 +45,46 @@ class FileTestBaseTest : public FileTestBase {
 
     auto filename = path().filename();
     if (filename == "args.carbon") {
-      EXPECT_THAT(test_files, ElementsAre(HasFilename("args.carbon")));
       return true;
     } else if (filename == "example.carbon") {
-      EXPECT_THAT(test_files, ElementsAre(HasFilename("example.carbon")));
+      int delta_line = 10;
       stdout << "something\n"
-                "\n"
-                "9: Line delta\n"
-                "8: Negative line delta\n"
-                "+*[]{}\n"
-                "Foo baz\n";
+             << "\n"
+             << "example.carbon:" << delta_line + 1 << ": Line delta\n"
+             << "example.carbon:" << delta_line << ": Negative line delta\n"
+             << "+*[]{}\n"
+             << "Foo baz\n";
       return true;
     } else if (filename == "fail_example.carbon") {
-      EXPECT_THAT(test_files, ElementsAre(HasFilename("fail_example.carbon")));
       stderr << "Oops\n";
       return false;
     } else if (filename == "two_files.carbon") {
       int i = 0;
       for (const auto& file : test_files) {
         // Prints line numbers to validate per-file.
-        stdout << file.filename << ": " << ++i << "\n";
+        stdout << file.filename << ":2: " << ++i << "\n";
       }
+      return true;
+    } else {
+      return ErrorBuilder() << "Unexpected file: " << filename;
+    }
+  }
+
+  auto ValidateRun(const llvm::SmallVector<TestFile>& test_files)
+      -> void override {
+    auto filename = path().filename();
+    if (filename == "two_files.carbon") {
       EXPECT_THAT(
           test_files,
           ElementsAre(
               AllOf(HasFilename("a.carbon"),
-                    HasContent("// CHECK:STDOUT: a.carbon: [[@LINE+0]]\n\n")),
+                    HasContent(
+                        "// CHECK:STDOUT: a.carbon:[[@LINE+1]]: 1\naaa\n\n")),
               AllOf(HasFilename("b.carbon"),
-                    HasContent("// CHECK:STDOUT: b.carbon: [[@LINE+1]]\n"))));
-      return true;
+                    HasContent(
+                        "// CHECK:STDOUT: b.carbon:[[@LINE+1]]: 2\nbbb\n"))));
     } else {
-      ADD_FAILURE() << "Unexpected file: " << filename;
-      return false;
+      EXPECT_THAT(test_files, ElementsAre(HasFilename(filename)));
     }
   }
 

testing/file_test/line.h

@@ -0,0 +1,46 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TESTING_FILE_TEST_LINE_H_
+#define CARBON_TESTING_FILE_TEST_LINE_H_
+
+#include "common/ostream.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace Carbon::Testing {
+
+// Interface for lines.
+class FileTestLineBase {
+ public:
+  explicit FileTestLineBase(int line_number) : line_number_(line_number) {}
+  virtual ~FileTestLineBase() {}
+
+  // Prints the autoupdated line.
+  virtual auto Print(llvm::raw_ostream& out) const -> void = 0;
+
+  auto line_number() const -> int { return line_number_; }
+
+ private:
+  int line_number_;
+};
+
+// A line in the original file test.
+class FileTestLine : public FileTestLineBase {
+ public:
+  explicit FileTestLine(int line_number, llvm::StringRef line)
+      : FileTestLineBase(line_number), line_(line) {}
+
+  auto Print(llvm::raw_ostream& out) const -> void override { out << line_; }
+
+  auto indent() const -> llvm::StringRef {
+    return line_.substr(0, line_.find_first_not_of(" \n"));
+  }
+
+ private:
+  llvm::StringRef line_;
+};
+
+}  // namespace Carbon::Testing
+
+#endif  // CARBON_TESTING_FILE_TEST_LINE_H_

testing/file_test/testdata/args.carbon

@@ -3,4 +3,5 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 // ARGS: abc file=%t %s
+// AUTOUPDATE
 // CHECK:STDOUT: 3 args: `abc`, `file={{.+}}/temp_file`, `args.carbon`

testing/file_test/testdata/example.carbon

@@ -2,10 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+// Can't autoupdate due to the `{{.+}}` check.
+// NOAUTOUPDATE
 // CHECK:STDOUT: 2 args: `default_args`, `example.carbon`
 // CHECK:STDOUT: something
 // CHECK:STDOUT:
-// CHECK:STDOUT: [[@LINE+1]]: Line delta
-// CHECK:STDOUT: [[@LINE-1]]: Negative line delta
+// CHECK:STDOUT: example.carbon:[[@LINE+1]]: Line delta
+// CHECK:STDOUT: example.carbon:[[@LINE-1]]: Negative line delta
 // CHECK:STDOUT: +*[]{}
 // CHECK:STDOUT: F{{.+}}z

testing/file_test/testdata/fail_example.carbon

@@ -2,5 +2,6 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+// AUTOUPDATE
 // CHECK:STDOUT: 2 args: `default_args`, `fail_example.carbon`
 // CHECK:STDERR: Oops

testing/file_test/testdata/two_files.carbon

@@ -2,10 +2,13 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+// AUTOUPDATE
 // CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon`
 
 // --- a.carbon
-// CHECK:STDOUT: a.carbon: [[@LINE+0]]
+// CHECK:STDOUT: a.carbon:[[@LINE+1]]: 1
+aaa
 
 // --- b.carbon
-// CHECK:STDOUT: b.carbon: [[@LINE+1]]
+// CHECK:STDOUT: b.carbon:[[@LINE+1]]: 2
+bbb

toolchain/autoupdate_testdata.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+"""Autoupdates testdata in toolchain."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import argparse
+import subprocess
+
+TARGETS = {
+    "codegen": "//toolchain/codegen:codegen_file_test",
+    "driver": "//toolchain/driver:driver_file_test",
+    "lexer": "//toolchain/lexer:lexer_file_test",
+    "lowering": "//toolchain/lowering:lowering_file_test",
+    "parser": "//toolchain/parser:parse_tree_file_test",
+    "semantics": "//toolchain/semantics:semantics_file_test",
+}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "dirs",
+        # We don't use `choices` because it seems to conflict with "*".
+        nargs="*",
+        default=TARGETS.keys(),
+        help="Optionally restrict directories to update. Defaults to all.",
+    )
+    parsed_args = parser.parse_args()
+
+    # Deduplicate and validate arguments.
+    dirs = set(parsed_args.dirs)
+    invalid_dirs = dirs.difference(TARGETS.keys())
+    if invalid_dirs:
+        exit(
+            f"Invalid dirs: {', '.join(invalid_dirs)}; "
+            f"allowed dirs are {', '.join(TARGETS.keys())}."
+        )
+
+    # Build the targets together if there's more than one. Otherwise, we may as
+    # well build and run together.
+    if len(dirs) > 1:
+        subprocess.check_call(
+            ["bazel", "build", "-c", "opt"] + [TARGETS[d] for d in dirs]
+        )
+    for d in dirs:
+        subprocess.check_call(
+            ["bazel", "run", "-c", "opt", TARGETS[d], "--", "--autoupdate"]
+        )
+
+
+if __name__ == "__main__":
+    main()

toolchain/codegen/testdata/assembly/fail_target_triple.carbon

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // ARGS: dump assembly --target_triple=x86_687-unknown-linux-gnu %s
-// TODO: Support autoupdate with ARGS.
+// No autoupdate because the message comes from LLVM.
 // NOAUTOUPDATE
 // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}}
 

toolchain/codegen/testdata/objcode/basic.carbon

@@ -3,8 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu --output_file=%t %s
-// TODO: Support autoupdate with ARGS.
-// NOAUTOUPDATE
+// AUTOUPDATE
 // CHECK:STDOUT: Success: Object file is generated!
 
 fn Main() -> i32 { return 0; }

toolchain/codegen/testdata/objcode/fail_no_input_file.carbon

@@ -3,8 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // ARGS: dump objcode --output_file=%t --target_triple=x86_64-unknown-linux-gnu
-// TODO: Support autoupdate with ARGS.
-// NOAUTOUPDATE
+// AUTOUPDATE
 // CHECK:STDERR: ERROR: No input file specified.
 
 fn Main() -> i32 { return 0; }

toolchain/codegen/testdata/objcode/fail_no_output_file.carbon

@@ -3,8 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu %s
-// TODO: Support autoupdate with ARGS.
-// NOAUTOUPDATE
+// AUTOUPDATE
 // CHECK:STDERR: ERROR: Must provide an output file.
 
 fn Main() -> i32 { return 0; }

toolchain/codegen/testdata/objcode/fail_target_triple.carbon

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // ARGS: dump objcode --target_triple=x86_684-unknown-linux-gnu --output_file=%t %s
-// TODO: Support autoupdate with ARGS.
+// No autoupdate because the message comes from LLVM.
 // NOAUTOUPDATE
 // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}}
 

toolchain/driver/driver_file_test.cpp

@@ -17,6 +17,13 @@ class DriverFileTest : public DriverFileTestBase {
   auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
     CARBON_FATAL() << "ARGS is always set in these tests";
   }
+
+  auto DoExtraCheckReplacements(std::string& check_line) -> void override {
+    // TODO: Disable token output, it's not interesting for these tests.
+    if (llvm::StringRef(check_line).starts_with("// CHECK:STDOUT: {")) {
+      check_line = "// CHECK:STDOUT: {{.*}}";
+    }
+  }
 };
 
 }  // namespace

toolchain/driver/driver_file_test_base.h

@@ -23,17 +23,16 @@ class DriverFileTestBase : public FileTestBase {
  public:
   using FileTestBase::FileTestBase;
 
-  auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
-                    const llvm::SmallVector<TestFile>& test_files,
-                    llvm::raw_pwrite_stream& stdout,
-                    llvm::raw_pwrite_stream& stderr) -> bool override {
+  auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
+           const llvm::SmallVector<TestFile>& test_files,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<bool> override {
     // Create the files in-memory.
     llvm::vfs::InMemoryFileSystem fs;
     for (const auto& test_file : test_files) {
       if (!fs.addFile(test_file.filename, /*ModificationTime=*/0,
                       llvm::MemoryBuffer::getMemBuffer(test_file.content))) {
-        ADD_FAILURE() << "File is repeated: " << test_file.filename;
-        return false;
+        return ErrorBuilder() << "File is repeated: " << test_file.filename;
       }
     }
 

toolchain/driver/testdata/fail_errors_sorted.carbon

@@ -4,9 +4,8 @@
 //
 // ARGS: dump tokens %s
 //
-// TODO: Support autoupdate with ARGS.
-// NOAUTOUPDATE
 // TODO: Disable token output, it's not interesting for these tests.
+// AUTOUPDATE
 // CHECK:STDOUT: [
 // CHECK:STDOUT: {{.*}}
 // CHECK:STDOUT: {{.*}}

toolchain/driver/testdata/fail_errors_streamed.carbon

@@ -4,9 +4,8 @@
 //
 // ARGS: --print-errors=streamed dump tokens %s
 //
-// TODO: Support autoupdate with ARGS.
-// NOAUTOUPDATE
 // TODO: Disable token output, it's not interesting for these tests.
+// AUTOUPDATE
 // CHECK:STDOUT: [
 // CHECK:STDOUT: {{.*}}
 // CHECK:STDOUT: {{.*}}
@@ -30,11 +29,10 @@
 fn run(String program) {
   return True;
 
-var x = 3a;
-
-// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-2]]:10: Invalid digit 'a' in decimal numeric literal.
+// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE+6]]:10: Invalid digit 'a' in decimal numeric literal.
 // CHECK:STDERR: var x = 3a;
 // CHECK:STDERR:          ^
-// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-8]]:24: Closing symbol does not match most recent opening symbol.
+// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-6]]:24: Closing symbol does not match most recent opening symbol.
 // CHECK:STDERR: fn run(String program) {
 // CHECK:STDERR:                        ^
+var x = 3a;

toolchain/lexer/BUILD

@@ -238,6 +238,7 @@ file_test(
     tests = glob(["testdata/**/*.carbon"]),
     deps = [
         "//toolchain/driver:driver_file_test_base",
+        "@com_googlesource_code_re2//:re2",
         "@llvm-project//llvm:Support",
     ],
 )

toolchain/lexer/autoupdate_testdata.py

@@ -1,44 +0,0 @@
-#!/usr/bin/env python3
-
-"""Updates the CHECK: lines in tests with an AUTOUPDATE line."""
-
-__copyright__ = """
-Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-Exceptions. See /LICENSE for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-import subprocess
-import sys
-from pathlib import Path
-
-
-def main() -> None:
-    # Subprocess to the main script in order to avoid Python import behaviors.
-    this_py = Path(__file__).resolve()
-    autoupdate_py = this_py.parent.parent.parent.joinpath(
-        "testing", "scripts", "autoupdate_testdata_base.py"
-    )
-    args = [
-        str(autoupdate_py),
-        # Flags to configure for lexer testing.
-        "--tool=carbon",
-        "--autoupdate_arg=dump",
-        "--autoupdate_arg=tokens",
-        # Ignore the resulting column of EndOfFile because it's typically the
-        # end of the CHECK comment.
-        "--extra_check_replacement",
-        ".*'EndOfFile'",
-        r"column: (?:\d+)",
-        "column: {{[0-9]+}}",
-        # Ignore spaces that are used to columnize lines.
-        "--line_number_delta_prefix={{ *}}",
-        "--line_number_pattern="
-        r"(?P<prefix> line: )(?P<line> *\d+)(?P<suffix>,)",
-        "--testdata=toolchain/lexer/testdata",
-    ] + sys.argv[1:]
-    exit(subprocess.call(args))
-
-
-if __name__ == "__main__":
-    main()

toolchain/lexer/lexer_file_test.cpp

@@ -5,6 +5,7 @@
 #include <string>
 
 #include "llvm/ADT/SmallVector.h"
+#include "re2/re2.h"
 #include "toolchain/driver/driver_file_test_base.h"
 
 namespace Carbon::Testing {
@@ -12,11 +13,30 @@ namespace {
 
 class LexerFileTest : public DriverFileTestBase {
  public:
-  using DriverFileTestBase::DriverFileTestBase;
+  explicit LexerFileTest(std::filesystem::path path)
+      : DriverFileTestBase(std::move(path)),
+        end_of_file_re_((R"((EndOfFile.*column: )( *\d+))")) {}
 
   auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
     return {"dump", "tokens", "%s"};
   }
+
+  auto GetLineNumberReplacement(llvm::ArrayRef<llvm::StringRef> /*filenames*/)
+      -> LineNumberReplacement override {
+    return {.has_file = false,
+            .pattern = R"(line: +(\d+))",
+            // The `{{{{` becomes `{{`.
+            .sub_for_formatv = "line: {{{{ *}}{0}"};
+  }
+
+  auto DoExtraCheckReplacements(std::string& check_line) -> void override {
+    // Ignore the resulting column of EndOfFile because it's often the end of
+    // the CHECK comment.
+    RE2::Replace(&check_line, end_of_file_re_, R"(\1{{ *\\d+}})");
+  }
+
+ private:
+  RE2 end_of_file_re_;
 };
 
 }  // namespace

toolchain/lexer/testdata/basic_syntax.carbon

@@ -18,6 +18,6 @@ fn run(String program) {
   // CHECK:STDOUT: { index:  9, kind:              'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true },
   return True;
 // CHECK:STDOUT: { index: 10, kind:   'CloseCurlyBrace', line: {{ *}}[[@LINE+3]], column:  1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true },
-// CHECK:STDOUT: { index: 11, kind:         'EndOfFile', line: {{ *}}[[@LINE+2]], column:  2, indent: 1, spelling: '' },
+// CHECK:STDOUT: { index: 11, kind:         'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' },
 // CHECK:STDOUT: ]
 }

toolchain/lexer/testdata/keywords.carbon

@@ -130,6 +130,6 @@ where
 while
 
 // CHECK:STDOUT: { index: 61, kind:          'Identifier', line: {{ *}}[[@LINE+3]], column:  1, indent: 1, spelling: 'notakeyword', identifier: 0, has_trailing_space: true },
-// CHECK:STDOUT: { index: 62, kind:           'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: { index: 62, kind:           'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' },
 // CHECK:STDOUT: ]
 notakeyword

toolchain/lexer/testdata/printing_digit_padding.carbon

@@ -12,6 +12,6 @@
         // CHECK:STDOUT: { index: 1, kind:      'Semi', line: {{ *}}[[@LINE+5]], column:  9, indent: 9, spelling: ';' },
         // CHECK:STDOUT: { index: 2, kind:      'Semi', line: {{ *}}[[@LINE+4]], column: 10, indent: 9, spelling: ';' },
         // CHECK:STDOUT: { index: 3, kind:      'Semi', line: {{ *}}[[@LINE+3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true },
-        // CHECK:STDOUT: { index: 4, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 9, spelling: '' },
+        // CHECK:STDOUT: { index: 4, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 9, spelling: '' },
         // CHECK:STDOUT: ]
         ;;;

toolchain/lexer/testdata/printing_integer_literal.carbon

@@ -6,6 +6,6 @@
 // CHECK:STDOUT: [
 
 // CHECK:STDOUT: { index: 0, kind: 'IntegerLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '123', value: `123`, has_trailing_space: true },
-// CHECK:STDOUT: { index: 1, kind:      'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: { index: 1, kind:      'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' },
 // CHECK:STDOUT: ]
 123

toolchain/lexer/testdata/printing_real_literal.carbon

@@ -6,6 +6,6 @@
 // CHECK:STDOUT: [
 
 // CHECK:STDOUT: { index: 0, kind: 'RealLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true },
-// CHECK:STDOUT: { index: 1, kind:   'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: { index: 1, kind:   'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' },
 // CHECK:STDOUT: ]
 2.5

toolchain/lexer/testdata/printing_token.carbon

@@ -6,6 +6,6 @@
 // CHECK:STDOUT: [
 
 // CHECK:STDOUT: { index: 0, kind: 'IntegerLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_trailing_space: true },
-// CHECK:STDOUT: { index: 1, kind:      'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: { index: 1, kind:      'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' },
 // CHECK:STDOUT: ]
 0x9

toolchain/lowering/autoupdate_testdata.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-
-"""Updates the CHECK: lines in tests with an AUTOUPDATE line."""
-
-__copyright__ = """
-Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-Exceptions. See /LICENSE for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-import subprocess
-import sys
-from pathlib import Path
-
-
-def main() -> None:
-    # Subprocess to the main script in order to avoid Python import behaviors.
-    this_py = Path(__file__).resolve()
-    autoupdate_py = this_py.parent.parent.parent.joinpath(
-        "testing", "scripts", "autoupdate_testdata_base.py"
-    )
-    args = [
-        str(autoupdate_py),
-        # Flags to configure for lowering testing.
-        "--tool=carbon",
-        "--autoupdate_arg=dump",
-        "--autoupdate_arg=llvm-ir",
-        "--testdata=toolchain/lowering/testdata",
-    ] + sys.argv[1:]
-    exit(subprocess.call(args))
-
-
-if __name__ == "__main__":
-    main()

toolchain/parser/autoupdate_testdata.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-
-"""Updates the CHECK: lines in tests with an AUTOUPDATE line."""
-
-__copyright__ = """
-Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-Exceptions. See /LICENSE for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-import subprocess
-import sys
-from pathlib import Path
-
-
-def main() -> None:
-    # Subprocess to the main script in order to avoid Python import behaviors.
-    this_py = Path(__file__).resolve()
-    autoupdate_py = this_py.parent.parent.parent.joinpath(
-        "testing", "scripts", "autoupdate_testdata_base.py"
-    )
-    args = [
-        str(autoupdate_py),
-        # Flags to configure for parser testing.
-        "--tool=carbon",
-        "--autoupdate_arg=dump",
-        "--autoupdate_arg=parse-tree",
-        "--testdata=toolchain/parser/testdata",
-    ] + sys.argv[1:]
-    exit(subprocess.call(args))
-
-
-if __name__ == "__main__":
-    main()

toolchain/semantics/autoupdate_testdata.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-
-"""Updates the CHECK: lines in tests with an AUTOUPDATE line."""
-
-__copyright__ = """
-Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-Exceptions. See /LICENSE for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-import subprocess
-import sys
-from pathlib import Path
-
-
-def main() -> None:
-    # Subprocess to the main script in order to avoid Python import behaviors.
-    this_py = Path(__file__).resolve()
-    autoupdate_py = this_py.parent.parent.parent.joinpath(
-        "testing", "scripts", "autoupdate_testdata_base.py"
-    )
-    args = [
-        str(autoupdate_py),
-        # Flags to configure for explorer testing.
-        "--tool=carbon",
-        "--autoupdate_arg=dump",
-        "--autoupdate_arg=semantics-ir",
-        "--testdata=toolchain/semantics/testdata",
-    ] + sys.argv[1:]
-    exit(subprocess.call(args))
-
-
-if __name__ == "__main__":
-    main()

toolchain/semantics/testdata/basics/builtin_nodes.carbon

@@ -4,8 +4,7 @@
 //
 // ARGS: dump semantics-ir --include_builtins %s
 //
-// TODO: Support autoupdate with ARGS
-// NOAUTOUPDATE
+// AUTOUPDATE
 // CHECK:STDOUT: cross_reference_irs_size: 1
 // CHECK:STDOUT: functions: [
 // CHECK:STDOUT: ]