Add file_test support for specifying arguments and only checking a subset of output. (#3018)

This is sufficient to convert remaining toolchain tests to file_test. %s
and %t are currently used. While #2978 might use %T with lit, I think
that's resolving a lit-specific issue that's not necessary when
TEST_TMPDIR is readily available for any necessary operations (i.e.,
setting the working directory). With ARGS, I think it's feasible to
switch to file_test without %T.

Also adds documentation to file_test_base.h, which was starting to feel
like a significant gap.

This is not yet handled by autoupdate, but I'm eyeing that next.

explorer/file_test.cpp

@@ -42,9 +42,13 @@ class ParseAndExecuteTestFile : public FileTestBase {
     }
   }
 
-  auto RunWithFiles(const llvm::SmallVector<TestFile>& test_files,
+  auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
+                    const llvm::SmallVector<TestFile>& test_files,
                     llvm::raw_pwrite_stream& stdout,
                     llvm::raw_pwrite_stream& stderr) -> bool override {
+    CARBON_CHECK(test_args.empty())
+        << "ARGS are not currently used in explorer's file_test.";
+
     if (test_files.size() != 1) {
       ADD_FAILURE() << "Only 1 file is supported: " << test_files.size()
                     << " provided";
@@ -88,6 +92,10 @@ class ParseAndExecuteTestFile : public FileTestBase {
     return result.ok();
   }
 
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {};
+  }
+
  private:
   bool trace_;
   bool is_trace_test = false;

testing/file_test/file_test_base.cpp

@@ -8,7 +8,9 @@
 #include <fstream>
 
 #include "common/check.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/InitLLVM.h"
 #include "testing/util/test_raw_ostream.h"
 
@@ -20,6 +22,9 @@ static int base_dir_len = 0;
 static std::string* subset_target = nullptr;
 
 using ::testing::Eq;
+using ::testing::Matcher;
+using ::testing::MatchesRegex;
+using ::testing::StrEq;
 
 void FileTestBase::RegisterTests(
     const char* fixture_label,
@@ -70,18 +75,37 @@ auto FileTestBase::TestBody() -> void {
   std::string test_content = ReadFile(path());
 
   // Load expected output.
+  llvm::SmallVector<std::string> test_args;
   llvm::SmallVector<TestFile> test_files;
-  llvm::SmallVector<testing::Matcher<std::string>> expected_stdout;
-  llvm::SmallVector<testing::Matcher<std::string>> expected_stderr;
-  ProcessTestFile(test_content, test_files, expected_stdout, expected_stderr);
+  llvm::SmallVector<Matcher<std::string>> expected_stdout;
+  llvm::SmallVector<Matcher<std::string>> expected_stderr;
+  bool check_subset = false;
+  ProcessTestFile(test_content, test_args, test_files, expected_stdout,
+                  expected_stderr, check_subset);
   if (HasFailure()) {
     return;
   }
 
+  // Process arguments.
+  if (test_args.empty()) {
+    test_args = GetDefaultArgs();
+  }
+  DoArgReplacements(test_args, test_files);
+  if (HasFailure()) {
+    return;
+  }
+
+  // Pass arguments as StringRef.
+  llvm::SmallVector<llvm::StringRef> test_args_ref;
+  test_args_ref.reserve(test_args.size());
+  for (const auto& arg : test_args) {
+    test_args_ref.push_back(arg);
+  }
+
   // Capture trace streaming, but only when in debug mode.
   TestRawOstream stdout;
   TestRawOstream stderr;
-  bool run_succeeded = RunWithFiles(test_files, stdout, stderr);
+  bool run_succeeded = RunWithFiles(test_args_ref, test_files, stdout, stderr);
   if (HasFailure()) {
     return;
   }
@@ -91,14 +115,63 @@ auto FileTestBase::TestBody() -> void {
          "is expected to fail.";
 
   // Check results.
-  EXPECT_THAT(SplitOutput(stdout.TakeStr()), ElementsAreArray(expected_stdout));
-  EXPECT_THAT(SplitOutput(stderr.TakeStr()), ElementsAreArray(expected_stderr));
+  if (check_subset) {
+    EXPECT_THAT(SplitOutput(stdout.TakeStr()), IsSupersetOf(expected_stdout));
+    EXPECT_THAT(SplitOutput(stderr.TakeStr()), IsSupersetOf(expected_stderr));
+
+  } else {
+    EXPECT_THAT(SplitOutput(stdout.TakeStr()),
+                ElementsAreArray(expected_stdout));
+    EXPECT_THAT(SplitOutput(stderr.TakeStr()),
+                ElementsAreArray(expected_stderr));
+  }
+}
+
+auto FileTestBase::DoArgReplacements(
+    llvm::SmallVector<std::string>& test_args,
+    const llvm::SmallVector<TestFile>& test_files) -> void {
+  for (auto* it = test_args.begin(); it != test_args.end(); ++it) {
+    auto percent = it->find("%");
+    if (percent == std::string::npos) {
+      continue;
+    }
+
+    if (percent + 1 >= it->size()) {
+      FAIL() << "% is not allowed on its own: " << *it;
+    }
+    char c = (*it)[percent + 1];
+    switch (c) {
+      case 's': {
+        if (*it != "%s") {
+          FAIL() << "%s must be the full argument: " << *it;
+        }
+        it = test_args.erase(it);
+        for (const auto& file : test_files) {
+          it = test_args.insert(it, file.filename);
+          ++it;
+        }
+        // Back up once because the for loop will advance.
+        --it;
+        break;
+      }
+      case 't': {
+        char* temp = getenv("TEST_TMPDIR");
+        CARBON_CHECK(temp != nullptr);
+        it->replace(percent, 2, llvm::formatv("{0}/temp_file", temp));
+        break;
+      }
+      default:
+        FAIL() << "%" << c << " is not supported: " << *it;
+    }
+  }
 }
 
 auto FileTestBase::ProcessTestFile(
-    llvm::StringRef file_content, llvm::SmallVector<TestFile>& test_files,
-    llvm::SmallVector<testing::Matcher<std::string>>& expected_stdout,
-    llvm::SmallVector<testing::Matcher<std::string>>& expected_stderr) -> void {
+    llvm::StringRef file_content, llvm::SmallVector<std::string>& test_args,
+    llvm::SmallVector<TestFile>& test_files,
+    llvm::SmallVector<Matcher<std::string>>& expected_stdout,
+    llvm::SmallVector<Matcher<std::string>>& expected_stderr,
+    bool& check_subset) -> void {
   llvm::StringRef cursor = file_content;
   bool found_content_pre_split = false;
   int line_index = 0;
@@ -135,15 +208,34 @@ auto FileTestBase::ProcessTestFile(
 
     // Process expectations when found.
     auto line_trimmed = line.ltrim();
-    if (!line_trimmed.consume_front("// CHECK")) {
-      continue;
-    }
-    if (line_trimmed.consume_front(":STDOUT:")) {
-      expected_stdout.push_back(TransformExpectation(line_index, line_trimmed));
-    } else if (line_trimmed.consume_front(":STDERR:")) {
-      expected_stderr.push_back(TransformExpectation(line_index, line_trimmed));
-    } else {
-      FAIL() << "Unexpected CHECK in input: " << line.str();
+    if (line_trimmed.consume_front("// ARGS: ")) {
+      if (test_args.empty()) {
+        // Split the line into arguments.
+        std::pair<llvm::StringRef, llvm::StringRef> cursor =
+            llvm::getToken(line_trimmed);
+        while (!cursor.first.empty()) {
+          test_args.push_back(std::string(cursor.first));
+          cursor = llvm::getToken(cursor.second);
+        }
+      } else {
+        FAIL() << "ARGS was specified multiple times: " << line.str();
+      }
+    } else if (line_trimmed == "// SET-CHECK-SUBSET") {
+      if (!check_subset) {
+        check_subset = true;
+      } else {
+        FAIL() << "SET-CHECK-SUBSET was specified multiple times";
+      }
+    } else if (line_trimmed.consume_front("// CHECK")) {
+      if (line_trimmed.consume_front(":STDOUT:")) {
+        expected_stdout.push_back(
+            TransformExpectation(line_index, line_trimmed));
+      } else if (line_trimmed.consume_front(":STDERR:")) {
+        expected_stderr.push_back(
+            TransformExpectation(line_index, line_trimmed));
+      } else {
+        FAIL() << "Unexpected CHECK in input: " << line.str();
+      }
     }
   }
 
@@ -159,17 +251,17 @@ auto FileTestBase::ProcessTestFile(
 
   // Assume there is always a suffix `\n` in output.
   if (!expected_stdout.empty()) {
-    expected_stdout.push_back(testing::StrEq(""));
+    expected_stdout.push_back(StrEq(""));
   }
   if (!expected_stderr.empty()) {
-    expected_stderr.push_back(testing::StrEq(""));
+    expected_stderr.push_back(StrEq(""));
   }
 }
 
 auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in)
-    -> testing::Matcher<std::string> {
+    -> Matcher<std::string> {
   if (in.empty()) {
-    return testing::StrEq("");
+    return StrEq("");
   }
   CARBON_CHECK(in[0] == ' ') << "Malformated input: " << in;
   std::string str = in.substr(1).str();
@@ -245,7 +337,7 @@ auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in)
     }
   }
 
-  return testing::MatchesRegex(str);
+  return MatchesRegex(str);
 }
 
 }  // namespace Carbon::Testing

testing/file_test/file_test_base.h

@@ -25,14 +25,57 @@ namespace Carbon::Testing {
 // individual test executions. This framework includes a `main` implementation,
 // so users must not provide one.
 //
-// Tests should have CHECK lines similar to `FileCheck` syntax:
-//   https://llvm.org/docs/CommandGuide/FileCheck.html
+// Settings in files are provided in comments, similar to `FileCheck` syntax.
+// `autoupdate_testdata.py` automatically constructs compatible CHECK:STDOUT:
+// and CHECK:STDERR: lines.
 //
-// Special nuances are that stdout and stderr will look like `// CHECK:STDOUT:
-// ...` and `// CHECK:STDERR: ...` respectively. `[[@LINE+offset]` and
-// `{{regex}}` syntaxes should also work.
+// Supported comment markers are:
 //
-// `autoupdate_testdata.py` automatically constructs compatible lines.
+// - // ARGS: <arguments>
+//
+//   Provides a space-separated list of arguments, which will be passed to
+//   RunWithFiles as test_args. These are intended for use by the command as
+//   arguments.
+//
+//   Supported replacements within arguments are:
+//
+//   - %s
+//
+//     Replaced with the list of files. Currently only allowed as a standalone
+//     argument, not a substring.
+//
+//   - %t
+//
+//     Replaced with `${TEST_TMPDIR}/temp_file`.
+//
+//   ARGS can be specified at most once. If not provided, the FileTestBase child
+//   is responsible for providing default arguments.
+//
+// - // SET-CHECK-SUBSET
+//
+//   By default, all lines of output must have a CHECK match. Adding this as a
+//   flag sets it so that non-matching lines are ignored. All provided
+//   CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output.
+//
+//   SET-CHECK-SUBSET can be specified at most once.
+//
+// - // --- <filename>
+//
+//   By default, all file content is provided to the test as a single file in
+//   test_files. Using this marker allows the file to be split into multiple
+//   files which will all be passed to test_files.
+//
+//   Files are not created on disk; it's expected the child will create an
+//   InMemoryFilesystem if needed.
+//
+// - // CHECK:STDOUT: <output line>
+//   // CHECK:STDERR: <output line>
+//
+//   These provides a match for output from the command. See SET-CHECK-SUBSET
+//   for how to change from full to subset matching of output.
+//
+//   Output line matchers may contain `[[@LINE+offset]` and
+//   `{{regex}}` syntaxes, similar to `FileCheck`.
 class FileTestBase : public testing::Test {
  public:
   struct TestFile {
@@ -72,10 +115,14 @@ class FileTestBase : public testing::Test {
   // Implemented by children to run the test. Called by the TestBody
   // implementation, which will validate stdout and stderr. The return value
   // should be false when "fail_" is in the filename.
-  virtual auto RunWithFiles(const llvm::SmallVector<TestFile>& test_files,
+  virtual auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
+                            const llvm::SmallVector<TestFile>& test_files,
                             llvm::raw_pwrite_stream& stdout,
                             llvm::raw_pwrite_stream& stderr) -> bool = 0;
 
+  // Returns default arguments. Only called when a file doesn't set ARGS.
+  virtual auto GetDefaultArgs() -> llvm::SmallVector<std::string> = 0;
+
   // Runs a test and compares output. This keeps output split by line so that
   // issues are a little easier to identify by the different line.
   auto TestBody() -> void final;
@@ -84,12 +131,17 @@ class FileTestBase : public testing::Test {
   auto path() -> const std::filesystem::path& { return *path_; };
 
  private:
+  // Does replacements in ARGS for %s and %t.
+  auto DoArgReplacements(llvm::SmallVector<std::string>& test_args,
+                         const llvm::SmallVector<TestFile>& test_files) -> void;
+
   // Processes the test input, producing test files and expected output.
   auto ProcessTestFile(
-      llvm::StringRef file_content, llvm::SmallVector<TestFile>& test_files,
+      llvm::StringRef file_content, llvm::SmallVector<std::string>& test_args,
+      llvm::SmallVector<TestFile>& test_files,
       llvm::SmallVector<testing::Matcher<std::string>>& expected_stdout,
-      llvm::SmallVector<testing::Matcher<std::string>>& expected_stderr)
-      -> void;
+      llvm::SmallVector<testing::Matcher<std::string>>& expected_stderr,
+      bool& check_subset) -> void;
 
   // Transforms an expectation on a given line from `FileCheck` syntax into a
   // standard regex matcher.

testing/file_test/file_test_base_test.cpp

@@ -10,7 +10,7 @@
 #include <fstream>
 #include <vector>
 
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace Carbon::Testing {
@@ -20,30 +20,44 @@ using ::testing::AllOf;
 using ::testing::ElementsAre;
 using ::testing::Eq;
 using ::testing::Field;
+using ::testing::Matcher;
 
 class FileTestBaseTest : public FileTestBase {
  public:
   explicit FileTestBaseTest(const std::filesystem::path& path)
       : FileTestBase(path) {}
 
-  static auto HasFilename(std::string filename) -> testing::Matcher<TestFile> {
+  static auto HasFilename(std::string filename) -> Matcher<TestFile> {
     return Field("filename", &TestFile::filename, Eq(filename));
   }
 
-  static auto HasContent(std::string content) -> testing::Matcher<TestFile> {
+  static auto HasContent(std::string content) -> Matcher<TestFile> {
     return Field("content", &TestFile::content, Eq(content));
   }
 
-  auto RunWithFiles(const llvm::SmallVector<TestFile>& test_files,
+  auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
+                    const llvm::SmallVector<TestFile>& test_files,
                     llvm::raw_pwrite_stream& stdout,
                     llvm::raw_pwrite_stream& stderr) -> bool override {
+    if (!test_args.empty()) {
+      llvm::ListSeparator sep;
+      stdout << test_args.size() << " args: ";
+      for (const auto& arg : test_args) {
+        stdout << sep << "`" << arg << "`";
+      }
+      stdout << "\n";
+    }
+
     auto filename = path().filename();
-    if (filename == "example.carbon") {
+    if (filename == "args.carbon") {
+      EXPECT_THAT(test_files, ElementsAre(HasFilename("args.carbon")));
+      return true;
+    } else if (filename == "example.carbon") {
       EXPECT_THAT(test_files, ElementsAre(HasFilename("example.carbon")));
       stdout << "something\n"
                 "\n"
-                "8: Line delta\n"
-                "7: Negative line delta\n"
+                "9: Line delta\n"
+                "8: Negative line delta\n"
                 "+*[]{}\n"
                 "Foo baz\n";
       return true;
@@ -70,6 +84,10 @@ class FileTestBaseTest : public FileTestBase {
       return false;
     }
   }
+
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {"default_args", "%s"};
+  }
 };
 
 }  // namespace

testing/file_test/testdata/args.carbon

@@ -0,0 +1,6 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// ARGS: abc file=%t %s
+// CHECK:STDOUT: 3 args: `abc`, `file={{.+}}/temp_file`, `args.carbon`

testing/file_test/testdata/example.carbon

@@ -2,6 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+// CHECK:STDOUT: 2 args: `default_args`, `example.carbon`
 // CHECK:STDOUT: something
 // CHECK:STDOUT:
 // CHECK:STDOUT: [[@LINE+1]]: Line delta

testing/file_test/testdata/fail_example.carbon

@@ -2,4 +2,5 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+// CHECK:STDOUT: 2 args: `default_args`, `fail_example.carbon`
 // CHECK:STDERR: Oops

testing/file_test/testdata/two_files.carbon

@@ -2,6 +2,8 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+// CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon`
+
 // --- a.carbon
 // CHECK:STDOUT: a.carbon: [[@LINE+0]]
 

toolchain/codegen/BUILD

@@ -2,6 +2,8 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+load("//testing/file_test:rules.bzl", "file_test")
+
 package(default_visibility = ["//visibility:public"])
 
 cc_library(
@@ -18,3 +20,13 @@ cc_library(
         "@llvm-project//llvm:TargetParser",
     ],
 )
+
+file_test(
+    name = "codegen_file_test",
+    srcs = ["codegen_file_test.cpp"],
+    tests = glob(["testdata/**/*.carbon"]),
+    deps = [
+        "//toolchain/driver:driver_file_test_base",
+        "@llvm-project//llvm:Support",
+    ],
+)

toolchain/codegen/codegen_file_test.cpp

@@ -0,0 +1,31 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <filesystem>
+#include <string>
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "toolchain/driver/driver_file_test_base.h"
+
+namespace Carbon::Testing {
+namespace {
+
+class CodeGenFileTest : public DriverFileTestBase {
+ public:
+  using DriverFileTestBase::DriverFileTestBase;
+
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    CARBON_FATAL() << "ARGS is always set in these tests";
+  }
+};
+
+}  // namespace
+
+auto RegisterFileTests(const llvm::SmallVector<std::filesystem::path>& paths)
+    -> void {
+  CodeGenFileTest::RegisterTests<CodeGenFileTest>("CodeGenFileTest", paths);
+}
+
+}  // namespace Carbon::Testing

toolchain/driver/testdata/codegen_assembly.carbon → toolchain/codegen/testdata/assembly/basic.carbon

@@ -2,7 +2,9 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{carbon} dump assembly --target_triple=x86_64-unknown-linux-gnu %s | %{FileCheck-allow-unmatched}
+// ARGS: dump assembly --target_triple=x86_64-unknown-linux-gnu %s
+// NOAUTOUPDATE
+// SET-CHECK-SUBSET
 // CHECK:STDOUT: Main:
 
 fn Main() -> i32 { return 0; }

toolchain/driver/testdata/error_codegen_objcode_target_triple.carbon → toolchain/codegen/testdata/assembly/fail_target_triple.carbon

@@ -2,7 +2,9 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{not} %{carbon} dump objcode --target_triple=x86_684-unknown-linux-gnu --output_file=%t %s | %{FileCheck-strict}
+// ARGS: dump assembly --target_triple=x86_687-unknown-linux-gnu %s
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
 // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}}
 
 fn Main() -> i32 { return 0; }

toolchain/driver/testdata/success_codegen_objfile.carbon → toolchain/codegen/testdata/objcode/basic.carbon

@@ -2,7 +2,9 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{carbon} dump objcode --target_triple=x86_64-unknown-linux-gnu --output_file=%t %s | %{FileCheck-strict}
+// ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu --output_file=%t %s
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
 // CHECK:STDOUT: Success: Object file is generated!
 
 fn Main() -> i32 { return 0; }

toolchain/driver/testdata/error_codegen_objcode_no_input_file.carbon → toolchain/codegen/testdata/objcode/fail_no_input_file.carbon

@@ -2,7 +2,9 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{not} %{carbon} dump objcode --output_file=%t --target_triple=x86_64-unknown-linux-gnu | %{FileCheck-strict}
+// ARGS: dump objcode --output_file=%t --target_triple=x86_64-unknown-linux-gnu
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
 // CHECK:STDERR: ERROR: No input file specified.
 
 fn Main() -> i32 { return 0; }

toolchain/driver/testdata/error_codegen_objcode_no_output_file.carbon → toolchain/codegen/testdata/objcode/fail_no_output_file.carbon

@@ -2,7 +2,9 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{not} %{carbon} dump objcode --target_triple=x86_64-unknown-linux-gnu %s | %{FileCheck-strict}
+// ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu %s
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
 // CHECK:STDERR: ERROR: Must provide an output file.
 
 fn Main() -> i32 { return 0; }

toolchain/driver/testdata/error_codegen_assembly_target_triple.carbon → toolchain/codegen/testdata/objcode/fail_target_triple.carbon

@@ -2,7 +2,9 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{not} %{carbon} dump assembly --target_triple=x86_687-unknown-linux-gnu %s | %{FileCheck-strict}
+// ARGS: dump objcode --target_triple=x86_684-unknown-linux-gnu --output_file=%t %s
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
 // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}}
 
 fn Main() -> i32 { return 0; }

toolchain/driver/BUILD

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 load("//bazel/cc_toolchains:defs.bzl", "cc_env")
+load("//testing/file_test:rules.bzl", "file_test")
 load("//testing/fuzzing:rules.bzl", "cc_fuzz_test")
 
 package(default_visibility = ["//visibility:public"])
@@ -27,6 +28,16 @@ cc_library(
     ],
 )
 
+file_test(
+    name = "driver_file_test",
+    srcs = ["driver_file_test.cpp"],
+    tests = glob(["testdata/**/*.carbon"]),
+    deps = [
+        ":driver_file_test_base",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_test(
     name = "driver_test",
     size = "small",

toolchain/driver/driver_file_test.cpp

@@ -0,0 +1,30 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <filesystem>
+#include <string>
+
+#include "llvm/ADT/SmallVector.h"
+#include "toolchain/driver/driver_file_test_base.h"
+
+namespace Carbon::Testing {
+namespace {
+
+class DriverFileTest : public DriverFileTestBase {
+ public:
+  using DriverFileTestBase::DriverFileTestBase;
+
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    CARBON_FATAL() << "ARGS is always set in these tests";
+  }
+};
+
+}  // namespace
+
+auto RegisterFileTests(const llvm::SmallVector<std::filesystem::path>& paths)
+    -> void {
+  DriverFileTest::RegisterTests<DriverFileTest>("DriverFileTest", paths);
+}
+
+}  // namespace Carbon::Testing

toolchain/driver/driver_file_test_base.h

@@ -24,16 +24,13 @@ class DriverFileTestBase : public FileTestBase {
  public:
   using FileTestBase::FileTestBase;
 
-  auto RunWithFiles(const llvm::SmallVector<TestFile>& test_files,
+  auto RunWithFiles(const llvm::SmallVector<llvm::StringRef>& test_args,
+                    const llvm::SmallVector<TestFile>& test_files,
                     llvm::raw_pwrite_stream& stdout,
                     llvm::raw_pwrite_stream& stderr) -> bool override {
-    // Prepare a list of filenames for MakeArgs. Also create the files
-    // in-memory.
-    llvm::SmallVector<llvm::StringRef> test_file_names;
+    // Create the files in-memory.
     llvm::vfs::InMemoryFileSystem fs;
     for (const auto& test_file : test_files) {
-      test_file_names.push_back(test_file.filename);
-
       if (!fs.addFile(test_file.filename, /*ModificationTime=*/0,
                       llvm::MemoryBuffer::getMemBuffer(test_file.content))) {
         ADD_FAILURE() << "File is repeated: " << test_file.filename;
@@ -42,11 +39,8 @@ class DriverFileTestBase : public FileTestBase {
     }
 
     Driver driver(fs, stdout, stderr);
-    return driver.RunFullCommand(MakeArgs(test_file_names));
+    return driver.RunFullCommand(test_args);
   }
-
-  virtual auto MakeArgs(const llvm::SmallVector<llvm::StringRef>& test_files)
-      -> llvm::SmallVector<llvm::StringRef> = 0;
 };
 
 }  // namespace Carbon::Testing

toolchain/driver/testdata/BUILD

@@ -1,17 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-load("//testing/lit_test:rules.bzl", "glob_lit_tests")
-
-glob_lit_tests(
-    name = "all_lit_tests",
-    data = [
-        "//testing/lit_test:merge_output",
-        "//toolchain/driver:carbon",
-        "@llvm-project//llvm:FileCheck",
-        "@llvm-project//llvm:not",
-    ],
-    driver = "lit.cfg.py",
-    test_file_exts = ["carbon"],
-)

toolchain/driver/testdata/errors_sorted_test.carbon

@@ -1,19 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// RUN: %{not} %{carbon} dump tokens %s |  %{FileCheck-strict}
-// CHECK:STDOUT: [
-// CHECK-COUNT-17:STDOUT: {{.*}}
-// CHECK:STDOUT: ]
-
-// CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+3]]:24: Closing symbol does not match most recent opening symbol.
-// CHECK:STDERR: fn run(String program) {
-// CHECK:STDERR:                        ^
-fn run(String program) {
-  return True;
-
-// CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+3]]:10: Invalid digit 'a' in decimal numeric literal.
-// CHECK:STDERR: var x = 3a;
-// CHECK:STDERR:          ^
-var x = 3a;

toolchain/driver/testdata/errors_streamed_test.carbon

@@ -1,21 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// RUN: %{not} %{carbon} --print-errors=streamed dump tokens %s | \
-// RUN:   %{FileCheck-strict}
-// CHECK:STDOUT: [
-// CHECK-COUNT-17:STDOUT: {{.*}}
-// CHECK:STDOUT: ]
-
-fn run(String program) {
-  return True;
-
-var x = 3a;
-
-// CHECK:STDERR: {{.*}}/errors_streamed_test.carbon:[[@LINE-2]]:10: Invalid digit 'a' in decimal numeric literal.
-// CHECK:STDERR: var x = 3a;
-// CHECK:STDERR:          ^
-// CHECK:STDERR: {{.*}}/errors_streamed_test.carbon:[[@LINE-8]]:24: Closing symbol does not match most recent opening symbol.
-// CHECK:STDERR: fn run(String program) {
-// CHECK:STDERR:                        ^

toolchain/driver/testdata/fail_errors_sorted.carbon

@@ -0,0 +1,39 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: dump tokens %s
+//
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
+// TODO: Disable token output, it's not interesting for these tests.
+// CHECK:STDOUT: [
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: ]
+
+// CHECK:STDERR: fail_errors_sorted.carbon:[[@LINE+3]]:24: Closing symbol does not match most recent opening symbol.
+// CHECK:STDERR: fn run(String program) {
+// CHECK:STDERR:                        ^
+fn run(String program) {
+  return True;
+
+// CHECK:STDERR: fail_errors_sorted.carbon:[[@LINE+3]]:10: Invalid digit 'a' in decimal numeric literal.
+// CHECK:STDERR: var x = 3a;
+// CHECK:STDERR:          ^
+var x = 3a;

toolchain/driver/testdata/fail_errors_streamed.carbon

@@ -0,0 +1,40 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --print-errors=streamed dump tokens %s
+//
+// TODO: Support autoupdate with ARGS.
+// NOAUTOUPDATE
+// TODO: Disable token output, it's not interesting for these tests.
+// CHECK:STDOUT: [
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: {{.*}}
+// CHECK:STDOUT: ]
+
+fn run(String program) {
+  return True;
+
+var x = 3a;
+
+// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-2]]:10: Invalid digit 'a' in decimal numeric literal.
+// CHECK:STDERR: var x = 3a;
+// CHECK:STDERR:          ^
+// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-8]]:24: Closing symbol does not match most recent opening symbol.
+// CHECK:STDERR: fn run(String program) {
+// CHECK:STDERR:                        ^

toolchain/driver/testdata/lit.cfg.py

@@ -1 +0,0 @@
-../../../testing/lit_test/lit.cfg.py

toolchain/lexer/lexer_file_test.cpp

@@ -16,11 +16,8 @@ class LexerFileTest : public DriverFileTestBase {
  public:
   using DriverFileTestBase::DriverFileTestBase;
 
-  auto MakeArgs(const llvm::SmallVector<llvm::StringRef>& test_files)
-      -> llvm::SmallVector<llvm::StringRef> override {
-    llvm::SmallVector<llvm::StringRef> args({"dump", "tokens"});
-    args.insert(args.end(), test_files.begin(), test_files.end());
-    return args;
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {"dump", "tokens", "%s"};
   }
 };
 

toolchain/lowering/lowering_file_test.cpp

@@ -16,11 +16,8 @@ class LoweringFileTest : public DriverFileTestBase {
  public:
   using DriverFileTestBase::DriverFileTestBase;
 
-  auto MakeArgs(const llvm::SmallVector<llvm::StringRef>& test_files)
-      -> llvm::SmallVector<llvm::StringRef> override {
-    llvm::SmallVector<llvm::StringRef> args({"dump", "llvm-ir"});
-    args.insert(args.end(), test_files.begin(), test_files.end());
-    return args;
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {"dump", "llvm-ir", "%s"};
   }
 };
 

toolchain/parser/parse_tree_file_test.cpp

@@ -16,11 +16,8 @@ class ParseTreeFileTest : public DriverFileTestBase {
  public:
   using DriverFileTestBase::DriverFileTestBase;
 
-  auto MakeArgs(const llvm::SmallVector<llvm::StringRef>& test_files)
-      -> llvm::SmallVector<llvm::StringRef> override {
-    llvm::SmallVector<llvm::StringRef> args({"dump", "parse-tree"});
-    args.insert(args.end(), test_files.begin(), test_files.end());
-    return args;
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {"dump", "parse-tree", "%s"};
   }
 };
 

toolchain/semantics/semantics_file_test.cpp

@@ -16,11 +16,8 @@ class SemanticsFileTest : public DriverFileTestBase {
  public:
   using DriverFileTestBase::DriverFileTestBase;
 
-  auto MakeArgs(const llvm::SmallVector<llvm::StringRef>& test_files)
-      -> llvm::SmallVector<llvm::StringRef> override {
-    llvm::SmallVector<llvm::StringRef> args({"dump", "semantics-ir"});
-    args.insert(args.end(), test_files.begin(), test_files.end());
-    return args;
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    return {"dump", "semantics-ir", "%s"};
   }
 };
 

toolchain/driver/testdata/semantics_builtin_nodes.carbon → toolchain/semantics/testdata/basics/builtin_nodes.carbon

@@ -2,8 +2,10 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{carbon} dump semantics-ir --include_builtins %s | \
-// RUN:   %{FileCheck-strict}
+// ARGS: dump semantics-ir --include_builtins %s
+//
+// TODO: Support autoupdate with ARGS
+// NOAUTOUPDATE
 // CHECK:STDOUT: cross_reference_irs_size: 1
 // CHECK:STDOUT: functions: [
 // CHECK:STDOUT: ]

toolchain/driver/testdata/semantics_verbose.carbon → toolchain/semantics/testdata/basics/verbose.carbon

@@ -2,9 +2,11 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{carbon} -v dump semantics-ir %s | %{FileCheck-allow-unmatched}
+// ARGS: -v dump semantics-ir %s
 //
 // Only checks a couple statements in order to minimize manual update churn.
+// NOAUTOUPDATE
+// SET-CHECK-SUBSET
 // CHECK:STDERR: Node Push 0: FunctionIntroducer -> <none>
 // CHECK:STDERR: AddNode block{{[0-9]+}}: {kind: FunctionDeclaration, arg0: function{{[0-9]+}}}