Augment the file_test framework to allow per-file fail checks. (#3747)

This handles toolchain failures per-file. The intent is to allow placing
both "success" and "fail" tests in the same file, using splits. However,
this PR only adds support and updates existing tests to continue
passing.

explorer/file_test.cpp

@@ -28,7 +28,7 @@ class ExplorerFileTest : public FileTestBase {
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
            llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> override {
+           llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
     // Add the prelude.
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
         llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon");
@@ -53,7 +53,7 @@ class ExplorerFileTest : public FileTestBase {
         args.size(), args.data(), /*install_path=*/"", PreludePath, stdout,
         stderr, check_trace_output() ? stdout : trace_stream_, fs);
 
-    return exit_code == EXIT_SUCCESS;
+    return {{.success = exit_code == EXIT_SUCCESS}};
   }
 
   auto ValidateRun() -> void override {

testing/file_test/file_test_base.cpp

@@ -15,6 +15,7 @@
 #include "common/check.h"
 #include "common/error.h"
 #include "common/init_llvm.h"
+#include "gmock/gmock.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -41,7 +42,6 @@ ABSL_FLAG(bool, dump_output, false,
 
 namespace Carbon::Testing {
 
-using ::testing::Eq;
 using ::testing::Matcher;
 using ::testing::MatchesRegex;
 using ::testing::StrEq;
@@ -66,6 +66,20 @@ static auto SplitOutput(llvm::StringRef output)
   return llvm::SmallVector<std::string_view>(lines.begin(), lines.end());
 }
 
+// Verify that the success and `fail_` prefix use correspond. Separately handle
+// both cases for clearer test failures.
+static auto CompareFailPrefix(llvm::StringRef filename, bool success) -> void {
+  if (success) {
+    EXPECT_FALSE(filename.starts_with("fail_"))
+        << "`" << filename
+        << "` succeeded; if this is correct, add a `fail_` prefix.";
+  } else {
+    EXPECT_TRUE(filename.starts_with("fail_"))
+        << "`" << filename
+        << "` failed; if this is correct, remove the `fail_` prefix.";
+  }
+}
+
 // Runs a test and compares output. This keeps output split by line so that
 // issues are a little easier to identify by the different line.
 auto FileTestBase::TestBody() -> void {
@@ -95,10 +109,30 @@ auto FileTestBase::TestBody() -> void {
   ASSERT_TRUE(run_result.ok()) << run_result.error();
   ValidateRun();
   auto test_filename = std::filesystem::path(test_name_.str()).filename();
-  EXPECT_THAT(!llvm::StringRef(test_filename).starts_with("fail_"),
-              Eq(context.exit_with_success))
-      << "Tests should be prefixed with `fail_` if and only if running them "
-         "is expected to fail.";
+
+  // Check success/failure against `fail_` prefixes.
+  if (context.run_result.per_file_success.empty()) {
+    CompareFailPrefix(test_filename.string(), context.run_result.success);
+  } else {
+    bool require_overall_failure = false;
+    for (const auto& [filename, success] :
+         context.run_result.per_file_success) {
+      CompareFailPrefix(filename, success);
+      if (!success) {
+        require_overall_failure = true;
+      }
+    }
+
+    if (require_overall_failure) {
+      EXPECT_FALSE(context.run_result.success)
+          << "There is a per-file failure expectation, so the overall result "
+             "should have been a failure.";
+    } else {
+      // Individual files all succeeded, so the prefix is enforced on the main
+      // test file.
+      CompareFailPrefix(test_filename.string(), context.run_result.success);
+    }
+  }
 
   // Check results. Include a reminder of the autoupdate command for any
   // stdout/stderr differences.
@@ -194,7 +228,7 @@ auto FileTestBase::DumpOutput() -> ErrorOr<Success> {
   llvm::errs() << banner << "= stderr\n"
                << banner << context.stderr << banner << "= stdout\n"
                << banner << context.stdout << banner << "= Exit with success: "
-               << (context.exit_with_success ? "true" : "false") << "\n"
+               << (context.run_result.success ? "true" : "false") << "\n"
                << banner;
   return Success();
 }
@@ -254,7 +288,7 @@ auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
   // Capture trace streaming, but only when in debug mode.
   llvm::raw_svector_ostream stdout(context.stdout);
   llvm::raw_svector_ostream stderr(context.stderr);
-  CARBON_ASSIGN_OR_RETURN(context.exit_with_success,
+  CARBON_ASSIGN_OR_RETURN(context.run_result,
                           Run(test_args_ref, fs, stdout, stderr));
   return Success();
 }

testing/file_test/file_test_base.h

@@ -39,6 +39,30 @@ class FileTestBase : public testing::Test {
   // Provided for child class convenience.
   using LineNumberReplacement = FileTestAutoupdater::LineNumberReplacement;
 
+  // The result of Run(), used to detect errors. Failing test files should be
+  // named with a `fail_` prefix to indicate an expectation of failure.
+  //
+  // If per_file_success is empty:
+  // - The main file has a `fail_` prefix if !success.
+  // - The prefix of split files is unused.
+  //
+  // If per_file_success is non-empty:
+  // - Each file has a `fail_` prefix if !per_file_success[i].second.
+  //   - Files may be in per_file_success that aren't part of the main test
+  //     file. This allows tracking success in handling files that are
+  //     well-known, such as standard libraries. It is still the responsibility
+  //     of callers to use a `fail_` prefix if !per_file_success[i].second.
+  // - If any file has a `fail_` prefix, success must be false, and the prefix
+  //   of the main file is unused.
+  // - If no file has a `fail_` prefix, the main file has a `fail_` prefix if
+  //   !success.
+  struct RunResult {
+    bool success;
+
+    // Per-file success results. May be empty.
+    llvm::SmallVector<std::pair<llvm::StringRef, bool>> per_file_success;
+  };
+
   explicit FileTestBase(llvm::StringRef test_name) : test_name_(test_name) {}
 
   // Implemented by children to run the test. For example, TestBody validates
@@ -47,13 +71,12 @@ class FileTestBase : public testing::Test {
   //
   // Any test expectations should be called from ValidateRun, not Run.
   //
-  // The return value should be an error if there was an abnormal error. It
-  // should be true if a binary would return EXIT_SUCCESS, and false for
-  // EXIT_FAILURE (which is a test success for `fail_*` tests).
+  // The return value should be an error if there was an abnormal error, and
+  // RunResult otherwise.
   virtual auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
                    llvm::vfs::InMemoryFileSystem& fs,
                    llvm::raw_pwrite_stream& stdout,
-                   llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> = 0;
+                   llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> = 0;
 
   // Implemented by children to do post-Run test expectations. Only called when
   // testing. Does not need to be provided if only CHECK test expectations are
@@ -130,8 +153,7 @@ class FileTestBase : public testing::Test {
     llvm::SmallString<16> stdout;
     llvm::SmallString<16> stderr;
 
-    // Whether Run exited with success.
-    bool exit_with_success = false;
+    RunResult run_result = {.success = false};
   };
 
   // Processes the test file and runs the test. Returns an error if something

testing/file_test/file_test_base_test.cpp

@@ -20,7 +20,7 @@ class FileTestBaseTest : public FileTestBase {
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
            llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> override {
+           llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
     llvm::ArrayRef<llvm::StringRef> args = test_args;
 
     llvm::ListSeparator sep;
@@ -34,7 +34,7 @@ class FileTestBaseTest : public FileTestBase {
     if (filename == "args.carbon") {
       // 'args.carbon' has custom arguments, so don't do regular argument
       // validation for it.
-      return true;
+      return {{.success = true}};
     }
 
     if (args.empty() || args.front() != "default_args") {
@@ -56,10 +56,10 @@ class FileTestBaseTest : public FileTestBase {
              << "example.carbon:" << delta_line << ": Negative line delta\n"
              << "+*[]{}\n"
              << "Foo baz\n";
-      return true;
+      return {{.success = true}};
     } else if (filename == "fail_example.carbon") {
       stderr << "Oops\n";
-      return false;
+      return {{.success = false}};
     } else if (filename == "two_files.carbon" ||
                filename == "not_split.carbon") {
       for (auto arg : args) {
@@ -74,7 +74,7 @@ class FileTestBaseTest : public FileTestBase {
         stdout.write_escaped(content.take_front(40));
         stdout << "\", length " << content.count('\n') << " lines\n";
       }
-      return true;
+      return {{.success = true}};
     } else if (filename == "alternating_files.carbon") {
       stdout << "unattached message 1\n"
              << "a.carbon:2: message 2\n"
@@ -88,19 +88,19 @@ class FileTestBaseTest : public FileTestBase {
              << "a.carbon:2: message 4\n"
              << "b.carbon:5: message 5\n"
              << "unattached message 6\n";
-      return true;
+      return {{.success = true}};
     } else if (filename == "unattached_multi_file.carbon") {
       stdout << "unattached message 1\n"
              << "unattached message 2\n";
       stderr << "unattached message 3\n"
              << "unattached message 4\n";
-      return true;
+      return {{.success = true}};
     } else if (filename == "file_only_re_one_file.carbon") {
       stdout << "unattached message 1\n"
              << "file: file_only_re_one_file.carbon\n"
              << "line: 1\n"
              << "unattached message 2\n";
-      return true;
+      return {{.success = true}};
     } else if (filename == "file_only_re_multi_file.carbon") {
       int msg_count = 0;
       stdout << "unattached message " << ++msg_count << "\n"
@@ -115,7 +115,22 @@ class FileTestBaseTest : public FileTestBase {
              << "unattached message " << ++msg_count << "\n"
              << "line: 7: late message " << ++msg_count << "\n"
              << "unattached message " << ++msg_count << "\n";
-      return true;
+      return {{.success = true}};
+    } else if (filename == "fail_multi_success_overall_fail.carbon") {
+      RunResult result = {.success = false};
+      result.per_file_success.push_back({"a.carbon", true});
+      result.per_file_success.push_back({"b.carbon", true});
+      return result;
+    } else if (filename == "multi_success.carbon") {
+      RunResult result = {.success = true};
+      result.per_file_success.push_back({"a.carbon", true});
+      result.per_file_success.push_back({"b.carbon", true});
+      return result;
+    } else if (filename == "multi_success_and_fail.carbon") {
+      RunResult result = {.success = false};
+      result.per_file_success.push_back({"a.carbon", true});
+      result.per_file_success.push_back({"fail_b.carbon", false});
+      return result;
     } else {
       return ErrorBuilder() << "Unexpected file: " << filename;
     }

testing/file_test/testdata/fail_multi_success_overall_fail.carbon

@@ -0,0 +1,13 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// AUTOUPDATE
+
+// --- a.carbon
+aaa
+
+// --- b.carbon
+bbb
+
+// CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon`

testing/file_test/testdata/multi_success.carbon

@@ -0,0 +1,13 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// AUTOUPDATE
+
+// --- a.carbon
+aaa
+
+// --- b.carbon
+bbb
+
+// CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon`

testing/file_test/testdata/multi_success_and_fail.carbon

@@ -0,0 +1,13 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// AUTOUPDATE
+
+// --- a.carbon
+aaa
+
+// --- fail_b.carbon
+bbb
+
+// CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `fail_b.carbon`

toolchain/check/check_fuzzer.cpp

@@ -31,7 +31,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
 
   // TODO: Get checking to a point where it can handle invalid parse trees
   // without crashing.
-  if (!driver.RunCommand({"compile", "--phase=parse", TestFileName})) {
+  if (!driver.RunCommand({"compile", "--phase=parse", TestFileName}).success) {
     return 0;
   }
   driver.RunCommand({"compile", "--phase=check", TestFileName});

toolchain/check/testdata/class/fail_import_misuses.carbon

@@ -13,13 +13,13 @@ class Empty {
 
 class Incomplete;
 
-// --- b.carbon
+// --- fail_b.carbon
 
 library "b" api;
 
 import library "a";
 
-// CHECK:STDERR: b.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_b.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: class Empty {
 // CHECK:STDERR: ^~~~~~~~~~~~~
 // CHECK:STDERR: a.carbon:4:1: Name is previously declared here.
@@ -28,10 +28,10 @@ import library "a";
 class Empty {
 }
 
-// CHECK:STDERR: b.carbon:[[@LINE+4]]:8: ERROR: Variable has incomplete type `Incomplete`.
+// CHECK:STDERR: fail_b.carbon:[[@LINE+4]]:8: ERROR: Variable has incomplete type `Incomplete`.
 // CHECK:STDERR: var a: Incomplete;
 // CHECK:STDERR:        ^~~~~~~~~~
-// CHECK:STDERR: b.carbon: Class was forward declared here.
+// CHECK:STDERR: fail_b.carbon: Class was forward declared here.
 var a: Incomplete;
 
 // CHECK:STDOUT: --- a.carbon
@@ -58,7 +58,7 @@ var a: Incomplete;
 // CHECK:STDOUT:
 // CHECK:STDOUT: class @Incomplete;
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- b.carbon
+// CHECK:STDOUT: --- fail_b.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: constants {
 // CHECK:STDOUT:   %Empty: type = class_type @Empty [template]

toolchain/check/testdata/class/fail_todo_import_forward_decl.carbon

@@ -10,14 +10,14 @@ library "a" api;
 
 class ForwardDecl;
 
-// --- b.carbon
+// --- fail_b.carbon
 
 library "b" api;
 
 import library "a";
 
 // TODO: This should probably have a valid form.
-// CHECK:STDERR: b.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_b.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: class ForwardDecl {
 // CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~
 // CHECK:STDERR: a.carbon:4:1: Name is previously declared here.
@@ -41,7 +41,7 @@ class ForwardDecl {
 // CHECK:STDOUT:
 // CHECK:STDOUT: class @ForwardDecl;
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- b.carbon
+// CHECK:STDOUT: --- fail_b.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: constants {
 // CHECK:STDOUT:   %ForwardDecl: type = class_type @ForwardDecl [template]

toolchain/check/testdata/function/definition/fail_todo_import_forward_decl.carbon

@@ -10,7 +10,7 @@ library "a" api;
 
 fn Foo();
 
-// --- b.carbon
+// --- fail_b.carbon
 
 library "b" api;
 
@@ -18,7 +18,7 @@ import library "a";
 
 // TODO: This should be an error until we can be clear whether `a` or `b` should
 // own the definition, as there might be a separate definition in `a`'s impl.
-// CHECK:STDERR: b.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_b.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: fn Foo() {}
 // CHECK:STDERR: ^~~~~~~~~~
 // CHECK:STDERR: a.carbon:4:1: Name is previously declared here.
@@ -37,7 +37,7 @@ fn Foo() {}
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @Foo();
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- b.carbon
+// CHECK:STDOUT: --- fail_b.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/interface/fail_todo_import.carbon

@@ -3,14 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // AUTOUPDATE
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
-// CHECK:STDERR: b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntityType`.
+// CHECK:STDERR: fail_b.carbon: ERROR: Semantics TODO: `TryResolveInst on AssociatedEntity`.
 
 // --- a.carbon
 
@@ -33,7 +33,7 @@ interface ForwardDeclared {
 
 var f_ref: {.f: ForwardDeclared};
 
-// --- b.carbon
+// --- fail_b.carbon
 
 library "b" api;
 
@@ -127,7 +127,7 @@ var f: ForwardDeclared* = &f_ref.f;
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @F.2();
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- b.carbon
+// CHECK:STDOUT: --- fail_b.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: constants {
 // CHECK:STDOUT:   %.1: type = interface_type @Empty [template]

toolchain/check/testdata/let/fail_generic_import.carbon

@@ -10,12 +10,12 @@ package Implicit api;
 
 let T:! type = i32;
 
-// --- implicit.impl.carbon
+// --- fail_implicit.impl.carbon
 
 package Implicit impl;
 
 // TODO: Should this be valid?
-// CHECK:STDERR: implicit.impl.carbon:[[@LINE+3]]:1: ERROR: Cannot implicitly convert from `i32` to `T`.
+// CHECK:STDERR: fail_implicit.impl.carbon:[[@LINE+3]]:1: ERROR: Cannot implicitly convert from `i32` to `T`.
 // CHECK:STDERR: let a: T = 0;
 // CHECK:STDERR: ^~~~~~~~~~~~~
 let a: T = 0;
@@ -27,7 +27,7 @@ let a: T = 0;
 // CHECK:STDOUT:   %T: type = bind_symbolic_name T, i32 [symbolic]
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- implicit.impl.carbon
+// CHECK:STDOUT: --- fail_implicit.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: constants {
 // CHECK:STDOUT:   %.1: i32 = int_literal 0 [template]

toolchain/check/testdata/namespace/fail_conflict_after_merge.carbon

@@ -10,7 +10,7 @@ package Example library "namespace" api;
 
 namespace NS;
 
-// --- conflict.carbon
+// --- fail_conflict.carbon
 
 package Example api;
 
@@ -20,10 +20,10 @@ import library "namespace";
 // imported declaration.
 namespace NS;
 
-// CHECK:STDERR: conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: fn NS();
 // CHECK:STDERR: ^~~~~~~~
-// CHECK:STDERR: conflict.carbon:[[@LINE-5]]:1: Name is previously declared here.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE-5]]:1: Name is previously declared here.
 // CHECK:STDERR: namespace NS;
 // CHECK:STDERR: ^~~~~~~~~~~~~
 fn NS();
@@ -32,10 +32,10 @@ fn NS();
 // we don't move it.
 namespace NS;
 
-// CHECK:STDERR: conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: fn NS();
 // CHECK:STDERR: ^~~~~~~~
-// CHECK:STDERR: conflict.carbon:[[@LINE-17]]:1: Name is previously declared here.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE-17]]:1: Name is previously declared here.
 // CHECK:STDERR: namespace NS;
 // CHECK:STDERR: ^~~~~~~~~~~~~
 fn NS();
@@ -49,7 +49,7 @@ fn NS();
 // CHECK:STDOUT:   %.loc4: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- conflict.carbon
+// CHECK:STDOUT: --- fail_conflict.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/namespace/fail_conflict_imported_namespace_first.carbon

@@ -10,13 +10,13 @@ package Example library "namespace" api;
 
 namespace NS;
 
-// --- conflict.carbon
+// --- fail_conflict.carbon
 
 package Example api;
 
 import library "namespace";
 
-// CHECK:STDERR: conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: fn NS();
 // CHECK:STDERR: ^~~~~~~~
 // CHECK:STDERR: namespace.carbon:4:1: Name is previously declared here.
@@ -35,7 +35,7 @@ fn NS.Foo();
 // CHECK:STDOUT:   %.loc4: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- conflict.carbon
+// CHECK:STDOUT: --- fail_conflict.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/namespace/fail_conflict_imported_namespace_second.carbon

@@ -10,13 +10,13 @@ package Example library "fn" api;
 
 fn NS();
 
-// --- conflict.carbon
+// --- fail_conflict.carbon
 
 package Example api;
 
 import library "fn";
 
-// CHECK:STDERR: conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE+6]]:1: ERROR: Duplicate name being declared in the same scope.
 // CHECK:STDERR: namespace NS;
 // CHECK:STDERR: ^~~~~~~~~~~~~
 // CHECK:STDERR: fn.carbon:4:1: Name is previously declared here.
@@ -24,10 +24,10 @@ import library "fn";
 // CHECK:STDERR: ^~~~~~~~
 namespace NS;
 
-// CHECK:STDERR: conflict.carbon:[[@LINE+6]]:7: ERROR: Name qualifiers are only allowed for entities that provide a scope.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE+6]]:7: ERROR: Name qualifiers are only allowed for entities that provide a scope.
 // CHECK:STDERR: fn NS.Foo();
 // CHECK:STDERR:       ^~~
-// CHECK:STDERR: conflict.carbon:[[@LINE+3]]:4: Non-scope entity referenced here.
+// CHECK:STDERR: fail_conflict.carbon:[[@LINE+3]]:4: Non-scope entity referenced here.
 // CHECK:STDERR: fn NS.Foo();
 // CHECK:STDERR:    ^~
 fn NS.Foo();
@@ -43,7 +43,7 @@ fn NS.Foo();
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @NS();
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- conflict.carbon
+// CHECK:STDOUT: --- fail_conflict.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/namespace/fail_conflict_in_imports_namespace_first.carbon

@@ -23,7 +23,7 @@ package Example library "fn" api;
 // CHECK:STDERR: ^~~~~~~~~~~~~
 fn NS() {}
 
-// --- conflict.carbon
+// --- fail_conflict.carbon
 
 package Example library "namespace_then_fn" api;
 
@@ -63,7 +63,7 @@ fn NS.Bar() {}
 // CHECK:STDOUT:   return
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- conflict.carbon
+// CHECK:STDOUT: --- fail_conflict.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/namespace/fail_conflict_in_imports_namespace_second.carbon

@@ -23,7 +23,7 @@ package Example library "namespace" api;
 namespace NS;
 fn NS.Foo() {}
 
-// --- conflict.carbon
+// --- fail_conflict.carbon
 
 package Example library "fn_then_namespace" api;
 
@@ -63,7 +63,7 @@ fn NS.Bar() {}
 // CHECK:STDOUT:   return
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- conflict.carbon
+// CHECK:STDOUT: --- fail_conflict.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/packages/fail_api_not_found.carbon

@@ -4,28 +4,28 @@
 //
 // AUTOUPDATE
 
-// --- no_api.impl.carbon
+// --- fail_no_api.impl.carbon
 
-// CHECK:STDERR: no_api.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
+// CHECK:STDERR: fail_no_api.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
 // CHECK:STDERR: package Foo impl;
 // CHECK:STDERR: ^~~~~~~
 package Foo impl;
 
-// --- no_api_lib.impl.carbon
+// --- fail_no_api_lib.impl.carbon
 
-// CHECK:STDERR: no_api_lib.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
+// CHECK:STDERR: fail_no_api_lib.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
 // CHECK:STDERR: package Foo library "Bar" impl;
 // CHECK:STDERR: ^~~~~~~
 package Foo library "Bar" impl;
 
-// --- no_api_main_lib.impl.carbon
+// --- fail_no_api_main_lib.impl.carbon
 
-// CHECK:STDERR: no_api_main_lib.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
+// CHECK:STDERR: fail_no_api_main_lib.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
 // CHECK:STDERR: library "Bar" impl;
 // CHECK:STDERR: ^~~~~~~
 library "Bar" impl;
 
-// CHECK:STDOUT: --- no_api.impl.carbon
+// CHECK:STDOUT: --- fail_no_api.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {
@@ -33,7 +33,7 @@ library "Bar" impl;
 // CHECK:STDOUT:   }
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- no_api_lib.impl.carbon
+// CHECK:STDOUT: --- fail_no_api_lib.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {
@@ -41,7 +41,7 @@ library "Bar" impl;
 // CHECK:STDOUT:   }
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- no_api_main_lib.impl.carbon
+// CHECK:STDOUT: --- fail_no_api_main_lib.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/packages/fail_conflict_no_namespaces.carbon

@@ -22,7 +22,7 @@ package Example library "var" api;
 // CHECK:STDERR: ^~~~~~~~~
 var Foo: i32;
 
-// --- conflict.carbon
+// --- fail_conflict.carbon
 
 package Example library "conflict" api;
 
@@ -50,7 +50,7 @@ import library "var";
 // CHECK:STDOUT:   %Foo: ref i32 = bind_name Foo, %Foo.var
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- conflict.carbon
+// CHECK:STDOUT: --- fail_conflict.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/packages/fail_cycle.carbon

@@ -4,50 +4,50 @@
 //
 // AUTOUPDATE
 
-// --- a.carbon
+// --- fail_a.carbon
 
 package A api;
 
-// CHECK:STDERR: a.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
+// CHECK:STDERR: fail_a.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
 // CHECK:STDERR: import B;
 // CHECK:STDERR: ^~~~~~
 import B;
 
-// --- b.carbon
+// --- fail_b.carbon
 
 package B api;
 
-// CHECK:STDERR: b.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
+// CHECK:STDERR: fail_b.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
 // CHECK:STDERR: import C;
 // CHECK:STDERR: ^~~~~~
 import C;
 
-// --- c.carbon
+// --- fail_c.carbon
 
 package C api;
 
-// CHECK:STDERR: c.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
+// CHECK:STDERR: fail_c.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
 // CHECK:STDERR: import A;
 // CHECK:STDERR: ^~~~~~
 import A;
 
-// --- c.impl.carbon
+// --- fail_c.impl.carbon
 
-// CHECK:STDERR: c.impl.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
+// CHECK:STDERR: fail_c.impl.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
 // CHECK:STDERR: package C impl;
 // CHECK:STDERR: ^~~~~~~
 package C impl;
 
-// --- cycle_child.carbon
+// --- fail_cycle_child.carbon
 
 package CycleChild api;
 
-// CHECK:STDERR: cycle_child.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
+// CHECK:STDERR: fail_cycle_child.carbon:[[@LINE+3]]:1: ERROR: Import cannot be used due to a cycle. Cycle must be fixed to import.
 // CHECK:STDERR: import B;
 // CHECK:STDERR: ^~~~~~
 import B;
 
-// CHECK:STDOUT: --- a.carbon
+// CHECK:STDOUT: --- fail_a.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -55,7 +55,7 @@ import B;
 // CHECK:STDOUT:   %B: <namespace> = bind_name B, %import
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- b.carbon
+// CHECK:STDOUT: --- fail_b.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -63,7 +63,7 @@ import B;
 // CHECK:STDOUT:   %C: <namespace> = bind_name C, %import
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- c.carbon
+// CHECK:STDOUT: --- fail_c.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -71,7 +71,7 @@ import B;
 // CHECK:STDOUT:   %A: <namespace> = bind_name A, %import
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- c.impl.carbon
+// CHECK:STDOUT: --- fail_c.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {
@@ -79,7 +79,7 @@ import B;
 // CHECK:STDOUT:   }
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- cycle_child.carbon
+// CHECK:STDOUT: --- fail_cycle_child.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/check/testdata/packages/fail_duplicate_api.carbon

@@ -3,19 +3,19 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // AUTOUPDATE
-// CHECK:STDERR: main2.carbon: ERROR: Main//default previously provided by `main1.carbon`.
+// CHECK:STDERR: fail_main2.carbon: ERROR: Main//default previously provided by `main1.carbon`.
 
 // --- main1.carbon
 
-// --- main2.carbon
+// --- fail_main2.carbon
 
 // --- main_lib1.carbon
 
 library "lib" api;
 
-// --- main_lib2.carbon
+// --- fail_main_lib2.carbon
 
-// CHECK:STDERR: main_lib2.carbon:[[@LINE+3]]:1: ERROR: Library's API previously provided by `main_lib1.carbon`.
+// CHECK:STDERR: fail_main_lib2.carbon:[[@LINE+3]]:1: ERROR: Library's API previously provided by `main_lib1.carbon`.
 // CHECK:STDERR: library "lib" api;
 // CHECK:STDERR: ^~~~~~~
 library "lib" api;
@@ -24,9 +24,9 @@ library "lib" api;
 
 package Package api;
 
-// --- package2.carbon
+// --- fail_package2.carbon
 
-// CHECK:STDERR: package2.carbon:[[@LINE+3]]:1: ERROR: Library's API previously provided by `package1.carbon`.
+// CHECK:STDERR: fail_package2.carbon:[[@LINE+3]]:1: ERROR: Library's API previously provided by `package1.carbon`.
 // CHECK:STDERR: package Package api;
 // CHECK:STDERR: ^~~~~~~
 package Package api;
@@ -35,9 +35,9 @@ package Package api;
 
 package Package library "lib" api;
 
-// --- package_lib2.carbon
+// --- fail_package_lib2.carbon
 
-// CHECK:STDERR: package_lib2.carbon:[[@LINE+3]]:1: ERROR: Library's API previously provided by `package_lib1.carbon`.
+// CHECK:STDERR: fail_package_lib2.carbon:[[@LINE+3]]:1: ERROR: Library's API previously provided by `package_lib1.carbon`.
 // CHECK:STDERR: package Package library "lib" api;
 // CHECK:STDERR: ^~~~~~~
 package Package library "lib" api;
@@ -48,7 +48,7 @@ package Package library "lib" api;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main2.carbon
+// CHECK:STDOUT: --- fail_main2.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -60,7 +60,7 @@ package Package library "lib" api;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_lib2.carbon
+// CHECK:STDOUT: --- fail_main_lib2.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -72,7 +72,7 @@ package Package library "lib" api;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- package2.carbon
+// CHECK:STDOUT: --- fail_package2.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -84,7 +84,7 @@ package Package library "lib" api;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- package_lib2.carbon
+// CHECK:STDOUT: --- fail_package_lib2.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/check/testdata/packages/fail_extension.carbon

@@ -3,128 +3,128 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // AUTOUPDATE
-// CHECK:STDERR: main.incorrect: ERROR: File extension of `.carbon` required for `api`.
-// CHECK:STDERR: main_redundant_with_swapped_ext.impl.carbon: ERROR: Main//default previously provided by `main.incorrect`.
-// CHECK:STDERR: main_redundant_with_swapped_ext.impl.carbon: ERROR: File extension of `.carbon` required for `api`.
-// CHECK:STDERR: main_redundant_with_swapped_ext.impl.carbon: File extension of `.impl.carbon` only allowed for `impl`.
+// CHECK:STDERR: fail_main.incorrect: ERROR: File extension of `.carbon` required for `api`.
+// CHECK:STDERR: fail_main_redundant_with_swapped_ext.impl.carbon: ERROR: Main//default previously provided by `fail_main.incorrect`.
+// CHECK:STDERR: fail_main_redundant_with_swapped_ext.impl.carbon: ERROR: File extension of `.carbon` required for `api`.
+// CHECK:STDERR: fail_main_redundant_with_swapped_ext.impl.carbon: File extension of `.impl.carbon` only allowed for `impl`.
 
-// --- main.incorrect
+// --- fail_main.incorrect
 
-// --- main_redundant_with_swapped_ext.impl.carbon
+// --- fail_main_redundant_with_swapped_ext.impl.carbon
 
-// --- main_lib.incorrect
+// --- fail_main_lib.incorrect
 
-// CHECK:STDERR: main_lib.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.carbon` required for `api`.
+// CHECK:STDERR: fail_main_lib.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.carbon` required for `api`.
 // CHECK:STDERR: library "lib" api;
 // CHECK:STDERR: ^~~~~~~
 library "lib" api;
 
-// --- main_lib_impl.incorrect
+// --- fail_main_lib_impl.incorrect
 
-// CHECK:STDERR: main_lib_impl.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
+// CHECK:STDERR: fail_main_lib_impl.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
 // CHECK:STDERR: library "lib" impl;
 // CHECK:STDERR: ^~~~~~~
 library "lib" impl;
 
-// --- package.incorrect
+// --- fail_package.incorrect
 
-// CHECK:STDERR: package.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.carbon` required for `api`.
+// CHECK:STDERR: fail_package.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.carbon` required for `api`.
 // CHECK:STDERR: package Package api;
 // CHECK:STDERR: ^~~~~~~
 package Package api;
 
-// --- package_impl.incorrect
+// --- fail_package_impl.incorrect
 
-// CHECK:STDERR: package_impl.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
+// CHECK:STDERR: fail_package_impl.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
 // CHECK:STDERR: package Package impl;
 // CHECK:STDERR: ^~~~~~~
 package Package impl;
 
-// --- package_lib.incorrect
+// --- fail_package_lib.incorrect
 
-// CHECK:STDERR: package_lib.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.carbon` required for `api`.
+// CHECK:STDERR: fail_package_lib.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.carbon` required for `api`.
 // CHECK:STDERR: package Package library "lib" api;
 // CHECK:STDERR: ^~~~~~~
 package Package library "lib" api;
 
-// --- package_lib_impl.incorrect
+// --- fail_package_lib_impl.incorrect
 
-// CHECK:STDERR: package_lib_impl.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
+// CHECK:STDERR: fail_package_lib_impl.incorrect:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
 // CHECK:STDERR: package Package library "lib" impl;
 // CHECK:STDERR: ^~~~~~~
 package Package library "lib" impl;
 
-// --- swapped_ext.impl.carbon
+// --- fail_swapped_ext.impl.carbon
 
-// CHECK:STDERR: swapped_ext.impl.carbon:[[@LINE+4]]:1: ERROR: File extension of `.carbon` required for `api`.
+// CHECK:STDERR: fail_swapped_ext.impl.carbon:[[@LINE+4]]:1: ERROR: File extension of `.carbon` required for `api`.
 // CHECK:STDERR: package SwappedExt api;
 // CHECK:STDERR: ^~~~~~~
-// CHECK:STDERR: swapped_ext.impl.carbon: File extension of `.impl.carbon` only allowed for `impl`.
+// CHECK:STDERR: fail_swapped_ext.impl.carbon: File extension of `.impl.carbon` only allowed for `impl`.
 package SwappedExt api;
 
-// --- swapped_ext.carbon
+// --- fail_swapped_ext.carbon
 
-// CHECK:STDERR: swapped_ext.carbon:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
+// CHECK:STDERR: fail_swapped_ext.carbon:[[@LINE+3]]:1: ERROR: File extension of `.impl.carbon` required for `impl`.
 // CHECK:STDERR: package SwappedExt impl;
 // CHECK:STDERR: ^~~~~~~
 package SwappedExt impl;
 
 
-// CHECK:STDOUT: --- main.incorrect
+// CHECK:STDOUT: --- fail_main.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_redundant_with_swapped_ext.impl.carbon
+// CHECK:STDOUT: --- fail_main_redundant_with_swapped_ext.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_lib.incorrect
+// CHECK:STDOUT: --- fail_main_lib.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_lib_impl.incorrect
+// CHECK:STDOUT: --- fail_main_lib_impl.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- package.incorrect
+// CHECK:STDOUT: --- fail_package.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- package_impl.incorrect
+// CHECK:STDOUT: --- fail_package_impl.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- package_lib.incorrect
+// CHECK:STDOUT: --- fail_package_lib.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- package_lib_impl.incorrect
+// CHECK:STDOUT: --- fail_package_lib_impl.incorrect
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- swapped_ext.impl.carbon
+// CHECK:STDOUT: --- fail_swapped_ext.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- swapped_ext.carbon
+// CHECK:STDOUT: --- fail_swapped_ext.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/check/testdata/packages/fail_import_default.carbon

@@ -4,59 +4,59 @@
 //
 // AUTOUPDATE
 
-// --- default_api.carbon
+// --- fail_default_api.carbon
 
 package A api;
 
-// CHECK:STDERR: default_api.carbon:[[@LINE+3]]:1: ERROR: File cannot import itself.
+// CHECK:STDERR: fail_default_api.carbon:[[@LINE+3]]:1: ERROR: File cannot import itself.
 // CHECK:STDERR: import library default;
 // CHECK:STDERR: ^~~~~~
 import library default;
 
-// --- default.impl.carbon
+// --- fail_default.impl.carbon
 
 package A impl;
 
-// CHECK:STDERR: default.impl.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
+// CHECK:STDERR: fail_default.impl.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
 // CHECK:STDERR: import library default;
 // CHECK:STDERR: ^~~~~~
 import library default;
 
-// --- main_import_default.carbon
+// --- fail_main_import_default.carbon
 
-// CHECK:STDERR: main_import_default.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
+// CHECK:STDERR: fail_main_import_default.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
 // CHECK:STDERR: import library default;
 // CHECK:STDERR: ^~~~~~
 import library default;
 
-// --- main_lib_import_default.carbon
+// --- fail_main_lib_import_default.carbon
 
 library "lib" api;
 
-// CHECK:STDERR: main_lib_import_default.carbon:[[@LINE+3]]:1: ERROR: Cannot import `Main//default`.
+// CHECK:STDERR: fail_main_lib_import_default.carbon:[[@LINE+3]]:1: ERROR: Cannot import `Main//default`.
 // CHECK:STDERR: import library default;
 // CHECK:STDERR: ^~~~~~
 import library default;
 
-// CHECK:STDOUT: --- default_api.carbon
+// CHECK:STDOUT: --- fail_default_api.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- default.impl.carbon
+// CHECK:STDOUT: --- fail_default.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_import_default.carbon
+// CHECK:STDOUT: --- fail_main_import_default.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_lib_import_default.carbon
+// CHECK:STDOUT: --- fail_main_lib_import_default.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/check/testdata/packages/fail_import_invalid.carbon

@@ -4,46 +4,46 @@
 //
 // AUTOUPDATE
 
-// --- main.carbon
+// --- fail_main.carbon
 
-// CHECK:STDERR: main.carbon:[[@LINE+3]]:1: ERROR: Imports from the current package must omit the package name.
+// CHECK:STDERR: fail_main.carbon:[[@LINE+3]]:1: ERROR: Imports from the current package must omit the package name.
 // CHECK:STDERR: import Main;
 // CHECK:STDERR: ^~~~~~
 import Main;
 
-// CHECK:STDERR: main.carbon:[[@LINE+3]]:1: ERROR: Imports from the current package must omit the package name.
+// CHECK:STDERR: fail_main.carbon:[[@LINE+3]]:1: ERROR: Imports from the current package must omit the package name.
 // CHECK:STDERR: import Main library "lib";
 // CHECK:STDERR: ^~~~~~
 import Main library "lib";
 
-// --- not_main.carbon
+// --- fail_not_main.carbon
 
 package NotMain api;
 
-// CHECK:STDERR: not_main.carbon:[[@LINE+3]]:1: ERROR: Cannot import `Main` from other packages.
+// CHECK:STDERR: fail_not_main.carbon:[[@LINE+3]]:1: ERROR: Cannot import `Main` from other packages.
 // CHECK:STDERR: import Main;
 // CHECK:STDERR: ^~~~~~
 import Main;
 
-// CHECK:STDERR: not_main.carbon:[[@LINE+3]]:1: ERROR: Cannot import `Main` from other packages.
+// CHECK:STDERR: fail_not_main.carbon:[[@LINE+3]]:1: ERROR: Cannot import `Main` from other packages.
 // CHECK:STDERR: import Main library "lib";
 // CHECK:STDERR: ^~~~~~
 import Main library "lib";
 
-// --- this.carbon
+// --- fail_this.carbon
 
 package This api;
 
-// CHECK:STDERR: this.carbon:[[@LINE+3]]:1: ERROR: File cannot import itself.
+// CHECK:STDERR: fail_this.carbon:[[@LINE+3]]:1: ERROR: File cannot import itself.
 // CHECK:STDERR: import This;
 // CHECK:STDERR: ^~~~~~
 import This;
 
-// --- this_lib.carbon
+// --- fail_this_lib.carbon
 
 package This library "lib" api;
 
-// CHECK:STDERR: this_lib.carbon:[[@LINE+3]]:1: ERROR: File cannot import itself.
+// CHECK:STDERR: fail_this_lib.carbon:[[@LINE+3]]:1: ERROR: File cannot import itself.
 // CHECK:STDERR: import library "lib";
 // CHECK:STDERR: ^~~~~~
 import library "lib";
@@ -52,11 +52,11 @@ import library "lib";
 
 package Implicit api;
 
-// --- implicit.impl.carbon
+// --- fail_implicit.impl.carbon
 
 package Implicit impl;
 
-// CHECK:STDERR: implicit.impl.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
+// CHECK:STDERR: fail_implicit.impl.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
 // CHECK:STDERR: import Implicit;
 // CHECK:STDERR: ^~~~~~
 import Implicit;
@@ -65,42 +65,42 @@ import Implicit;
 
 package Implicit library "lib" api;
 
-// --- implicit_lib.impl.carbon
+// --- fail_implicit_lib.impl.carbon
 
 package Implicit library "lib" impl;
 
-// CHECK:STDERR: implicit_lib.impl.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
+// CHECK:STDERR: fail_implicit_lib.impl.carbon:[[@LINE+3]]:1: ERROR: Explicit import of `api` from `impl` file is redundant with implicit import.
 // CHECK:STDERR: import Implicit library "lib";
 // CHECK:STDERR: ^~~~~~
 import Implicit library "lib";
 
-// --- not_found.carbon
+// --- fail_not_found.carbon
 package NotFound api;
 
-// CHECK:STDERR: not_found.carbon:[[@LINE+3]]:1: ERROR: Imported API not found.
+// CHECK:STDERR: fail_not_found.carbon:[[@LINE+3]]:1: ERROR: Imported API not found.
 // CHECK:STDERR: import ImportNotFound;
 // CHECK:STDERR: ^~~~~~
 import ImportNotFound;
 
-// CHECK:STDOUT: --- main.carbon
+// CHECK:STDOUT: --- fail_main.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- not_main.carbon
+// CHECK:STDOUT: --- fail_not_main.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- this.carbon
+// CHECK:STDOUT: --- fail_this.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- this_lib.carbon
+// CHECK:STDOUT: --- fail_this_lib.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -112,7 +112,7 @@ import ImportNotFound;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- implicit.impl.carbon
+// CHECK:STDOUT: --- fail_implicit.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -124,13 +124,13 @@ import ImportNotFound;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- implicit_lib.impl.carbon
+// CHECK:STDOUT: --- fail_implicit_lib.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- not_found.carbon
+// CHECK:STDOUT: --- fail_not_found.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/check/testdata/packages/fail_import_repeat.carbon

@@ -16,44 +16,44 @@ package Api library "lib" api;
 
 library "lib" api;
 
-// --- import.carbon
+// --- fail_import.carbon
 
 import Api;
-// CHECK:STDERR: import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
+// CHECK:STDERR: fail_import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
 // CHECK:STDERR: import Api;
 // CHECK:STDERR: ^~~~~~
-// CHECK:STDERR: import.carbon:[[@LINE-4]]:1: First import here.
+// CHECK:STDERR: fail_import.carbon:[[@LINE-4]]:1: First import here.
 // CHECK:STDERR: import Api;
 // CHECK:STDERR: ^~~~~~
 import Api;
 
 import Api library "lib";
-// CHECK:STDERR: import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
+// CHECK:STDERR: fail_import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
 // CHECK:STDERR: import Api library "lib";
 // CHECK:STDERR: ^~~~~~
-// CHECK:STDERR: import.carbon:[[@LINE-4]]:1: First import here.
+// CHECK:STDERR: fail_import.carbon:[[@LINE-4]]:1: First import here.
 // CHECK:STDERR: import Api library "lib";
 // CHECK:STDERR: ^~~~~~
 import Api library "lib";
 
 import library "lib";
-// CHECK:STDERR: import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
+// CHECK:STDERR: fail_import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
 // CHECK:STDERR: import library "lib";
 // CHECK:STDERR: ^~~~~~
-// CHECK:STDERR: import.carbon:[[@LINE-4]]:1: First import here.
+// CHECK:STDERR: fail_import.carbon:[[@LINE-4]]:1: First import here.
 // CHECK:STDERR: import library "lib";
 // CHECK:STDERR: ^~~~~~
 import library "lib";
 
-// --- default_import.carbon
+// --- fail_default_import.carbon
 
 package Api library "not_default" api;
 
 import library default;
-// CHECK:STDERR: default_import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
+// CHECK:STDERR: fail_default_import.carbon:[[@LINE+6]]:1: ERROR: Library imported more than once.
 // CHECK:STDERR: import library default;
 // CHECK:STDERR: ^~~~~~
-// CHECK:STDERR: default_import.carbon:[[@LINE-4]]:1: First import here.
+// CHECK:STDERR: fail_default_import.carbon:[[@LINE-4]]:1: First import here.
 // CHECK:STDERR: import library default;
 // CHECK:STDERR: ^~~~~~
 import library default;
@@ -76,7 +76,7 @@ import library default;
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- import.carbon
+// CHECK:STDOUT: --- fail_import.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
@@ -84,7 +84,7 @@ import library default;
 // CHECK:STDOUT:   %Api: <namespace> = bind_name Api, %import
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- default_import.carbon
+// CHECK:STDOUT: --- fail_default_import.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/check/testdata/packages/fail_import_type_error.carbon

@@ -4,23 +4,23 @@
 //
 // AUTOUPDATE
 
-// --- implicit.carbon
+// --- fail_implicit.carbon
 
 package Implicit api;
 
-// CHECK:STDERR: implicit.carbon:[[@LINE+3]]:12: ERROR: Name `x` not found.
+// CHECK:STDERR: fail_implicit.carbon:[[@LINE+3]]:12: ERROR: Name `x` not found.
 // CHECK:STDERR: var a_ref: x;
 // CHECK:STDERR:            ^
 var a_ref: x;
-// CHECK:STDERR: implicit.carbon:[[@LINE+3]]:18: ERROR: Name `x` not found.
+// CHECK:STDERR: fail_implicit.carbon:[[@LINE+3]]:18: ERROR: Name `x` not found.
 // CHECK:STDERR: var b_ref: {.a = x};
 // CHECK:STDERR:                  ^
 var b_ref: {.a = x};
-// CHECK:STDERR: implicit.carbon:[[@LINE+3]]:13: ERROR: Name `x` not found.
+// CHECK:STDERR: fail_implicit.carbon:[[@LINE+3]]:13: ERROR: Name `x` not found.
 // CHECK:STDERR: var c_ref: (x,);
 // CHECK:STDERR:             ^
 var c_ref: (x,);
-// CHECK:STDERR: implicit.carbon:[[@LINE+3]]:12: ERROR: Name `x` not found.
+// CHECK:STDERR: fail_implicit.carbon:[[@LINE+3]]:12: ERROR: Name `x` not found.
 // CHECK:STDERR: var d_ref: x*;
 // CHECK:STDERR:            ^
 var d_ref: x*;
@@ -36,7 +36,7 @@ var b: i32 = b_ref;
 var c: i32 = c_ref;
 var d: i32 = d_ref;
 
-// CHECK:STDOUT: --- implicit.carbon
+// CHECK:STDOUT: --- fail_implicit.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {

toolchain/check/testdata/packages/fail_name_with_import_failure.carbon

@@ -4,16 +4,16 @@
 //
 // AUTOUPDATE
 
-// --- implicit.impl.carbon
+// --- fail_implicit.impl.carbon
 
-// CHECK:STDERR: implicit.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
+// CHECK:STDERR: fail_implicit.impl.carbon:[[@LINE+3]]:1: ERROR: Corresponding API not found.
 // CHECK:STDERR: package Implicit impl;
 // CHECK:STDERR: ^~~~~~~
 package Implicit impl;
 
 var a: () = A();
 
-// CHECK:STDOUT: --- implicit.impl.carbon
+// CHECK:STDOUT: --- fail_implicit.impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: constants {
 // CHECK:STDOUT:   %.1: type = tuple_type () [template]

toolchain/check/testdata/packages/fail_package_main.carbon

@@ -4,54 +4,54 @@
 //
 // AUTOUPDATE
 
-// --- main.carbon
+// --- fail_main.carbon
 
-// CHECK:STDERR: main.carbon:[[@LINE+3]]:1: ERROR: `Main//default` must omit `package` directive.
+// CHECK:STDERR: fail_main.carbon:[[@LINE+3]]:1: ERROR: `Main//default` must omit `package` directive.
 // CHECK:STDERR: package Main api;
 // CHECK:STDERR: ^~~~~~~
 package Main api;
 
-// --- main_impl.carbon
+// --- fail_main_impl.carbon
 
-// CHECK:STDERR: main_impl.carbon:[[@LINE+3]]:1: ERROR: `Main//default` must omit `package` directive.
+// CHECK:STDERR: fail_main_impl.carbon:[[@LINE+3]]:1: ERROR: `Main//default` must omit `package` directive.
 // CHECK:STDERR: package Main impl;
 // CHECK:STDERR: ^~~~~~~
 package Main impl;
 
-// --- raw_main.carbon
+// --- fail_raw_main.carbon
 
 // `Main` isn't a keyword, so this fails the same way.
-// CHECK:STDERR: raw_main.carbon:[[@LINE+3]]:1: ERROR: `Main//default` must omit `package` directive.
+// CHECK:STDERR: fail_raw_main.carbon:[[@LINE+3]]:1: ERROR: `Main//default` must omit `package` directive.
 // CHECK:STDERR: package r#Main api;
 // CHECK:STDERR: ^~~~~~~
 package r#Main api;
 
-// --- main_lib.carbon
+// --- fail_main_lib.carbon
 
-// CHECK:STDERR: main_lib.carbon:[[@LINE+3]]:1: ERROR: Use `library` directive in `Main` package libraries.
+// CHECK:STDERR: fail_main_lib.carbon:[[@LINE+3]]:1: ERROR: Use `library` directive in `Main` package libraries.
 // CHECK:STDERR: package Main library "lib" api;
 // CHECK:STDERR: ^~~~~~~
 package Main library "lib" api;
 
-// CHECK:STDOUT: --- main.carbon
+// CHECK:STDOUT: --- fail_main.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_impl.carbon
+// CHECK:STDOUT: --- fail_main_impl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- raw_main.carbon
+// CHECK:STDOUT: --- fail_raw_main.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
-// CHECK:STDOUT: --- main_lib.carbon
+// CHECK:STDOUT: --- fail_main_lib.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: file {
 // CHECK:STDOUT:   package: <namespace> = namespace [template] {}

toolchain/driver/driver.cpp

@@ -4,6 +4,7 @@
 
 #include "toolchain/driver/driver.h"
 
+#include <algorithm>
 #include <memory>
 #include <optional>
 
@@ -333,13 +334,13 @@ auto Driver::ParseArgs(llvm::ArrayRef<llvm::StringRef> args, Options& options)
       [&](CommandLine::CommandBuilder& b) { options.Build(b); });
 }
 
-auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
+auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> RunResult {
   Options options;
   CommandLine::ParseResult result = ParseArgs(args, options);
   if (result == CommandLine::ParseResult::Error) {
-    return false;
+    return {.success = false};
   } else if (result == CommandLine::ParseResult::MetaSuccess) {
-    return true;
+    return {.success = true};
   }
 
   if (options.verbose) {
@@ -409,7 +410,7 @@ class Driver::CompilationUnit {
   }
 
   // Loads source and lexes it. Returns true on success.
-  auto RunLex() -> bool {
+  auto RunLex() -> void {
     LogCall("SourceBuffer::MakeFromFile", [&] {
       if (input_file_name_ == "-") {
         source_ = SourceBuffer::MakeFromStdin(*consumer_);
@@ -419,7 +420,8 @@ class Driver::CompilationUnit {
       }
     });
     if (!source_) {
-      return false;
+      success_ = false;
+      return;
     }
     CARBON_VLOG() << "*** SourceBuffer ***\n```\n"
                   << source_->text() << "\n```\n";
@@ -431,11 +433,13 @@ class Driver::CompilationUnit {
       driver_->output_stream_ << tokens_;
     }
     CARBON_VLOG() << "*** Lex::TokenizedBuffer ***\n" << tokens_;
-    return !tokens_->has_errors();
+    if (tokens_->has_errors()) {
+      success_ = false;
+    }
   }
 
   // Parses tokens. Returns true on success.
-  auto RunParse() -> bool {
+  auto RunParse() -> void {
     CARBON_CHECK(tokens_);
 
     LogCall("Parse::Parse", [&] {
@@ -446,7 +450,9 @@ class Driver::CompilationUnit {
       parse_tree_->Print(driver_->output_stream_, options_.preorder_parse_tree);
     }
     CARBON_VLOG() << "*** Parse::Tree ***\n" << parse_tree_;
-    return !parse_tree_->has_errors();
+    if (parse_tree_->has_errors()) {
+      success_ = false;
+    }
   }
 
   // Returns information needed to check this unit.
@@ -460,7 +466,7 @@ class Driver::CompilationUnit {
   }
 
   // Runs post-check logic. Returns true if checking succeeded for the IR.
-  auto PostCheck() -> bool {
+  auto PostCheck() -> void {
     CARBON_CHECK(sem_ir_);
 
     // We've finished all steps that can produce diagnostics. Emit the
@@ -484,7 +490,9 @@ class Driver::CompilationUnit {
       SemIR::FormatFile(*tokens_, *parse_tree_, *sem_ir_,
                         driver_->output_stream_);
     }
-    return !sem_ir_->has_errors();
+    if (sem_ir_->has_errors()) {
+      success_ = false;
+    }
   }
 
   // Lower SemIR to LLVM IR.
@@ -508,11 +516,26 @@ class Driver::CompilationUnit {
     }
   }
 
-  // Do codegen. Returns true on success.
-  auto RunCodeGen() -> bool {
+  auto RunCodeGen() -> void {
     CARBON_CHECK(module_);
+    LogCall("CodeGen", [&] { success_ = RunCodeGenHelper(); });
+  }
+
+  // Flushes output.
+  auto Flush() -> void { consumer_->Flush(); }
+
+  auto PrintSharedValues() const -> void {
+    Yaml::Print(driver_->output_stream_,
+                value_stores_.OutputYaml(input_file_name_));
+  }
+
+  auto input_file_name() -> llvm::StringRef { return input_file_name_; }
+  auto success() -> bool { return success_; }
+  auto has_source() -> bool { return source_.has_value(); }
 
-    CARBON_VLOG() << "*** CodeGen ***\n";
+ private:
+  // Do codegen. Returns true on success.
+  auto RunCodeGenHelper() -> bool {
     std::optional<CodeGen> codegen =
         CodeGen::Make(*module_, options_.target, driver_->error_stream_);
     if (!codegen) {
@@ -577,21 +600,9 @@ class Driver::CompilationUnit {
         }
       }
     }
-    CARBON_VLOG() << "*** CodeGen done ***\n";
     return true;
   }
 
-  // Flushes output.
-  auto Flush() -> void { consumer_->Flush(); }
-
-  auto PrintSharedValues() const -> void {
-    Yaml::Print(driver_->output_stream_,
-                value_stores_.OutputYaml(input_file_name_));
-  }
-
-  auto has_source() -> bool { return source_.has_value(); }
-
- private:
   // Wraps a call with log statements to indicate start and end.
   auto LogCall(llvm::StringLiteral label, llvm::function_ref<void()> fn)
       -> void {
@@ -613,6 +624,8 @@ class Driver::CompilationUnit {
   std::optional<SortingDiagnosticConsumer> sorting_consumer_;
   DiagnosticConsumer* consumer_;
 
+  bool success_ = true;
+
   // These are initialized as steps are run.
   std::optional<SourceBuffer> source_;
   std::optional<Lex::TokenizedBuffer> tokens_;
@@ -622,12 +635,21 @@ class Driver::CompilationUnit {
   std::unique_ptr<llvm::Module> module_;
 };
 
-auto Driver::Compile(const CompileOptions& options) -> bool {
+auto Driver::Compile(const CompileOptions& options) -> RunResult {
   if (!ValidateCompileOptions(options)) {
-    return false;
+    return {.success = false};
   }
 
   llvm::SmallVector<std::unique_ptr<CompilationUnit>> units;
+  auto make_result = [&]() {
+    RunResult result = {.success = true};
+    for (const auto& unit : units) {
+      result.success &= unit->success();
+      result.per_file_success.push_back(
+          {unit->input_file_name(), unit->success()});
+    }
+    return result;
+  };
   auto flush = llvm::make_scope_exit([&]() {
     // The diagnostics consumer must be flushed before compilation artifacts are
     // destructed, because diagnostics can refer to their state. This ensures
@@ -650,12 +672,11 @@ auto Driver::Compile(const CompileOptions& options) -> bool {
   }
 
   // Lex.
-  bool success_before_lower = true;
   for (auto& unit : units) {
-    success_before_lower &= unit->RunLex();
+    unit->RunLex();
   }
   if (options.phase == CompileOptions::Phase::Lex) {
-    return success_before_lower;
+    return make_result();
   }
   // Parse and check phases examine `has_source` because they want to proceed if
   // lex failed, but not if source doesn't exist. Later steps are skipped if
@@ -664,11 +685,11 @@ auto Driver::Compile(const CompileOptions& options) -> bool {
   // Parse.
   for (auto& unit : units) {
     if (unit->has_source()) {
-      success_before_lower &= unit->RunParse();
+      unit->RunParse();
     }
   }
   if (options.phase == CompileOptions::Phase::Parse) {
-    return success_before_lower;
+    return make_result();
   }
 
   // Check.
@@ -686,17 +707,18 @@ auto Driver::Compile(const CompileOptions& options) -> bool {
   CARBON_VLOG() << "*** Check::CheckParseTrees done ***\n";
   for (auto& unit : units) {
     if (unit->has_source()) {
-      success_before_lower &= unit->PostCheck();
+      unit->PostCheck();
     }
   }
   if (options.phase == CompileOptions::Phase::Check) {
-    return success_before_lower;
+    return make_result();
   }
 
   // Unlike previous steps, errors block further progress.
-  if (!success_before_lower) {
+  if (std::any_of(units.begin(), units.end(),
+                  [&](const auto& unit) { return !unit->success(); })) {
     CARBON_VLOG() << "*** Stopping before lowering due to errors ***";
-    return false;
+    return make_result();
   }
 
   // Lower.
@@ -704,17 +726,16 @@ auto Driver::Compile(const CompileOptions& options) -> bool {
     unit->RunLower();
   }
   if (options.phase == CompileOptions::Phase::Lower) {
-    return true;
+    return make_result();
   }
   CARBON_CHECK(options.phase == CompileOptions::Phase::CodeGen)
       << "CodeGen should be the last stage";
 
   // Codegen.
-  bool codegen_success = true;
   for (auto& unit : units) {
-    codegen_success &= unit->RunCodeGen();
+    unit->RunCodeGen();
   }
-  return codegen_success;
+  return make_result();
 }
 
 }  // namespace Carbon

toolchain/driver/driver.h

@@ -20,13 +20,21 @@ namespace Carbon {
 // with the language.
 class Driver {
  public:
+  // The result of RunCommand().
+  struct RunResult {
+    // Overall success result.
+    bool success;
+
+    // Per-file success results. May be empty if files aren't individually
+    // processed.
+    llvm::SmallVector<std::pair<llvm::StringRef, bool>> per_file_success;
+  };
+
   // Constructs a driver with any error or informational output directed to a
   // specified stream.
   Driver(llvm::vfs::FileSystem& fs, llvm::raw_pwrite_stream& output_stream,
          llvm::raw_pwrite_stream& error_stream)
-      : fs_(fs), output_stream_(output_stream), error_stream_(error_stream) {
-    (void)fs_;
-  }
+      : fs_(fs), output_stream_(output_stream), error_stream_(error_stream) {}
 
   // Parses the given arguments into both a subcommand to select the operation
   // to perform and any arguments to that subcommand.
@@ -34,7 +42,7 @@ class Driver {
   // Returns true if the operation succeeds. If the operation fails, returns
   // false and any information about the failure is printed to the registered
   // error stream (stderr by default).
-  auto RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+  auto RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> RunResult;
 
  private:
   struct Options;
@@ -51,7 +59,7 @@ class Driver {
   auto ValidateCompileOptions(const CompileOptions& options) const -> bool;
 
   // Implements the compile subcommand of the driver.
-  auto Compile(const CompileOptions& options) -> bool;
+  auto Compile(const CompileOptions& options) -> RunResult;
 
   llvm::vfs::FileSystem& fs_;
   llvm::raw_pwrite_stream& output_stream_;

toolchain/driver/driver_fuzzer.cpp

@@ -69,7 +69,7 @@ extern "C" auto LLVMFuzzerTestOneInput(const unsigned char* data, size_t size)
   TestRawOstream error_stream;
   llvm::raw_null_ostream dest;
   Driver d(fs, dest, error_stream);
-  if (!d.RunCommand(args)) {
+  if (!d.RunCommand(args).success) {
     if (error_stream.TakeStr().find("ERROR:") == std::string::npos) {
       llvm::errs() << "No error message on a failure!\n";
       return 1;

toolchain/driver/driver_main.cpp

@@ -26,6 +26,6 @@ auto main(int argc, char** argv) -> int {
   llvm::SmallVector<llvm::StringRef> args(argv + 1, argv + argc);
   auto fs = llvm::vfs::getRealFileSystem();
   Carbon::Driver driver(*fs, llvm::outs(), llvm::errs());
-  bool success = driver.RunCommand(args);
+  bool success = driver.RunCommand(args).success;
   return success ? EXIT_SUCCESS : EXIT_FAILURE;
 }

toolchain/driver/driver_test.cpp

@@ -97,19 +97,19 @@ class DriverTest : public testing::Test {
 };
 
 TEST_F(DriverTest, BadCommandErrors) {
-  EXPECT_FALSE(driver_.RunCommand({}));
+  EXPECT_FALSE(driver_.RunCommand({}).success);
   EXPECT_THAT(test_error_stream_.TakeStr(), HasSubstr("ERROR"));
 
-  EXPECT_FALSE(driver_.RunCommand({"foo"}));
+  EXPECT_FALSE(driver_.RunCommand({"foo"}).success);
   EXPECT_THAT(test_error_stream_.TakeStr(), HasSubstr("ERROR"));
 
-  EXPECT_FALSE(driver_.RunCommand({"foo --bar --baz"}));
+  EXPECT_FALSE(driver_.RunCommand({"foo --bar --baz"}).success);
   EXPECT_THAT(test_error_stream_.TakeStr(), HasSubstr("ERROR"));
 }
 
 TEST_F(DriverTest, CompileCommandErrors) {
   // No input file. This error message is important so check all of it.
-  EXPECT_FALSE(driver_.RunCommand({"compile"}));
+  EXPECT_FALSE(driver_.RunCommand({"compile"}).success);
   EXPECT_THAT(
       test_error_stream_.TakeStr(),
       StrEq("ERROR: Not all required positional arguments were provided. First "
@@ -119,7 +119,8 @@ TEST_F(DriverTest, CompileCommandErrors) {
   // TODO: Likely want a different filename on Windows.
   auto empty_file = MakeTestFile("");
   EXPECT_FALSE(
-      driver_.RunCommand({"compile", "--output=/dev/empty", empty_file}));
+      driver_.RunCommand({"compile", "--output=/dev/empty", empty_file})
+          .success);
   EXPECT_THAT(test_error_stream_.TakeStr(),
               ContainsRegex("ERROR: .*/dev/empty.*"));
 }
@@ -127,7 +128,8 @@ TEST_F(DriverTest, CompileCommandErrors) {
 TEST_F(DriverTest, DumpTokens) {
   auto file = MakeTestFile("Hello World");
   EXPECT_TRUE(
-      driver_.RunCommand({"compile", "--phase=lex", "--dump-tokens", file}));
+      driver_.RunCommand({"compile", "--phase=lex", "--dump-tokens", file})
+          .success);
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // Verify there is output without examining it.
   EXPECT_THAT(Yaml::Value::FromText(test_output_stream_.TakeStr()),
@@ -136,8 +138,10 @@ TEST_F(DriverTest, DumpTokens) {
 
 TEST_F(DriverTest, DumpParseTree) {
   auto file = MakeTestFile("var v: i32 = 42;");
-  EXPECT_TRUE(driver_.RunCommand(
-      {"compile", "--phase=parse", "--dump-parse-tree", file}));
+  EXPECT_TRUE(
+      driver_
+          .RunCommand({"compile", "--phase=parse", "--dump-parse-tree", file})
+          .success);
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // Verify there is output without examining it.
   EXPECT_THAT(Yaml::Value::FromText(test_output_stream_.TakeStr()),
@@ -148,13 +152,16 @@ TEST_F(DriverTest, StdoutOutput) {
   // Use explicit filenames so we can look for those to validate output.
   MakeTestFile("fn Main() -> i32 { return 0; }", "test.carbon");
 
-  EXPECT_TRUE(driver_.RunCommand({"compile", "--output=-", "test.carbon"}));
+  EXPECT_TRUE(
+      driver_.RunCommand({"compile", "--output=-", "test.carbon"}).success);
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // The default is textual assembly.
   EXPECT_THAT(test_output_stream_.TakeStr(), ContainsRegex("Main:"));
 
-  EXPECT_TRUE(driver_.RunCommand(
-      {"compile", "--output=-", "--force-obj-output", "test.carbon"}));
+  EXPECT_TRUE(driver_
+                  .RunCommand({"compile", "--output=-", "--force-obj-output",
+                               "test.carbon"})
+                  .success);
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   std::string output = test_output_stream_.TakeStr();
   auto result =
@@ -174,7 +181,7 @@ TEST_F(DriverTest, FileOutput) {
 
   // Object output (the default) uses `.o`.
   // TODO: This should actually reflect the platform defaults.
-  EXPECT_TRUE(driver_.RunCommand({"compile", "test.carbon"}));
+  EXPECT_TRUE(driver_.RunCommand({"compile", "test.carbon"}).success);
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // Ensure we wrote an object file of some form with the correct name.
   auto result = llvm::object::createBinary("test.o");
@@ -185,7 +192,8 @@ TEST_F(DriverTest, FileOutput) {
 
   // Assembly output uses `.s`.
   // TODO: This should actually reflect the platform defaults.
-  EXPECT_TRUE(driver_.RunCommand({"compile", "--asm-output", "test.carbon"}));
+  EXPECT_TRUE(
+      driver_.RunCommand({"compile", "--asm-output", "test.carbon"}).success);
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // TODO: This may need to be tailored to other assembly formats.
   EXPECT_THAT(ReadFile("test.s"), ContainsRegex("Main:"));

toolchain/driver/testdata/fail_errors_in_two_files.carbon

@@ -6,17 +6,17 @@
 //
 // AUTOUPDATE
 
-// --- file1.carbon
+// --- fail_file1.carbon
 
-// CHECK:STDERR: file1.carbon:[[@LINE+3]]:24: ERROR: Opening symbol without a corresponding closing symbol.
+// CHECK:STDERR: fail_file1.carbon:[[@LINE+3]]:24: ERROR: Opening symbol without a corresponding closing symbol.
 // CHECK:STDERR: fn run(String program) {
 // CHECK:STDERR:                        ^
 fn run(String program) {
   return True;
 
-// --- file2.carbon
+// --- fail_file2.carbon
 
-// CHECK:STDERR: file2.carbon:[[@LINE+3]]:10: ERROR: Invalid digit 'a' in decimal numeric literal.
+// CHECK:STDERR: fail_file2.carbon:[[@LINE+3]]:10: ERROR: Invalid digit 'a' in decimal numeric literal.
 // CHECK:STDERR: var x = 3a;
 // CHECK:STDERR:          ^
 var x = 3a;

toolchain/lex/testdata/fail_multifile.carbon

@@ -4,11 +4,11 @@
 //
 // AUTOUPDATE
 
-// --- a.carbon
-// CHECK:STDOUT: - filename: a.carbon
+// --- fail_a.carbon
+// CHECK:STDOUT: - filename: fail_a.carbon
 // CHECK:STDOUT:   tokens: [
 // CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '', has_trailing_space: true },
-// CHECK:STDERR: a.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal.
+// CHECK:STDERR: fail_a.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal.
 // CHECK:STDERR: 1.a
 // CHECK:STDERR:   ^
 1.a
@@ -16,11 +16,11 @@
 
 // CHECK:STDOUT:     { index: 2, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
 // CHECK:STDOUT:   ]
-// --- b.carbon
-// CHECK:STDOUT: - filename: b.carbon
+// --- fail_b.carbon
+// CHECK:STDOUT: - filename: fail_b.carbon
 // CHECK:STDOUT:   tokens: [
 // CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '', has_trailing_space: true },
-// CHECK:STDERR: b.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal.
+// CHECK:STDERR: fail_b.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal.
 // CHECK:STDERR: 2.b
 // CHECK:STDERR:   ^
 2.b

toolchain/parse/testdata/array/fail_syntax.carbon

@@ -4,66 +4,66 @@
 //
 // AUTOUPDATE
 
-// --- type.carbon
+// --- fail_type.carbon
 
-// CHECK:STDERR: type.carbon:[[@LINE+3]]:12: ERROR: Expected `;` in array type.
+// CHECK:STDERR: fail_type.carbon:[[@LINE+3]]:12: ERROR: Expected `;` in array type.
 // CHECK:STDERR: var x: [i32];
 // CHECK:STDERR:            ^
 var x: [i32];
 
-// --- invalid_char.carbon
+// --- fail_invalid_char.carbon
 
-// CHECK:STDERR: invalid_char.carbon:[[@LINE+12]]:12: ERROR: Expected expression.
+// CHECK:STDERR: fail_invalid_char.carbon:[[@LINE+12]]:12: ERROR: Expected expression.
 // CHECK:STDERR: fn X() -> [:];
 // CHECK:STDERR:            ^
-// CHECK:STDERR: invalid_char.carbon:[[@LINE+9]]:12: ERROR: Expected `;` in array type.
+// CHECK:STDERR: fail_invalid_char.carbon:[[@LINE+9]]:12: ERROR: Expected `;` in array type.
 // CHECK:STDERR: fn X() -> [:];
 // CHECK:STDERR:            ^
-// CHECK:STDERR: invalid_char.carbon:[[@LINE+6]]:12: ERROR: Expected expression.
+// CHECK:STDERR: fail_invalid_char.carbon:[[@LINE+6]]:12: ERROR: Expected expression.
 // CHECK:STDERR: fn X() -> [:];
 // CHECK:STDERR:            ^
-// CHECK:STDERR: invalid_char.carbon:[[@LINE+3]]:12: ERROR: Unexpected tokens before `]`.
+// CHECK:STDERR: fail_invalid_char.carbon:[[@LINE+3]]:12: ERROR: Unexpected tokens before `]`.
 // CHECK:STDERR: fn X() -> [:];
 // CHECK:STDERR:            ^
 fn X() -> [:];
 
-// --- unlexed_expr.carbon
+// --- fail_unlexed_expr.carbon
 
-// CHECK:STDERR: unlexed_expr.carbon:[[@LINE+15]]:9: ERROR: Encountered unrecognized characters while parsing.
+// CHECK:STDERR: fail_unlexed_expr.carbon:[[@LINE+15]]:9: ERROR: Encountered unrecognized characters while parsing.
 // CHECK:STDERR: var y: [`];
 // CHECK:STDERR:         ^
-// CHECK:STDERR: unlexed_expr.carbon:[[@LINE+12]]:9: ERROR: Expected expression.
+// CHECK:STDERR: fail_unlexed_expr.carbon:[[@LINE+12]]:9: ERROR: Expected expression.
 // CHECK:STDERR: var y: [`];
 // CHECK:STDERR:         ^
-// CHECK:STDERR: unlexed_expr.carbon:[[@LINE+9]]:9: ERROR: Expected `;` in array type.
+// CHECK:STDERR: fail_unlexed_expr.carbon:[[@LINE+9]]:9: ERROR: Expected `;` in array type.
 // CHECK:STDERR: var y: [`];
 // CHECK:STDERR:         ^
-// CHECK:STDERR: unlexed_expr.carbon:[[@LINE+6]]:9: ERROR: Expected expression.
+// CHECK:STDERR: fail_unlexed_expr.carbon:[[@LINE+6]]:9: ERROR: Expected expression.
 // CHECK:STDERR: var y: [`];
 // CHECK:STDERR:         ^
-// CHECK:STDERR: unlexed_expr.carbon:[[@LINE+3]]:9: ERROR: Unexpected tokens before `]`.
+// CHECK:STDERR: fail_unlexed_expr.carbon:[[@LINE+3]]:9: ERROR: Unexpected tokens before `]`.
 // CHECK:STDERR: var y: [`];
 // CHECK:STDERR:         ^
 var y: [`];
 
-// --- no_close_bracket.carbon
+// --- fail_no_close_bracket.carbon
 
-// CHECK:STDERR: no_close_bracket.carbon:[[@LINE+6]]:8: ERROR: Opening symbol without a corresponding closing symbol.
+// CHECK:STDERR: fail_no_close_bracket.carbon:[[@LINE+6]]:8: ERROR: Opening symbol without a corresponding closing symbol.
 // CHECK:STDERR: var x: [i32;;
 // CHECK:STDERR:        ^
-// CHECK:STDERR: no_close_bracket.carbon:[[@LINE+3]]:8: ERROR: Expected expression.
+// CHECK:STDERR: fail_no_close_bracket.carbon:[[@LINE+3]]:8: ERROR: Expected expression.
 // CHECK:STDERR: var x: [i32;;
 // CHECK:STDERR:        ^
 var x: [i32;;
 
-// --- no_semi.carbon
+// --- fail_no_semi.carbon
 
-// CHECK:STDERR: no_semi.carbon:[[@LINE+3]]:12: ERROR: Expected `;` in array type.
+// CHECK:STDERR: fail_no_semi.carbon:[[@LINE+3]]:12: ERROR: Expected `;` in array type.
 // CHECK:STDERR: var x: [i32];
 // CHECK:STDERR:            ^
 var x: [i32];
 
-// CHECK:STDOUT: - filename: type.carbon
+// CHECK:STDOUT: - filename: fail_type.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'VariableIntroducer', text: 'var'},
@@ -76,7 +76,7 @@ var x: [i32];
 // CHECK:STDOUT:     {kind: 'VariableDecl', text: ';', subtree_size: 8},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: invalid_char.carbon
+// CHECK:STDOUT: - filename: fail_invalid_char.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
@@ -92,7 +92,7 @@ var x: [i32];
 // CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 11},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: unlexed_expr.carbon
+// CHECK:STDOUT: - filename: fail_unlexed_expr.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'VariableIntroducer', text: 'var'},
@@ -106,7 +106,7 @@ var x: [i32];
 // CHECK:STDOUT:     {kind: 'VariableDecl', text: ';', subtree_size: 9},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: no_close_bracket.carbon
+// CHECK:STDOUT: - filename: fail_no_close_bracket.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'VariableIntroducer', text: 'var'},
@@ -117,7 +117,7 @@ var x: [i32];
 // CHECK:STDOUT:     {kind: 'EmptyDecl', text: ';'},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: no_semi.carbon
+// CHECK:STDOUT: - filename: fail_no_semi.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'VariableIntroducer', text: 'var'},

toolchain/parse/testdata/packages/library/fail_invalid_name.carbon

@@ -4,70 +4,70 @@
 //
 // AUTOUPDATE
 
-// --- identifier.carbon
+// --- fail_identifier.carbon
 
-// CHECK:STDERR: identifier.carbon:[[@LINE+3]]:9: ERROR: Expected `default` or a string literal to specify the library name.
+// CHECK:STDERR: fail_identifier.carbon:[[@LINE+3]]:9: ERROR: Expected `default` or a string literal to specify the library name.
 // CHECK:STDERR: library Shapes api;
 // CHECK:STDERR:         ^~~~~~
 library Shapes api;
 
-// --- raw_identifier.carbon
+// --- fail_raw_identifier.carbon
 
-// CHECK:STDERR: raw_identifier.carbon:[[@LINE+3]]:9: ERROR: Expected `default` or a string literal to specify the library name.
+// CHECK:STDERR: fail_raw_identifier.carbon:[[@LINE+3]]:9: ERROR: Expected `default` or a string literal to specify the library name.
 // CHECK:STDERR: library r#default api;
 // CHECK:STDERR:         ^~~~~~~
 library r#default api;
 
-// --- missing.carbon
+// --- fail_missing.carbon
 
-// CHECK:STDERR: missing.carbon:[[@LINE+3]]:8: ERROR: Expected `default` or a string literal to specify the library name.
+// CHECK:STDERR: fail_missing.carbon:[[@LINE+3]]:8: ERROR: Expected `default` or a string literal to specify the library name.
 // CHECK:STDERR: library;
 // CHECK:STDERR:        ^
 library;
 
-// --- missing_with_api.carbon
+// --- fail_missing_with_api.carbon
 
-// CHECK:STDERR: missing_with_api.carbon:[[@LINE+3]]:9: ERROR: Expected `default` or a string literal to specify the library name.
+// CHECK:STDERR: fail_missing_with_api.carbon:[[@LINE+3]]:9: ERROR: Expected `default` or a string literal to specify the library name.
 // CHECK:STDERR: library api;
 // CHECK:STDERR:         ^~~
 library api;
 
-// --- no_semi.carbon
+// --- fail_no_semi.carbon
 
 library "NoSemi" api
 
-// CHECK:STDERR: no_semi.carbon:[[@LINE+39]]:21: ERROR: `library` declarations must end with a `;`.
+// CHECK:STDERR: fail_no_semi.carbon:[[@LINE+39]]:21: ERROR: `library` declarations must end with a `;`.
 // CHECK:STDERR: // CHECK:STDOUT:   ]
 // CHECK:STDERR:                     ^
-// CHECK:STDOUT: - filename: identifier.carbon
+// CHECK:STDOUT: - filename: fail_identifier.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'LibraryIntroducer', text: 'library'},
 // CHECK:STDOUT:     {kind: 'LibraryDirective', text: ';', has_error: yes, subtree_size: 2},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: raw_identifier.carbon
+// CHECK:STDOUT: - filename: fail_raw_identifier.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'LibraryIntroducer', text: 'library'},
 // CHECK:STDOUT:     {kind: 'LibraryDirective', text: ';', has_error: yes, subtree_size: 2},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: missing.carbon
+// CHECK:STDOUT: - filename: fail_missing.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'LibraryIntroducer', text: 'library'},
 // CHECK:STDOUT:     {kind: 'LibraryDirective', text: ';', has_error: yes, subtree_size: 2},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: missing_with_api.carbon
+// CHECK:STDOUT: - filename: fail_missing_with_api.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'LibraryIntroducer', text: 'library'},
 // CHECK:STDOUT:     {kind: 'LibraryDirective', text: ';', has_error: yes, subtree_size: 2},
 // CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
 // CHECK:STDOUT:   ]
-// CHECK:STDOUT: - filename: no_semi.carbon
+// CHECK:STDOUT: - filename: fail_no_semi.carbon
 // CHECK:STDOUT:   parse_tree: [
 // CHECK:STDOUT:     {kind: 'FileStart', text: ''},
 // CHECK:STDOUT:       {kind: 'LibraryIntroducer', text: 'library'},

toolchain/testing/file_test.cpp

@@ -26,9 +26,19 @@ class ToolchainFileTest : public FileTestBase {
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
            llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> override {
+           llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
     Driver driver(fs, stdout, stderr);
-    return driver.RunCommand(test_args);
+    auto driver_result = driver.RunCommand(test_args);
+    if (std::find(test_args.begin(), test_args.end(), "%s") ==
+        test_args.end()) {
+      // Files weren't forwarded as an argument, so don't use per_file_success.
+      // This primarily occurs in driver tests with invalid filename arguments,
+      // which we wouldn't want to try validating.
+      return {{.success = driver_result.success}};
+    } else {
+      return {{.success = driver_result.success,
+               .per_file_success = std::move(driver_result.per_file_success)}};
+    }
   }
 
   auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {