Add tests to catch untested diagnostics. (#4426)

Use the diagnostic kind printing in #4425 to catch when we have
diagnostics with no tests.

This merges a couple other use-cases of filegroup manifests into a
common rule.

Note I do add a few tests for things, and also some things are
_actually_ unit tested (just not in the file_test structure). But I
stopped when I realized that dealing with merge conflicts is going to be
a pain. I might end up reverting test changes (as part of merge conflict
resolution) and doing narrow test additions in a separate PR, after both
this and #4425 are merged.

bazel/manifest/BUILD

@@ -0,0 +1,5 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Empty; only defs.bzl is needed.

bazel/manifest/defs.bzl

@@ -0,0 +1,36 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Rule for producing a manifest for a filegroup."""
+
+def _manifest(ctx):
+    out = ctx.actions.declare_file(ctx.label.name)
+
+    files = []
+    for src in ctx.attr.srcs:
+        files.extend([f.path for f in src.files.to_list()])
+        files.extend([f.path for f in src.default_runfiles.files.to_list()])
+
+    if ctx.attr.strip_package_dir:
+        package_dir = ctx.label.package + "/"
+        content = [f.removeprefix(package_dir) for f in files]
+    else:
+        content = files
+
+    ctx.actions.write(out, "\n".join(content) + "\n")
+
+    return [
+        DefaultInfo(
+            files = depset(direct = [out]),
+            default_runfiles = ctx.runfiles(files = [out]),
+        ),
+    ]
+
+manifest = rule(
+    implementation = _manifest,
+    attrs = {
+        "srcs": attr.label_list(allow_files = True, mandatory = True),
+        "strip_package_dir": attr.bool(default = False),
+    },
+)

core/BUILD

@@ -2,7 +2,7 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("manifest.bzl", "manifest")
+load("//bazel/manifest:defs.bzl", "manifest")
 
 # Raw prelude files.
 filegroup(
@@ -13,9 +13,9 @@ filegroup(
 # A list of prelude inputs.
 # This is consumed by //toolchain/install:install_paths.
 manifest(
-    name = "prelude_manifest",
+    name = "prelude_manifest.txt",
     srcs = [":prelude_files"],
-    out = "prelude_manifest.txt",
+    strip_package_dir = True,
 )
 
 # All files for the toolchain install.

core/manifest.bzl

@@ -1,25 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-"""Rule for producing a manifest for a filegroup."""
-
-def _manifest(ctx):
-    dir = ctx.label.package + "/"
-    content = [f.path.removeprefix(dir) for f in ctx.files.srcs]
-    ctx.actions.write(ctx.outputs.out, "\n".join(content) + "\n")
-
-    return [
-        DefaultInfo(
-            files = depset(direct = [ctx.outputs.out]),
-            default_runfiles = ctx.runfiles(files = [ctx.outputs.out]),
-        ),
-    ]
-
-manifest = rule(
-    implementation = _manifest,
-    attrs = {
-        "out": attr.output(mandatory = True),
-        "srcs": attr.label_list(mandatory = True),
-    },
-)

testing/file_test/rules.bzl

@@ -9,26 +9,7 @@ a file which can be accessed as a list. This avoids long argument parsing.
 """
 
 load("@rules_cc//cc:defs.bzl", "cc_test")
-
-def _tests_as_input_file_rule_impl(ctx):
-    out = ctx.actions.declare_file(ctx.label.name + ".txt")
-    data_files = []
-    for tests in ctx.attr.data:
-        data_files.extend(
-            [f.path for f in tests[DefaultInfo].data_runfiles.files.to_list()],
-        )
-        data_files.extend(
-            [f.path for f in tests[DefaultInfo].files.to_list()],
-        )
-    ctx.actions.write(out, "\n".join(data_files) + "\n")
-    return [DefaultInfo(files = depset([out]))]
-
-_tests_as_input_file_rule = rule(
-    attrs = {
-        "data": attr.label_list(allow_files = True),
-    },
-    implementation = _tests_as_input_file_rule_impl,
-)
+load("//bazel/manifest:defs.bzl", "manifest")
 
 def file_test(
         name,
@@ -55,9 +36,9 @@ def file_test(
 
     # Ensure tests are always a filegroup for tests_as_input_file_rule.
     tests_file = "{0}.tests".format(name)
-    _tests_as_input_file_rule(
+    manifest(
         name = tests_file,
-        data = tests,
+        srcs = tests,
         testonly = 1,
     )
     args = ["--test_targets_file=$(rootpath :{0})".format(tests_file)] + args

toolchain/check/testdata/class/fail_incomplete.carbon

@@ -8,51 +8,55 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/check/testdata/class/fail_incomplete.carbon
 
+// --- fail_forward_decl.carbon
+
+library "[[@TEST_NAME]]";
+
 class Class;
 
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:4: error(QualifiedDeclInIncompleteClassScope): cannot declare a member of incomplete class `Class`
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:4: error(QualifiedDeclInIncompleteClassScope): cannot declare a member of incomplete class `Class`
 // CHECK:STDERR: fn Class.Function() {}
 // CHECK:STDERR:    ^~~~~
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE-5]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-5]]:1: note(ClassForwardDeclaredHere): class was forward declared here
 // CHECK:STDERR: class Class;
 // CHECK:STDERR: ^~~~~~~~~~~~
 // CHECK:STDERR:
 fn Class.Function() {}
 
 fn CallClassFunction() {
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:3: error(QualifiedExprInIncompleteClassScope): member access into incomplete class `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:3: error(QualifiedExprInIncompleteClassScope): member access into incomplete class `Class`
   // CHECK:STDERR:   Class.Function();
   // CHECK:STDERR:   ^~~~~~~~~~~~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-15]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-15]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
   // CHECK:STDERR:
   Class.Function();
 }
 
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:17: error(IncompleteTypeInVarDecl): variable has incomplete type `Class`
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:17: error(IncompleteTypeInVarDecl): variable has incomplete type `Class`
 // CHECK:STDERR: var global_var: Class;
 // CHECK:STDERR:                 ^~~~~
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE-25]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-25]]:1: note(ClassForwardDeclaredHere): class was forward declared here
 // CHECK:STDERR: class Class;
 // CHECK:STDERR: ^~~~~~~~~~~~
 // CHECK:STDERR:
 var global_var: Class;
 
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:24: error(IncompleteTypeInFunctionReturnType): function returns incomplete type `Class`
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:24: error(IncompleteTypeInFunctionReturnType): function returns incomplete type `Class`
 // CHECK:STDERR: fn ConvertFromStruct() -> Class { return {}; }
 // CHECK:STDERR:                        ^~~~~~~~
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE-34]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-34]]:1: note(ClassForwardDeclaredHere): class was forward declared here
 // CHECK:STDERR: class Class;
 // CHECK:STDERR: ^~~~~~~~~~~~
 // CHECK:STDERR:
 fn ConvertFromStruct() -> Class { return {}; }
 
 fn G(p: Class*) -> i32 {
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:10: error(IncompleteTypeInMemberAccess): member access into object of incomplete type `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:10: error(IncompleteTypeInMemberAccess): member access into object of incomplete type `Class`
   // CHECK:STDERR:   return p->n;
   // CHECK:STDERR:          ^~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-44]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-44]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
   // CHECK:STDERR:
@@ -60,20 +64,20 @@ fn G(p: Class*) -> i32 {
 }
 
 fn MemberAccess(p: Class*) -> i32 {
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:11: error(IncompleteTypeInMemberAccess): member access into object of incomplete type `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:11: error(IncompleteTypeInMemberAccess): member access into object of incomplete type `Class`
   // CHECK:STDERR:   return (*p).n;
   // CHECK:STDERR:           ^~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-55]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-55]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
   // CHECK:STDERR:
   return (*p).n;
 }
 
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:20: error(IncompleteTypeInFunctionReturnType): function returns incomplete type `Class`
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:20: error(IncompleteTypeInFunctionReturnType): function returns incomplete type `Class`
 // CHECK:STDERR: fn Copy(p: Class*) -> Class {
 // CHECK:STDERR:                    ^~~~~~~~
-// CHECK:STDERR: fail_incomplete.carbon:[[@LINE-65]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+// CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-65]]:1: note(ClassForwardDeclaredHere): class was forward declared here
 // CHECK:STDERR: class Class;
 // CHECK:STDERR: ^~~~~~~~~~~~
 // CHECK:STDERR:
@@ -82,10 +86,10 @@ fn Copy(p: Class*) -> Class {
 }
 
 fn Let(p: Class*) {
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+7]]:10: error(IncompleteTypeInLetDecl): `let` binding has incomplete type `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+7]]:10: error(IncompleteTypeInLetDecl): `let` binding has incomplete type `Class`
   // CHECK:STDERR:   let c: Class = *p;
   // CHECK:STDERR:          ^~~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-77]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-77]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
   // CHECK:STDERR:
@@ -97,25 +101,25 @@ fn TakeIncomplete(c: Class);
 fn ReturnIncomplete() -> Class;
 
 fn CallTakeIncomplete(p: Class*) {
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+10]]:18: error(IncompleteTypeInValueConversion): forming value of incomplete type `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+10]]:18: error(IncompleteTypeInValueConversion): forming value of incomplete type `Class`
   // CHECK:STDERR:   TakeIncomplete(*p);
   // CHECK:STDERR:                  ^~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-92]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-92]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-11]]:19: note(InCallToFunctionParam): initializing function parameter
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-11]]:19: note(InCallToFunctionParam): initializing function parameter
   // CHECK:STDERR: fn TakeIncomplete(c: Class);
   // CHECK:STDERR:                   ^~~~~~~~
   // CHECK:STDERR:
   TakeIncomplete(*p);
 
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+10]]:18: error(IncompleteTypeInValueConversion): forming value of incomplete type `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+10]]:18: error(IncompleteTypeInValueConversion): forming value of incomplete type `Class`
   // CHECK:STDERR:   TakeIncomplete({});
   // CHECK:STDERR:                  ^~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-104]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-104]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-23]]:19: note(InCallToFunctionParam): initializing function parameter
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-23]]:19: note(InCallToFunctionParam): initializing function parameter
   // CHECK:STDERR: fn TakeIncomplete(c: Class);
   // CHECK:STDERR:                   ^~~~~~~~
   // CHECK:STDERR:
@@ -123,19 +127,34 @@ fn CallTakeIncomplete(p: Class*) {
 }
 
 fn CallReturnIncomplete() {
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE+9]]:3: error(IncompleteTypeInFunctionReturnType): function returns incomplete type `Class`
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE+10]]:3: error(IncompleteTypeInFunctionReturnType): function returns incomplete type `Class`
   // CHECK:STDERR:   ReturnIncomplete();
   // CHECK:STDERR:   ^~~~~~~~~~~~~~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-118]]:1: note(ClassForwardDeclaredHere): class was forward declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-118]]:1: note(ClassForwardDeclaredHere): class was forward declared here
   // CHECK:STDERR: class Class;
   // CHECK:STDERR: ^~~~~~~~~~~~
-  // CHECK:STDERR: fail_incomplete.carbon:[[@LINE-35]]:23: note(IncompleteReturnTypeHere): return type declared here
+  // CHECK:STDERR: fail_forward_decl.carbon:[[@LINE-35]]:23: note(IncompleteReturnTypeHere): return type declared here
   // CHECK:STDERR: fn ReturnIncomplete() -> Class;
   // CHECK:STDERR:                       ^~~~~~~~
+  // CHECK:STDERR:
   ReturnIncomplete();
 }
 
-// CHECK:STDOUT: --- fail_incomplete.carbon
+// --- fail_in_definition.carbon
+
+library "[[@TEST_NAME]]";
+
+class C {
+  // CHECK:STDERR: fail_in_definition.carbon:[[@LINE+6]]:10: error(IncompleteTypeInVarDecl): field has incomplete type `C`
+  // CHECK:STDERR:   var c: C;
+  // CHECK:STDERR:          ^
+  // CHECK:STDERR: fail_in_definition.carbon:[[@LINE-4]]:1: note(ClassIncompleteWithinDefinition): class is incomplete within its definition
+  // CHECK:STDERR: class C {
+  // CHECK:STDERR: ^~~~~~~~~
+  var c: C;
+}
+
+// CHECK:STDOUT: --- fail_forward_decl.carbon
 // CHECK:STDOUT:
 // CHECK:STDOUT: constants {
 // CHECK:STDOUT:   %Class: type = class_type @Class [template]
@@ -215,10 +234,10 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:     %p.param_patt: %.4 = param_pattern %p.patt, runtime_param0
 // CHECK:STDOUT:   } {
 // CHECK:STDOUT:     %Class.ref: type = name_ref Class, file.%Class.decl [template = constants.%Class]
-// CHECK:STDOUT:     %.loc51_14: type = ptr_type %Class [template = constants.%.4]
+// CHECK:STDOUT:     %.loc44_14: type = ptr_type %Class [template = constants.%.4]
 // CHECK:STDOUT:     %int.make_type_32: init type = call constants.%Int32() [template = i32]
-// CHECK:STDOUT:     %.loc51_20.1: type = value_of_initializer %int.make_type_32 [template = i32]
-// CHECK:STDOUT:     %.loc51_20.2: type = converted %int.make_type_32, %.loc51_20.1 [template = i32]
+// CHECK:STDOUT:     %.loc44_20.1: type = value_of_initializer %int.make_type_32 [template = i32]
+// CHECK:STDOUT:     %.loc44_20.2: type = converted %int.make_type_32, %.loc44_20.1 [template = i32]
 // CHECK:STDOUT:     %return: ref i32 = var <return slot>
 // CHECK:STDOUT:     %param: %.4 = param runtime_param0
 // CHECK:STDOUT:     %p: %.4 = bind_name p, %param
@@ -228,10 +247,10 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:     %p.param_patt: %.4 = param_pattern %p.patt, runtime_param0
 // CHECK:STDOUT:   } {
 // CHECK:STDOUT:     %Class.ref: type = name_ref Class, file.%Class.decl [template = constants.%Class]
-// CHECK:STDOUT:     %.loc62_25: type = ptr_type %Class [template = constants.%.4]
+// CHECK:STDOUT:     %.loc55_25: type = ptr_type %Class [template = constants.%.4]
 // CHECK:STDOUT:     %int.make_type_32: init type = call constants.%Int32() [template = i32]
-// CHECK:STDOUT:     %.loc62_31.1: type = value_of_initializer %int.make_type_32 [template = i32]
-// CHECK:STDOUT:     %.loc62_31.2: type = converted %int.make_type_32, %.loc62_31.1 [template = i32]
+// CHECK:STDOUT:     %.loc55_31.1: type = value_of_initializer %int.make_type_32 [template = i32]
+// CHECK:STDOUT:     %.loc55_31.2: type = converted %int.make_type_32, %.loc55_31.1 [template = i32]
 // CHECK:STDOUT:     %return: ref i32 = var <return slot>
 // CHECK:STDOUT:     %param: %.4 = param runtime_param0
 // CHECK:STDOUT:     %p: %.4 = bind_name p, %param
@@ -240,9 +259,9 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:     %p.patt: %.4 = binding_pattern p
 // CHECK:STDOUT:     %p.param_patt: %.4 = param_pattern %p.patt, runtime_param0
 // CHECK:STDOUT:   } {
-// CHECK:STDOUT:     %Class.ref.loc80_12: type = name_ref Class, file.%Class.decl [template = constants.%Class]
-// CHECK:STDOUT:     %.loc80: type = ptr_type %Class [template = constants.%.4]
-// CHECK:STDOUT:     %Class.ref.loc80_23: type = name_ref Class, file.%Class.decl [template = constants.%Class]
+// CHECK:STDOUT:     %Class.ref.loc73_12: type = name_ref Class, file.%Class.decl [template = constants.%Class]
+// CHECK:STDOUT:     %.loc73: type = ptr_type %Class [template = constants.%.4]
+// CHECK:STDOUT:     %Class.ref.loc73_23: type = name_ref Class, file.%Class.decl [template = constants.%Class]
 // CHECK:STDOUT:     %return: ref %Class = var <return slot>
 // CHECK:STDOUT:     %param: %.4 = param runtime_param0
 // CHECK:STDOUT:     %p: %.4 = bind_name p, %param
@@ -251,8 +270,8 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:     %p.patt: %.4 = binding_pattern p
 // CHECK:STDOUT:     %p.param_patt: %.4 = param_pattern %p.patt, runtime_param0
 // CHECK:STDOUT:   } {
-// CHECK:STDOUT:     %Class.ref.loc84: type = name_ref Class, file.%Class.decl [template = constants.%Class]
-// CHECK:STDOUT:     %.loc84: type = ptr_type %Class [template = constants.%.4]
+// CHECK:STDOUT:     %Class.ref.loc77: type = name_ref Class, file.%Class.decl [template = constants.%Class]
+// CHECK:STDOUT:     %.loc77: type = ptr_type %Class [template = constants.%.4]
 // CHECK:STDOUT:     %param: %.4 = param runtime_param0
 // CHECK:STDOUT:     %p: %.4 = bind_name p, %param
 // CHECK:STDOUT:   }
@@ -273,7 +292,7 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:     %p.param_patt: %.4 = param_pattern %p.patt, runtime_param0
 // CHECK:STDOUT:   } {
 // CHECK:STDOUT:     %Class.ref: type = name_ref Class, file.%Class.decl [template = constants.%Class]
-// CHECK:STDOUT:     %.loc99: type = ptr_type %Class [template = constants.%.4]
+// CHECK:STDOUT:     %.loc92: type = ptr_type %Class [template = constants.%.4]
 // CHECK:STDOUT:     %param: %.4 = param runtime_param0
 // CHECK:STDOUT:     %p: %.4 = bind_name p, %param
 // CHECK:STDOUT:   }
@@ -296,7 +315,7 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @ConvertFromStruct() -> %Class {
 // CHECK:STDOUT: !entry:
-// CHECK:STDOUT:   %.loc49: %.3 = struct_literal ()
+// CHECK:STDOUT:   %.loc42: %.3 = struct_literal ()
 // CHECK:STDOUT:   return <error>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
@@ -305,29 +324,29 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT: fn @G(%p.param_patt: %.4) -> i32 {
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %p.ref: %.4 = name_ref p, %p
-// CHECK:STDOUT:   %.loc59: ref %Class = deref %p.ref
+// CHECK:STDOUT:   %.loc52: ref %Class = deref %p.ref
 // CHECK:STDOUT:   return <error>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @MemberAccess(%p.param_patt: %.4) -> i32 {
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %p.ref: %.4 = name_ref p, %p
-// CHECK:STDOUT:   %.loc70: ref %Class = deref %p.ref
+// CHECK:STDOUT:   %.loc63: ref %Class = deref %p.ref
 // CHECK:STDOUT:   return <error>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @Copy(%p.param_patt: %.4) -> %Class {
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %p.ref: %.4 = name_ref p, %p
-// CHECK:STDOUT:   %.loc81: ref %Class = deref %p.ref
+// CHECK:STDOUT:   %.loc74: ref %Class = deref %p.ref
 // CHECK:STDOUT:   return <error>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @Let(%p.param_patt: %.4) {
 // CHECK:STDOUT: !entry:
-// CHECK:STDOUT:   %Class.ref.loc92: type = name_ref Class, file.%Class.decl [template = constants.%Class]
+// CHECK:STDOUT:   %Class.ref.loc85: type = name_ref Class, file.%Class.decl [template = constants.%Class]
 // CHECK:STDOUT:   %p.ref: %.4 = name_ref p, %p
-// CHECK:STDOUT:   %.loc92: ref %Class = deref %p.ref
+// CHECK:STDOUT:   %.loc85: ref %Class = deref %p.ref
 // CHECK:STDOUT:   %c: <error> = bind_name c, <error>
 // CHECK:STDOUT:   return
 // CHECK:STDOUT: }
@@ -338,13 +357,13 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:
 // CHECK:STDOUT: fn @CallTakeIncomplete(%p.param_patt: %.4) {
 // CHECK:STDOUT: !entry:
-// CHECK:STDOUT:   %TakeIncomplete.ref.loc110: %TakeIncomplete.type = name_ref TakeIncomplete, file.%TakeIncomplete.decl [template = constants.%TakeIncomplete]
+// CHECK:STDOUT:   %TakeIncomplete.ref.loc103: %TakeIncomplete.type = name_ref TakeIncomplete, file.%TakeIncomplete.decl [template = constants.%TakeIncomplete]
 // CHECK:STDOUT:   %p.ref: %.4 = name_ref p, %p
-// CHECK:STDOUT:   %.loc110: ref %Class = deref %p.ref
-// CHECK:STDOUT:   %TakeIncomplete.call.loc110: init %.1 = call %TakeIncomplete.ref.loc110(<invalid>) [template = <error>]
-// CHECK:STDOUT:   %TakeIncomplete.ref.loc122: %TakeIncomplete.type = name_ref TakeIncomplete, file.%TakeIncomplete.decl [template = constants.%TakeIncomplete]
-// CHECK:STDOUT:   %.loc122: %.3 = struct_literal ()
-// CHECK:STDOUT:   %TakeIncomplete.call.loc122: init %.1 = call %TakeIncomplete.ref.loc122(<invalid>) [template = <error>]
+// CHECK:STDOUT:   %.loc103: ref %Class = deref %p.ref
+// CHECK:STDOUT:   %TakeIncomplete.call.loc103: init %.1 = call %TakeIncomplete.ref.loc103(<invalid>) [template = <error>]
+// CHECK:STDOUT:   %TakeIncomplete.ref.loc115: %TakeIncomplete.type = name_ref TakeIncomplete, file.%TakeIncomplete.decl [template = constants.%TakeIncomplete]
+// CHECK:STDOUT:   %.loc115: %.3 = struct_literal ()
+// CHECK:STDOUT:   %TakeIncomplete.call.loc115: init %.1 = call %TakeIncomplete.ref.loc115(<invalid>) [template = <error>]
 // CHECK:STDOUT:   return
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
@@ -355,3 +374,41 @@ fn CallReturnIncomplete() {
 // CHECK:STDOUT:   return
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
+// CHECK:STDOUT: --- fail_in_definition.carbon
+// CHECK:STDOUT:
+// CHECK:STDOUT: constants {
+// CHECK:STDOUT:   %C: type = class_type @C [template]
+// CHECK:STDOUT: }
+// CHECK:STDOUT:
+// CHECK:STDOUT: imports {
+// CHECK:STDOUT:   %Core: <namespace> = namespace file.%Core.import, [template] {
+// CHECK:STDOUT:     import Core//prelude
+// CHECK:STDOUT:     import Core//prelude/operators
+// CHECK:STDOUT:     import Core//prelude/types
+// CHECK:STDOUT:     import Core//prelude/operators/arithmetic
+// CHECK:STDOUT:     import Core//prelude/operators/as
+// CHECK:STDOUT:     import Core//prelude/operators/bitwise
+// CHECK:STDOUT:     import Core//prelude/operators/comparison
+// CHECK:STDOUT:     import Core//prelude/types/bool
+// CHECK:STDOUT:   }
+// CHECK:STDOUT: }
+// CHECK:STDOUT:
+// CHECK:STDOUT: file {
+// CHECK:STDOUT:   package: <namespace> = namespace [template] {
+// CHECK:STDOUT:     .Core = imports.%Core
+// CHECK:STDOUT:     .C = %C.decl
+// CHECK:STDOUT:   }
+// CHECK:STDOUT:   %Core.import = import Core
+// CHECK:STDOUT:   %C.decl: type = class_decl @C [template = constants.%C] {} {}
+// CHECK:STDOUT: }
+// CHECK:STDOUT:
+// CHECK:STDOUT: class @C {
+// CHECK:STDOUT:   %C.ref: type = name_ref C, file.%C.decl [template = constants.%C]
+// CHECK:STDOUT:   %.loc11: <error> = field_decl c, element0 [template]
+// CHECK:STDOUT:   %.loc12: <witness> = complete_type_witness <error> [template = <error>]
+// CHECK:STDOUT:
+// CHECK:STDOUT: !members:
+// CHECK:STDOUT:   .Self = constants.%C
+// CHECK:STDOUT:   .c = %.loc11
+// CHECK:STDOUT: }
+// CHECK:STDOUT:

toolchain/diagnostics/BUILD

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+load("//bazel/manifest:defs.bzl", "manifest")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -57,6 +58,29 @@ cc_library(
     ],
 )
 
+manifest(
+    name = "all_testdata.txt",
+    srcs = ["//toolchain/testing:all_testdata"],
+)
+
+cc_test(
+    name = "emitted_diagnostics_test",
+    size = "small",
+    srcs = ["emitted_diagnostics_test.cpp"],
+    data = [
+        "all_testdata.txt",
+        "//toolchain/testing:all_testdata",
+    ],
+    deps = [
+        ":diagnostic_kind",
+        "//common:set",
+        "//testing/base:gtest_main",
+        "@googletest//:gtest",
+        "@llvm-project//llvm:Support",
+        "@re2",
+    ],
+)
+
 cc_library(
     name = "format_providers",
     srcs = ["format_providers.cpp"],

toolchain/diagnostics/emitted_diagnostics_test.cpp

@@ -0,0 +1,122 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <gtest/gtest.h>
+
+#include <fstream>
+#include <string>
+
+#include "common/set.h"
+#include "llvm/ADT/StringExtras.h"
+#include "re2/re2.h"
+#include "toolchain/diagnostics/diagnostic_kind.h"
+
+namespace Carbon {
+namespace {
+
+constexpr DiagnosticKind Diagnostics[] = {
+#define CARBON_DIAGNOSTIC_KIND(Name) DiagnosticKind::Name,
+#include "toolchain/diagnostics/diagnostic_kind.def"
+};
+
+// Returns true for diagnostics which have no tests. In general, diagnostics
+// should be tested.
+static auto IsUntestedDiagnostic(DiagnosticKind diagnostic_kind) -> bool {
+  switch (diagnostic_kind) {
+    case DiagnosticKind::TestDiagnostic:
+    case DiagnosticKind::TestDiagnosticNote:
+      // These exist only for unit tests.
+      return true;
+    case DiagnosticKind::ErrorReadingFile:
+    case DiagnosticKind::ErrorStattingFile:
+    case DiagnosticKind::FileTooLarge:
+      // These diagnose filesystem issues that are hard to unit test.
+      return true;
+    case DiagnosticKind::ArrayBoundTooLarge:
+      // Int literals are currently limited to i32. Once that's fixed, this
+      // should be tested.
+      return true;
+    case DiagnosticKind::ExternLibraryInImporter:
+    case DiagnosticKind::ExternLibraryOnDefinition:
+    case DiagnosticKind::HexadecimalEscapeMissingDigits:
+    case DiagnosticKind::ImplOfUndefinedInterface:
+    case DiagnosticKind::IncompleteTypeInFunctionParam:
+    case DiagnosticKind::InvalidDigit:
+    case DiagnosticKind::InvalidDigitSeparator:
+    case DiagnosticKind::InvalidHorizontalWhitespaceInString:
+    case DiagnosticKind::MismatchedIndentInString:
+    case DiagnosticKind::ModifierPrivateNotAllowed:
+    case DiagnosticKind::MultiLineStringWithDoubleQuotes:
+    case DiagnosticKind::NameAmbiguousDueToExtend:
+    case DiagnosticKind::TooManyDigits:
+    case DiagnosticKind::UnaryOperatorRequiresWhitespace:
+    case DiagnosticKind::UnicodeEscapeSurrogate:
+    case DiagnosticKind::UnicodeEscapeTooLarge:
+    case DiagnosticKind::UnknownBaseSpecifier:
+    case DiagnosticKind::UnsupportedCRLineEnding:
+    case DiagnosticKind::UnsupportedLFCRLineEnding:
+      // TODO: Should look closer at these, but adding tests is a high risk of
+      // loss in merge conflicts due to the amount of tests being changed right
+      // now.
+      return true;
+    default:
+      return false;
+  }
+}
+
+TEST(EmittedDiagnostics, Verify) {
+  std::ifstream manifest_in("toolchain/diagnostics/all_testdata.txt");
+  ASSERT_TRUE(manifest_in.good());
+
+  RE2 diagnostic_re(R"(\w\((\w+)\): )");
+  ASSERT_TRUE(diagnostic_re.ok()) << diagnostic_re.error();
+
+  Set<std::string> emitted_diagnostics;
+
+  std::string test_filename;
+  while (std::getline(manifest_in, test_filename)) {
+    std::ifstream test_in(test_filename);
+    ASSERT_TRUE(test_in.good());
+
+    std::string line;
+    while (std::getline(test_in, line)) {
+      std::string diagnostic;
+      if (RE2::PartialMatch(line, diagnostic_re, &diagnostic)) {
+        emitted_diagnostics.Insert(diagnostic);
+      }
+    }
+  }
+
+  llvm::SmallVector<llvm::StringRef> missing_diagnostics;
+  for (auto diagnostic_kind : Diagnostics) {
+    if (IsUntestedDiagnostic(diagnostic_kind)) {
+      EXPECT_FALSE(emitted_diagnostics.Erase(diagnostic_kind.name()))
+          << diagnostic_kind
+          << " was previously untested, and is now tested. That's good, but "
+             "please remove it from IsUntestedDiagnostic.";
+      continue;
+    }
+    if (!emitted_diagnostics.Erase(diagnostic_kind.name())) {
+      missing_diagnostics.push_back(diagnostic_kind.name());
+    }
+  }
+
+  constexpr llvm::StringLiteral Bullet = "\n  - ";
+
+  std::sort(missing_diagnostics.begin(), missing_diagnostics.end());
+  EXPECT_TRUE(missing_diagnostics.empty())
+      << "Some diagnostics have no tests:" << Bullet
+      << llvm::join(missing_diagnostics, Bullet);
+
+  llvm::SmallVector<std::string> unexpected_matches;
+  emitted_diagnostics.ForEach(
+      [&](const std::string& match) { unexpected_matches.push_back(match); });
+  std::sort(unexpected_matches.begin(), unexpected_matches.end());
+  EXPECT_TRUE(unexpected_matches.empty())
+      << "Matched things that don't appear to be diagnostics:" << Bullet
+      << llvm::join(unexpected_matches, Bullet);
+}
+
+}  // namespace
+}  // namespace Carbon

toolchain/driver/testdata/fail_missing_file.carbon

@@ -2,11 +2,11 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile not_file.carbon
+// ARGS: compile --include-diagnostic-kind not_file.carbon
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_missing_file.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_missing_file.carbon
-// CHECK:STDERR: not_file.carbon: error: error opening file for read: No such file or directory
+// CHECK:STDERR: not_file.carbon: error(ErrorOpeningFile): error opening file for read: No such file or directory

toolchain/lex/testdata/numeric_literals.carbon

@@ -7,10 +7,13 @@
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/lex/testdata/numeric_literals.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/numeric_literals.carbon
-// CHECK:STDOUT: - filename: numeric_literals.carbon
+
+// --- valid.carbon
+// CHECK:STDOUT: - filename: valid.carbon
 // CHECK:STDOUT:   tokens: [
 // CHECK:STDOUT:     { index:  0, kind:          'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
 
+
 fn F() {
 // CHECK:STDOUT:     { index:  1, kind:                 'Fn', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'fn', has_leading_space: true },
 // CHECK:STDOUT:     { index:  2, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true },
@@ -86,5 +89,33 @@ fn F() {
 }
 // CHECK:STDOUT:     { index: 54, kind:    'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '}', opening_token: 5, has_leading_space: true },
 
+
 // CHECK:STDOUT:     { index: 55, kind:            'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
 // CHECK:STDOUT:   ]
+// --- fail_binary_real.carbon
+// CHECK:STDOUT: - filename: fail_binary_real.carbon
+// CHECK:STDOUT:   tokens: [
+// CHECK:STDOUT:     { index: 0, kind:   'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+
+// CHECK:STDERR: fail_binary_real.carbon:[[@LINE+4]]:4: error(BinaryRealLiteral): binary real number literals are not supported
+// CHECK:STDERR: 0b1.0
+// CHECK:STDERR:    ^
+// CHECK:STDERR:
+0b1.0
+// CHECK:STDOUT:     { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '0b1.0', value: `2*2^-1`, has_leading_space: true },
+
+// CHECK:STDOUT:     { index: 2, kind:     'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
+// CHECK:STDOUT:   ]
+// --- fail_wrong_real_exponent
+// CHECK:STDOUT: - filename: fail_wrong_real_exponent
+// CHECK:STDOUT:   tokens: [
+// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+
+// CHECK:STDERR: fail_wrong_real_exponent:[[@LINE+3]]:4: error(WrongRealLiteralExponent): expected 'e' to introduce exponent
+// CHECK:STDERR: 1.0r3
+// CHECK:STDERR:    ^
+1.0r3
+// CHECK:STDOUT:     { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '1.0r3', has_leading_space: true },
+
+// CHECK:STDOUT:     { index: 2, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
+// CHECK:STDOUT:   ]

toolchain/testing/BUILD

@@ -8,6 +8,20 @@ load("//testing/file_test:rules.bzl", "file_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "all_testdata",
+    data = [
+        "//toolchain/check:testdata",
+        "//toolchain/codegen:testdata",
+        "//toolchain/diagnostics:testdata",
+        "//toolchain/driver:testdata",
+        "//toolchain/format:testdata",
+        "//toolchain/lex:testdata",
+        "//toolchain/lower:testdata",
+        "//toolchain/parse:testdata",
+    ],
+)
+
 cc_library(
     name = "compile_helper",
     testonly = 1,
@@ -29,16 +43,7 @@ file_test(
     timeout = "moderate",  # Taking >60 seconds in GitHub actions
     srcs = ["file_test.cpp"],
     env = cc_env(),
-    tests = [
-        "//toolchain/check:testdata",
-        "//toolchain/codegen:testdata",
-        "//toolchain/diagnostics:testdata",
-        "//toolchain/driver:testdata",
-        "//toolchain/format:testdata",
-        "//toolchain/lex:testdata",
-        "//toolchain/lower:testdata",
-        "//toolchain/parse:testdata",
-    ],
+    tests = [":all_testdata"],
     deps = [
         "//common:all_llvm_targets",
         "//common:error",