Refactor update_checks into a more generic lit_autoupdate (#2277)

I've refactored the script in order to make it work in more contexts, which is why the delta is lost. I've actually refactored a significant amount with the intent of making the logic easier to understand, because I was also adjusting bits of it.

Some key notes:

- Removes the multi-pass update that was dealing with unfixed line numbers in explorer (I think the current script should work in one pass)
  - Fixed explorer to handle multiple line numbers on the same line (turns out we can rely on local format for line numbers).
- Using execv instead of imports because making Python imports work in a setup like this feels like it's not worth it; only a nuisance.
- Adding __init__.py to satisfy mypy, which otherwise considers the lit_autoupdate.py scripts to be issues.
- Using py because I was thinking sh would be more platform-dependent. py should port better to Windows.
- Getting rid of [[ID#]] capture groups in the semantics-ir tests because with the autoupdate it's kind of moot (also, hard to autogenerate the pairs without relying on the %### value).

Note this does mean tests switch to more of a "make a change, see which tests change" setup. I don't know that that's a _bad_ thing though -- it's pretty much how tests are being written right now, which is why I went down this rabbit hole. It's a nuisance to make a change then _manually_ have to update a bunch of code.

My intent is to use this for to convert parse-tree tests to lit, but I wanted to do this with _existing_ tests first as a proof of concept and to make sure there's agreement.

bazel/testing/lit_autoupdate_base.py

@@ -0,0 +1,416 @@
+#!/usr/bin/env python3
+
+"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+from abc import ABC, abstractmethod
+import argparse
+from concurrent import futures
+import os
+from pathlib import Path
+import re
+import subprocess
+from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple
+
+# A prefix followed by a command to run for autoupdating checked output.
+AUTOUPDATE_MARKER = "// AUTOUPDATE: "
+
+# Indicates no autoupdate is requested.
+NOAUTOUPDATE_MARKER = "// NOAUTOUPDATE"
+
+
+class ParsedArgs(NamedTuple):
+    build_mode: str
+    build_target: str
+    cmd_replace: Tuple[str, str]
+    extra_check_replacements: List[Tuple[Pattern, Pattern, str]]
+    line_number_format: str
+    line_number_pattern: Pattern
+    testdata: str
+    tests: List[Path]
+
+
+def parse_args() -> ParsedArgs:
+    """Parses command-line arguments and flags."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("tests", nargs="*")
+    parser.add_argument(
+        "--build_mode",
+        metavar="MODE",
+        default="opt",
+        help="The build mode to use. Defaults to opt for faster execution.",
+    )
+    parser.add_argument(
+        "--build_target",
+        metavar="TARGET",
+        required=True,
+        help="The target to build.",
+    )
+    parser.add_argument(
+        "--cmd_replace",
+        nargs=2,
+        metavar=("BEFORE", "AFTER"),
+        required=True,
+        help="Adds a command replacement of `BEFORE` with `AFTER`. Typically "
+        "`BEFORE` will look like `%{name}`",
+    )
+    parser.add_argument(
+        "--extra_check_replacement",
+        nargs=3,
+        metavar=("MATCHING", "BEFORE", "AFTER"),
+        default=[],
+        action="append",
+        help="On a CHECK line with MATCHING, does a regex replacement of "
+        "BEFORE with AFTER.",
+    )
+    parser.add_argument(
+        "--line_number_format",
+        metavar="FORMAT",
+        default="[[@LINE%(delta)s]]",
+        help="An optional format string for line number delta replacements.",
+    )
+    parser.add_argument(
+        "--line_number_pattern",
+        metavar="PATTERN",
+        required=True,
+        help="A regular expression which matches line numbers to update as its "
+        "only group.",
+    )
+    parser.add_argument(
+        "--testdata",
+        metavar="PATH",
+        required=True,
+        help="The path to the testdata to update, relative to the workspace "
+        "root.",
+    )
+    parsed_args = parser.parse_args()
+    extra_check_replacements = [
+        (re.compile(line_matcher), re.compile(before), after)
+        for line_matcher, before, after in parsed_args.extra_check_replacement
+    ]
+    return ParsedArgs(
+        build_mode=parsed_args.build_mode,
+        build_target=parsed_args.build_target,
+        cmd_replace=parsed_args.cmd_replace,
+        extra_check_replacements=extra_check_replacements,
+        line_number_format=parsed_args.line_number_format,
+        line_number_pattern=re.compile(parsed_args.line_number_pattern),
+        testdata=parsed_args.testdata,
+        tests=[Path(test).resolve() for test in parsed_args.tests],
+    )
+
+
+def get_tests(testdata: str) -> Set[Path]:
+    """Get the list of tests from the filesystem."""
+    tests = set()
+    for root, _, files in os.walk(testdata):
+        for f in files:
+            if f in {"lit.cfg.py", "BUILD"}:
+                # Ignore the lit config.
+                continue
+            if os.path.splitext(f)[1] == ".carbon":
+                tests.add(Path(root).joinpath(f))
+            else:
+                exit(f"Unrecognized file type in testdata: {f}")
+    return tests
+
+
+class Line(ABC):
+    """A line that may appear in the resulting test file."""
+
+    @abstractmethod
+    def format(
+        self, *, output_line_number: int, line_number_remap: Dict[int, int]
+    ) -> str:
+        raise NotImplementedError
+
+
+class OriginalLine(Line):
+    """A line that was copied from the original test file."""
+
+    def __init__(self, line_number: int, text: str) -> None:
+        self.line_number = line_number
+        self.text = text
+
+    def format(self, **kwargs: Any) -> str:
+        return self.text
+
+
+class CheckLine(Line):
+    """A `// CHECK:` line generated from the test output.
+
+    If there's a line number, it'll be fixed up after we've figured out which
+    lines to include in the resulting test file and in what order, because
+    their contents depend on where an original input line appears in the output.
+    """
+
+    def __init__(
+        self,
+        out_line: str,
+        line_number_format: str,
+        line_number_pattern: Pattern,
+    ) -> None:
+        super().__init__()
+        self.indent = ""
+        self.out_line = out_line.rstrip()
+        self.line_number_format = line_number_format
+        self.line_number_pattern = line_number_pattern
+        self.line_numbers = [
+            int(n) - 1 for n in line_number_pattern.findall(self.out_line)
+        ]
+
+    def format(
+        self, *, output_line_number: int, line_number_remap: Dict[int, int]
+    ) -> str:
+        if not self.out_line:
+            return f"{self.indent}// CHECK-EMPTY:\n"
+        result = self.out_line
+        for line_number in self.line_numbers:
+            delta = line_number_remap[line_number] - output_line_number
+            # We use `:+d` here to produce `LINE-n` or `LINE+n` as appropriate.
+            result = self.line_number_pattern.sub(
+                self.line_number_format % {"delta": f"{delta:+d}"},
+                result,
+                count=1,
+            )
+        return f"{self.indent}// CHECK:{result}\n"
+
+
+class Autoupdate(NamedTuple):
+    line_number: int
+    cmd: str
+
+
+def find_autoupdate(test: str, orig_lines: List[str]) -> Optional[Autoupdate]:
+    """Figures out whether autoupdate should occur.
+
+    For AUTOUPDATE, returns the line and command. For NOAUTOUPDATE, returns
+    None.
+    """
+    found = 0
+    result = None
+    for line_number, line in enumerate(orig_lines):
+        if line.startswith(AUTOUPDATE_MARKER):
+            found += 1
+            result = Autoupdate(line_number, line[len(AUTOUPDATE_MARKER) :])
+        elif line.startswith(NOAUTOUPDATE_MARKER):
+            found += 1
+    if found == 0:
+        raise ValueError(
+            f"{test} must have either '{AUTOUPDATE_MARKER}' or "
+            f"'{NOAUTOUPDATE_MARKER}'"
+        )
+    elif found > 1:
+        raise ValueError(
+            f"{test} must have only one of '{AUTOUPDATE_MARKER}' or "
+            f"'{NOAUTOUPDATE_MARKER}'"
+        )
+    return result
+
+
+def replace_all(s: str, replacements: List[Tuple[str, str]]) -> str:
+    """Runs multiple replacements on a string."""
+    for before, after in replacements:
+        s = s.replace(before, after)
+    return s
+
+
+def get_matchable_test_output(
+    parsed_args: ParsedArgs,
+    test: str,
+    autoupdate_cmd: str,
+    extra_check_replacements: List[Tuple[Pattern, Pattern, str]],
+) -> List[str]:
+    """Runs the autoupdate command and returns the output lines."""
+    # Mirror lit.cfg.py substitutions; bazel runs don't need --prelude.
+    # Also replaces `%s` with the test file.
+    autoupdate_cmd = replace_all(
+        autoupdate_cmd, [parsed_args.cmd_replace, ("%s", test)]
+    )
+
+    # Run the autoupdate command to generate output.
+    # (`bazel run` would serialize)
+    p = subprocess.run(
+        autoupdate_cmd,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    out = p.stdout.decode("utf-8")
+
+    # `lit` uses full paths to the test file, so use a regex to ignore paths
+    # when used.
+    out = replace_all(
+        out,
+        [
+            ("{{", "{{[{][{]}}"),
+            ("[[", "{{[[][[]}}"),
+            # TODO: Maybe revisit and see if lit can be convinced to give a
+            # root-relative path.
+            (test, f"{{{{.*}}}}/{test}"),
+        ],
+    )
+    out_lines = out.splitlines()
+
+    for i, line in enumerate(out_lines):
+        for line_matcher, before, after in extra_check_replacements:
+            if line_matcher.match(line):
+                out_lines[i] = before.sub(after, line)
+
+    return out_lines
+
+
+def merge_lines(
+    line_number_format: str,
+    line_number_pattern: Pattern,
+    autoupdate_line_number: int,
+    raw_orig_lines: List[str],
+    out_lines: List[str],
+) -> List[Line]:
+    """Merges the original output and new lines."""
+    orig_lines = [
+        OriginalLine(i, line)
+        for i, line in enumerate(raw_orig_lines)
+        # Remove CHECK lines in the original output.
+        if not line.lstrip().startswith("// CHECK")
+    ]
+    check_lines = [
+        CheckLine(out_line, line_number_format, line_number_pattern)
+        for out_line in out_lines
+    ]
+
+    result_lines: List[Line] = []
+    # CHECK lines must go after AUTOUPDATE.
+    while orig_lines and orig_lines[0].line_number <= autoupdate_line_number:
+        result_lines.append(orig_lines.pop(0))
+    # Interleave the original lines and the CHECK: lines.
+    while orig_lines and check_lines:
+        # Original lines go first when the CHECK line is known and later.
+        if (
+            check_lines[0].line_numbers
+            and check_lines[0].line_numbers[0] > orig_lines[0].line_number
+        ):
+            result_lines.append(orig_lines.pop(0))
+        else:
+            check_line = check_lines.pop(0)
+            # Indent to match the next original line.
+            check_line.indent = re.findall("^ *", orig_lines[0].text)[0]
+            result_lines.append(check_line)
+    # One list is non-empty; append remaining lines from both to catch it.
+    result_lines.extend(orig_lines)
+    result_lines.extend(check_lines)
+
+    return result_lines
+
+
+def update_check(parsed_args: ParsedArgs, test: Path) -> bool:
+    """Updates the CHECK: lines for `test` by running explorer.
+
+    Returns true if a change was made.
+    """
+    with test.open() as f:
+        orig_lines = f.readlines()
+
+    # Make sure we're supposed to autoupdate.
+    autoupdate = find_autoupdate(str(test), orig_lines)
+    if autoupdate is None:
+        return False
+
+    # Determine the merged output lines.
+    out_lines = get_matchable_test_output(
+        parsed_args,
+        str(test),
+        autoupdate.cmd,
+        parsed_args.extra_check_replacements,
+    )
+    result_lines = merge_lines(
+        parsed_args.line_number_format,
+        parsed_args.line_number_pattern,
+        autoupdate.line_number,
+        orig_lines,
+        out_lines,
+    )
+
+    # Calculate the remap for original lines.
+    line_number_remap = dict(
+        [
+            (line.line_number, i)
+            for i, line in enumerate(result_lines)
+            if isinstance(line, OriginalLine)
+        ]
+    )
+    # If the last line of the original output was a CHECK, replace it with an
+    # empty line.
+    if orig_lines[-1].lstrip().startswith("// CHECK"):
+        line_number_remap[len(orig_lines) - 1] = len(result_lines) - 1
+
+    # Generate contents for any lines that depend on line numbers.
+    formatted_result_lines = [
+        line.format(output_line_number=i, line_number_remap=line_number_remap)
+        for i, line in enumerate(result_lines)
+    ]
+
+    # If nothing's changed, we're done.
+    if formatted_result_lines == orig_lines:
+        return False
+
+    # Interleave the new CHECK: lines with the tested content.
+    with test.open("w") as f:
+        f.writelines(formatted_result_lines)
+        return True
+
+
+def update_checks(parsed_args: ParsedArgs, tests: Set[Path]) -> None:
+    """Updates CHECK: lines in lit tests."""
+
+    def map_helper(test: Path) -> bool:
+        updated = update_check(parsed_args, test)
+        print(".", end="", flush=True)
+        return updated
+
+    print(f"Updating {len(tests)} lit test(s)...")
+    with futures.ThreadPoolExecutor() as exec:
+        # list() iterates in order to immediately propagate exceptions.
+        results = list(exec.map(map_helper, tests))
+
+    # Each update call indicates progress with a dot without a newline, so put a
+    # newline to wrap.
+    print(f"\nUpdated {results.count(True)} lit test(s).")
+
+
+def main() -> None:
+    # Parse arguments relative to the working directory.
+    parsed_args = parse_args()
+
+    # Remaining script logic should be relative to the repository root.
+    os.chdir(Path(__file__).parent.parent.parent)
+
+    if parsed_args.tests:
+        tests = set(parsed_args.tests)
+    else:
+        print("HINT: run `update_checks.py f1 f2 ...` to update specific tests")
+        tests = get_tests(parsed_args.testdata)
+
+    # Build inputs.
+    print("Building explorer...")
+    subprocess.check_call(
+        [
+            "bazel",
+            "build",
+            "-c",
+            parsed_args.build_mode,
+            parsed_args.build_target,
+        ]
+    )
+
+    # Run updates.
+    update_checks(parsed_args, tests)
+
+
+if __name__ == "__main__":
+    main()

explorer/README.md

@@ -99,7 +99,7 @@ To explain this boilerplate:
         [`lit` substitution](https://llvm.org/docs/CommandGuide/lit.html#substitutions)
         for the path to the given test file.
 -   The `AUTOUPDATE` line indicates that `CHECK` lines will be automatically
-    inserted immediately below by the `./update_checks.py` script.
+    inserted immediately below by the `./lit_autoupdate.py` script.
 -   The `CHECK` lines indicate expected output, verified by `FileCheck`.
     -   Where a `CHECK` line contains text like `{{.*}}`, the double curly
         braces indicate a contained regular expression.
@@ -107,7 +107,7 @@ To explain this boilerplate:
 
 ### Useful commands
 
--   `./update_checks.py` -- Updates expected output.
+-   `./lit_autodupate.py` -- Updates expected output.
 -   `bazel test ... --test_output=errors` -- Runs tests and prints any errors.
 
 ### Updating fuzzer logic after making AST changes

explorer/lit_autoupdate.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    # Calls the main script with explorer settings. This uses execv in order to
+    # avoid Python import behaviors.
+    actual_py = Path(__file__).parent.parent.joinpath(
+        "bazel", "testing", "lit_autoupdate_base.py"
+    )
+    args = [
+        sys.argv[0],
+        # Flags to configure for explorer testing.
+        "--build_target",
+        "//explorer",
+        "--cmd_replace",
+        "%{explorer}",
+        "./bazel-bin/explorer/explorer",
+        "--testdata",
+        "explorer/testdata",
+        "--line_number_pattern",
+        r"(?<=\.carbon:)(\d+)(?=(?:\D|$))",
+    ] + sys.argv[1:]
+    os.execv(actual_py, args)
+
+
+if __name__ == "__main__":
+    main()

explorer/testdata/assoc_const/fail_redefined.carbon

@@ -10,7 +10,7 @@ package ExplorerTest api;
 
 interface Iface {
   let T:! Type;
-  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:[[@LINE+1]]: Duplicate name `T` also found at {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:12
+  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:[[@LINE+1]]: Duplicate name `T` also found at {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:[[@LINE-1]]
   let T:! Type;
 }
 

explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon

@@ -14,7 +14,7 @@ class A{
     }
     destructor[me: Self]{
         Print("DESTRUCTOR A2 {0}",me.n);
-    // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:[[@LINE+1]]: Duplicate name `destructor` also found at {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:14
+    // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:[[@LINE+1]]: Duplicate name `destructor` also found at {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:[[@LINE-3]]
     }
     var n: i32;
 }

explorer/testdata/mixin/fail_field_member_name_clash.carbon

@@ -17,7 +17,7 @@ __mixin M {
 class C {
   let F: i32 = 0;
   __mix M;
-// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:14) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:18)
+// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE-6]]) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE-2]])
 }
 
 fn Main() -> i32 {

explorer/testdata/mixin/fail_method_member_name_clash.carbon

@@ -19,7 +19,7 @@ class C {
      return x;
   }
   __mix M;
-// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:14) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:20)
+// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE-8]]) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE-2]])
 }
 
 fn Main() -> i32 {

explorer/testdata/mixin/fail_mix_diamond_clash.carbon

@@ -28,7 +28,7 @@ __mixin M3 {
 class C {
   __mix M2;
   __mix M3;
-// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:13) is being mixed multiple times into C
+// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:[[@LINE-18]]) is being mixed multiple times into C
 }
 
 fn Main() -> i32 {

explorer/testdata/mixin/fail_mix_members_clash.carbon

@@ -21,7 +21,7 @@ __mixin M2 {
 class C {
   __mix M1;
   __mix M2;
-// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:18) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:13)
+// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE-6]]) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE-11]])
 }
 
 fn Main() -> i32 {

explorer/testdata/mixin/fail_recursive_mixing.carbon

@@ -12,7 +12,7 @@ __mixin M1 {
      return x;
   }
   __mix M1;
-// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:13) is being mixed multiple times into M1
+// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:[[@LINE-2]]) is being mixed multiple times into M1
 }
 
 class C {

explorer/testdata/name_lookup/fail_block_duplicate.carbon

@@ -10,7 +10,7 @@ package ExplorerTest api;
 
 fn Main() -> i32 {
   var x: i32 = 0;
-  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:12
+  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:[[@LINE-1]]
   var x: i32 = 0;
   return 0;
 }

explorer/testdata/name_lookup/fail_class_duplicate.carbon

@@ -10,7 +10,7 @@ package ExplorerTest api;
 
 class Foo {
   var x: i32;
-  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:12
+  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:[[@LINE-1]]
   var x: i32;
 }
 

explorer/testdata/name_lookup/fail_global_duplicate.carbon

@@ -9,7 +9,7 @@
 package ExplorerTest api;
 
 var x: i32 = 0;
-// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:11
+// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:[[@LINE-1]]
 var x: i32 = 0;
 
 fn Main() -> i32 {

explorer/testdata/returned_var/fail_duplicate_return_var.carbon

@@ -11,7 +11,7 @@ package ExplorerTest api;
 fn AddInt(a: i32, b: i32) -> i32 {
   returned var ret: i32 = a + b;
   if (a == b) {
-    // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:[[@LINE+1]]: Duplicate definition of returned var also found at {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:12
+    // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:[[@LINE+1]]: Duplicate definition of returned var also found at {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:[[@LINE-2]]
     returned var ret2: i32 = a + b;
   }
   return var;

explorer/testdata/returned_var/fail_missing_return.carbon

@@ -10,7 +10,7 @@ package ExplorerTest api;
 
 fn AddInt(a: i32, b: i32) -> i32 {
   returned var ret: i32 = a + b;
-  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:[[@LINE+1]]: `return <expression>` is not allowed with a returned var defined in scope: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:12
+  // CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:[[@LINE+1]]: `return <expression>` is not allowed with a returned var defined in scope: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:[[@LINE-1]]
   return ret;
 }
 

explorer/update_checks.py

@@ -1,303 +0,0 @@
-#!/usr/bin/env python3
-
-"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
-
-__copyright__ = """
-Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-Exceptions. See /LICENSE for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-import argparse
-from concurrent import futures
-import os
-import re
-import subprocess
-import sys
-from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Set
-
-_BIN = "./bazel-bin/explorer/explorer"
-_TESTDATA = "explorer/testdata"
-
-# A prefix followed by a command to run for autoupdating checked output.
-_AUTOUPDATE_MARKER = "// AUTOUPDATE: "
-
-# Indicates no autoupdate is requested.
-_NOAUTOUPDATE_MARKER = "// NOAUTOUPDATE"
-
-# A regexp matching lines that contain line number references.
-_LINE_NUMBER_RE = (
-    r"((?:SYNTAX|COMPILATION|RUNTIME) ERROR: [^:]*:)([1-9][0-9]*)(:.*)"
-)
-
-
-def _get_tests() -> Set[str]:
-    """Get the list of tests from the filesystem."""
-    tests = set()
-    for root, _, files in os.walk(_TESTDATA):
-        for f in files:
-            if f in {"lit.cfg.py", "BUILD"}:
-                # Ignore the lit config.
-                continue
-            if os.path.splitext(f)[1] == ".carbon":
-                tests.add(os.path.join(root, f))
-            else:
-                sys.exit("Unrecognized file type in testdata: %s" % f)
-    return tests
-
-
-class Line(ABC):
-    """A line that may appear in the resulting test file."""
-
-    @abstractmethod
-    def format(
-        self, *, output_line_number: int, line_number_remap: Dict[int, int]
-    ) -> str:
-        raise NotImplementedError
-
-
-class OriginalLine(Line):
-    """A line that was copied from the original test file."""
-
-    def __init__(self, line_number: int, text: str) -> None:
-        self.line_number = line_number
-        self.text = text
-
-    def format(self, **kwargs: Any) -> str:
-        return self.text
-
-
-class CheckLine(Line):
-    """A `// CHECK:` line generated from the test output."""
-
-    def __init__(self) -> None:
-        self.indent = ""
-
-    @staticmethod
-    def escape(s: str) -> str:
-        """Escape any FileCheck special characters in `s`."""
-        return s.replace("{{", "{{[{][{]}}").replace("[[", "{{[[][[]}}")
-
-    def print_before_line(self, line: int) -> bool:
-        """Determine if we'd prefer to print this CHECK before line `line`."""
-        return True
-
-
-class SimpleCheckLine(CheckLine):
-    """A `// CHECK:` line that checks for an exact string."""
-
-    def __init__(self, expected: str) -> None:
-        super().__init__()
-        self.expected = expected
-
-    def format(self, **kwargs: Any) -> str:
-        if self.expected:
-            return f"{self.indent}// CHECK:{self.expected}\n"
-        else:
-            return f"{self.indent}// CHECK-EMPTY:\n"
-
-
-class CheckLineWithLineNumber(CheckLine):
-    """A `// CHECK:` line where the expected output includes a line number.
-
-    Such result lines need to be fixed up after we've figured out which lines
-    to include in the resulting test file and in what order, because their
-    contents depend on where an original input line appears in the output.
-    """
-
-    def __init__(self, before: str, line_number: int, after: str) -> None:
-        super().__init__()
-        self.before = before
-        self.line_number = line_number
-        self.after = after
-
-    def format(
-        self, *, output_line_number: int, line_number_remap: Dict[int, int]
-    ) -> str:
-        delta = line_number_remap[self.line_number] - output_line_number
-        # We use `:+d` here to produce `LINE-n` or `LINE+n` as appropriate.
-        return (
-            f"{self.indent}// CHECK:{self.before}[[@LINE{delta:+d}]]"
-            + f"{self.after}\n"
-        )
-
-    def print_before_line(self, line: int) -> bool:
-        return line >= self.line_number
-
-
-def _make_check_line(out_line: str) -> CheckLine:
-    """Given a line of output, determine what CHECK line to produce."""
-    out_line = out_line.rstrip()
-    match = re.match(_LINE_NUMBER_RE, out_line)
-    if match:
-        # Convert from 1-based line numbers to 0-based indexes.
-        diagnostic_line_number = int(match[2]) - 1
-        return CheckLineWithLineNumber(
-            match[1], diagnostic_line_number, match[3]
-        )
-    else:
-        return SimpleCheckLine(out_line)
-
-
-def _should_produce_check_line(
-    check_line: CheckLine,
-    orig_line: Optional[OriginalLine],
-    autoupdate_index: int,
-) -> bool:
-    """Determine whether it's time to produce a given CHECK line."""
-    if not orig_line:
-        # If there's no original line, we have no choice.
-        return True
-    if orig_line.line_number <= autoupdate_index:
-        # Don't put any CHECK lines before the AUTOUPDATE line.
-        return False
-    return check_line.print_before_line(orig_line.line_number)
-
-
-def _update_check_once(test: str) -> bool:
-    """Updates the CHECK: lines for `test` by running explorer.
-
-    Returns True if the number of lines changes.
-    """
-    with open(test) as f:
-        orig_lines = f.readlines()
-
-    # Remove old OUT.
-    autoupdate_index = None
-    noautoupdate_index = None
-    for line_index, line in enumerate(orig_lines):
-        if line.startswith(_AUTOUPDATE_MARKER):
-            autoupdate_index = line_index
-            autoupdate_cmd = line[len(_AUTOUPDATE_MARKER) :]
-        if line.startswith(_NOAUTOUPDATE_MARKER):
-            noautoupdate_index = line_index
-    if autoupdate_index is None:
-        if noautoupdate_index is None:
-            raise ValueError(
-                "%s must have either '%s' or '%s'"
-                % (test, _AUTOUPDATE_MARKER, _NOAUTOUPDATE_MARKER)
-            )
-        else:
-            return False
-    elif noautoupdate_index is not None:
-        raise ValueError(
-            "%s has both '%s' and '%s', must have only one"
-            % (test, _AUTOUPDATE_MARKER, _NOAUTOUPDATE_MARKER)
-        )
-
-    # Mirror lit.cfg.py substitutions; bazel runs don't need --prelude.
-    autoupdate_cmd = autoupdate_cmd.replace("%{explorer}", _BIN)
-
-    # Run the autoupdate command to generate output.
-    # (`bazel run` would serialize)
-    p = subprocess.run(
-        autoupdate_cmd % test,
-        shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    out = p.stdout.decode("utf-8")
-
-    # `lit` uses full paths to the test file, so use a regex to ignore paths
-    # when used.
-    # TODO: Maybe revisit and see if lit can be convinced to give a
-    # root-relative path.
-    out = CheckLine.escape(out).replace(test, "{{.*}}/%s" % test)
-    out_lines = out.splitlines()
-
-    orig_line_iter = iter(
-        OriginalLine(i, line) for i, line in enumerate(orig_lines)
-    )
-    check_line_iter = iter(_make_check_line(out_line) for out_line in out_lines)
-    next_orig_line: Optional[OriginalLine] = next(orig_line_iter, None)
-    next_check_line: Optional[CheckLine] = next(check_line_iter, None)
-
-    # Interleave the original lines and the CHECK: lines into a list of
-    # `result_lines`.
-    result_lines: List[Line] = []
-    # Mapping from `orig_lines` indexes to `result_lines` indexes.
-    line_number_remap: Dict[int, int] = {}
-    while next_orig_line or next_check_line:
-        if next_check_line and _should_produce_check_line(
-            next_check_line, next_orig_line, autoupdate_index
-        ):
-            # Indent the CHECK: line to match the next original line.
-            if next_orig_line:
-                match = re.match(" *", next_orig_line.text)
-                if match:
-                    next_check_line.indent = match[0]
-            result_lines.append(next_check_line)
-            next_check_line = next(check_line_iter, None)
-        else:
-            assert next_orig_line, "no lines left"
-            # Include this original line if it isn't a CHECK: line.
-            if not re.match(" *// CHECK", next_orig_line.text):
-                line_number_remap[next_orig_line.line_number] = len(
-                    result_lines
-                )
-                result_lines.append(next_orig_line)
-            next_orig_line = next(orig_line_iter, None)
-
-    # Generate contents for any lines that depend on line numbers.
-    formatted_result_lines = [
-        line.format(output_line_number=i, line_number_remap=line_number_remap)
-        for i, line in enumerate(result_lines)
-    ]
-
-    # If nothing's changed, we're done.
-    if formatted_result_lines == orig_lines:
-        return False
-
-    # Interleave the new CHECK: lines with the tested content.
-    with open(test, "w") as f:
-        f.writelines(formatted_result_lines)
-    return True
-
-
-def _update_check(test: str) -> None:
-    """Wraps CHECK: updates for test files."""
-    # If the number of output lines changes, run again because output can be
-    # line-specific. However, output should stabilize quickly.
-    if (
-        _update_check_once(test)
-        and _update_check_once(test)
-        and _update_check_once(test)
-    ):
-        raise ValueError("The output of %s kept changing" % test)
-    print(".", end="", flush=True)
-
-
-def _update_checks(tests: Set[str]) -> None:
-    """Runs bazel to update CHECK: lines in lit tests."""
-
-    # Build all tests at once in order to allow parallel updates.
-    print("Building explorer...")
-    subprocess.check_call(["bazel", "build", "//explorer"])
-
-    print("Updating %d lit test(s)..." % len(tests))
-    with futures.ThreadPoolExecutor() as exec:
-        # list() iterates to propagate exceptions.
-        list(exec.map(_update_check, tests))
-    # Each update call indicates progress with a dot without a newline, so put a
-    # newline to wrap.
-    print("\nUpdated lit tests.")
-
-
-def main() -> None:
-    # Go to the repository root so that paths will match bazel's view.
-    os.chdir(os.path.join(os.path.dirname(__file__), ".."))
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("tests", nargs="*")
-    args = parser.parse_args()
-    if args.tests:
-        tests = set(args.tests)
-    else:
-        print("HINT: run `update_checks.py f1 f2 ...` to update specific tests")
-        tests = _get_tests()
-    _update_checks(tests)
-
-
-if __name__ == "__main__":
-    main()

toolchain/lexer/lit_autoupdate.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    # Calls the main script with explorer settings. This uses execv in order to
+    # avoid Python import behaviors.
+    actual_py = Path(__file__).parent.parent.parent.joinpath(
+        "bazel", "testing", "lit_autoupdate_base.py"
+    )
+    args = [
+        sys.argv[0],
+        # Flags to configure for explorer testing.
+        "--build_target",
+        "//toolchain/driver:carbon",
+        "--cmd_replace",
+        "%{carbon}",
+        "./bazel-bin/toolchain/driver/carbon",
+        # Ignore the resulting column of EndOfFile because it's typically the
+        # end of the CHECK comment.
+        "--extra_check_replacement",
+        ".*'EndOfFile'",
+        r"column: (?:\d+)",
+        "column: {{[0-9]+}}",
+        # Ignore spaces that are used to columnize lines.
+        "--line_number_format",
+        "{{ *}}[[@LINE%(delta)s]]",
+        "--line_number_pattern",
+        r"(?<= line: )( *\d+)(?=,)",
+        "--testdata",
+        "toolchain/lexer/testdata",
+    ] + sys.argv[1:]
+    os.execv(actual_py, args)
+
+
+if __name__ == "__main__":
+    main()

toolchain/lexer/testdata/BUILD

@@ -0,0 +1,15 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("//bazel/testing:lit.bzl", "glob_lit_tests")
+
+glob_lit_tests(
+    data = [
+        "//toolchain/driver:carbon",
+        "@llvm-project//llvm:FileCheck",
+        "@llvm-project//llvm:not",
+    ],
+    driver = "lit.cfg.py",
+    test_file_exts = ["carbon"],
+)

toolchain/driver/testdata/carbon_test.carbon → toolchain/lexer/testdata/carbon_test.carbon

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump tokens %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump tokens %s
 
 // CHECK:token: { index:  0, kind:              'Fn', line: {{ *}}[[@LINE+7]], column:   1, indent: 1, spelling: 'fn', has_trailing_space: true }
 // CHECK:token: { index:  1, kind:      'Identifier', line: {{ *}}[[@LINE+6]], column:   4, indent: 1, spelling: 'run', identifier: 0 }
@@ -18,4 +19,4 @@ fn run(String program) {
   return True;
 // CHECK:token: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+1]], column:   1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true }
 }
-// CHECK:token: { index: 11, kind:       'EndOfFile', line: {{ *}}[[@LINE]], column: {{[0-9]+}}, indent: 1, spelling: '' }
+// CHECK:token: { index: 11, kind:       'EndOfFile', line: {{ *}}[[@LINE+0]], column: {{[0-9]+}}, indent: 1, spelling: '' }

toolchain/lexer/testdata/lit.cfg.py

@@ -0,0 +1 @@
+../../../bazel/testing/lit.cfg.py

toolchain/semantics/lit_autoupdate.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    # Calls the main script with explorer settings. This uses execv in order to
+    # avoid Python import behaviors.
+    actual_py = Path(__file__).parent.parent.parent.joinpath(
+        "bazel", "testing", "lit_autoupdate_base.py"
+    )
+    args = [
+        sys.argv[0],
+        # Flags to configure for explorer testing.
+        "--build_target",
+        "//toolchain/driver:carbon",
+        "--cmd_replace",
+        "%{carbon}",
+        "./bazel-bin/toolchain/driver/carbon",
+        # TODO: This should eventually have lines in output, but it doesn't
+        # right now.
+        "--line_number_pattern",
+        "UNUSED",
+        "--testdata",
+        "toolchain/semantics/testdata",
+    ] + sys.argv[1:]
+    os.execv(actual_py, args)
+
+
+if __name__ == "__main__":
+    main()

toolchain/semantics/testdata/empty.carbon

@@ -3,5 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump semantics-ir %s
 // CHECK:{
 // CHECK:}

toolchain/semantics/testdata/function/basic.carbon

@@ -3,12 +3,13 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump semantics-ir %s
 // CHECK:{
 // CHECK:  Function(
-// CHECK:      %[[ID0:[0-9]+]],
+// CHECK:      %0,
 // CHECK:      {
 // CHECK:      }),
-// CHECK:  SetName(`Foo`, %[[ID0]]),
+// CHECK:  SetName(`Foo`, %0),
 // CHECK:}
 
 fn Foo() {}

toolchain/semantics/testdata/function/order.carbon

@@ -3,22 +3,23 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump semantics-ir %s
 // CHECK:{
 // CHECK:  Function(
-// CHECK:      %[[ID0:[0-9]+]],
+// CHECK:      %2,
 // CHECK:      {
 // CHECK:      }),
-// CHECK:  SetName(`Foo`, %[[ID0]]),
+// CHECK:  SetName(`Foo`, %2),
 // CHECK:  Function(
-// CHECK:      %[[ID1:[0-9]+]],
+// CHECK:      %1,
 // CHECK:      {
 // CHECK:      }),
-// CHECK:  SetName(`Bar`, %[[ID1]]),
+// CHECK:  SetName(`Bar`, %1),
 // CHECK:  Function(
-// CHECK:      %[[ID2:[0-9]+]],
+// CHECK:      %0,
 // CHECK:      {
 // CHECK:      }),
-// CHECK:  SetName(`Baz`, %[[ID2]]),
+// CHECK:  SetName(`Baz`, %0),
 // CHECK:}
 
 fn Foo() {}

toolchain/semantics/testdata/return/binary_op.carbon

@@ -3,16 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump semantics-ir %s
 // CHECK:{
 // CHECK:  Function(
-// CHECK:      %[[ID0:[0-9]+]],
+// CHECK:      %0,
 // CHECK:      {
-// CHECK:        IntegerLiteral(%[[ID1:[0-9]+]], 12),
-// CHECK:        IntegerLiteral(%[[ID2:[0-9]+]], 34),
-// CHECK:        BinaryOperator(%[[ID3:[0-9]+]], +, %[[ID1]], %[[ID2]]),
-// CHECK:        Return(%[[ID3]]),
+// CHECK:        IntegerLiteral(%3, 12),
+// CHECK:        IntegerLiteral(%2, 34),
+// CHECK:        BinaryOperator(%1, +, %3, %2),
+// CHECK:        Return(%1),
 // CHECK:      }),
-// CHECK:  SetName(`Main`, %[[ID0]]),
+// CHECK:  SetName(`Main`, %0),
 // CHECK:}
 
 fn Main() {

toolchain/semantics/testdata/return/literal.carbon

@@ -3,14 +3,15 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump semantics-ir %s
 // CHECK:{
 // CHECK:  Function(
-// CHECK:      %[[ID0:[0-9]+]],
+// CHECK:      %0,
 // CHECK:      {
-// CHECK:        IntegerLiteral(%[[ID1:[0-9]+]], 0),
-// CHECK:        Return(%[[ID1]]),
+// CHECK:        IntegerLiteral(%1, 0),
+// CHECK:        Return(%1),
 // CHECK:      }),
-// CHECK:  SetName(`Main`, %[[ID0]]),
+// CHECK:  SetName(`Main`, %0),
 // CHECK:}
 
 fn Main() {

toolchain/semantics/testdata/return/trivial.carbon

@@ -3,13 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
+// AUTOUPDATE: %{carbon} dump semantics-ir %s
 // CHECK:{
 // CHECK:  Function(
-// CHECK:      %[[ID0:[0-9]+]],
+// CHECK:      %0,
 // CHECK:      {
 // CHECK:        Return(None),
 // CHECK:      }),
-// CHECK:  SetName(`Main`, %[[ID0]]),
+// CHECK:  SetName(`Main`, %0),
 // CHECK:}
 
 fn Main() {