Migrate tree-sitter support to MODULE.bazel (#4783)

The WORKSPACE file is deprecated; support is already off by default, and
it'll be removed in the next major bazel release. Our main dependency is
tree-sitter, and I'm trying to address that here.

We're currently using https://github.com/elliottt/rules_tree_sitter, but
that hasn't been updated in a couple years, meaning it lacks
MODULE.bazel support. In the registry, there's
https://registry.bazel.build/modules/tree-sitter-bazel, but this is only
the *parser* libraries of tree-sitter, not the *generator*. I'm using it
for that much, at least.

For the *generator*, which transforms grammar.js to parser.c/h, I'm just
requiring a non-hermetic invocation (i.e., people who want to work on it
will need to install tree-sitter; see the README.md updates). I tried
running it manually, but parser.c is about 600 KB; pre-commit rejects
files that large and I don't think an exception makes sense to override
for this (it'd probably also grow substantially if the grammar were
updated to cover more syntax). In order to make the non-hermetic call
not break "bazel build //..." for most developers, I'm marking most
targets in the package as manual.

Note, I did look long and hard at using `aspect_rules_js`/`rules_nodejs`
to invoke npm. This took a lot of time, and I have a commit that's
mostly working, except I hit a point where it uses `declare_symlink`
which we disallow for compatibility reasons (commit "Lots of work for
figuring out rule_js uses declare_symlink" on the PR). As a consequence,
I think we can't use the primary supported ways to have hermetic npm
calls.

Also, `treesitter` -> `tree_sitter` because it's generally called
`tree-sitter`, two words. We even had a `treesitter/src/tree_sitter`
directory so it's a bit inconsistent.

As far as bugs here, the parser library breaks bazel queries, e.g. the
error:
```
ERROR: Evaluation of query "somepath(//..., @llvm-project//third-party/unittest:gtest)" failed: preloading transitive closure failed: no such package '@@[unknown repo 'platforms' requested from @@tree-sitter-bazel+]//': The repository '@@[unknown repo 'platforms' requested from @@tree-sitter-bazel+]' could not be resolved: No repository visible as '@platforms' from repository '@@tree-sitter-bazel+'
```

I'm just excluding tree_sitter from queries where I can to work around
the error.

.bazelrc

@@ -57,11 +57,6 @@ build --action_env=CMAKE_SYSROOT --host_action_env=CMAKE_SYSROOT
 build --per_file_copt=external/.*\.(c|cc|cpp|cxx)$@-w
 build --host_per_file_copt=external/.*\.(c|cc|cpp|cxx)$@-w
 
-# The `rules_treesitter` synthesized libraries don't allow us to inject flags,
-# and compile generated code where we can't fix warnings.
-build --per_file_copt=utils/treesitter/_treesitter.tree_sitter/.*\.c$@-w
-build --host_per_file_copt=utils/treesitter/_treesitter.tree_sitter/.*\.c$@-w
-
 # Default dynamic linking to off. While this can help build performance in some
 # edge cases with very large linked executables and a slow linker, between using
 # fast linkers on all platforms (LLD and the Apple linker), as well as having
@@ -128,17 +123,6 @@ build --allow_unresolved_symlinks=false
 # RC file here if present.
 try-import %workspace%/user.bazelrc
 
-# Incompatible with `rules_tree_sitter`.
-# TODO: WORKSPACE will be removed in bazel 9, and we need to move off.
-# TODO: The registry has a different treesitter rule set, and we should
-# investigate switching. See:
-# https://registry.bazel.build/modules/tree-sitter-bazel
-common --enable_workspace
-# This is on by default in bazel 8.
-common --incompatible_disallow_empty_glob=false
-# common --incompatible_auto_exec_groups
-# common --incompatible_disable_starlark_host_transitions
-
 # This excludes things like rules_android to reduce warnings.
 # TODO: There's a pending fix, so hopefully we can remove it soon.
 # - Issue: https://github.com/bazelbuild/bazel/issues/23929
@@ -150,11 +134,19 @@ common --incompatible_autoload_externally=+@rules_java,+@rules_python,+@rules_sh
 # TODO: Enable the flag once compatibility issues are fixed.
 # common --incompatible_disable_non_executable_java_binary
 
+# Incompatible with the clang-tidy build mode.
+# TODO: Enable the flag once compatibility issues are fixed.
+# common --incompatible_auto_exec_groups
+
 # Incompatible with `rules_cc`.
 # TODO: Enable the flag once compatibility issues are fixed.
 # common --incompatible_no_rule_outputs_param
 # common --incompatible_stop_exporting_language_modules
 
+# Incompatible with `rules_flex`.
+# TODO: Enable the flag once compatibility issues are fixed.
+# common --incompatible_disable_starlark_host_transitions
+
 # Incompatible with `rules_pkg`.
 # TODO: Enable the flag once compatibility issues are fixed.
 # common --incompatible_disable_target_default_provider_fields

.gitignore

@@ -35,7 +35,7 @@
 # vim temporary files
 .*.sw[a-p]
 
-# generated by utils/treesitter/helix.sh
+# generated by utils/tree_sitter/helix.sh
 /.helix/
 
 # Ignore .DS_Store files

MODULE.bazel

@@ -28,12 +28,13 @@ http_archive = use_repo_rule(
     "http_archive",
 )
 
-bazel_dep(name = "bazel_skylib", version = "1.7.1")
-bazel_dep(name = "rules_pkg", version = "1.0.1")
 bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2")
-bazel_dep(name = "re2", version = "2024-07-02.bcr.1")
-bazel_dep(name = "googletest", version = "1.15.2")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
 bazel_dep(name = "google_benchmark", version = "1.8.5")
+bazel_dep(name = "googletest", version = "1.15.2")
+bazel_dep(name = "re2", version = "2024-07-02.bcr.1")
+bazel_dep(name = "rules_pkg", version = "1.0.1")
+bazel_dep(name = "tree-sitter-bazel", version = "0.24.4")
 
 # The registry only has an old version. We use that here to avoid a miss but
 # override it with a newer version.

MODULE.bazel.lock

@@ -207,6 +207,8 @@
     "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7",
     "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5",
     "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216",
+    "https://bcr.bazel.build/modules/tree-sitter-bazel/0.24.4/MODULE.bazel": "5f23d216376ea547e4b8f8819c201e70c59010ca2badd2a89a226b03e048b847",
+    "https://bcr.bazel.build/modules/tree-sitter-bazel/0.24.4/source.json": "0ad3631cf2c38fb32445968acbc800f5bcff3503f7ca136c0b2d2c7b7d06a60c",
     "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
     "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9",
     "https://bcr.bazel.build/modules/xz/5.4.5.bcr.1/MODULE.bazel": "c037f75fa1b7e1ff15fbd15d807a8ce545e9b02f02df0a9777aa9aa7d8b268bb",
@@ -3713,6 +3715,36 @@
           ]
         ]
       }
+    },
+    "@@tree-sitter-bazel+//:extensions.bzl%tree_sitter_source_code": {
+      "general": {
+        "bzlTransitiveDigest": "7Be5pB4ksIvEy9VCPruRCllTYTjsiIJgaigRzR9e+u4=",
+        "usagesDigest": "+A8gOKWptG2ZhzIqOP3jb/D4vpatFdmbOLuveV6t1Yw=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "tree-sitter-raw": {
+            "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
+            "attributes": {
+              "urls": [
+                "https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v0.24.4.tar.gz"
+              ],
+              "sha256": "",
+              "integrity": "sha384-Af1RSXWjSyY7gKQ9W4kaUrJm6EaDFXfCDb1Ca25yZTBQ3hS9Ye7ev57dEGTvAbpz",
+              "strip_prefix": "tree-sitter-0.24.4",
+              "build_file_content": "exports_files(glob([\"lib/**\"]))"
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "tree-sitter-bazel+",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
     }
   }
 }

WORKSPACE

@@ -1,36 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-workspace(name = "carbon")
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-###############################################################################
-# Treesitter rules
-###############################################################################
-
-http_archive(
-    name = "rules_nodejs",
-    sha256 = "d124665ea12f89153086746821cf6c9ef93ab88360a50c1aeefa1fe522421704",
-    strip_prefix = "rules_nodejs-6.0.0-beta1",
-    url = "https://github.com/bazelbuild/rules_nodejs/releases/download/v6.0.0-beta1/rules_nodejs-v6.0.0-beta1.tar.gz",
-)
-
-load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains")
-
-nodejs_register_toolchains(
-    name = "nodejs",
-    node_version = DEFAULT_NODE_VERSION,
-)
-
-http_archive(
-    name = "rules_tree_sitter",
-    sha256 = "a09f177a2b8acb2f8a84def6ca0c41a5bd26b25634aa7313f22ade6c54e57ca1",
-    strip_prefix = "rules_tree_sitter-bc3a2131053207de7dfd9b24046b811ce770e35d",
-    urls = ["https://github.com/Maan2003/rules_tree_sitter/archive/bc3a2131053207de7dfd9b24046b811ce770e35d.tar.gz"],
-)
-
-load("@rules_tree_sitter//tree_sitter:tree_sitter.bzl", "tree_sitter_register_toolchains")
-
-tree_sitter_register_toolchains()

bazel/check_deps/BUILD

@@ -18,14 +18,17 @@ filegroup(
         "//migrate_cpp:rewriter",
         "//migrate_cpp/cpp_refactoring",
         "//toolchain/install:carbon-busybox",
-        "//utils/treesitter",
+        # The tree sitter rules can't be queried; evaluation fails on
+        # @platforms.
+        # "//utils/tree_sitter",
     ],
     tags = ["manual"],
 )
 
 genquery(
     name = "non_test_cc_deps.txt",
-    expression = "kind('cc.* rule', deps(//bazel/check_deps:non_test_cc_rules))",
+    expression =
+        "kind('cc.* rule', deps(//bazel/check_deps:non_test_cc_rules))",
     opts = [
         "--notool_deps",
         "--noimplicit_deps",

docs/design/lexical_conventions/words.md

@@ -38,7 +38,7 @@ in Unicode Normalization Form C (NFC).
 <!--
 Keep in sync:
 - utils/textmate/Syntaxes/Carbon.plist
-- utils/treesitter/queries/highlights.scm
+- utils/tree_sitter/queries/highlights.scm
 -->
 
 The following words are interpreted as keywords:

explorer/BUILD

@@ -95,12 +95,12 @@ file_test(
 )
 
 filegroup(
-    name = "treesitter_testdata",
+    name = "tree_sitter_testdata",
     srcs = glob(
         ["testdata/**/*.carbon"],
         exclude = [
             "testdata/**/fail_*",
         ],
     ),
-    visibility = ["//utils/treesitter:__pkg__"],
+    visibility = ["//utils/tree_sitter:__pkg__"],
 )

scripts/create_compdb.py

@@ -36,7 +36,9 @@ def _build_generated_files(bazel: str) -> None:
     kinds_query = (
         "filter("
         ' ".*\\.(h|cpp|cc|c|cxx|def|inc)$",'
-        ' kind("generated file", deps(//...))'
+        # tree_sitter is excluded here because it causes the query to failure on
+        # `@platforms`.
+        ' kind("generated file", deps(//... except //utils/tree_sitter/...))'
         ")"
     )
     generated_file_labels = subprocess.check_output(

scripts/forbid_llvm_googletest.py

@@ -38,7 +38,11 @@ def main() -> None:
     args = [
         scripts_utils.locate_bazel(),
         "query",
-        "somepath(//..., @llvm-project//third-party/unittest:gtest)",
+        "somepath("
+        # tree_sitter is excluded here because it causes the query to failure on
+        # `@platforms`.
+        + "//... except //utils/tree_sitter/..., "
+        + "@llvm-project//third-party/unittest:gtest)",
     ]
     p = subprocess.run(
         args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8"

utils/README.md

@@ -22,4 +22,4 @@ developers and developers writing Carbon code.
 Any editor that supports Language server protocol and/or tree-sitter is
 supported. The editor just needs to be configured manually.
 `bazel build //toolchain` produces the language server binary.
-`utils/treesitter` contains the treesitter grammar.
+`utils/tree_sitter` contains the tree-sitter grammar.

utils/nvim/setup.sh

@@ -11,7 +11,7 @@ ROOT="$(git rev-parse --show-toplevel)"
 mkdir -p ~/.config/nvim/{lua,parser,queries}
 
 # add highlight queries
-ln -sTf "$PWD/utils/treesitter/queries" ~/.config/nvim/queries/carbon
+ln -sTf "$PWD/utils/tree_sitter/queries" ~/.config/nvim/queries/carbon
 
 # add carbon.lua
 ln -sf "$PWD/utils/nvim/carbon.lua" ~/.config/nvim/lua/carbon.lua
@@ -19,7 +19,7 @@ ln -sf "$PWD/utils/nvim/carbon.lua" ~/.config/nvim/lua/carbon.lua
 # load carbon.lua on startup
 grep 'require "carbon"' ~/.config/nvim/init.lua || echo 'require "carbon"' >> ~/.config/nvim/init.lua
 
-# build treesitter
-cd utils/treesitter
+# build tree_sitter
+cd utils/tree_sitter
 tree-sitter generate
 clang -o ~/.config/nvim/parser/carbon.so -shared src/parser.c src/scanner.c -I ./src -Os -fPIC

utils/tree_sitter/BUILD

@@ -0,0 +1,118 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
+
+package(default_visibility = ["//bazel/check_deps:__pkg__"])
+
+# See README.md for instructions on tree-sitter setup and use. These rules are
+# manual because the tree-sitter invocation is non-hermetic, and most developers
+# won't have it installed; we don't want to break "bazel test //..." if we can
+# avoid it.
+#
+# We use tree-sitter non-hermetically for two key reasons:
+#
+# - The main way of hermetically using npms in bazel, `aspect_rules_js`, uses
+#   declare_symlink; we disallow that for important compatibility reasons.
+# - When generated, src/parser.c is over 500 KB, which is larger than we want to
+#   check in. It should also be expected to grow if the grammar becomes more complete.
+
+# Convenience target for running all tests, including manual tests.
+test_suite(
+    name = "tests",
+    tags = ["manual"],
+    tests = [
+        ":explorer_tests",
+        ":string_fail_tests",
+        ":string_tests",
+    ],
+)
+
+# Call tree-sitter to generate parser files.
+genrule(
+    name = "parser_files",
+    srcs = ["grammar.js"],
+    outs = [
+        "src/parser.c",
+        "src/tree_sitter/parser.h",
+    ],
+    cmd = "tree-sitter generate $(location grammar.js) &&\n" +
+          "cp src/parser.c $(location src/parser.c) &&\n" +
+          "cp src/tree_sitter/parser.h $(location src/tree_sitter/parser.h)",
+    tags = ["manual"],
+)
+
+cc_library(
+    name = "parser",
+    srcs = [
+        "src/scanner.c",
+        ":src/parser.c",
+    ],
+    hdrs = [":src/tree_sitter/parser.h"],
+    copts = ["-Wno-missing-prototypes"],
+    tags = ["manual"],
+    deps = ["@tree-sitter-bazel//:tree-sitter"],
+)
+
+cc_binary(
+    name = "test_runner",
+    testonly = 1,
+    srcs = ["test_runner.cpp"],
+    tags = ["manual"],
+    deps = [
+        ":parser",
+    ],
+)
+
+cc_test(
+    name = "explorer_tests",
+    size = "small",
+    srcs = ["test_runner.cpp"],
+    args = ["$(locations //explorer:tree_sitter_testdata)"],
+    data = ["//explorer:tree_sitter_testdata"],
+    tags = ["manual"],
+    deps = [
+        ":parser",
+    ],
+)
+
+filegroup(
+    name = "string_testdata",
+    srcs = glob(
+        ["testdata/string/*.carbon"],
+        exclude = ["testdata/string/fail_*.carbon"],
+    ),
+)
+
+filegroup(
+    name = "string_fail_testdata",
+    srcs = glob(["testdata/string/fail_*.carbon"]),
+)
+
+cc_test(
+    name = "string_tests",
+    size = "small",
+    srcs = ["test_runner.cpp"],
+    args = ["$(locations :string_testdata)"],
+    data = [":string_testdata"],
+    tags = ["manual"],
+    deps = [
+        ":parser",
+    ],
+)
+
+cc_test(
+    name = "string_fail_tests",
+    size = "small",
+    srcs = ["test_runner.cpp"],
+    args = ["$(locations :string_fail_testdata)"],
+    data = [":string_fail_testdata"],
+    env = {
+        "FAIL_TESTS": "1",
+    },
+    tags = ["manual"],
+    deps = [
+        ":parser",
+    ],
+)

utils/treesitter/README.md → utils/tree_sitter/README.md

@@ -1,13 +1,22 @@
+# Tree-sitter grammar for Carbon
+
 <!--
 Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 Exceptions. See /LICENSE for license information.
 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 -->
 
-# Tree-sitter grammar for Carbon
-
 Tree-sitter is currently used for syntax highlighting in supported editors.
 
+## Development
+
+We use a non-hermetic tree-sitter invocation, so it must be installed locally.
+To install tree-sitter, run:
+
+```
+npm install -g tree-sitter-cli
+```
+
 ## Editor Installation
 
 ### Helix

utils/treesitter/helix.sh → utils/tree_sitter/helix.sh

@@ -7,7 +7,7 @@
 set -euo pipefail
 
 ROOT="$(git rev-parse --show-toplevel)"
-cd "$ROOT/utils/treesitter"
+cd "$ROOT/utils/tree_sitter"
 
 tree-sitter generate --no-bindings
 

utils/treesitter/src/scanner.c → utils/tree_sitter/src/scanner.c

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "tree_sitter/parser.h"
+#include "utils/tree_sitter/src/tree_sitter/parser.h"
 
 enum TokenType {
   BINARY_STAR,

utils/treesitter/test_runner.cpp → utils/tree_sitter/test_runner.cpp

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 #include <tree_sitter/api.h>
-#include <tree_sitter/parser.h>
 
 #include <cstdlib>
 #include <filesystem>
@@ -13,6 +12,8 @@
 #include <string>
 #include <vector>
 
+#include "utils/tree_sitter/src/tree_sitter/parser.h"
+
 extern "C" {
 auto tree_sitter_carbon() -> TSLanguage*;
 }
@@ -29,7 +30,7 @@ static auto ReadFile(std::filesystem::path path) -> std::string {
 // TODO: use file_test.cpp
 auto main(int argc, char** argv) -> int {
   if (argc < 2) {
-    std::cerr << "Usage: treesitter_carbon_tester <file>...\n";
+    std::cerr << "Usage: test_runner <file>...\n";
     return 2;
   }
 

utils/treesitter/BUILD

@@ -1,75 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_test")
-load("@rules_tree_sitter//tree_sitter:tree_sitter.bzl", "tree_sitter_cc_library")
-
-package(default_visibility = [
-    "//bazel/check_deps:__pkg__",
-    "//installers:__subpackages__",
-])
-
-tree_sitter_cc_library(
-    name = "treesitter",
-    srcs = ["src/scanner.c"],
-    grammar = ["grammar.js"],
-)
-
-cc_binary(
-    name = "test_runner",
-    testonly = 1,
-    srcs = ["test_runner.cpp"],
-    deps = [
-        ":treesitter",
-    ],
-)
-
-cc_test(
-    name = "explorer_tests",
-    size = "small",
-    srcs = ["test_runner.cpp"],
-    args = ["$(locations //explorer:treesitter_testdata)"],
-    data = ["//explorer:treesitter_testdata"],
-    deps = [
-        ":treesitter",
-    ],
-)
-
-filegroup(
-    name = "string_testdata",
-    srcs = glob(
-        ["testdata/string/*.carbon"],
-        exclude = ["testdata/string/fail_*.carbon"],
-    ),
-)
-
-filegroup(
-    name = "string_fail_testdata",
-    srcs = glob(["testdata/string/fail_*.carbon"]),
-)
-
-cc_test(
-    name = "string_tests",
-    size = "small",
-    srcs = ["test_runner.cpp"],
-    args = ["$(locations :string_testdata)"],
-    data = [":string_testdata"],
-    deps = [
-        ":treesitter",
-    ],
-)
-
-cc_test(
-    name = "string_fail_tests",
-    size = "small",
-    srcs = ["test_runner.cpp"],
-    args = ["$(locations :string_fail_testdata)"],
-    data = [":string_fail_testdata"],
-    env = {
-        "FAIL_TESTS": "1",
-    },
-    deps = [
-        ":treesitter",
-    ],
-)