Start adapting to bzlmod configurations. (#3505)

Some background information is at https://bazel.build/external/migration

Trying to handle the simple cases first. This adds a requirement for
bazel 7 due to differences in bzlmod handling between 6 and 7 (also
discussed on
[#infra](https://discord.com/channels/655572317891461132/707150492370862090/1184942191412510720)).
Bazel seems to be okay with a partial migration such as ths.

The python import behavior has subtly shifted, so `carbon.` is no longer
part of import paths. There's a version-incompatible change for `@@`.
bzlmod makes repos sometimes show as `name~version`.

`target-determinator` seems to be okay with `@@` after a version update.

Things not moved here are things that basically need more dep work:

- clang_register_toolchains because I need to dive into its format.
- llvm-project because we need something slightly atypical, I need to
make sure patching and the repo work carries over.
- com_google_libprotobuf_mutator is sufficiently atypical that it
doesn't have a module already, but should be one of the easier things to
fix.
- brotli/woff2: I think we should actually consider removing these. But
again, they're not trivial moves.
- treesitter due to toolchain registration, which has shifted a bit.
- rules_nodejs because treesitter depends on it in an awkward way to
migrate.

.bazeliskrc

@@ -4,4 +4,4 @@
 
 # Keep pinned to a recent release, listed at
 # https://github.com/bazelbuild/bazel.
-USE_BAZEL_VERSION=6.0.0
+USE_BAZEL_VERSION=7.0.0

.bazelrc

@@ -2,10 +2,6 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# TODO: https://bazel.build/external/migration for
-# https://github.com/bazelbuild/bazel/issues/18958
-build --noenable_bzlmod
-
 # Default to using a disk cache to minimize re-building LLVM and Clang which we
 # try to avoid updating too frequently to minimize rebuild cost. The location
 # here can be overridden in the user configuration where needed.
@@ -19,25 +15,6 @@ build --disk_cache=~/.cache/carbon-lang-build-cache
 # future Bazel releases.
 build --experimental_guard_against_concurrent_changes
 
-# Allow the toolchain to configure itself differently in the host build from
-# the target build. Even when the host and target platforms are ostensibly the
-# same (and use identical toolchains), it is beneficial to be able to configure
-# the specific toolchain features enabled for the target separately from the
-# host. For example, sanitizers make sense for the target significantly more
-# than for the host.
-#
-# Bazel bug tracking undoing the default here:
-# https://github.com/bazelbuild/bazel/issues/13315
-#
-# TODO: Remove this once we require at least Bazel version 7, where
-# --incompatible_use_host_features is the default.
-build --incompatible_dont_enable_host_nonhost_crosstool_features=false
-
-# Opt into the new Bazel toolchain resolution mechanism.
-# TODO: Remove this once we require at least Bazel version 7, where it is the
-# default.
-build --incompatible_enable_cc_toolchain_resolution
-
 # Disable warnings for all external compilations. These involve code that isn't
 # developed as part of Carbon and may be difficult or impossible to patch, so
 # warnings aren't likely to be actionable.

.pre-commit-config.yaml

@@ -219,6 +219,7 @@ repos:
 # This excludes third-party code, and patches to third-party code.
 exclude: |
   (?x)^(
+      MODULE.bazel.lock|
       bazel/patches/.*\.patch|
       third_party/examples/.*/carbon/.*|
       third_party/llvm-project/.*|

MODULE.bazel

@@ -0,0 +1,64 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Bazel modules."""
+
+module(name = "carbon")
+
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(
+    name = "abseil-cpp",
+    version = "20230802.0",
+    repo_name = "com_google_absl",
+)
+bazel_dep(
+    name = "re2",
+    version = "2023-11-01",
+    repo_name = "com_googlesource_code_re2",
+)
+bazel_dep(
+    name = "googletest",
+    version = "1.14.0.bcr.1",
+    repo_name = "com_google_googletest",
+)
+bazel_dep(
+    name = "google_benchmark",
+    version = "1.8.3",
+    repo_name = "com_github_google_benchmark",
+)
+bazel_dep(name = "rules_bison", version = "0.2.2")
+bazel_dep(name = "rules_flex", version = "0.2.1")
+bazel_dep(name = "rules_m4", version = "0.2.3")
+bazel_dep(name = "rules_cc", version = "0.0.9")
+bazel_dep(name = "rules_proto", version = "6.0.0-rc1")
+bazel_dep(
+    name = "protobuf",
+    version = "21.7",
+    repo_name = "com_google_protobuf",
+)
+
+###############################################################################
+# Python
+###############################################################################
+
+bazel_dep(name = "rules_python", version = "0.27.1")
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    python_version = "3.11",
+)
+use_repo(python, "python_versions")
+
+pip = use_extension(
+    "@rules_python//python/extensions:pip.bzl",
+    "pip",
+)
+
+# Create a central repo that knows about the pip dependencies.
+pip.parse(
+    hub_name = "py_deps",
+    python_version = "3.11",
+    requirements_lock = "//github_tools:requirements.txt",
+)
+use_repo(pip, "py_deps")

WORKSPACE

@@ -7,54 +7,6 @@ workspace(name = "carbon")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
-skylib_version = "1.3.0"
-
-http_archive(
-    name = "bazel_skylib",
-    sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
-    urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(skylib_version),
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(skylib_version),
-    ],
-)
-
-###############################################################################
-# Python rules
-###############################################################################
-
-rules_python_version = "0.27.0"
-
-# Add Bazel's python rules and set up pip.
-http_archive(
-    name = "rules_python",
-    sha256 = "9acc0944c94adb23fba1c9988b48768b1bacc6583b52a2586895c5b7491e2e31",
-    strip_prefix = "rules_python-{0}".format(rules_python_version),
-    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/{0}.tar.gz".format(rules_python_version),
-)
-
-load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
-
-py_repositories()
-
-python_register_toolchains(
-    name = "python311",
-    python_version = "3.11",
-)
-
-load("@python311//:defs.bzl", "interpreter")
-load("@rules_python//python:pip.bzl", "pip_parse")
-
-# Create a central repo that knows about the pip dependencies.
-pip_parse(
-    name = "py_deps",
-    python_interpreter_target = interpreter,
-    requirements_lock = "//github_tools:requirements.txt",
-)
-
-load("@py_deps//:requirements.bzl", "install_deps")
-
-install_deps()
-
 ###############################################################################
 # C++ rules
 ###############################################################################
@@ -67,61 +19,6 @@ load(
 
 clang_register_toolchains(name = "bazel_cc_toolchain")
 
-###############################################################################
-# Abseil libraries
-###############################################################################
-
-# Head as of 2023-07-31.
-abseil_version = "407f2fdd5ec6f79287919486aa5869b346093906"
-
-http_archive(
-    name = "com_google_absl",
-    sha256 = "953a914ac42f87caf5ed6a86890e183ae4e2bc69666a90c67605091d6e77e502",
-    strip_prefix = "abseil-cpp-{0}".format(abseil_version),
-    urls = ["https://github.com/abseil/abseil-cpp/archive/{0}.tar.gz".format(abseil_version)],
-)
-
-###############################################################################
-# RE2 libraries
-###############################################################################
-
-# Head as of 2023-07-31.
-re2_version = "960c861764ff54c9a12ff683ba55ccaad1a8f73b"
-
-http_archive(
-    name = "com_googlesource_code_re2",
-    sha256 = "8315f22198c25e9f7f1a3754566824710c08ddbb39d93e9920f4a131e871fc15",
-    strip_prefix = "re2-{0}".format(re2_version),
-    urls = ["https://github.com/google/re2/archive/{0}.tar.gz".format(re2_version)],
-)
-
-###############################################################################
-# GoogleTest libraries
-###############################################################################
-
-# Head as of 2023-07-31.
-googletest_version = "c875c4e2249ec124c24f72141b3780c22256fd44"
-
-http_archive(
-    name = "com_google_googletest",
-    sha256 = "21e0cd1110ba534409facccdda1bad90174e7ee7ded60c00dd2b43b4df654080",
-    strip_prefix = "googletest-{0}".format(googletest_version),
-    urls = ["https://github.com/google/googletest/archive/{0}.tar.gz".format(googletest_version)],
-)
-
-###############################################################################
-# Google Benchmark libraries
-###############################################################################
-
-benchmark_version = "1.6.1"
-
-http_archive(
-    name = "com_github_google_benchmark",
-    sha256 = "6132883bc8c9b0df5375b16ab520fac1a85dc9e4cf5be59480448ece74b278d4",
-    strip_prefix = "benchmark-{0}".format(benchmark_version),
-    urls = ["https://github.com/google/benchmark/archive/refs/tags/v{0}.tar.gz".format(benchmark_version)],
-)
-
 ###############################################################################
 # LLVM libraries
 ###############################################################################
@@ -178,80 +75,6 @@ maybe(
     ],
 )
 
-###############################################################################
-# Flex/Bison rules
-###############################################################################
-
-rules_m4_version = "0.2.3"
-
-http_archive(
-    name = "rules_m4",
-    sha256 = "10ce41f150ccfbfddc9d2394ee680eb984dc8a3dfea613afd013cfb22ea7445c",
-    urls = ["https://github.com/jmillikin/rules_m4/releases/download/v{0}/rules_m4-v{0}.tar.xz".format(rules_m4_version)],
-)
-
-load("@rules_m4//m4:m4.bzl", "m4_register_toolchains")
-
-# When building M4, disable all compiler warnings as we can't realistically fix
-# them anyways.
-m4_register_toolchains(extra_copts = ["-w"])
-
-rules_flex_version = "0.2.1"
-
-http_archive(
-    name = "rules_flex",
-    sha256 = "8929fedc40909d19a4b42548d0785f796c7677dcef8b5d1600b415e5a4a7749f",
-    urls = ["https://github.com/jmillikin/rules_flex/releases/download/v{0}/rules_flex-v{0}.tar.xz".format(rules_flex_version)],
-)
-
-load("@rules_flex//flex:flex.bzl", "flex_register_toolchains")
-
-# When building Flex, disable all compiler warnings as we can't realistically
-# fix them anyways.
-flex_register_toolchains(extra_copts = ["-w"])
-
-rules_bison_version = "0.2.2"
-
-http_archive(
-    name = "rules_bison",
-    sha256 = "2279183430e438b2dc77cacd7b1dbb63438971b2411406570f1ddd920b7c9145",
-    urls = ["https://github.com/jmillikin/rules_bison/releases/download/v{0}/rules_bison-v{0}.tar.xz".format(rules_bison_version)],
-)
-
-load("@rules_bison//bison:bison.bzl", "bison_register_toolchains")
-
-# When building Bison, disable all compiler warnings as we can't realistically
-# fix them anyways.
-bison_register_toolchains(extra_copts = ["-w"])
-
-###############################################################################
-# Protocol buffers - for structured fuzzer testing.
-###############################################################################
-
-rules_cc_version = "0.0.4"
-
-http_archive(
-    name = "rules_cc",
-    sha256 = "af6cc82d87db94585bceeda2561cb8a9d55ad435318ccb4ddfee18a43580fb5d",
-    strip_prefix = "rules_cc-{0}".format(rules_cc_version),
-    urls = ["https://github.com/bazelbuild/rules_cc/releases/download/{0}/rules_cc-{0}.tar.gz".format(rules_cc_version)],
-)
-
-rules_proto_version = "5.3.0-21.7"
-
-http_archive(
-    name = "rules_proto",
-    sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
-    strip_prefix = "rules_proto-{0}".format(rules_proto_version),
-    urls = ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/{0}.tar.gz".format(rules_proto_version)],
-)
-
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-
-rules_proto_dependencies()
-
-rules_proto_toolchains()
-
 ###############################################################################
 # libprotobuf_mutator - for structured fuzzer testing.
 ###############################################################################

bazel-execroot

@@ -1 +1 @@
-bazel-out/../../carbon
+bazel-out/../../_main

bazel/check_deps/check_non_test_cc_deps.py

@@ -22,9 +22,7 @@ import sys
 from pathlib import Path
 
 runfiles = Path(os.environ["TEST_SRCDIR"])
-deps_path = (
-    runfiles / "carbon" / "bazel" / "check_deps" / "non_test_cc_deps.txt"
-)
+deps_path = runfiles / "_main" / "bazel" / "check_deps" / "non_test_cc_deps.txt"
 try:
     with deps_path.open() as deps_file:
         deps = deps_file.read().splitlines()
@@ -34,10 +32,12 @@ except FileNotFoundError:
 for dep in deps:
     print("Checking dependency: " + dep)
     repo, _, rule = dep.partition("//")
+    # Ignore the version, just use the repo name.
+    repo = repo.split("~")[0]
     if repo == "" and not rule.startswith("third_party"):
         # Carbon code is always allowed.
         continue
-    if repo == "@llvm-project":
+    if repo == "@@llvm-project":
         package, _, rule = rule.partition(":")
 
         # Other packages in the LLVM project shouldn't be accidentally used
@@ -56,7 +56,11 @@ for dep in deps:
 
         # The rest of LLVM, LLD, and Clang themselves are safe to depend on.
         continue
-    if repo in ("@llvm_terminfo", "@llvm_zlib", "@llvm_zstd"):
+    if repo == "@@rules_cc" and rule == ":link_extra_lib":
+        # An empty stub library added by rules_cc:
+        # https://github.com/bazelbuild/rules_cc/blob/main/BUILD
+        continue
+    if repo in ("@llvm_terminfo", "@@llvm_zlib", "@@llvm_zstd"):
         # These are stubs wrapping system libraries for LLVM. They aren't
         # distributed and so should be fine.
         continue

explorer/syntax/format_grammar_test.py

@@ -8,7 +8,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 import unittest
 
-from carbon.explorer.syntax import format_grammar
+from explorer.syntax import format_grammar
 
 
 class TestFormatGrammar(unittest.TestCase):

proposals/scripts/new_proposal_test.py

@@ -11,7 +11,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 import os
 import unittest
 
-from carbon.proposals.scripts import new_proposal
+from proposals.scripts import new_proposal
 
 
 class TestNewProposal(unittest.TestCase):

scripts/fix_cc_deps.py

@@ -16,7 +16,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 import re
 import subprocess
-from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple
+from typing import Callable, Dict, List, NamedTuple, Set, Tuple
 from xml.etree import ElementTree
 
 import scripts_utils
@@ -27,8 +27,6 @@ class ExternalRepo(NamedTuple):
     remap: Callable[[str], str]
     # The target expression to gather rules for within the repo.
     target: str
-    # The repo to use for dependencies.
-    dep_repo: Optional[str]
 
 
 # Maps external repository names to a method translating bazel labels to file
@@ -39,31 +37,25 @@ EXTERNAL_REPOS: Dict[str, ExternalRepo] = {
     "@llvm-project": ExternalRepo(
         lambda x: re.sub(":", "/", re.sub("^(.*:(lib|include))/", "", x)),
         "...",
-        None,
     ),
     # :src/google/protobuf/descriptor.h -> google/protobuf/descriptor.h
     # - protobuf_headers is specified because there are multiple overlapping
     #   targets.
-    # - @com_google_protobuf is the official dependency, and we use it, but it
-    #   aliases @com_github_protocolbuffers_protobuf.
-    "@com_github_protocolbuffers_protobuf": ExternalRepo(
+    "@com_google_protobuf": ExternalRepo(
         lambda x: re.sub("^(.*:src)/", "", x),
         ":protobuf_headers",
-        "@com_google_protobuf",
     ),
     # :src/libfuzzer/libfuzzer_macro.h -> libfuzzer/libfuzzer_macro.h
     "@com_google_libprotobuf_mutator": ExternalRepo(
-        lambda x: re.sub("^(.*:src)/", "", x), "...", None
+        lambda x: re.sub("^(.*:src)/", "", x), "..."
     ),
     # tools/cpp/runfiles:runfiles.h -> tools/cpp/runfiles/runfiles.h
-    "@bazel_tools": ExternalRepo(lambda x: re.sub(":", "/", x), "...", None),
+    "@bazel_tools": ExternalRepo(lambda x: re.sub(":", "/", x), "..."),
     # absl/flags:flag.h -> absl/flags/flag.h
-    "@com_google_absl": ExternalRepo(
-        lambda x: re.sub(":", "/", x), "...", None
-    ),
+    "@com_google_absl": ExternalRepo(lambda x: re.sub(":", "/", x), "..."),
     # :re2/re2.h -> re2/re2.h
     "@com_googlesource_code_re2": ExternalRepo(
-        lambda x: re.sub(":", "", x), ":re2", None
+        lambda x: re.sub(":", "", x), ":re2"
     ),
 }
 
@@ -94,6 +86,8 @@ def remap_file(label: str) -> str:
     repo, _, path = label.partition("//")
     if not repo:
         return path.replace(":", "/")
+    # Ignore the version, just use the repo name.
+    repo = repo.split("~", 1)[0]
     assert repo in EXTERNAL_REPOS, repo
     return EXTERNAL_REPOS[repo].remap(path)
 
@@ -175,9 +169,6 @@ def map_headers(
     """
     for rule_name, rule in rules.items():
         repo, _, path = rule_name.partition("//")
-        if repo and EXTERNAL_REPOS[repo].dep_repo:
-            rule_name = f"{EXTERNAL_REPOS[repo].dep_repo}//{path}"
-
         for header in rule.hdrs:
             if header in header_to_rule_map:
                 header_to_rule_map[header].add(rule_name)

scripts/scripts_utils.py

@@ -51,7 +51,7 @@ _BAZEL_TOOLS_VERSION_SHAS = {
     },
 }
 
-_TARGET_DETERMINATOR_URL = "https://github.com/bazel-contrib/target-determinator/releases/download/v0.23.0/"  # noqa: E501
+_TARGET_DETERMINATOR_URL = "https://github.com/bazel-contrib/target-determinator/releases/download/v0.25.0/"  # noqa: E501
 
 """Version SHAs.
 
@@ -59,15 +59,15 @@ Gather shas with:
     for v in darwin.amd64 darwin.arm64 linux.amd64 linux.arm64 \
         windows.amd64.exe
     do
-        echo "\"$v\": \"$(wget -q -O - https://github.com/bazel-contrib/target-determinator/releases/download/v0.23.0/target-determinator.$v | sha256sum | cut -d ' ' -f1)\", # noqa: E501"
+        echo "\"$v\": \"$(wget -q -O - https://github.com/bazel-contrib/target-determinator/releases/download/v0.25.0/target-determinator.$v | sha256sum | cut -d ' ' -f1)\", # noqa: E501"
     done
 """
 _TARGET_DETERMINATOR_SHAS = {
-    "darwin.amd64": "aba6dce8a978d2174b37dd1355eecba86db93be1ff77742d0753d8efd6a8a316",  # noqa: E501
-    "darwin.arm64": "6c3c308dcfc651408ed5490245ea3e0180fc49d4cc9b762ab84a4b979bcb07b8",  # noqa: E501
-    "linux.amd64": "5200dbca0dd4980690d5060cf8e04abac927efaca143567c51fe24cf973364d2",  # noqa: E501
-    "linux.arm64": "3c04f8bb2742219eb3415c6d675dcfe9175745eb7b1d6c3706085a9987f9f719",  # noqa: E501
-    "windows.amd64.exe": "3aea5bd52fdf29bfe6995ffcacc2b2c2299af02dc58f1039022ff758b58214c3",  # noqa: E501
+    "darwin.amd64": "8c7245603dede429b978e214ca327c3f3d686a1bc712c1298fca0396a0f25f23",  # noqa: E501
+    "darwin.arm64": "8f975b471c4a51d32781b757e1ece9700221bfd4c0ea507c18fa382360d1111f",  # noqa: E501
+    "linux.amd64": "c8a09143e9fe6eccc4b27a6be92c5929e5a78034a8d0b4c43dbed4ee539ec903",  # noqa: E501
+    "linux.arm64": "f34618c885d239d77a31f594daf73a67c1133ab4a0376d37a29dbe8d1d2b0b90",  # noqa: E501
+    "windows.amd64.exe": "e14fd75e33d193f579505cf3e641e07025904fc027686e13e154ba8e10ac0f58",  # noqa: E501
 }