Test and fix make-variable expansion in our toolchains (#7070)

This worked correctly in the system Clang toolchain, but was not
configured correctly in the Carbon toolchains. The test is designed to
let us cover all of these.

Assisted-by: Antigravity with Gemini

bazel/cc_toolchains/BUILD

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load(":carbon_cc_toolchain_config.bzl", "gen_cc_toolchain_paths_with_stage")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -69,3 +71,25 @@ filegroup(
         "cc_toolchain_carbon_project_features.bzl",
     ],
 )
+
+gen_cc_toolchain_paths_with_stage(
+    name = "gen_cc_tools_paths",
+    stage = 0,
+)
+
+# Test that the default toolchain's Make variables expand correctly.
+py_test(
+    name = "cc_tools_test",
+    srcs = ["cc_tools_test.py"],
+    args = ["$(location :gen_cc_tools_paths)"],
+    data = [":gen_cc_tools_paths"],
+    deps = [":cc_tools_test_lib"],
+)
+
+# Library containing the test logic, used by tests in other packages.
+py_library(
+    name = "cc_tools_test_lib",
+    srcs = ["cc_tools_test.py"],
+    visibility = ["//visibility:public"],
+    deps = ["@bazel_tools//tools/python/runfiles"],
+)

bazel/cc_toolchains/carbon_cc_toolchain_config.bzl

@@ -4,6 +4,7 @@
 
 """Starlark cc_toolchain configuration rules for using the Carbon toolchain"""
 
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
 load("@rules_cc//cc:action_names.bzl", "ACTION_NAMES")
 load(
     "@rules_cc//cc:cc_toolchain_config_lib.bzl",
@@ -33,6 +34,10 @@ load(
 )
 load("cc_toolchain_carbon_project_features.bzl", "carbon_project_features")
 load("cc_toolchain_features.bzl", "clang_cc_toolchain_features")
+load(
+    ":cc_toolchain_tools.bzl",
+    "llvm_tool_paths",
+)
 
 def _make_action_configs(tools, runtimes_path = None):
     runtimes_flag = "--no-build-runtimes"
@@ -100,12 +105,14 @@ def _compute_clang_system_include_dirs():
     return clang_include_dirs[system_include_dirs_start_index:]
 
 def _carbon_cc_toolchain_config_impl(ctx):
+    llvm_bindir = "llvm/bin"
+    clang_bindir = llvm_bindir
     tools = struct(
         carbon_busybox = tool(path = "carbon-busybox"),
-        clang = tool(path = "llvm/bin/clang"),
-        clangpp = tool(path = "llvm/bin/clang++"),
-        llvm_ar = tool(path = "llvm/bin/llvm-ar"),
-        llvm_strip = tool(path = "llvm/bin/llvm-strip"),
+        clang = tool(path = clang_bindir + "/clang"),
+        clangpp = tool(path = clang_bindir + "/clang++"),
+        llvm_ar = tool(path = llvm_bindir + "/llvm-ar"),
+        llvm_strip = tool(path = llvm_bindir + "/llvm-strip"),
     )
     if ctx.attr.bins:
         carbon_busybox = None
@@ -124,6 +131,10 @@ def _carbon_cc_toolchain_config_impl(ctx):
                 llvm_ar = f
             elif f.basename == "llvm-strip":
                 llvm_strip = f
+        if not all([carbon_busybox, clang, clangpp, llvm_ar, llvm_strip]):
+            fail("Missing required tool in bins: {0}".format(ctx.attr.bins))
+        llvm_bindir = llvm_ar.dirname
+        clang_bindir = clang.dirname
         tools = struct(
             carbon_busybox = tool(tool = carbon_busybox),
             clang = tool(tool = clang),
@@ -187,6 +198,10 @@ def _carbon_cc_toolchain_config_impl(ctx):
         # This is used to expose a "flag" that `config_setting` rules can use to
         # determine if the compiler is Clang.
         compiler = "clang",
+
+        # Pass in our tool paths to expose Make variables like $(NM) and
+        # $(OBJCOPY).
+        tool_paths = llvm_tool_paths(llvm_bindir, clang_bindir),
     )
 
 carbon_cc_toolchain_config = rule(
@@ -244,6 +259,43 @@ filegroup_with_stage = rule(
     """,
 )
 
+def _gen_cc_toolchain_paths_impl(ctx):
+    cc_toolchain = find_cpp_toolchain(ctx)
+
+    expanded_vars = [
+        ctx.expand_make_variables("vars", v, {})
+        for v in ctx.attr.vars
+    ]
+
+    out = ctx.actions.declare_file(ctx.attr.name + ".txt")
+    ctx.actions.write(out, "\n".join(expanded_vars) + "\n")
+
+    # Include all toolchain files in runfiles.
+    runfiles = ctx.runfiles(files = [out]).merge(
+        ctx.runfiles(transitive_files = cc_toolchain.all_files),
+    )
+
+    return [DefaultInfo(files = depset([out]), runfiles = runfiles)]
+
+gen_cc_toolchain_paths_with_stage = rule(
+    implementation = _gen_cc_toolchain_paths_impl,
+    attrs = {
+        "enable_runtimes_build": attr.bool(default = False),
+        "stage": attr.int(mandatory = True),
+        "vars": attr.string_list(
+            default = ["$(CC)", "$(AR)", "$(NM)", "$(OBJCOPY)", "$(STRIP)"],
+        ),
+        "_allowlist_function_transition": attr.label(
+            default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
+        ),
+        "_cc_toolchain": attr.label(
+            default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
+        ),
+    },
+    toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
+    cfg = _transition_with_stage,
+)
+
 def carbon_cc_toolchain_suite(
         name,
         all_hdrs,

bazel/cc_toolchains/cc_tools_test.py

@@ -0,0 +1,59 @@
+"""Tests that the C++ toolchain tools can be executed.
+
+This script reads a file containing paths to C++ tools (like clang++, llvm-ar)
+and attempts to run each with `--version` to verify they are functional.
+"""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import os
+import subprocess
+import sys
+from bazel_tools.tools.python.runfiles import runfiles
+
+
+def test_tools() -> None:
+    """Reads paths from file and runs each tool with --version."""
+    if len(sys.argv) < 2:
+        print("Usage: cc_tools_test.py <paths_file>")
+        sys.exit(1)
+
+    paths_file = sys.argv[1]
+    print(f"Reading tools from: {paths_file}")
+    with open(paths_file, "r") as f:
+        tools = [line.strip() for line in f if line.strip()]
+
+    print(f"Testing tools: {tools}")
+    r = runfiles.Create()
+    repo_name = os.environ.get("TEST_WORKSPACE") or "_main"
+
+    for tool in tools:
+        if "bazel-out/" in tool:
+            _, _, rest = tool.partition("bazel-out/")
+            _, sep, after = rest.partition("bin/")
+            if sep:
+                tool = after
+
+        rlocation_path = os.path.join(repo_name, tool)
+        tool = r.Rlocation(rlocation_path)
+
+        print(f"Running {tool} --version")
+        try:
+            res = subprocess.run(
+                [tool, "--version"],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            print(res.stdout)
+        except Exception as e:
+            print(f"Failed to run {tool}: {e}")
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    test_tools()

toolchain/install/BUILD

@@ -39,6 +39,7 @@ load(
     "//bazel/cc_toolchains:carbon_cc_toolchain_config.bzl",
     "carbon_cc_toolchain_suite",
     "filegroup_with_stage",
+    "gen_cc_toolchain_paths_with_stage",
 )
 load("//bazel/manifest:defs.bzl", "manifest")
 load(
@@ -753,3 +754,37 @@ py_test(
     ],
     deps = ["@bazel_tools//tools/python/runfiles"],
 )
+
+# Generate paths file for stage 1 toolchain.
+gen_cc_toolchain_paths_with_stage(
+    name = "gen_cc_tools_paths_stage1",
+    stage = 1,
+)
+
+# Test that stage 1 toolchain's Make variables expand correctly.
+py_test(
+    name = "cc_tools_stage1_test",
+    srcs = ["cc_tools_wrapper_test.py"],
+    args = ["$(location :gen_cc_tools_paths_stage1)"],
+    data = [":gen_cc_tools_paths_stage1"],
+    main = "cc_tools_wrapper_test.py",
+    deps = ["//bazel/cc_toolchains:cc_tools_test_lib"],
+)
+
+# Generate paths file for stage 2 toolchain.
+gen_cc_toolchain_paths_with_stage(
+    name = "gen_cc_tools_paths_stage2",
+    stage = 2,
+    tags = ["manual"],
+)
+
+# Test that stage 2 toolchain's Make variables expand correctly.
+py_test(
+    name = "cc_tools_stage2_test",
+    srcs = ["cc_tools_wrapper_test.py"],
+    args = ["$(location :gen_cc_tools_paths_stage2)"],
+    data = [":gen_cc_tools_paths_stage2"],
+    main = "cc_tools_wrapper_test.py",
+    tags = ["manual"],
+    deps = ["//bazel/cc_toolchains:cc_tools_test_lib"],
+)

toolchain/install/cc_tools_wrapper_test.py

@@ -0,0 +1,17 @@
+"""Wrapper script to run cc_tools_test from toolchain/install.
+
+This script imports `test_tools` from `bazel.cc_toolchains.cc_tools_test`
+and executes it. This allows running the test in `toolchain/install` package
+while keeping the main logic in `bazel/cc_toolchains`.
+"""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+from bazel.cc_toolchains.cc_tools_test import test_tools
+
+if __name__ == "__main__":
+    test_tools()