Enable using our own C++ runtimes across the board (#6549)

This enables on-demand building of runtimes by default, and enables
their header files for all of the Clang invocations. This also switches
the default flags to use the LLVM-provided runtimes (compiler-rt,
libunwind, and libcxx).

This also switches even `llvm_symlinks_test` to use the Bazel prebuilt
runtimes, which requires having a way to pass a Carbon flag even when
invoking the busybox as `clang` or `clang++`. This uses the pattern that
has worked for other Clang wrappers of spelling flags:
`-X<tool-name>=--flag=value`

Last but not least, this updates the Carbon Bazel rules to use our
installed and the Bazel prebuilt runtimes. With that, we make the C++
interop hello-world be enabled by default as this should pass reliably
on both Linux and macOS now.

bazel/carbon_rules/defs.bzl

@@ -6,20 +6,34 @@
 
 load("@rules_cc//cc/common:cc_info.bzl", "CcInfo")
 
+def _runtimes_path(runtimes_target):
+    path = None
+    for f in runtimes_target:
+        if f.short_path.endswith("clang_resource_dir/lib"):
+            path = f.path
+            break
+
+    if not path:
+        fail("Could not find the `clang_resource_dir` in target {}".format(runtimes_target.label))
+
+    return path[:-len("/clang_resource_dir/lib")]
+
 def _carbon_binary_impl(ctx):
     toolchain_driver = ctx.executable.internal_exec_toolchain_driver
     toolchain_data = ctx.files.internal_exec_toolchain_data
+    prebuilt_runtimes = ctx.files.internal_exec_prebuilt_runtimes
 
     # If the exec driver isn't provided, that means we're trying to use a target
     # config toolchain, likely to avoid build overhead of two configs.
     if toolchain_driver == None:
         toolchain_driver = ctx.executable.internal_target_toolchain_driver
         toolchain_data = ctx.files.internal_target_toolchain_data
+        prebuilt_runtimes = ctx.files.internal_target_prebuilt_runtimes
 
     # Pass any C++ flags from our dependencies onto Carbon.
     dep_flags = []
     dep_hdrs = []
-    dep_link_flags = ["-lc++"]
+    dep_link_flags = []
     dep_link_inputs = []
     for dep in ctx.attr.deps:
         if CcInfo in dep:
@@ -75,7 +89,7 @@ def _carbon_binary_impl(ctx):
                 inputs = depset(direct = srcs_reordered, transitive = dep_hdrs),
                 executable = toolchain_driver,
                 tools = depset(toolchain_data),
-                arguments = ["compile", "--output=" + out.path, "--clang-arg=-stdlib=libc++"] +
+                arguments = ["compile", "--output=" + out.path] +
                             [s.path for s in srcs_reordered] + extra_flags + ctx.attr.flags,
                 mnemonic = "CarbonCompile",
                 progress_message = "Compiling " + src.short_path,
@@ -86,8 +100,8 @@ def _carbon_binary_impl(ctx):
         outputs = [bin],
         inputs = objs + dep_link_inputs,
         executable = toolchain_driver,
-        tools = depset(toolchain_data),
-        arguments = ["link", "--output=" + bin.path] + ["--"] + dep_link_flags + [o.path for o in objs],
+        tools = depset(toolchain_data + prebuilt_runtimes),
+        arguments = ["--prebuilt-runtimes=" + _runtimes_path(prebuilt_runtimes), "link", "--output=" + bin.path] + ["--"] + dep_link_flags + [o.path for o in objs],
         mnemonic = "CarbonLink",
         progress_message = "Linking " + bin.short_path,
     )
@@ -98,11 +112,15 @@ _carbon_binary_internal = rule(
     attrs = {
         "deps": attr.label_list(allow_files = True, providers = [[CcInfo]]),
         "flags": attr.string_list(),
-        # The exec config toolchain driver and data. These will be `None` when
-        # using the target config and populated when using the exec config. We
-        # have to use duplicate attributes here and below to have different
-        # `cfg` settings, as that isn't `select`-able, and we'll use `select`s
-        # when populating these.
+
+        # The exec config toolchain attributes. These will be `None` when using
+        # the target config and populated when using the exec config. We have to
+        # use duplicate attributes here and below to have different `cfg`
+        # settings, as that isn't `select`-able, and we'll use `select`s when
+        # populating these.
+        "internal_exec_prebuilt_runtimes": attr.label(
+            cfg = "exec",
+        ),
         "internal_exec_toolchain_data": attr.label(
             cfg = "exec",
         ),
@@ -112,11 +130,14 @@ _carbon_binary_internal = rule(
             cfg = "exec",
         ),
 
-        # The target config toolchain driver and data. These will be 'None' when
+        # The target config toolchain attributes. These will be 'None' when
         # using the exec config and populated when using the target config. We
         # have to use duplicate attributes here and below to have different
         # `cfg` settings, as that isn't `select`-able, and we'll use `select`s
         # when populating these.
+        "internal_target_prebuilt_runtimes": attr.label(
+            cfg = "target",
+        ),
         "internal_target_toolchain_data": attr.label(
             cfg = "target",
         ),
@@ -162,6 +183,10 @@ def carbon_binary(name, srcs, deps = [], flags = [], tags = []):
             "//bazel/carbon_rules:use_target_config_carbon_rules_config": None,
             "//conditions:default": "//toolchain/install:install_data",
         }),
+        internal_exec_prebuilt_runtimes = select({
+            "//bazel/carbon_rules:use_target_config_carbon_rules_config": None,
+            "//conditions:default": "//toolchain/driver:prebuilt_runtimes",
+        }),
         internal_target_toolchain_driver = select({
             "//bazel/carbon_rules:use_target_config_carbon_rules_config": "//toolchain/install:prefix_root/bin/carbon",
             "//conditions:default": None,
@@ -170,4 +195,8 @@ def carbon_binary(name, srcs, deps = [], flags = [], tags = []):
             "//bazel/carbon_rules:use_target_config_carbon_rules_config": "//toolchain/install:install_data",
             "//conditions:default": None,
         }),
+        internal_target_prebuilt_runtimes = select({
+            "//bazel/carbon_rules:use_target_config_carbon_rules_config": "//toolchain/driver:prebuilt_runtimes",
+            "//conditions:default": None,
+        }),
     )

examples/interop/cpp/BUILD

@@ -7,6 +7,4 @@ load("//bazel/carbon_rules:defs.bzl", "carbon_binary")
 carbon_binary(
     name = "hello_world",
     srcs = ["hello_world.carbon"],
-    # TODO: Remove when macos can find cstdio.
-    tags = ["manual"],
 )

toolchain/base/clang_invocation.cpp

@@ -120,7 +120,7 @@ auto BuildClangInvocation(Diagnostics::Consumer& consumer,
   return invocation;
 }
 
-auto AppendDefaultClangArgs(const InstallPaths& /*install_paths*/,
+auto AppendDefaultClangArgs(const InstallPaths& install_paths,
                             llvm::StringRef target_str,
                             llvm::SmallVectorImpl<std::string>& args) -> void {
   args.append({
@@ -130,6 +130,15 @@ auto AppendDefaultClangArgs(const InstallPaths& /*install_paths*/,
       // TODO: Decide if we want this.
       "-fPIE",
 
+      // Override runtime library defaults.
+      //
+      // TODO: We should consider if there is a reasonable way to build Clang
+      // with its configuration macros set to establish these defaults rather
+      // than doing it with runtime flags.
+      "-rtlib=compiler-rt",
+      "-unwindlib=libunwind",
+      "-stdlib=libc++",
+
       // Override the default linker to use.
       "-fuse-ld=lld",
   });
@@ -162,7 +171,14 @@ auto AppendDefaultClangArgs(const InstallPaths& /*install_paths*/,
       break;
   }
 
-  // TODO: Add flags for the installed runtimes using `install_paths`.
+  // Append our exact header search paths for the various parts of the C++
+  // standard library headers as we don't build a single unified tree.
+  for (const std::filesystem::path& runtime_path :
+       {install_paths.libunwind_path(), install_paths.libcxx_path(),
+        install_paths.libcxxabi_path()}) {
+    args.push_back(
+        llvm::formatv("-stdlib++-isystem{0}", runtime_path / "include").str());
+  }
 }
 
 }  // namespace Carbon

toolchain/driver/clang_runner.cpp

@@ -44,6 +44,7 @@
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/LLVMDriver.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -166,7 +167,12 @@ auto ClangRunner::RunWithPrebuiltRuntimes(llvm::ArrayRef<llvm::StringRef> args,
 
   CARBON_ASSIGN_OR_RETURN(std::filesystem::path prebuilt_resource_dir_path,
                           prebuilt_runtimes.Get(Runtimes::ClangResourceDir));
+  CARBON_ASSIGN_OR_RETURN(std::filesystem::path libunwind_path,
+                          prebuilt_runtimes.Get(Runtimes::LibUnwind));
+  CARBON_ASSIGN_OR_RETURN(std::filesystem::path libcxx_path,
+                          prebuilt_runtimes.Get(Runtimes::Libcxx));
   return RunInternal(args, target, prebuilt_resource_dir_path.native(),
+                     std::move(libunwind_path), std::move(libcxx_path),
                      enable_leaking);
 }
 
@@ -184,7 +190,6 @@ auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args,
 
   std::string target = ComputeClangTarget(args);
 
-  CARBON_VLOG("Building target resource dir...\n");
   Runtimes::Cache::Features features = {.target = target};
   CARBON_ASSIGN_OR_RETURN(Runtimes runtimes, runtimes_cache.Lookup(features));
 
@@ -192,30 +197,46 @@ auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args,
   // requires a temporary directory as well as the destination directory for
   // the build. The temporary directory should only be used during the build,
   // not once we are running Clang with the built runtime.
-  std::filesystem::path resource_dir_path;
-  {
-    ClangResourceDirBuilder builder(this, &runtimes_build_thread_pool,
-                                    llvm::Triple(features.target), &runtimes);
-    CARBON_ASSIGN_OR_RETURN(resource_dir_path, std::move(builder).Wait());
-  }
+  CARBON_VLOG("Building target resource dir...\n");
+  ClangResourceDirBuilder builder(this, &runtimes_build_thread_pool,
+                                  llvm::Triple(features.target), &runtimes);
+  ClangArchiveRuntimesBuilder<Runtimes::LibUnwind> lib_unwind_builder(
+      this, &runtimes_build_thread_pool, llvm::Triple(features.target),
+      &runtimes);
+  ClangArchiveRuntimesBuilder<Runtimes::Libcxx> libcxx_builder(
+      this, &runtimes_build_thread_pool, llvm::Triple(features.target),
+      &runtimes);
+  CARBON_ASSIGN_OR_RETURN(std::filesystem::path resource_dir_path,
+                          std::move(builder).Wait());
+  CARBON_ASSIGN_OR_RETURN(std::filesystem::path libunwind_path,
+                          std::move(lib_unwind_builder).Wait());
+  CARBON_ASSIGN_OR_RETURN(std::filesystem::path libcxx_path,
+                          std::move(libcxx_builder).Wait());
 
   // Note that this function always successfully runs `clang` and returns a bool
   // to indicate whether `clang` itself succeeded, not whether the runner was
   // able to run it. As a consequence, even a `false` here is a non-`Error`
   // return.
-  return RunInternal(args, target, resource_dir_path.native(), enable_leaking);
+  return RunInternal(args, target, resource_dir_path.native(),
+                     std::move(libunwind_path), std::move(libcxx_path),
+                     enable_leaking);
 }
 
 auto ClangRunner::RunWithNoRuntimes(llvm::ArrayRef<llvm::StringRef> args,
                                     bool enable_leaking) -> bool {
   std::string target = ComputeClangTarget(args);
-  return RunInternal(args, target, std::nullopt, enable_leaking);
+  return RunInternal(args, target, /*target_resource_dir_path=*/std::nullopt,
+                     /*libunwind_path=*/std::nullopt,
+                     /*libcxx_path=*/std::nullopt, enable_leaking);
 }
 
 auto ClangRunner::RunInternal(
     llvm::ArrayRef<llvm::StringRef> args, llvm::StringRef target,
     std::optional<llvm::StringRef> target_resource_dir_path,
-    bool enable_leaking) -> bool {
+
+    std::optional<std::filesystem::path> libunwind_path,
+    std::optional<std::filesystem::path> libcxx_path, bool enable_leaking)
+    -> bool {
   // Rebuild the args as C-string args.
   llvm::OwningArrayRef<char> cstr_arg_storage;
 
@@ -258,6 +279,20 @@ auto ClangRunner::RunInternal(
 
   AppendDefaultClangArgs(*installation_, target, prefix_args);
 
+  // We don't have a direct way to configure the linker search paths in the
+  // Clang driver outside of command line flags, so we inject them here with
+  // flags. Note that we only inject these as _search_ paths to allow the normal
+  // linking rules to govern whether or not to link a given library. We also
+  // build our runtimes exclusively as static archives so we don't need to use
+  // command line flags to force static runtime linking to occur.
+  if (libunwind_path) {
+    prefix_args.push_back(
+        llvm::formatv("-L{0}/lib", *std::move(libunwind_path)).str());
+  }
+  if (libcxx_path) {
+    prefix_args.push_back(
+        llvm::formatv("-L{0}/lib", std::move(libcxx_path)).str());
+  }
   prefix_args.push_back("--end-no-unused-arguments");
 
   // Rebuild the args as C-string args.

toolchain/driver/clang_runner.h

@@ -110,6 +110,8 @@ class ClangRunner : ToolRunnerBase {
   // Handles building the Clang driver and passing the arguments down to it.
   auto RunInternal(llvm::ArrayRef<llvm::StringRef> args, llvm::StringRef target,
                    std::optional<llvm::StringRef> target_resource_dir_path,
+                   std::optional<std::filesystem::path> libunwind_path,
+                   std::optional<std::filesystem::path> libcxx_path,
                    bool enable_leaking) -> bool;
 
   // Returns the target-specific source files for the builtins runtime library.

toolchain/driver/clang_subcommand.cpp

@@ -30,9 +30,7 @@ the installation tree in the default searched locations.
 )""",
       },
       [&](auto& arg_b) {
-        // TODO: Once runtimes are cached properly, the plan is to enable this
-        // by default.
-        arg_b.Default(false);
+        arg_b.Default(true);
         arg_b.Set(&build_runtimes_on_demand);
       });
   b.AddStringPositionalArg(

toolchain/driver/link_subcommand.cpp

@@ -69,14 +69,6 @@ auto LinkSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
       llvm::formatv("--target={0}", options_.codegen_options.target).str();
   clang_args.push_back(target_arg);
 
-  // Disable linking the C++ standard library until can build and ship it as
-  // part of the Carbon toolchain. This clearly won't work once we get into
-  // interop, but for now it avoids spurious failures and distraction. The plan
-  // is to build and bundle libc++ at which point we can replace this with
-  // pointing at our bundled library.
-  // TODO: Replace this when ready.
-  clang_args.push_back("-nostdlib++");
-
   clang_args.push_back("-o");
   clang_args.push_back(options_.output_filename);
   clang_args.append(options_.object_filenames.begin(),
@@ -84,8 +76,13 @@ auto LinkSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
 
   ClangRunner runner(driver_env.installation, driver_env.fs,
                      driver_env.vlog_stream);
-  ErrorOr<bool> run_result = runner.Run(clang_args, driver_env.runtimes_cache,
-                                        *driver_env.thread_pool);
+  ErrorOr<bool> run_result =
+      driver_env.prebuilt_runtimes
+          ? runner.RunWithPrebuiltRuntimes(clang_args,
+                                           *driver_env.prebuilt_runtimes,
+                                           driver_env.enable_leaking)
+          : runner.Run(clang_args, driver_env.runtimes_cache,
+                       *driver_env.thread_pool, driver_env.enable_leaking);
   if (!run_result.ok()) {
     // This is not a Clang failure, but a failure to even run Clang, so we need
     // to diagnose it here.

toolchain/install/BUILD

@@ -395,7 +395,11 @@ py_test(
     name = "llvm_symlinks_test",
     size = "small",
     srcs = ["llvm_symlinks_test.py"],
-    data = [":install_data"],
+    data = [
+        ":install_data",
+        "//toolchain/driver:prebuilt_runtimes",
+    ],
+    deps = ["@bazel_tools//tools/python/runfiles"],
 )
 
 manifest(

toolchain/install/busybox_main.cpp

@@ -10,6 +10,7 @@
 #include "common/bazel_working_dir.h"
 #include "common/error.h"
 #include "common/init_llvm.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/LLVMDriver.h"
@@ -45,6 +46,9 @@ static auto Main(int argc, char** argv) -> ErrorOr<int> {
 
   auto fs = llvm::vfs::getRealFileSystem();
 
+  llvm::SmallVector<llvm::StringRef> raw_args;
+  raw_args.append(argv + 1, argv + argc);
+
   llvm::SmallVector<llvm::StringRef> args;
   args.reserve(argc + 1);
   if (busybox_info.mode) {
@@ -80,9 +84,24 @@ static auto Main(int argc, char** argv) -> ErrorOr<int> {
 #include "toolchain/base/llvm_tools.def"
 
             .Default({*busybox_info.mode, "--"});
+
+    // When we're operating as a busybox, we also support a special command line
+    // syntax for passing flags to the base Carbon driver as
+    // `-Xcarbon=--some-carbon-flag=some-value`. Extract any arguments of that
+    // form, remove the prefix, and prepend them to the arg list prior to the
+    // busybox subcommand arguments.
+    llvm::erase_if(raw_args, [&args](llvm::StringRef raw_arg) {
+      if (raw_arg.consume_front("-Xcarbon=")) {
+        args.push_back(raw_arg);
+        return true;
+      }
+      return false;
+    });
+
+    // And now append the subcommand args.
     args.append(subcommand_args);
   }
-  args.append(argv + 1, argv + argc);
+  args.append(raw_args);
 
   Driver driver(fs, &install_paths, stdin, &llvm::outs(), &llvm::errs(),
                 /*fuzzing=*/false, /*enable_leaking=*/true);

toolchain/install/llvm_symlinks_test.py

@@ -14,6 +14,7 @@ import os
 import platform
 import sys
 import unittest
+from bazel_tools.tools.python.runfiles import runfiles
 
 
 class LLVMSymlinksTest(unittest.TestCase):
@@ -23,12 +24,20 @@ class LLVMSymlinksTest(unittest.TestCase):
         self.tmpdir = Path(os.environ["TEST_TMPDIR"])
         self.test_o_file = self.tmpdir / "test.o"
         self.test_o_file.touch()
+        self.runfiles = runfiles.Create()
+        self.prebuilt_runtimes = self.runfiles.Rlocation(
+            "carbon/toolchain/driver/prebuilt_runtimes_tree"
+        )
 
     def get_link_cmd(self, clang: Path) -> list[str | Path]:
         return [
             clang,
             # Verbose printing to help with debugging.
             "-v",
+            # Pass a parameter to the underlying Carbon busybox using `-Xcarbon`
+            # to switch it to use the prebuilt runtimes rather than building
+            # runtimes on demand.
+            f"-Xcarbon=--prebuilt-runtimes={self.prebuilt_runtimes}",
             # Print out the link command rather than running it.
             "-###",
             # Give the link command an output.
@@ -56,12 +65,8 @@ class LLVMSymlinksTest(unittest.TestCase):
         )
 
         # Also ensure that it correctly didn't imply a C++ link.
-        if platform.system() == "Linux":
-            self.assertNotRegex(run.stderr, r'"-lstdc\+\+"')
-        elif platform.system() == "Darwin":
-            self.assertNotRegex(run.stderr, r'"-lc\+\+"')
-        else:
-            self.unsupported(run.stderr)
+        self.assertNotRegex(run.stderr, r'"-lc\+\+"')
+        self.assertNotRegex(run.stderr, r'"-lstdc\+\+"')
 
     # Note that we can't test `clang` vs. `clang++` portably. See the comment on
     # `test_clang` for details.
@@ -71,13 +76,9 @@ class LLVMSymlinksTest(unittest.TestCase):
             self.get_link_cmd(bin), check=True, capture_output=True, text=True
         )
 
-        # Ensure that this binary _does_ imply a C++ link.
-        if platform.system() == "Linux":
-            self.assertRegex(run.stderr, r'"-lstdc\+\+"')
-        elif platform.system() == "Darwin":
-            self.assertRegex(run.stderr, r'"-lc\+\+"')
-        else:
-            self.unsupported(run.stderr)
+        # Ensure that this binary _does_ imply a C++ link. Also ensure it uses
+        # `libc++`, as we default our Clang to use that on all platforms.
+        self.assertRegex(run.stderr, r'"-lc\+\+"')
 
     def test_clang_cl(self) -> None:
         bin = self.install_root / "lib/carbon/llvm/bin/clang-cl"