Refactor Clang runtimes building into async builder (#6380)

Previously, the Clang runtimes building only considered building the
target resource directory, and was only _internally_ asynchronous.
Because the asynchrony was only internal, it could use the function
frame as a context object throughout the build of the resource dir. This
is simple but doesn't generalize well to more runtimes: if we want to
add 2 or 3 more runtimes, we want them to _all_ build asynchronously.
That means using some asynchronous builder that maintains the context
and allows them to proceed concurrently with other work.

This also factors all the runtimes building code into a separate set of
files. These aren't separate libraries at this point due to the
`ClangRunner` in some cases wanting to build runtimes on-demand, but it
at least lets us organize the code more cleanly.

Because this splits code between `clang_runner.*` and
`clang_runtimes.*`, it also works to update the `#include`s for both to
be roughly accurate. I used ClangD's include cleaner for this and it
probably also did some latent cleaning as it went, but that's the reason
for the churn of `#include` lines.

The archive building is also factored out into a re-usable helper. This
is a bit "over factored" in this PR, but supports the next PR that uses
the same code to build archives for other runtimes.

This also overhauls the synchronization used -- it uses a simple `Latch`
construct introduced in a previous PR to coordinate between the steps of
building the runtimes.

Last but not least, it factors the "enable leaking" state out of a
boolean in the runner to a parameter. This is important in the face of
concurrent calls as otherwise toggling this boolean can create a race.

The next PR will layer building more runtimes on top of this new
factoring.

---------

Co-authored-by: David Blaikie <dblaikie@gmail.com>

toolchain/driver/BUILD

@@ -19,14 +19,22 @@ filegroup(
 
 cc_library(
     name = "clang_runner",
-    srcs = ["clang_runner.cpp"],
-    hdrs = ["clang_runner.h"],
+    srcs = [
+        "clang_runner.cpp",
+        "clang_runtimes.cpp",
+    ],
+    hdrs = [
+        "clang_runner.h",
+        "clang_runtimes.h",
+    ],
     deps = [
         ":llvm_runner",
         ":runtimes_cache",
         ":tool_runner_base",
+        "//common:check",
         "//common:error",
         "//common:filesystem",
+        "//common:latch",
         "//common:ostream",
         "//common:vlog",
         "//toolchain/base:runtime_sources",
@@ -53,6 +61,7 @@ cc_test(
     deps = [
         ":clang_runner",
         ":llvm_runner",
+        ":runtimes_cache",
         "//common:all_llvm_targets",
         "//common:check",
         "//common:ostream",
@@ -61,6 +70,33 @@ cc_test(
         "//testing/base:file_helpers",
         "//testing/base:global_exe_path",
         "//testing/base:gtest_main",
+        "//toolchain/install:install_paths",
+        "@googletest//:gtest",
+        "@llvm-project//llvm:Object",
+        "@llvm-project//llvm:Support",
+        "@llvm-project//llvm:TargetParser",
+    ],
+)
+
+cc_test(
+    name = "clang_runtimes_test",
+    size = "small",
+    srcs = ["clang_runtimes_test.cpp"],
+    data = ["//toolchain/install:install_data"],
+    deps = [
+        ":clang_runner",
+        ":llvm_runner",
+        ":runtimes_cache",
+        "//common:all_llvm_targets",
+        "//common:check",
+        "//common:ostream",
+        "//common:raw_string_ostream",
+        "//testing/base:capture_std_streams",
+        "//testing/base:file_helpers",
+        "//testing/base:global_exe_path",
+        "//testing/base:gtest_main",
+        "//toolchain/base:llvm_tools",
+        "//toolchain/install:install_paths",
         "@googletest//:gtest",
         "@llvm-project//llvm:Object",
         "@llvm-project//llvm:Support",

toolchain/driver/build_runtimes_subcommand.cpp

@@ -4,10 +4,9 @@
 
 #include "toolchain/driver/build_runtimes_subcommand.h"
 
-#include <variant>
-
 #include "llvm/TargetParser/Triple.h"
 #include "toolchain/driver/clang_runner.h"
+#include "toolchain/driver/clang_runtimes.h"
 
 namespace Carbon {
 
@@ -92,8 +91,13 @@ auto BuildRuntimesSubcommand::RunInternal(DriverEnv& driver_env)
                : Runtimes::Make(explicit_output_path, driver_env.vlog_stream));
   CARBON_ASSIGN_OR_RETURN(auto tmp_dir, Filesystem::MakeTmpDir());
 
-  return runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path(),
-                                       *driver_env.thread_pool);
+  ClangResourceDirBuilder resource_dir_builder(&runner, driver_env.thread_pool,
+                                               llvm::Triple(features.target),
+                                               &runtimes);
+
+  CARBON_RETURN_IF_ERROR(std::move(resource_dir_builder).Wait());
+
+  return runtimes.base_path();
 }
 
 }  // namespace Carbon

toolchain/driver/clang_runner.cpp

@@ -4,19 +4,18 @@
 
 #include "toolchain/driver/clang_runner.h"
 
+#include <stdlib.h>
 #include <unistd.h>
 
-#include <algorithm>
 #include <filesystem>
 #include <memory>
-#include <numeric>
 #include <optional>
 #include <string>
-#include <system_error>
 #include <utility>
-#include <variant>
 
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticDriver.h"
+#include "clang/Basic/DiagnosticIDs.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/CodeGen/ObjectFilePCHContainerWriter.h"
 #include "clang/Driver/Compilation.h"
@@ -28,25 +27,42 @@
 #include "clang/Frontend/Utils.h"
 #include "clang/FrontendTool/Utils.h"
 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
-#include "common/filesystem.h"
+#include "clang/Serialization/PCHContainerOperations.h"
+#include "common/check.h"
+#include "common/error.h"
 #include "common/vlog.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/BuryPointer.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/LLVMDriver.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/Program.h"
+#include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/Timer.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
-#include "toolchain/base/runtime_sources.h"
+#include "toolchain/driver/clang_runtimes.h"
+#include "toolchain/driver/runtimes_cache.h"
+#include "toolchain/driver/tool_runner_base.h"
+#include "toolchain/install/install_paths.h"
+
+// Defined in:
+// https://github.com/llvm/llvm-project/blob/main/clang/tools/driver/driver.cpp
+//
+// While not in a header, this is the API used by llvm-driver.cpp for
+// busyboxing.
+//
+// NOLINTNEXTLINE(readability-identifier-naming)
+auto clang_main(int Argc, char** Argv, const llvm::ToolContext& ToolContext)
+    -> int;
 
 namespace Carbon {
 
@@ -127,33 +143,35 @@ static auto IsNonLinkCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
 }
 
 auto ClangRunner::RunWithPrebuiltRuntimes(llvm::ArrayRef<llvm::StringRef> args,
-                                          Runtimes& prebuilt_runtimes)
+                                          Runtimes& prebuilt_runtimes,
+                                          bool enable_leaking)
     -> ErrorOr<bool> {
   // Check the args to see if we have a known target-independent command. If so,
   // directly dispatch it to avoid the cost of building the target resource
   // directory.
   // TODO: Maybe handle response file expansion similar to the Clang CLI?
   if (args.empty() || args[0].starts_with("-cc1") || IsNonLinkCommand(args)) {
-    return RunWithNoRuntimes(args);
+    return RunWithNoRuntimes(args, enable_leaking);
   }
 
   std::string target = ComputeClangTarget(args);
 
   CARBON_ASSIGN_OR_RETURN(std::filesystem::path prebuilt_resource_dir_path,
                           prebuilt_runtimes.Get(Runtimes::ClangResourceDir));
-  return RunInternal(args, target, prebuilt_resource_dir_path.native());
+  return RunInternal(args, target, prebuilt_resource_dir_path.native(),
+                     enable_leaking);
 }
 
 auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args,
                       Runtimes::Cache& runtimes_cache,
-                      llvm::ThreadPoolInterface& runtimes_build_thread_pool)
-    -> ErrorOr<bool> {
+                      llvm::ThreadPoolInterface& runtimes_build_thread_pool,
+                      bool enable_leaking) -> ErrorOr<bool> {
   // Check the args to see if we have a known target-independent command. If so,
   // directly dispatch it to avoid the cost of building the target resource
   // directory.
   // TODO: Maybe handle response file expansion similar to the Clang CLI?
   if (args.empty() || args[0].starts_with("-cc1") || IsNonLinkCommand(args)) {
-    return RunWithNoRuntimes(args);
+    return RunWithNoRuntimes(args, enable_leaking);
   }
 
   std::string target = ComputeClangTarget(args);
@@ -168,95 +186,26 @@ auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args,
   // not once we are running Clang with the built runtime.
   std::filesystem::path resource_dir_path;
   {
-    // Build the temporary directory and threadpool needed.
-    CARBON_ASSIGN_OR_RETURN(Filesystem::RemovingDir tmp_dir,
-                            Filesystem::MakeTmpDir());
-
-    CARBON_ASSIGN_OR_RETURN(
-        resource_dir_path,
-        BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path(),
-                               runtimes_build_thread_pool));
+    ClangResourceDirBuilder builder(this, &runtimes_build_thread_pool,
+                                    llvm::Triple(features.target), &runtimes);
+    CARBON_ASSIGN_OR_RETURN(resource_dir_path, std::move(builder).Wait());
   }
 
   // Note that this function always successfully runs `clang` and returns a bool
   // to indicate whether `clang` itself succeeded, not whether the runner was
   // able to run it. As a consequence, even a `false` here is a non-`Error`
   // return.
-  return RunInternal(args, target, resource_dir_path.native());
+  return RunInternal(args, target, resource_dir_path.native(), enable_leaking);
 }
 
-auto ClangRunner::RunWithNoRuntimes(llvm::ArrayRef<llvm::StringRef> args)
-    -> bool {
+auto ClangRunner::RunWithNoRuntimes(llvm::ArrayRef<llvm::StringRef> args,
+                                    bool enable_leaking) -> bool {
   std::string target = ComputeClangTarget(args);
-  return RunInternal(args, target, std::nullopt);
+  return RunInternal(args, target, std::nullopt, enable_leaking);
 }
 
-auto ClangRunner::BuildTargetResourceDir(
-    const Runtimes::Cache::Features& features, Runtimes& runtimes,
-    const std::filesystem::path& tmp_path, llvm::ThreadPoolInterface& threads)
-    -> ErrorOr<std::filesystem::path> {
-  // Disable any leaking of memory while building the target resource dir, and
-  // restore the previous setting at the end.
-  auto restore_leak_flag = llvm::make_scope_exit(
-      [&, orig_flag = enable_leaking_] { enable_leaking_ = orig_flag; });
-  enable_leaking_ = false;
-
-  CARBON_ASSIGN_OR_RETURN(auto build_dir,
-                          runtimes.Build(Runtimes::ClangResourceDir));
-  if (std::holds_alternative<std::filesystem::path>(build_dir)) {
-    // Found cached build.
-    return std::get<std::filesystem::path>(std::move(build_dir));
-  }
-
-  auto builder = std::get<Runtimes::Builder>(std::move(build_dir));
-  std::string target = features.target;
-
-  // Symlink the installation's `include` and `share` directories.
-  std::filesystem::path install_resource_path =
-      installation_->clang_resource_path();
-  CARBON_RETURN_IF_ERROR(
-      builder.dir().Symlink("include", install_resource_path / "include"));
-  CARBON_RETURN_IF_ERROR(
-      builder.dir().Symlink("share", install_resource_path / "share"));
-
-  // Create the target's `lib` directory.
-  std::filesystem::path lib_path =
-      std::filesystem::path("lib") / std::string_view(target);
-  CARBON_ASSIGN_OR_RETURN(Filesystem::Dir lib_dir,
-                          builder.dir().CreateDirectories(lib_path));
-
-  llvm::Triple target_triple(target);
-  if (target_triple.isOSWindows()) {
-    return Error("TODO: Windows runtimes are untested and not yet supported.");
-  }
-
-  llvm::ThreadPoolTaskGroup task_group(threads);
-
-  // For Linux targets, the system libc (typically glibc) doesn't necessarily
-  // provide the CRT begin/end files, and so we need to build them.
-  if (target_triple.isOSLinux()) {
-    task_group.async(
-        [this, target,
-         path = builder.path() / lib_path / "clang_rt.crtbegin.o"] {
-          BuildCrtFile(target, RuntimeSources::CrtBegin, path);
-        });
-    task_group.async(
-        [this, target, path = builder.path() / lib_path / "clang_rt.crtend.o"] {
-          BuildCrtFile(target, RuntimeSources::CrtEnd, path);
-        });
-  }
-
-  CARBON_RETURN_IF_ERROR(
-      BuildBuiltinsLib(target, target_triple, tmp_path, lib_dir, threads));
-
-  // Now wait for all the queued builds to complete before we commit the
-  // runtimes into the cache.
-  task_group.wait();
-
-  return std::move(builder).Commit();
-}
-
-auto ClangRunner::RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args) -> int {
+auto ClangRunner::RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args,
+                         bool enable_leaking) -> int {
   llvm::BumpPtrAllocator allocator;
   llvm::cl::ExpansionContext expansion_context(
       allocator, llvm::cl::TokenizeGNUCommandLine);
@@ -300,7 +249,7 @@ auto ClangRunner::RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args) -> int {
       std::move(invocation), std::move(pch_ops));
 
   // Override the disabling of free when we don't want to leak memory.
-  if (!enable_leaking_) {
+  if (!enable_leaking) {
     clang_instance->getFrontendOpts().DisableFree = false;
     clang_instance->getCodeGenOpts().DisableFree = false;
   }
@@ -400,7 +349,8 @@ auto ClangRunner::RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args) -> int {
 
 auto ClangRunner::RunInternal(
     llvm::ArrayRef<llvm::StringRef> args, llvm::StringRef target,
-    std::optional<llvm::StringRef> target_resource_dir_path) -> bool {
+    std::optional<llvm::StringRef> target_resource_dir_path,
+    bool enable_leaking) -> bool {
   std::string clang_path = installation_->clang_path();
 
   // Rebuild the args as C-string args.
@@ -410,8 +360,27 @@ auto ClangRunner::RunInternal(
 
   // Handle special dispatch for CC1 commands as they don't use the driver.
   if (!args.empty() && args[0].starts_with("-cc1")) {
-    CARBON_VLOG("Calling Clang's CC1...");
-    int exit_code = RunCC1(cstr_args);
+    if (args[0] == "-cc1") {
+      CARBON_VLOG("Dispatching `-cc1` command line");
+      int exit_code = RunCC1(cstr_args, enable_leaking);
+      // TODO: Should this be forwarding the full exit code?
+      return exit_code == 0;
+    }
+
+    // Other CC1-based invocations need to dispatch into the `clang_main`
+    // routine to work correctly. This means they're not reliable in a library
+    // context but currently there is too much logic to reasonably extract here.
+    // This at least allows simple cases (often when directly used on the
+    // command line) to work correctly.
+    //
+    // TODO: Factor the relevant code paths into a library API or move this into
+    // the busybox dispatch logic.
+    CARBON_VLOG("Calling clang_main for a cc1-based invocation...");
+    // cstr_args[0] will be the `clang_path` so we don't need the prepend arg.
+    llvm::ToolContext tool_context = {
+        .Path = cstr_args[0], .PrependArg = "clang", .NeedsPrependArg = false};
+    int exit_code = clang_main(
+        cstr_args.size(), const_cast<char**>(cstr_args.data()), tool_context);
     // TODO: Should this be forwarding the full exit code?
     return exit_code == 0;
   }
@@ -481,8 +450,9 @@ auto ClangRunner::RunInternal(
   //
   // Also note that we only do `-disable-free` filtering in the in-process
   // execution here, as subprocesses leaking memory won't impact this process.
-  auto cc1_main = [this](llvm::SmallVectorImpl<const char*>& cc1_args) -> int {
-    return RunCC1(cc1_args);
+  auto cc1_main = [this, enable_leaking](
+                      llvm::SmallVectorImpl<const char*>& cc1_args) -> int {
+    return RunCC1(cc1_args, enable_leaking);
   };
   driver.CC1Main = cc1_main;
 
@@ -525,180 +495,4 @@ auto ClangRunner::RunInternal(
   return result == 0 && failing_commands.empty();
 }
 
-auto ClangRunner::BuildCrtFile(llvm::StringRef target, llvm::StringRef src_file,
-                               const std::filesystem::path& out_path) -> void {
-  std::filesystem::path src_path =
-      installation_->llvm_runtime_srcs() / std::string_view(src_file);
-  CARBON_VLOG("Building `{0}' from `{1}`...\n", out_path, src_path);
-
-  std::string target_arg = llvm::formatv("--target={0}", target).str();
-  CARBON_CHECK(RunWithNoRuntimes({
-      "-no-canonical-prefixes",
-      target_arg,
-      "-DCRT_HAS_INITFINI_ARRAY",
-      "-DEH_USE_FRAME_REGISTRY",
-      "-O3",
-      "-fPIC",
-      "-ffreestanding",
-      "-std=c11",
-      "-w",
-      "-c",
-      "-o",
-      out_path.native(),
-      src_path.native(),
-  }));
-}
-
-auto ClangRunner::CollectBuiltinsSrcFiles(const llvm::Triple& target_triple)
-    -> llvm::SmallVector<llvm::StringRef> {
-  llvm::SmallVector<llvm::StringRef> src_files;
-  auto append_src_files =
-      [&](auto input_srcs,
-          llvm::function_ref<bool(llvm::StringRef)> filter_out = {}) {
-        for (llvm::StringRef input_src : input_srcs) {
-          if (!input_src.ends_with(".c") && !input_src.ends_with(".S")) {
-            // Not a compiled file.
-            continue;
-          }
-          if (filter_out && filter_out(input_src)) {
-            // Filtered out.
-            continue;
-          }
-
-          src_files.push_back(input_src);
-        }
-      };
-  append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsGenericSrcs));
-  append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsBf16Srcs));
-  if (target_triple.isArch64Bit()) {
-    append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsTfSrcs));
-  }
-  auto filter_out_chkstk = [&](llvm::StringRef src) {
-    return !target_triple.isOSWindows() || !src.ends_with("chkstk.S");
-  };
-  if (target_triple.isAArch64()) {
-    append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsAarch64Srcs),
-                     filter_out_chkstk);
-  } else if (target_triple.isX86()) {
-    append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsX86ArchSrcs));
-    if (target_triple.isArch64Bit()) {
-      append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsX86_64Srcs),
-                       filter_out_chkstk);
-    } else {
-      // TODO: This should be turned into a nice user-facing diagnostic about an
-      // unsupported target.
-      CARBON_CHECK(
-          target_triple.isArch32Bit(),
-          "The Carbon toolchain doesn't currently support 16-bit x86.");
-      append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsI386Srcs),
-                       filter_out_chkstk);
-    }
-  } else {
-    // TODO: This should be turned into a nice user-facing diagnostic about an
-    // unsupported target.
-    CARBON_FATAL("Target architecture is not supported: {0}",
-                 target_triple.str());
-  }
-  return src_files;
-}
-
-auto ClangRunner::BuildBuiltinsFile(llvm::StringRef target,
-                                    llvm::StringRef src_file,
-                                    const std::filesystem::path& out_path)
-    -> void {
-  std::filesystem::path src_path =
-      installation_->llvm_runtime_srcs() / std::string_view(src_file);
-  CARBON_VLOG("Building `{0}' from `{1}`...\n", out_path, src_path);
-
-  std::string target_arg = llvm::formatv("--target={0}", target).str();
-  CARBON_CHECK(RunWithNoRuntimes({
-      "-no-canonical-prefixes",
-      target_arg,
-      "-O3",
-      "-fPIC",
-      "-ffreestanding",
-      "-fno-builtin",
-      "-fomit-frame-pointer",
-      "-fvisibility=hidden",
-      "-std=c11",
-      "-w",
-      "-c",
-      "-o",
-      out_path.native(),
-      src_path.native(),
-  }));
-}
-
-auto ClangRunner::BuildBuiltinsLib(llvm::StringRef target,
-                                   const llvm::Triple& target_triple,
-                                   const std::filesystem::path& tmp_path,
-                                   Filesystem::DirRef lib_dir,
-                                   llvm::ThreadPoolInterface& threads)
-    -> ErrorOr<Success> {
-  llvm::SmallVector<llvm::StringRef> src_files =
-      CollectBuiltinsSrcFiles(target_triple);
-
-  CARBON_ASSIGN_OR_RETURN(Filesystem::Dir tmp_dir,
-                          Filesystem::Cwd().OpenDir(tmp_path));
-
-  // `NewArchiveMember` isn't default constructable unfortunately, so we first
-  // build the objects using an optional wrapper.
-  llvm::SmallVector<std::optional<llvm::NewArchiveMember>> objs;
-  objs.resize(src_files.size());
-  llvm::ThreadPoolTaskGroup member_group(threads);
-  for (auto [src_file, obj] : llvm::zip_equal(src_files, objs)) {
-    // Create any subdirectories needed for this file.
-    std::filesystem::path src_path = src_file.str();
-    if (src_path.has_parent_path()) {
-      CARBON_RETURN_IF_ERROR(tmp_dir.CreateDirectories(src_path.parent_path()));
-    }
-
-    member_group.async([this, target, src_file, &obj, &tmp_path] {
-      std::filesystem::path obj_path = tmp_path / std::string_view(src_file);
-      obj_path += ".o";
-      BuildBuiltinsFile(target, src_file, obj_path);
-
-      auto obj_result = llvm::NewArchiveMember::getFile(obj_path.native(),
-                                                        /*Deterministic=*/true);
-      CARBON_CHECK(obj_result, "TODO: Diagnose this: {0}",
-                   llvm::fmt_consume(obj_result.takeError()));
-      obj = std::move(*obj_result);
-    });
-  }
-
-  // Now build an archive out of the `.o` files for the builtins. Note that we
-  // build this directly into the `lib_dir` as this is expected to be a staging
-  // directory and cleaned up on errors.
-  std::filesystem::path builtins_a_path = "libclang_rt.builtins.a";
-  CARBON_ASSIGN_OR_RETURN(
-      Filesystem::WriteFile builtins_a_file,
-      lib_dir.OpenWriteOnly(builtins_a_path, Filesystem::CreateAlways));
-
-  // Wait for all the object compiles to complete, and then move the objects out
-  // of their optional wrappers to match the API required by the archive writer.
-  member_group.wait();
-  llvm::SmallVector<llvm::NewArchiveMember> unwrapped_objs;
-  unwrapped_objs.reserve(objs.size());
-  for (auto& obj : objs) {
-    unwrapped_objs.push_back(*std::move(obj));
-  }
-  objs.clear();
-
-  // Write the actual archive.
-  {
-    llvm::raw_fd_ostream builtins_a_os = builtins_a_file.WriteStream();
-    llvm::Error archive_err = llvm::writeArchiveToStream(
-        builtins_a_os, unwrapped_objs, llvm::SymtabWritingMode::NormalSymtab,
-        target_triple.isOSDarwin() ? llvm::object::Archive::K_DARWIN
-                                   : llvm::object::Archive::K_GNU,
-        /*Deterministic=*/true, /*Thin=*/false);
-    // The presence of an error is `true`.
-    if (archive_err) {
-      return Error(llvm::toString(std::move(archive_err)));
-    }
-  }
-  CARBON_RETURN_IF_ERROR(std::move(builtins_a_file).Close());
-  return Success();
-}
-
 }  // namespace Carbon

toolchain/driver/clang_runner.h

@@ -6,11 +6,13 @@
 #define CARBON_TOOLCHAIN_DRIVER_CLANG_RUNNER_H_
 
 #include <filesystem>
+#include <optional>
 
-#include "clang/Basic/DiagnosticIDs.h"
 #include "common/error.h"
 #include "common/ostream.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -78,15 +80,16 @@ class ClangRunner : ToolRunnerBase {
   // whether than Clang invocation succeeded is returned.
   auto Run(llvm::ArrayRef<llvm::StringRef> args,
            Runtimes::Cache& runtimes_cache,
-           llvm::ThreadPoolInterface& runtimes_build_thread_pool)
-      -> ErrorOr<bool>;
+           llvm::ThreadPoolInterface& runtimes_build_thread_pool,
+           bool enable_leaking = false) -> ErrorOr<bool>;
 
   // Run Clang with the provided arguments and prebuilt runtimes.
   //
   // Similar to `Run`, but requires and uses pre-built runtimes rather than a
   // cache or building them on demand.
   auto RunWithPrebuiltRuntimes(llvm::ArrayRef<llvm::StringRef> args,
-                               Runtimes& prebuilt_runtimes) -> ErrorOr<bool>;
+                               Runtimes& prebuilt_runtimes,
+                               bool enable_leaking = false) -> ErrorOr<bool>;
 
   // Run Clang with the provided arguments and without any target runtimes.
   //
@@ -97,68 +100,38 @@ class ClangRunner : ToolRunnerBase {
   //
   // This function simply returns true or false depending on whether Clang runs
   // successfully, as it should display any needed error messages.
-  auto RunWithNoRuntimes(llvm::ArrayRef<llvm::StringRef> args) -> bool;
-
-  // Builds the target-specific resource directory for Clang.
-  //
-  // There is a resource directory installed along side the Clang binary that
-  // contains all the target independent files such as headers. However, for
-  // target-specific files like runtimes, we build those on demand here and
-  // return the path.
-  auto BuildTargetResourceDir(const Runtimes::Cache::Features& features,
-                              Runtimes& runtimes,
-                              const std::filesystem::path& tmp_path,
-                              llvm::ThreadPoolInterface& threads)
-      -> ErrorOr<std::filesystem::path>;
-
-  // Enable leaking memory.
-  //
-  // Clang can avoid deallocating some of its memory to improve compile time.
-  // However, this isn't compatible with library-based invocations. When using
-  // the runner in a context where memory leaks are acceptable, such as from a
-  // command line driver, you can use this to enable that leaking behavior. Note
-  // that this will not override _explicit_ `args` in a run invocation that
-  // cause leaking, it will merely disable Clang's libraries injecting that
-  // behavior.
-  auto EnableLeakingMemory() -> void { enable_leaking_ = true; }
+  auto RunWithNoRuntimes(llvm::ArrayRef<llvm::StringRef> args,
+                         bool enable_leaking = false) -> bool;
 
  private:
+  friend class ClangRuntimesBuilderBase;
+
   // Emulates `cc1_main` but in a way that doesn't assume it is running in the
   // main thread and can more easily fit into library calls to do compiles.
   //
   // TODO: Much of the logic here should be factored out of the CC1
   // implementation in Clang's driver and into a reusable part of its libraries.
   // That should allow reducing the code here to a minimal amount.
-  auto RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args) -> int;
+  auto RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args, bool enable_leaking)
+      -> int;
 
   // Handles building the Clang driver and passing the arguments down to it.
   auto RunInternal(llvm::ArrayRef<llvm::StringRef> args, llvm::StringRef target,
-                   std::optional<llvm::StringRef> target_resource_dir_path)
-      -> bool;
-
-  // Helper to compile a single file of the CRT runtimes.
-  auto BuildCrtFile(llvm::StringRef target, llvm::StringRef src_file,
-                    const std::filesystem::path& out_path) -> void;
+                   std::optional<llvm::StringRef> target_resource_dir_path,
+                   bool enable_leaking) -> bool;
 
   // Returns the target-specific source files for the builtins runtime library.
   auto CollectBuiltinsSrcFiles(const llvm::Triple& target_triple)
       -> llvm::SmallVector<llvm::StringRef>;
 
-  // Helper to compile a single file of the compiler builtins runtimes.
-  auto BuildBuiltinsFile(llvm::StringRef target, llvm::StringRef src_file,
-                         const std::filesystem::path& out_path) -> void;
-
   // Builds the builtins runtime library into the provided archive file path,
   // using the provided objects path for intermediate object files.
   auto BuildBuiltinsLib(llvm::StringRef target,
                         const llvm::Triple& target_triple,
-                        const std::filesystem::path& tmp_path,
-                        Filesystem::DirRef lib_dir,
-                        llvm::ThreadPoolInterface& threads) -> ErrorOr<Success>;
+                        const std::filesystem::path& runtimes_path,
+                        llvm::ThreadPoolTaskGroup& threads) -> ErrorOr<Success>;
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs_;
-
-  bool enable_leaking_ = false;
 };
 
 }  // namespace Carbon

toolchain/driver/clang_runner_test.cpp

@@ -8,30 +8,27 @@
 #include <gtest/gtest.h>
 
 #include <filesystem>
-#include <fstream>
 #include <string>
 #include <utility>
 
-#include "common/check.h"
 #include "common/ostream.h"
 #include "common/raw_string_ostream.h"
-#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
-#include "llvm/Support/FormatVariadic.h"
-#include "llvm/Support/Program.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Host.h"
 #include "testing/base/capture_std_streams.h"
 #include "testing/base/file_helpers.h"
 #include "testing/base/global_exe_path.h"
-#include "toolchain/driver/llvm_runner.h"
+#include "toolchain/driver/runtimes_cache.h"
+#include "toolchain/install/install_paths.h"
 
 namespace Carbon {
 namespace {
 
-using ::testing::Eq;
 using ::testing::HasSubstr;
-using ::testing::IsSupersetOf;
 using ::testing::StrEq;
 
 // NOLINTNEXTLINE(modernize-use-trailing-return-type): Macro based function.
@@ -178,84 +175,6 @@ TEST_F(ClangRunnerTest, CompileMultipleFiles) {
   compile("test3.cpp", "int test3() { return 0; }");
 }
 
-TEST_F(ClangRunnerTest, BuildResourceDir) {
-  ClangRunner runner(&install_paths_, vfs_, &llvm::errs());
-
-  // Note that we can't test arbitrary targets here as we need to be able to
-  // compile the builtin functions for the target. We use the default target as
-  // the most likely to pass.
-  std::string target = llvm::sys::getDefaultTargetTriple();
-  llvm::Triple target_triple(target);
-  Runtimes::Cache::Features features = {.target = target};
-  auto runtimes = *runtimes_cache_.Lookup(features);
-  auto tmp_dir = *Filesystem::MakeTmpDir();
-  llvm::DefaultThreadPool threads(llvm::optimal_concurrency());
-  auto build_result = runner.BuildTargetResourceDir(
-      features, runtimes, tmp_dir.abs_path(), threads);
-  ASSERT_TRUE(build_result.ok()) << build_result.error();
-  std::filesystem::path resource_dir_path = std::move(*build_result);
-
-  // For Linux we can directly check the CRT begin/end object files.
-  if (target_triple.isOSLinux()) {
-    std::filesystem::path crt_begin_path =
-        resource_dir_path / "lib" / target / "clang_rt.crtbegin.o";
-    ASSERT_TRUE(std::filesystem::is_regular_file(crt_begin_path));
-    auto begin_result =
-        llvm::object::ObjectFile::createObjectFile(crt_begin_path.native());
-    llvm::object::ObjectFile& crtbegin = *begin_result->getBinary();
-    EXPECT_TRUE(crtbegin.isELF());
-    EXPECT_TRUE(crtbegin.isObject());
-    EXPECT_THAT(crtbegin.getArch(), Eq(target_triple.getArch()));
-
-    llvm::SmallVector<llvm::object::SymbolRef> symbols(crtbegin.symbols());
-    // The first symbol should come from the source file.
-    EXPECT_THAT(*symbols.front().getName(), Eq("crtbegin.c"));
-
-    // Check for representative symbols of `crtbegin.o` -- we always use
-    // `.init_array` in our runtimes build so we have predictable functions.
-    EXPECT_THAT(symbols, IsSupersetOf({TextSymbolNamed("__do_init"),
-                                       TextSymbolNamed("__do_fini")}));
-
-    std::filesystem::path crt_end_path =
-        resource_dir_path / "lib" / target / "clang_rt.crtend.o";
-    ASSERT_TRUE(std::filesystem::is_regular_file(crt_end_path));
-    auto end_result =
-        llvm::object::ObjectFile::createObjectFile(crt_end_path.native());
-    llvm::object::ObjectFile& crtend = *end_result->getBinary();
-    EXPECT_TRUE(crtend.isELF());
-    EXPECT_TRUE(crtend.isObject());
-    EXPECT_THAT(crtend.getArch(), Eq(target_triple.getArch()));
-
-    // Just check the source file symbol, not much of interest in the end.
-    llvm::object::SymbolRef crtend_front_symbol = *crtend.symbol_begin();
-    EXPECT_THAT(*crtend_front_symbol.getName(), Eq("crtend.c"));
-  }
-
-  // Across all targets, check that the builtins archive exists, and contains a
-  // relevant symbol by running the `llvm-nm` tool over it. Using `nm` rather
-  // than directly inspecting the objects is a bit awkward, but lets us easily
-  // ignore the wrapping in an archive file.
-  std::filesystem::path builtins_path =
-      resource_dir_path / "lib" / target / "libclang_rt.builtins.a";
-  LLVMRunner llvm_runner(&install_paths_, &llvm::errs());
-  std::string out;
-  std::string err;
-  EXPECT_TRUE(Testing::CallWithCapturedOutput(out, err, [&] {
-    return llvm_runner.Run(LLVMTool::Nm, {builtins_path.native()});
-  }));
-
-  // Check that we found a definition of `__mulodi4`, a builtin function
-  // provided by Compiler-RT, but not `libgcc` historically. Note that on macOS
-  // there is a leading `_` due to mangling.
-  EXPECT_THAT(out, HasSubstr(target_triple.isMacOSX() ? "T ___mulodi4\n"
-                                                      : "T __mulodi4\n"));
-
-  // Check that we don't include the `chkstk` builtins outside of Windows.
-  if (!target_triple.isOSWindows()) {
-    EXPECT_THAT(out, Not(HasSubstr("chkstk")));
-  }
-}
-
 // It's hard to write a portable and reliable unittest for all the layers of the
 // Clang driver because they work hard to interact with the underlying
 // filesystem and operating system. For now, we just check that a link command

toolchain/driver/clang_runtimes.cpp

@@ -0,0 +1,403 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/clang_runtimes.h"
+
+#include <unistd.h>
+
+#include <algorithm>
+#include <filesystem>
+#include <mutex>
+#include <optional>
+#include <string_view>
+#include <utility>
+#include <variant>
+
+#include "common/check.h"
+#include "common/error.h"
+#include "common/filesystem.h"
+#include "common/latch.h"
+#include "common/vlog.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FormatAdapters.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/Host.h"
+#include "llvm/TargetParser/Triple.h"
+#include "toolchain/base/runtime_sources.h"
+#include "toolchain/driver/clang_runner.h"
+#include "toolchain/driver/runtimes_cache.h"
+
+namespace Carbon {
+
+auto ClangRuntimesBuilderBase::ArchiveBuilder::Setup(Latch::Handle latch_handle)
+    -> void {
+  // `NewArchiveMember` isn't default constructable unfortunately, so we have to
+  // manually populate the vector with errors that we'll replace with the actual
+  // result in each thread.
+  objs_.reserve(src_files_.size());
+  for (auto _ : src_files_) {
+    objs_.push_back(Error("Never constructed archive member!"));
+  }
+
+  // Finish building the archive when the last compile finishes.
+  Latch::Handle comp_latch_handle =
+      compilation_latch_.Init([this, latch_handle] { result_ = Finish(); });
+
+  // Add all the compiles to the thread pool to run concurrently. The latch
+  // handle ensures the last one triggers the finishing closure above.
+  for (auto [src_file, obj] : llvm::zip_equal(src_files_, objs_)) {
+    builder_->tasks_.async([this, comp_latch_handle, src_file, &obj]() mutable {
+      obj = CompileMember(src_file);
+    });
+  }
+}
+
+auto ClangRuntimesBuilderBase::ArchiveBuilder::Finish() -> ErrorOr<Success> {
+  // We build this directly into the desired location as this is expected to be
+  // a staging directory and cleaned up on errors. We do need to create any
+  // intermediate directories.
+  Filesystem::DirRef runtimes_dir = builder_->runtimes_builder_->dir();
+  if (archive_path_.has_parent_path()) {
+    CARBON_RETURN_IF_ERROR(
+        runtimes_dir.CreateDirectories(archive_path_.parent_path()));
+  }
+
+  // Check if any compilations ended up producing an error. If so, return the
+  // first error for the entire function. Otherwise, move the archive member
+  // into a direct vector to match the required archive building API.
+  llvm::SmallVector<llvm::NewArchiveMember> unwrapped_objs;
+  unwrapped_objs.reserve(objs_.size());
+  for (auto& obj : objs_) {
+    if (!obj.ok()) {
+      return std::move(obj).error();
+    }
+    unwrapped_objs.push_back(*std::move(obj));
+  }
+  objs_.clear();
+
+  // Remove any directories created for the object files, the files should
+  // already be removed. We walk the sorted list of these in reverse so we
+  // remove child directories before parent directories.
+  for (const auto& obj_dir : llvm::reverse(obj_dirs_)) {
+    auto rmdir_result = runtimes_dir.Rmdir(obj_dir);
+    // Don't return an error on failure here as this has no problematic
+    // effect, just log that we couldn't clean up a directory.
+    if (!rmdir_result.ok()) {
+      CARBON_VLOG("Unable to remove object directory `{0}` in the runtime: {1}",
+                  obj_dir.native(), rmdir_result.error());
+    }
+  }
+
+  // Write the actual archive.
+  CARBON_ASSIGN_OR_RETURN(
+      Filesystem::WriteFile archive_file,
+      runtimes_dir.OpenWriteOnly(archive_path_, Filesystem::CreateAlways));
+  {
+    llvm::raw_fd_ostream archive_os = archive_file.WriteStream();
+    llvm::Error archive_err = llvm::writeArchiveToStream(
+        archive_os, unwrapped_objs, llvm::SymtabWritingMode::NormalSymtab,
+        builder_->target_triple_.isOSDarwin() ? llvm::object::Archive::K_DARWIN
+                                              : llvm::object::Archive::K_GNU,
+        /*Deterministic=*/true, /*Thin=*/false);
+    // The presence of an error is `true`.
+    if (archive_err) {
+      (void)std::move(archive_file).Close();
+      return Error(llvm::toString(std::move(archive_err)));
+    }
+  }
+  // Close and return any errors, potentially from the writes above.
+  CARBON_RETURN_IF_ERROR(std::move(archive_file).Close());
+  return Success();
+}
+
+auto ClangRuntimesBuilderBase::ArchiveBuilder::CreateObjDir(
+    const std::filesystem::path& src_path) -> ErrorOr<Success> {
+  auto obj_dir_path = src_path.parent_path();
+  if (obj_dir_path.empty()) {
+    return Success();
+  }
+
+  std::scoped_lock lock(obj_dirs_mu_);
+  auto* it = std::lower_bound(obj_dirs_.begin(), obj_dirs_.end(), obj_dir_path);
+  if (it != obj_dirs_.end() && *it == obj_dir_path) {
+    return Success();
+  }
+
+  auto create_result =
+      builder_->runtimes_builder_->dir().CreateDirectories(obj_dir_path);
+  if (!create_result.ok()) {
+    return Error(llvm::formatv(
+        "Unable to create object directory mirroring source file `{0}`: {1}",
+        src_path, create_result.error()));
+  }
+
+  it = obj_dirs_.insert(it, obj_dir_path);
+
+  // Also insert any parent paths. These should always sort earlier.
+  CARBON_DCHECK(!obj_dir_path.has_parent_path() ||
+                obj_dir_path.parent_path() < obj_dir_path);
+  obj_dir_path = obj_dir_path.parent_path();
+  while (!obj_dir_path.empty()) {
+    it = std::lower_bound(obj_dirs_.begin(), it, obj_dir_path);
+    if (*it != obj_dir_path) {
+      it = obj_dirs_.insert(it, obj_dir_path);
+    }
+    obj_dir_path = obj_dir_path.parent_path();
+  }
+  return Success();
+}
+
+auto ClangRuntimesBuilderBase::ArchiveBuilder::CompileMember(
+    llvm::StringRef src_file) -> ErrorOr<llvm::NewArchiveMember> {
+  // Create any obj subdirectories needed for this file.
+  CARBON_RETURN_IF_ERROR(CreateObjDir(src_file.str()));
+
+  std::filesystem::path obj_path =
+      builder_->runtimes_builder_->path() / std::string_view(src_file);
+  obj_path += ".o";
+  std::filesystem::path src_path = srcs_path_ / std::string_view(src_file);
+  CARBON_VLOG("Building `{0}' from `{1}`...\n", obj_path, src_path);
+
+  llvm::SmallVector<llvm::StringRef> args(cflags_);
+
+  // Add language-specific flags based on file extension.
+  if (src_file.ends_with(".c")) {
+    args.push_back("-std=c11");
+  } else if (src_file.ends_with(".cpp")) {
+    args.push_back("-std=c++20");
+  }
+
+  // Collect the additional required flags and dynamic flags for this builder.
+  args.append({
+      "-c",
+      builder_->target_flag_,
+      "-o",
+      obj_path.native(),
+      src_path.native(),
+  });
+  if (!builder_->clang_->RunWithNoRuntimes(args)) {
+    return Error(
+        llvm::formatv("Failed to compile runtime source file '{0}'", src_file));
+  }
+
+  auto obj_result = llvm::NewArchiveMember::getFile(obj_path.native(),
+                                                    /*Deterministic=*/true);
+  if (!obj_result) {
+    return Error(llvm::formatv("Unable to read `{0}` object file: {1}",
+                               src_file,
+                               llvm::fmt_consume(obj_result.takeError())));
+  }
+
+  // Unlink the object file once we've read it. However, we log and ignore
+  // any errors here as they aren't fatal.
+  auto unlink_result = builder_->runtimes_builder_->dir().Unlink(obj_path);
+  if (!unlink_result.ok()) {
+    CARBON_VLOG("Unable to unlink object file `{0}`: {1}\n", obj_path,
+                unlink_result.error());
+  }
+
+  return std::move(*obj_result);
+}
+
+ClangResourceDirBuilder::ClangResourceDirBuilder(
+    ClangRunner* clang, llvm::ThreadPoolInterface* threads,
+    llvm::Triple target_triple, Runtimes* runtimes)
+    : ClangRuntimesBuilderBase(clang, threads, std::move(target_triple)),
+      crt_begin_result_(Error("Never built CRT begin file!")),
+      crt_end_result_(Error("Never built CRT end file!")) {
+  // Ensure we're on a platform where we _can_ build a working runtime.
+  if (target_triple_.isOSWindows()) {
+    result_ =
+        Error("TODO: Windows runtimes are untested and not yet supported.");
+    return;
+  }
+
+  auto build_dir_or_error = runtimes->Build(Runtimes::ClangResourceDir);
+  if (!build_dir_or_error.ok()) {
+    result_ = std::move(build_dir_or_error).error();
+    return;
+  }
+  auto build_dir = *std::move(build_dir_or_error);
+  if (std::holds_alternative<std::filesystem::path>(build_dir)) {
+    // Found cached build.
+    result_ = std::get<std::filesystem::path>(std::move(build_dir));
+    return;
+  }
+
+  runtimes_builder_ = std::get<Runtimes::Builder>(std::move(build_dir));
+  lib_path_ = std::filesystem::path("lib") / target_triple_.str();
+  archive_.emplace(this, lib_path_ / "libclang_rt.builtins.a",
+                   installation().llvm_runtime_srcs(),
+                   CollectBuiltinsSrcFiles(), /*cflags=*/
+                   llvm::SmallVector<llvm::StringRef>{
+                       "-no-canonical-prefixes",
+                       "-O3",
+                       "-fPIC",
+                       "-ffreestanding",
+                       "-fno-builtin",
+                       "-fomit-frame-pointer",
+                       "-fvisibility=hidden",
+                       "-w",
+                   });
+  tasks_.async([this]() { Setup(); });
+}
+
+auto ClangResourceDirBuilder::CollectBuiltinsSrcFiles()
+    -> llvm::SmallVector<llvm::StringRef> {
+  llvm::SmallVector<llvm::StringRef> src_files;
+  auto append_src_files =
+      [&](auto input_srcs,
+          llvm::function_ref<bool(llvm::StringRef)> filter_out = {}) {
+        for (llvm::StringRef input_src : input_srcs) {
+          if (!input_src.ends_with(".c") && !input_src.ends_with(".S")) {
+            // Not a compiled file.
+            continue;
+          }
+          if (filter_out && filter_out(input_src)) {
+            // Filtered out.
+            continue;
+          }
+
+          src_files.push_back(input_src);
+        }
+      };
+  append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsGenericSrcs));
+  append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsBf16Srcs));
+  if (target_triple_.isArch64Bit()) {
+    append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsTfSrcs));
+  }
+  auto filter_out_chkstk = [&](llvm::StringRef src) {
+    return !target_triple_.isOSWindows() || !src.ends_with("chkstk.S");
+  };
+  if (target_triple_.isAArch64()) {
+    append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsAarch64Srcs),
+                     filter_out_chkstk);
+  } else if (target_triple_.isX86()) {
+    append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsX86ArchSrcs));
+    if (target_triple_.isArch64Bit()) {
+      append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsX86_64Srcs),
+                       filter_out_chkstk);
+    } else {
+      // TODO: This should be turned into a nice user-facing diagnostic about an
+      // unsupported target.
+      CARBON_CHECK(
+          target_triple_.isArch32Bit(),
+          "The Carbon toolchain doesn't currently support 16-bit x86.");
+      append_src_files(llvm::ArrayRef(RuntimeSources::BuiltinsI386Srcs),
+                       filter_out_chkstk);
+    }
+  } else {
+    // TODO: This should be turned into a nice user-facing diagnostic about an
+    // unsupported target.
+    CARBON_FATAL("Target architecture is not supported: {0}",
+                 target_triple_.str());
+  }
+  return src_files;
+}
+
+auto ClangResourceDirBuilder::Setup() -> void {
+  // Symlink the installation's `include` and `share` directories.
+  std::filesystem::path install_resource_path =
+      installation().clang_resource_path();
+  if (auto result = runtimes_builder_->dir().Symlink(
+          "include", install_resource_path / "include");
+      !result.ok()) {
+    result_ = std::move(result).error();
+    return;
+  }
+  if (auto result = runtimes_builder_->dir().Symlink(
+          "share", install_resource_path / "share");
+      !result.ok()) {
+    result_ = std::move(result).error();
+    return;
+  }
+
+  // Create the target's `lib` directory.
+  auto lib_dir_result = runtimes_builder_->dir().CreateDirectories(lib_path_);
+  if (!lib_dir_result.ok()) {
+    result_ = std::move(lib_dir_result).error();
+    return;
+  }
+  lib_dir_ = *std::move(lib_dir_result);
+
+  Latch::Handle latch_handle =
+      step_counter_.Init([this] { tasks_.async([this] { Finish(); }); });
+
+  // For Linux targets, the system libc (typically glibc) doesn't necessarily
+  // provide the CRT begin/end files, and so we need to build them.
+  if (target_triple_.isOSLinux()) {
+    tasks_.async([this, latch_handle] {
+      crt_begin_result_ = BuildCrtFile(RuntimeSources::CrtBegin);
+    });
+    tasks_.async([this, latch_handle] {
+      crt_end_result_ = BuildCrtFile(RuntimeSources::CrtEnd);
+    });
+  }
+
+  archive_->Setup(std::move(latch_handle));
+}
+
+auto ClangResourceDirBuilder::Finish() -> void {
+  CARBON_VLOG("Finished building resource dir...\n");
+  if (!archive_->result().ok()) {
+    result_ = std::move(archive_->result()).error();
+    return;
+  }
+  if (target_triple_.isOSLinux()) {
+    for (ErrorOr<Success>* result : {&crt_begin_result_, &crt_end_result_}) {
+      if (!result->ok()) {
+        result_ = std::move(*result).error();
+        return;
+      }
+    }
+  }
+
+  result_ = (*std::move(runtimes_builder_)).Commit();
+}
+
+auto ClangResourceDirBuilder::BuildCrtFile(llvm::StringRef src_file)
+    -> ErrorOr<Success> {
+  CARBON_CHECK(src_file == RuntimeSources::CrtBegin ||
+               src_file == RuntimeSources::CrtEnd);
+  std::filesystem::path out_path =
+      runtimes_builder_->path() / lib_path_ /
+      (src_file == RuntimeSources::CrtBegin ? "clang_rt.crtbegin.o"
+                                            : "clang_rt.crtend.o");
+  std::filesystem::path src_path =
+      installation().llvm_runtime_srcs() / std::string_view(src_file);
+  CARBON_VLOG("Building `{0}' from `{1}`...\n", out_path, src_path);
+
+  bool success = clang_->RunWithNoRuntimes({
+      "-no-canonical-prefixes",
+      "-DCRT_HAS_INITFINI_ARRAY",
+      "-DEH_USE_FRAME_REGISTRY",
+      "-O3",
+      "-fPIC",
+      "-ffreestanding",
+      "-std=c11",
+      "-w",
+      "-c",
+      target_flag_,
+      "-o",
+      out_path.native(),
+      src_path.native(),
+  });
+
+  if (success) {
+    return Success();
+  }
+  return Error(llvm::formatv("Failed to compile CRT file: {0}", src_file));
+}
+
+}  // namespace Carbon

toolchain/driver/clang_runtimes.h

@@ -0,0 +1,259 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_DRIVER_CLANG_RUNTIMES_H_
+#define CARBON_TOOLCHAIN_DRIVER_CLANG_RUNTIMES_H_
+
+#include <filesystem>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "common/error.h"
+#include "common/filesystem.h"
+#include "common/latch.h"
+#include "common/ostream.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/TargetParser/Triple.h"
+#include "toolchain/driver/clang_runner.h"
+#include "toolchain/driver/runtimes_cache.h"
+#include "toolchain/install/install_paths.h"
+
+namespace Carbon {
+
+// Common APIs and utilities factored out of all of the Clang runtimes builders.
+//
+// The Clang runtimes builders are _asynchronous_ builders at their core, and so
+// their primary API is to construct the builder (kicking off work across
+// threads) and then `Wait` for it to finish.
+class ClangRuntimesBuilderBase {
+ public:
+  // Waits until the runtimes are finished being built and then return either
+  // any error encountered or the path of the built runtimes.
+  //
+  // If a path is returned, it will be the same as `Runtimes::Builder::Commit`
+  // would return.
+  auto Wait() && -> ErrorOr<std::filesystem::path> {
+    tasks_.wait();
+    return std::move(result_);
+  }
+
+ protected:
+  class ArchiveBuilder;
+
+  // Initializes the common state of a runtimes builder.
+  //
+  // Both the `clang` runner and the `threads` need to outlive this object.
+  ClangRuntimesBuilderBase(ClangRunner* clang,
+                           llvm::ThreadPoolInterface* threads,
+                           llvm::Triple target_triple)
+      : clang_(clang),
+        vlog_stream_(clang_->vlog_stream_),
+        tasks_(*threads),
+        target_triple_(std::move(target_triple)),
+        target_flag_(llvm::formatv("--target={0}", target_triple_.str())),
+        result_(Error("Did not finish building the runtime!")) {}
+
+  auto installation() -> const InstallPaths& { return *clang_->installation_; }
+
+  // We use protected members as this base is just factoring out common
+  // implementation details of other runners.
+  //
+  // NOLINTBEGIN(misc-non-private-member-variables-in-classes)
+
+  ClangRunner* clang_;
+  llvm::raw_ostream* vlog_stream_;
+
+  // This task group will be used both to launch asynchronous work and to wait
+  // for all of the work for a particular build to complete.
+  llvm::ThreadPoolTaskGroup tasks_;
+
+  llvm::Triple target_triple_;
+  std::string target_flag_;
+
+  ErrorOr<std::filesystem::path> result_;
+
+  // If runtimes already exist, we may never need a builder for them. But if we
+  // do need to build the runtimes, store the `Runtimes::Builder` here.
+  std::optional<Runtimes::Builder> runtimes_builder_;
+
+  // When building runtimes, this latch synchronizes all the steps required
+  // to build the runtimes into the above builder's staging directory. Once
+  // satisfied, it is typically used to schedule committing the runtimes as the
+  // last task.
+  Latch step_counter_;
+  // NOLINTEND(misc-non-private-member-variables-in-classes)
+};
+
+// Helper class that factors out the logic to build an archive as part of a set
+// of Clang runtimes.
+//
+// Runtimes can consist of one or more archives, and potentially other
+// artifacts, but because archives are very common we factor out logic to build
+// them here.
+class ClangRuntimesBuilderBase::ArchiveBuilder {
+ public:
+  // Initialize an archive builder.
+  //
+  // - The `builder` must outlive this object.
+  // - `archive_path` is the _relative_ path of the archive file within the
+  // built
+  //   runtimes directory.
+  // - `srcs_path` is the _absolute_ root of source files used to build the
+  // archive.
+  // - `src_files` is a list of the file paths to build into the archive,
+  //   relative to the `srcs_path`. These are `StringRef`s so they can reference
+  //   constant `StringLiteral`s in common cases.
+  // - `cflags` are the compile flags that should be used for all the compiles
+  //   in this archive.
+  ArchiveBuilder(ClangRuntimesBuilderBase* builder,
+                 std::filesystem::path archive_path,
+                 std::filesystem::path srcs_path,
+                 llvm::ArrayRef<llvm::StringRef> src_files,
+                 llvm::ArrayRef<llvm::StringRef> cflags)
+      : builder_(builder),
+        vlog_stream_(builder_->vlog_stream_),
+        archive_path_(std::move(archive_path)),
+        srcs_path_(std::move(srcs_path)),
+        src_files_(src_files),
+        cflags_(cflags) {}
+
+  // Start building the archive, with a latch handle to signal its completion.
+  //
+  // This will launch asynchronous tasks on the `builder_->tasks` task group to
+  // first compile all the members of the archive, and once compiled to put them
+  // into the archive file. Only when this last step is complete will the
+  // provided handle be destroyed, signaling this step of any concurrent build
+  // is done.
+  //
+  // This must only be called once per instance.
+  auto Setup(Latch::Handle latch_handle) -> void;
+
+  // Accessor for the result of building the archive.
+  //
+  // This will return an error if accessed prior to `Start`-ing the archive's
+  // build.
+  //
+  // Once `Start` has been called, this must not be called until the
+  // `latch_handle` provided to `Start` is destroyed as doing so will race with
+  // producing the result.
+  auto result() -> ErrorOr<Success>& { return result_; }
+
+ private:
+  // Helper for finishing the build of the archive.
+  //
+  // Must only be called once all members have been compiled, and returns the
+  // result of forming the archive file from those members.
+  auto Finish() -> ErrorOr<Success>;
+
+  // Given a specific `src_path` relative to our `srcs_path`, create any
+  // necessary directories relative to the build's runtimes root to allow the
+  // object file for this source file to be written there.
+  //
+  // Returns any errors encountered creating the necessary directories.
+  //
+  // Uses a cache to avoid redundant directory creations, and stores a list of
+  // all the directories created so they can be removed at the end of the build.
+  //
+  // This method is thread-safe, and designed to be called concurrently from
+  // each source file's compilation.
+  auto CreateObjDir(const std::filesystem::path& src_path) -> ErrorOr<Success>;
+
+  // Compiles the `src_file` and read the resulting object as a new archive
+  // member.
+  //
+  // The argument should be one of the elements of `src_files_`. The compile is
+  // performed using the `builder_->clang_` runner and the provided `cflags_`.
+  //
+  // Any errors encountered are returned.
+  auto CompileMember(llvm::StringRef src_file)
+      -> ErrorOr<llvm::NewArchiveMember>;
+
+  ClangRuntimesBuilderBase* builder_;
+  llvm::raw_ostream* vlog_stream_;
+
+  std::filesystem::path archive_path_;
+
+  std::filesystem::path srcs_path_;
+  llvm::SmallVector<llvm::StringRef> src_files_;
+
+  llvm::SmallVector<llvm::StringRef> cflags_;
+
+  // A latch used to synchronize building the archive once all members have been
+  // compiled.
+  Latch compilation_latch_;
+  // Storage for either the archive members or the error encountered while
+  // compiling them.
+  llvm::SmallVector<ErrorOr<llvm::NewArchiveMember>> objs_;
+
+  // A mutex and vector used to maintain a thread-safe cache of directories
+  // created to hold object files when compiling.
+  std::mutex obj_dirs_mu_;
+  llvm::SmallVector<std::filesystem::path> obj_dirs_;
+
+  // Storage for the final result of building the requested archive.
+  ErrorOr<Success> result_ = Error("No archive built!");
+};
+
+// Builds the target-specific resource directory for Clang.
+//
+// There is a resource directory installed along side the Clang binary that
+// contains all the target independent files such as headers. However, for
+// target-specific files like the runtimes that are part of the resource
+// directory, we build those on demand as runtimes.
+//
+class ClangResourceDirBuilder : public ClangRuntimesBuilderBase {
+ public:
+  // Constructing this class will attempt to build the Clang resource directory
+  // into `runtimes`.
+  //
+  // If an existing build is found, it will immediately be available.
+  // Otherwise, constructing this class will schedule asynchronous work on
+  // `threads` to build them on-demand using `clang`.
+  //
+  // Once constructed, callers may call `Wait` (from the base class) to wait
+  // until the asynchronous work is complete and the runtimes are available. If
+  // they were already available, the call to `Wait` will not block.
+  ClangResourceDirBuilder(ClangRunner* clang,
+                          llvm::ThreadPoolInterface* threads,
+                          llvm::Triple target_triple, Runtimes* runtimes);
+
+ private:
+  // Helper method to encapsulate the logic of configuring the list of source
+  // files to use in the `builtins` archive within these runtimes on a
+  // particular target.
+  auto CollectBuiltinsSrcFiles() -> llvm::SmallVector<llvm::StringRef>;
+
+  // Helper to encapsulate the initial, but still asynchronous setup work.
+  auto Setup() -> void;
+
+  // Helper to encapsulate the final asynchronous step in building the resource
+  // directory.
+  auto Finish() -> void;
+
+  // Helper to compile a single file of the CRT runtimes.
+  auto BuildCrtFile(llvm::StringRef src_file) -> ErrorOr<Success>;
+
+  // The `lib` path and subdirectory of the being-built runtimes.
+  std::filesystem::path lib_path_;
+  Filesystem::Dir lib_dir_;
+
+  // The results of compiling the CRT `begin` and `end` files.
+  ErrorOr<Success> crt_begin_result_;
+  ErrorOr<Success> crt_end_result_;
+
+  // The archive builder for the builtins archive in the resource directory.
+  std::optional<ArchiveBuilder> archive_;
+};
+
+}  // namespace Carbon
+
+#endif  // CARBON_TOOLCHAIN_DRIVER_CLANG_RUNTIMES_H_

toolchain/driver/clang_runtimes_test.cpp

@@ -0,0 +1,153 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/clang_runtimes.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <string>
+#include <utility>
+
+#include "common/ostream.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/TargetParser/Host.h"
+#include "llvm/TargetParser/Triple.h"
+#include "testing/base/capture_std_streams.h"
+#include "testing/base/global_exe_path.h"
+#include "toolchain/base/llvm_tools.h"
+#include "toolchain/driver/clang_runner.h"
+#include "toolchain/driver/llvm_runner.h"
+#include "toolchain/driver/runtimes_cache.h"
+#include "toolchain/install/install_paths.h"
+
+namespace Carbon {
+namespace {
+
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::IsSupersetOf;
+
+// NOLINTNEXTLINE(modernize-use-trailing-return-type): Macro based function.
+MATCHER_P(TextSymbolNamed, name_matcher, "") {
+  llvm::Expected<llvm::StringRef> name = arg.getName();
+  if (auto error = name.takeError()) {
+    *result_listener << "with an error instead of a name: " << error;
+    return false;
+  }
+  if (!testing::ExplainMatchResult(name_matcher, *name, result_listener)) {
+    return false;
+  }
+  // We have to dig out the section to determine if this was a text symbol.
+  auto expected_section_it = arg.getSection();
+  if (auto error = expected_section_it.takeError()) {
+    *result_listener << "without a section: " << error;
+    return false;
+  }
+  llvm::object::SectionRef section = **expected_section_it;
+  if (!section.isText()) {
+    *result_listener << "in the non-text section: " << *section.getName();
+    return false;
+  }
+  return true;
+}
+
+class ClangRuntimesTest : public ::testing::Test {
+ public:
+  InstallPaths install_paths_ =
+      InstallPaths::MakeForBazelRunfiles(Testing::GetExePath());
+  Runtimes::Cache runtimes_cache_ =
+      *Runtimes::Cache::MakeSystem(install_paths_);
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs_ =
+      llvm::vfs::getRealFileSystem();
+};
+
+TEST_F(ClangRuntimesTest, ResourceDir) {
+  ClangRunner runner(&install_paths_, vfs_, &llvm::errs());
+
+  // Note that we can't test arbitrary targets here as we need to be able to
+  // compile the builtin functions for the target. We use the default target as
+  // the most likely to pass.
+  std::string target = llvm::sys::getDefaultTargetTriple();
+  llvm::Triple target_triple(target);
+  Runtimes::Cache::Features features = {.target = target};
+  auto runtimes = *runtimes_cache_.Lookup(features);
+  llvm::DefaultThreadPool threads(llvm::optimal_concurrency());
+
+  ClangResourceDirBuilder resource_dir_builder(&runner, &threads, target_triple,
+                                               &runtimes);
+  auto build_result = std::move(resource_dir_builder).Wait();
+  ASSERT_TRUE(build_result.ok()) << build_result.error();
+  std::filesystem::path resource_dir_path = std::move(*build_result);
+
+  // For Linux we can directly check the CRT begin/end object files.
+  if (target_triple.isOSLinux()) {
+    std::filesystem::path crt_begin_path =
+        resource_dir_path / "lib" / target / "clang_rt.crtbegin.o";
+    ASSERT_TRUE(std::filesystem::is_regular_file(crt_begin_path));
+    auto begin_result =
+        llvm::object::ObjectFile::createObjectFile(crt_begin_path.native());
+    llvm::object::ObjectFile& crtbegin = *begin_result->getBinary();
+    EXPECT_TRUE(crtbegin.isELF());
+    EXPECT_TRUE(crtbegin.isObject());
+    EXPECT_THAT(crtbegin.getArch(), Eq(target_triple.getArch()));
+
+    llvm::SmallVector<llvm::object::SymbolRef> symbols(crtbegin.symbols());
+    // The first symbol should come from the source file.
+    EXPECT_THAT(*symbols.front().getName(), Eq("crtbegin.c"));
+
+    // Check for representative symbols of `crtbegin.o` -- we always use
+    // `.init_array` in our runtimes build so we have predictable functions.
+    EXPECT_THAT(symbols, IsSupersetOf({TextSymbolNamed("__do_init"),
+                                       TextSymbolNamed("__do_fini")}));
+
+    std::filesystem::path crt_end_path =
+        resource_dir_path / "lib" / target / "clang_rt.crtend.o";
+    ASSERT_TRUE(std::filesystem::is_regular_file(crt_end_path));
+    auto end_result =
+        llvm::object::ObjectFile::createObjectFile(crt_end_path.native());
+    llvm::object::ObjectFile& crtend = *end_result->getBinary();
+    EXPECT_TRUE(crtend.isELF());
+    EXPECT_TRUE(crtend.isObject());
+    EXPECT_THAT(crtend.getArch(), Eq(target_triple.getArch()));
+
+    // Just check the source file symbol, not much of interest in the end.
+    llvm::object::SymbolRef crtend_front_symbol = *crtend.symbol_begin();
+    EXPECT_THAT(*crtend_front_symbol.getName(), Eq("crtend.c"));
+  }
+
+  // Across all targets, check that the builtins archive exists, and contains a
+  // relevant symbol by running the `llvm-nm` tool over it. Using `nm` rather
+  // than directly inspecting the objects is a bit awkward, but lets us easily
+  // ignore the wrapping in an archive file.
+  std::filesystem::path builtins_path =
+      resource_dir_path / "lib" / target / "libclang_rt.builtins.a";
+  LLVMRunner llvm_runner(&install_paths_, &llvm::errs());
+  std::string out;
+  std::string err;
+  EXPECT_TRUE(Testing::CallWithCapturedOutput(out, err, [&] {
+    return llvm_runner.Run(LLVMTool::Nm, {builtins_path.native()});
+  }));
+
+  // Check that we found a definition of `__mulodi4`, a builtin function
+  // provided by Compiler-RT, but not `libgcc` historically. Note that on macOS
+  // there is a leading `_` due to mangling.
+  EXPECT_THAT(out, HasSubstr(target_triple.isMacOSX() ? "T ___mulodi4\n"
+                                                      : "T __mulodi4\n"));
+
+  // Check that we don't include the `chkstk` builtins outside of Windows.
+  if (!target_triple.isOSWindows()) {
+    EXPECT_THAT(out, Not(HasSubstr("chkstk")));
+  }
+}
+
+}  // namespace
+}  // namespace Carbon

toolchain/driver/clang_subcommand.cpp

@@ -78,20 +78,17 @@ auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
     return {.success = false};
   }
 
-  // Only enable Clang's leaking of memory if the driver can support that.
-  if (driver_env.enable_leaking) {
-    runner.EnableLeakingMemory();
-  }
-
   ErrorOr<bool> run_result = false;
   if (driver_env.prebuilt_runtimes) {
     run_result = runner.RunWithPrebuiltRuntimes(options_.args,
-                                                *driver_env.prebuilt_runtimes);
+                                                *driver_env.prebuilt_runtimes,
+                                                driver_env.enable_leaking);
   } else if (options_.build_runtimes_on_demand) {
     run_result = runner.Run(options_.args, driver_env.runtimes_cache,
-                            *driver_env.thread_pool);
+                            *driver_env.thread_pool, driver_env.enable_leaking);
   } else {
-    run_result = runner.RunWithNoRuntimes(options_.args);
+    run_result =
+        runner.RunWithNoRuntimes(options_.args, driver_env.enable_leaking);
   }
   if (!run_result.ok()) {
     // This is not a Clang failure, but a failure to even run Clang, so we need