Add a config subcommand for exposing build system info (#6637)

This makes it easy to wire up build systems like Bazel that need to know
the actual include paths used. It also gives us a convenient place to
export any other information that build systems or integrations need,
and to get debugging info from users.

Most of the complexity is computing the Clang header search paths, but
I couldn't see a direct way to get closer to the source-of-truth than
this, and it doesn't seem _too_ unreasonable.

Depends on #6636 - start review at commit
[643fdab1](6637/commits/643fdab1)

toolchain/base/clang_invocation.cpp

@@ -21,53 +21,34 @@ namespace Carbon {
 // The fake file name to use for the synthesized includes file.
 static constexpr const char IncludesFileName[] = "<carbon Cpp imports>";
 
-namespace {
-
-// Used to convert diagnostics from the Clang driver to Carbon diagnostics.
-class ClangDriverDiagnosticConsumer : public clang::DiagnosticConsumer {
- public:
-  // Creates an instance with the location that triggers calling Clang.
-  // `context` must not be null.
-  explicit ClangDriverDiagnosticConsumer(Diagnostics::NoLocEmitter* emitter)
-      : emitter_(emitter) {}
-
-  // Generates a Carbon warning for each Clang warning and a Carbon error for
-  // each Clang error or fatal.
-  auto HandleDiagnostic(clang::DiagnosticsEngine::Level diag_level,
-                        const clang::Diagnostic& info) -> void override {
-    DiagnosticConsumer::HandleDiagnostic(diag_level, info);
-
-    llvm::SmallString<256> message;
-    info.FormatDiagnostic(message);
-
-    switch (diag_level) {
-      case clang::DiagnosticsEngine::Ignored:
-      case clang::DiagnosticsEngine::Note:
-      case clang::DiagnosticsEngine::Remark: {
-        // TODO: Emit notes and remarks.
-        break;
-      }
-      case clang::DiagnosticsEngine::Warning:
-      case clang::DiagnosticsEngine::Error:
-      case clang::DiagnosticsEngine::Fatal: {
-        CARBON_DIAGNOSTIC(CppInteropDriverWarning, Warning, "{0}", std::string);
-        CARBON_DIAGNOSTIC(CppInteropDriverError, Error, "{0}", std::string);
-        emitter_->Emit(diag_level == clang::DiagnosticsEngine::Warning
-                           ? CppInteropDriverWarning
-                           : CppInteropDriverError,
-                       message.str().str());
-        break;
-      }
+auto ClangDriverDiagnosticConsumer::HandleDiagnostic(
+    clang::DiagnosticsEngine::Level diag_level, const clang::Diagnostic& info)
+    -> void {
+  DiagnosticConsumer::HandleDiagnostic(diag_level, info);
+
+  llvm::SmallString<256> message;
+  info.FormatDiagnostic(message);
+
+  switch (diag_level) {
+    case clang::DiagnosticsEngine::Ignored:
+    case clang::DiagnosticsEngine::Note:
+    case clang::DiagnosticsEngine::Remark: {
+      // TODO: Emit notes and remarks.
+      break;
+    }
+    case clang::DiagnosticsEngine::Warning:
+    case clang::DiagnosticsEngine::Error:
+    case clang::DiagnosticsEngine::Fatal: {
+      CARBON_DIAGNOSTIC(CppInteropDriverWarning, Warning, "{0}", std::string);
+      CARBON_DIAGNOSTIC(CppInteropDriverError, Error, "{0}", std::string);
+      emitter_->Emit(diag_level == clang::DiagnosticsEngine::Warning
+                         ? CppInteropDriverWarning
+                         : CppInteropDriverError,
+                     message.str().str());
+      break;
     }
   }
-
- private:
-  // Diagnostic emitter. Note that driver diagnostics don't have meaningful
-  // locations attached.
-  Diagnostics::NoLocEmitter* emitter_;
-};
-
-}  // namespace
+}
 
 auto BuildClangInvocation(Diagnostics::Consumer& consumer,
                           llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,

toolchain/base/clang_invocation.h

@@ -7,6 +7,7 @@
 
 #include <string>
 
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -16,6 +17,25 @@
 
 namespace Carbon {
 
+// Converts diagnostics from the Clang driver to Carbon diagnostics.
+class ClangDriverDiagnosticConsumer : public clang::DiagnosticConsumer {
+ public:
+  // Creates an instance with the location that triggers calling Clang.
+  // `context` must not be null.
+  explicit ClangDriverDiagnosticConsumer(Diagnostics::NoLocEmitter* emitter)
+      : emitter_(emitter) {}
+
+  // Generates a Carbon warning for each Clang warning and a Carbon error for
+  // each Clang error or fatal.
+  auto HandleDiagnostic(clang::DiagnosticsEngine::Level diag_level,
+                        const clang::Diagnostic& info) -> void override;
+
+ private:
+  // Diagnostic emitter. Note that driver diagnostics don't have meaningful
+  // locations attached.
+  Diagnostics::NoLocEmitter* emitter_;
+};
+
 // Builds and returns a clang `CompilerInvocation` to use when building code for
 // interop, from a list of clang driver arguments. Emits diagnostics to
 // `consumer` if the arguments are invalid.

toolchain/base/install_paths.h

@@ -92,6 +92,9 @@ class InstallPaths {
     return error_;
   }
 
+  // The path to the root of this installation.
+  auto root() const -> std::filesystem::path { return root_; }
+
   // The directory containing the `Core` package. Computed on demand.
   auto core_package() const -> std::filesystem::path;
 

toolchain/diagnostics/coverage_test.cpp

@@ -27,6 +27,8 @@ constexpr Kind UntestedKinds[] = {
     // Diagnosing erroneous install conditions, but test environments are
     // typically correct.
     Kind::CompilePreludeManifestError,
+    Kind::ConfigFailedToReadDigest,
+    Kind::ConfigFailedToSetupTarget,
     Kind::DriverInstallInvalid,
 
     // These diagnose filesystem issues that are hard to unit test.

toolchain/diagnostics/diagnostic_kind.def

@@ -30,6 +30,8 @@ CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError)
 CARBON_DIAGNOSTIC_KIND(CompileInputNotRegularFile)
 CARBON_DIAGNOSTIC_KIND(CompileOutputFileOpenError)
 CARBON_DIAGNOSTIC_KIND(CompileTargetInvalid)
+CARBON_DIAGNOSTIC_KIND(ConfigFailedToReadDigest)
+CARBON_DIAGNOSTIC_KIND(ConfigFailedToSetupTarget)
 CARBON_DIAGNOSTIC_KIND(FailureBuildingRuntimes)
 CARBON_DIAGNOSTIC_KIND(FailureRunningClang)
 CARBON_DIAGNOSTIC_KIND(FailureRunningClangToLink)

toolchain/driver/BUILD

@@ -168,6 +168,8 @@ cc_library(
         "clang_subcommand.h",
         "compile_subcommand.cpp",
         "compile_subcommand.h",
+        "config_subcommand.cpp",
+        "config_subcommand.h",
         "driver.cpp",
         "driver_env.h",
         "driver_subcommand.cpp",
@@ -196,8 +198,10 @@ cc_library(
         ":lld_runner",
         ":llvm_runner",
         ":runtimes_cache",
+        "//common:check",
         "//common:command_line",
         "//common:error",
+        "//common:filesystem",
         "//common:ostream",
         "//common:pretty_stack_trace_function",
         "//common:raw_string_ostream",
@@ -223,6 +227,8 @@ cc_library(
         "//toolchain/sem_ir:typed_insts",
         "//toolchain/source:source_buffer",
         "@llvm-project//clang:codegen",
+        "@llvm-project//clang:frontend",
+        "@llvm-project//clang:lex",
         "@llvm-project//llvm:Core",
         "@llvm-project//llvm:MC",
         "@llvm-project//llvm:Passes",
@@ -238,6 +244,8 @@ cc_test(
     deps = [
         ":driver",
         "//common:all_llvm_targets",
+        "//common:error_test_helpers",
+        "//common:filesystem",
         "//common:raw_string_ostream",
         "//testing/base:file_helpers",
         "//testing/base:global_exe_path",

toolchain/driver/config_subcommand.cpp

@@ -0,0 +1,228 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/config_subcommand.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <variant>
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "common/check.h"
+#include "common/command_line.h"
+#include "common/filesystem.h"
+#include "common/version.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/Triple.h"
+#include "toolchain/base/clang_invocation.h"
+#include "toolchain/diagnostics/diagnostic.h"
+#include "toolchain/diagnostics/diagnostic_consumer.h"
+#include "toolchain/diagnostics/diagnostic_emitter.h"
+#include "toolchain/driver/clang_runner.h"
+#include "toolchain/driver/driver_env.h"
+#include "toolchain/driver/driver_subcommand.h"
+
+namespace Carbon {
+
+auto ConfigOptions::Build(CommandLine::CommandBuilder& b) -> void {
+  b.AddFlag(
+      {
+          .name = "json",
+          .help = R"""(
+Render output as a JSON map for easy parsing.
+)""",
+      },
+      [&](auto& arg_b) {
+        arg_b.Default(false);
+        arg_b.Set(&json_output);
+      });
+
+  codegen_options.Build(b);
+}
+
+static constexpr CommandLine::CommandInfo SubcommandInfo = {
+    .name = "config",
+    .help = R"""(
+Print configuration info for the Carbon toolchain.
+
+This subcommand displays configuration information for the Carbon toolchain.
+This can be useful for build systems as well as debugging issues.
+)""",
+};
+
+ConfigSubcommand::ConfigSubcommand() : DriverSubcommand(SubcommandInfo) {}
+
+namespace {
+struct ConfigDataEntry {
+  std::string key;
+  std::variant<std::string, llvm::SmallVector<std::string>> value;
+};
+}  // namespace
+
+// Creates a Clang invocation and queries it for the include directories
+// searched during compilation. If there are any errors setting up Clang, this
+// will diagnose them using `driver_env.consumer` and return `std::nullopt`.
+static auto ComputeClangIncludeDirs(DriverEnv& driver_env,
+                                    llvm::StringRef target_str)
+    -> std::optional<llvm::SmallVector<std::string>> {
+  // Build a library invocation of Clang in order to query its header search
+  // paths.
+  std::shared_ptr clang_invocation =
+      BuildClangInvocation(driver_env.consumer, driver_env.fs,
+                           *driver_env.installation, target_str, {});
+  clang_invocation->getFrontendOpts().DisableFree = false;
+
+  // Setup up a driver-style diagnostic engine for the compiler invocation and
+  // instance below as we won't go past that while computing the include dirs.
+  Diagnostics::ErrorTrackingConsumer error_tracker(driver_env.consumer);
+  Diagnostics::NoLocEmitter emitter(&error_tracker);
+  ClangDriverDiagnosticConsumer diagnostic_consumer(&emitter);
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diags(
+      clang::CompilerInstance::createDiagnostics(
+          *driver_env.fs, clang_invocation->getDiagnosticOpts(),
+          &diagnostic_consumer,
+          /*ShouldOwnClient=*/false));
+
+  auto clang_instance =
+      std::make_unique<clang::CompilerInstance>(clang_invocation);
+  clang_instance->setDiagnostics(diags);
+  clang_instance->setVirtualFileSystem(driver_env.fs);
+  clang_instance->createFileManager();
+  clang_instance->createSourceManager();
+  if (!clang_instance->createTarget()) {
+    CARBON_DIAGNOSTIC(ConfigFailedToSetupTarget, Error,
+                      "unable to setup the requested target `{0}`",
+                      std::string);
+    driver_env.emitter.Emit(ConfigFailedToSetupTarget, target_str.str());
+    return std::nullopt;
+  }
+
+  auto header_search = std::make_unique<clang::HeaderSearch>(
+      clang_instance->getHeaderSearchOpts(), clang_instance->getSourceManager(),
+      clang_instance->getDiagnostics(), clang_instance->getLangOpts(),
+      &clang_instance->getTarget());
+  clang::ApplyHeaderSearchOptions(
+      *header_search, clang_instance->getHeaderSearchOpts(),
+      clang_instance->getLangOpts(), clang_instance->getTarget().getTriple());
+
+  // If we ended up diagnosing any errors, just return. They will have been
+  // converted to Carbon diagnostics.
+  if (error_tracker.seen_error()) {
+    return std::nullopt;
+  }
+
+  llvm::SmallVector<std::string> search_paths;
+  for (const auto& search_dir : header_search->search_dir_range()) {
+    search_paths.push_back(search_dir.getName().str());
+  }
+  return search_paths;
+}
+
+static auto RenderDataAsJson(llvm::ArrayRef<ConfigDataEntry> data,
+                             llvm::raw_ostream& out) -> void {
+  out << "{\n";
+  llvm::ListSeparator data_sep(",\n");
+  for (const auto& entry : data) {
+    out << data_sep << "    \"" << entry.key << "\": ";
+
+    if (const auto* value = std::get_if<std::string>(&entry.value)) {
+      out << "\"" << *value << "\"";
+    } else if (const auto* value =
+                   std::get_if<llvm::SmallVector<std::string>>(&entry.value)) {
+      out << "[\n";
+      llvm::ListSeparator element_sep(",\n");
+      for (const std::string& value_element : *value) {
+        out << element_sep << "        \"" << value_element << "\"";
+      }
+      out << "\n    ]";
+    } else {
+      CARBON_FATAL("Invalid value in config data entry!");
+    }
+  }
+  out << "\n}\n";
+}
+
+static auto RenderData(llvm::ArrayRef<ConfigDataEntry> data,
+                       llvm::raw_ostream& out) -> void {
+  for (const auto& entry : data) {
+    out << entry.key << ":";
+    if (const auto* value = std::get_if<std::string>(&entry.value)) {
+      out << " " << *value << "\n";
+    } else if (const auto* value =
+                   std::get_if<llvm::SmallVector<std::string>>(&entry.value)) {
+      out << "\n";
+      for (const std::string& value_element : *value) {
+        out << "    " << value_element << "\n";
+      }
+    } else {
+      CARBON_FATAL("Invalid value in config data entry!");
+    }
+  }
+}
+
+auto ConfigSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
+  bool result = true;
+
+  // Start with basic data available from the driver or global constants.
+  llvm::SmallVector<ConfigDataEntry> data = {
+      {.key = "CLANG_RESOURCE_DIR",
+       .value = driver_env.installation->clang_resource_path()},
+      {.key = "INSTALL_ROOT", .value = driver_env.installation->root()},
+      {.key = "LLVM_BINDIR",
+       .value = driver_env.installation->llvm_install_bin()},
+      {.key = "VERSION", .value = Version::String.str()},
+  };
+
+  // Try to read the installation digest and include that.
+  auto read_result = Filesystem::Cwd().ReadFileToString(
+      driver_env.installation->digest_path());
+  if (!read_result.ok()) {
+    CARBON_DIAGNOSTIC(ConfigFailedToReadDigest, Error,
+                      "unable to read the installation's digest file: {0}",
+                      std::string);
+    driver_env.emitter.Emit(ConfigFailedToReadDigest,
+                            read_result.error().ToString());
+
+    // Remember that we encountered an error but continue to give a minimally
+    // useful `config` output.
+    result = false;
+  } else {
+    data.push_back({.key = "INSTALL_DIGEST",
+                    .value = llvm::StringRef(*read_result).rtrim().str()});
+  }
+
+  // Compute and print Clang's include dirs if we can.
+  std::optional<llvm::SmallVector<std::string>> clang_include_dirs =
+      ComputeClangIncludeDirs(driver_env, options_.codegen_options.target);
+  if (clang_include_dirs) {
+    data.push_back(
+        {.key = "CLANG_INCLUDE_DIRS", .value = *std::move(clang_include_dirs)});
+  } else {
+    // This will have been diagnosed while computing, continue with degraded
+    // data.
+    result = false;
+  }
+
+  llvm::sort(data, [](const ConfigDataEntry& lhs, const ConfigDataEntry& rhs) {
+    return lhs.key < rhs.key;
+  });
+
+  if (options_.json_output) {
+    RenderDataAsJson(data, *driver_env.output_stream);
+  } else {
+    RenderData(data, *driver_env.output_stream);
+  }
+
+  return {.success = result};
+}
+
+}  // namespace Carbon

toolchain/driver/config_subcommand.h

@@ -0,0 +1,44 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_DRIVER_CONFIG_SUBCOMMAND_H_
+#define CARBON_TOOLCHAIN_DRIVER_CONFIG_SUBCOMMAND_H_
+
+#include "common/command_line.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "toolchain/driver/codegen_options.h"
+#include "toolchain/driver/driver_env.h"
+#include "toolchain/driver/driver_subcommand.h"
+
+namespace Carbon {
+
+// Options for the link subcommand.
+//
+// See the implementation of `Build` for documentation on members.
+struct ConfigOptions {
+  auto Build(CommandLine::CommandBuilder& b) -> void;
+
+  CodegenOptions codegen_options;
+  bool json_output;
+};
+
+// Implements the link subcommand of the driver.
+class ConfigSubcommand : public DriverSubcommand {
+ public:
+  explicit ConfigSubcommand();
+
+  auto BuildOptions(CommandLine::CommandBuilder& b) -> void override {
+    options_.Build(b);
+  }
+
+  auto Run(DriverEnv& driver_env) -> DriverResult override;
+
+ private:
+  ConfigOptions options_;
+};
+
+}  // namespace Carbon
+
+#endif  // CARBON_TOOLCHAIN_DRIVER_CONFIG_SUBCOMMAND_H_

toolchain/driver/driver.cpp

@@ -15,6 +15,7 @@
 #include "toolchain/driver/build_runtimes_subcommand.h"
 #include "toolchain/driver/clang_subcommand.h"
 #include "toolchain/driver/compile_subcommand.h"
+#include "toolchain/driver/config_subcommand.h"
 #include "toolchain/driver/format_subcommand.h"
 #include "toolchain/driver/language_server_subcommand.h"
 #include "toolchain/driver/link_subcommand.h"
@@ -40,6 +41,7 @@ struct Options {
   BuildRuntimesSubcommand runtimes;
   ClangSubcommand clang;
   CompileSubcommand compile;
+  ConfigSubcommand config;
   FormatSubcommand format;
   LanguageServerSubcommand language_server;
   LinkSubcommand link;
@@ -147,6 +149,7 @@ when there are errors or other output.
   runtimes.AddTo(b, &selected_subcommand);
   clang.AddTo(b, &selected_subcommand);
   compile.AddTo(b, &selected_subcommand);
+  config.AddTo(b, &selected_subcommand);
   format.AddTo(b, &selected_subcommand);
   language_server.AddTo(b, &selected_subcommand);
   link.AddTo(b, &selected_subcommand);

toolchain/driver/driver_test.cpp

@@ -9,14 +9,19 @@
 
 #include <filesystem>
 #include <fstream>
+#include <optional>
 #include <string>
 #include <system_error>
 #include <utility>
 
+#include "common/error_test_helpers.h"
+#include "common/filesystem.h"
 #include "common/raw_string_ostream.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Object/Binary.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "testing/base/file_helpers.h"
 #include "testing/base/global_exe_path.h"
 #include "toolchain/testing/yaml_test_helpers.h"
@@ -27,6 +32,9 @@ namespace {
 using ::testing::_;
 using ::testing::ContainsRegex;
 using ::testing::HasSubstr;
+using Testing::IsSuccess;
+using ::testing::Ne;
+using ::testing::NotNull;
 using ::testing::StrEq;
 
 namespace Yaml = ::Carbon::Testing::Yaml;
@@ -216,5 +224,33 @@ TEST_F(DriverTest, FileOutput) {
   EXPECT_THAT(*Testing::ReadFile("test.s"), ContainsRegex("Main:"));
 }
 
+TEST_F(DriverTest, ConfigJson) {
+  // The command won't succeed because we won't find the installation digest to
+  // print. We can still test the overall output is valid JSON.
+  EXPECT_FALSE(driver_.RunCommand({"config", "--json"}).success);
+
+  // Ensure the failure was what we expected.
+  EXPECT_THAT(
+      test_error_stream_.TakeStr(),
+      HasSubstr("error: unable to read the installation's digest file"));
+
+  // Make sure the output parses as JSON.
+  std::string output = test_output_stream_.TakeStr();
+  llvm::Expected<llvm::json::Value> json_value = llvm::json::parse(output);
+  if (auto error = json_value.takeError()) {
+    FAIL() << "Unable to parse to JSON: " << toString(std::move(error))
+           << "\nOriginal text:\n"
+           << output << "\n";
+  }
+  llvm::json::Object* json_obj = json_value->getAsObject();
+  ASSERT_THAT(json_obj, NotNull());
+
+  // Check relevant paths in the output point to existing directories.
+  std::optional<llvm::StringRef> install_root =
+      json_obj->getString("INSTALL_ROOT");
+  ASSERT_THAT(install_root, Ne(std::nullopt));
+  EXPECT_THAT(Filesystem::Cwd().OpenDir(install_root->str()), IsSuccess(_));
+}
+
 }  // namespace
 }  // namespace Carbon

toolchain/driver/testdata/fail_config.carbon

@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: config --target=x86_64-unknown-linux-gnu
+//
+// NOAUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/config.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/config.carbon
+
+// We don't include the digest in `file_test` to avoid unnecessary dependencies.
+// CHECK:STDERR: error: unable to read the installation's digest file: {{.*}}
+// CHECK:STDERR:
+
+// The rest of the config should still be displayed:
+// CHECK:STDOUT: CLANG_INCLUDE_DIRS:
+// CHECK:STDOUT:     {{.*}}/toolchain/install/prefix/lib/carbon/llvm/lib/clang/22/include
+// CHECK:STDOUT: CLANG_RESOURCE_DIR: {{.*}}/toolchain/install/prefix/lib/carbon/llvm/lib/clang/22
+// CHECK:STDOUT: INSTALL_ROOT: {{.*}}/toolchain/install/prefix/lib/carbon/
+// CHECK:STDOUT: LLVM_BINDIR: {{.*}}/toolchain/install/prefix/lib/carbon/llvm/bin
+// CHECK:STDOUT: VERSION: {{[0-9]+}}.{{[0-9]+}}.{{[0-9]+}}-{{.*}}