Propagate `llvm::vfs::FileSystem` from `driver_env` to Clang (#4537)

As discussed in #4530 . This required switching to using
`llvm::IntrusiveRefCntPtr` in a number of places.

---------

Co-authored-by: Josh L <josh11b@users.noreply.github.com>

explorer/file_test.cpp

@@ -28,8 +28,9 @@ class ExplorerFileTest : public FileTestBase {
   }
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
-           llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
+           llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<RunResult> override {
     // Add the prelude.
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
         llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon");
@@ -41,7 +42,8 @@ class ExplorerFileTest : public FileTestBase {
     // here.
     static constexpr llvm::StringLiteral PreludePath =
         "/explorer/data/prelude.carbon";
-    if (!fs.addFile(PreludePath, /*ModificationTime=*/0, std::move(*prelude))) {
+    if (!fs->addFile(PreludePath, /*ModificationTime=*/0,
+                     std::move(*prelude))) {
       return ErrorBuilder() << "Duplicate prelude.carbon";
     }
 
@@ -52,7 +54,7 @@ class ExplorerFileTest : public FileTestBase {
 
     int exit_code = ExplorerMain(
         args.size(), args.data(), /*install_path=*/"", PreludePath, stdout,
-        stderr, check_trace_output() ? stdout : trace_stream_, fs);
+        stderr, check_trace_output() ? stdout : trace_stream_, *fs);
 
     return {{.success = exit_code == EXIT_SUCCESS}};
   }

testing/base/source_gen_test.cpp

@@ -143,15 +143,16 @@ TEST(SourceGenTest, UniqueIdentifiers) {
 
 // Check that the source code doesn't have compiler errors.
 auto TestCompile(llvm::StringRef source) -> bool {
-  llvm::vfs::InMemoryFileSystem fs;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs =
+      new llvm::vfs::InMemoryFileSystem;
   InstallPaths installation(
       InstallPaths::MakeForBazelRunfiles(Testing::GetExePath()));
   Driver driver(fs, &installation, llvm::outs(), llvm::errs());
 
-  AddPreludeFilesToVfs(installation, &fs);
+  AddPreludeFilesToVfs(installation, fs);
 
-  fs.addFile("test.carbon", /*ModificationTime=*/0,
-             llvm::MemoryBuffer::getMemBuffer(source));
+  fs->addFile("test.carbon", /*ModificationTime=*/0,
+              llvm::MemoryBuffer::getMemBuffer(source));
   return driver.RunCommand({"compile", "--phase=check", "test.carbon"}).success;
 }
 

testing/file_test/file_test_base.cpp

@@ -297,12 +297,13 @@ auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
       DoArgReplacements(context.test_args, context.test_files));
 
   // Create the files in-memory.
-  llvm::vfs::InMemoryFileSystem fs;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs =
+      new llvm::vfs::InMemoryFileSystem;
   for (const auto& test_file : context.test_files) {
-    if (!fs.addFile(test_file.filename, /*ModificationTime=*/0,
-                    llvm::MemoryBuffer::getMemBuffer(
-                        test_file.content, test_file.filename,
-                        /*RequiresNullTerminator=*/false))) {
+    if (!fs->addFile(test_file.filename, /*ModificationTime=*/0,
+                     llvm::MemoryBuffer::getMemBuffer(
+                         test_file.content, test_file.filename,
+                         /*RequiresNullTerminator=*/false))) {
       return ErrorBuilder() << "File is repeated: " << test_file.filename;
     }
   }

testing/file_test/file_test_base.h

@@ -77,7 +77,7 @@ class FileTestBase : public testing::Test {
   // The return value should be an error if there was an abnormal error, and
   // RunResult otherwise.
   virtual auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
-                   llvm::vfs::InMemoryFileSystem& fs,
+                   llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
                    llvm::raw_pwrite_stream& stdout,
                    llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> = 0;
 

testing/file_test/file_test_base_test.cpp

@@ -22,8 +22,9 @@ class FileTestBaseTest : public FileTestBase {
       : FileTestBase(output_mutex, test_name) {}
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
-           llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override;
+           llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<RunResult> override;
 
   auto GetArgReplacements() -> llvm::StringMap<std::string> override {
     return {{"replacement", "replaced"}};
@@ -217,10 +218,10 @@ static auto EchoFileContent(TestParams& params)
   return {{.success = true}};
 }
 
-auto FileTestBaseTest::Run(const llvm::SmallVector<llvm::StringRef>& test_args,
-                           llvm::vfs::InMemoryFileSystem& fs,
-                           llvm::raw_pwrite_stream& stdout,
-                           llvm::raw_pwrite_stream& stderr)
+auto FileTestBaseTest::Run(
+    const llvm::SmallVector<llvm::StringRef>& test_args,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
+    llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
     -> ErrorOr<RunResult> {
   PrintArgs(test_args, stdout);
 
@@ -260,8 +261,8 @@ auto FileTestBaseTest::Run(const llvm::SmallVector<llvm::StringRef>& test_args,
           .Default(&EchoFileContent);
 
   // Call the appropriate test function for the file.
-  TestParams params = {.fs = fs, .stdout = stdout, .stderr = stderr};
-  CARBON_ASSIGN_OR_RETURN(params.files, GetFilesFromArgs(test_args, fs));
+  TestParams params = {.fs = *fs, .stdout = stdout, .stderr = stderr};
+  CARBON_ASSIGN_OR_RETURN(params.files, GetFilesFromArgs(test_args, *fs));
   return test_fn(params);
 }
 

toolchain/check/check_fuzzer.cpp

@@ -31,9 +31,10 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
   }
 
   static constexpr llvm::StringLiteral TestFileName = "test.carbon";
-  llvm::vfs::InMemoryFileSystem fs;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs =
+      new llvm::vfs::InMemoryFileSystem;
   llvm::StringRef data_ref(reinterpret_cast<const char*>(data), size);
-  CARBON_CHECK(fs.addFile(
+  CARBON_CHECK(fs->addFile(
       TestFileName, /*ModificationTime=*/0,
       llvm::MemoryBuffer::getMemBuffer(data_ref, /*BufferName=*/TestFileName,
                                        /*RequiresNullTerminator=*/false)));

toolchain/driver/clang_runner.cpp

@@ -26,7 +26,6 @@
 #include "llvm/Support/LLVMDriver.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
-#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Host.h"
 
 // Defined in:
@@ -42,9 +41,12 @@ auto clang_main(int Argc, char** Argv, const llvm::ToolContext& ToolContext)
 namespace Carbon {
 
 ClangRunner::ClangRunner(const InstallPaths* install_paths,
-                         llvm::StringRef target, llvm::raw_ostream* vlog_stream)
+                         llvm::StringRef target,
+                         llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
+                         llvm::raw_ostream* vlog_stream)
     : installation_(install_paths),
       target_(target),
+      fs_(std::move(fs)),
       vlog_stream_(vlog_stream),
       diagnostic_ids_(new clang::DiagnosticIDs()) {}
 
@@ -123,10 +125,10 @@ auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args) -> bool {
   clang::DiagnosticsEngine diagnostics(
       diagnostic_ids_, diagnostic_options.get(), &diagnostic_client,
       /*ShouldOwnClient=*/false);
-  auto vfs = llvm::vfs::getRealFileSystem();
-  clang::ProcessWarningOptions(diagnostics, *diagnostic_options, *vfs);
+  clang::ProcessWarningOptions(diagnostics, *diagnostic_options, *fs_);
 
-  clang::driver::Driver driver(clang_path, target_, diagnostics);
+  clang::driver::Driver driver(clang_path, target_, diagnostics,
+                               "clang LLVM compiler", fs_);
 
   // Configure the install directory to find other tools and data files.
   //

toolchain/driver/clang_runner.h

@@ -9,6 +9,7 @@
 #include "common/ostream.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "toolchain/install/install_paths.h"
 
 namespace Carbon {
@@ -42,6 +43,7 @@ class ClangRunner {
   // If `verbose` is passed as true, will enable verbose logging to the
   // `err_stream` both from the runner and Clang itself.
   ClangRunner(const InstallPaths* install_paths, llvm::StringRef target,
+              llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
               llvm::raw_ostream* vlog_stream = nullptr);
 
   // Run Clang with the provided arguments.
@@ -51,6 +53,7 @@ class ClangRunner {
   const InstallPaths* installation_;
 
   llvm::StringRef target_;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs_;
   llvm::raw_ostream* vlog_stream_;
 
   llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> diagnostic_ids_;

toolchain/driver/clang_runner_test.cpp

@@ -58,7 +58,8 @@ TEST(ClangRunnerTest, Version) {
   const auto install_paths =
       InstallPaths::MakeForBazelRunfiles(Testing::GetExePath());
   std::string target = llvm::sys::getDefaultTargetTriple();
-  ClangRunner runner(&install_paths, target, &test_os);
+  auto vfs = llvm::vfs::getRealFileSystem();
+  ClangRunner runner(&install_paths, target, vfs, &test_os);
 
   std::string out;
   std::string err;
@@ -129,7 +130,8 @@ TEST(ClangRunnerTest, LinkCommandEcho) {
   std::string verbose_out;
   llvm::raw_string_ostream verbose_os(verbose_out);
   std::string target = llvm::sys::getDefaultTargetTriple();
-  ClangRunner runner(&install_paths, target, &verbose_os);
+  auto vfs = llvm::vfs::getRealFileSystem();
+  ClangRunner runner(&install_paths, target, vfs, &verbose_os);
   std::string out;
   std::string err;
   EXPECT_TRUE(RunWithCapturedOutput(out, err,
@@ -162,7 +164,8 @@ TEST(ClangRunnerTest, DashC) {
   std::string verbose_out;
   llvm::raw_string_ostream verbose_os(verbose_out);
   std::string target = llvm::sys::getDefaultTargetTriple();
-  ClangRunner runner(&install_paths, target, &verbose_os);
+  auto vfs = llvm::vfs::getRealFileSystem();
+  ClangRunner runner(&install_paths, target, vfs, &verbose_os);
   std::string out;
   std::string err;
   EXPECT_TRUE(RunWithCapturedOutput(out, err,

toolchain/driver/clang_subcommand.cpp

@@ -45,7 +45,8 @@ ClangSubcommand::ClangSubcommand() : DriverSubcommand(SubcommandInfo) {}
 // https://github.com/llvm/llvm-project/blob/main/clang/tools/driver/driver.cpp
 auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   std::string target = llvm::sys::getDefaultTargetTriple();
-  ClangRunner runner(driver_env.installation, target, driver_env.vlog_stream);
+  ClangRunner runner(driver_env.installation, target, driver_env.fs,
+                     driver_env.vlog_stream);
 
   // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing
   // due to many unfixed issues.

toolchain/driver/compile_benchmark.cpp

@@ -24,7 +24,7 @@ class CompileBenchmark {
   CompileBenchmark()
       : installation_(InstallPaths::MakeForBazelRunfiles(GetExePath())),
         driver_(fs_, &installation_, llvm::outs(), llvm::errs()) {
-    AddPreludeFilesToVfs(installation_, &fs_);
+    AddPreludeFilesToVfs(installation_, fs_);
   }
 
   // Setup a set of source files in the VFS for the driver. Each string input is
@@ -35,8 +35,8 @@ class CompileBenchmark {
     llvm::OwningArrayRef<std::string> file_names(sources.size());
     for (ssize_t i : llvm::seq<ssize_t>(sources.size())) {
       file_names[i] = llvm::formatv("file_{0}.carbon", i).str();
-      fs_.addFile(file_names[i], /*ModificationTime=*/0,
-                  llvm::MemoryBuffer::getMemBuffer(sources[i]));
+      fs_->addFile(file_names[i], /*ModificationTime=*/0,
+                   llvm::MemoryBuffer::getMemBuffer(sources[i]));
     }
     return file_names;
   }
@@ -45,7 +45,8 @@ class CompileBenchmark {
   auto gen() -> SourceGen& { return gen_; }
 
  private:
-  llvm::vfs::InMemoryFileSystem fs_;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs_ =
+      new llvm::vfs::InMemoryFileSystem;
   const InstallPaths installation_;
   Driver driver_;
 

toolchain/driver/compile_subcommand.cpp

@@ -364,7 +364,7 @@ class CompilationUnit {
   // Loads source and lexes it. Returns true on success.
   auto RunLex() -> void {
     LogCall("SourceBuffer::MakeFromFileOrStdin", "source", [&] {
-      source_ = SourceBuffer::MakeFromFileOrStdin(driver_env_->fs,
+      source_ = SourceBuffer::MakeFromFileOrStdin(*driver_env_->fs,
                                                   input_filename_, *consumer_);
     });
     if (mem_usage_) {

toolchain/driver/driver.h

@@ -22,7 +22,8 @@ class Driver {
  public:
   // Constructs a driver with any error or informational output directed to a
   // specified stream.
-  Driver(llvm::vfs::FileSystem& fs, const InstallPaths* installation,
+  Driver(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
+         const InstallPaths* installation,
          llvm::raw_pwrite_stream& output_stream,
          llvm::raw_pwrite_stream& error_stream)
       : driver_env_{.fs = fs,

toolchain/driver/driver_env.h

@@ -13,7 +13,7 @@ namespace Carbon {
 
 struct DriverEnv {
   // The filesystem for source code.
-  llvm::vfs::FileSystem& fs;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs;
 
   // Helper to locate the toolchain installation's files.
   const InstallPaths* installation;

toolchain/driver/driver_fuzzer.cpp

@@ -78,7 +78,8 @@ extern "C" auto LLVMFuzzerTestOneInput(const unsigned char* data, size_t size)
     size -= arg_length;
   }
 
-  llvm::vfs::InMemoryFileSystem fs;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs =
+      new llvm::vfs::InMemoryFileSystem;
   TestRawOstream error_stream;
   llvm::raw_null_ostream dest;
   Driver d(fs, install_paths, dest, error_stream);

toolchain/driver/driver_test.cpp

@@ -53,8 +53,8 @@ class DriverTest : public testing::Test {
   auto MakeTestFile(llvm::StringRef text,
                     llvm::StringRef filename = "test_file.carbon")
       -> llvm::StringRef {
-    fs_.addFile(filename, /*ModificationTime=*/0,
-                llvm::MemoryBuffer::getMemBuffer(text));
+    fs_->addFile(filename, /*ModificationTime=*/0,
+                 llvm::MemoryBuffer::getMemBuffer(text));
     return filename;
   }
 
@@ -91,7 +91,8 @@ class DriverTest : public testing::Test {
     });
   }
 
-  llvm::vfs::InMemoryFileSystem fs_;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs_ =
+      new llvm::vfs::InMemoryFileSystem;
   const InstallPaths installation_;
   TestRawOstream test_output_stream_;
   TestRawOstream test_error_stream_;

toolchain/driver/format_subcommand.cpp

@@ -74,7 +74,8 @@ auto FormatSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
 
     // TODO: Consider refactoring this for sharing with compile.
     // TODO: Decide what to do with `-` when there are multiple arguments.
-    auto source = SourceBuffer::MakeFromFileOrStdin(driver_env.fs, f, consumer);
+    auto source =
+        SourceBuffer::MakeFromFileOrStdin(*driver_env.fs, f, consumer);
     if (!source) {
       mark_per_file_error();
       continue;

toolchain/driver/link_subcommand.cpp

@@ -114,7 +114,7 @@ auto LinkSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
                     options_.object_filenames.end());
 
   ClangRunner runner(driver_env.installation, options_.codegen_options.target,
-                     driver_env.vlog_stream);
+                     driver_env.fs, driver_env.vlog_stream);
   return {.success = runner.Run(clang_args)};
 }
 

toolchain/install/busybox_main.cpp

@@ -45,7 +45,7 @@ static auto Main(int argc, char** argv) -> ErrorOr<int> {
   }
   args.append(argv + 1, argv + argc);
 
-  Driver driver(*fs, &install_paths, llvm::outs(), llvm::errs());
+  Driver driver(fs, &install_paths, llvm::outs(), llvm::errs());
   bool success = driver.RunCommand(args).success;
   return success ? EXIT_SUCCESS : EXIT_FAILURE;
 }

toolchain/install/install_paths_test_helpers.cpp

@@ -10,8 +10,9 @@ namespace Carbon::Testing {
 
 // Prepares the VFS with prelude files from the real filesystem. Primarily for
 // tests.
-auto AddPreludeFilesToVfs(InstallPaths install_paths,
-                          llvm::vfs::InMemoryFileSystem* vfs) -> void {
+auto AddPreludeFilesToVfs(
+    InstallPaths install_paths,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& vfs) -> void {
   // Load the prelude into the test VFS.
   auto real_fs = llvm::vfs::getRealFileSystem();
   auto prelude = install_paths.ReadPreludeManifest();

toolchain/install/install_paths_test_helpers.h

@@ -11,8 +11,9 @@
 namespace Carbon::Testing {
 
 // Prepares the VFS with prelude files from the real filesystem.
-auto AddPreludeFilesToVfs(InstallPaths install_paths,
-                          llvm::vfs::InMemoryFileSystem* vfs) -> void;
+auto AddPreludeFilesToVfs(
+    InstallPaths install_paths,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& vfs) -> void;
 
 }  // namespace Carbon::Testing
 

toolchain/sem_ir/yaml_test.cpp

@@ -31,8 +31,9 @@ using ::testing::SizeIs;
 namespace Yaml = ::Carbon::Testing::Yaml;
 
 TEST(SemIRTest, YAML) {
-  llvm::vfs::InMemoryFileSystem fs;
-  CARBON_CHECK(fs.addFile(
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs =
+      new llvm::vfs::InMemoryFileSystem;
+  CARBON_CHECK(fs->addFile(
       "test.carbon", /*ModificationTime=*/0,
       llvm::MemoryBuffer::getMemBuffer("fn F() { var x: () = (); return; }")));
   const auto install_paths =

toolchain/testing/file_test.cpp

@@ -34,11 +34,12 @@ class ToolchainFileTest : public FileTestBase {
   }
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
-           llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
+           llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
+           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+      -> ErrorOr<RunResult> override {
     CARBON_ASSIGN_OR_RETURN(auto prelude, installation_.ReadPreludeManifest());
     for (const auto& file : prelude) {
-      CARBON_RETURN_IF_ERROR(AddFile(fs, file));
+      CARBON_RETURN_IF_ERROR(AddFile(*fs, file));
     }
 
     Driver driver(fs, &installation_, stdout, stderr);