Change explorer's file_test to support argument passing to main. (#3032)

This shifts explorer's file_test to use ExplorerMain in order to be able
to pass arguments similar to how the command line would. It also means
that various output wrapping done by the command line is shared, no
longer copied by file_test.

In order to help make this work, I plugged vfs::FileSystem into
explorer, similar to how we're doing this on the toolchain side (might
try to share more code later). I was having trouble getting an overlay
working, I think due to how paths are specified -- but due to the issues
in fixing this, I'm just loading the prelude into the InMemoryFilesystem
for now.

Under this approach, I'm unifying more of the file versus string
handling. I think we should shift towards an approach where, like
toolchain code, anyone trying to use Explorer should use a vfs
implementation to supply code. This should reduce the number of distinct
code paths which we're maintaining/testing.

Short-term, this allows the trace testdata to be merged into file_test
with less special casing, using ARGS:. Long-term, it means that the
file_test knows the ARGS to pass to generate checks to match against,
which is the direction I'm planning for autoupdate.

explorer/BUILD

@@ -23,6 +23,7 @@ cc_library(
     hdrs = ["main.h"],
     deps = [
         "//common:error",
+        "//common:ostream",
         "//explorer/common:trace_stream",
         "//explorer/parse_and_execute",
         "@llvm-project//llvm:Support",
@@ -45,7 +46,8 @@ cc_library(
     testonly = 1,
     srcs = ["file_test.cpp"],
     deps = [
-        "//explorer/parse_and_execute",
+        ":main",
+        "//common:check",
         "//testing/file_test:file_test_base",
         "//testing/util:test_raw_ostream",
         "@com_google_absl//absl/flags:flag",

explorer/common/source_location.h

@@ -26,7 +26,7 @@ class SourceLocation {
   }
 
   // The filename should be eternal or arena-allocated to eliminate copies.
-  explicit constexpr SourceLocation(const char* filename, int line_num,
+  explicit constexpr SourceLocation(std::string_view filename, int line_num,
                                     FileKind file_kind)
       : filename_(filename), line_num_(line_num), file_kind_(file_kind) {}
   explicit SourceLocation(Nonnull<const std::string*> filename, int line_num,

explorer/file_test.cpp

@@ -3,7 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 #include "absl/flags/flag.h"
-#include "explorer/parse_and_execute/parse_and_execute.h"
+#include "common/check.h"
+#include "explorer/main.h"
 #include "testing/file_test/file_test_base.h"
 #include "testing/util/test_raw_ostream.h"
 
@@ -24,61 +25,65 @@ class ParseAndExecuteTestFile : public FileTestBase {
                     const llvm::SmallVector<TestFile>& test_files,
                     llvm::raw_pwrite_stream& stdout,
                     llvm::raw_pwrite_stream& stderr) -> bool override {
-    CARBON_CHECK(test_args.empty())
-        << "ARGS are not currently used in explorer's file_test.";
+    // Create the files in-memory.
+    llvm::vfs::InMemoryFileSystem fs(new llvm::vfs::InMemoryFileSystem());
+    for (const auto& test_file : test_files) {
+      if (!fs.addFile(test_file.filename, /*ModificationTime=*/0,
+                      llvm::MemoryBuffer::getMemBuffer(test_file.content))) {
+        ADD_FAILURE() << "File is repeated: " << test_file.filename;
+        return false;
+      }
+    }
 
-    if (test_files.size() != 1) {
-      ADD_FAILURE() << "Only 1 file is supported: " << test_files.size()
-                    << " provided";
+    // Add the prelude.
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
+        llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon");
+    if (prelude.getError()) {
+      ADD_FAILURE() << prelude.getError().message();
+      return false;
+    }
+    // TODO: This path is long with a prefix / because of the path expectations
+    // in tests. Change those to allow a shorter path (e.g., `prelude.carbon`)
+    // here.
+    static constexpr llvm::StringLiteral PreludePath =
+        "/explorer/data/prelude.carbon";
+    if (!fs.addFile(PreludePath, /*ModificationTime=*/0, std::move(*prelude))) {
+      ADD_FAILURE() << "Duplicate prelude.carbon";
       return false;
     }
 
-    // Trace output is only checked for a few tests.
-    bool check_trace_output =
-        path().string().find("/trace_testdata/") != std::string::npos;
-
-    // Capture trace streaming, but only when in debug mode.
-    TraceStream trace_stream;
-    TestRawOstream trace_stream_ostream;
-    if (check_trace_output || absl::GetFlag(FLAGS_trace)) {
-      trace_stream.set_stream(check_trace_output ? &stdout
-                                                 : &trace_stream_ostream);
-      trace_stream.set_allowed_phases({ProgramPhase::All});
-      trace_stream.set_allowed_file_kinds({FileKind::Main});
+    llvm::SmallVector<const char*> args = {"explorer"};
+    for (auto arg : test_args) {
+      args.push_back(arg.data());
     }
+    TestRawOstream trace_stream;
 
-    // Set the location of the prelude.
-    char* test_srcdir = getenv("TEST_SRCDIR");
-    CARBON_CHECK(test_srcdir != nullptr);
-    std::string prelude_path(test_srcdir);
-    prelude_path += "/carbon/explorer/data/prelude.carbon";
+    // Trace output is only checked for a few tests.
+    bool check_trace_output =
+        path().string().find("trace_testdata/") != std::string::npos;
 
-    // Run the parse. Parser debug output is always off because it's difficult
-    // to redirect.
-    auto result = ParseAndExecute(
-        prelude_path, test_files[0].filename, test_files[0].content,
-        /*parser_debug=*/false, &trace_stream, &stdout);
-    // This mirrors printing currently done by main.cpp.
-    if (result.ok()) {
-      stdout << "result: " << *result << "\n";
-    } else {
-      stderr << result.error() << "\n";
-    }
+    int exit_code = ExplorerMain(
+        args.size(), args.data(), /*install_path=*/"", PreludePath, stdout,
+        stderr, check_trace_output ? stdout : trace_stream, fs);
 
     // Skip trace test check as they use stdout stream instead of
     // trace_stream_ostream
     if (absl::GetFlag(FLAGS_trace)) {
-      CARBON_CHECK(!check_trace_output)
-          << "trace tests should only be run in the default mode.";
-      EXPECT_FALSE(trace_stream_ostream.TakeStr().empty())
+      EXPECT_FALSE(trace_stream.TakeStr().empty())
           << "Tracing should always do something";
     }
 
-    return result.ok();
+    return exit_code == EXIT_SUCCESS;
   }
 
   auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    return {};
+    llvm::SmallVector<std::string> args;
+    if (absl::GetFlag(FLAGS_trace)) {
+      args.push_back("--trace_file=-");
+      args.push_back("--trace_phase=all");
+    }
+    args.push_back("%s");
+    return args;
   }
 };
 

explorer/fuzzing/ast_to_proto_main.cpp

@@ -31,15 +31,10 @@ auto Main(int argc, char** argv) -> ErrorOr<Success> {
     return Error("Argument must be a file.");
   }
 
-  // Read the input file.
-  std::ifstream proto_file(argv[1]);
-  std::stringstream buffer;
-  buffer << proto_file.rdbuf();
-  proto_file.close();
-
   Arena arena;
-  const ErrorOr<AST> ast = Parse(&arena, argv[1], FileKind::Main,
-                                 /*parser_debug=*/false);
+  const ErrorOr<AST> ast =
+      Parse(*llvm::vfs::getRealFileSystem(), &arena, argv[1], FileKind::Main,
+            /*parser_debug=*/false);
   if (!ast.ok()) {
     return ErrorBuilder() << "Parsing failed: " << ast.error().message();
   }

explorer/fuzzing/ast_to_proto_test.cpp

@@ -99,8 +99,8 @@ TEST(AstToProtoTest, SetsAllProtoFields) {
   Fuzzing::Carbon merged_proto;
   for (const llvm::StringRef f : *carbon_files) {
     Arena arena;
-    const ErrorOr<AST> ast =
-        Parse(&arena, f, FileKind::Main, /*parser_debug=*/false);
+    const ErrorOr<AST> ast = Parse(*llvm::vfs::getRealFileSystem(), &arena, f,
+                                   FileKind::Main, /*parser_debug=*/false);
     if (ast.ok()) {
       merged_proto.MergeFrom(AstToProto(*ast));
     }
@@ -137,8 +137,8 @@ TEST(AstToProtoTest, Roundtrip) {
   int parsed_ok_count = 0;
   for (const llvm::StringRef f : *carbon_files) {
     Arena arena;
-    const ErrorOr<AST> ast =
-        Parse(&arena, f, FileKind::Main, /*parser_debug=*/false);
+    const ErrorOr<AST> ast = Parse(*llvm::vfs::getRealFileSystem(), &arena, f,
+                                   FileKind::Main, /*parser_debug=*/false);
     if (ast.ok()) {
       ++parsed_ok_count;
       const std::string source_from_proto =
@@ -177,8 +177,8 @@ TEST(AstToProtoTest, SameProtoAfterClone) {
   int parsed_ok_count = 0;
   for (const llvm::StringRef f : *carbon_files) {
     Arena arena;
-    const ErrorOr<AST> ast =
-        Parse(&arena, f, FileKind::Main, /*parser_debug=*/false);
+    const ErrorOr<AST> ast = Parse(*llvm::vfs::getRealFileSystem(), &arena, f,
+                                   FileKind::Main, /*parser_debug=*/false);
     if (ast.ok()) {
       ++parsed_ok_count;
       const AST clone = CloneAST(arena, *ast);

explorer/fuzzing/fuzzer_util.cpp

@@ -33,14 +33,24 @@ auto GetRunfilesFile(const std::string& file) -> ErrorOr<std::string> {
 }
 
 auto ParseAndExecuteProto(const Fuzzing::Carbon& carbon) -> ErrorOr<int> {
+  llvm::vfs::InMemoryFileSystem fs;
+
   const ErrorOr<std::string> prelude_path =
       GetRunfilesFile("carbon/explorer/data/prelude.carbon");
   // Can't do anything without a prelude, so it's a fatal error.
   CARBON_CHECK(prelude_path.ok()) << prelude_path.error();
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
+      llvm::MemoryBuffer::getFile(*prelude_path);
+  CARBON_CHECK(!prelude.getError()) << prelude.getError().message();
+  CARBON_CHECK(fs.addFile("prelude.carbon", /*ModificationTime=*/0,
+                          std::move(*prelude)));
 
   const std::string source = ProtoToCarbon(carbon, /*maybe_add_main=*/true);
+  CARBON_CHECK(fs.addFile("fuzzer.carbon", /*ModificationTime=*/0,
+                          llvm::MemoryBuffer::getMemBuffer(source)));
+
   TraceStream trace_stream;
-  return ParseAndExecute(*prelude_path, "fuzzer.carbon", source,
+  return ParseAndExecute(fs, "prelude.carbon", "fuzzer.carbon",
                          /*parser_debug=*/false, &trace_stream, &llvm::nulls());
 }
 

explorer/main.cpp

@@ -17,6 +17,7 @@
 #include "common/error.h"
 #include "explorer/common/trace_stream.h"
 #include "explorer/parse_and_execute/parse_and_execute.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
@@ -32,7 +33,6 @@ namespace path = llvm::sys::path;
 auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
                   llvm::StringRef relative_prelude_path) -> int {
   llvm::setBugReportMsg(
-
       "Please report issues to "
       "https://github.com/carbon-language/carbon-lang/issues and include the "
       "crash backtrace.\n");
@@ -42,6 +42,20 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
   // is piped to stdout.
   llvm::errs().tie(&llvm::outs());
 
+  std::string exe =
+      llvm::sys::fs::getMainExecutable(argv[0], static_for_main_addr);
+  llvm::StringRef install_path = path::parent_path(exe);
+
+  return ExplorerMain(argc, const_cast<const char**>(argv), install_path,
+                      relative_prelude_path, llvm::outs(), llvm::errs(),
+                      llvm::outs(), *llvm::vfs::getRealFileSystem());
+}
+
+auto ExplorerMain(int argc, const char** argv, llvm::StringRef install_path,
+                  llvm::StringRef relative_prelude_path,
+                  llvm::raw_ostream& out_stream, llvm::raw_ostream& err_stream,
+                  llvm::raw_ostream& out_stream_for_trace,
+                  llvm::vfs::FileSystem& fs) -> int {
   cl::opt<std::string> input_file_name(cl::Positional, cl::desc("<input file>"),
                                        cl::Required);
   cl::opt<bool> parser_debug("parser_debug",
@@ -98,10 +112,9 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
                      "Include trace output for all files")),
       cl::CommaSeparated);
 
+  CARBON_CHECK(argc > 0);
+
   // Use the executable path as a base for the relative prelude path.
-  std::string exe =
-      llvm::sys::fs::getMainExecutable(argv[0], static_for_main_addr);
-  llvm::StringRef install_path = path::parent_path(exe);
   llvm::SmallString<256> default_prelude_file(install_path);
   path::append(default_prelude_file,
                path::begin(relative_prelude_path, path::Style::posix),
@@ -111,6 +124,8 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
                                          cl::init(default_prelude_file_str));
 
   cl::ParseCommandLineOptions(argc, argv);
+  auto reset_parser =
+      llvm::make_scope_exit([] { cl::ResetCommandLineParser(); });
 
   // Set up a stream for trace output.
   std::unique_ptr<llvm::raw_ostream> scoped_trace_stream;
@@ -147,13 +162,13 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
     trace_stream.set_allowed_file_kinds(trace_file_kinds);
 
     if (trace_file_name == "-") {
-      trace_stream.set_stream(&llvm::outs());
+      trace_stream.set_stream(&out_stream_for_trace);
     } else {
       std::error_code err;
       scoped_trace_stream =
           std::make_unique<llvm::raw_fd_ostream>(trace_file_name, err);
       if (err) {
-        llvm::errs() << err.message() << "\n";
+        err_stream << err.message() << "\n";
         return EXIT_FAILURE;
       }
       trace_stream.set_stream(scoped_trace_stream.get());
@@ -161,14 +176,14 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
   }
 
   ErrorOr<int> result =
-      ParseAndExecuteFile(prelude_file_name, input_file_name, parser_debug,
-                          &trace_stream, &llvm::outs());
+      ParseAndExecute(fs, prelude_file_name, input_file_name, parser_debug,
+                      &trace_stream, &out_stream);
   if (result.ok()) {
     // Print the return code to stdout.
-    llvm::outs() << "result: " << *result << "\n";
+    out_stream << "result: " << *result << "\n";
     return EXIT_SUCCESS;
   } else {
-    llvm::errs() << result.error() << "\n";
+    err_stream << result.error() << "\n";
     return EXIT_FAILURE;
   }
 }

explorer/main.h

@@ -5,7 +5,9 @@
 #ifndef CARBON_EXPLORER_MAIN_H_
 #define CARBON_EXPLORER_MAIN_H_
 
+#include "common/ostream.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 namespace Carbon {
 
@@ -14,6 +16,19 @@ namespace Carbon {
 auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
                   llvm::StringRef relative_prelude_path) -> int;
 
+// Wrapped by the above, but without some process init. This is used directly by
+// tests, whereas the above is used directly by main binaries. The
+// out_stream_for_trace is only used when `--trace_file=-` is specified.
+//
+// TODO: We use argc/argv for parameters because command line parsing requires
+// it. When that's replaced, we should switch to
+// llvm::SmallVector<llvm::StringRef> args because it'll work better with tests.
+auto ExplorerMain(int argc, const char** argv, llvm::StringRef install_path,
+                  llvm::StringRef relative_prelude_path,
+                  llvm::raw_ostream& out_stream, llvm::raw_ostream& err_stream,
+                  llvm::raw_ostream& out_stream_for_trace,
+                  llvm::vfs::FileSystem& fs) -> int;
+
 }  // namespace Carbon
 
 #endif  // CARBON_EXPLORER_MAIN_H_

explorer/parse_and_execute/parse_and_execute.cpp

@@ -6,7 +6,6 @@
 
 #include <locale>
 
-#include "common/check.h"
 #include "common/error.h"
 #include "explorer/common/trace_stream.h"
 #include "explorer/interpreter/exec_program.h"
@@ -37,22 +36,22 @@ static auto PrintTimingOnExit(TraceStream* trace_stream, const char* label,
   return exit_scope_function;
 }
 
-static auto ParseAndExecuteHelper(std::function<ErrorOr<AST>(Arena*)> parse,
-                                  std::string_view prelude_path,
-                                  Nonnull<TraceStream*> trace_stream,
-                                  Nonnull<llvm::raw_ostream*> print_stream)
-    -> ErrorOr<int> {
+auto ParseAndExecute(llvm::vfs::FileSystem& fs, std::string_view prelude_path,
+                     std::string_view input_file_name, bool parser_debug,
+                     Nonnull<TraceStream*> trace_stream,
+                     Nonnull<llvm::raw_ostream*> print_stream) -> ErrorOr<int> {
   return RunWithExtraStack([&]() -> ErrorOr<int> {
     Arena arena;
     auto cursor = std::chrono::steady_clock::now();
 
-    ErrorOr<AST> parse_result = parse(&arena);
+    ErrorOr<AST> parse_result =
+        Parse(fs, &arena, input_file_name, FileKind::Main, parser_debug);
     auto print_parse_time = PrintTimingOnExit(trace_stream, "Parse", &cursor);
     if (!parse_result.ok()) {
       return ErrorBuilder() << "SYNTAX ERROR: " << parse_result.error();
     }
 
-    AddPrelude(prelude_path, &arena, &parse_result->declarations,
+    AddPrelude(fs, prelude_path, &arena, &parse_result->declarations,
                &parse_result->num_prelude_declarations);
     auto print_prelude_time =
         PrintTimingOnExit(trace_stream, "AddPrelude", &cursor);
@@ -87,27 +86,4 @@ static auto ParseAndExecuteHelper(std::function<ErrorOr<AST>(Arena*)> parse,
   });
 }
 
-auto ParseAndExecuteFile(std::string_view prelude_path,
-                         std::string_view input_file_name, bool parser_debug,
-                         Nonnull<TraceStream*> trace_stream,
-                         Nonnull<llvm::raw_ostream*> print_stream)
-    -> ErrorOr<int> {
-  auto parse = [&](Arena* arena) {
-    return Parse(arena, input_file_name, FileKind::Main, parser_debug);
-  };
-  return ParseAndExecuteHelper(parse, prelude_path, trace_stream, print_stream);
-}
-
-auto ParseAndExecute(std::string_view prelude_path,
-                     std::string_view input_file_name,
-                     std::string_view file_contents, bool parser_debug,
-                     Nonnull<TraceStream*> trace_stream,
-                     Nonnull<llvm::raw_ostream*> print_stream) -> ErrorOr<int> {
-  auto parse = [&](Arena* arena) {
-    return ParseFromString(arena, input_file_name, FileKind::Main,
-                           file_contents, parser_debug);
-  };
-  return ParseAndExecuteHelper(parse, prelude_path, trace_stream, print_stream);
-}
-
 }  // namespace Carbon

explorer/parse_and_execute/parse_and_execute.h

@@ -7,22 +7,13 @@
 
 #include "common/error.h"
 #include "explorer/common/trace_stream.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 namespace Carbon {
 
 // Parses and executes the input file, returning the program result on success.
-// This API is intended for use by main execution.
-auto ParseAndExecuteFile(std::string_view prelude_path,
-                         std::string_view input_file_name, bool parser_debug,
-                         Nonnull<TraceStream*> trace_stream,
-                         Nonnull<llvm::raw_ostream*> print_stream)
-    -> ErrorOr<int>;
-
-// Parses and executes the source, returning the program result on success.
-// Discards output.
-auto ParseAndExecute(std::string_view prelude_path,
-                     std::string_view input_file_name,
-                     std::string_view file_contents, bool parser_debug,
+auto ParseAndExecute(llvm::vfs::FileSystem& fs, std::string_view prelude_path,
+                     std::string_view input_file_name, bool parser_debug,
                      Nonnull<TraceStream*> trace_stream,
                      Nonnull<llvm::raw_ostream*> print_stream) -> ErrorOr<int>;
 

explorer/parse_and_execute/parse_and_execute_test.cpp

@@ -10,10 +10,16 @@
 namespace Carbon::Testing {
 namespace {
 
-using ::testing::Eq;
 using ::testing::MatchesRegex;
 
 TEST(ParseAndExecuteTest, Recursion) {
+  llvm::vfs::InMemoryFileSystem fs;
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
+      llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon");
+  ASSERT_FALSE(prelude.getError()) << prelude.getError().message();
+  ASSERT_TRUE(fs.addFile("prelude.carbon", /*ModificationTime=*/0,
+                         std::move(*prelude)));
+
   std::string source = R"(
     package Test api;
     fn Main() -> i32 {
@@ -32,9 +38,12 @@ TEST(ParseAndExecuteTest, Recursion) {
         ;
     }
   )";
+  ASSERT_TRUE(fs.addFile("test.carbon", /*ModificationTime=*/0,
+                         llvm::MemoryBuffer::getMemBuffer(source)));
+
   TraceStream trace_stream;
   auto err =
-      ParseAndExecute("explorer/data/prelude.carbon", "test.carbon", source,
+      ParseAndExecute(fs, "prelude.carbon", "test.carbon",
                       /*parser_debug=*/false, &trace_stream, &llvm::nulls());
   ASSERT_FALSE(err.ok());
   // Don't expect any particular source location for the error.

explorer/syntax/BUILD

@@ -44,6 +44,7 @@ cc_library(
         "//explorer/ast",
         "//explorer/common:arena",
         "//explorer/common:nonnull",
+        "@llvm-project//llvm:Support",
     ],
 )
 

explorer/syntax/parse.cpp

@@ -10,7 +10,6 @@
 #include "explorer/syntax/lexer.h"
 #include "explorer/syntax/parse_and_lex_context.h"
 #include "explorer/syntax/parser.h"
-#include "llvm/Support/Error.h"
 
 namespace Carbon {
 
@@ -42,30 +41,35 @@ static auto ParseImpl(yyscan_t scanner, Nonnull<Arena*> arena,
   return *ast;
 }
 
-auto Parse(Nonnull<Arena*> arena, std::string_view input_file_name,
-           FileKind file_kind, bool parser_debug) -> ErrorOr<AST> {
-  std::string name_str(input_file_name);
-  FILE* input_file = fopen(name_str.c_str(), "r");
-  if (input_file == nullptr) {
-    return ProgramError(SourceLocation(name_str.c_str(), 0, file_kind))
-           << "Error opening file: " << std::strerror(errno);
+auto Parse(llvm::vfs::FileSystem& fs, Nonnull<Arena*> arena,
+           std::string_view input_file_name, FileKind file_kind,
+           bool parser_debug) -> ErrorOr<AST> {
+  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> input_file =
+      fs.openFileForRead(input_file_name);
+  if (input_file.getError()) {
+    return ProgramError(SourceLocation(input_file_name, 0, file_kind))
+           << "Error opening file: " << input_file.getError().message();
   }
 
-  // Prepare the lexer.
-  yyscan_t scanner;
-  yylex_init(&scanner);
-  auto* buffer = yy_create_buffer(input_file, YY_BUF_SIZE, scanner);
-  yy_switch_to_buffer(buffer, scanner);
-
-  ErrorOr<AST> result =
-      ParseImpl(scanner, arena, input_file_name, file_kind, parser_debug);
+  llvm::ErrorOr<llvm::vfs::Status> status = (*input_file)->status();
+  if (status.getError()) {
+    return Error(status.getError().message());
+  }
+  auto size = status->getSize();
+  if (size >= std::numeric_limits<int32_t>::max()) {
+    return ProgramError(SourceLocation(input_file_name, 0, file_kind))
+           << "File is over the 2GiB input limit.";
+  }
 
-  // Clean up the lexer.
-  yy_delete_buffer(buffer, scanner);
-  yylex_destroy(scanner);
-  fclose(input_file);
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+      (*input_file)
+          ->getBuffer(input_file_name, size, /*RequiresNullTerminator=*/false);
+  if (buffer.getError()) {
+    return Error(buffer.getError().message());
+  }
 
-  return result;
+  return ParseFromString(arena, input_file_name, file_kind,
+                         (*buffer)->getBuffer(), parser_debug);
 }
 
 auto ParseFromString(Nonnull<Arena*> arena, std::string_view input_file_name,

explorer/syntax/parse.h

@@ -11,13 +11,15 @@
 #include "explorer/ast/ast.h"
 #include "explorer/common/arena.h"
 #include "explorer/common/source_location.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 namespace Carbon {
 
 // Returns the AST representing the contents of the named file, or an error code
 // if parsing fails. Allocations go into the provided arena.
-auto Parse(Nonnull<Arena*> arena, std::string_view input_file_name,
-           FileKind file_kind, bool parser_debug) -> ErrorOr<Carbon::AST>;
+auto Parse(llvm::vfs::FileSystem& fs, Nonnull<Arena*> arena,
+           std::string_view input_file_name, FileKind file_kind,
+           bool parser_debug) -> ErrorOr<Carbon::AST>;
 
 // Equivalent to `Parse`, but parses the contents of `file_contents`.
 // `input_file_name` is used only for reporting source locations, and does

explorer/syntax/prelude.cpp

@@ -9,15 +9,16 @@
 namespace Carbon {
 
 // Adds the Carbon prelude to `declarations`.
-void AddPrelude(std::string_view prelude_file_name, Nonnull<Arena*> arena,
+void AddPrelude(llvm::vfs::FileSystem& fs, std::string_view prelude_file_name,
+                Nonnull<Arena*> arena,
                 std::vector<Nonnull<Declaration*>>* declarations,
                 int* num_prelude_declarations) {
   ErrorOr<AST> parse_result =
-      Parse(arena, prelude_file_name, FileKind::Prelude, false);
+      Parse(fs, arena, prelude_file_name, FileKind::Prelude, false);
   if (!parse_result.ok()) {
     // Try again with tracing, to help diagnose the problem.
     ErrorOr<AST> trace_parse_result =
-        Parse(arena, prelude_file_name, FileKind::Prelude, true);
+        Parse(fs, arena, prelude_file_name, FileKind::Prelude, true);
     CARBON_FATAL() << "Failed to parse prelude:\n"
                    << trace_parse_result.error();
   }

explorer/syntax/prelude.h

@@ -10,11 +10,13 @@
 #include "explorer/ast/declaration.h"
 #include "explorer/common/arena.h"
 #include "explorer/common/nonnull.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 namespace Carbon {
 
 // Adds the Carbon prelude to `declarations`.
-void AddPrelude(std::string_view prelude_file_name, Nonnull<Arena*> arena,
+void AddPrelude(llvm::vfs::FileSystem& fs, std::string_view prelude_file_name,
+                Nonnull<Arena*> arena,
                 std::vector<Nonnull<Declaration*>>* declarations,
                 int* num_prelude_declarations);
 

explorer/trace_testdata/full_trace.carbon

@@ -18,6 +18,7 @@ fn Main() -> i32 {
 }
 
 // Place checks after code so that line numbers are stable, reducing merge conflicts.
+// ARGS: --trace_file=- --trace_phase=all %s
 // AUTOUPDATE
 // CHECK:STDOUT: ********** source program **********
 // CHECK:STDOUT: interface TestInterface {

toolchain/driver/driver_file_test_base.h

@@ -8,7 +8,6 @@
 #include <cstdio>
 #include <fstream>
 
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"