Add a rough outline of a command line driver. (#548)

This is actually code that I wrote a long time ago but didn't get added
to the repository, trying to tidy that up. This is the last code.
Nothing much interesting here, just a skeleton of a CLI. But seemed
better to add it than to not.

driver/BUILD

@@ -0,0 +1,63 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
+load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test")
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "driver",
+    srcs = ["driver.cpp"],
+    hdrs = ["driver.h"],
+    textual_hdrs = ["flags.def"],
+    deps = [
+        "//diagnostics:diagnostic_emitter",
+        "//lexer:tokenized_buffer",
+        "//source:source_buffer",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_test(
+    name = "driver_test",
+    srcs = ["driver_test.cpp"],
+    deps = [
+        ":driver",
+        "//lexer:tokenized_buffer_test_helpers",
+        "@llvm-project//llvm:Support",
+        "@llvm-project//llvm:gmock",
+        "@llvm-project//llvm:gtest",
+        "@llvm-project//llvm:gtest_main",
+    ],
+)
+
+cc_fuzz_test(
+    name = "driver_fuzzer",
+    srcs = ["driver_fuzzer.cpp"],
+    corpus = glob(["fuzzer_corpus/*"]),
+    deps = [
+        ":driver",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_binary(
+    name = "carbon",
+    srcs = ["driver_main.cpp"],
+    deps = [
+        ":driver",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+# FIXME: No support for LLVM's lit-style command line & FileCheck tests.
+#
+#lit_test(
+#    name = "carbon_test.carbon",
+#    data = [
+#        ":carbon",
+#        "@llvm-project//llvm:FileCheck",
+#    ],
+#)

driver/carbon_test.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// FIXME: We need to figure out how to integrate this style of integration
+// testing with Bazel.
+//
+// RUN: driver/carbon dump-tokens %s | FileCheck %s --check-prefix=TOKENS
+
+fn run(String program) {
+  return True;
+}
+
+// TOKENS: token: { index:  0, kind:       'FnKeyword', line: 4, column:  1, indent: 1, spelling: 'fn' }
+// TOKENS: token: { index:  1, kind:      'Identifier', line: 4, column:  4, indent: 1, spelling: 'run', identifier: 0 }
+// TOKENS: token: { index:  2, kind:       'OpenParen', line: 4, column:  7, indent: 1, spelling: '(', closing_token: 5 }
+// TOKENS: token: { index:  3, kind:      'Identifier', line: 4, column:  8, indent: 1, spelling: 'String', identifier: 1 }
+// TOKENS: token: { index:  4, kind:      'Identifier', line: 4, column: 15, indent: 1, spelling: 'program', identifier: 2 }
+// TOKENS: token: { index:  5, kind:      'CloseParen', line: 4, column: 22, indent: 1, spelling: ')', opening_token: 2 }
+// TOKENS: token: { index:  6, kind:  'OpenCurlyBrace', line: 4, column: 24, indent: 1, spelling: '{', closing_token: 10 }
+// TOKENS: token: { index:  7, kind:   'ReturnKeyword', line: 5, column:  3, indent: 3, spelling: 'return' }
+// TOKENS: token: { index:  8, kind:      'Identifier', line: 5, column: 10, indent: 3, spelling: 'True', identifier: 3 }
+// TOKENS: token: { index:  9, kind:            'Semi', line: 5, column: 14, indent: 3, spelling: ';' }
+// TOKENS: token: { index: 10, kind: 'CloseCurlyBrace', line: 6, column:  1, indent: 1, spelling: '}', opening_token: 6 }

driver/driver.cpp

@@ -0,0 +1,141 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "driver/driver.h"
+
+#include "diagnostics/diagnostic_emitter.h"
+#include "lexer/tokenized_buffer.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
+#include "source/source_buffer.h"
+
+namespace Carbon {
+
+namespace {
+
+enum class Subcommand {
+#define CARBON_SUBCOMMAND(Name, ...) Name,
+#include "driver/flags.def"
+  Unknown,
+};
+
+auto GetSubcommand(llvm::StringRef name) -> Subcommand {
+  return llvm::StringSwitch<Subcommand>(name)
+#define CARBON_SUBCOMMAND(Name, Spelling, ...) .Case(Spelling, Subcommand::Name)
+#include "driver/flags.def"
+      .Default(Subcommand::Unknown);
+}
+
+}  // namespace
+
+auto Driver::RunFullCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
+  if (args.empty()) {
+    error_stream << "ERROR: No subcommand specified.\n";
+    return false;
+  }
+
+  llvm::StringRef subcommand_text = args[0];
+  llvm::SmallVector<llvm::StringRef, 16> subcommand_args(
+      std::next(args.begin()), args.end());
+  switch (GetSubcommand(subcommand_text)) {
+    case Subcommand::Unknown:
+      error_stream << "ERROR: Unknown subcommand '" << subcommand_text
+                   << "'.\n";
+      return false;
+
+#define CARBON_SUBCOMMAND(Name, ...) \
+  case Subcommand::Name:             \
+    return Run##Name##Subcommand(subcommand_args);
+#include "driver/flags.def"
+  }
+  llvm_unreachable("All subcommands handled!");
+}
+
+auto Driver::RunHelpSubcommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
+  // FIXME: We should support getting detailed help on a subcommand by looking
+  // for it as a positional parameter here.
+  if (!args.empty()) {
+    ReportExtraArgs("help", args);
+    return false;
+  }
+
+  output_stream << "List of subcommands:\n\n";
+
+  constexpr llvm::StringLiteral SubcommandsAndHelp[][2] = {
+#define CARBON_SUBCOMMAND(Name, Spelling, HelpText) {Spelling, HelpText},
+#include "driver/flags.def"
+  };
+
+  int max_subcommand_width = 0;
+  for (auto subcommand_and_help : SubcommandsAndHelp) {
+    max_subcommand_width = std::max(
+        max_subcommand_width, static_cast<int>(subcommand_and_help[0].size()));
+  }
+
+  for (auto subcommand_and_help : SubcommandsAndHelp) {
+    llvm::StringRef subcommand_text = subcommand_and_help[0];
+    // FIXME: We should wrap this to the number of columns left after the
+    // subcommand on the terminal, and using a hanging indent.
+    llvm::StringRef help_text = subcommand_and_help[1];
+    output_stream << "  "
+                  << llvm::left_justify(subcommand_text, max_subcommand_width)
+                  << " - " << help_text << "\n";
+  }
+
+  output_stream << "\n";
+  return true;
+}
+
+auto Driver::RunDumpTokensSubcommand(llvm::ArrayRef<llvm::StringRef> args)
+    -> bool {
+  if (args.empty()) {
+    error_stream << "ERROR: No input file specified.\n";
+    return false;
+  }
+
+  llvm::StringRef input_file_name = args.front();
+  args = args.drop_front();
+  if (!args.empty()) {
+    ReportExtraArgs("dump-tokens", args);
+    return false;
+  }
+
+  auto source = SourceBuffer::CreateFromFile(input_file_name);
+  if (!source) {
+    error_stream << "ERROR: Unable to open input source file: ";
+    llvm::handleAllErrors(source.takeError(),
+                          [&](const llvm::ErrorInfoBase& ei) {
+                            ei.log(error_stream);
+                            error_stream << "\n";
+                          });
+    return false;
+  }
+  auto tokenized_source =
+      TokenizedBuffer::Lex(*source, ConsoleDiagnosticConsumer());
+  if (tokenized_source.HasErrors()) {
+    error_stream << "ERROR: Unable to tokenize source file '" << input_file_name
+                 << "'!\n";
+    return false;
+  }
+  tokenized_source.Print(output_stream);
+  return true;
+}
+
+auto Driver::ReportExtraArgs(llvm::StringRef subcommand_text,
+                             llvm::ArrayRef<llvm::StringRef> args) -> void {
+  error_stream << "ERROR: Unexpected additional arguments to the '"
+               << subcommand_text << "' subcommand:";
+  for (auto arg : args) {
+    error_stream << " " << arg;
+  }
+
+  error_stream << "\n";
+}
+
+}  // namespace Carbon

driver/driver.h

@@ -0,0 +1,70 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef DRIVER_DRIVER_H_
+#define DRIVER_DRIVER_H_
+
+#include <cstdint>
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Carbon {
+
+// Command line interface driver.
+//
+// Provides simple API to parse and run command lines for Carbon.  It is
+// generally expected to be used to implement command line tools for working
+// with the language.
+class Driver {
+ public:
+  // Default constructed driver uses stderr for all error and informational
+  // output.
+  Driver() : output_stream(llvm::outs()), error_stream(llvm::errs()) {}
+
+  // Constructs a driver with any error or informational output directed to a
+  // specified stream.
+  Driver(llvm::raw_ostream& output_stream, llvm::raw_ostream& error_stream)
+      : output_stream(output_stream), error_stream(error_stream) {}
+
+  // Parses the given arguments into both a subcommand to select the operation
+  // to perform and any arguments to that subcommand.
+  //
+  // Returns true if the operation succeeds. If the operation fails, returns
+  // false and any information about the failure is printed to the registered
+  // error stream (stderr by default).
+  auto RunFullCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+
+  // Subcommand that prints available help text to the error stream.
+  //
+  // Optionally one positional parameter may be provided to select a particular
+  // subcommand or detailed section of help to print.
+  //
+  // Returns true if appropriate help text was found and printed. If an invalid
+  // positional parameter (or flag) is provided, returns false.
+  auto RunHelpSubcommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+
+  // Subcommand that dumps the token information for the provided source file.
+  //
+  // Requires exactly one positional parameter to designate the source file to
+  // read. May be `-` to read from stdin.
+  //
+  // Returns true if the operation succeeds. If the operation fails, this
+  // returns false and any information about the failure is printed to the
+  // registered error stream (stderr by default).
+  auto RunDumpTokensSubcommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+
+ private:
+  auto ReportExtraArgs(llvm::StringRef subcommand_text,
+                       llvm::ArrayRef<llvm::StringRef> args) -> void;
+
+  llvm::raw_ostream& output_stream;
+  llvm::raw_ostream& error_stream;
+};
+
+}  // namespace Carbon
+
+#endif  // DRIVER_DRIVER_H_

driver/driver_fuzzer.cpp

@@ -0,0 +1,82 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cstdint>
+#include <cstring>
+#include <numeric>
+#include <string>
+
+#include "driver/driver.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Carbon {
+
+static auto Read(const unsigned char*& data, size_t& size, int& output)
+    -> bool {
+  if (size < sizeof(output)) {
+    return false;
+  }
+  std::memcpy(&output, data, sizeof(output));
+  size -= sizeof(output);
+  data += sizeof(output);
+  return true;
+}
+
+extern "C" auto LLVMFuzzerTestOneInput(const unsigned char* data, size_t size)
+    -> int {
+  // First use the data to compute the number of arguments. Note that for
+  // scaling reasons we don't allow 2^31 arguments, even empty ones. Simply
+  // creating the vector of those won't work. We limit this to 2^20 arguments
+  // total.
+  int num_args;
+  if (!Read(data, size, num_args) || num_args < 0 || num_args > (1 << 20)) {
+    return 0;
+  }
+
+  // Now use the data to compute the length of each argument. We don't want to
+  // exhaust all memory, so bound the search space to using 2^17 bytes of
+  // memory for the argument text itself.
+  size_t arg_length_sum = 0;
+  llvm::SmallVector<int, 16> arg_lengths(num_args);
+  for (int& arg_length : arg_lengths) {
+    if (!Read(data, size, arg_length) || arg_length < 0) {
+      return 0;
+    }
+    arg_length_sum += arg_length;
+    if (arg_length_sum > (1 << 17)) {
+      return 0;
+    }
+  }
+
+  // Ensure we have enough data for all the arguments.
+  if (size < arg_length_sum) {
+    return 0;
+  }
+
+  // Lastly, read the contents of each argument out of the data.
+  llvm::SmallVector<llvm::StringRef, 16> args;
+  args.reserve(num_args);
+  for (int arg_length : arg_lengths) {
+    args.push_back(
+        llvm::StringRef(reinterpret_cast<const char*>(data), arg_length));
+    data += arg_length;
+    size -= arg_length;
+  }
+
+  std::string error_text;
+  llvm::raw_string_ostream error_stream(error_text);
+  llvm::raw_null_ostream output_stream;
+  Driver d(output_stream, error_stream);
+  if (!d.RunFullCommand(args)) {
+    error_stream.flush();
+    if (error_text.find("ERROR:") == std::string::npos) {
+      llvm::errs() << "No error message on a failure!\n";
+      return 1;
+    }
+  }
+  return 0;
+}
+}  // namespace Carbon

driver/driver_main.cpp

@@ -0,0 +1,21 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cstdlib>
+
+#include "driver/driver.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+auto main(int argc, char** argv) -> int {
+  if (argc < 1) {
+    return EXIT_FAILURE;
+  }
+
+  llvm::SmallVector<llvm::StringRef, 16> args(argv + 1, argv + argc);
+  Carbon::Driver driver;
+  bool success = driver.RunFullCommand(args);
+  return success ? EXIT_SUCCESS : EXIT_FAILURE;
+}

driver/driver_test.cpp

@@ -0,0 +1,245 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "driver/driver.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "lexer/tokenized_buffer_test_helpers.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/YAMLParser.h"
+
+namespace Carbon {
+namespace {
+
+using Carbon::Testing::IsKeyValueScalars;
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::NotNull;
+using ::testing::StrEq;
+
+/// A raw_ostream that makes it easy to repeatedly check streamed output.
+class RawTestOstream : public llvm::raw_ostream {
+  std::string buffer;
+
+  void write_impl(const char* ptr, size_t size) override {
+    buffer.append(ptr, ptr + size);
+  }
+
+  [[nodiscard]] auto current_pos() const -> uint64_t override {
+    return buffer.size();
+  }
+
+ public:
+  ~RawTestOstream() override {
+    flush();
+    if (!buffer.empty()) {
+      ADD_FAILURE() << "Unchecked output:\n" << buffer;
+    }
+  }
+
+  /// Flushes the stream and returns the contents so far, clearing the stream
+  /// back to empty.
+  auto TakeStr() -> std::string {
+    flush();
+    std::string result = std::move(buffer);
+    buffer.clear();
+    return result;
+  }
+};
+
+TEST(DriverTest, FullCommandErrors) {
+  RawTestOstream test_output_stream;
+  RawTestOstream test_error_stream;
+  Driver driver = Driver(test_output_stream, test_error_stream);
+
+  EXPECT_FALSE(driver.RunFullCommand({}));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+
+  EXPECT_FALSE(driver.RunFullCommand({"foo"}));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+
+  EXPECT_FALSE(driver.RunFullCommand({"foo --bar --baz"}));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+}
+
+TEST(DriverTest, Help) {
+  RawTestOstream test_output_stream;
+  RawTestOstream test_error_stream;
+  Driver driver = Driver(test_output_stream, test_error_stream);
+
+  EXPECT_TRUE(driver.RunHelpSubcommand({}));
+  EXPECT_THAT(test_error_stream.TakeStr(), StrEq(""));
+  auto help_text = test_output_stream.TakeStr();
+
+  // Help text should mention each subcommand.
+#define CARBON_SUBCOMMAND(Name, Spelling, ...) \
+  EXPECT_THAT(help_text, HasSubstr(Spelling));
+#include "driver/flags.def"
+
+  // Check that the subcommand dispatch works.
+  EXPECT_TRUE(driver.RunFullCommand({"help"}));
+  EXPECT_THAT(test_error_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(help_text));
+}
+
+TEST(DriverTest, HelpErrors) {
+  RawTestOstream test_output_stream;
+  RawTestOstream test_error_stream;
+  Driver driver = Driver(test_output_stream, test_error_stream);
+
+  EXPECT_FALSE(driver.RunHelpSubcommand({"foo"}));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+
+  EXPECT_FALSE(driver.RunHelpSubcommand({"help"}));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+
+  EXPECT_FALSE(driver.RunHelpSubcommand({"--xyz"}));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+}
+
+auto CreateTestFile(llvm::StringRef text) -> std::string {
+  int fd = -1;
+  llvm::SmallString<1024> path;
+  auto ec = llvm::sys::fs::createTemporaryFile("test_file", ".txt", fd, path);
+  if (ec) {
+    llvm::report_fatal_error(llvm::Twine("Failed to create temporary file: ") +
+                             ec.message());
+  }
+
+  llvm::raw_fd_ostream s(fd, /*shouldClose=*/true);
+  s << text;
+  s.close();
+
+  return path.str().str();
+}
+
+TEST(DriverTest, DumpTokens) {
+  RawTestOstream test_output_stream;
+  RawTestOstream test_error_stream;
+  Driver driver = Driver(test_output_stream, test_error_stream);
+
+  auto test_file_path = CreateTestFile("Hello World");
+  EXPECT_TRUE(driver.RunDumpTokensSubcommand({test_file_path}));
+  EXPECT_THAT(test_error_stream.TakeStr(), StrEq(""));
+  auto tokenized_text = test_output_stream.TakeStr();
+
+  // Parse the output into a YAML stream. This will print errors to stderr and
+  // is the most stable view of the textual dumping API.
+  llvm::SourceMgr sm;
+  llvm::yaml::Stream yaml_stream(tokenized_text, sm);
+  auto yaml_it = yaml_stream.begin();
+  auto* root_node = llvm::dyn_cast<llvm::yaml::MappingNode>(yaml_it->getRoot());
+  ASSERT_THAT(root_node, NotNull());
+
+  // Walk the top-level mapping of tokens, dig out the sub-mapping of data for
+  // each taken, and then verify those entries.
+  auto mapping_it = llvm::cast<llvm::yaml::MappingNode>(root_node)->begin();
+  auto* token_node = llvm::dyn_cast<llvm::yaml::KeyValueNode>(&*mapping_it);
+  ASSERT_THAT(token_node, NotNull());
+  auto* token_key_node =
+      llvm::dyn_cast<llvm::yaml::ScalarNode>(token_node->getKey());
+  ASSERT_THAT(token_key_node, NotNull());
+  EXPECT_THAT(token_key_node->getRawValue(), StrEq("token"));
+  auto* token_value_node =
+      llvm::dyn_cast<llvm::yaml::MappingNode>(token_node->getValue());
+  ASSERT_THAT(token_value_node, NotNull());
+  auto token_it = token_value_node->begin();
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("index", "0"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("kind", "Identifier"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("line", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("column", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("indent", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("spelling", "Hello"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("identifier", "0"));
+  EXPECT_THAT(++token_it, Eq(token_value_node->end()));
+
+  ++mapping_it;
+  token_node = llvm::dyn_cast<llvm::yaml::KeyValueNode>(&*mapping_it);
+  ASSERT_THAT(token_node, NotNull());
+  token_key_node = llvm::dyn_cast<llvm::yaml::ScalarNode>(token_node->getKey());
+  ASSERT_THAT(token_key_node, NotNull());
+  EXPECT_THAT(token_key_node->getRawValue(), StrEq("token"));
+  token_value_node =
+      llvm::dyn_cast<llvm::yaml::MappingNode>(token_node->getValue());
+  ASSERT_THAT(token_value_node, NotNull());
+  token_it = token_value_node->begin();
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("index", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("kind", "Identifier"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("line", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("column", "7"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("indent", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("spelling", "World"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("identifier", "1"));
+  EXPECT_THAT(++token_it, Eq(token_value_node->end()));
+
+  ++mapping_it;
+  token_node = llvm::dyn_cast<llvm::yaml::KeyValueNode>(&*mapping_it);
+  ASSERT_THAT(token_node, NotNull());
+  token_key_node = llvm::dyn_cast<llvm::yaml::ScalarNode>(token_node->getKey());
+  ASSERT_THAT(token_key_node, NotNull());
+  EXPECT_THAT(token_key_node->getRawValue(), StrEq("token"));
+  token_value_node =
+      llvm::dyn_cast<llvm::yaml::MappingNode>(token_node->getValue());
+  ASSERT_THAT(token_value_node, NotNull());
+  token_it = token_value_node->begin();
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("index", "2"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("kind", "EndOfFile"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("line", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("column", "12"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("indent", "1"));
+  ++token_it;
+  EXPECT_THAT(&*token_it, IsKeyValueScalars("spelling", ""));
+  EXPECT_THAT(++token_it, Eq(token_value_node->end()));
+
+  ASSERT_THAT(++mapping_it, Eq(root_node->end()));
+  ASSERT_THAT(++yaml_it, Eq(yaml_stream.end()));
+
+  // Check that the subcommand dispatch works.
+  EXPECT_TRUE(driver.RunFullCommand({"dump-tokens", test_file_path}));
+  EXPECT_THAT(test_error_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(tokenized_text));
+}
+
+TEST(DriverTest, DumpTokenErrors) {
+  RawTestOstream test_output_stream;
+  RawTestOstream test_error_stream;
+  Driver driver = Driver(test_output_stream, test_error_stream);
+
+  EXPECT_FALSE(driver.RunDumpTokensSubcommand({}));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+
+  EXPECT_FALSE(driver.RunDumpTokensSubcommand({"--xyz"}));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+
+  EXPECT_FALSE(driver.RunDumpTokensSubcommand({"/not/a/real/file/name"}));
+  EXPECT_THAT(test_output_stream.TakeStr(), StrEq(""));
+  EXPECT_THAT(test_error_stream.TakeStr(), HasSubstr("ERROR"));
+}
+
+}  // namespace
+}  // namespace Carbon

driver/flags.def

@@ -0,0 +1,19 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Note: X-macro header. Do not use include guards.
+
+#ifndef CARBON_SUBCOMMAND
+// FIXME: We should expand `HelpText` to be a short help name, a synopsis, and a
+// long-form description.
+#define CARBON_SUBCOMMAND(Name, Spelling, HelpText)
+#endif
+
+CARBON_SUBCOMMAND(Help, "help",
+                  "Display help information about the driver options.")
+CARBON_SUBCOMMAND(
+    DumpTokens, "dump-tokens",
+    "Dumps the sequence of tokens lexed out of the input source file.")
+
+#undef CARBON_SUBCOMMAND

driver/fuzzer_corpus/3dbc1b67cab9a71bac41d7ee17fa31a8b32a9904

@@ -0,0 +1,3 @@
+
+
+'�

driver/fuzzer_corpus/8b0653417087d3d7fd049166a72cd7bcbc5f6dd1

@@ -0,0 +1,3 @@
+ｽ
+
+*

driver/fuzzer_corpus/adc83b19e793491b1c6ea0fd8b46cd9f32e592fc

@@ -0,0 +1 @@
+