Bracket the tokenized buffer output. (#2446)

This makes TokenizedBuffer more consistent with ParseTree and SemanticsIR, which also wrap with [] to produce a sequence value.

It also makes it possible in driver.cpp to just prefix the line with a name, so it ends up with:

var_name: [
  (content)
]
Noticed this due to bracketing comments on #2443 and trying to think of better answers. With this, we can also say that the [] bracket a variable.

toolchain/driver/driver.cpp

@@ -171,7 +171,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
     tokenized_source.Print(output_stream_);
     return !tokenized_source.has_errors();
   }
-  CARBON_VLOG() << tokenized_source;
+  CARBON_VLOG() << "tokenized_buffer: " << tokenized_source;
 
   CARBON_VLOG() << "*** ParseTree::Parse ***\n";
   auto parse_tree = ParseTree::Parse(tokenized_source, consumer, vlog_stream_);
@@ -181,7 +181,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
     parse_tree.Print(output_stream_, parse_tree_preorder);
     return !tokenized_source.has_errors() && !parse_tree.has_errors();
   }
-  CARBON_VLOG() << parse_tree;
+  CARBON_VLOG() << "parse_tree: " << parse_tree;
 
   const SemanticsIR builtin_ir = SemanticsIR::MakeBuiltinIR();
   CARBON_VLOG() << "*** SemanticsIR::MakeFromParseTree ***\n";
@@ -194,7 +194,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
     // TODO: Return false when SemanticsIR has errors (not supported right now).
     return !tokenized_source.has_errors() && !parse_tree.has_errors();
   }
-  CARBON_VLOG() << semantics_ir;
+  CARBON_VLOG() << "semantics_ir: " << semantics_ir;
 
   llvm_unreachable("should handle all dump modes");
 }

toolchain/driver/driver_test.cpp

@@ -133,29 +133,29 @@ TEST(DriverTest, DumpTokens) {
   auto tokenized_text = test_output_stream.TakeStr();
 
   EXPECT_THAT(Yaml::Value::FromText(tokenized_text),
-              ElementsAre(Yaml::MappingValue{
-                  {"token", Yaml::MappingValue{{"index", "0"},
-                                               {"kind", "Identifier"},
-                                               {"line", "1"},
-                                               {"column", "1"},
-                                               {"indent", "1"},
-                                               {"spelling", "Hello"},
-                                               {"identifier", "0"},
-                                               {"has_trailing_space", "true"}}},
-                  {"token", Yaml::MappingValue{{"index", "1"},
-                                               {"kind", "Identifier"},
-                                               {"line", "1"},
-                                               {"column", "7"},
-                                               {"indent", "1"},
-                                               {"spelling", "World"},
-                                               {"identifier", "1"},
-                                               {"has_trailing_space", "true"}}},
-                  {"token", Yaml::MappingValue{{"index", "2"},
-                                               {"kind", "EndOfFile"},
-                                               {"line", "1"},
-                                               {"column", "12"},
-                                               {"indent", "1"},
-                                               {"spelling", ""}}}}));
+              ElementsAre(Yaml::SequenceValue{
+                  Yaml::MappingValue{{"index", "0"},
+                                     {"kind", "Identifier"},
+                                     {"line", "1"},
+                                     {"column", "1"},
+                                     {"indent", "1"},
+                                     {"spelling", "Hello"},
+                                     {"identifier", "0"},
+                                     {"has_trailing_space", "true"}},
+                  Yaml::MappingValue{{"index", "1"},
+                                     {"kind", "Identifier"},
+                                     {"line", "1"},
+                                     {"column", "7"},
+                                     {"indent", "1"},
+                                     {"spelling", "World"},
+                                     {"identifier", "1"},
+                                     {"has_trailing_space", "true"}},
+                  Yaml::MappingValue{{"index", "2"},
+                                     {"kind", "EndOfFile"},
+                                     {"line", "1"},
+                                     {"column", "12"},
+                                     {"indent", "1"},
+                                     {"spelling", ""}}}));
 
   // Check that the subcommand dispatch works.
   EXPECT_TRUE(driver.RunFullCommand({"dump", "tokens", test_file_path}));

toolchain/driver/testdata/errors_sorted_test.carbon

@@ -3,7 +3,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: %{not} %{carbon-run-tokens}
-// CHECK-COUNT-17:STDOUT: token: {{.*}}
+// CHECK:STDOUT: [
+// CHECK-COUNT-17:STDOUT: {{.*}}
+// CHECK:STDOUT: ]
 
 // CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+1]]:24: Closing symbol does not match most recent opening symbol.
 fn run(String program) {

toolchain/driver/testdata/errors_streamed_test.carbon

@@ -4,7 +4,9 @@
 //
 // RUN: %{not} %{carbon} --print-errors=streamed dump tokens %s | \
 // RUN:   %{FileCheck-strict}
-// CHECK-COUNT-17:STDOUT: token: {{.*}}
+// CHECK:STDOUT: [
+// CHECK-COUNT-17:STDOUT: {{.*}}
+// CHECK:STDOUT: ]
 
 fn run(String program) {
   return True;

toolchain/lexer/testdata/carbon_test.carbon

@@ -4,19 +4,21 @@
 //
 // AUTOUPDATE
 // RUN: %{carbon-run-tokens}
+// CHECK:STDOUT: [
 
-// CHECK:STDOUT: token: { index:  0, kind:              'Fn', line: {{ *}}[[@LINE+7]], column:  1, indent: 1, spelling: 'fn', has_trailing_space: true }
-// CHECK:STDOUT: token: { index:  1, kind:      'Identifier', line: {{ *}}[[@LINE+6]], column:  4, indent: 1, spelling: 'run', identifier: 0 }
-// CHECK:STDOUT: token: { index:  2, kind:       'OpenParen', line: {{ *}}[[@LINE+5]], column:  7, indent: 1, spelling: '(', closing_token: 5 }
-// CHECK:STDOUT: token: { index:  3, kind:      'Identifier', line: {{ *}}[[@LINE+4]], column:  8, indent: 1, spelling: 'String', identifier: 1, has_trailing_space: true }
-// CHECK:STDOUT: token: { index:  4, kind:      'Identifier', line: {{ *}}[[@LINE+3]], column: 15, indent: 1, spelling: 'program', identifier: 2 }
-// CHECK:STDOUT: token: { index:  5, kind:      'CloseParen', line: {{ *}}[[@LINE+2]], column: 22, indent: 1, spelling: ')', opening_token: 2, has_trailing_space: true }
-// CHECK:STDOUT: token: { index:  6, kind:  'OpenCurlyBrace', line: {{ *}}[[@LINE+1]], column: 24, indent: 1, spelling: '{', closing_token: 10, has_trailing_space: true }
+// CHECK:STDOUT: { index:  0, kind:              'Fn', line: {{ *}}[[@LINE+7]], column:  1, indent: 1, spelling: 'fn', has_trailing_space: true },
+// CHECK:STDOUT: { index:  1, kind:      'Identifier', line: {{ *}}[[@LINE+6]], column:  4, indent: 1, spelling: 'run', identifier: 0 },
+// CHECK:STDOUT: { index:  2, kind:       'OpenParen', line: {{ *}}[[@LINE+5]], column:  7, indent: 1, spelling: '(', closing_token: 5 },
+// CHECK:STDOUT: { index:  3, kind:      'Identifier', line: {{ *}}[[@LINE+4]], column:  8, indent: 1, spelling: 'String', identifier: 1, has_trailing_space: true },
+// CHECK:STDOUT: { index:  4, kind:      'Identifier', line: {{ *}}[[@LINE+3]], column: 15, indent: 1, spelling: 'program', identifier: 2 },
+// CHECK:STDOUT: { index:  5, kind:      'CloseParen', line: {{ *}}[[@LINE+2]], column: 22, indent: 1, spelling: ')', opening_token: 2, has_trailing_space: true },
+// CHECK:STDOUT: { index:  6, kind:  'OpenCurlyBrace', line: {{ *}}[[@LINE+1]], column: 24, indent: 1, spelling: '{', closing_token: 10, has_trailing_space: true },
 fn run(String program) {
-  // CHECK:STDOUT: token: { index:  7, kind:          'Return', line: {{ *}}[[@LINE+3]], column:  3, indent: 3, spelling: 'return', has_trailing_space: true }
-  // CHECK:STDOUT: token: { index:  8, kind:      'Identifier', line: {{ *}}[[@LINE+2]], column: 10, indent: 3, spelling: 'True', identifier: 3 }
-  // CHECK:STDOUT: token: { index:  9, kind:            'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true }
+  // CHECK:STDOUT: { index:  7, kind:          'Return', line: {{ *}}[[@LINE+3]], column:  3, indent: 3, spelling: 'return', has_trailing_space: true },
+  // CHECK:STDOUT: { index:  8, kind:      'Identifier', line: {{ *}}[[@LINE+2]], column: 10, indent: 3, spelling: 'True', identifier: 3 },
+  // CHECK:STDOUT: { index:  9, kind:            'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true },
   return True;
-// CHECK:STDOUT: token: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+2]], column:  1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true }
-// CHECK:STDOUT: token: { index: 11, kind:       'EndOfFile', line: {{ *}}[[@LINE+1]], column:  2, indent: 1, spelling: '' }
+// CHECK:STDOUT: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+3]], column:  1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true },
+// CHECK:STDOUT: { index: 11, kind:       'EndOfFile', line: {{ *}}[[@LINE+2]], column:  2, indent: 1, spelling: '' },
+// CHECK:STDOUT: ]
 }

toolchain/lexer/tokenized_buffer.cpp

@@ -788,10 +788,12 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
     widths.Widen(GetTokenPrintWidths(token));
   }
 
+  output_stream << "[\n";
   for (Token token : tokens()) {
     PrintToken(output_stream, token, widths);
     output_stream << "\n";
   }
+  output_stream << "]\n";
 }
 
 auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
@@ -810,7 +812,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
   // justification manually in order to use the dynamically computed widths
   // and get the quotes included.
   output_stream << llvm::formatv(
-      "token: { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
+      "{ index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
       "spelling: '{5}'",
       llvm::format_decimal(token_index, widths.index),
       llvm::right_justify(
@@ -855,7 +857,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
     output_stream << ", recovery: true";
   }
 
-  output_stream << " }";
+  output_stream << " },";
 }
 
 auto TokenizedBuffer::GetLineInfo(Line line) -> LineInfo& {

toolchain/lexer/tokenized_buffer_test.cpp

@@ -1032,22 +1032,6 @@ auto GetAndDropLine(llvm::StringRef& text) -> std::string {
   return line.str();
 }
 
-TEST_F(LexerTest, PrintingBasic) {
-  auto buffer = Lex(";");
-  ASSERT_FALSE(buffer.has_errors());
-  std::string print_storage;
-  llvm::raw_string_ostream print_stream(print_storage);
-  buffer.Print(print_stream);
-  llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 0, kind:      'Semi', line: 1, column: 1, "
-                    "indent: 1, spelling: ';', has_trailing_space: true }"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 1, kind: 'EndOfFile', line: 1, column: 2, "
-                    "indent: 1, spelling: '' }"));
-  EXPECT_TRUE(print.empty()) << print;
-}
-
 TEST_F(LexerTest, PrintingInteger) {
   auto buffer = Lex("123");
   ASSERT_FALSE(buffer.has_errors());
@@ -1055,11 +1039,13 @@ TEST_F(LexerTest, PrintingInteger) {
   llvm::raw_string_ostream print_stream(print_storage);
   buffer.Print(print_stream);
   llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, "
+              StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
                     "column: 1, indent: 1, spelling: '123', value: `123`, "
-                    "has_trailing_space: true }"));
+                    "has_trailing_space: true },"));
   EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
+  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
   EXPECT_TRUE(print.empty()) << print;
 }
 
@@ -1070,12 +1056,14 @@ TEST_F(LexerTest, PrintingReal) {
   llvm::raw_string_ostream print_stream(print_storage);
   buffer.Print(print_stream);
   llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
   EXPECT_THAT(
       GetAndDropLine(print),
-      StrEq(
-          "token: { index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
-          "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }"));
+      StrEq("{ index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
+            "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true "
+            "},"));
   EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
+  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
   EXPECT_TRUE(print.empty()) << print;
 }
 
@@ -1087,25 +1075,27 @@ TEST_F(LexerTest, PrintingPadding) {
   llvm::raw_string_ostream print_stream(print_storage);
   buffer.Print(print_stream);
   llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 0, kind:  'OpenParen', line: 1, column: "
-                    "1, indent: 1, spelling: '(', closing_token: 4 }"));
+              StrEq("{ index: 0, kind:  'OpenParen', line: 1, column: "
+                    "1, indent: 1, spelling: '(', closing_token: 4 },"));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 1, kind:       'Semi', line: 1, column: "
-                    "2, indent: 1, spelling: ';' }"));
+              StrEq("{ index: 1, kind:       'Semi', line: 1, column: "
+                    "2, indent: 1, spelling: ';' },"));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 2, kind: 'Identifier', line: 1, column: "
-                    "3, indent: 1, spelling: 'foo', identifier: 0 }"));
+              StrEq("{ index: 2, kind: 'Identifier', line: 1, column: "
+                    "3, indent: 1, spelling: 'foo', identifier: 0 },"));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 3, kind:       'Semi', line: 1, column: "
-                    "6, indent: 1, spelling: ';' }"));
+              StrEq("{ index: 3, kind:       'Semi', line: 1, column: "
+                    "6, indent: 1, spelling: ';' },"));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 4, kind: 'CloseParen', line: 1, column: "
+              StrEq("{ index: 4, kind: 'CloseParen', line: 1, column: "
                     "7, indent: 1, spelling: ')', opening_token: 0, "
-                    "has_trailing_space: true }"));
+                    "has_trailing_space: true },"));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 5, kind:  'EndOfFile', line: 1, column: "
-                    "8, indent: 1, spelling: '' }"));
+              StrEq("{ index: 5, kind:  'EndOfFile', line: 1, column: "
+                    "8, indent: 1, spelling: '' },"));
+  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
   EXPECT_TRUE(print.empty()) << print;
 }
 
@@ -1117,22 +1107,20 @@ TEST_F(LexerTest, PrintingPaddingDigits) {
   llvm::raw_string_ostream print_stream(print_storage);
   buffer.Print(print_stream);
   llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(
-      GetAndDropLine(print),
-      StrEq("token: { index: 0, kind:      'Semi', line:  1, column:  1, "
-            "indent: 1, spelling: ';', has_trailing_space: true }"));
-  EXPECT_THAT(
-      GetAndDropLine(print),
-      StrEq("token: { index: 1, kind:      'Semi', line: 11, column:  9, "
-            "indent: 9, spelling: ';' }"));
-  EXPECT_THAT(
-      GetAndDropLine(print),
-      StrEq("token: { index: 2, kind:      'Semi', line: 11, column: 10, "
-            "indent: 9, spelling: ';', has_trailing_space: true }"));
-  EXPECT_THAT(
-      GetAndDropLine(print),
-      StrEq("token: { index: 3, kind: 'EndOfFile', line: 11, column: 11, "
-            "indent: 9, spelling: '' }"));
+  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
+  EXPECT_THAT(GetAndDropLine(print),
+              StrEq("{ index: 0, kind:      'Semi', line:  1, column:  1, "
+                    "indent: 1, spelling: ';', has_trailing_space: true },"));
+  EXPECT_THAT(GetAndDropLine(print),
+              StrEq("{ index: 1, kind:      'Semi', line: 11, column:  9, "
+                    "indent: 9, spelling: ';' },"));
+  EXPECT_THAT(GetAndDropLine(print),
+              StrEq("{ index: 2, kind:      'Semi', line: 11, column: 10, "
+                    "indent: 9, spelling: ';', has_trailing_space: true },"));
+  EXPECT_THAT(GetAndDropLine(print),
+              StrEq("{ index: 3, kind: 'EndOfFile', line: 11, column: 11, "
+                    "indent: 9, spelling: '' },"));
+  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
   EXPECT_TRUE(print.empty()) << print;
 }
 
@@ -1146,34 +1134,34 @@ TEST_F(LexerTest, PrintingAsYaml) {
   print_stream.flush();
 
   EXPECT_THAT(Yaml::Value::FromText(print_output),
-              ElementsAre(Yaml::MappingValue{
-                  {"token", Yaml::MappingValue{{"index", "0"},
-                                               {"kind", "Semi"},
-                                               {"line", "2"},
-                                               {"column", "2"},
-                                               {"indent", "2"},
-                                               {"spelling", ";"},
-                                               {"has_trailing_space", "true"}}},
-                  {"token", Yaml::MappingValue{{"index", "1"},
-                                               {"kind", "Semi"},
-                                               {"line", "5"},
-                                               {"column", "1"},
-                                               {"indent", "1"},
-                                               {"spelling", ";"},
-                                               {"has_trailing_space", "true"}}},
-                  {"token", Yaml::MappingValue{{"index", "2"},
-                                               {"kind", "Semi"},
-                                               {"line", "5"},
-                                               {"column", "3"},
-                                               {"indent", "1"},
-                                               {"spelling", ";"},
-                                               {"has_trailing_space", "true"}}},
-                  {"token", Yaml::MappingValue{{"index", "3"},
-                                               {"kind", "EndOfFile"},
-                                               {"line", "15"},
-                                               {"column", "1"},
-                                               {"indent", "1"},
-                                               {"spelling", ""}}}}));
+              ElementsAre(Yaml::SequenceValue{
+                  Yaml::MappingValue{{"index", "0"},
+                                     {"kind", "Semi"},
+                                     {"line", "2"},
+                                     {"column", "2"},
+                                     {"indent", "2"},
+                                     {"spelling", ";"},
+                                     {"has_trailing_space", "true"}},
+                  Yaml::MappingValue{{"index", "1"},
+                                     {"kind", "Semi"},
+                                     {"line", "5"},
+                                     {"column", "1"},
+                                     {"indent", "1"},
+                                     {"spelling", ";"},
+                                     {"has_trailing_space", "true"}},
+                  Yaml::MappingValue{{"index", "2"},
+                                     {"kind", "Semi"},
+                                     {"line", "5"},
+                                     {"column", "3"},
+                                     {"indent", "1"},
+                                     {"spelling", ";"},
+                                     {"has_trailing_space", "true"}},
+                  Yaml::MappingValue{{"index", "3"},
+                                     {"kind", "EndOfFile"},
+                                     {"line", "15"},
+                                     {"column", "1"},
+                                     {"indent", "1"},
+                                     {"spelling", ""}}}));
 }
 
 TEST_F(LexerTest, PrintToken) {
@@ -1183,10 +1171,11 @@ TEST_F(LexerTest, PrintToken) {
   llvm::raw_string_ostream print_stream(print_output);
   buffer.Print(print_stream);
   llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
   EXPECT_THAT(GetAndDropLine(print),
-              StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, "
+              StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
                     "column: 1, indent: 1, spelling: '0x9', value: `9`, "
-                    "has_trailing_space: true }"));
+                    "has_trailing_space: true },"));
 }
 
 }  // namespace