Add int and real printing (#1116)

toolchain/lexer/tokenized_buffer.cpp

@@ -820,18 +820,29 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
                            widths.indent),
       token_text);
 
-  if (token_info.kind == TokenKind::Identifier()) {
-    output_stream << ", identifier: " << GetIdentifier(token).index_;
-  } else if (token_info.kind.IsOpeningSymbol()) {
-    output_stream << ", closing_token: "
-                  << GetMatchedClosingToken(token).index_;
-  } else if (token_info.kind.IsClosingSymbol()) {
-    output_stream << ", opening_token: "
-                  << GetMatchedOpeningToken(token).index_;
-  } else if (token_info.kind == TokenKind::StringLiteral()) {
-    output_stream << ", value: `" << GetStringLiteral(token) << "`";
+  switch (token_info.kind) {
+    case TokenKind::Identifier():
+      output_stream << ", identifier: " << GetIdentifier(token).index_;
+      break;
+    case TokenKind::IntegerLiteral():
+      output_stream << ", value: `" << GetIntegerLiteral(token) << "`";
+      break;
+    case TokenKind::RealLiteral():
+      output_stream << ", value: `" << GetRealLiteral(token) << "`";
+      break;
+    case TokenKind::StringLiteral():
+      output_stream << ", value: `" << GetStringLiteral(token) << "`";
+      break;
+    default:
+      if (token_info.kind.IsOpeningSymbol()) {
+        output_stream << ", closing_token: "
+                      << GetMatchedClosingToken(token).index_;
+      } else if (token_info.kind.IsClosingSymbol()) {
+        output_stream << ", opening_token: "
+                      << GetMatchedOpeningToken(token).index_;
+      }
+      break;
   }
-  // TODO: Include value for numeric literals.
 
   if (token_info.has_trailing_space) {
     output_stream << ", has_trailing_space: true";

toolchain/lexer/tokenized_buffer.h

@@ -227,6 +227,11 @@ class TokenizedBuffer {
     // If true, the value is mantissa * 10^exponent.
     [[nodiscard]] auto IsDecimal() const -> bool { return is_decimal_; }
 
+    void Print(llvm::raw_ostream& output_stream) const {
+      output_stream << Mantissa() << "*" << (is_decimal_ ? "10" : "2") << "^"
+                    << Exponent();
+    }
+
    private:
     friend class TokenizedBuffer;
 

toolchain/lexer/tokenized_buffer_test.cpp

@@ -989,7 +989,7 @@ auto GetAndDropLine(llvm::StringRef& text) -> std::string {
   return line.str();
 }
 
-TEST_F(LexerTest, Printing) {
+TEST_F(LexerTest, PrintingBasic) {
   auto buffer = Lex(";");
   ASSERT_FALSE(buffer.HasErrors());
   std::string print_storage;
@@ -1003,13 +1003,47 @@ TEST_F(LexerTest, Printing) {
               StrEq("token: { index: 1, kind: 'EndOfFile', line: 1, column: 2, "
                     "indent: 1, spelling: '' }"));
   EXPECT_TRUE(print.empty()) << print;
+}
+
+TEST_F(LexerTest, PrintingInteger) {
+  auto buffer = Lex("123");
+  ASSERT_FALSE(buffer.HasErrors());
+  std::string print_storage;
+  llvm::raw_string_ostream print_stream(print_storage);
+  buffer.Print(print_stream);
+  llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(GetAndDropLine(print),
+              StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, "
+                    "column: 1, indent: 1, spelling: '123', value: `123`, "
+                    "has_trailing_space: true }"));
+  EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
+  EXPECT_TRUE(print.empty()) << print;
+}
+
+TEST_F(LexerTest, PrintingReal) {
+  auto buffer = Lex("2.5");
+  ASSERT_FALSE(buffer.HasErrors());
+  std::string print_storage;
+  llvm::raw_string_ostream print_stream(print_storage);
+  buffer.Print(print_stream);
+  llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(
+      GetAndDropLine(print),
+      StrEq(
+          "token: { index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
+          "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }"));
+  EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
+  EXPECT_TRUE(print.empty()) << print;
+}
 
+TEST_F(LexerTest, PrintingPadding) {
   // Test kind padding.
-  buffer = Lex("(;foo;)");
+  auto buffer = Lex("(;foo;)");
   ASSERT_FALSE(buffer.HasErrors());
-  print_storage.clear();
+  std::string print_storage;
+  llvm::raw_string_ostream print_stream(print_storage);
   buffer.Print(print_stream);
-  print = print_stream.str();
+  llvm::StringRef print = print_stream.str();
   EXPECT_THAT(GetAndDropLine(print),
               StrEq("token: { index: 0, kind:  'OpenParen', line: 1, column: "
                     "1, indent: 1, spelling: '(', closing_token: 4 }"));
@@ -1030,13 +1064,16 @@ TEST_F(LexerTest, Printing) {
               StrEq("token: { index: 5, kind:  'EndOfFile', line: 1, column: "
                     "8, indent: 1, spelling: '' }"));
   EXPECT_TRUE(print.empty()) << print;
+}
 
+TEST_F(LexerTest, PrintingPaddingDigits) {
   // Test digit padding with max values of 9, 10, and 11.
-  buffer = Lex(";\n\n\n\n\n\n\n\n\n\n        ;;");
+  auto buffer = Lex(";\n\n\n\n\n\n\n\n\n\n        ;;");
   ASSERT_FALSE(buffer.HasErrors());
-  print_storage.clear();
+  std::string print_storage;
+  llvm::raw_string_ostream print_stream(print_storage);
   buffer.Print(print_stream);
-  print = print_stream.str();
+  llvm::StringRef print = print_stream.str();
   EXPECT_THAT(
       GetAndDropLine(print),
       StrEq("token: { index: 0, kind:      'Semi', line:  1, column:  1, "