Browse Source

Fix integer literal token printing. (#2050)

Summary:
An `llvm::APInt` is always treated as a signed value by `operator<<`;
check [1]. This resulted in printing incorrect values for tokens that
have their MSB set to 1. For example, a value 9 would be printed as -7
since its `APInt` object would be 4-bits wide. However, integer literals
are always tokenized without the sign character so it is safe to treat
the values as unsigned for printing pruposes.

[1] https://llvm.org/doxygen/APInt_8h_source.html

Co-authored-by: ergawy <kareem.ergawy@guardsquare.com>
Kareem Ergawy 3 năm trước cách đây
mục cha
commit
3d44169199

+ 3 - 1
toolchain/lexer/tokenized_buffer.cpp

@@ -828,7 +828,9 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
       output_stream << ", identifier: " << GetIdentifier(token).index_;
       output_stream << ", identifier: " << GetIdentifier(token).index_;
       break;
       break;
     case TokenKind::IntegerLiteral():
     case TokenKind::IntegerLiteral():
-      output_stream << ", value: `" << GetIntegerLiteral(token) << "`";
+      output_stream << ", value: `";
+      GetIntegerLiteral(token).print(output_stream, /*isSigned=*/false);
+      output_stream << "`";
       break;
       break;
     case TokenKind::RealLiteral():
     case TokenKind::RealLiteral():
       output_stream << ", value: `" << GetRealLiteral(token) << "`";
       output_stream << ", value: `" << GetRealLiteral(token) << "`";

+ 13 - 0
toolchain/lexer/tokenized_buffer_test.cpp

@@ -1175,5 +1175,18 @@ TEST_F(LexerTest, PrintingAsYaml) {
                                                {"spelling", ""}}}}));
                                                {"spelling", ""}}}}));
 }
 }
 
 
+TEST_F(LexerTest, PrintToken) {
+  auto buffer = Lex("0x9");
+  ASSERT_FALSE(buffer.has_errors());
+  std::string print_output;
+  llvm::raw_string_ostream print_stream(print_output);
+  buffer.Print(print_stream);
+  llvm::StringRef print = print_stream.str();
+  EXPECT_THAT(GetAndDropLine(print),
+              StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, "
+                    "column: 1, indent: 1, spelling: '0x9', value: `9`, "
+                    "has_trailing_space: true }"));
+}
+
 }  // namespace
 }  // namespace
 }  // namespace Carbon::Testing
 }  // namespace Carbon::Testing