Rename {String,Numeric}LiteralToken -> Lexed{String,Numeric}Literal. (#373)

This avoids using a name too similar to Token::StringLiteral (the token
kind) and TokenizedBuffer::Token (which is not related to these
classes).

lexer/numeric_literal.cpp

@@ -81,9 +81,9 @@ struct WrongRealLiteralExponent {
 };
 }  // namespace
 
-auto NumericLiteralToken::Lex(llvm::StringRef source_text)
-    -> llvm::Optional<NumericLiteralToken> {
-  NumericLiteralToken result;
+auto LexedNumericLiteral::Lex(llvm::StringRef source_text)
+    -> llvm::Optional<LexedNumericLiteral> {
+  LexedNumericLiteral result;
 
   if (source_text.empty() || !IsDecimalDigit(source_text.front())) {
     return llvm::None;
@@ -145,8 +145,8 @@ auto NumericLiteralToken::Lex(llvm::StringRef source_text)
   return result;
 }
 
-NumericLiteralToken::Parser::Parser(DiagnosticEmitter& emitter,
-                                    NumericLiteralToken literal)
+LexedNumericLiteral::Parser::Parser(DiagnosticEmitter& emitter,
+                                    LexedNumericLiteral literal)
     : emitter(emitter), literal(literal) {
   int_part = literal.text.substr(0, literal.radix_point);
   if (int_part.consume_front("0x")) {
@@ -166,7 +166,7 @@ NumericLiteralToken::Parser::Parser(DiagnosticEmitter& emitter,
 
 // Check that the numeric literal token is syntactically valid and meaningful,
 // and diagnose if not.
-auto NumericLiteralToken::Parser::Check() -> CheckResult {
+auto LexedNumericLiteral::Parser::Check() -> CheckResult {
   if (!CheckLeadingZero() || !CheckIntPart() || !CheckFractionalPart() ||
       !CheckExponentPart()) {
     return UnrecoverableError;
@@ -201,13 +201,13 @@ static auto ParseInteger(llvm::StringRef digits, int radix, bool needs_cleaning)
   return value;
 }
 
-auto NumericLiteralToken::Parser::GetMantissa() -> llvm::APInt {
+auto LexedNumericLiteral::Parser::GetMantissa() -> llvm::APInt {
   const char* end = IsInteger() ? int_part.end() : fract_part.end();
   llvm::StringRef digits(int_part.begin(), end - int_part.begin());
   return ParseInteger(digits, radix, mantissa_needs_cleaning);
 }
 
-auto NumericLiteralToken::Parser::GetExponent() -> llvm::APInt {
+auto LexedNumericLiteral::Parser::GetExponent() -> llvm::APInt {
   // Compute the effective exponent from the specified exponent, if any,
   // and the position of the radix point.
   llvm::APInt exponent(64, 0);
@@ -246,7 +246,7 @@ auto NumericLiteralToken::Parser::GetExponent() -> llvm::APInt {
 // Check that a digit sequence is valid: that it contains one or more digits,
 // contains only digits in the specified base, and that any digit separators
 // are present and correctly positioned.
-auto NumericLiteralToken::Parser::CheckDigitSequence(
+auto LexedNumericLiteral::Parser::CheckDigitSequence(
     llvm::StringRef text, int radix, bool allow_digit_separators)
     -> CheckDigitSequenceResult {
   assert((radix == 2 || radix == 10 || radix == 16) && "unknown radix");
@@ -305,7 +305,7 @@ auto NumericLiteralToken::Parser::CheckDigitSequence(
 
 // Given a number with digit separators, check that the digit separators are
 // correctly positioned.
-auto NumericLiteralToken::Parser::CheckDigitSeparatorPlacement(
+auto LexedNumericLiteral::Parser::CheckDigitSeparatorPlacement(
     llvm::StringRef text, int radix, int num_digit_separators) -> void {
   assert(std::count(text.begin(), text.end(), '_') == num_digit_separators &&
          "given wrong number of digit separators");
@@ -346,7 +346,7 @@ auto NumericLiteralToken::Parser::CheckDigitSeparatorPlacement(
 };
 
 // Check that we don't have a '0' prefix on a non-zero decimal integer.
-auto NumericLiteralToken::Parser::CheckLeadingZero() -> bool {
+auto LexedNumericLiteral::Parser::CheckLeadingZero() -> bool {
   if (radix == 10 && int_part.startswith("0") && int_part != "0") {
     emitter.EmitError<UnknownBaseSpecifier>();
     return false;
@@ -355,7 +355,7 @@ auto NumericLiteralToken::Parser::CheckLeadingZero() -> bool {
 }
 
 // Check the integer part (before the '.', if any) is valid.
-auto NumericLiteralToken::Parser::CheckIntPart() -> bool {
+auto LexedNumericLiteral::Parser::CheckIntPart() -> bool {
   auto int_result = CheckDigitSequence(int_part, radix);
   mantissa_needs_cleaning |= int_result.has_digit_separators;
   return int_result.ok;
@@ -363,7 +363,7 @@ auto NumericLiteralToken::Parser::CheckIntPart() -> bool {
 
 // Check the fractional part (after the '.' and before the exponent, if any)
 // is valid.
-auto NumericLiteralToken::Parser::CheckFractionalPart() -> bool {
+auto LexedNumericLiteral::Parser::CheckFractionalPart() -> bool {
   if (IsInteger()) {
     return true;
   }
@@ -383,7 +383,7 @@ auto NumericLiteralToken::Parser::CheckFractionalPart() -> bool {
 }
 
 // Check the exponent part (if any) is valid.
-auto NumericLiteralToken::Parser::CheckExponentPart() -> bool {
+auto LexedNumericLiteral::Parser::CheckExponentPart() -> bool {
   if (literal.exponent == static_cast<int>(literal.text.size())) {
     return true;
   }

lexer/numeric_literal.h

@@ -15,7 +15,7 @@
 namespace Carbon {
 
 // A numeric literal token that has been extracted from a source buffer.
-class NumericLiteralToken {
+class LexedNumericLiteral {
  public:
   // Get the text corresponding to this literal.
   auto Text() const -> llvm::StringRef { return text; }
@@ -24,12 +24,12 @@ class NumericLiteralToken {
   //
   // The supplied `source_text` must outlive the return value.
   static auto Lex(llvm::StringRef source_text)
-      -> llvm::Optional<NumericLiteralToken>;
+      -> llvm::Optional<LexedNumericLiteral>;
 
   class Parser;
 
  private:
-  NumericLiteralToken() {}
+  LexedNumericLiteral() {}
 
   // The text of the token.
   llvm::StringRef text;
@@ -49,9 +49,9 @@ class NumericLiteralToken {
 //
 // Responsible for checking that a numeric literal is valid and meaningful and
 // either diagnosing or extracting its meaning.
-class NumericLiteralToken::Parser {
+class LexedNumericLiteral::Parser {
  public:
-  Parser(DiagnosticEmitter& emitter, NumericLiteralToken literal);
+  Parser(DiagnosticEmitter& emitter, LexedNumericLiteral literal);
 
   auto IsInteger() -> bool {
     return literal.radix_point == static_cast<int>(literal.text.size());
@@ -99,7 +99,7 @@ class NumericLiteralToken::Parser {
 
  private:
   DiagnosticEmitter& emitter;
-  NumericLiteralToken literal;
+  LexedNumericLiteral literal;
 
   // The radix of the literal: 2, 10, or 16, for a prefix of '0b', no prefix,
   // or '0x', respectively.

lexer/numeric_literal_fuzzer.cpp

@@ -14,15 +14,15 @@ namespace Carbon {
 // NOLINTNEXTLINE: Match the documented fuzzer entry point declaration style.
 extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
                                       std::size_t size) {
-  auto token = NumericLiteralToken::Lex(
+  auto token = LexedNumericLiteral::Lex(
       llvm::StringRef(reinterpret_cast<const char*>(data), size));
   if (!token) {
     // Lexically not a numeric literal.
     return 0;
   }
 
-  NumericLiteralToken::Parser parser(NullDiagnosticEmitter(), *token);
-  if (parser.Check() == NumericLiteralToken::Parser::UnrecoverableError) {
+  LexedNumericLiteral::Parser parser(NullDiagnosticEmitter(), *token);
+  if (parser.Check() == LexedNumericLiteral::Parser::UnrecoverableError) {
     // Lexically OK, but token is meaningless.
     return 0;
   }

lexer/numeric_literal_test.cpp

@@ -14,15 +14,15 @@ namespace Carbon {
 namespace {
 
 struct NumericLiteralTest : ::testing::Test {
-  auto Lex(llvm::StringRef text) -> NumericLiteralToken {
-    llvm::Optional<NumericLiteralToken> result = NumericLiteralToken::Lex(text);
+  auto Lex(llvm::StringRef text) -> LexedNumericLiteral {
+    llvm::Optional<LexedNumericLiteral> result = LexedNumericLiteral::Lex(text);
     assert(result);
     EXPECT_EQ(result->Text(), text);
     return *result;
   }
 
-  auto Parse(llvm::StringRef text) -> NumericLiteralToken::Parser {
-    return NumericLiteralToken::Parser(ConsoleDiagnosticEmitter(), Lex(text));
+  auto Parse(llvm::StringRef text) -> LexedNumericLiteral::Parser {
+    return LexedNumericLiteral::Parser(ConsoleDiagnosticEmitter(), Lex(text));
   }
 };
 

lexer/string_literal.cpp

@@ -94,8 +94,8 @@ static auto TakeMultiLineStringLiteralPrefix(llvm::StringRef source_text)
 
 // If source_text begins with a string literal token, extract and return
 // information on that token.
-auto StringLiteralToken::Lex(llvm::StringRef source_text)
-    -> llvm::Optional<StringLiteralToken> {
+auto LexedStringLiteral::Lex(llvm::StringRef source_text)
+    -> llvm::Optional<LexedStringLiteral> {
   const char* begin = source_text.begin();
 
   int hash_level = 0;
@@ -144,7 +144,7 @@ auto StringLiteralToken::Lex(llvm::StringRef source_text)
     content_end = source_text.begin();
   }
 
-  return StringLiteralToken(
+  return LexedStringLiteral(
       llvm::StringRef(begin, source_text.begin() - begin),
       llvm::StringRef(content_begin, content_end - content_begin), hash_level,
       multi_line);
@@ -305,7 +305,7 @@ static auto ExpandEscapeSequencesAndRemoveIndent(DiagnosticEmitter& emitter,
                                                  llvm::StringRef contents,
                                                  int hash_level,
                                                  llvm::StringRef indent)
-    -> StringLiteralToken::ExpandedValue {
+    -> LexedStringLiteral::ExpandedValue {
   std::string result;
   result.reserve(contents.size());
   bool has_errors = false;
@@ -369,7 +369,7 @@ static auto ExpandEscapeSequencesAndRemoveIndent(DiagnosticEmitter& emitter,
   }
 }
 
-auto StringLiteralToken::ComputeValue(DiagnosticEmitter& emitter) const
+auto LexedStringLiteral::ComputeValue(DiagnosticEmitter& emitter) const
     -> ExpandedValue {
   auto indent = multi_line ? CheckIndent(emitter, text, content) : Indent();
   auto result = ExpandEscapeSequencesAndRemoveIndent(emitter, content,

lexer/string_literal.h

@@ -10,7 +10,7 @@
 
 namespace Carbon {
 
-class StringLiteralToken {
+class LexedStringLiteral {
  public:
   // Get the text corresponding to this literal.
   auto Text() const -> llvm::StringRef { return text; }
@@ -21,7 +21,7 @@ class StringLiteralToken {
   // Extract a string literal token from the given text, if it has a suitable
   // form.
   static auto Lex(llvm::StringRef source_text)
-      -> llvm::Optional<StringLiteralToken>;
+      -> llvm::Optional<LexedStringLiteral>;
 
   // The result of expanding escape sequences in a string literal.
   struct ExpandedValue {
@@ -34,7 +34,7 @@ class StringLiteralToken {
   auto ComputeValue(DiagnosticEmitter& emitter) const -> ExpandedValue;
 
  private:
-  StringLiteralToken(llvm::StringRef text, llvm::StringRef content,
+  LexedStringLiteral(llvm::StringRef text, llvm::StringRef content,
                      int hash_level, bool multi_line)
       : text(text),
         content(content),

lexer/string_literal_fuzzer.cpp

@@ -14,7 +14,7 @@ namespace Carbon {
 // NOLINTNEXTLINE: Match the documented fuzzer entry point declaration style.
 extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
                                       std::size_t size) {
-  auto token = StringLiteralToken::Lex(
+  auto token = LexedStringLiteral::Lex(
       llvm::StringRef(reinterpret_cast<const char*>(data), size));
   if (!token) {
     // Lexically not a string literal.

lexer/string_literal_test.cpp

@@ -12,15 +12,15 @@ namespace Carbon {
 namespace {
 
 struct StringLiteralTest : ::testing::Test {
-  auto Lex(llvm::StringRef text) -> StringLiteralToken {
-    llvm::Optional<StringLiteralToken> result = StringLiteralToken::Lex(text);
+  auto Lex(llvm::StringRef text) -> LexedStringLiteral {
+    llvm::Optional<LexedStringLiteral> result = LexedStringLiteral::Lex(text);
     assert(result);
     EXPECT_EQ(result->Text(), text);
     return *result;
   }
 
-  auto Parse(llvm::StringRef text) -> StringLiteralToken::ExpandedValue {
-    StringLiteralToken token = Lex(text);
+  auto Parse(llvm::StringRef text) -> LexedStringLiteral::ExpandedValue {
+    LexedStringLiteral token = Lex(text);
     return token.ComputeValue(ConsoleDiagnosticEmitter());
   }
 };
@@ -73,7 +73,7 @@ TEST_F(StringLiteralTest, StringLiteralBounds) {
   };
 
   for (llvm::StringLiteral test : valid) {
-    llvm::Optional<StringLiteralToken> result = StringLiteralToken::Lex(test);
+    llvm::Optional<LexedStringLiteral> result = LexedStringLiteral::Lex(test);
     EXPECT_TRUE(result.hasValue()) << test;
     if (result) {
       EXPECT_EQ(result->Text(), test);
@@ -97,7 +97,7 @@ TEST_F(StringLiteralTest, StringLiteralBounds) {
   };
 
   for (llvm::StringLiteral test : invalid) {
-    EXPECT_FALSE(StringLiteralToken::Lex(test).hasValue())
+    EXPECT_FALSE(LexedStringLiteral::Lex(test).hasValue())
         << "`" << test << "`";
   }
 }

lexer/tokenized_buffer.cpp

@@ -178,8 +178,8 @@ class TokenizedBuffer::Lexer {
   }
 
   auto LexNumericLiteral(llvm::StringRef& source_text) -> LexResult {
-    llvm::Optional<NumericLiteralToken> literal =
-        NumericLiteralToken::Lex(source_text);
+    llvm::Optional<LexedNumericLiteral> literal =
+        LexedNumericLiteral::Lex(source_text);
     if (!literal) {
       return LexResult::NoMatch();
     }
@@ -194,10 +194,10 @@ class TokenizedBuffer::Lexer {
       set_indent = true;
     }
 
-    NumericLiteralToken::Parser literal_parser(emitter, *literal);
+    LexedNumericLiteral::Parser literal_parser(emitter, *literal);
 
     switch (literal_parser.Check()) {
-      case NumericLiteralToken::Parser::UnrecoverableError: {
+      case LexedNumericLiteral::Parser::UnrecoverableError: {
         auto token = buffer.AddToken({
             .kind = TokenKind::Error(),
             .token_line = current_line,
@@ -208,11 +208,11 @@ class TokenizedBuffer::Lexer {
         return token;
       }
 
-      case NumericLiteralToken::Parser::RecoverableError:
+      case LexedNumericLiteral::Parser::RecoverableError:
         buffer.has_errors = true;
         break;
 
-      case NumericLiteralToken::Parser::Valid:
+      case LexedNumericLiteral::Parser::Valid:
         break;
     }
 
@@ -237,8 +237,8 @@ class TokenizedBuffer::Lexer {
   }
 
   auto LexStringLiteral(llvm::StringRef& source_text) -> LexResult {
-    llvm::Optional<StringLiteralToken> literal =
-        StringLiteralToken::Lex(source_text);
+    llvm::Optional<LexedStringLiteral> literal =
+        LexedStringLiteral::Lex(source_text);
     if (!literal) {
       return LexResult::NoMatch();
     }
@@ -520,8 +520,8 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
       token_info.kind == TokenKind::RealLiteral()) {
     auto& line_info = GetLineInfo(token_info.token_line);
     int64_t token_start = line_info.start + token_info.column;
-    llvm::Optional<NumericLiteralToken> relexed_token =
-        NumericLiteralToken::Lex(source->Text().substr(token_start));
+    llvm::Optional<LexedNumericLiteral> relexed_token =
+        LexedNumericLiteral::Lex(source->Text().substr(token_start));
     assert(relexed_token && "Could not reform numeric literal token.");
     return relexed_token->Text();
   }
@@ -531,8 +531,8 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
   if (token_info.kind == TokenKind::StringLiteral()) {
     auto& line_info = GetLineInfo(token_info.token_line);
     int64_t token_start = line_info.start + token_info.column;
-    llvm::Optional<StringLiteralToken> relexed_token =
-        StringLiteralToken::Lex(source->Text().substr(token_start));
+    llvm::Optional<LexedStringLiteral> relexed_token =
+        LexedStringLiteral::Lex(source->Text().substr(token_start));
     assert(relexed_token && "Could not reform string literal token.");
     return relexed_token->Text();
   }