|
|
@@ -17,6 +17,7 @@
|
|
|
#include "toolchain/diagnostics/mocks.h"
|
|
|
#include "toolchain/lex/lex.h"
|
|
|
#include "toolchain/lex/tokenized_buffer_test_helpers.h"
|
|
|
+#include "toolchain/testing/compile_helper.h"
|
|
|
#include "toolchain/testing/yaml_test_helpers.h"
|
|
|
|
|
|
namespace Carbon::Lex {
|
|
|
@@ -35,29 +36,11 @@ namespace Yaml = ::Carbon::Testing::Yaml;
|
|
|
|
|
|
class LexerTest : public ::testing::Test {
|
|
|
protected:
|
|
|
- auto GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
|
|
|
- std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
|
|
|
- CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
|
|
|
- llvm::MemoryBuffer::getMemBuffer(text)));
|
|
|
- source_storage_.push_front(std::move(*SourceBuffer::MakeFromFile(
|
|
|
- fs_, filename, ConsoleDiagnosticConsumer())));
|
|
|
- return source_storage_.front();
|
|
|
- }
|
|
|
-
|
|
|
- auto Lex(llvm::StringRef text,
|
|
|
- DiagnosticConsumer& consumer = ConsoleDiagnosticConsumer())
|
|
|
- -> TokenizedBuffer {
|
|
|
- return Lex::Lex(value_stores_, GetSourceBuffer(text), consumer);
|
|
|
- }
|
|
|
-
|
|
|
- SharedValueStores value_stores_;
|
|
|
- llvm::vfs::InMemoryFileSystem fs_;
|
|
|
- int file_index_ = 0;
|
|
|
- std::forward_list<SourceBuffer> source_storage_;
|
|
|
+ Testing::CompileHelper compile_helper_;
|
|
|
};
|
|
|
|
|
|
TEST_F(LexerTest, HandlesEmptyBuffer) {
|
|
|
- auto buffer = Lex("");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer("");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart},
|
|
|
@@ -65,7 +48,8 @@ TEST_F(LexerTest, HandlesEmptyBuffer) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
|
- auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(
|
|
|
+ "\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
|
@@ -102,8 +86,8 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, TracksLinesAndColumnsCRLF) {
|
|
|
- auto buffer =
|
|
|
- Lex("\r\n ;;\r\n ;;;\r\n x\"foo\" '''baz\r\n a\r\n ''' y");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(
|
|
|
+ "\r\n ;;\r\n ;;;\r\n x\"foo\" '''baz\r\n a\r\n ''' y");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
|
@@ -140,7 +124,7 @@ TEST_F(LexerTest, TracksLinesAndColumnsCRLF) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, InvalidCR) {
|
|
|
- auto buffer = Lex("\n ;;\r ;\n x");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer("\n ;;\r ;\n x");
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
|
@@ -162,7 +146,7 @@ TEST_F(LexerTest, InvalidCR) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, InvalidLFCR) {
|
|
|
- auto buffer = Lex("\n ;;\n\r ;\n x");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer("\n ;;\n\r ;\n x");
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
|
@@ -184,7 +168,9 @@ TEST_F(LexerTest, InvalidLFCR) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, HandlesNumericLiteral) {
|
|
|
- auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
|
|
|
+ auto [buffer, value_stores] =
|
|
|
+ compile_helper_.GetTokenizedBufferWithSharedValueStore(
|
|
|
+ "12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
@@ -237,32 +223,33 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
|
|
|
}));
|
|
|
auto token_start = buffer.tokens().begin();
|
|
|
auto token_12 = token_start + 1;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_12)), 12);
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_12)), 12);
|
|
|
auto token_578 = token_12 + 2;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_578)), 578);
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_578)), 578);
|
|
|
auto token_1 = token_578 + 1;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_1)), 1);
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_1)), 1);
|
|
|
auto token_2 = token_1 + 1;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_2)), 2);
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_2)), 2);
|
|
|
auto token_0x12_3abc = token_2 + 1;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_0x12_3abc)),
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_0x12_3abc)),
|
|
|
0x12'3abc);
|
|
|
auto token_0b10_10_11 = token_0x12_3abc + 1;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_0b10_10_11)),
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_0b10_10_11)),
|
|
|
0b10'10'11);
|
|
|
auto token_1_234_567 = token_0b10_10_11 + 1;
|
|
|
- EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_1_234_567)),
|
|
|
+ EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_1_234_567)),
|
|
|
1'234'567);
|
|
|
auto token_1_5e9 = token_1_234_567 + 1;
|
|
|
auto value_1_5e9 =
|
|
|
- value_stores_.reals().Get(buffer.GetRealLiteral(*token_1_5e9));
|
|
|
+ value_stores.reals().Get(buffer.GetRealLiteral(*token_1_5e9));
|
|
|
EXPECT_EQ(value_1_5e9.mantissa.getZExtValue(), 15);
|
|
|
EXPECT_EQ(value_1_5e9.exponent.getSExtValue(), 8);
|
|
|
EXPECT_EQ(value_1_5e9.is_decimal, true);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
|
|
|
- auto buffer = Lex("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
|
|
|
+ auto& buffer =
|
|
|
+ compile_helper_.GetTokenizedBuffer("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
@@ -312,7 +299,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
|
|
12e+1
|
|
|
13._
|
|
|
)";
|
|
|
- auto buffer = Lex(source_text);
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(source_text);
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart},
|
|
|
@@ -370,7 +357,8 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
|
|
|
|
|
TEST_F(LexerTest, HandlesGarbageCharacters) {
|
|
|
constexpr char GarbageText[] = "$$💩-$\n$\0$12$\n\\\"\\\n\"x";
|
|
|
- auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(
|
|
|
+ llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
|
@@ -403,104 +391,104 @@ TEST_F(LexerTest, Symbols) {
|
|
|
// We don't need to exhaustively test symbols here as they're handled with
|
|
|
// common code, but we want to check specific patterns to verify things like
|
|
|
// max-munch rule and handling of interesting symbols.
|
|
|
- auto buffer = Lex("<<<");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::LessLess},
|
|
|
- {.kind = TokenKind::Less},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("<<=>>");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::LessLessEqual},
|
|
|
- {.kind = TokenKind::GreaterGreater},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("< <=> >");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Less},
|
|
|
- {.kind = TokenKind::LessEqualGreater},
|
|
|
- {.kind = TokenKind::Greater},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("\\/?@&^!");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Backslash},
|
|
|
- {.kind = TokenKind::Slash},
|
|
|
- {.kind = TokenKind::Question},
|
|
|
- {.kind = TokenKind::At},
|
|
|
- {.kind = TokenKind::Amp},
|
|
|
- {.kind = TokenKind::Caret},
|
|
|
- {.kind = TokenKind::Exclaim},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
+ auto& buffer1 = compile_helper_.GetTokenizedBuffer("<<<");
|
|
|
+ EXPECT_FALSE(buffer1.has_errors());
|
|
|
+ EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::LessLess},
|
|
|
+ {.kind = TokenKind::Less},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer2 = compile_helper_.GetTokenizedBuffer("<<=>>");
|
|
|
+ EXPECT_FALSE(buffer2.has_errors());
|
|
|
+ EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::LessLessEqual},
|
|
|
+ {.kind = TokenKind::GreaterGreater},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer3 = compile_helper_.GetTokenizedBuffer("< <=> >");
|
|
|
+ EXPECT_FALSE(buffer3.has_errors());
|
|
|
+ EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Less},
|
|
|
+ {.kind = TokenKind::LessEqualGreater},
|
|
|
+ {.kind = TokenKind::Greater},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer4 = compile_helper_.GetTokenizedBuffer("\\/?@&^!");
|
|
|
+ EXPECT_FALSE(buffer4.has_errors());
|
|
|
+ EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Backslash},
|
|
|
+ {.kind = TokenKind::Slash},
|
|
|
+ {.kind = TokenKind::Question},
|
|
|
+ {.kind = TokenKind::At},
|
|
|
+ {.kind = TokenKind::Amp},
|
|
|
+ {.kind = TokenKind::Caret},
|
|
|
+ {.kind = TokenKind::Exclaim},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, Parens) {
|
|
|
- auto buffer = Lex("()");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::OpenParen},
|
|
|
- {.kind = TokenKind::CloseParen},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("((()()))");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::OpenParen},
|
|
|
- {.kind = TokenKind::OpenParen},
|
|
|
- {.kind = TokenKind::OpenParen},
|
|
|
- {.kind = TokenKind::CloseParen},
|
|
|
- {.kind = TokenKind::OpenParen},
|
|
|
- {.kind = TokenKind::CloseParen},
|
|
|
- {.kind = TokenKind::CloseParen},
|
|
|
- {.kind = TokenKind::CloseParen},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
+ auto& buffer1 = compile_helper_.GetTokenizedBuffer("()");
|
|
|
+ EXPECT_FALSE(buffer1.has_errors());
|
|
|
+ EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::OpenParen},
|
|
|
+ {.kind = TokenKind::CloseParen},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer2 = compile_helper_.GetTokenizedBuffer("((()()))");
|
|
|
+ EXPECT_FALSE(buffer2.has_errors());
|
|
|
+ EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::OpenParen},
|
|
|
+ {.kind = TokenKind::OpenParen},
|
|
|
+ {.kind = TokenKind::OpenParen},
|
|
|
+ {.kind = TokenKind::CloseParen},
|
|
|
+ {.kind = TokenKind::OpenParen},
|
|
|
+ {.kind = TokenKind::CloseParen},
|
|
|
+ {.kind = TokenKind::CloseParen},
|
|
|
+ {.kind = TokenKind::CloseParen},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, CurlyBraces) {
|
|
|
- auto buffer = Lex("{}");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::OpenCurlyBrace},
|
|
|
- {.kind = TokenKind::CloseCurlyBrace},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("{{{}{}}}");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::OpenCurlyBrace},
|
|
|
- {.kind = TokenKind::OpenCurlyBrace},
|
|
|
- {.kind = TokenKind::OpenCurlyBrace},
|
|
|
- {.kind = TokenKind::CloseCurlyBrace},
|
|
|
- {.kind = TokenKind::OpenCurlyBrace},
|
|
|
- {.kind = TokenKind::CloseCurlyBrace},
|
|
|
- {.kind = TokenKind::CloseCurlyBrace},
|
|
|
- {.kind = TokenKind::CloseCurlyBrace},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
+ auto& buffer1 = compile_helper_.GetTokenizedBuffer("{}");
|
|
|
+ EXPECT_FALSE(buffer1.has_errors());
|
|
|
+ EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::OpenCurlyBrace},
|
|
|
+ {.kind = TokenKind::CloseCurlyBrace},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer2 = compile_helper_.GetTokenizedBuffer("{{{}{}}}");
|
|
|
+ EXPECT_FALSE(buffer2.has_errors());
|
|
|
+ EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::OpenCurlyBrace},
|
|
|
+ {.kind = TokenKind::OpenCurlyBrace},
|
|
|
+ {.kind = TokenKind::OpenCurlyBrace},
|
|
|
+ {.kind = TokenKind::CloseCurlyBrace},
|
|
|
+ {.kind = TokenKind::OpenCurlyBrace},
|
|
|
+ {.kind = TokenKind::CloseCurlyBrace},
|
|
|
+ {.kind = TokenKind::CloseCurlyBrace},
|
|
|
+ {.kind = TokenKind::CloseCurlyBrace},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, MatchingGroups) {
|
|
|
{
|
|
|
- TokenizedBuffer buffer = Lex("(){}");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer("(){}");
|
|
|
ASSERT_FALSE(buffer.has_errors());
|
|
|
auto it = ++buffer.tokens().begin();
|
|
|
auto open_paren_token = *it++;
|
|
|
@@ -521,14 +509,15 @@ TEST_F(LexerTest, MatchingGroups) {
|
|
|
}
|
|
|
|
|
|
{
|
|
|
- TokenizedBuffer buffer = Lex("({x}){(y)} {{((z))}}");
|
|
|
+ auto [buffer, value_stores] =
|
|
|
+ compile_helper_.GetTokenizedBufferWithSharedValueStore(
|
|
|
+ "({x}){(y)} {{((z))}}");
|
|
|
ASSERT_FALSE(buffer.has_errors());
|
|
|
auto it = ++buffer.tokens().begin();
|
|
|
auto open_paren_token = *it++;
|
|
|
auto open_curly_token = *it++;
|
|
|
|
|
|
- ASSERT_EQ("x",
|
|
|
- value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
|
|
|
+ ASSERT_EQ("x", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
|
|
|
auto close_curly_token = *it++;
|
|
|
auto close_paren_token = *it++;
|
|
|
EXPECT_EQ(close_paren_token,
|
|
|
@@ -542,8 +531,7 @@ TEST_F(LexerTest, MatchingGroups) {
|
|
|
|
|
|
open_curly_token = *it++;
|
|
|
open_paren_token = *it++;
|
|
|
- ASSERT_EQ("y",
|
|
|
- value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
|
|
|
+ ASSERT_EQ("y", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
|
|
|
close_paren_token = *it++;
|
|
|
close_curly_token = *it++;
|
|
|
EXPECT_EQ(close_curly_token,
|
|
|
@@ -559,8 +547,7 @@ TEST_F(LexerTest, MatchingGroups) {
|
|
|
auto inner_open_curly_token = *it++;
|
|
|
open_paren_token = *it++;
|
|
|
auto inner_open_paren_token = *it++;
|
|
|
- ASSERT_EQ("z",
|
|
|
- value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
|
|
|
+ ASSERT_EQ("z", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
|
|
|
auto inner_close_paren_token = *it++;
|
|
|
close_paren_token = *it++;
|
|
|
auto inner_close_curly_token = *it++;
|
|
|
@@ -589,26 +576,26 @@ TEST_F(LexerTest, MatchingGroups) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, MismatchedGroups) {
|
|
|
- auto buffer = Lex("{");
|
|
|
- EXPECT_TRUE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Error, .text = "{"},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("}");
|
|
|
- EXPECT_TRUE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Error, .text = "}"},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("{(}");
|
|
|
- EXPECT_TRUE(buffer.has_errors());
|
|
|
+ auto& buffer1 = compile_helper_.GetTokenizedBuffer("{");
|
|
|
+ EXPECT_TRUE(buffer1.has_errors());
|
|
|
+ EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Error, .text = "{"},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer2 = compile_helper_.GetTokenizedBuffer("}");
|
|
|
+ EXPECT_TRUE(buffer2.has_errors());
|
|
|
+ EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Error, .text = "}"},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer3 = compile_helper_.GetTokenizedBuffer("{(}");
|
|
|
+ EXPECT_TRUE(buffer3.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
- buffer,
|
|
|
+ buffer3,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart},
|
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
|
|
|
@@ -618,10 +605,10 @@ TEST_F(LexerTest, MismatchedGroups) {
|
|
|
{.kind = TokenKind::FileEnd},
|
|
|
}));
|
|
|
|
|
|
- buffer = Lex(")({)");
|
|
|
- EXPECT_TRUE(buffer.has_errors());
|
|
|
+ auto& buffer4 = compile_helper_.GetTokenizedBuffer(")({)");
|
|
|
+ EXPECT_TRUE(buffer4.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
- buffer,
|
|
|
+ buffer4,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Error, .column = 1, .text = ")"},
|
|
|
@@ -634,7 +621,7 @@ TEST_F(LexerTest, MismatchedGroups) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, Whitespace) {
|
|
|
- auto buffer = Lex("{( } {(");
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer("{( } {(");
|
|
|
|
|
|
// Whether there should be whitespace before/after each token.
|
|
|
bool space[] = {false,
|
|
|
@@ -675,7 +662,7 @@ TEST_F(LexerTest, Keywords) {
|
|
|
#include "toolchain/lex/token_kind.def"
|
|
|
};
|
|
|
for (const auto& keyword : keywords) {
|
|
|
- auto buffer = Lex(keyword.fixed_spelling());
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(keyword.fixed_spelling());
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart},
|
|
|
@@ -686,10 +673,10 @@ TEST_F(LexerTest, Keywords) {
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, Comments) {
|
|
|
- auto buffer = Lex(" ;\n // foo\n ;\n");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
+ auto& buffer1 = compile_helper_.GetTokenizedBuffer(" ;\n // foo\n ;\n");
|
|
|
+ EXPECT_FALSE(buffer1.has_errors());
|
|
|
EXPECT_THAT(
|
|
|
- buffer,
|
|
|
+ buffer1,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
|
|
|
@@ -697,25 +684,26 @@ TEST_F(LexerTest, Comments) {
|
|
|
{.kind = TokenKind::FileEnd, .line = 3, .column = 4},
|
|
|
}));
|
|
|
|
|
|
- buffer = Lex("// foo\n//\n// bar");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::FileEnd}}));
|
|
|
+ auto& buffer2 = compile_helper_.GetTokenizedBuffer("// foo\n//\n// bar");
|
|
|
+ EXPECT_FALSE(buffer2.has_errors());
|
|
|
+ EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::FileEnd}}));
|
|
|
|
|
|
// Make sure weird characters aren't a problem.
|
|
|
- buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::FileEnd}}));
|
|
|
+ auto& buffer3 =
|
|
|
+ compile_helper_.GetTokenizedBuffer(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
|
|
+ EXPECT_FALSE(buffer3.has_errors());
|
|
|
+ EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::FileEnd}}));
|
|
|
|
|
|
// Make sure we can lex a comment at the end of the input.
|
|
|
- buffer = Lex("//");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::FileEnd}}));
|
|
|
+ auto& buffer4 = compile_helper_.GetTokenizedBuffer("//");
|
|
|
+ EXPECT_FALSE(buffer4.has_errors());
|
|
|
+ EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::FileEnd}}));
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, InvalidComments) {
|
|
|
@@ -726,53 +714,54 @@ TEST_F(LexerTest, InvalidComments) {
|
|
|
" //world",
|
|
|
};
|
|
|
for (llvm::StringLiteral testcase : testcases) {
|
|
|
- auto buffer = Lex(testcase);
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(testcase);
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, Identifiers) {
|
|
|
- auto buffer = Lex(" foobar");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Identifier,
|
|
|
- .column = 4,
|
|
|
- .indent_column = 4,
|
|
|
- .text = "foobar"},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
+ auto& buffer1 = compile_helper_.GetTokenizedBuffer(" foobar");
|
|
|
+ EXPECT_FALSE(buffer1.has_errors());
|
|
|
+ EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Identifier,
|
|
|
+ .column = 4,
|
|
|
+ .indent_column = 4,
|
|
|
+ .text = "foobar"},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
|
|
|
// Check different kinds of identifier character sequences.
|
|
|
- buffer = Lex("_foo_bar");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
-
|
|
|
- buffer = Lex("foo2bar00");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
+ auto& buffer2 = compile_helper_.GetTokenizedBuffer("_foo_bar");
|
|
|
+ EXPECT_FALSE(buffer2.has_errors());
|
|
|
+ EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
+
|
|
|
+ auto& buffer3 = compile_helper_.GetTokenizedBuffer("foo2bar00");
|
|
|
+ EXPECT_FALSE(buffer3.has_errors());
|
|
|
+ EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
|
|
|
// Check that we can parse identifiers that start with a keyword.
|
|
|
- buffer = Lex("fnord");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::FileStart},
|
|
|
- {.kind = TokenKind::Identifier, .text = "fnord"},
|
|
|
- {.kind = TokenKind::FileEnd},
|
|
|
- }));
|
|
|
+ auto& buffer4 = compile_helper_.GetTokenizedBuffer("fnord");
|
|
|
+ EXPECT_FALSE(buffer4.has_errors());
|
|
|
+ EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
+ {.kind = TokenKind::Identifier, .text = "fnord"},
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
+ }));
|
|
|
|
|
|
// Check multiple identifiers with indent and interning.
|
|
|
- buffer = Lex(" foo;bar\nbar \n foo\tfoo");
|
|
|
- EXPECT_FALSE(buffer.has_errors());
|
|
|
- EXPECT_THAT(buffer,
|
|
|
+ auto& buffer5 =
|
|
|
+ compile_helper_.GetTokenizedBuffer(" foo;bar\nbar \n foo\tfoo");
|
|
|
+ EXPECT_FALSE(buffer5.has_errors());
|
|
|
+ EXPECT_THAT(buffer5,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::Identifier,
|
|
|
@@ -823,7 +812,8 @@ TEST_F(LexerTest, StringLiterals) {
|
|
|
"""x"""
|
|
|
)";
|
|
|
|
|
|
- auto buffer = Lex(testcase);
|
|
|
+ auto [buffer, value_stores] =
|
|
|
+ compile_helper_.GetTokenizedBufferWithSharedValueStore(testcase);
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
@@ -832,13 +822,13 @@ TEST_F(LexerTest, StringLiterals) {
|
|
|
.line = 2,
|
|
|
.column = 5,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {"hello world\n"}},
|
|
|
{.kind = TokenKind::StringLiteral,
|
|
|
.line = 4,
|
|
|
.column = 5,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {" test \xAB\n"}},
|
|
|
{.kind = TokenKind::Identifier,
|
|
|
.line = 7,
|
|
|
@@ -849,19 +839,19 @@ TEST_F(LexerTest, StringLiterals) {
|
|
|
.line = 9,
|
|
|
.column = 7,
|
|
|
.indent_column = 7,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {"\""}},
|
|
|
{.kind = TokenKind::StringLiteral,
|
|
|
.line = 11,
|
|
|
.column = 5,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = llvm::StringLiteral::withInnerNUL("\0")},
|
|
|
{.kind = TokenKind::StringLiteral,
|
|
|
.line = 13,
|
|
|
.column = 5,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {"\\0\"foo\"\\1"}},
|
|
|
|
|
|
// """x""" is three string literals, not one invalid
|
|
|
@@ -870,19 +860,19 @@ TEST_F(LexerTest, StringLiterals) {
|
|
|
.line = 15,
|
|
|
.column = 5,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {""}},
|
|
|
{.kind = TokenKind::StringLiteral,
|
|
|
.line = 15,
|
|
|
.column = 7,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {"x"}},
|
|
|
{.kind = TokenKind::StringLiteral,
|
|
|
.line = 15,
|
|
|
.column = 10,
|
|
|
.indent_column = 5,
|
|
|
- .value_stores = &value_stores_,
|
|
|
+ .value_stores = &value_stores,
|
|
|
.string_contents = {""}},
|
|
|
{.kind = TokenKind::FileEnd, .line = 16, .column = 3},
|
|
|
}));
|
|
|
@@ -909,7 +899,7 @@ TEST_F(LexerTest, InvalidStringLiterals) {
|
|
|
|
|
|
for (llvm::StringLiteral test : invalid) {
|
|
|
SCOPED_TRACE(test);
|
|
|
- auto buffer = Lex(test);
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(test);
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
|
|
|
// We should have formed at least one error token.
|
|
|
@@ -932,7 +922,8 @@ TEST_F(LexerTest, TypeLiterals) {
|
|
|
s1
|
|
|
)";
|
|
|
|
|
|
- auto buffer = Lex(testcase);
|
|
|
+ auto [buffer, value_stores] =
|
|
|
+ compile_helper_.GetTokenizedBufferWithSharedValueStore(testcase);
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
@@ -1017,7 +1008,7 @@ TEST_F(LexerTest, TypeLiterals) {
|
|
|
|
|
|
auto type_size = [&](int token_index) {
|
|
|
auto token = buffer.tokens().begin()[token_index];
|
|
|
- return value_stores_.ints().Get(buffer.GetTypeLiteralSize(token));
|
|
|
+ return value_stores.ints().Get(buffer.GetTypeLiteralSize(token));
|
|
|
};
|
|
|
|
|
|
EXPECT_EQ(type_size(2), 1);
|
|
|
@@ -1040,7 +1031,7 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
|
|
|
HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::TooManyDigits, DiagnosticLevel::Error, 1, 2,
|
|
|
HasSubstr(llvm::formatv(" {0} ", Count)))));
|
|
|
- auto buffer = Lex(code, consumer);
|
|
|
+ auto& buffer = compile_helper_.GetTokenizedBuffer(code, &consumer);
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
@@ -1064,7 +1055,7 @@ TEST_F(LexerTest, DiagnosticTrailingComment) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::TrailingComment,
|
|
|
DiagnosticLevel::Error, 3, 19, _)));
|
|
|
- Lex(testcase, consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer(testcase, &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticWhitespace) {
|
|
|
@@ -1072,7 +1063,7 @@ TEST_F(LexerTest, DiagnosticWhitespace) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::NoWhitespaceAfterCommentIntroducer,
|
|
|
DiagnosticLevel::Error, 1, 3, _)));
|
|
|
- Lex("//no space after comment", consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer("//no space after comment", &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
|
|
|
@@ -1080,7 +1071,7 @@ TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::UnknownEscapeSequence,
|
|
|
DiagnosticLevel::Error, 1, 8, HasSubstr("`b`"))));
|
|
|
- Lex(R"("hello\bworld")", consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer(R"("hello\bworld")", &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticBadHex) {
|
|
|
@@ -1088,7 +1079,7 @@ TEST_F(LexerTest, DiagnosticBadHex) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::HexadecimalEscapeMissingDigits,
|
|
|
DiagnosticLevel::Error, 1, 9, _)));
|
|
|
- Lex(R"("hello\xabworld")", consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer(R"("hello\xabworld")", &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticInvalidDigit) {
|
|
|
@@ -1096,7 +1087,7 @@ TEST_F(LexerTest, DiagnosticInvalidDigit) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::InvalidDigit,
|
|
|
DiagnosticLevel::Error, 1, 6, HasSubstr("'a'"))));
|
|
|
- Lex("0x123abc", consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer("0x123abc", &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticMissingTerminator) {
|
|
|
@@ -1104,7 +1095,7 @@ TEST_F(LexerTest, DiagnosticMissingTerminator) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::UnterminatedString,
|
|
|
DiagnosticLevel::Error, 1, 1, _)));
|
|
|
- Lex(R"(#" ")", consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer(R"(#" ")", &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
|
|
|
@@ -1112,12 +1103,13 @@ TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
|
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
|
|
DiagnosticKind::UnrecognizedCharacters,
|
|
|
DiagnosticLevel::Error, 1, 1, _)));
|
|
|
- Lex("\b", consumer);
|
|
|
+ compile_helper_.GetTokenizedBuffer("\b", &consumer);
|
|
|
}
|
|
|
|
|
|
TEST_F(LexerTest, PrintingOutputYaml) {
|
|
|
// Test that we can parse this into YAML and verify line and indent data.
|
|
|
- auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
|
|
|
+ auto& buffer =
|
|
|
+ compile_helper_.GetTokenizedBuffer("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
|
|
|
ASSERT_FALSE(buffer.has_errors());
|
|
|
TestRawOstream print_stream;
|
|
|
buffer.Print(print_stream);
|
|
|
@@ -1125,7 +1117,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
|
|
|
EXPECT_THAT(
|
|
|
Yaml::Value::FromText(print_stream.TakeStr()),
|
|
|
IsYaml(ElementsAre(Yaml::Sequence(ElementsAre(Yaml::Mapping(ElementsAre(
|
|
|
- Pair("filename", source_storage_.front().filename().str()),
|
|
|
+ Pair("filename", buffer.source().filename().str()),
|
|
|
Pair("tokens", Yaml::Sequence(ElementsAre(
|
|
|
Yaml::Mapping(ElementsAre(
|
|
|
Pair("index", "0"), Pair("kind", "FileStart"),
|