|
@@ -60,7 +60,7 @@ TEST_F(LexerTest, HandlesEmptyBuffer) {
|
|
|
auto buffer = Lex("");
|
|
auto buffer = Lex("");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
|
|
|
|
|
|
|
+ {TokenKind::FileStart}, {TokenKind::FileEnd}}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
@@ -69,7 +69,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
|
EXPECT_THAT(
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile,
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart,
|
|
|
.line = 1,
|
|
.line = 1,
|
|
|
.column = 1,
|
|
.column = 1,
|
|
|
.indent_column = 1},
|
|
.indent_column = 1},
|
|
@@ -96,7 +96,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
|
.column = 6,
|
|
.column = 6,
|
|
|
.indent_column = 11,
|
|
.indent_column = 11,
|
|
|
.text = "y"},
|
|
.text = "y"},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 6, .column = 7},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 6, .column = 7},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -105,7 +105,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
|
.line = 1,
|
|
.line = 1,
|
|
|
.column = 1,
|
|
.column = 1,
|
|
@@ -150,7 +150,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
|
|
|
.column = 1,
|
|
.column = 1,
|
|
|
.indent_column = 1,
|
|
.indent_column = 1,
|
|
|
.text = "1.5e9"},
|
|
.text = "1.5e9"},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 6, .column = 6},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 6, .column = 6},
|
|
|
}));
|
|
}));
|
|
|
auto token_start = buffer.tokens().begin();
|
|
auto token_start = buffer.tokens().begin();
|
|
|
auto token_12 = token_start + 1;
|
|
auto token_12 = token_start + 1;
|
|
@@ -190,7 +190,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::Error,
|
|
{.kind = TokenKind::Error,
|
|
|
.line = 1,
|
|
.line = 1,
|
|
|
.column = 1,
|
|
.column = 1,
|
|
@@ -216,7 +216,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
|
|
|
.column = 28,
|
|
.column = 28,
|
|
|
.indent_column = 1,
|
|
.indent_column = 1,
|
|
|
.text = "0ops"},
|
|
.text = "0ops"},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 1, .column = 32},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 1, .column = 32},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -239,7 +239,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
|
|
auto buffer = Lex(source_text);
|
|
auto buffer = Lex(source_text);
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart},
|
|
|
{.kind = TokenKind::IntegerLiteral, .text = "1"},
|
|
{.kind = TokenKind::IntegerLiteral, .text = "1"},
|
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::Period},
|
|
|
// newline
|
|
// newline
|
|
@@ -288,7 +288,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::Period},
|
|
|
{.kind = TokenKind::Underscore},
|
|
{.kind = TokenKind::Underscore},
|
|
|
// newline
|
|
// newline
|
|
|
- {.kind = TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -299,7 +299,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
|
|
|
EXPECT_THAT(
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::Error,
|
|
{.kind = TokenKind::Error,
|
|
|
.line = 1,
|
|
.line = 1,
|
|
|
.column = 1,
|
|
.column = 1,
|
|
@@ -322,7 +322,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
|
|
|
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
|
|
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
|
|
|
// newline
|
|
// newline
|
|
|
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
|
|
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 4, .column = 3},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 4, .column = 3},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -333,35 +333,35 @@ TEST_F(LexerTest, Symbols) {
|
|
|
auto buffer = Lex("<<<");
|
|
auto buffer = Lex("<<<");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::LessLess},
|
|
{TokenKind::LessLess},
|
|
|
{TokenKind::Less},
|
|
{TokenKind::Less},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("<<=>>");
|
|
buffer = Lex("<<=>>");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::LessLessEqual},
|
|
{TokenKind::LessLessEqual},
|
|
|
{TokenKind::GreaterGreater},
|
|
{TokenKind::GreaterGreater},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("< <=> >");
|
|
buffer = Lex("< <=> >");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::Less},
|
|
{TokenKind::Less},
|
|
|
{TokenKind::LessEqualGreater},
|
|
{TokenKind::LessEqualGreater},
|
|
|
{TokenKind::Greater},
|
|
{TokenKind::Greater},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("\\/?@&^!");
|
|
buffer = Lex("\\/?@&^!");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::Backslash},
|
|
{TokenKind::Backslash},
|
|
|
{TokenKind::Slash},
|
|
{TokenKind::Slash},
|
|
|
{TokenKind::Question},
|
|
{TokenKind::Question},
|
|
@@ -369,7 +369,7 @@ TEST_F(LexerTest, Symbols) {
|
|
|
{TokenKind::Amp},
|
|
{TokenKind::Amp},
|
|
|
{TokenKind::Caret},
|
|
{TokenKind::Caret},
|
|
|
{TokenKind::Exclaim},
|
|
{TokenKind::Exclaim},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -377,16 +377,16 @@ TEST_F(LexerTest, Parens) {
|
|
|
auto buffer = Lex("()");
|
|
auto buffer = Lex("()");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::OpenParen},
|
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::CloseParen},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("((()()))");
|
|
buffer = Lex("((()()))");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::OpenParen},
|
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::OpenParen},
|
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::OpenParen},
|
|
@@ -395,7 +395,7 @@ TEST_F(LexerTest, Parens) {
|
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::CloseParen},
|
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::CloseParen},
|
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::CloseParen},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -403,16 +403,16 @@ TEST_F(LexerTest, CurlyBraces) {
|
|
|
auto buffer = Lex("{}");
|
|
auto buffer = Lex("{}");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("{{{}{}}}");
|
|
buffer = Lex("{{{}{}}}");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
@@ -421,7 +421,7 @@ TEST_F(LexerTest, CurlyBraces) {
|
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -443,7 +443,7 @@ TEST_F(LexerTest, MatchingGroups) {
|
|
|
EXPECT_EQ(open_curly_token,
|
|
EXPECT_EQ(open_curly_token,
|
|
|
buffer.GetMatchedOpeningToken(close_curly_token));
|
|
buffer.GetMatchedOpeningToken(close_curly_token));
|
|
|
auto eof_token = *it++;
|
|
auto eof_token = *it++;
|
|
|
- EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
|
|
|
|
|
|
|
+ EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
|
|
|
EXPECT_EQ(buffer.tokens().end(), it);
|
|
EXPECT_EQ(buffer.tokens().end(), it);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -510,7 +510,7 @@ TEST_F(LexerTest, MatchingGroups) {
|
|
|
buffer.GetMatchedOpeningToken(inner_close_paren_token));
|
|
buffer.GetMatchedOpeningToken(inner_close_paren_token));
|
|
|
|
|
|
|
|
auto eof_token = *it++;
|
|
auto eof_token = *it++;
|
|
|
- EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
|
|
|
|
|
|
|
+ EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
|
|
|
EXPECT_EQ(buffer.tokens().end(), it);
|
|
EXPECT_EQ(buffer.tokens().end(), it);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -520,18 +520,18 @@ TEST_F(LexerTest, MismatchedGroups) {
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer,
|
|
EXPECT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
|
{.kind = TokenKind::CloseCurlyBrace, .recovery = true},
|
|
{.kind = TokenKind::CloseCurlyBrace, .recovery = true},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("}");
|
|
buffer = Lex("}");
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Error, .text = "}"},
|
|
{.kind = TokenKind::Error, .text = "}"},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("{(}");
|
|
buffer = Lex("{(}");
|
|
@@ -539,12 +539,12 @@ TEST_F(LexerTest, MismatchedGroups) {
|
|
|
EXPECT_THAT(
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
|
|
|
{.kind = TokenKind::OpenParen, .column = 2},
|
|
{.kind = TokenKind::OpenParen, .column = 2},
|
|
|
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
|
|
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
|
|
|
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
|
|
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex(")({)");
|
|
buffer = Lex(")({)");
|
|
@@ -552,13 +552,13 @@ TEST_F(LexerTest, MismatchedGroups) {
|
|
|
EXPECT_THAT(
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Error, .column = 1, .text = ")"},
|
|
{.kind = TokenKind::Error, .column = 1, .text = ")"},
|
|
|
{.kind = TokenKind::OpenParen, .column = 2},
|
|
{.kind = TokenKind::OpenParen, .column = 2},
|
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
|
|
|
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
|
|
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
|
|
|
{.kind = TokenKind::CloseParen, .column = 4},
|
|
{.kind = TokenKind::CloseParen, .column = 4},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -606,15 +606,15 @@ TEST_F(LexerTest, Keywords) {
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer,
|
|
EXPECT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Fn, .column = 4, .indent_column = 4},
|
|
{.kind = TokenKind::Fn, .column = 4, .indent_column = 4},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("and or not if else for return var break continue _");
|
|
buffer = Lex("and or not if else for return var break continue _");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{TokenKind::And},
|
|
{TokenKind::And},
|
|
|
{TokenKind::Or},
|
|
{TokenKind::Or},
|
|
|
{TokenKind::Not},
|
|
{TokenKind::Not},
|
|
@@ -626,7 +626,7 @@ TEST_F(LexerTest, Keywords) {
|
|
|
{TokenKind::Break},
|
|
{TokenKind::Break},
|
|
|
{TokenKind::Continue},
|
|
{TokenKind::Continue},
|
|
|
{TokenKind::Underscore},
|
|
{TokenKind::Underscore},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -636,28 +636,28 @@ TEST_F(LexerTest, Comments) {
|
|
|
EXPECT_THAT(
|
|
EXPECT_THAT(
|
|
|
buffer,
|
|
buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
|
|
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
|
|
|
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
|
|
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 3, .column = 4},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 3, .column = 4},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("// foo\n//\n// bar");
|
|
buffer = Lex("// foo\n//\n// bar");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
|
|
|
|
|
|
|
+ {TokenKind::FileStart}, {TokenKind::FileEnd}}));
|
|
|
|
|
|
|
|
// Make sure weird characters aren't a problem.
|
|
// Make sure weird characters aren't a problem.
|
|
|
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
|
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
|
|
|
|
|
|
|
+ {TokenKind::FileStart}, {TokenKind::FileEnd}}));
|
|
|
|
|
|
|
|
// Make sure we can lex a comment at the end of the input.
|
|
// Make sure we can lex a comment at the end of the input.
|
|
|
buffer = Lex("//");
|
|
buffer = Lex("//");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
|
|
|
|
|
|
|
+ {TokenKind::FileStart}, {TokenKind::FileEnd}}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
TEST_F(LexerTest, InvalidComments) {
|
|
TEST_F(LexerTest, InvalidComments) {
|
|
@@ -677,38 +677,38 @@ TEST_F(LexerTest, Identifiers) {
|
|
|
auto buffer = Lex(" foobar");
|
|
auto buffer = Lex(" foobar");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Identifier,
|
|
{.kind = TokenKind::Identifier,
|
|
|
.column = 4,
|
|
.column = 4,
|
|
|
.indent_column = 4,
|
|
.indent_column = 4,
|
|
|
.text = "foobar"},
|
|
.text = "foobar"},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
// Check different kinds of identifier character sequences.
|
|
// Check different kinds of identifier character sequences.
|
|
|
buffer = Lex("_foo_bar");
|
|
buffer = Lex("_foo_bar");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
|
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
buffer = Lex("foo2bar00");
|
|
buffer = Lex("foo2bar00");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
|
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
// Check that we can parse identifiers that start with a keyword.
|
|
// Check that we can parse identifiers that start with a keyword.
|
|
|
buffer = Lex("fnord");
|
|
buffer = Lex("fnord");
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {TokenKind::StartOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileStart},
|
|
|
{.kind = TokenKind::Identifier, .text = "fnord"},
|
|
{.kind = TokenKind::Identifier, .text = "fnord"},
|
|
|
- {TokenKind::EndOfFile},
|
|
|
|
|
|
|
+ {TokenKind::FileEnd},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
// Check multiple identifiers with indent and interning.
|
|
// Check multiple identifiers with indent and interning.
|
|
@@ -716,7 +716,7 @@ TEST_F(LexerTest, Identifiers) {
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer,
|
|
EXPECT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::Identifier,
|
|
{.kind = TokenKind::Identifier,
|
|
|
.line = 1,
|
|
.line = 1,
|
|
|
.column = 4,
|
|
.column = 4,
|
|
@@ -743,7 +743,7 @@ TEST_F(LexerTest, Identifiers) {
|
|
|
.column = 7,
|
|
.column = 7,
|
|
|
.indent_column = 3,
|
|
.indent_column = 3,
|
|
|
.text = "foo"},
|
|
.text = "foo"},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 3, .column = 10},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 3, .column = 10},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -769,7 +769,7 @@ TEST_F(LexerTest, StringLiterals) {
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
EXPECT_THAT(buffer,
|
|
EXPECT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
{.kind = TokenKind::StringLiteral,
|
|
{.kind = TokenKind::StringLiteral,
|
|
|
.line = 2,
|
|
.line = 2,
|
|
|
.column = 5,
|
|
.column = 5,
|
|
@@ -826,7 +826,7 @@ TEST_F(LexerTest, StringLiterals) {
|
|
|
.indent_column = 5,
|
|
.indent_column = 5,
|
|
|
.value_stores = &value_stores_,
|
|
.value_stores = &value_stores_,
|
|
|
.string_contents = {""}},
|
|
.string_contents = {""}},
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 16, .column = 3},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 16, .column = 3},
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -878,7 +878,7 @@ TEST_F(LexerTest, TypeLiterals) {
|
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
|
ASSERT_THAT(buffer,
|
|
ASSERT_THAT(buffer,
|
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
|
|
|
|
|
{.kind = TokenKind::Identifier,
|
|
{.kind = TokenKind::Identifier,
|
|
|
.line = 2,
|
|
.line = 2,
|
|
@@ -954,7 +954,7 @@ TEST_F(LexerTest, TypeLiterals) {
|
|
|
.indent_column = 5,
|
|
.indent_column = 5,
|
|
|
.text = {"s1"}},
|
|
.text = {"s1"}},
|
|
|
|
|
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 6, .column = 3},
|
|
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 6, .column = 3},
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
auto token_i1 = buffer.tokens().begin() + 2;
|
|
auto token_i1 = buffer.tokens().begin() + 2;
|
|
@@ -987,17 +987,16 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
|
|
|
HasSubstr(llvm::formatv(" {0} ", Count)))));
|
|
HasSubstr(llvm::formatv(" {0} ", Count)))));
|
|
|
auto buffer = Lex(code, consumer);
|
|
auto buffer = Lex(code, consumer);
|
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
- ASSERT_THAT(
|
|
|
|
|
- buffer,
|
|
|
|
|
- HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
|
|
- {.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
|
|
|
|
|
- {.kind = TokenKind::Error,
|
|
|
|
|
- .line = 1,
|
|
|
|
|
- .column = 1,
|
|
|
|
|
- .indent_column = 1,
|
|
|
|
|
- .text = {code}},
|
|
|
|
|
- {.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2},
|
|
|
|
|
- }));
|
|
|
|
|
|
|
+ ASSERT_THAT(buffer,
|
|
|
|
|
+ HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
|
|
|
+ {.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
|
|
|
|
+ {.kind = TokenKind::Error,
|
|
|
|
|
+ .line = 1,
|
|
|
|
|
+ .column = 1,
|
|
|
|
|
+ .indent_column = 1,
|
|
|
|
|
+ .text = {code}},
|
|
|
|
|
+ {.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2},
|
|
|
|
|
+ }));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
TEST_F(LexerTest, DiagnosticTrailingComment) {
|
|
TEST_F(LexerTest, DiagnosticTrailingComment) {
|
|
@@ -1075,7 +1074,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
|
|
|
Pair("tokens",
|
|
Pair("tokens",
|
|
|
Yaml::Sequence(ElementsAre(
|
|
Yaml::Sequence(ElementsAre(
|
|
|
Yaml::Mapping(ElementsAre(
|
|
Yaml::Mapping(ElementsAre(
|
|
|
- Pair("index", "0"), Pair("kind", "StartOfFile"),
|
|
|
|
|
|
|
+ Pair("index", "0"), Pair("kind", "FileStart"),
|
|
|
Pair("line", "1"), Pair("column", "1"),
|
|
Pair("line", "1"), Pair("column", "1"),
|
|
|
Pair("indent", "1"), Pair("spelling", ""),
|
|
Pair("indent", "1"), Pair("spelling", ""),
|
|
|
Pair("has_trailing_space", "true"))),
|
|
Pair("has_trailing_space", "true"))),
|
|
@@ -1095,7 +1094,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
|
|
|
Pair("indent", "1"), Pair("spelling", ";"),
|
|
Pair("indent", "1"), Pair("spelling", ";"),
|
|
|
Pair("has_trailing_space", "true"))),
|
|
Pair("has_trailing_space", "true"))),
|
|
|
Yaml::Mapping(ElementsAre(
|
|
Yaml::Mapping(ElementsAre(
|
|
|
- Pair("index", "4"), Pair("kind", "EndOfFile"),
|
|
|
|
|
|
|
+ Pair("index", "4"), Pair("kind", "FileEnd"),
|
|
|
Pair("line", "15"), Pair("column", "1"),
|
|
Pair("line", "15"), Pair("column", "1"),
|
|
|
Pair("indent", "1"), Pair("spelling", "")))))))))))));
|
|
Pair("indent", "1"), Pair("spelling", "")))))))))))));
|
|
|
}
|
|
}
|