[Toolchain][Parser] Add support for `package` directive. (#2144)

Summary:

Adds parsing support for the `package` directive as specified by the
`Code and name organization` design doc.

Co-authored-by: ergawy <kareem.ergawy@guardsquare.com>

toolchain/diagnostics/diagnostic_registry.def

@@ -67,6 +67,13 @@ CARBON_DIAGNOSTIC_KIND(UnexpectedTokenAfterListElement)
 CARBON_DIAGNOSTIC_KIND(UnexpectedTokenInCodeBlock)
 CARBON_DIAGNOSTIC_KIND(UnrecognizedDeclaration)
 
+// Packaged-related diagnostics
+CARBON_DIAGNOSTIC_KIND(ExpectedIdentifierAfterPackage)
+CARBON_DIAGNOSTIC_KIND(ExpectedLibraryName)
+CARBON_DIAGNOSTIC_KIND(MissingLibraryKeyword)
+CARBON_DIAGNOSTIC_KIND(ExpectedApiOrImpl)
+CARBON_DIAGNOSTIC_KIND(ExpectedSemiToEndPackageDirective)
+
 // ============================================================================
 // Other diagnostics
 // ============================================================================

toolchain/parser/parse_node_kind.def

@@ -17,6 +17,13 @@
 CARBON_PARSE_NODE_KIND(DeclarationEnd)
 CARBON_PARSE_NODE_KIND(EmptyDeclaration)
 CARBON_PARSE_NODE_KIND(DeclaredName)
+
+CARBON_PARSE_NODE_KIND(PackageDirective)
+CARBON_PARSE_NODE_KIND(PackageApi)
+CARBON_PARSE_NODE_KIND(PackageImpl)
+CARBON_PARSE_NODE_KIND(PackageLibrary)
+CARBON_PARSE_NODE_KIND(PackageEnd)
+
 CARBON_PARSE_NODE_KIND(FunctionDeclaration)
 CARBON_PARSE_NODE_KIND(ParameterList)
 CARBON_PARSE_NODE_KIND(ParameterListComma)

toolchain/parser/parse_tree_test.cpp

@@ -1293,5 +1293,68 @@ TEST_F(ParseTreeTest, ParsePostfixExpressionRegression) {
   }
 }
 
+TEST_F(ParseTreeTest, Package) {
+  TokenizedBuffer tokens = GetTokenizedBuffer(R"(
+    package Geometry api;
+    package Geometry impl;
+    package Geometry library "Shapes" api;
+    package Geometry library "Shapes" impl;
+  )");
+
+  ParseTree tree = ParseTree::Parse(tokens, consumer);
+
+  EXPECT_THAT(tree,
+              MatchParseTreeNodes(
+                  {MatchPackageDirective(MatchDeclaredName("Geometry"),
+                                         MatchPackageApi(), MatchPackageEnd()),
+
+                   MatchPackageDirective(MatchDeclaredName("Geometry"),
+                                         MatchPackageImpl(), MatchPackageEnd()),
+
+                   MatchPackageDirective(
+                       MatchDeclaredName("Geometry"),
+                       MatchPackageLibrary(MatchLiteral("\"Shapes\"")),
+                       MatchPackageApi(), MatchPackageEnd()),
+
+                   MatchPackageDirective(
+                       MatchDeclaredName("Geometry"),
+                       MatchPackageLibrary(MatchLiteral("\"Shapes\"")),
+                       MatchPackageImpl(), MatchPackageEnd()),
+
+                   MatchFileEnd()}));
+}
+
+TEST_F(ParseTreeTest, PackageErrors) {
+  struct TestCase {
+    llvm::StringLiteral input;
+    ::testing::Matcher<const Diagnostic&> diag_matcher;
+  };
+
+  TestCase testcases[] = {
+      {"package;", IsDiagnosticMessage("Expected identifier after `package`.")},
+      {"package fn;",
+       IsDiagnosticMessage("Expected identifier after `package`.")},
+      {"package library \"Shapes\" api;",
+       IsDiagnosticMessage("Expected identifier after `package`.")},
+      {"package Geometry library Shapes api;",
+       IsDiagnosticMessage(
+           "Expected a string literal to specify the library name.")},
+      {"package Geometry \"Shapes\" api;",
+       IsDiagnosticMessage("Missing `library` keyword.")},
+      {"package Geometry api",
+       IsDiagnosticMessage("Expected `;` to end package directive.")},
+      {"package Geometry;", IsDiagnosticMessage("Expected a `api` or `impl`.")},
+      {R"(package Foo library "bar" "baz";)",
+       IsDiagnosticMessage("Expected a `api` or `impl`.")}};
+
+  for (const TestCase& testcase : testcases) {
+    TokenizedBuffer tokens = GetTokenizedBuffer(testcase.input);
+    Testing::MockDiagnosticConsumer consumer;
+    EXPECT_CALL(consumer, HandleDiagnostic(testcase.diag_matcher));
+    ParseTree tree = ParseTree::Parse(tokens, consumer);
+    EXPECT_TRUE(tree.has_errors());
+  }
+}
+
 }  // namespace
 }  // namespace Carbon::Testing

toolchain/parser/parser_impl.cpp

@@ -448,6 +448,87 @@ auto ParseTree::Parser::ParseCodeBlock() -> llvm::Optional<Node> {
   return AddNode(ParseNodeKind::CodeBlock(), open_curly, start, has_errors);
 }
 
+auto ParseTree::Parser::ParsePackageDirective() -> Node {
+  TokenizedBuffer::Token package_intro_token = Consume(TokenKind::Package());
+  auto package_start = GetSubtreeStartPosition();
+  auto create_error_node = [&]() {
+    return AddNode(ParseNodeKind::PackageDirective(), package_intro_token,
+                   package_start,
+                   /*has_error=*/true);
+  };
+
+  CARBON_RETURN_IF_STACK_LIMITED(create_error_node());
+
+  auto exit_on_parse_error = [&]() {
+    SkipPastLikelyEnd(package_intro_token, [&](TokenizedBuffer::Token semi) {
+      return AddLeafNode(ParseNodeKind::PackageEnd(), semi);
+    });
+
+    return create_error_node();
+  };
+
+  if (!NextTokenIs(TokenKind::Identifier())) {
+    CARBON_DIAGNOSTIC(ExpectedIdentifierAfterPackage, Error,
+                      "Expected identifier after `package`.");
+    emitter_.Emit(*position_, ExpectedIdentifierAfterPackage);
+    return exit_on_parse_error();
+  }
+
+  AddLeafNode(ParseNodeKind::DeclaredName(), Consume(TokenKind::Identifier()));
+  bool library_parsed = false;
+
+  if (tokens_.GetKind(*(position_)) == TokenKind::Library()) {
+    auto library_start = GetSubtreeStartPosition();
+    auto library_decl_token = Consume(TokenKind::Library());
+
+    if (tokens_.GetKind(*(position_)) != TokenKind::StringLiteral()) {
+      CARBON_DIAGNOSTIC(
+          ExpectedLibraryName, Error,
+          "Expected a string literal to specify the library name.");
+      emitter_.Emit(*position_, ExpectedLibraryName);
+      return exit_on_parse_error();
+    }
+
+    AddLeafNode(ParseNodeKind::Literal(), Consume(TokenKind::StringLiteral()));
+    AddNode(ParseNodeKind::PackageLibrary(), library_decl_token, library_start,
+            /*has_error=*/false);
+    library_parsed = true;
+  }
+
+  auto api_or_impl_token = tokens_.GetKind(*(position_));
+
+  if (api_or_impl_token == TokenKind::Api()) {
+    AddLeafNode(ParseNodeKind::PackageApi(), Consume(TokenKind::Api()));
+  } else if (api_or_impl_token == TokenKind::Impl()) {
+    AddLeafNode(ParseNodeKind::PackageImpl(), Consume(TokenKind::Impl()));
+  } else if (!library_parsed &&
+             api_or_impl_token == TokenKind::StringLiteral()) {
+    // If we come acroess a string literal and we didn't parse `library "..."`
+    // yet, then most probably the user forgot to add `library` before the
+    // library name.
+    CARBON_DIAGNOSTIC(MissingLibraryKeyword, Error,
+                      "Missing `library` keyword.");
+    emitter_.Emit(*position_, MissingLibraryKeyword);
+    return exit_on_parse_error();
+  } else {
+    CARBON_DIAGNOSTIC(ExpectedApiOrImpl, Error, "Expected a `api` or `impl`.");
+    emitter_.Emit(*position_, ExpectedApiOrImpl);
+    return exit_on_parse_error();
+  }
+
+  if (tokens_.GetKind(*(position_)) != TokenKind::Semi()) {
+    CARBON_DIAGNOSTIC(ExpectedSemiToEndPackageDirective, Error,
+                      "Expected `;` to end package directive.");
+    emitter_.Emit(*position_, ExpectedSemiToEndPackageDirective);
+    return exit_on_parse_error();
+  }
+
+  AddLeafNode(ParseNodeKind::PackageEnd(), Consume(TokenKind::Semi()));
+
+  return AddNode(ParseNodeKind::PackageDirective(), package_intro_token,
+                 package_start, /*has_error=*/false);
+}
+
 auto ParseTree::Parser::ParseFunctionDeclaration() -> Node {
   TokenizedBuffer::Token function_intro_token = Consume(TokenKind::Fn());
   auto start = GetSubtreeStartPosition();
@@ -561,6 +642,8 @@ auto ParseTree::Parser::ParseEmptyDeclaration() -> Node {
 auto ParseTree::Parser::ParseDeclaration() -> llvm::Optional<Node> {
   CARBON_RETURN_IF_STACK_LIMITED(llvm::None);
   switch (NextTokenKind()) {
+    case TokenKind::Package():
+      return ParsePackageDirective();
     case TokenKind::Fn():
       return ParseFunctionDeclaration();
     case TokenKind::Var():
@@ -574,6 +657,7 @@ auto ParseTree::Parser::ParseDeclaration() -> llvm::Optional<Node> {
       break;
   }
 
+  // Should happen for packages now.
   // We didn't recognize an introducer for a valid declaration.
   CARBON_DIAGNOSTIC(UnrecognizedDeclaration, Error,
                     "Unrecognized declaration introducer.");

toolchain/parser/parser_impl.h

@@ -66,12 +66,12 @@ class ParseTree::Parser {
   // tree's preorder sequence.
   auto AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token) -> Node;
 
-  // Composes `consumeIf` and `addLeafNode`, propagating the failure case
+  // Composes `ConsumeIf` and `AddLeafNode`, propagating the failure case
   // through the optional.
   auto ConsumeAndAddLeafNodeIf(TokenKind t_kind, ParseNodeKind n_kind)
       -> llvm::Optional<Node>;
 
-  // Marks the node `N` as having some parse errors and that the tree contains
+  // Marks the node `n` as having some parse errors and that the tree contains
   // a node with a parse error.
   auto MarkNodeError(Node n) -> void;
 
@@ -178,6 +178,9 @@ class ParseTree::Parser {
   // Parses and returns an empty declaration node from a single semicolon token.
   auto ParseEmptyDeclaration() -> Node;
 
+  // Parses a package directive.
+  auto ParsePackageDirective() -> Node;
+
   // Tries to parse a declaration. If a declaration, even an empty one after
   // skipping errors, can be parsed, it is returned. There may be parse errors
   // even when a node is returned.