Add parse support for multiple requirements after `where` separated by `and` (#4298)

Follow on to #4275 that added `where` parse support.

---------

Co-authored-by: Josh L <josh11b@users.noreply.github.com>
Co-authored-by: Richard Smith <richard@metafoo.co.uk>

toolchain/check/handle_where.cpp

@@ -36,6 +36,11 @@ auto HandleParseNode(Context& context, Parse::RequirementImplsId node_id)
   return context.TODO(node_id, "HandleRequirementImpls");
 }
 
+auto HandleParseNode(Context& context, Parse::RequirementAndId node_id)
+    -> bool {
+  return context.TODO(node_id, "HandleRequirementAnd");
+}
+
 auto HandleParseNode(Context& context, Parse::WhereExprId node_id) -> bool {
   return context.TODO(node_id, "HandleWhereExpr");
 }

toolchain/check/node_stack.h

@@ -619,6 +619,7 @@ class NodeStack {
         case Parse::NodeKind::PrivateModifier:
         case Parse::NodeKind::ProtectedModifier:
         case Parse::NodeKind::RealLiteral:
+        case Parse::NodeKind::RequirementAnd:
         case Parse::NodeKind::RequirementEqual:
         case Parse::NodeKind::RequirementEqualEqual:
         case Parse::NodeKind::RequirementImpls:

toolchain/lex/token_kind.h

@@ -94,7 +94,7 @@ class TokenKind : public CARBON_ENUM_BASE(TokenKind) {
 
   // Get the expected number of parse tree nodes that will be created for this
   // token.
-  auto expected_parse_tree_size() const -> int {
+  auto expected_max_parse_tree_size() const -> int {
     return ExpectedParseTreeSize[AsInt()];
   }
 

toolchain/lex/tokenized_buffer.cpp

@@ -365,7 +365,7 @@ auto TokenizedBuffer::GetTokenInfo(TokenIndex token) const -> const TokenInfo& {
 
 auto TokenizedBuffer::AddToken(TokenInfo info) -> TokenIndex {
   token_infos_.push_back(info);
-  expected_parse_tree_size_ += info.kind().expected_parse_tree_size();
+  expected_max_parse_tree_size_ += info.kind().expected_max_parse_tree_size();
   return TokenIndex(static_cast<int>(token_infos_.size()) - 1);
 }
 

toolchain/lex/tokenized_buffer.h

@@ -219,8 +219,10 @@ class TokenizedBuffer : public Printable<TokenizedBuffer> {
 
   auto size() const -> int { return token_infos_.size(); }
 
-  auto expected_parse_tree_size() const -> int {
-    return expected_parse_tree_size_;
+  // This is an upper bound on the number of output parse nodes in the absence
+  // of errors.
+  auto expected_max_parse_tree_size() const -> int {
+    return expected_max_parse_tree_size_;
   }
 
   auto source() const -> const SourceBuffer& { return *source_; }
@@ -456,9 +458,9 @@ class TokenizedBuffer : public Printable<TokenizedBuffer> {
 
   llvm::SmallVector<LineInfo> line_infos_;
 
-  // The number of parse tree nodes that we expect to be created for the tokens
-  // in this buffer.
-  int expected_parse_tree_size_ = 0;
+  // An upper bound on the number of parse tree nodes that we expect to be
+  // created for the tokens in this buffer.
+  int expected_max_parse_tree_size_ = 0;
 
   bool has_errors_ = false;
 

toolchain/parse/handle_requirement.cpp

@@ -10,6 +10,7 @@ namespace Carbon::Parse {
 
 auto HandleRequirementBegin(Context& context) -> void {
   context.PopAndDiscardState();
+  // TODO: Peek ahead for `.designator = ...`, and give it special handling.
   context.PushState(State::RequirementOperator);
   context.PushStateForExpr(PrecedenceGroup::ForRequirements());
 }
@@ -69,7 +70,13 @@ auto HandleRequirementOperatorFinish(Context& context) -> void {
                      << token_kind;
       return;
   }
-  // TODO: Handle `and` token.
+  if (state.has_error) {
+    context.ReturnErrorOnState();
+  }
+  if (auto token = context.ConsumeIf(Lex::TokenKind::And)) {
+    context.AddNode(NodeKind::RequirementAnd, *token, /*has_error=*/false);
+    context.PushState(State::RequirementBegin);
+  }
 }
 
 auto HandleWhereFinish(Context& context) -> void {

toolchain/parse/node_kind.def

@@ -274,8 +274,9 @@ CARBON_PARSE_NODE_KIND(ShortCircuitOperandOr)
 CARBON_PARSE_NODE_KIND(ShortCircuitOperatorAnd)
 CARBON_PARSE_NODE_KIND(ShortCircuitOperatorOr)
 
-CARBON_PARSE_NODE_KIND(SelfTypeName)
 CARBON_PARSE_NODE_KIND(DesignatorExpr)
+CARBON_PARSE_NODE_KIND(SelfTypeName)
+CARBON_PARSE_NODE_KIND(RequirementAnd)
 CARBON_PARSE_NODE_KIND(RequirementEqual)
 CARBON_PARSE_NODE_KIND(RequirementEqualEqual)
 CARBON_PARSE_NODE_KIND(RequirementImpls)

toolchain/parse/state.def

@@ -645,6 +645,13 @@ CARBON_PARSE_STATE(RequirementOperator)
 // expr where expr == expr
 //                        ^
 //   (state done)
+// expr where expr impls expr and
+//                            ^~~
+// expr where expr = expr and
+//                        ^~~
+// expr where expr == expr and
+//                         ^~~
+//   1. RequirementBegin
 CARBON_PARSE_STATE(RequirementOperatorFinish)
 
 // Finishes an `where` expression.

toolchain/parse/testdata/where_expr/fail_todo_where_and.carbon

@@ -1,36 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// AUTOUPDATE
-// TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/parse/testdata/where_expr/fail_todo_where_and.carbon
-// TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/parse/testdata/where_expr/fail_todo_where_and.carbon
-
-// TODO: Support `and` to allow multiple requirements in `where` expressions.
-
-// CHECK:STDERR: fail_todo_where_and.carbon:[[@LINE+3]]:31: ERROR: Expected `,` or `)`.
-// CHECK:STDERR: fn Foo(T: type where .U = i32 and .V == .W and .X impls I);
-// CHECK:STDERR:                               ^~~
-fn Foo(T: type where .U = i32 and .V == .W and .X impls I);
-
-// CHECK:STDOUT: - filename: fail_todo_where_and.carbon
-// CHECK:STDOUT:   parse_tree: [
-// CHECK:STDOUT:     {kind: 'FileStart', text: ''},
-// CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
-// CHECK:STDOUT:       {kind: 'IdentifierName', text: 'Foo'},
-// CHECK:STDOUT:         {kind: 'TuplePatternStart', text: '('},
-// CHECK:STDOUT:           {kind: 'IdentifierName', text: 'T'},
-// CHECK:STDOUT:               {kind: 'TypeTypeLiteral', text: 'type'},
-// CHECK:STDOUT:             {kind: 'WhereOperand', text: 'where', subtree_size: 2},
-// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'U'},
-// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
-// CHECK:STDOUT:               {kind: 'IntTypeLiteral', text: 'i32'},
-// CHECK:STDOUT:             {kind: 'RequirementEqual', text: '=', subtree_size: 4},
-// CHECK:STDOUT:           {kind: 'WhereExpr', text: 'where', subtree_size: 7},
-// CHECK:STDOUT:         {kind: 'BindingPattern', text: ':', subtree_size: 9},
-// CHECK:STDOUT:       {kind: 'TuplePattern', text: ')', has_error: yes, subtree_size: 11},
-// CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 14},
-// CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
-// CHECK:STDOUT:   ]

toolchain/parse/testdata/where_expr/where_and.carbon

@@ -0,0 +1,143 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/parse/testdata/where_expr/where_and.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/parse/testdata/where_expr/where_and.carbon
+
+// --- success.carbon
+
+fn OneAnd(T:! type where .U = i32 and .V == .W);
+
+fn TwoAnd(Y:! I where .J impls K and .L == .M and .N = bool);
+
+// --- fail_and_prefix.carbon
+
+// CHECK:STDERR: fail_and_prefix.carbon:[[@LINE+4]]:29: ERROR: Expected expression.
+// CHECK:STDERR: fn AndPrefix(T:! type where and .V == .W);
+// CHECK:STDERR:                             ^~~
+// CHECK:STDERR:
+fn AndPrefix(T:! type where and .V == .W);
+
+// --- fail_and_suffix.carbon
+
+// CHECK:STDERR: fail_and_suffix.carbon:[[@LINE+4]]:40: ERROR: Expected expression.
+// CHECK:STDERR: fn AndSuffix(Y:! I where .J impls K and);
+// CHECK:STDERR:                                        ^
+// CHECK:STDERR:
+fn AndSuffix(Y:! I where .J impls K and);
+
+// --- fail_and_early.carbon
+
+// CHECK:STDERR: fail_and_early.carbon:[[@LINE+3]]:28: ERROR: Requirement should use `impls`, `=`, or `==` operator.
+// CHECK:STDERR: fn AndEarly(Z:! L where .M and N);
+// CHECK:STDERR:                            ^~~
+fn AndEarly(Z:! L where .M and N);
+
+// CHECK:STDOUT: - filename: success.carbon
+// CHECK:STDOUT:   parse_tree: [
+// CHECK:STDOUT:     {kind: 'FileStart', text: ''},
+// CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:       {kind: 'IdentifierName', text: 'OneAnd'},
+// CHECK:STDOUT:         {kind: 'TuplePatternStart', text: '('},
+// CHECK:STDOUT:           {kind: 'IdentifierName', text: 'T'},
+// CHECK:STDOUT:               {kind: 'TypeTypeLiteral', text: 'type'},
+// CHECK:STDOUT:             {kind: 'WhereOperand', text: 'where', subtree_size: 2},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'U'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:               {kind: 'IntTypeLiteral', text: 'i32'},
+// CHECK:STDOUT:             {kind: 'RequirementEqual', text: '=', subtree_size: 4},
+// CHECK:STDOUT:             {kind: 'RequirementAnd', text: 'and'},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'V'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'W'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:             {kind: 'RequirementEqualEqual', text: '==', subtree_size: 5},
+// CHECK:STDOUT:           {kind: 'WhereExpr', text: 'where', subtree_size: 13},
+// CHECK:STDOUT:         {kind: 'CompileTimeBindingPattern', text: ':!', subtree_size: 15},
+// CHECK:STDOUT:       {kind: 'TuplePattern', text: ')', subtree_size: 17},
+// CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 20},
+// CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:       {kind: 'IdentifierName', text: 'TwoAnd'},
+// CHECK:STDOUT:         {kind: 'TuplePatternStart', text: '('},
+// CHECK:STDOUT:           {kind: 'IdentifierName', text: 'Y'},
+// CHECK:STDOUT:               {kind: 'IdentifierNameExpr', text: 'I'},
+// CHECK:STDOUT:             {kind: 'WhereOperand', text: 'where', subtree_size: 2},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'J'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:               {kind: 'IdentifierNameExpr', text: 'K'},
+// CHECK:STDOUT:             {kind: 'RequirementImpls', text: 'impls', subtree_size: 4},
+// CHECK:STDOUT:             {kind: 'RequirementAnd', text: 'and'},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'L'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'M'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:             {kind: 'RequirementEqualEqual', text: '==', subtree_size: 5},
+// CHECK:STDOUT:             {kind: 'RequirementAnd', text: 'and'},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'N'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:               {kind: 'BoolTypeLiteral', text: 'bool'},
+// CHECK:STDOUT:             {kind: 'RequirementEqual', text: '=', subtree_size: 4},
+// CHECK:STDOUT:           {kind: 'WhereExpr', text: 'where', subtree_size: 18},
+// CHECK:STDOUT:         {kind: 'CompileTimeBindingPattern', text: ':!', subtree_size: 20},
+// CHECK:STDOUT:       {kind: 'TuplePattern', text: ')', subtree_size: 22},
+// CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 25},
+// CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT:   ]
+// CHECK:STDOUT: - filename: fail_and_prefix.carbon
+// CHECK:STDOUT:   parse_tree: [
+// CHECK:STDOUT:     {kind: 'FileStart', text: ''},
+// CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:       {kind: 'IdentifierName', text: 'AndPrefix'},
+// CHECK:STDOUT:         {kind: 'TuplePatternStart', text: '('},
+// CHECK:STDOUT:           {kind: 'IdentifierName', text: 'T'},
+// CHECK:STDOUT:               {kind: 'TypeTypeLiteral', text: 'type'},
+// CHECK:STDOUT:             {kind: 'WhereOperand', text: 'where', subtree_size: 2},
+// CHECK:STDOUT:             {kind: 'InvalidParse', text: 'and', has_error: yes},
+// CHECK:STDOUT:           {kind: 'WhereExpr', text: 'where', has_error: yes, subtree_size: 4},
+// CHECK:STDOUT:         {kind: 'CompileTimeBindingPattern', text: ':!', has_error: yes, subtree_size: 6},
+// CHECK:STDOUT:       {kind: 'TuplePattern', text: ')', has_error: yes, subtree_size: 8},
+// CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 11},
+// CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT:   ]
+// CHECK:STDOUT: - filename: fail_and_suffix.carbon
+// CHECK:STDOUT:   parse_tree: [
+// CHECK:STDOUT:     {kind: 'FileStart', text: ''},
+// CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:       {kind: 'IdentifierName', text: 'AndSuffix'},
+// CHECK:STDOUT:         {kind: 'TuplePatternStart', text: '('},
+// CHECK:STDOUT:           {kind: 'IdentifierName', text: 'Y'},
+// CHECK:STDOUT:               {kind: 'IdentifierNameExpr', text: 'I'},
+// CHECK:STDOUT:             {kind: 'WhereOperand', text: 'where', subtree_size: 2},
+// CHECK:STDOUT:                 {kind: 'IdentifierName', text: 'J'},
+// CHECK:STDOUT:               {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:               {kind: 'IdentifierNameExpr', text: 'K'},
+// CHECK:STDOUT:             {kind: 'RequirementImpls', text: 'impls', subtree_size: 4},
+// CHECK:STDOUT:             {kind: 'RequirementAnd', text: 'and'},
+// CHECK:STDOUT:             {kind: 'InvalidParse', text: ')', has_error: yes},
+// CHECK:STDOUT:           {kind: 'WhereExpr', text: 'where', has_error: yes, subtree_size: 9},
+// CHECK:STDOUT:         {kind: 'CompileTimeBindingPattern', text: ':!', has_error: yes, subtree_size: 11},
+// CHECK:STDOUT:       {kind: 'TuplePattern', text: ')', has_error: yes, subtree_size: 13},
+// CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 16},
+// CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT:   ]
+// CHECK:STDOUT: - filename: fail_and_early.carbon
+// CHECK:STDOUT:   parse_tree: [
+// CHECK:STDOUT:     {kind: 'FileStart', text: ''},
+// CHECK:STDOUT:       {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:       {kind: 'IdentifierName', text: 'AndEarly'},
+// CHECK:STDOUT:         {kind: 'TuplePatternStart', text: '('},
+// CHECK:STDOUT:           {kind: 'IdentifierName', text: 'Z'},
+// CHECK:STDOUT:               {kind: 'IdentifierNameExpr', text: 'L'},
+// CHECK:STDOUT:             {kind: 'WhereOperand', text: 'where', subtree_size: 2},
+// CHECK:STDOUT:               {kind: 'IdentifierName', text: 'M'},
+// CHECK:STDOUT:             {kind: 'DesignatorExpr', text: '.', subtree_size: 2},
+// CHECK:STDOUT:           {kind: 'WhereExpr', text: 'where', has_error: yes, subtree_size: 5},
+// CHECK:STDOUT:         {kind: 'CompileTimeBindingPattern', text: ':!', has_error: yes, subtree_size: 7},
+// CHECK:STDOUT:       {kind: 'TuplePattern', text: ')', has_error: yes, subtree_size: 9},
+// CHECK:STDOUT:     {kind: 'FunctionDecl', text: ';', subtree_size: 12},
+// CHECK:STDOUT:     {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT:   ]

toolchain/parse/tree.cpp

@@ -45,12 +45,20 @@ auto Tree::Verify() const -> ErrorOr<Success> {
     }
   }
 
-  if (!has_errors() &&
-      static_cast<int32_t>(size()) != tokens_->expected_parse_tree_size()) {
+  // Not every token that can produce a virtual node will, so we only check that
+  // the number of nodes is in a range.
+  int32_t num_nodes = size();
+  if (!has_errors() && num_nodes > tokens_->expected_max_parse_tree_size()) {
     return Error(llvm::formatv(
         "Tree has {0} nodes and no errors, but "
-        "Lex::TokenizedBuffer expected {1} nodes for {2} tokens.",
-        size(), tokens_->expected_parse_tree_size(), tokens_->size()));
+        "Lex::TokenizedBuffer expected up to {1} nodes for {2} tokens.",
+        num_nodes, tokens_->expected_max_parse_tree_size(), tokens_->size()));
+  }
+  if (!has_errors() && num_nodes < tokens_->size()) {
+    return Error(
+        llvm::formatv("Tree has {0} nodes and no errors, but expected at least "
+                      "{1} nodes to match the number of tokens.",
+                      num_nodes, tokens_->size()));
   }
 
 #ifndef NDEBUG

toolchain/parse/tree.h

@@ -100,7 +100,7 @@ class Tree : public Printable<Tree> {
   // be used to actually parse the tokens into a tree.
   explicit Tree(Lex::TokenizedBuffer& tokens_arg) : tokens_(&tokens_arg) {
     // If the tree is valid, there will be one node per token, so reserve once.
-    node_impls_.reserve(tokens_->expected_parse_tree_size());
+    node_impls_.reserve(tokens_->expected_max_parse_tree_size());
   }
 
   auto has_errors() const -> bool { return has_errors_; }

toolchain/parse/tree_and_subtrees.cpp

@@ -29,8 +29,9 @@ TreeAndSubtrees::TreeAndSubtrees(const Lex::TokenizedBuffer& tokens,
         size += subtree_sizes_[child.index];
         if (kind.has_bracket() && i == kind.child_count() - 1) {
           CARBON_CHECK(kind.bracket() == tree.node_kind(child))
-              << "Node " << kind << " needs bracket " << kind.bracket()
-              << ", found wrong bracket " << tree.node_kind(child);
+              << "Node " << kind << " with child count " << kind.child_count()
+              << " needs bracket " << kind.bracket() << ", found wrong bracket "
+              << tree.node_kind(child);
         }
       }
     } else {

toolchain/parse/typed_nodes.h

@@ -1059,6 +1059,9 @@ struct RequirementImpls {
   AnyExprId rhs;
 };
 
+// An `and` token separating requirements in a `where` expression.
+using RequirementAnd = LeafNode<NodeKind::RequirementAnd, Lex::AndTokenIndex>;
+
 struct WhereOperand {
   static constexpr auto Kind =
       NodeKind::WhereOperand.Define({.child_count = 1});
@@ -1069,13 +1072,11 @@ struct WhereOperand {
 };
 
 struct WhereExpr {
-  static constexpr auto Kind =
-      NodeKind::WhereExpr.Define({.category = NodeCategory::Expr,
-                                  .bracketed_by = WhereOperand::Kind,
-                                  .child_count = 2});
+  static constexpr auto Kind = NodeKind::WhereExpr.Define(
+      {.category = NodeCategory::Expr, .bracketed_by = WhereOperand::Kind});
   WhereOperandId introducer;
   Lex::WhereTokenIndex token;
-  AnyRequirementId requirements;
+  CommaSeparatedList<AnyRequirementId, RequirementAndId> requirements;
 };
 
 // Choice nodes