Reformat lexer/parser tokens. (#772)

Related to discussion on #738, but also trying to standardize handling of everything and make placement of token spellings (`AND "and"`) consistently in lexer.lpp.

I could have gone the other direction, removing the list of things in lexer.lpp, but this feels like it does more to use the compiler to detect skew between lexer.lpp and parser.ypp.

executable_semantics/syntax/lexer.lpp

@@ -23,51 +23,67 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 /* Lexing a token immediately after consuming some whitespace. */
 %s AFTER_WHITESPACE
-/* Lexing a token immediately after consuming an operand-ending token:
+/*
+ * Lexing a token immediately after consuming an operand-ending token:
  * a closing bracket, identifier, or literal.
  */
 %s AFTER_OPERAND
 
-AND               "and"
-ARROW             "->"
-AUTO              "auto"
-BOOL              "Bool"
-BREAK             "break"
-CASE              "case"
-CHOICE            "choice"
-ONE_LINE_COMMENT  \/\/[^\n]*\n
-CONTINUE          "continue"
-DBLARROW          "=>"
-DEFAULT           "default"
-ELSE              "else"
-EQUAL_EQUAL       "=="
-FALSE             "false"
-FN                "fn"
-FNTY              "fnty"
-IF                "if"
-MATCH             "match"
-NOT               "not"
-OR                "or"
-RETURN            "return"
-STRING            "String"
-CLASS             "class"
-TRUE              "true"
-TYPE              "Type"
-VAR               "var"
-WHILE             "while"
-CONTINUATION_TYPE "__Continuation"
-CONTINUATION      "__continuation"
-RUN               "__run"
-AWAIT             "__await"
-UNDERSCORE        "_"
-
-identifier    [A-Za-z_][A-Za-z0-9_]*
-sized_type_literal [iuf][1-9][0-9]*
-integer_literal   [0-9]+
-string_literal    \"([^\\\"\n\t]|\\.)*\"
+AND                  "and"
+ARROW                "->"
+AUTO                 "auto"
+AWAIT                "__await"
+BOOL                 "Bool"
+BREAK                "break"
+CASE                 "case"
+CHOICE               "choice"
+CLASS                "class"
+COLON                ":"
+COLON_BANG           ":!"
+COMMA                ","
+CONTINUATION         "__continuation"
+CONTINUATION_TYPE    "__Continuation"
+CONTINUE             "continue"
+DEFAULT              "default"
+DOUBLE_ARROW         "=>"
+ELSE                 "else"
+EQUAL                "="
+EQUAL_EQUAL          "=="
+FALSE                "false"
+FN                   "fn"
+FNTY                 "fnty"
+IF                   "if"
+LEFT_CURLY_BRACE     "{"
+LEFT_PARENTHESIS     "("
+LEFT_SQUARE_BRACKET  "["
+MATCH                "match"
+MINUS                "-"
+NOT                  "not"
+OR                   "or"
+PERIOD               "."
+PLUS                 "+"
+RETURN               "return"
+RIGHT_CURLY_BRACE    "}"
+RIGHT_PARENTHESIS    ")"
+RIGHT_SQUARE_BRACKET "]"
+RUN                  "__run"
+SEMICOLON            ";"
+SLASH                "/"
+STRING               "String"
+TRUE                 "true"
+TYPE                 "Type"
+UNDERSCORE           "_"
+VAR                  "var"
+WHILE                "while"
+
+identifier            [A-Za-z_][A-Za-z0-9_]*
+sized_type_literal    [iuf][1-9][0-9]*
+integer_literal       [0-9]+
+string_literal        \"([^\\\"\n\t]|\\.)*\"
 horizontal_whitespace [ \t\r]
-whitespace [ \t\r\n]
-operand_start [(A-Za-z0-9_"]
+whitespace            [ \t\r\n]
+one_line_comment      \/\/[^\n]*\n
+operand_start         [(A-Za-z0-9_"]
 
 %{
   // This macro is expanded immediately before each action specified below.
@@ -81,6 +97,12 @@ operand_start [(A-Za-z0-9_"]
           YY_START == AFTER_OPERAND) { \
         BEGIN(INITIAL); \
       }
+
+  #define SIMPLE_TOKEN(name) \
+      Carbon::Parser::make_##name(context.current_token_position);
+
+  #define ARG_TOKEN(name, arg) \
+      Carbon::Parser::make_##name(arg, context.current_token_position);
 %}
 
 %%
@@ -92,104 +114,124 @@ operand_start [(A-Za-z0-9_"]
   context.current_token_position.step();
 %}
 
-{AND}      { return Carbon::Parser::make_AND(context.current_token_position); }
-{ARROW}    { return Carbon::Parser::make_ARROW(context.current_token_position); }
-{AUTO}     { return Carbon::Parser::make_AUTO(context.current_token_position); }
-{BOOL}     { return Carbon::Parser::make_BOOL(context.current_token_position); }
-{BREAK}    { return Carbon::Parser::make_BREAK(context.current_token_position); }
-{CASE}     { return Carbon::Parser::make_CASE(context.current_token_position); }
-{CHOICE}   { return Carbon::Parser::make_CHOICE(context.current_token_position); }
-{CLASS}    { return Carbon::Parser::make_CLASS(context.current_token_position); }
-{CONTINUE} { return Carbon::Parser::make_CONTINUE(context.current_token_position); }
-{DBLARROW} { return Carbon::Parser::make_DBLARROW(context.current_token_position); }
-{DEFAULT}  { return Carbon::Parser::make_DEFAULT(context.current_token_position); }
-{ELSE}     { return Carbon::Parser::make_ELSE(context.current_token_position); }
-"=="       { return Carbon::Parser::make_EQUAL_EQUAL(context.current_token_position); }
-{FALSE}    { return Carbon::Parser::make_FALSE(context.current_token_position); }
-{FN}       { return Carbon::Parser::make_FN(context.current_token_position); }
-{FNTY}     { return Carbon::Parser::make_FNTY(context.current_token_position); }
-{IF}       { return Carbon::Parser::make_IF(context.current_token_position); }
-{MATCH}    { return Carbon::Parser::make_MATCH(context.current_token_position); }
-{NOT}      { return Carbon::Parser::make_NOT(context.current_token_position); }
-{OR}       { return Carbon::Parser::make_OR(context.current_token_position); }
-{RETURN}   { return Carbon::Parser::make_RETURN(context.current_token_position); }
-{TRUE}     { return Carbon::Parser::make_TRUE(context.current_token_position); }
-{TYPE}     { return Carbon::Parser::make_TYPE(context.current_token_position); }
-{VAR}      { return Carbon::Parser::make_VAR(context.current_token_position); }
-{WHILE}    { return Carbon::Parser::make_WHILE(context.current_token_position); }
-{CONTINUATION_TYPE} { return Carbon::Parser::make_CONTINUATION_TYPE(context.current_token_position); }
-{CONTINUATION}    { return Carbon::Parser::make_CONTINUATION(context.current_token_position); }
-{RUN}      { return Carbon::Parser::make_RUN(context.current_token_position); }
-{AWAIT}    { return Carbon::Parser::make_AWAIT(context.current_token_position); }
-{UNDERSCORE}      { return Carbon::Parser::make_UNDERSCORE(context.current_token_position); }
-{STRING}   { return Carbon::Parser::make_STRING(context.current_token_position); }
-
-{sized_type_literal} { return Carbon::Parser::make_sized_type_literal(yytext, context.current_token_position); }
-
-"=" return Carbon::Parser::make_EQUAL(context.current_token_position);
-"-" return Carbon::Parser::make_MINUS(context.current_token_position);
-"+" return Carbon::Parser::make_PLUS(context.current_token_position);
-"/" return Carbon::Parser::make_SLASH(context.current_token_position);
-"(" return Carbon::Parser::make_LEFT_PARENTHESIS(context.current_token_position);
-")" { BEGIN(AFTER_OPERAND); return Carbon::Parser::make_RIGHT_PARENTHESIS(context.current_token_position); }
-"{" return Carbon::Parser::make_LEFT_CURLY_BRACE(context.current_token_position);
-"}" { BEGIN(AFTER_OPERAND); return Carbon::Parser::make_RIGHT_CURLY_BRACE(context.current_token_position); }
-"[" return Carbon::Parser::make_LEFT_SQUARE_BRACKET(context.current_token_position);
-"]" { BEGIN(AFTER_OPERAND); return Carbon::Parser::make_RIGHT_SQUARE_BRACKET(context.current_token_position); }
-"." return Carbon::Parser::make_PERIOD(context.current_token_position);
-"," return Carbon::Parser::make_COMMA(context.current_token_position);
-";" return Carbon::Parser::make_SEMICOLON(context.current_token_position);
-":!" return Carbon::Parser::make_COLON_BANG(context.current_token_position);
-":" return Carbon::Parser::make_COLON(context.current_token_position);
+{AND}                 { return SIMPLE_TOKEN(AND); }
+{ARROW}               { return SIMPLE_TOKEN(ARROW); }
+{AUTO}                { return SIMPLE_TOKEN(AUTO); }
+{AWAIT}               { return SIMPLE_TOKEN(AWAIT); }
+{BOOL}                { return SIMPLE_TOKEN(BOOL); }
+{BREAK}               { return SIMPLE_TOKEN(BREAK); }
+{CASE}                { return SIMPLE_TOKEN(CASE); }
+{CHOICE}              { return SIMPLE_TOKEN(CHOICE); }
+{CLASS}               { return SIMPLE_TOKEN(CLASS); }
+{COLON_BANG}          { return SIMPLE_TOKEN(COLON_BANG); }
+{COLON}               { return SIMPLE_TOKEN(COLON); }
+{COMMA}               { return SIMPLE_TOKEN(COMMA); }
+{CONTINUATION_TYPE}   { return SIMPLE_TOKEN(CONTINUATION_TYPE); }
+{CONTINUATION}        { return SIMPLE_TOKEN(CONTINUATION); }
+{CONTINUE}            { return SIMPLE_TOKEN(CONTINUE); }
+{DEFAULT}             { return SIMPLE_TOKEN(DEFAULT); }
+{DOUBLE_ARROW}        { return SIMPLE_TOKEN(DOUBLE_ARROW); }
+{ELSE}                { return SIMPLE_TOKEN(ELSE); }
+{EQUAL_EQUAL}         { return SIMPLE_TOKEN(EQUAL_EQUAL); }
+{EQUAL}               { return SIMPLE_TOKEN(EQUAL); }
+{FALSE}               { return SIMPLE_TOKEN(FALSE); }
+{FNTY}                { return SIMPLE_TOKEN(FNTY); }
+{FN}                  { return SIMPLE_TOKEN(FN); }
+{IF}                  { return SIMPLE_TOKEN(IF); }
+{LEFT_PARENTHESIS}    { return SIMPLE_TOKEN(LEFT_PARENTHESIS); }
+{LEFT_CURLY_BRACE}    { return SIMPLE_TOKEN(LEFT_CURLY_BRACE); }
+{LEFT_SQUARE_BRACKET} { return SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); }
+{MATCH}               { return SIMPLE_TOKEN(MATCH); }
+{MINUS}               { return SIMPLE_TOKEN(MINUS); }
+{NOT}                 { return SIMPLE_TOKEN(NOT); }
+{OR}                  { return SIMPLE_TOKEN(OR); }
+{PERIOD}              { return SIMPLE_TOKEN(PERIOD); }
+{PLUS}                { return SIMPLE_TOKEN(PLUS); }
+{RETURN}              { return SIMPLE_TOKEN(RETURN); }
+{RUN}                 { return SIMPLE_TOKEN(RUN); }
+{SEMICOLON}           { return SIMPLE_TOKEN(SEMICOLON); }
+{SLASH}               { return SIMPLE_TOKEN(SLASH); }
+{STRING}              { return SIMPLE_TOKEN(STRING); }
+{TRUE}                { return SIMPLE_TOKEN(TRUE); }
+{TYPE}                { return SIMPLE_TOKEN(TYPE); }
+{UNDERSCORE}          { return SIMPLE_TOKEN(UNDERSCORE); }
+{VAR}                 { return SIMPLE_TOKEN(VAR); }
+{WHILE}               { return SIMPLE_TOKEN(WHILE); }
+
+ /* More modern Bisons provide make_EOF. */
+<<EOF>>               { return SIMPLE_TOKEN(END_OF_FILE); }
+
+{RIGHT_PARENTHESIS} {
+  BEGIN(AFTER_OPERAND);
+  return SIMPLE_TOKEN(RIGHT_PARENTHESIS);
+}
+{RIGHT_CURLY_BRACE} {
+  BEGIN(AFTER_OPERAND);
+  return SIMPLE_TOKEN(RIGHT_CURLY_BRACE);
+}
+{RIGHT_SQUARE_BRACKET} {
+  BEGIN(AFTER_OPERAND);
+  return SIMPLE_TOKEN(RIGHT_SQUARE_BRACKET);
+}
 
  /*
-For a `*` operator, we look at whitespace and local context to determine the
-arity and fixity. There are two ways to write a binary operator:
-
- 1) Whitespace on both sides.
- 2) Whitespace on neither side, and the previous token is considered to be
-    the end of an operand, and the next token is considered to be the start
-    of an operand.
-
-Otherwise, the operator is unary, but we also check for whitespace to help
-the parser enforce the rule that whitespace is not permitted between the
-operator and its operand, leading to three more cases:
-
- 3) Whitespace before (but implicitly not after, because that would give a
-    longer match and hit case 1): this can only be a prefix operator.
- 4) Whitespace after and not before: this can only be a postfix operator.
- 5) No whitespace on either side (otherwise the longest match would take us
-    to case 4): this is a unary operator and could be either prefix or
-    postfix.
-*/
-<AFTER_WHITESPACE>"*"{whitespace}+ /*case 1*/ {
+  * For a `*` operator, we look at whitespace and local context to determine the
+  * arity and fixity. There are two ways to write a binary operator:
+  *
+  * 1) Whitespace on both sides.
+  * 2) Whitespace on neither side, and the previous token is considered to be
+  *    the end of an operand, and the next token is considered to be the start
+  *    of an operand.
+  *
+  * Otherwise, the operator is unary, but we also check for whitespace to help
+  * the parser enforce the rule that whitespace is not permitted between the
+  * operator and its operand, leading to three more cases:
+  *
+  * 3) Whitespace before (but implicitly not after, because that would give a
+  *    longer match and hit case 1): this can only be a prefix operator.
+  * 4) Whitespace after and not before: this can only be a postfix operator.
+  * 5) No whitespace on either side (otherwise the longest match would take us
+  *    to case 4): this is a unary operator and could be either prefix or
+  *    postfix.
+  */
+
+ /* `*` operator case 1: */
+<AFTER_WHITESPACE>"*"{whitespace}+ {
   BEGIN(AFTER_WHITESPACE);
-  return Carbon::Parser::make_BINARY_STAR(context.current_token_position);
+  return SIMPLE_TOKEN(BINARY_STAR);
 }
-<AFTER_OPERAND>"*"/{operand_start} /*case 2*/ {
-  return Carbon::Parser::make_BINARY_STAR(context.current_token_position);
+ /* `*` operator case 2: */
+<AFTER_OPERAND>"*"/{operand_start} {
+  return SIMPLE_TOKEN(BINARY_STAR);
 }
-<AFTER_WHITESPACE>"*" /*case 3*/ {
-  return Carbon::Parser::make_PREFIX_STAR(context.current_token_position);
+ /* `*` operator case 3: */
+<AFTER_WHITESPACE>"*" {
+  return SIMPLE_TOKEN(PREFIX_STAR);
 }
-<INITIAL,AFTER_OPERAND>"*"{whitespace}+ /*case 4*/ {
+ /* `*` operator case 4: */
+<INITIAL,AFTER_OPERAND>"*"{whitespace}+ {
   BEGIN(AFTER_WHITESPACE);
-  return Carbon::Parser::make_POSTFIX_STAR(context.current_token_position);
+  return SIMPLE_TOKEN(POSTFIX_STAR);
 }
-<INITIAL,AFTER_OPERAND>"*" /*case 5*/ {
-  return Carbon::Parser::make_UNARY_STAR(context.current_token_position);
+ /* `*` operator case 5: */
+<INITIAL,AFTER_OPERAND>"*" {
+  return SIMPLE_TOKEN(UNARY_STAR);
+}
+
+{sized_type_literal} {
+  return ARG_TOKEN(sized_type_literal, yytext);
 }
 
 {identifier} {
   BEGIN(AFTER_OPERAND);
-  return Carbon::Parser::make_identifier(yytext, context.current_token_position);
+  return ARG_TOKEN(identifier, yytext);
 }
 
 {integer_literal} {
   BEGIN(AFTER_OPERAND);
   int val;
   CHECK(llvm::to_integer(yytext, val));
-  return Carbon::Parser::make_integer_literal(val, context.current_token_position);
+  return ARG_TOKEN(integer_literal, val);
 }
 
 {string_literal} {
@@ -205,11 +247,10 @@ operator and its operand, leading to three more cases:
     FATAL_COMPILATION_ERROR(context.SourceLoc())
         << "Invalid escaping in string: " << yytext;
   }
-  return Carbon::Parser::make_string_literal(
-      *unescaped, context.current_token_position);
+  return ARG_TOKEN(string_literal, *unescaped);
 }
 
-{ONE_LINE_COMMENT} {
+{one_line_comment} {
   // Advance end by 1 line, resetting the column to zero.
   context.current_token_position.lines(1);
   // Make the span empty by setting start to end.
@@ -240,9 +281,4 @@ operator and its operand, leading to three more cases:
             << llvm::toHex(llvm::StringRef(yytext, 1)) << "' in source file.";
 }
 
-<<EOF>>    {
-  // A more modern Bison would give us make_EOF.
-  return Carbon::Parser::make_END_OF_FILE(context.current_token_position);
-}
-
 %%

executable_semantics/syntax/parser.ypp

@@ -133,43 +133,59 @@ void Carbon::Parser::error(const location_type&, const std::string& message) {
 %type <std::list<std::pair<std::string, Ptr<const Expression>>>> alternative_list
 %type <std::pair<Ptr<const Pattern>, Ptr<const Statement>>*> clause
 %type <std::list<std::pair<Ptr<const Pattern>, Ptr<const Statement>>>*> clause_list
-%token END_OF_FILE 0
-%token AND
-%token OR
-%token NOT
-%token STRING
-%token BOOL
-%token TYPE
-%token FN
-%token FNTY
-%token ARROW "->"
-%token FNARROW "-> in return type"
-%token VAR
-%token EQUAL_EQUAL
-%token IF
-%token ELSE
-%token WHILE
-%token CONTINUATION_TYPE
-%token CONTINUATION
-%token RUN
-%token AWAIT
-%token BREAK
-%token CONTINUE
-%token RETURN
-%token TRUE
-%token FALSE
-%token CLASS
-%token CHOICE
-%token MATCH
-%token CASE
-%token DBLARROW "=>"
-%token DEFAULT
-%token AUTO
-%token UNDERSCORE
+
 %token
-  EQUAL  "="
-  MINUS  "-"
-  PLUS   "+"
+  // Most tokens have their spelling defined in lexer.lpp.
+  AND
+  ARROW
+  AUTO
+  AWAIT
+  BOOL
+  BREAK
+  CASE
+  CHOICE
+  CLASS
+  COLON
+  COLON_BANG
+  COMMA
+  CONTINUATION
+  CONTINUATION_TYPE
+  CONTINUE
+  DEFAULT
+  DOUBLE_ARROW
+  ELSE
+  EQUAL
+  EQUAL_EQUAL
+  FALSE
+  FN
+  FNTY
+  IF
+  LEFT_CURLY_BRACE
+  LEFT_PARENTHESIS
+  LEFT_SQUARE_BRACKET
+  MATCH
+  MINUS
+  NOT
+  OR
+  PERIOD
+  PLUS
+  RETURN
+  RIGHT_CURLY_BRACE
+  RIGHT_PARENTHESIS
+  RIGHT_SQUARE_BRACKET
+  RUN
+  SEMICOLON
+  SLASH
+  STRING
+  TRUE
+  TYPE
+  UNDERSCORE
+  VAR
+  WHILE
+  // Used to track EOF.
+  END_OF_FILE 0
+  // Only used for precedence.
+  FNARROW "-> in return type"
   // The lexer determines the arity and fixity of each `*` based on whitespace
   // and adjacent tokens. UNARY_STAR indicates that the operator is unary but
   // could be either prefix or postfix.
@@ -177,26 +193,14 @@ void Carbon::Parser::error(const location_type&, const std::string& message) {
   PREFIX_STAR "prefix *"
   POSTFIX_STAR "postfix *"
   BINARY_STAR "binary *"
-  SLASH  "/"
-  LEFT_PARENTHESIS "("
-  RIGHT_PARENTHESIS ")"
-  LEFT_CURLY_BRACE "{"
-  RIGHT_CURLY_BRACE "}"
-  LEFT_SQUARE_BRACKET "["
-  RIGHT_SQUARE_BRACKET "]"
-  PERIOD "."
-  COMMA ","
-  SEMICOLON ";"
-  COLON_BANG ":!"
-  COLON ":"
 ;
 
 %precedence FNARROW
-%precedence "{" "}"
-%precedence ":!" ":" "," DBLARROW
+%precedence LEFT_CURLY_BRACE RIGHT_CURLY_BRACE
+%precedence COLON_BANG COLON COMMA DOUBLE_ARROW
 %left OR AND
 %nonassoc EQUAL_EQUAL
-%left "+" "-"
+%left PLUS MINUS
 %left BINARY_STAR
 %precedence NOT UNARY_MINUS PREFIX_STAR
 // We need to give the `UNARY_STAR` token a precedence, rather than overriding
@@ -207,8 +211,13 @@ void Carbon::Parser::error(const location_type&, const std::string& message) {
 // is the final token of a rule, it must be a postfix usage, so we give it the
 // same precedence as POSTFIX_STAR.
 %precedence POSTFIX_STAR UNARY_STAR
-%left "." ARROW
-%precedence "(" ")" "[" "]"
+%left PERIOD ARROW
+%precedence
+  LEFT_PARENTHESIS
+  RIGHT_PARENTHESIS
+  LEFT_SQUARE_BRACKET
+  RIGHT_SQUARE_BRACKET
+;
 
 %start input
 %%
@@ -220,7 +229,7 @@ expression:
     { $$ = global_arena->New<IdentifierExpression>(context.SourceLoc(), $1); }
 | expression designator
     { $$ = global_arena->New<FieldAccessExpression>(context.SourceLoc(), $1, $2); }
-| expression "[" expression "]"
+| expression LEFT_SQUARE_BRACKET expression RIGHT_SQUARE_BRACKET
     { $$ = global_arena->New<IndexExpression>(context.SourceLoc(), $1, $3); }
 | integer_literal
     { $$ = global_arena->New<IntLiteral>(context.SourceLoc(), $1); }
@@ -249,10 +258,10 @@ expression:
 | expression EQUAL_EQUAL expression
     { $$ = global_arena->New<PrimitiveOperatorExpression>(
         context.SourceLoc(), Operator::Eq, std::vector<Ptr<const Expression>>({$1, $3})); }
-| expression "+" expression
+| expression PLUS expression
     { $$ = global_arena->New<PrimitiveOperatorExpression>(
         context.SourceLoc(), Operator::Add, std::vector<Ptr<const Expression>>({$1, $3})); }
-| expression "-" expression
+| expression MINUS expression
     { $$ = global_arena->New<PrimitiveOperatorExpression>(
         context.SourceLoc(), Operator::Sub, std::vector<Ptr<const Expression>>({$1, $3})); }
 | expression BINARY_STAR expression
@@ -267,7 +276,7 @@ expression:
 | NOT expression
     { $$ = global_arena->New<PrimitiveOperatorExpression>(
         context.SourceLoc(), Operator::Not, std::vector<Ptr<const Expression>>({$2})); }
-| "-" expression %prec UNARY_MINUS
+| MINUS expression %prec UNARY_MINUS
     { $$ = global_arena->New<PrimitiveOperatorExpression>(
         context.SourceLoc(), Operator::Neg, std::vector<Ptr<const Expression>>({$2})); }
 | PREFIX_STAR expression
@@ -290,7 +299,7 @@ expression:
       $$ = global_arena->New<FunctionTypeLiteral>(
         context.SourceLoc(), $2, return_exp, is_omitted_exp); }
 ;
-designator: "." identifier { $$ = $2; }
+designator: PERIOD identifier { $$ = $2; }
 ;
 paren_expression: paren_expression_base
     { $$ = ExpressionFromParenContents(context.SourceLoc(), $1); }
@@ -301,15 +310,15 @@ tuple: paren_expression_base
 paren_expression_element:
   expression
     { $$ = {.name = std::nullopt, .term = $1}; }
-| designator "=" expression
+| designator EQUAL expression
     { $$ = {.name = $1, .term = $3}; }
 ;
 paren_expression_base:
-  "(" ")"
+  LEFT_PARENTHESIS RIGHT_PARENTHESIS
     { $$ = {.elements = {}, .has_trailing_comma = false}; }
-| "(" paren_expression_contents ")"
+| LEFT_PARENTHESIS paren_expression_contents RIGHT_PARENTHESIS
     { $$ = $2; }
-| "(" paren_expression_contents "," ")"
+| LEFT_PARENTHESIS paren_expression_contents COMMA RIGHT_PARENTHESIS
     {
       $$ = $2;
       $$.has_trailing_comma = true;
@@ -318,7 +327,7 @@ paren_expression_base:
 paren_expression_contents:
   paren_expression_element
     { $$ = {.elements = {$1}, .has_trailing_comma = false}; }
-| paren_expression_contents "," paren_expression_element
+| paren_expression_contents COMMA paren_expression_element
     {
       $$ = $1;
       $$.elements.push_back($3);
@@ -340,7 +349,7 @@ pattern:
 non_expression_pattern:
   AUTO
     { $$ = global_arena->New<AutoPattern>(context.SourceLoc()); }
-| binding_lhs ":" pattern
+| binding_lhs COLON pattern
     { $$ = global_arena->New<BindingPattern>(context.SourceLoc(), $1, $3); }
 | paren_pattern
     { $$ = $1; }
@@ -355,9 +364,9 @@ paren_pattern: paren_pattern_base
     { $$ = PatternFromParenContents(context.SourceLoc(), $1); }
 ;
 paren_pattern_base:
-  "(" paren_pattern_contents ")"
+  LEFT_PARENTHESIS paren_pattern_contents RIGHT_PARENTHESIS
     { $$ = $2; }
-| "(" paren_pattern_contents "," ")"
+| LEFT_PARENTHESIS paren_pattern_contents COMMA RIGHT_PARENTHESIS
     {
       $$ = $2;
       $$.has_trailing_comma = true;
@@ -371,18 +380,18 @@ paren_pattern_base:
 paren_pattern_contents:
   paren_pattern_element
     { $$ = {.elements = {$1}, .has_trailing_comma = false }; }
-| paren_expression_contents "," paren_pattern_element
+| paren_expression_contents COMMA paren_pattern_element
     {
       $$ = ParenExpressionToParenPattern($1);
       $$.elements.push_back($3);
     }
-| paren_pattern_contents "," paren_expression_element
+| paren_pattern_contents COMMA paren_expression_element
     {
       $$ = $1;
       auto el = $3.Release();
       $$.elements.push_back({.name = el.name, .term = global_arena->New<ExpressionPattern>(el.term)});
     }
-| paren_pattern_contents "," paren_pattern_element
+| paren_pattern_contents COMMA paren_pattern_element
     {
       $$ = $1;
       $$.elements.push_back($3);
@@ -391,7 +400,7 @@ paren_pattern_contents:
 paren_pattern_element:
   non_expression_pattern
     { $$ = {.name = std::nullopt, .term = $1}; }
-| designator "=" non_expression_pattern
+| designator EQUAL non_expression_pattern
     { $$ = {.name = $1, .term = $3}; }
 ;
 tuple_pattern: paren_pattern_base
@@ -401,15 +410,15 @@ tuple_pattern: paren_pattern_base
 // so it should be used only when prior context (such as an introducer)
 // rules out the possibility of an `expression` at this point.
 maybe_empty_tuple_pattern:
-  "(" ")"
+  LEFT_PARENTHESIS RIGHT_PARENTHESIS
     { $$ = global_arena->New<TuplePattern>(context.SourceLoc(), std::vector<TuplePattern::Field>()); }
 | tuple_pattern
     { $$ = $1; }
 ;
 clause:
-  CASE pattern DBLARROW statement
+  CASE pattern DOUBLE_ARROW statement
     { $$ = global_arena->RawNew<std::pair<Ptr<const Pattern>, Ptr<const Statement>>>($2, $4); }
-| DEFAULT DBLARROW statement
+| DEFAULT DOUBLE_ARROW statement
     {
       auto vp = global_arena->New<BindingPattern>(
           context.SourceLoc(), std::nullopt, global_arena->New<AutoPattern>(context.SourceLoc()));
@@ -426,38 +435,39 @@ clause_list:
     { $$ = $2; $$->push_front(*$1); }
 ;
 statement:
-  expression "=" expression ";"
+  expression EQUAL expression SEMICOLON
     { $$ = global_arena->New<Assign>(context.SourceLoc(), $1, $3); }
-| VAR pattern "=" expression ";"
+| VAR pattern EQUAL expression SEMICOLON
     { $$ = global_arena->New<VariableDefinition>(context.SourceLoc(), $2, $4); }
-| expression ";"
+| expression SEMICOLON
     { $$ = global_arena->New<ExpressionStatement>(context.SourceLoc(), $1); }
 | if_statement
     { $$ = $1; }
-| WHILE "(" expression ")" block
+| WHILE LEFT_PARENTHESIS expression RIGHT_PARENTHESIS block
     { $$ = global_arena->New<While>(context.SourceLoc(), $3, $5); }
-| BREAK ";"
+| BREAK SEMICOLON
     { $$ = global_arena->New<Break>(context.SourceLoc()); }
-| CONTINUE ";"
+| CONTINUE SEMICOLON
     { $$ = global_arena->New<Continue>(context.SourceLoc()); }
-| RETURN return_expression ";"
+| RETURN return_expression SEMICOLON
     {
       auto [return_exp, is_omitted_exp] = $2.Release();
       $$ = global_arena->New<Return>(context.SourceLoc(), return_exp, is_omitted_exp);
     }
 | block
     { $$ = $1; }
-| MATCH "(" expression ")" "{" clause_list "}"
+| MATCH LEFT_PARENTHESIS expression RIGHT_PARENTHESIS LEFT_CURLY_BRACE
+  clause_list RIGHT_CURLY_BRACE
     { $$ = global_arena->New<Match>(context.SourceLoc(), $3, $6); }
 | CONTINUATION identifier statement
     { $$ = global_arena->New<Continuation>(context.SourceLoc(), $2, $3); }
-| RUN expression ";"
+| RUN expression SEMICOLON
     { $$ = global_arena->New<Run>(context.SourceLoc(), $2); }
-| AWAIT ";"
+| AWAIT SEMICOLON
     { $$ = global_arena->New<Await>(context.SourceLoc()); }
 ;
 if_statement:
-  IF "(" expression ")" block optional_else
+  IF LEFT_PARENTHESIS expression RIGHT_PARENTHESIS block optional_else
     { $$ = global_arena->New<If>(context.SourceLoc(), $3, $5, $6); }
 ;
 optional_else:
@@ -481,7 +491,7 @@ statement_list:
     { $$ = global_arena->New<Sequence>(context.SourceLoc(), $1, $2); }
 ;
 block:
-  "{" statement_list "}"
+  LEFT_CURLY_BRACE statement_list RIGHT_CURLY_BRACE
     { $$ = global_arena->New<Block>(context.SourceLoc(), $2); }
 ;
 return_type:
@@ -491,7 +501,7 @@ return_type:
     { $$ = {$2, false}; }
 ;
 generic_binding:
-  identifier ":!" expression
+  identifier COLON_BANG expression
     {
       $$ = GenericBinding({.name = std::move($1), .type = $3});
     }
@@ -504,7 +514,7 @@ deduced_param_list:
       $$ = std::vector<GenericBinding>();
       $$.push_back($1);
     }
-| generic_binding "," deduced_param_list
+| generic_binding COMMA deduced_param_list
     {
       $$ = $3;
       $$.push_back($1);
@@ -513,7 +523,7 @@ deduced_param_list:
 deduced_params:
   // Empty
     { $$ = std::vector<GenericBinding>(); }
-| "[" deduced_param_list "]"
+| LEFT_SQUARE_BRACKET deduced_param_list RIGHT_SQUARE_BRACKET
     { $$ = $2; }
 ;
 function_definition:
@@ -525,7 +535,8 @@ function_definition:
           global_arena->New<ExpressionPattern>(return_exp),
           is_omitted_exp, $6);
     }
-| FN identifier deduced_params maybe_empty_tuple_pattern DBLARROW expression ";"
+| FN identifier deduced_params maybe_empty_tuple_pattern DOUBLE_ARROW expression
+  SEMICOLON
     {
       // The return type is not considered "omitted" because it's automatic from
       // the expression.
@@ -536,7 +547,7 @@ function_definition:
     }
 ;
 function_declaration:
-  FN identifier deduced_params maybe_empty_tuple_pattern return_type ";"
+  FN identifier deduced_params maybe_empty_tuple_pattern return_type SEMICOLON
     {
       auto [return_exp, is_omitted_exp] = $5.Release();
       $$ = global_arena->New<FunctionDefinition>(
@@ -545,10 +556,10 @@ function_declaration:
           is_omitted_exp, std::nullopt);
     }
 ;
-variable_declaration: identifier ":" pattern
+variable_declaration: identifier COLON pattern
     { $$ = global_arena->New<BindingPattern>(context.SourceLoc(), $1, $3); }
 ;
-member: VAR variable_declaration ";"
+member: VAR variable_declaration SEMICOLON
     { $$ = global_arena->New<FieldMember>(context.SourceLoc(), $2); }
 ;
 member_list:
@@ -574,7 +585,7 @@ alternative_list:
       $$ = std::list<std::pair<std::string, Ptr<const Expression>>>();
       $$.push_front($1);
     }
-| alternative "," alternative_list
+| alternative COMMA alternative_list
     { $$ = std::move($3); $$.push_front($1); }
 ;
 declaration:
@@ -582,15 +593,15 @@ declaration:
     { $$ = global_arena->New<FunctionDeclaration>($1); }
 | function_declaration
     { $$ = global_arena->New<FunctionDeclaration>($1); }
-| CLASS identifier "{" member_list "}"
+| CLASS identifier LEFT_CURLY_BRACE member_list RIGHT_CURLY_BRACE
     {
       $$ = global_arena->New<ClassDeclaration>(context.SourceLoc(), $2, $4);
     }
-| CHOICE identifier "{" alternative_list "}"
+| CHOICE identifier LEFT_CURLY_BRACE alternative_list RIGHT_CURLY_BRACE
     {
       $$ = global_arena->New<ChoiceDeclaration>(context.SourceLoc(), $2, $4);
     }
-| VAR variable_declaration "=" expression ";"
+| VAR variable_declaration EQUAL expression SEMICOLON
     {
       $$ = global_arena->New<VariableDeclaration>(context.SourceLoc(), $2, $4);
     }

executable_semantics/testdata/pattern_variable_fail.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: executable_semantics/testdata/pattern_variable_fail.carbon:7: syntax error, unexpected :
+COMPILATION ERROR: executable_semantics/testdata/pattern_variable_fail.carbon:7: syntax error, unexpected COLON
 EXIT CODE: 255