Issue #2121: Support new block string literal syntax in explorer (#2399)

common/string_helpers.cpp

@@ -14,7 +14,7 @@
 
 namespace Carbon {
 
-static constexpr llvm::StringRef TripleQuotes = R"(""")";
+static constexpr llvm::StringRef TripleQuotes = "'''";
 static constexpr llvm::StringRef HorizontalWhitespaceChars = " \t";
 
 // Carbon only takes uppercase hex input.

common/string_helpers_test.cpp

@@ -95,57 +95,57 @@ TEST(ParseBlockStringLiteral, FailNoLeadingTripleQuotes) {
 }
 
 TEST(ParseBlockStringLiteral, FailInvalideFiletypeIndicator) {
-  EXPECT_THAT(ParseBlockStringLiteral("\"\"\"carbon file\n").error().message(),
+  EXPECT_THAT(ParseBlockStringLiteral("'''carbon file\n").error().message(),
               Eq("Invalid characters in file type indicator: carbon file"));
 }
 
 TEST(ParseBlockStringLiteral, FailEndingTripleQuotes) {
-  EXPECT_THAT(ParseBlockStringLiteral("\"\"\"\n").error().message(),
+  EXPECT_THAT(ParseBlockStringLiteral("'''\n").error().message(),
               Eq("Should end with triple quotes: "));
 }
 
 TEST(ParseBlockStringLiteral, FailWrongIndent) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal
     with wrong indent
-     """)";
+     ''')";
   EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
               Eq("Wrong indent for line:     with wrong indent, expected 5"));
 }
 
 TEST(ParseBlockStringLiteral, FailInvalidEscaping) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      \q
-     """)";
+     ''')";
   EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
               Eq("Invalid escaping in \\q"));
-  constexpr char InputRaw[] = R"("""
+  constexpr char InputRaw[] = R"('''
      \#q
-     """)";
+     ''')";
   EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(),
               Eq("Invalid escaping in \\#q"));
 }
 
 TEST(ParseBlockStringLiteral, OkEmptyString) {
-  constexpr char Input[] = R"("""
-""")";
+  constexpr char Input[] = R"('''
+''')";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(""));
 }
 
 TEST(ParseBlockStringLiteral, OkOneLineString) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal
 )";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
 }
 
 TEST(ParseBlockStringLiteral, OkTwoLineString) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal
        with indent.
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal
   with indent.
 )";
@@ -153,10 +153,10 @@ TEST(ParseBlockStringLiteral, OkTwoLineString) {
 }
 
 TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) {
-  constexpr char Input[] = R"("""carbon
+  constexpr char Input[] = R"('''carbon
      A block string literal
        with file type indicator.
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal
   with file type indicator.
 )";
@@ -164,16 +164,16 @@ TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) {
 }
 
 TEST(ParseBlockStringLiteral, OkWhitespaceAfterOpeningQuotes) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal
 )";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
 }
 
 TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal
 
        with
@@ -181,7 +181,7 @@ TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
        empty
 
        lines.
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal
 
   with
@@ -194,37 +194,37 @@ TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
 }
 
 TEST(ParseBlockStringLiteral, OkWithSlashNewlineEscape) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal\
-     """)";
+     ''')";
   constexpr char Expected[] = "A block string literal";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
 }
 
 TEST(ParseBlockStringLiteral, OkWithDoubleSlashNewline) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal\\
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal\
 )";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
 }
 
 TEST(ParseBlockStringLiteral, OkWithTripleSlashNewline) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal\\\
-     """)";
+     ''')";
   constexpr char Expected[] = R"(A block string literal\)";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
 }
 
 TEST(ParseBlockStringLiteral, OkMultipleSlashes) {
-  constexpr char Input[] = R"("""
+  constexpr char Input[] = R"('''
      A block string literal\
      \
      \
      \
-     """)";
+     ''')";
   constexpr char Expected[] = "A block string literal";
   EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
 }

explorer/syntax/lexer.lpp

@@ -308,29 +308,15 @@ operand_start         [(A-Za-z0-9_\"]
   return CARBON_ARG_TOKEN(integer_literal, val);
 }
 
-#*(\"\"\"|\") {
-  // Raw string literal.
-  // yytext (the token that matches the above regex) and chars scanned by
-  // str_lex_helper hold the source text, not the string the source represents.
+#*\" {
+  // Found a double quote character.
   Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
   const std::string& s = str_lex_helper.str();
   const int hashtag_num = s.find_first_of('"');
-  const int leading_quotes = s.size() - hashtag_num;
-  if (leading_quotes == 3 && hashtag_num > 0) {
-    // Check if it's a single-line string, like #"""#.
-    // TODO: Extend with other single-line string cases, like #""""#, based on
-    // the definition of block string in the design doc.
-    if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
-      return Carbon::ProcessSingleLineString(str_lex_helper.str(), context,
-                                             hashtag_num);
-    } else if (str_lex_helper.is_eof()) {
-      return CARBON_SIMPLE_TOKEN(END_OF_FILE);
-    }
-  } else if (!str_lex_helper.Advance()) {
+
+  if (!str_lex_helper.Advance()) {
     return CARBON_SIMPLE_TOKEN(END_OF_FILE);
   }
-  // 3 quotes indicates multi-line, otherwise it'll be one.
-  const bool multi_line = leading_quotes == 3;
 
   while (!str_lex_helper.is_eof()) {
     switch (str_lex_helper.last_char()) {
@@ -338,36 +324,60 @@ operand_start         [(A-Za-z0-9_\"]
       case '\v':  // Fall through.
       case '\f':  // Fall through.
       case '\r':
-        if (!multi_line) {
-          return context.RecordSyntaxError(
-              llvm::formatv("missing closing quote in single-line string: {0}",
-                            str_lex_helper.str()));
-        }
-        str_lex_helper.Advance();
+        return context.RecordSyntaxError(
+            llvm::formatv("missing closing quote in single-line string: {0}",
+                          str_lex_helper.str()));
         break;
       case '"':
-        if (multi_line) {
-          // Check for 2 more '"'s on block string.
-          if (!(str_lex_helper.Advance() &&
-                str_lex_helper.last_char() == '"')) {
-            continue;
-          }
-          if (!(str_lex_helper.Advance() &&
-                str_lex_helper.last_char() == '"')) {
+        if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
+          // Reach closing quotes, break out of the loop.
+          return Carbon::ProcessSingleLineString(str_lex_helper.str(), context,
+                                                 hashtag_num);
+        }
+        break;
+      case '\\':
+        if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
+          // Read the escaped char.
+          if (!str_lex_helper.Advance()) {
             continue;
           }
-          // Now we are at the last " of """.
+          // Read the next char.
+          str_lex_helper.Advance();
         }
+        break;
+      default:
+        str_lex_helper.Advance();
+    }
+  }
+
+  return CARBON_SIMPLE_TOKEN(END_OF_FILE);
+}
+
+#*\'\'\' {
+  // Multi-line string literal.
+  Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
+  const std::string& s = str_lex_helper.str();
+  const int hashtag_num = s.find_first_of('\'');
 
+  if (!str_lex_helper.Advance()) {
+    return CARBON_SIMPLE_TOKEN(END_OF_FILE);
+  }
+
+  while (!str_lex_helper.is_eof()) {
+    switch (str_lex_helper.last_char()) {
+      case '\'':
+        // Check for 2 more '\''s on block string.
+        if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) {
+          continue;
+        }
+        if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) {
+          continue;
+        }
+        // Now we are at the last ' of '''.
         if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
           // Reach closing quotes, break out of the loop.
-          if (leading_quotes == 3) {
-            return Carbon::ProcessMultiLineString(str_lex_helper.str(), context,
-                                                  hashtag_num);
-          } else {
-            return Carbon::ProcessSingleLineString(str_lex_helper.str(),
-                                                   context, hashtag_num);
-          }
+          return Carbon::ProcessMultiLineString(str_lex_helper.str(), context,
+                                                hashtag_num);
         }
         break;
       case '\\':

explorer/testdata/string/block.carbon

@@ -10,10 +10,10 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  var s: String = """
+  var s: String = '''
     A "block" ""string"" literal
       with indent.
-    """;
+    ''';
   if (s == "A \"block\" \"\"string\"\" literal\n  with indent.\n") {
     return 0;
   } else {

explorer/testdata/string/block_escaped_triple_quotes.carbon

@@ -10,11 +10,11 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  var s: String = """
+  var s: String = '''
     A block string literal
-    \"""
-    """;
-  if (s == "A block string literal\n\"\"\"\n") {
+    \'''
+    ''';
+  if (s == "A block string literal\n'''\n") {
     return 0;
   } else {
     return 1;

explorer/testdata/string/block_file_type_indicator.carbon

@@ -10,10 +10,10 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  var s: String = """filetype
+  var s: String = '''filetype
     A "block" ""string"" literal
       with file type indicator.
-    """;
+    ''';
   if (s == "A \"block\" \"\"string\"\" literal\n  with file type indicator.\n") {
     return 0;
   } else {

explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon

@@ -9,10 +9,10 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """
-  var s: String = """
-    error: closing """ is not on its own line.
-  """;
+  // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing '''
+  var s: String = '''
+    error: closing ''' is not on its own line.
+  ''';
 
   return 0;
 }

explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon

@@ -10,9 +10,9 @@ package ExplorerTest api;
 
 fn Main() -> i32 {
   // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon:[[@LINE+1]]: Unexpected end of file
-  var s: String = ##"""
+  var s: String = ##'''
     error: there are more #s on the left than the right.
-  """#;
+  '''#;
 
   return 0;
 }

explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon

@@ -10,9 +10,9 @@ package ExplorerTest api;
 
 fn Main() -> i32 {
   // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file.
-  var s: String = #"""
+  var s: String = #'''
     error: there are more #s on the right than the left.
-  """##;
+  '''##;
 
   return 0;
 }

explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon

@@ -9,10 +9,10 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """
-  var s: String = #"""
-    error: closing """# is not on its own line.
-  """#;
+  // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing '''
+  var s: String = #'''
+    error: closing '''# is not on its own line.
+  '''#;
 
   return 0;
 }

explorer/testdata/string/fail_raw_block_single_line.carbon

@@ -10,7 +10,7 @@ package ExplorerTest api;
 
 fn CompareStr(s: String) -> i32 {
   // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_single_line.carbon:[[@LINE+1]]: Invalid block string: Too few lines
-  if (s == #"""raw string literal starting with """#) {
+  if (s == #'''raw string literal starting with '''#) {
     return 0;
   }
   return 1;

explorer/testdata/string/raw_block.carbon

@@ -10,10 +10,10 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  var s: String = #"""
+  var s: String = #'''
     A "block" ""string"" literal
       with indent.
-    """#;
+    '''#;
   if (s == #"A \#"block\#" \#"\#"string\#"\#" literal\#n  with indent.\#n"#) {
     return 0;
   } else {

explorer/testdata/string/raw_block_escaped_back_slash.carbon

@@ -10,10 +10,10 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  var s: String = #"""
+  var s: String = #'''
     A block string literal
     \
-    """#;
+    '''#;
   if (s == "A block string literal\n\\\n") {
     return 0;
   } else {

explorer/testdata/string/raw_block_escaped_triple_quotes.carbon

@@ -10,11 +10,11 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  var s: String = #"""
+  var s: String = #'''
     A block string literal
-    \#"""#
-    """#;
-  if (s == "A block string literal\n\"\"\"#\n") {
+    \#'''#
+    '''#;
+  if (s == "A block string literal\n'''#\n") {
     return 0;
   } else {
     return 1;