Initial implementation of raw string literals (#1304)

* test cases for raw string literals

* raw string literal implementation

* match as block string if starting with triple ", and better error message for simple string

except for *#"""#*

* fix broken test case

block string  literal cannot be one line

* test cases for raw string literals

* raw string literal implementation

* match as block string if starting with triple ", and better error message for simple string

except for *#"""#*

* fix broken test case

block string  literal cannot be one line

* removed unused initial value

* rename flag to indicate multi-line string and remove comment

* use * to get value from std::optional

* clean-ups

* removed skip_scan flag and directly return in case of a single line string starting with #+\'\'\'

* Updated error message: simple string -> single-line string.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

* Updated test cases according to changes in error message

* Removed counting_hashtag flag.

* Implemented ScanHelper class to handle scanning

* Fixed explanation of ReadHashTags.

* Addressed PR comment.

* Clarify that scan_helper holds the source text.

* Addressed PR comments.

* Updated error messages in test cases.

* Added const keyword to return type of GetCurrentStr().

* addressed PR comments.

1. Moved ScanHelper class to lex_scan_helper.h and lex_scan_helper.cpp.
2. Moved ReadHashTags and Process* functions to lex_scan_helper.cpp. Moved YY_USER_ACTION, SIMPLE_TOKEN and ARG_TOKEN to lex_helper.h. Added a wrapper function YyinputWrapper to call static function yyinput in lexer.lpp.
3. Renamed ScanHelper with StringLexHelper.
4. Modified BUILD accordingly.
5. Renamed data members and functions.

* Addressed PR comments.

1. Adjusted order to keep ret usage close.
2. Used resize to construct the string to avoid creation of temp string.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Removed the multi_line flag and skip_read field to improve readability.

* Copied default parameter value to definition of UnescapeStringLiteral.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Copied default parameter value to definition of ParseBlockStringLiteral.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Prefix CARBON_ to SIMPLE_TOKEN and ARG_TOKEN macros.

* Rollback redefinition of arguments.

* Updated comment on the flex macro.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Updated wording.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Moved the EOF error out of the loop.

* Removed duplicated declaration.

* Changed type of `hashtag_num` and `leading_quotes` to int.

* Minor fix: string copy.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Added comment on YyinputWrapper.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Garmmar in comment.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Added check of eof before readling next char.

* Minor updates based on PR comments.

* Minor changes to address PR comments.

* Used a clearer way to calculate `hashtag_num` and `leading_quotes`. Switched back to indicate muti-line string with a flag.

* Directly copy StringRef for compilation error message.

* Make str_with_quote const as we don't change it.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

* Added TODO for unsupported cases.

* Fixed a typo.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

Co-authored-by: josh11b <josh11b@users.noreply.github.com>
Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

common/string_helpers.cpp

@@ -27,87 +27,86 @@ static auto FromHex(char c) -> std::optional<char> {
   return std::nullopt;
 }
 
-auto UnescapeStringLiteral(llvm::StringRef source, bool is_block_string)
-    -> std::optional<std::string> {
+auto UnescapeStringLiteral(llvm::StringRef source, const int hashtag_num,
+                           bool is_block_string) -> std::optional<std::string> {
   std::string ret;
   ret.reserve(source.size());
+  std::string escape = "\\";
+  escape.resize(hashtag_num + 1, '#');
   size_t i = 0;
   while (i < source.size()) {
     char c = source[i];
-    switch (c) {
-      case '\\':
-        ++i;
-        if (i == source.size()) {
-          return std::nullopt;
-        }
-        switch (source[i]) {
-          case 'n':
-            ret.push_back('\n');
-            break;
-          case 'r':
-            ret.push_back('\r');
-            break;
-          case 't':
-            ret.push_back('\t');
-            break;
-          case '0':
-            if (i + 1 < source.size() && llvm::isDigit(source[i + 1])) {
-              // \0[0-9] is reserved.
-              return std::nullopt;
-            }
-            ret.push_back('\0');
-            break;
-          case '"':
-            ret.push_back('"');
-            break;
-          case '\'':
-            ret.push_back('\'');
-            break;
-          case '\\':
-            ret.push_back('\\');
-            break;
-          case 'x': {
-            i += 2;
-            if (i >= source.size()) {
-              return std::nullopt;
-            }
-            std::optional<char> c1 = FromHex(source[i - 1]);
-            std::optional<char> c2 = FromHex(source[i]);
-            if (c1 == std::nullopt || c2 == std::nullopt) {
-              return std::nullopt;
-            }
-            ret.push_back(16 * *c1 + *c2);
-            break;
+    if (i + hashtag_num < source.size() &&
+        source.slice(i, i + hashtag_num + 1).equals(escape)) {
+      i += hashtag_num + 1;
+      if (i == source.size()) {
+        return std::nullopt;
+      }
+      switch (source[i]) {
+        case 'n':
+          ret.push_back('\n');
+          break;
+        case 'r':
+          ret.push_back('\r');
+          break;
+        case 't':
+          ret.push_back('\t');
+          break;
+        case '0':
+          if (i + 1 < source.size() && llvm::isDigit(source[i + 1])) {
+            // \0[0-9] is reserved.
+            return std::nullopt;
+          }
+          ret.push_back('\0');
+          break;
+        case '"':
+          ret.push_back('"');
+          break;
+        case '\'':
+          ret.push_back('\'');
+          break;
+        case '\\':
+          ret.push_back('\\');
+          break;
+        case 'x': {
+          i += 2;
+          if (i >= source.size()) {
+            return std::nullopt;
           }
-          case 'u':
-            CARBON_FATAL() << "\\u is not yet supported in string literals";
-          case '\n':
-            if (!is_block_string) {
-              return std::nullopt;
-            }
-            break;
-          default:
-            // Unsupported.
+          std::optional<char> c1 = FromHex(source[i - 1]);
+          std::optional<char> c2 = FromHex(source[i]);
+          if (c1 == std::nullopt || c2 == std::nullopt) {
             return std::nullopt;
+          }
+          ret.push_back(16 * *c1 + *c2);
+          break;
         }
-        break;
-
-      case '\t':
-        // Disallow non-` ` horizontal whitespace:
-        // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/whitespace.md
-        // TODO: This doesn't handle unicode whitespace.
-        return std::nullopt;
-
-      default:
-        ret.push_back(c);
-        break;
+        case 'u':
+          CARBON_FATAL() << "\\u is not yet supported in string literals";
+        case '\n':
+          if (!is_block_string) {
+            return std::nullopt;
+          }
+          break;
+        default:
+          // Unsupported.
+          return std::nullopt;
+      }
+    } else if (c == '\t') {
+      // Disallow non-` ` horizontal whitespace:
+      // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/whitespace.md
+      // TODO: This doesn't handle unicode whitespace.
+      return std::nullopt;
+    } else {
+      ret.push_back(c);
     }
     ++i;
   }
   return ret;
 }
 
-auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr<std::string> {
+auto ParseBlockStringLiteral(llvm::StringRef source, const int hashtag_num)
+    -> ErrorOr<std::string> {
   llvm::SmallVector<llvm::StringRef> lines;
   source.split(lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/true);
   if (lines.size() < 2) {
@@ -150,8 +149,9 @@ auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr<std::string> {
     }
     // Unescaping with \n appended to handle things like \\<newline>.
     llvm::SmallVector<char> buffer;
-    std::optional<std::string> unescaped = UnescapeStringLiteral(
-        (line + "\n").toStringRef(buffer), /*is_block_string=*/true);
+    std::optional<std::string> unescaped =
+        UnescapeStringLiteral((line + "\n").toStringRef(buffer), hashtag_num,
+                              /*is_block_string=*/true);
     if (!unescaped.has_value()) {
       return Error("Invalid escaping in " + line);
     }

common/string_helpers.h

@@ -19,11 +19,13 @@ namespace Carbon {
 // Unescapes Carbon escape sequences in the source string. Returns std::nullopt
 // on bad input. `is_block_string` enables escaping unique to block string
 // literals, such as \<newline>.
-auto UnescapeStringLiteral(llvm::StringRef source, bool is_block_string = false)
+auto UnescapeStringLiteral(llvm::StringRef source, int hashtag_num = 0,
+                           bool is_block_string = false)
     -> std::optional<std::string>;
 
 // Parses a block string literal in `source`.
-auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr<std::string>;
+auto ParseBlockStringLiteral(llvm::StringRef source, int hashtag_num = 0)
+    -> ErrorOr<std::string>;
 
 // Returns true if the pointer is in the string ref (including equality with
 // `ref.end()`). This should be used instead of `<=` comparisons for

common/string_helpers_test.cpp

@@ -28,6 +28,8 @@ TEST(UnescapeStringLiteral, Valid) {
   EXPECT_THAT(UnescapeStringLiteral("test\\\\n"), Optional(Eq("test\\n")));
   EXPECT_THAT(UnescapeStringLiteral("\\xAA"), Optional(Eq("\xAA")));
   EXPECT_THAT(UnescapeStringLiteral("\\x12"), Optional(Eq("\x12")));
+  EXPECT_THAT(UnescapeStringLiteral("test", 1), Optional(Eq("test")));
+  EXPECT_THAT(UnescapeStringLiteral("test\\#n", 1), Optional(Eq("test\n")));
 }
 
 TEST(UnescapeStringLiteral, Invalid) {
@@ -43,6 +45,7 @@ TEST(UnescapeStringLiteral, Invalid) {
   EXPECT_THAT(UnescapeStringLiteral("\\xaa"), Eq(std::nullopt));
   // Reserved.
   EXPECT_THAT(UnescapeStringLiteral("\\00"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\#00", 1), Eq(std::nullopt));
 }
 
 TEST(UnescapeStringLiteral, Nul) {
@@ -90,6 +93,11 @@ TEST(ParseBlockStringLiteral, FailInvalidEscaping) {
      """)";
   EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
               Eq("Invalid escaping in \\q"));
+  constexpr char InputRaw[] = R"("""
+     \#q
+     """)";
+  EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(),
+              Eq("Invalid escaping in \\#q"));
 }
 
 TEST(ParseBlockStringLiteral, OkEmptyString) {

explorer/syntax/BUILD

@@ -53,6 +53,9 @@ cc_library(
 cc_library(
     name = "syntax",
     srcs = [
+        "lex_helper.h",
+        "lex_scan_helper.cpp",
+        "lex_scan_helper.h",
         "lexer.cpp",
         "lexer.h",
         "parse.cpp",

explorer/syntax/lex_helper.h

@@ -0,0 +1,25 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_
+#define CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_
+
+// Flex expands this macro immediately before each action.
+//
+// Advances the current token position by yyleng columns without changing
+// the line number, and takes us out of the after-whitespace / after-operand
+// state.
+#define YY_USER_ACTION                                             \
+  context.current_token_position.columns(yyleng);                  \
+  if (YY_START == AFTER_WHITESPACE || YY_START == AFTER_OPERAND) { \
+    BEGIN(INITIAL);                                                \
+  }
+
+#define CARBON_SIMPLE_TOKEN(name) \
+  Carbon::Parser::make_##name(context.current_token_position);
+
+#define CARBON_ARG_TOKEN(name, arg) \
+  Carbon::Parser::make_##name(arg, context.current_token_position);
+
+#endif  // CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_

explorer/syntax/lex_scan_helper.cpp

@@ -0,0 +1,68 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "explorer/syntax/lex_scan_helper.h"
+
+#include "common/string_helpers.h"
+#include "explorer/syntax/lex_helper.h"
+#include "llvm/Support/FormatVariadic.h"
+
+namespace Carbon {
+
+auto StringLexHelper::Advance() -> bool {
+  CARBON_CHECK(is_eof_ == false);
+  const char c = YyinputWrapper(yyscanner_);
+  if (c <= 0) {
+    context_.RecordSyntaxError("Unexpected end of file");
+    is_eof_ = true;
+    return false;
+  }
+  str_.push_back(c);
+  return true;
+}
+
+auto ReadHashTags(Carbon::StringLexHelper& scan_helper,
+                  const size_t hashtag_num) -> bool {
+  for (size_t i = 0; i < hashtag_num; ++i) {
+    if (!scan_helper.Advance() || scan_helper.last_char() != '#') {
+      return false;
+    }
+  }
+  return true;
+}
+
+auto ProcessSingleLineString(llvm::StringRef str,
+                             Carbon::ParseAndLexContext& context,
+                             const size_t hashtag_num)
+    -> Carbon::Parser::symbol_type {
+  std::string hashtags(hashtag_num, '#');
+  const auto str_with_quote = str;
+  CARBON_CHECK(str.consume_front(hashtags + "\"") &&
+               str.consume_back("\"" + hashtags));
+
+  std::optional<std::string> unescaped =
+      Carbon::UnescapeStringLiteral(str, hashtag_num);
+  if (unescaped == std::nullopt) {
+    return context.RecordSyntaxError(
+        llvm::formatv("Invalid escaping in string: {0}", str_with_quote));
+  }
+  return CARBON_ARG_TOKEN(string_literal, *unescaped);
+}
+
+auto ProcessMultiLineString(llvm::StringRef str,
+                            Carbon::ParseAndLexContext& context,
+                            const size_t hashtag_num)
+    -> Carbon::Parser::symbol_type {
+  std::string hashtags(hashtag_num, '#');
+  CARBON_CHECK(str.consume_front(hashtags) && str.consume_back(hashtags));
+  Carbon::ErrorOr<std::string> block_string =
+      Carbon::ParseBlockStringLiteral(str, hashtag_num);
+  if (!block_string.ok()) {
+    return context.RecordSyntaxError(llvm::formatv(
+        "Invalid block string: {0}", block_string.error().message()));
+  }
+  return CARBON_ARG_TOKEN(string_literal, *block_string);
+}
+
+}  // namespace Carbon

explorer/syntax/lex_scan_helper.h

@@ -0,0 +1,58 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_
+#define CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_
+
+#include <string>
+
+#include "explorer/syntax/parse_and_lex_context.h"
+#include "explorer/syntax/parser.h"
+
+// Exposes yyinput; defined in lexer.lpp.
+extern auto YyinputWrapper(yyscan_t yyscanner) -> int;
+
+namespace Carbon {
+
+class StringLexHelper {
+ public:
+  StringLexHelper(const char* text, yyscan_t yyscanner,
+                  Carbon::ParseAndLexContext& context)
+      : str_(text), yyscanner_(yyscanner), context_(context), is_eof_(false) {}
+  // Advances yyscanner by one char. Sets is_eof to true and returns false on
+  // EOF.
+  auto Advance() -> bool;
+  // Returns the last scanned char.
+  auto last_char() -> char { return str_.back(); };
+  // Returns the scanned string.
+  auto str() -> const std::string& { return str_; };
+
+  auto is_eof() -> bool { return is_eof_; };
+
+ private:
+  std::string str_;
+  yyscan_t yyscanner_;
+  Carbon::ParseAndLexContext& context_;
+  // Skips reading next char.
+  bool is_eof_;
+};
+
+// Tries to Read `hashtag_num` hashtags. Returns true on success.
+// Reads `hashtag_num` characters on success, and number of consecutive hashtags
+// (< `hashtag_num`) + 1 characters on failure.
+auto ReadHashTags(Carbon::StringLexHelper& scan_helper, size_t hashtag_num)
+    -> bool;
+
+// Removes quotes and escapes a single line string. Reports an error on
+// invalid escaping.
+auto ProcessSingleLineString(llvm::StringRef str,
+                             Carbon::ParseAndLexContext& context,
+                             size_t hashtag_num) -> Carbon::Parser::symbol_type;
+auto ProcessMultiLineString(llvm::StringRef str,
+                            Carbon::ParseAndLexContext& context,
+                            size_t hashtag_num) -> Carbon::Parser::symbol_type;
+
+}  // namespace Carbon
+
+#endif  // CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_

explorer/syntax/lexer.lpp

@@ -9,14 +9,12 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
   #include "common/check.h"
   #include "common/error.h"
-  #include "common/string_helpers.h"
+  #include "explorer/syntax/lex_helper.h"
+  #include "explorer/syntax/lex_scan_helper.h"
   #include "explorer/syntax/parse_and_lex_context.h"
   #include "explorer/syntax/parser.h"
   #include "llvm/ADT/StringExtras.h"
   #include "llvm/Support/FormatVariadic.h"
-
-  // Reads and returns a single character. Reports an error on EOF.
-  auto ReadChar(yyscan_t yyscanner, Carbon::ParseAndLexContext& context) -> int;
 %}
 
 /* Turn off legacy bits we don't need. */
@@ -109,28 +107,6 @@ whitespace            [ \t\r\n]
 one_line_comment      \/\/[^\n]*\n
 operand_start         [(A-Za-z0-9_\"]
 
-/* Single-line string literals should reject vertical whitespace. */
-string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
-
-%{
-  // This macro is expanded immediately before each action specified below.
-  //
-  // Advances the current token position by yyleng columns without changing
-  // the line number, and takes us out of the after-whitespace / after-operand
-  // state.
-  #define YY_USER_ACTION                                             \
-    context.current_token_position.columns(yyleng);                  \
-    if (YY_START == AFTER_WHITESPACE || YY_START == AFTER_OPERAND) { \
-      BEGIN(INITIAL);                                                \
-    }
-
-  #define SIMPLE_TOKEN(name) \
-    Carbon::Parser::make_##name(context.current_token_position);
-
-  #define ARG_TOKEN(name, arg) \
-    Carbon::Parser::make_##name(arg, context.current_token_position);
-%}
-
 %%
 
 %{
@@ -141,83 +117,83 @@ string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
 %}
 
  /* table-begin */
-{ADDR}                { return SIMPLE_TOKEN(ADDR);                }
-{ALIAS}               { return SIMPLE_TOKEN(ALIAS);               }
-{AMPERSAND}           { return SIMPLE_TOKEN(AMPERSAND);           }
-{AND}                 { return SIMPLE_TOKEN(AND);                 }
-{API}                 { return SIMPLE_TOKEN(API);                 }
-{ARROW}               { return SIMPLE_TOKEN(ARROW);               }
-{AS}                  { return SIMPLE_TOKEN(AS);                  }
-{AUTO}                { return SIMPLE_TOKEN(AUTO);                }
-{AWAIT}               { return SIMPLE_TOKEN(AWAIT);               }
-{BOOL}                { return SIMPLE_TOKEN(BOOL);                }
-{BREAK}               { return SIMPLE_TOKEN(BREAK);               }
-{CASE}                { return SIMPLE_TOKEN(CASE);                }
-{CHOICE}              { return SIMPLE_TOKEN(CHOICE);              }
-{CLASS}               { return SIMPLE_TOKEN(CLASS);               }
-{COLON_BANG}          { return SIMPLE_TOKEN(COLON_BANG);          }
-{COLON}               { return SIMPLE_TOKEN(COLON);               }
-{COMMA}               { return SIMPLE_TOKEN(COMMA);               }
-{CONTINUATION_TYPE}   { return SIMPLE_TOKEN(CONTINUATION_TYPE);   }
-{CONTINUATION}        { return SIMPLE_TOKEN(CONTINUATION);        }
-{CONTINUE}            { return SIMPLE_TOKEN(CONTINUE);            }
-{DEFAULT}             { return SIMPLE_TOKEN(DEFAULT);             }
-{DOUBLE_ARROW}        { return SIMPLE_TOKEN(DOUBLE_ARROW);        }
-{ELSE}                { return SIMPLE_TOKEN(ELSE);                }
-{EQUAL_EQUAL}         { return SIMPLE_TOKEN(EQUAL_EQUAL);         }
-{EQUAL}               { return SIMPLE_TOKEN(EQUAL);               }
-{EXTERNAL}            { return SIMPLE_TOKEN(EXTERNAL);            }
-{FALSE}               { return SIMPLE_TOKEN(FALSE);               }
-{FN_TYPE}             { return SIMPLE_TOKEN(FN_TYPE);             }
-{FN}                  { return SIMPLE_TOKEN(FN);                  }
-{FORALL}              { return SIMPLE_TOKEN(FORALL);              }
-{IF}                  { return SIMPLE_TOKEN(IF);                  }
-{IMPL}                { return SIMPLE_TOKEN(IMPL);                }
-{IMPORT}              { return SIMPLE_TOKEN(IMPORT);              }
-{INTERFACE}           { return SIMPLE_TOKEN(INTERFACE);           }
-{IS}                  { return SIMPLE_TOKEN(IS);                  }
-{LEFT_CURLY_BRACE}    { return SIMPLE_TOKEN(LEFT_CURLY_BRACE);    }
-{LEFT_PARENTHESIS}    { return SIMPLE_TOKEN(LEFT_PARENTHESIS);    }
-{LEFT_SQUARE_BRACKET} { return SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); }
-{LET}                 { return SIMPLE_TOKEN(LET);                 }
-{LIBRARY}             { return SIMPLE_TOKEN(LIBRARY);             }
-{MATCH}               { return SIMPLE_TOKEN(MATCH);               }
-{MINUS}               { return SIMPLE_TOKEN(MINUS);               }
-{NOT}                 { return SIMPLE_TOKEN(NOT);                 }
-{OR}                  { return SIMPLE_TOKEN(OR);                  }
-{PACKAGE}             { return SIMPLE_TOKEN(PACKAGE);             }
-{PERIOD}              { return SIMPLE_TOKEN(PERIOD);              }
-{PLUS}                { return SIMPLE_TOKEN(PLUS);                }
-{RETURN}              { return SIMPLE_TOKEN(RETURN);              }
-{RUN}                 { return SIMPLE_TOKEN(RUN);                 }
-{SELF}                { return SIMPLE_TOKEN(SELF);                }
-{SEMICOLON}           { return SIMPLE_TOKEN(SEMICOLON);           }
-{SLASH}               { return SIMPLE_TOKEN(SLASH);               }
-{STRING}              { return SIMPLE_TOKEN(STRING);              }
-{THEN}                { return SIMPLE_TOKEN(THEN);                }
-{TRUE}                { return SIMPLE_TOKEN(TRUE);                }
-{TYPE}                { return SIMPLE_TOKEN(TYPE);                }
-{UNDERSCORE}          { return SIMPLE_TOKEN(UNDERSCORE);          }
-{UNIMPL_EXAMPLE}      { return SIMPLE_TOKEN(UNIMPL_EXAMPLE);      }
-{VAR}                 { return SIMPLE_TOKEN(VAR);                 }
-{WHERE}               { return SIMPLE_TOKEN(WHERE);               }
-{WHILE}               { return SIMPLE_TOKEN(WHILE);               }
+{ADDR}                { return CARBON_SIMPLE_TOKEN(ADDR);                }
+{ALIAS}               { return CARBON_SIMPLE_TOKEN(ALIAS);               }
+{AMPERSAND}           { return CARBON_SIMPLE_TOKEN(AMPERSAND);           }
+{AND}                 { return CARBON_SIMPLE_TOKEN(AND);                 }
+{API}                 { return CARBON_SIMPLE_TOKEN(API);                 }
+{ARROW}               { return CARBON_SIMPLE_TOKEN(ARROW);               }
+{AS}                  { return CARBON_SIMPLE_TOKEN(AS);                  }
+{AUTO}                { return CARBON_SIMPLE_TOKEN(AUTO);                }
+{AWAIT}               { return CARBON_SIMPLE_TOKEN(AWAIT);               }
+{BOOL}                { return CARBON_SIMPLE_TOKEN(BOOL);                }
+{BREAK}               { return CARBON_SIMPLE_TOKEN(BREAK);               }
+{CASE}                { return CARBON_SIMPLE_TOKEN(CASE);                }
+{CHOICE}              { return CARBON_SIMPLE_TOKEN(CHOICE);              }
+{CLASS}               { return CARBON_SIMPLE_TOKEN(CLASS);               }
+{COLON_BANG}          { return CARBON_SIMPLE_TOKEN(COLON_BANG);          }
+{COLON}               { return CARBON_SIMPLE_TOKEN(COLON);               }
+{COMMA}               { return CARBON_SIMPLE_TOKEN(COMMA);               }
+{CONTINUATION_TYPE}   { return CARBON_SIMPLE_TOKEN(CONTINUATION_TYPE);   }
+{CONTINUATION}        { return CARBON_SIMPLE_TOKEN(CONTINUATION);        }
+{CONTINUE}            { return CARBON_SIMPLE_TOKEN(CONTINUE);            }
+{DEFAULT}             { return CARBON_SIMPLE_TOKEN(DEFAULT);             }
+{DOUBLE_ARROW}        { return CARBON_SIMPLE_TOKEN(DOUBLE_ARROW);        }
+{ELSE}                { return CARBON_SIMPLE_TOKEN(ELSE);                }
+{EQUAL_EQUAL}         { return CARBON_SIMPLE_TOKEN(EQUAL_EQUAL);         }
+{EQUAL}               { return CARBON_SIMPLE_TOKEN(EQUAL);               }
+{EXTERNAL}            { return CARBON_SIMPLE_TOKEN(EXTERNAL);            }
+{FALSE}               { return CARBON_SIMPLE_TOKEN(FALSE);               }
+{FN_TYPE}             { return CARBON_SIMPLE_TOKEN(FN_TYPE);             }
+{FN}                  { return CARBON_SIMPLE_TOKEN(FN);                  }
+{FORALL}              { return CARBON_SIMPLE_TOKEN(FORALL);              }
+{IF}                  { return CARBON_SIMPLE_TOKEN(IF);                  }
+{IMPL}                { return CARBON_SIMPLE_TOKEN(IMPL);                }
+{IMPORT}              { return CARBON_SIMPLE_TOKEN(IMPORT);              }
+{INTERFACE}           { return CARBON_SIMPLE_TOKEN(INTERFACE);           }
+{IS}                  { return CARBON_SIMPLE_TOKEN(IS);                  }
+{LEFT_CURLY_BRACE}    { return CARBON_SIMPLE_TOKEN(LEFT_CURLY_BRACE);    }
+{LEFT_PARENTHESIS}    { return CARBON_SIMPLE_TOKEN(LEFT_PARENTHESIS);    }
+{LEFT_SQUARE_BRACKET} { return CARBON_SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); }
+{LET}                 { return CARBON_SIMPLE_TOKEN(LET);                 }
+{LIBRARY}             { return CARBON_SIMPLE_TOKEN(LIBRARY);             }
+{MATCH}               { return CARBON_SIMPLE_TOKEN(MATCH);               }
+{MINUS}               { return CARBON_SIMPLE_TOKEN(MINUS);               }
+{NOT}                 { return CARBON_SIMPLE_TOKEN(NOT);                 }
+{OR}                  { return CARBON_SIMPLE_TOKEN(OR);                  }
+{PACKAGE}             { return CARBON_SIMPLE_TOKEN(PACKAGE);             }
+{PERIOD}              { return CARBON_SIMPLE_TOKEN(PERIOD);              }
+{PLUS}                { return CARBON_SIMPLE_TOKEN(PLUS);                }
+{RETURN}              { return CARBON_SIMPLE_TOKEN(RETURN);              }
+{RUN}                 { return CARBON_SIMPLE_TOKEN(RUN);                 }
+{SELF}                { return CARBON_SIMPLE_TOKEN(SELF);                }
+{SEMICOLON}           { return CARBON_SIMPLE_TOKEN(SEMICOLON);           }
+{SLASH}               { return CARBON_SIMPLE_TOKEN(SLASH);               }
+{STRING}              { return CARBON_SIMPLE_TOKEN(STRING);              }
+{THEN}                { return CARBON_SIMPLE_TOKEN(THEN);                }
+{TRUE}                { return CARBON_SIMPLE_TOKEN(TRUE);                }
+{TYPE}                { return CARBON_SIMPLE_TOKEN(TYPE);                }
+{UNDERSCORE}          { return CARBON_SIMPLE_TOKEN(UNDERSCORE);          }
+{UNIMPL_EXAMPLE}      { return CARBON_SIMPLE_TOKEN(UNIMPL_EXAMPLE);      }
+{VAR}                 { return CARBON_SIMPLE_TOKEN(VAR);                 }
+{WHERE}               { return CARBON_SIMPLE_TOKEN(WHERE);               }
+{WHILE}               { return CARBON_SIMPLE_TOKEN(WHILE);               }
  /* table-end */
 
  /* More modern Bisons provide make_EOF. */
-<<EOF>>               { return SIMPLE_TOKEN(END_OF_FILE); }
+<<EOF>>               { return CARBON_SIMPLE_TOKEN(END_OF_FILE); }
 
 {RIGHT_PARENTHESIS} {
   BEGIN(AFTER_OPERAND);
-  return SIMPLE_TOKEN(RIGHT_PARENTHESIS);
+  return CARBON_SIMPLE_TOKEN(RIGHT_PARENTHESIS);
 }
 {RIGHT_CURLY_BRACE} {
   BEGIN(AFTER_OPERAND);
-  return SIMPLE_TOKEN(RIGHT_CURLY_BRACE);
+  return CARBON_SIMPLE_TOKEN(RIGHT_CURLY_BRACE);
 }
 {RIGHT_SQUARE_BRACKET} {
   BEGIN(AFTER_OPERAND);
-  return SIMPLE_TOKEN(RIGHT_SQUARE_BRACKET);
+  return CARBON_SIMPLE_TOKEN(RIGHT_SQUARE_BRACKET);
 }
 
  /*
@@ -244,23 +220,23 @@ string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
  /* `*` operator case 1: */
 <AFTER_WHITESPACE>"*"{whitespace}+ {
   BEGIN(AFTER_WHITESPACE);
-  return SIMPLE_TOKEN(BINARY_STAR);
+  return CARBON_SIMPLE_TOKEN(BINARY_STAR);
 }
  /* `*` operator case 2: */
-<AFTER_OPERAND>"*"/{operand_start} { return SIMPLE_TOKEN(BINARY_STAR); }
+<AFTER_OPERAND>"*"/{operand_start} { return CARBON_SIMPLE_TOKEN(BINARY_STAR); }
  /* `*` operator case 3: */
-<AFTER_WHITESPACE>"*" { return SIMPLE_TOKEN(PREFIX_STAR); }
+<AFTER_WHITESPACE>"*" { return CARBON_SIMPLE_TOKEN(PREFIX_STAR); }
  /* `*` operator case 4: */
 <INITIAL,AFTER_OPERAND>"*"{whitespace}+ {
   BEGIN(AFTER_WHITESPACE);
-  return SIMPLE_TOKEN(POSTFIX_STAR);
+  return CARBON_SIMPLE_TOKEN(POSTFIX_STAR);
 }
  /* `*` operator case 5: */
-<INITIAL,AFTER_OPERAND>"*" { return SIMPLE_TOKEN(UNARY_STAR); }
+<INITIAL,AFTER_OPERAND>"*" { return CARBON_SIMPLE_TOKEN(UNARY_STAR); }
 
 {sized_type_literal} {
   BEGIN(AFTER_OPERAND);
-  return ARG_TOKEN(sized_type_literal, yytext);
+  return CARBON_ARG_TOKEN(sized_type_literal, yytext);
 }
 
 {intrinsic_identifier} {
@@ -268,7 +244,7 @@ string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
   Carbon::ErrorOr<Carbon::IntrinsicExpression::Intrinsic> intrinsic =
       Carbon::IntrinsicExpression::FindIntrinsic(yytext, context.source_loc());
   if (intrinsic.ok()) {
-    return ARG_TOKEN(intrinsic_identifier, *intrinsic);
+    return CARBON_ARG_TOKEN(intrinsic_identifier, *intrinsic);
   } else {
     return context.RecordSyntaxError(intrinsic.error().message());
   }
@@ -276,7 +252,7 @@ string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
 
 {identifier} {
   BEGIN(AFTER_OPERAND);
-  return ARG_TOKEN(identifier, yytext);
+  return CARBON_ARG_TOKEN(identifier, yytext);
 }
 
 {integer_literal} {
@@ -286,69 +262,86 @@ string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
     return context.RecordSyntaxError(
         llvm::formatv("Invalid integer literal: {0}", yytext));
   }
-  return ARG_TOKEN(integer_literal, val);
-}
-
-{string_literal} {
-  llvm::StringRef str(yytext);
-  CARBON_CHECK(str.consume_front("\"") && str.consume_back("\""));
-  std::optional<std::string> unescaped = Carbon::UnescapeStringLiteral(str);
-  if (unescaped == std::nullopt) {
-    return context.RecordSyntaxError(
-        llvm::formatv("Invalid escaping in string: {0}", yytext));
-  }
-  return ARG_TOKEN(string_literal, *unescaped);
+  return CARBON_ARG_TOKEN(integer_literal, val);
 }
 
-\"\"\" {
-  // Block string literal.
-  std::string s(yytext);
-  // Scans for the closing """, checking for possible escape sequences
-  // like \""".
-  for (;;) {
-    int c = ReadChar(yyscanner, context);
-    if (c <= 0) {
-      return SIMPLE_TOKEN(END_OF_FILE);
-    }
-    s.push_back(c);
-    if (c != '"' && c != '\\') {
-      continue;
-    }
-    if (c == '\\') {
-      // \" in \""" is not a terminator.
-      c = ReadChar(yyscanner, context);
-      if (c <= 0) {
-        return SIMPLE_TOKEN(END_OF_FILE);
-      }
-      s.push_back(c);
-      continue;
-    }
-
-    c = ReadChar(yyscanner, context);
-    if (c <= 0) {
-      return SIMPLE_TOKEN(END_OF_FILE);
-    }
-    s.push_back(c);
-    if (c != '"') {
-      continue;
-    }
-
-    c = ReadChar(yyscanner, context);
-    if (c <= 0) {
-      return SIMPLE_TOKEN(END_OF_FILE);
-    }
-    s.push_back(c);
-    if (c == '"') {
-      break;
+#*(\"\"\"|\") {
+  // Raw string literal.
+  // yytext (the token that matches the above regex) and chars scanned by
+  // str_lex_helper hold the source text, not the string the source represents.
+  Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
+  const std::string& s = str_lex_helper.str();
+  const int hashtag_num = s.find_first_of('"');
+  const int leading_quotes = s.size() - hashtag_num;
+  if (leading_quotes == 3 && hashtag_num > 0) {
+    // Check if it's a single-line string, like #"""#.
+    // TODO: Extend with other single-line string cases, like #""""#, based on
+    // the definition of block string in the design doc.
+    if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
+      return Carbon::ProcessSingleLineString(str_lex_helper.str(), context,
+                                             hashtag_num);
+    } else if (str_lex_helper.is_eof()) {
+      return CARBON_SIMPLE_TOKEN(END_OF_FILE);
     }
+  } else if (!str_lex_helper.Advance()) {
+    return CARBON_SIMPLE_TOKEN(END_OF_FILE);
   }
-  Carbon::ErrorOr<std::string> block_string =
-      Carbon::ParseBlockStringLiteral(s);
-  if (!block_string.ok()) {
-    return context.RecordSyntaxError(llvm::formatv(
-        "Invalid block string: {0}", block_string.error().message()));
+  // 3 quotes indicates multi-line, otherwise it'll be one.
+  const bool multi_line = leading_quotes == 3;
+
+  while (!str_lex_helper.is_eof()) {
+    switch (str_lex_helper.last_char()) {
+      case '\n':  // Fall through.
+      case '\v':  // Fall through.
+      case '\f':  // Fall through.
+      case '\r':
+        if (!multi_line) {
+          return context.RecordSyntaxError(
+              llvm::formatv("missing closing quote in single-line string: {0}",
+                            str_lex_helper.str()));
+        }
+        str_lex_helper.Advance();
+        break;
+      case '"':
+        if (multi_line) {
+          // Check for 2 more '"'s on block string.
+          if (!(str_lex_helper.Advance() &&
+                str_lex_helper.last_char() == '"')) {
+            continue;
+          }
+          if (!(str_lex_helper.Advance() &&
+                str_lex_helper.last_char() == '"')) {
+            continue;
+          }
+          // Now we are at the last " of """.
+        }
+
+        if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
+          // Reach closing quotes, break out of the loop.
+          if (leading_quotes == 3) {
+            return Carbon::ProcessMultiLineString(str_lex_helper.str(), context,
+                                                  hashtag_num);
+          } else {
+            return Carbon::ProcessSingleLineString(str_lex_helper.str(),
+                                                   context, hashtag_num);
+          }
+        }
+        break;
+      case '\\':
+        if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
+          // Read the escaped char.
+          if (!str_lex_helper.Advance()) {
+            continue;
+          }
+          // Read the next char.
+          str_lex_helper.Advance();
+        }
+        break;
+      default:
+        str_lex_helper.Advance();
+    }
   }
-  return ARG_TOKEN(string_literal, *block_string);
+  return CARBON_SIMPLE_TOKEN(END_OF_FILE);
 }
 
 {one_line_comment} {
@@ -380,10 +373,4 @@ string_literal        \"([^\\\"\n\v\f\r]|\\.)*\"
 
 %%
 
-auto ReadChar(yyscan_t yyscanner, Carbon::ParseAndLexContext& context) -> int {
-  const int c = yyinput(yyscanner);
-  if (c <= 0) {
-    context.RecordSyntaxError("Unexpected end of file");
-  }
-  return c;
-}
+auto YyinputWrapper(yyscan_t yyscanner) -> int { return yyinput(yyscanner); }

explorer/testdata/string/block_file_type_indicator.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  var s: String = """filetype
+    A "block" ""string"" literal
+      with file type indicator.
+    """;
+  if (s == "A \"block\" \"\"string\"\" literal\n  with file type indicator.\n") {
+    return 0;
+  } else {
+    return 1;
+  }
+}

explorer/testdata/string/fail_newline.carbon

@@ -11,7 +11,8 @@
 package ExplorerTest api;
 
 fn Main() -> i32 {
-  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+1]]: invalid character '\x22' in source file.
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+2]]: missing closing quote in single-line string: "new
+  // CHECK-EMPTY:
   Print("new
 line");
   return 0;

explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{not} %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon:[[@LINE+1]]: Unexpected end of file
+  var s: String = ##"""
+    error: there are more #s on the left than the right.
+  """#;
+
+  return 0;
+}

explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{not} %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file.
+  var s: String = #"""
+    error: there are more #s on the right than the left.
+  """##;
+
+  return 0;
+}

explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{not} %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """
+  var s: String = #"""
+    error: closing """# is not on its own line.
+  """#;
+
+  return 0;
+}

explorer/testdata/string/fail_raw_block_single_line.carbon

@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{not} %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_single_line.carbon:[[@LINE+1]]: Invalid block string: Too few lines
+  if (s == #"""raw string literal starting with """#) {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr("\"\"raw string literal starting with \"\"");
+}

explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{not} %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon:[[@LINE+2]]: missing closing quote in single-line string: ##"str"#) {
+  // CHECK-EMPTY:
+  if (s == ##"str"#) {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr("str");
+}

explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon

@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{not} %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file.
+  if (s == "str"#) {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr("str");
+}

explorer/testdata/string/raw_basic.carbon

@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  if (s == #"str"#) {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr("str");
+}

explorer/testdata/string/raw_basic_multi_hashtag.carbon

@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  if (s == ##"str"##) {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr(#####"str"#####);
+}

explorer/testdata/string/raw_block.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  var s: String = #"""
+    A "block" ""string"" literal
+      with indent.
+    """#;
+  if (s == #"A \#"block\#" \#"\#"string\#"\#" literal\#n  with indent.\#n"#) {
+    return 0;
+  } else {
+    return 1;
+  }
+}

explorer/testdata/string/raw_block_escaped_back_slash.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  var s: String = #"""
+    A block string literal
+    \
+    """#;
+  if (s == "A block string literal\n\\\n") {
+    return 0;
+  } else {
+    return 1;
+  }
+}

explorer/testdata/string/raw_block_escaped_triple_quotes.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn Main() -> i32 {
+  var s: String = #"""
+    A block string literal
+    \#"""#
+    """#;
+  if (s == "A block string literal\n\"\"\"#\n") {
+    return 0;
+  } else {
+    return 1;
+  }
+}

explorer/testdata/string/raw_nesting.carbon

@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  if (s == ##"#"str"#"##) {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr("#\"str\"#");
+}