Get the VS Code extension working again. (#4037)

VSCode's JSON parser no longer accepts comments in at least textmate
grammars, so remove the comments. It's unclear whether VSCode changed
here or whether this hasn't worked since the comments were added.

Clean up the textmate grammar: refactor, add some missing operators and
keywords, classify keywords as introducer / modifier as appropriate, fix
comment grammar to require space after `//`, and generalize recognition
of iN, fN, uN to recognize all such type literals.

Be more cautious in language server -- I was seeing frequent crashes for
`IdentifierName` parse nodes whose token was not actually an identifier,
presumably due to error recovery.

.pre-commit-config.yaml

@@ -179,10 +179,6 @@ repos:
             Part of the Carbon Language project, under the Apache License v2.0 with LLVM
             Exceptions. See /LICENSE for license information.
             SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-          - --skip_pattern
-          # The default skip pattern skips JSON which we use with VSCode and
-          # it, like most file-based parsers, can support comments easily.
-          - '(^LICENSE|\.(ico))$'
           - --custom_format
           - '\.(carbon|c|json|proto|ypp)(\.tmpl)?$'
           - ''
@@ -217,11 +213,9 @@ repos:
           (?x)^(
               .bazelversion|
               .github/pull_request_template.md|
-              .vscode/extensions.json|
               compile_flags.txt|
               github_tools/requirements.txt|
               third_party/.*|
-              utils/vscode/package(-lock)?.json|
               .*\.def|
               .*\.svg|
               .*/fuzzer_corpus/.*|

language_server/language_server.cpp

@@ -83,8 +83,10 @@ static auto GetIdentifierName(const SharedValueStores& value_stores,
     -> std::optional<llvm::StringRef> {
   for (auto ch : p.children(node)) {
     if (p.node_kind(ch) == Parse::NodeKind::IdentifierName) {
-      return value_stores.identifiers().Get(
-          tokens.GetIdentifier(p.node_token(ch)));
+      auto token = p.node_token(ch);
+      if (tokens.GetKind(token) == Lex::TokenKind::Identifier) {
+        return value_stores.identifiers().Get(tokens.GetIdentifier(token));
+      }
     }
   }
   return std::nullopt;

utils/textmate/Syntaxes/carbon.tmLanguage.json

@@ -1,9 +1,7 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 {
   "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
   "name": "carbon",
+  "scopeName": "source.carbon",
   "foldingStartMarker": "\\{\\s*$",
   "foldingStopMarker": "^\\s*\\}",
   "fileTypes": ["carbon"],
@@ -11,6 +9,9 @@
     {
       "include": "#comments"
     },
+    {
+      "include": "#strings"
+    },
     {
       "include": "#operators"
     },
@@ -21,78 +22,88 @@
       "include": "#numbers"
     },
     {
-      "include": "#reserved-words"
+      "include": "#introducer-keywords"
     },
     {
-      "include": "#operator-dedicated-keywords-statements"
+      "include": "#modifier-keywords"
     },
     {
-      "include": "#ctrl-statements"
+      "include": "#misc-keywords"
     },
     {
-      "include": "#true-false"
+      "include": "#self-keywords"
     },
     {
-      "include": "#functions"
+      "include": "#underscore"
     },
     {
-      "name": "string.quoted.triple.carbon",
-      "begin": "'''([^\\s'#]*\\n)?",
-      "end": "'''",
-      "beginCaptures": {
-        "1": {
-          "name": "constant.character.escape.carbon"
-        }
-      },
-      "patterns": [
-        {
-          "include": "#string_escapes"
-        }
-      ]
+      "include": "#true-false"
     },
     {
-      "name": "string.quoted.double.carbon",
-      "begin": "\"",
-      "end": "\"",
-      "patterns": [
-        {
-          "include": "#string_escapes"
-        }
-      ]
+      "include": "#type-literals"
+    },
+    {
+      "include": "#functions"
     },
     {
       "include": "#customs"
     }
   ],
   "repository": {
-    "operators": {
+    "comments": {
       "patterns": [
         {
-          "name": "keyword.operator.carbon",
-          "match": "\\b(\\+|-|\\*|/|!)\\b"
+          "name": "comment.line.carbon",
+          "match": "^\\s*(?=//\\s).*$"
         }
       ]
     },
-    "comments": {
+    "string_escapes": {
+      "patterns": [
+        {
+          "name": "constant.character.escape.carbon",
+          "match": "\\\\([tnr'\"0\\0]|x[0-9A-F]{2}|u\\{[0-9A-F]{4,}\\})"
+        }
+      ]
+    },
+    "strings": {
       "patterns": [
         {
-          "begin": "(^[ \\t]+)?(?=//)",
-          "end": "(?!\\G)",
+          "name": "string.quoted.triple.carbon",
+          "begin": "'''([^\\s'#]*\\n)?",
+          "end": "'''",
+          "beginCaptures": {
+            "1": {
+              "name": "constant.character.escape.carbon"
+            }
+          },
+          "patterns": [
+            {
+              "include": "#string_escapes"
+            }
+          ]
+        },
+        {
+          "name": "string.quoted.double.carbon",
+          "begin": "\"",
+          "end": "\"",
           "patterns": [
             {
-              "name": "comment.line.carbon",
-              "begin": "//",
-              "end": "$"
+              "include": "#string_escapes"
             }
           ]
         }
       ]
     },
-    "string_escapes": {
+    "operators": {
       "patterns": [
         {
-          "name": "constant.character.escape.carbon",
-          "match": "\\\\([tnr'\"0\\0]|x[0-9A-F]{2}|u\\{[0-9A-F]{4,}\\})"
+          "name": "keyword.operator.carbon",
+          "match": "\\b(>>=|<=>|<<=|\\&=|==|!=|>=|>>|<=|<<|-=|->|--|%=|\\|=|\\+=|\\+\\+|/=|\\*=|\\&|\\^|=|>|<|-|%|.|\\||\\+|/|\\*)\\b"
+        },
+        {
+          "name": "keyword.operator.carbon",
+          "match": "\\b(addr|and|as|impls|in|like|not|or|partial|template|where)\\b"
         }
       ]
     },
@@ -104,19 +115,75 @@
         }
       ]
     },
-    "operator-dedicated-keywords-statements": {
+    "introducer-keywords": {
+      "patterns": [
+        {
+          "name": "storage.type.carbon",
+          "match": "\\b(adapt|alias|choice|class|constraint|fn|import|interface|let|library|namespace|var)\\b"
+        },
+        {
+          "name": "storage.type.carbon",
+          "match": "\\b(base)\\b(?!\\s*class\\b)"
+        },
+        {
+          "name": "storage.type.carbon",
+          "match": "\\b(export)\\b(?!\\s*import\\b)"
+        },
+        {
+          "name": "storage.type.carbon",
+          "match": "\\b(impl)\\b(?!\\s*(fn|library|package)\\b)"
+        },
+        {
+          "name": "storage.type.carbon",
+          "match": "\\b(package)\\b(?!\\.)"
+        }
+      ]
+    },
+    "modifier-keywords": {
+      "patterns": [
+        {
+          "name": "storage.modifier.carbon",
+          "match": "\\b(abstract|default|extend|extern|final|private|protected|virtual)\\b"
+        },
+        {
+          "name": "storage.modifier.carbon",
+          "match": "\\b(base)\\b(?=\\s*class\\b)"
+        },
+        {
+          "name": "storage.modifier.carbon",
+          "match": "\\b(export)\\b(?=\\s*import\\b)"
+        },
+        {
+          "name": "storage.modifier.carbon",
+          "match": "\\b(impl)\\b(?=\\s*(fn|library|package)\\b)"
+        }
+      ]
+    },
+    "misc-keywords": {
       "patterns": [
         {
           "name": "keyword.other.carbon",
-          "match": "\\b(abstract|adapt|addr|alias|and|api|as|auto|base|choice|class|constraint|destructor|extend|final|fn|forall|friend|impl|impls|import|in|interface|let|library|like|namespace|not|observe|or|override|package|partial|private|protected|require|Self|template|type|var|virtual|where|_)\\b"
+          "match": "\\b(auto|destructor|forall|friend|observe|override|require)\\b"
+        },
+        {
+          "name": "keyword.other.carbon",
+          "match": "(?<=\\.)\\b(base)\\b"
+        }
+      ]
+    },
+    "self-keywords": {
+      "patterns": [
+        {
+          "name": "variable.language.carbon",
+          "match": "\\b(self|Self)\\b"
         }
       ]
     },
-    "reserved-words": {
+    "underscore": {
       "patterns": [
         {
-          "name": "support.type.carbon",
-          "match": "\\b(As|bool|Carbon\\.Int|Carbon\\.UInt|f16|f32|f64|f128|i8|i16|i32|i64|i128|i256|Slice|String|StringView|type|u8|u16|u32|u64|u128|u256)\\b"
+          "name": "variable.language.carbon",
+          "match": "\\b(_)\\b"
         }
       ]
     },
@@ -128,6 +195,14 @@
         }
       ]
     },
+    "type-literals": {
+      "patterns": [
+        {
+          "name": "constant.language.carbon",
+          "match": "\\b(bool|[iuf][1-9][0-9]*|type)\\b"
+        }
+      ]
+    },
     "functions": {
       "patterns": [
         {
@@ -196,6 +271,5 @@
         }
       ]
     }
-  },
-  "scopeName": "source.carbon"
+  }
 }

utils/vscode/language-configuration.json

@@ -1,18 +1,12 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
 {
   "comments": {
     "lineComment": "//"
   },
-  // Symbols used as brackets.
   "brackets": [
     ["{", "}"],
     ["[", "]"],
     ["(", ")"]
   ],
-  // Symbols that are auto closed when typing.
   "autoClosingPairs": [
     ["{", "}"],
     ["[", "]"],
@@ -20,7 +14,6 @@
     ["\"", "\""],
     ["'", "'"]
   ],
-  // Symbols that can be used to surround a selection.
   "surroundingPairs": [
     ["{", "}"],
     ["[", "]"],

utils/vscode/package.json

@@ -39,9 +39,10 @@
     "package": "mkdir -p out && vsce package -o out/carbon.vsix"
   },
   "devDependencies": {
-    "@vscode/vsce": "^2.19.0"
+    "@vscode/vsce": "^2.27.0"
   },
   "dependencies": {
+    "vsce": "^2.15.0",
     "vscode-languageclient": "^8.1.0"
   }
 }