Improve vscode syntax highlighting (#6543)

This PR improves the vscode syntax highlighting.

- Added `comment` keys.
- Added highlighting of invalid numbers such as `0x`, `0b`, `0xa`, etc.
- Restricted highlighting of numeric type literals to common types to
avoid highlighting identifiers such as `i1`.
- Changed the highlighting of named operators (e.g., `as`).
- Added `char` and `str` to type literals.
- Added `const` to modifier keywords.
- Removed `addr` keyword.
- Refactored some rules to use `begin`/`end` to handle line breaks.
- Added highlighting to `choice` values as `enum` values.
- Updated the rules for matching `types`.
- Added highlighting to rhs of `adapt`, `alias`, `choice`, `constraint`,
`impl`, `interface`, `as`, and `impls`.
    - Added highlighting to rhs of bindings.
    - Added highlighting to function return types.
- Updated the rules for matching `functions`.
- Updated the rules for matching `variables`.
- Added highlighting unidentified words as `variable`. 
- Added examples and before/after screenshots.

| Before | After |
| :---: | :---: |
| <img width="424" alt="before"
src="https://github.com/user-attachments/assets/e84d0ff9-237b-40c2-845b-ec550b8f7bea">
| <img width="431" alt="now"
src="https://github.com/user-attachments/assets/2c18640b-318a-4cd5-952c-bad61d3fdbca">
|

---------

Co-authored-by: Dana Jansens <danakj@orodu.net>

utils/textmate/Samples/choices.carbon

@@ -0,0 +1,9 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+choice SomeChoice(T:! type) {
+    Val1,
+    Val2(a: i32, b: array(T, 5), t: T),
+    Val3
+}

utils/textmate/Samples/comments.carbon

@@ -0,0 +1,14 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This is a comment.
+    //    Comment with whitespace.
+// class C; fn F[T:! A](a: T) -> i32 { return 86; }
+//@dump-sem-ir-begin
+//@dump-sem-ir-end
+//@dump-sem-ir-
+//no whitespace after double slash
+/* invalid comment pattern. */
+/// invalid comment pattern.
+class C; // Comment after a statement/expression/declaration.

utils/textmate/Samples/customs.carbon

@@ -0,0 +1,11 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+package SomePackage api;
+Core SomeCore;
+
+import SomePackage;
+
+CallFunction(a, b, c.d);
+unidentified;

utils/textmate/Samples/functions_variables.carbon

@@ -0,0 +1,39 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+fn F;
+fn F1();
+fn Func(a: i32, b: A);
+fn Generic[T:! type, U:! I](a: T, b: U) -> U;
+fn Lambda(a: i32) -> i32 => return a + 5;
+fn
+
+    MultiLine[
+        A:! type,
+        B:! C1,
+        C:! C2
+    ](a: A(B, C),
+      b: B,
+      c: i32);
+fn Invalid#+?Name();
+x.(I.F)();
+
+a: A;
+a: A = (b as A);
+a: i32;
+a: array(A, 5);
+a:! type;
+a:! T = b;
+a: A in some_list;
+var a: T;
+let b: T;
+binding: T;
+binding:! T;
+var
+    multiline: T;
+
+fn F(ref a: A, const ref b:! B);
+fn F() -> T;
+fn F() -> T.U { return val; }
+fn F() -> i32 => return val;

utils/textmate/Samples/keywords.carbon

@@ -0,0 +1,46 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// --- operators
+
+>>= <=> <<= &= == != >= >> <= << -= -> -- %= \ = += ++ /= *= & ^ = > < - % . | + / *
+and as impls in like not or partial ref template where
+
+// --- special-keywords
+
+break case continue else if for match return returned then while
+default =>
+
+// --- introducer-keywords
+
+adapt alias choice class constraint fn import interface let library namespace var
+base
+export
+impl
+package
+
+// --- modifier-keywords
+
+abstract const extend extern final private protected virtual
+base class
+default
+export import
+impl fn
+impl library
+impl package
+
+// --- misc-keywords
+
+auto destructor forall friend observe override require
+.base
+
+// --- self-keywords
+
+self Self
+
+// --- underscore
+
+;
+_
+_some_var

utils/textmate/Samples/literals.carbon

@@ -0,0 +1,46 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// --- string
+
+'C'
+"Hello\nWorld\n"
+"\"\'\t\r\n"
+'''Single line string'''
+''' Multi line
+    string
+'''
+
+// --- true-false
+
+true false
+
+// --- type-literals
+
+array bool char str type ;
+i8 i16 i32 i64 i128 i10
+u8 u16 u32 u64 u128 u11
+f8 f16 f32 f64 f128 f12
+
+// --- numbers
+
+12345 6789 00012345
+
+1234.56789
+0.875
+0.3e5
+1e5
+.123
+.1e3
+
+0b1011_0010
+0b_010
+0b2345
+0B011
+
+0x2AE5
+0x_F109
+0xD8_E2.5F_7Bp8
+0xabcdef.abcdef
+0XA85CE

utils/textmate/Samples/types.carbon

@@ -0,0 +1,41 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+class C1;
+class C2(T1:! type, T2:! C1) {}
+class C3 {
+    extend base: C1;
+}
+class
+        (C4(
+            (T1:! type),
+            T2:!
+                    (A.(B(E, F(G)).C)
+                                        .D))) ;
+
+class C5 {
+    impl as I;
+    extend impl as I;
+}
+
+alias a1 = C1;
+alias a2 = i32;
+
+adapt SomeClass;
+adapt {.a: SomeClass, .b: i32};
+adapt i32;
+
+interface I;
+impl C as (I(A, B, C.(D.E(F, G)).H)) {}
+impl C as I where .A = (B where .Self = {}) {}
+impl C as I where .A = {} and .B = C {}
+impl Self as I;
+impl as I;
+
+constraint Constraint {
+    require T impls I;
+    require Self impls I;
+    require impls I;
+    extern require impls I;
+}

utils/textmate/Syntaxes/carbon.tmLanguage.json

@@ -13,13 +13,19 @@
       "include": "#strings"
     },
     {
-      "include": "#operators"
+      "include": "#types"
     },
     {
-      "include": "#special-keywords"
+      "include": "#functions"
     },
     {
-      "include": "#numbers"
+      "include": "#variables"
+    },
+    {
+      "include": "#operators"
+    },
+    {
+      "include": "#special-keywords"
     },
     {
       "include": "#introducer-keywords"
@@ -43,7 +49,10 @@
       "include": "#type-literals"
     },
     {
-      "include": "#functions"
+      "include": "#numbers"
+    },
+    {
+      "include": "#parameter-bindings"
     },
     {
       "include": "#customs"
@@ -103,15 +112,149 @@
         }
       ]
     },
+    "types": {
+      "name": "meta.type.carbon",
+      "patterns": [
+        {
+          "comment": "Matches type, constraint, and implementation declarations, and explicit casts.",
+          "begin": "\\b((adapt|alias|class|constraint|interface|impl)|(as|impls))\\b",
+          "beginCaptures": {
+            "2": { "name": "storage.type.carbon" },
+            "3": { "name": "keyword.control.carbon" }
+          },
+          "end": "(?=[;{])",
+          "patterns": [
+            { "include": "#common" },
+            { "include": "#parameter-bindings" },
+            {
+              "name": "support.type.carbon",
+              "match": "\\w+"
+            }
+          ]
+        },
+        {
+          "comment": "Matches types in bindings.",
+          "begin": ":!?",
+          "end": "(?=([;{=]|->|in|where))",
+          "patterns": [
+            { "include": "#common" },
+            { "include": "#variables" },
+            { "include": "#parameter-bindings" },
+            {
+              "name": "support.type.carbon",
+              "match": "\\w+"
+            }
+          ]
+        },
+        {
+          "comment": "Matches function return types.",
+          "begin": "->",
+          "end": "(?=([;{]|=>))",
+          "patterns": [
+            { "include": "#common" },
+            {
+              "name": "support.type.carbon",
+              "match": "\\w+"
+            }
+          ]
+        },
+        {
+          "comment": "Matches types in where clause.",
+          "begin": "\\bwhere\\b",
+          "beginCaptures": {
+            "0": { "name": "keyword.control.carbon" }
+          },
+          "end": "(?=[;{])",
+          "patterns": [
+            { "include": "#common" },
+            { "include": "#variables" },
+            { "include": "#parameter-bindings" },
+            {
+              "name": "support.type.carbon",
+              "match": "\\w+"
+            }
+          ]
+        },
+        {
+          "comment": "Matches choices.",
+          "begin": "\\bchoice\\b",
+          "beginCaptures": {
+            "0": { "name": "storage.type.carbon" }
+          },
+          "end": "(?=[;}])",
+          "patterns": [
+            { "include": "#common" },
+            { "include": "#variables" },
+            { "include": "#parameter-bindings" },
+            {
+              "name": "support.type.carbon",
+              "match": "\\w+"
+            },
+            {
+              "comment": "Matches the choice values.",
+              "begin": "\\{",
+              "end": "(?=})",
+              "patterns": [
+                { "include": "#common" },
+                {
+                  "name": "variable.other.enummember.carbon",
+                  "match": "\\w+"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    "functions": {
+      "name": "meta.function.carbon",
+      "patterns": [
+        {
+          "comment": "Matches function declarations.",
+          "begin": "\\bfn\\b",
+          "beginCaptures": {
+            "0": { "name": "storage.type.carbon" }
+          },
+          "end": "(?=[\\[\\(])",
+          "patterns": [
+            { "include": "#common" },
+            {
+              "name": "support.function.carbon",
+              "match": "\\w+"
+            }
+          ]
+        }
+      ]
+    },
+    "variables": {
+      "name": "meta.variable.carbon",
+      "patterns": [
+        {
+          "comment": "Matches variable declarations.",
+          "begin": "\\b(var|let)\\b",
+          "beginCaptures": {
+            "0": { "name": "storage.type.carbon" }
+          },
+          "end": "(?=:)",
+          "patterns": [
+            { "include": "#common" },
+            {
+              "name": "support.variable.carbon",
+              "match": "\\w+"
+            }
+          ]
+        }
+      ]
+    },
     "operators": {
       "patterns": [
         {
           "name": "keyword.operator.carbon",
-          "match": "\\b(>>=|<=>|<<=|\\&=|==|!=|>=|>>|<=|<<|-=|->|--|%=|\\|=|\\+=|\\+\\+|/=|\\*=|\\&|\\^|=|>|<|-|%|.|\\||\\+|/|\\*)\\b"
+          "match": "\\b(>>=|<=>|<<=|\\&=|==|!=|>=|>>|<=|<<|-=|->|--|%=|\\|=|\\+=|\\+\\+|/=|\\*=|\\&|\\^|=|>|<|-|%|\\.|\\||\\+|/|\\*)\\b"
         },
         {
-          "name": "keyword.operator.carbon",
-          "match": "\\b(addr|and|as|impls|in|like|not|or|partial|template|where)\\b"
+          "name": "keyword.control.carbon",
+          "match": "\\b(and|as|impls|in|like|not|or|partial|ref|template|where)\\b"
         }
       ]
     },
@@ -155,7 +298,7 @@
       "patterns": [
         {
           "name": "storage.modifier.carbon",
-          "match": "\\b(abstract|extend|extern|final|private|protected|virtual)\\b"
+          "match": "\\b(abstract|const|extend|extern|final|private|protected|virtual)\\b"
         },
         {
           "name": "storage.modifier.carbon",
@@ -215,76 +358,84 @@
       "patterns": [
         {
           "name": "constant.language.carbon",
-          "match": "\\b(bool|[iuf][1-9][0-9]*|type)\\b"
+          "match": "\\b(array|bool|char|str|type)\\b"
+        },
+        {
+          "name": "constant.language.carbon",
+          "comment": "Match common numeric type literals. Language server should highlight other valid literals.",
+          "match": "\\b[iuf](8|16|32|64|128)\\b"
         }
       ]
     },
-    "functions": {
+    "numbers": {
+      "comment": "Invalid formats are also highlighted. Language server handles error highlighting",
       "patterns": [
         {
-          "name": "support.function.carbon",
-          "match": "\\b[a-zA-Z]+\\s*\\("
+          "name": "constant.numeric.carbon",
+          "match": "0x[_0-9a-fA-F]*(\\.[_0-9a-fA-F]+(p[-+]?[0-9]+)?)?"
+        },
+        {
+          "name": "constant.numeric.carbon",
+          "match": "0b[_01]*"
+        },
+        {
+          "name": "constant.numeric.carbon",
+          "match": "[0-9][_0-9]*(\\.[_0-9]+(e[-+]?[0-9]+)?)?"
+        }
+      ]
+    },
+    "parameter-bindings": {
+      "patterns": [
+        {
+          "comment": "Matches parameter bindings.",
+          "match": "\\b(\\w+)\\s*(?=:)",
+          "captures": {
+            "1": { "name": "support.variable.carbon" }
+          }
         }
       ]
     },
     "customs": {
+      "comment": "For rules that should be checked last.",
       "patterns": [
         {
           "name": "support.class.carbon",
           "match": "(?<=\\b(package|Core)\\s)\\w+"
         },
         {
-          "name": "support.variable.carbon",
-          "match": "(?<=\\s*\\.)\\w+"
-        },
-        {
-          "name": "support.variable.carbon",
-          "match": "(?<=\\bas\\s)\\w+"
-        },
-        {
-          "name": "support.variable.carbon",
-          "match": "\\w+(?=\\s*:)"
-        },
-        {
-          "name": "support.variable.carbon",
-          "match": "(?<=\\blet\\s)\\w+"
-        },
-        {
-          "name": "support.variable.carbon",
-          "match": "(?<=\\bvar\\s)\\w+"
+          "name": "support.type.property-name.carbon",
+          "match": "(?<=\\bimport\\s)\\w+"
         },
         {
+          "comment": "Matches function calls.",
           "name": "support.function.carbon",
-          "match": "(?<=\\bfn\\s)\\w+"
-        },
-        {
-          "name": "support.other.carbon",
-          "match": "(?<=\\bimpl\\s)\\w+"
+          "match": "\\b\\w+\\s*\\("
         },
         {
-          "name": "support.type.property-name.carbon",
-          "match": "(?<=\\bimport\\s)\\w+"
-        },
-        {
-          "name": "support.class.carbon",
-          "match": "(?<=\\bclass\\s)\\w+"
+          "comment": "Matches unidentified words as variables.",
+          "name": "support.variable.carbon",
+          "match": "\\b\\w+\\b"
         }
       ]
     },
-    "numbers": {
+    "common": {
+      "comments": [
+        "For including comments, operators, keywords, literals, and language constants ",
+        "in other rules that use begin/end. This way we do not accidentally override their ",
+        "highlighting in complex rules."
+      ],
       "patterns": [
-        {
-          "name": "constant.numeric.carbon",
-          "match": "0x[_0-9A-F]+(\\.[_0-9A-F]+(p[-+]?[1-9][0-9]*)?)?"
-        },
-        {
-          "name": "constant.numeric.carbon",
-          "match": "0b[_01]+"
-        },
-        {
-          "name": "constant.numeric.carbon",
-          "match": "[1-9][_0-9]*(\\.[_0-9]+(e[-+]?[1-9][0-9]*)?)?"
-        }
+        { "include": "#comments" },
+        { "include": "#strings" },
+        { "include": "#operators" },
+        { "include": "#special-keywords" },
+        { "include": "#introducer-keywords" },
+        { "include": "#modifier-keywords" },
+        { "include": "#misc-keywords" },
+        { "include": "#self-keywords" },
+        { "include": "#true-false" },
+        { "include": "#type-literals" },
+        { "include": "#numbers" }
       ]
     }
   }