improve strings and scanner for text block

grammar.js

@@ -137,7 +137,7 @@ module.exports = grammar({
     _text_block_definition: ($) =>
       seq(
         optional($.language),
-        $._eol,
+        /\s/,
         optional(alias($._text_block_raw, $.raw_text))
       ),
 
@@ -285,8 +285,44 @@ module.exports = grammar({
 
     string: ($) =>
       choice(
-        seq("'", repeat(token.immediate(/[^'\n]+/)), "'"),
+        seq(
-        seq('"', repeat(token.immediate(/[^"\n]+/)), '"')
+          "'",
+          repeat(
+            choice(
+              alias($._unescaped_single_string_fragment, $.string_fragment),
+              $.escape_sequence
+            )
+          ),
+          "'"
+        ),
+        seq(
+          '"',
+          repeat(
+            choice(
+              alias($._unescaped_double_string_fragment, $.string_fragment),
+              $.escape_sequence
+            )
+          ),
+          '"'
+        )
+      ),
+
+    _unescaped_single_string_fragment: ($) => token.immediate(/[^'\\\n]+/),
+
+    _unescaped_double_string_fragment: ($) => token.immediate(/[^"\\\n]+/),
+
+    escape_sequence: ($) =>
+      token.immediate(
+        seq(
+          "\\",
+          choice(
+            /[^xu0-7]/,
+            /[0-7]{1,3}/,
+            /x[0-9a-fA-F]{2}/,
+            /u[0-9a-fA-F]{4}/,
+            /u{[0-9a-fA-F]+}/
+          )
+        )
       ),
 
     boolean: ($) => choice("true", "false"),

queries/highlights.scm

@@ -8,10 +8,11 @@
 ; Literals
 ;-------------------------------------------------------------------------------
 
-(language) @type.qualifier
+(string) @string
-(container_key (string) @string)
+(container_key (string (string_fragment) @string))
-(shape_key (string) @string)
+(shape_key (string (string_fragment) @string))
-(label) @string.special
+(escape_sequence) @string.escape
+(label) @text.title
 (attr_value) @string
 (integer) @number
 (float) @float
@@ -20,6 +21,7 @@
 ; Comments
 ;-------------------------------------------------------------------------------
 
+(language) @comment
 (line_comment) @comment.line
 
 ; Punctiation

scripts/pre-push

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+# Can be used as a pre-push hook
+# Just symlink this file to .git/hooks/pre-push
+
+set -xe
+
+if [ ! -z $(git diff --cached --name-only | grep -e "^grammar.js$") ]
+then
+  make build
+  make build-wasm
+fi
+

src/grammar.json

@@ -592,8 +592,8 @@
           ]
         },
         {
-          "type": "SYMBOL",
+          "type": "PATTERN",
-          "name": "_eol"
+          "value": "\\s"
         },
         {
           "type": "CHOICE",
@@ -1244,11 +1244,22 @@
             {
               "type": "REPEAT",
               "content": {
-                "type": "IMMEDIATE_TOKEN",
+                "type": "CHOICE",
+                "members": [
+                  {
+                    "type": "ALIAS",
                     "content": {
-                  "type": "PATTERN",
+                      "type": "SYMBOL",
-                  "value": "[^'\\n]+"
+                      "name": "_unescaped_single_string_fragment"
+                    },
+                    "named": true,
+                    "value": "string_fragment"
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "escape_sequence"
                   }
+                ]
               }
             },
             {
@@ -1267,11 +1278,22 @@
             {
               "type": "REPEAT",
               "content": {
-                "type": "IMMEDIATE_TOKEN",
+                "type": "CHOICE",
+                "members": [
+                  {
+                    "type": "ALIAS",
                     "content": {
-                  "type": "PATTERN",
+                      "type": "SYMBOL",
-                  "value": "[^\"\\n]+"
+                      "name": "_unescaped_double_string_fragment"
+                    },
+                    "named": true,
+                    "value": "string_fragment"
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "escape_sequence"
                   }
+                ]
               }
             },
             {
@@ -1282,6 +1304,57 @@
         }
       ]
     },
+    "_unescaped_single_string_fragment": {
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "PATTERN",
+        "value": "[^'\\\\\\n]+"
+      }
+    },
+    "_unescaped_double_string_fragment": {
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "PATTERN",
+        "value": "[^\"\\\\\\n]+"
+      }
+    },
+    "escape_sequence": {
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "SEQ",
+        "members": [
+          {
+            "type": "STRING",
+            "value": "\\"
+          },
+          {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "PATTERN",
+                "value": "[^xu0-7]"
+              },
+              {
+                "type": "PATTERN",
+                "value": "[0-7]{1,3}"
+              },
+              {
+                "type": "PATTERN",
+                "value": "x[0-9a-fA-F]{2}"
+              },
+              {
+                "type": "PATTERN",
+                "value": "u[0-9a-fA-F]{4}"
+              },
+              {
+                "type": "PATTERN",
+                "value": "u{[0-9a-fA-F]+}"
+              }
+            ]
+          }
+        ]
+      }
+    },
     "boolean": {
       "type": "CHOICE",
       "members": [

src/node-types.json

@@ -289,7 +289,21 @@
   {
     "type": "string",
     "named": true,
-    "fields": {}
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "escape_sequence",
+          "named": true
+        },
+        {
+          "type": "string_fragment",
+          "named": true
+        }
+      ]
+    }
   },
   {
     "type": "text_block",
@@ -366,6 +380,10 @@
     "type": "dot",
     "named": true
   },
+  {
+    "type": "escape_sequence",
+    "named": true
+  },
   {
     "type": "false",
     "named": false
@@ -434,6 +452,10 @@
     "type": "source-arrowhead",
     "named": false
   },
+  {
+    "type": "string_fragment",
+    "named": true
+  },
   {
     "type": "stroke",
     "named": false

src/scanner.c

@@ -31,10 +31,6 @@ static bool scan_raw_text(TSLexer *lexer) {
       return has_content;
     }
 
-    if (lexer->lookahead == '\n') {
-      advance(lexer);
-      lexer->mark_end(lexer);
-
     while (iswspace(lexer->lookahead)) {
       advance(lexer);
       lexer->mark_end(lexer);
@@ -45,6 +41,8 @@ static bool scan_raw_text(TSLexer *lexer) {
     }
 
     if (lexer->lookahead == '|') {
+      advance(lexer);
+      if (lexer->lookahead == '\n') {
         return has_content;
       }
     }

test/corpus/attributes.txt

@@ -26,7 +26,7 @@ near: abc
   (attribute (attr_key) (attr_value))
 
   (attribute (attr_key (reserved)) (attr_value))
-  (attribute (attr_key (reserved)) (attr_value (string)))
+  (attribute (attr_key (reserved)) (attr_value (string (string_fragment))))
   (attribute (attr_key (reserved)) (attr_value))
   (attribute (attr_key (reserved)) (attr_value))
   (attribute (attr_key (reserved)) (attr_value (float)))
@@ -134,7 +134,7 @@ foo: {
     (container_key)
     (block
       (attribute (attr_key) (attr_value))
-      (attribute (attr_key) (attr_value (string)))
+      (attribute (attr_key) (attr_value (string (string_fragment))))
       (attribute (attr_key) (attr_value))
       (attribute (attr_key) (attr_value))
       (attribute (attr_key) (attr_value (integer)))
@@ -381,7 +381,7 @@ foo -> bar: {
         (attr_key)
         (block
           (attribute (attr_key) (attr_value))
-          (attribute (attr_key) (attr_value (string)))
+          (attribute (attr_key) (attr_value (string (string_fragment))))
           (attribute (attr_key) (attr_value))
           (attribute (attr_key) (attr_value))
           (attribute (attr_key) (attr_value (integer)))

test/corpus/container.txt

@@ -29,8 +29,8 @@ Use quoted string as keys
 
 (source_file
   (container
-    (container_key (string)) (dot)
+    (container_key (string (string_fragment))) (dot)
-    (shape (shape_key (string)))
+    (shape (shape_key (string (string_fragment))))
   )
 )
 

test/corpus/shape.txt

@@ -54,7 +54,7 @@ Use quoted string as a shape key
 --------------------------------------------------------------------------------
 
 (source_file
-  (shape (shape_key (string)))
+  (shape (shape_key (string (string_fragment))))
 )
 
 ================================================================================
@@ -108,7 +108,9 @@ Use quoted string as shape key and label
 --------------------------------------------------------------------------------
 
 (source_file
-  (shape (shape_key (string)) (label (string)))
+  (shape
+    (shape_key (string (string_fragment)))
+    (label (string (string_fragment))))
 )
 
 ================================================================================

test/highlight/shape.d2

@@ -2,21 +2,28 @@ foo.'baz'.biz
 # <- constant
 #  ^ punctuation.delimiter
 #   ^ string
+#     ^ string
 #        ^ punctuation.delimiter
 #         ^ variable
 
-'biz': 'Baz'
+'b\nz'
+# ^ string.escape
+#  ^ string.escape
+
+'biz': 'Baz\nBiz'
 # <- string
-#      ^ string.special
+#      ^ string
+#        ^ string
+#           ^ string.escape
 
 foo: Foo Bar
 # <- variable
 #  ^ punctuation.delimiter
-#    ^ string.special
+#    ^ text.title
 
 foo: Foo Bar {
   # <- constant
-  #  ^ string.special
+  #  ^ text.title
   #          ^ punctuation.bracket
 
   bar.baz