add language injections

Closes #6

binding.gyp

@@ -9,7 +9,7 @@
       "sources": [
         "bindings/node/binding.cc",
         "src/parser.c",
-        # If your language uses an external scanner, add it here.
+        "src/scanner.c",
       ],
       "cflags_c": [
         "-std=c99",

bindings/rust/build.rs

@@ -13,11 +13,9 @@ fn main() {
     // If your language uses an external scanner written in C,
     // then include this block of code:
 
-    /*
     let scanner_path = src_dir.join("scanner.c");
     c_config.file(&scanner_path);
     println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
 
     c_config.compile("parser");
     println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());

dev/nvim-ts.nix

@@ -10,7 +10,7 @@ let
   };
 
   tree-sitter = (pkgs.tree-sitter.override { inherit extraGrammars; });
-  grammars = tree-sitter.withPlugins (g: [ g.tree-sitter-d2 g.tree-sitter-javascript ]);
+  grammars = tree-sitter.withPlugins (g: tree-sitter.allGrammars);
 
   nvim-treesitter = pkgs.vimPlugins.nvim-treesitter.overrideAttrs (oldAttrs: {
     postPatch = ''

examples/twitter.d2

@@ -1,11 +1,12 @@
 timeline mixer: "" {
   explanation: |md
-    ## **Timeline mixer**
-    - Inject ads, who-to-follow, onboarding
-    - Conversation module
-    - Cursoring,pagination
-    - Tweat deduplication
-    - Served data logging
+## **Timeline mixer**
+
+- **Inject ads**, who-to-follow, onboarding
+- Conversation module
+- Cursoring,pagination
+- Tweat deduplication
+- Served data logging
   |
 }
 People discovery: "People discovery \nservice"
@@ -27,7 +28,7 @@ container0.graphql: GraphQL\nFederated Strato Column {
   icon: https://upload.wikimedia.org/wikipedia/commons/thumb/1/17/GraphQL_Logo.svg/1200px-GraphQL_Logo.svg.png
 }
 container0.comment: |md
-  ## Tweet/user content hydration, visibility filtering
+## Tweet/user content hydration, visibility filtering
 |
 container0.tlsapi: TLS-API (being deprecated)
 container0.graphql -> timeline mixer

grammar.js

@@ -15,6 +15,8 @@ const PREC = {
 module.exports = grammar({
   name: "d2",
 
+  externals: ($) => [$._text_block_raw],
+
   extras: ($) => [
     /[ \f\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/,
     $.line_comment,
@@ -132,10 +134,11 @@ module.exports = grammar({
       ),
 
     _text_block_definition: ($) =>
-      seq(optional($.language), $._eol, optional($.text_block_content)),
-
-    text_block_content: ($) =>
-      repeat1(token(prec(PREC.TEXT_BLOCK_CONTENT, /.*?[^`|]/))),
+      seq(
+        optional($.language),
+        $._eol,
+        optional(alias($._text_block_raw, $.raw_text))
+      ),
 
     language: ($) => /\w+/,
 

queries/highlights.scm

@@ -8,6 +8,7 @@
 ; Literals
 ;-------------------------------------------------------------------------------
 
+(language) @string.special
 (container_key (string) @string.special)
 (shape_key (string) @string)
 (label) @string 

queries/injections.scm

@@ -0,0 +1,35 @@
+; for tree-sitter
+
+(
+  (text_block (raw_text) @injection.content)
+  (#set! injection.language "markdown")
+)
+
+(text_block
+  (language) @injection.language
+  (raw_text) @injection.content
+)
+
+;; overwrite for nvim-treesitter
+; use markdown as default
+(text_block . (raw_text) @markdown)
+
+; add alias for markdown
+(text_block
+  (language) @_language
+  (raw_text) @markdown
+  (#eq? @_language "md")
+)
+
+; add alias for javascript
+(text_block
+  (language) @_language
+  (raw_text) @javascript
+  (#eq? @_language "js")
+)
+
+(text_block
+  (language) @language
+  (raw_text) @content
+)
+

src/grammar.json

@@ -569,8 +569,13 @@
           "type": "CHOICE",
           "members": [
             {
-              "type": "SYMBOL",
-              "name": "text_block_content"
+              "type": "ALIAS",
+              "content": {
+                "type": "SYMBOL",
+                "name": "_text_block_raw"
+              },
+              "named": true,
+              "value": "raw_text"
             },
             {
               "type": "BLANK"
@@ -579,20 +584,6 @@
         }
       ]
     },
-    "text_block_content": {
-      "type": "REPEAT1",
-      "content": {
-        "type": "TOKEN",
-        "content": {
-          "type": "PREC",
-          "value": -1,
-          "content": {
-            "type": "PATTERN",
-            "value": ".*?[^`|]"
-          }
-        }
-      }
-    },
     "language": {
       "type": "PATTERN",
       "value": "\\w+"
@@ -1339,7 +1330,12 @@
     ]
   ],
   "precedences": [],
-  "externals": [],
+  "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "_text_block_raw"
+    }
+  ],
   "inline": [],
   "supertypes": []
 }

src/node-types.json

@@ -287,17 +287,12 @@
           "named": true
         },
         {
-          "type": "text_block_content",
+          "type": "raw_text",
           "named": true
         }
       ]
     }
   },
-  {
-    "type": "text_block_content",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "\u0000",
     "named": false
@@ -394,6 +389,10 @@
     "type": "opacity",
     "named": false
   },
+  {
+    "type": "raw_text",
+    "named": true
+  },
   {
     "type": "shadow",
     "named": false

src/scanner.c

@@ -0,0 +1,68 @@
+#include <tree_sitter/parser.h>
+#include <wctype.h>
+
+enum TokenType {
+  RAW_TEXT
+};
+
+void *tree_sitter_d2_external_scanner_create() { return NULL; }
+void tree_sitter_d2_external_scanner_destroy(void *payload) { }
+unsigned tree_sitter_d2_external_scanner_serialize(void *p, char *buffer) { return 0; }
+void tree_sitter_d2_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
+
+static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+static bool scan_raw_text(TSLexer *lexer) {
+  lexer->result_symbol = RAW_TEXT;
+
+  if (lexer->lookahead == '`') {
+    advance(lexer);
+  }
+
+  if (lexer->lookahead == '|') {
+    return false;
+  }
+
+  for (bool has_content = false;; has_content = true) {
+    lexer->mark_end(lexer);
+
+    if (lexer->lookahead == '\0') {
+      return has_content;
+    }
+
+    if (lexer->lookahead == '\n') {
+      advance(lexer);
+      lexer->mark_end(lexer);
+
+      while (iswspace(lexer->lookahead)) {
+        advance(lexer);
+        lexer->mark_end(lexer);
+      }
+
+      if (lexer->lookahead == '`') {
+        advance(lexer);
+      }
+
+      if (lexer->lookahead == '|') {
+        return has_content;
+      }
+    }
+
+    advance(lexer);
+  }
+
+  return false;
+}
+
+bool tree_sitter_d2_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  if (valid_symbols[RAW_TEXT]) {
+    return scan_raw_text(lexer);
+  }
+
+  return false;
+}

test/corpus/shape.txt

@@ -122,7 +122,7 @@ foo: |
   (shape
     (shape_key)
     (text_block
-      (text_block_content)
+      (raw_text)
     )
   )
 )
@@ -149,7 +149,7 @@ foo: |go
     (shape_key)
     (text_block
       (language)
-      (text_block_content)
+      (raw_text)
     )
   )
 )
@@ -176,7 +176,7 @@ foo: |`go
     (shape_key)
     (text_block
       (language)
-      (text_block_content)
+      (raw_text)
     )
   )
 )