diff --git a/assets/screenshot.png b/assets/screenshot.png index aa55d09..e744b25 100644 Binary files a/assets/screenshot.png and b/assets/screenshot.png differ diff --git a/binding.gyp b/binding.gyp index 919050d..efea429 100644 --- a/binding.gyp +++ b/binding.gyp @@ -9,7 +9,7 @@ "sources": [ "bindings/node/binding.cc", "src/parser.c", - # If your language uses an external scanner, add it here. + "src/scanner.c", ], "cflags_c": [ "-std=c99", diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index c6061f0..f7aaf16 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -13,11 +13,9 @@ fn main() { // If your language uses an external scanner written in C, // then include this block of code: - /* let scanner_path = src_dir.join("scanner.c"); c_config.file(&scanner_path); println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); - */ c_config.compile("parser"); println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); diff --git a/dev/nvim-ts.nix b/dev/nvim-ts.nix index db59445..3f4c835 100644 --- a/dev/nvim-ts.nix +++ b/dev/nvim-ts.nix @@ -10,7 +10,7 @@ let }; tree-sitter = (pkgs.tree-sitter.override { inherit extraGrammars; }); - grammars = tree-sitter.withPlugins (g: [ g.tree-sitter-d2 g.tree-sitter-javascript ]); + grammars = tree-sitter.withPlugins (g: tree-sitter.allGrammars); nvim-treesitter = pkgs.vimPlugins.nvim-treesitter.overrideAttrs (oldAttrs: { postPatch = '' diff --git a/examples/twitter.d2 b/examples/twitter.d2 index 6b67852..c3e2128 100644 --- a/examples/twitter.d2 +++ b/examples/twitter.d2 @@ -1,11 +1,12 @@ timeline mixer: "" { explanation: |md - ## **Timeline mixer** - - Inject ads, who-to-follow, onboarding - - Conversation module - - Cursoring,pagination - - Tweat deduplication - - Served data logging +## **Timeline mixer** + +- **Inject ads**, who-to-follow, onboarding +- Conversation module +- Cursoring,pagination +- Tweat deduplication +- Served data logging | } People discovery: "People discovery \nservice" @@ -27,7 +28,7 @@ container0.graphql: GraphQL\nFederated Strato Column { icon: https://upload.wikimedia.org/wikipedia/commons/thumb/1/17/GraphQL_Logo.svg/1200px-GraphQL_Logo.svg.png } container0.comment: |md - ## Tweet/user content hydration, visibility filtering +## Tweet/user content hydration, visibility filtering | container0.tlsapi: TLS-API (being deprecated) container0.graphql -> timeline mixer diff --git a/grammar.js b/grammar.js index 8bff710..2ea0227 100644 --- a/grammar.js +++ b/grammar.js @@ -15,6 +15,8 @@ const PREC = { module.exports = grammar({ name: "d2", + externals: ($) => [$._text_block_raw], + extras: ($) => [ /[ \f\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/, $.line_comment, @@ -132,10 +134,11 @@ module.exports = grammar({ ), _text_block_definition: ($) => - seq(optional($.language), $._eol, optional($.text_block_content)), - - text_block_content: ($) => - repeat1(token(prec(PREC.TEXT_BLOCK_CONTENT, /.*?[^`|]/))), + seq( + optional($.language), + $._eol, + optional(alias($._text_block_raw, $.raw_text)) + ), language: ($) => /\w+/, diff --git a/queries/highlights.scm b/queries/highlights.scm index 54a6672..d983af3 100644 --- a/queries/highlights.scm +++ b/queries/highlights.scm @@ -8,6 +8,7 @@ ; Literals ;------------------------------------------------------------------------------- +(language) @string.special (container_key (string) @string.special) (shape_key (string) @string) (label) @string diff --git a/queries/injections.scm b/queries/injections.scm new file mode 100644 index 0000000..b9ec8c7 --- /dev/null +++ b/queries/injections.scm @@ -0,0 +1,35 @@ +; for tree-sitter + +( + (text_block (raw_text) @injection.content) + (#set! injection.language "markdown") +) + +(text_block + (language) @injection.language + (raw_text) @injection.content +) + +;; overwrite for nvim-treesitter +; use markdown as default +(text_block . (raw_text) @markdown) + +; add alias for markdown +(text_block + (language) @_language + (raw_text) @markdown + (#eq? @_language "md") +) + +; add alias for javascript +(text_block + (language) @_language + (raw_text) @javascript + (#eq? @_language "js") +) + +(text_block + (language) @language + (raw_text) @content +) + diff --git a/src/grammar.json b/src/grammar.json index 767bdf0..e89c666 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -569,8 +569,13 @@ "type": "CHOICE", "members": [ { - "type": "SYMBOL", - "name": "text_block_content" + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "_text_block_raw" + }, + "named": true, + "value": "raw_text" }, { "type": "BLANK" @@ -579,20 +584,6 @@ } ] }, - "text_block_content": { - "type": "REPEAT1", - "content": { - "type": "TOKEN", - "content": { - "type": "PREC", - "value": -1, - "content": { - "type": "PATTERN", - "value": ".*?[^`|]" - } - } - } - }, "language": { "type": "PATTERN", "value": "\\w+" @@ -1339,7 +1330,12 @@ ] ], "precedences": [], - "externals": [], + "externals": [ + { + "type": "SYMBOL", + "name": "_text_block_raw" + } + ], "inline": [], "supertypes": [] } diff --git a/src/node-types.json b/src/node-types.json index f39fe3c..4ec46b8 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -287,17 +287,12 @@ "named": true }, { - "type": "text_block_content", + "type": "raw_text", "named": true } ] } }, - { - "type": "text_block_content", - "named": true, - "fields": {} - }, { "type": "\u0000", "named": false @@ -394,6 +389,10 @@ "type": "opacity", "named": false }, + { + "type": "raw_text", + "named": true + }, { "type": "shadow", "named": false diff --git a/src/parser.c b/src/parser.c index bc09a54..09532e9 100644 Binary files a/src/parser.c and b/src/parser.c differ diff --git a/src/scanner.c b/src/scanner.c new file mode 100644 index 0000000..b9ab9b2 --- /dev/null +++ b/src/scanner.c @@ -0,0 +1,68 @@ +#include +#include + +enum TokenType { + RAW_TEXT +}; + +void *tree_sitter_d2_external_scanner_create() { return NULL; } +void tree_sitter_d2_external_scanner_destroy(void *payload) { } +unsigned tree_sitter_d2_external_scanner_serialize(void *p, char *buffer) { return 0; } +void tree_sitter_d2_external_scanner_deserialize(void *p, const char *b, unsigned n) {} + +static void advance(TSLexer *lexer) { lexer->advance(lexer, false); } +static void skip(TSLexer *lexer) { lexer->advance(lexer, true); } + +static bool scan_raw_text(TSLexer *lexer) { + lexer->result_symbol = RAW_TEXT; + + if (lexer->lookahead == '`') { + advance(lexer); + } + + if (lexer->lookahead == '|') { + return false; + } + + for (bool has_content = false;; has_content = true) { + lexer->mark_end(lexer); + + if (lexer->lookahead == '\0') { + return has_content; + } + + if (lexer->lookahead == '\n') { + advance(lexer); + lexer->mark_end(lexer); + + while (iswspace(lexer->lookahead)) { + advance(lexer); + lexer->mark_end(lexer); + } + + if (lexer->lookahead == '`') { + advance(lexer); + } + + if (lexer->lookahead == '|') { + return has_content; + } + } + + advance(lexer); + } + + return false; +} + +bool tree_sitter_d2_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + if (valid_symbols[RAW_TEXT]) { + return scan_raw_text(lexer); + } + + return false; +} diff --git a/test/corpus/shape.txt b/test/corpus/shape.txt index 2ff0098..7413d6d 100644 --- a/test/corpus/shape.txt +++ b/test/corpus/shape.txt @@ -122,7 +122,7 @@ foo: | (shape (shape_key) (text_block - (text_block_content) + (raw_text) ) ) ) @@ -149,7 +149,7 @@ foo: |go (shape_key) (text_block (language) - (text_block_content) + (raw_text) ) ) ) @@ -176,7 +176,7 @@ foo: |`go (shape_key) (text_block (language) - (text_block_content) + (raw_text) ) ) ) diff --git a/tree-sitter-d2.wasm b/tree-sitter-d2.wasm index 024038d..410e154 100755 Binary files a/tree-sitter-d2.wasm and b/tree-sitter-d2.wasm differ