diff --git a/grammar.js b/grammar.js index a93f834..8588958 100644 --- a/grammar.js +++ b/grammar.js @@ -137,7 +137,7 @@ module.exports = grammar({ _text_block_definition: ($) => seq( optional($.language), - $._eol, + /\s/, optional(alias($._text_block_raw, $.raw_text)) ), @@ -285,8 +285,44 @@ module.exports = grammar({ string: ($) => choice( - seq("'", repeat(token.immediate(/[^'\n]+/)), "'"), - seq('"', repeat(token.immediate(/[^"\n]+/)), '"') + seq( + "'", + repeat( + choice( + alias($._unescaped_single_string_fragment, $.string_fragment), + $.escape_sequence + ) + ), + "'" + ), + seq( + '"', + repeat( + choice( + alias($._unescaped_double_string_fragment, $.string_fragment), + $.escape_sequence + ) + ), + '"' + ) + ), + + _unescaped_single_string_fragment: ($) => token.immediate(/[^'\\\n]+/), + + _unescaped_double_string_fragment: ($) => token.immediate(/[^"\\\n]+/), + + escape_sequence: ($) => + token.immediate( + seq( + "\\", + choice( + /[^xu0-7]/, + /[0-7]{1,3}/, + /x[0-9a-fA-F]{2}/, + /u[0-9a-fA-F]{4}/, + /u{[0-9a-fA-F]+}/ + ) + ) ), boolean: ($) => choice("true", "false"), diff --git a/queries/highlights.scm b/queries/highlights.scm index b84bd1c..504fa9c 100644 --- a/queries/highlights.scm +++ b/queries/highlights.scm @@ -8,10 +8,11 @@ ; Literals ;------------------------------------------------------------------------------- -(language) @type.qualifier -(container_key (string) @string) -(shape_key (string) @string) -(label) @string.special +(string) @string +(container_key (string (string_fragment) @string)) +(shape_key (string (string_fragment) @string)) +(escape_sequence) @string.escape +(label) @text.title (attr_value) @string (integer) @number (float) @float @@ -20,6 +21,7 @@ ; Comments ;------------------------------------------------------------------------------- +(language) @comment (line_comment) @comment.line ; Punctiation diff --git a/scripts/pre-push b/scripts/pre-push new file mode 100644 index 0000000..368012b --- /dev/null +++ b/scripts/pre-push @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +# Can be used as a pre-push hook +# Just symlink this file to .git/hooks/pre-push + +set -xe + +if [ ! -z $(git diff --cached --name-only | grep -e "^grammar.js$") ] +then + make build + make build-wasm +fi + diff --git a/src/grammar.json b/src/grammar.json index 51a09fd..cf506aa 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -592,8 +592,8 @@ ] }, { - "type": "SYMBOL", - "name": "_eol" + "type": "PATTERN", + "value": "\\s" }, { "type": "CHOICE", @@ -1244,11 +1244,22 @@ { "type": "REPEAT", "content": { - "type": "IMMEDIATE_TOKEN", - "content": { - "type": "PATTERN", - "value": "[^'\\n]+" - } + "type": "CHOICE", + "members": [ + { + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "_unescaped_single_string_fragment" + }, + "named": true, + "value": "string_fragment" + }, + { + "type": "SYMBOL", + "name": "escape_sequence" + } + ] } }, { @@ -1267,11 +1278,22 @@ { "type": "REPEAT", "content": { - "type": "IMMEDIATE_TOKEN", - "content": { - "type": "PATTERN", - "value": "[^\"\\n]+" - } + "type": "CHOICE", + "members": [ + { + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "_unescaped_double_string_fragment" + }, + "named": true, + "value": "string_fragment" + }, + { + "type": "SYMBOL", + "name": "escape_sequence" + } + ] } }, { @@ -1282,6 +1304,57 @@ } ] }, + "_unescaped_single_string_fragment": { + "type": "IMMEDIATE_TOKEN", + "content": { + "type": "PATTERN", + "value": "[^'\\\\\\n]+" + } + }, + "_unescaped_double_string_fragment": { + "type": "IMMEDIATE_TOKEN", + "content": { + "type": "PATTERN", + "value": "[^\"\\\\\\n]+" + } + }, + "escape_sequence": { + "type": "IMMEDIATE_TOKEN", + "content": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "\\" + }, + { + "type": "CHOICE", + "members": [ + { + "type": "PATTERN", + "value": "[^xu0-7]" + }, + { + "type": "PATTERN", + "value": "[0-7]{1,3}" + }, + { + "type": "PATTERN", + "value": "x[0-9a-fA-F]{2}" + }, + { + "type": "PATTERN", + "value": "u[0-9a-fA-F]{4}" + }, + { + "type": "PATTERN", + "value": "u{[0-9a-fA-F]+}" + } + ] + } + ] + } + }, "boolean": { "type": "CHOICE", "members": [ diff --git a/src/node-types.json b/src/node-types.json index 61493c8..cfd17d8 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -289,7 +289,21 @@ { "type": "string", "named": true, - "fields": {} + "fields": {}, + "children": { + "multiple": true, + "required": false, + "types": [ + { + "type": "escape_sequence", + "named": true + }, + { + "type": "string_fragment", + "named": true + } + ] + } }, { "type": "text_block", @@ -366,6 +380,10 @@ "type": "dot", "named": true }, + { + "type": "escape_sequence", + "named": true + }, { "type": "false", "named": false @@ -434,6 +452,10 @@ "type": "source-arrowhead", "named": false }, + { + "type": "string_fragment", + "named": true + }, { "type": "stroke", "named": false diff --git a/src/parser.c b/src/parser.c index cde28cf..4115a50 100644 Binary files a/src/parser.c and b/src/parser.c differ diff --git a/src/scanner.c b/src/scanner.c index b9ab9b2..b3962b2 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -31,20 +31,18 @@ static bool scan_raw_text(TSLexer *lexer) { return has_content; } - if (lexer->lookahead == '\n') { + while (iswspace(lexer->lookahead)) { advance(lexer); lexer->mark_end(lexer); + } - while (iswspace(lexer->lookahead)) { - advance(lexer); - lexer->mark_end(lexer); - } + if (lexer->lookahead == '`') { + advance(lexer); + } - if (lexer->lookahead == '`') { - advance(lexer); - } - - if (lexer->lookahead == '|') { + if (lexer->lookahead == '|') { + advance(lexer); + if (lexer->lookahead == '\n') { return has_content; } } diff --git a/test/corpus/attributes.txt b/test/corpus/attributes.txt index e6f46f5..a8e62e7 100644 --- a/test/corpus/attributes.txt +++ b/test/corpus/attributes.txt @@ -26,7 +26,7 @@ near: abc (attribute (attr_key) (attr_value)) (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value (string))) + (attribute (attr_key (reserved)) (attr_value (string (string_fragment)))) (attribute (attr_key (reserved)) (attr_value)) (attribute (attr_key (reserved)) (attr_value)) (attribute (attr_key (reserved)) (attr_value (float))) @@ -134,7 +134,7 @@ foo: { (container_key) (block (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (string))) + (attribute (attr_key) (attr_value (string (string_fragment)))) (attribute (attr_key) (attr_value)) (attribute (attr_key) (attr_value)) (attribute (attr_key) (attr_value (integer))) @@ -381,7 +381,7 @@ foo -> bar: { (attr_key) (block (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (string))) + (attribute (attr_key) (attr_value (string (string_fragment)))) (attribute (attr_key) (attr_value)) (attribute (attr_key) (attr_value)) (attribute (attr_key) (attr_value (integer))) diff --git a/test/corpus/container.txt b/test/corpus/container.txt index 704d2b5..f0696e4 100644 --- a/test/corpus/container.txt +++ b/test/corpus/container.txt @@ -29,8 +29,8 @@ Use quoted string as keys (source_file (container - (container_key (string)) (dot) - (shape (shape_key (string))) + (container_key (string (string_fragment))) (dot) + (shape (shape_key (string (string_fragment)))) ) ) diff --git a/test/corpus/shape.txt b/test/corpus/shape.txt index f1f6bb1..29c2d87 100644 --- a/test/corpus/shape.txt +++ b/test/corpus/shape.txt @@ -54,7 +54,7 @@ Use quoted string as a shape key -------------------------------------------------------------------------------- (source_file - (shape (shape_key (string))) + (shape (shape_key (string (string_fragment)))) ) ================================================================================ @@ -108,7 +108,9 @@ Use quoted string as shape key and label -------------------------------------------------------------------------------- (source_file - (shape (shape_key (string)) (label (string))) + (shape + (shape_key (string (string_fragment))) + (label (string (string_fragment)))) ) ================================================================================ diff --git a/test/highlight/shape.d2 b/test/highlight/shape.d2 index c0b279f..2e44801 100644 --- a/test/highlight/shape.d2 +++ b/test/highlight/shape.d2 @@ -2,21 +2,28 @@ foo.'baz'.biz # <- constant # ^ punctuation.delimiter # ^ string +# ^ string # ^ punctuation.delimiter # ^ variable -'biz': 'Baz' +'b\nz' +# ^ string.escape +# ^ string.escape + +'biz': 'Baz\nBiz' # <- string -# ^ string.special +# ^ string +# ^ string +# ^ string.escape foo: Foo Bar # <- variable # ^ punctuation.delimiter -# ^ string.special +# ^ text.title foo: Foo Bar { # <- constant - # ^ string.special + # ^ text.title # ^ punctuation.bracket bar.baz diff --git a/tree-sitter-d2.wasm b/tree-sitter-d2.wasm index cd7e07b..3160194 100755 Binary files a/tree-sitter-d2.wasm and b/tree-sitter-d2.wasm differ