pleshevskiy/tree-sitter-d2
1
0
Fork 0
Code Issues 9 Pull requests Releases Wiki Activity

improve strings and scanner for text block

Browse source
This commit is contained in:
Dmitriy Pleshevskiy 2022-12-10 05:23:27 +03:00
parent 1f0e76c937
commit 24cdec0cb3
Signed by: pleshevskiy
GPG key ID: 1B59187B161C0215
12 changed files with 194 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
grammar.js
View file

@ -137,7 +137,7 @@ module.exports = grammar({
_text_block_definition: ($) =>
seq(
optional($.language),
$._eol,
/\s/,
optional(alias($._text_block_raw, $.raw_text))
),
@ -285,8 +285,44 @@ module.exports = grammar({
string: ($) =>
choice(
seq("'", repeat(token.immediate(/[^'\n]+/)), "'"),
seq('"', repeat(token.immediate(/[^"\n]+/)), '"')
seq(
"'",
repeat(
choice(
alias($._unescaped_single_string_fragment, $.string_fragment),
$.escape_sequence
)
),
"'"
),
seq(
'"',
repeat(
choice(
alias($._unescaped_double_string_fragment, $.string_fragment),
$.escape_sequence
)
),
'"'
)
),
_unescaped_single_string_fragment: ($) => token.immediate(/[^'\\\n]+/),
_unescaped_double_string_fragment: ($) => token.immediate(/[^"\\\n]+/),
escape_sequence: ($) =>
token.immediate(
seq(
"\\",
choice(
/[^xu0-7]/,
/[0-7]{1,3}/,
/x[0-9a-fA-F]{2}/,
/u[0-9a-fA-F]{4}/,
/u{[0-9a-fA-F]+}/
)
)
),
boolean: ($) => choice("true", "false"),

10
queries/highlights.scm
View file

@ -8,10 +8,11 @@
; Literals
;-------------------------------------------------------------------------------
(language) @type.qualifier
(container_key (string) @string)
(shape_key (string) @string)
(label) @string.special
(string) @string
(container_key (string (string_fragment) @string))
(shape_key (string (string_fragment) @string))
(escape_sequence) @string.escape
(label) @text.title
(attr_value) @string
(integer) @number
(float) @float
@ -20,6 +21,7 @@
; Comments
;-------------------------------------------------------------------------------
(language) @comment
(line_comment) @comment.line
; Punctiation

13
scripts/pre-push Normal file
View file

@ -0,0 +1,13 @@
#! /usr/bin/env bash
# Can be used as a pre-push hook
# Just symlink this file to .git/hooks/pre-push
set -xe
if [ ! -z $(git diff --cached --name-only | grep -e "^grammar.js$") ]
then
make build
make build-wasm
fi

97
src/grammar.json
View file

@ -592,8 +592,8 @@
]
},
{
"type": "SYMBOL",
"name": "_eol"
"type": "PATTERN",
"value": "\\s"
},
{
"type": "CHOICE",
@ -1244,11 +1244,22 @@
{
"type": "REPEAT",
"content": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "[^'\\n]+"
}
"type": "CHOICE",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_unescaped_single_string_fragment"
},
"named": true,
"value": "string_fragment"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
@ -1267,11 +1278,22 @@
{
"type": "REPEAT",
"content": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "[^\"\\n]+"
}
"type": "CHOICE",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_unescaped_double_string_fragment"
},
"named": true,
"value": "string_fragment"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
@ -1282,6 +1304,57 @@
}
]
},
"_unescaped_single_string_fragment": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "[^'\\\\\\n]+"
}
},
"_unescaped_double_string_fragment": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "[^\"\\\\\\n]+"
}
},
"escape_sequence": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\\"
},
{
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^xu0-7]"
},
{
"type": "PATTERN",
"value": "[0-7]{1,3}"
},
{
"type": "PATTERN",
"value": "x[0-9a-fA-F]{2}"
},
{
"type": "PATTERN",
"value": "u[0-9a-fA-F]{4}"
},
{
"type": "PATTERN",
"value": "u{[0-9a-fA-F]+}"
}
]
}
]
}
},
"boolean": {
"type": "CHOICE",
"members": [

24
src/node-types.json
View file

@ -289,7 +289,21 @@
{
"type": "string",
"named": true,
"fields": {}
"fields": {},
"children": {
"multiple": true,
"required": false,
"types": [
{
"type": "escape_sequence",
"named": true
},
{
"type": "string_fragment",
"named": true
}
]
}
},
{
"type": "text_block",
@ -366,6 +380,10 @@
"type": "dot",
"named": true
},
{
"type": "escape_sequence",
"named": true
},
{
"type": "false",
"named": false
@ -434,6 +452,10 @@
"type": "source-arrowhead",
"named": false
},
{
"type": "string_fragment",
"named": true
},
{
"type": "stroke",
"named": false

BIN
src/parser.c
View file

Binary file not shown.

18
src/scanner.c
View file

@ -31,20 +31,18 @@ static bool scan_raw_text(TSLexer *lexer) {
return has_content;
}
if (lexer->lookahead == '\n') {
while (iswspace(lexer->lookahead)) {
advance(lexer);
lexer->mark_end(lexer);
}
while (iswspace(lexer->lookahead)) {
advance(lexer);
lexer->mark_end(lexer);
}
if (lexer->lookahead == '`') {
advance(lexer);
}
if (lexer->lookahead == '`') {
advance(lexer);
}
if (lexer->lookahead == '|') {
if (lexer->lookahead == '|') {
advance(lexer);
if (lexer->lookahead == '\n') {
return has_content;
}
}

6
test/corpus/attributes.txt
View file

@ -26,7 +26,7 @@ near: abc
(attribute (attr_key) (attr_value))
(attribute (attr_key (reserved)) (attr_value))
(attribute (attr_key (reserved)) (attr_value (string)))
(attribute (attr_key (reserved)) (attr_value (string (string_fragment))))
(attribute (attr_key (reserved)) (attr_value))
(attribute (attr_key (reserved)) (attr_value))
(attribute (attr_key (reserved)) (attr_value (float)))
@ -134,7 +134,7 @@ foo: {
(container_key)
(block
(attribute (attr_key) (attr_value))
(attribute (attr_key) (attr_value (string)))
(attribute (attr_key) (attr_value (string (string_fragment))))
(attribute (attr_key) (attr_value))
(attribute (attr_key) (attr_value))
(attribute (attr_key) (attr_value (integer)))
@ -381,7 +381,7 @@ foo -> bar: {
(attr_key)
(block
(attribute (attr_key) (attr_value))
(attribute (attr_key) (attr_value (string)))
(attribute (attr_key) (attr_value (string (string_fragment))))
(attribute (attr_key) (attr_value))
(attribute (attr_key) (attr_value))
(attribute (attr_key) (attr_value (integer)))

4
test/corpus/container.txt
View file

@ -29,8 +29,8 @@ Use quoted string as keys
(source_file
(container
(container_key (string)) (dot)
(shape (shape_key (string)))
(container_key (string (string_fragment))) (dot)
(shape (shape_key (string (string_fragment))))
)
)

6
test/corpus/shape.txt
View file

@ -54,7 +54,7 @@ Use quoted string as a shape key
--------------------------------------------------------------------------------
(source_file
(shape (shape_key (string)))
(shape (shape_key (string (string_fragment))))
)
================================================================================
@ -108,7 +108,9 @@ Use quoted string as shape key and label
--------------------------------------------------------------------------------
(source_file
(shape (shape_key (string)) (label (string)))
(shape
(shape_key (string (string_fragment)))
(label (string (string_fragment))))
)
================================================================================

15
test/highlight/shape.d2
View file

@ -2,21 +2,28 @@ foo.'baz'.biz
# <- constant
# ^ punctuation.delimiter
# ^ string
# ^ string
# ^ punctuation.delimiter
# ^ variable
'biz': 'Baz'
'b\nz'
# ^ string.escape
# ^ string.escape
'biz': 'Baz\nBiz'
# <- string
# ^ string.special
# ^ string
# ^ string
# ^ string.escape
foo: Foo Bar
# <- variable
# ^ punctuation.delimiter
# ^ string.special
# ^ text.title
foo: Foo Bar {
# <- constant
# ^ string.special
# ^ text.title
# ^ punctuation.bracket
bar.baz

BIN
tree-sitter-d2.wasm
View file

Binary file not shown.