diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d3a8b5b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,39 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{json,toml,yml,gyp}] +indent_style = space +indent_size = 2 + +[*.js] +indent_style = space +indent_size = 2 + +[*.rs] +indent_style = space +indent_size = 4 + +[*.{c,cc,h}] +indent_style = space +indent_size = 4 + +[*.{py,pyi}] +indent_style = space +indent_size = 4 + +[*.swift] +indent_style = space +indent_size = 4 + +[*.go] +indent_style = tab +indent_size = 8 + +[Makefile] +indent_style = tab +indent_size = 8 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ffb52ab --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +* text eol=lf + +src/*.json linguist-generated +src/parser.c linguist-generated +src/tree_sitter/* linguist-generated + +bindings/** linguist-generated +binding.gyp linguist-generated +setup.py linguist-generated +Makefile linguist-generated +Package.swift linguist-generated diff --git a/Makefile b/Makefile index 5829278..4001f42 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ playground: build-wasm tree-sitter playground --quiet build-wasm: build - tree-sitter build-wasm + tree-sitter build --wasm build: tree-sitter generate diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..3388ee9 --- /dev/null +++ b/Package.swift @@ -0,0 +1,47 @@ +// swift-tools-version:5.3 +import PackageDescription + +let package = Package( + name: "TreeSitterD2", + products: [ + .library(name: "TreeSitterD2", targets: ["TreeSitterD2"]), + ], + dependencies: [], + targets: [ + .target(name: "TreeSitterD2", + path: ".", + exclude: [ + "Cargo.toml", + "Makefile", + "binding.gyp", + "bindings/c", + "bindings/go", + "bindings/node", + "bindings/python", + "bindings/rust", + "prebuilds", + "grammar.js", + "package.json", + "package-lock.json", + "pyproject.toml", + "setup.py", + "test", + "examples", + ".editorconfig", + ".github", + ".gitignore", + ".gitattributes", + ".gitmodules", + ], + sources: [ + "src/parser.c", + // NOTE: if your language has an external scanner, add it here. + ], + resources: [ + .copy("queries") + ], + publicHeadersPath: "bindings/swift", + cSettings: [.headerSearchPath("src")]) + ], + cLanguageStandard: .c11 +) diff --git a/README.md b/README.md index 1614ef7..c75fd1a 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ parser_config.d2 = { install_info = { url = 'https://git.pleshevski.ru/pleshevskiy/tree-sitter-d2', revision = 'main', - files = { 'src/parser.c', 'src/scanner.cc' }, + files = { 'src/parser.c', 'src/scanner.c' }, }, filetype = 'd2', }; diff --git a/binding.gyp b/binding.gyp index 493bc06..cd55f42 100644 --- a/binding.gyp +++ b/binding.gyp @@ -2,18 +2,29 @@ "targets": [ { "target_name": "tree_sitter_d2_binding", + "dependencies": [ + " -#include "nan.h" +#include -using namespace v8; +typedef struct TSLanguage TSLanguage; -extern "C" TSLanguage * tree_sitter_d2(); +extern "C" TSLanguage *tree_sitter_d2(); -namespace { +// "tree-sitter", "language" hashed with BLAKE2 +const napi_type_tag LANGUAGE_TYPE_TAG = { + 0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16 +}; -NAN_METHOD(New) {} - -void Init(Local exports, Local module) { - Local tpl = Nan::New(New); - tpl->SetClassName(Nan::New("Language").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); - - Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); - Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); - Nan::SetInternalFieldPointer(instance, 0, tree_sitter_d2()); - - Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("d2").ToLocalChecked()); - Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports["name"] = Napi::String::New(env, "d2"); + auto language = Napi::External::New(env, tree_sitter_d2()); + language.TypeTag(&LANGUAGE_TYPE_TAG); + exports["language"] = language; + return exports; } -NODE_MODULE(tree_sitter_d2_binding, Init) - -} // namespace +NODE_API_MODULE(tree_sitter_d2_binding, Init) diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts new file mode 100644 index 0000000..efe259e --- /dev/null +++ b/bindings/node/index.d.ts @@ -0,0 +1,28 @@ +type BaseNode = { + type: string; + named: boolean; +}; + +type ChildNode = { + multiple: boolean; + required: boolean; + types: BaseNode[]; +}; + +type NodeInfo = + | (BaseNode & { + subtypes: BaseNode[]; + }) + | (BaseNode & { + fields: { [name: string]: ChildNode }; + children: ChildNode[]; + }); + +type Language = { + name: string; + language: unknown; + nodeTypeInfo: NodeInfo[]; +}; + +declare const language: Language; +export = language; diff --git a/bindings/node/index.js b/bindings/node/index.js index 46e96c8..6657bcf 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -1,18 +1,6 @@ -try { - module.exports = require("../../build/Release/tree_sitter_d2_binding"); -} catch (error1) { - if (error1.code !== 'MODULE_NOT_FOUND') { - throw error1; - } - try { - module.exports = require("../../build/Debug/tree_sitter_d2_binding"); - } catch (error2) { - if (error2.code !== 'MODULE_NOT_FOUND') { - throw error2; - } - throw error1 - } -} +const root = require("path").join(__dirname, "..", ".."); + +module.exports = require("node-gyp-build")(root); try { module.exports.nodeTypeInfo = require("../../src/node-types.json"); diff --git a/bindings/python/tree_sitter_d2/__init__.py b/bindings/python/tree_sitter_d2/__init__.py new file mode 100644 index 0000000..da12613 --- /dev/null +++ b/bindings/python/tree_sitter_d2/__init__.py @@ -0,0 +1,5 @@ +"D2 grammar for tree-sitter" + +from ._binding import language + +__all__ = ["language"] diff --git a/bindings/python/tree_sitter_d2/__init__.pyi b/bindings/python/tree_sitter_d2/__init__.pyi new file mode 100644 index 0000000..5416666 --- /dev/null +++ b/bindings/python/tree_sitter_d2/__init__.pyi @@ -0,0 +1 @@ +def language() -> int: ... diff --git a/bindings/python/tree_sitter_d2/binding.c b/bindings/python/tree_sitter_d2/binding.c new file mode 100644 index 0000000..33aa69c --- /dev/null +++ b/bindings/python/tree_sitter_d2/binding.c @@ -0,0 +1,27 @@ +#include + +typedef struct TSLanguage TSLanguage; + +TSLanguage *tree_sitter_d2(void); + +static PyObject* _binding_language(PyObject *self, PyObject *args) { + return PyLong_FromVoidPtr(tree_sitter_d2()); +} + +static PyMethodDef methods[] = { + {"language", _binding_language, METH_NOARGS, + "Get the tree-sitter language for this grammar."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_binding", + .m_doc = NULL, + .m_size = -1, + .m_methods = methods +}; + +PyMODINIT_FUNC PyInit__binding(void) { + return PyModule_Create(&module); +} diff --git a/bindings/python/tree_sitter_d2/py.typed b/bindings/python/tree_sitter_d2/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index 618e90a..404fd0e 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -7,6 +7,9 @@ fn main() { .flag_if_supported("-Wno-unused-parameter") .flag_if_supported("-Wno-unused-but-set-variable") .flag_if_supported("-Wno-trigraphs"); + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + let parser_path = src_dir.join("parser.c"); c_config.file(&parser_path); @@ -31,7 +34,7 @@ fn main() { cpp_config .flag_if_supported("-Wno-unused-parameter") .flag_if_supported("-Wno-unused-but-set-variable"); - let scanner_path = src_dir.join("scanner.cc"); + let scanner_path = src_dir.join("scanner.c"); cpp_config.file(&scanner_path); cpp_config.compile("scanner"); println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); diff --git a/bindings/swift/TreeSitterD2/d2.h b/bindings/swift/TreeSitterD2/d2.h new file mode 100644 index 0000000..2dfaf8e --- /dev/null +++ b/bindings/swift/TreeSitterD2/d2.h @@ -0,0 +1,16 @@ +#ifndef TREE_SITTER_D2_H_ +#define TREE_SITTER_D2_H_ + +typedef struct TSLanguage TSLanguage; + +#ifdef __cplusplus +extern "C" { +#endif + +const TSLanguage *tree_sitter_d2(void); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_D2_H_ diff --git a/flake.lock b/flake.lock index 77f47b2..5b00bc9 100644 --- a/flake.lock +++ b/flake.lock @@ -32,16 +32,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686736559, - "narHash": "sha256-YyUSVoOKIDAscTx7IZhF9x3qgZ9dPNF19fKk+4c5irc=", + "lastModified": 1720687749, + "narHash": "sha256-nqJ+iK/zyqCJ/YShqCpZ2cJKE1UtjZIEUWLUFZqvxcA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ddf4688dc7aeb14e8a3c549cb6aa6337f187a884", + "rev": "6af55cb91ca2005516b9562f707bb99c8f79bf77", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 7470b53..e6d9597 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,6 @@ { inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; tools.url = "git+https://git.pleshevski.ru/mynix/tools"; }; diff --git a/grammar.js b/grammar.js index e8cb50b..506b939 100644 --- a/grammar.js +++ b/grammar.js @@ -31,7 +31,7 @@ const attrAlias = mkAlias(($) => $.attribute); // mkAttrCont :: ($ -> Rule) -> ($ -> Rule) -> $ -> Rule const mkAttrCont = (onValue) => (onKey) => ($) => - seq(onKey($), $._colon, onValue($)); + seq(onKey($), $.colon, onValue($)); const mkAttr = (onKey) => mkAttrCont(($) => $.attr_value)(attrKeyAlias(onKey)); const mkListAttr = (onKey) => mkAttrCont( @@ -98,7 +98,7 @@ module.exports = grammar({ $._full_connection_path, alias($._referencing_full_connection_path, $.referencing) ), - optional(seq($._colon, optional($.label))), + optional(seq($.colon, optional($.label))), optional(seq(alias($._connection_block, $.block))) ), @@ -132,7 +132,7 @@ module.exports = grammar({ choice( optional(seq($.dot, $._classes_item)), seq( - optional(seq($._colon, optional($.label))), + optional(seq($.colon, optional($.label))), optional(alias($._classes_block, $.block)) ) ) @@ -147,7 +147,7 @@ module.exports = grammar({ choice( optional(seq($.dot, $._shape_attribute)), seq( - optional(seq($._colon, optional($.label))), + optional(seq($.colon, optional($.label))), optional(alias($._classes_item_block, $.class_block)) ) ) @@ -169,7 +169,7 @@ module.exports = grammar({ choice( seq($.dot, choice($.shape, $.container)), seq( - optional(seq($._colon, optional($.label))), + optional(seq($.colon, optional($.label))), optional(alias($._container_block, $.block)) ) ) @@ -192,7 +192,7 @@ module.exports = grammar({ choice( seq($.dot, $._shape_attribute), seq( - $._colon, + $.colon, choice( $.label, seq( @@ -307,7 +307,7 @@ module.exports = grammar({ $.keyword_style, choice( seq($.dot, alias($._inner_style_attribute, $.attribute)), - seq($._colon, alias($._style_attribute_block, $.block)) + seq($.colon, alias($._style_attribute_block, $.block)) ) ) ), @@ -364,7 +364,7 @@ module.exports = grammar({ choice( seq($.dot, alias($._style_attribute, $.attribute)), seq( - optional(seq($._colon, optional($.label))), + optional(seq($.colon, optional($.label))), optional(seq(alias($._container_block, $.block))) ) ) @@ -393,7 +393,7 @@ module.exports = grammar({ _dash: ($) => token.immediate("-"), - _colon: ($) => token(":"), + colon: ($) => token(":"), arrow: ($) => token(prec(PREC.ARROW, choice(/-+>/, /--+/, /<-+/, /<-+>/))), @@ -444,7 +444,7 @@ module.exports = grammar({ /[0-7]{1,3}/, /x[0-9a-fA-F]{2}/, /u[0-9a-fA-F]{4}/, - /u{[0-9a-fA-F]+}/ + /u\{[0-9a-fA-F]+\}/ ) ) ), diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..243ab5c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "tree-sitter-d2", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-d2", + "version": "1.0.0", + "license": "ISC" + } + } +} diff --git a/package.json b/package.json index 2b76bf5..de9857b 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "description": "", "main": "grammar.js", + "types": "bindings/node", "author": "", "license": "ISC", "tree-sitter": [ @@ -13,5 +14,31 @@ ], "injection-regex": "^d2$" } + ], + "dependencies": { + "node-gyp-build": "^4.8.0" + }, + "peerDependencies": { + "tree-sitter": "^0.21.0" + }, + "peerDependenciesMeta": { + "tree_sitter": { + "optional": true + } + }, + "devDependencies": { + "prebuildify": "^6.0.0" + }, + "scripts": { + "install": "node-gyp-build", + "prebuildify": "prebuildify --napi --strip" + }, + "files": [ + "grammar.js", + "binding.gyp", + "prebuilds/**", + "bindings/node/*", + "queries/*", + "src/**" ] } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..84ee9eb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tree-sitter-d2" +description = "D2 grammar for tree-sitter" +version = "0.0.1" +keywords = ["incremental", "parsing", "tree-sitter", "d2"] +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Compilers", + "Topic :: Text Processing :: Linguistic", + "Typing :: Typed" +] +requires-python = ">=3.8" +license.text = "MIT" +readme = "README.md" + +[project.urls] +Homepage = "https://github.com/tree-sitter/tree-sitter-d2" + +[project.optional-dependencies] +core = ["tree-sitter~=0.21"] + +[tool.cibuildwheel] +build = "cp38-*" +build-frontend = "build" diff --git a/queries/highlights.scm b/queries/highlights.scm index 63383e3..0c13680 100644 --- a/queries/highlights.scm +++ b/queries/highlights.scm @@ -43,7 +43,7 @@ [ (dot) - ":" + (colon) ";" ] @punctuation.delimiter diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fa81c90 --- /dev/null +++ b/setup.py @@ -0,0 +1,60 @@ +from os.path import isdir, join +from platform import system + +from setuptools import Extension, find_packages, setup +from setuptools.command.build import build +from wheel.bdist_wheel import bdist_wheel + + +class Build(build): + def run(self): + if isdir("queries"): + dest = join(self.build_lib, "tree_sitter_d2", "queries") + self.copy_tree("queries", dest) + super().run() + + +class BdistWheel(bdist_wheel): + def get_tag(self): + python, abi, platform = super().get_tag() + if python.startswith("cp"): + python, abi = "cp38", "abi3" + return python, abi, platform + + +setup( + packages=find_packages("bindings/python"), + package_dir={"": "bindings/python"}, + package_data={ + "tree_sitter_d2": ["*.pyi", "py.typed"], + "tree_sitter_d2.queries": ["*.scm"], + }, + ext_package="tree_sitter_d2", + ext_modules=[ + Extension( + name="_binding", + sources=[ + "bindings/python/tree_sitter_d2/binding.c", + "src/parser.c", + # NOTE: if your language uses an external scanner, add it here. + ], + extra_compile_args=[ + "-std=c11", + ] if system() != "Windows" else [ + "/std:c11", + "/utf-8", + ], + define_macros=[ + ("Py_LIMITED_API", "0x03080000"), + ("PY_SSIZE_T_CLEAN", None) + ], + include_dirs=["src"], + py_limited_api=True, + ) + ], + cmdclass={ + "build": Build, + "bdist_wheel": BdistWheel + }, + zip_safe=False +) diff --git a/src/grammar.json b/src/grammar.json index e6a0560..6c548c7 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -94,7 +94,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -367,7 +367,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -521,7 +521,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -701,7 +701,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -868,7 +868,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -1194,7 +1194,7 @@ }, { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "SYMBOL", @@ -1284,7 +1284,7 @@ }, { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -1394,7 +1394,7 @@ }, { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -1425,7 +1425,7 @@ }, { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "SYMBOL", @@ -1528,7 +1528,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "ALIAS", @@ -1637,7 +1637,7 @@ }, { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "SYMBOL", @@ -1778,7 +1778,7 @@ }, { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "SYMBOL", @@ -1852,7 +1852,7 @@ "members": [ { "type": "SYMBOL", - "name": "_colon" + "name": "colon" }, { "type": "CHOICE", @@ -2109,7 +2109,7 @@ "value": "-" } }, - "_colon": { + "colon": { "type": "TOKEN", "content": { "type": "STRING", @@ -2279,7 +2279,7 @@ }, { "type": "PATTERN", - "value": "u{[0-9a-fA-F]+}" + "value": "u\\{[0-9a-fA-F]+\\}" } ] } @@ -2421,4 +2421,3 @@ "inline": [], "supertypes": [] } - diff --git a/src/node-types.json b/src/node-types.json index 5193755..2ccf8a6 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -96,6 +96,10 @@ "type": "class_name", "named": true }, + { + "type": "colon", + "named": true + }, { "type": "dot", "named": true @@ -135,6 +139,10 @@ "type": "class_name", "named": true }, + { + "type": "colon", + "named": true + }, { "type": "connection", "named": true @@ -240,6 +248,10 @@ "type": "class_name", "named": true }, + { + "type": "colon", + "named": true + }, { "type": "dot", "named": true @@ -271,6 +283,10 @@ "type": "block", "named": true }, + { + "type": "colon", + "named": true + }, { "type": "container_key", "named": true @@ -306,6 +322,10 @@ "type": "block", "named": true }, + { + "type": "colon", + "named": true + }, { "type": "container", "named": true @@ -438,6 +458,10 @@ "type": "block", "named": true }, + { + "type": "colon", + "named": true + }, { "type": "dot", "named": true @@ -605,6 +629,10 @@ "type": "border-radius", "named": false }, + { + "type": "colon", + "named": true + }, { "type": "constraint", "named": false diff --git a/src/parser.c b/src/parser.c index 497b52a..8e0da7e 100644 Binary files a/src/parser.c and b/src/parser.c differ diff --git a/src/scanner.c b/src/scanner.c new file mode 100644 index 0000000..3b09e09 --- /dev/null +++ b/src/scanner.c @@ -0,0 +1,182 @@ +#include +#include +#include +#include +#include +#include + +enum TokenType { + TEXT_BLOCK_START, + TEXT_BLOCK_END, + TEXT_BLOCK_RAW_TEXT, + BLOCK_COMMENT, +}; + +typedef struct { + int16_t *escape_char_stack; + size_t escape_char_stack_size; + size_t escape_char_stack_capacity; +} Scanner; + +static void vector_push(Scanner *scanner, int16_t value) { + if (scanner->escape_char_stack_size == scanner->escape_char_stack_capacity) { + scanner->escape_char_stack_capacity = scanner->escape_char_stack_capacity * 2 + 1; + scanner->escape_char_stack = realloc(scanner->escape_char_stack, scanner->escape_char_stack_capacity * sizeof(int16_t)); + } + scanner->escape_char_stack[scanner->escape_char_stack_size++] = value; +} + +static void vector_clear(Scanner *scanner) { + scanner->escape_char_stack_size = 0; +} + +static void advance(TSLexer *lexer) { + lexer->advance(lexer, false); +} + +static void skip(TSLexer *lexer) { + lexer->advance(lexer, true); +} + +static void skip_whitespaces(TSLexer *lexer) { + while (lexer->lookahead != 0 && iswspace(lexer->lookahead)) { + skip(lexer); + } +} + +static bool is_text_block_end(Scanner *scanner, TSLexer *lexer) { + for (int i = scanner->escape_char_stack_size - 1; i >= 0; i--) { + if (lexer->lookahead != scanner->escape_char_stack[i]) { + return false; + } + advance(lexer); + } + return true; +} + +static bool is_triple_double_quote(TSLexer *lexer) { + for (int i = 0; i < 3; ++i) { + if (lexer->lookahead != '"') { + return false; + } + advance(lexer); + } + return true; +} + +static bool scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { + Scanner *scanner = (Scanner *)payload; + + if (valid_symbols[TEXT_BLOCK_START] && scanner->escape_char_stack_size == 0) { + lexer->result_symbol = TEXT_BLOCK_START; + lexer->mark_end(lexer); + + skip_whitespaces(lexer); + + if (lexer->lookahead != '|') { + return false; + } + + advance(lexer); + vector_push(scanner, '|'); + + if (iswalnum(lexer->lookahead) || iswspace(lexer->lookahead)) { + lexer->mark_end(lexer); + return true; + } + + int16_t escape_char = lexer->lookahead; + while (lexer->lookahead == escape_char) { + vector_push(scanner, escape_char); + advance(lexer); + } + + lexer->mark_end(lexer); + + return true; + } else if (valid_symbols[TEXT_BLOCK_END] && scanner->escape_char_stack_size > 0) { + lexer->result_symbol = TEXT_BLOCK_END; + lexer->mark_end(lexer); + + skip_whitespaces(lexer); + + if (is_text_block_end(scanner, lexer)) { + lexer->mark_end(lexer); + vector_clear(scanner); + return true; + } + } else if (valid_symbols[TEXT_BLOCK_RAW_TEXT] && scanner->escape_char_stack_size > 0) { + lexer->result_symbol = TEXT_BLOCK_RAW_TEXT; + lexer->mark_end(lexer); + + while (lexer->lookahead != 0 && !is_text_block_end(scanner, lexer)) { + advance(lexer); + lexer->mark_end(lexer); + } + + return true; + } else if (valid_symbols[BLOCK_COMMENT]) { + lexer->result_symbol = BLOCK_COMMENT; + lexer->mark_end(lexer); + + skip_whitespaces(lexer); + // Check start of block comment + if (!is_triple_double_quote(lexer)) { + return false; + } + + // Search end of block comment + while (!is_triple_double_quote(lexer)) { + // d2 expects closed tag for block comment + if (lexer->lookahead == 0) return false; + advance(lexer); + } + + lexer->mark_end(lexer); + + return true; + } + + return false; +} + +void *tree_sitter_d2_external_scanner_create() { + Scanner *scanner = calloc(1, sizeof(Scanner)); + scanner->escape_char_stack_capacity = 10; + scanner->escape_char_stack = malloc(scanner->escape_char_stack_capacity * sizeof(int16_t)); + return scanner; +} + +bool tree_sitter_d2_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { + return scan(payload, lexer, valid_symbols); +} + +unsigned tree_sitter_d2_external_scanner_serialize(void *payload, char *buffer) { + Scanner *scanner = (Scanner *)payload; + size_t i = 0; + buffer[i++] = scanner->escape_char_stack_size; + + for (size_t j = 0; j < scanner->escape_char_stack_size && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++j) { + buffer[i++] = scanner->escape_char_stack[j]; + } + + return i; +} + +void tree_sitter_d2_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) { + Scanner *scanner = (Scanner *)payload; + vector_clear(scanner); + if (length == 0) return; + size_t i = 0; + size_t escape_char_count = (uint8_t)buffer[i++]; + for (; i < escape_char_count + 1 && i < length; i++) { + vector_push(scanner, buffer[i]); + } +} + +void tree_sitter_d2_external_scanner_destroy(void *payload) { + Scanner *scanner = (Scanner *)payload; + free(scanner->escape_char_stack); + free(scanner); +} + diff --git a/src/tree_sitter/alloc.h b/src/tree_sitter/alloc.h new file mode 100644 index 0000000..1f4466d --- /dev/null +++ b/src/tree_sitter/alloc.h @@ -0,0 +1,54 @@ +#ifndef TREE_SITTER_ALLOC_H_ +#define TREE_SITTER_ALLOC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +// Allow clients to override allocation functions +#ifdef TREE_SITTER_REUSE_ALLOCATOR + +extern void *(*ts_current_malloc)(size_t); +extern void *(*ts_current_calloc)(size_t, size_t); +extern void *(*ts_current_realloc)(void *, size_t); +extern void (*ts_current_free)(void *); + +#ifndef ts_malloc +#define ts_malloc ts_current_malloc +#endif +#ifndef ts_calloc +#define ts_calloc ts_current_calloc +#endif +#ifndef ts_realloc +#define ts_realloc ts_current_realloc +#endif +#ifndef ts_free +#define ts_free ts_current_free +#endif + +#else + +#ifndef ts_malloc +#define ts_malloc malloc +#endif +#ifndef ts_calloc +#define ts_calloc calloc +#endif +#ifndef ts_realloc +#define ts_realloc realloc +#endif +#ifndef ts_free +#define ts_free free +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ALLOC_H_ diff --git a/src/tree_sitter/array.h b/src/tree_sitter/array.h new file mode 100644 index 0000000..15a3b23 --- /dev/null +++ b/src/tree_sitter/array.h @@ -0,0 +1,290 @@ +#ifndef TREE_SITTER_ARRAY_H_ +#define TREE_SITTER_ARRAY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./alloc.h" + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(disable : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + +#define Array(T) \ + struct { \ + T *contents; \ + uint32_t size; \ + uint32_t capacity; \ + } + +/// Initialize an array. +#define array_init(self) \ + ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) + +/// Create an empty array. +#define array_new() \ + { NULL, 0, 0 } + +/// Get a pointer to the element at a given `index` in the array. +#define array_get(self, _index) \ + (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) + +/// Get a pointer to the first element in the array. +#define array_front(self) array_get(self, 0) + +/// Get a pointer to the last element in the array. +#define array_back(self) array_get(self, (self)->size - 1) + +/// Clear the array, setting its size to zero. Note that this does not free any +/// memory allocated for the array's contents. +#define array_clear(self) ((self)->size = 0) + +/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is +/// less than the array's current capacity, this function has no effect. +#define array_reserve(self, new_capacity) \ + _array__reserve((Array *)(self), array_elem_size(self), new_capacity) + +/// Free any memory allocated for this array. Note that this does not free any +/// memory allocated for the array's contents. +#define array_delete(self) _array__delete((Array *)(self)) + +/// Push a new `element` onto the end of the array. +#define array_push(self, element) \ + (_array__grow((Array *)(self), 1, array_elem_size(self)), \ + (self)->contents[(self)->size++] = (element)) + +/// Increase the array's size by `count` elements. +/// New elements are zero-initialized. +#define array_grow_by(self, count) \ + do { \ + if ((count) == 0) break; \ + _array__grow((Array *)(self), count, array_elem_size(self)); \ + memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ + (self)->size += (count); \ + } while (0) + +/// Append all elements from one array to the end of another. +#define array_push_all(self, other) \ + array_extend((self), (other)->size, (other)->contents) + +/// Append `count` elements to the end of the array, reading their values from the +/// `contents` pointer. +#define array_extend(self, count, contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), (self)->size, \ + 0, count, contents \ + ) + +/// Remove `old_count` elements from the array starting at the given `index`. At +/// the same index, insert `new_count` new elements, reading their values from the +/// `new_contents` pointer. +#define array_splice(self, _index, old_count, new_count, new_contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), _index, \ + old_count, new_count, new_contents \ + ) + +/// Insert one `element` into the array at the given `index`. +#define array_insert(self, _index, element) \ + _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) + +/// Remove one element from the array at the given `index`. +#define array_erase(self, _index) \ + _array__erase((Array *)(self), array_elem_size(self), _index) + +/// Pop the last element off the array, returning the element by value. +#define array_pop(self) ((self)->contents[--(self)->size]) + +/// Assign the contents of one array to another, reallocating if necessary. +#define array_assign(self, other) \ + _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) + +/// Swap one array with another +#define array_swap(self, other) \ + _array__swap((Array *)(self), (Array *)(other)) + +/// Get the size of the array contents +#define array_elem_size(self) (sizeof *(self)->contents) + +/// Search a sorted array for a given `needle` value, using the given `compare` +/// callback to determine the order. +/// +/// If an existing element is found to be equal to `needle`, then the `index` +/// out-parameter is set to the existing value's index, and the `exists` +/// out-parameter is set to true. Otherwise, `index` is set to an index where +/// `needle` should be inserted in order to preserve the sorting, and `exists` +/// is set to false. +#define array_search_sorted_with(self, compare, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare, , needle, _index, _exists) + +/// Search a sorted array for a given `needle` value, using integer comparisons +/// of a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_with`. +#define array_search_sorted_by(self, field, needle, _index, _exists) \ + _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) + +/// Insert a given `value` into a sorted array, using the given `compare` +/// callback to determine the order. +#define array_insert_sorted_with(self, compare, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +/// Insert a given `value` into a sorted array, using integer comparisons of +/// a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_by`. +#define array_insert_sorted_by(self, field, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +// Private + +typedef Array(void) Array; + +/// This is not what you're looking for, see `array_delete`. +static inline void _array__delete(Array *self) { + if (self->contents) { + ts_free(self->contents); + self->contents = NULL; + self->size = 0; + self->capacity = 0; + } +} + +/// This is not what you're looking for, see `array_erase`. +static inline void _array__erase(Array *self, size_t element_size, + uint32_t index) { + assert(index < self->size); + char *contents = (char *)self->contents; + memmove(contents + index * element_size, contents + (index + 1) * element_size, + (self->size - index - 1) * element_size); + self->size--; +} + +/// This is not what you're looking for, see `array_reserve`. +static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { + if (new_capacity > self->capacity) { + if (self->contents) { + self->contents = ts_realloc(self->contents, new_capacity * element_size); + } else { + self->contents = ts_malloc(new_capacity * element_size); + } + self->capacity = new_capacity; + } +} + +/// This is not what you're looking for, see `array_assign`. +static inline void _array__assign(Array *self, const Array *other, size_t element_size) { + _array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + +/// This is not what you're looking for, see `array_swap`. +static inline void _array__swap(Array *self, Array *other) { + Array swap = *other; + *other = *self; + *self = swap; +} + +/// This is not what you're looking for, see `array_push` or `array_grow_by`. +static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; + if (new_size > self->capacity) { + uint32_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + if (new_capacity < new_size) new_capacity = new_size; + _array__reserve(self, element_size, new_capacity); + } +} + +/// This is not what you're looking for, see `array_splice`. +static inline void _array__splice(Array *self, size_t element_size, + uint32_t index, uint32_t old_count, + uint32_t new_count, const void *elements) { + uint32_t new_size = self->size + new_count - old_count; + uint32_t old_end = index + old_count; + uint32_t new_end = index + new_count; + assert(old_end <= self->size); + + _array__reserve(self, element_size, new_size); + + char *contents = (char *)self->contents; + if (self->size > old_end) { + memmove( + contents + new_end * element_size, + contents + old_end * element_size, + (self->size - old_end) * element_size + ); + } + if (new_count > 0) { + if (elements) { + memcpy( + (contents + index * element_size), + elements, + new_count * element_size + ); + } else { + memset( + (contents + index * element_size), + 0, + new_count * element_size + ); + } + } + self->size += new_count - old_count; +} + +/// A binary search routine, based on Rust's `std::slice::binary_search_by`. +/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. +#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ + do { \ + *(_index) = start; \ + *(_exists) = false; \ + uint32_t size = (self)->size - *(_index); \ + if (size == 0) break; \ + int comparison; \ + while (size > 1) { \ + uint32_t half_size = size / 2; \ + uint32_t mid_index = *(_index) + half_size; \ + comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ + if (comparison <= 0) *(_index) = mid_index; \ + size -= half_size; \ + } \ + comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ + if (comparison == 0) *(_exists) = true; \ + else if (comparison < 0) *(_index) += 1; \ + } while (0) + +/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) +/// parameter by reference in order to work with the generic sorting function above. +#define _compare_int(a, b) ((int)*(a) - (int)(b)) + +#ifdef _MSC_VER +#pragma warning(default : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ARRAY_H_ diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h index 2b14ac1..17f0e94 100644 --- a/src/tree_sitter/parser.h +++ b/src/tree_sitter/parser.h @@ -13,9 +13,8 @@ extern "C" { #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 -typedef uint16_t TSStateId; - #ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; typedef uint16_t TSSymbol; typedef uint16_t TSFieldId; typedef struct TSLanguage TSLanguage; @@ -87,6 +86,11 @@ typedef union { } entry; } TSParseActionEntry; +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + struct TSLanguage { uint32_t version; uint32_t symbol_count; @@ -126,13 +130,38 @@ struct TSLanguage { const TSStateId *primary_state_ids; }; +static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + /* * Lexer Macros */ +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + #define START_LEXER() \ bool result = false; \ bool skip = false; \ + UNUSED \ bool eof = false; \ int32_t lookahead; \ goto start; \ @@ -148,6 +177,17 @@ struct TSLanguage { goto next_state; \ } +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + #define SKIP(state_value) \ { \ skip = true; \ @@ -166,7 +206,7 @@ struct TSLanguage { * Parse Table Macros */ -#define SMALL_STATE(id) id - LARGE_STATE_COUNT +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) #define STATE(id) id @@ -176,7 +216,7 @@ struct TSLanguage { {{ \ .shift = { \ .type = TSParseActionTypeShift, \ - .state = state_value \ + .state = (state_value) \ } \ }} @@ -184,7 +224,7 @@ struct TSLanguage { {{ \ .shift = { \ .type = TSParseActionTypeShift, \ - .state = state_value, \ + .state = (state_value), \ .repetition = true \ } \ }} @@ -197,14 +237,15 @@ struct TSLanguage { } \ }} -#define REDUCE(symbol_val, child_count_val, ...) \ - {{ \ - .reduce = { \ - .type = TSParseActionTypeReduce, \ - .symbol = symbol_val, \ - .child_count = child_count_val, \ - __VA_ARGS__ \ - }, \ +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ }} #define RECOVER() \ diff --git a/test/corpus/attributes.txt b/test/corpus/attributes.txt index 0915414..9a034dd 100644 --- a/test/corpus/attributes.txt +++ b/test/corpus/attributes.txt @@ -25,26 +25,26 @@ near: abc -------------------------------------------------------------------------------- (source_file - (attribute (attr_key) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value (string (string_fragment)))) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value (float))) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value (integer))) - (attribute (attr_key (reserved)) (attr_value (integer))) - (attribute (attr_key (reserved)) (attr_value (integer))) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value (integer))) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value (boolean))) - (attribute (attr_key (reserved)) (attr_value (boolean))) - (attribute (attr_key (reserved)) (attr_value (boolean))) - (attribute (attr_key (reserved)) (attr_value)) - (attribute (attr_key (reserved)) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value (string (string_fragment)))) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value (float))) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value (integer))) + (attribute (attr_key (reserved)) (colon) (attr_value (integer))) + (attribute (attr_key (reserved)) (colon) (attr_value (integer))) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value (integer))) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value (boolean))) + (attribute (attr_key (reserved)) (colon) (attr_value (boolean))) + (attribute (attr_key (reserved)) (colon) (attr_value (boolean))) + (attribute (attr_key (reserved)) (colon) (attr_value)) + (attribute (attr_key (reserved)) (colon) (attr_value)) ) ================================================================================ @@ -66,18 +66,18 @@ foo.style.text-transform: uppercase -------------------------------------------------------------------------------- (source_file - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (float))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value)))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value)))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (integer))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (integer))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (integer))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value)))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (boolean))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (boolean))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (boolean))))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value)))) - (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value)))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (float))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value)))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value)))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (integer))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (integer))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (integer))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value)))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (boolean))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (boolean))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (boolean))))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value)))) + (shape (shape_key) (dot) (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value)))) ) ================================================================================ @@ -105,19 +105,20 @@ foo.style: { (shape_key) (dot) (attribute (keyword_style) + (colon) (block - (attribute (attr_key) (attr_value (float))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) + (attribute (attr_key) (colon) (attr_value (float))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) ) ) ) @@ -142,15 +143,16 @@ foo: { (source_file (container (container_key) + (colon) (block - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (string (string_fragment)))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (escape_sequence))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (string (string_fragment)))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (escape_sequence))) ) ) ) @@ -177,18 +179,19 @@ foo: { (source_file (container (container_key) + (colon) (block - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (float)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (integer)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (integer)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (integer)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (boolean)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (boolean)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value (boolean)))) - (attribute (keyword_style) (dot) (attribute (attr_key) (attr_value))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (float)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (integer)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (integer)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (integer)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (boolean)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (boolean)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value (boolean)))) + (attribute (keyword_style) (dot) (attribute (attr_key) (colon) (attr_value))) ) ) ) @@ -217,21 +220,23 @@ foo: { (source_file (container (container_key) + (colon) (block (attribute (keyword_style) + (colon) (block - (attribute (attr_key) (attr_value (float))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value)) + (attribute (attr_key) (colon) (attr_value (float))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value)) ) ) ) @@ -264,21 +269,23 @@ foo -> bar: { (shape_key) (arrow) (shape_key) + (colon) (block (attribute (keyword_style) + (colon) (block - (attribute (attr_key) (attr_value (float))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value)) + (attribute (attr_key) (colon) (attr_value (float))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value)) ) ) ) @@ -300,15 +307,10 @@ foo -> bar: { (shape_key) (arrow) (shape_key) + (colon) (block - (attribute - (attr_key) - (label) - ) - (attribute - (attr_key) - (label) - ) + (attribute (attr_key) (colon) (label)) + (attribute (attr_key) (colon) (label)) ) ) ) @@ -325,15 +327,10 @@ foo -> bar: {source-arrowhead: 0; target-arrowhead: 1} (shape_key) (arrow) (shape_key) + (colon) (block - (attribute - (attr_key) - (label) - ) - (attribute - (attr_key) - (label) - ) + (attribute (attr_key) (colon) (label)) + (attribute (attr_key) (colon) (label)) ) ) ) @@ -366,24 +363,27 @@ foo -> bar: { (shape_key) (arrow) (shape_key) + (colon) (block (attribute (attr_key) + (colon) (block (attribute (keyword_style) + (colon) (block - (attribute (attr_key) (attr_value (float))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value (boolean))) - (attribute (attr_key) (attr_value)) + (attribute (attr_key) (colon) (attr_value (float))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value (boolean))) + (attribute (attr_key) (colon) (attr_value)) ) ) ) @@ -413,16 +413,18 @@ foo -> bar: { (shape_key) (arrow) (shape_key) + (colon) (block (attribute (attr_key) + (colon) (block - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (string (string_fragment)))) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (string (string_fragment)))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) ) ) ) @@ -441,9 +443,10 @@ foo.style: {opacity: 0.5; fill: red} (shape_key) (dot) (attribute (keyword_style) + (colon) (block - (attribute (attr_key) (attr_value (float))) - (attribute (attr_key) (attr_value)) + (attribute (attr_key) (colon) (attr_value (float))) + (attribute (attr_key) (colon) (attr_value)) ) ) ) @@ -471,16 +474,17 @@ footer -------------------------------------------------------------------------------- (source_file - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) (shape (shape_key)) (container (container_key) + (colon) (label (string)) (block - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) - (attribute (attr_key) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) + (attribute (attr_key) (colon) (attr_value (integer))) (shape (shape_key)) (shape (shape_key)) ) @@ -506,11 +510,12 @@ block: { (source_file (container (container_key) + (colon) (block - (attribute (attr_key) (attr_value)) - (attribute (attr_key) (attr_value_list (attr_value))) - (attribute (attr_key) (attr_value_list (attr_value) (attr_value))) - (attribute (attr_key) (attr_value_list (attr_value) (attr_value))) + (attribute (attr_key) (colon) (attr_value)) + (attribute (attr_key) (colon) (attr_value_list (attr_value))) + (attribute (attr_key) (colon) (attr_value_list (attr_value) (attr_value))) + (attribute (attr_key) (colon) (attr_value_list (attr_value) (attr_value))) ) ) ) diff --git a/test/corpus/classes.txt b/test/corpus/classes.txt index c29122c..a996f48 100644 --- a/test/corpus/classes.txt +++ b/test/corpus/classes.txt @@ -25,14 +25,18 @@ classes: { (source_file (classes (keyword_classes) + (colon) (block (class_name) + (colon) (class_block (attribute (keyword_style) + (colon) (block (attribute (attr_key) + (colon) (attr_value (string (string_fragment)) ) @@ -58,14 +62,17 @@ classes: { (source_file (classes (keyword_classes) + (colon) (block (class_name) (dot) (attribute (keyword_style) + (colon) (block (attribute (attr_key) + (colon) (attr_value (string (string_fragment)) ) @@ -93,9 +100,11 @@ classes.foo.style: { (dot) (attribute (keyword_style) + (colon) (block (attribute (attr_key) + (colon) (attr_value (string (string_fragment)) ) @@ -121,12 +130,15 @@ classes.foo: { (keyword_classes) (dot) (class_name) + (colon) (class_block (attribute (keyword_style) + (colon) (block (attribute (attr_key) + (colon) (attr_value (string (string_fragment)) ) @@ -150,6 +162,7 @@ foo.class: biz (dot) (attribute (keyword_class) + (colon) (class_name) ) ) @@ -169,6 +182,7 @@ foo.class: [biz; baz] (dot) (attribute (keyword_class) + (colon) (class_list (class_name) ) @@ -179,6 +193,7 @@ foo.class: [biz; baz] (dot) (attribute (keyword_class) + (colon) (class_list (class_name) (class_name) @@ -205,19 +220,23 @@ foo: { (source_file (container (container_key) + (colon) (block (attribute (keyword_class) + (colon) (class_name) ) (attribute (keyword_class) + (colon) (class_list (class_name) ) ) (attribute (keyword_class) + (colon) (class_list (class_name) (class_name) @@ -225,6 +244,7 @@ foo: { ) (attribute (keyword_class) + (colon) (class_list (class_name) (class_name) diff --git a/test/corpus/connection.txt b/test/corpus/connection.txt index 4c68529..3d1790a 100644 --- a/test/corpus/connection.txt +++ b/test/corpus/connection.txt @@ -175,12 +175,14 @@ bar -> baz: Yes (shape_key) (arrow) (shape_key) + (colon) (label) ) (connection (shape_key) (arrow) (shape_key) + (colon) (label) ) ) @@ -197,6 +199,7 @@ foo.biz.baz -> bar.baz.biz: Label (container_key) (dot) (container_key) (dot) (shape_key) (arrow) (container_key) (dot) (container_key) (dot) (shape_key) + (colon) (label) ) ) @@ -215,11 +218,13 @@ foo.baz: { (container_key) (dot) (container (container_key) + (colon) (block (connection (shape_key) (arrow) (shape_key) + (colon) (label) ) ) @@ -270,7 +275,7 @@ Diclare a connection with espaced key fragments ) ================================================================================ -Declare a referencing connection +Declare a referencing connection ================================================================================ (x() -> y()) (x() -> y())[0] @@ -310,6 +315,7 @@ Declare a referencing connection with label (arrow) (shape_key) ) + (colon) (label) ) (connection @@ -319,6 +325,7 @@ Declare a referencing connection with label (shape_key) (index (integer)) ) + (colon) (label) ) ) @@ -342,12 +349,14 @@ Declare a referencing connection with block (arrow) (shape_key) ) + (colon) (block (attribute (keyword_style) (dot) (attribute (attr_key) + (colon) (attr_value (escape_sequence) ) @@ -362,12 +371,14 @@ Declare a referencing connection with block (shape_key) (index (integer)) ) + (colon) (block (attribute (keyword_style) (dot) (attribute (attr_key) + (colon) (attr_value (escape_sequence) ) diff --git a/test/corpus/container.txt b/test/corpus/container.txt index 5742d7a..11add4a 100644 --- a/test/corpus/container.txt +++ b/test/corpus/container.txt @@ -51,12 +51,15 @@ foo: { (source_file (container (container_key) + (colon) (block (container (container_key) + (colon) (block (container (container_key) + (colon) (block (shape (shape_key)) ) @@ -84,17 +87,20 @@ foo: Foo { (source_file (container (container_key) + (colon) (label) (block (container (container_key) + (colon) (label) (block (container (container_key) + (colon) (label) (block - (shape (shape_key) (label)) + (shape (shape_key) (colon) (label)) ) ) ) @@ -118,6 +124,7 @@ foo: { (source_file (container (container_key) + (colon) (block (shape (shape_key)) (shape (shape_key)) @@ -144,12 +151,15 @@ Foo biz bar: { (source_file (container (container_key) + (colon) (block (container (container_key) + (colon) (block (container (container_key) + (colon) (block (shape (shape_key)) ) @@ -177,14 +187,17 @@ Foo biz bar: Biz biz Bar { (source_file (container (container_key) + (colon) (label) (block (container (container_key) + (colon) (label) (block (container (container_key) + (colon) (label) (block (shape (shape_key)) @@ -216,6 +229,7 @@ Foo: Baz { (source_file (container (container_key) + (colon) (label) (block (shape (shape_key)) @@ -241,10 +255,12 @@ primty: Primitive types { (source_file (container (container_key) + (colon) (label) (block (container (container_key) + (colon) (block (shape (shape_key (string (string_fragment))) @@ -253,12 +269,14 @@ primty: Primitive types { ) (container (container_key) + (colon) (block (shape (shape_key)) ) ) (container (container_key) + (colon) (block (shape (shape_key)) (shape (shape_key)) @@ -280,11 +298,12 @@ container: { (source_file (container (container_key) + (colon) (block (container (container_key (keyword_underscore)) (dot) - (shape (shape_key) (label)) + (shape (shape_key) (colon) (label)) ) ) ) diff --git a/test/corpus/shape.txt b/test/corpus/shape.txt index 6ef4ba0..b8dc65e 100644 --- a/test/corpus/shape.txt +++ b/test/corpus/shape.txt @@ -55,7 +55,7 @@ bar ================================================================================ Complex shape key ================================================================================ - Foo bar + Foo bar -Biz-baz- imAShape im_a$_shape @@ -129,9 +129,9 @@ a: Foo Bar; b: Biz Baz -------------------------------------------------------------------------------- (source_file - (shape (shape_key) (label)) - (shape (shape_key) (label)) - (shape (shape_key) (label)) + (shape (shape_key) (colon) (label)) + (shape (shape_key) (colon) (label)) + (shape (shape_key) (colon) (label)) ) ================================================================================ @@ -143,9 +143,9 @@ a: Foo\nB\@r; b: Biz\nBaz -------------------------------------------------------------------------------- (source_file - (shape (shape_key) (label (escape_sequence) (escape_sequence))) - (shape (shape_key) (label (escape_sequence) (escape_sequence))) - (shape (shape_key) (label (escape_sequence))) + (shape (shape_key) (colon) (label (escape_sequence) (escape_sequence))) + (shape (shape_key) (colon) (label (escape_sequence) (escape_sequence))) + (shape (shape_key) (colon) (label (escape_sequence))) ) @@ -161,7 +161,7 @@ bar : Foo Bar; baz (source_file (shape (shape_key)) - (shape (shape_key) (label)) + (shape (shape_key) (colon) (label)) (shape (shape_key)) ) @@ -175,6 +175,7 @@ Use quoted string as shape key and label (source_file (shape (shape_key (string (string_fragment))) + (colon) (label (string (string_fragment)))) ) @@ -190,6 +191,7 @@ foo: | (source_file (shape (shape_key) + (colon) (text_block (raw_text) ) @@ -216,6 +218,7 @@ foo: |go (source_file (shape (shape_key) + (colon) (text_block (language) (raw_text) @@ -243,6 +246,7 @@ $$$| (source_file (shape (shape_key) + (colon) (text_block (language) (raw_text) @@ -262,12 +266,14 @@ bar: |##md ## hello world ##| (source_file (shape (shape_key) + (colon) (text_block (raw_text) ) ) (shape (shape_key) + (colon) (text_block (language) (raw_text) diff --git a/tree-sitter-d2.wasm b/tree-sitter-d2.wasm index 3efa1ad..a135dca 100755 Binary files a/tree-sitter-d2.wasm and b/tree-sitter-d2.wasm differ