diff --git a/spec/wasm-tree-sitter-language-mode-spec.js b/spec/wasm-tree-sitter-language-mode-spec.js index 0ed5eb1289..c1c4119d9b 100644 --- a/spec/wasm-tree-sitter-language-mode-spec.js +++ b/spec/wasm-tree-sitter-language-mode-spec.js @@ -77,29 +77,24 @@ describe('WASMTreeSitterLanguageMode', () => { describe('highlighting', () => { it('applies the most specific scope mapping to each node in the syntax tree', async () => { - jasmine.useRealClock(); - grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (member_expression object: (identifier) @support) - - (call_expression - function: (identifier) @support) + buffer.setText('aa.bbb = cc(d.eee());'); - (assignment_expression - left: (member_expression - property: (property_identifier) @variable)) + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (member_expression object: (identifier) @support) - ["="] @keyword + (call_expression + function: (identifier) @support) - ["." "(" ")" ";"] @punctuation - `); + (assignment_expression + left: (member_expression + property: (property_identifier) @variable)) - buffer.setText('aa.bbb = cc(d.eee());'); + ["="] @keyword - const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + ["." "(" ")" ";"] @punctuation + ` + }); await wait(0); @@ -125,38 +120,31 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('can start or end multiple scopes at the same position', async () => { - jasmine.useRealClock(); - grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (member_expression object: (identifier) @support) - - (call_expression - function: (identifier) @call) + buffer.setText('a = bb.ccc();'); - (call_expression - function: (member_expression - property: (property_identifier) @call)) + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (member_expression object: (identifier) @support) - (assignment_expression left: (identifier) @variable) - (assignment_expression - left: (member_expression - property: (property_identifier) @variable)) + (call_expression + function: (identifier) @call) - (member_expression object: (identifier) @object - property: (_) @member) + (call_expression + function: (member_expression + property: (property_identifier) @call)) - "(" @open-paren - ")" @close-paren - `) + (assignment_expression left: (identifier) @variable) + (assignment_expression + left: (member_expression + property: (property_identifier) @variable)) - buffer.setText('a = bb.ccc();'); + (member_expression object: (identifier) @object + property: (_) @member) - const languageMode = new WASMTreeSitterLanguageMode({ - grammar, buffer + "(" @open-paren + ")" @close-paren + ` }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; await wait(0); expectTokensToEqual(editor, [ @@ -255,19 +243,14 @@ describe('WASMTreeSitterLanguageMode', () => { }); it("updates lines' highlighting when they are affected by distant changes", async () => { - jasmine.useRealClock(); - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (call_expression (identifier) @function) - (property_identifier) @member - `); - buffer.setText('a(\nb,\nc\n'); - const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (call_expression (identifier) @function) + (property_identifier) @member + ` + }); await wait(0); // missing closing paren @@ -295,18 +278,6 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('updates the range of the current node in the tree when highlight.invalidateOnChange is set', async () => { - jasmine.useRealClock(); - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - ((template_string) @lorem - (#match? @lorem "lorem") - (#set! highlight.invalidateOnChange true)) - ((template_string) @ipsum - (#not-match? @ipsum "lorem") - (#set! highlight.invalidateOnChange true)) - `); - buffer.setText(dedent`\` @@ -315,11 +286,16 @@ describe('WASMTreeSitterLanguageMode', () => { \``); - - - const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + ((template_string) @lorem + (#match? @lorem "lorem") + (#set! highlight.invalidateOnChange true)) + ((template_string) @ipsum + (#not-match? @ipsum "lorem") + (#set! highlight.invalidateOnChange true)) + ` + }); await wait(0); expectTokensToEqual(editor, [ @@ -373,21 +349,15 @@ describe('WASMTreeSitterLanguageMode', () => { }) it('handles edits after tokens that end between CR and LF characters (regression)', async () => { - jasmine.useRealClock(); - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - - await grammar.setQueryForTest('highlightsQuery', ` - (comment) @comment - (string) @string - (property_identifier) @property - `); - buffer.setText(['// abc', '', 'a("b").c'].join('\r\n')); - const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (comment) @comment + (string) @string + (property_identifier) @property + ` + }); await wait(0); expectTokensToEqual(editor, [ @@ -419,19 +389,14 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('handles multi-line nodes with children on different lines (regression)', async () => { - jasmine.useRealClock(); - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (template_string) @string - ["\${" "}"] @interpolation - `); - buffer.setText('`\na${1}\nb${2}\n`;'); - const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (template_string) @string + ["\${" "}"] @interpolation + ` + }); await wait(0); expectTokensToEqual(editor, [ @@ -453,14 +418,6 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('handles folds inside of highlighted tokens', async () => { - jasmine.useRealClock(); - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (comment) @comment - (call_expression (identifier) @function) - `); - buffer.setText(dedent` /* * Hello @@ -469,9 +426,12 @@ describe('WASMTreeSitterLanguageMode', () => { hello(); `); - const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (comment) @comment + (call_expression (identifier) @function) + ` + }); await wait(0); editor.foldBufferRange([[0, 2], [2, 0]]); @@ -2972,17 +2932,13 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('includes the root scope name even when the given position is in trailing whitespace at EOF', async () => { - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (property_identifier) @property.name - `); - buffer.setText('a; '); - let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (property_identifier) @property.name + ` + }); expect( editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray() @@ -2990,17 +2946,13 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('works when the given position is between tokens', async () => { - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (comment) @comment.block - `); - buffer.setText('a // b'); - let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (comment) @comment.block + ` + }); expect( editor.scopeDescriptorForBufferPosition([0, 2]).getScopesArray() @@ -3011,19 +2963,15 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('works when a scope range has been adjusted', async () => { - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (comment) @comment.block - ((comment) @punctuation.definition.comment.begin - (#set! adjust.startAndEndAroundFirstMatchOf "^/\\\\*")) - `); - buffer.setText('\n/* lorem ipsum dolor sit amet */'); - let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (comment) @comment.block + ((comment) @punctuation.definition.comment.begin + (#set! adjust.startAndEndAroundFirstMatchOf "^/\\\\*")) + ` + }); expect( editor.scopeDescriptorForBufferPosition([1, 0]).getScopesArray() @@ -3037,56 +2985,21 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('ignores a parent\'s scopes if an injection layer sets `coverShallowerScopes`', async () => { - jasmine.useRealClock(); - const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - let tempJsRegexConfig = { - ...jsRegexConfig, - injectionRegex: '^(js-regex-for-test)$' - }; - - const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig); - - await regexGrammar.setQueryForTest('highlightsQuery', ` - (pattern) @string.regexp - (optional "?" @keyword.operator.optional) - `); - - jsGrammar.addInjectionPoint({ - type: 'regex_pattern', - language(regex) { - return 'js-regex-for-test'; - }, - content(regex) { - return regex; - }, - includeChildren: true, - languageScope: null, - coverShallowerScopes: true - }); - - await jsGrammar.setQueryForTest('highlightsQuery', ` - ((regex) @gadfly - (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em")) - (regex) @regex-outer - (regex_pattern) @regex-inner - `); - - atom.grammars.addGrammar(regexGrammar); - atom.grammars.addGrammar(jsGrammar); - buffer.setText(dedent` let foo = /patt.lor?em.ern/; `); - - const languageMode = new WASMTreeSitterLanguageMode({ - grammar: jsGrammar, - buffer, - config: atom.config, - grammars: atom.grammars - }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + regexQuery: ` + (pattern) @string.regexp + (optional "?" @keyword.operator.optional) + `, + jsQuery: ` + ((regex) @gadfly + (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em")) + (regex) @regex-outer + (regex_pattern) @regex-inner + ` + }) // Wait for injections. await wait(100); @@ -3100,55 +3013,51 @@ describe('WASMTreeSitterLanguageMode', () => { expect(scopes.includes('regex-inner')).toBe(false); }); - it('arranges scopes in the proper order when scopes from several layers were already open at a given point', async () => { - jasmine.useRealClock(); - const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - let tempJsRegexConfig = { - ...jsRegexConfig, - injectionRegex: '^(js-regex-for-test)$' - }; - - const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig); - - await regexGrammar.setQueryForTest('highlightsQuery', ` - (pattern) @string.regexp - `); - - jsGrammar.addInjectionPoint({ - type: 'regex_pattern', - language(regex) { - return 'js-regex-for-test'; - }, - content(regex) { - return regex; - }, - includeChildren: true, - languageScope: null - }); + it('ignores an empty scope map from an earlier layer without a tree', async () => { + const languageMode = await prepareTSLanguageMode(buffer, { + regexQuery: ` (pattern) @string.regexp `, + jsQuery: ` (regex) @regex-outer (regex_pattern) @regex-inner ` + }) - await jsGrammar.setQueryForTest('highlightsQuery', ` - ((regex_pattern) @gadfly - (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em")) - (regex) @regex-outer - (regex_pattern) @regex-inner - `); + buffer.setText('let foo = /pattern/;'); + // Wait for the injection layer to be populated. + await wait(100); - atom.grammars.addGrammar(regexGrammar); - atom.grammars.addGrammar(jsGrammar); + const rootLayer = languageMode.rootLanguageLayer; + const rootTree = rootLayer.tree; + rootLayer.tree = null; + + try { + const rootIterator = rootLayer.buildHighlightIterator(); + const [hasBoundaries, openScopes] = rootIterator.seek(new Point(0, 15), 0); + expect(hasBoundaries).toBe(false); + expect(openScopes.size).toBe(0); + + expect(() => { + languageMode.scopeDescriptorForPosition(new Point(0, 15)); + }).not.toThrow(); + } finally { + rootLayer.tree = rootTree; + } + }); + it('arranges scopes in the proper order when scopes from several layers were already open at a given point', async () => { buffer.setText(dedent` let foo = /patt.lor?em.ern/; `); - const languageMode = new WASMTreeSitterLanguageMode({ - grammar: jsGrammar, - buffer, - config: atom.config, - grammars: atom.grammars + const languageMode = await prepareTSLanguageMode(buffer, { + coverShallowerScopes: false, + regexQuery: ` + (pattern) @string.regexp + `, + jsQuery: ` + ((regex_pattern) @gadfly + (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em")) + (regex) @regex-outer + (regex_pattern) @regex-inner + ` }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; // Wait for injections. await wait(100); @@ -3340,18 +3249,14 @@ describe('WASMTreeSitterLanguageMode', () => { describe('with a selector', () => { it('returns the range of the smallest matching node at position', async () => { - const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await grammar.setQueryForTest('highlightsQuery', ` - (property_identifier) @variable.other.object.property - (template_string) @string.quoted.template - `); - buffer.setText('a(`${b({ccc: ddd})} eee`);'); - let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + (property_identifier) @variable.other.object.property + (template_string) @string.quoted.template + ` + }); expect( editor.bufferRangeForScopeAtPosition('.variable.property', [0, 9]) @@ -3419,33 +3324,21 @@ describe('WASMTreeSitterLanguageMode', () => { }); it('reports results correctly when scope ranges have been adjusted', async () => { - jasmine.useRealClock(); - const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - await jsGrammar.setQueryForTest('highlightsQuery', ` - ((regex) @keyword.operator.optional - (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?")) - (regex) @string.regexp.js - ((comment) @comment.block.js) - ((comment) @punctuation.definition.comment.begin.js - (#set! adjust.endAfterFirstMatchOf "^/\\\\*")) - `); - - atom.grammars.addGrammar(jsGrammar); - buffer.setText(dedent` let foo = /patt?ern/; /* this is a block comment */ `); - const languageMode = new WASMTreeSitterLanguageMode({ - grammar: jsGrammar, - buffer, - config: atom.config, - grammars: atom.grammars + const languageMode = await prepareTSLanguageMode(buffer, { + jsQuery: ` + ((regex) @keyword.operator.optional + (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?")) + (regex) @string.regexp.js + ((comment) @comment.block.js) + ((comment) @punctuation.definition.comment.begin.js + (#set! adjust.endAfterFirstMatchOf "^/\\\\*")) + ` }); - buffer.setLanguageMode(languageMode); - await languageMode.ready; let range = languageMode.bufferRangeForScopeAtPosition('keyword', new Point(0, 15)); expect(range.toString()).toBe(`[(0, 15) - (0, 16)]`); @@ -3460,54 +3353,21 @@ describe('WASMTreeSitterLanguageMode', () => { it('ignores scopes that are not present because they are covered by a deeper layer', async () => { // A similar test to the one above, except now we expect not to see the // scope because it's being covered by the injection layer. - jasmine.useRealClock(); - const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); - - let tempJsRegexConfig = { - ...jsRegexConfig, - injectionRegex: '^(js-regex-for-test)$' - }; - - const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig); - - await regexGrammar.setQueryForTest('highlightsQuery', ` - (pattern) @string.regexp - `); - - jsGrammar.addInjectionPoint({ - type: 'regex_pattern', - language(regex) { - return 'js-regex-for-test'; - }, - content(regex) { - return regex; - }, - languageScope: null, - coverShallowerScopes: true - }); - - await jsGrammar.setQueryForTest('highlightsQuery', ` - ((regex) @keyword.operator.optional - (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?")) - ((regex_pattern) @string.regexp.js) - `); - - atom.grammars.addGrammar(regexGrammar); - atom.grammars.addGrammar(jsGrammar); - buffer.setText(dedent` let foo = /patt?ern/; `); - const languageMode = new WASMTreeSitterLanguageMode({ - grammar: jsGrammar, - buffer, - config: atom.config, - grammars: atom.grammars + const languageMode = await prepareTSLanguageMode(buffer, { + includeChildren: false, + regexQuery: ` + (pattern) @string.regexp + `, + jsQuery: ` + ((regex) @keyword.operator.optional + (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?")) + ((regex_pattern) @string.regexp.js) + ` }); - buffer.setLanguageMode(languageMode); - - await languageMode.ready; await wait(100); let point = new Point(0, 15); @@ -5046,3 +4906,50 @@ function stringFragmentsOfTemplateString(templateStringNode) { c => c.type === 'string_fragment' ); } + +async function prepareTSLanguageMode(buffer, { + regexQuery = '', + jsQuery, + includeChildren = true, + coverShallowerScopes = true +}) { + jasmine.useRealClock(); + const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); + + let tempJsRegexConfig = { + ...jsRegexConfig, + injectionRegex: '^(js-regex-for-test)$' + }; + + const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig); + + await regexGrammar.setQueryForTest('highlightsQuery', regexQuery); + + jsGrammar.addInjectionPoint({ + type: 'regex_pattern', + language(regex) { + return 'js-regex-for-test'; + }, + content(regex) { + return regex; + }, + includeChildren, + languageScope: null, + coverShallowerScopes + }); + + await jsGrammar.setQueryForTest('highlightsQuery', jsQuery); + + atom.grammars.addGrammar(regexGrammar); + atom.grammars.addGrammar(jsGrammar); + + const languageMode = new WASMTreeSitterLanguageMode({ + grammar: jsGrammar, + buffer, + config: atom.config, + grammars: atom.grammars + }); + buffer.setLanguageMode(languageMode); + await languageMode.ready; + return languageMode; +} diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index 1e4a7a4e57..9f70a91c26 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -2029,6 +2029,11 @@ class NullLanguageModeHighlightIterator { } class NullLayerHighlightIterator { + constructor(languageLayer) { + this.languageLayer = languageLayer; + this.depth = languageLayer.depth; + this.coverShallowerScopes = false; + } seek() { return [false, new OpenScopeMap]; } @@ -3134,7 +3139,7 @@ class LanguageLayer { if (this.tree) { return new LayerHighlightIterator(this, this.tree); } else { - return new NullLayerHighlightIterator(); + return new NullLayerHighlightIterator(this); } }