Heredoc rework

.gitattributes

@@ -1,10 +1,6 @@
-/src/** linguist-vendored
 /examples/* linguist-vendored
 
++src/tree_sitter/* linguist-generated
 src/grammar.json linguist-generated
 src/node-types.json linguist-generated
 src/parser.c linguist-generated
-
-src/grammar.json -diff
-src/node-types.json -diff
-src/parser.c -di

.github/workflows/pack.yml

@@ -0,0 +1,93 @@
+name: Pack
+
+on:
+  release:
+    types:
+      - released
+
+env:
+  NODE_PREBUILD_CMD: npx prebuild -t 10.0.0 -t 12.0.0 -t 14.0.0 -t 16.0.0 -t 18.0.0 -t 20.0.0 --strip -u ${{ secrets.GH_TOKEN }}
+  ELECTRON_PREBUILD_CMD: npx prebuild -r electron -t 3.0.0 -t 4.0.0 -t 5.0.0 -t 6.0.0 -t 7.0.0 -t 8.0.0 -t 9.0.0 -t 10.0.0 -t 11.0.0 -t 12.0.0 -t 13.0.0 -t 14.0.0 -t 15.0.0 -t 16.0.0 -t 17.0.0 -t 18.0.0 -t 19.0.0 -t 20.0.0 -t 21.0.0 -t 22.0.0 -t 23.0.0 -t 24.0.0 -t 25.0.0 --strip -u ${{ secrets.GH_TOKEN }}
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+        node:
+          - 10
+          - 12
+          - 14
+          - 16
+          - 18
+          - 20
+      fail-fast: false
+    name: Testing Node ${{ matrix.node }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm install
+      - run: npm test
+
+  test-windows:
+    strategy:
+      matrix:
+        os:
+          - windows-2019
+        node:
+          - 10
+          - 12
+          - 14
+          - 16
+          - 18
+          - 20
+      fail-fast: false
+    name: Testing Node ${{ matrix.node }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm install
+      - run: npm run test-windows
+
+  prebuild:
+    strategy:
+      matrix:
+        os:
+          - windows-2019
+          - macos-latest
+          - ubuntu-latest
+      fail-fast: false
+    name: Prebuild for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: [test, test-windows]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+      - run: npm install
+      - if: runner.os == 'macOS'
+        run: ${{ env.NODE_PREBUILD_CMD }} --arch arm64
+      - if: runner.os == 'macOS'
+        run: ${{ env.ELECTRON_PREBUILD_CMD }} --arch arm64
+      - run: ${{ env.NODE_PREBUILD_CMD }}
+      - run: ${{ env.ELECTRON_PREBUILD_CMD }}

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-bash"
 description = "Bash grammar for tree-sitter"
-version = "0.20.1"
+version = "0.20.2"
 authors = ["Max Brunsfeld <[email protected]"]
 license = "MIT"
 readme = "bindings/rust/README.md"

binding.gyp

@@ -15,5 +15,8 @@
         "-std=c99",
       ]
     }
-  ]
+  ],
+  'variables': {
+    'openssl_fips': '',
+  }
 }

bindings/rust/README.md

@@ -9,7 +9,7 @@ way.)
 ```toml
 [dependencies]
 tree-sitter = "0.20.10"
-tree-sitter-bash = "0.20.1"
+tree-sitter-bash = "0.20.2"
 ```
 
 Typically, you will use the [language][language func] function to add this

grammar.js

@@ -31,6 +31,7 @@ module.exports = grammar({
     [$.compound_statement],
     [$.redirected_statement, $.command],
     [$.redirected_statement, $.command_substitution],
+    [$._expansion_body],
   ],
 
   inline: $ => [
@@ -48,15 +49,17 @@ module.exports = grammar({
 
   externals: $ => [
     $.heredoc_start,
-    $._simple_heredoc_body,
+    $.simple_heredoc_body,
     $._heredoc_body_beginning,
     $._heredoc_body_middle,
-    $._heredoc_body_end,
+    $.heredoc_end,
     $.file_descriptor,
     $._empty_value,
     $._concat,
     $.variable_name, // Variable name followed by an operator like '=' or '+='
     $.regex,
+    $._regex_no_slash,
+    $._regex_no_space,
     $.extglob_pattern,
     $._bare_dollar,
     $._brace_start,
@@ -88,17 +91,14 @@ module.exports = grammar({
     _statements: $ => prec(1, seq(
       repeat(seq(
         $._statement,
-        optional(seq('\n', $.heredoc_body)),
         $._terminator,
       )),
       $._statement,
-      optional(seq('\n', $.heredoc_body)),
       optional($._terminator),
     )),
 
     _statements2: $ => repeat1(seq(
       $._statement,
-      optional(seq('\n', $.heredoc_body)),
       $._terminator,
     )),
 
@@ -436,20 +436,36 @@ module.exports = grammar({
       field('descriptor', optional($.file_descriptor)),
       choice('<<', '<<-'),
       $.heredoc_start,
+      optional(seq(
+        choice(alias($._heredoc_pipeline, $.pipeline), $.file_redirect),
+      )),
+      '\n',
+      choice($._heredoc_body, $._simple_heredoc_body),
     ),
 
-    heredoc_body: $ => choice(
-      $._simple_heredoc_body,
-      seq(
-        $._heredoc_body_beginning,
-        repeat(choice(
-          $.expansion,
-          $.simple_expansion,
-          $.command_substitution,
-          $._heredoc_body_middle,
-        )),
-        $._heredoc_body_end,
-      ),
+    _heredoc_pipeline: $ => seq(
+      choice('|', '|&'),
+      $._statement,
+    ),
+
+    _heredoc_body: $ => seq(
+      $.heredoc_body,
+      $.heredoc_end,
+    ),
+
+    heredoc_body: $ => seq(
+      $._heredoc_body_beginning,
+      repeat(choice(
+        $.expansion,
+        $.simple_expansion,
+        $.command_substitution,
+        $._heredoc_body_middle,
+      )),
+    ),
+
+    _simple_heredoc_body: $ => seq(
+      $.simple_heredoc_body,
+      $.heredoc_end,
     ),
 
     herestring_redirect: $ => seq(
@@ -489,7 +505,7 @@ module.exports = grammar({
       seq(
         field('left', $._expression),
         field('operator', choice('==', '=~', '!=')),
-        field('right', $.regex),
+        field('right', alias($._regex_no_space, $.regex)),
       ),
     )),
 
@@ -549,7 +565,7 @@ module.exports = grammar({
       $.brace_expression,
     ),
 
-    arithmetic_expansion: $ => seq('$((', $._arithmetic_expression, '))'),
+    arithmetic_expansion: $ => seq(choice('$((', '(('), commaSep1($._arithmetic_expression), '))'),
 
     brace_expression: $ => seq(
       alias($._brace_start, '{'),
@@ -570,10 +586,12 @@ module.exports = grammar({
 
     _arithmetic_literal: $ => prec(1, choice(
       $.number,
+      $.test_operator,
       $.subscript,
       $.simple_expansion,
       $.expansion,
       $._simple_variable_name,
+      $.variable_name,
     )),
 
     _arithmetic_binary_expression: $ => prec.left(2, choice(
@@ -606,12 +624,12 @@ module.exports = grammar({
     ),
 
     _arithmetic_unary_expression: $ => choice(
-      prec(1, seq(
+      prec(3, seq(
         token(prec(1, choice('-', '+', '~', '++', '--'))),
         $._arithmetic_expression,
       )),
-      prec.right(1, seq(
-        choice('!', $.test_operator),
+      prec.right(3, seq(
+        '!',
         $._arithmetic_expression,
       )),
     ),
@@ -703,9 +721,45 @@ module.exports = grammar({
     ),
     _expansion_body: $ => choice(
       seq(
-        $.variable_name,
-        '=',
-        optional($._literal),
+        choice($.variable_name, $._special_variable_name),
+        choice(
+          seq(
+            field('operator', choice('=', ':=', '-', ':-', '+', ':+', '?', ':?')),
+            repeat(choice($._literal, $.array)),
+          ),
+          seq(
+            field('operator', choice('#', '##', '%', '%%')),
+            choice($.regex, alias(')', $.regex), $.string, $.raw_string),
+          ),
+          seq(
+            choice('/', '//', '/#', '/%'),
+            alias($._regex_no_slash, $.regex),
+            // This can be elided
+            optional(seq(
+              '/',
+              optional(seq(
+                $._literal,
+                optional('/'),
+              )),
+            )),
+          ),
+          seq(
+            choice(',', ',,', '^', '^^'),
+            $.regex,
+          ),
+          seq(
+            ':',
+            choice($._simple_variable_name, $.number, $.arithmetic_expansion, $.expansion, '\n'),
+            optional(seq(
+              ':',
+              optional(choice($._simple_variable_name, $.number, $.arithmetic_expansion, '\n')),
+            )),
+          ),
+          seq(
+            '@',
+            field('operator', choice('U', 'u', 'L', 'Q', 'E', 'P', 'A', 'K', 'a', 'k')),
+          ),
+        ),
       ),
       seq(
         choice(
@@ -716,7 +770,6 @@ module.exports = grammar({
         ),
         optional(seq(
           choice(
-            alias(token(prec(1, '/')), '/'),
             alias(token(prec(1, ',')), ','),
             alias(token(prec(1, ',,')), ',,'),
             alias(token(prec(1, '^')), '^'),

package.json

@@ -1,11 +1,12 @@
 {
   "name": "tree-sitter-bash",
-  "version": "0.20.1",
+  "version": "0.20.2",
   "description": "Bash grammar for tree-sitter",
   "main": "bindings/node",
   "keywords": [
     "parser",
-    "lexer"
+    "lexer",
+    "bash"
   ],
   "author": "Max Brunsfeld",
   "license": "MIT",