feat: migrate to antlr4ng (#267)

* feat: replace antlr4ts with antlr4ng

* feat: switch caseInsensitive option on

* feat: recompile all g4 file

* feat:  update parser to fit antlr4ng

* test: update test to fit antlr4ng

Author: HaydenOrz

jest.config.js

@@ -17,13 +17,13 @@ module.exports = {
     clearMocks: true,
 
     // Indicates whether the coverage information should be collected while executing the test
-    collectCoverage: true,
+    collectCoverage: false,
 
     // An array of glob patterns indicating a set of files for which coverage information should be collected
     // collectCoverageFrom: undefined,
 
     // The directory where Jest should output its coverage files
-    coverageDirectory: 'coverage',
+    // coverageDirectory: 'coverage',
 
     // An array of regexp pattern strings used to skip coverage collection
     // coveragePathIgnorePatterns: [
@@ -76,7 +76,7 @@ module.exports = {
     // ],
 
     // An array of file extensions your modules use
-    moduleFileExtensions: ['js', 'mjs', 'cjs', 'jsx', 'ts', 'tsx', 'json', 'node'],
+    moduleFileExtensions: ['js', 'mjs', 'ts'],
 
     // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
     moduleNameMapper: {
@@ -166,10 +166,11 @@ module.exports = {
     // A map from regular expressions to paths to transformers
     transform: {
         '\\.[jt]sx?$': ['@swc/jest'],
+        '\\.mjs$': ['@swc/jest'],
     },
 
     // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
-    extensionsToTreatAsEsm: ['.ts', '.tsx'],
+    extensionsToTreatAsEsm: ['.ts'],
     // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
     // unmockedModulePathPatterns: undefined,
 
@@ -181,4 +182,6 @@ module.exports = {
 
     // Whether to use watchman for file crawling
     // watchman: true
+
+    transformIgnorePatterns: ['dist/', '<rootDir>/node_modules/.pnpm/(?!(antlr4ng|antlr4-c3)@)'],
 };

package.json

@@ -41,7 +41,7 @@
     "@types/jest": "^29.5.1",
     "@types/node": "^18.15.11",
     "antlr-format-cli": "^1.2.1",
-    "antlr4ts-cli": "^0.5.0-alpha.4",
+    "antlr4ng-cli": "^1.0.7",
     "chalk": "4.1.2",
     "commitizen": "^4.3.0",
     "glob": "^10.3.10",
@@ -60,8 +60,8 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "antlr4-c3": "3.1.1",
-    "antlr4ts": "0.5.0-alpha.4"
+    "antlr4-c3": "3.3.7",
+    "antlr4ng": "2.0.11"
   },
   "sideEffects": false
 }

pnpm-lock.yaml

@@ -11,7 +11,7 @@ const outputPath = path.resolve(__dirname, '../src/lib');
 
 const languageEntries = fs.readdirSync(grammarsPath);
 
-const baseCmd = 'antlr4ts -visitor -listener -Xexact-output-dir -o';
+const baseCmd = 'antlr4ng -Dlanguage=TypeScript -visitor -listener -Xexact-output-dir -o';
 
 function compile(language) {
     const cmd = `${baseCmd} ${outputPath}/${language} ${grammarsPath}/${language}/*.g4`;
@@ -31,7 +31,7 @@ function compile(language) {
             );
         } else {
             cleanComment(language);
-            console.log(chalk.greenBright(`Compile ${language} succeeded!`));
+            console.info(chalk.greenBright(`Compile ${language} succeeded!`));
         }
     });
 }

scripts/antlr4.js

@@ -53,15 +53,13 @@ function execStandardVersion(res) {
     cmd += ` --tag-prefix ${tagPrefix} `;
     cmd += ' --infile CHANGELOG.md ';
 
-    console.log(`Executing: ${cmd} \n`);
+    console.info(`Executing: ${cmd} \n`);
 
     runCommand(cmd)
         .then(({ message }) => {
-            console.log('Please checkout recent commit, and then');
-            console.log(
-                'Push branch and new tag to github, publish package to npm'
-            );
-            // message && console.log(message)
+            console.info('Please checkout recent commit, and then');
+            console.info('Push branch and new tag to github, publish package to npm');
+            // message && console.info(message)
         })
         .catch(({ error, code }) => {
             code && console.error('Error: process exit code' + code);

scripts/release.js

@@ -5,6 +5,10 @@
 
 lexer grammar FlinkSqlLexer;
 
+options {
+    caseInsensitive= true;
+}
+
 // SKIP
 
 SPACE         : [ \t\r\n]+                                                         -> channel(HIDDEN);
@@ -585,9 +589,9 @@ ID_LITERAL : ID_LITERAL_FRAG;
 
 fragment JAR_FILE_PARTTARN : '`' ( '\\' . | '``' | ~('`' | '\\'))* '`';
 fragment EXPONENT_NUM_PART : 'E' [-+]? DEC_DIGIT+;
-fragment ID_LITERAL_FRAG   : [A-Z_0-9a-z]*? [A-Z_a-z]+? [A-Z_0-9a-z]*;
+fragment ID_LITERAL_FRAG   : [A-Z_0-9]*? [A-Z_]+? [A-Z_0-9]*;
 fragment DEC_DIGIT         : [0-9];
-fragment DEC_LETTER        : [A-Za-z];
+fragment DEC_LETTER        : [A-Z];
 fragment DQUOTA_STRING     : '"' ( '\\' . | '""' | ~('"' | '\\'))* '"';
 fragment SQUOTA_STRING     : '\'' ('\\' . | '\'\'' | ~('\'' | '\\'))* '\'';
 fragment BIT_STRING_L      : 'B' '\'' [01]+ '\'';

src/grammar/flinksql/FlinkSqlLexer.g4

@@ -6,6 +6,7 @@ parser grammar FlinkSqlParser;
 
 options {
     tokenVocab=FlinkSqlLexer;
+    caseInsensitive= true;
 }
 
 program

src/grammar/flinksql/FlinkSqlParser.g4

@@ -27,8 +27,9 @@
 
 lexer grammar HiveSqlLexer;
 
-// unsupported option caseInsensitive in [email protected]
-// options { caseInsensitive = true; }
+options {
+    caseInsensitive= true;
+}
 
 // Keywords
 KW_ABORT                : 'ABORT';
@@ -502,7 +503,7 @@ Identifier: (Letter | Digit) (Letter | Digit | '_')* | QuotedIdentifier | '`' Re
 
 fragment QuotedIdentifier: '`' ('``' | ~'`')* '`';
 
-fragment Letter: 'A' ..'Z' | 'a' ..'z';
+fragment Letter: 'A' ..'Z';
 
 fragment HexDigit: 'A' ..'F';
 

src/grammar/hive/HiveSqlLexer.g4

@@ -29,6 +29,7 @@ parser grammar HiveSqlParser;
 options
 {
     tokenVocab=HiveSqlLexer;
+    caseInsensitive= true;
 }
 
 program

src/grammar/hive/HiveSqlParser.g4

@@ -27,7 +27,7 @@ THE SOFTWARE.
 lexer grammar ImpalaSqlLexer;
 
 options {
-    caseInsensitive=true;
+    caseInsensitive= true;
 }
 
 KW_ADD               : 'ADD';

src/grammar/impala/ImpalaSqlLexer.g4

@@ -21,6 +21,7 @@ parser grammar ImpalaSqlParser;
 options
 {
     tokenVocab=ImpalaSqlLexer;
+    caseInsensitive= true;
 }
 
 program
@@ -873,7 +874,7 @@ booleanExpression
     | left=booleanExpression operator=KW_OR right=booleanExpression  # logicalBinary
     ;
 
-predicate[ParserRuleContext value]
+predicate[antlr.ParserRuleContext value]
     : comparisonOperator right=valueExpression                              # comparison
     | comparisonOperator comparisonQuantifier subQueryRelation              # quantifiedComparison
     | KW_NOT? KW_BETWEEN lower=valueExpression KW_AND upper=valueExpression # between

src/grammar/impala/ImpalaSqlParser.g4

@@ -35,6 +35,7 @@ parser grammar MySqlParser;
 
 options {
     tokenVocab= MySqlLexer;
+    caseInsensitive= true;
 }
 
 // Top Level Description

src/grammar/mysql/MySqlParser.g4

@@ -36,6 +36,10 @@
 
 lexer grammar PostgreSQLLexer;
 
+options {
+    caseInsensitive= true;
+}
+
 /**
  * Reference Doc: https://www.postgresql.org/docs/16.1/sql-commands.html
  */
@@ -673,9 +677,9 @@ KW_BUFFER_USAGE_LIMIT : 'BUFFER_USAGE_LIMIT';
 Identifier: IdentifierStartChar IdentifierChar*;
 
 fragment IdentifierStartChar: // these are the valid identifier start characters below 0x7F
-    [a-zA-Z_]
+    [A-Z_]
     | // these are the valid characters from 0x80 to 0xFF
-    [\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]
+    [\u00AA\u00B5\u00BA\u00C0-\u00D6\u00F8-\u00FF]
     | // these are the letters above 0xFF which only need a single UTF-16 code unit
     [\u0100-\uD7FF\uE000-\uFFFF]
     | // letters which require multiple UTF-16 code units
@@ -771,7 +775,7 @@ InvalidUnterminatedBinaryStringConstant: 'B' UnterminatedStringConstant;
 
 HexadecimalStringConstant: UnterminatedHexadecimalStringConstant '\'';
 
-UnterminatedHexadecimalStringConstant: 'X' '\'' [0-9a-fA-F]*;
+UnterminatedHexadecimalStringConstant: 'X' '\'' [0-9A-F]*;
 
 InvalidHexadecimalStringConstant: InvalidUnterminatedHexadecimalStringConstant '\'';
 
@@ -791,7 +795,7 @@ Numeric:
 
 fragment Digits: [0-9]+;
 
-PLSQLVARIABLENAME: ':' [a-zA-Z_] [a-zA-Z_0-9$]*;
+PLSQLVARIABLENAME: ':' [A-Z_] [A-Z_0-9$]*;
 
 PLSQLIDENTIFIER: ':"' ('\\' . | '""' | ~ ('"' | '\\'))* '"';
 //
@@ -861,13 +865,13 @@ fragment EscapeStringText: (
         '\'\''
         | '\\' (
             // two-digit hex escapes are still valid when treated as single-digit escapes
-            'x' [0-9a-fA-F]
-            | 'u' [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]
-            | 'U' [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]
+            'x' [0-9A-F]
+            | 'u' [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F]
+            | 'U' [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F]
             |
             // Any character other than the Unicode escapes can follow a backslash. Some have
             // special meaning, but that doesn't affect the syntax.
-            ~ [xuU]
+            ~ [xu]
         )
         | ~ ['\\]
     )*;

src/grammar/pgsql/PostgreSQLLexer.g4

@@ -41,6 +41,7 @@ parser grammar PostgreSQLParser;
 
 options {
     tokenVocab= PostgreSQLLexer;
+    caseInsensitive= true;
 }
 
 program

src/grammar/pgsql/PostgreSQLParser.g4

@@ -2366,7 +2366,7 @@ BIT_STRING_LIT: 'B' ('\'' [01]* '\'')+;
 //  Rule #284 <HEX_STRING_LIT> - subtoken typecast in <REGULAR_ID>
 //  Lowercase 'x' is a usual addition to the standard
 
-HEX_STRING_LIT : 'X' ('\'' [A-Fa-f0-9]* '\'')+;
+HEX_STRING_LIT : 'X' ('\'' [A-F0-9]* '\'')+;
 DOUBLE_PERIOD  : '..';
 PERIOD         : '.';
 
@@ -2468,10 +2468,8 @@ REMARK_COMMENT:
 PROMPT_MESSAGE: 'PRO' {this.IsNewlineAtPos(-4)}? 'MPT'? (' ' ~('\r' | '\n')*)? NEWLINE_EOF;
 
 // TODO: should starts with newline
-START_CMD
-    //: 'STA' 'RT'? SPACE ~('\r' | '\n')* NEWLINE_EOF
-    : // https://docs.oracle.com/cd/B19306_01/server.102/b14357/ch12002.htm
-    '@' {this.IsNewlineAtPos(-2)}? '@'? ~('\r' | '\n')* NEWLINE_EOF; // https://docs.oracle.com/cd/B19306_01/server.102/b14357/ch12003.htm
+START_CMD: // https://docs.oracle.com/cd/B19306_01/server.102/b14357/ch12002.htm
+    '@'    {this.IsNewlineAtPos(-2)}? '@'? ~('\r' | '\n')* NEWLINE_EOF; // https://docs.oracle.com/cd/B19306_01/server.102/b14357/ch12003.htm
 
 REGULAR_ID: SIMPLE_LETTER (SIMPLE_LETTER | '$' | '_' | '#' | [0-9])*;
 
@@ -2481,7 +2479,7 @@ SPACES: [ \t\r\n]+ -> channel(HIDDEN);
 
 fragment NEWLINE_EOF    : NEWLINE | EOF;
 fragment QUESTION_MARK  : '?';
-fragment SIMPLE_LETTER  : [a-zA-Z];
+fragment SIMPLE_LETTER  : [A-Z];
 fragment FLOAT_FRAGMENT : UNSIGNED_INTEGER* '.'? UNSIGNED_INTEGER+;
 fragment NEWLINE        : '\r'? '\n';
 fragment SPACE          : [ \t];