From d928e192c7c8da0685b589211ff8ce8225f5a39d Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Fri, 28 Jul 2017 16:16:09 -0400 Subject: [PATCH 1/3] basic mvp of bash repr --- app/__init__.py | 17 ++++++++++++----- app/ast_dump.py | 20 ++++++++++++++++++++ app/bash_dump_node.py | 24 ++++++++++++++++++++++++ app/static/src/editor.vue | 15 ++++++++++++++- requirements.txt | 1 + 5 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 app/bash_dump_node.py diff --git a/app/__init__.py b/app/__init__.py index efc632d..fda9fc9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -10,7 +10,8 @@ from antlr_plsql import ast as plsql_ast from antlr_tsql import ast as tsql_ast import ast as python_ast -from .ast_dump import dump_node +from .ast_dump import dump_node, dump_bash +import bashlex def get_parser(parser_name): parsers = {'plsql': plsql_ast, 'tsql': tsql_ast} @@ -24,6 +25,10 @@ def get_ast(code, start, parser_name): return tsql_ast.parse(code, start) elif parser_name == "python": return python_ast.parse(code) + elif parser_name == "bash-simple": + return bashlex.parse(code) + elif parser_name == "bash-verbose": + return bashlex.parse(code) return None @@ -31,8 +36,10 @@ def get_ast(code, start, parser_name): from flask import Flask, request, url_for, redirect, jsonify, make_response import yaml -def str_or_dump(ast): - if isinstance(ast, str): return {'type': 'PYTHON_OBJECT', 'data': {"": ast}} +def str_or_dump(ast, parser): + if parser == 'bash-simple': return dump_bash(ast) + elif parser == 'bash-verbose': return dump_bash(ast, v = True) + elif isinstance(ast, str): return {'type': 'PYTHON_OBJECT', 'data': {"": ast}} elif hasattr(ast, '_dump'): return ast._dump() else: return dump_node(ast) @@ -48,7 +55,7 @@ def ast_postgres(): ast = get_ast(args['code'], args['start'], args['parser']) if ast is None: return make_response("Incorrect parser name", 400) - return jsonify(str_or_dump(ast)) + return jsonify(str_or_dump(ast, args['parser'])) @app.route('/ast-from-config', methods = ['GET', 'POST']) def ast_from_config(): @@ -63,7 +70,7 @@ def ast_from_config(): out = {} for k, v in trees.items(): - json_asts = [str_or_dump(tree) for tree in v] + json_asts = [str_or_dump(tree, args['parser_name']) for tree in v] sql_cmds = code[k] zipped = zip(sql_cmds, json_asts) # entry with attrs code: sql_cmd, ast: json_ast diff --git a/app/ast_dump.py b/app/ast_dump.py index ed92762..d7c84a2 100644 --- a/app/ast_dump.py +++ b/app/ast_dump.py @@ -15,3 +15,23 @@ def dump_node(obj): return [dump_node(x) for x in obj] else: return obj + +import bashlex +def dump_bash(obj, parent_cls = bashlex.ast.node, v = False): + # pull element out of single entry lists + if isinstance(obj, (list, tuple)) and len(obj) == 1: obj = obj[0] + # dump to dict + if isinstance(obj, parent_cls): + if obj.kind in ['word', 'reservedword'] and not v: + return obj.word + fields = OrderedDict() + for name in [el for el in obj.__dict__.keys() if el not in ('kind', 'pos')]: + attr = getattr(obj, name) + if isinstance(attr, parent_cls): fields[name] = dump_bash(attr, parent_cls, v) + elif isinstance(attr, list) and len(attr) == 0: continue + elif isinstance(attr, list): fields[name] = [dump_bash(x, parent_cls, v) for x in attr] + else: fields[name] = attr + return {'type': obj.kind, 'data': fields} + elif isinstance(obj, list): + return [dump_bash(x, parent_cls, v) for x in obj] + else: raise Exception("received non-node object?") diff --git a/app/bash_dump_node.py b/app/bash_dump_node.py new file mode 100644 index 0000000..22fb5ff --- /dev/null +++ b/app/bash_dump_node.py @@ -0,0 +1,24 @@ +from collections import OrderedDict + +def dump_node(obj, parent_cls): + # pull element out of single entry lists + if isinstance(obj, (list, tuple)) and len(obj) == 1: obj = obj[0] + # dump to dict + if isinstance(obj, parent_cls): + fields = OrderedDict() + for name in [el for el in obj.__dict__.keys() if el != 'kind']: + attr = getattr(obj, name) + if isinstance(attr, parent_cls): fields[name] = dump_node(attr, parent_cls) + elif isinstance(attr, list) and len(attr) == 0: continue + elif isinstance(attr, list): fields[name] = [dump_node(x, parent_cls) for x in attr] + else: fields[name] = attr + return {'type': obj.kind, 'data': fields} + elif isinstance(obj, list): + return [dump_node(x) for x in obj] + else: raise Exception("received non-node object?") + +import bashlex + +parts = bashlex.parse('true && echo hey') + +out = dump_node(parts, bashlex.ast.node) diff --git a/app/static/src/editor.vue b/app/static/src/editor.vue index 00d943f..40c3dfa 100644 --- a/app/static/src/editor.vue +++ b/app/static/src/editor.vue @@ -38,16 +38,29 @@ var request = require('superagent') var grammars = [ { name: 'plsql', + show_parse: true, funcs: require('../grammar/antlr_plsql/js/index.js').default, start: 'sql_script' }, { name: 'tsql', + show_parse: true, funcs: require('../grammar/antlr_tsql/js/index.js').default, start: 'tsql_file' }, { name: 'python', + show_parse: false, + start: 'NA' + }, + { + name: 'bash-simple', + show_parse: false, + start: 'NA' + }, + { + name: 'bash-verbose', + show_parse: false, start: 'NA' } ] @@ -128,7 +141,7 @@ export default { parseCode () { var grammar = this.crntGrammar.funcs - if (this.crntGrammar.name != "python") + if (this.crntGrammar.show_parse) this.codeData = parseFromGrammar(grammar, this.code, this.parserStart) else this.codeData = {} diff --git a/requirements.txt b/requirements.txt index d374fae..edd3d0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ gevent flask whitenoise pyyaml +bashlex From 390899ee184cc80faddcc6ce2a6daa788347fa37 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 8 Aug 2017 17:26:07 -0400 Subject: [PATCH 2/3] add vorpal bash-parser for examination --- app/__init__.py | 5 ++++- app/ast_dump.py | 29 +++++++++++++++++++++++++++++ app/static/src/editor.vue | 5 +++++ package.json | 4 +++- requirements.txt | 1 + 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index fda9fc9..4a2019c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -10,7 +10,7 @@ from antlr_plsql import ast as plsql_ast from antlr_tsql import ast as tsql_ast import ast as python_ast -from .ast_dump import dump_node, dump_bash +from .ast_dump import dump_node, dump_bash, dump_vorpal_bash, get_vorpal_bash_ast import bashlex def get_parser(parser_name): @@ -29,6 +29,8 @@ def get_ast(code, start, parser_name): return bashlex.parse(code) elif parser_name == "bash-verbose": return bashlex.parse(code) + elif parser_name == "bash-vorpal": + return get_vorpal_bash_ast(code) return None @@ -39,6 +41,7 @@ def get_ast(code, start, parser_name): def str_or_dump(ast, parser): if parser == 'bash-simple': return dump_bash(ast) elif parser == 'bash-verbose': return dump_bash(ast, v = True) + elif parser == 'bash-vorpal': return dump_vorpal_bash(ast) elif isinstance(ast, str): return {'type': 'PYTHON_OBJECT', 'data': {"": ast}} elif hasattr(ast, '_dump'): return ast._dump() else: return dump_node(ast) diff --git a/app/ast_dump.py b/app/ast_dump.py index d7c84a2..4faa66d 100644 --- a/app/ast_dump.py +++ b/app/ast_dump.py @@ -35,3 +35,32 @@ def dump_bash(obj, parent_cls = bashlex.ast.node, v = False): elif isinstance(obj, list): return [dump_bash(x, parent_cls, v) for x in obj] else: raise Exception("received non-node object?") + + +def dump_vorpal_bash(obj, v = False): + # pull element out of single entry lists + if isinstance(obj, (list, tuple)) and len(obj) == 1: obj = obj[0] + # dump to dict + if isinstance(obj, list): + return [dump_vorpal_bash(x, v) for x in obj] + elif isinstance(obj, dict): + if obj['type'] in ['Word'] and not v: + return obj['text'] + fields = OrderedDict() + for name in [el for el in obj.keys() if el not in ('loc', 'type')]: + attr = obj[name] + if isinstance(attr, dict): fields[name] = dump_vorpal_bash(attr, v) + elif isinstance(attr, list) and len(attr) == 0: continue + elif isinstance(attr, list): fields[name] = [dump_vorpal_bash(x, v) for x in attr] + else: fields[name] = attr + return {'type': obj['type'], 'data': fields} + else: raise Exception("received non-node object?") + +import execjs +import os +node_path = os.getcwd() + '/node_modules' +os.environ['NODE_PATH'] = node_path + ':' + os.environ.get('NODE_PATH', "") +def get_vorpal_bash_ast(src): + node = execjs.get('node') + return node.eval("""require('bash-parser')("%s")"""%src.replace('"', r'\"')) + diff --git a/app/static/src/editor.vue b/app/static/src/editor.vue index 40c3dfa..2529179 100644 --- a/app/static/src/editor.vue +++ b/app/static/src/editor.vue @@ -62,6 +62,11 @@ var grammars = [ name: 'bash-verbose', show_parse: false, start: 'NA' + }, + { + name: 'bash-vorpal', + show_parse: false, + start: 'NA' } ] diff --git a/package.json b/package.json index 67d4880..9bdbf28 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "description": "", "main": "index.js", - "dependencies": {}, + "dependencies": { + "bash-parser": "^0.5.0" + }, "devDependencies": {}, "scripts": { "test": "echo \"Error: no test specified\" && exit 1" diff --git a/requirements.txt b/requirements.txt index edd3d0c..f95b590 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ flask whitenoise pyyaml bashlex +PyExecJS From 1f074ca8a28d02d0defab472592434d260368274 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 8 Aug 2017 17:38:40 -0400 Subject: [PATCH 3/3] allow multiline vorpal bash --- app/ast_dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ast_dump.py b/app/ast_dump.py index 4faa66d..92f11ca 100644 --- a/app/ast_dump.py +++ b/app/ast_dump.py @@ -62,5 +62,5 @@ def dump_vorpal_bash(obj, v = False): os.environ['NODE_PATH'] = node_path + ':' + os.environ.get('NODE_PATH', "") def get_vorpal_bash_ast(src): node = execjs.get('node') - return node.eval("""require('bash-parser')("%s")"""%src.replace('"', r'\"')) + return node.eval("""require('bash-parser')(`%s`)"""%src.replace('"', r'\"'))