site-packages/bin console-scripts support

* check python syntax with ast.parse() for non .py files (maybe blacklist needed)
* site-packages/bin scripts completions
* avoid the KeyboardInterrupt error in Pythonista 3.4: fix from ywangd#499
* updated scripts matching logics, now able to find console-scripts installed with `pip` in `site-packages/bin`

Author: o-murphy

__main__.py

@@ -6,6 +6,10 @@
 import sys
 import argparse
 
+# fix from https://github.com/ywangd/stash/pull/499
+if hasattr(sys, "_exit"):
+    sys.exit = sys._exit
+
 module_names = (
     "stash",
     "system.shcommon",

system/shcommon.py

@@ -3,6 +3,7 @@
 The Control, Escape and Graphics are taken from pyte (https://github.com/selectel/pyte)
 """
 
+import ast
 import os
 import sys
 import platform
@@ -70,7 +71,7 @@
 
 # directory for stash extensions
 _STASH_EXTENSION_PATH = os.path.abspath(
-    os.path.join(os.getenv("HOME"), "Documents", "stash_extensions"),
+    os.path.join(os.path.expanduser("~"), "Documents", "stash_extensions"),
 )
 # directory for stash bin extensions
 _STASH_EXTENSION_BIN_PATH = os.path.join(_STASH_EXTENSION_PATH, "bin")
@@ -181,6 +182,22 @@ def writelines(self, lines):
 _OS_ENVIRON = os.environ
 
 
+def is_true_python_file(filename):
+    try:
+        with open(filename, "rb", encoding="utf-8") as fp:
+            ast.parse(fp.read(), filename)
+            return True
+    except SyntaxError:
+        return False
+    except Exception:
+        return False
+
+
+def has_py_extension(filename):
+    if filename.endswith(".py"):
+        return True
+
+
 def is_binary_file(filename, nbytes=1024):
     """
     An approximate way to tell whether a file is binary.

system/shruntime.py

@@ -37,7 +37,12 @@
 
 # noinspection PyProtectedMember
 from .shcommon import _STASH_ROOT, _STASH_HISTORY_FILE, _SYS_STDOUT, _SYS_STDERR
-from .shcommon import is_binary_file, _STASH_EXTENSION_BIN_PATH
+from .shcommon import (
+    is_binary_file,
+    is_true_python_file,
+    has_py_extension,
+    _STASH_EXTENSION_BIN_PATH,
+)
 from .shparsers import ShPipeSequence
 from .shthreads import (
     ShBaseThread,
@@ -48,6 +53,11 @@
 )
 from .shhistory import ShHistory
 
+_HOME2 = os.path.join(os.path.expanduser("~"), "Documents")
+_SITE_PACKAGES = os.path.join(_HOME2, "site-packages")
+_SITE_PACKAGES_BIN = os.path.join(_SITE_PACKAGES, "bin")
+_SITE_PACKAGES_COMPLETER_MAX_FILES_LIMIT = 100
+
 # Default .stashrc file
 _DEFAULT_RC = r"""BIN_PATH=~/Documents/bin:{bin_ext}:$BIN_PATH
 SELFUPDATE_TARGET=master
@@ -79,14 +89,16 @@ def __init__(self, stash, parser, expander, no_historyfile=False, debug=False):
         self.state = ShState(
             environ=dict(
                 os.environ,
-                HOME2=os.path.join(os.environ["HOME"], "Documents"),
+                HOME2=_HOME2,
                 STASH_ROOT=_STASH_ROOT,
                 STASH_PY_VERSION=platform.python_version(),
                 BIN_PATH=os.path.join(_STASH_ROOT, "bin"),
                 # Must have a placeholder because it is needed before _DEFAULT_RC is loaded
                 PROMPT=r"[\W]$ ",
                 PYTHONISTA_ROOT=os.path.dirname(sys.executable),
                 TMPDIR=os.environ.get("TMPDIR", tempfile.gettempdir()),
+                # site-packages/bin (console scripts added via pip
+                SITE_PACKAGES_BIN=_SITE_PACKAGES_BIN,
             ),
             sys_stdin=self.stash.io,
             sys_stdout=self.stash.io,
@@ -170,30 +182,46 @@ def write_error_message(self, stream, msg, prefix=None, log=True):
     def find_script_file(self, filename):
         _, current_state = self.get_current_worker_and_state()
 
-        dir_match_found = False
         # direct match of the filename, e.g. full path, relative path etc.
         for fname in (filename, filename + ".py", filename + ".sh"):
             if os.path.exists(fname):
                 if os.path.isdir(fname):
-                    dir_match_found = True
+                    raise ShIsDirectory("%s: is a directory" % filename)
                 else:
                     return fname
 
         # Match for commands in current dir and BIN_PATH
         # Effectively, current dir is always the first in BIN_PATH
         for path in ["."] + current_state.environ_get("BIN_PATH").split(":"):
             path = os.path.abspath(os.path.expanduser(path))
-            if os.path.exists(path):
-                for f in os.listdir(path):
-                    if f == filename or f == filename + ".py" or f == filename + ".sh":
-                        if os.path.isdir(f):
-                            dir_match_found = True
-                        else:
-                            return os.path.join(path, f)
-        if dir_match_found:
-            raise ShIsDirectory("%s: is a directory" % filename)
-        else:
-            raise ShFileNotFound("%s: command not found" % filename)
+            file_match_found = self._find_matching_executable(filename, path)
+            if file_match_found:
+                return file_match_found
+
+        # match for commands in site-packages/bin
+        # is nothing was found in BIN_PATH
+        # assume all files in site-packages/bin is executable
+        path = current_state.environ_get("SITE_PACKAGES_BIN")
+        path = os.path.abspath(path)
+        file_match_found = self._find_matching_executable(filename, path)
+        if file_match_found:
+            return file_match_found
+
+        raise ShFileNotFound("%s: command not found" % filename)
+
+    @staticmethod
+    def _find_matching_executable(filename, path):
+        if not os.path.exists(path):
+            return None
+
+        for f in os.listdir(path):
+            if f in (filename, filename + ".py", filename + ".sh"):
+                found = os.path.join(path, f)
+                if os.path.isdir(found):
+                    raise ShIsDirectory("%s: is a directory" % filename)
+                return found
+
+        return None
 
     def get_all_script_names(self):
         """This function used for completer, whitespaces in names are escaped"""
@@ -207,6 +235,22 @@ def get_all_script_names(self):
                         f.endswith(".py") or f.endswith(".sh")
                     ):
                         all_names.append(f.replace(" ", "\\ "))
+
+        # find executables in site-packages/bin
+        path = current_state.environ_get("SITE_PACKAGES_BIN")
+        if os.path.exists(path):
+            count_unknown = 0
+            for f in os.listdir(path):
+                if not os.path.isdir(f):
+                    if f.endswith(".py") or f.endswith(".sh"):
+                        all_names.append(f.replace(" ", "\\ "))
+                    else:
+                        # assume we check only some range of .undefined
+                        if count_unknown < _SITE_PACKAGES_COMPLETER_MAX_FILES_LIMIT:
+                            if is_true_python_file(f):
+                                all_names.append(f.replace(" ", "\\ "))
+                            count_unknown += 1
+
         return all_names
 
     def run(
@@ -503,7 +547,8 @@ def run_pipe_sequence(
                     else:
                         simple_command_args = simple_command.args
 
-                    if script_file.endswith(".py"):
+                    # check file extension or if syntax match
+                    if has_py_extension(script_file) or is_true_python_file(script_file):
                         self.exec_py_file(
                             script_file, simple_command_args, ins, outs, errs
                         )