Hermetic Python 3 runtime by building from source + jupyter notebook (UBC-Thunderbots#1828)

* pull python3 and build it in bazel for a hermetic python runtime

* pull python3 and build it in bazel for a hermetic python runtime

* run jupyter

* remvoe macos stuff for python build and make the python build a little faster

Author: jordanzliu

environment_setup/setup_software.sh

@@ -58,6 +58,7 @@ host_software_packages=(
     python3-yaml # Load dynamic parameter configuration files
     qt5-default # The GUI library for our visualizer
     valgrind # Checks for memory leaks
+    libsqlite3-dev # needed to build Python 3 with sqlite support
 )
 
 if [[ $(lsb_release -rs) == "20.04" ]]; then

src/BUILD

@@ -0,0 +1,23 @@
+load("@rules_python//python:defs.bzl", "py_runtime")
+load("@rules_python//python:defs.bzl", "py_runtime_pair")
+
+# targets for hermetic Python runtime
+py_runtime(
+    name = "python3_runtime",
+    files = ["@python_interpreter//:files"],
+    interpreter = "@python_interpreter//:python_bin",
+    python_version = "PY3",
+    visibility = ["//visibility:public"],
+)
+
+py_runtime_pair(
+    name = "py_runtime_pair",
+    py2_runtime = None,
+    py3_runtime = ":python3_runtime",
+)
+
+toolchain(
+    name = "py_toolchain",
+    toolchain = ":py_runtime_pair",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)

src/WORKSPACE

@@ -192,35 +192,65 @@ http_archive(
 
 git_repository(
     name = "rules_python",
-    commit = "a0fbf98d4e3a232144df4d0d80b577c7a693b570",
+    commit = "c8c79aae9aa1b61d199ad03d5fe06338febd0774",
     remote = "https://github.com/bazelbuild/rules_python.git",
     shallow_since = "1586444447 +0200",
 )
 
-load("@rules_python//python:repositories.bzl", "py_repositories")
+# Based on the hermetic Python 3 guide found here
+# https://thethoughtfulkoala.com/posts/2020/05/16/bazel-hermetic-python.html
+_py_configure = "./configure --prefix=$(pwd)/bazel_install"
 
-py_repositories()
+http_archive(
+    name = "python_interpreter",
+    build_file_content = """
+exports_files(["python_bin"])
+filegroup(
+    name = "files",
+    srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
+    visibility = ["//visibility:public"],
+)
+""",
+    patch_cmds = [
+        "mkdir $(pwd)/bazel_install",
+        _py_configure,
+        "make -j$(nproc)",
+        "make install",
+        "ln -s bazel_install/bin/python3 python_bin",
+    ],
+    sha256 = "91923007b05005b5f9bd46f3b9172248aea5abc1543e8a636d59e629c3331b01",
+    strip_prefix = "Python-3.7.9",
+    urls = ["https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tar.xz"],
+)
 
-load("@rules_python//python:pip.bzl", "pip_repositories")
+register_toolchains("//:py_toolchain")
 
-pip_repositories()
+load("@rules_python//python:repositories.bzl", "py_repositories")
 
-load("@rules_python//python:pip.bzl", "pip3_import")
+py_repositories()
+
+load("@rules_python//python:pip.bzl", "pip_install")
 
-pip3_import(
+pip_install(
     name = "python_control_deps",
+    python_interpreter_target = "@python_interpreter//:python_bin",
     requirements = "//firmware/app/control/python_controller:requirements.txt",
 )
 
-load("@python_control_deps//:requirements.bzl", "pip_install")
-
-pip_install()
-
-pip3_import(
+pip_install(
     name = "dynamic_parameter_deps",
+    python_interpreter_target = "@python_interpreter//:python_bin",
     requirements = "//shared/parameter_v2/generation_scripts:requirements.txt",
 )
 
-load("@dynamic_parameter_deps//:requirements.bzl", "pip_install")
+pip_install(
+    name = "nanopb_deps",
+    python_interpreter_target = "@python_interpreter//:python_bin",
+    requirements = "//external:nanopb_requirements.txt",
+)
 
-pip_install()
+pip_install(
+    name = "python_tools_deps",
+    python_interpreter_target = "@python_interpreter//:python_bin",
+    requirements = "//python_tools:requirements.txt",
+)

src/external/nanopb.BUILD

@@ -4,6 +4,8 @@ exports_files(["LICENSE.txt"])
 
 package(default_visibility = ["//visibility:public"])
 
+load("@nanopb_deps//:requirements.bzl", "requirement")
+
 common_defines = [
     # By default, NanoPb only supports 8-bit tags. This define changes the tag type to
     # one that supports 16-bit tags.
@@ -52,7 +54,9 @@ cc_library(
 py_binary(
     name = "nanopb_generator",
     srcs = ["generator/nanopb_generator.py"],
-    imports = ["proto"],
+    deps = [
+        requirement("protobuf"),
+    ],
 )
 
 proto_library(

src/external/nanopb_requirements.txt

@@ -0,0 +1 @@
+protobuf==3.6.1

src/python_tools/BUILD

@@ -0,0 +1,17 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@python_tools_deps//:requirements.bzl", "requirement")
+
+py_test(
+    name = "test_hermetic_python_environment",
+    srcs = ["test_hermetic_python_environment.py"],
+)
+
+py_binary(
+    name = "jupyter",
+    srcs = ["jupyter.py"],
+    deps = [
+        requirement("jupyter"),
+        requirement("notebook"),
+    ],
+)

src/python_tools/jupyter.py

@@ -0,0 +1,5 @@
+from notebook.notebookapp import main
+import sys
+
+if __name__ == "__main__":
+    sys.exit(main())

src/python_tools/requirements.txt

@@ -0,0 +1,2 @@
+jupyter==1.0.0
+notebook==6.1.5

src/python_tools/test_hermetic_python_environment.py

@@ -0,0 +1,4 @@
+import sys
+
+# a basic test to make sure we're not running the system python interpreter
+assert "/usr/bin/python" not in sys.executable