WIP - JS/Python package manager migrations: npm -> bun, pip -> uv

.github/workflows/docs.yaml

@@ -21,7 +21,7 @@ jobs:
             ~/.cache/sphinx
             ~/.ccache
             ~/.local
-            ~/.npm
+            ~/.bun
           key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }}
 
       - uses: actions/setup-python@v5
@@ -32,11 +32,14 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.docs.txt
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv --python 3.10 && source .venv/bin/activate
+          uv pip install --upgrade pip
+          uv pip install -r requirements.docs.txt
 
       - name: Build docs
         run: |
+          source .venv/bin/activate
           cd doc
           make html
           touch _build/html/.nojekyll

.github/workflows/test.yaml

@@ -36,9 +36,7 @@ jobs:
         with:
           python-version: "3.10"
 
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 20
+      - uses: oven-sh/setup-bun@v2
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -51,8 +49,8 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: |
-            ~/.npm
-          key: ${{ runner.os }}-npm-${{ hashFiles('baselayer_template_app/package.json') }}
+            ~/.bun
+          key: ${{ runner.os }}-bun-${{ hashFiles('baselayer_template_app/package.json') }}
 
       - uses: actions/cache@v4
         with:
@@ -66,16 +64,14 @@ jobs:
 
       - name: Install system dependencies
         run: |
-          cd baselayer
-
           sudo apt update -y
 
           ### firefox installation
           sudo snap remove firefox
           sudo add-apt-repository ppa:mozillateam/ppa
           printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox
 
-          sudo apt install -y wget nodejs unzip firefox
+          sudo apt install -y wget unzip firefox
 
           # if nginx is already installed, remove it
           sudo apt remove -y nginx nginx-common nginx-core nginx-full
@@ -86,31 +82,35 @@ jobs:
           sudo apt update -y
           sudo apt install nginx libnginx-mod-http-brotli-static libnginx-mod-http-brotli-filter -y
 
+          ### UV python package manager installation
+          curl -LsSf https://astral.sh/uv/install.sh | sh
 
-          pip install --upgrade pip
-          pip install wheel
+          cd baselayer_template_app
 
-          export NPM_PACKAGES="${HOME}/.npm-packages"
-          export PATH=${NPM_PACKAGES}/bin:$PATH
-          export NODE_PATH="$NPM_PACKAGES/lib/node_modules:$NODE_PATH"
+          uv venv --python 3.10 && source .venv/bin/activate
 
-          sudo npm -g install npm@latest
+          uv pip install --upgrade pip && uv pip install wheel
+
+          bun upgrade
 
           which python; python --version
-          echo npm $(npm --version)
-          echo node $(node --version)
+          echo uv $(uv --version)
+          echo bun $(bun --version)
           nginx -v
           firefox --version
 
       - name: Install Geckodriver / Selenium
         run: |
+          cd baselayer_template_app
+          source .venv/bin/activate
           geckodriver --version
-          pip install selenium==4.8.3
+          uv pip install selenium==4.8.3
           python -c "import selenium; print(f'Selenium {selenium.__version__}')"
 
       - name: Write configuration & build DB
         run: |
           cd baselayer_template_app
+          source .venv/bin/activate
 
           cat << EOF > config.yaml
             database:
@@ -144,6 +144,7 @@ jobs:
       - name: Test template app
         run: |
           cd baselayer_template_app
+          source .venv/bin/activate
           make test_headless
 
       - name: Upload logs

.gitignore

@@ -13,6 +13,7 @@ __pycache__/
 node_modules/
 *.pyc
 doc/_build
+bun.lockb
 
 # These are built from templates
 conf/supervisor/supervisor.conf

Makefile

@@ -1,5 +1,5 @@
 SHELL = /bin/bash
-ESLINT=npx eslint
+ESLINT=bun eslint
 
 .DEFAULT_GOAL := help
 
@@ -10,6 +10,11 @@ FLAGS:=$(if $(FLAGS),$(FLAGS),--config=config.yaml)
 PYTHON=PYTHONPATH=. python
 ENV_SUMMARY=$(PYTHON) baselayer/tools/env_summary.py $(FLAGS)
 
+# Set the Python package installer to use
+# if uv is available set it to uv pip, otherwise use pip
+# for that simply call the uv command, and if we get a non-zero exit code, we know it's not available
+PYTHON_PGK_INSTALLER := $(shell command -v uv 2>&1 >/dev/null && echo "uv pip" || echo "pip")
+
 # Flags are propagated to supervisord via the FLAGS environment variable
 # Inside of supervisord configuration files, you may reference them using
 # %(ENV_FLAGS)s
@@ -25,7 +30,7 @@ B=\033[1m
 N=\033[0m
 
 bundle = static/build/main.bundle.js
-rspack = npx rspack
+rspack = bun rspack
 
 # NOTE: These targets are meant to be *included* in the parent app
 #       Makefile.  See end of this file for baselayer specific targets.
@@ -41,7 +46,7 @@ help:
 	@python ./baselayer/tools/makefile_to_help.py $(MAKEFILE_LIST)
 
 dependencies: README.md
-	@PYTHONPATH=. pip install packaging
+	@PYTHONPATH=. $(PYTHON_PGK_INSTALLER) install packaging setuptools wheel
 	@baselayer/tools/check_app_environment.py
 	@PYTHONPATH=. python baselayer/tools/pip_install_requirements.py baselayer/requirements.txt requirements.txt
 	@./baselayer/tools/silent_monitor.py baselayer/tools/check_js_deps.sh
@@ -146,10 +151,6 @@ test_report: ## Print report on failed tests
 test_report:
 	@PYTHONPATH='.' baselayer/tools/junitxml_report.py test-results/junit.xml
 
-# Call this target to see which Javascript dependencies are not up to date
-check-js-updates:
-	./baselayer/tools/check_js_updates.sh
-
 # Lint targets
 lint-install: ## Install ESLint and a git pre-commit hook.
 lint-install: cp-lint-yaml lint-githook
@@ -182,7 +183,7 @@ lint-githook:
 
 # Documentation targets, run from the `baselayer` directory
 baselayer_doc_reqs:
-	pip install -q -r requirements.docs.txt
+	$(PYTHON_PGK_INSTALLER) install -q -r requirements.docs.txt
 
 baselayer_html: | baselayer_doc_reqs
 	export SPHINXOPTS=-W; make -C doc html

doc/dev.md

@@ -129,10 +129,3 @@ We use ESLint to ensure that our JavaScript & JSX code is consistent and conform
 
 - Install ESLint using `make lint-install`. This will also install a git pre-commit hook so that any commit is linted before it is checked in.
 - Run `make lint` to perform a style check
-
-## Upgrading Javascript dependencies
-
-The `./tools/check_js_updates.sh` script uses
-[`npm-check`](https://github.com/dylang/npm-check) to search updates
-for packages defined in `package.json`. It then provides an
-interactive interface for selecting new versions and performing the upgrade.

doc/setup.md

@@ -38,7 +38,7 @@ See [below](#configuration) for more information on modifying the baselayer conf
 ### On Linux
 
 - Using `apt-get`:
-  `sudo apt-get install supervisor postgresql libpq-dev npm nodejs-legacy`
+  `sudo apt-get install supervisor postgresql libpq-dev nodejs-legacy`
 
   If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you have to install NGINX and the brotli module from another source with:
 
@@ -51,6 +51,12 @@ See [below](#configuration) for more information on modifying the baselayer conf
 
   Otherwise, you can install NGINX normally with `sudo apt-get install nginx`.
 
+  Then, we install `Bun` (a javascript runtime, faster equivalent to `node+npm`):
+
+  ```
+  curl -fsSL https://bun.sh/install | bash
+  ```
+
 - It may be necessary to configure your database permissions: at
   the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main` or `/var/lib/pgsql/data`),
   add the following lines and restart PostgreSQL

services/rspack/rspack.py

@@ -20,7 +20,7 @@ def run(cmd):
 
 if env.debug:
     log("Debug mode detected, launching rspack monitor")
-    p = run(["npx", "rspack", "--watch"])
+    p = run(["bun", "rspack", "--watch"])
     sys.exit(p.returncode)
 else:
     log("Production mode; not building JavaScript bundle")

tools/check_app_environment.py

@@ -29,8 +29,7 @@ def output(cmd):
         lambda v: v.split("\n")[-1].split()[2],
         "12.0",
     ),
-    "npm": (["npm", "-v"], lambda v: v, "8.3.2"),
-    "node": (["node", "-v"], lambda v: v[1:], "16.14.0"),
+    "bun": (["bun", "-v"], lambda v: v, "1.1.33"),
     "python": (["python", "--version"], lambda v: v.split()[1], "3.8"),
 }
 

tools/check_js_deps.sh

@@ -2,18 +2,28 @@
 
 set -e
 
-CHECKER="node_modules/.bin/check-dependencies"
+# in this script, we want to use bun install to check if all the dependencies are installed
+# if we run bun install and:
+# - it contains and error message, print it out and exit
+# - it contains "(no changes)" in the output, then we know all the dependencies are installed
+# - otherwise, we tell the user that there are missing dependencies that were installed
 
-if [[ ! -x ${CHECKER} ]]; then
-    npm install check-dependencies --legacy-peer-deps
-fi
+INSTALLER="bun install"
+
+# first run it and save the output
+output=$(${INSTALLER} 2>&1)
 
-# We suppress output for the next command because, annoyingly, it reports
-# that a dependency is unsatisfied even if the --install flag is specified,
-# and that package has been successfully installed
-${CHECKER} --install
+# if we got an error message, print it out and exit with an error
+if [ $? -ne 0 ]; then
+    echo "✗ Error installing Javascript dependencies:"
+    echo "${output}"
+    exit 1
+fi
 
-# Print report, if any unsatisfied dependencies remain
-if ${CHECKER}; then
-    echo "✓ All Javascript dependencies satisfied."
+# check if the output contains "(no changes)"
+if echo "${output}" | grep -q "(no changes)"; then
+     echo "✓ All Javascript dependencies satisfied."
+else
+    echo "✗ Some Javascript dependencies are unsatisfied."
+    echo "✓ Missing dependencies have been installed."
 fi

tools/pip_install_requirements.py

@@ -5,6 +5,14 @@
 from pkg_resources import DistributionNotFound, Requirement, VersionConflict
 from status import status
 
+uv = True
+# check if the uv command is available
+# if not, use pip directly
+try:
+    subprocess.run(["uv", "--version"], stdout=subprocess.PIPE)
+except FileNotFoundError:
+    uv = False
+
 if len(sys.argv) < 2:
     print(
         "Usage: pip_install_requirements.py requirements.txt [requirements_other.txt]"
@@ -19,9 +27,14 @@
 
 
 def pip(req_files):
-    args = ["pip", "install"]
+    if uv:
+        args = ["uv", "pip", "install"]
+    else:
+        args = ["pip", "install"]
+
     for req_file in req_files:
         args.extend(["-r", req_file])
+
     p = subprocess.Popen(
         args,
         stdout=subprocess.PIPE,