Trac #33739: Migrate gitpod to conda

The current solution to prebuild the gitpod docker image using github
actions does not work nicely, since prebuilds triggered shortly after a
new release still use the old gitpod image (and it may take up to a few
days until the new image is available). This is especially notable for
the `develop` branch, which is the default entry point if you want to
start a new branch on gitpod.

In this ticket, we migrate the gitpod config to use the conda
environment, which is now generally stable enough for serious
development (e.g. almost all tests pass). As a positive side effect, the
gitpod config simplifies and startup times are considerably reduced.

URL: https://trac.sagemath.org/33739
Reported by: gh-tobiasdiez
Ticket author(s): Tobias Diez
Reviewer(s): Matthias Koeppe

Author: Release Manager

.github/workflows/tox.yml

@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
       max-parallel: 20
       matrix:
-        tox_system_factor: [gitpod, ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy,  debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
+        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy,  debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
         tox_packages_factor: [minimal, standard]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}

.gitpod.yml

@@ -1,41 +1,32 @@
 # Use custom docker image. https://www.gitpod.io/docs/config-docker
 image:
-# Each of these two options works:
-# - Directly use the "-with-targets" image built by .github/workflows/tox.yml
-#   image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev
-# - or go through a custom Dockerfile, which builds a smaller image
-#   based on the "-with-system-packages" image built by .github/workflows/tox.yml
-#   with the built SAGE_LOCAL from the "-with-targets" image copied in.
-    file: docker/.gitpod.Dockerfile
-
-# We use the following layout:
-#
-# $HOME/sage            Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod).
-#                       - We delete it in every invocation of the 'before' script
-#                         and replace it by a symlink to /workspace/...
-#                         (This symlink is needed because the package-removal scripts
-#                         ({prefix,venv}/var/lib/sage/scripts/*/{spkg-piprm,spkg-prerm,spkg-postrm)
-#                         hardcode SAGE_ROOT and SAGE_SRC from package installation time)
-# $HOME/sage/logs       Logs of the Docker build.
-#                       - In the first invocation of the 'before' script, we move it
-#                         to /workspace/.../logs
-# $HOME/sage-local      The configured prefix (SAGE_LOCAL) of the Sage installation.
-#                       - During the Docker build, this is the physical location.
-#                       - In the first invocation of the 'before' script, we move it
-#                         to the new physical location /workspace/.../local
-#                         (because gitpod only preserves the contents of /workspace)
-#                         and replace it by a symlink to the new physical location.
-#                       - In subsequent invocations of the 'before' script, we
-#                         remove it and replace it by a symlink to the physical
-#                         location /workspace/.../local
-# /worktree/.../local   The physical location of the Sage installation,
-#                       established in the first run of the 'before' script and
-#                       preserved by gitpod.
+  file: docker/.gitpod.Dockerfile
 
 # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/
 tasks:
   - name: Setup
-    before: |
+    init: |
+      # Create conda environment
+      ./bootstrap-conda
+      mamba env create --file src/environment-dev.yml --prefix venv
+      conda config --append envs_dirs /workspace/sagetrac-mirror
+      conda activate /workspace/sagetrac-mirror/venv
+      
+      # Build sage
+      ./bootstrap
+      ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX 
+      pip install --no-build-isolation -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup 
+      pip install --no-build-isolation -v -v -e ./src
+
+    command: |
+      # Activate conda environment
+      conda config --append envs_dirs /workspace/sagetrac-mirror
+      conda activate /workspace/sagetrac-mirror/venv
+
+      # RestructuredText extension recommends python extension, although we have already installed it
+      ## So disable the recommendation dialog
+      echo "{\"restructuredtext.pythonRecommendation.disabled\": true}" > /workspace/.vscode-remote/data/Machine/settings.json
+
       # Setup trac as remote
       ## In order to push to trac, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$(<tempkey)"` (or by following https://www.gitpod.io/docs/environment-variables#using-the-account-settings)
       ## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac.
@@ -47,7 +38,7 @@ tasks:
         echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa
         sed -i '/^$/d' ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
-        unset PRIVATE_SSH_KEY
+        echo "PubkeyAcceptedKeyTypes +ssh-rsa" > ~/.ssh/config
         ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts
 
         # Setup trac repo
@@ -60,43 +51,7 @@ tasks:
         git remote add trac https://github.com/sagemath/sagetrac-mirror.git -t master -t develop
         git remote set-url --push trac pushing-needs-ssh-key
       fi
-      
-      ## No need for pyenv
-      pyenv shell --unset 2> /dev/null
-      pyenv global system 2> /dev/null
-      if [ -d local ]; then
-        mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log
-        # The init script has already populated the SAGE_LOCAL,
-        # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected
-        # copy of sage-local.  Replace it again by a symlink.
-        rm -Rf $HOME/sage-local
-        ln -sf $(pwd)/local $HOME/sage-local
-        # Now run make. No timeout here.
-        MAKE='make -j24' make build V=0
-      else
-        # Prebuild init script has not been run
-        # Only /workspace is preserved during build.
-        # If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace.
-        if [ -d $HOME/sage-local ]; then
-          mv $HOME/sage-local local
-        fi
-        rm -Rf $HOME/sage-local
-        ln -sf $(pwd)/local $HOME/sage-local
-        # Save the logs of the source tree used by the Docker build
-        if [ -d $HOME/sage/logs ]; then
-          mv $HOME/sage/logs logs
-        fi
-      fi
-      # Remove the source tree used by the Docker build and replace it by a symlink
-      rm -Rf $HOME/sage
-      ln -s $(pwd) $HOME/sage
-    init: |
-      # Start build
-      mkdir -p logs && echo '### .gitpod.yml Setup.init: Starting build' >> logs/install.log
-      ./bootstrap
-      ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv
-      ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful
-      MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)"
+
     env:
       SAGE_NUM_THREADS: 8
 
@@ -109,6 +64,8 @@ vscode:
     - lextudio.restructuredtext
     - streetsidesoftware.code-spell-checker
     - ms-toolsai.jupyter
+    - ms-toolsai.jupyter-keymap
+    - ms-toolsai.jupyter-renderers
 
 # https://www.gitpod.io/docs/prebuilds#github-specific-configuration
 github:

.vscode/settings.json

@@ -21,6 +21,8 @@
         "--doctest-modules",
     ],
     "python.testing.unittestEnabled": false,
+    "python.linting.pycodestyleEnabled": true,
+    "python.linting.enabled": true,
     "cSpell.words": [
         "furo"
         "Conda",

bootstrap-conda

@@ -65,6 +65,7 @@ echo "  # Additional dev tools" >> src/environment-dev.yml
 echo "  - openssh" >> src/environment-dev.yml
 echo "  - pycodestyle" >> src/environment-dev.yml
 echo "  - pytest" >> src/environment-dev.yml
+echo "  - esbonio" >> src/environment-dev.yml
 
 cp environment.yml environment-optional.yml
     echo "  # optional packages" >> environment-optional.yml

docker/.gitpod.Dockerfile

@@ -1,12 +1,6 @@
-ARG BASE_GITHUB_REPOSITORY=sagemath/sage
-ARG BASE_TAG=dev
-FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets
-RUN sudo rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /tmp/* /var/tmp/*
-# Fast doc rebuilds do not work because
-# "loading pickled environment... failed; source directory has changed"
-# Until this is fixed, we can as well remove the whole documentation, which saves a lot of space.
-RUN rm -Rf /home/gitpod/sage-local/share/doc/sage
-
-FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-system-packages:${BASE_TAG}
-COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage/logs  /home/gitpod/sage/logs
-COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage-local /home/gitpod/sage-local
+# Use minimal Ubuntu installation that includes mamba
+FROM condaforge/mambaforge
+# Workaround so that vscode internals (such as git) find things installed in the conda env (notably ssh which is required to contribute to trac)
+ENV PATH $PATH:/workspace/sagetrac-mirror/venv/bin
+# Default to non-admin user
+USER gitpod

tox.ini

@@ -437,22 +437,6 @@ setenv =
     ubuntu-bionic-nvidia-cuda:      BASE_TAG=ubuntu18.04
     ubuntu-xenial-nvidia-cuda:      BASE_TAG=ubuntu16.04
     #
-    # https://hub.docker.com/r/gitpod/
-    #
-    gitpod:           SYSTEM=debian
-    gitpod:             BASE_IMAGE=gitpod/workspace-base
-    gitpod-full:        BASE_IMAGE=gitpod/workspace-full
-    gitpod:               CONFIG_CONFIGURE_ARGS_ROOT=--prefix=/home/gitpod/sage-local --with-sage-venv
-    gitpod:                 __SUDO=--sudo
-    gitpod:                   __CHOWN=--chown=gitpod:gitpod
-                                # Play safe and make sure that the image build succeeds even
-                                # if packages that do not exist on ubuntu-focal are added to debian.txt
-    gitpod:                     IGNORE_MISSING_SYSTEM_PACKAGES=yes
-                                  # As of 2022-01, gitpod/workspace-base is based on ubuntu-focal.
-                                  # To save space, we filter out some packages that are too old and
-                                  # will be rejected by our configure script.
-    gitpod-standard:              SAGE_PACKAGE_LIST_ARGS=--has-file=spkg-configure.m4 :standard: --exclude pari --exclude eclib --exclude lcalc --exclude giac --exclude singular
-    #
     # Resulting full image:tag name
     #
     docker:           FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:}