Install RabbitMQ from conda on arm64

.github/workflows/main.yml

@@ -58,9 +58,9 @@ jobs:
     # To save arm64 runner resources, we run the tests only on main
     # and only for full-stack image (same for integration tests below).
     test-arm64:
-        if: >-
-            github.repository == 'aiidalab/aiidalab-docker-stack'
-            && (github.ref_type == 'tag' || github.ref_name == 'main')
+        #if: >-
+        #    github.repository == 'aiidalab/aiidalab-docker-stack'
+        #    && (github.ref_type == 'tag' || github.ref_name == 'main')
         needs: build
         uses: ./.github/workflows/test.yml
         with:
@@ -80,9 +80,9 @@ jobs:
                 runner: [ubuntu-22.04, buildjet-4vcpu-ubuntu-2204-arm]
                 isPR:
                     - ${{ github.event_name == 'pull_request' }}
-                exclude:
-                    - isPR: true
-                      runner: buildjet-4vcpu-ubuntu-2204-arm
+                #exclude:
+                    #    - isPR: true
+                    #  runner: buildjet-4vcpu-ubuntu-2204-arm
 
         uses: ./.github/workflows/test.yml
         with:

build.json

@@ -7,14 +7,17 @@
       "default": "15"
     },
     "RMQ_VERSION": {
-      "default": "3.9.13"
+      "default": "3.13.5"
     },
     "AIIDA_VERSION": {
       "default": "2.6.2"
     },
     "AIIDALAB_VERSION": {
       "default": "24.09.0"
     },
+    "AWB_VERSION": {
+      "default": "2.3.0a2"
+    },
     "AIIDALAB_HOME_VERSION": {
       "default": "24.09.0"
     }

docker-bake.hcl

@@ -17,6 +17,9 @@ variable "AIIDA_VERSION" {
 variable "AIIDALAB_VERSION" {
 }
 
+variable "AWB_VERSION" {
+}
+
 variable "AIIDALAB_HOME_VERSION" {
 }
 
@@ -74,7 +77,6 @@ target "base-with-services" {
   }
   platforms = "${PLATFORMS}"
   args = {
-    "AIIDA_VERSION" = "${AIIDA_VERSION}"
     "PGSQL_VERSION" = "${PGSQL_VERSION}"
     "RMQ_VERSION" = "${RMQ_VERSION}"
   }
@@ -88,6 +90,7 @@ target "lab" {
   platforms = "${PLATFORMS}"
   args = {
     "AIIDALAB_VERSION"      = "${AIIDALAB_VERSION}"
+    "AWB_VERSION"      = "${AWB_VERSION}"
     "AIIDALAB_HOME_VERSION" = "${AIIDALAB_HOME_VERSION}"
     "PYTHON_MINOR_VERSION" = get_python_minor_version("${PYTHON_VERSION}")
   }

stack/base-with-services/Dockerfile

@@ -6,44 +6,20 @@ LABEL maintainer="AiiDAlab Team <[email protected]>"
 USER root
 WORKDIR /opt/
 
-ARG AIIDA_VERSION
 ARG PGSQL_VERSION
 ARG RMQ_VERSION
-ARG TARGETARCH
 
 # Location of the Postgresql DB
 # This variable is automatically picked up by initdb and pg_ctl
 # WARNING: If you change this, you have to change it in full-stack as well!
 ENV PGDATA=/home/${NB_USER}/.postgresql
 
 # Install RabbitMQ and PostgreSQL in a dedicated conda environment.
-#
-# RabbitMQ is currently not available on conda-forge for arm64, see:
-# https://github.com/conda-forge/rabbitmq-server-feedstock/issues/67If
-# Instead we need install erlang via apt and RabbitMQ as a "Generic Unix Build", see:
-# https://www.rabbitmq.com/install-generic-unix.html
-# Note that this version must be compatible with system's erlang version.
-# Currently installed Erlang version is 23.3, so the latest supported RMQ version is 3.9.21
-# https://www.rabbitmq.com/docs/which-erlang#old-timers
-# Note that system erlang from arm64 is already installed in the base image,
-# together with other APT dependencies to save build time.
-RUN if [ "$TARGETARCH" = "amd64" ]; then \
-     mamba create -p /opt/conda/envs/aiida-core-services --yes \
-         postgresql=${PGSQL_VERSION} \
-         rabbitmq-server=${RMQ_VERSION} && \
-     mamba clean --all -f -y && \
-     fix-permissions "${CONDA_DIR}"; \
-  elif [ "$TARGETARCH" = "arm64" ]; then \
-     mamba create -p /opt/conda/envs/aiida-core-services --yes \
-         postgresql=${PGSQL_VERSION} && \
-     mamba clean --all -f -y && \
-     wget -c https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \
-     tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \
-     rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \
-     mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/rabbitmq_server && \
-     ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server/sbin/* /opt/conda/envs/aiida-core-services/bin/ && \
-     fix-permissions "${CONDA_DIR}"; \
-  fi
+RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \
+        postgresql=${PGSQL_VERSION} \
+        rabbitmq-server=${RMQ_VERSION} && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}"
 
 # Configure AiiDA profile.
 COPY config-quick-setup.yaml .

stack/base-with-services/before-notebook.d/30_start-rabbitmq.sh

@@ -4,7 +4,4 @@ set -emx
 # Fix issue where the erlang cookie permissions are corrupted.
 chmod 400 "/home/${NB_USER}/.erlang.cookie" || echo "erlang cookie not created yet."
 
-# NOTE: In arm64 build, rabbitmq is not installed via conda,
-# but the following incantation still works since
-# rabbitmq-server is available globally.
 mamba run -n aiida-core-services rabbitmq-server -detached

stack/base-with-services/configure_rabbitmq.sh

@@ -1,14 +1,9 @@
 #!/bin/bash
 set -emx
 
-RMQ_ETC_DIR_ARM64="/opt/conda/envs/aiida-core-services/rabbitmq_server/etc/rabbitmq"
-RMQ_ETC_DIR_AMD64="/opt/conda/envs/aiida-core-services/etc/rabbitmq"
-if [[ -d $RMQ_ETC_DIR_ARM64 ]]; then
-    RMQ_ETC_DIR="$RMQ_ETC_DIR_ARM64"
-elif [[ -d $RMQ_ETC_DIR_AMD64 ]]; then
-    RMQ_ETC_DIR="$RMQ_ETC_DIR_AMD64"
-else
-    echo "ERROR: Could not find RabbitMQ etc directory"
+RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/etc/rabbitmq"
+if [[ ! -d $RMQ_ETC_DIR ]]; then
+    echo "ERROR: directory '$RMQ_ETC_DIR' does not exist!"
     exit 1
 fi
 

stack/base/Dockerfile

@@ -5,19 +5,15 @@ LABEL maintainer="AiiDAlab Team <[email protected]>"
 
 USER root
 
-# build-essential: includes GCC compilers that are needed when building
-# pip packages from sources, which often seems to happen for pymatgen:
-# https://pymatgen.org/installation.html#installation-tips-for-optional-libraries
 # rsync: needed to support the new AiiDA backup command
 # povray: rendering engine used in aiidalab-widgets-base
-ENV EXTRA_APT_PACKAGES "curl povray rsync build-essential"
+ENV EXTRA_APT_PACKAGES "curl povray rsync"
 
-# For ARM64 we need to install erlang as it is not available on conda-forge
-# (this is needed later as rabbitmq dependency in base-with-services image,
-# but we install it here so that we don't have to invoke apt multiple times.
+# For ARM64 we also install build-essential, which includes GCC compilers that are needed when building
+# pip packages from sources, which is more often needed on ARM64.
 ARG TARGETARCH
 RUN if [ "$TARGETARCH" = "arm64" ]; then \
-        EXTRA_APT_PACKAGES="erlang libhdf5-serial-dev pkg-config ${EXTRA_APT_PACKAGES}"; \
+        EXTRA_APT_PACKAGES="build-essential libhdf5-serial-dev pkg-config ${EXTRA_APT_PACKAGES}"; \
     fi;\
     apt-get update --yes && \
     apt-get install --yes --no-install-recommends ${EXTRA_APT_PACKAGES} && \
@@ -51,7 +47,7 @@ COPY pip.conf /etc/pip.conf
 # Install aiida-core and other shared requirements.
 RUN mamba update -y pip mamba zstandard async_generator certipy && \
      mamba install --yes \
-     aiida-core==${AIIDA_VERSION} \
+     aiida-core.atomic_tools==${AIIDA_VERSION} \
      mamba-bash-completion \
      && mamba clean --all -f -y && \
      fix-permissions "${CONDA_DIR}"

stack/lab/Dockerfile

@@ -13,10 +13,11 @@ ENV DOCKER_STACKS_JUPYTER_CMD=notebook
 USER root
 WORKDIR /opt/
 
-# Install aiidalab package
 ARG AIIDALAB_VERSION
+ARG AWB_VERSION
 RUN mamba install --yes \
      aiidalab=${AIIDALAB_VERSION} \
+     aiidalab-widgets-base=${AWB_VERSION} \
      && mamba clean --all -f -y && \
      fix-permissions "${CONDA_DIR}" && \
      fix-permissions "/home/${NB_USER}"

stack/lab/before-notebook.d/60_prepare-aiidalab.sh

@@ -6,11 +6,6 @@ set -x
 # Environment.
 export SHELL=/bin/bash
 
-# Fix https://github.com/aiidalab/aiidalab-docker-stack/issues/225
-if [ -L /home/${NB_USER}/${NB_USER} ]; then
-  rm /home/${NB_USER}/${NB_USER}
-fi
-
 # Check whether this is likely the first time we run this script.
 [[ -z $(ls -A "/home/${NB_USER}/apps/") ]] && INITIAL_SETUP=1 || INITIAL_SETUP=0