From 0363dbc85a2f9cb42756d59d5e5c278b7f772d40 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 9 Apr 2026 18:26:08 +0200
Subject: [PATCH 01/22] feat: enable remote store support by adding boto3

Add boto3 and its dependencies (botocore, s3transfer, jmespath) to the
alibuild and bitsorg recipes. Add python-dateutil as an explicit
dependency for botocore's runtime needs.

This enables the --remote-store flag for S3-backed package caching.
---
 alibuild.sh | 5 +++--
 bitsorg.sh  | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/alibuild.sh b/alibuild.sh
index 3b7a31fe..b9ee9a0a 100644
--- a/alibuild.sh
+++ b/alibuild.sh
@@ -9,7 +9,7 @@ requires:
   - python-requests
   - python-distro
   - python-jinja2
-  # boto3 is not included; remote store support is unavailable.
+  - python-dateutil
 build_requires:
   - uv
   - alibuild-recipe-tools
@@ -26,7 +26,8 @@ pyver=$(python3 -c 'import sysconfig; print(sysconfig.get_python_version())')
 TARGET="$INSTALLROOT/lib/python$pyver/site-packages"
 mkdir -p "$TARGET"
 
-uv pip install --no-deps --no-cache-dir --target="$TARGET" --python="$(command -v python3)" "alibuild==$PKGVERSION"
+uv pip install --no-deps --no-cache-dir --target="$TARGET" --python="$(command -v python3)" \
+  "alibuild==$PKGVERSION" boto3 botocore s3transfer jmespath
 
 ln -snf "python$pyver" "$INSTALLROOT/lib/python"
 
diff --git a/bitsorg.sh b/bitsorg.sh
index 50de25d5..ed16c6dd 100644
--- a/bitsorg.sh
+++ b/bitsorg.sh
@@ -9,7 +9,7 @@ requires:
   - python-requests
   - python-distro
   - python-jinja2
-  # boto3 is not included; remote store support is unavailable.
+  - python-dateutil
 build_requires:
   - uv
   - alibuild-recipe-tools
@@ -24,7 +24,8 @@ pyver=$(python3 -c 'import sysconfig; print(sysconfig.get_python_version())')
 TARGET="$INSTALLROOT/lib/python$pyver/site-packages"
 mkdir -p "$TARGET"
 
-uv pip install --no-deps --no-cache-dir --target="$TARGET" --python="$(command -v python3)" "bitsorg==$PKGVERSION"
+uv pip install --no-deps --no-cache-dir --target="$TARGET" --python="$(command -v python3)" \
+  "bitsorg==$PKGVERSION" boto3 botocore s3transfer jmespath
 
 ln -snf "python$pyver" "$INSTALLROOT/lib/python"
 

From 9b46a61f8425849aec33b81d1dd366b416678da1 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 9 Apr 2026 18:26:53 +0200
Subject: [PATCH 02/22] feat: add build workflow for CI package builds

GitHub Actions workflow that builds FairShip using aliBuild with S3
remote store caching. Triggers on pushes to main, PRs, weekly schedule,
and manual dispatch. PRs use read-only store; main/tags use read-write.
---
 .github/workflows/build.yml | 92 +++++++++++++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 .github/workflows/build.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 00000000..c9ecdd64
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,92 @@
+---
+name: Build packages
+
+'on':
+  push:
+    branches: [main]
+    paths:
+      - '*.sh'
+  pull_request:
+    paths:
+      - '*.sh'
+  schedule:
+    # Weekly rebuild on Monday at 04:00 UTC
+    - cron: '0 4 * * 1'
+  workflow_dispatch:
+    inputs:
+      package:
+        description: Package to build
+        default: FairShip
+      defaults:
+        description: Defaults to use
+        default: release
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REMOTE_STORE: https://s3.cern.ch/swift/v1/ship-packages
+  WORK_DIR: /opt/ship
+  DEFAULTS: release
+
+jobs:
+  build:
+    name: Build ${{ inputs.package || 'FairShip' }}
+    runs-on: [self-hosted, alma9, x86_64]
+    container:
+      image: registry.cern.ch/ship/gha-runner:latest
+      volumes:
+        - /cvmfs:/cvmfs:ro
+        - shipbuild:/opt/ship
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build
+        run: |
+          PACKAGE="${{ inputs.package || 'FairShip' }}"
+          DEFAULTS="${{ inputs.defaults || env.DEFAULTS }}"
+          STORE_MODE=""
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            STORE_MODE=""
+          else
+            STORE_MODE="::rw"
+          fi
+          aliBuild build "$PACKAGE" \
+            --defaults "$DEFAULTS" \
+            --remote-store "${REMOTE_STORE}${STORE_MODE}" \
+            --work-dir "$WORK_DIR" \
+            -j "$(nproc)"
+
+  build-container:
+    name: Build container image
+    needs: build
+    if: github.event_name == 'push' || startsWith(github.ref, 'refs/tags/')
+    runs-on: [self-hosted, alma9, x86_64]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Determine image tag
+        id: tag
+        run: |
+          if [[ "$GITHUB_REF" == refs/tags/* ]]; then
+            echo "tag=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT"
+            echo "latest=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "tag=main-${GITHUB_SHA::8}" >> "$GITHUB_OUTPUT"
+            echo "latest=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build and push
+        run: |
+          IMAGE=registry.cern.ch/ship/ship-sim
+          TAG="${{ steps.tag.outputs.tag }}"
+          podman build -t "$IMAGE:$TAG" .
+          podman push "$IMAGE:$TAG"
+          if [ "${{ steps.tag.outputs.latest }}" = "true" ]; then
+            podman tag "$IMAGE:$TAG" "$IMAGE:latest"
+            podman push "$IMAGE:latest"
+          fi

From 2a404084c55ed98eff5fb5e1c8a094a6f0880e44 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 9 Apr 2026 18:27:34 +0200
Subject: [PATCH 03/22] feat: add CVMFS-based container image

Minimal AlmaLinux 9 container with CVMFS client and environment modules.
Expects /cvmfs to be bind-mounted at runtime. Entrypoint loads the
FairShip environment from CVMFS automatically.

Container files are in container/ to avoid alidistlint treating scripts
as recipes.
---
 .github/workflows/build.yml |  2 +-
 .pre-commit-config.yaml     |  1 +
 container/Dockerfile        | 14 ++++++++++++++
 container/cvmfs-ship.conf   |  2 ++
 container/entrypoint.sh     | 12 ++++++++++++
 5 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 container/Dockerfile
 create mode 100644 container/cvmfs-ship.conf
 create mode 100644 container/entrypoint.sh

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c9ecdd64..f9f22710 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -84,7 +84,7 @@ jobs:
         run: |
           IMAGE=registry.cern.ch/ship/ship-sim
           TAG="${{ steps.tag.outputs.tag }}"
-          podman build -t "$IMAGE:$TAG" .
+          podman build -t "$IMAGE:$TAG" container/
           podman push "$IMAGE:$TAG"
           if [ "${{ steps.tag.outputs.latest }}" = "true" ]; then
             podman tag "$IMAGE:$TAG" "$IMAGE:latest"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 02815e51..7dcb779a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,6 +18,7 @@ repos:
   hooks:
   - id: alidistlint
     verbose: true
+    exclude: ^container/
 ci:
   autofix_prs: true
   autoupdate_commit_msg: "chore(deps): update pre-commit hooks"
diff --git a/container/Dockerfile b/container/Dockerfile
new file mode 100644
index 00000000..864dfb34
--- /dev/null
+++ b/container/Dockerfile
@@ -0,0 +1,14 @@
+FROM almalinux:9-minimal
+
+RUN microdnf install -y \
+      https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm && \
+    microdnf install -y cvmfs fuse environment-modules && \
+    microdnf clean all
+
+COPY cvmfs-ship.conf /etc/cvmfs/default.d/ship.conf
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/bin/bash"]
diff --git a/container/cvmfs-ship.conf b/container/cvmfs-ship.conf
new file mode 100644
index 00000000..aee722e6
--- /dev/null
+++ b/container/cvmfs-ship.conf
@@ -0,0 +1,2 @@
+CVMFS_REPOSITORIES=ship.cern.ch
+CVMFS_HTTP_PROXY=DIRECT
diff --git a/container/entrypoint.sh b/container/entrypoint.sh
new file mode 100644
index 00000000..cde967a7
--- /dev/null
+++ b/container/entrypoint.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+# Source environment modules
+source /etc/profile.d/modules.sh 2>/dev/null || true
+
+# Load FairShip environment from CVMFS if available
+if [ -d /cvmfs/ship.cern.ch ]; then
+  eval "$(/cvmfs/ship.cern.ch/bin/alienv printenv FairShip/latest 2>/dev/null)" || true
+fi
+
+exec "$@"

From f76629479e9dcb2240fca72134bae78efb387c97 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 9 Apr 2026 18:34:42 +0200
Subject: [PATCH 04/22] docs: document CI setup and remaining steps

---
 README.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/README.md b/README.md
index b3e1e78f..500485ec 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,27 @@ Defaults in use:
 
 #### `aliBuild init` and local development packages
 
+## CI
+
+### What's in place
+
+- **Recipe linting** (`.github/workflows/recipe-checks.yml`): runs on every push/PR
+- **Build workflow** (`.github/workflows/build.yml`): builds FairShip on self-hosted runners with S3 remote store caching
+- **Container image** (`container/`): minimal AlmaLinux 9 image with CVMFS client, loads FairShip from `/cvmfs/ship.cern.ch`
+
+### Setup required before first use
+
+1. **S3 credentials**: add `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` as GitHub Actions secrets (for `s3.cern.ch/swift/v1/ship-packages`)
+2. **Seed the remote store**: run a full build with `--remote-store https://s3.cern.ch/swift/v1/ship-packages::rw` to populate the cache
+3. **Container registry**: authenticate the runner to push to `registry.cern.ch/ship/ship-sim`
+
+### Remaining work
+
+4. **CVMFS publishing**: set up `ship-cvmfs-builder-slc9` as a CVMFS Stratum-0 publisher for `ship.cern.ch`, then add a publish job to the workflow (following the LCG bits pattern: fetch tarballs from S3, unpack to CVMFS, `cvmfs_server transaction`/`publish`)
+5. **unpacked.cern.ch**: submit `registry.cern.ch/ship/ship-sim` to the DUCC wishlist for CVMFS distribution of the container image
+6. **Branch protection**: require the `build` check to pass before merging
+7. **Failure notifications**: add alerting for scheduled build failures
+
 ## Platform specific information
 
 Information for different platforms available below.

From a8494f24a2fae6ba74a9acaa3c1f877866823e81 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 15:04:24 +0200
Subject: [PATCH 05/22] fix: use s3:// URL scheme for remote store

The HTTPS URL format doesn't use the S3 API for uploads. Use the s3://
scheme with S3_ENDPOINT_URL instead.
---
 .github/workflows/build.yml | 3 ++-
 README.md                   | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f9f22710..951161d7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -26,7 +26,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  REMOTE_STORE: https://s3.cern.ch/swift/v1/ship-packages
+  REMOTE_STORE: s3://ship-packages
   WORK_DIR: /opt/ship
   DEFAULTS: release
 
@@ -42,6 +42,7 @@ jobs:
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
+      S3_ENDPOINT_URL: https://s3.cern.ch
     steps:
       - uses: actions/checkout@v4
 
diff --git a/README.md b/README.md
index 500485ec..8fa4e6d1 100644
--- a/README.md
+++ b/README.md
@@ -33,8 +33,8 @@ Defaults in use:
 
 ### Setup required before first use
 
-1. **S3 credentials**: add `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` as GitHub Actions secrets (for `s3.cern.ch/swift/v1/ship-packages`)
-2. **Seed the remote store**: run a full build with `--remote-store https://s3.cern.ch/swift/v1/ship-packages::rw` to populate the cache
+1. **S3 credentials**: add `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY` as GitHub Actions secrets (EC2 credentials for `s3.cern.ch`)
+2. **Seed the remote store**: run a full build with `S3_ENDPOINT_URL=https://s3.cern.ch aliBuild build FairShip --remote-store s3://ship-packages::rw` to populate the cache
 3. **Container registry**: authenticate the runner to push to `registry.cern.ch/ship/ship-sim`
 
 ### Remaining work

From 6ee05d23b3028bc1c704c402e3d1af26e28bd37d Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 17:39:07 +0200
Subject: [PATCH 06/22] fix: generate .s3cfg from secrets for s3cmd

aliBuild uses s3cmd (not boto3) to download from S3 stores. s3cmd
needs a .s3cfg file with host config and credentials. Generate it from
the GitHub Actions secrets at build time.
---
 .github/workflows/build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 951161d7..b49e7ab6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -46,6 +46,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Configure S3 credentials
+        run: |
+          printf 'host_base = s3.cern.ch\nhost_bucket = %%(bucket)s.s3.cern.ch\naccess_key = %s\nsecret_key = %s\n' \
+            "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" > ~/.s3cfg
+
       - name: Build
         run: |
           PACKAGE="${{ inputs.package || 'FairShip' }}"

From 3bc0f477cb5af171cdf0f0286e085e0dd7770c96 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 18:06:16 +0200
Subject: [PATCH 07/22] fix: address review feedback

- Run container as non-root user (shipuser, uid 1000)
- Mask S3 secrets in workflow logs before writing .s3cfg
- Simplify STORE_MODE conditional assignment
- Surface alienv failures in entrypoint instead of silencing them
---
 .github/workflows/build.yml | 6 +++---
 container/Dockerfile        | 3 +++
 container/entrypoint.sh     | 6 +++++-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b49e7ab6..cf27cac5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -48,6 +48,8 @@ jobs:
 
       - name: Configure S3 credentials
         run: |
+          echo "::add-mask::${AWS_ACCESS_KEY_ID}"
+          echo "::add-mask::${AWS_SECRET_ACCESS_KEY}"
           printf 'host_base = s3.cern.ch\nhost_bucket = %%(bucket)s.s3.cern.ch\naccess_key = %s\nsecret_key = %s\n' \
             "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" > ~/.s3cfg
 
@@ -55,11 +57,9 @@ jobs:
         run: |
           PACKAGE="${{ inputs.package || 'FairShip' }}"
           DEFAULTS="${{ inputs.defaults || env.DEFAULTS }}"
-          STORE_MODE=""
+          STORE_MODE="::rw"
           if [ "${{ github.event_name }}" = "pull_request" ]; then
             STORE_MODE=""
-          else
-            STORE_MODE="::rw"
           fi
           aliBuild build "$PACKAGE" \
             --defaults "$DEFAULTS" \
diff --git a/container/Dockerfile b/container/Dockerfile
index 864dfb34..01d0dc93 100644
--- a/container/Dockerfile
+++ b/container/Dockerfile
@@ -10,5 +10,8 @@ COPY cvmfs-ship.conf /etc/cvmfs/default.d/ship.conf
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
+RUN useradd -m -u 1000 shipuser
+USER shipuser
+
 ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/bin/bash"]
diff --git a/container/entrypoint.sh b/container/entrypoint.sh
index cde967a7..f3af8389 100644
--- a/container/entrypoint.sh
+++ b/container/entrypoint.sh
@@ -6,7 +6,11 @@ source /etc/profile.d/modules.sh 2>/dev/null || true
 
 # Load FairShip environment from CVMFS if available
 if [ -d /cvmfs/ship.cern.ch ]; then
-  eval "$(/cvmfs/ship.cern.ch/bin/alienv printenv FairShip/latest 2>/dev/null)" || true
+  alienv_output=$(/cvmfs/ship.cern.ch/bin/alienv printenv FairShip/latest 2>&1)
+  if ! eval "$alienv_output" 2>/dev/null; then
+    echo "Warning: failed to load FairShip environment from CVMFS" >&2
+    echo "$alienv_output" >&2
+  fi
 fi
 
 exec "$@"

From af4693e979f1ee07d0148b9508ad7d0efea148b5 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 18:12:29 +0200
Subject: [PATCH 08/22] feat: install aliBuild via uvx in build workflow

The runner image doesn't include aliBuild, so install uv and use uvx to
run it. Also add -c . to point aliBuild at the local recipes.
---
 .github/workflows/build.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cf27cac5..72a5b03a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -46,6 +46,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install uv and aliBuild
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          "$HOME/.local/bin/uvx" --from alibuild aliBuild version
+
       - name: Configure S3 credentials
         run: |
           echo "::add-mask::${AWS_ACCESS_KEY_ID}"
@@ -61,10 +67,11 @@ jobs:
           if [ "${{ github.event_name }}" = "pull_request" ]; then
             STORE_MODE=""
           fi
-          aliBuild build "$PACKAGE" \
+          uvx --from alibuild aliBuild build "$PACKAGE" \
             --defaults "$DEFAULTS" \
             --remote-store "${REMOTE_STORE}${STORE_MODE}" \
             --work-dir "$WORK_DIR" \
+            -c . \
             -j "$(nproc)"
 
   build-container:

From 8389597ac4aa1c9a546515e39cb47d51bb5ab5c7 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 18:34:51 +0200
Subject: [PATCH 09/22] fix: use debug output for CI build

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 72a5b03a..b9be6cd0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -71,6 +71,7 @@ jobs:
             --defaults "$DEFAULTS" \
             --remote-store "${REMOTE_STORE}${STORE_MODE}" \
             --work-dir "$WORK_DIR" \
+            --debug \
             -c . \
             -j "$(nproc)"
 

From 65355c326477bf984c02c459e8051ca1371c0d6a Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 18:56:45 +0200
Subject: [PATCH 10/22] Revert "fix: use debug output for CI build"

This reverts commit 8389597ac4aa1c9a546515e39cb47d51bb5ab5c7.
---
 .github/workflows/build.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b9be6cd0..72a5b03a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -71,7 +71,6 @@ jobs:
             --defaults "$DEFAULTS" \
             --remote-store "${REMOTE_STORE}${STORE_MODE}" \
             --work-dir "$WORK_DIR" \
-            --debug \
             -c . \
             -j "$(nproc)"
 

From b89e6bbb8137cebb52d90771e9b00c7c8b871b6e Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 19:10:40 +0200
Subject: [PATCH 11/22] feat: print build log on failure

Find the most recently modified log in the BUILD directory and print it
in a collapsible group when the build step fails.
---
 .github/workflows/build.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 72a5b03a..b6c295af 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -74,6 +74,17 @@ jobs:
             -c . \
             -j "$(nproc)"
 
+      - name: Print build log on failure
+        if: failure()
+        run: |
+          log=$(find "$WORK_DIR/BUILD" -name log -printf '%T@ %p\n' 2>/dev/null |
+                sort -rn | head -1 | cut -d' ' -f2-)
+          if [ -n "$log" ]; then
+            echo "::group::Build log: $log"
+            cat "$log"
+            echo "::endgroup::"
+          fi
+
   build-container:
     name: Build container image
     needs: build

From 8a295e17c4e4494157608fc58ee28c5256017196 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 19:26:11 +0200
Subject: [PATCH 12/22] fix: use ls -t instead of find -printf for build log

The find -printf approach produced no output on the runner. Use ls -t
on the BUILD/*/log glob to find the most recently modified log instead.
---
 .github/workflows/build.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b6c295af..2cd3e64b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -77,9 +77,8 @@ jobs:
       - name: Print build log on failure
         if: failure()
         run: |
-          log=$(find "$WORK_DIR/BUILD" -name log -printf '%T@ %p\n' 2>/dev/null |
-                sort -rn | head -1 | cut -d' ' -f2-)
-          if [ -n "$log" ]; then
+          log=$(ls -t "$WORK_DIR"/BUILD/*/log 2>/dev/null | head -1)
+          if [ -f "$log" ]; then
             echo "::group::Build log: $log"
             cat "$log"
             echo "::endgroup::"

From 145c78f9b123a3718bcefd001cee569f6962c1a2 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 19:33:49 +0200
Subject: [PATCH 13/22] feat: run aliBuild with --debug, print tail on failure

Redirect --debug output to a file to avoid filling runner storage.
On failure, print the last 3000 lines of aliBuild debug output and the
failed package's build log in collapsible groups.
---
 .github/workflows/build.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2cd3e64b..d11984a8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -72,14 +72,20 @@ jobs:
             --remote-store "${REMOTE_STORE}${STORE_MODE}" \
             --work-dir "$WORK_DIR" \
             -c . \
-            -j "$(nproc)"
+            -j "$(nproc)" \
+            --debug > "$WORK_DIR/aliBuild.log" 2>&1
 
-      - name: Print build log on failure
+      - name: Print debug log on failure
         if: failure()
         run: |
+          if [ -f "$WORK_DIR/aliBuild.log" ]; then
+            echo "::group::aliBuild debug output (last 3000 lines)"
+            tail -3000 "$WORK_DIR/aliBuild.log"
+            echo "::endgroup::"
+          fi
           log=$(ls -t "$WORK_DIR"/BUILD/*/log 2>/dev/null | head -1)
           if [ -f "$log" ]; then
-            echo "::group::Build log: $log"
+            echo "::group::Package build log: $log"
             cat "$log"
             echo "::endgroup::"
           fi

From da61588d670362c558af00cafbaa73bbe7675590 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 20:41:39 +0200
Subject: [PATCH 14/22] fix(acts): use relative symlink for lib -> lib64

The absolute symlink caused 'Too many levels of symbolic links' during
the relocate-me.sh step when unpacking from the remote store, because
the absolute path embedded in the symlink no longer matched after
relocation.
---
 acts.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acts.sh b/acts.sh
index 2054f74d..05df94db 100644
--- a/acts.sh
+++ b/acts.sh
@@ -41,7 +41,7 @@ cmake "$SOURCEDIR" -DCMAKE_INSTALL_PREFIX="$INSTALLROOT" \
 
 cmake --build . -- ${JOBS:+-j$JOBS} install
 
-[[ -d "$INSTALLROOT/lib64" ]] && [[ ! -d "$INSTALLROOT/lib" ]] && ln -sf "${INSTALLROOT}/lib64" "$INSTALLROOT/lib"
+[[ -d "$INSTALLROOT/lib64" ]] && [[ ! -d "$INSTALLROOT/lib" ]] && ln -sf lib64 "$INSTALLROOT/lib"
 
 # Modulefile
 MODULEDIR="$INSTALLROOT/etc/modulefiles"

From cc66bef2ea174f726aa9b31f2d433deedd3d2d03 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Tue, 14 Apr 2026 21:03:58 +0200
Subject: [PATCH 15/22] fix: drop --debug flag to avoid filling runner disk

aliBuild --debug enables bash set -x tracing which produces gigabytes
of output even when redirected to a file. Rely on the per-package build
log instead, which contains the actual compilation output.
---
 .github/workflows/build.yml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d11984a8..2cd3e64b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -72,20 +72,14 @@ jobs:
             --remote-store "${REMOTE_STORE}${STORE_MODE}" \
             --work-dir "$WORK_DIR" \
             -c . \
-            -j "$(nproc)" \
-            --debug > "$WORK_DIR/aliBuild.log" 2>&1
+            -j "$(nproc)"
 
-      - name: Print debug log on failure
+      - name: Print build log on failure
         if: failure()
         run: |
-          if [ -f "$WORK_DIR/aliBuild.log" ]; then
-            echo "::group::aliBuild debug output (last 3000 lines)"
-            tail -3000 "$WORK_DIR/aliBuild.log"
-            echo "::endgroup::"
-          fi
           log=$(ls -t "$WORK_DIR"/BUILD/*/log 2>/dev/null | head -1)
           if [ -f "$log" ]; then
-            echo "::group::Package build log: $log"
+            echo "::group::Build log: $log"
             cat "$log"
             echo "::endgroup::"
           fi

From 24c872a1bee1fdd6b8a3daa8dbac886fb41be501 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Wed, 15 Apr 2026 10:24:51 +0200
Subject: [PATCH 16/22] fix: debug build log discovery on failure

List BUILD directory contents to diagnose why logs aren't found.
Use *-latest/log glob to follow aliBuild's symlink convention.
---
 .github/workflows/build.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2cd3e64b..d8a759cf 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -77,12 +77,15 @@ jobs:
       - name: Print build log on failure
         if: failure()
         run: |
-          log=$(ls -t "$WORK_DIR"/BUILD/*/log 2>/dev/null | head -1)
-          if [ -f "$log" ]; then
-            echo "::group::Build log: $log"
-            cat "$log"
-            echo "::endgroup::"
-          fi
+          echo "Searching for build logs in $WORK_DIR/BUILD/"
+          ls -la "$WORK_DIR/BUILD/" 2>&1 || true
+          for log in "$WORK_DIR"/BUILD/*-latest/log; do
+            if [ -f "$log" ]; then
+              echo "::group::Build log: $log"
+              cat "$log"
+              echo "::endgroup::"
+            fi
+          done
 
   build-container:
     name: Build container image

From 916e83f05e474877634a29458749d2d8b2c34f2b Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 16 Apr 2026 11:07:11 +0200
Subject: [PATCH 17/22] feat: read from CVMFS, write to S3

Read cached tarballs from /cvmfs/ship-nightlies.cern.ch/main/sw (local,
fast, no download needed). New packages built by CI are uploaded to S3
via --write-store. PRs don't write to either store.
---
 .github/workflows/build.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d8a759cf..0377daa1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -26,7 +26,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  REMOTE_STORE: s3://ship-packages
+  REMOTE_STORE: /cvmfs/ship-nightlies.cern.ch/main/sw
+  WRITE_STORE: s3://ship-packages
   WORK_DIR: /opt/ship
   DEFAULTS: release
 
@@ -63,13 +64,14 @@ jobs:
         run: |
           PACKAGE="${{ inputs.package || 'FairShip' }}"
           DEFAULTS="${{ inputs.defaults || env.DEFAULTS }}"
-          STORE_MODE="::rw"
-          if [ "${{ github.event_name }}" = "pull_request" ]; then
-            STORE_MODE=""
+          WRITE_STORE_FLAG=""
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            WRITE_STORE_FLAG="--write-store ${WRITE_STORE}"
           fi
           uvx --from alibuild aliBuild build "$PACKAGE" \
             --defaults "$DEFAULTS" \
-            --remote-store "${REMOTE_STORE}${STORE_MODE}" \
+            --remote-store "${REMOTE_STORE}" \
+            $WRITE_STORE_FLAG \
             --work-dir "$WORK_DIR" \
             -c . \
             -j "$(nproc)"

From 62872803428486b48d39d054f9c48694b2b0589c Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 16 Apr 2026 16:18:19 +0200
Subject: [PATCH 18/22] feat: read cached packages from CVMFS via cvmfs://
 scheme

Use the cvmfs:// remote store scheme which reads the aliPublish layout
directly from /cvmfs/ship-nightlies.cern.ch/main. This avoids S3
downloads and local disk usage for cached packages.

The cvmfs:// backend does not support --write-store, so new packages
are only built locally. They must be uploaded to S3 and published to
CVMFS separately.
---
 .github/workflows/build.yml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0377daa1..a2bc880b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -26,8 +26,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  REMOTE_STORE: /cvmfs/ship-nightlies.cern.ch/main/sw
-  WRITE_STORE: s3://ship-packages
+  # cvmfs:// scheme uses CVMFSRemoteSync which reads the aliPublish layout
+  # (el9-x86_64/Packages/<pkg>/<ver>/ with .meta.json) from CVMFS directly.
+  # Note: cvmfs:// does not support --write-store.
+  REMOTE_STORE: cvmfs:///cvmfs/ship-nightlies.cern.ch/main
   WORK_DIR: /opt/ship
   DEFAULTS: release
 
@@ -64,14 +66,9 @@ jobs:
         run: |
           PACKAGE="${{ inputs.package || 'FairShip' }}"
           DEFAULTS="${{ inputs.defaults || env.DEFAULTS }}"
-          WRITE_STORE_FLAG=""
-          if [ "${{ github.event_name }}" != "pull_request" ]; then
-            WRITE_STORE_FLAG="--write-store ${WRITE_STORE}"
-          fi
           uvx --from alibuild aliBuild build "$PACKAGE" \
             --defaults "$DEFAULTS" \
             --remote-store "${REMOTE_STORE}" \
-            $WRITE_STORE_FLAG \
             --work-dir "$WORK_DIR" \
             -c . \
             -j "$(nproc)"

From 14a7c54dbf61c0763100f0a36b4c6758fba5b347 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 16 Apr 2026 16:21:11 +0200
Subject: [PATCH 19/22] fix: adjust mounts

---
 .github/workflows/build.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a2bc880b..97291fdd 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -40,8 +40,7 @@ jobs:
     container:
       image: registry.cern.ch/ship/gha-runner:latest
       volumes:
-        - /cvmfs:/cvmfs:ro
-        - shipbuild:/opt/ship
+        - /cvmfs:/cvmfs
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}

From b8d39b2be8cc1625eeea0600ce7a06158516cd05 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 16 Apr 2026 16:28:13 +0200
Subject: [PATCH 20/22] fix: patch aliBuild CVMFSRemoteSync {} bug

aliBuild's CVMFSRemoteSync.fetch_symlinks has an unescaped {} in a
find -exec command which crashes Python's str.format(). Patch it after
installation until the upstream fix lands.
---
 .github/workflows/build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 97291fdd..513484e5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -53,6 +53,10 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
           "$HOME/.local/bin/uvx" --from alibuild aliBuild version
+          # Workaround for https://github.com/alisw/alibuild/issues/XXX:
+          # CVMFSRemoteSync.fetch_symlinks has unescaped {} in find -exec
+          SYNC=$(find "$HOME/.cache/uv" -path "*/alibuild_helpers/sync.py" | head -1)
+          sed -i 's/-exec ln -sf {} /-exec ln -sf {{}} /' "$SYNC"
 
       - name: Configure S3 credentials
         run: |

From 0d6ae68c2be4ed7532dd5ab29726db3c2445fc46 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 16 Apr 2026 16:39:48 +0200
Subject: [PATCH 21/22] fix: use aliBuild branch with CVMFSRemoteSync fix

Use the format_find_argument branch from alisw/alibuild#1023 which
fixes the unescaped {} in find -exec. Replaces the sed workaround.
---
 .github/workflows/build.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 513484e5..062c1da8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -52,11 +52,8 @@ jobs:
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
-          "$HOME/.local/bin/uvx" --from alibuild aliBuild version
-          # Workaround for https://github.com/alisw/alibuild/issues/XXX:
-          # CVMFSRemoteSync.fetch_symlinks has unescaped {} in find -exec
-          SYNC=$(find "$HOME/.cache/uv" -path "*/alibuild_helpers/sync.py" | head -1)
-          sed -i 's/-exec ln -sf {} /-exec ln -sf {{}} /' "$SYNC"
+          # Use branch with fix for alisw/alibuild#1023
+          "$HOME/.local/bin/uvx" --from "alibuild @ git+https://github.com/alisw/alibuild@format_find_argument" aliBuild version
 
       - name: Configure S3 credentials
         run: |

From 6afd0715165f4e56158f01842bcc03c70f074075 Mon Sep 17 00:00:00 2001
From: Oliver Lantwin <oliver.lantwin@cern.ch>
Date: Thu, 16 Apr 2026 16:47:47 +0200
Subject: [PATCH 22/22] fixup! fix: use aliBuild branch with CVMFSRemoteSync
 fix

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 062c1da8..f81f3b6b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -53,7 +53,7 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
           # Use branch with fix for alisw/alibuild#1023
-          "$HOME/.local/bin/uvx" --from "alibuild @ git+https://github.com/alisw/alibuild@format_find_argument" aliBuild version
+          "$HOME/.local/bin/uvx" --from "alibuild @ git+https://github.com/m-fol/alibuild@format_find_argument" aliBuild version
 
       - name: Configure S3 credentials
         run: |