Skip to content

Commit 6ce2fb2

Browse files
mattcieslakmattcieslak
committed
ci: handle cross-compile linux containers (no /opt/python)
1 parent 78eaca5 commit 6ce2fb2

1 file changed

Lines changed: 88 additions & 37 deletions

File tree

‎.github/workflows/release.yml‎

Lines changed: 88 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -90,67 +90,105 @@ jobs:
9090
manylinux: ${{ matrix.libc == 'manylinux' && 'auto' || 'musllinux_1_2' }}
9191
before-script-linux: |
9292
# hdf5-metno-src builds HDF5 from source via cmake; it requires
93-
# CMake >= 3.26. Both manylinux_2_17 (AlmaLinux yum cmake = 3.20)
94-
# and musllinux_1_2 (Alpine apk cmake = 3.22) ship versions that
95-
# are too old. Install a recent cmake from PyPI and pin its
96-
# path via CMAKE=... (set on the step's env above) so the
97-
# cmake-rs crate uses it deterministically.
93+
# CMake >= 3.26. Both manylinux_2_17 (AlmaLinux yum cmake 3.20)
94+
# and musllinux_1_2 (Alpine apk cmake 3.22) ship too-old
95+
# versions, and cross-compile images from messense (used for
96+
# aarch64) don't have /opt/python/ at all. Install a recent
97+
# cmake via pip from whatever Python we can find, and pin its
98+
# path via the CMAKE env var.
9899
#
99100
# `set -ex` prints every command so failures are diagnosable
100101
# from the workflow log without re-running.
101102
set -ex
102103
103104
uname -a
104105
cat /etc/os-release || true
106+
ls -d /opt/python 2>&1 || echo "(no /opt/python — likely a cross-compile image)"
105107
106-
# Detect package manager and install minimal build tools for
107-
# the HDF5 source compile.
108+
# Install minimal build tools + a system Python if needed.
109+
# The cross-compile images have package managers but often no
110+
# Python preinstalled, so add it here.
108111
if command -v apk >/dev/null 2>&1; then
109-
# Alpine / musllinux. apk update so package metadata is fresh.
110112
apk update
111-
apk add --no-cache make gcc g++ musl-dev linux-headers perl bash
113+
apk add --no-cache make gcc g++ musl-dev linux-headers perl bash python3 py3-pip
112114
elif command -v dnf >/dev/null 2>&1; then
113-
# AlmaLinux 9 / manylinux_2_28
114-
dnf install -y make gcc gcc-c++ perl
115+
dnf install -y make gcc gcc-c++ perl python3 python3-pip
115116
elif command -v yum >/dev/null 2>&1; then
116-
# AlmaLinux 8 / manylinux_2_17
117-
yum install -y make gcc gcc-c++ perl
117+
yum install -y make gcc gcc-c++ perl python3 python3-pip
118+
elif command -v apt-get >/dev/null 2>&1; then
119+
apt-get update
120+
apt-get install -y make gcc g++ perl python3 python3-pip
118121
fi
119122
120-
# Find a CPython 3.x in the standard PyPA layout. Prefer 3.10
121-
# since that's the abi3 minimum we promise; any cp3* works.
123+
# Resolve a pip we can use. Prefer the PyPA-style /opt/python
124+
# bundled CPython (manylinux_2_17 / musllinux_1_2 native images);
125+
# fall back to system python3 (typical of cross-compile images).
126+
PIP_CMD=""
122127
PYBIN=""
123-
for d in /opt/python/cp310-cp310/bin /opt/python/cp311-cp311/bin \
124-
/opt/python/cp312-cp312/bin /opt/python/cp39-cp39/bin \
125-
/opt/python/cp313-cp313/bin; do
126-
if [ -d "$d" ]; then PYBIN="$d"; break; fi
128+
for cand in /opt/python/cp310-cp310/bin /opt/python/cp311-cp311/bin \
129+
/opt/python/cp312-cp312/bin /opt/python/cp39-cp39/bin \
130+
/opt/python/cp313-cp313/bin; do
131+
if [ -d "$cand" ]; then
132+
PYBIN="$cand"
133+
PIP_CMD="$cand/pip"
134+
break
135+
fi
127136
done
128-
if [ -z "$PYBIN" ]; then
129-
echo "::error::No /opt/python/cp3*/bin found"
130-
ls -la /opt/python/ 2>&1 || true
131-
exit 1
137+
if [ -z "$PIP_CMD" ]; then
138+
if command -v pip3 >/dev/null 2>&1; then
139+
PIP_CMD="pip3"
140+
elif command -v python3 >/dev/null 2>&1; then
141+
python3 -m ensurepip --default-pip 2>/dev/null || true
142+
PIP_CMD="python3 -m pip"
143+
else
144+
echo "::error::No Python/pip found in build container"
145+
exit 1
146+
fi
132147
fi
133-
echo "Selected PYBIN=$PYBIN"
134-
"$PYBIN/python" --version
148+
echo "Using PIP_CMD='$PIP_CMD' (PYBIN='${PYBIN:-system}')"
135149
136-
# Install a recent cmake from PyPI. Use --no-cache-dir so we
137-
# don't blow up the build cache.
138-
"$PYBIN/pip" install --no-cache-dir --upgrade "cmake>=3.26"
150+
# Install cmake>=3.26 from PyPI. `--break-system-packages` is
151+
# for PEP 668-marked Pythons (some distros); harmless on others.
152+
$PIP_CMD install --no-cache-dir --upgrade --break-system-packages "cmake>=3.26" 2>/dev/null \
153+
|| $PIP_CMD install --no-cache-dir --upgrade "cmake>=3.26"
139154
140-
# Pin the cmake path to match the CMAKE env var set above.
155+
# Find where the pip-installed cmake landed. With /opt/python
156+
# it's in $PYBIN/cmake; with system pip it's wherever pip puts
157+
# console scripts — discoverable via `command -v cmake` after
158+
# install (PATH is updated by the install).
159+
INSTALLED_CMAKE=""
160+
if [ -n "$PYBIN" ] && [ -x "$PYBIN/cmake" ]; then
161+
INSTALLED_CMAKE="$PYBIN/cmake"
162+
elif command -v cmake >/dev/null 2>&1; then
163+
INSTALLED_CMAKE=$(command -v cmake)
164+
else
165+
# Last-ditch: ask pip where it put the script.
166+
INSTALLED_CMAKE=$($PIP_CMD show -f cmake 2>/dev/null | awk '/Location:/ {loc=$2} /bin\/cmake/ {print loc"/"$1}' | head -1)
167+
# That gives a path relative to site-packages; resolve to abs.
168+
if [ -n "$INSTALLED_CMAKE" ]; then
169+
INSTALLED_CMAKE=$(realpath "$INSTALLED_CMAKE" 2>/dev/null || echo "$INSTALLED_CMAKE")
170+
fi
171+
fi
172+
if [ -z "$INSTALLED_CMAKE" ] || [ ! -x "$INSTALLED_CMAKE" ]; then
173+
echo "::error::Could not locate pip-installed cmake binary"
174+
$PIP_CMD show -f cmake || true
175+
exit 1
176+
fi
177+
echo "Pip-installed cmake at: $INSTALLED_CMAKE"
178+
179+
# Pin the path to match the CMAKE env var set on the step.
141180
mkdir -p /opt/_pipcmake/bin
142-
ln -sf "$PYBIN/cmake" /opt/_pipcmake/bin/cmake
181+
ln -sf "$INSTALLED_CMAKE" /opt/_pipcmake/bin/cmake
143182
144-
# Shadow common system locations as a fallback for any tool
145-
# that ignores CMAKE and just resolves `cmake` via PATH.
183+
# Shadow common system locations as a PATH fallback.
146184
for target in /usr/local/bin/cmake /usr/bin/cmake; do
147185
parent=$(dirname "$target")
148-
[ -d "$parent" ] && ln -sf "$PYBIN/cmake" "$target" || true
186+
[ -d "$parent" ] && ln -sf "$INSTALLED_CMAKE" "$target" || true
149187
done
150188
151-
# Diagnostic: show what's where so future failures are obvious.
152-
command -v cmake || true
153-
"$PYBIN/cmake" --version
189+
# Diagnostic: confirm the right cmake is found via every route.
190+
echo "--- cmake versions:"
191+
"$INSTALLED_CMAKE" --version
154192
/opt/_pipcmake/bin/cmake --version
155193
cmake --version
156194
- uses: actions/upload-artifact@v4
@@ -159,9 +197,22 @@ jobs:
159197
path: python/dist
160198

161199
# ─── macOS wheels (x86_64 + Apple Silicon) ─────────────────────────────────
200+
#
201+
# Runner mapping:
202+
# - aarch64 → macos-14 (Apple Silicon, free, ample capacity)
203+
# - x86_64 → macos-13-large (Intel, paid 6-core, much better queue
204+
# priority than the free macos-13).
205+
#
206+
# macos-13 is the newest Intel macOS image GitHub offers (Apple stopped
207+
# making Intel Macs), and the free macos-13 runner pool is usually
208+
# under-provisioned. macos-13-large costs billing minutes but reliably
209+
# picks up jobs in seconds rather than tens of minutes. If you'd rather
210+
# not consume the budget, the cleanest alternative is to drop the
211+
# x86_64 wheel altogether — the sdist + arm64 wheel cover ~all current
212+
# macOS Python users.
162213
macos:
163214
name: macOS ${{ matrix.target }}
164-
runs-on: ${{ matrix.target == 'aarch64' && 'macos-14' || 'macos-13' }}
215+
runs-on: ${{ matrix.target == 'aarch64' && 'macos-14' || 'macos-13-large' }}
165216
needs: [verify-version]
166217
strategy:
167218
fail-fast: false

0 commit comments

Comments
 (0)