Build Python Package #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Python Package | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| python_version: | |
| description: 'Python version to build (e.g., 3.13.11)' | |
| required: true | |
| default: '3.13.11' | |
| type: string | |
| permissions: | |
| contents: write | |
| packages: write | |
| actions: write | |
| env: | |
| PYTHON_VERSION: ${{ inputs.python_version }} | |
| jobs: | |
| # ============================================================================= | |
| # BUILD: Amazon Linux 2023 - RPM | |
| # ============================================================================= | |
| build-amazonlinux: | |
| name: Build RPM (Amazon Linux) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amazonlinux:2023 | |
| outputs: | |
| package_filename: ${{ steps.build.outputs.package_filename }} | |
| package_sha256: ${{ steps.build.outputs.package_sha256 }} | |
| steps: | |
| - name: Install build dependencies | |
| run: | | |
| yum install -y --allowerasing \ | |
| rpm-build rpmdevtools curl tar gzip git \ | |
| gcc gcc-c++ make openssl-devel bzip2-devel libffi-devel \ | |
| zlib-devel readline-devel sqlite-devel ncurses-devel \ | |
| xz-devel tk-devel gdbm-devel libuuid-devel findutils \ | |
| && yum clean all | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up rpmbuild tree | |
| run: rpmdev-setuptree | |
| - name: Download Python source | |
| run: | | |
| curl -L --fail -o ~/rpmbuild/SOURCES/Python-${PYTHON_VERSION}.tgz \ | |
| "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" | |
| - name: Build RPM | |
| id: build | |
| run: | | |
| cp rpm/python.spec ~/rpmbuild/SPECS/ | |
| echo "Building Python ${PYTHON_VERSION} RPM..." | |
| rpmbuild -bb ~/rpmbuild/SPECS/python.spec \ | |
| --define "runtime_version ${PYTHON_VERSION}" \ | |
| --define "_topdir $HOME/rpmbuild" | |
| PKG_FILE=$(find ~/rpmbuild/RPMS -name "python-${PYTHON_VERSION}-*.rpm" -type f | head -1) | |
| PKG_FILENAME=$(basename "$PKG_FILE") | |
| PKG_SHA256=$(sha256sum "$PKG_FILE" | cut -d' ' -f1) | |
| cp "$PKG_FILE" $GITHUB_WORKSPACE/ | |
| echo "package_filename=${PKG_FILENAME}" >> $GITHUB_OUTPUT | |
| echo "package_sha256=${PKG_SHA256}" >> $GITHUB_OUTPUT | |
| echo "Built: ${PKG_FILENAME}" | |
| echo "SHA256: ${PKG_SHA256}" | |
| rpm -qip "$GITHUB_WORKSPACE/$PKG_FILENAME" | |
| - name: Upload build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: python-amazonlinux | |
| path: ${{ steps.build.outputs.package_filename }} | |
| retention-days: 30 | |
| # ============================================================================= | |
| # BUILD: Alpine Linux - Tarball | |
| # ============================================================================= | |
| build-alpine: | |
| name: Build Tarball (Alpine) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: alpine:3.19 | |
| outputs: | |
| package_filename: ${{ steps.build.outputs.package_filename }} | |
| package_sha256: ${{ steps.build.outputs.package_sha256 }} | |
| steps: | |
| - name: Install build dependencies | |
| run: | | |
| apk add --no-cache \ | |
| alpine-sdk sudo curl tar gzip git \ | |
| gcc g++ make musl-dev linux-headers \ | |
| openssl-dev bzip2-dev libffi-dev zlib-dev \ | |
| readline-dev sqlite-dev ncurses-dev xz-dev \ | |
| tk-dev gdbm-dev libuuid util-linux-dev \ | |
| tcl-dev expat-dev | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Download Python source | |
| run: | | |
| curl -L --fail -o /tmp/Python-${PYTHON_VERSION}.tgz \ | |
| "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" | |
| - name: Build Python | |
| id: build | |
| run: | | |
| cd /tmp | |
| tar -xzf Python-${PYTHON_VERSION}.tgz | |
| cd Python-${PYTHON_VERSION} | |
| # Note: --enable-optimizations is disabled for Alpine because PGO tests | |
| # fail due to musl libc locale limitations. LTO is still enabled. | |
| ./configure \ | |
| --prefix=/export/apps/citools/python/python-${PYTHON_VERSION} \ | |
| --with-lto \ | |
| --with-computed-gotos \ | |
| --enable-ipv6 \ | |
| --enable-loadable-sqlite-extensions \ | |
| --with-ensurepip=upgrade | |
| make -j$(nproc) | |
| DESTDIR=/tmp/python-install make install | |
| cd /tmp/python-install | |
| PKG_FILENAME="python-${PYTHON_VERSION}-alpine319-x86_64.tar.gz" | |
| tar -czf /tmp/${PKG_FILENAME} . | |
| PKG_SHA256=$(sha256sum /tmp/${PKG_FILENAME} | cut -d' ' -f1) | |
| cp /tmp/${PKG_FILENAME} $GITHUB_WORKSPACE/ | |
| echo "package_filename=${PKG_FILENAME}" >> $GITHUB_OUTPUT | |
| echo "package_sha256=${PKG_SHA256}" >> $GITHUB_OUTPUT | |
| echo "Built: ${PKG_FILENAME}" | |
| echo "SHA256: ${PKG_SHA256}" | |
| - name: Upload build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: python-alpine | |
| path: ${{ steps.build.outputs.package_filename }} | |
| retention-days: 30 | |
| # ============================================================================= | |
| # TEST: Fresh Amazon Linux container (no build deps) | |
| # ============================================================================= | |
| test-amazonlinux: | |
| name: Test on Fresh Amazon Linux | |
| needs: build-amazonlinux | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amazonlinux:2023 | |
| steps: | |
| - name: Install minimal tools for checkout | |
| run: yum install -y tar gzip && yum clean all | |
| - name: Checkout (for test script only) | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: rpm/test-python.py | |
| sparse-checkout-cone-mode: false | |
| - name: Download package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: python-amazonlinux | |
| - name: Install package (no build deps) | |
| run: | | |
| echo "=== Fresh Amazon Linux 2023 container ===" | |
| echo "Installed packages before:" | |
| rpm -qa | wc -l | |
| echo "" | |
| echo "Installing Python RPM..." | |
| rpm -ivh ${{ needs.build-amazonlinux.outputs.package_filename }} | |
| echo "" | |
| echo "Installation complete." | |
| - name: Run functional tests | |
| run: | | |
| PYTHON_BIN="/export/apps/citools/python/python-${PYTHON_VERSION}/bin/python3" | |
| echo "=== Testing Python on fresh system ===" | |
| echo "Python binary: $PYTHON_BIN" | |
| "$PYTHON_BIN" --version | |
| "$PYTHON_BIN" rpm/test-python.py "${PYTHON_VERSION}" | |
| # ============================================================================= | |
| # TEST: Fresh Alpine container (no build deps) | |
| # ============================================================================= | |
| test-alpine: | |
| name: Test on Fresh Alpine | |
| needs: build-alpine | |
| runs-on: ubuntu-latest | |
| container: | |
| image: alpine:3.19 | |
| steps: | |
| - name: Install minimal tools for checkout | |
| run: apk add --no-cache tar gzip | |
| - name: Checkout (for test script only) | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: rpm/test-python.py | |
| sparse-checkout-cone-mode: false | |
| - name: Download package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: python-alpine | |
| - name: Install package (no build deps) | |
| run: | | |
| echo "=== Fresh Alpine 3.19 container ===" | |
| echo "Installed packages before:" | |
| apk list --installed 2>/dev/null | wc -l | |
| echo "" | |
| echo "Installing Python tarball..." | |
| tar -xzf ${{ needs.build-alpine.outputs.package_filename }} -C / | |
| echo "" | |
| echo "Installation complete." | |
| - name: Run functional tests | |
| run: | | |
| PYTHON_BIN="/export/apps/citools/python/python-${PYTHON_VERSION}/bin/python3" | |
| echo "=== Testing Python on fresh system ===" | |
| echo "Python binary: $PYTHON_BIN" | |
| "$PYTHON_BIN" --version | |
| "$PYTHON_BIN" rpm/test-python.py "${PYTHON_VERSION}" | |
| # ============================================================================= | |
| # PUBLISH: Amazon Linux RPM to GitHub Releases | |
| # ============================================================================= | |
| publish-amazonlinux: | |
| name: Publish Amazon Linux RPM | |
| runs-on: ubuntu-latest | |
| needs: [build-amazonlinux, test-amazonlinux] | |
| outputs: | |
| release_url: ${{ steps.release.outputs.release_url }} | |
| asset_url: ${{ steps.release.outputs.asset_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Download package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: python-amazonlinux | |
| - name: Verify package | |
| run: | | |
| ACTUAL_SHA256=$(sha256sum "${{ needs.build-amazonlinux.outputs.package_filename }}" | cut -d' ' -f1) | |
| if [[ "$ACTUAL_SHA256" != "${{ needs.build-amazonlinux.outputs.package_sha256 }}" ]]; then | |
| echo "ERROR: SHA256 mismatch!" | |
| exit 1 | |
| fi | |
| echo "SHA256 verified: $ACTUAL_SHA256" | |
| - name: Create GitHub Release | |
| id: release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| RELEASE_TAG="python-amazonlinux-${PYTHON_VERSION}" | |
| RELEASE_NAME="Python ${PYTHON_VERSION} (Amazon Linux 2023)" | |
| PKG_FILE="${{ needs.build-amazonlinux.outputs.package_filename }}" | |
| PKG_SHA256="${{ needs.build-amazonlinux.outputs.package_sha256 }}" | |
| RELEASE_BODY=$(cat <<EOF | |
| ## Python ${PYTHON_VERSION} for Amazon Linux 2023 | |
| **Tested on:** Fresh Amazon Linux 2023 container (no build dependencies) | |
| **Install location:** \`/export/apps/citools/python/python-${PYTHON_VERSION}/\` | |
| **SHA256:** \`${PKG_SHA256}\` | |
| **Build date:** $(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| ### Tests Passed | |
| - Version check | |
| - Standard library imports (ssl, sqlite3, asyncio, etc.) | |
| - SSL/TLS functionality | |
| - SQLite operations | |
| - File I/O | |
| - Subprocess execution | |
| - pip available | |
| ### Installation | |
| \`\`\`bash | |
| curl -LO https://github.com/${{ github.repository }}/releases/download/${RELEASE_TAG}/${PKG_FILE} | |
| rpm -ivh ${PKG_FILE} | |
| /export/apps/citools/python/python-${PYTHON_VERSION}/bin/python3 --version | |
| \`\`\` | |
| EOF | |
| ) | |
| gh release delete "$RELEASE_TAG" --yes 2>/dev/null || true | |
| gh release create "$RELEASE_TAG" --title "$RELEASE_NAME" --notes "$RELEASE_BODY" "$PKG_FILE" | |
| RELEASE_URL=$(gh release view "$RELEASE_TAG" --json url -q '.url') | |
| ASSET_URL="https://github.com/${{ github.repository }}/releases/download/${RELEASE_TAG}/${PKG_FILE}" | |
| echo "release_url=${RELEASE_URL}" >> $GITHUB_OUTPUT | |
| echo "asset_url=${ASSET_URL}" >> $GITHUB_OUTPUT | |
| # ============================================================================= | |
| # PUBLISH: Alpine Tarball to GitHub Releases | |
| # ============================================================================= | |
| publish-alpine: | |
| name: Publish Alpine Tarball | |
| runs-on: ubuntu-latest | |
| needs: [build-alpine, test-alpine] | |
| outputs: | |
| release_url: ${{ steps.release.outputs.release_url }} | |
| asset_url: ${{ steps.release.outputs.asset_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Download package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: python-alpine | |
| - name: Verify package | |
| run: | | |
| ACTUAL_SHA256=$(sha256sum "${{ needs.build-alpine.outputs.package_filename }}" | cut -d' ' -f1) | |
| if [[ "$ACTUAL_SHA256" != "${{ needs.build-alpine.outputs.package_sha256 }}" ]]; then | |
| echo "ERROR: SHA256 mismatch!" | |
| exit 1 | |
| fi | |
| echo "SHA256 verified: $ACTUAL_SHA256" | |
| - name: Create GitHub Release | |
| id: release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| RELEASE_TAG="python-alpine-${PYTHON_VERSION}" | |
| RELEASE_NAME="Python ${PYTHON_VERSION} (Alpine Linux 3.19)" | |
| PKG_FILE="${{ needs.build-alpine.outputs.package_filename }}" | |
| PKG_SHA256="${{ needs.build-alpine.outputs.package_sha256 }}" | |
| RELEASE_BODY=$(cat <<EOF | |
| ## Python ${PYTHON_VERSION} for Alpine Linux 3.19 | |
| **Tested on:** Fresh Alpine Linux 3.19 container (no build dependencies) | |
| **Install location:** \`/export/apps/citools/python/python-${PYTHON_VERSION}/\` | |
| **SHA256:** \`${PKG_SHA256}\` | |
| **Build date:** $(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| ### Tests Passed | |
| - Version check | |
| - Standard library imports (ssl, sqlite3, asyncio, etc.) | |
| - SSL/TLS functionality | |
| - SQLite operations | |
| - File I/O | |
| - Subprocess execution | |
| - pip available | |
| ### Installation | |
| \`\`\`bash | |
| curl -LO https://github.com/${{ github.repository }}/releases/download/${RELEASE_TAG}/${PKG_FILE} | |
| tar -xzf ${PKG_FILE} -C / | |
| /export/apps/citools/python/python-${PYTHON_VERSION}/bin/python3 --version | |
| \`\`\` | |
| EOF | |
| ) | |
| gh release delete "$RELEASE_TAG" --yes 2>/dev/null || true | |
| gh release create "$RELEASE_TAG" --title "$RELEASE_NAME" --notes "$RELEASE_BODY" "$PKG_FILE" | |
| RELEASE_URL=$(gh release view "$RELEASE_TAG" --json url -q '.url') | |
| ASSET_URL="https://github.com/${{ github.repository }}/releases/download/${RELEASE_TAG}/${PKG_FILE}" | |
| echo "release_url=${RELEASE_URL}" >> $GITHUB_OUTPUT | |
| echo "asset_url=${ASSET_URL}" >> $GITHUB_OUTPUT | |
| # ============================================================================= | |
| # UPDATE INDEX: Register both releases in database | |
| # ============================================================================= | |
| update-index: | |
| name: Update Downloads Index | |
| runs-on: ubuntu-latest | |
| needs: [build-amazonlinux, build-alpine, publish-amazonlinux, publish-alpine] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up ORAS | |
| run: | | |
| curl -LO "https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz" | |
| tar -xzf oras_1.2.0_linux_amd64.tar.gz | |
| sudo mv oras /usr/local/bin/ | |
| - name: Login to GHCR | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | oras login ghcr.io -u ${{ github.actor }} --password-stdin | |
| - name: Pull downloads.db | |
| run: | | |
| REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | |
| oras pull ghcr.io/${REPO_LOWER}/downloads-db:latest -o . || echo "No existing database" | |
| - name: Download packages for size | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: packages | |
| - name: Insert Amazon Linux release record | |
| run: | | |
| if [[ ! -f downloads.db ]]; then | |
| echo "No database to update" | |
| exit 0 | |
| fi | |
| RELEASE_TAG="python-amazonlinux-${PYTHON_VERSION}" | |
| RELEASE_URL="${{ needs.publish-amazonlinux.outputs.release_url }}" | |
| ASSET_URL="${{ needs.publish-amazonlinux.outputs.asset_url }}" | |
| PKG_FILENAME="${{ needs.build-amazonlinux.outputs.package_filename }}" | |
| PKG_SHA256="${{ needs.build-amazonlinux.outputs.package_sha256 }}" | |
| PLATFORM="linux-rpm-x86_64" | |
| MAJOR=$(echo "${PYTHON_VERSION}" | cut -d. -f1) | |
| MINOR=$(echo "${PYTHON_VERSION}" | cut -d. -f2) | |
| PATCH=$(echo "${PYTHON_VERSION}" | cut -d. -f3) | |
| FILE_SIZE=0 | |
| if [[ -f "packages/python-amazonlinux/$PKG_FILENAME" ]]; then | |
| FILE_SIZE=$(stat -c%s "packages/python-amazonlinux/$PKG_FILENAME") | |
| fi | |
| ARTIFACTS_JSON=$(cat <<EOF | |
| {"platforms":[{"platform":"${PLATFORM}","platform_os":"linux","platform_arch":"x86_64","binary":{"filename":"${PKG_FILENAME}","size":${FILE_SIZE},"sha256":"${PKG_SHA256}","url":"${ASSET_URL}","uploaded_at":"$(date -u +%Y-%m-%dT%H:%M:%SZ)"}}],"common_files":[],"metadata":{"total_platforms":1,"total_size":${FILE_SIZE}}} | |
| EOF | |
| ) | |
| ARTIFACTS_ESCAPED=$(echo "$ARTIFACTS_JSON" | sed "s/'/''/g") | |
| EXISTING=$(sqlite3 downloads.db "SELECT COUNT(*) FROM releases WHERE release_tag='${RELEASE_TAG}';") | |
| if [[ "$EXISTING" -gt 0 ]]; then | |
| sqlite3 downloads.db "UPDATE releases SET release_url='${RELEASE_URL}', artifacts='${ARTIFACTS_ESCAPED}', created_at=datetime('now') WHERE release_tag='${RELEASE_TAG}';" | |
| echo "Updated existing record for ${RELEASE_TAG}" | |
| else | |
| sqlite3 downloads.db "INSERT INTO releases (runtime, version, semver_major, semver_minor, semver_patch, release_tag, release_url, artifacts, created_at) VALUES ('python', '${PYTHON_VERSION}', ${MAJOR}, ${MINOR}, ${PATCH}, '${RELEASE_TAG}', '${RELEASE_URL}', '${ARTIFACTS_ESCAPED}', datetime('now'));" | |
| echo "Inserted new record for ${RELEASE_TAG}" | |
| fi | |
| - name: Insert Alpine release record | |
| run: | | |
| if [[ ! -f downloads.db ]]; then | |
| echo "No database to update" | |
| exit 0 | |
| fi | |
| RELEASE_TAG="python-alpine-${PYTHON_VERSION}" | |
| RELEASE_URL="${{ needs.publish-alpine.outputs.release_url }}" | |
| ASSET_URL="${{ needs.publish-alpine.outputs.asset_url }}" | |
| PKG_FILENAME="${{ needs.build-alpine.outputs.package_filename }}" | |
| PKG_SHA256="${{ needs.build-alpine.outputs.package_sha256 }}" | |
| PLATFORM="linux-alpine-x86_64" | |
| MAJOR=$(echo "${PYTHON_VERSION}" | cut -d. -f1) | |
| MINOR=$(echo "${PYTHON_VERSION}" | cut -d. -f2) | |
| PATCH=$(echo "${PYTHON_VERSION}" | cut -d. -f3) | |
| FILE_SIZE=0 | |
| if [[ -f "packages/python-alpine/$PKG_FILENAME" ]]; then | |
| FILE_SIZE=$(stat -c%s "packages/python-alpine/$PKG_FILENAME") | |
| fi | |
| ARTIFACTS_JSON=$(cat <<EOF | |
| {"platforms":[{"platform":"${PLATFORM}","platform_os":"linux","platform_arch":"x86_64","binary":{"filename":"${PKG_FILENAME}","size":${FILE_SIZE},"sha256":"${PKG_SHA256}","url":"${ASSET_URL}","uploaded_at":"$(date -u +%Y-%m-%dT%H:%M:%SZ)"}}],"common_files":[],"metadata":{"total_platforms":1,"total_size":${FILE_SIZE}}} | |
| EOF | |
| ) | |
| ARTIFACTS_ESCAPED=$(echo "$ARTIFACTS_JSON" | sed "s/'/''/g") | |
| EXISTING=$(sqlite3 downloads.db "SELECT COUNT(*) FROM releases WHERE release_tag='${RELEASE_TAG}';") | |
| if [[ "$EXISTING" -gt 0 ]]; then | |
| sqlite3 downloads.db "UPDATE releases SET release_url='${RELEASE_URL}', artifacts='${ARTIFACTS_ESCAPED}', created_at=datetime('now') WHERE release_tag='${RELEASE_TAG}';" | |
| echo "Updated existing record for ${RELEASE_TAG}" | |
| else | |
| sqlite3 downloads.db "INSERT INTO releases (runtime, version, semver_major, semver_minor, semver_patch, release_tag, release_url, artifacts, created_at) VALUES ('python', '${PYTHON_VERSION}', ${MAJOR}, ${MINOR}, ${PATCH}, '${RELEASE_TAG}', '${RELEASE_URL}', '${ARTIFACTS_ESCAPED}', datetime('now'));" | |
| echo "Inserted new record for ${RELEASE_TAG}" | |
| fi | |
| - name: Show database state | |
| run: | | |
| echo "=== Python releases in database ===" | |
| sqlite3 downloads.db "SELECT runtime, version, release_tag, created_at FROM releases WHERE runtime='python' ORDER BY created_at DESC LIMIT 10;" | |
| - name: Push downloads.db | |
| run: | | |
| if [[ -f downloads.db ]]; then | |
| REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | |
| oras push ghcr.io/${REPO_LOWER}/downloads-db:latest ./downloads.db | |
| fi | |
| - name: Trigger site deployment | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh workflow run run.yaml --ref ${{ github.ref_name }} || echo "Could not trigger workflow" |