Publish Index #874
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Index | |
| # Keeps the published package index current by ingesting new channel-release | |
| # snapshots from releases.nixos.org (see docs/indexer-rewrite/DESIGN.md). | |
| # No nixpkgs clone, no commit walking: each run diffs the S3 listing against | |
| # the index's release ledger and ingests whatever is new — typically 0-2 | |
| # releases per 6h window, seconds to minutes of work. | |
| # | |
| # The initial full v4 rebuild is an out-of-band, operator-driven run (hours); | |
| # this workflow only does incremental updates against an existing index. | |
| on: | |
| # Run every 6 hours to keep the index fresh | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| # Allow manual triggering | |
| workflow_dispatch: | |
| inputs: | |
| force_publish: | |
| description: "Publish even when no new releases were ingested this run" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: publish-index | |
| cancel-in-progress: false # Let runs complete, queue new ones | |
| env: | |
| CARGO_TERM_COLOR: always | |
| # Index is stored in a GitHub release with this tag | |
| # Forks can override INDEX_URL_PREFIX via repository variable | |
| INDEX_RELEASE_TAG: index-latest | |
| INDEX_URL_PREFIX: ${{ vars.INDEX_URL_PREFIX || format('https://github.com/{0}/releases/download/index-latest', github.repository) }} | |
| INDEX_ASSET_PREFIX: index-${{ github.run_id }}-${{ github.run_attempt }}- | |
| jobs: | |
| update-index: | |
| name: Update Package Index | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout nxv | |
| uses: actions/checkout@v4 | |
| # Nix is only needed for the --head-eval fallback (channel-stuck | |
| # periods); the snapshot path itself does no evaluation. | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@v30 | |
| with: | |
| github_access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-indexer-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-indexer- | |
| - name: Build nxv with indexer | |
| run: cargo build --release --features indexer | |
| - name: Download existing index | |
| run: | | |
| set -euo pipefail | |
| mkdir -p ~/.local/share/nxv | |
| echo "Downloading existing manifest from ${INDEX_URL_PREFIX}..." | |
| # Incremental updates REQUIRE an existing index. The initial v4 | |
| # full rebuild is run out-of-band and uploaded manually. | |
| manifest_path="$(mktemp)" | |
| curl -sSfL "${INDEX_URL_PREFIX}/manifest.json" -o "$manifest_path" | |
| index_url="$(jq -r '.full_index.url // empty' "$manifest_path")" | |
| if [ -z "$index_url" ]; then | |
| echo "::error::Published manifest is missing full_index.url" | |
| exit 1 | |
| fi | |
| echo "Downloading existing index from ${index_url}..." | |
| curl -sSfL "$index_url" -o ~/.local/share/nxv/index.db.zst | |
| zstd -d -f ~/.local/share/nxv/index.db.zst -o ~/.local/share/nxv/index.db | |
| rm ~/.local/share/nxv/index.db.zst | |
| echo "Index ready: $(du -h ~/.local/share/nxv/index.db | cut -f1)" | |
| - name: Ingest new channel releases | |
| id: ingest | |
| env: | |
| # Used by --head-eval to resolve master HEAD without rate limits | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| # Not --strict: gate-failed releases never write rows, so | |
| # publishing the successfully ingested progress is always safe and | |
| # strictly fresher than not publishing. Failures are alerted on in | |
| # the final step (after publish) instead of blocking it. | |
| ./target/release/nxv index --head-eval --report report.json | |
| echo "ingested=$(jq -r '.ingested' report.json)" >> "$GITHUB_OUTPUT" | |
| echo "healthy=$(jq -r '.healthy // false' report.json)" >> "$GITHUB_OUTPUT" | |
| echo "failed=$(jq -r '.failed' report.json)" >> "$GITHUB_OUTPUT" | |
| - name: Decide whether to publish | |
| id: gate | |
| run: | | |
| INGESTED="${{ steps.ingest.outputs.ingested }}" | |
| FORCE="${{ inputs.force_publish }}" | |
| if [ "${INGESTED:-0}" -gt 0 ] || [ "$FORCE" = "true" ]; then | |
| echo "publish=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "No new releases this run; skipping publish." | |
| echo "publish=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Generate publishable artifacts | |
| if: steps.gate.outputs.publish == 'true' | |
| env: | |
| NXV_SECRET_KEY: ${{ secrets.NXV_SIGNING_KEY }} | |
| run: | | |
| # --min-version defaults to the schema version in code; passed | |
| # explicitly as belt-and-braces (pre-v4 clients must be gated | |
| # BEFORE download or they overwrite a working index). | |
| ./target/release/nxv publish \ | |
| --output ./publish \ | |
| --url-prefix "${INDEX_URL_PREFIX}" \ | |
| --artifact-name-prefix "${INDEX_ASSET_PREFIX}" \ | |
| --min-version 4 \ | |
| --sign | |
| - name: Upload workflow artifacts | |
| if: steps.gate.outputs.publish == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nxv-index | |
| path: publish/ | |
| retention-days: 7 | |
| - name: Upload to GitHub Release | |
| if: steps.gate.outputs.publish == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if ! gh release view "$INDEX_RELEASE_TAG" > /dev/null 2>&1; then | |
| echo "Creating release $INDEX_RELEASE_TAG..." | |
| gh release create "$INDEX_RELEASE_TAG" \ | |
| --title "Package Index" \ | |
| --notes "nxv package index files. Updated automatically." \ | |
| --latest=false | |
| fi | |
| replace_stable_asset() { | |
| local path="$1" | |
| local name="$2" | |
| local backup_dir | |
| backup_dir="$(mktemp -d)" | |
| if gh release view "$INDEX_RELEASE_TAG" --json assets --jq '.assets[].name' | grep -Fxq "$name"; then | |
| gh release download "$INDEX_RELEASE_TAG" --pattern "$name" --dir "$backup_dir" | |
| fi | |
| if gh release upload "$INDEX_RELEASE_TAG" "${path}#${name}" --clobber; then | |
| return 0 | |
| fi | |
| echo "::error::Failed to replace ${name}; attempting to restore previous asset" | |
| if [ -f "${backup_dir}/${name}" ]; then | |
| gh release upload "$INDEX_RELEASE_TAG" "${backup_dir}/${name}#${name}" --clobber || true | |
| fi | |
| return 1 | |
| } | |
| echo "Uploading immutable index files to release..." | |
| immutable_upload_dir="$(mktemp -d)" | |
| cp publish/index.db.zst "${immutable_upload_dir}/${INDEX_ASSET_PREFIX}index.db.zst" | |
| cp publish/bloom.bin "${immutable_upload_dir}/${INDEX_ASSET_PREFIX}bloom.bin" | |
| gh release upload "$INDEX_RELEASE_TAG" \ | |
| "${immutable_upload_dir}/${INDEX_ASSET_PREFIX}index.db.zst" \ | |
| "${immutable_upload_dir}/${INDEX_ASSET_PREFIX}bloom.bin" | |
| echo "Verifying immutable assets are present before moving the manifest pointer..." | |
| gh release view "$INDEX_RELEASE_TAG" --json assets \ | |
| --jq '.assets[].name' \ | |
| | grep -Fx "${INDEX_ASSET_PREFIX}index.db.zst" | |
| gh release view "$INDEX_RELEASE_TAG" --json assets \ | |
| --jq '.assets[].name' \ | |
| | grep -Fx "${INDEX_ASSET_PREFIX}bloom.bin" | |
| # GitHub release assets cannot replace manifest.json and its signature | |
| # atomically. Payload assets are already present, and the restore logic | |
| # keeps stable assets from staying missing; clients may see a brief | |
| # signature mismatch until manifest.json is replaced as the final step. | |
| replace_stable_asset publish/manifest.json.minisig manifest.json.minisig | |
| replace_stable_asset publish/manifest.json manifest.json | |
| echo "Verifying published manifest points at this run's immutable assets..." | |
| published_manifest="$(mktemp)" | |
| expected_index_url="${INDEX_URL_PREFIX}/${INDEX_ASSET_PREFIX}index.db.zst" | |
| expected_bloom_url="${INDEX_URL_PREFIX}/${INDEX_ASSET_PREFIX}bloom.bin" | |
| manifest_verified=false | |
| for attempt in {1..12}; do | |
| if curl -sSfL "${INDEX_URL_PREFIX}/manifest.json" -o "$published_manifest" \ | |
| && jq -e \ | |
| --arg index_url "$expected_index_url" \ | |
| --arg bloom_url "$expected_bloom_url" \ | |
| '.full_index.url == $index_url and .bloom_filter.url == $bloom_url' \ | |
| "$published_manifest" >/dev/null; then | |
| manifest_verified=true | |
| break | |
| fi | |
| echo "Manifest pointer has not propagated yet (attempt ${attempt}/12); retrying..." | |
| sleep 5 | |
| done | |
| if [ "$manifest_verified" != "true" ]; then | |
| echo "::error::Published manifest did not point at this run's immutable assets after waiting for propagation." | |
| jq . "$published_manifest" || cat "$published_manifest" | |
| exit 1 | |
| fi | |
| # AFTER publishing: turn anomalies into a red workflow run so they | |
| # page the operator. Failed releases stay in the (published) ledger | |
| # with retry/backoff state; head-lag breaches and gate failures must | |
| # not rot silently — that is how the old indexer lost four months. | |
| - name: Alert on anomalies | |
| if: always() && steps.ingest.outcome == 'success' | |
| run: | | |
| if [ "${{ steps.ingest.outputs.healthy }}" != "true" ] || [ "${{ steps.ingest.outputs.failed }}" != "0" ]; then | |
| echo "::error::Index run reported anomalies (healthy=${{ steps.ingest.outputs.healthy }}, failed=${{ steps.ingest.outputs.failed }}) — see report in the job summary." | |
| exit 1 | |
| fi | |
| - name: Summary | |
| if: always() | |
| run: | | |
| echo "## Index Update Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f report.json ]; then | |
| echo '```json' >> $GITHUB_STEP_SUMMARY | |
| cat report.json >> $GITHUB_STEP_SUMMARY | |
| echo '' >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "_No report generated (run failed before ingest)._" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ "${{ steps.gate.outputs.publish }}" = "true" ] && [ -f publish/manifest.json ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Published:** yes ($(du -h publish/index.db.zst | cut -f1))" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Published:** no" >> $GITHUB_STEP_SUMMARY | |
| fi |