perf: use float32 for CSR data array to halve memory #133
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync OpenAlex Snapshot | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'sync/**' | |
| - '.github/workflows/sync.yml' | |
| schedule: | |
| - cron: '0 6 * * 1' # Weekly Monday 06:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| entity: | |
| description: 'Entity to sync (empty = all)' | |
| required: false | |
| default: '' | |
| self_hosted_runner: | |
| description: 'Self-hosted runner label (empty = managed only)' | |
| required: false | |
| default: '' | |
| concurrency: | |
| group: sync-openalex | |
| cancel-in-progress: true | |
| jobs: | |
| # ── Detect ─────────────────────────────────────────────────────────── | |
| # Runs once. Finds entities with new/incomplete shards AND orphan files | |
| # on HF that have no S3 manifest entry. Outputs entity list as a matrix | |
| # for per-entity sync jobs. Writes detect results (including orphans) for | |
| # the cleanup and sync jobs to reuse. | |
| detect: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| outputs: | |
| has_new: ${{ steps.detect.outputs.has_new }} | |
| matrix: ${{ steps.detect.outputs.matrix }} | |
| max_parallel: ${{ steps.detect.outputs.max_parallel }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| cache: pip | |
| - run: pip install -r sync/requirements.txt | |
| - name: Restore HF listing cache | |
| id: hf-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: .cache/hf-listings | |
| key: hf-listings-${{ github.event.inputs.entity || 'all' }}-${{ github.run_id }} | |
| - name: Detect new shards | |
| id: detect | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| SELF_HOSTED_RUNNER: ${{ github.event.inputs.self_hosted_runner || '' }} | |
| run: | | |
| ENTITY_ARG="" | |
| if [ -n "${{ github.event.inputs.entity }}" ]; then | |
| ENTITY_ARG="--entity ${{ github.event.inputs.entity }}" | |
| fi | |
| python3 -m sync.ci_sync detect-entities \ | |
| --cache-dir .cache/hf-listings \ | |
| $ENTITY_ARG | |
| - name: Save HF listing cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: .cache/hf-listings | |
| key: hf-listings-${{ github.event.inputs.entity || 'all' }}-${{ github.run_id }} | |
| - name: Upload detect results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: detect-results | |
| path: detect_results.json | |
| retention-days: 1 | |
| # ── Sync (per-entity matrix) ───────────────────────────────────────── | |
| # Each matrix entry is one entity. The job handles ALL of that entity's | |
| # shards internally — download, extract, upload — with periodic commits | |
| # so progress survives timeouts. Completeness-aware detect means the | |
| # next run picks up any shards that weren't finished. | |
| sync: | |
| needs: detect | |
| if: needs.detect.outputs.has_new == 'true' | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 360 | |
| strategy: | |
| max-parallel: ${{ fromJSON(needs.detect.outputs.max_parallel) }} | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} | |
| name: sync / ${{ matrix.label }} (@${{ matrix.runner }}) | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python (managed) | |
| if: matrix.runner == 'ubuntu-latest' | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| cache: pip | |
| - name: Set up Python (self-hosted) | |
| if: matrix.runner != 'ubuntu-latest' | |
| run: | | |
| python3 --version || { echo "Python 3 required on self-hosted runner"; exit 1; } | |
| python3 -m pip --version || { echo "pip required on self-hosted runner"; exit 1; } | |
| - run: pip install -r sync/requirements.txt | |
| - name: Download detect results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: detect-results | |
| - name: Sync ${{ matrix.entity }} | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| HF_XET_HIGH_PERFORMANCE: "1" | |
| # Stagger starts to avoid all jobs hitting HF API simultaneously. | |
| # Each job sleeps a random 0–30s before starting work. | |
| SYNC_STAGGER_MAX: "30" | |
| run: | | |
| STAGGER=$(( RANDOM % ${SYNC_STAGGER_MAX:-0} )) | |
| if [ "$STAGGER" -gt 0 ]; then | |
| echo "Staggering start by ${STAGGER}s to spread HF API load" | |
| sleep "$STAGGER" | |
| fi | |
| python3 -m sync.ci_sync sync \ | |
| --entity "${{ matrix.entity }}" \ | |
| --batch "${{ matrix.batch }}" \ | |
| --detect-file detect_results.json | |
| - name: Upload sync results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sync-results-${{ matrix.label }} | |
| path: sync_results.jsonl | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| # ── Cleanup ────────────────────────────────────────────────────── | |
| # Runs AFTER sync. Re-scans HF against S3 manifests and deletes any | |
| # orphan files (source or parquet) not backed by the current manifest. | |
| # S3 is the single source of truth. Running after sync ensures the | |
| # final HF state is fully consistent — no manual cycling needed. | |
| cleanup: | |
| needs: sync | |
| if: always() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| cache: pip | |
| - run: pip install -r sync/requirements.txt | |
| - name: Restore HF listing cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: .cache/hf-listings | |
| key: hf-listings-${{ github.event.inputs.entity || 'all' }}-${{ github.run_id }} | |
| - name: Cleanup orphan files | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: python3 -m sync.ci_sync cleanup --cache-dir .cache/hf-listings | |
| # ── Aggregate results ──────────────────────────────────────────────── | |
| # Collects all per-entity results into a single summary. | |
| report: | |
| needs: cleanup | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: sync-results-* | |
| merge-multiple: true | |
| path: results | |
| - name: Summary | |
| run: | | |
| if [ -d results ] && ls results/sync_results.jsonl 1>/dev/null 2>&1; then | |
| cat results/sync_results.jsonl > sync_results.jsonl | |
| echo "## Sync results" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat sync_results.jsonl >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| FAILED=$(grep -c '"status": "error"' sync_results.jsonl || true) | |
| OK=$(grep -c '"status": "ok"' sync_results.jsonl || true) | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**${OK} succeeded, ${FAILED} failed**" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No results files found" >> $GITHUB_STEP_SUMMARY | |
| fi |