Sync OpenAlex Snapshot #41
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync OpenAlex Snapshot | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' # Weekly Monday 06:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| entity: | |
| description: 'Entity to sync (empty = all)' | |
| required: false | |
| default: '' | |
| concurrency: | |
| group: sync-openalex | |
| cancel-in-progress: true | |
| jobs: | |
| # ── Detect ─────────────────────────────────────────────────────────── | |
| # Runs once. Finds entities with new/incomplete shards. | |
| # Outputs entity list as a matrix for per-entity sync jobs. | |
| detect: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| outputs: | |
| has_new: ${{ steps.detect.outputs.has_new }} | |
| matrix: ${{ steps.detect.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| cache: pip | |
| - run: pip install -r sync/requirements.txt | |
| - name: Restore HF listing cache | |
| id: hf-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: .cache/hf-listings | |
| key: hf-listings-${{ github.event.inputs.entity || 'all' }}-${{ github.run_id }} | |
| restore-keys: | | |
| hf-listings-${{ github.event.inputs.entity || 'all' }}- | |
| - name: Detect new shards | |
| id: detect | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| ENTITY_ARG="" | |
| if [ -n "${{ github.event.inputs.entity }}" ]; then | |
| ENTITY_ARG="--entity ${{ github.event.inputs.entity }}" | |
| fi | |
| python3 -m sync.ci_sync detect-entities \ | |
| --cache-dir .cache/hf-listings \ | |
| $ENTITY_ARG | |
| - name: Save HF listing cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: .cache/hf-listings | |
| key: hf-listings-${{ github.event.inputs.entity || 'all' }}-${{ github.run_id }} | |
| - name: Upload detect results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: detect-results | |
| path: detect_results.json | |
| retention-days: 1 | |
| # ── Sync (per-entity matrix) ───────────────────────────────────────── | |
| # Each matrix entry is one entity. The job handles ALL of that entity's | |
| # shards internally — download, extract, upload — with periodic commits | |
| # so progress survives timeouts. Completeness-aware detect means the | |
| # next run picks up any shards that weren't finished. | |
| sync: | |
| needs: detect | |
| if: needs.detect.outputs.has_new == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} | |
| name: sync / ${{ matrix.label }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| cache: pip | |
| - run: pip install -r sync/requirements.txt | |
| - name: Download detect results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: detect-results | |
| - name: Sync ${{ matrix.entity }} | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| python3 -m sync.ci_sync sync \ | |
| --entity "${{ matrix.entity }}" \ | |
| --batch "${{ matrix.batch }}" \ | |
| --detect-file detect_results.json | |
| - name: Upload sync results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sync-results-${{ matrix.label }} | |
| path: sync_results.jsonl | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| # ── Aggregate results ──────────────────────────────────────────────── | |
| # Collects all per-entity results into a single summary. | |
| report: | |
| needs: sync | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: sync-results-* | |
| merge-multiple: true | |
| path: results | |
| - name: Summary | |
| run: | | |
| if [ -d results ] && ls results/sync_results.jsonl 1>/dev/null 2>&1; then | |
| cat results/sync_results.jsonl > sync_results.jsonl | |
| echo "## Sync results" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat sync_results.jsonl >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| FAILED=$(grep -c '"status": "error"' sync_results.jsonl || true) | |
| OK=$(grep -c '"status": "ok"' sync_results.jsonl || true) | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**${OK} succeeded, ${FAILED} failed**" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No results files found" >> $GITHUB_STEP_SUMMARY | |
| fi |