Sync OpenAlex Snapshot #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync OpenAlex Snapshot | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' # Weekly Monday 06:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| entity: | |
| description: 'Entity to sync (empty = all)' | |
| required: false | |
| default: '' | |
| shards-per-batch: | |
| description: 'Shards per matrix entry (default 200)' | |
| required: false | |
| default: '200' | |
| concurrency: | |
| group: sync-openalex | |
| cancel-in-progress: true | |
| jobs: | |
| # ── Detect ─────────────────────────────────────────────────────────── | |
| # Runs once. Generates the dynamic matrix for parallel sync jobs. | |
| # If no new shards, the sync job is skipped entirely. | |
| detect: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| outputs: | |
| has_new: ${{ steps.matrix.outputs.has_new }} | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install -r sync/requirements.txt | |
| - name: Generate sync matrix | |
| id: matrix | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| ARGS="" | |
| if [ -n "${{ github.event.inputs.entity }}" ]; then | |
| ARGS="$ARGS --entity ${{ github.event.inputs.entity }}" | |
| fi | |
| if [ -n "${{ github.event.inputs.shards-per-batch }}" ]; then | |
| ARGS="$ARGS --shards-per-batch ${{ github.event.inputs.shards-per-batch }}" | |
| fi | |
| python3 -m sync.ci_sync prepare-matrix $ARGS | |
| - name: Upload detect results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: detect-results | |
| path: detect_results.json | |
| retention-days: 1 | |
| # ── Sync (matrix) ──────────────────────────────────────────────────── | |
| # Each matrix entry processes a slice: (entity, batch). | |
| # Runs in parallel up to max-parallel jobs. | |
| sync: | |
| needs: detect | |
| if: needs.detect.outputs.has_new == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} | |
| name: sync / ${{ matrix.label }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install -r sync/requirements.txt | |
| - name: Download detect results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: detect-results | |
| - name: Sync ${{ matrix.label }} | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| python3 -m sync.ci_sync sync \ | |
| --entity "${{ matrix.entity }}" \ | |
| --batch-index "${{ matrix.batch_index }}" \ | |
| --detect-file detect_results.json | |
| - name: Upload sync results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sync-results-${{ matrix.label }} | |
| path: sync_results.jsonl | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| # ── Aggregate results ──────────────────────────────────────────────── | |
| # Collects all per-job results into a single summary. | |
| report: | |
| needs: sync | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: sync-results-* | |
| merge-multiple: true | |
| path: results | |
| - name: Summary | |
| run: | | |
| if [ -d results ] && ls results/sync_results.jsonl 1>/dev/null 2>&1; then | |
| cat results/sync_results.jsonl > sync_results.jsonl | |
| echo "## Sync results" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat sync_results.jsonl >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| FAILED=$(grep -c '"status": "error"' sync_results.jsonl || true) | |
| OK=$(grep -c '"status": "ok"' sync_results.jsonl || true) | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**${OK} succeeded, ${FAILED} failed**" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No results files found" >> $GITHUB_STEP_SUMMARY | |
| fi |