OpenAlex/.github/workflows/sync.yml at 49bfee9bc11d126b98d024224a6cac710a76acf6 · Mearman/OpenAlex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
name: Sync OpenAlex Snapshot

on:
  schedule:
    - cron: '0 6 * * 1'  # Weekly Monday 06:00 UTC
  workflow_dispatch:
    inputs:
      entity:
        description: 'Entity to sync (empty = all)'
        required: false
        default: ''
      shards-per-batch:
        description: 'Shards per matrix entry (default 200)'
        required: false
        default: '200'

concurrency:
  group: sync-openalex
  cancel-in-progress: true

jobs:
  # ── Detect ───────────────────────────────────────────────────────────
  # Runs once. Generates the dynamic matrix for parallel sync jobs.
  # If no new shards, the sync job is skipped entirely.

  detect:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    outputs:
      has_new: ${{ steps.matrix.outputs.has_new }}
      matrix: ${{ steps.matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: '3.13'

      - run: pip install -r sync/requirements.txt

      - name: Generate sync matrix
        id: matrix
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          ARGS=""
          if [ -n "${{ github.event.inputs.entity }}" ]; then
            ARGS="$ARGS --entity ${{ github.event.inputs.entity }}"
          fi
          if [ -n "${{ github.event.inputs.shards-per-batch }}" ]; then
            ARGS="$ARGS --shards-per-batch ${{ github.event.inputs.shards-per-batch }}"
          fi
          python3 -m sync.ci_sync prepare-matrix $ARGS

      - name: Upload detect results
        uses: actions/upload-artifact@v7
        with:
          name: detect-results
          path: detect_results.json
          retention-days: 1

  # ── Sync (matrix) ────────────────────────────────────────────────────
  # Each matrix entry processes a slice: (entity, batch).
  # Runs in parallel up to max-parallel jobs.

  sync:
    needs: detect
    if: needs.detect.outputs.has_new == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 120
    strategy:
      max-parallel: 10
      fail-fast: false
      matrix: ${{ fromJSON(needs.detect.outputs.matrix) }}
    name: sync / ${{ matrix.label }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: '3.13'

      - run: pip install -r sync/requirements.txt

      - name: Download detect results
        uses: actions/download-artifact@v8
        with:
          name: detect-results

      - name: Sync ${{ matrix.label }}
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          python3 -m sync.ci_sync sync \
            --entity "${{ matrix.entity }}" \
            --batch-index "${{ matrix.batch_index }}" \
            --detect-file detect_results.json

      - name: Upload sync results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: sync-results-${{ matrix.label }}
          path: sync_results.jsonl
          retention-days: 30
          if-no-files-found: ignore

  # ── Aggregate results ────────────────────────────────────────────────
  # Collects all per-job results into a single summary.

  report:
    needs: sync
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Download all results
        uses: actions/download-artifact@v8
        with:
          pattern: sync-results-*
          merge-multiple: true
          path: results

      - name: Summary
        run: |
          if [ -d results ] && ls results/sync_results.jsonl 1>/dev/null 2>&1; then
            cat results/sync_results.jsonl > sync_results.jsonl
            echo "## Sync results" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            cat sync_results.jsonl >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY

            FAILED=$(grep -c '"status": "error"' sync_results.jsonl || true)
            OK=$(grep -c '"status": "ok"' sync_results.jsonl || true)
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "**${OK} succeeded, ${FAILED} failed**" >> $GITHUB_STEP_SUMMARY
          else
            echo "No results files found" >> $GITHUB_STEP_SUMMARY
          fi