datasets/.github/workflows/daily.yml at main · dataresearchcenter/datasets · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
name: datasets_daily

on:
  schedule:
    - cron: "13 12 * * *"
  workflow_dispatch: {}

jobs:
  data:
    runs-on: ubuntu-latest
    container: ghcr.io/dataresearchcenter/datasets:latest
    env:
      HOME: /home/1000
      LAKEHOUSE_URI: ${{ secrets.LAKEHOUSE_URI }}
      LAKEHOUSE_API_KEY: ${{ secrets.LAKEHOUSE_API_KEY }}
      LAKEHOUSE_API_SECRET: ${{ secrets.LAKEHOUSE_API_SECRET }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - dataset: ec_meetings
            runner: investigraph
          - dataset: eu_transparency_register
            runner: investigraph
          - dataset: eu_ec_ease
            runner: memorious
          - dataset: de_lobbyregister
            runner: investigraph
          - dataset: de_bundestag_dip
            runner: memorious
          - dataset: de_sehrgutachten
            runner: memorious
          - dataset: de_bgh
            runner: memorious
          - dataset: de_parlamentsspiegel
            runner: memorious
    steps:
      - name: Run ${{ matrix.runner }} for ${{ matrix.dataset }}
        run: make ${{ matrix.runner }} dataset=${{ matrix.dataset }} DATASETS_DIR=.
        working-directory: /datasets

  # catalog:
  #   needs: data
  #   if: always()
  #   runs-on: ubuntu-latest
  #   container: ghcr.io/dataresearchcenter/datasets:latest
  #   defaults:
  #     run:
  #       working-directory: /datasets
  #   env:
  #     AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  #     AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  #     FSSPEC_S3_ENDPOINT_URL: ${{ secrets.FSSPEC_S3_ENDPOINT_URL }}
  #   steps:
  #     - name: Update the catalog
  #       run: "make publish"
  #     - name: Notify success
  #       run: "anystore io -i ${{ secrets.DATASETS_DAILY_HEARTBEAT }}"