Merge pull request #440 from stackql/dev #901
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy Registry Providers | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - dev | |
| pull_request: | |
| branches: | |
| - main | |
| - dev | |
| workflow_dispatch: | |
| jobs: | |
| build-and-deploy: | |
| name: build-and-deploy | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| AWS_DEFAULT_REGION: us-west-1 | |
| REG_MAX_VERSIONS: 3 | |
| REG_MAX_AGE_MONTHS: 6 | |
| REG_WEBSITE_DIR: _website | |
| REG_PROVIDER_PATH: providers/dist | |
| REG_ARTIFACT_REPO_BUCKET: stackql-registry-artifacts | |
| steps: | |
| - uses: actions/checkout@v7 | |
| name: "[SETUP] checkout repo" | |
| with: | |
| fetch-depth: 0 | |
| - name: "[SETUP] setup job" | |
| run: | | |
| echo "REG_COMMIT_DATETIME=$(date -d @`git show -s --format=%ct` +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| echo "REG_COMMIT_YEAR=$(date -d @`git show -s --format=%ct` +'%y')" >> $GITHUB_ENV | |
| echo "REG_COMMIT_MONTH=$(date -d @`git show -s --format=%ct` +'%m')" >> $GITHUB_ENV | |
| cd scripts/setup-js; npm i | |
| node setup-job.js | |
| - name: "[SETUP] print env vars" | |
| run: | | |
| node scripts/setup-js/print-env-vars.js | |
| - name: "[SETUP] get version" | |
| run: | | |
| node scripts/setup-js/get-version.js | |
| - name: "[SETUP] find changed files" | |
| run: | | |
| cd $GITHUB_WORKSPACE; git diff --name-status --diff-filter=ACMRT $REG_BASE_SHA $REG_COMMIT_SHA > diff.txt | |
| - name: "[SETUP] get updated providers" | |
| run: | | |
| pip install -q -r requirements.txt | |
| python scripts/setup/get-updated-providers.py | |
| - name: "[SETUP] prepare dist and test dirs" | |
| run: | | |
| mkdir -p "${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH}" | |
| mkdir -p "provider-tests/src" | |
| - name: '[PACKAGE] set up golang' | |
| if: env.NUM_PROVIDERS > 0 | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: ^1.19 | |
| check-latest: true | |
| cache: true | |
| id: go | |
| - name: "[PACKAGE] build sign tool" | |
| if: env.NUM_PROVIDERS > 0 | |
| run: | | |
| go build -o ed25519tool ./signing/Ed25519/app/cmd/main | |
| - name: "[PACKAGE] prepare provider dirs" | |
| if: env.NUM_PROVIDERS > 0 | |
| run: | | |
| while IFS= read -r provider_dir | |
| do | |
| echo "creating dist dir: ${provider_dir}..." | |
| mkdir "${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH}/${provider_dir}" | |
| mkdir "provider-tests/src/${provider_dir}" | |
| done < provider_dirs.txt | |
| - name: "[PACKAGE] update versions" | |
| if: env.NUM_PROVIDERS > 0 | |
| run: | | |
| python scripts/package/update-versions.py | |
| - name: "[PACKAGE] sign provider docs" | |
| if: env.NUM_PROVIDERS > 0 | |
| env: | |
| SIGNING_VERSION: v1 | |
| SIGNING_CERT: ${{ secrets.V1_SIGNING_CERT }} | |
| SIGNING_PRIV_KEY: '${{ secrets.V1_SIGNING_PRIV_KEY }}' | |
| SIGNING_PUB_KEY: ${{ secrets.V1_SIGNING_PUB_KEY }} | |
| run: | | |
| python scripts/package/sign-provider-docs.py | |
| - name: "[PACKAGE] package provider docs" | |
| if: env.NUM_PROVIDERS > 0 | |
| run: | | |
| python scripts/package/package-provider-docs.py | |
| - name: "[TESTS] simulate REGISTRY PULL" | |
| if: env.NUM_PROVIDERS > 0 | |
| run: | | |
| python scripts/tests/simulate-REGISTRY-PULL.py | |
| - name: "[TESTS] test provider(s)" | |
| if: env.NUM_PROVIDERS > 0 | |
| run: | | |
| echo "cloning test repo" | |
| providersdir="$(pwd)/provider-tests" | |
| git clone https://github.com/stackql/stackql-provider-tests.git | |
| cd stackql-provider-tests | |
| while IFS= read -r provider | |
| do | |
| echo "testing ${provider}..." | |
| sh test-provider.sh $provider false $providersdir | |
| done < ../providers.txt | |
| # | |
| # run the following steps only on pushes to protected branches (merge commits) | |
| # | |
| - name: "[PUBLISH] configure aws credentials" | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| if: env.REG_EVENT == 'push' | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_DEFAULT_REGION }} | |
| - name: "[PUBLISH] publish provider docs to artifact repo" | |
| if: env.NUM_PROVIDERS > 0 && env.REG_EVENT == 'push' | |
| run: | | |
| python scripts/publish/publish-provider-docs-to-artifact-repo.py | |
| - name: "[DEPLOY] pull additional docs from artifact repo" | |
| if: env.REG_EVENT == 'push' | |
| run: | | |
| python scripts/deploy/pull-additional-docs-from-artifact-repo.py | |
| # | |
| # Cloudflare serving layer. S3 (stackql-registry-artifacts) remains the | |
| # master/archive: the steps above publish changed providers to S3 and | |
| # reconstruct the full docs tree from it. Here that full tree (changed | |
| # providers + everything pulled from the artifact repo, plus the freshly | |
| # generated providers.yaml) lives at ${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH} | |
| # and is mirrored to R2, which the Worker serves from at the edge. | |
| # | |
| - name: "[DEPLOY-CF] install worker deps" | |
| if: env.REG_EVENT == 'push' | |
| run: | | |
| cd origin && npm install | |
| - name: "[DEPLOY-CF] sync docs to R2 (dev)" | |
| if: env.REG_TARGET_BRANCH == 'dev' && env.REG_EVENT == 'push' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_SESSION_TOKEN: "" | |
| AWS_DEFAULT_REGION: auto | |
| AWS_REGION: auto | |
| R2_BUCKET: stackql-provider-registry-dev | |
| run: | | |
| # version-pinned .tgz are immutable -> --size-only keeps R2 ops low and skips re-uploads | |
| aws s3 sync "${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH}" "s3://${R2_BUCKET}/${REG_PROVIDER_PATH}" \ | |
| --endpoint-url "https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com" \ | |
| --region auto --size-only --delete --no-progress | |
| # providers.yaml can change without changing size -> always overwrite it | |
| aws s3 cp "${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH}/providers.yaml" "s3://${R2_BUCKET}/${REG_PROVIDER_PATH}/providers.yaml" \ | |
| --endpoint-url "https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com" \ | |
| --region auto --no-progress | |
| - name: "[DEPLOY-CF] sync docs to R2 (prod)" | |
| if: env.REG_TARGET_BRANCH == 'main' && env.REG_EVENT == 'push' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_SESSION_TOKEN: "" | |
| AWS_DEFAULT_REGION: auto | |
| AWS_REGION: auto | |
| R2_BUCKET: stackql-provider-registry | |
| run: | | |
| aws s3 sync "${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH}" "s3://${R2_BUCKET}/${REG_PROVIDER_PATH}" \ | |
| --endpoint-url "https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com" \ | |
| --size-only --delete --no-progress | |
| aws s3 cp "${REG_WEBSITE_DIR}/${REG_PROVIDER_PATH}/providers.yaml" "s3://${R2_BUCKET}/${REG_PROVIDER_PATH}/providers.yaml" \ | |
| --endpoint-url "https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com" \ | |
| --no-progress | |
| - name: "[DEPLOY-CF] deploy worker (dev)" | |
| if: env.REG_TARGET_BRANCH == 'dev' && env.REG_EVENT == 'push' | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: | | |
| cd origin && npx wrangler deploy --env dev | |
| - name: "[DEPLOY-CF] deploy worker (prod)" | |
| if: env.REG_TARGET_BRANCH == 'main' && env.REG_EVENT == 'push' | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: | | |
| cd origin && npx wrangler deploy --env production |