Skip to content

Commit 5db80c7

Browse files
mikealfaremikealfare
authored
Update release workflow (#732)
1 parent fea86ff commit 5db80c7

File tree

127 files changed

+687
-649
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

127 files changed

+687
-649
lines changed

‎.github/workflows/ci.yml

-36
This file was deleted.

‎.github/workflows/code-quality.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: "Code quality"
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
branch:
7+
description: "Choose the branch to check"
8+
type: string
9+
default: "main"
10+
repository:
11+
description: "Choose the repository to check, when using a fork"
12+
type: string
13+
default: "dbt-labs/dbt-athena"
14+
workflow_dispatch:
15+
inputs:
16+
branch:
17+
description: "Choose the branch to check"
18+
type: string
19+
default: "main"
20+
repository:
21+
description: "Choose the repository to check, when using a fork"
22+
type: string
23+
default: "dbt-labs/dbt-athena"
24+
25+
permissions:
26+
contents: read
27+
28+
jobs:
29+
code-quality:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v4
33+
with:
34+
ref: ${{ inputs.branch }}
35+
repository: ${{ inputs.repository }}
36+
- uses: actions/setup-python@v5
37+
with:
38+
python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
39+
- uses: pre-commit/[email protected]

‎.github/workflows/functional-tests-workflow.yml

-54
This file was deleted.

‎.github/workflows/functional-tests.yml

-48
This file was deleted.

‎.github/workflows/integration-tests.yml

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: "Integration tests"
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
package:
7+
description: "Choose the package to test"
8+
type: string
9+
default: "dbt-athena"
10+
branch:
11+
description: "Choose the branch to test"
12+
type: string
13+
default: "main"
14+
repository:
15+
description: "Choose the repository to test, when using a fork"
16+
type: string
17+
default: "dbt-labs/dbt-athena"
18+
workflow_dispatch:
19+
inputs:
20+
package:
21+
description: "Choose the package to test"
22+
type: choice
23+
options: ["dbt-athena", "dbt-athena-community"]
24+
branch:
25+
description: "Choose the branch to test"
26+
type: string
27+
default: "main"
28+
repository:
29+
description: "Choose the repository to test, when using a fork"
30+
type: string
31+
default: "dbt-labs/dbt-athena"
32+
33+
permissions:
34+
id-token: write
35+
contents: read
36+
37+
jobs:
38+
integration-tests:
39+
runs-on: ubuntu-latest
40+
env:
41+
DBT_TEST_ATHENA_S3_STAGING_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/staging/
42+
DBT_TEST_ATHENA_S3_TMP_TABLE_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/tmp_tables/
43+
DBT_TEST_ATHENA_REGION_NAME: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }}
44+
DBT_TEST_ATHENA_DATABASE: awsdatacatalog
45+
DBT_TEST_ATHENA_SCHEMA: dbt-tests
46+
DBT_TEST_ATHENA_WORK_GROUP: athena-dbt-tests
47+
DBT_TEST_ATHENA_THREADS: 16
48+
DBT_TEST_ATHENA_POLL_INTERVAL: 0.5
49+
DBT_TEST_ATHENA_NUM_RETRIES: 3
50+
steps:
51+
- uses: actions/checkout@v4
52+
with:
53+
ref: ${{ inputs.branch }}
54+
repository: ${{ inputs.repository }}
55+
- uses: actions/setup-python@v5
56+
with:
57+
python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
58+
- uses: pypa/hatch@install
59+
- uses: aws-actions/configure-aws-credentials@v2
60+
with:
61+
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }}
62+
aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }}
63+
- run: hatch run integration-tests
64+
working-directory: ./${{ inputs.package }}

‎.github/workflows/publish-pypi.yml

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: "Publish to PyPI"
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
package:
7+
description: "Choose the package to publish"
8+
type: string
9+
default: "dbt-athena"
10+
deploy-to:
11+
description: "Choose whether to publish to test or prod"
12+
type: string
13+
default: "prod"
14+
branch:
15+
description: "Choose the branch to publish"
16+
type: string
17+
default: "main"
18+
19+
permissions:
20+
contents: read
21+
22+
defaults:
23+
run:
24+
shell: bash
25+
26+
jobs:
27+
publish:
28+
runs-on: ubuntu-latest
29+
environment:
30+
name: ${{ inputs.deploy-to }}
31+
url: ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}
32+
permissions:
33+
# this permission is required for trusted publishing
34+
# see https://github.com/marketplace/actions/pypi-publish
35+
id-token: write
36+
steps:
37+
- uses: actions/checkout@v4
38+
with:
39+
ref: ${{ inputs.branch }}
40+
- uses: actions/setup-python@v5
41+
with:
42+
python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
43+
- uses: pypa/hatch@install
44+
# hatch will build using test PyPI first and fall back to prod PyPI when deploying to test
45+
# this is done via environment variables in the test environment in GitHub
46+
- run: hatch build && hatch run build:check-all
47+
working-directory: ./${{ inputs.package }}
48+
- uses: pypa/gh-action-pypi-publish@release/v1
49+
with:
50+
repository-url: ${{ vars.PYPI_REPOSITORY_URL }}
51+
packages-dir: ${{ inputs.package }}/dist/
52+
53+
verify:
54+
runs-on: ubuntu-latest
55+
needs: publish
56+
# check the correct index
57+
environment:
58+
name: ${{ inputs.deploy-to }}
59+
steps:
60+
- uses: actions/checkout@v4
61+
with:
62+
ref: ${{ inputs.branch }}
63+
- id: version
64+
run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT
65+
working-directory: ./${{ inputs.package }}
66+
- uses: nick-fields/retry@v3
67+
with:
68+
timeout_seconds: 10
69+
retry_wait_seconds: 10
70+
max_attempts: 15 # 5 minutes: (10s timeout + 10s delay) * 15 attempts
71+
command: wget ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}/${{ steps.version.outputs.version }}

‎.github/workflows/publish.yml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: "Publish"
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
deploy-to:
7+
description: "Choose whether to deploy to test or prod"
8+
type: environment
9+
default: "prod"
10+
branch:
11+
description: "Choose the branch to release from"
12+
type: string
13+
default: "main"
14+
15+
# don't attempt to release the same target in parallel
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ inputs.deploy-to }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
unit-tests:
22+
uses: ./.github/workflows/unit-tests.yml
23+
with:
24+
branch: ${{ inputs.branch }}
25+
26+
integration-tests:
27+
uses: ./.github/workflows/integration-tests.yml
28+
with:
29+
branch: ${{ inputs.branch }}
30+
repository: ${{ github.repository }}
31+
secrets: inherit
32+
33+
publish-dbt-athena:
34+
needs: [unit-tests, integration-tests]
35+
uses: ./.github/workflows/publish-pypi.yml
36+
with:
37+
deploy-to: ${{ inputs.deploy-to }}
38+
branch: ${{ inputs.branch }}
39+
40+
publish-dbt-athena-community:
41+
# dbt-athena-community is hard pinned to dbt-athena to ensure they are the same
42+
# this means we need to finish publishing dbt-athena before starting to build dbt-athena-community
43+
needs: publish-dbt-athena
44+
uses: ./.github/workflows/publish-pypi.yml
45+
with:
46+
package: "dbt-athena-community"
47+
deploy-to: ${{ inputs.deploy-to }}
48+
branch: ${{ inputs.branch }}

0 commit comments

Comments
 (0)