Remove DSSKey references from SSH provider, bump paramiko to 4.0+ #31307
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| # AMD CI workflow. Triggered on every PR and on push to release branches. | |
| # Companion to `ci-arm.yml`, which runs the same pipeline on ARM as the | |
| # scheduled canary. The two workflows are physical copies of each other | |
| # (GitHub Actions has no cross-file YAML include), differing only in their | |
| # trigger surface, the hardcoded `runner-type` / `platform` values in | |
| # `build-info`'s outputs, and the architecture-specific print-platform job. | |
| # Edit both files together when changing pipeline shape. | |
| name: Tests (AMD) | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| # AMD canary runs 2x/day (01:58, 13:58), interleaved with ARM's 2x/day (07:28, 19:28) | |
| # in `ci-arm.yml` so a full-matrix canary still runs roughly every ~6h (alternating | |
| # architecture) while halving the scheduled AMD compute. The `:58` vs ARM's `:28` | |
| # offset keeps the two from competing for runners at exactly the same minute. | |
| - cron: '58 1,13 * * *' | |
| pull_request: | |
| branches: | |
| - main | |
| - v[0-9]+-[0-9]+-test | |
| - v[0-9]+-[0-9]+-stable | |
| - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ | |
| types: [opened, reopened, synchronize, ready_for_review] | |
| push: | |
| # Post-merge pushes to release-prep / providers branches run on both | |
| # AMD and ARM (the matching block lives in the other wrapper too — | |
| # `check-ci-workflows-in-sync` enforces they stay identical). `main` is | |
| # intentionally NOT in this list: main pushes stay AMD-only via | |
| # `ci-amd.yml`; ARM main coverage is the cron-driven canary above. | |
| branches: | |
| - v[0-9]+-[0-9]+-test | |
| - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ | |
| workflow_dispatch: | |
| permissions: | |
| # All other permissions are set to none by default | |
| contents: read | |
| env: | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| JAVA_VERSION: '11' | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| VERBOSE: "true" | |
| concurrency: | |
| group: ci-amd-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| name: "Build info" | |
| # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners | |
| runs-on: ["ubuntu-22.04"] | |
| outputs: | |
| all-python-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
| arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} | |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} | |
| canary-run: ${{ steps.source-run-info.outputs.canary-run }} | |
| ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} | |
| core-test-types-list-as-strings-in-json: >- | |
| ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} | |
| debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} | |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} | |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} | |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} | |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} | |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} | |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
| disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} | |
| docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} | |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} | |
| docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} | |
| excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }} | |
| force-pip: ${{ steps.selective-checks.outputs.force-pip }} | |
| full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} | |
| has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} | |
| helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} | |
| helm-test-kubernetes-versions: ${{ steps.selective-checks.outputs.helm-test-kubernetes-versions }} | |
| include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} | |
| individual-providers-test-types-list-as-strings-in-json: >- | |
| ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} | |
| kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} | |
| kubernetes-combos-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} | |
| kubernetes-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} | |
| kustomize-overlay-names: ${{ steps.selective-checks.outputs.kustomize-overlay-names }} | |
| run-kustomize-overlays-tests: ${{ steps.selective-checks.outputs.run-kustomize-overlays-tests }} | |
| latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} | |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} | |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} | |
| # platform / runner-type are hardcoded per workflow file (this is the | |
| # AMD copy). The matching ARM copy lives in ci-arm.yml. | |
| platform: "linux/amd64" | |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} | |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} | |
| prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} | |
| # yamllint disable rule:line-length | |
| providers-compatibility-tests-matrix: > | |
| ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} | |
| providers-test-types-list-as-strings-in-json: >- | |
| ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} | |
| pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} | |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} | |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} | |
| run-airflow-ctl-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-tests }} | |
| run-airflow-ctl-integration-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-integration-tests }} | |
| run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} | |
| run-api-codegen: ${{ steps.selective-checks.outputs.run-api-codegen }} | |
| run-api-tests: ${{ steps.selective-checks.outputs.run-api-tests }} | |
| run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} | |
| run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }} | |
| run-java-sdk-tests: ${{ steps.selective-checks.outputs.run-java-sdk-tests }} | |
| run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }} | |
| run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }} | |
| run-remote-logging-opensearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-opensearch-e2e-tests }} | |
| run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }} | |
| run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }} | |
| run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }} | |
| run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }} | |
| run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} | |
| run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} | |
| run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} | |
| run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} | |
| run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} | |
| runner-type: '["ubuntu-22.04"]' | |
| run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} | |
| run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} | |
| run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} | |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} | |
| selected-providers-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} | |
| shared-distributions-as-json: ${{ steps.selective-checks.outputs.shared-distributions-as-json }} | |
| skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} | |
| skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} | |
| source-head-repo: ${{ steps.source-run-info.outputs.head-repo }} | |
| source-head-ref: ${{ steps.source-run-info.outputs.head-ref }} | |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} | |
| testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} | |
| testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} | |
| use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} | |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: sudo rm -rf ${GITHUB_WORKSPACE}/* | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: "Save github context to file" | |
| # The github context can be very large for PRs that touch many files (e.g. Dependabot | |
| # bumps with dozens of packages). Passing it as an env var would exceed ARG_MAX and | |
| # prevent subsequent steps from starting bash, so we write it to a file once here and | |
| # let downstream breeze invocations read it via GITHUB_CONTEXT_INPUT. | |
| # | |
| # Zizmor flags this as template-injection because attacker-controllable fields in the | |
| # github context (PR title, body, branch names) are pasted into a `run:` script. It is | |
| # a false positive in this specific shape: | |
| # * the substituted JSON sits inside a SINGLE-QUOTED heredoc (`<< '__GITHUB_CONTEXT_END__'`), | |
| # which disables ALL bash expansion -- $(...), `...`, ${...}, and even `\` are literal, | |
| # so no shell metacharacter in the JSON can be evaluated; | |
| # * the only remaining escape is the delimiter `__GITHUB_CONTEXT_END__` appearing on a | |
| # line by itself in the JSON. `toJson` escapes newlines inside string values as `\n` | |
| # (so they are NOT real newlines in the script source), and the delimiter is unique | |
| # enough that github will never produce it as a structural line of the JSON output; | |
| # * `cat` writes the bytes verbatim to a file -- it is not eval'd. | |
| # Whatever junk a malicious PR puts in title/body therefore lands inside a JSON file | |
| # consumed by breeze, never executed as shell. | |
| shell: bash | |
| run: | # zizmor: ignore[template-injection] | |
| cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' | |
| ${{ toJson(github) }} | |
| __GITHUB_CONTEXT_END__ | |
| - name: "Get information about the Workflow" | |
| id: source-run-info | |
| run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} | |
| env: | |
| SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" | |
| GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" | |
| COMMIT_REF: "${{ github.sha }}" | |
| VERBOSE: "false" | |
| GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" | |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
| - name: env | |
| run: printenv | |
| env: | |
| PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} | |
| print-platform: | |
| name: "Platform: AMD" | |
| needs: [build-info] | |
| runs-on: ["ubuntu-22.04"] | |
| steps: | |
| - name: "Print architecture" | |
| run: "echo '## Architecture: AMD' >> $GITHUB_STEP_SUMMARY" | |
| basic-tests: | |
| name: "Basic tests" | |
| needs: [build-info] | |
| uses: ./.github/workflows/basic-tests.yml | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} | |
| run-www-tests: ${{needs.build-info.outputs.run-www-tests}} | |
| run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} | |
| skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} | |
| canary-run: ${{needs.build-info.outputs.canary-run}} | |
| run-breeze-integration-tests: ${{needs.build-info.outputs.run-breeze-integration-tests}} | |
| run-scripts-tests: ${{needs.build-info.outputs.run-scripts-tests}} | |
| latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} | |
| use-uv: ${{needs.build-info.outputs.use-uv}} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}} | |
| build-ci-images: | |
| name: Build CI images | |
| needs: [build-info] | |
| uses: ./.github/workflows/ci-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| packages: write | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| push-image: "false" | |
| upload-image-artifact: "true" | |
| upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
| disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| additional-ci-image-checks: | |
| name: "Additional CI image checks" | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/additional-ci-image-checks.yml | |
| permissions: | |
| contents: read | |
| packages: write | |
| id-token: write | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} | |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
| disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| generate-constraints: | |
| name: "Generate constraints" | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/generate-constraints.yml | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| generate-pypi-constraints: "true" | |
| # generate no providers constraints only in canary builds - they take quite some time to generate | |
| # they are not needed for regular builds, they are only needed to update constraints in canaries | |
| generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| ci-image-checks: | |
| name: "CI image checks" | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/ci-image-checks.yml | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} | |
| latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} | |
| basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} | |
| ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| docs-build: ${{ needs.build-info.outputs.docs-build }} | |
| run-api-codegen: ${{ needs.build-info.outputs.run-api-codegen }} | |
| default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| source-head-repo: ${{ needs.build-info.outputs.source-head-repo }} | |
| source-head-ref: ${{ needs.build-info.outputs.source-head-ref }} | |
| secrets: | |
| DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} | |
| DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| mypy-providers: | |
| timeout-minutes: 45 | |
| name: "MyPy providers checks" | |
| needs: [build-info, build-ci-images] | |
| runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} | |
| if: needs.build-info.outputs.run-mypy-providers == 'true' | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: sudo rm -rf ${GITHUB_WORKSPACE}/* | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| with: | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python: "${{ needs.build-info.outputs.default-python-version }}" | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| make-mnt-writeable-and-cleanup: true | |
| id: breeze | |
| - name: "Install prek" | |
| uses: ./.github/actions/install-prek | |
| id: prek | |
| with: | |
| python-version: ${{steps.breeze.outputs.host-python-version}} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| save-cache: false | |
| - name: "MyPy checks for providers" | |
| run: prek --color always --verbose --stage manual mypy-providers --all-files | |
| env: | |
| VERBOSE: "false" | |
| COLUMNS: "202" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| RUFF_FORMAT: "github" | |
| INCLUDE_MYPY_VOLUME: "false" | |
| migration-round-trip: | |
| timeout-minutes: 20 | |
| name: "Migration round-trip check" | |
| needs: [build-info, build-ci-images] | |
| runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} | |
| if: needs.build-info.outputs.has-migrations == 'true' | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: sudo rm -rf ${GITHUB_WORKSPACE}/* | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| with: | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python: "${{ needs.build-info.outputs.default-python-version }}" | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| make-mnt-writeable-and-cleanup: true | |
| id: breeze | |
| - name: "Install prek" | |
| uses: ./.github/actions/install-prek | |
| id: prek | |
| with: | |
| python-version: ${{steps.breeze.outputs.host-python-version}} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| save-cache: false | |
| - name: "Migration round-trip check" | |
| run: prek --color always --verbose --stage manual migration-round-trip --all-files | |
| env: | |
| VERBOSE: "false" | |
| COLUMNS: "202" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| providers: | |
| name: "provider distributions tests" | |
| uses: ./.github/workflows/test-providers.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: > | |
| needs.build-info.outputs.skip-providers-tests != 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' && | |
| needs.build-info.outputs.run-unit-tests == 'true' | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} | |
| # yamllint disable rule:line-length | |
| providers-compatibility-tests-matrix: > | |
| ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| providers-test-types-list-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| tests-helm: | |
| name: "Helm tests" | |
| uses: ./.github/workflows/helm-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} | |
| helm-test-kubernetes-versions: ${{ needs.build-info.outputs.helm-test-kubernetes-versions }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| if: > | |
| needs.build-info.outputs.run-helm-tests == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| tests-kustomize-overlays: | |
| name: "Kustomize overlays smoke tests" | |
| uses: ./.github/workflows/kustomize-overlays-tests.yml | |
| needs: [build-info, build-prod-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| overlays: ${{ needs.build-info.outputs.kustomize-overlay-names }} | |
| # Gated by run-kustomize-overlays-tests (computed by SelectiveChecks | |
| # from changes under chart/kustomize-overlays/ and the narrow set of | |
| # runner files), NOT by run-helm-tests — a values.yaml tweak should | |
| # not pull in a 30-40 min kind cluster spin-up for unrelated overlays. | |
| # AMD-only via the platform guard below: the smoke test is | |
| # CPU-architecture-neutral (kustomize render + apply + STATUS | |
| # verify-block) and the overlays' third-party images (e.g. | |
| # gcavalcante8808/krb5-server) are not published for arm64, so running it | |
| # on the ARM lane would only add cost and arch-specific flakes for no | |
| # extra signal. The job stays defined in both ci-arm.yml/ci-amd.yml (kept | |
| # in sync) and is skipped on ARM through build-info.platform, mirroring | |
| # the tests-unit-* jobs. | |
| if: > | |
| needs.build-info.outputs.run-kustomize-overlays-tests == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' && | |
| needs.build-info.outputs.latest-versions-only != 'true' && | |
| needs.build-info.outputs.kustomize-overlay-names != '[]' && | |
| needs.build-info.outputs.platform == 'linux/amd64' | |
| tests-postgres-core: | |
| name: "Postgres tests: core" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "postgres" | |
| test-name: "Postgres" | |
| test-scope: "DB" | |
| test-group: "core" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: ${{ needs.build-info.outputs.postgres-versions }} | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.postgres-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-migration-tests: "true" | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-postgres-providers: | |
| name: "Postgres tests: providers" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "postgres" | |
| test-name: "Postgres" | |
| test-scope: "DB" | |
| test-group: "providers" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: ${{ needs.build-info.outputs.postgres-versions }} | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.postgres-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-migration-tests: "true" | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-mysql-core: | |
| name: "MySQL tests: core" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "mysql" | |
| test-name: "MySQL" | |
| test-scope: "DB" | |
| test-group: "core" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: ${{ needs.build-info.outputs.mysql-versions }} | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.mysql-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| run-migration-tests: "true" | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64' | |
| tests-mysql-providers: | |
| name: "MySQL tests: providers" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "mysql" | |
| test-name: "MySQL" | |
| test-scope: "DB" | |
| test-group: "providers" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: ${{ needs.build-info.outputs.mysql-versions }} | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.mysql-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| run-migration-tests: "true" | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64' | |
| tests-sqlite-core: | |
| name: "Sqlite tests: core" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "sqlite" | |
| test-name: "Sqlite" | |
| test-name-separator: "" | |
| test-scope: "DB" | |
| test-group: "core" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| # No versions for sqlite | |
| backend-versions: "['']" | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| run-migration-tests: "true" | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-sqlite-providers: | |
| name: "Sqlite tests: providers" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "sqlite" | |
| test-name: "Sqlite" | |
| test-name-separator: "" | |
| test-scope: "DB" | |
| test-group: "providers" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| # No versions for sqlite | |
| backend-versions: "['']" | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| run-migration-tests: "true" | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-non-db-core: | |
| name: "Non-DB tests: core" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "sqlite" | |
| test-name: "" | |
| test-name-separator: "" | |
| test-scope: "Non-DB" | |
| test-group: "core" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| # No versions for non-db | |
| backend-versions: "['']" | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-non-db-providers: | |
| name: "Non-DB tests: providers" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| backend: "sqlite" | |
| test-name: "" | |
| test-name-separator: "" | |
| test-scope: "Non-DB" | |
| test-group: "providers" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| # No versions for non-db | |
| backend-versions: "['']" | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-special: | |
| name: "Special tests" | |
| uses: ./.github/workflows/special-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: > | |
| needs.build-info.outputs.run-unit-tests == 'true' && | |
| (needs.build-info.outputs.canary-run == 'true' || | |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || | |
| needs.build-info.outputs.full-tests-needed == 'true') | |
| with: | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| core-test-types-list-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} | |
| providers-test-types-list-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| tests-integration-system: | |
| name: Integration and System Tests | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/integration-system-tests.yml | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} | |
| testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} | |
| run-system-tests: ${{ needs.build-info.outputs.run-system-tests }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
| default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| tests-with-lowest-direct-resolution-core: | |
| name: "Low dep tests:core" | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: > | |
| needs.build-info.outputs.run-unit-tests == 'true' | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| test-name: "LowestDeps" | |
| force-lowest-dependencies: "true" | |
| test-scope: "All" | |
| test-group: "core" | |
| backend: "sqlite" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" | |
| excluded-providers-as-string: "" | |
| excludes: "[]" | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| monitor-delay-time-in-seconds: 120 | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| tests-with-lowest-direct-resolution-providers: | |
| name: "Low dep tests: providers" | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| test-name: "LowestDeps" | |
| force-lowest-dependencies: "true" | |
| test-scope: "All" | |
| test-group: "providers" | |
| backend: "sqlite" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" | |
| excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
| excludes: "[]" | |
| test-types-as-strings-in-json: > | |
| ${{ needs.build-info.outputs.individual-providers-test-types-list-as-strings-in-json }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| monitor-delay-time-in-seconds: 120 | |
| skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| build-prod-images: | |
| name: Build PROD images | |
| needs: [build-info, build-ci-images, generate-constraints] | |
| uses: ./.github/workflows/prod-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| packages: write | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| build-type: "Regular" | |
| push-image: "false" | |
| upload-image-artifact: "true" | |
| upload-package-artifact: "true" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
| disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
| prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} | |
| additional-prod-image-tests: | |
| name: "Additional PROD image tests" | |
| needs: [build-info, build-prod-images, generate-constraints] | |
| uses: ./.github/workflows/additional-prod-image-tests.yml | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| default-branch: ${{ needs.build-info.outputs.default-branch }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
| disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| run-task-sdk-integration-tests: ${{ needs.build-info.outputs.run-task-sdk-integration-tests }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| run-remote-logging-elasticsearch-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }} | |
| run-remote-logging-opensearch-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-opensearch-e2e-tests }} | |
| run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }} | |
| run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }} | |
| run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} | |
| run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} | |
| run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| tests-kubernetes: | |
| name: "Kubernetes tests" | |
| uses: ./.github/workflows/k8s-tests.yml | |
| needs: [build-info, build-prod-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} | |
| if: > | |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || | |
| needs.build-info.outputs.run-helm-tests == 'true') | |
| tests-task-sdk: | |
| name: "Task SDK tests" | |
| uses: ./.github/workflows/airflow-distributions-tests.yml | |
| needs: [build-info, build-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| distribution-name: "task-sdk" | |
| distribution-cmd-format: "prepare-task-sdk-distributions" | |
| test-type: "task-sdk-tests" | |
| use-local-venv: 'false' | |
| test-timeout: 20 | |
| if: needs.build-info.outputs.run-task-sdk-tests == 'true' | |
| tests-go-sdk: | |
| name: "Go SDK tests" | |
| needs: [build-info] | |
| runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: needs.build-info.outputs.run-go-sdk-tests == 'true' | |
| env: | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| VERBOSE: "true" | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| # keep this in sync with go.mod in go-sdk/ | |
| - name: Setup Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version: 1.25 | |
| cache-dependency-path: go-sdk/go.sum | |
| # keep this in sync with go.mod in go-sdk/ | |
| - name: Setup Gotestsum | |
| shell: bash | |
| run: | | |
| go install gotest.tools/gotestsum@c4a0df2e75a225d979a444342dd3db752b53619f # v1.13.0 | |
| gotestsum --version | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: Run Go tests | |
| working-directory: ./go-sdk | |
| run: gotestsum --format github-actions ./... | |
| tests-java-sdk: | |
| name: "Java SDK tests" | |
| needs: [build-info] | |
| runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: needs.build-info.outputs.run-java-sdk-tests == 'true' | |
| env: | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| VERBOSE: "true" | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Setup Java | |
| uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: Run Java SDK tests | |
| working-directory: ./java-sdk | |
| run: ./gradlew test | |
| build-java-sdk-docs: | |
| name: "Java SDK docs" | |
| needs: [build-info] | |
| runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| packages: read | |
| if: needs.build-info.outputs.run-java-sdk-tests == 'true' | |
| env: | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| VERBOSE: "true" | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Build Java SDK Javadoc" | |
| run: breeze build-docs --sdk-docs-only --sdk=java | |
| - name: "Upload Javadoc artifact" | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: java-sdk-docs | |
| path: generated/_build/docs/java-sdk/ | |
| retention-days: 7 | |
| if-no-files-found: error | |
| overwrite: 'true' | |
| tests-airflow-ctl: | |
| name: "Airflow CTL tests" | |
| uses: ./.github/workflows/airflow-distributions-tests.yml | |
| needs: [build-info] | |
| permissions: | |
| contents: read | |
| packages: read | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| distribution-name: "airflow-ctl" | |
| distribution-cmd-format: "prepare-airflow-ctl-distributions" | |
| test-type: "airflow-ctl-tests" | |
| use-local-venv: 'true' | |
| test-timeout: 20 | |
| if: needs.build-info.outputs.run-airflow-ctl-tests == 'true' | |
| finalize-tests: | |
| name: Finalize tests | |
| permissions: | |
| contents: write | |
| packages: write | |
| # This will fire when all the jobs from "needs" are either successful or skipped | |
| if: always() && !failure() && !cancelled() | |
| needs: | |
| - additional-ci-image-checks | |
| - additional-prod-image-tests | |
| - basic-tests | |
| - build-info | |
| - build-prod-images | |
| - ci-image-checks | |
| - generate-constraints | |
| - migration-round-trip | |
| - mypy-providers | |
| - providers | |
| - tests-helm | |
| - tests-integration-system | |
| - tests-kubernetes | |
| - tests-mysql-core | |
| - tests-mysql-providers | |
| - tests-non-db-core | |
| - tests-non-db-providers | |
| - tests-postgres-core | |
| - tests-postgres-providers | |
| - tests-sqlite-core | |
| - tests-sqlite-providers | |
| - tests-task-sdk | |
| - tests-airflow-ctl | |
| - tests-go-sdk | |
| - tests-java-sdk | |
| - build-java-sdk-docs | |
| - tests-with-lowest-direct-resolution-core | |
| - tests-with-lowest-direct-resolution-providers | |
| uses: ./.github/workflows/finalize-tests.yml | |
| with: | |
| runners: ${{ needs.build-info.outputs.runner-type }} | |
| platform: ${{ needs.build-info.outputs.platform }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| default-python-version: "${{ needs.build-info.outputs.default-python-version }}" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
| disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
| canary-run: ${{ needs.build-info.outputs.canary-run }} | |
| use-uv: ${{ needs.build-info.outputs.use-uv }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| notify-slack: | |
| name: "Notify Slack" | |
| needs: | |
| - build-info | |
| - finalize-tests | |
| if: >- | |
| always() && | |
| !cancelled() && | |
| github.event_name == 'schedule' && | |
| github.run_attempt == 1 | |
| runs-on: ["ubuntu-22.04"] | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: "Get failing jobs" | |
| id: get-failures | |
| shell: bash | |
| run: | | |
| FAILED_JOBS=$(gh run view "${{ github.run_id }}" \ | |
| --repo "${{ github.repository }}" \ | |
| --json jobs \ | |
| --jq '[.jobs[] | select(.conclusion == "failure") | .name] | sort | .[]') | |
| echo "failed-jobs<<EOF" >> "${GITHUB_OUTPUT}" | |
| echo "${FAILED_JOBS}" >> "${GITHUB_OUTPUT}" | |
| echo "EOF" >> "${GITHUB_OUTPUT}" | |
| if [[ -n "${FAILED_JOBS}" ]]; then | |
| echo "has-failures=true" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "has-failures=false" >> "${GITHUB_OUTPUT}" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: "Determine notification action" | |
| id: notification | |
| shell: bash | |
| run: | | |
| command -v gh >/dev/null 2>&1 || { echo "::error::gh CLI is not installed"; exit 1; } | |
| gh auth status || gh auth login --with-token <<< "${GITHUB_TOKEN}" | |
| python3 scripts/ci/slack_notification_state.py | |
| env: | |
| ARTIFACT_NAME: "slack-state-tests-${{ github.ref_name }}-amd" | |
| CURRENT_FAILURES: "${{ steps.get-failures.outputs.failed-jobs }}" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: "Upload notification state" | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "slack-state-tests-${{ github.ref_name }}-amd" | |
| path: ./slack-state/ | |
| retention-days: 7 | |
| overwrite: true | |
| - name: "Notify Slack (new/changed failures)" | |
| if: steps.notification.outputs.action == 'notify_new' | |
| uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 | |
| with: | |
| method: chat.postMessage | |
| token: ${{ env.SLACK_BOT_TOKEN }} | |
| # yamllint disable rule:line-length | |
| payload: | | |
| channel: "internal-airflow-ci-cd" | |
| text: "🚨 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "🚨 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>" | |
| # yamllint enable rule:line-length | |
| - name: "Notify Slack (still not fixed)" | |
| if: steps.notification.outputs.action == 'notify_reminder' | |
| uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 | |
| with: | |
| method: chat.postMessage | |
| token: ${{ env.SLACK_BOT_TOKEN }} | |
| # yamllint disable rule:line-length | |
| payload: | | |
| channel: "internal-airflow-ci-cd" | |
| text: "🚨🔁 Still not fixed: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "🚨🔁 Still not fixed: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>" | |
| # yamllint enable rule:line-length | |
| - name: "Notify Slack (all tests passing)" | |
| if: steps.notification.outputs.action == 'notify_recovery' | |
| uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 | |
| with: | |
| method: chat.postMessage | |
| token: ${{ env.SLACK_BOT_TOKEN }} | |
| # yamllint disable rule:line-length | |
| payload: | | |
| channel: "internal-airflow-ci-cd" | |
| text: "✅ All tests passing: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the run log>" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "✅ All tests passing: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the run log>" | |
| # yamllint enable rule:line-length | |
| summarize-warnings: | |
| timeout-minutes: 15 | |
| name: "Summarize warnings" | |
| needs: | |
| - build-info | |
| - tests-mysql-core | |
| - tests-mysql-providers | |
| - tests-non-db-core | |
| - tests-non-db-providers | |
| - tests-postgres-core | |
| - tests-postgres-providers | |
| - tests-sqlite-core | |
| - tests-sqlite-providers | |
| - tests-task-sdk | |
| - tests-airflow-ctl | |
| - tests-special | |
| - tests-with-lowest-direct-resolution-core | |
| - tests-with-lowest-direct-resolution-providers | |
| runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} | |
| if: needs.build-info.outputs.run-unit-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: sudo rm -rf ${GITHUB_WORKSPACE}/* | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: "Free up disk space" | |
| shell: bash | |
| run: ./scripts/tools/free_up_disk_space.sh | |
| - name: "Download all test warning artifacts from the current build" | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ./artifacts | |
| pattern: test-warnings-* | |
| - name: "Setup python" | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: "${{ inputs.default-python-version }}" | |
| - name: "Summarize all warnings" | |
| run: | | |
| ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ | |
| --pattern "**/warnings-*.txt" \ | |
| --output ./files | |
| - name: "Upload artifact for summarized warnings" | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "test-summarized-warnings" | |
| path: ./files/warn-summary-*.txt | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| overwrite: true |