py-tests-postgres.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: py-tests-postgres
on:
  merge_group:
  workflow_dispatch:
  pull_request_target:
    types: [labeled, opened, synchronize, reopened, ready_for_review]

permissions:
  contents: read
  pull-requests: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}

jobs:
  # Authorize pull request code once before the matrix fans out, then detect
  # whether relevant paths changed.
  changes:
    name: Authorize & Detect Changes
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
    outputs:
      python: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.python }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        if: ${{ github.event_name == 'merge_group' }}
        with:
          fetch-depth: 0
          filter: blob:none
          persist-credentials: false
      - uses: dorny/paths-filter@v4
        id: filter
        if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'merge_group' }}
        with:
          base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }}
          filters: |
            python:
              - 'ingestion/**'
              - 'openmetadata-spec/**'
              - 'openmetadata-service/src/main/java/org/openmetadata/service/resources/**'
              - 'openmetadata-service/src/main/java/org/openmetadata/service/JsonPatchMessageBodyReader.java'
              - 'openmetadata-service/src/main/java/org/openmetadata/service/JsonPatchProvider.java'
              - 'openmetadata-service/src/main/java/org/openmetadata/service/ResourceRegistry.java'

      - name: Wait for the labeler
        uses: lewagon/wait-on-check-action@v1.7.0
        if: ${{ github.event_name == 'pull_request_target' && steps.filter.outputs.python == 'true' }}
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          check-name: Team Label
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          wait-interval: 30

      - name: Verify PR labels
        uses: jesusvasquez333/verify-pr-label-action@v1.4.0
        if: ${{ github.event_name == 'pull_request_target' && steps.filter.outputs.python == 'true' }}
        with:
          github-token: "${{ secrets.GITHUB_TOKEN }}"
          valid-labels: "safe to test"
          pull-request-number: "${{ github.event.pull_request.number }}"
          disable-reviews: true

  py-integration-tests:
    name: "Integration Tests (${{ matrix.shard.name }}, ${{ matrix.py-version }})"
    needs: changes
    if: ${{ needs.changes.outputs.python == 'true' }}
    timeout-minutes: 180
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        py-version: ["3.10", "3.11", "3.12"]
        shard:
          - name: "shard-1"
            nox-args: >-
              tests/integration/ometa
              tests/integration/postgres
              tests/integration/mysql
              tests/integration/profiler
              tests/integration/data_quality
              tests/integration/sql_server
          - name: "shard-2"
            nox-args: >-
              --ignore=tests/integration/ometa
              --ignore=tests/integration/postgres
              --ignore=tests/integration/mysql
              --ignore=tests/integration/profiler
              --ignore=tests/integration/data_quality
              --ignore=tests/integration/sql_server
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          swap-storage: true
          docker-images: false

      - name: Checkout
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
          allow-unsafe-pr-checkout: true
          persist-credentials: false
      - name: Use runner data disk for Docker
        if: matrix.shard.name == 'shard-2'
        run: bash .github/scripts/configure_docker_storage.sh

      - name: Setup Openmetadata Test Environment
        uses: ./.github/actions/setup-openmetadata-test-environment
        with:
          python-version: ${{ matrix.py-version}}
          args: "-m no-ui -d postgresql"
          ingestion_dependency: "mysql,elasticsearch,sample-data"

      - name: Reclaim unused disk for the Exasol shard
        if: matrix.shard.name == 'shard-2'
        continue-on-error: true
        run: |
          {
            echo "Before Exasol shard disk reclaim"
            df -h
            docker system df
            sudo rm -rf /opt/hostedtoolcache/CodeQL
            docker image prune --all --force || true
            docker builder prune --all --force || true
            echo "After Exasol shard disk reclaim"
            df -h
            docker system df
          } 2>&1 | tee "${RUNNER_TEMP}/exasol-disk-reclaim.log"
        shell: bash

      - name: Run Integration Tests
        id: integration-tests
        run: |
          set -o pipefail
          source env/bin/activate
          cd ingestion
          read -r -a shard_args <<< "$NOX_ARGS"
          nox --no-venv -s integration-tests -- --standalone --durations=5 "${shard_args[@]}" 2>&1 | tee integration-test-run.log
        env:
          NOX_ARGS: ${{ matrix.shard.nox-args }}
          TESTCONTAINERS_RYUK_DISABLED: true
        shell: bash

      - name: Classify PostgreSQL Python integration test outcome
        if: ${{ always() }}
        env:
          TEST_LANE: ${{ matrix.shard.name }}-python-${{ matrix.py-version }}
          TEST_PROFILE: python-postgresql-integration
          TEST_STEP_OUTCOME: ${{ steps.integration-tests.outcome }}
        run: |
          python3 .github/scripts/classify_test_outcome.py \
            --step-outcome "$TEST_STEP_OUTCOME" \
            --report-glob 'ingestion/junit/test-results-*.xml' \
            --diagnostic-glob 'ingestion/integration-test-run.log' \
            --output ingestion/ci-diagnostics/outcome.json

      - name: Upload test results
        if: ${{ always() }}
        continue-on-error: true
        uses: actions/upload-artifact@v6
        with:
          name: py-postgres-integration-test-results-${{ matrix.shard.name }}-${{ matrix.py-version }}
          path: |
            ingestion/junit/
            ingestion/ci-diagnostics/outcome.json
            ingestion/integration-test-run.log
          if-no-files-found: warn
          retention-days: 14
          compression-level: 1

      - name: Collect runner and service diagnostics
        if: ${{ failure() || cancelled() }}
        continue-on-error: true
        run: |
          diagnostics="${RUNNER_TEMP}/python-ci-diagnostics"
          rm -rf "$diagnostics"
          mkdir -p "$diagnostics"
          df -h > "$diagnostics/disk-usage.txt" 2>&1 || true
          free -h > "$diagnostics/memory-usage.txt" 2>&1 || true
          docker info > "$diagnostics/docker-info.txt" 2>&1 || true
          docker system df > "$diagnostics/docker-disk-usage.txt" 2>&1 || true
          docker ps --all --no-trunc > "$diagnostics/docker-containers.txt" 2>&1 || true
          if [ -d docker/development ]; then
            (cd docker/development && docker compose ps --all) > "$diagnostics/docker-compose-ps.txt" 2>&1 || true
            (cd docker/development && docker compose logs --no-color --tail 2000) > "$diagnostics/docker-compose.log" 2>&1 || true
          fi
          coredumpctl --no-pager list > "$diagnostics/coredumps.txt" 2>&1 || true
          coredumpctl --no-pager info >> "$diagnostics/coredumps.txt" 2>&1 || true
          find "$GITHUB_WORKSPACE" /tmp /var/crash -maxdepth 3 -type f \
            \( -name 'core' -o -name 'core.*' -o -name '*.core' -o -name '*.dmp' -o -name '*.crash' \) \
            -printf '%p %s bytes\n' >> "$diagnostics/core-files.txt" 2>/dev/null || true
          if [ -f "${RUNNER_TEMP}/exasol-disk-reclaim.log" ]; then
            cp "${RUNNER_TEMP}/exasol-disk-reclaim.log" "$diagnostics/"
          fi
        shell: bash

      - name: Upload runner and service diagnostics
        if: ${{ failure() || cancelled() }}
        continue-on-error: true
        uses: actions/upload-artifact@v6
        with:
          name: py-postgres-integration-diagnostics-${{ matrix.shard.name }}-${{ matrix.py-version }}
          path: ${{ runner.temp }}/python-ci-diagnostics/
          if-no-files-found: warn
          retention-days: 14
          compression-level: 1

      - name: Clean Up
        if: ${{ always() }}
        continue-on-error: true
        run: |
          cd ./docker/development
          docker compose down --remove-orphans || true
          sudo rm -rf "${PWD}/docker-volume"

  # Keep the existing required context until the repository ruleset is migrated atomically.
  py-tests-status:
    name: ${{ github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name != 'safe to test' && 'py-tests-status (ignored label)' || 'py-tests-status' }}
    needs: [changes, py-integration-tests]
    if: ${{ always() && !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
    runs-on: ubuntu-latest
    steps:
      - name: Verify all expected PostgreSQL Python jobs completed
        env:
          CHANGES_RESULT: ${{ needs.changes.result }}
          PYTHON_CHANGED: ${{ needs.changes.outputs.python }}
          INTEGRATION_RESULT: ${{ needs.py-integration-tests.result }}
        run: |
          if [ "$CHANGES_RESULT" != "success" ]; then
            echo "Python preflight did not succeed: $CHANGES_RESULT"
            exit 1
          fi

          if [ "$PYTHON_CHANGED" = "true" ] && [ "$INTEGRATION_RESULT" != "success" ]; then
            echo "Expected PostgreSQL Python jobs did not succeed: integration=$INTEGRATION_RESULT"
            exit 1
          fi

          if [ "$PYTHON_CHANGED" = "false" ] && [ "$INTEGRATION_RESULT" != "skipped" ]; then
            echo "Unexpected PostgreSQL Python job result for an unchanged tree: integration=$INTEGRATION_RESULT"
            exit 1
          fi

          if [ "$PYTHON_CHANGED" != "true" ] && [ "$PYTHON_CHANGED" != "false" ]; then
            echo "Python change detection produced an invalid result: $PYTHON_CHANGED"
            exit 1
          fi
        shell: bash