Optimize CI: consolidate workflows, fix caching, speed up e2e tests (47min → 15min) #248
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests and Code Coverage | |
| permissions: | |
| contents: read | |
| id-token: write | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| test-with-coverage: | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| DATABRICKS_CATALOG: peco | |
| DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libkrb5-dev | |
| - name: Setup Poetry | |
| uses: ./.github/actions/setup-poetry | |
| with: | |
| python-version: "3.10" | |
| install-args: "--all-extras" | |
| - name: Run all tests with coverage | |
| continue-on-error: false | |
| run: | | |
| poetry run pytest tests/unit tests/e2e \ | |
| -n 4 \ | |
| --dist=loadgroup \ | |
| --cov=src \ | |
| --cov-report=xml \ | |
| --cov-report=term \ | |
| -v | |
| - name: Check for coverage override | |
| id: override | |
| env: | |
| PR_BODY: ${{ github.event.pull_request.body }} | |
| run: | | |
| OVERRIDE_COMMENT=$(echo "$PR_BODY" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "") | |
| if [ -n "$OVERRIDE_COMMENT" ]; then | |
| echo "override=true" >> $GITHUB_OUTPUT | |
| REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/') | |
| echo "reason=$REASON" >> $GITHUB_OUTPUT | |
| echo "Coverage override found in PR description: $REASON" | |
| else | |
| echo "override=false" >> $GITHUB_OUTPUT | |
| echo "No coverage override found" | |
| fi | |
| - name: Check coverage percentage | |
| if: steps.override.outputs.override == 'false' | |
| run: | | |
| COVERAGE_FILE="coverage.xml" | |
| if [ ! -f "$COVERAGE_FILE" ]; then | |
| echo "ERROR: Coverage file not found at $COVERAGE_FILE" | |
| exit 1 | |
| fi | |
| if ! command -v xmllint &> /dev/null; then | |
| sudo apt-get update && sudo apt-get install -y libxml2-utils | |
| fi | |
| COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE") | |
| TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE") | |
| PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))") | |
| echo "Branch Coverage: $PERCENTAGE%" | |
| echo "Required Coverage: 85%" | |
| python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)" | |
| if [ $? -eq 1 ]; then | |
| echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%" | |
| exit 1 | |
| else | |
| echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%" | |
| fi | |
| - name: Coverage enforcement summary | |
| env: | |
| OVERRIDE: ${{ steps.override.outputs.override }} | |
| REASON: ${{ steps.override.outputs.reason }} | |
| run: | | |
| if [ "$OVERRIDE" == "true" ]; then | |
| echo "Coverage checks bypassed: $REASON" | |
| echo "Please ensure this override is justified and temporary" | |
| else | |
| echo "Coverage checks enforced - minimum 85% required" | |
| fi |