perf(aicontext): strip render-only markup from context documents #18467
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2026 Collate | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # Runs the full integration test suite with the Redis cache enabled (postgres + elasticsearch + | |
| # redis), via the cache-tests Maven profile. Catches cache-invalidation and stale-data bugs that | |
| # only surface when every test path goes through the cache layer. | |
| # | |
| # Security note (CodeQL "pull_request_target + checkout untrusted code"): | |
| # This workflow uses `pull_request_target` so PRs from forks can produce a required check. | |
| # CodeQL flags the pattern as risky because it checks out PR-controlled code while having | |
| # access to secrets. The mitigation is the explicit `safe to test` label gate below — the | |
| # verify-pr-label step rejects the workflow run before any PR code is checked out unless a | |
| # maintainer has applied the label. This matches the mitigation used by every other | |
| # integration-tests-*.yml workflow in this repo. If you remove the label gate, you reopen | |
| # the vulnerability. | |
| name: Integration Tests - PostgreSQL + Elasticsearch + Redis | |
| on: | |
| merge_group: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "openmetadata-service/**" | |
| - "openmetadata-integration-tests/**" | |
| - "openmetadata-spec/pom.xml" | |
| - "openmetadata-spec/src/**" | |
| - "openmetadata-sdk/**" | |
| - "openmetadata-shaded-deps/**" | |
| - "openmetadata-k8s-operator/**" | |
| - "openmetadata-mcp/**" | |
| - "common/**" | |
| - "pom.xml" | |
| - "Makefile" | |
| - "bootstrap/**" | |
| - ".github/scripts/classify_test_outcome.py" | |
| - ".github/workflows/integration-tests-postgres-elasticsearch-redis.yml" | |
| # `pull_request_target` is intentional and required so the workflow runs against PRs from | |
| # forks (which `pull_request` cannot for security reasons). The `safe to test` label gate | |
| # below is what makes this safe — see security note in the file header. | |
| pull_request_target: | |
| types: [labeled, opened, synchronize, reopened, ready_for_review] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| checks: write | |
| concurrency: | |
| group: integration-tests-pg-es-redis-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }} | |
| jobs: | |
| # Detect whether relevant paths changed. When no matching files are modified | |
| # the downstream job is skipped via its `if` condition. | |
| # A job skipped by `if` reports as "Success", so required checks still pass. | |
| changes: | |
| name: Detect Changes | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }} | |
| # pull_request_target uses the GitHub API and does not require a checkout. Merge | |
| # queue runs check out the merge-queue commit so dorny can diff against | |
| # merge_group.base_sha and skip the heavy work when nothing backend-relevant | |
| # changed. Push and workflow_dispatch remain force-true — push is already | |
| # gated by the trigger paths above, and workflow_dispatch is an explicit | |
| # "run the full suite" request. Keep the trigger paths and filter list in sync. | |
| outputs: | |
| backend: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'true' || steps.filter.outputs.backend }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| if: ${{ github.event_name == 'merge_group' }} | |
| with: | |
| fetch-depth: 0 | |
| filter: blob:none | |
| persist-credentials: false | |
| - uses: dorny/paths-filter@v4 | |
| id: filter | |
| if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'merge_group' }} | |
| with: | |
| base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }} | |
| filters: | | |
| backend: | |
| - 'openmetadata-service/**' | |
| - 'openmetadata-integration-tests/**' | |
| - 'openmetadata-spec/pom.xml' | |
| - 'openmetadata-spec/src/**' | |
| - 'openmetadata-sdk/**' | |
| - 'openmetadata-shaded-deps/**' | |
| - 'openmetadata-k8s-operator/**' | |
| - 'openmetadata-mcp/**' | |
| - 'common/**' | |
| - 'pom.xml' | |
| - 'Makefile' | |
| - 'bootstrap/**' | |
| - '.github/scripts/classify_test_outcome.py' | |
| - '.github/workflows/integration-tests-postgres-elasticsearch-redis.yml' | |
| integration-test-build: | |
| name: Build Integration Test Runtime | |
| needs: changes | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| if: ${{ needs.changes.outputs.backend == 'true' }} | |
| steps: | |
| - name: Wait for the labeler | |
| uses: lewagon/wait-on-check-action@v1.7.0 | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| check-name: Team Label | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| wait-interval: 90 | |
| - name: Verify PR labels | |
| uses: jesusvasquez333/verify-pr-label-action@v1.4.0 | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| with: | |
| github-token: '${{ secrets.GITHUB_TOKEN }}' | |
| valid-labels: 'safe to test' | |
| pull-request-number: '${{ github.event.pull_request.number }}' | |
| disable-reviews: true | |
| # SECURITY: this step checks out PR-controlled code while the workflow runs with | |
| # `pull_request_target` privileges. The label verification above must succeed first. | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }} | |
| allow-unsafe-pr-checkout: true | |
| persist-credentials: false | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@v5 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| - name: Build and Bundle Integration Test Runtime | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SOURCE_SHA: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }} | |
| run: | | |
| project_version="$(mvn --quiet --non-recursive help:evaluate \ | |
| -Dexpression=project.version -DforceStdout -Dstyle.color=never)" | |
| if [[ ! "$project_version" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]*$ ]]; then | |
| echo "Invalid Maven project version: $project_version" | |
| exit 1 | |
| fi | |
| project_repository="$HOME/.m2/repository/org/open-metadata" | |
| if [[ -d "$project_repository" ]]; then | |
| find "$project_repository" -mindepth 2 -maxdepth 2 -type d \ | |
| -name "$project_version" -prune -exec rm -rf -- {} + | |
| fi | |
| mvn -DskipTests -DskipITs -Dspring-boot.repackage.skip=true \ | |
| clean install -pl :openmetadata-integration-tests -am | |
| staging="$RUNNER_TEMP/java-it-reactor-repository" | |
| bundle="$RUNNER_TEMP/java-it-build" | |
| mkdir -p "$staging/org/open-metadata" "$bundle" | |
| version_directory_count=0 | |
| while IFS= read -r -d '' version_directory; do | |
| artifact_id="$(basename "$(dirname "$version_directory")")" | |
| mkdir -p "$staging/org/open-metadata/$artifact_id" | |
| cp -a "$version_directory" "$staging/org/open-metadata/$artifact_id/" | |
| version_directory_count=$((version_directory_count + 1)) | |
| done < <(find "$project_repository" -mindepth 2 -maxdepth 2 -type d \ | |
| -name "$project_version" -print0) | |
| if (( version_directory_count == 0 )); then | |
| echo "The Maven build did not install any reactor artifacts." | |
| exit 1 | |
| fi | |
| find "$staging" -type f -name '*-boot.jar' -delete | |
| tar -C "$staging" -czf "$bundle/reactor-repository.tar.gz" org | |
| ( | |
| cd "$bundle" | |
| sha256sum reactor-repository.tar.gz > reactor-repository.sha256 | |
| ) | |
| printf '%s\n' "$project_version" > "$bundle/project-version.txt" | |
| printf '%s\n' "$SOURCE_SHA" > "$bundle/source-sha.txt" | |
| du -h "$bundle/reactor-repository.tar.gz" | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: Upload Integration Test Runtime | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: java-it-build-postgres-elasticsearch-redis-${{ github.run_id }} | |
| overwrite: true | |
| path: ${{ runner.temp }}/java-it-build | |
| if-no-files-found: error | |
| retention-days: 1 | |
| compression-level: 0 | |
| integration-test-lanes: | |
| name: Integration Test Lane (${{ matrix.lane.name }}) | |
| needs: [changes, integration-test-build] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| if: ${{ needs.changes.outputs.backend == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| lane: | |
| - name: parallel | |
| - name: global-state | |
| - name: multi-node | |
| - name: retry-queue | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: false | |
| swap-storage: true | |
| # The required build job verifies `safe to test` before this fan-out can run. | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }} | |
| allow-unsafe-pr-checkout: true | |
| persist-credentials: false | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@v5 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| - name: Install Ubuntu dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ | |
| libevent-dev jq | |
| sudo make install_antlr_cli | |
| - name: Download Integration Test Runtime | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: java-it-build-postgres-elasticsearch-redis-${{ github.run_id }} | |
| path: ${{ runner.temp }}/java-it-build | |
| - name: Restore Integration Test Runtime | |
| run: | | |
| bundle="$RUNNER_TEMP/java-it-build" | |
| ( | |
| cd "$bundle" | |
| sha256sum --check reactor-repository.sha256 | |
| ) | |
| project_version="$(<"$bundle/project-version.txt")" | |
| source_sha="$(<"$bundle/source-sha.txt")" | |
| if [[ ! "$project_version" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]*$ ]]; then | |
| echo "Invalid Maven project version: $project_version" | |
| exit 1 | |
| fi | |
| if [[ "$(git rev-parse HEAD)" != "$source_sha" ]]; then | |
| echo "The build artifact does not match the checked-out commit." | |
| exit 1 | |
| fi | |
| project_repository="$HOME/.m2/repository/org/open-metadata" | |
| if [[ -d "$project_repository" ]]; then | |
| find "$project_repository" -mindepth 2 -maxdepth 2 -type d \ | |
| -name "$project_version" -prune -exec rm -rf -- {} + | |
| fi | |
| mkdir -p "$HOME/.m2/repository" | |
| tar -C "$HOME/.m2/repository" -xzf "$bundle/reactor-repository.tar.gz" | |
| test -s "$project_repository/platform/$project_version/platform-$project_version.pom" | |
| test -s "$project_repository/openmetadata-service/$project_version/openmetadata-service-$project_version.jar" | |
| - name: Run Integration Tests (PostgreSQL + Elasticsearch + Redis) | |
| id: integration-tests | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IT_LANE: ${{ matrix.lane.name }} | |
| IT_PARALLELISM: '4' | |
| IT_MAX_POOL_SIZE: '16' | |
| run: | | |
| { | |
| echo "### Java integration configuration" | |
| echo "- Profile: cache-tests" | |
| echo "- Lane: ${IT_LANE}" | |
| echo "- Parallel workers: ${IT_PARALLELISM}" | |
| echo "- Compensation-thread ceiling: ${IT_MAX_POOL_SIZE}" | |
| echo "- Runner processors: $(nproc)" | |
| } | tee -a "$GITHUB_STEP_SUMMARY" | |
| diagnostics="openmetadata-integration-tests/target/ci-diagnostics" | |
| mkdir -p "$diagnostics" | |
| maven_log="$diagnostics/maven.log" | |
| set +e | |
| timeout --signal=TERM --kill-after=60s 65m \ | |
| mvn --no-snapshot-updates verify -pl :openmetadata-integration-tests -Pcache-tests \ | |
| -DintegrationTests.lane="${IT_LANE}" \ | |
| -DintegrationTests.parallelStrategy=fixed \ | |
| -DintegrationTests.parallelism="${IT_PARALLELISM}" \ | |
| -DintegrationTests.maxPoolSize="${IT_MAX_POOL_SIZE}" \ | |
| 2>&1 | tee "$maven_log" | |
| pipeline_status=("${PIPESTATUS[@]}") | |
| exit_code="${pipeline_status[0]}" | |
| if (( exit_code == 0 && pipeline_status[1] != 0 )); then | |
| exit_code="${pipeline_status[1]}" | |
| fi | |
| set -e | |
| max_log_bytes=20971520 | |
| log_bytes="$(stat --format=%s "$maven_log")" | |
| if (( log_bytes > max_log_bytes )); then | |
| tail -c "$max_log_bytes" "$maven_log" > "${maven_log}.tail" | |
| mv "${maven_log}.tail" "$maven_log" | |
| fi | |
| echo "exit_code=${exit_code}" >> "$GITHUB_OUTPUT" | |
| exit "$exit_code" | |
| - name: Capture Integration Test Diagnostics | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| env: | |
| TEST_STEP_OUTCOME: ${{ steps.integration-tests.outcome }} | |
| run: | | |
| diagnostics="openmetadata-integration-tests/target/ci-diagnostics" | |
| mkdir -p "$diagnostics" | |
| { | |
| date -u | |
| uname -a | |
| java -version | |
| echo "processors=$(nproc)" | |
| df -h | |
| free -h | |
| docker system df | |
| docker ps -a --no-trunc | |
| } > "$diagnostics/environment.txt" 2>&1 | |
| capture_container_logs=false | |
| if [ "$TEST_STEP_OUTCOME" != "success" ]; then | |
| capture_container_logs=true | |
| elif ! find openmetadata-integration-tests/target/failsafe-reports \ | |
| -name 'TEST-*.xml' -print -quit 2>/dev/null | grep -q .; then | |
| capture_container_logs=true | |
| fi | |
| if [ "$capture_container_logs" = "true" ]; then | |
| while IFS= read -r container; do | |
| [[ -n "$container" ]] || continue | |
| name=$(docker inspect --format '{{.Name}}' "$container" | tr -cd '[:alnum:]_.-') | |
| docker logs --timestamps --since 30m --tail 10000 "$container" 2>&1 \ | |
| | tail -c 10485760 > "$diagnostics/docker-${name}.log" || true | |
| done < <(docker ps -aq) | |
| fi | |
| - name: Classify Integration Test Outcome | |
| if: ${{ always() }} | |
| env: | |
| TEST_STEP_OUTCOME: ${{ steps.integration-tests.outcome }} | |
| TEST_EXIT_CODE: ${{ steps.integration-tests.outputs.exit_code }} | |
| TEST_PROFILE: cache-tests | |
| TEST_LANE: ${{ matrix.lane.name }} | |
| run: | | |
| python3 .github/scripts/classify_test_outcome.py \ | |
| --step-outcome "$TEST_STEP_OUTCOME" \ | |
| --exit-code "$TEST_EXIT_CODE" \ | |
| --report-glob 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml' \ | |
| --diagnostic-glob 'openmetadata-integration-tests/target/ci-diagnostics/**/*' \ | |
| --diagnostic-glob 'openmetadata-integration-tests/target/failsafe-reports/*.dump*' \ | |
| --diagnostic-glob 'openmetadata-integration-tests/target/failsafe-reports/*.txt' \ | |
| --output openmetadata-integration-tests/target/ci-diagnostics/outcome.json | |
| - name: Upload Integration Test Diagnostics | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: java-it-postgres-elasticsearch-redis-${{ matrix.lane.name }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| openmetadata-integration-tests/target/ci-diagnostics | |
| openmetadata-integration-tests/target/failsafe-reports/*.dump* | |
| openmetadata-integration-tests/target/failsafe-reports/*.txt | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| - name: Upload Integration Test Reports | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: java-it-reports-postgres-elasticsearch-redis-${{ matrix.lane.name }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Clean Up | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| run: | | |
| cd ./docker/development | |
| docker compose down --remove-orphans | |
| sudo rm -rf "${PWD}/docker-volume" | |
| - name: Publish Test Report | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: ScalableCapital/action-surefire-report@v2 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| check_name: Integration Test Report (postgres-elasticsearch-redis, ${{ matrix.lane.name }}) | |
| fail_on_test_failures: false | |
| fail_if_no_tests: false | |
| report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml' | |
| integration-tests-postgres-elasticsearch-redis: | |
| name: ${{ github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name != 'safe to test' && 'integration-tests-postgres-elasticsearch-redis (ignored label)' || 'integration-tests-postgres-elasticsearch-redis' }} | |
| needs: [changes, integration-test-build, integration-test-lanes] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: ${{ always() && !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }} | |
| steps: | |
| - name: Verify Maven Integration Test Lanes | |
| env: | |
| CHANGES_RESULT: ${{ needs.changes.result }} | |
| BACKEND_CHANGED: ${{ needs.changes.outputs.backend }} | |
| BUILD_RESULT: ${{ needs.integration-test-build.result }} | |
| LANES_RESULT: ${{ needs.integration-test-lanes.result }} | |
| run: | | |
| if [[ "$CHANGES_RESULT" == "success" && "$BACKEND_CHANGED" != "true" ]]; then | |
| echo "No backend integration-test paths changed." | |
| exit 0 | |
| fi | |
| if [[ "$CHANGES_RESULT" != "success" || "$BUILD_RESULT" != "success" || "$LANES_RESULT" != "success" ]]; then | |
| echo "Java integration jobs did not complete successfully: changes=${CHANGES_RESULT}, build=${BUILD_RESULT}, lanes=${LANES_RESULT}" | |
| exit 1 | |
| fi |