vvbandeira started CI #197
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ORAssistant Secret CI | |
| # Quoted: an unquoted ' #' would start a YAML comment. | |
| run-name: "${{ github.actor }} started CI${{ inputs.pr && format(' for PR #{0}', inputs.pr) || '' }}" | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - 'backend/**' | |
| - 'frontend/**' | |
| - 'evaluation/**' | |
| - 'Makefile' | |
| - 'docker-compose.yml' | |
| - 'docker-compose.ci.yml' | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| description: Open pull request number to test instead of the selected branch | |
| required: false | |
| type: string | |
| default: '' | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| # Pick the code under test: the pushed commit, or a pull request's merge result. | |
| # Only people with write access can dispatch this workflow, so the secrets stay | |
| # with reviewed code. | |
| resolve: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| statuses: write | |
| outputs: | |
| ref: ${{ steps.target.outputs.ref }} | |
| pr: ${{ steps.target.outputs.pr }} | |
| head_sha: ${{ steps.target.outputs.head_sha }} | |
| steps: | |
| - name: Resolve the code under test | |
| id: target | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR: ${{ inputs.pr }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [ -z "$PR" ]; then | |
| echo "ref=$GITHUB_SHA" >> "$GITHUB_OUTPUT" | |
| echo "pr=" >> "$GITHUB_OUTPUT" | |
| echo "head_sha=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # GitHub computes mergeability lazily; the first answer can be null. | |
| for attempt in 1 2 3 4 5 6; do | |
| pr_json=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR") | |
| mergeable=$(jq -r '.mergeable' <<< "$pr_json") | |
| [ "$mergeable" != "null" ] && break | |
| sleep 5 | |
| done | |
| state=$(jq -r '.state' <<< "$pr_json") | |
| head_sha=$(jq -r '.head.sha' <<< "$pr_json") | |
| if [ "$state" != "open" ]; then | |
| echo "::error::PR #$PR is $state, not open" | |
| exit 1 | |
| fi | |
| if [ "$mergeable" != "true" ]; then | |
| echo "::error::PR #$PR is not mergeable (mergeable=$mergeable); update it first" | |
| exit 1 | |
| fi | |
| gh api "repos/$GITHUB_REPOSITORY/statuses/$head_sha" \ | |
| -f state=pending -f context="Secret CI" \ | |
| -f target_url="$RUN_URL" -f description="Full suite running" | |
| echo "ref=refs/pull/$PR/merge" >> "$GITHUB_OUTPUT" | |
| echo "pr=$PR" >> "$GITHUB_OUTPUT" | |
| echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT" | |
| lint-backend: | |
| needs: [resolve] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.resolve.outputs.ref }} | |
| - name: Install deps | |
| run: cd backend && make init-dev | |
| - name: Lint | |
| run: cd backend && make check | |
| lint-frontend: | |
| needs: [resolve] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.resolve.outputs.ref }} | |
| - name: Install deps | |
| run: cd frontend && make init-dev | |
| - name: Lint | |
| run: cd frontend && make check | |
| lint-evaluation: | |
| needs: [resolve] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.resolve.outputs.ref }} | |
| - name: Install deps | |
| run: cd evaluation && make init-dev | |
| - name: Lint | |
| run: cd evaluation && make check | |
| test: | |
| needs: [resolve, lint-backend] | |
| runs-on: self-hosted | |
| steps: | |
| - name: Setup python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.resolve.outputs.ref }} | |
| - name: Install deps | |
| run: cd backend && make init-dev | |
| - name: Run unit tests | |
| working-directory: backend | |
| run: | | |
| cp .env.test .env | |
| make test | |
| docker-eval: | |
| needs: [resolve, test, lint-frontend, lint-evaluation] | |
| runs-on: self-hosted | |
| steps: | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.resolve.outputs.ref }} | |
| - name: Download HF dataset | |
| run: | | |
| if [ ! -d "data" ] || [ -z "$(ls -A data 2>/dev/null)" ]; then | |
| uv tool install huggingface-hub | |
| hf download The-OpenROAD-Project/ORAssistant_RAG_Dataset \ | |
| --repo-type dataset --local-dir ./data | |
| else | |
| echo "Dataset already cached on runner, skipping download" | |
| fi | |
| - name: Populate environment variables | |
| run: | | |
| cp backend/.env.example backend/.env | |
| sed -i 's|{{GOOGLE_API_KEY}}|${{ secrets.GOOGLE_API_KEY }}|g' backend/.env | |
| sed -i 's|{{GOOGLE_PROJECT_ID}}|${{ secrets.GOOGLE_PROJECT_ID }}|g' backend/.env | |
| sed -i 's|{{PATH_TO_GOOGLE_APPLICATION_CREDENTIALS}}|src/secret.json|g' backend/.env | |
| sed -i 's|HF_TOKEN=|HF_TOKEN=${{ secrets.HF_TOKEN }}|g' backend/.env | |
| cp backend/.env.example evaluation/.env | |
| sed -i 's|{{GOOGLE_API_KEY}}|${{ secrets.GOOGLE_API_KEY }}|g' evaluation/.env | |
| sed -i 's|{{GOOGLE_PROJECT_ID}}|${{ secrets.GOOGLE_PROJECT_ID }}|g' evaluation/.env | |
| sed -i 's|{{PATH_TO_GOOGLE_APPLICATION_CREDENTIALS}}|src/secret.json|g' evaluation/.env | |
| sed -i 's|HF_TOKEN=|HF_TOKEN=${{ secrets.HF_TOKEN }}|g' evaluation/.env | |
| - name: Copy Google credentials | |
| env: | |
| GOOGLE_SECRET_JSON: ${{ secrets.PATH_TO_GOOGLE_APPLICATION_CREDENTIALS }} | |
| run: | | |
| make seed-credentials | |
| - name: Build and start Docker stack | |
| run: | | |
| make docker-up-ci | |
| - name: Wait for graph readiness | |
| run: | | |
| echo "Waiting for graph to finish initializing..." | |
| for i in $(seq 1 360); do | |
| if curl -sf http://localhost:8000/conversations/ready | grep -q '"ready"'; then | |
| echo "Graph is ready" | |
| exit 0 | |
| fi | |
| echo "Waiting for graph initialization... ($i/360)" | |
| sleep 10 | |
| done | |
| echo "Graph did not become ready after 60 minutes" | |
| exit 1 | |
| - name: Run LLM CI | |
| id: llm_tests | |
| working-directory: evaluation | |
| run: | | |
| make llm-tests | |
| - name: Capture Docker logs on failure | |
| if: failure() | |
| run: | | |
| mkdir -p docker-logs | |
| echo "=== Docker Container Status ===" | |
| docker ps -a | |
| echo "" | |
| echo "=== COMPLETE BACKEND LOGS ===" | |
| docker logs backend 2>&1 || echo "Failed to get backend logs" | |
| echo "" | |
| echo "=== COMPLETE FRONTEND LOGS ===" | |
| docker logs frontend 2>&1 || echo "Failed to get frontend logs" | |
| echo "" | |
| echo "=== COMPLETE POSTGRES LOGS ===" | |
| docker logs postgres 2>&1 || echo "Failed to get postgres logs" | |
| docker logs backend > docker-logs/backend.log 2>&1 || echo "Failed to capture backend logs" | |
| docker logs frontend > docker-logs/frontend.log 2>&1 || echo "Failed to capture frontend logs" | |
| docker logs postgres > docker-logs/postgres.log 2>&1 || echo "Failed to capture postgres logs" | |
| - name: Upload Docker logs as artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: docker-logs-${{ github.run_id }} | |
| path: docker-logs/ | |
| retention-days: 7 | |
| # The full output is several hundred KB, far above GitHub's 65536-character | |
| # comment limit. Comments get the aggregate metrics at its end. | |
| - name: Summarize evaluation output | |
| if: always() | |
| working-directory: evaluation/auto_evaluation | |
| run: | | |
| [ -s llm_tests_output.txt ] || exit 0 | |
| start=$(grep -n 'Aggregate Metrics' llm_tests_output.txt | tail -n 1 | cut -d: -f1 || true) | |
| { | |
| echo '```text' | |
| if [ -n "$start" ]; then | |
| tail -n +"$((start - 1))" llm_tests_output.txt | |
| else | |
| tail -n 100 llm_tests_output.txt | |
| fi | tail -c 60000 | |
| echo '```' | |
| } > llm_tests_summary.md | |
| - name: Upload evaluation output as artifact | |
| if: failure() || needs.resolve.outputs.pr != '' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: evaluation-output-${{ github.run_id }} | |
| path: | | |
| evaluation/auto_evaluation/llm_tests_output.txt | |
| evaluation/auto_evaluation/llm_tests_summary.md | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Create commit comment | |
| if: needs.resolve.outputs.pr == '' | |
| uses: peter-evans/commit-comment@f6d60c65d05bb59f750fa51ad3de1d443ba0eb52 # v4.0.0 | |
| with: | |
| token: ${{ secrets.GH_PAT }} | |
| body-path: evaluation/auto_evaluation/llm_tests_summary.md | |
| - name: Teardown | |
| if: always() | |
| run: | | |
| make docker-down-ci | |
| # Report a pull request run on the PR. This runs on a GitHub-hosted runner, | |
| # which has the gh CLI, and after every other job, so an early failure is | |
| # reported too. | |
| report: | |
| if: always() && needs.resolve.outputs.pr != '' | |
| needs: [resolve, lint-backend, lint-frontend, lint-evaluation, test, docker-eval] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| pull-requests: write | |
| statuses: write | |
| steps: | |
| - name: Download evaluation output | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: evaluation-output-${{ github.run_id }} | |
| path: evaluation-output | |
| - name: Report the result on the pull request | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| NEEDS_JSON: ${{ toJSON(needs) }} | |
| PR: ${{ needs.resolve.outputs.pr }} | |
| HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if jq -e 'all(.[]; .result == "success")' <<< "$NEEDS_JSON" > /dev/null; then | |
| state=success | |
| else | |
| state=failure | |
| fi | |
| gh api "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ | |
| -f state="$state" -f context="Secret CI" \ | |
| -f target_url="$RUN_URL" -f description="Full suite $state" | |
| summary=evaluation-output/llm_tests_summary.md | |
| if [ -s "$summary" ]; then | |
| { echo "Secret CI $state: $RUN_URL"; echo; cat "$summary"; } > comment.md | |
| else | |
| echo "Secret CI $state with no evaluation output: $RUN_URL" > comment.md | |
| fi | |
| gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body-file comment.md |