ci-secret.yaml

name: ORAssistant Secret CI
# Quoted: an unquoted ' #' would start a YAML comment.
run-name: "${{ github.actor }} started CI${{ inputs.pr && format(' for PR #{0}', inputs.pr) || '' }}"

on:
  push:
    branches:
      - master
    paths:
      - 'backend/**'
      - 'frontend/**'
      - 'evaluation/**'
      - 'Makefile'
      - 'docker-compose.yml'
      - 'docker-compose.ci.yml'
  workflow_dispatch:
    inputs:
      pr:
        description: Open pull request number to test instead of the selected branch
        required: false
        type: string
        default: ''

defaults:
  run:
    shell: bash

jobs:
  # Pick the code under test: the pushed commit, or a pull request's merge result.
  # Only people with write access can dispatch this workflow, so the secrets stay
  # with reviewed code.
  resolve:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
      statuses: write
    outputs:
      ref: ${{ steps.target.outputs.ref }}
      pr: ${{ steps.target.outputs.pr }}
      head_sha: ${{ steps.target.outputs.head_sha }}
    steps:
    - name: Resolve the code under test
      id: target
      env:
        GH_TOKEN: ${{ github.token }}
        PR: ${{ inputs.pr }}
        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
      run: |
        if [ -z "$PR" ]; then
          echo "ref=$GITHUB_SHA" >> "$GITHUB_OUTPUT"
          echo "pr=" >> "$GITHUB_OUTPUT"
          echo "head_sha=" >> "$GITHUB_OUTPUT"
          exit 0
        fi
        # GitHub computes mergeability lazily; the first answer can be null.
        for attempt in 1 2 3 4 5 6; do
          pr_json=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR")
          mergeable=$(jq -r '.mergeable' <<< "$pr_json")
          [ "$mergeable" != "null" ] && break
          sleep 5
        done
        state=$(jq -r '.state' <<< "$pr_json")
        head_sha=$(jq -r '.head.sha' <<< "$pr_json")
        if [ "$state" != "open" ]; then
          echo "::error::PR #$PR is $state, not open"
          exit 1
        fi
        if [ "$mergeable" != "true" ]; then
          echo "::error::PR #$PR is not mergeable (mergeable=$mergeable); update it first"
          exit 1
        fi
        gh api "repos/$GITHUB_REPOSITORY/statuses/$head_sha" \
          -f state=pending -f context="Secret CI" \
          -f target_url="$RUN_URL" -f description="Full suite running"
        echo "ref=refs/pull/$PR/merge" >> "$GITHUB_OUTPUT"
        echo "pr=$PR" >> "$GITHUB_OUTPUT"
        echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"

  lint-backend:
    needs: [resolve]
    runs-on: ubuntu-latest
    steps:
    - name: Setup python
      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
      with:
        python-version: '3.12'
    - name: Install uv
      uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4  # v10.1.0
    - name: Checkout code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      with:
        ref: ${{ needs.resolve.outputs.ref }}
    - name: Install deps
      run: cd backend && make init-dev
    - name: Lint
      run: cd backend && make check

  lint-frontend:
    needs: [resolve]
    runs-on: ubuntu-latest
    steps:
    - name: Setup python
      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
      with:
        python-version: '3.12'
    - name: Install uv
      uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4  # v10.1.0
    - name: Checkout code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      with:
        ref: ${{ needs.resolve.outputs.ref }}
    - name: Install deps
      run: cd frontend && make init-dev
    - name: Lint
      run: cd frontend && make check

  lint-evaluation:
    needs: [resolve]
    runs-on: ubuntu-latest
    steps:
    - name: Setup python
      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
      with:
        python-version: '3.12'
    - name: Install uv
      uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4  # v10.1.0
    - name: Checkout code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      with:
        ref: ${{ needs.resolve.outputs.ref }}
    - name: Install deps
      run: cd evaluation && make init-dev
    - name: Lint
      run: cd evaluation && make check

  test:
    needs: [resolve, lint-backend]
    runs-on: self-hosted
    steps:
    - name: Setup python
      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
      with:
        python-version: '3.12'
    - name: Install uv
      uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4  # v10.1.0
    - name: Checkout code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      with:
        ref: ${{ needs.resolve.outputs.ref }}
    - name: Install deps
      run: cd backend && make init-dev
    - name: Run unit tests
      working-directory: backend
      run: |
        cp .env.test .env
        make test

  docker-eval:
    needs: [resolve, test, lint-frontend, lint-evaluation]
    runs-on: self-hosted
    steps:
    - name: Install uv
      uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4  # v10.1.0

    - name: Checkout code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      with:
        ref: ${{ needs.resolve.outputs.ref }}

    - name: Download HF dataset
      run: |
        if [ ! -d "data" ] || [ -z "$(ls -A data 2>/dev/null)" ]; then
          uv tool install huggingface-hub
          hf download The-OpenROAD-Project/ORAssistant_RAG_Dataset \
            --repo-type dataset --local-dir ./data
        else
          echo "Dataset already cached on runner, skipping download"
        fi

    - name: Populate environment variables
      run: |
        cp backend/.env.example backend/.env
        sed -i 's|{{GOOGLE_API_KEY}}|${{ secrets.GOOGLE_API_KEY }}|g' backend/.env
        sed -i 's|{{GOOGLE_PROJECT_ID}}|${{ secrets.GOOGLE_PROJECT_ID }}|g' backend/.env
        sed -i 's|{{PATH_TO_GOOGLE_APPLICATION_CREDENTIALS}}|src/secret.json|g' backend/.env
        sed -i 's|HF_TOKEN=|HF_TOKEN=${{ secrets.HF_TOKEN }}|g' backend/.env

        cp backend/.env.example evaluation/.env
        sed -i 's|{{GOOGLE_API_KEY}}|${{ secrets.GOOGLE_API_KEY }}|g' evaluation/.env
        sed -i 's|{{GOOGLE_PROJECT_ID}}|${{ secrets.GOOGLE_PROJECT_ID }}|g' evaluation/.env
        sed -i 's|{{PATH_TO_GOOGLE_APPLICATION_CREDENTIALS}}|src/secret.json|g' evaluation/.env
        sed -i 's|HF_TOKEN=|HF_TOKEN=${{ secrets.HF_TOKEN }}|g' evaluation/.env

    - name: Copy Google credentials
      env:
        GOOGLE_SECRET_JSON: ${{ secrets.PATH_TO_GOOGLE_APPLICATION_CREDENTIALS }}
      run: |
        make seed-credentials

    - name: Build and start Docker stack
      run: |
        make docker-up-ci

    - name: Wait for graph readiness
      run: |
        echo "Waiting for graph to finish initializing..."
        for i in $(seq 1 360); do
          if curl -sf http://localhost:8000/conversations/ready | grep -q '"ready"'; then
            echo "Graph is ready"
            exit 0
          fi
          echo "Waiting for graph initialization... ($i/360)"
          sleep 10
        done
        echo "Graph did not become ready after 60 minutes"
        exit 1

    - name: Run LLM CI
      id: llm_tests
      working-directory: evaluation
      run: |
        make llm-tests

    - name: Capture Docker logs on failure
      if: failure()
      run: |
        mkdir -p docker-logs
        echo "=== Docker Container Status ==="
        docker ps -a
        echo ""
        echo "=== COMPLETE BACKEND LOGS ==="
        docker logs backend 2>&1 || echo "Failed to get backend logs"
        echo ""
        echo "=== COMPLETE FRONTEND LOGS ==="
        docker logs frontend 2>&1 || echo "Failed to get frontend logs"
        echo ""
        echo "=== COMPLETE POSTGRES LOGS ==="
        docker logs postgres 2>&1 || echo "Failed to get postgres logs"

        docker logs backend > docker-logs/backend.log 2>&1 || echo "Failed to capture backend logs"
        docker logs frontend > docker-logs/frontend.log 2>&1 || echo "Failed to capture frontend logs"
        docker logs postgres > docker-logs/postgres.log 2>&1 || echo "Failed to capture postgres logs"

    - name: Upload Docker logs as artifacts
      if: failure()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
      with:
        name: docker-logs-${{ github.run_id }}
        path: docker-logs/
        retention-days: 7

    # The full output is several hundred KB, far above GitHub's 65536-character
    # comment limit. Comments get the aggregate metrics at its end.
    - name: Summarize evaluation output
      if: always()
      working-directory: evaluation/auto_evaluation
      run: |
        [ -s llm_tests_output.txt ] || exit 0
        start=$(grep -n 'Aggregate Metrics' llm_tests_output.txt | tail -n 1 | cut -d: -f1 || true)
        {
          echo '```text'
          if [ -n "$start" ]; then
            tail -n +"$((start - 1))" llm_tests_output.txt
          else
            tail -n 100 llm_tests_output.txt
          fi | tail -c 60000
          echo '```'
        } > llm_tests_summary.md

    - name: Upload evaluation output as artifact
      if: failure() || needs.resolve.outputs.pr != ''
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
      with:
        name: evaluation-output-${{ github.run_id }}
        path: |
          evaluation/auto_evaluation/llm_tests_output.txt
          evaluation/auto_evaluation/llm_tests_summary.md
        retention-days: 7
        if-no-files-found: ignore

    - name: Create commit comment
      if: needs.resolve.outputs.pr == ''
      uses: peter-evans/commit-comment@f6d60c65d05bb59f750fa51ad3de1d443ba0eb52  # v4.0.0
      with:
        token: ${{ secrets.GH_PAT }}
        body-path: evaluation/auto_evaluation/llm_tests_summary.md

    - name: Teardown
      if: always()
      run: |
        make docker-down-ci

  # Report a pull request run on the PR. This runs on a GitHub-hosted runner,
  # which has the gh CLI, and after every other job, so an early failure is
  # reported too.
  report:
    if: always() && needs.resolve.outputs.pr != ''
    needs: [resolve, lint-backend, lint-frontend, lint-evaluation, test, docker-eval]
    runs-on: ubuntu-latest
    permissions:
      actions: read
      pull-requests: write
      statuses: write
    steps:
    - name: Download evaluation output
      continue-on-error: true
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
      with:
        name: evaluation-output-${{ github.run_id }}
        path: evaluation-output

    - name: Report the result on the pull request
      env:
        GH_TOKEN: ${{ github.token }}
        NEEDS_JSON: ${{ toJSON(needs) }}
        PR: ${{ needs.resolve.outputs.pr }}
        HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
      run: |
        if jq -e 'all(.[]; .result == "success")' <<< "$NEEDS_JSON" > /dev/null; then
          state=success
        else
          state=failure
        fi
        gh api "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \
          -f state="$state" -f context="Secret CI" \
          -f target_url="$RUN_URL" -f description="Full suite $state"
        summary=evaluation-output/llm_tests_summary.md
        if [ -s "$summary" ]; then
          { echo "Secret CI $state: $RUN_URL"; echo; cat "$summary"; } > comment.md
        else
          echo "Secret CI $state with no evaluation output: $RUN_URL" > comment.md
        fi
        gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body-file comment.md