base-std-fork-tests.yml

name: Base Std Fork Tests

on:
  pull_request:
  merge_group:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  BASE_REF: main
  # Branch to clone; SHA below is pinned for reproducibility and verified after clone.
  BASE_ANVIL_BRANCH: base-anvil-fork
  BASE_ANVIL_SHA: 14a43909528eb7ff4bafab0e5fee9b3b651bcd50
  CARGO_TERM_COLOR: always

permissions:
  contents: read
  pull-requests: write

jobs:
  fork-tests:
    name: Base Std Fork Tests
    runs-on: ubuntu-latest
    timeout-minutes: 120
    # Advisory: base-std is the spec forerunner. Divergences mean base/base needs to catch up,
    # not that base-std is wrong. This job surfaces failures without blocking PR merges.
    continue-on-error: true
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout base-std
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 1
          submodules: recursive

      - name: Free disk space
        shell: bash
        run: |
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          df -h

      - name: Install native dependencies
        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
        with:
          packages: libsqlite3-dev clang libclang-dev llvm llvm-dev build-essential pkg-config protobuf-compiler
          version: 1.4

      - name: Install Rust
        shell: bash
        run: |
          rustup default stable
          rustup show active-toolchain

      - name: Set LIBCLANG_PATH
        shell: bash
        run: |
          LLVM_BIN=$(command -v llvm-config || ls /usr/bin/llvm-config-* 2>/dev/null | sort -V | tail -1)
          LLVM_VERSION=$($LLVM_BIN --version | cut -d. -f1)
          echo "LIBCLANG_PATH=/usr/lib/llvm-${LLVM_VERSION}/lib" >> "$GITHUB_ENV"

      - name: Install mold
        uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1

      - name: Setup sccache
        uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9

      - name: Install Foundry
        uses: foundry-rs/foundry-toolchain@50d5a8956f2e319df19e6b57539d7e2acb9f8c1e # v1.5.0
        with:
          version: stable

      - name: Clone fork-test repositories
        shell: bash
        run: |
          set -euo pipefail

          workdir="$RUNNER_TEMP/base-std-fork-tests"
          rm -rf "$workdir"
          mkdir -p "$workdir"

          git clone --depth 1 --single-branch --branch "$BASE_REF" \
            https://github.com/base/base.git "$workdir/base"

          git clone --depth 1 --single-branch --branch "$BASE_ANVIL_BRANCH" \
            https://github.com/base/base-anvil.git "$workdir/base-anvil"

          actual_sha=$(git -C "$workdir/base-anvil" rev-parse HEAD)
          if [ "$actual_sha" != "$BASE_ANVIL_SHA" ]; then
            echo "::warning::base-anvil HEAD ($actual_sha) differs from pinned SHA ($BASE_ANVIL_SHA) — update BASE_ANVIL_SHA in the workflow"
          fi

          echo "BASE_DIR=$workdir/base" >> "$GITHUB_ENV"
          echo "BASE_ANVIL_DIR=$workdir/base-anvil" >> "$GITHUB_ENV"

      - name: Build patched base-anvil binaries
        shell: bash
        env:
          CARGO_PROFILE_RELEASE_LTO: "false"
          RUSTC_WRAPPER: "sccache"
          SCCACHE_GHA_ENABLED: "true"
        run: |
          set -euo pipefail

          cd "$BASE_ANVIL_DIR"
          rustup show active-toolchain
          cargo \
            --config "patch.\"https://github.com/base/base.git\".base-common-precompiles.path=\"$BASE_DIR/crates/common/precompiles\"" \
            --config "patch.\"https://github.com/base/base.git\".base-common-chains.path=\"$BASE_DIR/crates/common/chains\"" \
            build --release --no-default-features --features anvil/cli -p anvil -p forge

          "$BASE_ANVIL_DIR/target/release/anvil" --version
          "$BASE_ANVIL_DIR/target/release/forge" --version

      - name: Set up Python 3.13
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.13'

      - name: Set up fork-test runner venv
        shell: bash
        # setup-python makes python3 == 3.13; make python-check enforces it.
        run: make smoke-setup PYTHON=python3

      - name: Run base-std fork tests
        id: fork_tests
        shell: bash
        run: |
          set -euo pipefail

          ANVIL_BIN="$BASE_ANVIL_DIR/target/release/anvil" \
            FORGE_BIN="$BASE_ANVIL_DIR/target/release/forge" \
            ANVIL_LOG="$RUNNER_TEMP/base-std-anvil.log" \
            make fork-tests 2>&1 | tee "$RUNNER_TEMP/fork-test-output.txt"
          echo "fork_tests_exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
        continue-on-error: true

      - name: Summarize fork test results
        if: always()
        shell: bash
        run: |
          output="$RUNNER_TEMP/fork-test-output.txt"
          passed=$(grep -c '\[PASS\]' "$output" 2>/dev/null || true); passed=${passed:-0}
          failed=$(grep -c '\[FAIL' "$output" 2>/dev/null || true); failed=${failed:-0}

          {
            echo "## Fork Test Results"
            echo ""
            if [ "$failed" -eq 0 ]; then
              echo "✅ All **${passed}** tests passed — base/base is fully in sync with base-std."
            else
              echo "⚠️ **${failed}** test(s) failed, **${passed}** passed."
              echo ""
              echo "These failures indicate divergences where **base/base needs to catch up** to the base-std spec."
              echo "base-std PRs are not blocked by this check."
              echo ""
              echo "### Failing tests"
              echo '```'
              grep '\[FAIL' "$output" | sed 's/\x1B\[[0-9;]*m//g' || true
              echo '```'
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Comment fork test results on PR
        if: github.event_name == 'pull_request' && always()
        shell: bash
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          output="$RUNNER_TEMP/fork-test-output.txt"
          passed=$(grep -c '\[PASS\]' "$output" 2>/dev/null || true); passed=${passed:-0}
          failed=$(grep -c '\[FAIL' "$output" 2>/dev/null || true); failed=${failed:-0}
          marker="<!-- fork-test-results -->"

          if [ ! -s "$output" ] || [ "$((passed + failed))" -eq 0 ]; then
            body="${marker}
          ### ❌ Fork tests did not run
          The build or setup step failed before any tests could execute. Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
          elif [ "$failed" -eq 0 ]; then
            body="${marker}
          ### ✅ Fork tests: all ${passed} passed
          base/base is fully in sync with the base-std spec."
          else
            # Extract test name and full error for each failure, deduplicated
            failing=$(grep '\[FAIL' "$output" \
              | sed 's/\x1B\[[0-9;]*m//g' \
              | sed 's/^\[FAIL: \(.*\)\] \(.*\) (runs.*$/- **\2**: `\1`/' \
              | sort -u || true)
            body="${marker}
          ### ⚠️ Fork tests: ${failed} failed, ${passed} passed

          These failures indicate divergences where **base/base needs to catch up** to the base-std spec. This check is advisory and does not block merging.

          <details>
          <summary>Failing tests</summary>

          ${failing}

          </details>"
          fi

          pr="${{ github.event.pull_request.number }}"
          repo="${{ github.repository }}"

          # Find existing comment with our marker and update it, or post a new one
          existing_id=$(gh api "repos/${repo}/issues/${pr}/comments" \
            --jq ".[] | select(.body | startswith(\"${marker}\")) | .id" \
            | head -1)

          if [ -n "$existing_id" ]; then
            gh api "repos/${repo}/issues/comments/${existing_id}" \
              -X PATCH --field body="$body" > /dev/null
          else
            gh api "repos/${repo}/issues/${pr}/comments" \
              --field body="$body" > /dev/null
          fi

      - name: Upload anvil log
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: base-std-anvil-log
          path: ${{ runner.temp }}/base-std-anvil.log
          if-no-files-found: ignore