feat(evaluate): gate vote scoring on the merits stage #2562

Workflow file for this run

	name: ci

	# The local gate, enforced on every PR and on pushes to main. The gate job below
	# invokes scripts/gate.sh — the single definition AGENTS.md and README.md also
	# point to — so "green CI" and "passes the local gate" cannot drift apart. Uses
	# no secrets, so it does not declare the `prod` environment.
	on:
	pull_request:
	# `edited` re-evaluates the base-sensitive checks (main-base,
	# promotion-gate, paths, cleanup-paths) when a PR is retargeted. For the
	# required contexts a stale `skipped` run would satisfy the requirement
	# outright; cleanup-paths is review-time defense rather than required, so
	# there the stale run instead misleads the reviewer.
	types: [opened, synchronize, reopened, edited]
	push:
	branches: [main]

	permissions: {}

	concurrency:
	group: ci-${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	gate:
	runs-on: ubuntu-latest
	# The whole suite (lint, mypy, pytest, validation) runs in a few minutes; this
	# cap is generous headroom so a hung step is cancelled long before the 6h default.
	timeout-minutes: 20
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: ./.github/actions/setup-python-env
	# Each step invokes one stage of scripts/gate.sh — the single definition of
	# the gate — so the individual step names and their tool annotations survive
	# while the commands live in exactly one place.
	- name: lint
	run: scripts/gate.sh lint
	- name: types
	run: scripts/gate.sh types
	- name: test
	# GATE_COV=1 runs pytest under coverage, which the summary step below reads.
	env:
	GATE_COV: "1"
	run: scripts/gate.sh test
	- name: coverage summary
	# Reporting, not gating: surface coverage to maintainers in the run's job
	# summary (no badge, no commit, no upload). Depends on the test step having
	# run with GATE_COV=1 so a .coverage file exists. Needs no permissions —
	# $GITHUB_STEP_SUMMARY is a runner-local file — so this stays fork-safe.
	run: \|
	{
	echo "## Coverage: $(uv run coverage report --format=total)%"
	echo ""
	echo "<details><summary>Per-file coverage</summary>"
	echo ""
	uv run coverage report --format=markdown
	echo ""
	echo "</details>"
	} >> "$GITHUB_STEP_SUMMARY"
	- name: data
	run: scripts/gate.sh data
	- name: schemas
	run: scripts/gate.sh schemas

	paths:
	# Defense-in-depth path jail for the auto-merged data-production PRs: a
	# predict/evaluate run branch may only add files under data/, so a
	# PR that auto-merges without a human cannot carry a code, workflow, or
	# config change. A no-op that passes for every other branch, so it is safe to
	# make a required status check on all PRs. The jail also runs producer-side in
	# each collect job; this required check enforces it independently of the
	# workflow that produced the branch. Logic lives in the tested `assert-paths`
	# command; this step only runs git.
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	fetch-depth: 0
	- uses: ./.github/actions/setup-python-env
	- name: Enforce the data/ path jail on data-production branches
	env:
	HEAD_REF: ${{ github.head_ref }}
	BASE_SHA: ${{ github.event.pull_request.base.sha }}
	run: \|
	set -euo pipefail
	case "$HEAD_REF" in
	predict/run-\|evaluate/run-) ;;
	*) echo "not a data-production branch; path jail not applicable"; exit 0 ;;
	esac
	git diff --name-status "${BASE_SHA}...HEAD" > changes.txt
	uv run fedcourts assert-paths --name-status-file changes.txt

	cleanup-paths:
	# The destructive counterpart of `paths`: a cleanup-sweep branch may only delete
	# files under a data/cases/*/events//predictions/ subtree. A no-op that passes
	# for every other branch, so it is safe to require on all PRs — but it is
	# deliberately left out of the required list: a cleanup PR is never
	# auto-merged, so this is review-time defense.
	# A maintainer runs the jail locally before opening the sweep PR; this enforces
	# it independently.
	# Logic lives in the tested `assert-cleanup-paths` command; this step only runs git.
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	fetch-depth: 0
	- uses: ./.github/actions/setup-python-env
	- name: Enforce the cleanup jail on cleanup branches
	env:
	HEAD_REF: ${{ github.head_ref }}
	BASE_SHA: ${{ github.event.pull_request.base.sha }}
	run: \|
	set -euo pipefail
	case "$HEAD_REF" in
	cleanup/*) ;;
	*) echo "not a cleanup branch; cleanup jail not applicable"; exit 0 ;;
	esac
	git diff --name-status "${BASE_SHA}...HEAD" > changes.txt
	uv run fedcourts assert-cleanup-paths --name-status-file changes.txt

	promotion-gate:
	# The staging→main promotion PR's gate: quiescence plus integration-run
	# freshness at the PR head, via scripts/promotion-gate.sh — the same
	# script the `promote` dispatch runs, so the pre-flight and this required
	# check cannot drift apart. Skipped — and therefore green as a required
	# check — on every other PR. Gate state is point-in-time: re-run this
	# check immediately before merging the promotion. Same-repo head only: a
	# fork branch named `staging` gets the skip, never the gate (and a fork's
	# read-only token could not satisfy it anyway).
	if: >-
	github.event_name == 'pull_request' &&
	github.base_ref == 'main' &&
	github.head_ref == 'staging' &&
	github.event.pull_request.head.repo.full_name == github.repository
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	actions: read
	issues: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- name: Quiescence and freshness at the PR head
	env:
	GH_TOKEN: ${{ github.token }}
	HEAD_SHA: ${{ github.event.pull_request.head.sha }}
	run: scripts/promotion-gate.sh all "$HEAD_SHA"

	main-base:
	# Merge routing, on the `paths` pattern: code reaches `main` only through
	# the staging→main promotion or a reviewed non-feature lane — the collect
	# run branches, the maintainer's cleanup sweep, the metrics-refresh /
	# cert-backtest / salience-replay PRs, and the qp-topic labeling run's
	# labels PR. This job runs (and fails) only on a
	# PR to `main` from any other head; every legitimate PR skips it, and the
	# skipped run satisfies the requirement. It is among `main`'s required
	# contexts (with `gate`, `paths`, `promotion-gate`): a `pull_request` runs
	# the workflow from the merge ref, the legitimate lanes are all cut from
	# `main`, and `main`'s own ci.yml carries this job, so the context reports
	# on every lane — and a mis-routed feature PR runs it, fails, and cannot
	# merge. Fork heads never match the allowlist — outside contributions
	# route through `staging` too. The emergency escape hatch is a repo admin
	# editing the ruleset: deliberate and auditable. Expression
	# `==`/`startsWith` compare case-insensitively, so a write-access branch
	# named `Staging` skips the jail — a hygiene gap, not a hole: the PR still
	# needs the human merge.
	if: >-
	github.event_name == 'pull_request' &&
	github.base_ref == 'main' &&
	!(
	github.event.pull_request.head.repo.full_name == github.repository &&
	(
	github.head_ref == 'staging' \|\|
	startsWith(github.head_ref, 'predict/run-') \|\|
	startsWith(github.head_ref, 'evaluate/run-') \|\|
	startsWith(github.head_ref, 'cleanup/') \|\|
	github.head_ref == 'metrics/refresh' \|\|
	github.head_ref == 'metrics/cert-backtest' \|\|
	github.head_ref == 'metrics/salience-replay' \|\|
	github.head_ref == 'qp-topics/refresh'
	)
	)
	runs-on: ubuntu-latest
	timeout-minutes: 5
	permissions: {}
	steps:
	- run: \|
	set -euo pipefail
	echo "::error::PRs to main carry only promotion batches (head 'staging') or the bot lanes; feature PRs target 'staging' (AGENTS.md)."
	exit 1

Navigation Menu

Uh oh!

feat(evaluate): gate vote scoring on the merits stage #2562

Workflow file

feat(evaluate): gate vote scoring on the merits stage #2562

Uh oh!

Workflow file for this run