Skip to content

Scenario Suite

Scenario Suite #160

name: Scenario Suite
on:
workflow_dispatch:
inputs:
build_timeout_minutes:
description: Minutes to wait for build orchestration
type: string
default: "10"
dispatch_timeout_minutes:
description: Minutes to wait for dispatch confirmation
type: string
default: "5"
cascade_version:
description: 'cascade rc tag to self-repin to (e.g. v0.16.6). Empty runs committed defaults.'
required: false
default: ''
cascade_version_sha:
description: 'Peeled commit SHA paired with cascade_version. Empty runs committed defaults.'
required: false
default: ''
schedule:
- cron: '30 6 * * *'
permissions:
contents: write
actions: write
pull-requests: write
concurrency:
group: scenario-suite
cancel-in-progress: false
env:
MANIFEST_FILE: .github/manifest.yaml
MANIFEST_KEY: ci
TAG_PREFIX: v
BOT_NAME: github-actions[bot]
BOT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com
jobs:
scenario-suite:
name: Scenario Suite
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
# Captured before anything is dispatched or merged. The reconcile job
# enumerates every run this repo produced at or after this instant and
# fails if any is unaccounted for in the ledger. Recorded first so no run
# the suite causes can fall outside the window.
window-start: ${{ steps.window.outputs.window-start }}
steps:
- name: Install gh transient-retry wrapper
run: |
cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY'
_gh_is_transient() {
local out="$1"
if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then
return 0
fi
if printf '%s' "$out" | grep -qiE 'HTTP 403'; then
if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then
return 0
fi
fi
return 1
}
gh() {
local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc
while :; do
out="$(command gh "$@" 2>&1)" && rc=0 || rc=$?
if [ "$rc" -eq 0 ]; then
printf '%s\n' "$out"
return 0
fi
if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then
printf '%s\n' "$out" >&2
return "$rc"
fi
printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
GHRETRY
echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV"
# The reconcile window opens here, before the seed merge that fires the
# orchestrate run. Every run the suite goes on to cause in this repo lands
# at or after this timestamp, so the reconcile job sees all of them.
- name: Open reconcile window
id: window
run: |
set -euo pipefail
WINDOW_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "window-start=$WINDOW_START" >> "$GITHUB_OUTPUT"
echo "reconcile window opened at $WINDOW_START"
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.CASCADE_STATE_TOKEN }}
- name: Setup cascade CLI
uses: stablekernel/cascade/.github/actions/setup-cli@v1.1.2
with:
token: ${{ secrets.GITHUB_TOKEN }}
version: ${{ inputs.cascade_version || 'v0.8.0' }}
- name: Configure git identity
run: |
git config user.name "$BOT_NAME"
git config user.email "$BOT_EMAIL"
- name: Log cascade version mode
run: |
if [ -n "${{ inputs.cascade_version }}" ]; then
echo "Running against dispatched rc: ${{ inputs.cascade_version }} (sha ${{ inputs.cascade_version_sha }})"
else
echo "Running against committed defaults (no rc dispatched)"
fi
- name: Self-repin manifest to the dispatched rc
uses: stablekernel/cascade/.github/actions/fleet-repin@main
with:
cascade_version: ${{ inputs.cascade_version }}
cascade_version_sha: ${{ inputs.cascade_version_sha }}
token: ${{ secrets.CASCADE_STATE_TOKEN }}
- name: Clean slate - delete leftover releases and tags
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
run: |
gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \
| while read -r t; do gh release delete "$t" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag 2>/dev/null || true; done
git fetch --tags --quiet || true
for t in $(git tag -l 'v*' 'rel-*'); do git push origin --delete "$t" 2>/dev/null || true; done
- name: Seed build via PR
id: seed
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
run: |
set -euo pipefail
git config user.name "$BOT_NAME"
git config user.email "$BOT_EMAIL"
BRANCH="scenario/src-$(date +%s)-$RANDOM"
git fetch origin main --quiet
git checkout -B "$BRANCH" origin/main
mkdir -p src
echo "scenario change $(date -u +%FT%TWZ)" >> src/build-trigger.txt
git add src/build-trigger.txt
git commit --no-gpg-sign -m "feat: trigger build run for scenario suite"
git push origin "$BRANCH"
gh pr create --base main --head "$BRANCH" --title "feat: trigger build run for scenario suite" --body "Automated scenario run; drives orchestrate on merge."
gh pr merge "$BRANCH" --rebase --delete-branch
git fetch origin main --quiet
MERGE_SHA="$(git rev-parse origin/main)"
echo "merge_sha=$MERGE_SHA" >> "$GITHUB_OUTPUT"
# Stamp a UTC boundary before orchestrate can fan out to primary, so the
# primary external-update wait correlates the run this merge triggered and
# never an older external-update run that happens to be newest.
DISPATCH_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "dispatch_ts=$DISPATCH_TS" >> "$GITHUB_OUTPUT"
- name: Wait for orchestrate run
id: wait
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
MERGE_SHA: ${{ steps.seed.outputs.merge_sha }}
run: |
set -euo pipefail
RUN_ID=""
for _i in $(seq 1 6); do
RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow=orchestrate.yaml --branch=main \
--json databaseId,headSha,status \
--jq ".[] | select(.headSha==\"$MERGE_SHA\") | .databaseId" | head -n1)"
[ -n "$RUN_ID" ] && break
sleep 60
done
[ -n "$RUN_ID" ] || { echo "::error::no orchestrate run found for merge SHA $MERGE_SHA"; exit 1; }
echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
MAX_WAIT=$(( ${{ inputs.build_timeout_minutes || 10 }} * 60 ))
ELAPSED=0
INTERVAL=15
while [ $ELAPSED -lt $MAX_WAIT ]; do
STATUS=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID" --jq '.status')
CONCLUSION=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID" --jq '.conclusion // empty')
echo "Status: $STATUS, Conclusion: $CONCLUSION"
if [ "$STATUS" = "completed" ]; then
if [ "$CONCLUSION" = "success" ]; then
echo "Orchestrate run completed successfully"
break
else
echo "::error::Orchestrate run completed with conclusion: $CONCLUSION"
exit 1
fi
fi
sleep $INTERVAL
ELAPSED=$(( ELAPSED + INTERVAL ))
done
if [ $ELAPSED -ge $MAX_WAIT ]; then
echo "::error::Timed out waiting for orchestrate run to complete"
exit 1
fi
# The merge above fires exactly one own-repo run: the push-triggered
# Orchestrate run on src/** (resolved as steps.wait.outputs.run_id). The
# wait step already gates it to success; register it so the reconcile job
# accounts for it. This is the only own-repo run the suite causes besides
# itself - external-update.yaml runs land in the primary repo, which the
# reconcile gate (scoped to this repo) does not enumerate.
- name: Register orchestrate run
uses: stablekernel/cascade/.github/actions/register-run@main
with:
run-id: ${{ steps.wait.outputs.run_id }}
expected-conclusion: success
reason: orchestrate-merge
upload: 'true'
# The manifest sets notify.deploy_name and notify.environment as explicit
# overrides. This satellite is build-only (one build "shared", no deploys, no
# environments), so WITHOUT the overrides the generator would emit
# deploy_name 'shared' and environment 'dev' in the Notify Primary Repo step
# (the build-name / default-env fallbacks). With the overrides it must emit
# 'artifact-a' and 'staging'. The fleet repin regenerates and commits
# orchestrate.yaml from this manifest before the suite runs, so the
# checked-out file is the authoritative regenerated artifact. Assert the
# overrides flowed through generation and the unconfigured defaults did not.
- name: Assert notify overrides reach the generated dispatch
env:
EXPECTED_DEPLOY_NAME: artifact-a
EXPECTED_ENVIRONMENT: staging
FALLBACK_DEPLOY_NAME: shared
FALLBACK_ENVIRONMENT: dev
run: |
set -euo pipefail
ORCHESTRATE=".github/workflows/orchestrate.yaml"
[ -f "$ORCHESTRATE" ] || { echo "::error::$ORCHESTRATE missing"; exit 1; }
# The Notify Primary Repo dispatch inputs are emitted as
# deploy_name: '<value>',
# environment: '<value>',
got_deploy="$(grep -oE "deploy_name: '[^']*'" "$ORCHESTRATE" | head -n1 | sed "s/.*'\\(.*\\)'.*/\\1/")"
got_env="$(grep -oE "environment: '[^']*'" "$ORCHESTRATE" | head -n1 | sed "s/.*'\\(.*\\)'.*/\\1/")"
if [ "$got_deploy" != "$EXPECTED_DEPLOY_NAME" ]; then
echo "::error::notify.deploy_name override did not reach dispatch: got '$got_deploy' want '$EXPECTED_DEPLOY_NAME'"
exit 1
fi
if [ "$got_env" != "$EXPECTED_ENVIRONMENT" ]; then
echo "::error::notify.environment override did not reach dispatch: got '$got_env' want '$EXPECTED_ENVIRONMENT'"
exit 1
fi
# Guard against a regression where the unconfigured fallbacks leak through.
if [ "$got_deploy" = "$FALLBACK_DEPLOY_NAME" ] || [ "$got_env" = "$FALLBACK_ENVIRONMENT" ]; then
echo "::error::generated dispatch carried the unconfigured fallback rather than the override"
exit 1
fi
echo "OK: notify overrides reached dispatch (deploy_name=$got_deploy environment=$got_env)"
{
echo "## Notify overrides"
echo "- generated deploy_name == override ($EXPECTED_DEPLOY_NAME): yes"
echo "- generated environment == override ($EXPECTED_ENVIRONMENT): yes"
echo "- unconfigured fallbacks ($FALLBACK_DEPLOY_NAME/$FALLBACK_ENVIRONMENT) not used: yes"
} >> "$GITHUB_STEP_SUMMARY"
- name: Assert sentinel external-update lands verbatim in primary state
id: assert_dispatch
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
PRIMARY_REPO: stablekernel/cascade-example-primary
# Primary-side external-deploy key and environment for THIS satellite.
# These are the keys the primary's external-update consumer requires
# (primary manifest external[].deploys[].name == "artifact-a", envs
# [staging, prod]); the cascade external update command rejects a
# deploy_name/environment that is not present in the primary config
# (external/command.go:139,150). Dispatch with the exact keys plus a
# UNIQUE sentinel sha, then correlate on that sentinel value - never on
# run ordering - so a stale primary run can never satisfy the assertion.
PRIMARY_DEPLOY_NAME: artifact-a
PRIMARY_ENV: staging
run: |
set -euo pipefail
# Unique sentinel SHA for this run. The dispatched --sha lands verbatim
# in primary state.<env>.external.<deploy_name>.sha (external/command.go:187),
# so the value itself is the correlation key.
SENTINEL="a$(date -u +%Y%m%d%H%M%S)$RANDOM"
echo "Sentinel sha for this run: $SENTINEL"
# primary's `external update` requires state.<env> to pre-exist
# (external/command.go:149). Ensure state.staging is present on primary
# before dispatching, writing only when absent so we never clobber a
# live promote/build state. Uses the Contents API so the write is a
# single atomic commit on primary main.
ensure_primary_env() {
local tmp resp cur_sha b64
tmp="$(mktemp)"
resp="$(gh api "repos/$PRIMARY_REPO/contents/.github/manifest.yaml?ref=main")"
cur_sha="$(echo "$resp" | jq -r '.sha')"
echo "$resp" | jq -r '.content' | base64 -d > "$tmp"
if [ "$(yq eval '.ci.state.'"$PRIMARY_ENV"' // ""' "$tmp")" != "" ]; then
echo "primary state.$PRIMARY_ENV already present; no seed needed"
return 0
fi
echo "seeding primary state.$PRIMARY_ENV (absent)"
yq eval -i '.ci.state.'"$PRIMARY_ENV"'.version = "v0.0.0-seed"' "$tmp"
b64="$(base64 -w0 "$tmp" 2>/dev/null || base64 "$tmp" | tr -d '\n')"
gh api "repos/$PRIMARY_REPO/contents/.github/manifest.yaml" -X PUT \
-f "message=chore: seed $PRIMARY_ENV state for external-update test [skip ci]" \
-f "content=$b64" \
-f "branch=main" \
-f "sha=$cur_sha" >/dev/null
}
ensure_primary_env
# Boundary used only to avoid resolving runs from earlier in this job.
# Sibling artifact suites dispatch into the same primary concurrently,
# so "newest run since boundary" can resolve a FOREIGN run (a different
# deploy_name). We therefore never gate on a sibling run's conclusion;
# the unique sentinel landing verbatim in our own slot is the only
# authoritative signal (external/command.go:187 writes --sha verbatim).
DISPATCH_SINCE="$(date -u -d '120 seconds ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \
|| date -u -v-120S +%Y-%m-%dT%H:%M:%SZ)"
gh workflow run external-update.yaml --repo "$PRIMARY_REPO" --ref main \
-f source_repo="$GITHUB_REPOSITORY" \
-f deploy_name="$PRIMARY_DEPLOY_NAME" \
-f environment="$PRIMARY_ENV" \
-f sha="$SENTINEL" \
-f version=v1.0.0 \
-f artifacts='{"artifact_id":"artifact-a-'"$SENTINEL"'"}'
# Best-effort: surface this dispatch's own run for log triage. Match the
# run whose displayTitle carries OUR sentinel so concurrent sibling runs
# are never mistaken for ours. A failure here is informational only and
# must NOT abort the assertion; the slot poll below is the hard gate.
RUN_ID=""
for i in $(seq 1 3); do
RUN_ID="$(gh run list --repo "$PRIMARY_REPO" --workflow=external-update.yaml \
--branch=main --created=">=$DISPATCH_SINCE" --limit=20 \
--json databaseId,displayTitle \
--jq 'map(select(.displayTitle | contains("'"$SENTINEL"'"))) | .[0].databaseId // empty')"
[ -n "$RUN_ID" ] && break
echo "attempt ${i}: our external-update run (sentinel $SENTINEL) not yet visible"
sleep 60
done
if [ -n "$RUN_ID" ]; then
echo "Our primary external-update run: ${RUN_ID} (watch is informational)"
gh run watch "$RUN_ID" --repo "$PRIMARY_REPO" --interval 60 || \
echo "note: watch on ${RUN_ID} returned non-zero; deferring to slot poll"
else
echo "note: could not correlate our run by sentinel; deferring to slot poll"
fi
# Read back the EXACT slot and assert the sentinel landed verbatim and
# that .repo records THIS source repo. Poll primary main until the
# finalize commit replicates.
GOT_SHA=""
GOT_REPO=""
for i in $(seq 1 3); do
resp="$(gh api "repos/$PRIMARY_REPO/contents/.github/manifest.yaml?ref=main")"
echo "$resp" | jq -r '.content' | base64 -d > /tmp/primary-manifest.yaml
GOT_SHA="$(yq eval '.ci.state.'"$PRIMARY_ENV"'.external.'"$PRIMARY_DEPLOY_NAME"'.sha // ""' /tmp/primary-manifest.yaml)"
GOT_REPO="$(yq eval '.ci.state.'"$PRIMARY_ENV"'.external.'"$PRIMARY_DEPLOY_NAME"'.repo // ""' /tmp/primary-manifest.yaml)"
[ "$GOT_SHA" = "$SENTINEL" ] && break
echo "attempt $i: primary slot sha='$GOT_SHA' (want '$SENTINEL')"
sleep 60
done
if [ "$GOT_SHA" != "$SENTINEL" ]; then
echo "::error::sentinel sha did not land in primary state.$PRIMARY_ENV.external.$PRIMARY_DEPLOY_NAME.sha: want '$SENTINEL' got '$GOT_SHA'"
exit 1
fi
if [ "$GOT_REPO" != "$GITHUB_REPOSITORY" ]; then
echo "::error::primary slot .repo mismatch: want '$GITHUB_REPOSITORY' got '$GOT_REPO'"
exit 1
fi
echo "OK: sentinel $SENTINEL landed verbatim in primary state.$PRIMARY_ENV.external.$PRIMARY_DEPLOY_NAME (repo=$GOT_REPO)"
# cascade #213 deploy-on-update: the primary external[].deploys[].on_update.deploy
# block makes the receiver run a scoped deploy_<deploy_name> job in the SAME
# external-update run after recording the update. Assert that scoped deploy
# actually ran and succeeded - record-only would skip it.
#
# Deterministic correlation: our --sha is the UNIQUE sentinel, and the
# receiver run carries it in its displayTitle, so we resolve OUR receiver
# run id by that sentinel (never latest-run, never log-grep). The sentinel
# already landed verbatim above, so the run provably exists; failing to
# resolve it is a hard error.
RECEIVER_RUN_ID=""
for i in $(seq 1 5); do
RECEIVER_RUN_ID="$(gh run list --repo "$PRIMARY_REPO" --workflow=external-update.yaml \
--branch=main --created=">=$DISPATCH_SINCE" --limit=20 \
--json databaseId,displayTitle \
--jq 'map(select(.displayTitle | contains("'"$SENTINEL"'"))) | .[0].databaseId // empty')"
[ -n "$RECEIVER_RUN_ID" ] && break
echo "attempt $i: receiver run carrying sentinel $SENTINEL not yet listable"
sleep 60
done
if [ -z "$RECEIVER_RUN_ID" ]; then
echo "::error::deploy-on-update (#213): could not resolve receiver external-update run for sentinel $SENTINEL; expected a run whose displayTitle contains the sentinel, got none"
exit 1
fi
echo "Receiver external-update run for deploy assertion: $RECEIVER_RUN_ID"
# Let the receiver run (update + scoped deploy_<name>) finish before reading
# job conclusions; watch is the completion gate, not the correctness gate.
gh run watch "$RECEIVER_RUN_ID" --repo "$PRIMARY_REPO" --interval 60 || \
echo "note: watch on receiver $RECEIVER_RUN_ID returned non-zero; asserting job conclusion directly"
# Assert the scoped deploy job for THIS component exists in the receiver run
# and concluded success. The generator emits job id deploy_<deploy_name> with
# name "Deploy <deploy_name>" (internal/generate/external.go writeDeployJob).
DEPLOY_CONCLUSION="$(gh run view "$RECEIVER_RUN_ID" --repo "$PRIMARY_REPO" --json jobs \
--jq '.jobs[] | select(.name | startswith("Deploy '"$PRIMARY_DEPLOY_NAME"'")) | .conclusion' | head -n1)"
if [ -z "$DEPLOY_CONCLUSION" ]; then
echo "::error::deploy-on-update (#213): receiver run $RECEIVER_RUN_ID has no scoped deploy job for '$PRIMARY_DEPLOY_NAME'; expected a 'Deploy $PRIMARY_DEPLOY_NAME' job, found none"
exit 1
fi
if [ "$DEPLOY_CONCLUSION" != "success" ]; then
echo "::error::deploy-on-update (#213): scoped deploy job for '$PRIMARY_DEPLOY_NAME' in receiver run $RECEIVER_RUN_ID concluded '$DEPLOY_CONCLUSION', expected 'success'"
exit 1
fi
echo "OK: deploy-on-update (#213) scoped deploy job 'Deploy $PRIMARY_DEPLOY_NAME' concluded success in receiver run $RECEIVER_RUN_ID"
- name: Write summary
if: always()
run: |
{
echo "## Scenario Suite Results"
echo ""
echo "| Check | Result |"
echo "|-------|--------|"
echo "| Build seeded via PR | ${{ steps.seed.outcome }} |"
echo "| Orchestrate completed | ${{ steps.wait.outcome }} |"
echo "| Sentinel landed verbatim in primary state | ${{ steps.assert_dispatch.outcome }} |"
} >> "$GITHUB_STEP_SUMMARY"
# ---------------------------------------------------------------------
# Reconcile gate: structural coverage backstop. The suite's own asserts above
# still gate each run they wait on; this job is additive. It enumerates EVERY
# run this repo produced since window-start (captured before the seed merge)
# and fails if any is unaccounted for in the ledger the register-run step
# uploaded - an unregistered non-success run, or a registered run that
# concluded other than its expected conclusion. That turns any fire-and-forget
# run the suite forgets to gate into a hard red.
reconcile:
name: Reconcile scenario-window runs
needs: [scenario-suite]
if: always()
uses: stablekernel/cascade/.github/workflows/fleet-reconcile.yaml@main
permissions:
contents: read
actions: read
with:
window-start: ${{ needs.scenario-suite.outputs.window-start }}
# Artifact mode: the register-run step uploaded the ledger under the
# default cascade-run-ledger-* name; reconcile globs and merges it.
cascade-ref: main