Skip to content

Scenario Suite

Scenario Suite #154

name: Scenario Suite
# Bespoke end-to-end validation driver for the release-only example.
# It exercises the full lifecycle against the real repository: reset to a
# clean slate, fire orchestrate by committing under src/, assert the RC
# draft/prerelease is minted, dispatch promote to publish, then assert the
# published release and tag cleanup. This file is hand-written and is not
# part of the workflow set that cascade produces.
on:
workflow_dispatch:
inputs:
orchestrate_timeout_minutes:
description: 'Max minutes to wait for the orchestrate run to mint an RC'
type: string
default: '8'
promote_timeout_minutes:
description: 'Max minutes to wait for the promote run to publish'
type: string
default: '8'
cascade_version:
description: 'cascade rc tag to self-repin to (e.g. v0.16.2-rc.2). Empty runs committed defaults.'
required: false
default: ''
cascade_version_sha:
description: 'Peeled commit SHA paired with cascade_version. Empty runs committed defaults.'
required: false
default: ''
schedule:
# Daily smoke run at 06:30 UTC.
- cron: '30 6 * * *'
permissions:
contents: write
actions: write
pull-requests: write
concurrency:
group: scenario-suite
cancel-in-progress: false
jobs:
scenario:
name: Release lifecycle scenario
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
# Captured before anything is dispatched. The reconcile job enumerates
# every run this repo produced at or after this instant and fails if any
# is unaccounted for in the ledger. Recorded first so no run the suite
# causes can fall outside the window.
window-start: ${{ steps.window.outputs.window-start }}
env:
MANIFEST_FILE: .github/manifest.yaml
MANIFEST_KEY: ci
TAG_PREFIX: rel-
BOT_NAME: 'github-actions[bot]'
BOT_EMAIL: '41898282+github-actions[bot]@users.noreply.github.com'
steps:
- name: Install gh transient-retry wrapper
run: |
cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY'
_gh_is_transient() {
local out="$1"
if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then
return 0
fi
if printf '%s' "$out" | grep -qiE 'HTTP 403'; then
if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then
return 0
fi
fi
return 1
}
gh() {
local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc
while :; do
out="$(command gh "$@" 2>&1)" && rc=0 || rc=$?
if [ "$rc" -eq 0 ]; then
printf '%s\n' "$out"
return 0
fi
if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then
printf '%s\n' "$out" >&2
return "$rc"
fi
printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
GHRETRY
echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV"
# The reconcile window opens here, before the first dispatch or merge.
# Every run the suite goes on to cause lands at or after this timestamp,
# so the reconcile job sees all of them.
- name: Open reconcile window
id: window
run: |
set -euo pipefail
WINDOW_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "window-start=$WINDOW_START" >> "$GITHUB_OUTPUT"
echo "reconcile window opened at $WINDOW_START"
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.CASCADE_STATE_TOKEN }}
- name: Log cascade version mode
run: |
if [ -n "${{ inputs.cascade_version }}" ]; then
echo "Running against dispatched rc: ${{ inputs.cascade_version }} (sha ${{ inputs.cascade_version_sha }})"
else
echo "Running against committed defaults (no rc dispatched)"
fi
- name: Setup CLI
uses: stablekernel/cascade/.github/actions/setup-cli@v0.16.1
with:
token: ${{ secrets.GITHUB_TOKEN }}
version: ${{ inputs.cascade_version || 'v0.8.0' }}
- name: Configure git identity
run: |
git config user.name "$BOT_NAME"
git config user.email "$BOT_EMAIL"
- name: Self-repin manifest to the dispatched rc
uses: stablekernel/cascade/.github/actions/fleet-repin@main
with:
cascade_version: ${{ inputs.cascade_version }}
cascade_version_sha: ${{ inputs.cascade_version_sha }}
token: ${{ secrets.CASCADE_STATE_TOKEN }}
# ------------------------------------------------------------------
# Stage 0 - RESET: wipe every release, tag, and state entry so the run
# always starts from a clean slate.
# ------------------------------------------------------------------
- name: Reset to clean slate
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
run: |
set -euo pipefail
echo "Resetting releases, tags, and manifest state..."
gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \
| while read -r t; do gh release delete "$t" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag 2>/dev/null || true; done
git fetch --tags --quiet || true
for t in $(git tag -l 'v*' 'rel-*'); do git push origin --delete "$t" 2>/dev/null || true; done
# cascade reset reads trunk, rewrites manifest state, and pushes the
# result. On the live fleet a concurrent writer can advance main between
# the read and the push, so a plain push is rejected non-fast-forward.
# Retry with bounded, growing backoff: on a rejected push, refresh the
# checkout to the current trunk tip and re-run reset so it re-reads the
# now-current main, re-applies the reset, and re-pushes. Fail closed if
# trunk keeps advancing past the attempt budget.
reset_pushed=false
for attempt in 1 2 3 4 5; do
if cascade reset --state --push; then
reset_pushed=true
break
fi
echo "reset push attempt ${attempt} rejected; refreshing trunk and retrying"
git fetch origin main --quiet
git reset --hard origin/main >/dev/null
sleep "$((attempt * 5))"
done
if [ "${reset_pushed}" != true ]; then
echo "::error::cascade reset --state --push failed after 5 attempts; trunk kept advancing"
exit 1
fi
# Re-sync the local checkout to whatever reset pushed to trunk.
git fetch origin "${GITHUB_REF_NAME}"
git reset --hard "origin/${GITHUB_REF_NAME}"
echo "Reset complete."
# ------------------------------------------------------------------
# Stage 1 - FIRE ORCHESTRATE: open a PR with a src/ change, rebase-merge
# it so the branch commit lands on main carrying a feat: subject, which
# matches the orchestrate path filter and triggers a version bump.
# ------------------------------------------------------------------
- name: Open and merge src-change PR to fire orchestrate
id: fire
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
run: |
set -euo pipefail
git config user.name "$BOT_NAME"
git config user.email "$BOT_EMAIL"
# Unique branch so re-runs never collide.
STAMP="$(date -u +%Y%m%d%H%M%S)-${GITHUB_RUN_ID}"
BRANCH="scenario/src-${STAMP}"
git fetch origin main --quiet
git checkout -B "$BRANCH" origin/main
# Write under src/ to match the orchestrate path filter.
mkdir -p src
printf 'scenario run %s\n' "$STAMP" > src/scenario.txt
git add src/scenario.txt
# The branch commit message carries the conventional type and, under
# rebase, is preserved as the trunk commit subject.
git commit --no-gpg-sign -m "feat: exercise release orchestration ${STAMP}"
git push origin "$BRANCH"
# Open PR and rebase-merge so the branch commit lands on main and the
# push event fires orchestrate.
gh pr create \
--base main \
--head "$BRANCH" \
--title "feat: exercise release orchestration ${STAMP}" \
--body "Automated scenario run; drives orchestrate on merge."
gh pr merge "$BRANCH" --rebase --delete-branch
# Capture the exact merge SHA on trunk - used to key the wait loop.
git fetch origin main --quiet
MERGE_SHA="$(git rev-parse origin/main)"
echo "merge_sha=${MERGE_SHA}" >> "$GITHUB_OUTPUT"
echo "Rebase-merged ${BRANCH} -> main at ${MERGE_SHA}."
- name: Wait for orchestrate run to complete
id: orchestrate_wait
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
WAIT_MINUTES: ${{ github.event.inputs.orchestrate_timeout_minutes || '8' }}
run: |
set -euo pipefail
MERGE_SHA="${{ steps.fire.outputs.merge_sha }}"
MAX_ATTEMPTS=$(( WAIT_MINUTES )) # one attempt every 60s
# Locate the orchestrate run triggered by this exact merge commit.
RUN_ID=""
for i in $(seq 1 "$MAX_ATTEMPTS"); do
RUN_ID="$(gh run list \
--repo "${GITHUB_REPOSITORY}" \
--workflow orchestrate.yaml \
--branch main \
--json databaseId,headSha,status \
--jq ".[] | select(.headSha==\"${MERGE_SHA}\") | .databaseId" \
2>/dev/null | head -n1 || true)"
if [ -n "$RUN_ID" ]; then
echo "Found orchestrate run ${RUN_ID} for ${MERGE_SHA}."
break
fi
echo "attempt ${i}/${MAX_ATTEMPTS}: no orchestrate run for ${MERGE_SHA} yet; waiting..."
sleep 60
done
if [ -z "$RUN_ID" ]; then
echo "::error::No orchestrate run found for merge SHA ${MERGE_SHA} after ${WAIT_MINUTES}m."
exit 1
fi
# Surface the resolved run id so the fleet reconcile gate can register
# it. Emitted as soon as it is known, before the poll below, so the
# ledger captures it even if the conclusion poll later fails it.
echo "orchestrate_run_id=${RUN_ID}" >> "$GITHUB_OUTPUT"
# Poll the located run to completion.
attempt=0
while [ "$attempt" -lt "$MAX_ATTEMPTS" ]; do
attempt=$(( attempt + 1 ))
LINE="$(gh run view "$RUN_ID" \
--repo "${GITHUB_REPOSITORY}" \
--json status,conclusion \
--jq '"\(.status) \(.conclusion)"' 2>/dev/null || true)"
status="${LINE%% *}"
conclusion="${LINE##* }"
echo "poll ${attempt}/${MAX_ATTEMPTS}: run ${RUN_ID} status='${status}' conclusion='${conclusion}'"
if [ "$status" = "completed" ]; then
if [ "$conclusion" = "success" ]; then
echo "Orchestrate run completed successfully."
exit 0
fi
echo "::error::Orchestrate run ${RUN_ID} completed with conclusion '${conclusion}' (expected success)."
exit 1
fi
sleep 60
done
echo "::error::Timed out after ${WAIT_MINUTES}m waiting for orchestrate run ${RUN_ID} to complete."
exit 1
# Register the orchestrate run in the fleet ledger so the reconcile gate
# accounts for it. Runs whenever the run id resolved, even if the wait
# step above failed it, so a registered run is never silently dropped.
- name: Register orchestrate run
if: always() && steps.orchestrate_wait.outputs.orchestrate_run_id != ''
uses: stablekernel/cascade/.github/actions/register-run@main
with:
run-id: ${{ steps.orchestrate_wait.outputs.orchestrate_run_id }}
expected-conclusion: success
reason: orchestrate-merge
upload: 'true'
# ------------------------------------------------------------------
# Stage 1 assertions: an RC draft/prerelease exists, a matching rel-*-rc.*
# tag exists, and manifest prerelease state is populated.
# ------------------------------------------------------------------
- name: Assert RC draft minted
id: stage1
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
# Pull the latest trunk so we read the state the orchestrate run wrote.
git fetch origin "${GITHUB_REF_NAME}"
git reset --hard "origin/${GITHUB_REF_NAME}"
# 1) A GitHub release tagged rel-*-rc.* that is a draft or prerelease.
RC_RELEASE=""
for attempt in 1 2; do
RC_RELEASE="$(gh release list \
--repo "${GITHUB_REPOSITORY}" \
--limit 50 \
--json tagName,isDraft,isPrerelease \
--jq "[.[] | select(.tagName | test(\"^${TAG_PREFIX}.*-rc\\\\.\")) | select(.isDraft or .isPrerelease)] | .[0].tagName // \"\"" 2>/dev/null || true)"
if [ -n "$RC_RELEASE" ]; then
break
fi
echo "RC release not visible yet (attempt ${attempt}); waiting..."
sleep 60
done
if [ -z "$RC_RELEASE" ]; then
echo "::error::No draft/prerelease release tagged ${TAG_PREFIX}*-rc.* was found."
gh release list --repo "${GITHUB_REPOSITORY}" --limit 50 || true
exit 1
fi
echo "Observed RC release: ${RC_RELEASE}"
echo "rc_tag=${RC_RELEASE}" >> "$GITHUB_OUTPUT"
# 2) A matching rel-*-rc.* git tag exists on the remote.
RC_GIT_TAG="$(git ls-remote --tags origin \
| sed -n 's@.*refs/tags/@@p' \
| grep -E "^${TAG_PREFIX}.*-rc\." \
| grep -v '\^{}$' \
| head -n1 || true)"
if [ -z "$RC_GIT_TAG" ]; then
echo "::error::No remote git tag matching ${TAG_PREFIX}*-rc.* was found."
exit 1
fi
echo "Observed RC git tag: ${RC_GIT_TAG}"
# 3) Manifest prerelease state is populated.
STATE_VERSION="$(yq eval ".${MANIFEST_KEY}.state.prerelease.version // \"\"" "$MANIFEST_FILE")"
STATE_BY="$(yq eval ".${MANIFEST_KEY}.state.prerelease.committed_by // \"\"" "$MANIFEST_FILE")"
if [ -z "$STATE_VERSION" ] || [ "$STATE_VERSION" = "null" ]; then
echo "::error::ci.state.prerelease.version is empty after orchestrate."
exit 1
fi
if [ -z "$STATE_BY" ] || [ "$STATE_BY" = "null" ]; then
echo "::error::ci.state.prerelease.committed_by is empty after orchestrate."
exit 1
fi
echo "Observed prerelease state: version=${STATE_VERSION} committed_by=${STATE_BY}"
{
echo "stage1_release=${RC_RELEASE}"
echo "stage1_tag=${RC_GIT_TAG}"
echo "stage1_version=${STATE_VERSION}"
} >> "$GITHUB_OUTPUT"
echo "Stage 1 PASS: RC draft '${RC_RELEASE}' minted, tag '${RC_GIT_TAG}' present, state version '${STATE_VERSION}'."
# ------------------------------------------------------------------
# Stage 2 - PROMOTE: dispatch promote.yaml to publish the prerelease
# into a published, latest release and clean up the RC tag(s).
# ------------------------------------------------------------------
- name: Dispatch promote workflow
id: dispatch
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
# Capture the UTC instant immediately before dispatch so the wait step
# can select only the run this step triggered, never a prior promote.
DISPATCH_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "dispatch_ts=${DISPATCH_TS}" >> "$GITHUB_OUTPUT"
echo "Dispatching promote.yaml (mode=default, dry_run=false) at ${DISPATCH_TS}..."
gh workflow run promote.yaml \
--repo "${GITHUB_REPOSITORY}" \
--ref "${GITHUB_REF_NAME}" \
-f mode=default \
-f dry_run=false
- name: Wait for promote run to complete
id: promote_wait
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WAIT_MINUTES: ${{ github.event.inputs.promote_timeout_minutes || '8' }}
run: |
set -euo pipefail
DISPATCH_TS="${{ steps.dispatch.outputs.dispatch_ts }}"
MAX_ATTEMPTS=$(( WAIT_MINUTES )) # one attempt every 60s
# Locate the promote run this suite dispatched: the first run created
# at or after the captured dispatch timestamp.
RUN_ID=""
for i in $(seq 1 "$MAX_ATTEMPTS"); do
RUN_ID="$(gh run list \
--repo "${GITHUB_REPOSITORY}" \
--workflow promote.yaml \
--created ">=${DISPATCH_TS}" \
--json databaseId,status \
--jq '.[0].databaseId // empty' \
2>/dev/null || true)"
if [ -n "$RUN_ID" ]; then
echo "Found promote run ${RUN_ID} dispatched at ${DISPATCH_TS}."
break
fi
echo "attempt ${i}/${MAX_ATTEMPTS}: no promote run created since ${DISPATCH_TS} yet; waiting..."
sleep 60
done
if [ -z "$RUN_ID" ]; then
echo "::error::No promote run created at or after ${DISPATCH_TS} after ${WAIT_MINUTES}m."
exit 1
fi
# Surface the resolved run id for the fleet reconcile gate, before the
# conclusion poll so the ledger captures it regardless of outcome.
echo "promote_run_id=${RUN_ID}" >> "$GITHUB_OUTPUT"
# Poll the located run to completion.
attempt=0
while [ "$attempt" -lt "$MAX_ATTEMPTS" ]; do
attempt=$(( attempt + 1 ))
LINE="$(gh run view "$RUN_ID" \
--repo "${GITHUB_REPOSITORY}" \
--json status,conclusion \
--jq '"\(.status) \(.conclusion)"' 2>/dev/null || true)"
status="${LINE%% *}"
conclusion="${LINE##* }"
echo "poll ${attempt}/${MAX_ATTEMPTS}: run ${RUN_ID} status='${status}' conclusion='${conclusion}'"
if [ "$status" = "completed" ]; then
if [ "$conclusion" = "success" ]; then
echo "Promote run completed successfully."
exit 0
fi
echo "::error::Promote run ${RUN_ID} completed with conclusion '${conclusion}' (expected success)."
exit 1
fi
sleep 60
done
echo "::error::Timed out after ${WAIT_MINUTES}m waiting for promote run ${RUN_ID} to complete."
exit 1
# Register the promote run in the fleet ledger so the reconcile gate
# accounts for it. Runs whenever the run id resolved, even if the wait
# step above failed it.
- name: Register promote run
if: always() && steps.promote_wait.outputs.promote_run_id != ''
uses: stablekernel/cascade/.github/actions/register-run@main
with:
run-id: ${{ steps.promote_wait.outputs.promote_run_id }}
expected-conclusion: success
reason: promote-publish
upload: 'true'
# ------------------------------------------------------------------
# Stage 2 assertions: a published latest release tagged rel-<semver>
# (no -rc.) exists, the RC tag(s) are deleted, and the body carries
# inline contributor attribution (the (@username) form produced by
# changelog.contributors: true).
# ------------------------------------------------------------------
- name: Assert published release
id: stage2
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
# 1) A published (not draft, not prerelease) release tagged
# rel-<semver> with no -rc. segment, marked latest.
PUB_TAG=""
for attempt in 1 2; do
PUB_TAG="$(gh release list \
--repo "${GITHUB_REPOSITORY}" \
--limit 50 \
--json tagName,isDraft,isPrerelease,isLatest \
--jq "[.[] | select(.tagName | test(\"^${TAG_PREFIX}\")) | select((.tagName | test(\"-rc\\\\.\")) | not) | select((.isDraft or .isPrerelease) | not)] | .[0].tagName // \"\"" 2>/dev/null || true)"
if [ -n "$PUB_TAG" ]; then
break
fi
echo "Published release not visible yet (attempt ${attempt}); waiting..."
sleep 60
done
if [ -z "$PUB_TAG" ]; then
echo "::error::No published (non-draft, non-prerelease) release tagged ${TAG_PREFIX}<semver> was found."
gh release list --repo "${GITHUB_REPOSITORY}" --limit 50 || true
exit 1
fi
echo "Observed published release: ${PUB_TAG}"
# Confirm it is flagged latest.
IS_LATEST="$(gh release list \
--repo "${GITHUB_REPOSITORY}" \
--limit 50 \
--json tagName,isLatest \
--jq "[.[] | select(.tagName == \"${PUB_TAG}\")] | .[0].isLatest // false")"
if [ "$IS_LATEST" != "true" ]; then
echo "::error::Published release ${PUB_TAG} is not marked latest."
exit 1
fi
# 2) The published semver git tag exists and the RC tag(s) are gone.
git fetch --prune --tags origin
REMOTE_TAGS="$(git ls-remote --tags origin \
| sed -n 's@.*refs/tags/@@p' \
| grep -v '\^{}$' || true)"
if ! printf '%s\n' "$REMOTE_TAGS" | grep -qx "$PUB_TAG"; then
echo "::error::Published semver tag ${PUB_TAG} is not present on the remote."
exit 1
fi
LEFTOVER_RC="$(printf '%s\n' "$REMOTE_TAGS" | grep -E "^${TAG_PREFIX}.*-rc\." || true)"
if [ -n "$LEFTOVER_RC" ]; then
echo "::error::RC tag(s) were not cleaned up after publish: ${LEFTOVER_RC}"
exit 1
fi
echo "RC tags cleaned up; published tag ${PUB_TAG} present."
# 3) The release body carries inline contributor attribution. With
# changelog.contributors: true, cascade appends a "(@username)"
# suffix to each changelog entry; it does NOT emit a separate
# "Contributors" heading. Assert the inline (@username) form.
BODY="$(gh release view "$PUB_TAG" \
--repo "${GITHUB_REPOSITORY}" \
--json body \
--jq '.body // ""')"
if ! printf '%s\n' "$BODY" | grep -qE '\(@[A-Za-z0-9-]+\)'; then
echo "::error::Published release body lacks inline (@username) contributor attribution."
printf '%s\n' "$BODY"
exit 1
fi
echo "Inline contributor attribution present in release body."
{
echo "stage2_release=${PUB_TAG}"
echo "stage2_tag=${PUB_TAG}"
} >> "$GITHUB_OUTPUT"
echo "Stage 2 PASS: published release '${PUB_TAG}' is latest, RC tags cleaned, inline attribution present."
- name: Write run summary
if: always()
env:
STAGE1_RELEASE: ${{ steps.stage1.outputs.stage1_release }}
STAGE1_TAG: ${{ steps.stage1.outputs.stage1_tag }}
STAGE1_VERSION: ${{ steps.stage1.outputs.stage1_version }}
STAGE2_RELEASE: ${{ steps.stage2.outputs.stage2_release }}
STAGE2_TAG: ${{ steps.stage2.outputs.stage2_tag }}
run: |
{
echo "## Scenario Suite"
echo ""
echo "| Stage | Release stage | GitHub release | Tag |"
echo "|-------|---------------|----------------|-----|"
echo "| 1 | RC draft / prerelease | ${STAGE1_RELEASE:-n/a} | ${STAGE1_TAG:-n/a} |"
echo "| 2 | Published (latest) | ${STAGE2_RELEASE:-n/a} | ${STAGE2_TAG:-n/a} |"
echo ""
if [ -n "${STAGE2_RELEASE:-}" ]; then
echo "**Overall: PASS** - prerelease ${STAGE1_VERSION:-?} published as ${STAGE2_RELEASE}."
else
echo "**Overall: see step logs for the failing assertion.**"
fi
} >> "$GITHUB_STEP_SUMMARY"
# ---------------------------------------------------------------------
# Reconcile gate: structural coverage backstop. The scenario job's own
# asserts above still gate each run they wait on; this job is additive. It
# enumerates EVERY run this repo produced since window-start (captured before
# the first merge) and fails if any is unaccounted for in the ledger the
# register-run steps uploaded - an unregistered non-success run, or a
# registered run that concluded other than its expected conclusion. That turns
# any fire-and-forget run the suite forgot to gate into a hard red.
reconcile:
name: Reconcile scenario-window runs
needs: [scenario]
if: always()
uses: stablekernel/cascade/.github/workflows/fleet-reconcile.yaml@main
permissions:
contents: read
actions: read
with:
window-start: ${{ needs.scenario.outputs.window-start }}
# Artifact mode: each register-run step uploaded a per-job ledger under
# the default cascade-run-ledger-* name; reconcile globs and merges them.
cascade-ref: main