Feat:Add B200 test and SgLang test results #32

Workflow file for this run

.github/workflows/validate_pr.yml at a2ec8f6

	name: Validate Submission

	# Triggers when results/, runner metadata, the platforms catalogue, the
	# README (which contains the auto-generated matrix), or the matrix
	# generator itself are touched in a PR.
	on:
	pull_request:
	paths:
	- 'results/**'
	- 'runners/**'
	- 'suites/**'
	- 'schema/**'
	- 'tools/generate_platforms_matrix.py'
	- 'README.md'
	- 'leaderboard/site/**'

	# Lock the default GITHUB_TOKEN down to read-only on repo contents for every
	# job (GitHub's hardening recommendation for OSS repos). Jobs that need more
	# (e.g. posting a PR comment) opt in explicitly via their own `permissions:`.
	permissions:
	contents: read

	jobs:
	validate-runners:
	name: Validate runner folders
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip

	- name: Install dependencies
	run: pip install jsonschema

	- name: Detect changed runner folders
	id: changed
	run: \|
	# Runner folders always match the schema pattern: {platform}_{name}_{8hexchars}
	# Filter on that regex instead of an allowlist — nothing to keep in sync.
	CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
	FOLDERS=""
	while IFS= read -r f; do
	if echo "$f" \| grep -qE '^runners/[^/]+/'; then
	folder=$(echo "$f" \| cut -d'/' -f2)
	if echo "$folder" \| grep -qE '^[a-z][a-z0-9]_[a-z][a-z0-9_]_[0-9a-f]{8}$'; then
	FOLDERS="$FOLDERS $folder"
	fi
	fi
	done <<< "$CHANGED"
	FOLDERS=$(echo "$FOLDERS" \| tr ' ' '\n' \| sort -u \| grep -v '^$' \| tr '\n' ' ')
	echo "folders=$FOLDERS" >> $GITHUB_OUTPUT
	echo "Detected runner folders: ${FOLDERS:-none}"

	- name: Validate changed runner folders
	if: steps.changed.outputs.folders != ''
	id: validate_runners
	run: \|
	FAILED=0
	for folder in ${{ steps.changed.outputs.folders }}; do
	runner_path="runners/$folder"
	[ -d "$runner_path" ] \|\| continue
	echo "::group::Validating $folder"
	if ! python runners/validate_runners.py --dir "$runner_path"; then
	echo "::error file=runners/$folder/meta.json::Validation failed for $folder"
	FAILED=1
	fi
	echo "::endgroup::"
	done
	echo "failed=$FAILED" >> $GITHUB_OUTPUT
	[ $FAILED -eq 0 ] \|\| exit 1

	- name: No runner folders changed
	if: steps.changed.outputs.folders == ''
	run: echo "No runner folders changed in this PR — skipping."

	# Always validate every runner folder (not just the ones touched in
	# this PR). This catches drift introduced by shared changes — e.g.
	# a meta.schema.json edit that breaks an unrelated existing runner.
	- name: Validate all runner folders (drift check)
	run: \|
	echo "::group::Validating every runner folder in the repo"
	python runners/validate_runners.py
	echo "::endgroup::"

	# README "Supported platforms" matrix is generated from each runner's
	# meta.json. If a PR changes a runner's suite_support / hardware_label
	# or adds a new runner without regenerating the table, fail.
	- name: README platforms matrix is in sync
	run: \|
	echo "::group::tools/generate_platforms_matrix.py --check"
	python tools/generate_platforms_matrix.py --check
	echo "::endgroup::"

	validate-suites:
	name: Validate suite definitions
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip

	- name: Install dependencies
	run: pip install jsonschema

	# Always validate every suite (and re-validate on schema changes too).
	# This catches drift introduced by shared changes — e.g. a
	# suite.schema.json edit that breaks an unrelated existing suite.
	- name: Validate all suite folders (drift check)
	run: \|
	echo "::group::Validating every suite folder in the repo"
	python runners/validate_suites.py
	echo "::endgroup::"

	validate:
	name: Validate result submissions
	runs-on: ubuntu-latest
	# Needs pull-requests:write to post the validation summary comment
	# via actions/github-script. After the FreedomIntelligence transfer
	# the default GITHUB_TOKEN is read-only on issues/pull-requests, so
	# this must be granted explicitly.
	permissions:
	contents: read
	pull-requests: write

	steps:
	- name: Checkout PR
	uses: actions/checkout@v4
	with:
	# Fetch base branch so we can diff against it
	fetch-depth: 0

	- name: Set up Python 3.11
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'

	- name: Install dependencies
	run: \|
	pip install jsonschema --quiet

	- name: Find changed submission directories
	id: find_dirs
	run: \|
	# Get list of changed files vs base branch
	git fetch origin ${{ github.base_ref }} --quiet
	CHANGED_DIRS=$(
	git diff --name-only origin/${{ github.base_ref }}...HEAD \
	\| grep '^results/' \
	\| awk -F'/' 'NF>=3 {print $1"/"$2"/"$3}' \
	\| sort -u
	)
	echo "Found changed directories:"
	echo "$CHANGED_DIRS"
	# Write to file to handle spaces/newlines safely
	echo "$CHANGED_DIRS" > /tmp/changed_dirs.txt
	# Also export count
	COUNT=$(echo "$CHANGED_DIRS" \| grep -c . \|\| true)
	echo "count=$COUNT" >> $GITHUB_OUTPUT

	- name: Validate each submission
	run: \|
	FAILED=0
	PASSED=0
	SKIPPED=0

	while IFS= read -r DIR; do
	# Skip empty lines
	[ -z "$DIR" ] && continue

	# Skip .gitkeep and non-directories
	if [ ! -d "$DIR" ]; then
	echo "Skipping non-directory: $DIR"
	SKIPPED=$((SKIPPED + 1))
	continue
	fi

	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "Validating: $DIR"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

	if python runners/validate_submission.py --dir "$DIR"; then
	echo "✓ PASSED: $DIR"
	PASSED=$((PASSED + 1))
	else
	echo "✗ FAILED: $DIR"
	FAILED=$((FAILED + 1))
	fi

	done < /tmp/changed_dirs.txt

	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "Summary: $PASSED passed, $FAILED failed, $SKIPPED skipped"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

	if [ "$FAILED" -gt 0 ]; then
	echo ""
	echo "PR blocked: $FAILED submission(s) failed validation."
	echo "Fix the errors listed above before merging."
	exit 1
	fi

	- name: Comment on PR with validation summary
	if: always()
	uses: actions/github-script@v7
	with:
	script: \|
	const outcome = '${{ job.status }}';
	const icon = outcome === 'success' ? '✅' : '❌';
	const status = outcome === 'success' ? 'All submissions valid' : 'Validation failed';
	const marker = '<!-- accelmark-validation-comment -->';
	const body = `${marker}\n## ${icon} AccelMark Validation: ${status}\n\nSee the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.`;

	// Update existing bot comment rather than accumulating new ones
	const { data: comments } = await github.rest.issues.listComments({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	});
	const existing = comments.find(c =>
	c.user.type === 'Bot' && c.body.includes(marker)
	);
	if (existing) {
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: existing.id,
	body,
	});
	} else {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body,
	});
	}

	frontend-tests:
	name: Frontend unit tests (modal viz)
	runs-on: ubuntu-latest
	# Frontend tests don't need full git history; a shallow checkout
	# plus Node 20 (which ships node:test) is enough.
	steps:
	- uses: actions/checkout@v4

	- uses: actions/setup-node@v4
	with:
	node-version: '20'

	# No npm install — the suite intentionally has zero runtime deps
	# (a hand-written DOM stub stands in for jsdom / happy-dom). Add
	# extra files to leaderboard/site/test/ to widen coverage; the
	# glob below picks them up automatically.
	- name: Run leaderboard frontend tests
	run: node --test leaderboard/site/test/*.test.mjs

	python-tests:
	name: Python unit tests (serve + skill)
	runs-on: ubuntu-latest
	# Lightweight checks for the FastAPI serve layer and the OpenClaw skill
	# entry point. No GPU, no real model — everything is mocked. Tests are
	# opt-in per package so missing deps in one folder don't take the rest
	# of the suite down with them.
	steps:
	- uses: actions/checkout@v4

	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip

	- name: Install test dependencies
	# numpy is pulled in transitively by loadgen (imported when serve.server
	# touches runners.benchmark_runner). Keep this list lean — these are the
	# only packages required to collect and run the unit tests; no torch,
	# no vendor SDKs, no real runner.
	run: \|
	pip install --quiet pytest pydantic fastapi httpx pyyaml jsonschema numpy

	- name: Run serve unit tests
	run: \|
	if [ -d serve/tests ]; then
	echo "::group::pytest serve/tests"
	python -m pytest serve/tests -q --no-header --color=no
	echo "::endgroup::"
	else
	echo "serve/tests/ not present — skipping."
	fi

	- name: Run OpenClaw skill unit tests
	run: \|
	if [ -d openclaw_skill/tests ]; then
	echo "::group::pytest openclaw_skill/tests"
	python -m pytest openclaw_skill/tests -q --no-header --color=no
	echo "::endgroup::"
	else
	echo "openclaw_skill/tests/ not present — skipping."
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Feat:Add B200 test and SgLang test results #32

Workflow file

Feat:Add B200 test and SgLang test results #32

Uh oh!

Workflow file for this run