Merge pull request #390 from kmarchais/release/v0.17.0 #1002

Workflow file for this run

.github/workflows/benchmark.yml at 418d170

	name: Benchmarks

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	workflow_dispatch:

	permissions:
	contents: write
	deployments: write
	pull-requests: write

	jobs:
	benchmark:
	runs-on: ubuntu-latest
	if: github.event_name != 'pull_request'
	# Share the gh-pages mutex with mike deploys (docs.yml, redeploy-docs.yml,
	# build-wheels.yml deploy-versioned-docs). Without this, the benchmark
	# auto-push step can interleave with a mike push and reject it with a
	# non-fast-forward error.
	concurrency:
	group: docs-deploy
	cancel-in-progress: false

	steps:
	- uses: actions/checkout@v7

	- name: Cache apt packages
	uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3
	with:
	packages: qtbase5-dev qt5-qmake libgl1-mesa-dev libglu1-mesa-dev freeglut3-dev
	version: 1.0

	- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
	with:
	python-version: "3.12"
	enable-cache: true

	- name: Install dependencies
	run: uv sync -v

	- name: Run benchmarks
	run: \|
	uv run pytest benchmarks/ \
	--benchmark-only \
	--benchmark-json=benchmark-results.json \
	--benchmark-group-by=group \
	--benchmark-sort=mean \
	--benchmark-warmup=on \
	--benchmark-min-rounds=5 \
	--benchmark-disable-gc

	- name: Store benchmark result
	uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
	with:
	tool: "pytest"
	output-file-path: benchmark-results.json
	benchmark-data-dir-path: bench
	github-token: ${{ secrets.GITHUB_TOKEN }}
	auto-push: true
	alert-threshold: "200%"
	comment-on-alert: true
	fail-on-alert: false

	- name: Upload benchmark results
	uses: actions/upload-artifact@v7
	with:
	name: benchmark-results
	path: benchmark-results.json
	retention-days: 30

	benchmark-pr:
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request'

	steps:
	# --- Setup ---
	- uses: actions/checkout@v7
	with:
	ref: ${{ github.event.pull_request.base.sha }}
	fetch-depth: 0

	- name: Cache apt packages
	uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3
	with:
	packages: qtbase5-dev qt5-qmake libgl1-mesa-dev libglu1-mesa-dev freeglut3-dev
	version: 1.0

	- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
	with:
	python-version: "3.12"
	enable-cache: true

	# --- Phase 1: Baseline (main) ---
	- name: Install dependencies (baseline)
	run: uv sync -v

	- name: Run baseline benchmarks
	run: \|
	uv run pytest benchmarks/ \
	--benchmark-only \
	--benchmark-json=baseline-results.json \
	--benchmark-group-by=group \
	--benchmark-sort=mean \
	--benchmark-warmup=on \
	--benchmark-min-rounds=5 \
	--benchmark-disable-gc

	# --- Phase 2: Current (PR branch) ---
	- uses: actions/checkout@v7
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	clean: false

	- name: Install dependencies (PR branch)
	run: uv sync -v

	- name: Run PR benchmarks
	run: \|
	uv run pytest benchmarks/ \
	--benchmark-only \
	--benchmark-json=current-results.json \
	--benchmark-group-by=group \
	--benchmark-sort=mean \
	--benchmark-warmup=on \
	--benchmark-min-rounds=5 \
	--benchmark-disable-gc

	# --- Phase 3: Compare ---
	- name: Compare benchmarks
	id: compare
	run: \|
	uv run python benchmarks/scripts/compare_benchmarks.py \
	--baseline baseline-results.json \
	--current current-results.json \
	--threshold 1.3 \
	--github-output comparison-report.md \
	--fail-on-regression

	- name: Generate benchmark summary
	run: \|
	uv run python << 'EOF'
	import json
	from pathlib import Path

	with open("current-results.json") as f:
	data = json.load(f)

	benchmarks = data.get("benchmarks", [])
	if not benchmarks:
	print("No benchmark results found")
	exit(0)

	# Group by benchmark group
	groups = {}
	for b in benchmarks:
	group = b.get("group", "default")
	if group not in groups:
	groups[group] = []
	groups[group].append(b)

	# Generate summary
	summary_lines = ["### Benchmark Results Summary\n"]
	summary_lines.append("\| Group \| Benchmarks \| Mean Time Range \|")
	summary_lines.append("\|-------\|------------\|-----------------\|")

	for group, benches in sorted(groups.items()):
	times = [b["stats"]["mean"] * 1000 for b in benches]
	min_t, max_t = min(times), max(times)
	summary_lines.append(f"\| {group} \| {len(benches)} \| {min_t:.2f}ms - {max_t:.2f}ms \|")

	summary_lines.append(f"\nTotal: {len(benchmarks)} benchmarks")

	# Add comparison results
	comparison_file = Path("comparison-report.md")
	if comparison_file.exists():
	summary_lines.append("\n---\n")
	summary_lines.append(comparison_file.read_text())

	summary = "\n".join(summary_lines)
	print(summary)

	Path("benchmark-summary.md").write_text(summary)
	EOF

	- name: Comment benchmark summary on PR
	uses: actions/github-script@v9
	with:
	script: \|
	const fs = require('fs');
	const summary = fs.readFileSync('benchmark-summary.md', 'utf8');

	const { data: comments } = await github.rest.issues.listComments({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	});

	const botComment = comments.find(c =>
	c.user.type === 'Bot' &&
	c.body.includes('### Benchmark Results Summary')
	);

	if (botComment) {
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: botComment.id,
	body: summary
	});
	} else {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body: summary
	});
	}

	- name: Upload benchmark results
	uses: actions/upload-artifact@v7
	with:
	name: benchmark-results
	path: \|
	baseline-results.json
	current-results.json
	comparison-report.md
	retention-days: 30

Navigation Menu

Merge pull request #390 from kmarchais/release/v0.17.0 #1002

Workflow file

Merge pull request #390 from kmarchais/release/v0.17.0 #1002

Uh oh!

Workflow file for this run