Benchmark #9

Workflow file for this run

.github/workflows/benchmark.yml at 63efd0f

	name: Benchmark

	# Nightly run of the HTTP user-journey performance benchmark
	# (dev/benchmarks/omnigent). Seeds a sizeable corpus, boots a real server
	# against it, drives the journeys, and uploads the JSON report as an artifact.
	# Runs a backend matrix — SQLite (in-process) and Postgres (a service
	# container, matching prod's Lakebase/Postgres round-trip + pooling profile).
	# A workspace Databricks notebook pulls these artifacts via the GitHub API into
	# a Delta table for the trend dashboard (see dev/benchmarks/omnigent/README.md)
	# — so this workflow only produces artifacts; it never touches Databricks.
	#
	# Scheduled -> runs on the trusted default branch with the repo GITHUB_TOKEN;
	# it reads no PR-authored code. Also dispatchable for an ad-hoc run.

	on:
	schedule:
	- cron: "37 7 * * *" # 07:37 UTC nightly (off-peak, off the :00 mark)
	workflow_dispatch:
	inputs:
	iterations:
	description: "Requests per run"
	required: false
	default: "100"
	runs:
	description: "Timed runs per journey"
	required: false
	default: "3"
	sessions:
	description: "Seeded sessions"
	required: false
	default: "5000"
	items_per_session:
	description: "Seeded items per session"
	required: false
	default: "200"

	permissions:
	contents: read

	env:
	# No web SPA build during `uv sync` (setup.py _build_web_ui): this job never
	# serves the bundle, and the build otherwise times out on public npm.
	OMNIGENT_SKIP_WEB_UI: "true"
	UV_INDEX_URL: https://pypi.org/simple
	PIP_INDEX_URL: https://pypi.org/simple
	ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations \|\| '100' }}
	RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs \|\| '3' }}
	SESSIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.sessions \|\| '5000' }}
	ITEMS: ${{ github.event_name == 'workflow_dispatch' && inputs.items_per_session \|\| '200' }}

	concurrency:
	# Never cancel a scheduled run mid-flight (each is a distinct data point);
	# coalesce manual dispatches per ref.
	group: benchmark-${{ github.event_name }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}

	jobs:
	benchmark:
	name: Run benchmark (${{ matrix.backend }})
	if: github.repository == 'omnigent-ai/omnigent'
	runs-on: ubuntu-latest
	timeout-minutes: 30
	strategy:
	fail-fast: false
	matrix:
	backend: [sqlite, postgres]
	services:
	# A Postgres service is defined unconditionally (GitHub Actions has no
	# per-matrix-value service gating), but only the postgres leg connects to
	# it — the sqlite leg simply ignores it. postgres:16 mirrors Lakebase's
	# major version.
	postgres:
	image: postgres:16
	env:
	POSTGRES_PASSWORD: bench
	POSTGRES_DB: benchdb
	ports:
	- 5432:5432
	options: >-
	--health-cmd "pg_isready -U postgres"
	--health-interval 5s
	--health-timeout 5s
	--health-retries 10
	steps:
	- name: Check out repo
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
	with:
	persist-credentials: false

	- name: Set up Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version-file: ".python-version"

	- name: Set up uv
	uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v3
	with:
	enable-cache: true

	- name: Install dependencies
	# `databricks` extra carries psycopg[binary] for the Postgres backend.
	run: uv sync --extra dev --extra databricks

	# Resolve the DB URI + a stable seed-cache key for this backend. The
	# cache key binds the DB schema head + seed.py contents + corpus config,
	# so a schema change or seed edit busts the cache and forces a reseed —
	# the "you changed the schema, refresh the seed" contract (SQLite only;
	# the Postgres service is fresh each run so its DB is never cached).
	- name: Resolve DB target
	id: db
	run: \|
	HEAD="$(uv run --no-sync dev/benchmarks/omnigent/seed.py --print-head)"
	if [[ "${{ matrix.backend }}" == "postgres" ]]; then
	echo "uri=postgresql+psycopg://postgres:bench@localhost:5432/benchdb" >> "$GITHUB_OUTPUT"
	echo "cache_path=" >> "$GITHUB_OUTPUT"
	else
	echo "uri=sqlite:///$PWD/bench.db" >> "$GITHUB_OUTPUT"
	echo "cache_path=bench.db" >> "$GITHUB_OUTPUT"
	fi
	echo "cache_key=benchdb-${{ matrix.backend }}-$HEAD-${SESSIONS}x${ITEMS}-${{ hashFiles('dev/benchmarks/omnigent/seed.py') }}" >> "$GITHUB_OUTPUT"

	# Reuse a previously-seeded SQLite corpus when schema + seed + config are
	# unchanged. No-op for the postgres leg (empty path).
	- name: Restore seeded SQLite corpus
	if: matrix.backend == 'sqlite'
	id: seedcache
	uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
	with:
	path: ${{ steps.db.outputs.cache_path }}
	key: ${{ steps.db.outputs.cache_key }}

	- name: Seed corpus
	# Postgres always seeds (fresh service each run); SQLite seeds only on a
	# cache miss. seed.py is itself idempotent, so a stray hit is harmless.
	if: matrix.backend == 'postgres' \|\| steps.seedcache.outputs.cache-hit != 'true'
	run: \|
	uv run --no-sync dev/benchmarks/omnigent/seed.py \
	--database-uri "${{ steps.db.outputs.uri }}" \
	--sessions "$SESSIONS" --items-per-session "$ITEMS"

	- name: Run benchmark
	run: \|
	uv run --no-sync dev/benchmarks/omnigent/run.py \
	--database-uri "${{ steps.db.outputs.uri }}" \
	--iterations "$ITERATIONS" \
	--runs "$RUNS" \
	--output "benchmark-results-${{ matrix.backend }}.json"

	- name: Upload benchmark results
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	if: always()
	with:
	name: benchmark-results-${{ matrix.backend }}-${{ github.run_id }}
	path: benchmark-results-${{ matrix.backend }}.json
	retention-days: 90
	if-no-files-found: warn

Navigation Menu

Uh oh!

Benchmark #9

Workflow file

Benchmark #9

Uh oh!

Workflow file for this run