Engine Comparison (TensorSharp vs llama.cpp) #260
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Engine Comparison (TensorSharp vs llama.cpp) | |
| # Cross-engine benchmark: TensorSharp vs llama.cpp on the same GGUF files. | |
| # | |
| # This replaces the old TensorSharp.TestMatrix-driven matrix (which fanned out | |
| # over backends x features x env-var sweeps and regularly timed out). The | |
| # simplified workflow does exactly four things: | |
| # | |
| # 1. Clones and builds llama.cpp (CUDA) and sets it up for the harness. | |
| # 2. Downloads the benchmark models from their Hugging Face pointers (the | |
| # `_hf` fields in benchmarks/engine_comparison/benchmark_config_ci.json) | |
| # and runs benchmarks/engine_comparison/run_matrix.py to compare | |
| # TensorSharp vs llama.cpp performance. | |
| # 3. Compares output quality between the two engines (report.py's | |
| # "Output quality" section: cross-engine similarity of greedy outputs, | |
| # JSON validity, tool-call correctness). | |
| # 4. Generates a single markdown report covering both performance and | |
| # quality (uploaded as an artifact + rendered into the job summary). | |
| # | |
| # Two profiles: | |
| # - smoke (pull_request): one small model (gemma4-12b), four cheap scenarios | |
| # (text_short, function_call, json_mode, prefill_4k). The report is posted | |
| # as a PR comment. Superseded runs on the same PR are cancelled. | |
| # - full (workflow_dispatch / weekly schedule): the whole CI model + | |
| # scenario set; dispatch inputs can select a custom subset. | |
| # | |
| # llama.cpp sources/build and the model files live in a persistent directory | |
| # on the runner ($HOME/tensorsharp-bench by default, overridable with the | |
| # BENCH_HOME repository variable), so only the first run on a fresh runner | |
| # pays the llama.cpp build + ~20 GB model download; repeat runs (including PR | |
| # smoke runs) only pay incremental costs. | |
| # | |
| # Self-hosted runner required: [self-hosted, tensorsharp-cuda] | |
| # - Linux + NVIDIA GPU with the CUDA toolkit (nvcc) on PATH | |
| # - .NET 10 SDK, cmake, git, python3 + pip on PATH | |
| # - ~40 GB free disk for llama.cpp builds + model files | |
| on: | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| models: | |
| description: "Comma-separated model ids from benchmark_config_ci.json (empty = default set)" | |
| required: false | |
| default: "" | |
| scenarios: | |
| description: "Comma-separated scenario ids (empty = default set)" | |
| required: false | |
| default: "" | |
| llama_ref: | |
| description: "llama.cpp git ref to build (branch, tag or SHA)" | |
| required: false | |
| default: "master" | |
| schedule: | |
| # Weekly regression point - Monday 03:00 UTC. | |
| - cron: "0 3 * * 1" | |
| concurrency: | |
| # One benchmark per ref. PR smoke runs are cancelled when new commits land; | |
| # full runs (dispatch / schedule) are long and expensive, so later triggers | |
| # queue behind them instead of cancelling. | |
| group: engine-comparison-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| benchmark: | |
| name: Benchmark (tensorsharp-cuda) | |
| runs-on: [self-hosted, tensorsharp-cuda] | |
| # PR smoke runs are small; only full runs need the long ceiling. | |
| timeout-minutes: ${{ github.event_name == 'pull_request' && 120 || 300 }} | |
| env: | |
| # pull_request -> trimmed smoke profile; dispatch/schedule -> inputs or | |
| # the full CI default set. | |
| BENCH_PROFILE: ${{ github.event_name == 'pull_request' && 'smoke (pull request)' || github.event_name == 'schedule' && 'full (weekly schedule)' || 'full / custom (manual dispatch)' }} | |
| BENCH_MODELS: ${{ github.event_name == 'pull_request' && 'gemma4-12b' || github.event.inputs.models || 'gemma4-12b,qwen36-35b-a3b' }} | |
| BENCH_SCENARIOS: ${{ github.event_name == 'pull_request' && 'text_short,function_call,json_mode,prefill_4k' || github.event.inputs.scenarios || 'text_short,text_long,multi_turn,function_call,json_mode,prefill_2k,prefill_4k,prefill_8k' }} | |
| LLAMA_REF: ${{ github.event.inputs.llama_ref || 'master' }} | |
| steps: | |
| # clean: false keeps the untracked incremental state that makes repeat | |
| # runs fast (TensorSharp.GGML.Native/build, ExternalProjects/ggml). | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| clean: false | |
| - name: Resolve persistent benchmark directory | |
| run: | | |
| BENCH_HOME="${{ vars.BENCH_HOME }}" | |
| BENCH_HOME="${BENCH_HOME:-$HOME/tensorsharp-bench}" | |
| mkdir -p "$BENCH_HOME" | |
| { | |
| echo "BENCH_HOME=$BENCH_HOME" | |
| echo "BENCH_MODEL_ROOT=$BENCH_HOME/models" | |
| echo "BENCH_RESULTS=$GITHUB_WORKSPACE/bench-results" | |
| } >> "$GITHUB_ENV" | |
| { | |
| echo "### Run profile" | |
| echo "" | |
| echo "- profile: \`$BENCH_PROFILE\`" | |
| echo "- models: \`$BENCH_MODELS\`" | |
| echo "- scenarios: \`$BENCH_SCENARIOS\`" | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Show host info | |
| run: | | |
| uname -a | |
| dotnet --version | |
| cmake --version | head -1 | |
| python3 --version | |
| nvidia-smi || true | |
| df -h "$BENCH_HOME" . | |
| - name: Build TensorSharp native GGML library (CUDA) | |
| run: bash TensorSharp.GGML.Native/build-linux.sh --cuda | |
| - name: Build TensorSharp.Server | |
| run: dotnet build TensorSharp.Server/TensorSharp.Server.csproj -c Release | |
| # ----- 1. Clone and build llama.cpp, set up its environment ----------- | |
| - name: Clone / update llama.cpp | |
| run: | | |
| set -euo pipefail | |
| LLAMA_SRC="$BENCH_HOME/llama.cpp" | |
| if [ ! -d "$LLAMA_SRC/.git" ]; then | |
| git clone https://github.com/ggml-org/llama.cpp "$LLAMA_SRC" | |
| fi | |
| git -C "$LLAMA_SRC" fetch --tags origin "$LLAMA_REF" | |
| git -C "$LLAMA_SRC" checkout --detach FETCH_HEAD | |
| echo "LLAMA_SRC=$LLAMA_SRC" >> "$GITHUB_ENV" | |
| echo "llama.cpp @ $(git -C "$LLAMA_SRC" log -1 --format='%h %s')" | |
| - name: Build llama-server (CUDA) | |
| run: | | |
| set -euo pipefail | |
| cmake -S "$LLAMA_SRC" -B "$LLAMA_SRC/build" \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DGGML_CUDA=ON \ | |
| -DLLAMA_CURL=OFF \ | |
| -DLLAMA_BUILD_SERVER=ON | |
| cmake --build "$LLAMA_SRC/build" --config Release --target llama-server -j "$(nproc)" | |
| LLAMA_BIN="$LLAMA_SRC/build/bin/llama-server" | |
| test -x "$LLAMA_BIN" | |
| echo "BENCH_LLAMA_SERVER=$LLAMA_BIN" >> "$GITHUB_ENV" | |
| { | |
| echo "### Engine versions" | |
| echo "" | |
| echo "- TensorSharp: \`$(git rev-parse --short HEAD)\`" | |
| echo "- llama.cpp: \`$(git -C "$LLAMA_SRC" rev-parse --short HEAD)\` (ref \`$LLAMA_REF\`)" | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # ----- 2. Download models from their Hugging Face pointers ------------ | |
| - name: Download benchmark models | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| python3 -m pip install --user -q -U huggingface_hub requests | |
| python3 benchmarks/engine_comparison/download_models.py \ | |
| --config benchmarks/engine_comparison/benchmark_config_ci.json \ | |
| --models "$BENCH_MODELS" | |
| # ----- 2. Run the performance comparison ------------------------------ | |
| - name: Run benchmark matrix (TensorSharp vs llama.cpp) | |
| id: bench | |
| working-directory: benchmarks/engine_comparison | |
| run: | | |
| set -euo pipefail | |
| rm -rf "$BENCH_RESULTS" | |
| python3 run_matrix.py --config benchmark_config_ci.json \ | |
| --models "$BENCH_MODELS" \ | |
| --scenarios "$BENCH_SCENARIOS" | |
| # ----- 3 + 4. Quality comparison + combined report --------------------- | |
| # report.py aggregates the per-cell JSONs into one markdown report with | |
| # both the performance tables/ratios and the output-quality section. | |
| # These steps run even when the benchmark step itself failed partway | |
| # (some cells may still be reportable) but not when it never ran. | |
| - name: Generate performance + quality report | |
| if: ${{ !cancelled() && steps.bench.outcome != 'skipped' }} | |
| working-directory: benchmarks/engine_comparison | |
| run: | | |
| python3 report.py --config benchmark_config_ci.json | |
| cp ../../docs/engine_comparison_report.md "$GITHUB_WORKSPACE/engine-comparison-report.md" | |
| - name: Upload report | |
| if: ${{ !cancelled() && steps.bench.outcome != 'skipped' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: engine-comparison-report | |
| path: engine-comparison-report.md | |
| retention-days: 90 | |
| - name: Upload per-cell results (JSON, logs, CSV) | |
| if: ${{ !cancelled() && steps.bench.outcome != 'skipped' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: engine-comparison-results | |
| path: bench-results | |
| retention-days: 30 | |
| - name: Render report into job summary | |
| if: ${{ !cancelled() && steps.bench.outcome != 'skipped' }} | |
| run: | | |
| if [ -f engine-comparison-report.md ]; then | |
| # GitHub job summaries cap at 1 MiB. | |
| head -c 900000 engine-comparison-report.md >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "(no report produced)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Fail on errored or empty benchmark cells | |
| run: | | |
| python3 - <<'EOF' | |
| import json, os, sys | |
| from pathlib import Path | |
| results = Path(os.environ["BENCH_RESULTS"]) | |
| cells = [json.loads(p.read_text(encoding="utf-8")) | |
| for p in results.glob("*.json")] | |
| failed = [c for c in cells if c.get("status") == "fail"] | |
| ok = [c for c in cells if c.get("status") == "ok"] | |
| for c in failed: | |
| print(f"FAIL {c['engine']}/{c['backend']}/{c['model']}/{c['scenario']}: " | |
| f"{c.get('detail', '')[:300]}") | |
| if failed: | |
| sys.exit(f"{len(failed)} benchmark cell(s) failed - see bench-results logs") | |
| if not ok: | |
| sys.exit("no benchmark cell ran successfully - check model/binary setup") | |
| print(f"{len(ok)} ok cells, 0 failed") | |
| EOF | |
| pr-comment: | |
| name: PR comment | |
| needs: benchmark | |
| if: github.event_name == 'pull_request' && always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: engine-comparison-report | |
| merge-multiple: true | |
| path: report | |
| - name: Post or update PR comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const marker = '<!-- engine-comparison-report -->'; | |
| let body = marker + '\n## Engine comparison — TensorSharp vs llama.cpp (PR smoke)\n\n'; | |
| const f = 'report/engine-comparison-report.md'; | |
| if (fs.existsSync(f)) { | |
| const txt = fs.readFileSync(f, 'utf8'); | |
| // PR comment bodies cap at 65536 chars. | |
| body += txt.length > 60000 | |
| ? txt.slice(0, 60000) + '\n\n_(truncated — see the engine-comparison-report artifact for the full report)_' | |
| : txt; | |
| } else { | |
| body += '_No report artifact was produced — the benchmark failed before generating results (see the workflow logs)._'; | |
| } | |
| const { owner, repo } = context.repo; | |
| const issue_number = context.issue.number; | |
| const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number }); | |
| const existing = comments.find(c => c.body && c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); | |
| } else { | |
| await github.rest.issues.createComment({ owner, repo, issue_number, body }); | |
| } |