chore: elevate client-read-fail log to WARN to surface bench failure … #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| format-check: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install clang-format | |
| run: sudo apt-get update && sudo apt-get install -y clang-format-15 | |
| - name: Check formatting | |
| run: | | |
| shopt -s globstar | |
| clang-format-15 --dry-run -Werror src/**/*.cpp src/**/*.h tools/**/*.cpp tests/**/*.cpp bench/**/*.cpp | |
| build-release: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cxx: [g++, clang++] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential cmake clang libssl-dev | |
| - name: Configure | |
| run: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} | |
| - name: Build | |
| run: cmake --build build -j | |
| - name: Run tests | |
| run: ctest --test-dir build --output-on-failure --timeout 90 | |
| build-asan: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake libssl-dev | |
| - name: Configure ASan | |
| run: cmake -S . -B build-asan -DCMAKE_BUILD_TYPE=Debug -DIR_ENABLE_ASAN=ON | |
| - name: Build | |
| run: cmake --build build-asan -j | |
| - name: Run tests under ASan/UBSan | |
| env: | |
| ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1 | |
| UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 | |
| run: ctest --test-dir build-asan --output-on-failure --timeout 120 | |
| chaos-smoke: | |
| runs-on: ubuntu-22.04 | |
| needs: build-release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake libssl-dev | |
| - name: Build | |
| run: | | |
| cmake -S . -B build -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build -j --target chaos | |
| - name: Run chaos smoke (10 clients x 50 requests, drain at t=1.5s) | |
| run: | | |
| mkdir -p bench | |
| ./build/chaos --clients 10 --requests 50 --sigterm-after-ms 1500 \ | |
| --threads 4 --pool-size 4 --shutdown-grace 30000 \ | |
| --out bench/chaos-smoke.json | |
| cat bench/chaos-smoke.json | |
| - name: Assert dropped == 0 | |
| run: | | |
| dropped=$(grep -o '"dropped_total": [0-9]*' bench/chaos-smoke.json | awk '{print $2}') | |
| echo "dropped=$dropped" | |
| test "$dropped" = "0" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: chaos-smoke | |
| path: bench/chaos-smoke.json | |
| chaos-tls: | |
| runs-on: ubuntu-22.04 | |
| needs: build-release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake libssl-dev openssl | |
| - name: Build | |
| run: | | |
| cmake -S . -B build -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build -j --target chaos | |
| - name: Generate test cert | |
| run: | | |
| mkdir -p tests/certs | |
| openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ | |
| -keyout tests/certs/key.pem -out tests/certs/cert.pem \ | |
| -subj "/C=US/O=inference-router/CN=localhost" | |
| - name: Run chaos over TLS (10 clients x 50 requests, drain at t=1.5s) | |
| run: | | |
| mkdir -p bench | |
| ./build/chaos --clients 10 --requests 50 --sigterm-after-ms 1500 \ | |
| --threads 4 --pool-size 4 --shutdown-grace 30000 \ | |
| --tls --tls-cert tests/certs/cert.pem --tls-key tests/certs/key.pem \ | |
| --out bench/chaos-tls.json | |
| cat bench/chaos-tls.json | |
| - name: Assert dropped == 0 | |
| run: | | |
| dropped=$(grep -o '"dropped_total": [0-9]*' bench/chaos-tls.json | awk '{print $2}') | |
| echo "dropped=$dropped" | |
| test "$dropped" = "0" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: chaos-tls | |
| path: bench/chaos-tls.json | |
| bench-smoke: | |
| runs-on: ubuntu-22.04 | |
| needs: build-release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake python3 libssl-dev | |
| - name: Build | |
| run: | | |
| cmake -S . -B build -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build -j --target load_bench | |
| - name: Run smoke bench (200 clients x 20 requests) | |
| run: | | |
| mkdir -p bench/results | |
| ./build/load_bench --clients 200 --reqs-per-client 20 \ | |
| --router-threads 16 --router-pool 64 --payload-bytes 64 \ | |
| --out bench/results/smoke.json | |
| cat bench/results/smoke.json | |
| - name: Assert JSON shape | |
| run: | | |
| grep -q '"throughput_rps":' bench/results/smoke.json | |
| grep -q '"p99_us":' bench/results/smoke.json | |
| grep -q '"connect_rate_per_s":' bench/results/smoke.json | |
| - name: Bench regression gate (30% drift) | |
| run: | | |
| python3 bench/regress.py --baseline bench/results/baseline.json \ | |
| --candidate bench/results/smoke.json --threshold 0.30 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-smoke | |
| path: bench/results/smoke.json | |
| bench-10k: | |
| runs-on: ubuntu-22.04 | |
| needs: build-release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake python3 libssl-dev | |
| - name: Raise file descriptor limit | |
| run: | | |
| ulimit -n 65536 || true | |
| ulimit -u 65536 || true | |
| - name: Build | |
| run: | | |
| cmake -S . -B build -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build -j --target load_bench | |
| - name: Run 10k-client bench (10000 clients x 50 requests = 500k total) | |
| run: | | |
| mkdir -p bench/results | |
| # Best-effort raise of soft limits; GH runners may refuse some, that's fine. | |
| ulimit -n 65536 || true | |
| ulimit -u 65536 || true | |
| ulimit -s 1024 || true | |
| # Sysctl tweaks to help the kernel reuse ephemeral ports faster under | |
| # the 500k-conn hammer; runners may not let us set these (rootless). | |
| sudo sysctl -w net.ipv4.tcp_tw_reuse=1 || true | |
| sudo sysctl -w net.ipv4.tcp_fin_timeout=15 || true | |
| sudo sysctl -w net.ipv4.ip_local_port_range="10000 65535" || true | |
| ./build/load_bench --clients 10000 --reqs-per-client 50 \ | |
| --router-threads 64 --router-pool 512 --client-stack-kb 96 \ | |
| --payload-bytes 64 --out bench/results/bench-10k.json | |
| cat bench/results/bench-10k.json | |
| - name: Assert correctness | |
| run: | | |
| ok=$(grep -oE '"ok": [0-9]+' bench/results/bench-10k.json | head -1 | awk '{print $2}') | |
| err=$(grep -oE '"err": [0-9]+' bench/results/bench-10k.json | head -1 | awk '{print $2}') | |
| throughput=$(grep -oE '"throughput_rps": [0-9.]+' bench/results/bench-10k.json | head -1 | awk '{print $2}') | |
| echo "ok=$ok err=$err throughput=$throughput" | |
| # The 10k-client bench saturates the ephemeral-port pool, the router's | |
| # worker queue, and the backend pool simultaneously. The test asserts | |
| # the floor — at least 50% of requests complete (the rest is queue | |
| # overflow / IO timeouts). The strict-success path is bench-smoke. | |
| # The bench's primary observable is the captured throughput / percentile | |
| # numbers, not pass-rate at 500k. | |
| test "$ok" -ge 250000 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-10k | |
| path: bench/results/bench-10k.json | |
| build-tsan: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake clang libssl-dev | |
| - name: Configure TSan | |
| run: >- | |
| cmake -S . -B build-tsan -DCMAKE_BUILD_TYPE=Debug -DIR_ENABLE_TSAN=ON | |
| -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ | |
| - name: Build | |
| run: cmake --build build-tsan -j | |
| - name: Run tests under TSan (50 client x 100 requests stress) | |
| env: | |
| TSAN_OPTIONS: halt_on_error=1:second_deadlock_stack=1:exitcode=66 | |
| run: ctest --test-dir build-tsan --output-on-failure --timeout 240 | |
| fuzz-wire: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake clang libssl-dev | |
| - name: Configure fuzz | |
| env: | |
| CC: clang | |
| CXX: clang++ | |
| CFLAGS: "-fsanitize=fuzzer-no-link,address,undefined -fno-omit-frame-pointer -g -O1" | |
| CXXFLAGS: "-fsanitize=fuzzer-no-link,address,undefined -fno-omit-frame-pointer -g -O1" | |
| LDFLAGS: "-fsanitize=fuzzer-no-link,address,undefined" | |
| run: >- | |
| cmake -S . -B build-fuzz -DCMAKE_BUILD_TYPE=Debug -DIR_BUILD_FUZZ=ON | |
| -DIR_BUILD_TESTS=OFF -DIR_BUILD_BENCH=OFF | |
| - name: Build fuzz target | |
| run: cmake --build build-fuzz -j --target fuzz_wire | |
| - name: Run wire-protocol fuzz (60s budget) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1 | |
| UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 | |
| run: | | |
| mkdir -p tests/fuzz/corpus-run | |
| cp tests/fuzz/corpus/* tests/fuzz/corpus-run/ | |
| ./build-fuzz/fuzz_wire -max_total_time=60 -print_final_stats=1 \ | |
| tests/fuzz/corpus-run tests/fuzz/corpus | tee fuzz.log | |
| echo "corpus_size_after=$(ls tests/fuzz/corpus-run | wc -l)" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: fuzz-corpus | |
| path: | | |
| tests/fuzz/corpus-run/ | |
| fuzz.log | |
| coverage: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: sudo apt-get update && sudo apt-get install -y build-essential cmake lcov libssl-dev | |
| - name: Configure coverage | |
| run: >- | |
| cmake -S . -B build-cov -DCMAKE_BUILD_TYPE=Debug -DIR_ENABLE_COVERAGE=ON | |
| -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ | |
| - name: Build | |
| run: cmake --build build-cov -j | |
| - name: Run tests | |
| run: ctest --test-dir build-cov --output-on-failure --timeout 240 | |
| - name: Capture coverage | |
| run: | | |
| # Ubuntu 22.04 ships lcov 1.15 which doesn't accept newer --ignore-errors | |
| # flags (mismatch/inconsistent/empty). Stick to the lowest-common-denominator | |
| # invocation; we exclude generated and third-party paths via --exclude. | |
| lcov --capture --directory build-cov --output-file build-cov/coverage.info \ | |
| --rc geninfo_unexecuted_blocks=1 | |
| lcov --remove build-cov/coverage.info \ | |
| '*/_deps/*' '*/tests/*' '/usr/*' '*/build-cov/*' \ | |
| --output-file build-cov/coverage.info | |
| lcov --summary build-cov/coverage.info 2>&1 | tee build-cov/coverage-summary.txt | |
| - name: Enforce coverage threshold | |
| # v1 establishes the coverage gate. Prior to v1 there was no measurement at | |
| # all; the new floor is set 5+ percentage points above the previous "0% gated" | |
| # baseline and intentionally below the measured line coverage so transient | |
| # variation (e.g. line counting differences across gcc minor versions) does | |
| # not fail builds. Future layers may raise this value monotonically. | |
| env: | |
| IR_COVERAGE_MIN: "60" | |
| run: | | |
| line_pct=$(grep -i '^[[:space:]]*lines' build-cov/coverage-summary.txt \ | |
| | head -1 | grep -oE '[0-9]+\.[0-9]+' | head -1) | |
| echo "measured_line_coverage=${line_pct}% threshold=${IR_COVERAGE_MIN}%" | |
| if [ -z "$line_pct" ]; then | |
| echo "FAIL: could not parse coverage from lcov output" >&2 | |
| cat build-cov/coverage-summary.txt | |
| exit 1 | |
| fi | |
| awk -v p="$line_pct" -v m="$IR_COVERAGE_MIN" \ | |
| 'BEGIN { if (p+0 < m+0) { printf "FAIL: %s%% < %s%%\n", p, m; exit 1 } else { printf "OK: %s%% >= %s%%\n", p, m } }' | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: coverage | |
| path: | | |
| build-cov/coverage.info | |
| build-cov/coverage-summary.txt |