diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 143f6bf1..6c315cad 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -562,6 +562,16 @@ def _build_phases( ) ) + # Accuracy mirrors the perf load pattern so evaluation exercises the + # endpoint the same way it was benchmarked. AGENTIC_INFERENCE can't drive + # the (non-agentic) accuracy datasets — create_load_strategy rejects it — + # so it (and a missing perf pattern) falls back to MAX_THROUGHPUT. + perf_lp = ctx.rt_settings.load_pattern if ctx.rt_settings is not None else None + if perf_lp is None or perf_lp.type == LoadPatternType.AGENTIC_INFERENCE: + acc_load_pattern = LoadPattern(type=LoadPatternType.MAX_THROUGHPUT) + else: + acc_load_pattern = perf_lp + # Accuracy phases — use eval_cfg.dataset_name as phase name so it matches # what Scorer._load_sample_index_map() looks up in sample_idx_map.json for eval_cfg in ctx.eval_configs: @@ -574,15 +584,14 @@ def _build_phases( "AgenticInferenceDataset, which is not yet supported for " "accuracy evaluation." ) - # Accuracy phases run at MAX_THROUGHPUT; inheriting perf_lp (e.g. POISSON) - # would silently rate-limit evaluation until an agentic inference accuracy strategy - # and QPS-budgeting support are added. + logger.info( + "Accuracy issuer '%s' load mode: %s", + eval_cfg.dataset_name, + acc_load_pattern, + ) rng_settings = ctx.rt_settings or RuntimeSettings.from_config( ctx.config, acc_ds.num_samples() ) - acc_load_pattern: LoadPattern | None = LoadPattern( - type=LoadPatternType.MAX_THROUGHPUT - ) acc_settings = RuntimeSettings( metric_target=rng_settings.metric_target, reported_metrics=rng_settings.reported_metrics, diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index d1a834b1..b96843fe 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -718,6 +718,20 @@ def _validate_completeness(self) -> Self: ) return self + def __str__(self) -> str: + """Human-readable "type (param=value)" form for logging, e.g. + ``concurrency (target_concurrency=7)`` / ``poisson (target_qps=10.0)``. + Patterns without a driving parameter render as just the type name. + """ + if self.type in ( + LoadPatternType.CONCURRENCY, + LoadPatternType.AGENTIC_INFERENCE, + ): + return f"{self.type.value} (target_concurrency={self.target_concurrency})" + if self.type == LoadPatternType.POISSON: + return f"{self.type.value} (target_qps={self.target_qps})" + return self.type.value + @cyclopts.Parameter(name="*") class WarmupConfig(BaseModel): diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index 722b6864..491dc72b 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -16,8 +16,10 @@ """Tests for benchmark CLI models, config building, and command handlers.""" import asyncio +import dataclasses import io import json +import logging import random import tempfile from pathlib import Path @@ -1165,6 +1167,151 @@ def test_accuracy_drain_timeout_defaults_to_unbounded( acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY) assert acc.drain_timeout is None + @pytest.mark.unit + def test_accuracy_phase_inherits_perf_concurrency( + self, base_rt_settings, simple_dataset + ): + """When the perf phase runs CONCURRENCY, the accuracy phase mirrors the + same fixed concurrency instead of bursting at MAX_THROUGHPUT.""" + rt = dataclasses.replace( + base_rt_settings, + load_pattern=LoadPattern( + type=LoadPatternType.CONCURRENCY, target_concurrency=7 + ), + ) + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, rt, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + phases = _build_phases(ctx) + + acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY) + assert acc.runtime_settings.load_pattern is not None + assert acc.runtime_settings.load_pattern.type == LoadPatternType.CONCURRENCY + assert acc.runtime_settings.load_pattern.target_concurrency == 7 + + @pytest.mark.unit + def test_accuracy_phase_inherits_perf_poisson( + self, base_rt_settings, simple_dataset + ): + """POISSON perf: accuracy mirrors the same POISSON config (target_qps).""" + rt = dataclasses.replace( + base_rt_settings, + load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=10.0), + ) + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, rt, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + phases = _build_phases(ctx) + + acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY) + assert acc.runtime_settings.load_pattern is not None + assert acc.runtime_settings.load_pattern.type == LoadPatternType.POISSON + assert acc.runtime_settings.load_pattern.target_qps == 10.0 + + @pytest.mark.unit + def test_accuracy_phase_max_throughput_when_perf_offline( + self, base_rt_settings, simple_dataset + ): + """Offline (MAX_THROUGHPUT) perf leaves accuracy at MAX_THROUGHPUT.""" + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, base_rt_settings, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + phases = _build_phases(ctx) + + acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY) + assert acc.runtime_settings.load_pattern is not None + assert acc.runtime_settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT + + @pytest.mark.unit + def test_accuracy_phase_max_throughput_when_perf_agentic( + self, base_rt_settings, simple_dataset + ): + """AGENTIC_INFERENCE can't drive a non-agentic accuracy dataset, so the + accuracy phase falls back to MAX_THROUGHPUT instead of crashing.""" + rt = dataclasses.replace( + base_rt_settings, + load_pattern=LoadPattern( + type=LoadPatternType.AGENTIC_INFERENCE, target_concurrency=8 + ), + ) + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, rt, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + phases = _build_phases(ctx) + + acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY) + assert acc.runtime_settings.load_pattern is not None + assert acc.runtime_settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT + + @pytest.mark.unit + def test_accuracy_phase_max_throughput_when_perf_none( + self, base_rt_settings, simple_dataset + ): + """A missing perf load pattern falls back to MAX_THROUGHPUT for accuracy.""" + rt = dataclasses.replace(base_rt_settings, load_pattern=None) + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, rt, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + phases = _build_phases(ctx) + + acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY) + assert acc.runtime_settings.load_pattern is not None + assert acc.runtime_settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT + + @pytest.mark.unit + def test_accuracy_issuer_logs_load_mode( + self, base_rt_settings, simple_dataset, caplog + ): + """The accuracy issuer logs which load mode it will run in.""" + rt = dataclasses.replace( + base_rt_settings, + load_pattern=LoadPattern( + type=LoadPatternType.CONCURRENCY, target_concurrency=4 + ), + ) + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, rt, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + + with caplog.at_level( + logging.INFO, logger="inference_endpoint.commands.benchmark.execute" + ): + _build_phases(ctx) + + msgs = [r.getMessage() for r in caplog.records] + assert any( + "load mode" in m and "concurrency" in m and "4" in m for m in msgs + ), msgs + + @pytest.mark.unit + def test_accuracy_issuer_logs_poisson_when_perf_poisson( + self, base_rt_settings, simple_dataset, caplog + ): + """POISSON perf logs the inherited poisson mode with its target_qps and + must NOT emit a target_concurrency suffix (the concurrency-only branch).""" + rt = dataclasses.replace( + base_rt_settings, + load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=10.0), + ) + config = OfflineConfig(**_OFFLINE_KWARGS) + ctx = self._make_ctx(config, rt, simple_dataset) + ctx.eval_configs = [self._make_eval_config(simple_dataset)] + + with caplog.at_level( + logging.INFO, logger="inference_endpoint.commands.benchmark.execute" + ): + _build_phases(ctx) + + msgs = [r.getMessage() for r in caplog.records] + load_mode_msgs = [m for m in msgs if "load mode" in m] + assert load_mode_msgs, msgs + assert any( + "poisson" in m and "target_qps=10.0" in m for m in load_mode_msgs + ), load_mode_msgs + assert all( + "target_concurrency" not in m for m in load_mode_msgs + ), load_mode_msgs + @pytest.mark.unit def test_warmup_uses_independent_rng_instances( self, base_rt_settings, simple_dataset