diff --git a/src/inference_endpoint/config/ruleset_registry.py b/src/inference_endpoint/config/ruleset_registry.py index 2769585d..99974037 100644 --- a/src/inference_endpoint/config/ruleset_registry.py +++ b/src/inference_endpoint/config/ruleset_registry.py @@ -28,6 +28,7 @@ from typing import TYPE_CHECKING +from .rulesets.mlcommons.rules import ALL_ROUNDS as mlcommons_rounds from .rulesets.mlcommons.rules import CURRENT as mlcommons_current from .rulesets.mlcommons.rules import EDGE_CURRENT as mlcommons_edge_current @@ -44,7 +45,16 @@ def register_ruleset(name: str, ruleset: BenchmarkSuiteRuleset) -> None: Args: name: Unique name/version identifier (e.g., "mlperf-inference-v5.1") ruleset: The ruleset instance to register + + Raises: + ValueError: If a ruleset is already registered under ``name`` (prevents + an accidental overwrite via this function). The MLCommons + auto-registration path uses ``setdefault`` and does not hit this + guard; round-version uniqueness is enforced by + ``test_round_versions_are_unique``. """ + if name in _RULESET_REGISTRY: + raise ValueError(f"Ruleset {name!r} is already registered") _RULESET_REGISTRY[name] = ruleset @@ -79,16 +89,22 @@ def list_rulesets() -> list[str]: # Auto-register MLCommons rulesets def _auto_register_mlcommons(): - """Auto-register MLCommons rulesets.""" - # Register with version-specific name - register_ruleset(f"mlperf-inference-{mlcommons_current.version}", mlcommons_current) - # Also register as "mlcommons-current" for convenience - register_ruleset("mlcommons-current", mlcommons_current) - # Edge-Agentic (BFCL v4) ruleset - register_ruleset( + """Auto-register MLCommons rulesets (idempotent). + + Uses ``setdefault`` so re-running (e.g. a module re-import) is a no-op + rather than tripping ``register_ruleset``'s duplicate guard. Uniqueness of + round versions is enforced separately by ``test_round_versions_are_unique``. + """ + # Register every known round under its version-specific name + for ruleset in mlcommons_rounds: + _RULESET_REGISTRY.setdefault(f"mlperf-inference-{ruleset.version}", ruleset) + # Also register the current round as "mlcommons-current" for convenience + _RULESET_REGISTRY.setdefault("mlcommons-current", mlcommons_current) + # Edge-agentic ruleset: by version and as the edge "current". + _RULESET_REGISTRY.setdefault( f"mlperf-{mlcommons_edge_current.version}", mlcommons_edge_current ) # -> "mlperf-edge-v0.1" - register_ruleset("mlperf-edge-current", mlcommons_edge_current) + _RULESET_REGISTRY.setdefault("mlperf-edge-current", mlcommons_edge_current) # Auto-register on import diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index e28d777e..2625d514 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -19,6 +19,7 @@ https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc """ +import copy import random from dataclasses import dataclass, field from enum import Enum @@ -272,7 +273,30 @@ def apply_user_config( ) -CURRENT = _v5_1 +# v6.1 per-model latency targets are unchanged from v5.1 (they match the +# current mlperf.conf); only the round RNG seeds rotate. Seeds are the +# schedule_rng_seed / sample_index_rng_seed values from loadgen/mlperf.conf, +# pinned to a specific upstream commit for traceability: +# https://github.com/mlcommons/inference/blob/10f823448fd38bb739e52690efe8191c3a55412b/loadgen/mlperf.conf#L42-L43 +# qsl_rng_seed is intentionally not modeled: it only selects the load order of +# the first-N working set, which the sample-index shuffle already covers. +_v6_1 = RoundRuleset( + version="v6.1", + scheduler_rng_seed=16159082839903944936, + sample_index_rng_seed=2747215439041700203, + benchmark_rulesets={ + # Keep the model-singleton keys; deep-copy only the per-model rules so + # v6.1's mutable leaves (e.g. reported_metrics) can't leak into v5.1. + model: copy.deepcopy(rules) + for model, rules in _v5_1.benchmark_rulesets.items() + }, +) + + +# All known rounds, registered by version in the ruleset registry. +ALL_ROUNDS = [_v5_1, _v6_1] + +CURRENT = _v6_1 # --- Edge-Agentic (BFCL v4) ruleset --- diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index c2b52053..f9fe0ed9 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -22,6 +22,7 @@ from __future__ import annotations +import logging from collections import Counter from enum import Enum from pathlib import Path @@ -50,6 +51,8 @@ from .ruleset_base import BenchmarkSuiteRuleset from .utils import parse_dataset_string, resolve_env_vars +logger = logging.getLogger(__name__) + class SystemDefaults(BaseModel): DEFAULT_TIMEOUT: ClassVar[float] = 300.0 @@ -1029,8 +1032,92 @@ def _resolve_and_validate(self) -> Self: f"got '{lp.type}'" ) + # Pin RNG seeds from the submission ruleset. Done last so the values + # are baked into the config before any consumer reads them — the config + # dump to the report dir, RuntimeSettings.from_config, and the report + # seeds block all see the pinned values. + self._apply_ruleset_seed_overrides() + return self + def _apply_ruleset_seed_overrides(self) -> None: + """Override runtime + warmup RNG seeds from the selected submission ruleset. + + MLPerf rounds pin the RNG seeds; this mirrors LoadGen locking the core + seeds from ``user.conf`` (a submitter cannot substitute their own). + If ``submission_ref`` is unset, the config is left unchanged. If it + names an unregistered ruleset, a ``type=SUBMISSION`` config errors (a + submission cannot silently fall back to default seeds), while any other + type is left unchanged so non-submission/placeholder configs still work. + + The warmup phase is reseeded from the sample-index (dataloader) seed so + its sample order derives from the same pinned seed as the perf phase. + Only the seed *value* is propagated — each phase builds its own + ``random.Random`` downstream, so the RNG object is never shared. + """ + if self.submission_ref is None: + return + try: + ruleset = self.submission_ref.get_ruleset_instance() + except KeyError as e: + if self.type == TestType.SUBMISSION: + raise ValueError( + f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not " + "registered; a submission must pin official RNG seeds and cannot " + "fall back to defaults." + ) from e + logger.warning( + "submission_ref.ruleset %r is not registered; skipping ruleset " + "seed overrides.", + self.submission_ref.ruleset, + ) + return + + # A ruleset used as a submission_ref must pin both seeds. ``None`` means + # "unseeded" in the general ruleset contract (ruleset_base.py), but an + # unseeded submission is incoherent and would silently diverge from the + # random.Random(None) path in RoundRuleset.apply_user_config. Reject it. + if ruleset.scheduler_rng_seed is None or ruleset.sample_index_rng_seed is None: + raise ValueError( + f"submission_ref.ruleset {self.submission_ref.ruleset!r} leaves an " + "RNG seed unset; a pinned ruleset must define both the scheduler " + "and sample-index seeds." + ) + + # Rebuild through model_validate (not model_copy(update=)): with + # extra='forbid' this validates seed *values* and rejects renamed/unknown + # fields. model_copy(update=) writes straight into __dict__, so a + # wrong-typed (e.g. str) or renamed seed would slip through unchecked. + runtime = self.settings.runtime + new_runtime = type(runtime).model_validate( + { + **runtime.model_dump(), + "scheduler_random_seed": ruleset.scheduler_rng_seed, + "dataloader_random_seed": ruleset.sample_index_rng_seed, + } + ) + warmup = self.settings.warmup + new_warmup = type(warmup).model_validate( + { + **warmup.model_dump(), + "warmup_random_seed": ruleset.sample_index_rng_seed, + } + ) + object.__setattr__( + self, + "settings", + self.settings.model_copy( + update={"runtime": new_runtime, "warmup": new_warmup} + ), + ) + logger.debug( + "Pinned RNG seeds from ruleset %r: scheduler=%s sample_index=%s " + "(warmup reseeded from sample_index)", + self.submission_ref.ruleset, + ruleset.scheduler_rng_seed, + ruleset.sample_index_rng_seed, + ) + @model_validator(mode="after") def _propagate_client_api_type(self) -> Self: """Sync client.api_type from endpoint_config.api_type at construction. diff --git a/src/inference_endpoint/config/templates/submission_template.yaml b/src/inference_endpoint/config/templates/submission_template.yaml index 793c047d..ac3c2b11 100644 --- a/src/inference_endpoint/config/templates/submission_template.yaml +++ b/src/inference_endpoint/config/templates/submission_template.yaml @@ -7,7 +7,7 @@ benchmark_mode: "offline" # Required for submission: "offline" or "online" # Submission reference (model + ruleset for compliance checks) submission_ref: model: "llama-2-70b" - ruleset: "mlperf-inference-v6.0" + ruleset: "mlperf-inference-v6.1" model_params: name: "meta-llama/Llama-2-70B-Chat-HF" diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index e8a27b5e..2abcea32 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -382,7 +382,7 @@ def test_from_config_submission_defaults_to_both(self, mock_run, tmp_path): - path: "test.jsonl" submission_ref: model: "test-model" - ruleset: "test" + ruleset: "mlperf-inference-v6.1" """ config_file = tmp_path / "sub.yaml" config_file.write_text(yaml_content) diff --git a/tests/unit/commands/test_util_commands.py b/tests/unit/commands/test_util_commands.py index e1e3a3cc..96c897b4 100644 --- a/tests/unit/commands/test_util_commands.py +++ b/tests/unit/commands/test_util_commands.py @@ -81,12 +81,12 @@ def test_submission_ref_logging(self, tmp_path, caplog): - path: "test.jsonl" submission_ref: model: "llama" - ruleset: "mlperf-v6" + ruleset: "mlperf-inference-v6.1" """) with caplog.at_level("INFO"): execute_validate(config_file) assert "Submission:" in caplog.text - assert "mlperf-v6" in caplog.text + assert "mlperf-inference-v6.1" in caplog.text @pytest.mark.unit def test_invalid_yaml(self, tmp_path): diff --git a/tests/unit/config/rulesets/mlcommons/test_rules.py b/tests/unit/config/rulesets/mlcommons/test_rules.py index 8a9d4bc6..7a515fd3 100644 --- a/tests/unit/config/rulesets/mlcommons/test_rules.py +++ b/tests/unit/config/rulesets/mlcommons/test_rules.py @@ -15,9 +15,14 @@ import pytest from inference_endpoint import metrics -from inference_endpoint.config.ruleset_registry import get_ruleset, list_rulesets +from inference_endpoint.config.ruleset_registry import ( + get_ruleset, + list_rulesets, + register_ruleset, +) from inference_endpoint.config.rulesets.mlcommons import datasets, models from inference_endpoint.config.rulesets.mlcommons.rules import ( + ALL_ROUNDS, CURRENT, EDGE_CURRENT, OptimizationPriority, @@ -116,6 +121,62 @@ def test_apply_user_config_min_sample_count_override(): @pytest.mark.unit +def test_current_round_is_v6_1(): + assert CURRENT.version == "v6.1" + + +@pytest.mark.unit +def test_v6_1_official_seeds(): + """Seeds are the schedule/sample_index values from loadgen/mlperf.conf.""" + v6_1 = get_ruleset("mlperf-inference-v6.1") + assert v6_1.scheduler_rng_seed == 16159082839903944936 + assert v6_1.sample_index_rng_seed == 2747215439041700203 + + +@pytest.mark.unit +def test_all_rounds_registered_by_version(): + names = list_rulesets() + for ruleset in ALL_ROUNDS: + assert f"mlperf-inference-{ruleset.version}" in names + # Both the prior and current round remain resolvable. + assert "mlperf-inference-v5.1" in names + assert "mlperf-inference-v6.1" in names + assert get_ruleset("mlcommons-current") is CURRENT + + +@pytest.mark.unit +def test_round_versions_are_unique(): + """Each registered round must own a distinct version string, else the + registry would silently overwrite one round's entry with another's.""" + assert len(ALL_ROUNDS) == len({r.version for r in ALL_ROUNDS}) + + +@pytest.mark.unit +def test_register_ruleset_rejects_duplicate(): + """Re-registering an existing name raises rather than clobbering it.""" + with pytest.raises(ValueError): + register_ruleset("mlperf-inference-v6.1", CURRENT) + + +@pytest.mark.unit +def test_v6_1_latency_targets_match_v5_1(): + """Only the round seeds rotate; per-model targets are identical.""" + v5_1 = get_ruleset("mlperf-inference-v5.1") + v6_1 = get_ruleset("mlperf-inference-v6.1") + assert v6_1.benchmark_rulesets == v5_1.benchmark_rulesets + # v6.1 holds an independent copy, not a shared reference to v5.1's dict. + assert v6_1.benchmark_rulesets is not v5_1.benchmark_rulesets + # Anchor on a concrete target so the equality above verifies real structure. + assert ( + v6_1.benchmark_rulesets[models.Llama3_1_405b][ + OptimizationPriority.THROUGHPUT + ].max_tpot_latency_ms + == 175 + ) + assert v6_1.scheduler_rng_seed != v5_1.scheduler_rng_seed + assert v6_1.sample_index_rng_seed != v5_1.sample_index_rng_seed + + def test_edge_ruleset_registered(): # Resolvable by version-specific name and the "current" alias. assert get_ruleset("mlperf-edge-v0.1") is EDGE_CURRENT diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index 85155989..613f7ec3 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -20,6 +20,9 @@ import pytest from inference_endpoint import metrics +from inference_endpoint.config import ruleset_registry +from inference_endpoint.config.ruleset_registry import register_ruleset +from inference_endpoint.config.rulesets.mlcommons.rules import RoundRuleset from inference_endpoint.config.runtime_settings import RuntimeSettings from inference_endpoint.config.schema import ( APIType, @@ -161,12 +164,12 @@ def test_submission_with_ref(self): benchmark_mode=TestType.OFFLINE, endpoint_config={"endpoints": ["http://localhost:8000"]}, submission_ref=SubmissionReference( - model="llama-2-70b", ruleset="mlperf-inference-v6.0" + model="llama-2-70b", ruleset="mlperf-inference-v6.1" ), datasets=[{"path": "perf.jsonl"}], ) assert config.model_params.name == "llama-2-70b" - assert config.submission_ref.ruleset == "mlperf-inference-v6.0" + assert config.submission_ref.ruleset == "mlperf-inference-v6.1" @pytest.mark.unit def test_multiple_accuracy_datasets(self): @@ -292,7 +295,7 @@ def test_get_benchmark_mode_submission(self): model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - submission_ref={"model": "M", "ruleset": "R"}, + submission_ref={"model": "M", "ruleset": "mlperf-inference-v6.1"}, ) assert config.get_benchmark_mode() == TestType.OFFLINE @@ -729,3 +732,149 @@ def test_url_scheme_rejected_without_scheme(self, ctor): def test_valid_urls_accepted(self): cfg = ProfilingConfig(engine="vllm", urls=["http://h:8001/v1"]) assert cfg.urls == ["http://h:8001/v1"] + + +# Official v6.1 seeds from loadgen/mlperf.conf L41-43 (schedule / sample_index). +_V6_1_SCHED_SEED = 16159082839903944936 +_V6_1_SAMPLE_SEED = 2747215439041700203 + + +class TestRulesetSeedOverride: + """A submission ruleset pins the runtime RNG seeds at config construction, + before the config is dumped to the report dir.""" + + def _submission(self, ruleset: str, runtime: dict | None = None) -> BenchmarkConfig: + settings = {"runtime": runtime} if runtime is not None else {} + return BenchmarkConfig( + type=TestType.SUBMISSION, + benchmark_mode=TestType.OFFLINE, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + submission_ref=SubmissionReference(model="llama-2-70b", ruleset=ruleset), + datasets=[{"path": "perf.jsonl"}], + settings=settings, + ) + + @pytest.mark.unit + def test_registered_ruleset_pins_seeds(self): + cfg = self._submission("mlperf-inference-v6.1") + assert cfg.settings.runtime.scheduler_random_seed == _V6_1_SCHED_SEED + assert cfg.settings.runtime.dataloader_random_seed == _V6_1_SAMPLE_SEED + # Warmup is reseeded from the sample-index (dataloader) seed so its + # sample order derives from the same pinned seed as the perf phase. + assert cfg.settings.warmup.warmup_random_seed == _V6_1_SAMPLE_SEED + + @pytest.mark.unit + def test_override_precedes_config_dump(self, tmp_path): + """The dumped config.yaml carries the pinned seeds, so the report dir's + config is representative of what actually ran.""" + cfg = self._submission("mlperf-inference-v6.1") + out = tmp_path / "config.yaml" + cfg.to_yaml_file(out) + reloaded = BenchmarkConfig.from_yaml_file(out) + assert reloaded.settings.runtime.scheduler_random_seed == _V6_1_SCHED_SEED + assert reloaded.settings.runtime.dataloader_random_seed == _V6_1_SAMPLE_SEED + + @pytest.mark.unit + def test_ruleset_seed_wins_over_user_value(self): + """Compliance rounds lock the seeds — a user-supplied seed is overridden + (mirrors LoadGen locking core seeds from user.conf).""" + cfg = self._submission( + "mlperf-inference-v6.1", + runtime={"scheduler_random_seed": 7, "dataloader_random_seed": 9}, + ) + assert cfg.settings.runtime.scheduler_random_seed == _V6_1_SCHED_SEED + assert cfg.settings.runtime.dataloader_random_seed == _V6_1_SAMPLE_SEED + + @pytest.mark.unit + def test_unregistered_ruleset_submission_raises(self): + """A submission naming an unregistered ruleset must fail loudly rather + than silently falling back to default seeds.""" + with pytest.raises(ValidationError): + self._submission("does-not-exist") + + @pytest.mark.unit + def test_unregistered_ruleset_non_submission_is_lenient(self): + """Non-submission configs are unaffected: an unknown ruleset leaves the + runtime seeds at their defaults rather than erroring.""" + cfg = BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "test.jsonl"}], + submission_ref=SubmissionReference(model="test", ruleset="does-not-exist"), + ) + assert cfg.settings.runtime.scheduler_random_seed == 42 + assert cfg.settings.runtime.dataloader_random_seed == 42 + + @pytest.mark.unit + def test_no_submission_ref_keeps_defaults(self): + cfg = BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "test.jsonl"}], + ) + assert cfg.settings.runtime.scheduler_random_seed == 42 + assert cfg.settings.runtime.dataloader_random_seed == 42 + + @pytest.fixture + def register_temp_ruleset(self): + """Register a throwaway RoundRuleset for the test, then unregister it.""" + registered: list[str] = [] + + def _register(name, *, scheduler_rng_seed, sample_index_rng_seed): + register_ruleset( + name, + RoundRuleset( + version=name, + scheduler_rng_seed=scheduler_rng_seed, + sample_index_rng_seed=sample_index_rng_seed, + benchmark_rulesets={}, + ), + ) + registered.append(name) + return name + + yield _register + for name in registered: + ruleset_registry._RULESET_REGISTRY.pop(name, None) + + @pytest.mark.unit + def test_partial_none_seed_sample_index_rejected(self, register_temp_ruleset): + """A submission ruleset must pin both seeds. ``None`` (unseeded) is a + valid value in the general ruleset contract but incoherent for a pinned + submission, so a partially-unseeded ruleset is rejected loudly.""" + name = register_temp_ruleset( + "test-partial-seed", scheduler_rng_seed=None, sample_index_rng_seed=999 + ) + with pytest.raises(ValidationError, match="leaves an RNG seed unset"): + self._submission(name) + + @pytest.mark.unit + def test_partial_none_seed_scheduler_rejected(self, register_temp_ruleset): + """Mirror of the scheduler-None case: scheduler set, sample_index None.""" + name = register_temp_ruleset( + "test-partial-sched", scheduler_rng_seed=777, sample_index_rng_seed=None + ) + with pytest.raises(ValidationError, match="leaves an RNG seed unset"): + self._submission(name) + + @pytest.mark.unit + def test_all_none_seed_rejected(self, register_temp_ruleset): + """A fully unseeded ruleset (both seeds ``None``) is rejected.""" + name = register_temp_ruleset( + "test-no-seed", scheduler_rng_seed=None, sample_index_rng_seed=None + ) + with pytest.raises(ValidationError, match="leaves an RNG seed unset"): + self._submission(name) + + @pytest.mark.unit + def test_non_int_seed_rejected(self, register_temp_ruleset): + """Seed *values* are validated: a wrong-typed ruleset seed fails config + construction instead of silently landing a ``str`` on the runtime. + Guards against the model_copy(update=) __dict__ bypass.""" + name = register_temp_ruleset( + "test-bad-seed", scheduler_rng_seed="oops", sample_index_rng_seed=999 + ) + with pytest.raises(ValidationError): + self._submission(name)