diff --git a/src/inference_endpoint/config/ruleset_registry.py b/src/inference_endpoint/config/ruleset_registry.py index 99974037a..0a16a8718 100644 --- a/src/inference_endpoint/config/ruleset_registry.py +++ b/src/inference_endpoint/config/ruleset_registry.py @@ -31,6 +31,8 @@ from .rulesets.mlcommons.rules import ALL_ROUNDS as mlcommons_rounds from .rulesets.mlcommons.rules import CURRENT as mlcommons_current from .rulesets.mlcommons.rules import EDGE_CURRENT as mlcommons_edge_current +from .rulesets.mlcommons.rules import ENDPOINTS_ALL as mlcommons_endpoints_all +from .rulesets.mlcommons.rules import ENDPOINTS_CURRENT as mlcommons_endpoints_current if TYPE_CHECKING: from .ruleset_base import BenchmarkSuiteRuleset @@ -105,6 +107,14 @@ def _auto_register_mlcommons(): f"mlperf-{mlcommons_edge_current.version}", mlcommons_edge_current ) # -> "mlperf-edge-v0.1" _RULESET_REGISTRY.setdefault("mlperf-edge-current", mlcommons_edge_current) + # Endpoints seed sets: by cohort-qualified version and as the endpoints + # "current". A later cohort registers alongside rather than replacing, so a + # submission bound to an earlier set stays resolvable for its full window. + for ruleset in mlcommons_endpoints_all: + _RULESET_REGISTRY.setdefault(f"mlperf-{ruleset.version}", ruleset) + _RULESET_REGISTRY.setdefault( + "mlperf-endpoints-current", mlcommons_endpoints_current + ) # Auto-register on import diff --git a/src/inference_endpoint/config/rulesets/mlcommons/__init__.py b/src/inference_endpoint/config/rulesets/mlcommons/__init__.py index 3164cb287..e23a3c723 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/__init__.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/__init__.py @@ -15,6 +15,18 @@ """MLCommons benchmark ruleset implementation.""" -from .rules import CURRENT, EDGE_CURRENT, OptimizationPriority, RoundRuleset +from .rules import ( + CURRENT, + EDGE_CURRENT, + ENDPOINTS_CURRENT, + OptimizationPriority, + RoundRuleset, +) -__all__ = ["CURRENT", "EDGE_CURRENT", "OptimizationPriority", "RoundRuleset"] +__all__ = [ + "CURRENT", + "EDGE_CURRENT", + "ENDPOINTS_CURRENT", + "OptimizationPriority", + "RoundRuleset", +] diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index 826368e4e..86cfe0c57 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -13,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""This module contains a code representation of the rules for the current round of MLPerf Inference. +"""Code representation of the MLCommons benchmark rounds and their pinned seeds. -These values are derived directly from the MLPerf Inference Policies document: +The MLPerf Inference rounds derive from the MLPerf Inference Policies document: https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc + +The MLPerf Endpoints seed sets come from a separate repository on an +independent rotation; each block cites its own pinned upstream source. """ import copy @@ -330,3 +333,43 @@ def apply_user_config( ) EDGE_CURRENT = _edge_v0_1 + + +# --- MLPerf Endpoints v1.0 seed set --- +# +# Endpoints publishes its own seeds on its own rotation, independent of the +# MLPerf Inference rounds above: a seed set per publication cohort, refreshed +# every two cohorts, each set adoptable for four. A submission binds to one set +# and keeps it for its full Pareto-update window. Values are transcribed +# verbatim from cohort 2026-10-C1, set A, pinned to a specific upstream commit +# so the transcription stays re-verifiable (the branch itself is mutable): +# https://github.com/mlcommons/endpoints_policies/blob/5279b845b8492b02742a66d31cee09d9512f7e1d/seedset.yaml +# +# The cohort ID and set ID are both carried in the version string: `seed_sets` +# is a list keyed by `id`, so a cohort may publish more than one set, and the +# version is the registry key. A later cohort — or a second set within this +# cohort — adds a sibling round rather than editing this one. Endpoints +# characterizes a Pareto curve rather than gating per-model TTFT/TPOT, so there +# are no per-model rulesets to declare: this round exists to pin seeds, and the +# legacy per-model `apply_user_config` path correctly refuses it. +# +# The set also publishes a third seed, `model_seed: 9315206023656308754`, which +# is not pinned here. Its purpose is undocumented: `model_seed` appears nowhere +# in endpoints_rules.md or endpoints_submission_rules.md at the commit above — +# only in seedset.yaml, which gives no definition. Submission rules §4.6 says +# the set drives "request-issue / sample order, and the per-query salt"; this +# round maps the first two, leaving the salt RNG unaccounted for. Pinning the +# third seed awaits a definition from MLCommons of what it seeds. +_endpoints_v1_0_2026_10_C1_A = RoundRuleset( + version="endpoints-v1.0-2026-10-C1-A", + scheduler_rng_seed=10487924139932647040, + sample_index_rng_seed=586478644936801402, + benchmark_rulesets={}, +) + +# Every published cohort stays registered: a submission binds to one set for +# its full update window, so an older set must keep resolving after a newer +# one is published. Add new sets here, do not replace. +ENDPOINTS_ALL = [_endpoints_v1_0_2026_10_C1_A] + +ENDPOINTS_CURRENT = _endpoints_v1_0_2026_10_C1_A diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index b2f261fc7..71bd3c400 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -1315,10 +1315,11 @@ def _apply_ruleset_seed_overrides(self) -> None: MLPerf rounds pin the RNG seeds; this mirrors LoadGen locking the core seeds from ``user.conf`` (a submitter cannot substitute their own). - If ``submission_ref`` is unset, the config is left unchanged. If it - names an unregistered ruleset, a ``type=SUBMISSION`` config errors (a - submission cannot silently fall back to default seeds), while any other - type is left unchanged so non-submission/placeholder configs still work. + If ``submission_ref`` is unset, the config is left unchanged. Naming an + unregistered ruleset is always an error, for every test type: declaring + a ``submission_ref`` asserts the run is bound to that ruleset's seeds, + so a typo that silently fell back to the defaults would produce a run + that looks bound while issuing load from seed 42. The warmup phase is reseeded from the sample-index (dataloader) seed so its sample order derives from the same pinned seed as the perf phase. @@ -1330,18 +1331,19 @@ def _apply_ruleset_seed_overrides(self) -> None: try: ruleset = self.submission_ref.get_ruleset_instance() except KeyError as e: - if self.type == TestType.SUBMISSION: - raise ValueError( - f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not " - "registered; a submission must pin official RNG seeds and cannot " - "fall back to defaults." - ) from e - logger.warning( - "submission_ref.ruleset %r is not registered; skipping ruleset " - "seed overrides.", - self.submission_ref.ruleset, - ) - return + # Imported here, not at module scope: ruleset_registry imports the + # rulesets package, which imports this module (same reason + # SubmissionReference.get_ruleset_instance defers it). + from .ruleset_registry import list_rulesets + + # The likely mistake is a near-miss on a cohort-qualified name, so + # name the alternatives rather than only the rejected value. + raise ValueError( + f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not " + "registered, so the run cannot be bound to its RNG seeds and " + "would silently fall back to the defaults. Registered rulesets: " + + ", ".join(sorted(list_rulesets())) + ) from e # A ruleset used as a submission_ref must pin both seeds. ``None`` means # "unseeded" in the general ruleset contract (ruleset_base.py), but an diff --git a/tests/unit/config/rulesets/mlcommons/test_rules.py b/tests/unit/config/rulesets/mlcommons/test_rules.py index 5d7b65ec0..0efcd5b15 100644 --- a/tests/unit/config/rulesets/mlcommons/test_rules.py +++ b/tests/unit/config/rulesets/mlcommons/test_rules.py @@ -20,13 +20,23 @@ list_rulesets, register_ruleset, ) +from inference_endpoint.config.rulesets.mlcommons import ( + ENDPOINTS_CURRENT as package_endpoints_current, +) from inference_endpoint.config.rulesets.mlcommons import datasets, models from inference_endpoint.config.rulesets.mlcommons.rules import ( ALL_ROUNDS, CURRENT, EDGE_CURRENT, + ENDPOINTS_ALL, + ENDPOINTS_CURRENT, OptimizationPriority, ) +from inference_endpoint.config.schema import ( + BenchmarkConfig, + SubmissionReference, + TestType, +) from inference_endpoint.config.user_config import UserConfig @@ -233,3 +243,112 @@ def test_edge_ruleset_apply_user_config(): assert rt_settings.min_issue_duration_ms == 0 assert rt_settings.max_issue_duration_ms == 4 * 60 * 60 * 1000 assert rt_settings.n_samples_from_dataset == 995 + + +# Verbatim from mlcommons/endpoints_policies seedset.yaml, cohort 2026-10-C1 set A. +_EP_SCHED_SEED = 10487924139932647040 +_EP_SAMPLE_SEED = 586478644936801402 + + +@pytest.mark.unit +def test_endpoints_v1_0_official_seeds(): + """Seeds are the published Endpoints v1.0 cohort 2026-10-C1 set A values. + + A drift here means a run would issue load from seeds MLCommons never + published, which the reviewer-side seeded-RNG check would reject. + """ + ep = get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") + assert ep.scheduler_rng_seed == _EP_SCHED_SEED + assert ep.sample_index_rng_seed == _EP_SAMPLE_SEED + + +@pytest.mark.unit +def test_endpoints_ruleset_registered(): + assert get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") is ENDPOINTS_CURRENT + assert get_ruleset("mlperf-endpoints-current") is ENDPOINTS_CURRENT + assert "mlperf-endpoints-v1.0-2026-10-C1-A" in list_rulesets() + assert ENDPOINTS_CURRENT.version == "endpoints-v1.0-2026-10-C1-A" + + +@pytest.mark.unit +def test_endpoints_seeds_differ_from_every_other_registered_ruleset(): + """Endpoints cohorts rotate independently of the other rulesets; a shared + value would mean one of the two was transcribed from the wrong source.""" + for other in [*ALL_ROUNDS, EDGE_CURRENT]: + assert ENDPOINTS_CURRENT.scheduler_rng_seed != other.scheduler_rng_seed + assert ENDPOINTS_CURRENT.sample_index_rng_seed != other.sample_index_rng_seed + + +@pytest.mark.unit +def test_every_published_endpoints_cohort_stays_registered(): + """A submission keeps its bound seed set for its full update window, so + publishing a newer cohort must not unregister an older one.""" + names = list_rulesets() + for ruleset in ENDPOINTS_ALL: + assert f"mlperf-{ruleset.version}" in names + assert ENDPOINTS_CURRENT in ENDPOINTS_ALL + + +@pytest.mark.unit +def test_endpoints_cohort_versions_are_unique(): + """Duplicate versions would make the registry silently drop a cohort.""" + assert len(ENDPOINTS_ALL) == len({r.version for r in ENDPOINTS_ALL}) + + +@pytest.mark.unit +def test_endpoints_round_refuses_the_per_model_config_path(): + """The round declares no per-model rules, so the legacy per-model path must + refuse it rather than emit runtime settings with no rules behind them.""" + with pytest.raises(ValueError, match="not found in rules"): + ENDPOINTS_CURRENT.apply_user_config( + model=models.Llama3_1_8b, user_config=UserConfig(1.0) + ) + + +@pytest.mark.unit +def test_endpoints_current_is_re_exported_from_the_package(): + assert package_endpoints_current is ENDPOINTS_CURRENT + + +@pytest.mark.unit +@pytest.mark.parametrize( + "ruleset_name", + ["mlperf-endpoints-v1.0-2026-10-C1-A", "mlperf-endpoints-current"], +) +def test_binding_the_round_pins_the_published_seeds_on_a_config(ruleset_name): + """Closes the loop through the only consumer that matters: the seeds must + survive pydantic revalidation in _apply_ruleset_seed_overrides and land on + the runtime config. scheduler_rng_seed exceeds int64, so a future bound on + that field would break this round while every other test stayed green. + """ + cfg = BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test-model"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "perf.jsonl"}], + submission_ref=SubmissionReference(model="test-model", ruleset=ruleset_name), + ) + assert cfg.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED + assert cfg.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED + # Warmup derives its sample order from the same pinned seed as the perf phase. + assert cfg.settings.warmup.warmup_random_seed == _EP_SAMPLE_SEED + + +@pytest.mark.unit +def test_pinned_seeds_survive_a_yaml_round_trip(tmp_path): + """config.yaml in the report dir is the reproducibility record, so it must + carry the pinned values rather than the pre-resolution defaults.""" + cfg = BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test-model"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "perf.jsonl"}], + submission_ref=SubmissionReference( + model="test-model", ruleset="mlperf-endpoints-current" + ), + ) + out = tmp_path / "config.yaml" + cfg.to_yaml_file(out) + reloaded = BenchmarkConfig.from_yaml_file(out) + assert reloaded.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED + assert reloaded.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index d1e109e2b..9a881c184 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1081,18 +1081,52 @@ def test_unregistered_ruleset_submission_raises(self): self._submission("does-not-exist") @pytest.mark.unit - def test_unregistered_ruleset_non_submission_is_lenient(self): - """Non-submission configs are unaffected: an unknown ruleset leaves the - runtime seeds at their defaults rather than erroring.""" - cfg = BenchmarkConfig( - type=TestType.OFFLINE, - model_params={"name": "test"}, - endpoint_config={"endpoints": ["http://localhost:8000"]}, - datasets=[{"path": "test.jsonl"}], - submission_ref=SubmissionReference(model="test", ruleset="does-not-exist"), - ) - assert cfg.settings.runtime.scheduler_random_seed == 42 - assert cfg.settings.runtime.dataloader_random_seed == 42 + @pytest.mark.parametrize( + ("test_type", "settings"), + [ + (TestType.OFFLINE, {}), + ( + TestType.ONLINE, + { + "load_pattern": { + "type": LoadPatternType.CONCURRENCY, + "target_concurrency": 1, + } + }, + ), + ], + ) + def test_unregistered_ruleset_raises_for_any_type(self, test_type, settings): + """Naming a ruleset is a declaration that the run is bound to its seeds. + A typo must fail loudly rather than silently running on the defaults — + the run would otherwise look bound while issuing load from seed 42. + """ + with pytest.raises(ValidationError, match="not registered"): + BenchmarkConfig( + type=test_type, + model_params={"name": "test"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "test.jsonl"}], + settings=settings, + submission_ref=SubmissionReference( + model="test", ruleset="does-not-exist" + ), + ) + + @pytest.mark.unit + def test_unregistered_ruleset_error_names_the_available_rulesets(self): + """The failure has to be actionable: a near-miss on a cohort-qualified + name is the likely mistake, so the message lists what is registered.""" + with pytest.raises(ValidationError, match="mlperf-endpoints-current"): + BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "test.jsonl"}], + submission_ref=SubmissionReference( + model="test", ruleset="mlperf-endpoints-v1.0-2026-10-C1" + ), + ) @pytest.mark.unit def test_no_submission_ref_keeps_defaults(self):