From 670600015eb2523cd94ffcd03d2914a2112bdd60 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:59:13 -0500 Subject: [PATCH 1/2] feat(config): register MLPerf Endpoints v1.0 seed set as a ruleset round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the published Endpoints v1.0 seed set (cohort 2026-10-C1, set A) as the round `endpoints-v1.0-2026-10-C1-A`, selectable via `submission_ref.ruleset` as `mlperf-endpoints-v1.0-2026-10-C1-A` or the `mlperf-endpoints-current` alias. The two seeds are transcribed verbatim from `seedset.yaml` in mlcommons/endpoints_policies at commit 5279b845 and reach the runtime config through the existing `_apply_ruleset_seed_overrides` path, so no config schema changes are needed. Endpoints rotates independently of the MLPerf Inference rounds: MLCommons publishes a seed set every two publication cohorts and a submission keeps its bound set for its full Pareto-update window. `ENDPOINTS_ALL` therefore registers every published set rather than replacing the previous one, and the version string carries both the cohort ID and the set ID because `seed_sets` is a list keyed by `id` and a cohort may publish more than one set. The cohort's third seed, `model_seed`, is deliberately not pinned. Its purpose is undocumented upstream: the string appears in neither `endpoints_rules.md` nor `endpoints_submission_rules.md` at that commit, and submission rules §4.6 enumerates three RNG purposes (request-issue, sample order, and the per-query salt) of which this round maps two. Recorded as a comment pending a definition from MLCommons. Co-Authored-By: Claude Opus 5 (1M context) --- .../config/ruleset_registry.py | 10 ++ .../config/rulesets/mlcommons/__init__.py | 16 ++- .../config/rulesets/mlcommons/rules.py | 47 ++++++- .../config/rulesets/mlcommons/test_rules.py | 119 ++++++++++++++++++ 4 files changed, 188 insertions(+), 4 deletions(-) diff --git a/src/inference_endpoint/config/ruleset_registry.py b/src/inference_endpoint/config/ruleset_registry.py index 99974037a..0a16a8718 100644 --- a/src/inference_endpoint/config/ruleset_registry.py +++ b/src/inference_endpoint/config/ruleset_registry.py @@ -31,6 +31,8 @@ from .rulesets.mlcommons.rules import ALL_ROUNDS as mlcommons_rounds from .rulesets.mlcommons.rules import CURRENT as mlcommons_current from .rulesets.mlcommons.rules import EDGE_CURRENT as mlcommons_edge_current +from .rulesets.mlcommons.rules import ENDPOINTS_ALL as mlcommons_endpoints_all +from .rulesets.mlcommons.rules import ENDPOINTS_CURRENT as mlcommons_endpoints_current if TYPE_CHECKING: from .ruleset_base import BenchmarkSuiteRuleset @@ -105,6 +107,14 @@ def _auto_register_mlcommons(): f"mlperf-{mlcommons_edge_current.version}", mlcommons_edge_current ) # -> "mlperf-edge-v0.1" _RULESET_REGISTRY.setdefault("mlperf-edge-current", mlcommons_edge_current) + # Endpoints seed sets: by cohort-qualified version and as the endpoints + # "current". A later cohort registers alongside rather than replacing, so a + # submission bound to an earlier set stays resolvable for its full window. + for ruleset in mlcommons_endpoints_all: + _RULESET_REGISTRY.setdefault(f"mlperf-{ruleset.version}", ruleset) + _RULESET_REGISTRY.setdefault( + "mlperf-endpoints-current", mlcommons_endpoints_current + ) # Auto-register on import diff --git a/src/inference_endpoint/config/rulesets/mlcommons/__init__.py b/src/inference_endpoint/config/rulesets/mlcommons/__init__.py index 3164cb287..e23a3c723 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/__init__.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/__init__.py @@ -15,6 +15,18 @@ """MLCommons benchmark ruleset implementation.""" -from .rules import CURRENT, EDGE_CURRENT, OptimizationPriority, RoundRuleset +from .rules import ( + CURRENT, + EDGE_CURRENT, + ENDPOINTS_CURRENT, + OptimizationPriority, + RoundRuleset, +) -__all__ = ["CURRENT", "EDGE_CURRENT", "OptimizationPriority", "RoundRuleset"] +__all__ = [ + "CURRENT", + "EDGE_CURRENT", + "ENDPOINTS_CURRENT", + "OptimizationPriority", + "RoundRuleset", +] diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index 826368e4e..86cfe0c57 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -13,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""This module contains a code representation of the rules for the current round of MLPerf Inference. +"""Code representation of the MLCommons benchmark rounds and their pinned seeds. -These values are derived directly from the MLPerf Inference Policies document: +The MLPerf Inference rounds derive from the MLPerf Inference Policies document: https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc + +The MLPerf Endpoints seed sets come from a separate repository on an +independent rotation; each block cites its own pinned upstream source. """ import copy @@ -330,3 +333,43 @@ def apply_user_config( ) EDGE_CURRENT = _edge_v0_1 + + +# --- MLPerf Endpoints v1.0 seed set --- +# +# Endpoints publishes its own seeds on its own rotation, independent of the +# MLPerf Inference rounds above: a seed set per publication cohort, refreshed +# every two cohorts, each set adoptable for four. A submission binds to one set +# and keeps it for its full Pareto-update window. Values are transcribed +# verbatim from cohort 2026-10-C1, set A, pinned to a specific upstream commit +# so the transcription stays re-verifiable (the branch itself is mutable): +# https://github.com/mlcommons/endpoints_policies/blob/5279b845b8492b02742a66d31cee09d9512f7e1d/seedset.yaml +# +# The cohort ID and set ID are both carried in the version string: `seed_sets` +# is a list keyed by `id`, so a cohort may publish more than one set, and the +# version is the registry key. A later cohort — or a second set within this +# cohort — adds a sibling round rather than editing this one. Endpoints +# characterizes a Pareto curve rather than gating per-model TTFT/TPOT, so there +# are no per-model rulesets to declare: this round exists to pin seeds, and the +# legacy per-model `apply_user_config` path correctly refuses it. +# +# The set also publishes a third seed, `model_seed: 9315206023656308754`, which +# is not pinned here. Its purpose is undocumented: `model_seed` appears nowhere +# in endpoints_rules.md or endpoints_submission_rules.md at the commit above — +# only in seedset.yaml, which gives no definition. Submission rules §4.6 says +# the set drives "request-issue / sample order, and the per-query salt"; this +# round maps the first two, leaving the salt RNG unaccounted for. Pinning the +# third seed awaits a definition from MLCommons of what it seeds. +_endpoints_v1_0_2026_10_C1_A = RoundRuleset( + version="endpoints-v1.0-2026-10-C1-A", + scheduler_rng_seed=10487924139932647040, + sample_index_rng_seed=586478644936801402, + benchmark_rulesets={}, +) + +# Every published cohort stays registered: a submission binds to one set for +# its full update window, so an older set must keep resolving after a newer +# one is published. Add new sets here, do not replace. +ENDPOINTS_ALL = [_endpoints_v1_0_2026_10_C1_A] + +ENDPOINTS_CURRENT = _endpoints_v1_0_2026_10_C1_A diff --git a/tests/unit/config/rulesets/mlcommons/test_rules.py b/tests/unit/config/rulesets/mlcommons/test_rules.py index 5d7b65ec0..0efcd5b15 100644 --- a/tests/unit/config/rulesets/mlcommons/test_rules.py +++ b/tests/unit/config/rulesets/mlcommons/test_rules.py @@ -20,13 +20,23 @@ list_rulesets, register_ruleset, ) +from inference_endpoint.config.rulesets.mlcommons import ( + ENDPOINTS_CURRENT as package_endpoints_current, +) from inference_endpoint.config.rulesets.mlcommons import datasets, models from inference_endpoint.config.rulesets.mlcommons.rules import ( ALL_ROUNDS, CURRENT, EDGE_CURRENT, + ENDPOINTS_ALL, + ENDPOINTS_CURRENT, OptimizationPriority, ) +from inference_endpoint.config.schema import ( + BenchmarkConfig, + SubmissionReference, + TestType, +) from inference_endpoint.config.user_config import UserConfig @@ -233,3 +243,112 @@ def test_edge_ruleset_apply_user_config(): assert rt_settings.min_issue_duration_ms == 0 assert rt_settings.max_issue_duration_ms == 4 * 60 * 60 * 1000 assert rt_settings.n_samples_from_dataset == 995 + + +# Verbatim from mlcommons/endpoints_policies seedset.yaml, cohort 2026-10-C1 set A. +_EP_SCHED_SEED = 10487924139932647040 +_EP_SAMPLE_SEED = 586478644936801402 + + +@pytest.mark.unit +def test_endpoints_v1_0_official_seeds(): + """Seeds are the published Endpoints v1.0 cohort 2026-10-C1 set A values. + + A drift here means a run would issue load from seeds MLCommons never + published, which the reviewer-side seeded-RNG check would reject. + """ + ep = get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") + assert ep.scheduler_rng_seed == _EP_SCHED_SEED + assert ep.sample_index_rng_seed == _EP_SAMPLE_SEED + + +@pytest.mark.unit +def test_endpoints_ruleset_registered(): + assert get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") is ENDPOINTS_CURRENT + assert get_ruleset("mlperf-endpoints-current") is ENDPOINTS_CURRENT + assert "mlperf-endpoints-v1.0-2026-10-C1-A" in list_rulesets() + assert ENDPOINTS_CURRENT.version == "endpoints-v1.0-2026-10-C1-A" + + +@pytest.mark.unit +def test_endpoints_seeds_differ_from_every_other_registered_ruleset(): + """Endpoints cohorts rotate independently of the other rulesets; a shared + value would mean one of the two was transcribed from the wrong source.""" + for other in [*ALL_ROUNDS, EDGE_CURRENT]: + assert ENDPOINTS_CURRENT.scheduler_rng_seed != other.scheduler_rng_seed + assert ENDPOINTS_CURRENT.sample_index_rng_seed != other.sample_index_rng_seed + + +@pytest.mark.unit +def test_every_published_endpoints_cohort_stays_registered(): + """A submission keeps its bound seed set for its full update window, so + publishing a newer cohort must not unregister an older one.""" + names = list_rulesets() + for ruleset in ENDPOINTS_ALL: + assert f"mlperf-{ruleset.version}" in names + assert ENDPOINTS_CURRENT in ENDPOINTS_ALL + + +@pytest.mark.unit +def test_endpoints_cohort_versions_are_unique(): + """Duplicate versions would make the registry silently drop a cohort.""" + assert len(ENDPOINTS_ALL) == len({r.version for r in ENDPOINTS_ALL}) + + +@pytest.mark.unit +def test_endpoints_round_refuses_the_per_model_config_path(): + """The round declares no per-model rules, so the legacy per-model path must + refuse it rather than emit runtime settings with no rules behind them.""" + with pytest.raises(ValueError, match="not found in rules"): + ENDPOINTS_CURRENT.apply_user_config( + model=models.Llama3_1_8b, user_config=UserConfig(1.0) + ) + + +@pytest.mark.unit +def test_endpoints_current_is_re_exported_from_the_package(): + assert package_endpoints_current is ENDPOINTS_CURRENT + + +@pytest.mark.unit +@pytest.mark.parametrize( + "ruleset_name", + ["mlperf-endpoints-v1.0-2026-10-C1-A", "mlperf-endpoints-current"], +) +def test_binding_the_round_pins_the_published_seeds_on_a_config(ruleset_name): + """Closes the loop through the only consumer that matters: the seeds must + survive pydantic revalidation in _apply_ruleset_seed_overrides and land on + the runtime config. scheduler_rng_seed exceeds int64, so a future bound on + that field would break this round while every other test stayed green. + """ + cfg = BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test-model"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "perf.jsonl"}], + submission_ref=SubmissionReference(model="test-model", ruleset=ruleset_name), + ) + assert cfg.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED + assert cfg.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED + # Warmup derives its sample order from the same pinned seed as the perf phase. + assert cfg.settings.warmup.warmup_random_seed == _EP_SAMPLE_SEED + + +@pytest.mark.unit +def test_pinned_seeds_survive_a_yaml_round_trip(tmp_path): + """config.yaml in the report dir is the reproducibility record, so it must + carry the pinned values rather than the pre-resolution defaults.""" + cfg = BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test-model"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "perf.jsonl"}], + submission_ref=SubmissionReference( + model="test-model", ruleset="mlperf-endpoints-current" + ), + ) + out = tmp_path / "config.yaml" + cfg.to_yaml_file(out) + reloaded = BenchmarkConfig.from_yaml_file(out) + assert reloaded.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED + assert reloaded.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED From 3b57ddabe38ce90aae92ff3517f40f899b27d1ac Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:58:36 -0500 Subject: [PATCH 2/2] fix(config): fail loudly on an unregistered submission_ref ruleset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unregistered ruleset previously raised only for `type: submission`; every other test type logged a warning and carried on with the default seeds 42/42. That makes a typo invisible in exactly the case it matters: the run reports itself as bound to a ruleset, dumps a config.yaml naming it, and issues load from seeds nobody chose. Naming a `submission_ref.ruleset` is a declaration that the run is bound to that ruleset's seeds, so an unresolvable name is now an error for every test type. The message lists the registered rulesets, because the likely mistake is a near-miss on a cohort-qualified name — `mlperf-endpoints-v1.0-2026-10-C1` against the registered `...-C1-A`, say, which is hard to spot unaided. `list_rulesets` is imported inside the handler rather than at module scope: `ruleset_registry` imports the rulesets package, which imports this module. This is the same deferral `SubmissionReference.get_ruleset_instance` already uses for `get_ruleset`. Co-Authored-By: Claude Opus 5 (1M context) --- src/inference_endpoint/config/schema.py | 34 ++++++++------- tests/unit/config/test_schema.py | 58 ++++++++++++++++++++----- 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index b2f261fc7..71bd3c400 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -1315,10 +1315,11 @@ def _apply_ruleset_seed_overrides(self) -> None: MLPerf rounds pin the RNG seeds; this mirrors LoadGen locking the core seeds from ``user.conf`` (a submitter cannot substitute their own). - If ``submission_ref`` is unset, the config is left unchanged. If it - names an unregistered ruleset, a ``type=SUBMISSION`` config errors (a - submission cannot silently fall back to default seeds), while any other - type is left unchanged so non-submission/placeholder configs still work. + If ``submission_ref`` is unset, the config is left unchanged. Naming an + unregistered ruleset is always an error, for every test type: declaring + a ``submission_ref`` asserts the run is bound to that ruleset's seeds, + so a typo that silently fell back to the defaults would produce a run + that looks bound while issuing load from seed 42. The warmup phase is reseeded from the sample-index (dataloader) seed so its sample order derives from the same pinned seed as the perf phase. @@ -1330,18 +1331,19 @@ def _apply_ruleset_seed_overrides(self) -> None: try: ruleset = self.submission_ref.get_ruleset_instance() except KeyError as e: - if self.type == TestType.SUBMISSION: - raise ValueError( - f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not " - "registered; a submission must pin official RNG seeds and cannot " - "fall back to defaults." - ) from e - logger.warning( - "submission_ref.ruleset %r is not registered; skipping ruleset " - "seed overrides.", - self.submission_ref.ruleset, - ) - return + # Imported here, not at module scope: ruleset_registry imports the + # rulesets package, which imports this module (same reason + # SubmissionReference.get_ruleset_instance defers it). + from .ruleset_registry import list_rulesets + + # The likely mistake is a near-miss on a cohort-qualified name, so + # name the alternatives rather than only the rejected value. + raise ValueError( + f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not " + "registered, so the run cannot be bound to its RNG seeds and " + "would silently fall back to the defaults. Registered rulesets: " + + ", ".join(sorted(list_rulesets())) + ) from e # A ruleset used as a submission_ref must pin both seeds. ``None`` means # "unseeded" in the general ruleset contract (ruleset_base.py), but an diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index d1e109e2b..9a881c184 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1081,18 +1081,52 @@ def test_unregistered_ruleset_submission_raises(self): self._submission("does-not-exist") @pytest.mark.unit - def test_unregistered_ruleset_non_submission_is_lenient(self): - """Non-submission configs are unaffected: an unknown ruleset leaves the - runtime seeds at their defaults rather than erroring.""" - cfg = BenchmarkConfig( - type=TestType.OFFLINE, - model_params={"name": "test"}, - endpoint_config={"endpoints": ["http://localhost:8000"]}, - datasets=[{"path": "test.jsonl"}], - submission_ref=SubmissionReference(model="test", ruleset="does-not-exist"), - ) - assert cfg.settings.runtime.scheduler_random_seed == 42 - assert cfg.settings.runtime.dataloader_random_seed == 42 + @pytest.mark.parametrize( + ("test_type", "settings"), + [ + (TestType.OFFLINE, {}), + ( + TestType.ONLINE, + { + "load_pattern": { + "type": LoadPatternType.CONCURRENCY, + "target_concurrency": 1, + } + }, + ), + ], + ) + def test_unregistered_ruleset_raises_for_any_type(self, test_type, settings): + """Naming a ruleset is a declaration that the run is bound to its seeds. + A typo must fail loudly rather than silently running on the defaults — + the run would otherwise look bound while issuing load from seed 42. + """ + with pytest.raises(ValidationError, match="not registered"): + BenchmarkConfig( + type=test_type, + model_params={"name": "test"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "test.jsonl"}], + settings=settings, + submission_ref=SubmissionReference( + model="test", ruleset="does-not-exist" + ), + ) + + @pytest.mark.unit + def test_unregistered_ruleset_error_names_the_available_rulesets(self): + """The failure has to be actionable: a near-miss on a cohort-qualified + name is the likely mistake, so the message lists what is registered.""" + with pytest.raises(ValidationError, match="mlperf-endpoints-current"): + BenchmarkConfig( + type=TestType.OFFLINE, + model_params={"name": "test"}, + endpoint_config={"endpoints": ["http://localhost:8000"]}, + datasets=[{"path": "test.jsonl"}], + submission_ref=SubmissionReference( + model="test", ruleset="mlperf-endpoints-v1.0-2026-10-C1" + ), + ) @pytest.mark.unit def test_no_submission_ref_keeps_defaults(self):