From 8999334fa4b4acbf605fe01f0ab7376fa2622bd4 Mon Sep 17 00:00:00 2001
From: anandhu-eng <anandhusooraj011@gmail.com>
Date: Thu, 10 Sep 2026 18:23:33 +0530
Subject: [PATCH 1/2] Add submission cli test for create run

Run a sample pareto sweep against a dataset replay server on every PR, then
dry-run each resulting run folder through `endpoints-submission-cli runs
create` so the submission path stays exercised.

The replay server answers each prompt with its own reference output, so the
accuracy phase scores 100% by construction and no GPU or model is needed.
`runs create --dry-run` returns before it reads a token or calls the API, so
the job needs no PRISM credentials and creates nothing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../points_config/point_c1.yaml               |  80 +++++
 .../points_config/point_c128.yaml             |  80 +++++
 .../points_config/point_c16.yaml              |  80 +++++
 .../points_config/point_c256.yaml             |  80 +++++
 .../points_config/point_c32.yaml              |  80 +++++
 .../points_config/point_c4.yaml               |  80 +++++
 .../points_config/point_c7.yaml               |  80 +++++
 .../scripts/check_payload.py                  | 149 +++++++++
 .../scripts/make_system_desc.py               | 168 ++++++++++
 .../submission-cli-test/scripts/run_sweep.sh  | 110 ++++++
 .github/workflows/pareto-dry-run.yml          |  55 +++
 .../testing/dataset_replay_server.py          | 314 ++++++++++++++++++
 12 files changed, 1356 insertions(+)
 create mode 100644 .github/submission-cli-test/points_config/point_c1.yaml
 create mode 100644 .github/submission-cli-test/points_config/point_c128.yaml
 create mode 100644 .github/submission-cli-test/points_config/point_c16.yaml
 create mode 100644 .github/submission-cli-test/points_config/point_c256.yaml
 create mode 100644 .github/submission-cli-test/points_config/point_c32.yaml
 create mode 100644 .github/submission-cli-test/points_config/point_c4.yaml
 create mode 100644 .github/submission-cli-test/points_config/point_c7.yaml
 create mode 100644 .github/submission-cli-test/scripts/check_payload.py
 create mode 100644 .github/submission-cli-test/scripts/make_system_desc.py
 create mode 100755 .github/submission-cli-test/scripts/run_sweep.sh
 create mode 100644 .github/workflows/pareto-dry-run.yml
 create mode 100644 src/inference_endpoint/testing/dataset_replay_server.py

diff --git a/.github/submission-cli-test/points_config/point_c1.yaml b/.github/submission-cli-test/points_config/point_c1.yaml
new file mode 100644
index 000000000..f0072db93
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c1.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 1 (low_latency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c1"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 200
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 1
+  client:
+    num_workers: 2
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c1
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/points_config/point_c128.yaml b/.github/submission-cli-test/points_config/point_c128.yaml
new file mode 100644
index 000000000..83c2bb548
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c128.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 128 (high_concurrency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c128"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 4000
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 128
+  client:
+    num_workers: 4
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c128
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/points_config/point_c16.yaml b/.github/submission-cli-test/points_config/point_c16.yaml
new file mode 100644
index 000000000..556ff14c1
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c16.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 16 (med_concurrency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c16"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 2000
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 16
+  client:
+    num_workers: 2
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c16
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/points_config/point_c256.yaml b/.github/submission-cli-test/points_config/point_c256.yaml
new file mode 100644
index 000000000..5c5277722
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c256.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 256 (high_concurrency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c256"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 4000
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 256
+  client:
+    num_workers: 4
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c256
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/points_config/point_c32.yaml b/.github/submission-cli-test/points_config/point_c32.yaml
new file mode 100644
index 000000000..25626f424
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c32.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 32 (med_concurrency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c32"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 3000
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 32
+  client:
+    num_workers: 2
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c32
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/points_config/point_c4.yaml b/.github/submission-cli-test/points_config/point_c4.yaml
new file mode 100644
index 000000000..10490a955
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c4.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 4 (low_concurrency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c4"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 800
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 4
+  client:
+    num_workers: 2
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c4
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/points_config/point_c7.yaml b/.github/submission-cli-test/points_config/point_c7.yaml
new file mode 100644
index 000000000..f9d3b90b5
--- /dev/null
+++ b/.github/submission-cli-test/points_config/point_c7.yaml
@@ -0,0 +1,80 @@
+# MLPerf Endpoints sample pareto point - concurrency 7 (low_concurrency).
+#
+# One measurement point of the 7-point sample sweep. Region boundaries are
+# derived by hand from rules §5.5 for the envelope C_min=1, C_max=256 — nothing
+# recomputes them, and nothing checks §5.3 coverage:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+#
+# The endpoint is inference_endpoint.testing.dataset_replay_server, which replays each
+# prompt's own reference answer, so the accuracy phase scores 100% by
+# construction. This exercises the perf+accuracy pipeline without a GPU.
+#
+# NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
+# Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
+# here is sized for a fast PR gate instead.
+name: "oracle-sim-c7"
+version: "1.0"
+type: online
+
+model_params:
+  name: oracle-sim-model
+  # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
+  # network access and no gated HuggingFace repo in CI. Token counts are
+  # therefore character counts, not a real model's tokenization.
+  tokenizer_name: tests/assets/tokenizers/char_chat
+  temperature: 0.0
+  top_p: 1.0
+  max_new_tokens: 128
+  streaming: "on"
+
+datasets:
+  # Accuracy is a fixed correctness check, so it holds the same small sample
+  # count at every point rather than scaling with concurrency.
+  - name: pareto-ci-accuracy
+    type: accuracy
+    path: tests/assets/datasets/dummy_1k.jsonl
+    # NOTE: the accuracy phase always issues the whole dataset -
+    # execute.py computes n_samples_to_issue as num_samples() * repeats, and the
+    # `samples:` config field is never read anywhere in the codebase. So this
+    # phase issues all 1000 dummy_1k rows at every point; there is no knob to
+    # trim it.
+    parser:
+      prompt: text_input
+    accuracy_config:
+      eval_method: string_match
+      ground_truth: ref_output
+      extractor: identity_extractor
+      num_repeats: 1
+  - name: pareto-ci-performance
+    type: performance
+    path: tests/assets/datasets/dummy_1k.jsonl
+    parser:
+      prompt: text_input
+
+settings:
+  runtime:
+    n_samples_to_issue: 1200
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    # Rules §6.1: pareto points must use the ConcurrencyScheduler.
+    type: concurrency
+    target_concurrency: 7
+  client:
+    num_workers: 2
+    log_level: INFO
+    # Rules §6.5: performance runs stream every chunk, so per-token timing
+    # (and therefore TPOT / interactivity) is actually measurable.
+    stream_all_chunks: true
+
+endpoint_config:
+  endpoints:
+    - "http://127.0.0.1:8765"
+  api_key: null
+  api_type: openai
+
+report_dir: results/pareto_ci/point_c7
+# The runner pins load-generator threads, which is Linux-only; leaving it off
+# keeps these configs runnable on a developer macOS box as well as in CI.
+enable_cpu_affinity: false
+verbose: false
diff --git a/.github/submission-cli-test/scripts/check_payload.py b/.github/submission-cli-test/scripts/check_payload.py
new file mode 100644
index 000000000..cb5da5ec2
--- /dev/null
+++ b/.github/submission-cli-test/scripts/check_payload.py
@@ -0,0 +1,149 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""Assert that a ``runs create --dry-run`` payload is actually well-formed.
+
+``runs create --dry-run`` exits 0 as long as the three required files parse, so
+exit status alone would still pass on a run that measured nothing. This checks
+the payload's contents, which is what makes the CI gate meaningful:
+
+* the four API fields the submission API needs are present and non-empty,
+* the run window is ordered and non-zero,
+* the config that reached the payload is the concurrency point we intended,
+* the performance phase actually completed samples, with no failures.
+
+    python .github/submission-cli-test/scripts/check_payload.py --payload payload.json --concurrency 16
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+_REQUIRED_TOP_LEVEL = (
+    "benchmark_version",
+    "started_at",
+    "finished_at",
+    "system_info",
+    "config",
+    "result_summary",
+)
+
+
+def check(payload: dict[str, Any], concurrency: int) -> list[str]:
+    """Return a list of problems with *payload*; empty means it passed."""
+    problems: list[str] = []
+
+    for key in _REQUIRED_TOP_LEVEL:
+        if key not in payload:
+            problems.append(f"payload is missing {key!r}")
+        elif payload[key] in (None, "", {}, []):
+            problems.append(f"payload field {key!r} is empty")
+
+    if problems:
+        # Everything below indexes into these fields; bail rather than pile on
+        # cascading KeyErrors.
+        return problems
+
+    if payload["benchmark_version"] == "unknown":
+        problems.append(
+            "benchmark_version is 'unknown' (git_sha missing from result_summary)"
+        )
+
+    try:
+        started = datetime.fromisoformat(payload["started_at"])
+        finished = datetime.fromisoformat(payload["finished_at"])
+        if finished <= started:
+            problems.append(f"finished_at {finished} is not after started_at {started}")
+    except ValueError as exc:
+        problems.append(f"unparseable run window: {exc}")
+
+    config = payload["config"]
+    actual = config.get("settings", {}).get("load_pattern", {})
+    if actual.get("type") != "concurrency":
+        problems.append(
+            f"load_pattern.type is {actual.get('type')!r}, expected 'concurrency' (rules §6.1)"
+        )
+    if actual.get("target_concurrency") != concurrency:
+        problems.append(
+            f"target_concurrency is {actual.get('target_concurrency')!r}, expected {concurrency}"
+        )
+
+    client = config.get("settings", {}).get("client", {})
+    if client.get("stream_all_chunks") is not True:
+        problems.append(
+            "settings.client.stream_all_chunks is not true; per-token timing "
+            "(and therefore TPOT) is not measurable (rules §6.5)"
+        )
+
+    summary = payload["result_summary"]
+    completed = summary.get("n_samples_completed") or 0
+    failed = summary.get("n_samples_failed") or 0
+    if completed <= 0:
+        problems.append(
+            f"n_samples_completed is {completed}; the point measured nothing"
+        )
+    if failed:
+        problems.append(f"n_samples_failed is {failed}; the endpoint dropped requests")
+    if not summary.get("complete"):
+        problems.append(
+            "result_summary.complete is false; the run did not finish cleanly"
+        )
+
+    system_info = payload["system_info"]
+    for key in ("system_name", "division", "max_supported_concurrency"):
+        if not system_info.get(key):
+            problems.append(f"system_desc.json is missing {key!r}")
+
+    return problems
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--payload", type=Path, required=True, help="dry-run payload JSON"
+    )
+    parser.add_argument(
+        "--concurrency",
+        type=int,
+        required=True,
+        help="the point's expected concurrency",
+    )
+    args = parser.parse_args()
+
+    try:
+        payload = json.loads(args.payload.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        print(
+            f"FAIL c={args.concurrency}: dry-run output is not valid JSON: {exc}",
+            file=sys.stderr,
+        )
+        return 1
+
+    problems = check(payload, args.concurrency)
+    if problems:
+        print(f"FAIL c={args.concurrency}: payload did not validate", file=sys.stderr)
+        for problem in problems:
+            print(f"  - {problem}", file=sys.stderr)
+        return 1
+
+    summary = payload["result_summary"]
+
+    def fmt(value: Any) -> str:
+        # tps is None whenever no tokenizer was attached, so this must not
+        # assume a number is present.
+        return f"{value:,.1f}" if isinstance(value, int | float) else "n/a"
+
+    print(
+        f"OK c={args.concurrency}: payload valid "
+        f"({summary['n_samples_completed']} samples, qps={fmt(summary.get('qps'))}, "
+        f"tps={fmt(summary.get('tps'))})"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/.github/submission-cli-test/scripts/make_system_desc.py b/.github/submission-cli-test/scripts/make_system_desc.py
new file mode 100644
index 000000000..6a5efb328
--- /dev/null
+++ b/.github/submission-cli-test/scripts/make_system_desc.py
@@ -0,0 +1,168 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""Author the ``system_desc.json`` that ``runs create`` requires (rules §8.2).
+
+``system_desc.json`` is deliberately *not* an endpoints artifact — the submitter
+writes it and drops it into the run folder (see the submission CLI's
+``docs/endpoints-cli/reference/run-folder-layout.md``). For the sample sweep
+there is no real system to describe, so this fills the §8.2 schema with
+clearly-marked simulated values and pulls the few fields that must agree with
+the run itself out of the run's own artifacts.
+
+Fields derived from the run rather than hardcoded:
+
+* ``model_id`` / ``dataset_name`` — read from the run's ``config.yaml``.
+* ``input_token_average`` / ``output_token_average`` — read from
+  ``performance/result_summary.json``, so they describe the actual traffic.
+* ``measured_accuracy_score`` — read from ``accuracy/accuracy_results.json``.
+
+    python .github/submission-cli-test/scripts/make_system_desc.py --run-dir results/point_c16 --c-max 256
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+#: Marks every field that describes hardware which does not exist. Anything a
+#: reviewer might mistake for a real disclosure carries this string.
+_SIMULATED = "SIMULATED - dataset replay server, no physical system under test"
+
+
+def _mean(stat_block: Any) -> float | None:
+    """Pull the mean out of a result_summary stat block, tolerating absence.
+
+    Token-length blocks come back as an empty list when no tokenizer was
+    attached, so this must cope with more than just a missing key.
+    """
+    if isinstance(stat_block, dict):
+        value = stat_block.get("avg")
+        if isinstance(value, int | float):
+            return float(value)
+    return None
+
+
+def _accuracy_score(run_dir: Path) -> float | None:
+    """Return the first dataset's accuracy score, or None if no accuracy phase ran."""
+    path = run_dir / "accuracy" / "accuracy_results.json"
+    if not path.exists():
+        return None
+    data = json.loads(path.read_text(encoding="utf-8"))
+    scores = data.get("accuracy_scores") or []
+    for entry in scores:
+        value = entry.get("score")
+        if isinstance(value, int | float):
+            return float(value)
+    return None
+
+
+def build_system_desc(run_dir: Path, c_max: int, system_name: str) -> dict[str, Any]:
+    """Assemble the §8.2 document for a single measurement point."""
+    config = yaml.safe_load((run_dir / "config.yaml").read_text(encoding="utf-8"))
+    summary = json.loads(
+        (run_dir / "performance" / "result_summary.json").read_text(encoding="utf-8")
+    )
+
+    model_id = config.get("model_params", {}).get("name", "unknown")
+    datasets = config.get("datasets") or [{}]
+    dataset_name = datasets[0].get("name", "unknown")
+
+    isl_avg = _mean(summary.get("input_sequence_lengths"))
+    osl_avg = _mean(summary.get("output_sequence_lengths"))
+    accuracy = _accuracy_score(run_dir)
+
+    return {
+        # Org / submission metadata
+        "submitter_org_names": "MLCommons",
+        "submitter_contact": "endpoints-wg@mlcommons.org",
+        # System metadata
+        "system_name": system_name,
+        "shortened_system_name": system_name[:20],
+        "system_category": "datacenter",
+        "system_availability_status": "RDI",
+        "max_supported_concurrency": c_max,
+        "system_size": "1x node, 0x accelerators (simulated)",
+        "system_node_ensemble_count": 1,
+        "system_node_ensemble_total": 1,
+        "serving_framework": "inference_endpoint.testing.dataset_replay_server",
+        "node_types": [
+            {
+                "system_node_ensemble_id": 0,
+                "number_of_nodes": 1,
+                "host_processor_model_name": _SIMULATED,
+                "host_processors_per_node": 1,
+                "host_processor_vcpu_count": 4,
+                "host_memory_capacity": _SIMULATED,
+                "host_memory_configuration": _SIMULATED,
+                "accelerator_model_name": _SIMULATED,
+                "accelerators_per_node": 0,
+                "accelerator_memory_capacity": _SIMULATED,
+                "accelerator_memory_type": _SIMULATED,
+                "accelerator_interconnect": _SIMULATED,
+                "accelerator_host_interconnect": _SIMULATED,
+                "host_network_card_count": _SIMULATED,
+                "host_networking": _SIMULATED,
+                "host_storage_capacity": _SIMULATED,
+                "host_storage_type": _SIMULATED,
+                "other_hardware": _SIMULATED,
+                "cooling": _SIMULATED,
+                "hw_notes": _SIMULATED,
+                "inference_backend": "dataset_replay_server",
+                "driver": _SIMULATED,
+                "operating_system": _SIMULATED,
+                "filesystem": _SIMULATED,
+                "other_software_stack": _SIMULATED,
+                "sw_notes": _SIMULATED,
+            }
+        ],
+        # Division / model metadata
+        "division": "RDI",
+        "model_id": model_id,
+        "model_name": model_id,
+        "model_precision": "N/A (no model executed)",
+        "link_to_model": "https://github.com/mlcommons/endpoints",
+        "model_notes": _SIMULATED,
+        # Dataset metadata
+        "dataset_id": dataset_name,
+        "dataset_name": dataset_name,
+        "input_token_average": isl_avg if isl_avg is not None else 0.0,
+        "output_token_average": osl_avg if osl_avg is not None else 0.0,
+        "dataset_type": "performance",
+        "dataset_link": "tests/assets/datasets/dummy_1k.jsonl (committed)",
+        "measured_accuracy_score": accuracy if accuracy is not None else 0.0,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--run-dir", type=Path, required=True, help="a single run folder"
+    )
+    parser.add_argument(
+        "--c-max",
+        type=int,
+        required=True,
+        help="declared Maximum Supported Concurrency",
+    )
+    parser.add_argument(
+        "--system-name", default="oracle_sim_ci", help="system_name to record"
+    )
+    args = parser.parse_args()
+
+    desc = build_system_desc(args.run_dir, args.c_max, args.system_name)
+    out = args.run_dir / "system_desc.json"
+    out.write_text(json.dumps(desc, indent=2) + "\n", encoding="utf-8")
+    print(
+        f"Wrote {out} (isl_avg={desc['input_token_average']:.1f} "
+        f"osl_avg={desc['output_token_average']:.1f} "
+        f"accuracy={desc['measured_accuracy_score']})"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/.github/submission-cli-test/scripts/run_sweep.sh b/.github/submission-cli-test/scripts/run_sweep.sh
new file mode 100755
index 000000000..55f622282
--- /dev/null
+++ b/.github/submission-cli-test/scripts/run_sweep.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+#
+# Run the sample pareto sweep against the dataset replay server and dry-run every
+# resulting run folder through `endpoints-submission-cli runs create`.
+#
+# This is the whole gate in one script so it can be run identically on a laptop
+# and inside GitHub Actions — the workflow calls this rather than duplicating
+# the steps in YAML.
+#
+#   .github/submission-cli-test/scripts/run_sweep.sh
+#
+# Environment:
+#   RUNNER      command prefix for the endpoints CLI  (default: "uv run")
+#   SUBMIT_CLI  the submission CLI entry point        (default: endpoints-submission-cli)
+#   OUT_DIR     where run folders are written         (default: results/pareto_ci)
+
+set -euo pipefail
+
+C_MIN=1
+C_MAX=256
+POINTS=(1 4 7 16 32 128 256)
+
+RUNNER="${RUNNER:-uv run}"
+SUBMIT_CLI="${SUBMIT_CLI:-endpoints-submission-cli}"
+OUT_DIR="${OUT_DIR:-results/pareto_ci}"
+# The repo's committed smoke dataset — the same one the README's Quick Start
+# uses. Its `text_input` / `ref_output` columns are exactly what the replay
+# server serves back, so no dataset has to be generated for this sweep. Keep it in
+# step with the `path:` in .github/submission-cli-test/points_config/point_c*.yaml.
+DATASET="${PARETO_CI_DATASET:-tests/assets/datasets/dummy_1k.jsonl}"
+PORT="${REPLAY_PORT:-8765}"
+
+log() { printf '\n=== %s ===\n' "$*"; }
+
+# --------------------------------------------------------------------------
+# 1. Dataset replay server.
+# --------------------------------------------------------------------------
+# POINTS satisfies the rules §5.3 region coverage for the declared envelope
+# C_min=1 / C_max=256. Boundaries derived from §5.5 (log2-space thirds, banker's
+# rounding), fixed for this envelope:
+#   Ultra Low 1-32 (fixed, §5.4) | Low 2-7 | Medium 8-41 | High 42-256
+# Changing C_max or adding a point means re-deriving these by hand — nothing
+# checks region coverage automatically. Each point's config is validated against
+# this array by scripts/check_payload.py once the run folder exists.
+log "Starting dataset replay server on port $PORT"
+$RUNNER python -m inference_endpoint.testing.dataset_replay_server \
+  --port "$PORT" --dataset "$DATASET" \
+  --ttft-ms 40 --tpot-ms 6 --slots 64 > dataset_replay_server.log 2>&1 &
+SERVER_PID=$!
+
+# Always take the server down, including on a failed benchmark, so a local run
+# does not leave a port bound behind it.
+cleanup() {
+  if kill -0 "$SERVER_PID" 2>/dev/null; then
+    kill "$SERVER_PID" 2>/dev/null || true
+    wait "$SERVER_PID" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# Poll rather than sleeping a fixed span: startup is fast locally and slower on
+# a cold CI runner.
+for _ in $(seq 1 40); do
+  if curl -sf -m 2 -o /dev/null \
+      -H 'Content-Type: application/json' \
+      -d '{"model":"oracle-sim-model","messages":[{"role":"user","content":"ping"}]}' \
+      "http://127.0.0.1:$PORT/v1/chat/completions"; then
+    break
+  fi
+  sleep 0.5
+done
+log "Dataset replay server ready"
+
+# --------------------------------------------------------------------------
+# 2. One benchmark per measurement point.
+# --------------------------------------------------------------------------
+for c in "${POINTS[@]}"; do
+  log "Benchmarking concurrency $c"
+  $RUNNER inference-endpoint benchmark from-config \
+    -c ".github/submission-cli-test/points_config/point_c${c}.yaml" --mode both
+done
+
+# --------------------------------------------------------------------------
+# 3. Author system_desc.json, then dry-run each run folder through the
+#    submission CLI. `runs create --dry-run` prints the payload and exits
+#    before touching the API, so no PRISM token is needed.
+# --------------------------------------------------------------------------
+# Payloads go beside the run folders, not inside them: a real `runs create`
+# archives the whole run folder, and the dry-run's own output is not part of
+# the submission.
+payload_dir="${OUT_DIR}/payloads"
+mkdir -p "$payload_dir"
+
+for c in "${POINTS[@]}"; do
+  run_dir="${OUT_DIR}/point_c${c}"
+  payload="${payload_dir}/point_c${c}.json"
+
+  log "Preparing run folder $run_dir"
+  $RUNNER python .github/submission-cli-test/scripts/make_system_desc.py \
+    --run-dir "$run_dir" --c-max "$C_MAX" --system-name "oracle_sim_ci"
+
+  log "runs create --dry-run for concurrency $c"
+  $SUBMIT_CLI runs create --path "$run_dir" --dry-run > "$payload"
+  $RUNNER python .github/submission-cli-test/scripts/check_payload.py \
+    --payload "$payload" --concurrency "$c"
+done
+
+log "Sweep complete: ${#POINTS[@]} points measured and dry-run validated"
diff --git a/.github/workflows/pareto-dry-run.yml b/.github/workflows/pareto-dry-run.yml
new file mode 100644
index 000000000..d5b0a881d
--- /dev/null
+++ b/.github/workflows/pareto-dry-run.yml
@@ -0,0 +1,55 @@
+name: Pareto sample run (dry-run)
+
+# Exercises the full submission path on every PR: run a 7-point pareto sweep
+# against a simulated endpoint, then push each resulting run folder through
+# `endpoints-submission-cli runs create --dry-run`.
+#
+# `--dry-run` prints the parsed payload and returns before it reads a token or
+# calls the API, so this job needs no PRISM credentials and creates nothing.
+
+on:
+  pull_request:
+    branches: [main, master, "release/**"]
+  workflow_dispatch:
+
+jobs:
+  pareto-dry-run:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5
+        with:
+          python-version-file: .python-version
+
+      - name: Install endpoints
+        run: uv sync --frozen
+
+      - name: Install submission CLI
+        # Installed from main, not from PyPI, and that is load-bearing.
+        #
+        # The latest release (v1.0.0.0) still requires a flat
+        # `result_summary.json` at the run-folder root. endpoints writes it to
+        # `performance/result_summary.json`, so every dry-run against the
+        # release fails with:
+        #     missing required file(s): result_summary.json
+        # mlcommons/endpoints-submission-cli PR #78 fixed the parser to expect
+        # the layout endpoints actually writes, but landed after v1.0.0.0 and
+        # is not in any tag yet.
+        #
+        # SWAP THIS BACK once a release includes PR #78:
+        #   uv pip install 'endpoints-submission-cli==<version>'
+        run: uv pip install 'endpoints-submission-cli @ git+https://github.com/mlcommons/endpoints-submission-cli@main'
+
+      - name: Run pareto sweep and dry-run every point
+        env:
+          # --no-sync: the submission CLI is installed into the project venv with
+          # `uv pip install` and is not in pyproject.toml/uv.lock, so a re-sync
+          # here would uninstall it before the dry-run step could use it.
+          RUNNER: uv run --no-sync
+          SUBMIT_CLI: uv run --no-sync endpoints-submission-cli
+        run: |
+          chmod +x .github/submission-cli-test/scripts/run_sweep.sh
+          .github/submission-cli-test/scripts/run_sweep.sh
diff --git a/src/inference_endpoint/testing/dataset_replay_server.py b/src/inference_endpoint/testing/dataset_replay_server.py
new file mode 100644
index 000000000..9edae460f
--- /dev/null
+++ b/src/inference_endpoint/testing/dataset_replay_server.py
@@ -0,0 +1,314 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Dataset replay server — replays a dataset's reference answers with simulated pacing.
+
+Unlike :mod:`echo_server`, which returns the prompt back, this server looks the
+prompt up in the benchmark dataset and returns that row's *reference answer*.
+A benchmark run against it therefore scores 100% accuracy under any
+response-comparing scorer, which is what makes it useful for exercising the
+performance + accuracy pipeline end to end without a GPU or a real model.
+
+Pacing is simulated so the emitted metrics are shaped like a real serving stack
+rather than being uniformly zero:
+
+* ``--ttft-ms`` — base time before the first token of a response.
+* ``--tpot-ms`` — base time between subsequent tokens.
+* ``--slots``   — batch capacity. Up to this many concurrent requests are served
+  at the base rate; beyond it, per-token time degrades in proportion to the
+  overflow, the way continuous batching degrades under load.
+
+The degradation is what produces a non-trivial pareto curve: ``system_tps``
+saturates near ``slots * 1000 / tpot_ms`` while ``tps_per_user`` falls off as
+concurrency climbs, so the measurement points at different concurrency levels
+are actually distinguishable from one another.
+
+Usage::
+
+    python -m inference_endpoint.testing.dataset_replay_server \\
+        --port 8765 \\
+        --dataset /tmp/pareto_ci.jsonl \\
+        --prompt-key text_input --response-key ref_output
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import time
+from pathlib import Path
+
+from aiohttp import web
+
+from inference_endpoint.dataset_manager import Dataset
+from inference_endpoint.dataset_manager.transforms import ColumnRemap
+from inference_endpoint.openai.openai_types_gen import CreateChatCompletionRequest
+from inference_endpoint.testing.echo_server import EchoServer
+from inference_endpoint.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+#: Returned when a prompt is not found in the dataset. Kept distinctive so a
+#: lookup miss shows up as an accuracy failure rather than silently scoring.
+_MISS_RESPONSE = "REPLAY_LOOKUP_MISS"
+
+
+class DatasetReplayServer(EchoServer):
+    """Serves each prompt's reference answer, with optional simulated pacing.
+
+    Pacing defaults to zero, which makes the server a pure lookup table that
+    answers as fast as it can. :meth:`main` supplies non-zero defaults for the
+    standalone server, where realistic timing is the point.
+    """
+
+    def __init__(
+        self,
+        *,
+        answers: dict[str, str],
+        ttft_ms: float = 0.0,
+        tpot_ms: float = 0.0,
+        slots: int = 64,
+        **kwargs: object,
+    ):
+        super().__init__(**kwargs)  # type: ignore[arg-type]
+        self._answers = answers
+        self._ttft_s = ttft_ms / 1000.0
+        self._tpot_s = tpot_ms / 1000.0
+        self._slots = max(1, slots)
+        self._inflight = 0
+        self._misses = 0
+
+    @classmethod
+    def from_dataset(
+        cls,
+        dataset_path: str | Path,
+        *,
+        prompt_key: str = "text_input",
+        response_key: str = "ref_output",
+        **kwargs: object,
+    ) -> DatasetReplayServer:
+        """Build a server whose answers come from a benchmark dataset file.
+
+        Loads through :meth:`Dataset.load_from_file` rather than parsing the
+        file directly, so the server accepts every format the benchmark itself
+        accepts and applies the same column-remap machinery.
+
+        Raises:
+            ValueError: If no row yields both a prompt and an answer — otherwise
+                every request would miss, and the run would score 0% with no
+                indication of the cause.
+        """
+        loader = Dataset.load_from_file(
+            Path(dataset_path),
+            transforms=[ColumnRemap({prompt_key: "prompt", response_key: "output"})],
+        )
+        loader.load()
+
+        answers: dict[str, str] = {}
+        for i in range(loader.num_samples()):
+            sample = loader.load_sample(i)
+            prompt, answer = sample.get("prompt"), sample.get("output")
+            if prompt is None or answer is None:
+                continue
+            answers[str(prompt)] = str(answer)
+
+        if not answers:
+            raise ValueError(
+                f"{dataset_path}: no rows with both {prompt_key!r} and "
+                f"{response_key!r}; every request would score as a lookup miss"
+            )
+        return cls(answers=answers, **kwargs)  # type: ignore[arg-type]
+
+    @property
+    def answer_count(self) -> int:
+        """How many prompt -> answer pairs the server can serve."""
+        return len(self._answers)
+
+    @property
+    def miss_count(self) -> int:
+        """How many requests failed to match a known prompt."""
+        return self._misses
+
+    def get_response(self, request: str) -> str:
+        """Return the reference answer for *request*, or the miss sentinel.
+
+        A miss means the client sent something other than the raw dataset
+        column — every response would then be the sentinel and accuracy would
+        read 0% with no indication of why, so the first one is logged loudly
+        with both sides of the comparison.
+        """
+        answer = self._answers.get(request)
+        if answer is not None:
+            return answer
+
+        self._misses += 1
+        if self._misses == 1:
+            sample_key = next(iter(self._answers))
+            logger.warning(
+                "Prompt lookup MISS - the client is not sending the raw dataset column.\n"
+                "  received: %r\n"
+                "  a known key: %r",
+                request[:400],
+                sample_key[:400],
+            )
+        return _MISS_RESPONSE
+
+    def _load_factor(self) -> float:
+        """Per-token slowdown for the current in-flight count.
+
+        Returns 1.0 while the batch is within capacity, then grows linearly —
+        the shape that makes ``system_tps`` saturate instead of scaling forever.
+        """
+        return max(1.0, self._inflight / self._slots)
+
+    async def _handle_streaming_response(
+        self,
+        id: str,
+        request: web.Request,
+        completion_request: CreateChatCompletionRequest,
+        content: str,
+    ) -> web.StreamResponse:
+        """Stream the reference answer one whitespace token per SSE chunk.
+
+        ``content`` arrives already resolved: the caller in :class:`EchoServer`
+        has run it through :meth:`get_response`. Looking it up again here would
+        search for an *answer* in a map keyed by *prompts*, miss every time, and
+        silently drive accuracy to zero.
+
+        One chunk per token is what lets the client measure TPOT; batching
+        tokens into fewer chunks would collapse the inter-token timing that the
+        pareto curve's interactivity axis is built from.
+        """
+        response = web.StreamResponse(
+            status=200,
+            headers={
+                "Content-Type": "text/event-stream",
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+            },
+        )
+        await response.prepare(request)
+
+        model = str(completion_request.model.root)
+        tokens = content.split()
+
+        self._inflight += 1
+        try:
+            await asyncio.sleep(self._ttft_s * self._load_factor())
+
+            for i, token in enumerate(tokens):
+                if i > 0:
+                    await asyncio.sleep(self._tpot_s * self._load_factor())
+                chunk = {
+                    "id": id,
+                    "object": "chat.completion.chunk",
+                    "created": int(time.time()),
+                    "model": model,
+                    "choices": [
+                        {
+                            "index": 0,
+                            "delta": {"content": token if i == 0 else f" {token}"},
+                            "finish_reason": None,
+                        }
+                    ],
+                }
+                await response.write(f"data: {json.dumps(chunk)}\n\n".encode())
+
+            final = {
+                "id": id,
+                "object": "chat.completion.chunk",
+                "created": int(time.time()),
+                "model": model,
+                "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+            }
+            await response.write(f"data: {json.dumps(final)}\n\n".encode())
+            await response.write(b"data: [DONE]\n\n")
+            return response
+        finally:
+            self._inflight -= 1
+
+
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Dataset replay server — replays dataset reference answers with simulated pacing.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--host", default="127.0.0.1", help="hostname/address to bind to"
+    )
+    parser.add_argument("--port", type=int, default=8765, help="port to bind to")
+    parser.add_argument(
+        "--dataset",
+        type=Path,
+        required=True,
+        help="dataset file to replay answers from (any format Dataset accepts)",
+    )
+    parser.add_argument(
+        "--prompt-key", default="text_input", help="dataset column holding the prompt"
+    )
+    parser.add_argument(
+        "--response-key", default="ref_output", help="dataset column holding the answer"
+    )
+    parser.add_argument(
+        "--ttft-ms", type=float, default=40.0, help="base time to first token, ms"
+    )
+    parser.add_argument(
+        "--tpot-ms", type=float, default=8.0, help="base time per output token, ms"
+    )
+    parser.add_argument(
+        "--slots",
+        type=int,
+        default=64,
+        help="batch capacity; per-token time degrades linearly beyond it",
+    )
+    return parser
+
+
+def main() -> None:
+    setup_logging()
+    args = create_parser().parse_args()
+
+    server = DatasetReplayServer.from_dataset(
+        args.dataset,
+        prompt_key=args.prompt_key,
+        response_key=args.response_key,
+        ttft_ms=args.ttft_ms,
+        tpot_ms=args.tpot_ms,
+        slots=args.slots,
+        host=args.host,
+        port=args.port,
+    )
+    logger.info(
+        "Dataset replay server: %d reference answers from %s (ttft=%.1fms tpot=%.1fms slots=%d)",
+        server.answer_count,
+        args.dataset,
+        args.ttft_ms,
+        args.tpot_ms,
+        args.slots,
+    )
+    server.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        logger.info("Interrupted, shutting down")
+    finally:
+        server.stop()
+
+
+if __name__ == "__main__":
+    main()

From c79195bd406dfa5932569612b33cc2b1101aa351 Mon Sep 17 00:00:00 2001
From: anandhu-eng <anandhusooraj011@gmail.com>
Date: Fri, 11 Sep 2026 20:22:16 +0530
Subject: [PATCH 2/2] chore: rename oracle-sim-* to sim-* in the pareto dry-run
 gate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

'oracle' carried no meaning here beyond the test fixture's origin — the
endpoint is a dataset replay server, not an oracle. Renames the sample
point configs (sim-c<N>), the model name (sim-model), and the recorded
system_name (sim_ci).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .github/submission-cli-test/points_config/point_c1.yaml   | 4 ++--
 .github/submission-cli-test/points_config/point_c128.yaml | 4 ++--
 .github/submission-cli-test/points_config/point_c16.yaml  | 4 ++--
 .github/submission-cli-test/points_config/point_c256.yaml | 4 ++--
 .github/submission-cli-test/points_config/point_c32.yaml  | 4 ++--
 .github/submission-cli-test/points_config/point_c4.yaml   | 4 ++--
 .github/submission-cli-test/points_config/point_c7.yaml   | 4 ++--
 .github/submission-cli-test/scripts/make_system_desc.py   | 4 +---
 .github/submission-cli-test/scripts/run_sweep.sh          | 4 ++--
 9 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/.github/submission-cli-test/points_config/point_c1.yaml b/.github/submission-cli-test/points_config/point_c1.yaml
index f0072db93..3836fbd72 100644
--- a/.github/submission-cli-test/points_config/point_c1.yaml
+++ b/.github/submission-cli-test/points_config/point_c1.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c1"
+name: "sim-c1"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/points_config/point_c128.yaml b/.github/submission-cli-test/points_config/point_c128.yaml
index 83c2bb548..f2f39565c 100644
--- a/.github/submission-cli-test/points_config/point_c128.yaml
+++ b/.github/submission-cli-test/points_config/point_c128.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c128"
+name: "sim-c128"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/points_config/point_c16.yaml b/.github/submission-cli-test/points_config/point_c16.yaml
index 556ff14c1..158a6decd 100644
--- a/.github/submission-cli-test/points_config/point_c16.yaml
+++ b/.github/submission-cli-test/points_config/point_c16.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c16"
+name: "sim-c16"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/points_config/point_c256.yaml b/.github/submission-cli-test/points_config/point_c256.yaml
index 5c5277722..d898c7a94 100644
--- a/.github/submission-cli-test/points_config/point_c256.yaml
+++ b/.github/submission-cli-test/points_config/point_c256.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c256"
+name: "sim-c256"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/points_config/point_c32.yaml b/.github/submission-cli-test/points_config/point_c32.yaml
index 25626f424..f43aba0aa 100644
--- a/.github/submission-cli-test/points_config/point_c32.yaml
+++ b/.github/submission-cli-test/points_config/point_c32.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c32"
+name: "sim-c32"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/points_config/point_c4.yaml b/.github/submission-cli-test/points_config/point_c4.yaml
index 10490a955..9588de3cc 100644
--- a/.github/submission-cli-test/points_config/point_c4.yaml
+++ b/.github/submission-cli-test/points_config/point_c4.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c4"
+name: "sim-c4"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/points_config/point_c7.yaml b/.github/submission-cli-test/points_config/point_c7.yaml
index f9d3b90b5..4c985de34 100644
--- a/.github/submission-cli-test/points_config/point_c7.yaml
+++ b/.github/submission-cli-test/points_config/point_c7.yaml
@@ -12,12 +12,12 @@
 # NOT a rules-conformant measurement point. Rules §6.2 requires 600s (Ultra Low
 # Concurrency) / 1200s (all other regions) of steady state; n_samples_to_issue
 # here is sized for a fast PR gate instead.
-name: "oracle-sim-c7"
+name: "sim-c7"
 version: "1.0"
 type: online
 
 model_params:
-  name: oracle-sim-model
+  name: sim-model
   # Vendored character tokenizer: keeps ISL / OSL / TPOT / TPS populated with no
   # network access and no gated HuggingFace repo in CI. Token counts are
   # therefore character counts, not a real model's tokenization.
diff --git a/.github/submission-cli-test/scripts/make_system_desc.py b/.github/submission-cli-test/scripts/make_system_desc.py
index 6a5efb328..22758cb89 100644
--- a/.github/submission-cli-test/scripts/make_system_desc.py
+++ b/.github/submission-cli-test/scripts/make_system_desc.py
@@ -148,9 +148,7 @@ def main() -> int:
         required=True,
         help="declared Maximum Supported Concurrency",
     )
-    parser.add_argument(
-        "--system-name", default="oracle_sim_ci", help="system_name to record"
-    )
+    parser.add_argument("--system-name", default="sim_ci", help="system_name to record")
     args = parser.parse_args()
 
     desc = build_system_desc(args.run_dir, args.c_max, args.system_name)
diff --git a/.github/submission-cli-test/scripts/run_sweep.sh b/.github/submission-cli-test/scripts/run_sweep.sh
index 55f622282..7927e99e4 100755
--- a/.github/submission-cli-test/scripts/run_sweep.sh
+++ b/.github/submission-cli-test/scripts/run_sweep.sh
@@ -65,7 +65,7 @@ trap cleanup EXIT
 for _ in $(seq 1 40); do
   if curl -sf -m 2 -o /dev/null \
       -H 'Content-Type: application/json' \
-      -d '{"model":"oracle-sim-model","messages":[{"role":"user","content":"ping"}]}' \
+      -d '{"model":"sim-model","messages":[{"role":"user","content":"ping"}]}' \
       "http://127.0.0.1:$PORT/v1/chat/completions"; then
     break
   fi
@@ -99,7 +99,7 @@ for c in "${POINTS[@]}"; do
 
   log "Preparing run folder $run_dir"
   $RUNNER python .github/submission-cli-test/scripts/make_system_desc.py \
-    --run-dir "$run_dir" --c-max "$C_MAX" --system-name "oracle_sim_ci"
+    --run-dir "$run_dir" --c-max "$C_MAX" --system-name "sim_ci"
 
   log "runs create --dry-run for concurrency $c"
   $SUBMIT_CLI runs create --path "$run_dir" --dry-run > "$payload"