From bcb08f9b5ac516fce009c465d5205ce4365d057b Mon Sep 17 00:00:00 2001 From: Marwen Abid Date: Wed, 2 Sep 2026 11:15:23 -0700 Subject: [PATCH 1/3] Split query verdicts into SLA and end-to-end-budget families The read path answers two separate requirements, so the benchmark now judges and reports two verdicts instead of one. Neither is folded into the other. The SLA family covers all four endpoints. Each floor is that endpoint's share of the 500 rps Standard-tier watermark from the team RPC SLA doc section 3.4, with the mix getTransaction 0.60, getEvents 0.20, getTransactions 0.15, getLedgers 0.05. Each endpoint answers for the p99 of section 4.1, one per storage tier: hot is the Live window, cold is Recent on frozen NVMe. Historical on frozen EBS is out of scope, because the benchmark boxes carry no EBS tier. The floors hold in every phase and every dataset profile, so targets.json stores them once. The end-to-end-budget probe covers getTransaction alone, at the demand-derived floors of work item 856, and answers for one number: an in-RPC p99 of 10 ms, its slice of the transaction-lifecycle budget. The scheduled p99 of that cell is reported and not judged. getTransaction carries both families in one leg. Its rate list is the union of the two ladders, deduplicated and sorted, so no new leg machinery is needed. Where the two floors coincide the shared cell carries both verdicts. getEvents' mean-page budget is deleted. The SLA states a p99 per endpoint and nothing else, so the mean page is reported and judged by nothing. docs/sla-derivation.md records where every number comes from, including the Little's-law derivation that was rejected once section 3.4 became available. Published runs under docs/runs keep the verdicts they were converted with, and the viewers still render them. --- SCHEMA.md | 130 ++++++++--- converter/convert.py | 165 +++++++++----- converter/tests/fixtures.py | 20 +- converter/tests/test_phase.py | 53 +++-- converter/tests/test_queries_rps.py | 234 +++++++++++++------ docs/app.js | 290 +++++++++++++++++++----- docs/sla-derivation.md | 71 ++++++ docs/styles.css | 4 + docs/summary.js | 27 ++- docs/targets.json | 59 +++-- runner/README.md | 35 ++- runner/cmd/campaign/main_test.go | 8 +- runner/cmd/campaign/run_test.go | 2 +- runner/internal/plan/plan_test.go | 10 +- runner/internal/targets/targets.go | 155 ++++++++----- runner/internal/targets/targets_test.go | 91 ++++++-- tests/smoke/gen-fixtures.py | 175 +++++++++----- tests/smoke/smoke.mjs | 186 +++++++++++---- 18 files changed, 1234 insertions(+), 481 deletions(-) create mode 100644 docs/sla-derivation.md diff --git a/SCHEMA.md b/SCHEMA.md index c35f5e2..84ac974 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -91,9 +91,12 @@ query `events` rows, `n_items` may vary — keep the per-run array as `items_r`, // "Phase 1/2/3 performance targets" below. "query_load": { … }, // campaign layout only, and only when the run carries // open-loop query cells; targets.json's query_load - // block copied verbatim (RPS floors per profile × - // phase, the 0.5/1/2 ladder, and the two extra - // budgets) — see "queries" below. + // block copied verbatim: the 0.5/1/2 ladder, the + // sla family (one floor and one p99 per endpoint + // and tier), the e2e_probe family (getTransaction's + // demand floors per profile × phase and its in-RPC + // budget), and the derivation the numbers come + // from — see "queries" below. "name": "phase1-synthetic-minspec", // optional; metadata.json campaign.name "config_file": "…​.toml", // optional; metadata.json campaign.config_file "config": { … } // optional; remaining metadata.json campaign knobs, @@ -106,25 +109,37 @@ query `events` rows, `n_items` may vary — keep the per-run array as `items_r`, // among the published runs. }, "checks": // this run's PRIMARY pass/fail semantics AS DATA - { "kind": "query_p99_threshold", "threshold_ns": 500000000, - "label": "query p99 ≤ 500 ms", "applies_to": "queries" } + { "kind": "query_sla", + "targets_ns": { "": { "hot": int, "cold": int } }, + "floors_rps": { "": float }, + "label": "query p99 ≤ the per-endpoint SLA at the SLA rate", "applies_to": "queries" } + | { "kind": "query_e2e_probe", "threshold_ns": 10000000, + "label": "getTransaction in-RPC p99 ≤ 10 ms at the demand-derived rate", + "applies_to": "queries" } | { "kind": "block_keepup", "interval_ns": 600000000, "label": "600 ms block model", "applies_to": "ingest_hot" }, // block_keepup interval_ns: the legacy synthetic layout keeps the constant // 600 ms model. The campaign layout derives it from close_interval_ns — // label "Phase 1 block model (2 s)" on an exact phase match, "1.5 s pace" // otherwise. An unpaced campaign run emits no block_keepup check. - // query_p99_threshold threshold_ns comes from docs/targets.json - // (query_p99_target_ns) — one number for every phase. + // query_sla targets_ns and floors_rps are docs/targets.json + // query_load.sla verbatim: one arrival rate and one p99 per endpoint and + // storage tier, the same table for every phase and every profile. + // query_e2e_probe threshold_ns is query_load.e2e_probe.in_rpc_p99_ns. + // LEGACY: runs converted before 2026-09-01 carry a single + // "query_p99_threshold" check in place of both, with one + // "threshold_ns": 500000000 for every endpoint, tier and phase. Match on + // kind, and treat it as the SLA family's check. "checks_all": [ … ], // EVERY check the run earns, primary first // A run can earn more than one: a paced campaign that also swept queries - // is judged both on keeping up with the block model (applies_to - // "ingest_hot") and on the read-path target (applies_to "queries"), and - // the two answer different questions about different sections. Each entry - // has the shape of "checks", and checks_all[0] IS "checks" — the single - // object stays for readers written before the list, and for the published - // runs that predate it. Read a verdict by matching applies_to, never by - // position; treat an absent checks_all as the one-element list [checks]. + // is judged on keeping up with the block model (applies_to "ingest_hot"), + // on the read-path SLA, and on getTransaction's end-to-end budget — three + // questions about two sections, so TWO entries share applies_to + // "queries". Each entry has the shape of "checks", and checks_all[0] IS + // "checks" — the single object stays for readers written before the list, + // and for the published runs that predate it. Read a verdict by matching + // kind (applies_to alone no longer picks one out); treat an absent + // checks_all as the one-element list [checks]. "sections": ["ingest_cold", "ingest_hot", "queries", "golden"], // exactly the keys present "ingest_cold": { … }, "ingest_hot": { … }, "queries": { … }, // pubnet only @@ -250,7 +265,7 @@ converter warns. "items_s": V, "items_r": [int×5] // events only: n_items / wall per run; raw per-run n_items }, // OPEN-LOOP cells — one per paced target rate. The key is the rate token - // verbatim as the CSV spells it ("r0.5", "r3.75", "r300"); cells are + // verbatim as the CSV spells it ("r12.5", "r37.5", "r300"); cells are // ordered ascending BY VALUE, so r500 precedes r1000. "r": StageAgg & { // from .csv row total_r — the SCHEDULED // latency (due→done). THIS is the headline percentile: @@ -272,15 +287,22 @@ converter warns. "mean_page_ns": V, "items_r": […] // events only: service total ÷ n (a sequential // subscriber's mean page latency); raw per-run n_items }, - "verdict_1x": { // open-loop only; sibling of the rate cells, see below - "rate": "r1000", // the cell whose target_rps IS this phase's floor - "target_rps": 1000.0, // campaign.query_load.profiles.._rps[phase−1] + "verdict_sla": { // open-loop only; sibling of the rate cells, see below + "rate": "r75", // the cell whose target_rps IS this endpoint's SLA floor + "target_rps": 75.0, // campaign.query_load.sla.floors_rps[] + "achieved_rps_m": 74.998, // that cell's achieved_rps median (absent if unmeasured) + "p99_ns": 60000000, // that cell's SCHEDULED p99 median + "threshold_ns": 80000000, // query_load.sla.p99_ns[][] — the cell's own + // target: this row is txpage in the cold tier + "pass": true // p99_ns ≤ threshold_ns — the SLA verdict + }, + "verdict_e2e": { // txhash ONLY; a DIFFERENT requirement at a DIFFERENT rate + "rate": "r1000", // the cell at this profile and phase's demand-derived floor + "target_rps": 1000.0, // campaign.query_load.e2e_probe.floors_rps.[phase−1] "achieved_rps_m": 999.998, // that cell's achieved_rps median (absent if unmeasured) - "p99_ns": 200000000, // that cell's SCHEDULED p99 median - "threshold_ns": 500000000, // query_p99_target_ns - "pass": true, // p99_ns ≤ threshold_ns — the headline verdict - "in_rpc": { "p99_ns": …, "threshold_ns": 10000000, "pass": true }, // txhash only - "page_budget": { "mean_ns": …, "budget_ns": 100000000, "pass": true } // events only + "p99_ns": 15000000, // that cell's SCHEDULED p99 median — CONTEXT, never judged + "in_rpc": { "p99_ns": 5000000, "threshold_ns": 10000000, "pass": true }, // the only judged number + "pass": true // == in_rpc.pass; there is no threshold_ns on this object } }, "setup": { "": { …V of total_ns, "n_items": int } } // driver rows belonging to no cell @@ -302,18 +324,56 @@ open-loop field (`service`, `wall`, `achieved_rps`, `dispatch_lag`, `shed`, `mean_page_ns`) is omitted with a converter warning when its row is missing from any rep — never zero-filled, so an `r`-array is always one entry per rep. -`verdict_1x` is emitted for campaign-layout runs that have a goal phase (see -`campaign.phase` — the pace, else the manifest's `query_phase`). The 1× -cell is the one paced at the phase's RPS floor for that profile and endpoint — the -ladder's 0.5×/2× cells are context around it — and it is matched by VALUE, so the -token's spelling never has to be guessed. The profile key is the dataset MODEL name: -the unit id minus its `-c` suffix and minus the trailing per-ledger tx count -(`sac-6000-c1` → `sac`, `custom_token-3600-c1` → `custom_token`). `in_rpc` -(getTransaction's additional in-RPC p99 budget) and `page_budget` (getEvents' MEAN -page-latency budget, which a sequential subscriber accumulates as lag) are **separate -verdicts** — render them as their own columns and never fold them into `pass`. The -whole object is omitted, with a warning, when the run has no goal phase, the profile is -unknown to `targets.json`, or no cell sits at the floor; the data still converts. +### Query verdicts: two families + +An open-loop leg answers to **two separate requirements**, and carries one verdict for +each. They are measured at different rates and judged on different numbers, so a viewer +must render them apart — never as one row, and never as one pass/fail. Where every rate +and every latency target below comes from is recorded in +[`docs/sla-derivation.md`](docs/sla-derivation.md). + +`verdict_sla` is the read-path SLA. Every endpoint has one. The judged cell is the one +paced at that endpoint's share of the sustained request-rate watermark +(`query_load.sla.floors_rps[]`, the same rate in every phase and every profile), +and the judged number is the scheduled p99 against +`query_load.sla.p99_ns[][]`. The tier keys are the SLA's data-age windows: +`hot` is Live (hot storage tier), `cold` is Recent (frozen-NVMe). The SLA's third window, +Historical (frozen-EBS), has no tier here — the boxes carry no EBS store, so no leg +measures it. Two cells of the same endpoint therefore carry different thresholds +(getTransactions: 60 ms hot, 80 ms cold), and a viewer must state each row's own number +rather than one number for the run. + +`verdict_e2e` is the end-to-end-budget probe, and only `txhash` has one. The judged cell +is the one paced at the demand-derived floor for this profile and phase +(`query_load.e2e_probe.floors_rps.[phase−1]`), and the only judged number is +`in_rpc.p99_ns` against the 10 ms slice that getTransaction owns in the transaction +lifecycle. The object's `p99_ns` is the scheduled p99 of the same cell, carried as +context: it includes the client's queueing and the assumed network model, neither of +which that slice owns, so it has no threshold beside it and `pass` equals `in_rpc.pass`. + +`txhash` sweeps both families in ONE leg, so its rate list is the union of the two +ladders, deduplicated and sorted. Where the two floors coincide — sac at phase 1, where +the demand floor is also 300 rps — one cell carries both verdicts, each judged on its +own number. + +Both are emitted for campaign-layout runs that have a goal phase (see `campaign.phase` — +the pace, else the manifest's `query_phase`). Cells are matched to a floor by VALUE, so +the rate token's spelling never has to be guessed. The profile key is the dataset MODEL +name: the unit id minus its `-c` suffix and minus the trailing per-ledger tx count +(`sac-6000-c1` → `sac`, `custom_token-3600-c1` → `custom_token`); only `verdict_e2e` +depends on it. Either object is omitted, with a warning naming its family, when the run +has no goal phase, no cell sits at that family's floor, or (for the probe alone) the +profile is unknown to `targets.json`. The data still converts. + +`mean_page_ns` on the events cells is reported, never judged. + +**LEGACY**: runs converted before 2026-09-01 carry a single `verdict_1x` in place of +both. It plays the SLA role — same fields, same meaning — and folds the in-RPC budget in +as its own `in_rpc` object judged at the SAME cell. Runs converted before 2026-08-31 also +carry a `page_budget` object beside it (getEvents' mean-page budget) and one 500 ms +`threshold_ns` for every endpoint and tier. Read `verdict_sla || verdict_1x` for the SLA +family, and treat a `verdict_1x.in_rpc` as the probe result when there is no +`verdict_e2e`. ## Manifest — `docs/runs/index.json` diff --git a/converter/convert.py b/converter/convert.py index e16e80d..abdb517 100644 --- a/converter/convert.py +++ b/converter/convert.py @@ -424,21 +424,24 @@ def ingest_p99_target(block_time_ns, e2e_budget_ns, block_count=2): return e2e_budget_ns - block_count * block_time_ns - FIXED_E2E_NS -# Read-path design target: the p99 no query cell may exceed. One number for -# every phase, so it lives beside the phase list rather than inside it. -QUERY_P99_TARGET_NS = _TARGETS["query_p99_target_ns"] - -# Open-loop query load model: per endpoint x profile x phase RPS floors, the -# 0.5/1/2 ladder they are paced at, and the two budgets that are judged beside -# the headline p99 (getTransaction's in-RPC p99, getEvents' mean page latency). +# Open-loop query load model: the two families of requirement the read path +# answers to, and the 0.5/1/2 ladder every leg is paced on. # Copied verbatim into campaign runs that carry RPS query cells, the way # PHASE_TARGETS is, so the viewer reads the floors as data. QUERY_LOAD = _TARGETS["query_load"] -# targets.json keys its floors by endpoint; the bundle names its legs by query -# type. One mapping, in one place. -_QTYPE_RPS_KEY = {"ledgers": "ledgers_rps", "txpage": "txpage_rps", - "txhash": "txhash_rps", "events": "events_rps"} +# Family 1 — the SLA: one arrival rate per endpoint (its share of the 500 rps +# Standard watermark) and one p99 per endpoint and storage tier (hot = the SLA's +# Live window, cold = Recent). Neither depends on the phase or the profile. +SLA_FLOORS_RPS = QUERY_LOAD["sla"]["floors_rps"] +SLA_P99_NS = QUERY_LOAD["sla"]["p99_ns"] + +# Family 2 — the E2E-budget probe: getTransaction alone, at the demand-derived +# floors of work item 856 (per profile, indexed by phase), answering only for +# its in-RPC p99, which is its slice of the end-to-end lifecycle budget. The +# scheduled p99 of that cell is reported and never judged. +E2E_IN_RPC_P99_NS = QUERY_LOAD["e2e_probe"]["in_rpc_p99_ns"] +E2E_FLOORS_RPS = QUERY_LOAD["e2e_probe"]["floors_rps"] PHASE_TARGETS = _TARGETS["phases"] @@ -843,8 +846,8 @@ def query_profile(unit): def rps_cells(qout): """The rate cells of one qtype entry, keyed r. - The key match carries the weight: verdict_1x is a sibling of the cells that - also carries target_rps, so matching on that field alone would count the + The key match carries the weight: each verdict is a sibling of the cells + that also carries target_rps, so matching on that field alone would count a verdict as a cell on any second pass over converted data. """ return {k: v for k, v in qout.items() @@ -857,66 +860,97 @@ def has_rps_cells(queries): for qt, qout in entry.items() if qt != "setup") -def _verdict_1x(cell, rate_key, floor, qt, idx, prof): - """Pass/fail at the 1x cell. The headline is the SCHEDULED p99 against the - read-path SLA; getTransaction's in-RPC budget and getEvents' mean-page - budget are separate columns, never folded into it.""" - p99 = cell["p99"]["m"] +def _cell_at(cells, floor): + """The rate cell paced at `floor`, or None. Matched by VALUE, so the rate + token's spelling never has to be guessed.""" + return next((k for k, c in cells.items() + if abs(c["target_rps"] - floor) <= 1e-9), None) + + +def _verdict_head(cell, rate_key, floor): + """The fields both families state about the cell they judge.""" v = {"rate": rate_key, "target_rps": float(floor)} if "achieved_rps" in cell: v["achieved_rps_m"] = cell["achieved_rps"]["m"] - v["p99_ns"] = p99 - v["threshold_ns"] = QUERY_P99_TARGET_NS - v["pass"] = p99 <= QUERY_P99_TARGET_NS - if qt == "txhash" and "service" in cell: - budget = QUERY_LOAD["get_tx_in_rpc_p99_ns"] - in_rpc = cell["service"]["p99"]["m"] - v["in_rpc"] = {"p99_ns": in_rpc, "threshold_ns": budget, - "pass": in_rpc <= budget} - if qt == "events" and "mean_page_ns" in cell: - budgets = prof.get("events_page_budget_ns") or [] - if idx < len(budgets): - mean = cell["mean_page_ns"]["m"] - v["page_budget"] = {"mean_ns": mean, "budget_ns": budgets[idx], - "pass": mean <= budgets[idx]} + v["p99_ns"] = cell["p99"]["m"] + return v + + +def _verdict_sla(cell, rate_key, floor, qt, tier): + """The SLA verdict: the SCHEDULED p99 at this endpoint's SLA rate, against + the p99 that endpoint carries in this storage tier. Every endpoint has one, + getTransaction included.""" + v = _verdict_head(cell, rate_key, floor) + threshold = SLA_P99_NS[qt][tier] + v["threshold_ns"] = threshold + v["pass"] = v["p99_ns"] <= threshold + return v + + +def _verdict_e2e(cell, rate_key, floor): + """The E2E-budget verdict: getTransaction's time INSIDE the RPC at the + demand-derived rate, against its slice of the end-to-end budget. The + scheduled p99 rides along as context and is never judged — the arrival rate + this cell is paced at is a demand estimate, not an SLA the tail answers to.""" + v = _verdict_head(cell, rate_key, floor) + in_rpc = cell["service"]["p99"]["m"] + v["in_rpc"] = {"p99_ns": in_rpc, "threshold_ns": E2E_IN_RPC_P99_NS, + "pass": in_rpc <= E2E_IN_RPC_P99_NS} + v["pass"] = v["in_rpc"]["pass"] return v def attach_query_verdicts(queries, phase): - """Attach each qtype's verdict_1x beside its rate cells, in place. + """Attach each qtype's verdicts beside its rate cells, in place. - The 1x rate IS the floor the load model sets for this profile and phase; - the ladder's 0.5x/2x cells are context around it. A cell is matched to the - floor by value, so the rate token's spelling never has to be guessed. + Two families, never folded together. verdict_sla sits on every endpoint's + cell at that endpoint's SLA floor. verdict_e2e sits on getTransaction's cell + at the demand-derived floor for this profile and phase. Where the two floors + coincide the one cell carries both. """ idx = phase["phase"] - 1 for tier, units in queries.items(): for unit, entry in units.items(): profile = query_profile(unit) - prof = QUERY_LOAD["profiles"].get(profile) + e2e_floors = E2E_FLOORS_RPS.get(profile) for qt, qout in entry.items(): if qt == "setup": continue cells = rps_cells(qout) if not cells: continue - if prof is None: + # --- the SLA family: one floor and one p99 per endpoint --- + floor = SLA_FLOORS_RPS.get(qt) + if floor is None or tier not in SLA_P99_NS.get(qt, {}): + warn(f"query-{tier}-{unit} {qt}: no SLA floor or p99 in " + f"targets.json; no SLA verdict") + elif (hit := _cell_at(cells, floor)) is None: + warn(f"query-{tier}-{unit} {qt}: no cell at the SLA floor " + f"of {floor} rps; no SLA verdict") + else: + qout["verdict_sla"] = _verdict_sla(cells[hit], hit, floor, qt, tier) + # --- the E2E-budget probe: getTransaction alone --- + if qt != "txhash": + continue + if e2e_floors is None: warn(f"query-{tier}-{unit}: no query_load profile {profile!r} " - f"in targets.json; no 1x verdict") - break - floors = prof.get(_QTYPE_RPS_KEY.get(qt, "")) or [] - if idx >= len(floors): - warn(f"query-{tier}-{unit} {qt}: no phase-{phase['phase']} RPS " - f"floor in targets.json; no 1x verdict") + f"in targets.json; no E2E-budget verdict") continue - floor = floors[idx] - hit = next((k for k, c in cells.items() - if abs(c["target_rps"] - floor) <= 1e-9), None) + if idx >= len(e2e_floors): + warn(f"query-{tier}-{unit} {qt}: no phase-{phase['phase']} " + f"E2E-probe floor in targets.json; no E2E-budget verdict") + continue + e2e_floor = e2e_floors[idx] + hit = _cell_at(cells, e2e_floor) if hit is None: warn(f"query-{tier}-{unit} {qt}: no cell at the phase-" - f"{phase['phase']} floor of {floor} rps; no 1x verdict") - continue - qout["verdict_1x"] = _verdict_1x(cells[hit], hit, floor, qt, idx, prof) + f"{phase['phase']} E2E-probe floor of {e2e_floor} rps; " + f"no E2E-budget verdict") + elif "service" not in cells[hit]: + warn(f"query-{tier}-{unit} {qt}: no service row at the " + f"E2E-probe floor of {e2e_floor} rps; no E2E-budget verdict") + else: + qout["verdict_e2e"] = _verdict_e2e(cells[hit], hit, e2e_floor) def build_golden(results_dir, unit): @@ -1206,9 +1240,11 @@ def convert(args): if goal_phase is not None: campaign["phase"] = goal_phase["phase"] campaign["phase_targets"] = PHASE_TARGETS - # RPS query cells are judged against the load-model floors, which are - # per phase: embed the table (as phase_targets is) and, when the run - # has a goal phase, attach each qtype's verdict at its 1x cell. + # RPS query cells are judged against the load-model floors: embed the + # table (as phase_targets is) and, when the run has a goal phase, attach + # each qtype's verdicts at the cells those floors name. The SLA floors + # do not depend on the phase, but the E2E probe's do, so a run still + # needs a goal phase before either family can be judged. if has_rps_cells(queries): campaign["query_load"] = QUERY_LOAD if goal_phase is not None: @@ -1255,9 +1291,20 @@ def convert(args): # two answer different questions about different sections. `checks` carries # the first for readers that predate the list; `checks_all` carries them all. checks = [] - query_check = {"kind": "query_p99_threshold", "threshold_ns": QUERY_P99_TARGET_NS, - "label": f"query p99 ≤ {QUERY_P99_TARGET_NS // 1_000_000} ms", - "applies_to": "queries"} + # The read path answers to two requirements, so it earns two checks. The + # SLA one carries the whole target table rather than a single threshold_ns: + # every cell already states the number it was judged against. The probe one + # carries the single in-RPC budget, which is the same for every profile, + # phase and tier. The legacy query_p99_threshold kind is read-side only now. + query_checks = [ + {"kind": "query_sla", "targets_ns": SLA_P99_NS, "floors_rps": SLA_FLOORS_RPS, + "label": "query p99 ≤ the per-endpoint SLA at the SLA rate", + "applies_to": "queries"}, + {"kind": "query_e2e_probe", "threshold_ns": E2E_IN_RPC_P99_NS, + "label": "getTransaction in-RPC p99 ≤ " + f"{E2E_IN_RPC_P99_NS // 1_000_000} ms at the demand-derived rate", + "applies_to": "queries"}, + ] if layout == "campaign": # The keep-up check derives from the run's own pace: a matched phase # names it, any other pace is judged as itself, and an unpaced @@ -1269,14 +1316,14 @@ def convert(args): checks.append({"kind": "block_keepup", "interval_ns": close_interval_ns, "label": label, "applies_to": "ingest_hot"}) if queries: - checks.append(query_check) + checks.extend(query_checks) elif args.dataset_kind == "synthetic": checks.append({"kind": "block_keepup", "interval_ns": 600000000, "label": "600 ms block model", "applies_to": "ingest_hot"}) if queries: - checks.append(query_check) + checks.extend(query_checks) elif queries: - checks.append(query_check) + checks.extend(query_checks) if checks: data["checks"] = checks[0] diff --git a/converter/tests/fixtures.py b/converter/tests/fixtures.py index 123d9b4..35fe2db 100644 --- a/converter/tests/fixtures.py +++ b/converter/tests/fixtures.py @@ -260,19 +260,23 @@ def build_campaign_bundle(root, paced=True, reps=2, machine_commit="0" * 40, # leg (_r{,_millirps,_lag,_shed}) beside the usual setup rows. # Rates are spelled the way Go's strconv.FormatFloat(rps,'f',-1,64) spells them. # -# The default ladder is the sac profile's phase-3 floors x 0.5/1/2, so the -# MIDDLE rung of every type is the 1x cell a phase-3 run is judged at. +# The default ladder is a phase-3 sac run: every endpoint sweeps its SLA floor +# x 0.5/1/2, so the MIDDLE rung is the cell the SLA verdict lands on. txhash +# carries BOTH families in one leg, so its ladder is the SLA one (150/300/600) +# unioned with sac's phase-3 demand ladder (500/1000/2000): the SLA verdict +# lands on r300 and the E2E-budget verdict on r1000. RPS_DATASET = "sac-6000" RPS_RATES = { - "ledgers": ["0.835", "1.67", "3.34"], - "txpage": ["25", "50", "100"], - "txhash": ["500", "1000", "2000"], - "events": ["5", "10", "20"], + "ledgers": ["12.5", "25", "50"], + "txpage": ["37.5", "75", "150"], + "txhash": ["150", "300", "500", "600", "1000", "2000"], + "events": ["50", "100", "200"], } RPS_ANSWERED = 100 # answered requests per leg -RPS_SCHED_P99_NS = 200_000_000 # scheduled p99, inside the 500 ms read SLA +RPS_SCHED_P99_NS = 15_000_000 # scheduled p99, inside every endpoint's SLA + # (the tightest is getTransaction hot, 20 ms) RPS_SERVICE_P99_NS = 5_000_000 # service p99, inside the 10 ms in-RPC budget -RPS_MEAN_PAGE_NS = 80_000_000 # getEvents mean page, inside phase-3 sac's 100 ms +RPS_MEAN_PAGE_NS = 80_000_000 # getEvents mean page — reported, not judged RPS_SHED = 7 # dropped at the in-flight cap (top rung only) diff --git a/converter/tests/test_phase.py b/converter/tests/test_phase.py index 5b03379..ad57b06 100644 --- a/converter/tests/test_phase.py +++ b/converter/tests/test_phase.py @@ -127,10 +127,11 @@ def test_unpaced_no_phase_no_keepup(self): self.assertEqual(data["campaign"]["phase_targets"], convert.PHASE_TARGETS) self.assertNotIn("checks", data) - def test_paced_run_with_queries_carries_both_checks(self): - # A paced campaign that also swept queries is judged twice: it must keep - # up with the block model AND meet the read-path target. Emitting only - # the keep-up check would leave the query cells unjudged. + def test_paced_run_with_queries_carries_every_check(self): + # A paced campaign that also swept queries is judged three ways: it must + # keep up with the block model, meet the read-path SLA, and hold + # getTransaction inside its slice of the end-to-end budget. Emitting + # only the keep-up check would leave the query cells unjudged. tmp = tempfile.mkdtemp() fixtures.build_campaign_bundle(os.path.join(tmp, "b"), paced=True, close_interval="600ms", queries=True) @@ -140,23 +141,41 @@ def test_paced_run_with_queries_carries_both_checks(self): # list see exactly what they saw before. self.assertEqual(data["checks"]["kind"], "block_keepup") self.assertEqual([c["kind"] for c in data["checks_all"]], - ["block_keepup", "query_p99_threshold"]) + ["block_keepup", "query_sla", "query_e2e_probe"]) self.assertEqual(data["checks_all"][0], data["checks"]) - q = next(c for c in data["checks_all"] if c["applies_to"] == "queries") - self.assertEqual(q["threshold_ns"], convert.QUERY_P99_TARGET_NS) - self.assertEqual(q["label"], "query p99 ≤ 500 ms") - - def test_unpaced_run_with_queries_carries_only_the_query_check(self): + sla = next(c for c in data["checks_all"] if c["kind"] == "query_sla") + self.assertEqual(sla["targets_ns"], convert.SLA_P99_NS) + self.assertEqual(sla["floors_rps"], convert.SLA_FLOORS_RPS) + self.assertEqual(sla["applies_to"], "queries") + probe = next(c for c in data["checks_all"] if c["kind"] == "query_e2e_probe") + self.assertEqual(probe["threshold_ns"], 10_000_000) + self.assertEqual(probe["label"], + "getTransaction in-RPC p99 ≤ 10 ms at the demand-derived rate") + self.assertEqual(probe["applies_to"], "queries") + + def test_unpaced_run_with_queries_carries_only_the_query_checks(self): tmp = tempfile.mkdtemp() fixtures.build_campaign_bundle(os.path.join(tmp, "b"), paced=False, queries=True) data, _, _ = run_convert(os.path.join(tmp, "b")) - self.assertEqual([c["kind"] for c in data["checks_all"]], ["query_p99_threshold"]) - self.assertEqual(data["checks"]["kind"], "query_p99_threshold") - - def test_query_target_comes_from_targets_json(self): - # The 500 ms is a design target, not a converter constant: it lives in - # docs/targets.json beside the phase goals. - self.assertEqual(convert.QUERY_P99_TARGET_NS, 500_000_000) + self.assertEqual([c["kind"] for c in data["checks_all"]], + ["query_sla", "query_e2e_probe"]) + self.assertEqual(data["checks"]["kind"], "query_sla") + + def test_query_targets_come_from_targets_json(self): + # The read-path numbers are design targets, not converter constants: + # they live in docs/targets.json beside the phase goals. The SLA family + # states one rate and one p99 per endpoint and storage tier (hot = the + # SLA's Live window, cold = Recent); the probe states one in-RPC budget. + self.assertEqual(convert.SLA_FLOORS_RPS, + {"txhash": 300, "txpage": 75, "events": 100, "ledgers": 25}) + self.assertEqual(convert.SLA_P99_NS, { + "txhash": {"hot": 20_000_000, "cold": 30_000_000}, + "txpage": {"hot": 60_000_000, "cold": 80_000_000}, + "events": {"hot": 40_000_000, "cold": 40_000_000}, + "ledgers": {"hot": 150_000_000, "cold": 200_000_000}, + }) + self.assertEqual(convert.E2E_IN_RPC_P99_NS, 10_000_000) + self.assertEqual(convert.E2E_FLOORS_RPS["sac"], [300, 500, 1000]) def test_legacy_layouts_unchanged(self): tmp = tempfile.mkdtemp() diff --git a/converter/tests/test_queries_rps.py b/converter/tests/test_queries_rps.py index 7436749..f040b1a 100644 --- a/converter/tests/test_queries_rps.py +++ b/converter/tests/test_queries_rps.py @@ -38,6 +38,11 @@ def build_and_convert(**kw): UNIT = f"{fixtures.RPS_DATASET}-c1" +def rate_keys(qout): + """The qtype entry's rate cells, in order — the verdicts are siblings.""" + return [k for k in qout if not k.startswith("verdict_")] + + class ProfileKeyTests(unittest.TestCase): def test_dataset_model_name(self): self.assertEqual(convert.query_profile("sac-6000-c1"), "sac") @@ -81,11 +86,12 @@ def test_all_qtypes_and_setup(self): {"ledgers", "txpage", "txhash", "events", "setup"}) def test_cells_are_rate_tokens_sorted_ascending(self): - # sorted by VALUE, not by string: r500 < r1000 < r2000. - self.assertEqual([k for k in self.cold["txhash"] if k != "verdict_1x"], - ["r500", "r1000", "r2000"]) - self.assertEqual([k for k in self.cold["ledgers"] if k != "verdict_1x"], - ["r0.835", "r1.67", "r3.34"]) + # sorted by VALUE, not by string: r500 < r1000 < r2000. getTransaction + # sweeps both families in one leg, so its ladder is the union. + self.assertEqual(rate_keys(self.cold["txhash"]), + ["r150", "r300", "r500", "r600", "r1000", "r2000"]) + self.assertEqual(rate_keys(self.cold["ledgers"]), + ["r12.5", "r25", "r50"]) def test_headline_is_the_scheduled_latency(self): cell = self.cold["txhash"]["r1000"] @@ -100,7 +106,7 @@ def test_achieved_rps_is_millirps_over_1000(self): self.assertEqual(cell["achieved_rps"], {"m": 999.998, "lo": 999.998, "hi": 999.998, "r": [999.998, 999.998]}) - self.assertEqual(self.cold["ledgers"]["r0.835"]["achieved_rps"]["m"], 0.833) + self.assertEqual(self.cold["ledgers"]["r12.5"]["achieved_rps"]["m"], 12.498) def test_wall_dispatch_lag_and_shed(self): cell = self.cold["txhash"]["r1000"] @@ -112,7 +118,7 @@ def test_wall_dispatch_lag_and_shed(self): self.assertEqual(self.cold["txhash"]["r2000"]["shed"]["m"], fixtures.RPS_SHED) def test_events_mean_page_and_items(self): - cell = self.cold["events"]["r10"] + cell = self.cold["events"]["r100"] self.assertEqual(cell["mean_page_ns"]["m"], float(fixtures.RPS_MEAN_PAGE_NS)) self.assertEqual(cell["items_r"], [1000, 1000]) self.assertNotIn("mean_page_ns", self.cold["txhash"]["r1000"]) @@ -125,52 +131,98 @@ def test_setup_excludes_every_per_leg_row(self): def test_query_load_embedded_in_campaign(self): self.assertEqual(self.data["campaign"]["query_load"], convert.QUERY_LOAD) - def test_verdicts_pass_at_the_phase3_floors(self): - for qt, rate in (("ledgers", "r1.67"), ("txpage", "r50"), - ("txhash", "r1000"), ("events", "r10")): - v = self.cold[qt]["verdict_1x"] + def test_sla_verdict_on_every_endpoint_at_its_sla_floor(self): + # The SLA floors belong to the endpoint alone, so the same four rates + # are judged in every phase and every profile — getTransaction included. + for qt, rate in (("ledgers", "r25"), ("txpage", "r75"), + ("txhash", "r300"), ("events", "r100")): + v = self.cold[qt]["verdict_sla"] self.assertEqual(v["rate"], rate) self.assertTrue(v["pass"]) - self.assertEqual(v["threshold_ns"], convert.QUERY_P99_TARGET_NS) + self.assertEqual(v["threshold_ns"], convert.SLA_P99_NS[qt]["cold"]) self.assertEqual(v["p99_ns"], fixtures.RPS_SCHED_P99_NS) - self.assertEqual(self.cold["txhash"]["verdict_1x"]["target_rps"], 1000.0) - self.assertEqual(self.cold["txhash"]["verdict_1x"]["achieved_rps_m"], 999.998) - - def test_txhash_in_rpc_is_a_separate_column(self): - v = self.cold["txhash"]["verdict_1x"] + self.assertNotIn("in_rpc", v) + self.assertEqual(self.cold["txhash"]["verdict_sla"]["target_rps"], 300.0) + self.assertEqual(self.cold["txhash"]["verdict_sla"]["achieved_rps_m"], 299.998) + + def test_e2e_verdict_on_getTransaction_alone_at_the_demand_floor(self): + # sac phase 3 asks getTransaction for 1000 rps; the cell answers for its + # time inside the RPC, and for nothing else. + v = self.cold["txhash"]["verdict_e2e"] + self.assertEqual(v["rate"], "r1000") + self.assertEqual(v["target_rps"], 1000.0) self.assertEqual(v["in_rpc"], {"p99_ns": fixtures.RPS_SERVICE_P99_NS, "threshold_ns": 10_000_000, "pass": True}) - self.assertNotIn("in_rpc", self.cold["ledgers"]["verdict_1x"]) + self.assertTrue(v["pass"]) + # the scheduled p99 rides along as context, with no threshold beside it + self.assertEqual(v["p99_ns"], fixtures.RPS_SCHED_P99_NS) + self.assertNotIn("threshold_ns", v) + for qt in ("ledgers", "txpage", "events"): + self.assertNotIn("verdict_e2e", self.cold[qt]) + + def test_threshold_follows_the_storage_tier(self): + # The SLA reads the tiers as data-age windows — hot is Live, cold is + # Recent — so the same endpoint is judged differently in each. + hot = self.data["queries"]["hot"][UNIT] + self.assertEqual(hot["txpage"]["verdict_sla"]["threshold_ns"], 60_000_000) + self.assertEqual(self.cold["txpage"]["verdict_sla"]["threshold_ns"], 80_000_000) + self.assertEqual(hot["txhash"]["verdict_sla"]["threshold_ns"], 20_000_000) + self.assertEqual(self.cold["txhash"]["verdict_sla"]["threshold_ns"], 30_000_000) + # The E2E budget is one number for every tier, profile and phase. + for entry in (hot, self.cold): + self.assertEqual(entry["txhash"]["verdict_e2e"]["in_rpc"]["threshold_ns"], + 10_000_000) + + def test_the_legacy_single_verdict_is_gone(self): + for qt in ("ledgers", "txpage", "txhash", "events"): + self.assertNotIn("verdict_1x", self.cold[qt]) - def test_events_page_budget_is_a_separate_column(self): - v = self.cold["events"]["verdict_1x"] - self.assertEqual(v["page_budget"], {"mean_ns": 80_000_000.0, - "budget_ns": 100_000_000, "pass": True}) - self.assertNotIn("page_budget", self.cold["txpage"]["verdict_1x"]) + def test_no_page_budget_verdict(self): + # getEvents' mean page is reported as a cell column, never judged: the + # SLA states a p99 per endpoint and nothing else. + self.assertNotIn("page_budget", self.cold["events"]["verdict_sla"]) + self.assertIn("mean_page_ns", self.cold["events"]["r100"]) class RpsVerdictFailureTests(unittest.TestCase): def test_scheduled_p99_over_the_sla_fails(self): data, _, _ = build_and_convert(sched_p99={"txhash": 700_000_000}) - v = data["queries"]["cold"][UNIT]["txhash"]["verdict_1x"] + v = data["queries"]["cold"][UNIT]["txhash"]["verdict_sla"] self.assertEqual(v["p99_ns"], 700_000_000) self.assertFalse(v["pass"]) # a sibling type at the same rate ladder is unaffected - self.assertTrue(data["queries"]["cold"][UNIT]["ledgers"]["verdict_1x"]["pass"]) - - def test_in_rpc_budget_fails_without_failing_the_headline(self): + self.assertTrue(data["queries"]["cold"][UNIT]["ledgers"]["verdict_sla"]["pass"]) + + def test_a_p99_between_the_two_tier_targets_fails_hot_only(self): + # 70 ms clears getTransactions' 80 ms Recent target but breaches its + # 60 ms Live one, so the same latency passes cold and fails hot. + data, _, _ = build_and_convert(sched_p99={"txpage": 70_000_000}) + self.assertTrue(data["queries"]["cold"][UNIT]["txpage"]["verdict_sla"]["pass"]) + self.assertFalse(data["queries"]["hot"][UNIT]["txpage"]["verdict_sla"]["pass"]) + + def test_the_two_families_fail_independently(self): + # 40 ms is over getTransaction's SLA p99 in both tiers (20 hot, 30 cold) + # but the RPC itself answered in 5 ms: the SLA verdict fails, the + # E2E-budget verdict passes, and neither reads the other's number. + data, _, _ = build_and_convert(sched_p99={"txhash": 40_000_000}) + cold = data["queries"]["cold"][UNIT]["txhash"] + self.assertFalse(cold["verdict_sla"]["pass"]) + self.assertTrue(cold["verdict_e2e"]["pass"]) + self.assertEqual(cold["verdict_e2e"]["p99_ns"], 40_000_000) + + def test_in_rpc_budget_fails_the_e2e_verdict_alone(self): data, _, _ = build_and_convert(svc_p99={"txhash": 12_000_000}) - v = data["queries"]["cold"][UNIT]["txhash"]["verdict_1x"] - self.assertTrue(v["pass"]) # scheduled p99 still inside 500 ms - self.assertFalse(v["in_rpc"]["pass"]) # but over the 10 ms in-RPC budget - self.assertEqual(v["in_rpc"]["p99_ns"], 12_000_000) + cold = data["queries"]["cold"][UNIT]["txhash"] + self.assertTrue(cold["verdict_sla"]["pass"]) # the tail is still inside 30 ms + self.assertFalse(cold["verdict_e2e"]["pass"]) # but over the 10 ms in-RPC budget + self.assertEqual(cold["verdict_e2e"]["in_rpc"]["p99_ns"], 12_000_000) + self.assertNotIn("in_rpc", cold["verdict_sla"]) - def test_page_budget_fails_without_failing_the_headline(self): + def test_a_slow_mean_page_never_fails_a_verdict(self): data, _, _ = build_and_convert(mean_page={"events": 150_000_000}) - v = data["queries"]["cold"][UNIT]["events"]["verdict_1x"] + v = data["queries"]["cold"][UNIT]["events"]["verdict_sla"] self.assertTrue(v["pass"]) - self.assertFalse(v["page_budget"]["pass"]) - self.assertEqual(v["page_budget"]["mean_ns"], 150_000_000.0) + self.assertNotIn("page_budget", v) class RpsVerdictOmissionTests(unittest.TestCase): @@ -180,33 +232,50 @@ def test_no_phase_no_verdict(self): self.assertNotIn("phase", data["campaign"]) self.assertEqual(data["campaign"]["query_load"], convert.QUERY_LOAD) for qt in ("ledgers", "txpage", "txhash", "events"): - self.assertNotIn("verdict_1x", cold[qt]) + self.assertNotIn("verdict_sla", cold[qt]) + self.assertNotIn("verdict_e2e", cold[qt]) self.assertTrue(any("verdicts omitted" in w for w in warnings)) self.assertEqual(convert.validate_run(data, reps=2), []) - def test_unknown_profile_warns_once_per_unit(self): + def test_unknown_profile_costs_the_probe_verdict_only(self): + # The SLA floors belong to the endpoint, so an unknown dataset profile + # still earns every SLA verdict; only the demand-derived probe, which is + # keyed by profile, has nothing to judge against. data, warnings, _ = build_and_convert(dataset="mystery-9000") cold = data["queries"]["cold"]["mystery-9000-c1"] - self.assertNotIn("verdict_1x", cold["txhash"]) + self.assertIn("verdict_sla", cold["txhash"]) + self.assertNotIn("verdict_e2e", cold["txhash"]) hits = [w for w in warnings if "no query_load profile 'mystery'" in w] self.assertEqual(len(hits), 2) # one per tier, not per qtype self.assertEqual(convert.validate_run(data, reps=2), []) - def test_no_cell_at_the_floor(self): - # sac phase-3 txpage floor is 50 rps; this ladder never reaches it. + def test_no_cell_at_the_sla_floor(self): + # the txpage SLA floor is 75 rps; this ladder never reaches it. data, warnings, _ = build_and_convert(rates={"txpage": ["1", "2", "4"]}) - self.assertNotIn("verdict_1x", data["queries"]["cold"][UNIT]["txpage"]) - self.assertTrue(any("no cell at the phase-3 floor of 50 rps" in w + self.assertNotIn("verdict_sla", data["queries"]["cold"][UNIT]["txpage"]) + self.assertTrue(any("no cell at the SLA floor of 75 rps" in w for w in warnings)) + def test_no_cell_at_the_probe_floor(self): + # An SLA-only txhash ladder: the SLA verdict lands, the probe's + # phase-3 floor of 1000 rps has no cell and says so. + data, warnings, _ = build_and_convert( + rates={"txhash": ["150", "300", "600"]}) + cold = data["queries"]["cold"][UNIT]["txhash"] + self.assertEqual(cold["verdict_sla"]["rate"], "r300") + self.assertNotIn("verdict_e2e", cold) + self.assertTrue(any("E2E-probe floor of 1000 rps" in w for w in warnings)) + class RpsExplicitPhaseTests(unittest.TestCase): """An unpaced campaign has no pace to read a phase from — a cold-only query run has nothing to pace — so the manifest states the goal phase outright as campaign.query_phase, and the verdicts come from that.""" - # the sac phase-1 floors x the 0.5/1/2 ladder - PHASE1_RATES = {"ledgers": ["0.25", "0.5", "1"], "txpage": ["7.5", "15", "30"], - "txhash": ["150", "300", "600"], "events": ["1.5", "3", "6"]} + # A phase-1 sac ladder. sac's phase-1 demand floor is 300 rps, which IS the + # SLA floor, so the two txhash ladders coincide exactly and the leg sweeps + # one list — the case where a single cell carries both verdicts. + PHASE1_RATES = {"ledgers": ["12.5", "25", "50"], "txpage": ["37.5", "75", "150"], + "txhash": ["150", "300", "600"], "events": ["50", "100", "200"]} @classmethod def setUpClass(cls): @@ -223,23 +292,34 @@ def test_phase_from_the_manifest(self): self.assertEqual(self.data["campaign"]["phase"], 1) def test_verdicts_at_the_phase1_floors(self): - for qt, rate in (("ledgers", "r0.5"), ("txpage", "r15"), - ("txhash", "r300"), ("events", "r3")): - self.assertEqual(self.cold[qt]["verdict_1x"]["rate"], rate) - # phase 1's page budget is the loosest of the three - self.assertEqual(self.cold["events"]["verdict_1x"]["page_budget"]["budget_ns"], - 333_000_000) + for qt, rate in (("ledgers", "r25"), ("txpage", "r75"), + ("txhash", "r300"), ("events", "r100")): + self.assertEqual(self.cold[qt]["verdict_sla"]["rate"], rate) + # the latency target is a property of the endpoint and the tier, not of + # the phase: phase 1 is judged at the same p99 as phase 3. + self.assertEqual(self.cold["events"]["verdict_sla"]["threshold_ns"], + 40_000_000) + + def test_one_cell_carries_both_verdicts_when_the_floors_coincide(self): + txhash = self.cold["txhash"] + self.assertEqual(txhash["verdict_sla"]["rate"], "r300") + self.assertEqual(txhash["verdict_e2e"]["rate"], "r300") + # Same cell, two independent judgements against two different numbers. + self.assertEqual(txhash["verdict_sla"]["threshold_ns"], 30_000_000) + self.assertEqual(txhash["verdict_e2e"]["in_rpc"]["threshold_ns"], 10_000_000) + self.assertTrue(txhash["verdict_sla"]["pass"]) + self.assertTrue(txhash["verdict_e2e"]["pass"]) def test_unpaced_still_earns_no_keepup_check(self): # The goal phase judges the READ path; it does not invent a block model # for a run that paced nothing. self.assertEqual([c["kind"] for c in self.data["checks_all"]], - ["query_p99_threshold"]) + ["query_sla", "query_e2e_probe"]) def test_neither_pace_nor_manifest_phase(self): data, warnings, _ = build_and_convert(close_interval="0", query_phase=0) self.assertNotIn("phase", data["campaign"]) - self.assertNotIn("verdict_1x", data["queries"]["cold"][UNIT]["txhash"]) + self.assertNotIn("verdict_sla", data["queries"]["cold"][UNIT]["txhash"]) self.assertTrue(any("query_phase" in w and "verdicts omitted" in w for w in warnings)) self.assertEqual(convert.validate_run(data, reps=2), []) @@ -249,25 +329,28 @@ def test_the_pace_wins_over_the_manifest(self): # reaches here, the measured pace is the truth. data, _, _ = build_and_convert(close_interval="600ms", query_phase=1) self.assertEqual(data["campaign"]["phase"], 3) + # phase 3's probe floor, not phase 1's — the SLA rate is the same either way self.assertEqual( - data["queries"]["cold"][UNIT]["txhash"]["verdict_1x"]["rate"], "r1000") + data["queries"]["cold"][UNIT]["txhash"]["verdict_e2e"]["rate"], "r1000") class RpsCellScanTests(unittest.TestCase): - """rps_cells must survive a second pass over already-converted data: the - verdict is a sibling of the rate cells and carries target_rps too.""" + """rps_cells must survive a second pass over already-converted data: both + verdicts are siblings of the rate cells and carry target_rps too.""" def test_verdict_is_not_a_cell(self): data, _, _ = build_and_convert() qout = data["queries"]["cold"][UNIT]["txhash"] - self.assertIn("verdict_1x", qout) - self.assertEqual(list(convert.rps_cells(qout)), ["r500", "r1000", "r2000"]) + self.assertIn("verdict_sla", qout) + self.assertIn("verdict_e2e", qout) + self.assertEqual(list(convert.rps_cells(qout)), + ["r150", "r300", "r500", "r600", "r1000", "r2000"]) self.assertTrue(convert.has_rps_cells(data["queries"])) def test_only_rate_keys_count(self): cell = {"target_rps": 1.0} self.assertEqual( - list(convert.rps_cells({"r1": cell, "r0.5": cell, - "verdict_1x": cell, "setup_r1": cell})), + list(convert.rps_cells({"r1": cell, "r0.5": cell, "verdict_sla": cell, + "verdict_e2e": cell, "setup_r1": cell})), ["r1", "r0.5"]) @@ -277,24 +360,25 @@ class RpsRateTokenTests(unittest.TestCase): def setUpClass(cls): cls.data, cls.warnings, _ = build_and_convert( dataset="soroswap-1500", close_interval="2s", - rates={"ledgers": ["0.25", "0.5", "1"], - "txpage": ["3.75", "7.5", "15"]}) + rates={"ledgers": ["12.5", "25", "50"], + "txpage": ["37.5", "75", "150"]}) cls.cold = cls.data["queries"]["cold"]["soroswap-1500-c1"] def test_tokens_kept_verbatim(self): - self.assertEqual([k for k in self.cold["txpage"] if k != "verdict_1x"], - ["r3.75", "r7.5", "r15"]) - self.assertEqual(self.cold["txpage"]["r3.75"]["target_rps"], 3.75) - self.assertEqual(self.cold["ledgers"]["r0.5"]["target_rps"], 0.5) - - def test_phase1_soroswap_floors_matched(self): - # soroswap phase 1: txpage 3.75 rps, ledgers 0.5 rps — the 1x cell is - # the floor itself, wherever it sits in the ladder the leg swept. + self.assertEqual(rate_keys(self.cold["txpage"]), + ["r37.5", "r75", "r150"]) + self.assertEqual(self.cold["txpage"]["r37.5"]["target_rps"], 37.5) + self.assertEqual(self.cold["ledgers"]["r12.5"]["target_rps"], 12.5) + + def test_soroswap_sla_floors_matched(self): + # txpage 75 rps, ledgers 25 rps — the SLA cell is the floor itself, + # wherever it sits in the ladder the leg swept. The 0.5x rung is the + # fractional token the matching has to survive. self.assertEqual(self.data["campaign"]["phase"], 1) - self.assertEqual(self.cold["txpage"]["verdict_1x"]["rate"], "r3.75") - self.assertEqual(self.cold["txpage"]["verdict_1x"]["target_rps"], 3.75) - self.assertEqual(self.cold["ledgers"]["verdict_1x"]["rate"], "r0.5") - self.assertEqual(self.cold["ledgers"]["verdict_1x"]["target_rps"], 0.5) + self.assertEqual(self.cold["txpage"]["verdict_sla"]["rate"], "r75") + self.assertEqual(self.cold["txpage"]["verdict_sla"]["target_rps"], 75.0) + self.assertEqual(self.cold["ledgers"]["verdict_sla"]["rate"], "r25") + self.assertEqual(self.cold["ledgers"]["verdict_sla"]["target_rps"], 25.0) def test_clean(self): self.assertEqual(self.warnings, []) @@ -326,7 +410,7 @@ def setUpClass(cls): def test_concurrency_cells(self): for qt in ("ledgers", "txpage", "txhash", "events"): self.assertEqual(list(self.cold[qt]), ["c1", "c4"]) - self.assertNotIn("verdict_1x", self.cold[qt]) + self.assertNotIn("verdict_sla", self.cold[qt]) self.assertNotIn("target_rps", self.cold[qt]["c1"]) self.assertIn("ops_s", self.cold[qt]["c1"]) diff --git a/docs/app.js b/docs/app.js index 3838244..2568994 100644 --- a/docs/app.js +++ b/docs/app.js @@ -570,11 +570,40 @@ // AND on the read-path target — so checks_all is the list and checks is its // first entry. Runs published before the list carry only the single object, // hence the fallback: never read a verdict by position. + function allChecks(D) { + return Array.isArray(D.checks_all) ? D.checks_all : (D.checks ? [D.checks] : []); + } function checkFor(D, section) { - const all = Array.isArray(D.checks_all) ? D.checks_all : (D.checks ? [D.checks] : []); - return all.find(c => c && c.applies_to === section) || null; + return allChecks(D).find(c => c && c.applies_to === section) || null; + } + function checkOfKind(D, kind) { + return allChecks(D).find(c => c && c.kind === kind) || null; } + // The p99 one query cell answers to. The SLA states a target per endpoint AND + // storage tier, which the check carries as targets_ns; runs converted before + // that state one number for the whole run in threshold_ns. A cell that earned + // no verdict of its own has to resolve the same way its verdict would have, + // so every judgement outside a verdict goes through here. + function thresholdFor(check, qt, tier) { + const byTier = check && check.targets_ns && check.targets_ns[qt]; + if (byTier && byTier[tier] != null) return byTier[tier]; + return check && check.threshold_ns ? check.threshold_ns : 500e6; + } + + // The read path answers to two requirements and carries a verdict for each. + // verdict_sla is the SLA family — every endpoint, at its share of the + // sustained watermark, against the p99 of its storage tier. verdict_e2e is + // getTransaction's E2E-budget probe — the demand-derived rate, judged on the + // time inside the RPC alone. Runs converted before the split carry one + // verdict_1x that plays the SLA role and folds the in-RPC budget into itself; + // it is read here so those published runs still render, with their in-RPC + // chip intact. Their retired page_budget verdict is deliberately NOT chipped + // any more: the mean page is reported and judged by nothing, so those runs + // show one fewer check than they used to. + const slaVerdict = qout => (qout && (qout.verdict_sla || qout.verdict_1x)) || null; + const e2eVerdict = qout => (qout && qout.verdict_e2e) || null; + // ---- shared query rendering ---- // Both layouts sweep the same cell grid (type × tier × concurrency, per unit), // so the three query figures are built once here and pointed at whichever @@ -587,7 +616,7 @@ const qtLabel = qt => QT_LABEL[qt] || qt; // Open-loop vocabulary. The paced generation names its cells "r" and - // hangs a `verdict_1x` beside them; the section speaks the endpoint names the + // hangs its verdicts beside them; the section speaks the endpoint names the // RPC's public API uses, while the data keeps the CSV's machine names // everywhere. An endpoint the map does not know keeps its raw name. const QT_ENDPOINT = { @@ -600,7 +629,7 @@ // its requests, so the leg measured a queue rather than the endpoint. const SATURATED = 0.95; // A qtype entry's rate cells, ascending BY VALUE (so r500 precedes r1000). - // verdict_1x is a sibling of theirs, not one of them. + // The verdicts are siblings of theirs, not cells of their own. const rateKeys = qout => Object.keys(qout || {}) .filter(k => RATE_KEY.test(k)).sort((a, b) => +a.slice(1) - +b.slice(1)); const fmtRps = v => v == null || !isFinite(v) ? "—" @@ -609,22 +638,24 @@ const cellTarget = (qout, k) => (qout[k] && qout[k].target_rps != null ? qout[k].target_rps : +k.slice(1)); - // The judged rung of one endpoint entry: the cell the run names as this - // phase's floor, else the middle of whatever ladder it swept — an unfloored - // run still reports a measured number, just without a floor to judge it by. + // The judged rung of one endpoint entry: the cell the run names as the SLA + // floor, else the middle of whatever ladder it swept — an unfloored run still + // reports a measured number, just without a floor to judge it by. The SLA + // family names the rung everywhere the two families disagree: it covers all + // four endpoints, where the probe covers getTransaction alone. function judgedRate(qout) { - const v = qout && qout.verdict_1x; + const v = slaVerdict(qout); if (v && v.rate && qout[v.rate]) return v.rate; const ks = rateKeys(qout); return ks.length ? ks[Math.floor((ks.length - 1) / 2)] : null; } // One entry's ladder, labelled for a reader. The multiplier comes from the - // floor the run names; failing that, from the campaign's ladder by ascending - // position; failing that, the rung is labelled with its own rate. + // SLA floor the run names; failing that, from the campaign's ladder by + // ascending position; failing that, the rung is labelled with its own rate. function rungs(qout, ladder) { const ks = rateKeys(qout); - const v = qout && qout.verdict_1x; + const v = slaVerdict(qout); const floor = v && v.target_rps > 0 ? v.target_rps : null; return ks.map((k, i) => { const target = cellTarget(qout, k); @@ -745,7 +776,7 @@ // Query target table: rows are type · tier, columns the concurrency sweep, // each cell the worst unit's median-run p99 against the threshold. function queryTargetTable(tableId, footId, o) { - const { Q, QT, CONC, TIERS, C, units, thr } = o; + const { Q, QT, CONC, TIERS, C, units, thrFor, perEndpoint } = o; const t = document.getElementById(tableId); if (!t) return { over: [], breaches: 0 }; const tr = document.createElement("tr"); @@ -754,10 +785,21 @@ const over = []; let breaches = 0, unitBreaches = 0; for (const qt of QT) for (const tier of TIERS) { + // The target belongs to the endpoint and the tier, so it is a property of + // the ROW; the concurrency columns all answer to the same number. + const thr = thrFor(qt, tier); const r = document.createElement("tr"); const name = document.createElement("td"); const tag = document.createElement("span"); tag.className = "tier-tag"; tag.style.background = tier === "cold" ? C.cold : C.hot; - name.appendChild(tag); name.appendChild(document.createTextNode(`${qt} · ${tier}`)); r.appendChild(name); + name.appendChild(tag); name.appendChild(document.createTextNode(`${qt} · ${tier}`)); + // Only a run with per-endpoint targets needs the number here: with one + // target for the whole run the section intro already states it. + if (perEndpoint) { + const n = document.createElement("span"); + n.className = "cell-note"; n.textContent = ` (≤ ${fmtNsAxis(thr)})`; + name.appendChild(n); + } + r.appendChild(name); for (const cc of CONC) { const w = worstQueryCell(Q, units, tier, qt, cc, thr); const td = document.createElement("td"); @@ -785,17 +827,20 @@ } // ---- open-loop query rendering ---- - // The 1× verdict table: the run's whole read-path answer in one grid, one row - // per endpoint × profile × tier. Three budgets are judged SEPARATELY and never - // folded together — the scheduled-latency target every endpoint carries, plus - // getTransaction's tighter in-RPC budget and getEvents' mean-page budget. + // The SLA verdict table: the run's read-path answer to the sustained-load + // requirement, one row per endpoint × profile × tier, each judged on the + // scheduled p99 against the target its endpoint carries in its storage tier. + // getTransaction's E2E-budget probe is a DIFFERENT requirement measured at a + // different rate; it has its own table and never shares a row with this one. + // Published runs converted before the split carry a single verdict_1x with + // the in-RPC budget folded in as a column: that column stays for them. // Returns the counts so the caller's footnote can reconcile with the chips. function openVerdictTable(tableId, footId, o) { - const { Q, QT, TIERS, C, units, thr, short } = o; + const { Q, QT, TIERS, C, units, thrFor, short } = o; const t = document.getElementById(tableId); if (!t) return { checks: 0, fails: 0, rows: 0, unfloored: 0 }; const head = ["Endpoint · profile · tier", "Offered", "Served", - `Scheduled p99 (≤ ${fmtNsAxis(thr)})`, "In-RPC p99", "Mean page"]; + "Scheduled p99", "In-RPC p99", "Mean page"]; const tr = document.createElement("tr"); head.forEach(h => { const th = document.createElement("th"); th.textContent = h; tr.appendChild(th); }); t.appendChild(tr); @@ -809,7 +854,15 @@ s.textContent = pass ? " ✓" : " ▲"; return s; }; - const note = text => { const s = document.createElement("span"); s.className = "cell-note"; s.textContent = " " + text; return s; }; + // `extra` marks a note a reader has to be able to find on its own — the + // headline target, which is per endpoint and tier and so cannot be read off + // the column head. + const note = (text, extra) => { + const s = document.createElement("span"); + s.className = extra ? "cell-note " + extra : "cell-note"; + s.textContent = " " + text; + return s; + }; const td = (parent, ...kids) => { const c = document.createElement("td"); for (const k of kids) c.appendChild(typeof k === "string" ? document.createTextNode(k) : k); @@ -820,7 +873,7 @@ const qout = ((Q[tier] || {})[u] || {})[qt]; const key = qout && judgedRate(qout); if (!key) continue; - const cell = qout[key] || {}, v = qout.verdict_1x || null; + const cell = qout[key] || {}, v = slaVerdict(qout); if (!v) unfloored++; rows++; const r = document.createElement("tr"); @@ -848,25 +901,31 @@ // its schedule shows the delay it caused. const p99 = v && v.p99_ns != null ? v.p99_ns : (cell.p99 || {}).m; const pTd = td(r, p99 == null ? "—" : fmtNs(p99)); - if (p99 != null) pTd.appendChild(chip(v ? v.pass : p99 <= thr)); - // The two extra budgets belong to one endpoint each — getTransaction - // answers for its time inside the RPC, getEvents for the page a - // sequential subscriber waits on. Every other row leaves them empty - // rather than showing a number no budget applies to. + // Each row states the target it was judged against: the SLA sets one per + // endpoint and storage tier, so no single number heads the column. A row + // with no verdict is judged on the same target its verdict would have + // used, never on a run-wide number this endpoint does not answer to. + const rowThr = v && v.threshold_ns != null ? v.threshold_ns : thrFor(qt, tier); + if (p99 != null) { + pTd.appendChild(chip(v ? v.pass : p99 <= rowThr)); + pTd.appendChild(note(`(≤ ${fmtNsAxis(rowThr)})`, "q-thr")); + } + // The in-RPC time belongs to getTransaction alone; every other row leaves + // it empty rather than showing a number no budget applies to. It is + // JUDGED here only on the published runs that fold it into verdict_1x — + // where the run splits the families, the E2E-budget table judges it at + // its own rate and this column is context. getEvents' mean page is + // reported beside it, judged by nothing. const inR = v && v.in_rpc; const svc = (cell.service || {}).p99; if (inR) { const c = td(r, fmtNs(inR.p99_ns)); c.appendChild(chip(inR.pass)); c.appendChild(note(`(≤ ${fmtNsAxis(inR.threshold_ns)})`)); - } else if (qt === "txhash" && svc) td(r, fmtNs(svc.m), note("(no budget)")); - else td(r, "—"); - const pg = v && v.page_budget; - if (pg) { - const c = td(r, fmtNs(pg.mean_ns)); - c.appendChild(chip(pg.pass)); - c.appendChild(note(`(≤ ${fmtNsAxis(pg.budget_ns)})`)); - } else if (qt === "events" && cell.mean_page_ns) td(r, fmtNs(cell.mean_page_ns.m), note("(no budget)")); + } else if (qt === "txhash" && svc) { + td(r, fmtNs(svc.m), note(o.hasE2E ? "(judged as the E2E budget)" : "(no budget)")); + } else td(r, "—"); + if (qt === "events" && cell.mean_page_ns) td(r, fmtNs(cell.mean_page_ns.m)); else td(r, "—"); t.appendChild(r); } @@ -879,17 +938,94 @@ : `${checks - fails} of ${checks} checks pass at the 1× rate; ${fails} breach. `; foot.appendChild(verdict); foot.appendChild(document.createTextNode( - "Each row is judged on the scheduled p99; getTransaction also carries an in-RPC budget and getEvents a mean-page budget, counted here as checks of their own." - + (unfloored ? ` ${unfloored} of ${rows} rows carry no phase floor — those are judged on the latency target alone.` : ""))); + (o.hasE2E + ? "Each row is judged on the scheduled p99 against its own endpoint and tier target, shown beside it. getTransaction's time inside the RPC is shown as context here and judged in the E2E-budget table, at the rate that budget applies to." + : "Each row is judged on the scheduled p99 against its own endpoint and tier target, shown beside it; getTransaction also carries an in-RPC budget, counted here as a check of its own.") + + " getEvents' mean page is reported, not judged." + + (unfloored ? ` ${unfloored} of ${rows} rows carry no phase floor — those are judged on their endpoint and tier's latency target alone, with no load-model rate to meet it at.` : ""))); } return { checks, fails, rows, unfloored }; } + // The E2E-budget probe table: getTransaction alone, at the demand-derived + // rate this phase asks of this profile, judged on the time INSIDE the RPC — + // its slice of the end-to-end transaction-lifecycle budget. The scheduled p99 + // rides along as context and is deliberately unjudged: the arrival rate here + // is a demand estimate, not a rate the SLA states a tail for. Kept apart from + // the SLA table on purpose — the two answer different questions at different + // rates, and one row carrying both would read as one requirement. + function e2eProbeTable(tableId, footId, o) { + const { Q, TIERS, C, units, short, budgetNs } = o; + const t = document.getElementById(tableId); + if (!t) return { checks: 0, fails: 0, rows: 0 }; + const head = ["Profile · tier", "Offered", "Served", + `In-RPC p99 (≤ ${fmtNsAxis(budgetNs)})`, "Scheduled p99"]; + const tr = document.createElement("tr"); + head.forEach(h => { const th = document.createElement("th"); th.textContent = h; tr.appendChild(th); }); + t.appendChild(tr); + let checks = 0, fails = 0, rows = 0; + for (const u of units) for (const tier of TIERS) { + const qout = ((Q[tier] || {})[u] || {}).txhash; + const v = e2eVerdict(qout); + if (!v) continue; + const cell = qout[v.rate] || {}; + rows++; + const r = document.createElement("tr"); + const name = document.createElement("td"); + const tag = document.createElement("span"); + tag.className = "tier-tag"; tag.style.background = tier === "cold" ? C.cold : C.hot; + name.appendChild(tag); + name.appendChild(document.createTextNode(`${short(u)} · ${tier}`)); + r.appendChild(name); + const td = (...kids) => { + const c = document.createElement("td"); + for (const k of kids) c.appendChild(typeof k === "string" ? document.createTextNode(k) : k); + r.appendChild(c); + return c; + }; + const note = text => { + const s = document.createElement("span"); + s.className = "cell-note"; s.textContent = " " + text; return s; + }; + td(rpsTxt(v.target_rps), note("(demand floor)")); + const ach = cell.achieved_rps; + const sTd = td(ach ? rpsTxt(ach.m) : "—"); + if (ach && v.target_rps > 0 && ach.m < v.target_rps * SATURATED) { + const w = document.createElement("span"); w.className = "cell-warn"; + w.textContent = ` ▲ ${Math.round((1 - ach.m / v.target_rps) * 100)} % under`; + sTd.appendChild(w); + } + const inR = v.in_rpc || {}; + const iTd = td(inR.p99_ns == null ? "—" : fmtNs(inR.p99_ns)); + if (inR.p99_ns != null) { + checks++; if (!inR.pass) fails++; + const s = document.createElement("span"); + s.className = inR.pass ? "cell-ok e2e-pass" : "cell-warn e2e-fail"; + s.textContent = inR.pass ? " ✓" : " ▲"; + iTd.appendChild(s); + } + td(v.p99_ns == null ? "—" : fmtNs(v.p99_ns), note("(context, not judged)")); + t.appendChild(r); + } + const foot = document.getElementById(footId); + if (foot) { + foot.replaceChildren(); + const verdict = document.createElement("strong"); + verdict.textContent = fails === 0 + ? `All ${checks} getTransaction legs hold the in-RPC budget. ` + : `${checks - fails} of ${checks} getTransaction legs hold the in-RPC budget; ${fails} breach. `; + foot.appendChild(verdict); + foot.appendChild(document.createTextNode( + `The budget is ${fmtNsAxis(budgetNs)} — getTransaction's slice of the end-to-end transaction-lifecycle target, the one number this leg answers for. Scheduled p99 includes the client's queueing and the network model, neither of which the E2E slice owns, so it is reported and not judged. The same endpoint's SLA verdict, at a different rate and against a different number, is in the table above.`)); + } + return { checks, fails, rows }; + } + // The three profile-scoped open-loop figures. ids: [offered-vs-served, // p99 ladder, latency detail at the 1× rate]. o.unit() is read on every draw, // so the profile picker just re-invokes the returned function. function openQueryFigs(o) { - const { Q, QT, TIERS, C, ids, thr, ladder } = o; + const { Q, QT, TIERS, C, ids, thr, perEndpoint, ladder } = o; const tierColor = t => (t === "cold" ? C.cold : C.hot); const entry = (tier, qt) => ((Q[tier] || {})[o.unit()] || {})[qt] || null; // Panels share one x axis, so only entries that swept the same number of @@ -977,8 +1113,12 @@ if (lanes.length) rows.push({ label: endpointLabel(qt), sub, lanes }); } if (rows.length) { + // One reference line only when one number judges every lane. A run with + // per-endpoint targets draws none: the lanes are the endpoints, so a + // single line would name a target most of them are not judged against. dotRangeChart(ids[2] + "-body", rows, { - reflines: [{ ns: thr, label: fmtNsAxis(thr) + " — latency target" }], groupSeparators: true, + reflines: perEndpoint ? [] : [{ ns: thr, label: fmtNsAxis(thr) + " — latency target" }], + groupSeparators: true, }); } else document.getElementById(ids[2] + "-body").replaceChildren(); legend(ids[2] + "-legend", [...TIERS.map(t => ({ label: t + " tier", color: tierColor(t) })), @@ -1260,7 +1400,13 @@ // discover query types + concurrency levels from data (degrade gracefully) const { QT, CONC, TIERS } = queryGrid(Q, CH); const qCheck = checkFor(D, "queries"); - const thr = qCheck && qCheck.threshold_ns ? qCheck.threshold_ns : 500e6; + // The pubnet page draws the CLOSED-LOOP grid, whose cells carry no verdict + // of their own, so every judgement here resolves its own target from the + // check — per endpoint and tier when the run states it that way. + const thrFor = (qt, tier) => thresholdFor(qCheck, qt, tier); + const qPerEndpoint = !!(qCheck && qCheck.targets_ns); + // The one number the section intro names, for a run that states one. + const qThr = qPerEndpoint ? null : (qCheck && qCheck.threshold_ns) || 500e6; const qLabel = qCheck ? qCheck.label : "query p99 target"; const chunkSub = c => `${fmtK(um[c].events)} ev`; @@ -1276,10 +1422,11 @@ // A cell no chunk recorded is not a design target the run answered for. if (!w) continue; total++; - if (w.cell.p99.m <= thr) pass++; + if (w.cell.p99.m <= thrFor(qt, tier)) pass++; if (!worst || w.cell.p99.m > worst.cell.p99.m) worst = w; } const worstTxt = worst ? `${worst.tier}-tier ${worst.qt} at ${worst.cc.slice(1)}-way concurrency on chunk ${worst.c} — median-run p99 of ${fmtNs(worst.cell.p99.m)}` : ""; + const worstOk = worst && worst.cell.p99.m <= thrFor(worst.qt, worst.tier); // ---- section skeleton ---- reportEl.innerHTML = mastheadHTML(D) + ` @@ -1289,7 +1436,7 @@ ${PH ? phaseBlockHTML(PH) : ""} @@ -1347,7 +1494,7 @@
07

Design-target check — ${esc(qLabel)}

-

Each cell shows the worst chunk's median-run p99 for that query type, tier, and concurrency. The design target is met when p99 ≤ ${fmtMs(thr)} ms.

+

Each cell shows the worst chunk's median-run p99 for that query type, tier, and concurrency. The design target is met when p99 ≤ ${qPerEndpoint ? "the target that endpoint carries in that tier, printed beside each row" : fmtMs(qThr) + " ms"}.

@@ -1604,7 +1751,8 @@ })(); /* ---- design-target table ---- */ - queryTargetTable("target-table", "target-footnote", { Q, QT, CONC, TIERS, C, units: CH, thr }); + queryTargetTable("target-table", "target-footnote", + { Q, QT, CONC, TIERS, C, units: CH, thrFor, perEndpoint: qPerEndpoint }); document.getElementById("machine-metadata").textContent = (D.machine && D.machine.raw || "").trim(); } @@ -1659,7 +1807,26 @@ const qMode = qGrid ? qGrid.mode : null; const qLadder = (D.campaign && D.campaign.query_load && D.campaign.query_load.ladder) || null; const qCheck = checkFor(D, "queries"); - const qThr = qCheck && qCheck.threshold_ns ? qCheck.threshold_ns : 500e6; + // A run states its read-path target either as one number (the published + // generation) or as a table keyed by endpoint and storage tier (targets_ns, + // the SLA shape). qThrFor resolves whichever it carries, for every cell that + // earned no verdict of its own. + const qThrFor = (qt, tier) => thresholdFor(qCheck, qt, tier); + const qPerEndpoint = !!(qCheck && qCheck.targets_ns); + // One number for the figures that span every endpoint at once; the runs + // that state a target per endpoint draw no such line at all. + const qThr = qPerEndpoint ? null : (qCheck && qCheck.threshold_ns) || 500e6; + // The E2E-budget probe is a second, independent requirement: getTransaction + // at the demand-derived rate, judged on its time inside the RPC. It gets its + // own table, and the whole figure disappears on a run that carries no such + // verdict — every published run predating the split. + const e2eCheck = checkOfKind(D, "query_e2e_probe"); + const hasE2E = !!(Q && e2eCheck && Object.values(Q).some(units => + Object.values(units).some(entry => e2eVerdict((entry || {}).txhash)))); + // Figure numbers inside the queries section. Slot 2 is the E2E-budget + // table; a run without that family closes the gap rather than skipping a + // number, so every figure after it shifts down by one. + const qFig = slot => `Fig ${+secQueries}.${hasE2E || slot < 2 ? slot : slot - 1}`; let secN = 2; const secNum = () => String(++secN).padStart(2, "0"); const secCold = hasCold ? secNum() : null; @@ -1803,21 +1970,29 @@ ${hasQueries && qMode === "open" ? `
${secQueries}

Queries — the read path at a fixed request rate

-

The read side of the same stores the sections above filled, driven open-loop: every endpoint gets a fixed number of requests per second on a fixed schedule, rather than a fixed pool of clients that only asks again once the last answer came back. A server that slows down therefore builds a queue instead of quietly being asked for less. Each endpoint runs three legs — at half, exactly, and twice the request rate this phase asks of the profile in question — for a fixed span each. The latency reported everywhere below is the scheduled one: the clock starts when a request was due, not when the client got round to sending it, so queueing counts against the server rather than disappearing. The verdict is read at the 1× leg; the half and double legs are the context on either side of it. Cold legs drop the profile's files from the OS page cache before measuring; hot legs warm them first, because a warm cache is the hot tier's steady state.

+

The read side of the same stores the sections above filled, driven open-loop: every endpoint gets a fixed number of requests per second on a fixed schedule, rather than a fixed pool of clients that only asks again once the last answer came back. A server that slows down therefore builds a queue instead of quietly being asked for less. Each endpoint runs three legs — at half, exactly, and twice the request rate the load model asks of it — for a fixed span each. The latency reported everywhere below is the scheduled one: the clock starts when a request was due, not when the client got round to sending it, so queueing counts against the server rather than disappearing. The verdict is read at the 1× leg; the half and double legs are the context on either side of it. Cold legs drop the profile's files from the OS page cache before measuring; hot legs warm them first, because a warm cache is the hot tier's steady state.

+ ${hasE2E ? `

This section answers two separate requirements, and keeps them apart. The SLA asks every endpoint to hold its latency target while serving its share of the sustained request rate. The end-to-end budget asks one endpoint, getTransaction, to answer inside a fixed slice of the transaction-lifecycle target, at the rate the phase's own demand model predicts. The two are measured at different rates and judged on different numbers, so they get a table each and never share a row.

` : ""}
-
Fig ${+secQueries}.1Verdict at the 1× request rate
+
${qFig(1)}${hasE2E ? "SLA verdict at the 1× request rate" : "Verdict at the 1× request rate"}
-
One row per endpoint, profile and tier, at the rate this phase asks for. Offered is that rate; served is what the client actually managed to place (${medRuns}, spread in brackets) — well under offered means the endpoint saturated. Scheduled p99 is the headline, judged against ${esc(fmtNsAxis(qThr))}. getTransaction additionally answers for its time inside the RPC alone, and getEvents for the mean page a sequential subscriber waits on; both are separate verdicts and neither is folded into the headline.
+
One row per endpoint, profile and tier, at the rate the SLA asks of that endpoint. Offered is that rate; served is what the client actually managed to place (${medRuns}, spread in brackets) — well under offered means the endpoint saturated. Scheduled p99 is the headline, judged against ${qPerEndpoint ? "the target this endpoint carries in this tier, printed beside it" : esc(fmtNsAxis(qThr))}. ${hasE2E ? "getTransaction's time inside the RPC is context in this table; it is judged in the next one, at the rate its budget applies to." : "getTransaction additionally answers for its time inside the RPC alone — a separate verdict, never folded into the headline."} getEvents' mean page, the wait a sequential subscriber accumulates, is reported beside it and judged by nothing.

+ ${hasE2E ? ` +
+
${qFig(2)}End-to-end budget — getTransaction at the demand-derived rate
+
+
A different question from the table above, asked of one endpoint. Offered is the rate this phase's demand model predicts for this profile, not the SLA rate. In-RPC p99 is the only judged number: the time getTransaction spent inside the RPC, which is its slice of the end-to-end transaction-lifecycle budget. The scheduled p99 beside it carries the client's queueing and is reported for context alone.
+
+

` : ""}

The rate ladder, one profile at a time

Pick a profile to scope the three figures below to it.

${esc(ds.unit_label)}
- ${figHTML("figq2", `Fig ${+secQueries}.2`, "Offered vs served request rate", "figq2-legend", `What the client asked for against what it managed to place, at each rung (${medRuns}; log scale, shared across panels). The two lines sit on top of each other while the endpoint keeps up; the served line falling away is the endpoint refusing more load. The table view names the shortfall and the requests dropped at the in-flight cap.`)} - ${figHTML("figq3", `Fig ${+secQueries}.3`, "Scheduled p99 as the request rate climbs", "figq3-legend", `The tail latency each rung bought (${medRuns}; log scale, shared across panels). A rung far above its neighbour is the knee — past it the endpoint is queueing, not serving.`)} - ${figHTML("figq4", `Fig ${+secQueries}.4`, "Latency at the 1× rate — the whole distribution", "figq4-legend", `Scheduled-latency percentiles over every request of the 1× leg (${medRuns}; log scale). The dashed line is the ${esc(fmtNsAxis(qThr))} latency target. The table view adds each endpoint's time inside the RPC.`)} + ${figHTML("figq2", qFig(3), "Offered vs served request rate", "figq2-legend", `What the client asked for against what it managed to place, at each rung (${medRuns}; log scale, shared across panels). The two lines sit on top of each other while the endpoint keeps up; the served line falling away is the endpoint refusing more load. The table view names the shortfall and the requests dropped at the in-flight cap.`)} + ${figHTML("figq3", qFig(4), "Scheduled p99 as the request rate climbs", "figq3-legend", `The tail latency each rung bought (${medRuns}; log scale, shared across panels). A rung far above its neighbour is the knee — past it the endpoint is queueing, not serving.`)} + ${figHTML("figq4", qFig(5), "Latency at the 1× rate — the whole distribution", "figq4-legend", `Scheduled-latency percentiles over every request of the 1× leg (${medRuns}; log scale). ${qPerEndpoint ? "Each endpoint carries its own latency target, so " + qFig(1) + " states them row by row rather than one dashed line here." : "The dashed line is the " + esc(fmtNsAxis(qThr)) + " latency target."} The table view adds each endpoint's time inside the RPC.`)}
-
Fig ${+secQueries}.5Per-leg setup and event-scan detail
+
${qFig(6)}Per-leg setup and event-scan detail
Setup is the untimed work each leg does before it measures: open builds the read fixture, and evict drops the cold artifacts from the page cache once per leg — a hot leg has nothing to evict, so the row is absent. The event columns are what getEvents actually returned at the 1× rate.
@@ -1835,7 +2010,7 @@
Setup is the untimed work each leg does before it measures: open builds the read fixture, and evict drops the cold artifacts from the page cache once per cell — a hot leg has nothing to evict, so the row is absent. Event counts are the events one page actually returned.

Read-path target — ${esc(qCheck ? qCheck.label : "query p99")}

-

Each cell is the worst profile's median-run p99 for that query type, tier, and concurrency. The target is met when p99 ≤ ${fmtMs(qThr)} ms.

+

Each cell is the worst profile's median-run p99 for that query type, tier, and concurrency. The target is met when p99 ≤ ${qPerEndpoint ? "the target that endpoint carries in that tier, printed beside each row" : fmtMs(qThr) + " ms"}.

` : ""} @@ -2189,7 +2364,7 @@ const { QT, CONC, TIERS } = qGrid; let curProfile = ORDER[0]; const draw = qMode === "open" - ? openQueryFigs({ Q, QT, TIERS, C, ids: ["figq2", "figq3", "figq4"], thr: qThr, ladder: qLadder, unit: () => curProfile }) + ? openQueryFigs({ Q, QT, TIERS, C, ids: ["figq2", "figq3", "figq4"], thr: qThr, perEndpoint: qPerEndpoint, ladder: qLadder, unit: () => curProfile }) : queryFigs({ Q, QT, CONC, TIERS, C, ids: ["figq1", "figq2", "figq3"], unit: () => curProfile }); const holder = document.getElementById("profile-filter"); for (const p of ORDER) { @@ -2254,13 +2429,18 @@ })(); if (qMode === "open") { + const short = u => (parts(u) || {}).name || disp(u); openVerdictTable("query-verdict-table", "query-verdict-footnote", - { Q, QT, TIERS, C, units: ORDER, thr: qThr, short: u => (parts(u) || {}).name || disp(u) }); + { Q, QT, TIERS, C, units: ORDER, thrFor: qThrFor, hasE2E, short }); + if (hasE2E) { + e2eProbeTable("query-e2e-table", "query-e2e-footnote", + { Q, TIERS, C, units: ORDER, short, budgetNs: e2eCheck.threshold_ns }); + } return; } const { breaches, unitBreaches } = queryTargetTable("query-target-table", "query-target-footnote", - { Q, QT, CONC, TIERS, C, units: ORDER, thr: qThr }); + { Q, QT, CONC, TIERS, C, units: ORDER, thrFor: qThrFor, perEndpoint: qPerEndpoint }); const total = QT.length * TIERS.length * CONC.length; const foot = document.getElementById("query-target-footnote"); if (foot) { diff --git a/docs/sla-derivation.md b/docs/sla-derivation.md new file mode 100644 index 0000000..9d7775e --- /dev/null +++ b/docs/sla-derivation.md @@ -0,0 +1,71 @@ +# How the query targets were derived + +This is the record behind the numbers in [`targets.json`](targets.json) `query_load`. Derived 2026-09-01 from the team RPC SLA doc, sections 3.3, 3.4 and 4.1. Change a number here and in `targets.json` together; nothing else in this repo hardcodes one. + +The benchmark judges **two separate requirements**. They are measured at different rates and against different numbers, and the reports keep them apart. Do not fold them into one verdict. + +## Family 1 — the SLA + +### Request rates + +Section 3.4 states a sustained request-rate watermark per service tier, for one RPC node: Light approximately 250 rps, Standard approximately 500 rps, Heavy approximately 1000 rps. It also states the traffic mix that watermark assumes. + +| Endpoint | Mix share | Floor at Standard (rps) | +|---|---|---| +| getTransaction (`txhash`) | 0.60 | 300 | +| getEvents (`events`) | 0.20 | 100 | +| getTransactions (`txpage`) | 0.15 | 75 | +| getLedgers (`ledgers`) | 0.05 | 25 | + +Each floor is that endpoint's mix share times the Standard watermark of 500 rps. The mix is a property of the traffic, not of the dataset or the ledger close time, so one floor per endpoint holds in every phase and every dataset profile. + +The `0.5x / 1x / 2x` ladder every leg sweeps IS the tier set: `0.5x` is Light, `1x` is Standard, `2x` is Heavy. The verdict is read at `1x`; the other two rungs show where the endpoint sits relative to the tier above and below. + +Section 3.4 calls these watermarks conservative placeholders that need a load-test backstop before they ship. These benchmarks are that backstop. + +### Latency targets + +Section 4.1 states a P75 and a P99 per endpoint, per data-age window. The benchmark judges the P99: it is the SLA's binding number, and the bench emits p50, p90 and p99, so there is no p75 to compare against. + +The two data-age windows the boxes can measure map onto the two tiers the bench already runs. + +| SLA window | Bench tier | Why it matches | +|---|---|---| +| Live | `hot` | The active-chunk RocksDB store. The bench warms it before measuring, which is the hot-cache state the window assumes. | +| Recent | `cold` | Frozen artifacts on the local NVMe. The bench drops them from the page cache before measuring, which is the cold-cache state the window assumes. | +| Historical | none | Served from frozen artifacts on EBS. The benchmark boxes carry no EBS tier, so no leg measures this window. It is out of scope. | + +| Endpoint | Live (hot) P99 | Recent (cold) P99 | +|---|---|---| +| getTransaction (`txhash`) | 20 ms | 30 ms | +| getTransactions (`txpage`) | 60 ms | 80 ms | +| getEvents (`events`) | 40 ms | 40 ms | +| getLedgers (`ledgers`) | 150 ms | 200 ms | + +### Request shapes + +The section 4.1 rows state the request each latency applies to, so the bench drives the same shapes. + +- getLedgers: 10 ledgers per request. +- getTransactions: 200 transactions per page. +- getEvents: a small page, 10 matches — the shape the section 3.4 mix names. + +## Family 2 — the end-to-end-budget probe + +getTransaction also runs at the demand-derived floors of work item 856, per dataset profile and per phase: sac 300 / 500 / 1000 rps, custom_token 200 / 400 / 600, soroswap 75 / 150 / 300. + +That leg answers for one number only: an in-RPC P99 of 10 ms. This is getTransaction's slice of the end-to-end transaction-lifecycle budget in `targets.json` `fixed_estimates`, so it measures the time inside the RPC and excludes the client's queueing and the assumed network legs. The scheduled P99 of the same cell is reported for context and is not judged: the arrival rate here is a demand estimate, not a rate the SLA states a tail for. + +This is a different requirement from the SLA, at a different rate, against a different number. The two must not be conflated. + +getTransaction therefore sweeps both ladders in ONE leg — `bench-query --target-rps` takes a list and emits a cell per rate — as the sorted, deduplicated union of the two. Where the two floors coincide, as they do for sac at phase 1, the shared cell carries both verdicts. + +## Rejected alternative, kept for the record + +Before section 3.4 was available, the floors were derived by Little's law from the section 3.3 per-endpoint concurrency caps: floor = cap / P99. getLedgers gave 32 / 0.200 s = 160 rps, and the other endpoints gave figures of the same order. + +That method is discarded. Section 3.4 states the intended sustained load directly, so there is no need to infer it from a cap. The concurrency caps remain background context: they bound what a node accepts at once, not what it is expected to sustain. + +## Known modeling caveat + +Each leg drives one endpoint at a time. The mix rates are therefore applied per endpoint in sequence, not as one blended 500 rps stream, so contention between endpoints is not reproduced. A blended-mix leg would measure that; no leg does today. diff --git a/docs/styles.css b/docs/styles.css index 8193732..15ad7d7 100644 --- a/docs/styles.css +++ b/docs/styles.css @@ -179,6 +179,10 @@ code { font-family: ui-monospace, Menlo, Consolas, monospace; font-size: 0.9em; /* ---------- figures ---------- */ figure.fig { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; margin: 22px 0; padding: 18px 20px 12px; } +/* A figure answering a DIFFERENT requirement from the one above it. The rule + down the left edge is the whole signal: same shape, visibly not the same + question. */ +figure.fig.fig-alt { border-left: 3px solid var(--accent); background: color-mix(in srgb, var(--accent) 4%, var(--surface)); } .fig-head { display: flex; justify-content: space-between; gap: 16px; align-items: baseline; flex-wrap: wrap; margin-bottom: 4px; } .fig-title { font-weight: 650; font-size: 15px; } .fig-no { diff --git a/docs/summary.js b/docs/summary.js index 063f942..c73f524 100644 --- a/docs/summary.js +++ b/docs/summary.js @@ -692,9 +692,10 @@ const fmtRps = v => v == null || !isFinite(v) ? "—" : (v >= 100 ? fmtInt(v) : v >= 10 ? String(+v.toFixed(1)) : String(+v.toFixed(2))) + " rps"; - // Every (tier, profile, endpoint) leg carrying a 1× result, grouped by tier - // and ordered endpoint-first. The source field is named verdict_1x, but the - // summary presents its p99 and request rate without a read-path verdict. + // Every (tier, profile, endpoint) leg carrying an SLA result, grouped by tier + // and ordered endpoint-first. The summary presents its p99 and request rate + // without a read-path verdict. Runs converted before the SLA and E2E-budget + // families were split carry one verdict_1x that plays the SLA role. function queryVerdicts(Q, order) { const out = {}; for (const tier of TIER_ORDER) { @@ -709,8 +710,9 @@ const qts = [...QT_ORDER.filter(q => seen.includes(q)), ...seen.filter(q => !QT_ORDER.includes(q))]; const rows = []; for (const qt of qts) for (const u of order) { - const v = ((t[u] || {})[qt] || {}).verdict_1x; - if (v && v.p99_ns != null) rows.push({ unit: u, qt, v }); + const qout = (t[u] || {})[qt] || {}; + const v = qout.verdict_sla || qout.verdict_1x; + if (v && v.p99_ns != null) rows.push({ unit: u, qt, v, e2e: qout.verdict_e2e || null }); } if (rows.length) out[tier] = rows; } @@ -718,12 +720,19 @@ } // Worst in-RPC getTransaction p99 over the live-store legs — the E2E model // carries the network legs as constants, so only the in-RPC slice belongs in - // the budget. Null when the run measured none: the allocation then stands. + // the budget. It comes from the E2E-budget probe, which is the leg run at the + // demand-derived rate for exactly this purpose; runs predating the split + // carry it folded into their single verdict instead. Null when the run + // measured none: the allocation then stands. function measuredGetTx(QV) { let worst = null; for (const r of (QV.hot || [])) { - const ir = r.qt === "txhash" && r.v.in_rpc; - if (ir && typeof ir.p99_ns === "number" && (!worst || ir.p99_ns > worst.ns)) worst = { ns: ir.p99_ns, unit: r.unit }; + if (r.qt !== "txhash") continue; + const ir = (r.e2e && r.e2e.in_rpc) || r.v.in_rpc; + if (ir && typeof ir.p99_ns === "number" && (!worst || ir.p99_ns > worst.ns)) { + worst = { ns: ir.p99_ns, unit: r.unit, rps: (r.e2e || r.v).target_rps, + fromProbe: !!r.e2e }; + } } return worst; } @@ -1016,7 +1025,7 @@ ${qTiers.map(tier => `

${esc(TIER_TITLE[tier] || tier)}

${esc(TIER_NOTE[tier] || "")}

${queryTableHTML(tier)} - ${tier === "hot" && canBudget && MEASURED_GET ? `

The end-to-end budget in §${secNo("budget")} uses the highest measured getTransaction p99.

` : ""}`).join("")} + ${tier === "hot" && canBudget && MEASURED_GET ? `

The end-to-end budget in §${secNo("budget")} uses the highest measured getTransaction p99${MEASURED_GET.fromProbe ? `, taken from its end-to-end probe leg at ${esc(fmtRps(MEASURED_GET.rps))}` : ""}.

` : ""}`).join("")} ` : ""; /* ---- pacing prose ---- */ diff --git a/docs/targets.json b/docs/targets.json index 1a4fb2a..8b96d5d 100644 --- a/docs/targets.json +++ b/docs/targets.json @@ -7,40 +7,39 @@ "send_tx_p99_ns": 10000000, "get_tx_p99_ns": 10000000 }, - "query_p99_note": "Read-path design target, one number for every phase: no query cell (type x tier x concurrency) may exceed this p99. It sits outside fixed_estimates because it is not a slice of the E2E formula — the read benchmarks judge against it directly.", - "query_p99_target_ns": 500000000, - "query_load_note": "Open-loop query load model: per endpoint x profile x phase RPS floors on a single RPC node, derived 2026-08-27 from the management read/write patterns guideline (benchmarking hub, 856-query-load-targets.md). Load is an arrival rate, not a concurrency level: each cell is paced at ladder x floor (0.5x/1x/2x); the verdict cell is 1x. Profiles are keyed by dataset model name (the dataset name with its per-ledger tx count stripped: sac-6000 -> sac). Per-profile arrays are indexed by phase: [phase 1, phase 2, phase 3]. The p99 SLA at the target rate is query_p99_target_ns above. get_tx_in_rpc_p99_ns is getTransaction's additional in-RPC p99 target — a separate verdict from the E2E-slice estimate in fixed_estimates. events_page_budget_ns is getEvents' additional per-phase MEAN page-latency budget (a sequential subscriber needs each page's cursor, so mean overrun accumulates as lag), not a p99.", + "query_load_note": "Open-loop query load model, derived 2026-09-01. Two families of requirement, judged and reported separately — never folded into one verdict. Full derivation: docs/sla-derivation.md. (1) sla: every endpoint runs at its share of the team RPC SLA doc's Standard-tier sustained watermark (section 3.4: 500 rps aggregate on one node, mix txhash 0.60 / events 0.20 / txpage 0.15 / ledgers 0.05) and answers for the p99 of section 4.1. The floors are a property of the endpoint alone, so they are the same in every phase and every dataset profile, and they are stored once. p99_ns is keyed by endpoint then storage tier, and the tier keys are the SLA's data-age windows: hot = Live, cold = Recent (frozen-NVMe). The third window, Historical (frozen-EBS), is out of scope — the benchmark boxes carry no EBS tier. (2) e2e_probe: getTransaction ALSO runs at the demand-derived floors of work item 856, per profile and per phase, and answers only for in_rpc_p99_ns, its slice of the end-to-end transaction-lifecycle budget in fixed_estimates. The scheduled p99 of that cell is reported, never judged. Load is an arrival rate, not a concurrency level: each leg is paced at ladder x floor (0.5x/1x/2x) and the verdict cell is 1x. For the sla family the ladder IS the doc's tier set: 0.5x = Light (250 rps aggregate), 1x = Standard (500), 2x = Heavy (1000). getTransaction therefore sweeps ONE leg carrying both ladders, deduped and sorted ascending; where the two floors coincide the shared cell carries both verdicts. Profiles are keyed by dataset model name (the dataset name with its per-ledger tx count stripped: sac-6000 -> sac). e2e_probe.floors_rps arrays are indexed by phase: [phase 1, phase 2, phase 3]. A leg drives one endpoint at a time, so the mix rates apply per endpoint in sequence, not as one blended 500 rps stream.", "query_load": { "ladder": [0.5, 1, 2], - "derivation": { - "read_fraction": 0.1, - "txpage_page_limit": 200, - "events_page_limit": 1000, - "events_per_tx": {"sac": 1, "custom_token": 1, "soroswap": 5} + "sla": { + "floors_rps": {"txhash": 300, "txpage": 75, "events": 100, "ledgers": 25}, + "p99_ns": { + "txhash": { "hot": 20000000, "cold": 30000000 }, + "txpage": { "hot": 60000000, "cold": 80000000 }, + "events": { "hot": 40000000, "cold": 40000000 }, + "ledgers": { "hot": 150000000, "cold": 200000000 } + } }, - "get_tx_in_rpc_p99_ns": 10000000, - "profiles": { - "sac": { - "txhash_rps": [300, 500, 1000], - "txpage_rps": [15, 25, 50], - "ledgers_rps": [0.5, 1, 1.67], - "events_rps": [3, 5, 10], - "events_page_budget_ns": [333000000, 200000000, 100000000] - }, - "custom_token": { - "txhash_rps": [200, 400, 600], - "txpage_rps": [10, 20, 30], - "ledgers_rps": [0.5, 1, 1.67], - "events_rps": [2, 4, 6], - "events_page_budget_ns": [500000000, 250000000, 167000000] - }, - "soroswap": { - "txhash_rps": [75, 150, 300], - "txpage_rps": [3.75, 7.5, 15], - "ledgers_rps": [0.5, 1, 1.67], - "events_rps": [3.75, 7.5, 15], - "events_page_budget_ns": [267000000, 133000000, 67000000] + "e2e_probe": { + "in_rpc_p99_ns": 10000000, + "floors_rps": { + "sac": [300, 500, 1000], + "custom_token": [200, 400, 600], + "soroswap": [75, 150, 300] } + }, + "derivation": { + "full_record": "docs/sla-derivation.md", + "source": "team RPC SLA doc, sections 3.3, 3.4 and 4.1", + "derived": "2026-09-01", + "aggregate_rps": 500, + "mix": {"txhash": 0.6, "events": 0.2, "txpage": 0.15, "ledgers": 0.05}, + "sla_tiers_rps": {"light": 250, "standard": 500, "heavy": 1000}, + "tier_windows": {"hot": "Live", "cold": "Recent (frozen-NVMe)"}, + "out_of_scope": "Historical (frozen-EBS): the benchmark boxes carry no EBS tier, so no leg measures it", + "txpage_page_limit": 200, + "events_page_limit": 10, + "ledgers_span": 10, + "e2e_probe_floors": "demand-derived values from work item 856 — getTransaction is the E2E-budget probe" } }, "phases": [ diff --git a/runner/README.md b/runner/README.md index f4c7264..ad15a0b 100644 --- a/runner/README.md +++ b/runner/README.md @@ -229,6 +229,13 @@ under it. The rates are not a knob — they come from [`docs/targets.json`](../docs/targets.json), this repo's single source of truth for the goal, and no floor is hardcoded in the runner. +### Two families of floor + +The campaign answers two separate requirements, and the load model keeps them apart. The full derivation of both is in [`docs/sla-derivation.md`](../docs/sla-derivation.md). + +- **The SLA family** (`query_load.sla`) asks every endpoint to hold its latency target while it serves its share of the sustained request-rate watermark. Its floor is a property of the endpoint alone — `txhash` 300, `events` 100, `txpage` 75, `ledgers` 25 rps — so it is the same in every phase and every dataset profile. +- **The end-to-end-budget probe** (`query_load.e2e_probe`) asks one endpoint, `txhash`, to answer inside the 10 ms slice it owns in the transaction-lifecycle budget, at the rate the demand model of work item 856 predicts. Those floors ARE per profile and per phase: sac reads 300 / 500 / 1000, custom_token 200 / 400 / 600, soroswap 75 / 150 / 300. + Three things decide a leg's rate ladder: - **Endpoint type.** `ledgers`, `txpage`, `txhash`, `events` each have their own floor, so @@ -236,21 +243,39 @@ Three things decide a leg's rate ladder: therefore produces **four legs per tier**, in that order: `query-{cold,hot}--c--run`. - **Dataset profile.** The dataset name minus a trailing `-` is the - `query_load.profiles` key: `sac-6000` → `sac`, `soroswap-1500` → `soroswap`. A dataset - whose name has no profile is refused before the campaign starts, naming both. + `query_load.e2e_probe.floors_rps` key: `sac-6000` → `sac`, `soroswap-1500` → `soroswap`. A + dataset whose name has no profile is refused before the campaign starts, naming both. - **Phase.** `phase = 1|2|3`, or derived from a `close_interval` that is a phase's block time (`"2s"` → phase 1, `"1s"` → 2, `"600ms"` → 3). When both are set they must agree — pacing ledgers at one phase while measuring another phase's read targets is not a phase run. A query campaign with neither is refused. -Each leg then sweeps `query_load.ladder` (`0.5×`, `1×`, `2×`) around that cell's floor and -passes the result as one `--target-rps` list, so the plan reads: +Each leg then sweeps `query_load.ladder` (`0.5×`, `1×`, `2×`) around its floor and passes +the result as one `--target-rps` list, so a `getEvents` leg reads: + +``` +$ …/stellar-rpc- bench-query cold --cold-dir=… --start-chunk=1 --num-chunks=1 \ + --types=events --target-rps=50,100,200 --duration=60s --out=… +``` + +For the SLA family the ladder IS the SLA's load tiers: `0.5×` is Light (250 rps aggregate), +`1×` is Standard (500 rps, the sustained watermark), and `2×` is Heavy (1000 rps). The +verdict is read at `1×`; the other two rungs are the context around it. + +**`txhash` carries both families in one leg.** `bench-query --target-rps` already takes a +list and emits one cell per rate, so no second leg is needed: the ladder is the union of +the two, deduplicated and sorted ascending. For sac at phase 3 the SLA ladder is +`150,300,600` and the demand ladder is `500,1000,2000`, so the leg reads: ``` $ …/stellar-rpc- bench-query cold --cold-dir=… --start-chunk=1 --num-chunks=1 \ - --types=txhash --target-rps=500,1000,2000 --duration=60s --out=… + --types=txhash --target-rps=150,300,500,600,1000,2000 --duration=60s --out=… ``` +The converter then judges `r300` against the SLA p99 and `r1000` against the 10 ms in-RPC +budget. Where the two floors coincide — sac at phase 1, whose demand floor is also 300 rps +— the ladder collapses to `150,300,600` and the one `r300` cell carries both verdicts. + `--targets ` on `run`, `plan`, and `preflight` overrides which targets file is read; by default the runner walks up from the working directory to the `docs/targets.json` of the checkout it is running from. `preflight` resolves the whole load model, so a bad diff --git a/runner/cmd/campaign/main_test.go b/runner/cmd/campaign/main_test.go index df213f7..4cc2c56 100644 --- a/runner/cmd/campaign/main_test.go +++ b/runner/cmd/campaign/main_test.go @@ -194,8 +194,10 @@ func TestPlanCmdQueryLoad(t *testing.T) { } for _, want := range []string{ "== query-cold-sac-6000-c1-ledgers-run1\n", - "--types=ledgers --target-rps=0.835,1.67,3.34 --duration=60s", - "--types=txhash --target-rps=500,1000,2000 --duration=60s", + "--types=ledgers --target-rps=12.5,25,50 --duration=60s", + // One txhash leg carries both ladders: the SLA one (150, 300, 600) + // unioned with sac's phase-3 demand ladder (500, 1000, 2000). + "--types=txhash --target-rps=150,300,500,600,1000,2000 --duration=60s", } { if !strings.Contains(stdout, want) { t.Errorf("stdout missing %q, got:\n%s", want, stdout) @@ -208,7 +210,7 @@ func TestPlanCmdQueryLoad(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0 (stderr: %s)", code, stderr) } - if want := "--types=txhash --target-rps=250,500,1000 --duration=120s"; !strings.Contains(stdout, want) { + if want := "--types=txhash --target-rps=150,250,300,500,600,1000 --duration=120s"; !strings.Contains(stdout, want) { t.Errorf("stdout missing %q, got:\n%s", want, stdout) } }) diff --git a/runner/cmd/campaign/run_test.go b/runner/cmd/campaign/run_test.go index 754e740..3c1877f 100644 --- a/runner/cmd/campaign/run_test.go +++ b/runner/cmd/campaign/run_test.go @@ -46,7 +46,7 @@ func TestRunCmdDryRunTouchesNothing(t *testing.T) { "== query-cold-sac-6000-c1-txhash-run1\n", "--types=txhash --target-rps=150,300,600 --duration=60s", "== query-hot-soroswap-1500-c2-events-run2\n", - "--types=events --target-rps=1.875,3.75,7.5 --duration=60s", + "--types=events --target-rps=50,100,200 --duration=60s", "$ campaign publish " + filepath.Join(benchRoot, "results"), "dry run complete", } { diff --git a/runner/internal/plan/plan_test.go b/runner/internal/plan/plan_test.go index acbc00c..435857b 100644 --- a/runner/internal/plan/plan_test.go +++ b/runner/internal/plan/plan_test.go @@ -32,10 +32,12 @@ func goldenInputs() Inputs { } } -// goldenRates stands in for what the CLI resolves out of docs/targets.json: the -// phase-1 ladders of the profiles these tests' datasets name. Build never reads -// that file, so pinning the numbers keeps the golden plan a function of the -// config alone. +// goldenRates stands in for what the CLI resolves out of docs/targets.json. +// Build never reads that file, so pinning the numbers keeps the golden plan a +// function of the config alone — and pinned they stay: these are the phase-1 +// ladders of the demand-derived model, not of the SLA floors the file carries +// today. What the golden plan asserts is that Build lays a leg per endpoint and +// spells the ladder it is handed, whatever the numbers in it are. func goldenRates() map[string]map[string][]float64 { sac := map[string][]float64{ "ledgers": {0.25, 0.5, 1}, diff --git a/runner/internal/targets/targets.go b/runner/internal/targets/targets.go index 1433953..fe2ddf6 100644 --- a/runner/internal/targets/targets.go +++ b/runner/internal/targets/targets.go @@ -2,12 +2,14 @@ // for the performance goal, and answers the one question the runner asks of it: // how fast should each query leg be paced? // -// The floors differ per endpoint type, per dataset profile, and per phase, so -// nothing here is hardcoded in Go — the file is loaded at campaign time and the -// numbers it carries become the `--target-rps` ladders in the plan. Everything -// else in the file (latency budgets, verdict inputs) belongs to the converter -// and the viewer; this package models only the query_load section and the phase -// block times it is indexed by. +// Two families of floor answer that question. The SLA floors belong to the +// endpoint alone and hold in every phase and every dataset profile; the +// E2E-probe floors belong to getTransaction and differ per profile and per +// phase. Nothing here is hardcoded in Go — the file is loaded at campaign time +// and the numbers it carries become the `--target-rps` ladders in the plan. +// Everything else in the file (latency budgets, verdict inputs) belongs to the +// converter and the viewer; this package models only the query_load section and +// the phase block times it is indexed by. package targets import ( @@ -16,6 +18,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "sort" "strings" ) @@ -37,20 +40,30 @@ type Phase struct { BlockTimeNs int64 `json:"block_time_ns"` } -// QueryLoad is the open-loop load model: a floor per (profile, phase, endpoint -// type), and the ladder of multipliers every leg sweeps around that floor. +// QueryLoad is the open-loop load model: two families of floor and the ladder +// of multipliers every leg sweeps around them. SLA floors belong to the +// endpoint alone; the E2E probe's floors belong to the dataset profile and the +// phase. getTransaction carries both, so its leg sweeps both ladders. type QueryLoad struct { - Ladder []float64 `json:"ladder"` - Profiles map[string]Profile `json:"profiles"` + Ladder []float64 `json:"ladder"` + SLA SLA `json:"sla"` + E2EProbe E2EProbe `json:"e2e_probe"` } -// Profile is one dataset model's floors. Each array is indexed by phase minus -// one, so entry 0 is phase 1. -type Profile struct { - LedgersRPS []float64 `json:"ledgers_rps"` - TxpageRPS []float64 `json:"txpage_rps"` - TxhashRPS []float64 `json:"txhash_rps"` - EventsRPS []float64 `json:"events_rps"` +// SLA is the read-path requirement: one arrival rate and one p99 per endpoint, +// the same in every phase and every profile. P99Ns is keyed by endpoint then +// storage tier; the runner reads only the floors, the converter judges the p99. +type SLA struct { + FloorsRPS map[string]float64 `json:"floors_rps"` + P99Ns map[string]map[string]int64 `json:"p99_ns"` +} + +// E2EProbe is getTransaction's second leg: the demand-derived floors of work +// item 856, per profile, each array indexed by phase minus one so entry 0 is +// phase 1. It answers for its in-RPC p99 alone. +type E2EProbe struct { + InRPCP99Ns int64 `json:"in_rpc_p99_ns"` + FloorsRPS map[string][]float64 `json:"floors_rps"` } // Load reads and validates the targets file at path. The validation is @@ -68,23 +81,33 @@ func Load(path string) (*Targets, error) { if len(t.Phases) == 0 { return nil, fmt.Errorf("targets: %s: phases is empty", path) } - if len(t.QueryLoad.Profiles) == 0 { - return nil, fmt.Errorf("targets: %s: query_load.profiles is empty — this file predates the paced-RPS query model", path) + if len(t.QueryLoad.E2EProbe.FloorsRPS) == 0 { + return nil, fmt.Errorf("targets: %s: query_load.e2e_probe.floors_rps is empty — this file predates the two-family query model", path) } if len(t.QueryLoad.Ladder) == 0 { return nil, fmt.Errorf("targets: %s: query_load.ladder must list at least one multiplier", path) } - for _, name := range sortedKeys(t.QueryLoad.Profiles) { - p := t.QueryLoad.Profiles[name] - for _, qtype := range QueryTypes { - rps, err := p.rates(qtype) - if err != nil { - return nil, fmt.Errorf("targets: %s: %w", path, err) - } - if len(rps) != len(t.Phases) { - return nil, fmt.Errorf("targets: %s: query_load.profiles.%s.%s_rps has %d entries, want one per phase (%d)", - path, name, qtype, len(rps), len(t.Phases)) - } + // The SLA family names every endpoint, twice: once for the rate it is + // driven at and once for the p99 it answers for. A missing entry would + // otherwise surface hours later as a leg paced at zero. + for _, qtype := range QueryTypes { + if _, ok := t.QueryLoad.SLA.FloorsRPS[qtype]; !ok { + return nil, fmt.Errorf("targets: %s: query_load.sla.floors_rps has no %s floor, want one per endpoint (%s)", + path, qtype, strings.Join(QueryTypes, ", ")) + } + if len(t.QueryLoad.SLA.P99Ns[qtype]) == 0 { + return nil, fmt.Errorf("targets: %s: query_load.sla.p99_ns has no %s entry, want one per endpoint (%s)", + path, qtype, strings.Join(QueryTypes, ", ")) + } + } + if extra := extraKeys(t.QueryLoad.SLA.FloorsRPS); extra != "" { + return nil, fmt.Errorf("targets: %s: query_load.sla.floors_rps names %s, which is not a query type (%s)", + path, extra, strings.Join(QueryTypes, ", ")) + } + for _, name := range sortedKeys(t.QueryLoad.E2EProbe.FloorsRPS) { + if rps := t.QueryLoad.E2EProbe.FloorsRPS[name]; len(rps) != len(t.Phases) { + return nil, fmt.Errorf("targets: %s: query_load.e2e_probe.floors_rps.%s has %d entries, want one per phase (%d)", + path, name, len(rps), len(t.Phases)) } } return &t, nil @@ -137,50 +160,64 @@ func (t *Targets) MatchPhase(closeIntervalNs int64) int { // plan's leg order is its own; this list is the vocabulary of the file. var QueryTypes = []string{"ledgers", "txpage", "txhash", "events"} -// Rates is the RPS ladder one query leg targets: the (profile, phase, qtype) -// floor multiplied by each ladder step, in ladder order. +// Rates is the RPS ladder one query leg targets, ascending. +// +// Three of the four endpoints answer to the SLA family alone, so their ladder +// is the endpoint's SLA floor times each ladder step — the same list in every +// phase and every profile. getTransaction answers to both families and runs as +// ONE leg: its ladder is the union of the SLA ladder and the (profile, phase) +// E2E-probe ladder, deduplicated. Where the two floors coincide the shared cell +// carries both verdicts, which is why the union is deduplicated rather than +// concatenated. func (t *Targets) Rates(profileKey string, phase int, qtype string) ([]float64, error) { - profile, ok := t.QueryLoad.Profiles[profileKey] + if !slices.Contains(QueryTypes, qtype) { + return nil, fmt.Errorf("targets: unknown query type '%s' (known types: %s)", qtype, strings.Join(QueryTypes, ", ")) + } + // Every leg resolves the profile and the phase, even the three that do not + // pace by them: a run that names neither is a run nobody can judge. + e2eFloors, ok := t.QueryLoad.E2EProbe.FloorsRPS[profileKey] if !ok { return nil, fmt.Errorf("targets: no query_load profile '%s' (known profiles: %s)", - profileKey, strings.Join(sortedKeys(t.QueryLoad.Profiles), ", ")) + profileKey, strings.Join(sortedKeys(t.QueryLoad.E2EProbe.FloorsRPS), ", ")) } - floors, err := profile.rates(qtype) - if err != nil { - return nil, err + if phase < 1 || phase > len(e2eFloors) { + return nil, fmt.Errorf("targets: phase %d has no floors (query_load carries phases 1-%d)", phase, len(e2eFloors)) } - if phase < 1 || phase > len(floors) { - return nil, fmt.Errorf("targets: phase %d has no floors (query_load carries phases 1-%d)", phase, len(floors)) + rates := t.ladderAround(t.QueryLoad.SLA.FloorsRPS[qtype]) + if qtype == "txhash" { + rates = append(rates, t.ladderAround(e2eFloors[phase-1])...) + slices.Sort(rates) + rates = slices.Compact(rates) } - floor := floors[phase-1] + return rates, nil +} + +// ladderAround is one floor times every ladder step, in ladder order. +func (t *Targets) ladderAround(floor float64) []float64 { rates := make([]float64, len(t.QueryLoad.Ladder)) for i, step := range t.QueryLoad.Ladder { rates[i] = floor * step } - return rates, nil -} - -// rates picks the floor array one endpoint type is paced from. -func (p Profile) rates(qtype string) ([]float64, error) { - switch qtype { - case "ledgers": - return p.LedgersRPS, nil - case "txpage": - return p.TxpageRPS, nil - case "txhash": - return p.TxhashRPS, nil - case "events": - return p.EventsRPS, nil - } - return nil, fmt.Errorf("targets: unknown query type '%s' (known types: %s)", qtype, strings.Join(QueryTypes, ", ")) + return rates } // sortedKeys keeps every message that lists profiles stable. -func sortedKeys(profiles map[string]Profile) []string { - names := make([]string, 0, len(profiles)) - for name := range profiles { +func sortedKeys[V any](m map[string]V) []string { + names := make([]string, 0, len(m)) + for name := range m { names = append(names, name) } sort.Strings(names) return names } + +// extraKeys names the keys of m that are not query types, or "" when none are. +func extraKeys[V any](m map[string]V) string { + var extra []string + for _, name := range sortedKeys(m) { + if !slices.Contains(QueryTypes, name) { + extra = append(extra, "'"+name+"'") + } + } + return strings.Join(extra, ", ") +} diff --git a/runner/internal/targets/targets_test.go b/runner/internal/targets/targets_test.go index 7ea8f5a..a930429 100644 --- a/runner/internal/targets/targets_test.go +++ b/runner/internal/targets/targets_test.go @@ -29,10 +29,27 @@ func TestLoadRepoTargets(t *testing.T) { t.Errorf("ladder = %v, want [0.5 1 2]", tg.QueryLoad.Ladder) } for _, name := range []string{"sac", "custom_token", "soroswap"} { - if _, ok := tg.QueryLoad.Profiles[name]; !ok { - t.Errorf("query_load has no profile %q", name) + if _, ok := tg.QueryLoad.E2EProbe.FloorsRPS[name]; !ok { + t.Errorf("query_load.e2e_probe has no profile %q", name) } } + // The SLA family is stored once, not per profile: one floor and one p99 + // pair per endpoint. + for qtype, want := range map[string]float64{ + "txhash": 300, "txpage": 75, "events": 100, "ledgers": 25, + } { + if got := tg.QueryLoad.SLA.FloorsRPS[qtype]; got != want { + t.Errorf("sla floor for %s = %v, want %v", qtype, got, want) + } + for _, tier := range []string{"hot", "cold"} { + if tg.QueryLoad.SLA.P99Ns[qtype][tier] == 0 { + t.Errorf("sla p99 for %s/%s is missing", qtype, tier) + } + } + } + if got := tg.QueryLoad.E2EProbe.InRPCP99Ns; got != 10000000 { + t.Errorf("e2e_probe in_rpc_p99_ns = %d, want 10000000", got) + } if len(tg.Phases) != 3 { t.Fatalf("phases = %d, want 3", len(tg.Phases)) } @@ -51,14 +68,25 @@ func TestRates(t *testing.T) { qtype string want []float64 }{ - {"sac", 1, "txhash", []float64{150, 300, 600}}, - {"sac", 3, "txhash", []float64{500, 1000, 2000}}, - {"sac", 3, "ledgers", []float64{0.835, 1.67, 3.34}}, - {"sac", 2, "txpage", []float64{12.5, 25, 50}}, - {"sac", 1, "events", []float64{1.5, 3, 6}}, - {"custom_token", 2, "txhash", []float64{200, 400, 800}}, - {"soroswap", 1, "txpage", []float64{1.875, 3.75, 7.5}}, - {"soroswap", 3, "events", []float64{7.5, 15, 30}}, + // getTransaction sweeps ONE leg carrying both ladders: the SLA one + // (150, 300, 600 in every phase and profile) unioned with the + // demand-derived E2E-probe one, deduplicated and sorted. + {"sac", 1, "txhash", []float64{150, 300, 600}}, // e2e floor 300 == the SLA floor + {"sac", 2, "txhash", []float64{150, 250, 300, 500, 600, 1000}}, + {"sac", 3, "txhash", []float64{150, 300, 500, 600, 1000, 2000}}, + {"custom_token", 1, "txhash", []float64{100, 150, 200, 300, 400, 600}}, + {"custom_token", 2, "txhash", []float64{150, 200, 300, 400, 600, 800}}, + {"custom_token", 3, "txhash", []float64{150, 300, 600, 1200}}, + {"soroswap", 1, "txhash", []float64{37.5, 75, 150, 300, 600}}, + {"soroswap", 3, "txhash", []float64{150, 300, 600}}, // e2e floor 300 again + // The other three answer to the SLA family alone: the mix share of + // 500 rps, the same ladder in every phase and every profile. + {"sac", 3, "ledgers", []float64{12.5, 25, 50}}, + {"sac", 2, "txpage", []float64{37.5, 75, 150}}, + {"sac", 1, "events", []float64{50, 100, 200}}, + {"soroswap", 1, "txpage", []float64{37.5, 75, 150}}, + {"soroswap", 3, "events", []float64{50, 100, 200}}, + {"custom_token", 2, "ledgers", []float64{12.5, 25, 50}}, } for _, tc := range cases { t.Run(fmt.Sprintf("%s-phase%d-%s", tc.profile, tc.phase, tc.qtype), func(t *testing.T) { @@ -151,17 +179,22 @@ func fixture(t *testing.T, body string) string { } func TestLoadRejects(t *testing.T) { - // A two-phase file with one whole profile, edited per case below. + // A two-phase file with both families whole, edited per case below. const good = `{ "query_load": { "ladder": [1, 2], - "profiles": { - "sac": { - "ledgers_rps": [1, 2], - "txpage_rps": [10, 20], - "txhash_rps": [100, 200], - "events_rps": [3, 6] + "sla": { + "floors_rps": {"ledgers": 1, "txpage": 10, "txhash": 100, "events": 3}, + "p99_ns": { + "ledgers": {"hot": 1, "cold": 2}, + "txpage": {"hot": 1, "cold": 2}, + "txhash": {"hot": 1, "cold": 2}, + "events": {"hot": 1, "cold": 2} } + }, + "e2e_probe": { + "in_rpc_p99_ns": 10000000, + "floors_rps": {"sac": [100, 200]} } }, "phases": [{"phase": 1, "block_time_ns": 2000000000}, {"phase": 2, "block_time_ns": 1000000000}] @@ -179,7 +212,7 @@ func TestLoadRejects(t *testing.T) { { name: "no query_load", body: `{"phases": [{"phase": 1, "block_time_ns": 2000000000}]}`, - want: []string{"query_load.profiles is empty", "paced-RPS"}, + want: []string{"query_load.e2e_probe.floors_rps is empty", "two-family"}, }, { name: "no phases", @@ -192,14 +225,24 @@ func TestLoadRejects(t *testing.T) { want: []string{"query_load.ladder must list at least one multiplier"}, }, { - name: "short floor array", - body: strings.Replace(good, `"txhash_rps": [100, 200]`, `"txhash_rps": [100]`, 1), - want: []string{"query_load.profiles.sac.txhash_rps has 1 entries", "one per phase (2)"}, + name: "short e2e floor array", + body: strings.Replace(good, `"floors_rps": {"sac": [100, 200]}`, `"floors_rps": {"sac": [100]}`, 1), + want: []string{"query_load.e2e_probe.floors_rps.sac has 1 entries", "one per phase (2)"}, + }, + { + name: "missing sla floor", + body: strings.Replace(good, `"events": 3`, `"eventz": 3`, 1), + want: []string{"query_load.sla.floors_rps has no events floor", "one per endpoint"}, + }, + { + name: "sla floor that is not an endpoint", + body: strings.Replace(good, `"events": 3}`, `"events": 3, "blocks": 9}`, 1), + want: []string{"query_load.sla.floors_rps names 'blocks'", "not a query type"}, }, { - name: "missing floor array", - body: strings.Replace(good, `"events_rps": [3, 6]`, `"events_rps": []`, 1), - want: []string{"query_load.profiles.sac.events_rps has 0 entries"}, + name: "missing sla p99", + body: strings.Replace(good, `"txpage": {"hot": 1, "cold": 2},`, ``, 1), + want: []string{"query_load.sla.p99_ns has no txpage entry", "one per endpoint"}, }, } for _, tc := range cases { diff --git a/tests/smoke/gen-fixtures.py b/tests/smoke/gen-fixtures.py index 07441c7..034d9fe 100644 --- a/tests/smoke/gen-fixtures.py +++ b/tests/smoke/gen-fixtures.py @@ -7,7 +7,7 @@ fixture-rps-phase3 two profiles, phase 3, mixed 1x verdicts. fixture-rps-unfloored one profile, paced at an interval that matches no phase and with no query_phase in the manifest, so the - converter emits r-cells with NO verdict_1x and no + converter emits r-cells with NO verdict and no campaign.phase — the viewer must judge the rows on the latency target alone and say so. fixture-rps-partial two profiles, phase 3, first profile's COLD query legs @@ -55,25 +55,45 @@ EVENTS_ITEMS_PER_REQ = 10 # events pages carry ten records each SHED_TOP = 5 # dropped at the in-flight cap on the 2x rung -# Per-rung latencies in MILLISECONDS, by tier: [0.5x, 1x, 2x]. `sched` is the -# scheduled p99 (due->done, the headline), `svc` the service p99 (dispatch->done, -# getTransaction's in-RPC budget), `page` the mean getEvents page. Cold pays more -# than hot everywhere, and the 2x rung sits past the knee. +# Scheduled p99 (due->done, the headline) in MILLISECONDS, by endpoint, tier and +# LOAD LEVEL: [light, nominal, heavy]. The SLA states a p99 per endpoint and per +# tier, so one number per tier cannot clear every endpoint — getEvents answers +# at 40 ms where getLedgers has 200. Cold pays more than hot everywhere, the +# nominal level clears its own target, and the heavy level sits past the knee. +SCHED = { + "txhash": {"cold": [12, 18, 60], "hot": [8, 12, 45]}, # SLA 30 / 20 ms + "txpage": {"cold": [40, 60, 190], "hot": [28, 45, 150]}, # SLA 80 / 60 ms + "events": {"cold": [18, 30, 110], "hot": [12, 22, 90]}, # SLA 40 / 40 ms + "ledgers": {"cold": [80, 140, 380], "hot": [55, 100, 300]}, # SLA 200 / 150 ms +} +# Service p99 (dispatch->done — getTransaction's in-RPC budget) and mean +# getEvents page, by tier and load level. The mean page is reported, never +# judged; the service p99 is judged only where the E2E-budget probe lands. BASE = { - "cold": {"sched": [90, 180, 460], "svc": [3.0, 5.0, 9.5], "page": [35, 55, 95]}, - "hot": {"sched": [60, 120, 380], "svc": [2.0, 4.0, 9.0], "page": [25, 40, 80]}, + "cold": {"svc": [3.0, 5.0, 9.5], "page": [35, 55, 95]}, + "hot": {"svc": [2.0, 4.0, 9.0], "page": [25, 40, 80]}, } -# The mixed verdicts, one breach per line. `achieved` is the fraction of the -# target rate actually served (None = keeps up); `lag_p99` is the dispatch lag. +# The mixed verdicts, one designed breach per line, keyed by the rate token the +# cell was paced at (getTransaction sweeps two ladders in one leg, so a rung +# index would not name a cell). `achieved` is the fraction of the target rate +# actually served (None = keeps up); `lag_p99` is the dispatch lag. OVERRIDES = { - # 1x cells: one breached budget each, the other axes still clear. - ("cold", "sac", "txpage", 1): {"sched": 640}, # headline 500 ms - ("hot", "sac", "txhash", 1): {"sched": 130, "svc": 14.0}, # in-RPC 10 ms - ("cold", "soroswap", "events", 1): {"sched": 150, "page": 88}, # mean page 67 ms - # 2x saturation: sac's txhash rung cannot be served at 2000 rps. Cold cannot - # be faster than hot at the same rate, so both tiers saturate together. - ("cold", "sac", "txhash", 2): {"sched": 1400, "achieved": 0.60, "shed": 140, "lag_p99": 900}, - ("hot", "sac", "txhash", 2): {"sched": 1400, "achieved": 0.60, "shed": 140, "lag_p99": 900}, + # An SLA breach on a non-txhash endpoint: 640 ms against getTransactions' + # 80 ms cold target. + ("cold", "sac", "txpage", "75"): {"sched": 640}, + ("cold", "soroswap", "events", "100"): {"sched": 150}, # SLA, 40 ms cold + # The two families, proven independent on ONE cell: soroswap's phase-3 + # demand floor is 300 rps, which IS the SLA floor, so r300 carries both + # verdicts. 45 ms breaches getTransaction's 30 ms cold SLA while the RPC + # itself answers in 5 ms, well inside the 10 ms E2E slice. + ("cold", "soroswap", "txhash", "300"): {"sched": 45}, + # An E2E-budget breach at the demand-derived rate, on a cell the SLA does + # not judge at all: sac's phase-3 probe floor is 1000 rps. + ("hot", "sac", "txhash", "1000"): {"svc": 14.0}, + # Saturation at the top rung: sac's txhash leg cannot be served at 2000 rps. + # Cold cannot be faster than hot at the same rate, so both tiers go together. + ("cold", "sac", "txhash", "2000"): {"sched": 1400, "achieved": 0.60, "shed": 140, "lag_p99": 900}, + ("hot", "sac", "txhash", "2000"): {"sched": 1400, "achieved": 0.60, "shed": 140, "lag_p99": 900}, } @@ -88,31 +108,45 @@ def rate_token(v): def ladder(profile, qt, phase): - """This profile's floor for `phase` times the 0.5/1/2 ladder, as rate tokens. - Both come from docs/targets.json, so the middle rung IS the judged 1x cell - whenever the run has a goal phase to judge it against.""" - floors = C.QUERY_LOAD["profiles"][profile][C._QTYPE_RPS_KEY[qt]] - return [rate_token(floors[phase - 1] * x) for x in C.QUERY_LOAD["ladder"]] - - -def cell_spec(tier, profile, qt, i): - spec = {"sched": BASE[tier]["sched"][i], "svc": BASE[tier]["svc"][i], + """The rate tokens one leg sweeps, computed the way the runner computes + them: this endpoint's SLA floor times the 0.5/1/2 ladder, unioned for + getTransaction with the same ladder around this profile and phase's + E2E-probe floor, deduplicated and sorted ascending. Everything comes from + docs/targets.json, so both judged cells are in the list by construction.""" + steps = C.QUERY_LOAD["ladder"] + rates = [C.SLA_FLOORS_RPS[qt] * x for x in steps] + if qt == "txhash": + rates += [C.E2E_FLOORS_RPS[profile][phase - 1] * x for x in steps] + return [rate_token(r) for r in sorted(set(rates))] + + +def load_level(qt, tok): + """Which of the three latency levels a rate sits at, read off its distance + from that endpoint's SLA floor. getTransaction's demand rungs run far above + that floor, so they land on the heavy level and pay the heavy latency.""" + ratio = float(tok) / C.SLA_FLOORS_RPS[qt] + return 0 if ratio <= 0.75 else 1 if ratio <= 1.5 else 2 + + +def cell_spec(tier, profile, qt, tok, top=False): + i = load_level(qt, tok) + spec = {"sched": SCHED[qt][tier][i], "svc": BASE[tier]["svc"][i], "page": BASE[tier]["page"][i], "achieved": None, - "shed": SHED_TOP if i == len(C.QUERY_LOAD["ladder"]) - 1 else 0, + "shed": SHED_TOP if top else 0, "lag_p99": 3.0} - spec.update(OVERRIDES.get((tier, profile, qt, i), {})) + spec.update(OVERRIDES.get((tier, profile, qt, tok), {})) return spec def qtype_rows(qt, tokens, tier, profile, run): """.csv: a scheduled row and a service row per target rate.""" rows = [] - for i, tok in enumerate(tokens): - c = cell_spec(tier, profile, qt, i) + for tok in tokens: + c = cell_spec(tier, profile, qt, tok) n = ANSWERED items = n * EVENTS_ITEMS_PER_REQ if qt == "events" else n s, v = _ns(c["sched"]), _ns(c["svc"]) - # getEvents' mean page is a judged budget, so it is set outright; every + # getEvents reports a mean page of its own, so it is set outright; every # other type gets a plausible mean under its own service p99. mean = _ns(c["page"]) if qt == "events" else v // 2 rows.append((f"total_r{tok}",) + F._scale( @@ -134,7 +168,7 @@ def driver_rows(qt, tokens, tier, profile, run): if tier == "cold": rows.append(("evict", len(tokens), 6, 4000, 500, 600, 700, 700)) for i, tok in enumerate(tokens): - c = cell_spec(tier, profile, qt, i) + c = cell_spec(tier, profile, qt, tok, top=i == len(tokens) - 1) n, lag, shed = ANSWERED, _ns(c["lag_p99"]), c["shed"] # The achieved rate is carried x1000 as an int in the duration columns. # A keeping-up leg lands a hair under target; a saturated one lands far @@ -222,10 +256,11 @@ def build_bundle(root, spec): def rate_cells(qout): - """The r cells of one qtype entry. verdict_1x is a sibling of theirs - that also carries a target_rps, so it is named out rather than sniffed.""" + """The r cells of one qtype entry. The verdicts are siblings of theirs + that also carry a target_rps, so they are named out rather than sniffed.""" return {k: v for k, v in qout.items() - if k != "verdict_1x" and isinstance(v, dict) and "target_rps" in v} + if not k.startswith("verdict_") and isinstance(v, dict) + and "target_rps" in v} def qtype_entries(D): @@ -238,16 +273,17 @@ def qtype_entries(D): def verdict_failures(D): - """Every (tier, unit, qtype) whose 1x verdict breaches some budget, with the - axis that broke — the run's whole verdict picture in one comparable set.""" + """Every (tier, unit, qtype) whose verdict breaches its family's number, + with the family that broke — the run's whole verdict picture in one + comparable set. The two families are counted apart on purpose: they are + measured at different rates and judged on different budgets.""" out = set() for tier, unit, qt, qout in qtype_entries(D): - v = qout["verdict_1x"] - for axis, ok in [("p99", v["pass"]), - ("in_rpc", v.get("in_rpc", {}).get("pass", True)), - ("page_budget", v.get("page_budget", {}).get("pass", True))]: - if not ok: - out.add((tier, unit, qt, axis)) + if not qout["verdict_sla"]["pass"]: + out.add((tier, unit, qt, "sla")) + e2e = qout.get("verdict_e2e") + if e2e and not e2e["pass"]: + out.add((tier, unit, qt, "e2e")) return out @@ -258,11 +294,14 @@ def check_common(D, spec): for u in spec["units"]: meta = D["dataset"]["unit_meta"][u] assert all(k in meta for k in ("ledgers", "txs", "events")), meta - assert D["campaign"]["query_load"]["profiles"], "campaign.query_load missing" + assert D["campaign"]["query_load"]["sla"], "campaign.query_load missing" want = {"target_rps", "achieved_rps", "service", "dispatch_lag", "shed"} for tier, unit, qt, qout in qtype_entries(D): cells = rate_cells(qout) - assert len(cells) == len(C.QUERY_LOAD["ladder"]), (tier, unit, qt, list(cells)) + # getTransaction sweeps both ladders in one leg, so its cell count is + # the size of their union, not the ladder length. + wanted = ladder(C.query_profile(unit), qt, spec["ladder_phase"]) + assert list(cells) == [f"r{t}" for t in wanted], (tier, unit, qt, list(cells)) for key, cell in cells.items(): missing = want - set(cell) assert not missing, (tier, unit, qt, key, sorted(missing)) @@ -273,18 +312,35 @@ def check_phase3(D, spec): drift in the converter or in targets.json breaks generation, not the viewer.""" assert D["campaign"]["phase"] == spec["phase"], D["campaign"].get("phase") for tier, unit, qt, qout in qtype_entries(D): - v = qout.get("verdict_1x") - assert v, f"no verdict_1x on {tier}/{unit}/{qt}" + v = qout.get("verdict_sla") + assert v, f"no verdict_sla on {tier}/{unit}/{qt}" assert v["target_rps"] == float(v["rate"][1:]), v - - # The three designed breaches, and nothing else. + # Only getTransaction answers to the E2E budget, and it always does. + e2e = qout.get("verdict_e2e") + assert (e2e is not None) == (qt == "txhash"), (tier, unit, qt, e2e) + if e2e: + assert e2e["target_rps"] == float(e2e["rate"][1:]), e2e + assert "threshold_ns" not in e2e, "the probe judges in_rpc alone" + + # soroswap's phase-3 demand floor IS the SLA floor, so one cell carries both + # verdicts; sac's does not, so its two verdicts sit on different cells. + soro = D["queries"]["cold"]["soroswap-1500-c1"]["txhash"] + assert soro["verdict_sla"]["rate"] == soro["verdict_e2e"]["rate"] == "r300", soro["verdict_sla"] + sac = D["queries"]["hot"]["sac-6000-c1"]["txhash"] + assert sac["verdict_sla"]["rate"] == "r300", sac["verdict_sla"] + assert sac["verdict_e2e"]["rate"] == "r1000", sac["verdict_e2e"] + + # The four designed breaches, and nothing else. The soroswap pair is the + # point: the same cell fails the SLA and passes the E2E budget. expected = { - ("cold", "sac-6000-c1", "txpage", "p99"), - ("hot", "sac-6000-c1", "txhash", "in_rpc"), - ("cold", "soroswap-1500-c1", "events", "page_budget"), + ("cold", "sac-6000-c1", "txpage", "sla"), + ("cold", "soroswap-1500-c1", "events", "sla"), + ("cold", "soroswap-1500-c1", "txhash", "sla"), + ("hot", "sac-6000-c1", "txhash", "e2e"), } got = verdict_failures(D) assert got == expected, f"verdict drift: {sorted(got)}" + assert soro["verdict_e2e"]["pass"], "the shared cell must pass the E2E budget" # Saturation: the 2x sac txhash cell is served far below its target rate. for tier in ("cold", "hot"): @@ -296,11 +352,12 @@ def check_phase3(D, spec): def check_unfloored(D, spec): """No phase means no floor: every rate cell still converts, and not one of - them earns a verdict_1x for the viewer to judge against.""" + them earns a verdict for the viewer to judge against.""" assert D["campaign"].get("phase") is None, D["campaign"].get("phase") assert D["campaign"]["close_interval_ns"] > 0, "the fixture is paced, just off-phase" for tier, unit, qt, qout in qtype_entries(D): - assert "verdict_1x" not in qout, f"unexpected verdict on {tier}/{unit}/{qt}" + for family in ("verdict_sla", "verdict_e2e"): + assert family not in qout, f"unexpected {family} on {tier}/{unit}/{qt}" # The viewer labels rungs by multiplier off the campaign ladder when no floor # names them, so the two lengths have to agree. assert len(C.QUERY_LOAD["ladder"]) == len( @@ -318,16 +375,18 @@ def check_partial(D, spec): sorted(D["queries"]["hot"]) assert D["campaign"]["phase"] == spec["phase"], D["campaign"].get("phase") for qt in QTYPES: - assert D["queries"]["hot"][first][qt].get("verdict_1x"), qt - assert D["queries"]["cold"][second][qt].get("verdict_1x"), qt + assert D["queries"]["hot"][first][qt].get("verdict_sla"), qt + assert D["queries"]["cold"][second][qt].get("verdict_sla"), qt + assert D["queries"]["hot"][first]["txhash"].get("verdict_e2e"), "probe on the hot leg" SPECS = [ { "id": "fixture-rps-phase3", "name": "Fixture — open-loop query load (phase 3)", - # Two profiles: enough to exercise the profile picker, and soroswap's - # floors are the ones that produce fractional rate tokens (r7.5, r30). + # Two profiles: enough to exercise the profile picker. Every profile now + # shares the SLA floors, so the fractional rate tokens the viewer has to + # survive (r12.5, r37.5) come from the 0.5x rung. "units": ["sac-6000-c1", "soroswap-1500-c1"], "reps": 2, "close_interval": "600ms", # phase 3's block time diff --git a/tests/smoke/smoke.mjs b/tests/smoke/smoke.mjs index 10a3f91..26bd3dc 100644 --- a/tests/smoke/smoke.mjs +++ b/tests/smoke/smoke.mjs @@ -133,12 +133,28 @@ function queryShape(D) { if (Object.keys(e).some((k) => CONC_KEY.test(k))) return "closed"; return rateKeys(e).length ? "open" : null; } -/* The run's read-path latency target, and the rung a verdict table judges — - both read exactly the way docs/app.js reads them (checkFor / judgedRate). */ -const queryThr = (D) => ((D.checks_all || [D.checks || {}]) - .find((c) => c && c.applies_to === "queries") || {}).threshold_ns || 500e6; +/* The run's read-path latency target for one endpoint in one storage tier, and + the rung a verdict table judges — both read exactly the way docs/app.js reads + them (checkFor + thresholdFor / judgedRate). A run states its target either + per endpoint and tier (targets_ns, the SLA shape) or as one number for the + whole run (threshold_ns, the published generation). */ +const queryCheck = (D) => (D.checks_all || [D.checks || {}]) + .find((c) => c && c.applies_to === "queries") || {}; +const queryThr = (D, qt, tier) => { + const c = queryCheck(D); + const byTier = c.targets_ns && c.targets_ns[qt]; + if (byTier && byTier[tier] != null) return byTier[tier]; + return c.threshold_ns || 500e6; +}; +/* The two verdict families, read the way docs/app.js reads them. verdict_sla + covers every endpoint at its SLA rate; verdict_e2e covers getTransaction at + the demand-derived rate and judges the in-RPC time alone. Runs converted + before the split carry one verdict_1x that plays the SLA role. */ +const slaVerdict = (qout) => (qout && (qout.verdict_sla || qout.verdict_1x)) || null; +const e2eVerdict = (qout) => (qout && qout.verdict_e2e) || null; +const hasE2EFamily = (D) => qEntries(D).some((e) => e2eVerdict(e.qout)); const judgedKey = (qout) => { - const v = qout.verdict_1x; + const v = slaVerdict(qout); if (v && v.rate && qout[v.rate]) return v.rate; const ks = rateKeys(qout); return ks.length ? ks[Math.floor((ks.length - 1) / 2)] : null; @@ -162,6 +178,9 @@ function checkKind(kind, doc, group, data) { if (kind === "synthetic" && data.queries) { exp.sections += 1; exp.minFigures += queryShape(data) === "open" ? 5 : 4; + // A run that also carries the E2E-budget family draws its table as a + // figure of its own — the two requirements never share one. + if (hasE2EFamily(data)) exp.minFigures += 1; exp.minSvgs += 3; } const report = doc.getElementById("report"); @@ -199,10 +218,12 @@ function checkSyntheticQueries(doc, group, D) { if (!sec) return; const shape = queryShape(D); // Exact per-section counts, both generations: three charts either way, plus - // the table figures each shape adds. + // the table figures each shape adds — and one more when the run carries the + // E2E-budget family, which gets a table of its own rather than a column in + // the SLA one. const figs = sec.querySelectorAll("figure.fig").length; const svgs = sec.querySelectorAll("svg").length; - const wantFigs = shape === "open" ? 5 : 4; + const wantFigs = (shape === "open" ? 5 : 4) + (hasE2EFamily(D) ? 1 : 0); check(group, `queries section emits exactly ${wantFigs} figures (${shape}-loop)`, figs === wantFigs, figs + " figures"); check(group, "queries section emits exactly 3 charts", svgs === 3, svgs + " svgs"); const picker = sec.querySelectorAll("#profile-filter .chunk-btn").length; @@ -271,7 +292,6 @@ function checkOpenLoopQueries(doc, sec, group, D) { const tiers = Object.keys(D.queries); const units = D.dataset.unit_order; const types = qTypes(D); - const thr = queryThr(D); // Human endpoint names everywhere; the CSV's machine names nowhere. check(group, "endpoint display names rendered", types.every((t) => !ENDPOINT[t] || secTxt.includes(ENDPOINT[t])), @@ -280,17 +300,17 @@ function checkOpenLoopQueries(doc, sec, group, D) { check(group, "no raw CSV tokens in the rendered section", rawTokens.length === 0, rawTokens.slice(0, 6).join(",")); // Verdict table: one row per endpoint × profile × tier, and one chip per - // budget the run actually judged — headline, plus getTransaction's in-RPC - // budget and getEvents' page budget as separate verdicts. + // budget the run actually judged — the headline against this endpoint and + // tier's own target, plus getTransaction's in-RPC budget as its own verdict. const tbl = doc.getElementById("query-verdict-table"); check(group, "verdict table rendered", !!tbl, "missing"); if (!tbl) return; let checks = 0, fails = 0, rows = 0, unfloored = 0; - const judged = []; + const judged = [], thresholds = []; for (const qt of types) for (const u of units) for (const tier of tiers) { const qout = ((D.queries[tier] || {})[u] || {})[qt]; if (!qout) continue; - const v = qout.verdict_1x; + const v = slaVerdict(qout); const key = judgedKey(qout); if (!key) continue; rows++; @@ -298,12 +318,17 @@ function checkOpenLoopQueries(doc, sec, group, D) { const cell = qout[key] || {}; judged.push({ qt, u, tier, cell, v }); // A row with no phase floor still gets a headline chip — the viewer judges - // it on the latency target alone (docs/app.js openVerdictTable), so the - // hand count has to judge it the same way. + // it on this endpoint and tier's own latency target, with no rate to meet + // it at (docs/app.js openVerdictTable), so the hand count judges it the + // same way rather than against one number for the whole run. The in-RPC + // chip is counted here ONLY on the published runs that fold that budget + // into the same verdict; where the run splits the families it is judged in + // the E2E-budget table instead, and this table shows it as context. const p99 = v && v.p99_ns != null ? v.p99_ns : (cell.p99 || {}).m; - for (const ok of [p99 == null ? null : (v ? v.pass : p99 <= thr), - v && v.in_rpc ? v.in_rpc.pass : null, - v && v.page_budget ? v.page_budget.pass : null]) { + const rowThr = v && v.threshold_ns != null ? v.threshold_ns : queryThr(D, qt, tier); + thresholds.push(rowThr); + for (const ok of [p99 == null ? null : (v ? v.pass : p99 <= rowThr), + v && v.in_rpc ? v.in_rpc.pass : null]) { if (ok === null) continue; checks++; if (!ok) fails++; } @@ -329,10 +354,31 @@ function checkOpenLoopQueries(doc, sec, group, D) { check(group, "footnote states the unfloored row count", foot.includes(`${unfloored} of ${rows} rows carry no phase floor`), foot.slice(0, 260)); } - // The three budgets are separate columns, never folded into one verdict. + // The in-RPC budget and the mean page stay their own columns, never folded + // into the headline verdict. The headline column heads no single threshold: + // the SLA sets one per endpoint and tier, so every row prints its own. const heads = [...tbl.querySelectorAll("th")].map(txt); - check(group, "in-RPC and mean-page budgets are their own columns", + check(group, "in-RPC budget and mean page are their own columns", heads.some((h) => /In-RPC p99/.test(h)) && heads.some((h) => /Mean page/.test(h)), heads.join(" | ")); + // Every row prints the headline target it was judged against, in its own + // marked note (.q-thr) so it cannot be confused with the in-RPC one beside + // it. The rendered set has to be as varied as the data: as many distinct + // targets on the page as the run's own thresholds imply. + const verdictRows = [...tbl.querySelectorAll("tr")].slice(1); + const printed = verdictRows.map((r) => [...r.querySelectorAll(".q-thr")].map(txt)); + check(group, `every row prints its own headline target (${verdictRows.length} rows)`, + printed.length > 0 && printed.every((n) => n.length === 1 && /^\s*\(≤ \S+.*\)$/.test(n[0])), + printed.filter((n) => n.length !== 1).length + " rows without exactly one target note"); + const wantDistinct = new Set(thresholds).size; + const gotDistinct = new Set(printed.flat()).size; + check(group, `the page prints as many distinct targets as the run defines (${wantDistinct})`, + gotDistinct === wantDistinct, gotDistinct + " distinct printed"); + // The SLA shape is the point: a run keyed by endpoint and tier must not + // collapse to one number on the page. + if (queryCheck(D).targets_ns) { + check(group, "a per-endpoint run prints more than one distinct target", + wantDistinct > 1, wantDistinct + " distinct"); + } // Saturation at the top rung is legible: the ladder names the shortfall and // both the offered and served rates for the profile the picker opens on. @@ -351,15 +397,59 @@ function checkOpenLoopQueries(doc, sec, group, D) { check(group, "ladder shows the saturated rung's offered and served rates", sat.length === 0 || sat.every((s) => ladder.includes(fmtRps(s.target) + " rps") && ladder.includes(fmtRps(s.got) + " rps")), ladder.slice(0, 200)); - // Every rung of every endpoint × tier is listed, labelled by multiplier. - const rungRows = judged.filter((j) => j.u === first).length; - check(group, "ladder table lists 1× for every endpoint × tier of the profile", + // Every rung of every endpoint × tier is listed, labelled by multiplier — + // off the floor the run names, else off the campaign ladder by position. A + // leg that swept MORE rungs than the ladder has (getTransaction sweeping both + // families with no floor to anchor them) is labelled by rate instead, so it + // is not counted here. + const ladderLen = ((D.campaign || {}).query_load || {}).ladder; + const rungRows = judged.filter((j) => j.u === first && (j.v + || (Array.isArray(ladderLen) && rateKeys(((D.queries[j.tier] || {})[j.u] || {})[j.qt]).length === ladderLen.length))).length; + check(group, `ladder table lists 1× for every anchored endpoint × tier (${rungRows})`, (ladder.match(/1×/g) || []).length >= rungRows, ladder.slice(0, 160)); // Latency detail at 1×: the scheduled distribution plus the in-RPC column. const detail = txt(doc.querySelector("#figq4-tv")); check(group, "1× latency table carries p50/p90/p99/max + in-RPC p99", /p50/.test(detail) && /p99/.test(detail) && /In-RPC p99/.test(detail), detail.slice(0, 160)); + // The E2E-budget probe is a SECOND requirement, measured at a different rate + // and judged on a different number. It must have a table of its own, and the + // SLA table above must not chip getTransaction's in-RPC time as if the two + // were one verdict. + const e2eTbl = doc.getElementById("query-e2e-table"); + const wantE2E = qEntries(D).filter((e) => e2eVerdict(e.qout)); + check(group, wantE2E.length ? "E2E-budget table rendered" : "no E2E-budget table without the family", + !!e2eTbl === wantE2E.length > 0, e2eTbl ? "present" : "missing"); + if (e2eTbl) { + const e2eRows = [...e2eTbl.querySelectorAll("tr")].slice(1); + check(group, `E2E table has one row per profile × tier with a probe verdict (${wantE2E.length})`, + e2eRows.length === wantE2E.length, e2eRows.length + " rows"); + const wantFails = wantE2E.filter((e) => !e2eVerdict(e.qout).in_rpc.pass).length; + check(group, `E2E breach chips match a hand count of the probe verdicts (${wantFails})`, + e2eTbl.querySelectorAll(".e2e-fail").length === wantFails, + e2eTbl.querySelectorAll(".e2e-fail").length + " ▲ chips"); + check(group, "E2E table judges the in-RPC p99 and reports the scheduled one", + [...e2eTbl.querySelectorAll("th")].some((h) => /In-RPC p99/.test(txt(h))) + && e2eRows.every((r) => /not judged/.test(txt(r))), + [...e2eTbl.querySelectorAll("th")].map(txt).join(" | ")); + // The families never share a row: the SLA table shows getTransaction's + // in-RPC time without a verdict chip beside it. + // "getTransactions" starts with "getTransaction", so the label is matched + // up to its separator rather than as a substring. + const slaTxhashCells = [...tbl.querySelectorAll("tr")].slice(1) + .filter((r) => /^getTransaction · /.test(txt(r.querySelector("td")))); + check(group, "the SLA table chips no in-RPC budget once the families are split", + slaTxhashCells.length > 0 && slaTxhashCells.every((r) => /judged as the E2E budget/.test(txt(r))), + slaTxhashCells.map((r) => txt(r)).join(" | ").slice(0, 200)); + // Different requirement, different rate: the offered rates must not all be + // the SLA floor, or the split would be cosmetic. + const slaRates = new Set(wantE2E.map((e) => slaVerdict(e.qout).target_rps)); + const probeRates = new Set(wantE2E.map((e) => e2eVerdict(e.qout).target_rps)); + check(group, "the probe rate is read from the demand model, not the SLA floor", + [...probeRates].some((r) => !slaRates.has(r)), + `sla=[${[...slaRates].join(",")}] probe=[${[...probeRates].join(",")}]`); + } + // Setup/event-scan table still works off `setup`. const setup = txt(doc.getElementById("query-setup-table")); check(group, "setup table names open", /open/.test(setup), setup.slice(0, 160)); @@ -521,8 +611,8 @@ function summaryQueryRows(D) { for (const u of D.dataset.unit_order) { for (const [qt, qout] of Object.entries(t[u] || {})) { if (qt === "setup") continue; - const v = (qout || {}).verdict_1x; - if (v && v.p99_ns != null) rows.push({ unit: u, qt, v }); + const v = slaVerdict(qout); + if (v && v.p99_ns != null) rows.push({ unit: u, qt, v, e2e: e2eVerdict(qout) }); } } if (rows.length) out[tier] = rows; @@ -611,8 +701,11 @@ function checkSummarySections(sdoc, group, D) { /sendTransaction/.test(tileTxt) && /ingestion/.test(tileTxt) && /getTransaction/.test(tileTxt) && (tileTxt.match(/allocation/g) || []).length === 3, tileTxt.slice(0, 160)); // getTransaction is measured only when the run recorded an in-RPC p99 on a - // live-store getTransaction leg; otherwise its allocation stands. - const measuredGet = (QR.hot || []).some((r) => r.qt === "txhash" && r.v.in_rpc); + // live-store getTransaction leg; otherwise its allocation stands. The + // number comes from the E2E-budget probe, which is the leg run for exactly + // this purpose; runs predating the split fold it into their one verdict. + const measuredGet = (QR.hot || []).some((r) => r.qt === "txhash" + && (r.e2e ? r.e2e.in_rpc : r.v.in_rpc)); const f11t = txt(sdoc.querySelector("#fig11-tv")); const getRow = f11t.split("getTransaction round trip")[1] || ""; check(group, measuredGet ? "budget table: getTransaction allocated → measured" : "budget table: getTransaction stays an estimate", @@ -689,10 +782,10 @@ function checkSummarySections(sdoc, group, D) { const hotCell = sdoc.querySelector("#q-table-hot tbody tr:nth-child(1) td:nth-child(2)"); const coldCell = sdoc.querySelector("#q-table-cold tbody tr:nth-child(2) td:nth-child(2)"); check(group, "representative hot and cold matrix values render", - txt(hotCell && hotCell.querySelector(".q-latency")) === "130 ms" - && txt(hotCell && hotCell.querySelector(".q-rate")) === "1,000 rps" + txt(hotCell && hotCell.querySelector(".q-latency")) === "12.0 ms" + && txt(hotCell && hotCell.querySelector(".q-rate")) === "300 rps" && txt(coldCell && coldCell.querySelector(".q-latency")) === "640 ms" - && txt(coldCell && coldCell.querySelector(".q-rate")) === "50 rps", + && txt(coldCell && coldCell.querySelector(".q-rate")) === "75 rps", `${txt(hotCell)} | ${txt(coldCell)}`); } const measuredGet = (QR.hot || []).some((r) => r.qt === "txhash" && r.v.in_rpc); @@ -717,33 +810,48 @@ function checkSummarySections(sdoc, group, D) { const FIXTURE_CHECKS = { "fixture-rps-phase3": (doc, group, D) => { const es = qEntries(D); - check(group, "every endpoint × profile × tier carries a 1× verdict", - es.length > 0 && es.every((e) => e.qout.verdict_1x), - es.filter((e) => !e.qout.verdict_1x).length + " without a verdict"); + check(group, "every endpoint × profile × tier carries an SLA verdict", + es.length > 0 && es.every((e) => e.qout.verdict_sla), + es.filter((e) => !e.qout.verdict_sla).length + " without a verdict"); + // The second family covers getTransaction and nothing else. + const probe = es.filter((e) => e.qout.verdict_e2e); + check(group, `only getTransaction carries an E2E-budget verdict (${probe.length})`, + probe.length > 0 && probe.every((e) => e.qt === "txhash"), + probe.map((e) => e.qt).join(",")); check(group, "the run names its goal phase", D.campaign.phase === 3, String(D.campaign.phase)); }, /* No phase floor: the run is paced off-phase and names no query_phase, so the - converter emits r-cells with no verdict_1x at all. The section must still - render and must judge every row on the latency target alone, saying so. */ + converter emits r-cells with no verdict at all. The section must still + render and must judge every row on that endpoint and tier's own latency + target — never on one run-wide number — saying so. */ "fixture-rps-unfloored": (doc, group, D) => { const es = qEntries(D); check(group, "no run phase and no 1× floor anywhere", - D.campaign.phase == null && es.every((e) => !e.qout.verdict_1x), + D.campaign.phase == null && es.every((e) => !slaVerdict(e.qout) && !e2eVerdict(e.qout)), "phase " + D.campaign.phase); const tbl = doc.getElementById("query-verdict-table"); const rows = [...tbl.querySelectorAll("tr")].slice(1); check(group, "every verdict row is noted (no floor)", rows.length > 0 && rows.every((r) => /\(no floor\)/.test(txt(r))), rows.length + " rows"); check(group, "no row claims a 1× floor", !rows.some((r) => /\(1× floor\)/.test(txt(r))), "floor note present"); - // One headline chip per row, judged against the run's own query target. - const thr = queryThr(D); + // One headline chip per row, each judged against the target its own + // endpoint carries in its own tier — the check's targets_ns table, the same + // number a verdict would have stated had the run earned one. const want = es.filter((e) => judgedKey(e.qout)).length; const chips = tbl.querySelectorAll(".q-pass, .q-fail").length; check(group, `one latency-target chip per row, no budget chips (${want})`, chips === want, chips + " chips"); - const over = es.filter((e) => { const k = judgedKey(e.qout); return k && e.qout[k].p99.m > thr; }).length; - check(group, `breach chips match the ${Math.round(thr / 1e6)} ms target hand count (${over})`, + const over = es.filter((e) => { + const k = judgedKey(e.qout); + return k && e.qout[k].p99.m > queryThr(D, e.qt, e.tier); + }).length; + check(group, `breach chips match the per-endpoint target hand count (${over})`, tbl.querySelectorAll(".q-fail").length === over, tbl.querySelectorAll(".q-fail").length + " ▲ chips"); + // The regression this fixture exists to catch: a run with no verdicts must + // still show each endpoint's own target, not one fallback number. + const printed = new Set([...tbl.querySelectorAll(".q-thr")].map(txt)); + check(group, "unfloored rows print more than one distinct target", + printed.size > 1, [...printed].join(" | ")); // The three open-loop figures still draw, chart and table view alike. for (const id of ["figq2", "figq3", "figq4"]) { check(group, `${id} draws a chart and a table view`, From c8edcd62a0d9cf62ce113911c5351ee8e8c227e3 Mon Sep 17 00:00:00 2001 From: Marwen Abid Date: Wed, 2 Sep 2026 11:29:38 -0700 Subject: [PATCH 2/3] Simplify the two-family verdict split Viewer: the SLA check is looked up by kind, as SCHEMA.md now tells readers to, with applies_to kept only as the legacy fallback. One queryTargets(D) resolver replaces the thrFor / perEndpoint / qThr triad both renderers spelled out, so the 500 ms legacy default has one definition. The SLA and E2E-budget tables share cellTd / noteEl / tierTag / underTarget instead of each carrying its own copies. The SLA table decides per row whether the in-RPC time is judged elsewhere, from that row's own verdict_e2e, rather than from a run-level flag. The E2E table renders whatever entry carries a verdict_e2e instead of reading .txhash by name, and takes its header budget from the verdicts. Query figures are numbered by a counter in emission order rather than a hand-kept slot map. Converter: the query checks are appended once after the keep-up branch instead of in every arm. A test pins each SLA floor to aggregate_rps x mix. targets.json: the derivation block keeps only the numeric inputs; the prose lives in docs/sla-derivation.md and no longer rides into every run JSON. Runner: sortedKeys is slices.Sorted(maps.Keys); the one-use extraKeys helper is inlined; Rates reads the SLA floor with a map lookup instead of a separate membership test. Smoke: measuredGet is computed once (the second copy was not e2e-aware, so the footnote check never ran on split runs; it now does and allows the probe-leg suffix). hasE2EFamily and wantE2E share one selector; the ladder-row filter reuses the qout it already had. gen-fixtures picks the latency level as the nearest ladder rung in log space instead of hardcoded cutoffs, and reuses the converter's rps_cells. --- SCHEMA.md | 4 +- converter/convert.py | 14 +- converter/tests/test_phase.py | 5 + docs/app.js | 316 ++++++++++++++--------------- docs/targets.json | 6 +- runner/internal/targets/targets.go | 47 ++--- tests/smoke/gen-fixtures.py | 17 +- tests/smoke/smoke.mjs | 40 ++-- 8 files changed, 206 insertions(+), 243 deletions(-) diff --git a/SCHEMA.md b/SCHEMA.md index 84ac974..fe894b2 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -95,8 +95,8 @@ query `events` rows, `n_items` may vary — keep the per-run array as `items_r`, // sla family (one floor and one p99 per endpoint // and tier), the e2e_probe family (getTransaction's // demand floors per profile × phase and its in-RPC - // budget), and the derivation the numbers come - // from — see "queries" below. + // budget), and the derivation inputs (aggregate + // rate and mix) — see "queries" below. "name": "phase1-synthetic-minspec", // optional; metadata.json campaign.name "config_file": "…​.toml", // optional; metadata.json campaign.config_file "config": { … } // optional; remaining metadata.json campaign knobs, diff --git a/converter/convert.py b/converter/convert.py index abdb517..466ac72 100644 --- a/converter/convert.py +++ b/converter/convert.py @@ -1291,11 +1291,9 @@ def convert(args): # two answer different questions about different sections. `checks` carries # the first for readers that predate the list; `checks_all` carries them all. checks = [] - # The read path answers to two requirements, so it earns two checks. The - # SLA one carries the whole target table rather than a single threshold_ns: - # every cell already states the number it was judged against. The probe one - # carries the single in-RPC budget, which is the same for every profile, - # phase and tier. The legacy query_p99_threshold kind is read-side only now. + # The read path answers to two requirements, so it earns two checks: the SLA + # one carries the whole target table (every cell states its own number), the + # probe one the single in-RPC budget shared by every profile, phase and tier. query_checks = [ {"kind": "query_sla", "targets_ns": SLA_P99_NS, "floors_rps": SLA_FLOORS_RPS, "label": "query p99 ≤ the per-endpoint SLA at the SLA rate", @@ -1315,14 +1313,10 @@ def convert(args): else f"{format_interval(close_interval_ns)} pace") checks.append({"kind": "block_keepup", "interval_ns": close_interval_ns, "label": label, "applies_to": "ingest_hot"}) - if queries: - checks.extend(query_checks) elif args.dataset_kind == "synthetic": checks.append({"kind": "block_keepup", "interval_ns": 600000000, "label": "600 ms block model", "applies_to": "ingest_hot"}) - if queries: - checks.extend(query_checks) - elif queries: + if queries: checks.extend(query_checks) if checks: diff --git a/converter/tests/test_phase.py b/converter/tests/test_phase.py index ad57b06..bf777a1 100644 --- a/converter/tests/test_phase.py +++ b/converter/tests/test_phase.py @@ -176,6 +176,11 @@ def test_query_targets_come_from_targets_json(self): }) self.assertEqual(convert.E2E_IN_RPC_P99_NS, 10_000_000) self.assertEqual(convert.E2E_FLOORS_RPS["sac"], [300, 500, 1000]) + # The floors are derived numbers, so pin them to the inputs they come from. + deriv = convert.QUERY_LOAD["derivation"] + self.assertAlmostEqual(sum(deriv["mix"].values()), 1.0) + for qt, floor in convert.SLA_FLOORS_RPS.items(): + self.assertAlmostEqual(floor, deriv["aggregate_rps"] * deriv["mix"][qt]) def test_legacy_layouts_unchanged(self): tmp = tempfile.mkdtemp() diff --git a/docs/app.js b/docs/app.js index 2568994..d8e0586 100644 --- a/docs/app.js +++ b/docs/app.js @@ -565,11 +565,11 @@ } function verNum(s, re) { const m = (s || "").match(re); return m ? m[1] : ""; } - // The run's check for one section, matched on applies_to. A run can carry - // several — a paced campaign that also swept queries is judged on keeping up - // AND on the read-path target — so checks_all is the list and checks is its - // first entry. Runs published before the list carry only the single object, - // hence the fallback: never read a verdict by position. + // The run's checks. A run can carry several — a paced campaign that also + // swept queries is judged on keeping up AND on the read path — so checks_all + // is the list and checks is its first entry. Runs published before the list + // carry only the single object, hence the fallback: never read a verdict by + // position. function allChecks(D) { return Array.isArray(D.checks_all) ? D.checks_all : (D.checks ? [D.checks] : []); } @@ -579,28 +579,40 @@ function checkOfKind(D, kind) { return allChecks(D).find(c => c && c.kind === kind) || null; } - - // The p99 one query cell answers to. The SLA states a target per endpoint AND - // storage tier, which the check carries as targets_ns; runs converted before - // that state one number for the whole run in threshold_ns. A cell that earned - // no verdict of its own has to resolve the same way its verdict would have, - // so every judgement outside a verdict goes through here. + // The read-path SLA check. Two checks now share applies_to "queries", so it + // is matched on kind; the applies_to fallback picks up the single + // query_p99_threshold check of runs converted before the split. + const queryCheck = D => checkOfKind(D, "query_sla") || checkFor(D, "queries"); + + // The p99 one query cell answers to. The SLA check states a target per + // endpoint AND storage tier in targets_ns; older runs state one number for + // the whole run in threshold_ns. A cell that earned no verdict of its own + // resolves the same way its verdict would have, so every judgement outside a + // verdict goes through here. function thresholdFor(check, qt, tier) { const byTier = check && check.targets_ns && check.targets_ns[qt]; if (byTier && byTier[tier] != null) return byTier[tier]; return check && check.threshold_ns ? check.threshold_ns : 500e6; } + // Everything a renderer needs to judge query cells: the check, a per-row + // resolver, whether the run states targets per endpoint, and — when it does + // not — the single number the prose and the reference lines can name. + function queryTargets(D) { + const check = queryCheck(D); + const perEndpoint = !!(check && check.targets_ns); + return { + check, perEndpoint, + thrFor: (qt, tier) => thresholdFor(check, qt, tier), + single: perEndpoint ? null : thresholdFor(check), + }; + } - // The read path answers to two requirements and carries a verdict for each. - // verdict_sla is the SLA family — every endpoint, at its share of the - // sustained watermark, against the p99 of its storage tier. verdict_e2e is - // getTransaction's E2E-budget probe — the demand-derived rate, judged on the - // time inside the RPC alone. Runs converted before the split carry one - // verdict_1x that plays the SLA role and folds the in-RPC budget into itself; - // it is read here so those published runs still render, with their in-RPC - // chip intact. Their retired page_budget verdict is deliberately NOT chipped - // any more: the mean page is reported and judged by nothing, so those runs - // show one fewer check than they used to. + // The read path answers to two requirements and carries a verdict for each: + // verdict_sla (every endpoint, at its SLA rate, against its tier's p99) and + // verdict_e2e (getTransaction at the demand-derived rate, judged on its + // in-RPC time alone). Runs converted before the split carry one verdict_1x + // that plays the SLA role with the in-RPC budget folded in; it still renders, + // minus its retired page_budget chip. const slaVerdict = qout => (qout && (qout.verdict_sla || qout.verdict_1x)) || null; const e2eVerdict = qout => (qout && qout.verdict_e2e) || null; @@ -638,6 +650,35 @@ const cellTarget = (qout, k) => (qout[k] && qout[k].target_rps != null ? qout[k].target_rps : +k.slice(1)); + // Builders the query tables share. `cls` on a note marks one a reader (and + // the smoke test) has to find on its own, like each row's headline target. + const cellTd = (parent, ...kids) => { + const c = document.createElement("td"); + for (const k of kids) c.appendChild(typeof k === "string" ? document.createTextNode(k) : k); + parent.appendChild(c); + return c; + }; + const noteEl = (text, cls) => { + const s = document.createElement("span"); + s.className = cls ? "cell-note " + cls : "cell-note"; + s.textContent = " " + text; + return s; + }; + const tierTag = (tier, C) => { + const tag = document.createElement("span"); + tag.className = "tier-tag"; tag.style.background = tier === "cold" ? C.cold : C.hot; + return tag; + }; + // Served short of offered by more than the saturation margin: the shortfall + // as a whole percent, or null while the pacer kept up. + const underTarget = (ach, target) => + ach && target > 0 && ach.m < target * SATURATED ? Math.round((1 - ach.m / target) * 100) : null; + const underWarn = pct => { + const w = document.createElement("span"); + w.className = "cell-warn"; w.textContent = ` ▲ ${pct} % under`; + return w; + }; + // The judged rung of one endpoint entry: the cell the run names as the SLA // floor, else the middle of whatever ladder it swept — an unfloored run still // reports a measured number, just without a floor to judge it by. The SLA @@ -789,17 +830,10 @@ // the ROW; the concurrency columns all answer to the same number. const thr = thrFor(qt, tier); const r = document.createElement("tr"); - const name = document.createElement("td"); - const tag = document.createElement("span"); tag.className = "tier-tag"; tag.style.background = tier === "cold" ? C.cold : C.hot; - name.appendChild(tag); name.appendChild(document.createTextNode(`${qt} · ${tier}`)); + const name = cellTd(r, tierTag(tier, C), `${qt} · ${tier}`); // Only a run with per-endpoint targets needs the number here: with one // target for the whole run the section intro already states it. - if (perEndpoint) { - const n = document.createElement("span"); - n.className = "cell-note"; n.textContent = ` (≤ ${fmtNsAxis(thr)})`; - name.appendChild(n); - } - r.appendChild(name); + if (perEndpoint) name.appendChild(noteEl(`(≤ ${fmtNsAxis(thr)})`)); for (const cc of CONC) { const w = worstQueryCell(Q, units, tier, qt, cc, thr); const td = document.createElement("td"); @@ -808,7 +842,7 @@ td.textContent = fmtMs(w.cell.p99.m) + " ms"; // Name the worst unit, and say when it is not the only one over. const extra = w.overCount > 1 ? ` +${w.overCount - 1} more` : ""; - const note = document.createElement("span"); note.className = "cell-note"; note.textContent = ` (${w.unit}${extra})`; td.appendChild(note); + td.appendChild(noteEl(`(${w.unit}${extra})`)); const ok = document.createElement("span"); if (w.cell.p99.m > thr) { ok.className = "cell-warn"; ok.textContent = " ▲"; breaches++; unitBreaches += w.overCount; } else if (w.cell.p99.hi > thr) { ok.className = "cell-ok"; ok.textContent = " ✓†"; over.push(`${qt} · ${tier} · ${cc} on ${w.unit} (worst run ${fmtMs(w.cell.p99.hi)} ms)`); } @@ -844,7 +878,7 @@ const tr = document.createElement("tr"); head.forEach(h => { const th = document.createElement("th"); th.textContent = h; tr.appendChild(th); }); t.appendChild(tr); - let checks = 0, fails = 0, rows = 0, unfloored = 0; + let checks = 0, fails = 0, rows = 0, unfloored = 0, probed = 0; // A verdict chip is its own element so a hand count of breaches can be read // straight off the table. const chip = pass => { @@ -854,79 +888,56 @@ s.textContent = pass ? " ✓" : " ▲"; return s; }; - // `extra` marks a note a reader has to be able to find on its own — the - // headline target, which is per endpoint and tier and so cannot be read off - // the column head. - const note = (text, extra) => { - const s = document.createElement("span"); - s.className = extra ? "cell-note " + extra : "cell-note"; - s.textContent = " " + text; - return s; - }; - const td = (parent, ...kids) => { - const c = document.createElement("td"); - for (const k of kids) c.appendChild(typeof k === "string" ? document.createTextNode(k) : k); - parent.appendChild(c); - return c; - }; for (const qt of QT) for (const u of units) for (const tier of TIERS) { const qout = ((Q[tier] || {})[u] || {})[qt]; const key = qout && judgedRate(qout); if (!key) continue; - const cell = qout[key] || {}, v = slaVerdict(qout); + const cell = qout[key] || {}, v = slaVerdict(qout), e2e = e2eVerdict(qout); if (!v) unfloored++; + if (e2e) probed++; rows++; const r = document.createElement("tr"); - const name = td(r, ""); - const tag = document.createElement("span"); - tag.className = "tier-tag"; tag.style.background = tier === "cold" ? C.cold : C.hot; - name.appendChild(tag); - name.appendChild(document.createTextNode(`${endpointLabel(qt)} · ${short(u)} · ${tier}`)); + cellTd(r, tierTag(tier, C), `${endpointLabel(qt)} · ${short(u)} · ${tier}`); // Offered: the phase floor when the run names one, else the rate the leg // was actually paced at. const target = v && v.target_rps != null ? v.target_rps : cell.target_rps; - td(r, rpsTxt(target), v ? note("(1× floor)") : note("(no floor)")); + cellTd(r, rpsTxt(target), noteEl(v ? "(1× floor)" : "(no floor)")); // Served: the achieved arrival rate, flagged when the pacer fell behind. const ach = cell.achieved_rps; - const sTd = td(r, ach ? rpsTxt(ach.m) : "—"); + const sTd = cellTd(r, ach ? rpsTxt(ach.m) : "—"); if (ach) { - sTd.appendChild(note(`(${fmtRps(ach.lo)} – ${fmtRps(ach.hi)})`)); - if (target > 0 && ach.m < target * SATURATED) { - const w = document.createElement("span"); w.className = "cell-warn"; - w.textContent = ` ▲ ${Math.round((1 - ach.m / target) * 100)} % under`; - sTd.appendChild(w); - } + sTd.appendChild(noteEl(`(${fmtRps(ach.lo)} – ${fmtRps(ach.hi)})`)); + const under = underTarget(ach, target); + if (under != null) sTd.appendChild(underWarn(under)); } // Headline: the SCHEDULED p99 — queue-inclusive, so a leg that fell behind - // its schedule shows the delay it caused. + // its schedule shows the delay it caused. Each row states the target it + // was judged against: the SLA sets one per endpoint and storage tier, so + // no single number heads the column, and a row with no verdict is judged + // on the same target its verdict would have used. const p99 = v && v.p99_ns != null ? v.p99_ns : (cell.p99 || {}).m; - const pTd = td(r, p99 == null ? "—" : fmtNs(p99)); - // Each row states the target it was judged against: the SLA sets one per - // endpoint and storage tier, so no single number heads the column. A row - // with no verdict is judged on the same target its verdict would have - // used, never on a run-wide number this endpoint does not answer to. + const pTd = cellTd(r, p99 == null ? "—" : fmtNs(p99)); const rowThr = v && v.threshold_ns != null ? v.threshold_ns : thrFor(qt, tier); if (p99 != null) { pTd.appendChild(chip(v ? v.pass : p99 <= rowThr)); - pTd.appendChild(note(`(≤ ${fmtNsAxis(rowThr)})`, "q-thr")); + pTd.appendChild(noteEl(`(≤ ${fmtNsAxis(rowThr)})`, "q-thr")); } - // The in-RPC time belongs to getTransaction alone; every other row leaves - // it empty rather than showing a number no budget applies to. It is - // JUDGED here only on the published runs that fold it into verdict_1x — - // where the run splits the families, the E2E-budget table judges it at - // its own rate and this column is context. getEvents' mean page is - // reported beside it, judged by nothing. + // In-RPC time: judged here only where the run folds it into the SLA + // verdict (legacy verdict_1x). A row with its own verdict_e2e is judged in + // the E2E-budget table, so its number is context here. Every other row + // leaves the column empty rather than showing a number no budget applies + // to — except an unfloored getTransaction leg, which shows the measurement. const inR = v && v.in_rpc; const svc = (cell.service || {}).p99; if (inR) { - const c = td(r, fmtNs(inR.p99_ns)); + const c = cellTd(r, fmtNs(inR.p99_ns)); c.appendChild(chip(inR.pass)); - c.appendChild(note(`(≤ ${fmtNsAxis(inR.threshold_ns)})`)); - } else if (qt === "txhash" && svc) { - td(r, fmtNs(svc.m), note(o.hasE2E ? "(judged as the E2E budget)" : "(no budget)")); - } else td(r, "—"); - if (qt === "events" && cell.mean_page_ns) td(r, fmtNs(cell.mean_page_ns.m)); - else td(r, "—"); + c.appendChild(noteEl(`(≤ ${fmtNsAxis(inR.threshold_ns)})`)); + } else if (e2e) cellTd(r, svc ? fmtNs(svc.m) : "—", noteEl("(judged as the E2E budget)")); + else if (qt === "txhash" && svc) cellTd(r, fmtNs(svc.m), noteEl("(no budget)")); + else cellTd(r, "—"); + // getEvents' mean page (the only cells that carry one): reported, not judged. + cellTd(r, cell.mean_page_ns ? fmtNs(cell.mean_page_ns.m) : "—"); t.appendChild(r); } const foot = document.getElementById(footId); @@ -938,9 +949,10 @@ : `${checks - fails} of ${checks} checks pass at the 1× rate; ${fails} breach. `; foot.appendChild(verdict); foot.appendChild(document.createTextNode( - (o.hasE2E - ? "Each row is judged on the scheduled p99 against its own endpoint and tier target, shown beside it. getTransaction's time inside the RPC is shown as context here and judged in the E2E-budget table, at the rate that budget applies to." - : "Each row is judged on the scheduled p99 against its own endpoint and tier target, shown beside it; getTransaction also carries an in-RPC budget, counted here as a check of its own.") + "Each row is judged on the scheduled p99 against its own endpoint and tier target, shown beside it" + + (probed + ? ". getTransaction's time inside the RPC is shown as context here and judged in the E2E-budget table, at the rate that budget applies to." + : "; getTransaction also carries an in-RPC budget, counted here as a check of its own.") + " getEvents' mean page is reported, not judged." + (unfloored ? ` ${unfloored} of ${rows} rows carry no phase floor — those are judged on their endpoint and tier's latency target alone, with no load-model rate to meet it at.` : ""))); } @@ -955,48 +967,36 @@ // the SLA table on purpose — the two answer different questions at different // rates, and one row carrying both would read as one requirement. function e2eProbeTable(tableId, footId, o) { - const { Q, TIERS, C, units, short, budgetNs } = o; + const { Q, TIERS, C, units, short } = o; const t = document.getElementById(tableId); - if (!t) return { checks: 0, fails: 0, rows: 0 }; - const head = ["Profile · tier", "Offered", "Served", + if (!t) return; + // A row is whatever entry carries a verdict_e2e: the data names the + // endpoint, the table does not. + const probes = []; + for (const u of units) for (const tier of TIERS) { + for (const [qt, qout] of Object.entries((Q[tier] || {})[u] || {})) { + const v = e2eVerdict(qout); + if (v) probes.push({ qt, u, tier, v, cell: qout[v.rate] || {} }); + } + } + // Every probe verdict carries the one budget, so the header names it once. + const budgetNs = probes.length ? probes[0].v.in_rpc.threshold_ns : null; + const head = ["Endpoint · profile · tier", "Offered", "Served", `In-RPC p99 (≤ ${fmtNsAxis(budgetNs)})`, "Scheduled p99"]; const tr = document.createElement("tr"); head.forEach(h => { const th = document.createElement("th"); th.textContent = h; tr.appendChild(th); }); t.appendChild(tr); - let checks = 0, fails = 0, rows = 0; - for (const u of units) for (const tier of TIERS) { - const qout = ((Q[tier] || {})[u] || {}).txhash; - const v = e2eVerdict(qout); - if (!v) continue; - const cell = qout[v.rate] || {}; - rows++; + let checks = 0, fails = 0; + for (const { qt, u, tier, v, cell } of probes) { const r = document.createElement("tr"); - const name = document.createElement("td"); - const tag = document.createElement("span"); - tag.className = "tier-tag"; tag.style.background = tier === "cold" ? C.cold : C.hot; - name.appendChild(tag); - name.appendChild(document.createTextNode(`${short(u)} · ${tier}`)); - r.appendChild(name); - const td = (...kids) => { - const c = document.createElement("td"); - for (const k of kids) c.appendChild(typeof k === "string" ? document.createTextNode(k) : k); - r.appendChild(c); - return c; - }; - const note = text => { - const s = document.createElement("span"); - s.className = "cell-note"; s.textContent = " " + text; return s; - }; - td(rpsTxt(v.target_rps), note("(demand floor)")); + cellTd(r, tierTag(tier, C), `${endpointLabel(qt)} · ${short(u)} · ${tier}`); + cellTd(r, rpsTxt(v.target_rps), noteEl("(demand floor)")); const ach = cell.achieved_rps; - const sTd = td(ach ? rpsTxt(ach.m) : "—"); - if (ach && v.target_rps > 0 && ach.m < v.target_rps * SATURATED) { - const w = document.createElement("span"); w.className = "cell-warn"; - w.textContent = ` ▲ ${Math.round((1 - ach.m / v.target_rps) * 100)} % under`; - sTd.appendChild(w); - } + const sTd = cellTd(r, ach ? rpsTxt(ach.m) : "—"); + const under = underTarget(ach, v.target_rps); + if (under != null) sTd.appendChild(underWarn(under)); const inR = v.in_rpc || {}; - const iTd = td(inR.p99_ns == null ? "—" : fmtNs(inR.p99_ns)); + const iTd = cellTd(r, inR.p99_ns == null ? "—" : fmtNs(inR.p99_ns)); if (inR.p99_ns != null) { checks++; if (!inR.pass) fails++; const s = document.createElement("span"); @@ -1004,7 +1004,7 @@ s.textContent = inR.pass ? " ✓" : " ▲"; iTd.appendChild(s); } - td(v.p99_ns == null ? "—" : fmtNs(v.p99_ns), note("(context, not judged)")); + cellTd(r, v.p99_ns == null ? "—" : fmtNs(v.p99_ns), noteEl("(context, not judged)")); t.appendChild(r); } const foot = document.getElementById(footId); @@ -1018,14 +1018,13 @@ foot.appendChild(document.createTextNode( `The budget is ${fmtNsAxis(budgetNs)} — getTransaction's slice of the end-to-end transaction-lifecycle target, the one number this leg answers for. Scheduled p99 includes the client's queueing and the network model, neither of which the E2E slice owns, so it is reported and not judged. The same endpoint's SLA verdict, at a different rate and against a different number, is in the table above.`)); } - return { checks, fails, rows }; } // The three profile-scoped open-loop figures. ids: [offered-vs-served, // p99 ladder, latency detail at the 1× rate]. o.unit() is read on every draw, // so the profile picker just re-invokes the returned function. function openQueryFigs(o) { - const { Q, QT, TIERS, C, ids, thr, perEndpoint, ladder } = o; + const { Q, QT, TIERS, C, ids, thr, ladder } = o; const tierColor = t => (t === "cold" ? C.cold : C.hot); const entry = (tier, qt) => ((Q[tier] || {})[o.unit()] || {})[qt] || null; // Panels share one x axis, so only entries that swept the same number of @@ -1064,9 +1063,8 @@ g.all.flatMap(p => p.rows.map(r => { const c = r.cell || {}, ach = c.achieved_rps; let s = ach ? rpsTxt(ach.m) : "—"; - if (ach && r.target > 0 && ach.m < r.target * SATURATED) { - s += ` ▲ ${Math.round((1 - ach.m / r.target) * 100)} % under target`; - } + const under = underTarget(ach, r.target); + if (under != null) s += ` ▲ ${under} % under target`; return [endpointLabel(p.qt), p.tier, r.label, rpsTxt(r.target), s, c.p99 ? fmtNs(c.p99.m) : "—", c.shed ? fmtInt(c.shed.m) : "—"]; }))); @@ -1114,10 +1112,10 @@ } if (rows.length) { // One reference line only when one number judges every lane. A run with - // per-endpoint targets draws none: the lanes are the endpoints, so a - // single line would name a target most of them are not judged against. + // per-endpoint targets passes no thr and draws none: the lanes are the + // endpoints, so a single line would name a target most are not judged against. dotRangeChart(ids[2] + "-body", rows, { - reflines: perEndpoint ? [] : [{ ns: thr, label: fmtNsAxis(thr) + " — latency target" }], + reflines: thr == null ? [] : [{ ns: thr, label: fmtNsAxis(thr) + " — latency target" }], groupSeparators: true, }); } else document.getElementById(ids[2] + "-body").replaceChildren(); @@ -1399,14 +1397,10 @@ const PH = phaseState(D); // phase targets, campaign runs only (null otherwise) // discover query types + concurrency levels from data (degrade gracefully) const { QT, CONC, TIERS } = queryGrid(Q, CH); - const qCheck = checkFor(D, "queries"); // The pubnet page draws the CLOSED-LOOP grid, whose cells carry no verdict - // of their own, so every judgement here resolves its own target from the - // check — per endpoint and tier when the run states it that way. - const thrFor = (qt, tier) => thresholdFor(qCheck, qt, tier); - const qPerEndpoint = !!(qCheck && qCheck.targets_ns); - // The one number the section intro names, for a run that states one. - const qThr = qPerEndpoint ? null : (qCheck && qCheck.threshold_ns) || 500e6; + // of their own, so every judgement resolves its target from the check — + // per endpoint and tier when the run states it that way. + const { check: qCheck, thrFor, perEndpoint: qPerEndpoint, single: qThr } = queryTargets(D); const qLabel = qCheck ? qCheck.label : "query p99 target"; const chunkSub = c => `${fmtK(um[c].events)} ev`; @@ -1806,27 +1800,18 @@ // two share nothing but the cell grid, so the section branches whole. const qMode = qGrid ? qGrid.mode : null; const qLadder = (D.campaign && D.campaign.query_load && D.campaign.query_load.ladder) || null; - const qCheck = checkFor(D, "queries"); - // A run states its read-path target either as one number (the published - // generation) or as a table keyed by endpoint and storage tier (targets_ns, - // the SLA shape). qThrFor resolves whichever it carries, for every cell that - // earned no verdict of its own. - const qThrFor = (qt, tier) => thresholdFor(qCheck, qt, tier); - const qPerEndpoint = !!(qCheck && qCheck.targets_ns); - // One number for the figures that span every endpoint at once; the runs - // that state a target per endpoint draw no such line at all. - const qThr = qPerEndpoint ? null : (qCheck && qCheck.threshold_ns) || 500e6; - // The E2E-budget probe is a second, independent requirement: getTransaction - // at the demand-derived rate, judged on its time inside the RPC. It gets its - // own table, and the whole figure disappears on a run that carries no such - // verdict — every published run predating the split. - const e2eCheck = checkOfKind(D, "query_e2e_probe"); - const hasE2E = !!(Q && e2eCheck && Object.values(Q).some(units => - Object.values(units).some(entry => e2eVerdict((entry || {}).txhash)))); - // Figure numbers inside the queries section. Slot 2 is the E2E-budget - // table; a run without that family closes the gap rather than skipping a - // number, so every figure after it shifts down by one. - const qFig = slot => `Fig ${+secQueries}.${hasE2E || slot < 2 ? slot : slot - 1}`; + // qThr is the one number the prose and the reference lines can name; null + // when the run states a target per endpoint and tier instead. + const { check: qCheck, thrFor: qThrFor, perEndpoint: qPerEndpoint, single: qThr } = queryTargets(D); + // The E2E-budget probe is a second, independent requirement with a table + // of its own. The figure disappears on a run that carries no such verdict — + // every published run predating the split. + const hasE2E = !!Q && Object.values(Q).some(units => + Object.values(units || {}).some(entry => Object.values(entry || {}).some(e2eVerdict))); + // Figures in the queries section are numbered in the order the template + // emits them, so a run without the E2E-budget table closes the gap. + let qFigN = 0; + const qFig = () => `Fig ${+secQueries}.${++qFigN}`; let secN = 2; const secNum = () => String(++secN).padStart(2, "0"); const secCold = hasCold ? secNum() : null; @@ -1973,14 +1958,14 @@

The read side of the same stores the sections above filled, driven open-loop: every endpoint gets a fixed number of requests per second on a fixed schedule, rather than a fixed pool of clients that only asks again once the last answer came back. A server that slows down therefore builds a queue instead of quietly being asked for less. Each endpoint runs three legs — at half, exactly, and twice the request rate the load model asks of it — for a fixed span each. The latency reported everywhere below is the scheduled one: the clock starts when a request was due, not when the client got round to sending it, so queueing counts against the server rather than disappearing. The verdict is read at the 1× leg; the half and double legs are the context on either side of it. Cold legs drop the profile's files from the OS page cache before measuring; hot legs warm them first, because a warm cache is the hot tier's steady state.

${hasE2E ? `

This section answers two separate requirements, and keeps them apart. The SLA asks every endpoint to hold its latency target while serving its share of the sustained request rate. The end-to-end budget asks one endpoint, getTransaction, to answer inside a fixed slice of the transaction-lifecycle target, at the rate the phase's own demand model predicts. The two are measured at different rates and judged on different numbers, so they get a table each and never share a row.

` : ""}
-
${qFig(1)}${hasE2E ? "SLA verdict at the 1× request rate" : "Verdict at the 1× request rate"}
+
${qFig()}${hasE2E ? "SLA verdict at the 1× request rate" : "Verdict at the 1× request rate"}
One row per endpoint, profile and tier, at the rate the SLA asks of that endpoint. Offered is that rate; served is what the client actually managed to place (${medRuns}, spread in brackets) — well under offered means the endpoint saturated. Scheduled p99 is the headline, judged against ${qPerEndpoint ? "the target this endpoint carries in this tier, printed beside it" : esc(fmtNsAxis(qThr))}. ${hasE2E ? "getTransaction's time inside the RPC is context in this table; it is judged in the next one, at the rate its budget applies to." : "getTransaction additionally answers for its time inside the RPC alone — a separate verdict, never folded into the headline."} getEvents' mean page, the wait a sequential subscriber accumulates, is reported beside it and judged by nothing.

${hasE2E ? `
-
${qFig(2)}End-to-end budget — getTransaction at the demand-derived rate
+
${qFig()}End-to-end budget — getTransaction at the demand-derived rate
A different question from the table above, asked of one endpoint. Offered is the rate this phase's demand model predicts for this profile, not the SLA rate. In-RPC p99 is the only judged number: the time getTransaction spent inside the RPC, which is its slice of the end-to-end transaction-lifecycle budget. The scheduled p99 beside it carries the client's queueing and is reported for context alone.
@@ -1988,11 +1973,11 @@

The rate ladder, one profile at a time

Pick a profile to scope the three figures below to it.

${esc(ds.unit_label)}
- ${figHTML("figq2", qFig(3), "Offered vs served request rate", "figq2-legend", `What the client asked for against what it managed to place, at each rung (${medRuns}; log scale, shared across panels). The two lines sit on top of each other while the endpoint keeps up; the served line falling away is the endpoint refusing more load. The table view names the shortfall and the requests dropped at the in-flight cap.`)} - ${figHTML("figq3", qFig(4), "Scheduled p99 as the request rate climbs", "figq3-legend", `The tail latency each rung bought (${medRuns}; log scale, shared across panels). A rung far above its neighbour is the knee — past it the endpoint is queueing, not serving.`)} - ${figHTML("figq4", qFig(5), "Latency at the 1× rate — the whole distribution", "figq4-legend", `Scheduled-latency percentiles over every request of the 1× leg (${medRuns}; log scale). ${qPerEndpoint ? "Each endpoint carries its own latency target, so " + qFig(1) + " states them row by row rather than one dashed line here." : "The dashed line is the " + esc(fmtNsAxis(qThr)) + " latency target."} The table view adds each endpoint's time inside the RPC.`)} + ${figHTML("figq2", qFig(), "Offered vs served request rate", "figq2-legend", `What the client asked for against what it managed to place, at each rung (${medRuns}; log scale, shared across panels). The two lines sit on top of each other while the endpoint keeps up; the served line falling away is the endpoint refusing more load. The table view names the shortfall and the requests dropped at the in-flight cap.`)} + ${figHTML("figq3", qFig(), "Scheduled p99 as the request rate climbs", "figq3-legend", `The tail latency each rung bought (${medRuns}; log scale, shared across panels). A rung far above its neighbour is the knee — past it the endpoint is queueing, not serving.`)} + ${figHTML("figq4", qFig(), "Latency at the 1× rate — the whole distribution", "figq4-legend", `Scheduled-latency percentiles over every request of the 1× leg (${medRuns}; log scale). ${qPerEndpoint ? `Each endpoint carries its own latency target, so Fig ${+secQueries}.1 states them row by row rather than one dashed line here.` : "The dashed line is the " + esc(fmtNsAxis(qThr)) + " latency target."} The table view adds each endpoint's time inside the RPC.`)}
-
${qFig(6)}Per-leg setup and event-scan detail
+
${qFig()}Per-leg setup and event-scan detail
Setup is the untimed work each leg does before it measures: open builds the read fixture, and evict drops the cold artifacts from the page cache once per leg — a hot leg has nothing to evict, so the row is absent. The event columns are what getEvents actually returned at the 1× rate.
@@ -2364,7 +2349,7 @@ const { QT, CONC, TIERS } = qGrid; let curProfile = ORDER[0]; const draw = qMode === "open" - ? openQueryFigs({ Q, QT, TIERS, C, ids: ["figq2", "figq3", "figq4"], thr: qThr, perEndpoint: qPerEndpoint, ladder: qLadder, unit: () => curProfile }) + ? openQueryFigs({ Q, QT, TIERS, C, ids: ["figq2", "figq3", "figq4"], thr: qThr, ladder: qLadder, unit: () => curProfile }) : queryFigs({ Q, QT, CONC, TIERS, C, ids: ["figq1", "figq2", "figq3"], unit: () => curProfile }); const holder = document.getElementById("profile-filter"); for (const p of ORDER) { @@ -2431,11 +2416,8 @@ if (qMode === "open") { const short = u => (parts(u) || {}).name || disp(u); openVerdictTable("query-verdict-table", "query-verdict-footnote", - { Q, QT, TIERS, C, units: ORDER, thrFor: qThrFor, hasE2E, short }); - if (hasE2E) { - e2eProbeTable("query-e2e-table", "query-e2e-footnote", - { Q, TIERS, C, units: ORDER, short, budgetNs: e2eCheck.threshold_ns }); - } + { Q, QT, TIERS, C, units: ORDER, thrFor: qThrFor, short }); + if (hasE2E) e2eProbeTable("query-e2e-table", "query-e2e-footnote", { Q, TIERS, C, units: ORDER, short }); return; } diff --git a/docs/targets.json b/docs/targets.json index 8b96d5d..a4cec3d 100644 --- a/docs/targets.json +++ b/docs/targets.json @@ -29,17 +29,13 @@ }, "derivation": { "full_record": "docs/sla-derivation.md", - "source": "team RPC SLA doc, sections 3.3, 3.4 and 4.1", "derived": "2026-09-01", "aggregate_rps": 500, "mix": {"txhash": 0.6, "events": 0.2, "txpage": 0.15, "ledgers": 0.05}, "sla_tiers_rps": {"light": 250, "standard": 500, "heavy": 1000}, - "tier_windows": {"hot": "Live", "cold": "Recent (frozen-NVMe)"}, - "out_of_scope": "Historical (frozen-EBS): the benchmark boxes carry no EBS tier, so no leg measures it", "txpage_page_limit": 200, "events_page_limit": 10, - "ledgers_span": 10, - "e2e_probe_floors": "demand-derived values from work item 856 — getTransaction is the E2E-budget probe" + "ledgers_span": 10 } }, "phases": [ diff --git a/runner/internal/targets/targets.go b/runner/internal/targets/targets.go index fe2ddf6..c63d024 100644 --- a/runner/internal/targets/targets.go +++ b/runner/internal/targets/targets.go @@ -15,11 +15,11 @@ package targets import ( "encoding/json" "fmt" + "maps" "os" "path/filepath" "regexp" "slices" - "sort" "strings" ) @@ -100,9 +100,11 @@ func Load(path string) (*Targets, error) { path, qtype, strings.Join(QueryTypes, ", ")) } } - if extra := extraKeys(t.QueryLoad.SLA.FloorsRPS); extra != "" { - return nil, fmt.Errorf("targets: %s: query_load.sla.floors_rps names %s, which is not a query type (%s)", - path, extra, strings.Join(QueryTypes, ", ")) + for _, name := range sortedKeys(t.QueryLoad.SLA.FloorsRPS) { + if !slices.Contains(QueryTypes, name) { + return nil, fmt.Errorf("targets: %s: query_load.sla.floors_rps names '%s', which is not a query type (%s)", + path, name, strings.Join(QueryTypes, ", ")) + } } for _, name := range sortedKeys(t.QueryLoad.E2EProbe.FloorsRPS) { if rps := t.QueryLoad.E2EProbe.FloorsRPS[name]; len(rps) != len(t.Phases) { @@ -160,17 +162,14 @@ func (t *Targets) MatchPhase(closeIntervalNs int64) int { // plan's leg order is its own; this list is the vocabulary of the file. var QueryTypes = []string{"ledgers", "txpage", "txhash", "events"} -// Rates is the RPS ladder one query leg targets, ascending. -// -// Three of the four endpoints answer to the SLA family alone, so their ladder -// is the endpoint's SLA floor times each ladder step — the same list in every -// phase and every profile. getTransaction answers to both families and runs as -// ONE leg: its ladder is the union of the SLA ladder and the (profile, phase) -// E2E-probe ladder, deduplicated. Where the two floors coincide the shared cell -// carries both verdicts, which is why the union is deduplicated rather than -// concatenated. +// Rates is the RPS ladder one query leg targets, ascending: the endpoint's SLA +// floor times every ladder step. txhash runs as ONE leg answering to both +// families, so its ladder also unions the (profile, phase) E2E-probe ladder, +// deduplicated and sorted — a floor the two families share becomes one cell +// carrying both verdicts. func (t *Targets) Rates(profileKey string, phase int, qtype string) ([]float64, error) { - if !slices.Contains(QueryTypes, qtype) { + slaFloor, ok := t.QueryLoad.SLA.FloorsRPS[qtype] + if !ok { return nil, fmt.Errorf("targets: unknown query type '%s' (known types: %s)", qtype, strings.Join(QueryTypes, ", ")) } // Every leg resolves the profile and the phase, even the three that do not @@ -183,7 +182,7 @@ func (t *Targets) Rates(profileKey string, phase int, qtype string) ([]float64, if phase < 1 || phase > len(e2eFloors) { return nil, fmt.Errorf("targets: phase %d has no floors (query_load carries phases 1-%d)", phase, len(e2eFloors)) } - rates := t.ladderAround(t.QueryLoad.SLA.FloorsRPS[qtype]) + rates := t.ladderAround(slaFloor) if qtype == "txhash" { rates = append(rates, t.ladderAround(e2eFloors[phase-1])...) slices.Sort(rates) @@ -203,21 +202,5 @@ func (t *Targets) ladderAround(floor float64) []float64 { // sortedKeys keeps every message that lists profiles stable. func sortedKeys[V any](m map[string]V) []string { - names := make([]string, 0, len(m)) - for name := range m { - names = append(names, name) - } - sort.Strings(names) - return names -} - -// extraKeys names the keys of m that are not query types, or "" when none are. -func extraKeys[V any](m map[string]V) string { - var extra []string - for _, name := range sortedKeys(m) { - if !slices.Contains(QueryTypes, name) { - extra = append(extra, "'"+name+"'") - } - } - return strings.Join(extra, ", ") + return slices.Sorted(maps.Keys(m)) } diff --git a/tests/smoke/gen-fixtures.py b/tests/smoke/gen-fixtures.py index 034d9fe..0e57750 100644 --- a/tests/smoke/gen-fixtures.py +++ b/tests/smoke/gen-fixtures.py @@ -32,6 +32,7 @@ import contextlib import io import json +import math import os import re import shutil @@ -121,11 +122,12 @@ def ladder(profile, qt, phase): def load_level(qt, tok): - """Which of the three latency levels a rate sits at, read off its distance - from that endpoint's SLA floor. getTransaction's demand rungs run far above - that floor, so they land on the heavy level and pay the heavy latency.""" + """Which of the three latency levels a rate sits at: the ladder rung nearest + its multiple of that endpoint's SLA floor, measured in log space because the + ladder is geometric.""" ratio = float(tok) / C.SLA_FLOORS_RPS[qt] - return 0 if ratio <= 0.75 else 1 if ratio <= 1.5 else 2 + steps = C.QUERY_LOAD["ladder"] + return min(range(len(steps)), key=lambda i: abs(math.log(ratio / steps[i]))) def cell_spec(tier, profile, qt, tok, top=False): @@ -256,11 +258,8 @@ def build_bundle(root, spec): def rate_cells(qout): - """The r cells of one qtype entry. The verdicts are siblings of theirs - that also carry a target_rps, so they are named out rather than sniffed.""" - return {k: v for k, v in qout.items() - if not k.startswith("verdict_") and isinstance(v, dict) - and "target_rps" in v} + """The r cells of one qtype entry — the converter's own selector.""" + return C.rps_cells(qout) def qtype_entries(D): diff --git a/tests/smoke/smoke.mjs b/tests/smoke/smoke.mjs index 26bd3dc..52a83ff 100644 --- a/tests/smoke/smoke.mjs +++ b/tests/smoke/smoke.mjs @@ -135,11 +135,14 @@ function queryShape(D) { } /* The run's read-path latency target for one endpoint in one storage tier, and the rung a verdict table judges — both read exactly the way docs/app.js reads - them (checkFor + thresholdFor / judgedRate). A run states its target either + them (queryCheck + thresholdFor / judgedRate). A run states its target either per endpoint and tier (targets_ns, the SLA shape) or as one number for the whole run (threshold_ns, the published generation). */ -const queryCheck = (D) => (D.checks_all || [D.checks || {}]) - .find((c) => c && c.applies_to === "queries") || {}; +const allChecks = (D) => D.checks_all || [D.checks || {}]; +// Matched on kind first: two checks share applies_to "queries" once the +// families are split. The applies_to fallback is the legacy single check. +const queryCheck = (D) => allChecks(D).find((c) => c && c.kind === "query_sla") + || allChecks(D).find((c) => c && c.applies_to === "queries") || {}; const queryThr = (D, qt, tier) => { const c = queryCheck(D); const byTier = c.targets_ns && c.targets_ns[qt]; @@ -152,7 +155,8 @@ const queryThr = (D, qt, tier) => { before the split carry one verdict_1x that plays the SLA role. */ const slaVerdict = (qout) => (qout && (qout.verdict_sla || qout.verdict_1x)) || null; const e2eVerdict = (qout) => (qout && qout.verdict_e2e) || null; -const hasE2EFamily = (D) => qEntries(D).some((e) => e2eVerdict(e.qout)); +const e2eEntries = (D) => qEntries(D).filter((e) => e2eVerdict(e.qout)); +const hasE2EFamily = (D) => e2eEntries(D).length > 0; const judgedKey = (qout) => { const v = slaVerdict(qout); if (v && v.rate && qout[v.rate]) return v.rate; @@ -316,7 +320,7 @@ function checkOpenLoopQueries(doc, sec, group, D) { rows++; if (!v) unfloored++; const cell = qout[key] || {}; - judged.push({ qt, u, tier, cell, v }); + judged.push({ qt, u, tier, qout, cell, v }); // A row with no phase floor still gets a headline chip — the viewer judges // it on this endpoint and tier's own latency target, with no rate to meet // it at (docs/app.js openVerdictTable), so the hand count judges it the @@ -402,9 +406,9 @@ function checkOpenLoopQueries(doc, sec, group, D) { // leg that swept MORE rungs than the ladder has (getTransaction sweeping both // families with no floor to anchor them) is labelled by rate instead, so it // is not counted here. - const ladderLen = ((D.campaign || {}).query_load || {}).ladder; - const rungRows = judged.filter((j) => j.u === first && (j.v - || (Array.isArray(ladderLen) && rateKeys(((D.queries[j.tier] || {})[j.u] || {})[j.qt]).length === ladderLen.length))).length; + const ladderSteps = ((D.campaign || {}).query_load || {}).ladder; + const anchored = (j) => j.v || (Array.isArray(ladderSteps) && rateKeys(j.qout).length === ladderSteps.length); + const rungRows = judged.filter((j) => j.u === first && anchored(j)).length; check(group, `ladder table lists 1× for every anchored endpoint × tier (${rungRows})`, (ladder.match(/1×/g) || []).length >= rungRows, ladder.slice(0, 160)); @@ -417,7 +421,7 @@ function checkOpenLoopQueries(doc, sec, group, D) { // SLA table above must not chip getTransaction's in-RPC time as if the two // were one verdict. const e2eTbl = doc.getElementById("query-e2e-table"); - const wantE2E = qEntries(D).filter((e) => e2eVerdict(e.qout)); + const wantE2E = e2eEntries(D); check(group, wantE2E.length ? "E2E-budget table rendered" : "no E2E-budget table without the family", !!e2eTbl === wantE2E.length > 0, e2eTbl ? "present" : "missing"); if (e2eTbl) { @@ -643,6 +647,11 @@ function checkSummarySections(sdoc, group, D) { const sreport = sdoc.getElementById("report"); const wantBudget = summaryHasBudget(D); const QR = summaryQueryRows(D); + // getTransaction is measured only when a live-store leg recorded an in-RPC + // p99 — from the E2E-budget probe where the run has one, else folded into the + // single legacy verdict. Otherwise its allocation stands. + const measuredGet = (QR.hot || []).some((r) => r.qt === "txhash" && (r.e2e ? r.e2e.in_rpc : r.v.in_rpc)); + const probedGet = (QR.hot || []).some((r) => r.qt === "txhash" && r.e2e); const qTiers = ["hot", "cold"].filter((t) => QR[t]); const wantQueries = qTiers.length > 0; @@ -700,12 +709,6 @@ function checkSummarySections(sdoc, group, D) { check(group, "tiles name the three slices and their allocations", /sendTransaction/.test(tileTxt) && /ingestion/.test(tileTxt) && /getTransaction/.test(tileTxt) && (tileTxt.match(/allocation/g) || []).length === 3, tileTxt.slice(0, 160)); - // getTransaction is measured only when the run recorded an in-RPC p99 on a - // live-store getTransaction leg; otherwise its allocation stands. The - // number comes from the E2E-budget probe, which is the leg run for exactly - // this purpose; runs predating the split fold it into their one verdict. - const measuredGet = (QR.hot || []).some((r) => r.qt === "txhash" - && (r.e2e ? r.e2e.in_rpc : r.v.in_rpc)); const f11t = txt(sdoc.querySelector("#fig11-tv")); const getRow = f11t.split("getTransaction round trip")[1] || ""; check(group, measuredGet ? "budget table: getTransaction allocated → measured" : "budget table: getTransaction stays an estimate", @@ -788,11 +791,12 @@ function checkSummarySections(sdoc, group, D) { && txt(coldCell && coldCell.querySelector(".q-rate")) === "75 rps", `${txt(hotCell)} | ${txt(coldCell)}`); } - const measuredGet = (QR.hot || []).some((r) => r.qt === "txhash" && r.v.in_rpc); if (wantBudget && measuredGet) { + // A run with the probe family names the probe leg and its rate in the note. + const want = new RegExp("^The end-to-end budget in §01 uses the highest measured getTransaction p99" + + (probedGet ? ", taken from its end-to-end probe leg at [\\d,.]+ rps" : "") + "\\.$"); check(group, "hot matrix links its highest getTransaction p99 to the E2E budget", - txt(qsecLive.querySelector(".q-foot")) === "The end-to-end budget in §01 uses the highest measured getTransaction p99.", - txt(qsecLive.querySelector(".q-foot"))); + want.test(txt(qsecLive.querySelector(".q-foot"))), txt(qsecLive.querySelector(".q-foot"))); } } From f83b0576b93e51586d2620f01fb04b74bb91d3d3 Mon Sep 17 00:00:00 2001 From: Marwen Abid Date: Wed, 2 Sep 2026 13:17:38 -0700 Subject: [PATCH 3/3] Address Copilot review on the verdict split - Match the read-path check on kind only (query_sla, then the legacy query_p99_threshold) in the viewer and the smoke mirror; the applies_to fallback could have picked the E2E probe check if query_sla were absent. - Rates sorts and deduplicates every endpoint's ladder, not only txhash, so the doc comment's 'ascending' holds regardless of ladder order in the file. - The missing-profile message names query_load.e2e_probe, the block it actually looks in, in both the runner and the converter. --- converter/convert.py | 2 +- converter/tests/test_queries_rps.py | 2 +- docs/app.js | 8 ++++---- runner/cmd/campaign/main_test.go | 2 +- runner/internal/targets/targets.go | 17 ++++++++--------- runner/internal/targets/targets_test.go | 2 +- tests/smoke/smoke.mjs | 9 ++++----- 7 files changed, 20 insertions(+), 22 deletions(-) diff --git a/converter/convert.py b/converter/convert.py index 466ac72..c122011 100644 --- a/converter/convert.py +++ b/converter/convert.py @@ -933,7 +933,7 @@ def attach_query_verdicts(queries, phase): if qt != "txhash": continue if e2e_floors is None: - warn(f"query-{tier}-{unit}: no query_load profile {profile!r} " + warn(f"query-{tier}-{unit}: no query_load.e2e_probe profile {profile!r} " f"in targets.json; no E2E-budget verdict") continue if idx >= len(e2e_floors): diff --git a/converter/tests/test_queries_rps.py b/converter/tests/test_queries_rps.py index f040b1a..05b2651 100644 --- a/converter/tests/test_queries_rps.py +++ b/converter/tests/test_queries_rps.py @@ -245,7 +245,7 @@ def test_unknown_profile_costs_the_probe_verdict_only(self): cold = data["queries"]["cold"]["mystery-9000-c1"] self.assertIn("verdict_sla", cold["txhash"]) self.assertNotIn("verdict_e2e", cold["txhash"]) - hits = [w for w in warnings if "no query_load profile 'mystery'" in w] + hits = [w for w in warnings if "no query_load.e2e_probe profile 'mystery'" in w] self.assertEqual(len(hits), 2) # one per tier, not per qtype self.assertEqual(convert.validate_run(data, reps=2), []) diff --git a/docs/app.js b/docs/app.js index d8e0586..7abab64 100644 --- a/docs/app.js +++ b/docs/app.js @@ -579,10 +579,10 @@ function checkOfKind(D, kind) { return allChecks(D).find(c => c && c.kind === kind) || null; } - // The read-path SLA check. Two checks now share applies_to "queries", so it - // is matched on kind; the applies_to fallback picks up the single - // query_p99_threshold check of runs converted before the split. - const queryCheck = D => checkOfKind(D, "query_sla") || checkFor(D, "queries"); + // The read-path SLA check, matched on kind — two checks share applies_to + // "queries" now, so applies_to alone cannot pick it out. Runs converted + // before the split carry the single query_p99_threshold check in its place. + const queryCheck = D => checkOfKind(D, "query_sla") || checkOfKind(D, "query_p99_threshold"); // The p99 one query cell answers to. The SLA check states a target per // endpoint AND storage tier in targets_ns; older runs state one number for diff --git a/runner/cmd/campaign/main_test.go b/runner/cmd/campaign/main_test.go index 4cc2c56..9c65032 100644 --- a/runner/cmd/campaign/main_test.go +++ b/runner/cmd/campaign/main_test.go @@ -245,7 +245,7 @@ func TestPlanCmdQueryLoad(t *testing.T) { if code != 2 { t.Errorf("exit code = %d, want 2", code) } - for _, want := range []string{"dataset 'pubnet-63'", "load profile 'pubnet'", "known profiles:"} { + for _, want := range []string{"dataset 'pubnet-63'", "e2e_probe profile 'pubnet'", "known profiles:"} { if !strings.Contains(stderr, want) { t.Errorf("stderr missing %q, got:\n%s", want, stderr) } diff --git a/runner/internal/targets/targets.go b/runner/internal/targets/targets.go index c63d024..033e3b2 100644 --- a/runner/internal/targets/targets.go +++ b/runner/internal/targets/targets.go @@ -162,11 +162,11 @@ func (t *Targets) MatchPhase(closeIntervalNs int64) int { // plan's leg order is its own; this list is the vocabulary of the file. var QueryTypes = []string{"ledgers", "txpage", "txhash", "events"} -// Rates is the RPS ladder one query leg targets, ascending: the endpoint's SLA -// floor times every ladder step. txhash runs as ONE leg answering to both -// families, so its ladder also unions the (profile, phase) E2E-probe ladder, -// deduplicated and sorted — a floor the two families share becomes one cell -// carrying both verdicts. +// Rates is the RPS ladder one query leg targets, sorted ascending and +// deduplicated: the endpoint's SLA floor times every ladder step. txhash runs +// as ONE leg answering to both families, so its ladder also unions the +// (profile, phase) E2E-probe ladder — a floor the two families share becomes +// one cell carrying both verdicts. func (t *Targets) Rates(profileKey string, phase int, qtype string) ([]float64, error) { slaFloor, ok := t.QueryLoad.SLA.FloorsRPS[qtype] if !ok { @@ -176,7 +176,7 @@ func (t *Targets) Rates(profileKey string, phase int, qtype string) ([]float64, // pace by them: a run that names neither is a run nobody can judge. e2eFloors, ok := t.QueryLoad.E2EProbe.FloorsRPS[profileKey] if !ok { - return nil, fmt.Errorf("targets: no query_load profile '%s' (known profiles: %s)", + return nil, fmt.Errorf("targets: no query_load.e2e_probe profile '%s' (known profiles: %s)", profileKey, strings.Join(sortedKeys(t.QueryLoad.E2EProbe.FloorsRPS), ", ")) } if phase < 1 || phase > len(e2eFloors) { @@ -185,10 +185,9 @@ func (t *Targets) Rates(profileKey string, phase int, qtype string) ([]float64, rates := t.ladderAround(slaFloor) if qtype == "txhash" { rates = append(rates, t.ladderAround(e2eFloors[phase-1])...) - slices.Sort(rates) - rates = slices.Compact(rates) } - return rates, nil + slices.Sort(rates) + return slices.Compact(rates), nil } // ladderAround is one floor times every ladder step, in ladder order. diff --git a/runner/internal/targets/targets_test.go b/runner/internal/targets/targets_test.go index a930429..65901f3 100644 --- a/runner/internal/targets/targets_test.go +++ b/runner/internal/targets/targets_test.go @@ -110,7 +110,7 @@ func TestRatesRejects(t *testing.T) { qtype string want []string }{ - {"unknown profile", "pubnet", 1, "txhash", []string{"no query_load profile 'pubnet'", "sac"}}, + {"unknown profile", "pubnet", 1, "txhash", []string{"no query_load.e2e_probe profile 'pubnet'", "sac"}}, {"unknown query type", "sac", 1, "blocks", []string{"unknown query type 'blocks'", "ledgers"}}, {"phase below the range", "sac", 0, "txhash", []string{"phase 0 has no floors", "phases 1-3"}}, {"phase above the range", "sac", 4, "txhash", []string{"phase 4 has no floors", "phases 1-3"}}, diff --git a/tests/smoke/smoke.mjs b/tests/smoke/smoke.mjs index 52a83ff..eeb6f18 100644 --- a/tests/smoke/smoke.mjs +++ b/tests/smoke/smoke.mjs @@ -138,11 +138,10 @@ function queryShape(D) { them (queryCheck + thresholdFor / judgedRate). A run states its target either per endpoint and tier (targets_ns, the SLA shape) or as one number for the whole run (threshold_ns, the published generation). */ -const allChecks = (D) => D.checks_all || [D.checks || {}]; -// Matched on kind first: two checks share applies_to "queries" once the -// families are split. The applies_to fallback is the legacy single check. -const queryCheck = (D) => allChecks(D).find((c) => c && c.kind === "query_sla") - || allChecks(D).find((c) => c && c.applies_to === "queries") || {}; +const checkOfKind = (D, kind) => (D.checks_all || [D.checks || {}]).find((c) => c && c.kind === kind); +// Matched on kind: two checks share applies_to "queries" once the families +// are split. query_p99_threshold is the legacy single check. +const queryCheck = (D) => checkOfKind(D, "query_sla") || checkOfKind(D, "query_p99_threshold") || {}; const queryThr = (D, qt, tier) => { const c = queryCheck(D); const byTier = c.targets_ns && c.targets_ns[qt];