Skip to content

Commit 094d962

Browse files
committed
feat(ci): drift check warns past 1.3x measured/predicted, below the 1.5x red
WARN_MEASURED_OVER_PREDICTED sits beside MAX_MEASURED_OVER_PREDICTED: the same ratio over the same executed windows, an annotation-only tier under the red. renderDriftVerdict() returns the four verdicts without printing them, so the self-test reads the ::warning line a runner receives. Claude-Session: https://claude.ai/code/session_01VF48aw8RPG6wzDnMgp6rtw Co-authored-by: Claude <noreply@anthropic.com>
1 parent 74192b4 commit 094d962

1 file changed

Lines changed: 250 additions & 54 deletions

File tree

‎scripts/partition-test-shards.mjs‎

Lines changed: 250 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@
8282
// written (#16173). Every Test Core shard already passes `--summarize`, so the
8383
// run it just finished has written the measured truth to `.turbo/runs/`; this
8484
// mode reads that back, compares it to what this script PREDICTED for the same
85-
// packages, and reds past MAX_MEASURED_OVER_PREDICTED. Without it the dataset
85+
// packages, raises a `::warning::` past WARN_MEASURED_OVER_PREDICTED, and reds
86+
// past MAX_MEASURED_OVER_PREDICTED. Without it the dataset
8687
// rots silently in one direction and the only instrument that notices is a
8788
// shard killed by the job timeout -- which is a shard that produced NO reading
8889
// while the rollup read green.
@@ -173,6 +174,32 @@ export const MAX_SHARD_OVER_MEAN = 1.3;
173174
// number it would be raised past is a measurement of the dataset being wrong.
174175
export const MAX_MEASURED_OVER_PREDICTED = 1.5;
175176

177+
// The WARNING tier under that red (#16465): past this factor `--check-drift`
178+
// raises a `::warning::` annotation on the shard and the step stays green.
179+
// One quantity, two thresholds -- the same measured/predicted ratio over the
180+
// same executed windows, so the warning is the red's early half and never a
181+
// second rule. A shard past MAX_MEASURED_OVER_PREDICTED gets the red alone.
182+
//
183+
// ⚠ It is NOT MAX_SHARD_OVER_MEAN, though both read 1.3 today. That one bounds
184+
// the predicted bins against EACH OTHER (max/mean of one split); this one bounds
185+
// one shard's measurement against ITS OWN prediction. Neither is derived from
186+
// the other, so moving one never moves the other -- which is why this is its
187+
// own constant rather than a reference to that one.
188+
//
189+
// Why 1.3, and why a warning rather than a red:
190+
//
191+
// - it sits ABOVE the healthy population the red was measured against (the
192+
// five healthy shards of run 34013842594 topped out at 1.18x), so a green
193+
// dataset does not raise it;
194+
// - it sits AT the split's own balance tolerance, which is the point the
195+
// docblock above names as where drift stops being something the
196+
// partitioner absorbs -- past it the dataset is due a refresh, not yet
197+
// wrong enough to block anyone;
198+
// - it is a warning by ruling: one red rule per signal, and that rule is the
199+
// 1.5x above. The self-test pins the band between the two as non-empty, so
200+
// the warning cannot be made unreachable by moving either end.
201+
export const WARN_MEASURED_OVER_PREDICTED = 1.3;
202+
176203
// ── SHARDING ONE PACKAGE BELOW PACKAGE GRANULARITY (#16173) ────────────────
177204
//
178205
// The floor argument at the top of this file is not a caveat, it is a wall: a
@@ -792,14 +819,110 @@ export function driftReport(
792819
// the same way: NOT MEASURED is not a pass, and it is not a failure either.
793820
const measurable = rows.length > 0 && predictedTotal > 0;
794821
const ratio = measurable ? measuredTotal / predictedTotal : null;
822+
const drifted = measurable && ratio > factor;
795823
return {
796824
rows,
797825
unpredicted,
798826
predictedTotal,
799827
measuredTotal,
800828
ratio,
801829
measurable,
802-
drifted: measurable && ratio > factor,
830+
drifted,
831+
// The warning BAND, exclusive of the red: a drifted shard is reported by
832+
// the red alone, so one shard never carries two verdicts. Strict `>` like
833+
// the red, so a reading exactly at WARN_MEASURED_OVER_PREDICTED is green.
834+
warned: measurable && !drifted && ratio > WARN_MEASURED_OVER_PREDICTED,
835+
};
836+
}
837+
838+
// The four verdicts `--check-drift` can print, rendered without printing them,
839+
// so the self-test can read what a runner will receive. `out` is stdout, where
840+
// the runner parses workflow commands; the annotation is the warning tier's
841+
// ONLY effect, so it is pinned here rather than trusted to the caller.
842+
//
843+
// A workflow-command message is one line: `%`, CR and LF are escaped the way
844+
// the runner un-escapes them, so a package name or label can never end the
845+
// annotation early or start a second command.
846+
export function escapeWorkflowCommandMessage(text) {
847+
return String(text).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
848+
}
849+
850+
export function renderDriftVerdict(report, label) {
851+
const skipped =
852+
report.unpredicted.length === 0
853+
? ''
854+
: ` ${report.unpredicted.length} package(s) carry no dataset entry and were excluded ` +
855+
`(estimated, not predicted): ${report.unpredicted.join(', ')}.`;
856+
857+
if (!report.measurable) {
858+
return {
859+
verdict: 'NOT MEASURED',
860+
exitCode: 0,
861+
out: [],
862+
err: [
863+
`shard-timing-drift: NOT MEASURED -- ${label} finished no test task that was both a cache ` +
864+
'MISS and carried a dataset entry, so this run says nothing about whether ' +
865+
`scripts/test-shard-timings.json is still true.${skipped}`,
866+
],
867+
};
868+
}
869+
870+
const head =
871+
`${report.measuredTotal.toFixed(1)}s measured vs ${report.predictedTotal.toFixed(1)}s predicted ` +
872+
`across ${report.rows.length} package(s) = ${report.ratio.toFixed(2)}x ` +
873+
`(warning past ${WARN_MEASURED_OVER_PREDICTED}x, red past ${MAX_MEASURED_OVER_PREDICTED}x)`;
874+
const ratioOf = (r) => (r.predicted > 0 ? `${(r.measured / r.predicted).toFixed(2)}x, ` : '');
875+
const worst = report.rows
876+
.slice(0, 5)
877+
.map(
878+
(r) =>
879+
` ${r.name}: predicted ${r.predicted.toFixed(1)}s, measured ${r.measured.toFixed(1)}s ` +
880+
`(${ratioOf(r)}${r.overshoot >= 0 ? '+' : ''}${r.overshoot.toFixed(1)}s)`
881+
)
882+
.join('\n');
883+
884+
if (report.drifted) {
885+
return {
886+
verdict: 'DRIFT',
887+
exitCode: 1,
888+
out: [],
889+
err: [
890+
`shard-timing-drift: DRIFT -- ${label}, ${head}.${skipped}\n` +
891+
' Heaviest overshoots:\n' +
892+
`${worst}\n` +
893+
' scripts/test-shard-timings.json no longer describes this workspace, so the shard split\n' +
894+
' is balancing a quantity that is not the runtime. Refresh it -- see\n' +
895+
' scripts/measure-test-shard-timings.mjs for the two refresh paths -- and ⛔ do NOT\n' +
896+
' hand-edit the dataset or raise this bound to absorb the gap. Expect the refresh to red\n' +
897+
" this script's own balance pins if a single suite has outgrown the acceptance bound:\n" +
898+
' that is those pins working, and the remedy they name is splitting that suite below\n' +
899+
' package granularity, never a different shard count.',
900+
],
901+
};
902+
}
903+
904+
if (report.warned) {
905+
const top = report.rows[0];
906+
return {
907+
verdict: 'WARN',
908+
exitCode: 0,
909+
out: [
910+
`::warning title=Test Core shard timing drift::${escapeWorkflowCommandMessage(
911+
`${label}: ${head}. Heaviest overshoot: ${top.name}, predicted ${top.predicted.toFixed(1)}s, ` +
912+
`measured ${top.measured.toFixed(1)}s. scripts/test-shard-timings.json is drifting from the ` +
913+
'runtime; refresh it before it reaches the red (see scripts/measure-test-shard-timings.mjs). ' +
914+
'Not a failure: this step stays green until the red bound.'
915+
)}`,
916+
],
917+
err: [`shard-timing-drift: WARN -- ${label}, ${head}.${skipped}\n Heaviest overshoots:\n${worst}`],
918+
};
919+
}
920+
921+
return {
922+
verdict: 'OK',
923+
exitCode: 0,
924+
out: [],
925+
err: [`shard-timing-drift: OK -- ${label}, ${head}.${skipped}`],
803926
};
804927
}
805928

@@ -887,13 +1010,14 @@ const SELF_TEST_BATTERIES = Object.freeze({
8871010
// heaviest package do not depend on what the live map slices.
8881011
'the balancing pins (#10472)': 25,
8891012
'predicted-vs-measured drift (#16173)': 9,
1013+
'drift warning tier under the red (#16465)': 12,
8901014
'file-level slice items (#16173)': 20,
8911015
'file-level slices reach vitest through OS_TEST_SHARD (#19278)': 11,
8921016
});
8931017

8941018
// DELETING an entry silences that battery's floor exactly as effectively as
8951019
// zeroing it, so the roster's own size is pinned too.
896-
const SELF_TEST_BATTERY_FLOOR = 10;
1020+
const SELF_TEST_BATTERY_FLOOR = 11;
8971021

8981022
// The key an assertion is filed under when no battery is open. It is not a
8991023
// declared battery, so it reds by the same set difference rather than silently
@@ -1434,6 +1558,118 @@ function selfTest() {
14341558
}
14351559
});
14361560

1561+
// -- THE WARNING TIER UNDER THE RED (#16465) ----------------------------
1562+
//
1563+
// Same ratio, same executed windows, a lower threshold that annotates and
1564+
// never fails. Fixtures are scaled to the 100s `slow` entry above, so a
1565+
// measured N seconds reads as N/100x. What these hold: the band is
1566+
// (WARN, RED], it is exclusive of the red, it is non-empty, its effect is an
1567+
// annotation a runner will actually parse, and a replay never enters it.
1568+
battery('drift warning tier under the red (#16465)');
1569+
const tierOf = (seconds) => driftReport(measuredMap({ slow: seconds }), driftTimings);
1570+
1571+
check(() => {
1572+
const r = tierOf(131);
1573+
if (!r.warned || r.drifted) {
1574+
throw new Error(`drift warning: a 1.31x reading came back warned=${r.warned} drifted=${r.drifted}`);
1575+
}
1576+
});
1577+
// The lower edge: strict `>`, like the red, so exactly at the factor is green.
1578+
check(() => {
1579+
const r = tierOf(WARN_MEASURED_OVER_PREDICTED * 100);
1580+
if (r.warned || r.drifted) {
1581+
throw new Error('drift warning: a reading exactly AT the warning factor was flagged');
1582+
}
1583+
});
1584+
// The upper edge belongs to the warning: AT the red is not yet a red.
1585+
check(() => {
1586+
const r = tierOf(MAX_MEASURED_OVER_PREDICTED * 100);
1587+
if (!r.warned || r.drifted) {
1588+
throw new Error(`drift warning: a reading exactly AT the red came back warned=${r.warned} drifted=${r.drifted}`);
1589+
}
1590+
});
1591+
// Past the red the red alone speaks: one shard, one verdict.
1592+
check(() => {
1593+
const r = tierOf(151);
1594+
if (r.warned || !r.drifted) {
1595+
throw new Error(`drift warning: a 1.51x reading came back warned=${r.warned} drifted=${r.drifted}`);
1596+
}
1597+
});
1598+
// The healthy population the red was measured against topped out at 1.18x.
1599+
check(() => {
1600+
const r = tierOf(118);
1601+
if (r.warned || r.drifted) throw new Error('drift warning: a healthy 1.18x reading was flagged');
1602+
});
1603+
// A replay-only shard is NOT MEASURED, never a warning in either direction.
1604+
check(() => {
1605+
const r = driftReport(replayed.samples, driftTimings);
1606+
if (r.warned || r.drifted) throw new Error('drift warning: a summary of replays and failures was flagged');
1607+
});
1608+
// The band is non-empty, or the warning is a declared tier nothing can reach.
1609+
check(() => {
1610+
if (!(WARN_MEASURED_OVER_PREDICTED < MAX_MEASURED_OVER_PREDICTED)) {
1611+
throw new Error(
1612+
`drift warning: WARN_MEASURED_OVER_PREDICTED (${WARN_MEASURED_OVER_PREDICTED}) is not below ` +
1613+
`MAX_MEASURED_OVER_PREDICTED (${MAX_MEASURED_OVER_PREDICTED}) -- the warning band is empty`
1614+
);
1615+
}
1616+
});
1617+
// Not below the split's own tolerance: under it, the warning would fire on
1618+
// drift the partitioner is built to absorb, and a warning on healthy input is
1619+
// one everybody learns to scroll past. (A relation, not a reuse: see the
1620+
// constant's docblock.)
1621+
check(() => {
1622+
if (WARN_MEASURED_OVER_PREDICTED < MAX_SHARD_OVER_MEAN) {
1623+
throw new Error(
1624+
`drift warning: WARN_MEASURED_OVER_PREDICTED (${WARN_MEASURED_OVER_PREDICTED}) sits below ` +
1625+
`MAX_SHARD_OVER_MEAN (${MAX_SHARD_OVER_MEAN}), inside the drift the split absorbs`
1626+
);
1627+
}
1628+
});
1629+
1630+
// What the runner receives. The verdict is the whole effect of this tier, so
1631+
// it is read as rendered text, never printed here (a self-test line opening
1632+
// with a workflow command would mint a real annotation on every lint run).
1633+
const label = 'Test Core (9/9)';
1634+
check(() => {
1635+
const v = renderDriftVerdict(tierOf(131), label);
1636+
const commands = v.out.filter((line) => line.startsWith('::'));
1637+
if (v.verdict !== 'WARN' || v.exitCode !== 0 || commands.length !== 1) {
1638+
throw new Error(
1639+
`drift warning: rendered ${v.verdict} exit ${v.exitCode} with ${commands.length} workflow command(s)`
1640+
);
1641+
}
1642+
if (!commands[0].startsWith('::warning ') || !commands[0].includes(label) || commands[0].includes('\n')) {
1643+
throw new Error(`drift warning: the annotation is not one ::warning line naming the shard: ${commands[0]}`);
1644+
}
1645+
});
1646+
check(() => {
1647+
const v = renderDriftVerdict(tierOf(151), label);
1648+
if (v.verdict !== 'DRIFT' || v.exitCode !== 1 || v.out.some((line) => line.startsWith('::'))) {
1649+
throw new Error(`drift warning: a red rendered ${v.verdict} exit ${v.exitCode}, ${v.out.length} stdout line(s)`);
1650+
}
1651+
});
1652+
check(() => {
1653+
for (const [seconds, expected] of [[118, 'OK'], [WARN_MEASURED_OVER_PREDICTED * 100, 'OK']]) {
1654+
const v = renderDriftVerdict(tierOf(seconds), label);
1655+
if (v.verdict !== expected || v.exitCode !== 0 || v.out.length !== 0) {
1656+
throw new Error(`drift warning: ${seconds / 100}x rendered ${v.verdict} with ${v.out.length} stdout line(s)`);
1657+
}
1658+
}
1659+
const nm = renderDriftVerdict(driftReport(replayed.samples, driftTimings), label);
1660+
if (nm.verdict !== 'NOT MEASURED' || nm.exitCode !== 0 || nm.out.length !== 0) {
1661+
throw new Error(`drift warning: a replay-only shard rendered ${nm.verdict}`);
1662+
}
1663+
});
1664+
// One line per command: a newline smuggled in through a label or package
1665+
// name would end the annotation and open whatever followed it as a command.
1666+
check(() => {
1667+
const escaped = escapeWorkflowCommandMessage('50% done\r\n::error::x');
1668+
if (escaped !== '50%25 done%0D%0A::error::x') {
1669+
throw new Error(`drift warning: workflow-command escaping produced ${JSON.stringify(escaped)}`);
1670+
}
1671+
});
1672+
14371673
// -- FILE-LEVEL SLICE ITEMS (#16173) ------------------------------------
14381674
//
14391675
// The grammar, its refusals, and the two joins that read it. The balancing
@@ -1794,11 +2030,13 @@ function selfTest() {
17942030

17952031
// `--check-drift`: the shard just measured itself, so read that back.
17962032
//
1797-
// Every verdict this prints is one of exactly three, and NOT MEASURED is a
1798-
// first-class one rather than a quiet pass. A shard whose test tasks were all
1799-
// cache replays has said nothing about the dataset, and reporting that as OK is
1800-
// the #4690 shape -- a check that read nothing reporting as a check that found
1801-
// nothing wrong.
2033+
// Every verdict this prints is one of exactly four -- OK, WARN, DRIFT and NOT
2034+
// MEASURED (renderDriftVerdict) -- and only DRIFT exits non-zero. WARN is the
2035+
// annotation tier under the red (WARN_MEASURED_OVER_PREDICTED), and NOT
2036+
// MEASURED is a first-class verdict rather than a quiet pass. A shard whose
2037+
// test tasks were all cache replays has said nothing about the dataset, and
2038+
// reporting that as OK is the #4690 shape -- a check that read nothing
2039+
// reporting as a check that found nothing wrong.
18022040
function checkDrift(argv) {
18032041
const inputs = [];
18042042
let label = 'this shard';
@@ -1840,52 +2078,10 @@ function checkDrift(argv) {
18402078
}
18412079
const timings = loadTimings();
18422080
const report = driftReport(merged, timings, MAX_MEASURED_OVER_PREDICTED, observedSlices);
1843-
const skipped =
1844-
report.unpredicted.length === 0
1845-
? ''
1846-
: ` ${report.unpredicted.length} package(s) carry no dataset entry and were excluded ` +
1847-
`(estimated, not predicted): ${report.unpredicted.join(', ')}.`;
1848-
1849-
if (!report.measurable) {
1850-
console.error(
1851-
`shard-timing-drift: NOT MEASURED -- ${label} finished no test task that was both a cache ` +
1852-
'MISS and carried a dataset entry, so this run says nothing about whether ' +
1853-
`scripts/test-shard-timings.json is still true.${skipped}`
1854-
);
1855-
return;
1856-
}
1857-
1858-
const head =
1859-
`${report.measuredTotal.toFixed(1)}s measured vs ${report.predictedTotal.toFixed(1)}s predicted ` +
1860-
`across ${report.rows.length} package(s) = ${report.ratio.toFixed(2)}x ` +
1861-
`(bound ${MAX_MEASURED_OVER_PREDICTED}x)`;
1862-
const worst = report.rows
1863-
.slice(0, 5)
1864-
.map(
1865-
(r) =>
1866-
` ${r.name}: predicted ${r.predicted.toFixed(1)}s, measured ${r.measured.toFixed(1)}s ` +
1867-
`(${r.predicted > 0 ? `${(r.measured / r.predicted).toFixed(2)}x, ` : ''}` +
1868-
`${r.overshoot >= 0 ? '+' : ''}${r.overshoot.toFixed(1)}s)`
1869-
)
1870-
.join('\n');
1871-
1872-
if (!report.drifted) {
1873-
console.error(`shard-timing-drift: OK -- ${label}, ${head}.${skipped}`);
1874-
return;
1875-
}
1876-
console.error(
1877-
`shard-timing-drift: DRIFT -- ${label}, ${head}.${skipped}\n` +
1878-
' Heaviest overshoots:\n' +
1879-
`${worst}\n` +
1880-
' scripts/test-shard-timings.json no longer describes this workspace, so the shard split\n' +
1881-
' is balancing a quantity that is not the runtime. Refresh it -- see\n' +
1882-
' scripts/measure-test-shard-timings.mjs for the two refresh paths -- and ⛔ do NOT\n' +
1883-
' hand-edit the dataset or raise this bound to absorb the gap. Expect the refresh to red\n' +
1884-
" this script's own balance pins if a single suite has outgrown the acceptance bound:\n" +
1885-
' that is those pins working, and the remedy they name is splitting that suite below\n' +
1886-
' package granularity, never a different shard count.'
1887-
);
1888-
process.exit(1);
2081+
const rendered = renderDriftVerdict(report, label);
2082+
for (const line of rendered.out) console.log(line);
2083+
for (const line of rendered.err) console.error(line);
2084+
if (rendered.exitCode !== 0) process.exit(rendered.exitCode);
18892085
}
18902086

18912087
function main() {

0 commit comments

Comments
 (0)