|
82 | 82 | // written (#16173). Every Test Core shard already passes `--summarize`, so the |
83 | 83 | // run it just finished has written the measured truth to `.turbo/runs/`; this |
84 | 84 | // mode reads that back, compares it to what this script PREDICTED for the same |
85 | | -// packages, and reds past MAX_MEASURED_OVER_PREDICTED. Without it the dataset |
| 85 | +// packages, raises a `::warning::` past WARN_MEASURED_OVER_PREDICTED, and reds |
| 86 | +// past MAX_MEASURED_OVER_PREDICTED. Without it the dataset |
86 | 87 | // rots silently in one direction and the only instrument that notices is a |
87 | 88 | // shard killed by the job timeout -- which is a shard that produced NO reading |
88 | 89 | // while the rollup read green. |
@@ -173,6 +174,32 @@ export const MAX_SHARD_OVER_MEAN = 1.3; |
173 | 174 | // number it would be raised past is a measurement of the dataset being wrong. |
174 | 175 | export const MAX_MEASURED_OVER_PREDICTED = 1.5; |
175 | 176 |
|
| 177 | +// The WARNING tier under that red (#16465): past this factor `--check-drift` |
| 178 | +// raises a `::warning::` annotation on the shard and the step stays green. |
| 179 | +// One quantity, two thresholds -- the same measured/predicted ratio over the |
| 180 | +// same executed windows, so the warning is the red's early half and never a |
| 181 | +// second rule. A shard past MAX_MEASURED_OVER_PREDICTED gets the red alone. |
| 182 | +// |
| 183 | +// ⚠ It is NOT MAX_SHARD_OVER_MEAN, though both read 1.3 today. That one bounds |
| 184 | +// the predicted bins against EACH OTHER (max/mean of one split); this one bounds |
| 185 | +// one shard's measurement against ITS OWN prediction. Neither is derived from |
| 186 | +// the other, so moving one never moves the other -- which is why this is its |
| 187 | +// own constant rather than a reference to that one. |
| 188 | +// |
| 189 | +// Why 1.3, and why a warning rather than a red: |
| 190 | +// |
| 191 | +// - it sits ABOVE the healthy population the red was measured against (the |
| 192 | +// five healthy shards of run 34013842594 topped out at 1.18x), so a green |
| 193 | +// dataset does not raise it; |
| 194 | +// - it sits AT the split's own balance tolerance, which is the point the |
| 195 | +// docblock above names as where drift stops being something the |
| 196 | +// partitioner absorbs -- past it the dataset is due a refresh, not yet |
| 197 | +// wrong enough to block anyone; |
| 198 | +// - it is a warning by ruling: one red rule per signal, and that rule is the |
| 199 | +// 1.5x above. The self-test pins the band between the two as non-empty, so |
| 200 | +// the warning cannot be made unreachable by moving either end. |
| 201 | +export const WARN_MEASURED_OVER_PREDICTED = 1.3; |
| 202 | + |
176 | 203 | // ── SHARDING ONE PACKAGE BELOW PACKAGE GRANULARITY (#16173) ──────────────── |
177 | 204 | // |
178 | 205 | // The floor argument at the top of this file is not a caveat, it is a wall: a |
@@ -792,14 +819,110 @@ export function driftReport( |
792 | 819 | // the same way: NOT MEASURED is not a pass, and it is not a failure either. |
793 | 820 | const measurable = rows.length > 0 && predictedTotal > 0; |
794 | 821 | const ratio = measurable ? measuredTotal / predictedTotal : null; |
| 822 | + const drifted = measurable && ratio > factor; |
795 | 823 | return { |
796 | 824 | rows, |
797 | 825 | unpredicted, |
798 | 826 | predictedTotal, |
799 | 827 | measuredTotal, |
800 | 828 | ratio, |
801 | 829 | measurable, |
802 | | - drifted: measurable && ratio > factor, |
| 830 | + drifted, |
| 831 | + // The warning BAND, exclusive of the red: a drifted shard is reported by |
| 832 | + // the red alone, so one shard never carries two verdicts. Strict `>` like |
| 833 | + // the red, so a reading exactly at WARN_MEASURED_OVER_PREDICTED is green. |
| 834 | + warned: measurable && !drifted && ratio > WARN_MEASURED_OVER_PREDICTED, |
| 835 | + }; |
| 836 | +} |
| 837 | + |
| 838 | +// The four verdicts `--check-drift` can print, rendered without printing them, |
| 839 | +// so the self-test can read what a runner will receive. `out` is stdout, where |
| 840 | +// the runner parses workflow commands; the annotation is the warning tier's |
| 841 | +// ONLY effect, so it is pinned here rather than trusted to the caller. |
| 842 | +// |
| 843 | +// A workflow-command message is one line: `%`, CR and LF are escaped the way |
| 844 | +// the runner un-escapes them, so a package name or label can never end the |
| 845 | +// annotation early or start a second command. |
| 846 | +export function escapeWorkflowCommandMessage(text) { |
| 847 | + return String(text).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); |
| 848 | +} |
| 849 | + |
| 850 | +export function renderDriftVerdict(report, label) { |
| 851 | + const skipped = |
| 852 | + report.unpredicted.length === 0 |
| 853 | + ? '' |
| 854 | + : ` ${report.unpredicted.length} package(s) carry no dataset entry and were excluded ` + |
| 855 | + `(estimated, not predicted): ${report.unpredicted.join(', ')}.`; |
| 856 | + |
| 857 | + if (!report.measurable) { |
| 858 | + return { |
| 859 | + verdict: 'NOT MEASURED', |
| 860 | + exitCode: 0, |
| 861 | + out: [], |
| 862 | + err: [ |
| 863 | + `shard-timing-drift: NOT MEASURED -- ${label} finished no test task that was both a cache ` + |
| 864 | + 'MISS and carried a dataset entry, so this run says nothing about whether ' + |
| 865 | + `scripts/test-shard-timings.json is still true.${skipped}`, |
| 866 | + ], |
| 867 | + }; |
| 868 | + } |
| 869 | + |
| 870 | + const head = |
| 871 | + `${report.measuredTotal.toFixed(1)}s measured vs ${report.predictedTotal.toFixed(1)}s predicted ` + |
| 872 | + `across ${report.rows.length} package(s) = ${report.ratio.toFixed(2)}x ` + |
| 873 | + `(warning past ${WARN_MEASURED_OVER_PREDICTED}x, red past ${MAX_MEASURED_OVER_PREDICTED}x)`; |
| 874 | + const ratioOf = (r) => (r.predicted > 0 ? `${(r.measured / r.predicted).toFixed(2)}x, ` : ''); |
| 875 | + const worst = report.rows |
| 876 | + .slice(0, 5) |
| 877 | + .map( |
| 878 | + (r) => |
| 879 | + ` ${r.name}: predicted ${r.predicted.toFixed(1)}s, measured ${r.measured.toFixed(1)}s ` + |
| 880 | + `(${ratioOf(r)}${r.overshoot >= 0 ? '+' : ''}${r.overshoot.toFixed(1)}s)` |
| 881 | + ) |
| 882 | + .join('\n'); |
| 883 | + |
| 884 | + if (report.drifted) { |
| 885 | + return { |
| 886 | + verdict: 'DRIFT', |
| 887 | + exitCode: 1, |
| 888 | + out: [], |
| 889 | + err: [ |
| 890 | + `shard-timing-drift: DRIFT -- ${label}, ${head}.${skipped}\n` + |
| 891 | + ' Heaviest overshoots:\n' + |
| 892 | + `${worst}\n` + |
| 893 | + ' scripts/test-shard-timings.json no longer describes this workspace, so the shard split\n' + |
| 894 | + ' is balancing a quantity that is not the runtime. Refresh it -- see\n' + |
| 895 | + ' scripts/measure-test-shard-timings.mjs for the two refresh paths -- and ⛔ do NOT\n' + |
| 896 | + ' hand-edit the dataset or raise this bound to absorb the gap. Expect the refresh to red\n' + |
| 897 | + " this script's own balance pins if a single suite has outgrown the acceptance bound:\n" + |
| 898 | + ' that is those pins working, and the remedy they name is splitting that suite below\n' + |
| 899 | + ' package granularity, never a different shard count.', |
| 900 | + ], |
| 901 | + }; |
| 902 | + } |
| 903 | + |
| 904 | + if (report.warned) { |
| 905 | + const top = report.rows[0]; |
| 906 | + return { |
| 907 | + verdict: 'WARN', |
| 908 | + exitCode: 0, |
| 909 | + out: [ |
| 910 | + `::warning title=Test Core shard timing drift::${escapeWorkflowCommandMessage( |
| 911 | + `${label}: ${head}. Heaviest overshoot: ${top.name}, predicted ${top.predicted.toFixed(1)}s, ` + |
| 912 | + `measured ${top.measured.toFixed(1)}s. scripts/test-shard-timings.json is drifting from the ` + |
| 913 | + 'runtime; refresh it before it reaches the red (see scripts/measure-test-shard-timings.mjs). ' + |
| 914 | + 'Not a failure: this step stays green until the red bound.' |
| 915 | + )}`, |
| 916 | + ], |
| 917 | + err: [`shard-timing-drift: WARN -- ${label}, ${head}.${skipped}\n Heaviest overshoots:\n${worst}`], |
| 918 | + }; |
| 919 | + } |
| 920 | + |
| 921 | + return { |
| 922 | + verdict: 'OK', |
| 923 | + exitCode: 0, |
| 924 | + out: [], |
| 925 | + err: [`shard-timing-drift: OK -- ${label}, ${head}.${skipped}`], |
803 | 926 | }; |
804 | 927 | } |
805 | 928 |
|
@@ -887,13 +1010,14 @@ const SELF_TEST_BATTERIES = Object.freeze({ |
887 | 1010 | // heaviest package do not depend on what the live map slices. |
888 | 1011 | 'the balancing pins (#10472)': 25, |
889 | 1012 | 'predicted-vs-measured drift (#16173)': 9, |
| 1013 | + 'drift warning tier under the red (#16465)': 12, |
890 | 1014 | 'file-level slice items (#16173)': 20, |
891 | 1015 | 'file-level slices reach vitest through OS_TEST_SHARD (#19278)': 11, |
892 | 1016 | }); |
893 | 1017 |
|
894 | 1018 | // DELETING an entry silences that battery's floor exactly as effectively as |
895 | 1019 | // zeroing it, so the roster's own size is pinned too. |
896 | | -const SELF_TEST_BATTERY_FLOOR = 10; |
| 1020 | +const SELF_TEST_BATTERY_FLOOR = 11; |
897 | 1021 |
|
898 | 1022 | // The key an assertion is filed under when no battery is open. It is not a |
899 | 1023 | // declared battery, so it reds by the same set difference rather than silently |
@@ -1434,6 +1558,118 @@ function selfTest() { |
1434 | 1558 | } |
1435 | 1559 | }); |
1436 | 1560 |
|
| 1561 | + // -- THE WARNING TIER UNDER THE RED (#16465) ---------------------------- |
| 1562 | + // |
| 1563 | + // Same ratio, same executed windows, a lower threshold that annotates and |
| 1564 | + // never fails. Fixtures are scaled to the 100s `slow` entry above, so a |
| 1565 | + // measured N seconds reads as N/100x. What these hold: the band is |
| 1566 | + // (WARN, RED], it is exclusive of the red, it is non-empty, its effect is an |
| 1567 | + // annotation a runner will actually parse, and a replay never enters it. |
| 1568 | + battery('drift warning tier under the red (#16465)'); |
| 1569 | + const tierOf = (seconds) => driftReport(measuredMap({ slow: seconds }), driftTimings); |
| 1570 | + |
| 1571 | + check(() => { |
| 1572 | + const r = tierOf(131); |
| 1573 | + if (!r.warned || r.drifted) { |
| 1574 | + throw new Error(`drift warning: a 1.31x reading came back warned=${r.warned} drifted=${r.drifted}`); |
| 1575 | + } |
| 1576 | + }); |
| 1577 | + // The lower edge: strict `>`, like the red, so exactly at the factor is green. |
| 1578 | + check(() => { |
| 1579 | + const r = tierOf(WARN_MEASURED_OVER_PREDICTED * 100); |
| 1580 | + if (r.warned || r.drifted) { |
| 1581 | + throw new Error('drift warning: a reading exactly AT the warning factor was flagged'); |
| 1582 | + } |
| 1583 | + }); |
| 1584 | + // The upper edge belongs to the warning: AT the red is not yet a red. |
| 1585 | + check(() => { |
| 1586 | + const r = tierOf(MAX_MEASURED_OVER_PREDICTED * 100); |
| 1587 | + if (!r.warned || r.drifted) { |
| 1588 | + throw new Error(`drift warning: a reading exactly AT the red came back warned=${r.warned} drifted=${r.drifted}`); |
| 1589 | + } |
| 1590 | + }); |
| 1591 | + // Past the red the red alone speaks: one shard, one verdict. |
| 1592 | + check(() => { |
| 1593 | + const r = tierOf(151); |
| 1594 | + if (r.warned || !r.drifted) { |
| 1595 | + throw new Error(`drift warning: a 1.51x reading came back warned=${r.warned} drifted=${r.drifted}`); |
| 1596 | + } |
| 1597 | + }); |
| 1598 | + // The healthy population the red was measured against topped out at 1.18x. |
| 1599 | + check(() => { |
| 1600 | + const r = tierOf(118); |
| 1601 | + if (r.warned || r.drifted) throw new Error('drift warning: a healthy 1.18x reading was flagged'); |
| 1602 | + }); |
| 1603 | + // A replay-only shard is NOT MEASURED, never a warning in either direction. |
| 1604 | + check(() => { |
| 1605 | + const r = driftReport(replayed.samples, driftTimings); |
| 1606 | + if (r.warned || r.drifted) throw new Error('drift warning: a summary of replays and failures was flagged'); |
| 1607 | + }); |
| 1608 | + // The band is non-empty, or the warning is a declared tier nothing can reach. |
| 1609 | + check(() => { |
| 1610 | + if (!(WARN_MEASURED_OVER_PREDICTED < MAX_MEASURED_OVER_PREDICTED)) { |
| 1611 | + throw new Error( |
| 1612 | + `drift warning: WARN_MEASURED_OVER_PREDICTED (${WARN_MEASURED_OVER_PREDICTED}) is not below ` + |
| 1613 | + `MAX_MEASURED_OVER_PREDICTED (${MAX_MEASURED_OVER_PREDICTED}) -- the warning band is empty` |
| 1614 | + ); |
| 1615 | + } |
| 1616 | + }); |
| 1617 | + // Not below the split's own tolerance: under it, the warning would fire on |
| 1618 | + // drift the partitioner is built to absorb, and a warning on healthy input is |
| 1619 | + // one everybody learns to scroll past. (A relation, not a reuse: see the |
| 1620 | + // constant's docblock.) |
| 1621 | + check(() => { |
| 1622 | + if (WARN_MEASURED_OVER_PREDICTED < MAX_SHARD_OVER_MEAN) { |
| 1623 | + throw new Error( |
| 1624 | + `drift warning: WARN_MEASURED_OVER_PREDICTED (${WARN_MEASURED_OVER_PREDICTED}) sits below ` + |
| 1625 | + `MAX_SHARD_OVER_MEAN (${MAX_SHARD_OVER_MEAN}), inside the drift the split absorbs` |
| 1626 | + ); |
| 1627 | + } |
| 1628 | + }); |
| 1629 | + |
| 1630 | + // What the runner receives. The verdict is the whole effect of this tier, so |
| 1631 | + // it is read as rendered text, never printed here (a self-test line opening |
| 1632 | + // with a workflow command would mint a real annotation on every lint run). |
| 1633 | + const label = 'Test Core (9/9)'; |
| 1634 | + check(() => { |
| 1635 | + const v = renderDriftVerdict(tierOf(131), label); |
| 1636 | + const commands = v.out.filter((line) => line.startsWith('::')); |
| 1637 | + if (v.verdict !== 'WARN' || v.exitCode !== 0 || commands.length !== 1) { |
| 1638 | + throw new Error( |
| 1639 | + `drift warning: rendered ${v.verdict} exit ${v.exitCode} with ${commands.length} workflow command(s)` |
| 1640 | + ); |
| 1641 | + } |
| 1642 | + if (!commands[0].startsWith('::warning ') || !commands[0].includes(label) || commands[0].includes('\n')) { |
| 1643 | + throw new Error(`drift warning: the annotation is not one ::warning line naming the shard: ${commands[0]}`); |
| 1644 | + } |
| 1645 | + }); |
| 1646 | + check(() => { |
| 1647 | + const v = renderDriftVerdict(tierOf(151), label); |
| 1648 | + if (v.verdict !== 'DRIFT' || v.exitCode !== 1 || v.out.some((line) => line.startsWith('::'))) { |
| 1649 | + throw new Error(`drift warning: a red rendered ${v.verdict} exit ${v.exitCode}, ${v.out.length} stdout line(s)`); |
| 1650 | + } |
| 1651 | + }); |
| 1652 | + check(() => { |
| 1653 | + for (const [seconds, expected] of [[118, 'OK'], [WARN_MEASURED_OVER_PREDICTED * 100, 'OK']]) { |
| 1654 | + const v = renderDriftVerdict(tierOf(seconds), label); |
| 1655 | + if (v.verdict !== expected || v.exitCode !== 0 || v.out.length !== 0) { |
| 1656 | + throw new Error(`drift warning: ${seconds / 100}x rendered ${v.verdict} with ${v.out.length} stdout line(s)`); |
| 1657 | + } |
| 1658 | + } |
| 1659 | + const nm = renderDriftVerdict(driftReport(replayed.samples, driftTimings), label); |
| 1660 | + if (nm.verdict !== 'NOT MEASURED' || nm.exitCode !== 0 || nm.out.length !== 0) { |
| 1661 | + throw new Error(`drift warning: a replay-only shard rendered ${nm.verdict}`); |
| 1662 | + } |
| 1663 | + }); |
| 1664 | + // One line per command: a newline smuggled in through a label or package |
| 1665 | + // name would end the annotation and open whatever followed it as a command. |
| 1666 | + check(() => { |
| 1667 | + const escaped = escapeWorkflowCommandMessage('50% done\r\n::error::x'); |
| 1668 | + if (escaped !== '50%25 done%0D%0A::error::x') { |
| 1669 | + throw new Error(`drift warning: workflow-command escaping produced ${JSON.stringify(escaped)}`); |
| 1670 | + } |
| 1671 | + }); |
| 1672 | + |
1437 | 1673 | // -- FILE-LEVEL SLICE ITEMS (#16173) ------------------------------------ |
1438 | 1674 | // |
1439 | 1675 | // The grammar, its refusals, and the two joins that read it. The balancing |
@@ -1794,11 +2030,13 @@ function selfTest() { |
1794 | 2030 |
|
1795 | 2031 | // `--check-drift`: the shard just measured itself, so read that back. |
1796 | 2032 | // |
1797 | | -// Every verdict this prints is one of exactly three, and NOT MEASURED is a |
1798 | | -// first-class one rather than a quiet pass. A shard whose test tasks were all |
1799 | | -// cache replays has said nothing about the dataset, and reporting that as OK is |
1800 | | -// the #4690 shape -- a check that read nothing reporting as a check that found |
1801 | | -// nothing wrong. |
| 2033 | +// Every verdict this prints is one of exactly four -- OK, WARN, DRIFT and NOT |
| 2034 | +// MEASURED (renderDriftVerdict) -- and only DRIFT exits non-zero. WARN is the |
| 2035 | +// annotation tier under the red (WARN_MEASURED_OVER_PREDICTED), and NOT |
| 2036 | +// MEASURED is a first-class verdict rather than a quiet pass. A shard whose |
| 2037 | +// test tasks were all cache replays has said nothing about the dataset, and |
| 2038 | +// reporting that as OK is the #4690 shape -- a check that read nothing |
| 2039 | +// reporting as a check that found nothing wrong. |
1802 | 2040 | function checkDrift(argv) { |
1803 | 2041 | const inputs = []; |
1804 | 2042 | let label = 'this shard'; |
@@ -1840,52 +2078,10 @@ function checkDrift(argv) { |
1840 | 2078 | } |
1841 | 2079 | const timings = loadTimings(); |
1842 | 2080 | const report = driftReport(merged, timings, MAX_MEASURED_OVER_PREDICTED, observedSlices); |
1843 | | - const skipped = |
1844 | | - report.unpredicted.length === 0 |
1845 | | - ? '' |
1846 | | - : ` ${report.unpredicted.length} package(s) carry no dataset entry and were excluded ` + |
1847 | | - `(estimated, not predicted): ${report.unpredicted.join(', ')}.`; |
1848 | | - |
1849 | | - if (!report.measurable) { |
1850 | | - console.error( |
1851 | | - `shard-timing-drift: NOT MEASURED -- ${label} finished no test task that was both a cache ` + |
1852 | | - 'MISS and carried a dataset entry, so this run says nothing about whether ' + |
1853 | | - `scripts/test-shard-timings.json is still true.${skipped}` |
1854 | | - ); |
1855 | | - return; |
1856 | | - } |
1857 | | - |
1858 | | - const head = |
1859 | | - `${report.measuredTotal.toFixed(1)}s measured vs ${report.predictedTotal.toFixed(1)}s predicted ` + |
1860 | | - `across ${report.rows.length} package(s) = ${report.ratio.toFixed(2)}x ` + |
1861 | | - `(bound ${MAX_MEASURED_OVER_PREDICTED}x)`; |
1862 | | - const worst = report.rows |
1863 | | - .slice(0, 5) |
1864 | | - .map( |
1865 | | - (r) => |
1866 | | - ` ${r.name}: predicted ${r.predicted.toFixed(1)}s, measured ${r.measured.toFixed(1)}s ` + |
1867 | | - `(${r.predicted > 0 ? `${(r.measured / r.predicted).toFixed(2)}x, ` : ''}` + |
1868 | | - `${r.overshoot >= 0 ? '+' : ''}${r.overshoot.toFixed(1)}s)` |
1869 | | - ) |
1870 | | - .join('\n'); |
1871 | | - |
1872 | | - if (!report.drifted) { |
1873 | | - console.error(`shard-timing-drift: OK -- ${label}, ${head}.${skipped}`); |
1874 | | - return; |
1875 | | - } |
1876 | | - console.error( |
1877 | | - `shard-timing-drift: DRIFT -- ${label}, ${head}.${skipped}\n` + |
1878 | | - ' Heaviest overshoots:\n' + |
1879 | | - `${worst}\n` + |
1880 | | - ' scripts/test-shard-timings.json no longer describes this workspace, so the shard split\n' + |
1881 | | - ' is balancing a quantity that is not the runtime. Refresh it -- see\n' + |
1882 | | - ' scripts/measure-test-shard-timings.mjs for the two refresh paths -- and ⛔ do NOT\n' + |
1883 | | - ' hand-edit the dataset or raise this bound to absorb the gap. Expect the refresh to red\n' + |
1884 | | - " this script's own balance pins if a single suite has outgrown the acceptance bound:\n" + |
1885 | | - ' that is those pins working, and the remedy they name is splitting that suite below\n' + |
1886 | | - ' package granularity, never a different shard count.' |
1887 | | - ); |
1888 | | - process.exit(1); |
| 2081 | + const rendered = renderDriftVerdict(report, label); |
| 2082 | + for (const line of rendered.out) console.log(line); |
| 2083 | + for (const line of rendered.err) console.error(line); |
| 2084 | + if (rendered.exitCode !== 0) process.exit(rendered.exitCode); |
1889 | 2085 | } |
1890 | 2086 |
|
1891 | 2087 | function main() { |
|
0 commit comments