Skip to content

Commit 3993387

Browse files
committed
test(pm): pin the derived non-PR-event half of check-expected-skips
The recogniser admits only its one exact shape and refuses a pull_request / push / merge_group term, a negation and every live mixed if:; the derivation refuses a shared, a listed and an untellable name; the measured #20748 head judges 0 with its raw-named canary and 4 with a genuinely unexpected skip beside it, both in-process and through the CLI on a payload file. Claude-Session: https://claude.ai/code/session_01KTZmMfzVzjNvyaLyQ8mHvg Co-authored-by: Claude <noreply@anthropic.com>
1 parent 6c5772f commit 3993387

1 file changed

Lines changed: 178 additions & 2 deletions

File tree

‎scripts/pm/check-expected-skips.mjs‎

Lines changed: 178 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1083,6 +1083,68 @@ function measuredPayload() {
10831083
return { total_count: MEASURED_18315.length, check_runs: MEASURED_18315.map(([name, conclusion, suite]) => fixtureRun(name, conclusion, suite)) };
10841084
}
10851085

1086+
/**
1087+
* Measured 2026-09-30 on PR #20748's head `a84b73af13` (a `create-objectstack`
1088+
* landing, so `scaffold-e2e.yml` ran on `pull_request`): 42 check-runs, 34
1089+
* success / 8 skipped, full `GET /commits/{head}/check-runs` listing. Names,
1090+
* conclusions and check-suite ids VERBATIM. Its one skip outside the listed
1091+
* roster is `Registry canary: ${{ matrix.template }}` (run 36658032070) —
1092+
* the raw pre-expansion name the derived half exists to admit.
1093+
*/
1094+
const MEASURED_20748 = [
1095+
['Auto Label', 'success', 99276458818],
1096+
['Auto Label', 'skipped', 99276487096],
1097+
['Build Core', 'success', 99276458806],
1098+
['Build Docs', 'skipped', 99276458806],
1099+
['Check Changeset', 'skipped', 99276487096],
1100+
['Check Changeset', 'success', 99276458818],
1101+
['Check Documentation Links', 'success', 99276459150],
1102+
['Check PR Size', 'skipped', 99276487096],
1103+
['Check PR Size', 'success', 99276458818],
1104+
['Close issues referenced in other repositories', 'success', 99283715448],
1105+
['Console Pin Gate', 'skipped', 99276458806],
1106+
['Dogfood Regression Gate', 'success', 99276458806],
1107+
['Dogfood Regression Gate (1/3)', 'success', 99276458806],
1108+
['Dogfood Regression Gate (2/3)', 'success', 99276458806],
1109+
['Dogfood Regression Gate (3/3)', 'success', 99276458806],
1110+
['Dogfood Verify CLI', 'success', 99276458806],
1111+
['filter', 'success', 99276458806],
1112+
['Flag docs affected by code changes', 'success', 99276458808],
1113+
['Governed Surface Queue Guard', 'success', 99280116148],
1114+
['Governed Surface Queue Guard', 'success', 99276459094],
1115+
['Lint & Repo Gates', 'success', 99276458889],
1116+
['No other open PR may claim the same issue', 'success', 99276458886],
1117+
['No other open PR may claim the same single-writer path', 'success', 99276458829],
1118+
['Packed-tarball smoke (opt-in)', 'skipped', 99276487117],
1119+
['Packed-tarball smoke (opt-in)', 'skipped', 99276458917],
1120+
['Part-of PR must not also close its card', 'success', 99276458819],
1121+
['Registry canary: ${{ matrix.template }}', 'skipped', 99276458880],
1122+
['Scaffold with repo dist', 'success', 99276458880],
1123+
['Temporal Conformance (live PG + MySQL)', 'success', 99276458806],
1124+
['Test Core', 'success', 99276458806],
1125+
['Test Core (1/6)', 'success', 99276458806],
1126+
['Test Core (2/6)', 'success', 99276458806],
1127+
['Test Core (3/6)', 'success', 99276458806],
1128+
['Test Core (4/6)', 'success', 99276458806],
1129+
['Test Core (5/6)', 'success', 99276458806],
1130+
['Test Core (6/6)', 'success', 99276458806],
1131+
['The card this PR closes must claim this branch', 'success', 99276458997],
1132+
['Type Check · consumer gates', 'success', 99276458889],
1133+
['Type Check · debt ledger', 'success', 99276458889],
1134+
['Type Check · source gates', 'success', 99276458889],
1135+
['Type Check · workspace', 'success', 99276458889],
1136+
['TypeScript Type Check', 'success', 99276458889],
1137+
];
1138+
1139+
/** The measured #20748 head, plus any extra runs a control case adds. */
1140+
function measured20748Payload(extra = []) {
1141+
const runs = [...MEASURED_20748.map(([name, conclusion, suite]) => fixtureRun(name, conclusion, suite)), ...extra];
1142+
return { total_count: runs.length, check_runs: runs };
1143+
}
1144+
1145+
/** The raw pre-expansion name the measured #20748 head carries, spelled once. */
1146+
const CANARY_RAW_NAME = 'Registry canary: ${{ matrix.template }}';
1147+
10861148
/** Spawn this file's CLI on a payload file, offline; returns { status, stdout, stderr }. */
10871149
function spawnSelf(args) {
10881150
const child = spawnSync(process.execPath, [SELF_PATH, ...args], {
@@ -1185,6 +1247,104 @@ export function selfTest() {
11851247
t('fixture: a null payload → NOT MEASURED', verdictExit(judgeCheckRuns(null)), 3);
11861248
t('fixture: a roster override is honoured (the judge is pure over its roster)', verdictExit(judgeCheckRuns({ total_count: 1, check_runs: [fixtureRun('Build Core', 'skipped')] }, [])), 4);
11871249

1250+
// ---- the derived half: the recogniser, one exact shape ---------------------
1251+
const gate = (cond) => JSON.stringify(nonPrEventGate(cond));
1252+
const BOTH = '["schedule","workflow_dispatch"]';
1253+
t("derive: the measured canary if: reads as schedule | workflow_dispatch", gate("github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'"), BOTH);
1254+
t('derive: …the same, wrapped whole in one ${{ }}', gate("${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}"), BOTH);
1255+
t('derive: …the same, one term per line (a folded block scalar)', gate("github.event_name == 'schedule'\n || github.event_name == 'workflow_dispatch'\n"), BOTH);
1256+
t('derive: a single schedule term is admitted', gate("github.event_name == 'schedule'"), '["schedule"]');
1257+
t('derive: a repeated term de-duplicates', gate("github.event_name == 'schedule' || github.event_name == 'schedule'"), '["schedule"]');
1258+
t('derive: NARROWNESS — a pull_request term beside schedule is NOT admitted', gate("github.event_name == 'schedule' || github.event_name == 'pull_request'"), 'null');
1259+
t('derive: …nor a push term (push fires on a PR branch)', gate("github.event_name == 'schedule' || github.event_name == 'push'"), 'null');
1260+
t('derive: …nor merge_group (the queue build is a verdict, not a skip)', gate("github.event_name == 'merge_group'"), 'null');
1261+
t("derive: …nor a negation (scaffold-local's live `!= 'schedule'` runs on every PR)", gate("github.event_name != 'schedule'"), 'null');
1262+
t('derive: …nor an inputs conjunct (release.yml version-pr, live)', gate("github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.refresh_version_pr)"), 'null');
1263+
t(
1264+
'derive: …nor success() and inputs beside event terms (merged-branch-reaper.yml reap, live)',
1265+
gate("success() && github.event_name != 'pull_request' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false))"),
1266+
'null',
1267+
);
1268+
t('derive: …nor a needs output', gate("github.event_name == 'schedule' || needs.filter.outputs.core != 'false'"), 'null');
1269+
t('derive: …nor a matrix value', gate("github.event_name == 'schedule' || matrix.os == 'linux'"), 'null');
1270+
t('derive: …nor a label read', gate("github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')"), 'null');
1271+
t('derive: …nor a parenthesised term (the allow-list names one bare comparison)', gate("(github.event_name == 'schedule')"), 'null');
1272+
t("derive: …nor a case-folded event (GitHub's == folds case; the recogniser refuses rather than folds)", gate("github.event_name == 'Schedule'"), 'null');
1273+
t('derive: …nor a double-quoted literal (not a GitHub string)', gate('github.event_name == "schedule"'), 'null');
1274+
t('derive: …nor two ${{ }} blocks joined outside an expression', gate("${{ github.event_name == 'schedule' }} || ${{ github.event_name == 'workflow_dispatch' }}"), 'null');
1275+
t('derive: …nor an empty or absent if:', gate('') === 'null' && gate(undefined) === 'null' && gate(true) === 'null', true);
1276+
1277+
t('derive: a matrix name is kept verbatim, raw', skippedCheckRunName('registry-canary', { name: CANARY_RAW_NAME, strategy: { matrix: { template: ['blank'] } } }).name, CANARY_RAW_NAME);
1278+
t('derive: a job with no name: is named by its key', skippedCheckRunName('nightly', { if: "github.event_name == 'schedule'" }).name, 'nightly');
1279+
t('derive: a name holding a non-matrix expression is refused (its skipped spelling is unmeasured)', 'refused' in skippedCheckRunName('x', { name: 'Nightly ${{ inputs.target }}' }), true);
1280+
t('derive: a matrix reference on a job with no strategy.matrix is refused', 'refused' in skippedCheckRunName('x', { name: 'N: ${{ matrix.os }}' }), true);
1281+
t('derive: a non-string name: is refused', 'refused' in skippedCheckRunName('x', { name: 42 }), true);
1282+
1283+
// A synthetic tree: every admission and every refusal the derivation claims.
1284+
const synthetic = {
1285+
'a.yml': {
1286+
jobs: {
1287+
canary: { name: 'Nightly: ${{ matrix.os }}', if: "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'", strategy: { matrix: { os: ['linux'] } } },
1288+
mixed: { name: 'Mixed', if: "github.event_name == 'schedule' || github.event_name == 'pull_request'" },
1289+
plain: { name: 'Plain', if: "github.event_name == 'pull_request'" },
1290+
},
1291+
},
1292+
'b.yml': {
1293+
jobs: {
1294+
weekly: { name: 'Shared', if: "github.event_name == 'schedule'" },
1295+
always: { name: 'Shared' },
1296+
listed: { name: 'Build Core', if: "github.event_name == 'schedule'" },
1297+
opaque: { name: 'Nightly ${{ inputs.target }}', if: "github.event_name == 'workflow_dispatch'" },
1298+
},
1299+
},
1300+
};
1301+
const derivedSyn = deriveNonPrEventSkips(['a.yml', 'b.yml', 'c.yml'], (f) => synthetic[f] ?? null);
1302+
const refusedAt = (job) => derivedSyn.refused.find((r) => r.job === job)?.reason ?? '';
1303+
t('derive: the synthetic tree admits exactly the schedule/dispatch-only matrix job', JSON.stringify(derivedSyn.rows.map((r) => `${r.workflow} › ${r.job} › ${r.name}`)), JSON.stringify(['a.yml › canary › Nightly: ${{ matrix.os }}']));
1304+
t('derive: …as a derived row with the non-pr-event gate and its events', derivedSyn.rows[0]?.derived === true && derivedSyn.rows[0]?.gate.kind === 'non-pr-event' && JSON.stringify(derivedSyn.rows[0]?.gate.events) === BOTH, true);
1305+
t('derive: …whose reason names the raw template', derivedSyn.rows[0]?.reason.includes('raw `name:` template'), true);
1306+
t('derive: the mixed pull_request + schedule job is neither admitted nor refused (it is simply not this shape)', derivedSyn.rows.some((r) => r.job === 'mixed') || derivedSyn.refused.some((r) => r.job === 'mixed'), false);
1307+
t('derive: a name two jobs carry is refused (a skip of the other would read expected)', refusedAt('weekly').includes('2 jobs in the tree carry'), true);
1308+
t("derive: a name the listed roster carries is refused (derive, don't list)", refusedAt('listed').includes("derive, don't list"), true);
1309+
t('derive: a name that cannot be told is refused, with the reason', refusedAt('opaque').includes('inputs.target'), true);
1310+
t('derive: an unreadable workflow is refused by file', derivedSyn.refused.some((r) => r.workflow === 'c.yml' && r.job === null), true);
1311+
t('derive: …and the scan counts every file handed to it', derivedSyn.scanned, 3);
1312+
const synRoster = [...EXPECTED_SKIPS, ...derivedSyn.rows];
1313+
t('derive: the admitted raw-named skip judges exit 0', verdictExit(judgeCheckRuns({ total_count: 1, check_runs: [fixtureRun('Nightly: ${{ matrix.os }}', 'skipped')] }, synRoster)), 0);
1314+
t('derive: NARROWNESS — the mixed pull_request + schedule job skipped still judges exit 4', verdictExit(judgeCheckRuns({ total_count: 1, check_runs: [fixtureRun('Mixed', 'skipped')] }, synRoster)), 4);
1315+
t('derive: …and so does the refused shared name', verdictExit(judgeCheckRuns({ total_count: 1, check_runs: [fixtureRun('Shared', 'skipped')] }, synRoster)), 4);
1316+
1317+
// ---- the derived half on this checkout's workflows ---------------------------
1318+
const liveFiles = listWorkflowFiles(ROOT);
1319+
const live = expectedSkipRoster(ROOT);
1320+
const liveKeys = live.derivation.rows.map((r) => `${r.workflow} › ${r.job}`);
1321+
t('live derive: the workflow listing is not vacuous', liveFiles.length > 0 && live.derivation.scanned === liveFiles.length, true);
1322+
t(`live derive: nothing refused on this tree (${live.derivation.refused.map((r) => `${r.workflow} › ${r.job}: ${r.reason}`).join(' | ') || 'none'})`, live.derivation.refused.length, 0);
1323+
t('live derive: scaffold-e2e.yml › registry-canary is derived under its raw name', live.derivation.rows.find((r) => r.workflow === 'scaffold-e2e.yml' && r.job === 'registry-canary')?.name, CANARY_RAW_NAME);
1324+
t(
1325+
"live derive: the measured refusals stay out (release.yml version-pr, merged-branch-reaper.yml reap, scaffold-e2e.yml's PR-running scaffold-local)",
1326+
['release.yml › version-pr', 'merged-branch-reaper.yml › reap', 'scaffold-e2e.yml › scaffold-local'].some((k) => liveKeys.includes(k)),
1327+
false,
1328+
);
1329+
t('live derive: the listed and derived halves share no name', live.derivation.rows.some((r) => EXPECTED_SKIPS.some((l) => l.name === r.name)), false);
1330+
t('live derive: the joined roster is frozen and duplicate-free', Object.isFrozen(live.roster) && new Set(live.roster.map((r) => r.name)).size === live.roster.length, true);
1331+
1332+
// ---- the measured #20748 head: the card's pins -------------------------------
1333+
const m20748 = judgeCheckRuns(measured20748Payload(), live.roster);
1334+
t('measured #20748: the raw-named skipped canary judges exit 0 against the joined roster', verdictExit(m20748), 0);
1335+
t('measured #20748: 8 skipped, every one expected', m20748.expected.reduce((n, e) => n + e.count, 0) === 8 && m20748.unexpected.length === 0, true);
1336+
t('measured #20748: …the canary read as a derived expected skip', m20748.expected.find((e) => e.name === CANARY_RAW_NAME)?.derived, true);
1337+
t('measured #20748: …and the report tags it derived', renderReport(m20748).some((l) => l.includes(CANARY_RAW_NAME) && l.includes('; derived')), true);
1338+
t('measured #20748: the LISTED half alone still answers exit 4 on it (the derivation is what moved it)', judgeCheckRuns(measured20748Payload()).unexpected[0]?.name, CANARY_RAW_NAME);
1339+
const m20748Control = judgeCheckRuns(measured20748Payload([fixtureRun('TypeScript Type Check', 'skipped', 99276458889)]), live.roster);
1340+
t('measured #20748 CONTROL: a genuinely unexpected skip beside the canary still judges exit 4', verdictExit(m20748Control), 4);
1341+
t('measured #20748 CONTROL: …naming only it', JSON.stringify(m20748Control.unexpected.map((u) => u.name)), '["TypeScript Type Check"]');
1342+
t(
1343+
"measured #20748 CONTROL: the canary's PR-running sibling skipped still judges exit 4",
1344+
verdictExit(judgeCheckRuns(measured20748Payload([fixtureRun('Scaffold with repo dist', 'skipped', 99276458880)]), live.roster)),
1345+
4,
1346+
);
1347+
11881348
// ---- read classification ---------------------------------------------------
11891349
t('read: a network throw is unreachable', classifyRead({ networkError: 'ECONNREFUSED' }).kind, 'unreachable');
11901350
t('read: 422 is the garbage-sha answer', classifyRead({ status: 422 }).kind, 'no-such-commit');
@@ -1254,6 +1414,18 @@ export function selfTest() {
12541414
t('cli: no mode → usage on stderr, exit 2', usage.status === 2 && usage.stderr.includes('usage:'), true);
12551415
const roster = spawnSelf(['--roster']);
12561416
t('cli: --roster prints every row without a network', roster.status === 0 && EXPECTED_SKIPS.every((r) => roster.stdout.includes(r.name)), true);
1417+
t('cli: …the derived rows too, tagged derived', live.derivation.rows.length > 0 && live.derivation.rows.every((r) => roster.stdout.includes(`${r.name} [${r.workflow} › ${r.job}; gate: non-pr-event`)), true);
1418+
1419+
const canaryFile = join(dir, 'canary.json');
1420+
writeFileSync(canaryFile, JSON.stringify({ head_sha: 'a84b73af13', ...measured20748Payload() }));
1421+
const canary = spawnSelf(['--check-runs-json', canaryFile]);
1422+
t('cli: the measured #20748 head on disk (raw-named skipped canary) → exit 0', canary.status, 0);
1423+
t('cli: …with the canary on stdout as a derived expected skip', canary.stdout.includes(`${CANARY_RAW_NAME} [scaffold-e2e.yml › registry-canary; derived]`), true);
1424+
const canaryControlFile = join(dir, 'canary-control.json');
1425+
writeFileSync(canaryControlFile, JSON.stringify(measured20748Payload([fixtureRun('Lint & Repo Gates', 'skipped', 99276458889)])));
1426+
const canaryControl = spawnSelf(['--check-runs-json', canaryControlFile]);
1427+
t('cli: CONTROL — the same head plus a genuinely unexpected skip → exit 4', canaryControl.status, 4);
1428+
t('cli: …naming it under the ⛔ heading', canaryControl.stdout.includes('⛔ unexpected skips (1)') && canaryControl.stdout.includes('Lint & Repo Gates'), true);
12571429
} finally {
12581430
rmSync(dir, { recursive: true, force: true });
12591431
}
@@ -1286,8 +1458,12 @@ export function selfTest() {
12861458
'declared gate kinds — and its truth on this checkout\'s workflows, with the audit driven red on a deleted, renamed, un-gated and ' +
12871459
"re-gated job, a lost || 'true' widening and an unreadable workflow; the judge on the measured 39-run head and on fixtures for an " +
12881460
'expected skip, an unexpected skip named as a filter miss, a same-suite failure read as a dependency skip, a raw matrix template, ' +
1289-
'other conclusions, a pending run and an empty head; read classification for 422 / 404 / 401 / 403 / 5xx / network; argv; the real ' +
1290-
'CLI on payload files for 0 / 4 / 3 and --json; and the structural no-write-path, single-sourced transport and SKILL.md pointer pins).',
1461+
'other conclusions, a pending run and an empty head; the derived half — the non-PR-event recogniser admitting only its one shape and ' +
1462+
'refusing a pull_request / push / merge_group term, a negation and every live mixed if:, the raw-name rule, and the derivation ' +
1463+
'refusing a shared, a listed and an untellable name — on a synthetic tree and on this checkout\'s workflows; the measured #20748 ' +
1464+
'head judged 0 with its raw-named canary and 4 with a genuinely unexpected skip beside it; read classification for 422 / 404 / 401 / ' +
1465+
'403 / 5xx / network; argv; the real CLI on payload files for 0 / 4 / 3 and --json; and the structural no-write-path, ' +
1466+
'single-sourced transport and SKILL.md pointer pins).',
12911467
);
12921468
return { code: EXIT_OK, verdict: SELF_TEST_VERDICT };
12931469
}

0 commit comments

Comments
 (0)