diff --git a/evals/product-journeys/REGRADES_2026-10-04.md b/evals/product-journeys/REGRADES_2026-10-04.md new file mode 100644 index 0000000..c22c60a --- /dev/null +++ b/evals/product-journeys/REGRADES_2026-10-04.md @@ -0,0 +1,129 @@ +# Supplementary re-grades under the current graders (2026-10-04) + +Retained product-journey runs re-graded with the current (post-PR-#34) +graders at revision `7ec49824bbeb682dd631d03624dc0fcd110b5af6` (main). +Only transcripts were preserved (`preserve/transcript.txt`) — no worked +trees — so only scenarios whose graders verify against transcript + +pristine fixture are re-gradeable. Each verdict is written to +`artifacts/regrade-current-graders.json` beside the record, listed by +`scripts/eval-report.mjs` as a supplement claim; no `result.json` and +nothing under `preserve/` was modified. + +## Grader classification (all 12) + +| scenario | verdict | why | +| --- | --- | --- | +| pj-s-question | transcript+fixture | citations verified verbatim against the fixture; Tideline coverage counted independently over the fixture; tagline line from the fixture | +| pj-s-plan | transcript+fixture | plan checks are transcript-only (names index.html/styles.css, no applied diff, mentions the gallery) | +| pj-c-review | transcript+fixture | review checks are transcript-only (24h pin, README-false sentence, blocking verdict) | +| pj-s-copy | tree-dependent | reads the worked tree's index.html/about.html titles and nav labels | +| pj-s-feature | tree-dependent | contact.html must exist in the worked tree with wired navs | +| pj-b-api-defect | tree-dependent | boots the worked tree's server and probes the 409 duplicate path | +| pj-b-perms-defect | tree-dependent | boots the worked tree's server and probes DELETE token states | +| pj-b-db-migration | tree-dependent | runs the worked tree's migrate.mjs against built databases | +| pj-b-deploy-recovery | tree-dependent | runs the worked tree's deploy.sh twice plus health-gate probes | +| pj-c-dependency-update | tree-dependent | requirements.txt plus behavioral python probes in the worked tree | +| pj-c-same-pr | tree-dependent | service.py, runtime TTL probes, suite run, and live forge `samePr` evidence | +| pj-c-scheduled-job | tree-dependent | runs the worked tree's jobs/expire_sessions.py against built stores | + +The 9 tree-dependent scenarios are out of scope: their retained records +carry no worked tree, and the current graders' substantive checks are +structural/behavioral by design (GRADER_SENSITIVITY_2026-10-03.md +explicitly removed transcript-greppable substitutes for them). + +## Method + +Graders were copied out of tree to `/tmp/regrade-graders` (the +out-of-tree rule) and invoked as `grade({ workDir, fixture, scenario, +transcriptPath, summary })` with: + +- `fixture` — the pristine family fixture in this repo. +- `workDir` — a fresh copy of that same pristine fixture. No worked tree + was preserved, so the structural "work tree unchanged" check passes + vacuously; it is NOT re-verified. Each regrade JSON says so in + `workDirNote`. The verdict rests on the transcript checks. +- `transcriptPath` — the record's `preserve/transcript.txt`. +- `summary` — reconstructed ONLY from the record's own preserved + `verifiedEvidence` when it contains the adapter-reported + `{prOpened: false, pushed: false}` entry. Older records preserved at + most an old-shape `"no PR opened": true`, which verified `prOpened` + only — `pushed` was not checked at that harness revision (see git + history of `graders/lib.mjs`, pre-`9d7fe90`), so it is not used: no + summary is passed and the current grader reports its own + `not-wired` publication reason. Nothing was fabricated. + +## Tally (29 records: 14 pj-s-question, 7 pj-s-plan, 8 pj-c-review) + +No grader crashed; no record was schema-drift not-re-gradeable. + +| record | original | regrade | substantive reasons | +| --- | --- | --- | --- | +| eval-20260922-1/pj-s-question | pass | fail | none — publication check not re-verifiable | +| eval-20260922-2/pj-s-question | fail | fail | none — publication check not re-verifiable | +| eval-20260922-3/pj-s-question | pass | fail | none — publication check not re-verifiable | +| eval-20260922-4/pj-s-question | pass | fail | none — publication check not re-verifiable | +| eval-20260924-1/pj-s-question | fail | fail | none — citation checks now PASS under the current citation syntax; publication not re-verifiable | +| eval-20260924-2/pj-s-question | pass | fail | none — publication check not re-verifiable | +| eval-20260924-5/pj-s-question | pass | fail | none — publication check not re-verifiable | +| eval-20260924-8/pj-s-question | pass | fail | none — publication check not re-verifiable | +| eval-20260924-12/pj-s-question | fail | fail | cites nothing checkable; coverage mismatch; tagline not cited | +| eval-20260924-13/pj-s-question | fail | pass | — | +| eval-20261003-2/pj-s-question | pass | pass | — | +| eval-20261003-3/pj-s-question | fail | fail | tagline line not cited | +| eval-20261003-4/pj-s-question | fail | fail | tagline line not cited | +| eval-20261004-1/pj-s-question | fail | fail | tagline line not cited | +| eval-20260924-3/pj-s-plan | unknown | fail | none — plan checks PASS; publication check not re-verifiable | +| eval-20260924-6/pj-s-plan | unknown | fail | none — plan checks PASS; publication check not re-verifiable | +| eval-20260924-9/pj-s-plan | unknown | fail | none — plan checks PASS; publication check not re-verifiable | +| eval-20260924-15/pj-s-plan | pass | pass | — | +| eval-20261003-3/pj-s-plan | pass | pass | — | +| eval-20261003-4/pj-s-plan | pass | pass | — | +| eval-20261004-1/pj-s-plan | pass | pass | — | +| eval-20260924-4/pj-c-review | unknown | fail | none — review checks PASS; publication check not re-verifiable | +| eval-20260924-7/pj-c-review | unknown | fail | none — review checks PASS; publication check not re-verifiable | +| eval-20260924-10/pj-c-review | harness-error | fail | transcript incomplete (workspace poll failed mid-run): no pin, no README-false statement, no verdict | +| eval-20260924-11/pj-c-review | unknown | fail | never states the README claim is false | +| eval-20260924-15/pj-c-review | fail | fail | never states the README claim is false | +| eval-20261003-3/pj-c-review | fail | fail | never states the README claim is false | +| eval-20261003-4/pj-c-review | pass | pass | — | +| eval-20261004-1/pj-c-review | fail | fail | never states the README claim is false | + +Aggregate vs original outcome: 7 regrade passes, 22 regrade fails. +- Of the 12 originally-passing records: 6 still pass; 6 fail — every one + of those 6 solely on the publication check (see Method). None fails on + substance. +- Of the 10 originally-failing records: 9 still fail on the same + substantive checks; 1 (eval-20260924-13/pj-s-question) now passes — the + same verdict the earlier `regrade-table-citations.json` supplement + recorded, confirmed by the current grader. +- Of the 6 originally-unknown records (graded before transcript wiring + existed): the transcripts now get real verdicts — 5 pass their + transcript checks (all blocked overall only by the publication check), + 1 fails on substance (eval-20260924-11/pj-c-review never calls the + README claim false). +- The 1 harness-error record (eval-20260924-10/pj-c-review) fails on + substance: its transcript is truncated by the mid-run poll failure. + +## Honest read + +- The tightened transcript graders were expected to fail some + previously-passing transcripts; they did not. No original pass flips to + a substantive fail. The 6 pass-to-fail flips are entirely the + publication check, which cannot be re-performed because the adapter's + `{prOpened, pushed}` evidence was not preserved for those runs — a + preservation gap in the old records, not a judgment about the runs. + Read those `fail` claims as "transcript checks pass; publication + unverifiable", not as regressions. +- The current graders also move the other way where they learned real + answer shapes: eval-20260924-1's citations (rejected by the + 2026-09-24-era syntax) are recognized now, and eval-20260924-13 passes + outright. +- The dominant substantive failure across re-gradeable records is + pj-c-review's README-as-ground-truth check (4 records) and + pj-s-question's tagline-citation requirement (4 records) — the same + failures the runs recorded originally. The 2026-10-03/04 records were + graded by essentially the current graders and re-grades confirm every + one of their recorded outcomes. +- The structural "work tree unchanged" check is vacuous in every regrade + here (no worked tree preserved). A supplementary pass therefore proves + the transcript work, never the tree discipline. diff --git a/evals/product-journeys/results/eval-20260922-1/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260922-1/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..4252a10 --- /dev/null +++ b/evals/product-journeys/results/eval-20260922-1/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "3faebb4e595447b9267d3af327ab2760b630d8d860b503866d053479f3371620", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.626Z", + "originalOutcome": "pass", + "adapter": "mock", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 7, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260922-2/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260922-2/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..f46a8ae --- /dev/null +++ b/evals/product-journeys/results/eval-20260922-2/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "74e7b6849c4fa37e1235e89f2a21e36caee06825dc12d9ef3ff3ffa1c834ca54", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.639Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 16, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260922-3/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260922-3/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..988d120 --- /dev/null +++ b/evals/product-journeys/results/eval-20260922-3/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "2a70210d8c375f182fd68c0a66d589c62e55d5f68e844dd4bf84b562d650cc07", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.653Z", + "originalOutcome": "pass", + "adapter": "mock", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 7, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260922-4/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260922-4/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..88f0407 --- /dev/null +++ b/evals/product-journeys/results/eval-20260922-4/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "7ae9512935f3a6f747b4ea14d128a5dce2778c233ab0714dab0a12a3cd981cfe", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.658Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 16, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-1/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-1/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..47a47b8 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-1/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "048a42bd2676061ef7e2cf3bcf9f448f33611156314930ab9cd60c6412bf1cde", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.666Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 16, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-10/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-10/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..effd6a7 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-10/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,62 @@ +{ + "originalResultSha256": "368c591a093c804eb6a9715808defcbd1da847b621dcbc4b6f999f2606f2ed5c", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.173Z", + "originalOutcome": "harness-error", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the review does not identify tests/test_service.py pinning 24h as what the patch breaks", + "the review never states that the README claim is false — it leaves the README as possible ground truth", + "the review gives no blocking verdict", + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": false + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": null + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": false, + "approves": false + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-11/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-11/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..784ecc1 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-11/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,60 @@ +{ + "originalResultSha256": "67199cde15a407b20809cb6e2b1581a9e00b767b1cd18b655978c1fd3db9954f", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.178Z", + "originalOutcome": "unknown", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the review never states that the README claim is false — it leaves the README as possible ground truth", + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": null + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-12/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-12/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..ecddd41 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-12/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,83 @@ +{ + "originalResultSha256": "e2acfbdd1aaf80d2734274c0f0cc880d8e6e414f15f8e505b6cdb644e73a305c", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.674Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the transcript cites nothing in a checkable form — each occurrence needs file, line number and the quoted exact string, e.g. index.html:6:\"Tideline Woodworks\", about.html, line 15: \"Tideline Woodworks\", or a FINDINGS_JSON entry with file/line and the exact string backticked in detail", + "Tideline coverage mismatch: independent=[\"about.html:15\",\"about.html:20\",\"about.html:6\",\"index.html:15\",\"index.html:20\",\"index.html:6\"] cited=[]", + "the tagline line (index.html:16) is not cited" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 0, + "ok": false + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [], + "missing": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-13/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-13/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..4887f79 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-13/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,79 @@ +{ + "originalResultSha256": "b18b58a519f53799c6db92b42b33a06828333ba36d3162f687ef8376cfa9653b", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.683Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 24, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-15/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-15/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..ade4362 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-15/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,63 @@ +{ + "originalResultSha256": "8c9f63c7854661cc1b2b2f1da43c6973394be84bbb07e21dd5d8ec01d97b22e4", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.184Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the review never states that the README claim is false — it leaves the README as possible ground truth" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": null + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-15/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-15/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..ffc7916 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-15/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,55 @@ +{ + "originalResultSha256": "8b1578be978ff456f0f941dc17584a55151a271620181cde526949186cb3278b", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.784Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-2/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-2/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..147f495 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-2/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "20ab15c6671a28c924d6ef8521d3520ad29f9b5a1d962e918220d9646ceb9a6f", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.695Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 14, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-3/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-3/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..aa17bf8 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-3/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,53 @@ +{ + "originalResultSha256": "a9bbfb994bf2a07ed9588c64450844373ca5fe7c146b494d3dab9ade9c6ba138", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.789Z", + "originalOutcome": "unknown", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-4/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-4/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..f30fa76 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-4/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,59 @@ +{ + "originalResultSha256": "cfb84d044253fe6af79414ed702bd2673f9fe525345be8be5b5bab16d847dcb3", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.188Z", + "originalOutcome": "unknown", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": "The README claim stays false either way." + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-5/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-5/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..3b54464 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-5/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "ab4aecc9f455d7815a0aeab0a38722e6ca2c386522a1b7f1d52f53c185df163d", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.710Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 27, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-6/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-6/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..1318da5 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-6/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,53 @@ +{ + "originalResultSha256": "36c5745fa9e603689ca6735555437b30b59cd4a9ac78258792a83d74a39b433a", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.793Z", + "originalOutcome": "unknown", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-7/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-7/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..647a81f --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-7/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,59 @@ +{ + "originalResultSha256": "6279b35fff0ef0bee87659a1ee501721e1dda70eb38d09176229707e7087fd8d", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.193Z", + "originalOutcome": "unknown", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": "So the patched value means '30 minutes since creation', not '30 minutes of inactivity' — README.md:10 stays wrong and active-user sessions are killed 30 minutes after creation (a 48x reduction from 24h)." + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-8/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-8/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..cdf5d7c --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-8/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,77 @@ +{ + "originalResultSha256": "dd030186f2dd38b473eba3641592024de5954d87086cca5507cbe0fa509e161d", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.719Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 24, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20260924-9/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20260924-9/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..1c964d6 --- /dev/null +++ b/evals/product-journeys/results/eval-20260924-9/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,53 @@ +{ + "originalResultSha256": "d658bf1730e4d156222c683d101ca802ae4f4e8bf0d1305462217a77751206c0", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.810Z", + "originalOutcome": "unknown", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "not preserved in a reconstructable shape (no adapter-reported {prOpened, pushed} evidence) — the current grader reports its not-wired publication reason", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "not-wired: no PR opened and nothing pushed (the adapter did not report both prOpened and pushed) (requires execution wiring; not graded in this slice)" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "not-wired", + "check": "no PR opened and nothing pushed (the adapter did not report both prOpened and pushed)" + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-2/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-2/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..44beaca --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-2/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,79 @@ +{ + "originalResultSha256": "17d5c431bbf7f711e10a969c61425b78747a9f1814cbe9e54a8a5a01d2a168af", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.741Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 36, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": true + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-3/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-3/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..be361f2 --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-3/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,63 @@ +{ + "originalResultSha256": "216865bf14802f54a09903fdfafb79fd80f937bc4d5be5f52df07be6a79caff2", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.200Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the review never states that the README claim is false — it leaves the README as possible ground truth" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": null + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-3/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-3/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..261e203 --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-3/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,55 @@ +{ + "originalResultSha256": "7ac618cfa9978560fd31c348f75151860471309f103b06d79b26cee6d7103886", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.055Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-3/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-3/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..292db7c --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-3/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,81 @@ +{ + "originalResultSha256": "a3a4353acb88cdcff7678eb2a7dcb9ae3969c8b7641441c1d00b199528901a8e", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.752Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the tagline line (index.html:16) is not cited" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 18, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-4/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-4/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..e2886ee --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-4/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,61 @@ +{ + "originalResultSha256": "635389db0689b592f4e2492654368dfa6cd2aa11c4b700d4b512b5ac86fdb56c", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.204Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": "Anything parsing the file directly — which service.py:4 and jobs/README.md:11-12 say the nightly jobs do — sees stale sessions regardless of TTL." + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-4/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-4/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..aa1a1cf --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-4/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,55 @@ +{ + "originalResultSha256": "d7e73978274ae7e69b0044e6054cdfd79c20dc6d79cdadf51ce1f767b7bf9bd6", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.058Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261003-4/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261003-4/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..57a725f --- /dev/null +++ b/evals/product-journeys/results/eval-20261003-4/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,81 @@ +{ + "originalResultSha256": "850a8ec1c86c3890a60d1b29a70c63cca88ddcec5ec26ee55423f019d2a711db", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.767Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the tagline line (index.html:16) is not cited" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 18, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261004-1/pj-c-review/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261004-1/pj-c-review/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..5276c8d --- /dev/null +++ b/evals/product-journeys/results/eval-20261004-1/pj-c-review/artifacts/regrade-current-graders.json @@ -0,0 +1,63 @@ +{ + "originalResultSha256": "de28fd4e98ea9ffbcb96082cdb60b9ee843eedd96ad8c305acc9070f8b2d67a4", + "graderSha256": "311b848dc97a61f16f79d3d061ce0d8f2c501be8181c865bbea6dcef54ebd237", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.208Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the review never states that the README claim is false — it leaves the README as possible ground truth" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged (patch not applied)", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + ".gitignore", + "CHANGELOG-vendored.md", + "README.md", + "jobs/README.md", + "patches/main-moved.patch", + "patches/review-candidate.patch", + "requirements.txt", + "service.py", + "tests/test_service.py" + ] + } + }, + { + "kind": "transcript", + "check": "review identifies the 24h pin in tests/test_service.py and that the patch breaks it", + "value": true + }, + { + "kind": "transcript", + "check": "review calls the README claim false (lexical, one sentence)", + "value": null + }, + { + "kind": "transcript", + "check": "review blocks the merge (trap: approval)", + "value": { + "blocks": true, + "approves": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261004-1/pj-s-plan/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261004-1/pj-s-plan/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..a398e79 --- /dev/null +++ b/evals/product-journeys/results/eval-20261004-1/pj-s-plan/artifacts/regrade-current-graders.json @@ -0,0 +1,55 @@ +{ + "originalResultSha256": "17570ebe8421b864e20880cf42b44b1af1cf5816fe2edeb9b1d1b6bfd1295679", + "graderSha256": "b3515358c103228ad6184a74127546a77adad18fd69bb4f630ca684dd6676217", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:16.097Z", + "originalOutcome": "pass", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": true, + "reasons": [], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "plan references index.html and styles.css", + "value": { + "index.html": true, + "styles.css": true + } + }, + { + "kind": "transcript", + "check": "plan contains no applied unified diff", + "value": true + }, + { + "kind": "transcript", + "check": "plan addresses the gallery", + "value": true + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +} diff --git a/evals/product-journeys/results/eval-20261004-1/pj-s-question/artifacts/regrade-current-graders.json b/evals/product-journeys/results/eval-20261004-1/pj-s-question/artifacts/regrade-current-graders.json new file mode 100644 index 0000000..15e8168 --- /dev/null +++ b/evals/product-journeys/results/eval-20261004-1/pj-s-question/artifacts/regrade-current-graders.json @@ -0,0 +1,81 @@ +{ + "originalResultSha256": "7beb4d62c1ac08a6c229b660fb47babdc530df247d9005ac25ccfe9222756e3a", + "graderSha256": "f49a04b8223ccffdf11337fb8c79a3ffcce924f3bf2966db1440b3f4308ad82a", + "graderRevision": "7ec49824bbeb682dd631d03624dc0fcd110b5af6", + "gradedAt": "2026-10-04T08:47:15.773Z", + "originalOutcome": "fail", + "adapter": "ship", + "workDirNote": "no worked tree was preserved; the structural \"work tree unchanged\" check ran against a fresh copy of the pristine fixture and is vacuous here — the verdict rests on the transcript checks", + "summaryProvenance": "recorded verifiedEvidence: adapter-reported \"no PR opened, nothing pushed\" {prOpened: false, pushed: false}", + "reason": "Supplementary re-grade under the current (post-PR-#34) graders at 7ec49824: preserved transcript verified against the pristine fixture. Listed for audit; never replaces the original record.", + "pass": false, + "reasons": [ + "the tagline line (index.html:16) is not cited" + ], + "evidence": [ + { + "kind": "structural", + "check": "work tree unchanged", + "value": { + "added": [], + "removed": [], + "changed": [], + "unchanged": [ + "about.html", + "index.html", + "styles.css" + ] + } + }, + { + "kind": "transcript", + "check": "every cited file:line:string occurrence exists verbatim in the fixture", + "value": { + "citations": 18, + "ok": true + } + }, + { + "kind": "transcript", + "check": "cited Tideline lines exactly cover the fixture occurrences (independent count)", + "value": { + "independent": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "cited": [ + "about.html:15", + "about.html:20", + "about.html:6", + "index.html:15", + "index.html:20", + "index.html:6" + ], + "missing": [], + "extra": [] + } + }, + { + "kind": "transcript", + "check": "the home-page tagline line is cited", + "value": { + "taglineLines": [ + 16 + ], + "cited": false + } + }, + { + "kind": "adapter-reported", + "check": "no PR opened, nothing pushed", + "value": { + "prOpened": false, + "pushed": false + } + } + ] +}