diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 058b76e..2dfb57e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,6 +125,38 @@ jobs: - name: Tier 5 -- tool module tests run: .venv/bin/python -m pytest modules/tool-work-tracker/tests -v + # Tier 7 -- operator-surface conformance, Tier B (real browser). + # contracts/operator-surface.v1.md Freeze 2 requires the Tier-B kit to + # "run as its own CI tier", and this is that tier: a separate step, so a + # browser failure is legible at a glance as a RENDERED-SURFACE failure + # and its cost/flake profile never rides inside the fast tiers above. + # Chromium is installed here rather than in the shared setup step for + # the same reason -- nothing above this line needs a browser. + # + # The kit is deselected everywhere else by pyproject's `-m "not tier_b"` + # addopts; `-m tier_b` on this command line is what selects it. + # + # `if: always()` is deliberate: this tier's whole job is to report what + # the rendered surface actually does, and a red tier above (a unit-test + # regression, say) must not hide that report. + - name: Tier 7 -- operator-surface conformance (Tier B, browser) + if: always() + run: | + .venv/bin/playwright install --with-deps chromium + .venv/bin/python -m pytest -m tier_b tests/conformance/operator_surface/browser -v + + # The Tier-B artifacts (pixel counts, computed contrast ratios, bounding + # boxes, post-swap DOM snapshots, screenshots) -- Freeze 3's "artifacts + # the orchestrator re-checks itself". Uploaded on every run, pass or + # fail: a failing run's numbers are the ones worth reading. + - name: Tier 7 artifacts -- operator-surface Tier-B evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: operator-surface-tier-b-artifacts + path: tests/conformance/operator_surface/browser/_artifacts/ + if-no-files-found: warn + - name: Dolt server log (always, for debugging) if: always() run: cat /tmp/dolt-server.log || true diff --git a/.gitignore b/.gitignore index 84391ca..c9f78a5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ DONE.json # goal-batch orchestration scratch (per-run, never repo content) .amplifier/goals/ + +# Tier-B conformance artifacts -- a RUN's evidence (pixel counts, contrast +# numbers, post-swap DOM snapshots, screenshots), never repo content. A +# committed artifact is a number nobody re-measured. +tests/conformance/operator_surface/browser/_artifacts/ diff --git a/Makefile b/Makefile index 18546f2..5576242 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ -.PHONY: venv test test-unit test-integration test-cli test-ledger ledger-mutate test-conformance-a test-module check lint types doctor clean +.PHONY: venv playwright-install test test-unit test-integration test-cli test-ledger \ + test-conformance-a test-conformance-b ledger-mutate test-module check lint types \ + doctor clean PYTHON ?= python3.12 VENV := .venv @@ -61,6 +63,29 @@ ledger-mutate: test-conformance-a: $(PYTEST) tests/conformance/operator_surface -v +## Chromium for the Tier-B browser conformance kit. Separate from `venv` +## on purpose: `playwright` (the library) is a normal dev dependency and +## installs with everything else, but the BROWSER is a ~150MB download into +## `~/.cache/ms-playwright` that a contributor who never runs the browser +## tier should not pay for. `--with-deps` pulls the system libraries chromium +## needs on a bare CI runner (it is a no-op where they are already present). +playwright-install: + $(VENV)/bin/playwright install --with-deps chromium + +## Tier B -- operator-surface conformance in a REAL BROWSER +## (contracts/operator-surface.v1.md Freeze 2). Its own target, and +## deliberately NOT part of `test` below: it costs a chromium launch and a +## live app boot per scenario, and Freeze 2 requires it to run as its own +## tier so a browser tier's cost and flake profile never ride inside the fast +## ones. `-m tier_b` overrides the `-m "not tier_b"` in pyproject's addopts +## (pytest inserts addopts before the command line, and the last -m wins), so +## every other invocation deselects this tier automatically. +## +## Depends on `playwright-install`: a Tier-B run without chromium is not a +## pass and not a skip, it is an error naming the missing browser. +test-conformance-b: playwright-install + $(PYTEST) -m tier_b tests/conformance/operator_surface/browser -v + ## Tier 5 -- tool module: modules/tool-work-tracker's own suite, the only ## place the post-reclaim custody behaviour of the AGENT SEAM (work_claim / ## work_declare / work_resolve / work_release) is asserted mechanically. diff --git a/ledger/checks/mutation_harness.py b/ledger/checks/mutation_harness.py index f9f9eca..7cc08ac 100644 --- a/ledger/checks/mutation_harness.py +++ b/ledger/checks/mutation_harness.py @@ -81,6 +81,13 @@ CLI = probes.CLI +#: The Tier-B kit's committed run summary. Every OSV1 row that re-reads a +#: browser-produced number reads it from here, so every mutation for those +#: rows is a mutation OF THIS FILE -- the counterfactual is "the browser +#: measured something else", stated the only way an in-process harness can +#: state it. +TIER_B_SUMMARY = _support.REPO_ROOT / op_probes.TIER_B_SUMMARY + #: Every module that owns `test_row_*` probes. Readers are patched in ALL of #: them, so a mutation is seen the same way whichever family's probe runs -- #: the alternative (patching only the family under test) would let a probe that @@ -478,20 +485,15 @@ def _mo006_an_unregistered_computed_site_appears(w: World) -> None: def _mo007_a_get_handler_reaches_a_write(w: World) -> None: """REGRESSION: the L1 GET view acquires a mutating adapter call.""" + # Re-anchored 2026-09-04: this used to hang off the L1 view's `limit=0` + # call, which OSV1-015's fix (work_item_pipeline-8vv) removed. The anchor + # is incidental to what this mutation proves -- it needs ANY line inside + # the L1 GET handler to hang a write off -- so it moved to the bounded + # call that replaced it rather than the row being re-derived. w.replace( WEBBROWSE, - "all_matching = bd.list(status=status_filter, include_resolved=True, limit=0)", - "all_matching = bd.list(status=status_filter, include_resolved=True, limit=0)\n" - " bd.update(name)", - ) - - -def _mo008_swap_restores_the_pause_flag(w: World) -> None: - """FIXED: `restoreState` starts touching the pause control.""" - w.replace( - WEBTHEME, - "window.scrollTo(0, state.scrollY);", - "window.scrollTo(0, state.scrollY);\n window.__wtRefreshPaused;", + "query_capped = len(fetched) > _L1_ITEM_QUERY_LIMIT", + "query_capped = len(fetched) > _L1_ITEM_QUERY_LIMIT\n bd.update(name)", ) @@ -515,11 +517,6 @@ def _mo009b_the_attr_light_block_regresses(w: World) -> None: ) -def _mo010_a_browser_driver_appears(w: World) -> None: - """FIXED: something in the repo starts driving a browser.""" - w.append(PYPROJECT, '\n# test-only: "playwright>=1.40"\n') - - def _mo011_a_second_motion_block_appears(w: World) -> None: """REGRESSION: reduced motion becomes per-widget opt-in.""" w.append( @@ -555,19 +552,26 @@ def _mo014_a_chart_library_is_declared(w: World) -> None: w.replace(PYPROJECT, '"itsdangerous>=2.1",', '"itsdangerous>=2.1",\n "plotly>=5.20",') -def _mo015_the_unbounded_query_is_bounded(w: World) -> None: - """FIXED: the L1 view stops asking for everything.""" - w.replace(WEBBROWSE, "include_resolved=True, limit=0)", "include_resolved=True, limit=200)") +def _mo015_the_bounded_query_goes_unbounded_again(w: World) -> None: + """REGRESSION: the L1 view goes back to asking for everything -- literally + the shape this row closed (`limit=0` is bd's own "unlimited"). + """ + w.replace(WEBBROWSE, "limit=_L1_ITEM_QUERY_LIMIT + 1,", "limit=0,") -def _mo016_the_theme_starts_persisting(w: World) -> None: - """FIXED: a chosen theme survives a refresh.""" - w.replace( - WEBAPP, - "document.documentElement.setAttribute('data-theme', t);", - "document.documentElement.setAttribute('data-theme', t);\n" - " localStorage.setItem('wt-theme', t);", - ) +def _mo016_the_theme_stops_persisting(w: World) -> None: + """REGRESSION: the chosen theme goes back to dying on refresh -- the setter + applies it and remembers nothing, exactly as before dg3. + """ + w.replace(WEBAPP, " try{ localStorage.setItem(WT_THEME_KEY, t); }catch(e){}\n", "") + + +def _mo016_the_first_paint_resolver_moves_out_of_head(w: World) -> None: + """REGRESSION, the OTHER half: the choice is still stored, but nothing + reads it before the body paints -- persistence written and never applied + is not persistence. + """ + w.replace(WEBTHEME, ' f""\n', "") def _mo017_a_second_push_call_site_appears(w: World) -> None: @@ -577,10 +581,6 @@ def _mo017_a_second_push_call_site_appears(w: World) -> None: w.append(WEBAPP, "\ndef _ledger_mutation():\n WP.fire_reclaim_alarm(1, 2, 3)\n") -def _mo_tier_b_kit_appears(w: World) -> None: - w.touch(TIER_B_KIT) - - def _mo022_the_swap_mechanism_changes(w: World) -> None: """The substantive half of Conformance 3's pin: the whole-body innerHTML replacement IS the mechanism the bad fixture describes. @@ -616,9 +616,233 @@ def _mo027_ci_stops_running_the_kit(w: World) -> None: ) -def _mo029_an_artifact_directory_appears(w: World) -> None: - """The substantive half of Freeze 3: artifacts start being emitted.""" - w.touch("tests/conformance/operator_surface/browser/artifacts/.keep") +def _mo003_the_calm_page_stops_painting_blocked(w: World) -> None: + """FIXED: a calm L1 stops painting `--blocked`. + + The counterfactual a pinning row needs -- the browser measuring the FIXED + behaviour. Both themes measured 97, so both anchors move together. + """ + w.replace( + TIER_B_SUMMARY, + '"calm/L1/dark": {\n "alarm": 0,\n "blocked": 97,', + '"calm/L1/dark": {\n "alarm": 0,\n "blocked": 0,', + ) + + +def _mo008_the_swap_starts_restoring_the_disclosure(w: World) -> None: + """FIXED: an open `
` survives the body-swap on L0.""" + w.replace( + TIER_B_SUMMARY, + '"calm/L0/dark": {\n "details_with_id": 0,\n "live_regions_before": 1,\n' + ' "marked_live_regions_after": 0,\n "open_details_preserved": false,', + '"calm/L0/dark": {\n "details_with_id": 2,\n "live_regions_before": 1,\n' + ' "marked_live_regions_after": 0,\n "open_details_preserved": true,', + ) + + +def _mo008_the_announcement_survives_the_swap(w: World) -> None: + """FIXED: the live region tagged before the swap SURVIVES it on L0. + + Newly measurable since the hero rebuild: before it, L0 rendered no live + region at all and Core 6's announcement half had nothing to preserve. Now + there is exactly one (`role="status"`), the swap destroys it, and a fix + that carried it across would flip this half of the row. + """ + w.replace( + TIER_B_SUMMARY, + '"calm/L0/dark": {\n "details_with_id": 0,\n "live_regions_before": 1,\n' + ' "marked_live_regions_after": 0,', + '"calm/L0/dark": {\n "details_with_id": 0,\n "live_regions_before": 1,\n' + ' "marked_live_regions_after": 1,', + ) + + +def _mo008_the_pause_control_starts_surviving(w: World) -> None: + """FIXED: the pause CONTROL's own state survives the swap on L0. + + Separable from the disclosure half above, and pinned separately, because + the two are different fixes: one needs ids in the markup, the other needs + the re-rendered button to be re-synchronised with `window.__wtRefreshPaused`. + A row that noticed only one of them would absorb the other silently. + """ + w.replace( + TIER_B_SUMMARY, + '"calm/L0/dark": {\n "details_with_id": 0,\n "live_regions_before": 1,\n' + ' "marked_live_regions_after": 0,\n "open_details_preserved": false,\n' + ' "pause_control_preserved": false,', + '"calm/L0/dark": {\n "details_with_id": 0,\n "live_regions_before": 1,\n' + ' "marked_live_regions_after": 0,\n "open_details_preserved": false,\n' + ' "pause_control_preserved": true,', + ) + + +def _mo010_the_target_floor_is_met(w: World) -> None: + """FIXED: every interactive control on L0 reaches 44px.""" + w.replace( + TIER_B_SUMMARY, + '"calm/L0/1280/dark": {\n "client_width": 1280,\n "controls": 34,\n' + ' "controls_below_44px": 26,', + '"calm/L0/1280/dark": {\n "client_width": 1280,\n "controls": 34,\n' + ' "controls_below_44px": 0,', + ) + + +def _mo011_an_animation_runs_under_the_preference(w: World) -> None: + """REGRESSION: the browser measures an animation still RUNNING under + `prefers-reduced-motion: reduce` on one swept render. + + The half this row acquired on 2026-09-05. The static halves -- one + `@media` block, universal selector -- would be entirely unmoved by it, + which is the point: a stylesheet that says the right thing while the page + still animates satisfies a source check and fails an operator. + """ + w.replace( + TIER_B_SUMMARY, + '"calm/L0/430/dark": {\n "client_width": 430,\n "controls": 34,\n' + ' "controls_below_44px": 16,\n "elements_beyond_viewport": 0,\n' + ' "non_text_below_floor": 16,\n "non_text_measured": 77,\n' + ' "overflow_x_style": "clip",\n' + ' "running_animations_under_reduced_motion": 0,', + '"calm/L0/430/dark": {\n "client_width": 430,\n "controls": 34,\n' + ' "controls_below_44px": 16,\n "elements_beyond_viewport": 0,\n' + ' "non_text_below_floor": 16,\n "non_text_measured": 77,\n' + ' "overflow_x_style": "clip",\n' + ' "running_animations_under_reduced_motion": 6,', + ) + + +def _mo020_the_calm_bad_half_stops_discriminating(w: World) -> None: + """REGRESSION: Conformance 1's injected-alarm-chip bad half stops firing. + + A green fixture row whose bad half quietly stopped catching its defect is + the failure Freeze 4 exists to prevent, and it is invisible from the + browser tier's own green -- every good half still passes. + """ + w.replace( + TIER_B_SUMMARY, + '"bad-alarm-chip/L0/dark": {\n "alarm": 10531,', + '"bad-alarm-chip/L0/dark": {\n "alarm": 0,', + ) + + +def _mo021_the_rendered_chips_lose_their_word(w: World) -> None: + """REGRESSION on the half of Conformance 2 that HAS landed: a rendered + status chip stops carrying a word, which the Tier-B arm must notice even + while the row stays red for the Tier-A half.""" + w.replace( + TIER_B_SUMMARY, + '"alarm/L1/dark": {\n "status_elements": 19,\n "wordless": 0', + '"alarm/L1/dark": {\n "status_elements": 19,\n "wordless": 4', + ) + + +def _mo021_the_tier_a_good_half_is_deferred_again(w: World) -> None: + """REGRESSION on the half of Conformance 2 that Tier A owns: the rendered + accessible-name check is deferred behind an `xfail(strict)` again. + + A deferred good half is a check that does not currently pass, and a green + Conformance row whose good half does not pass is a claim. The Tier-B arm + would not notice: every browser measurement still holds. + """ + w.replace( + _support.REPO_ROOT / TIER_A_KIT, + "def test_state_not_colour_only(alarm_dataset, level: str) -> None:", + '@pytest.mark.xfail(strict=True, reason="OSV1-004 regressed")\n' + "def test_state_not_colour_only(alarm_dataset, level: str) -> None:", + ) + + +def _mo030_the_contrast_bad_half_borrows_the_live_token_again(w: World) -> None: + """REGRESSION: Conformance 4's contrast bad half goes back to injecting the + LIVE token pair instead of owning its specimen. + + Exactly how this row went red at the wave-2 integration: the bad half was + written when `var(--ink-quiet)` WAS the recorded 4.27:1 colour, the + contrast lane moved the token, and the fixture followed the fix around + until it named no defect at all. The run summary would not move at the + moment of the change -- which is why this is asserted on the fixture's + source, not only on its numbers. + """ + w.replace( + _support.REPO_ROOT / op_probes.TIER_B_PROBE_LIB, + "'bottom:0;color:#9aa3b2;background:#eef2fb'", + "'bottom:0;color:var(--ink-quiet);background:var(--ground)'", + ) + + +def _mo030_a_conformance_bad_half_stops_biting(w: World) -> None: + """REGRESSION, the other way in: a re-recorded run shows one of the + fourteen arms no longer failing against the defect it names. + + Conformance 4's contrast arm in LIGHT, chosen because it is the arm this + row was red for: a bad half measured at or above the floor is a fixture + that has stopped discriminating, whatever its source still says. + """ + w.replace( + TIER_B_SUMMARY, + '"bad-low-contrast/L0/light": {\n "control_ratio": 13.96,\n' + ' "min_ratio": 2.27', + '"bad-low-contrast/L0/light": {\n "control_ratio": 13.96,\n' + ' "min_ratio": 5.36', + ) + + +def _mo022_the_swap_bad_half_stops_discriminating(w: World) -> None: + """REGRESSION: the naive replacement stops losing the open disclosure, so + Conformance 3's bad half no longer differs from the shipped poller.""" + w.replace( + TIER_B_SUMMARY, + '"bad-naive-replacement/L0/dark": {\n "details_with_id": 0,\n' + ' "live_regions_before": 1,\n "marked_live_regions_after": 0,\n' + ' "open_details_preserved": false,', + '"bad-naive-replacement/L0/dark": {\n "details_with_id": 0,\n' + ' "live_regions_before": 1,\n "marked_live_regions_after": 0,\n' + ' "open_details_preserved": true,', + ) + + +def _mo023_the_contrast_bad_half_loses_its_control(w: World) -> None: + """REGRESSION: the contrast probe starts reporting the CONTROL pair -- a + literal 13.96:1 -- below the floor too. + + That is the shape of a probe that has stopped measuring and started always + saying no, and it would leave every good half in Conformance 4 looking + exactly as it does now. (Before 2026-09-05 the control was the same token + pair measured in dark; it became a literal above-floor pair when the bad + half stopped depending on the token set.) + """ + w.replace( + TIER_B_SUMMARY, + '"bad-low-contrast/L0/dark": {\n "control_ratio": 13.96,', + '"bad-low-contrast/L0/dark": {\n "control_ratio": 2.1,', + ) + + +def _mo028_the_browser_pin_moves_without_a_re_run(w: World) -> None: + """REGRESSION: the manifest's playwright pin moves while the recorded + numbers stay behind. + + The precise hazard Freeze 2's "pinned chromium" exists to prevent: every + contrast ratio and pixel count in the ledger would silently start + describing a browser build nobody ran. + """ + w.replace(PYPROJECT, '"playwright==1.60.0",', '"playwright==1.61.0",') + + +def _mo029_the_kit_stops_reading_its_artifacts_back(w: World) -> None: + """REGRESSION: ONE Tier-B check stops reading its artifact back and asserts + on the value still in its own local variable. + + Freeze 3's whole content, and the reason the probe counts read-backs + against writes rather than looking for the call anywhere: every other + check would still read back, the artifact would still be written, and the + browser tier would still be green. + """ + w.replace( + _support.REPO_ROOT / op_probes.TIER_B_KIT, + " cache[key] = _artifacts.read(path)", + ' cache[key] = {"measurement": measurement} # trust the local value', + ) def _mo031_a_red_core_row_goes_green(w: World) -> None: @@ -628,8 +852,8 @@ def _mo031_a_red_core_row_goes_green(w: World) -> None: """ w.replace( ROWS_PATH, - " disposition: VIOLATION\n work: work_item_pipeline-8vv", - " disposition: CONFORMS\n work: work_item_pipeline-8vv", + " disposition: VIOLATION\n work: work_item_pipeline-c1a", + " disposition: CONFORMS\n work: work_item_pipeline-c1a", ) @@ -795,6 +1019,11 @@ def _mo034_the_changelog_records_a_look(w: World) -> None: "the --amber alias acquires its own literal value (a bespoke fourth status hue)", _mo002_alias_becomes_a_bespoke_hue, ), + Mutation( + "OSV1-003", + "the browser measures a calm L1 painting ZERO --blocked pixels (the fix)", + _mo003_the_calm_page_stops_painting_blocked, + ), Mutation( "OSV1-003", "the retired palette a calm sweep would catch is tokenised away", @@ -829,8 +1058,20 @@ def _mo034_the_changelog_records_a_look(w: World) -> None: ), Mutation( "OSV1-008", - "the body-swap starts restoring the pause flag", - _mo008_swap_restores_the_pause_flag, + "the browser measures an open `
` surviving the swap (the fix)", + _mo008_the_swap_starts_restoring_the_disclosure, + ), + Mutation( + "OSV1-008", + "the browser measures the pause CONTROL surviving the swap (the fix)", + _mo008_the_pause_control_starts_surviving, + ), + Mutation( + "OSV1-008", + "the browser measures L0's live region surviving the swap (the fix, on the " + "half that only became measurable when the hero rebuild gave L0 a " + "`role=status` region to destroy)", + _mo008_the_announcement_survives_the_swap, ), Mutation( "OSV1-009", @@ -842,12 +1083,22 @@ def _mo034_the_changelog_records_a_look(w: World) -> None: '--ink-quiet falls back below the text floor in the :root[data-theme="light"] block', _mo009b_the_attr_light_block_regresses, ), - Mutation("OSV1-010", "a browser driver appears in the repo", _mo010_a_browser_driver_appears), + Mutation( + "OSV1-010", + "the browser measures every interactive control on L0 reaching 44px (the fix)", + _mo010_the_target_floor_is_met, + ), Mutation( "OSV1-011", "reduced motion becomes per-widget opt-in (a second @media block)", _mo011_a_second_motion_block_appears, ), + Mutation( + "OSV1-011", + "the browser measures an animation still RUNNING under the preference on a " + "swept render, while the stylesheet still says the right thing", + _mo011_an_animation_runs_under_the_preference, + ), Mutation( "OSV1-012", "the empty attention queue grows the sentence Core 8 asks for", @@ -861,28 +1112,58 @@ def _mo034_the_changelog_records_a_look(w: World) -> None: ), Mutation( "OSV1-015", - "the L1 view's unbounded query acquires a real bound", - _mo015_the_unbounded_query_is_bounded, + "the L1 view's bounded query goes unbounded again", + _mo015_the_bounded_query_goes_unbounded_again, + ), + Mutation( + "OSV1-016", + "the theme choice stops surviving a refresh", + _mo016_the_theme_stops_persisting, ), Mutation( "OSV1-016", - "the theme choice starts surviving a refresh", - _mo016_the_theme_starts_persisting, + "the first-paint resolver leaves , so the stored theme is never applied", + _mo016_the_first_paint_resolver_moves_out_of_head, ), Mutation( "OSV1-017", "the push channel acquires a second call site", _mo017_a_second_push_call_site_appears, ), - Mutation("OSV1-020", "the Tier-B kit file appears", _mo_tier_b_kit_appears), - Mutation("OSV1-021", "the Tier-B kit file appears", _mo_tier_b_kit_appears), - Mutation("OSV1-022", "the Tier-B kit file appears", _mo_tier_b_kit_appears), + Mutation( + "OSV1-020", + "Conformance 1's injected-alarm-chip bad half stops catching its defect", + _mo020_the_calm_bad_half_stops_discriminating, + ), + # OSV1-021 carried a "the OTHER tier's kit file appears" pin in each lane. + # Both kits landed at the wave-2 integration, so neither pin is true any + # more and the row is green: one REGRESSION mutation per half, so a row + # that spans two tiers cannot be held up by only one of them. + Mutation( + "OSV1-021", + "Conformance 2's Tier-A good half is deferred again behind an xfail", + _mo021_the_tier_a_good_half_is_deferred_again, + ), + Mutation( + "OSV1-021", + "a rendered status chip loses its word (the Tier-B half)", + _mo021_the_rendered_chips_lose_their_word, + ), + Mutation( + "OSV1-022", + "Conformance 3's naive-replacement bad half stops losing the open disclosure", + _mo022_the_swap_bad_half_stops_discriminating, + ), Mutation( "OSV1-022", "the whole-body innerHTML swap (the bad fixture's own premise) changes shape", _mo022_the_swap_mechanism_changes, ), - Mutation("OSV1-023", "the Tier-B kit file appears", _mo_tier_b_kit_appears), + Mutation( + "OSV1-023", + "Conformance 4's contrast bad half loses its dark-mode control", + _mo023_the_contrast_bad_half_loses_its_control, + ), Mutation( "OSV1-023", "a swept breakpoint disappears from the stylesheet", @@ -908,17 +1189,36 @@ def _mo034_the_changelog_records_a_look(w: World) -> None: "CI stops running the Tier-A kit (the 'runs in a gate' half)", _mo027_ci_stops_running_the_kit, ), - Mutation("OSV1-028", "the Tier-B kit file appears", _mo_tier_b_kit_appears), - Mutation("OSV1-029", "the Tier-B kit file appears", _mo_tier_b_kit_appears), + Mutation( + "OSV1-028", + "the manifest's chromium pin moves while the recorded numbers stay behind", + _mo028_the_browser_pin_moves_without_a_re_run, + ), Mutation( "OSV1-029", - "a Tier-B artifact directory appears", - _mo029_an_artifact_directory_appears, + "the Tier-B kit stops reading its own artifacts back before asserting", + _mo029_the_kit_stops_reading_its_artifacts_back, + ), + # OSV1-030 (Freeze 4) went GREEN at the second wave-2 integration pass: + # the contrast bad half was rebuilt to own its specimen and the kit was + # re-run. Both counterfactuals are therefore the KNOWN-WRONG shape it now + # forbids -- the fixture borrowing the live token again, and a recorded + # arm that no longer bites. + Mutation( + "OSV1-030", + "Conformance 4's contrast bad half borrows the LIVE token pair again, so its " + "defect goes back to following the product's own fixes around", + _mo030_the_contrast_bad_half_borrows_the_live_token_again, + ), + Mutation( + "OSV1-030", + "a re-recorded run shows one of the fourteen fixture arms measuring at or " + "above the floor it is supposed to fail", + _mo030_a_conformance_bad_half_stops_biting, ), - Mutation("OSV1-030", "the Tier-B kit file appears", _mo_tier_b_kit_appears), Mutation( "OSV1-031", - "one of the ten red Core-carrying rows flips to CONFORMS", + "one of the five red Core-carrying rows flips to CONFORMS", _mo031_a_red_core_row_goes_green, ), Mutation( diff --git a/ledger/checks/test_operator_rows.py b/ledger/checks/test_operator_rows.py index c8603fa..ad47800 100644 --- a/ledger/checks/test_operator_rows.py +++ b/ledger/checks/test_operator_rows.py @@ -42,9 +42,11 @@ from __future__ import annotations import ast +import json import re from ._support import ( + ADAPTER, CHARTSVG, GROUND_TOKENS, LITERAL, @@ -52,6 +54,7 @@ PINNING_DISPOSITIONS, PYPROJECT, REPO_ROOT, + ROUTE_MODULES, SUPERVISOR, WEBAPP, WEBBROWSE, @@ -60,6 +63,8 @@ WEBTHEME, WEBTRUST, WIDGETS, + _called_names, + _route_methods, collapse, contains, count, @@ -90,6 +95,54 @@ MAKEFILE = REPO_ROOT / "Makefile" CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" +#: The Tier-B kit's COMMITTED run summary -- the bridge that lets this +#: in-process, browserless ledger re-read numbers a real chromium produced +#: (Freeze 3, OSV1-029). Written by +#: `tests/conformance/operator_surface/browser/_artifacts.py`; the per-run +#: artifact directory beside it is gitignored and unreadable from here. +TIER_B_PROBE_LIB = "tests/conformance/operator_surface/browser/_probe.py" +TIER_B_SUMMARY = "tests/conformance/operator_surface/browser/LAST_RUN.json" +TIER_B_SUMMARY_SCHEMA = "operator-surface-tier-b/1" + + +def tier_b_summary() -> dict: + """The Tier-B run summary, parsed, with its envelope checked. + + Fails loud rather than returning `{}`: a probe that quietly read an empty + summary would report "no violation measured" for every Tier-B row at once, + which is precisely the hollow green Freeze 3 exists to prevent. Read + through `read()` so the mutation harness's injected world is what a + mutated run sees. + """ + path = REPO_ROOT / TIER_B_SUMMARY + assert path.exists(), ( + f"{TIER_B_SUMMARY} is missing. Every Tier-B row re-reads its verdict " + f"from it -- run `make test-conformance-b` to regenerate it." + ) + data = json.loads(read(path)) + assert data.get("schema") == TIER_B_SUMMARY_SCHEMA, ( + f"{TIER_B_SUMMARY} declares schema {data.get('schema')!r}, expected " + f"{TIER_B_SUMMARY_SCHEMA!r} -- the artifact shape moved under the ledger." + ) + assert data.get("checks"), f"{TIER_B_SUMMARY} carries no checks" + return data + + +def tier_b(check: str, scenario: str) -> dict: + """One headline out of the run summary, or a loud failure naming what is there.""" + checks = tier_b_summary()["checks"] + assert check in checks, ( + f"the Tier-B run summary has no `{check}` check (it has " + f"{sorted(checks)}). Either the kit stopped emitting it or the run was " + f"partial -- re-run `make test-conformance-b`." + ) + scenarios = checks[check] + assert scenario in scenarios, ( + f"the Tier-B run summary has no `{check}` / `{scenario}` scenario (it has " + f"{sorted(scenarios)})." + ) + return scenarios[scenario] + def _exists(rel: str) -> bool: return (REPO_ROOT / rel).exists() @@ -326,14 +379,28 @@ def test_row_osv1_002() -> None: def test_row_osv1_003() -> None: - """Core 2 GAP pin (calm pixels): no pixel sweep exists, and the retired - palette that a sweep would catch is still in the tree. + """Core 2 VIOLATION pin, RE-READ from the browser run's own numbers. + + Not "a file appeared" and not the browser tier's pass/fail: the calm sweep + wrote pixel counts, and this reads them back. L0 is clean in both themes; + L1 paints `--blocked` on a page with nothing blocked, and THAT is the + pinned violation. """ - assert not _exists(TIER_B_KIT), ( - f"OSV1-003 (Core 2): {TIER_B_KIT} now exists. Re-derive this row from the " - f"sweep's EMITTED ARTIFACT (Freeze 3 / ruling 6) -- never from the fact that " - f"a file appeared (work_item_pipeline-qgo)." - ) + for theme in ("dark", "light"): + clean = tier_b("calm.zero_alarm_pixels", f"calm/L0/{theme}") + assert clean["alarm"] == 0 and clean["blocked"] == 0, ( + f"OSV1-003 (Core 2): a calm L0 in {theme} now paints " + f"{clean['alarm']} --alarm and {clean['blocked']} --blocked pixels. " + f"L0 was the CLEAN half of this row -- a regression, not progress." + ) + dirty = tier_b("calm.zero_alarm_pixels", f"calm/L1/{theme}") + assert dirty["blocked"] == 97, ( + f"OSV1-003 (Core 2) PIN MOVED: a calm L1 in {theme} painted " + f"{dirty['blocked']} --blocked pixels, pinned at 97. If the legend " + f"swatch, the live dot and the danger button stopped painting " + f"`--blocked` on a calm page, re-derive this row from the new sweep " + f"(work_item_pipeline-qgo)." + ) assert contains(WEBPWA, "background:#0D0D0C;color:#F2EEE6"), ( "OSV1-003 (Core 2): the retired palette at webpwa.py:121-122 is gone. That is " "the Conformance 1 bad fixture's specimen and a Core 4 violation closing " @@ -346,9 +413,6 @@ def test_row_osv1_003() -> None: ) -# --------------------------------------------------------------- OSV1-004 - - def test_row_osv1_004() -> None: """Core 3 CONFORMS: the rendered check exists, is not deferred, and the chip vocabulary it walks still gives every status a WORD. @@ -417,8 +481,8 @@ def test_row_osv1_005() -> None: "OSV1-005 (Core 4): webpwa.py:121's retired-palette inline body is no longer " "counted as a literal site -- the worst specimen in the census. Re-derive." ) - assert "webbrowse.py:741" in literal, ( - "OSV1-005 (Core 4): webbrowse.py:741's textarea (literal font-size, max-width " + assert "webbrowse.py:814" in literal, ( + "OSV1-005 (Core 4): webbrowse.py:814's textarea (literal font-size, max-width " "and padding) is no longer counted. Re-derive." ) total = len(inline_style_sites()) @@ -496,11 +560,11 @@ def test_row_osv1_005() -> None: "webapp.py:2266", "webapp.py:2572", "webapp.py:3376", - "webapp.py:4393", - "webapp.py:4908", - "webtheme.py:4131", - "webtheme.py:4150", - "webtheme.py:4157", + "webapp.py:4421", + "webapp.py:4936", + "webtheme.py:4185", + "webtheme.py:4204", + "webtheme.py:4211", "widgets.py:704", "widgets.py:831", "widgets.py:834", @@ -570,33 +634,72 @@ def test_row_osv1_007() -> None: # --------------------------------------------------------------- OSV1-008 +#: Live regions present BEFORE the forced swap, per level, as the 2026-09-05 +#: re-recorded run measures them. L0 renders exactly ONE since the hero +#: rebuild landed (`widgets.py:1379`, the verdict hero's `role="status"`); L1 +#: still renders none. Pinned per level rather than as a single number, +#: because the two levels answer Core 6's announcement half differently and a +#: shared pin would let one move under the other. +_LIVE_REGIONS_BEFORE_SWAP = {"L0": 1, "L1": 0} + + def test_row_osv1_008() -> None: - """Core 6 GAP pin: no browser kit, and the swap restores exactly two - things -- neither of them the pause control or a live region. + """Core 6 VIOLATION pin: one of four survivals holds, RE-READ from the run. + + Pinned in BOTH directions per survival, because they are separable and a + fix to any one of them is progress this row must record rather than + absorb. """ - assert not _exists(TIER_B_KIT), ( - f"OSV1-008 (Core 6): {TIER_B_KIT} now exists -- re-derive this row from the " - f"post-swap DOM SNAPSHOT it emits (work_item_pipeline-qgo)." - ) - theme = read(WEBTHEME) - assert "function captureState()" in theme and "function restoreState(state)" in theme, ( - "OSV1-008 (Core 6): the swap's capture/restore pair is gone -- the mechanism " - "this row measures moved. Re-derive." - ) - restore = theme[theme.index("function restoreState(state)") :][:400] - assert "openIds" in restore and "scrollTo" in restore, ( - "OSV1-008 (Core 6): `restoreState` no longer restores open disclosures and scroll position." - ) - assert "__wtRefreshPaused" not in restore, ( - "OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY: `restoreState` now touches the " - "pause flag. The pause CONTROL's state surviving the swap is one of the four " - "things Core 6 names -- flip the row only after the browser kit MEASURES it, " - "and retarget this probe in the same change (work_item_pipeline-qgo)." - ) + for level in ("L0", "L1"): + m = tier_b("swap.survives", f"calm/{level}/dark") + assert m["scroll_preserved"], ( + f"OSV1-008 (Core 6): scroll offset stopped surviving the body-swap on " + f"{level}. That was the ONE of Core 6's four named survivals that held " + f"-- a regression." + ) + assert not m["open_details_preserved"], ( + f"OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY on {level}: an open " + f"`
` now survives the swap. Confirm it survives because the " + f"markup gained ids and `restoreState` reaches them, then re-derive " + f"this row (work_item_pipeline-qgo)." + ) + assert m["details_with_id"] == 0, ( + f"OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY on {level}: " + f"{m['details_with_id']} `
` now carry an id. `restoreState` " + f"only ever re-opens `details[id]`, so this is the mechanism acquiring " + f"its first targets -- re-derive from the new swap measurement." + ) + assert not m["pause_control_preserved"], ( + f"OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY on {level}: the pause " + f"CONTROL's state now survives the swap. Re-derive this row." + ) + assert m["pause_flag_preserved"], ( + f"OSV1-008 (Core 6): `window.__wtRefreshPaused` stopped surviving the " + f"swap on {level}. The flag living on `window` is why polling stays " + f"paused at all -- a regression." + ) + assert m["live_regions_before"] == _LIVE_REGIONS_BEFORE_SWAP[level], ( + f"OSV1-008 (Core 6) PIN MOVED on {level}: the page renders " + f"{m['live_regions_before']} live region(s) before the swap, pinned at " + f"{_LIVE_REGIONS_BEFORE_SWAP[level]}. Movement in either direction " + f"changes what Core 6's announcement half is even asking -- re-derive " + f"this row from the new swap measurement (work_item_pipeline-qgo)." + ) + assert m["marked_live_regions_after"] == 0, ( + f"OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY on {level}: " + f"{m['marked_live_regions_after']} of the live region(s) tagged before " + f"the swap SURVIVED it. On L0 that is the announcement half closing -- " + f"re-derive this row from the new measurement." + ) assert count(WEBAPP, "aria-live") == 0 and count(WEBTHEME, "aria-live") == 0, ( - "OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY: an `aria-live` region appeared. " - "Core 6's announcement half had NO implementation at seed -- if one landed, " - "re-derive this row from the Tier-B snapshot rather than from its presence." + "OSV1-008 (Core 6) PIN BROKE THE RIGHT WAY: an `aria-live` region appeared " + "in the source. Re-derive this row from the Tier-B snapshot rather than " + "from its presence." + ) + assert contains(WIDGETS, ' role="status">'), ( + 'OSV1-008 (Core 6): the verdict hero\'s `role="status"` region is gone -- ' + "that is the ONE live region L0 renders, and the thing the swap destroys. " + "Re-derive this row (and OSV1-001's hero rebuild) from a fresh run." ) @@ -694,33 +797,58 @@ def test_row_osv1_009() -> None: def test_row_osv1_010() -> None: - """Core 7 GAP pin (rendered half): nothing in this repo drives a browser.""" - assert not _exists(TIER_B_KIT), ( - f"OSV1-010 (Core 7): {TIER_B_KIT} now exists -- re-derive this row from the " - f"computed contrast ratios, bounding boxes and motion trace it EMITS, which " - f"the ledger must re-check itself (Freeze 3 / ruling 6)." - ) - for driver in BROWSER_DRIVERS: - assert not _repo_mentions(driver), ( - f"OSV1-010 (Core 7) PIN BROKE THE RIGHT WAY: {driver!r} now appears in the " - f"repo. A browser is being driven somewhere -- re-derive this row, and " - f"OSV1-003, -008, -020..-023, from what it actually measures." - ) + """Core 7 VIOLATION pin (rendered half), RE-READ from the browser run. + + Four floors, measured across 18 renders. Three fail and one passes, and + all four are pinned: a fix to any one is progress this row must record. + """ + l0 = tier_b("perception.floors", "calm/L0/1280/dark") + l1 = tier_b("perception.floors", "calm/L1/1280/dark") + l1_light = tier_b("perception.floors", "calm/L1/1280/light") + + assert l0["text_below_floor"] == 0, ( + f"OSV1-010 (Core 7): L0 now has {l0['text_below_floor']} text elements below " + f"4.5:1. L0 was the CLEAN level for text contrast -- a regression." + ) + assert l1["text_below_floor"] == 3 and l1_light["text_below_floor"] == 4, ( + f"OSV1-010 (Core 7) PIN MOVED: L1 text below 4.5:1 measured " + f"{l1['text_below_floor']} dark / {l1_light['text_below_floor']} light, " + f"pinned at 3 / 4 (light was 5 before the contrast lane moved " + f"`--ink-quiet`). Movement in either direction means the render changed " + f"-- re-derive (work_item_pipeline-qgo)." + ) + assert l0["controls_below_44px"] == 26 and l0["controls"] == 34, ( + f"OSV1-010 (Core 7) PIN MOVED: L0 measured {l0['controls_below_44px']} of " + f"{l0['controls']} interactive controls under 44px, pinned at 26 of 34 " + f"(35 before the hero rebuild replaced one control)." + ) + assert l0["non_text_below_floor"] > 0, ( + "OSV1-010 (Core 7) PIN BROKE THE RIGHT WAY: every measured control border " + "and icon stroke on L0 now meets 3:1. Re-derive this row." + ) + assert l0["running_animations_under_reduced_motion"] == 0, ( + f"OSV1-010 (Core 7): {l0['running_animations_under_reduced_motion']} " + f"animation(s) now run under `prefers-reduced-motion: reduce`. That floor " + f"PASSED at this measurement -- a regression, and Core 7's kernel-rule half " + f"(OSV1-011) with it." + ) assert contains(WEBTHEME, "--u:44px"), ( "OSV1-010 (Core 7): the 44px target token is gone -- the thing the Tier-B " "bounding-box check exists to verify." ) -# --------------------------------------------------------------- OSV1-011 - - def test_row_osv1_011() -> None: - """Core 7 CONFORMS: reduced motion is ONE kernel-level rule. - - Both halves matter: exactly one block, and its selector is universal. A - per-widget opt-in would satisfy a naive "is it handled?" check and is - precisely what the clause forbids. + """Core 7 CONFORMS: reduced motion is ONE kernel-level rule -- and the + browser confirms nothing runs under the preference. + + Three halves now, all load-bearing: exactly one block, its selector is + universal, and (RE-DERIVED 2026-09-05) the recorded browser run measures + ZERO running animations under `prefers-reduced-motion: reduce` in every + one of the 18 renders it sweeps. A per-widget opt-in would satisfy a naive + "is it handled?" check and is precisely what the clause forbids; a + stylesheet that says the right thing while the page still animates would + satisfy the static halves alone. """ theme = read(WEBTHEME) blocks = re.findall(r"@media \(prefers-reduced-motion:\s*reduce\)\s*\{", theme) @@ -745,6 +873,31 @@ def test_row_osv1_011() -> None: "webtheme.py. The rule is kernel-level and belongs in one place." ) + # The browser half, re-read from the run summary rather than trusted from + # the Tier-B tier's own green (Freeze 3). Every swept render, not one: the + # count is only meaningful measured on a QUIESCENT page, and reading a + # single scenario would let the other seventeen move unseen. + swept = { + scenario: headline["running_animations_under_reduced_motion"] + for scenario, headline in tier_b_summary()["checks"]["perception.floors"].items() + if scenario.startswith("calm/") + } + assert len(swept) == 18, ( + f"OSV1-011 (Core 7): the recorded run sweeps {len(swept)} renders, not the " + f"18 (L0/L1/L2 x 430/900/1280 x dark/light) this row reads. A narrowed " + f"sweep is a narrowed claim -- re-derive." + ) + still_running = {s: n for s, n in sorted(swept.items()) if n} + assert not still_running, ( + f"OSV1-011 (Core 7) REGRESSION: the recorded run measures animations still " + f"running under `prefers-reduced-motion: reduce`: {still_running}. The " + f"kernel rule collapses every duration to .001ms, so a running animation " + f"means something escapes it -- read the run's " + f"`motion`/`motion_at_preference_change` artifacts before flipping this row: " + f"a transition created BEFORE the preference was applied and still finishing " + f"is the page settling, and is recorded separately for exactly that reason." + ) + # --------------------------------------------------------------- OSV1-012 @@ -879,26 +1032,175 @@ def test_row_osv1_014() -> None: # --------------------------------------------------------------- OSV1-015 +#: Every adapter read that ACCEPTS a `limit` -- the calls Core 10's "every +#: adapter call reached from a view passes an explicit limit" is about. A +#: scalar read has nothing to bound, so it is not listed. +_BOUNDED_READS = frozenset( + { + "list", + "list_bounded", + "activity", + "attention_items", + "attention_items_from_rows", + "recent_activity_feed", + } +) + +#: A helper that makes an unbounded listing call but is reached by NO route. +#: Dead code is not "reached from a view", so the clause as written does not +#: condemn it -- but the exemption is re-earned every run below, by proving it +#: is still dead. +_UNREACHED_UNCAPPED = ("_oldest_ready_item", WEBAPP) + + +def _module_int_constants(path) -> dict[str, int]: # type: ignore[no-untyped-def] + """Module-level `NAME = ` assignments, plus one alias hop through the + adapter (`NAME = A.LIST_MAX_LIMIT`), read by PARSING -- never importing. + Static reading is what lets this kit measure source it does not execute. + """ + adapter_consts: dict[str, int] = {} + for node in ast.parse(read(ADAPTER)).body: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, int) + ): + adapter_consts[node.targets[0].id] = node.value.value + out: dict[str, int] = {} + for node in ast.parse(read(path)).body: + if not ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + continue + name = node.targets[0].id + value = node.value + if isinstance(value, ast.Constant) and isinstance(value.value, int): + out[name] = value.value + elif isinstance(value, ast.Attribute) and value.attr in adapter_consts: + out[name] = adapter_consts[value.attr] + return out + + +def _limit_passed(call: ast.Call, consts: dict[str, int]) -> object: + """The `limit=` this call passes: an int where it resolves, `None` when no + `limit` keyword is present at all, `"?"` when one IS passed from an + expression this static reading cannot evaluate. + + `"?"` is not a failure: the clause asks for an EXPLICIT limit at the call + site, and a value computed from a parameter is explicit there. `None` is + the failure -- an inherited default is a bound nobody at the call site + can see. + """ + + def evaluate(node: ast.expr) -> object: + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name): + return consts.get(node.id, "?") + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add | ast.Sub | ast.Mult): + left, right = evaluate(node.left), evaluate(node.right) + if isinstance(left, int) and isinstance(right, int): + if isinstance(node.op, ast.Add): + return left + right + return left - right if isinstance(node.op, ast.Sub) else left * right + return "?" + + for kw in call.keywords: + if kw.arg == "limit": + return evaluate(kw.value) + return None + + +def view_listing_calls() -> list[tuple[str, int, str, object]]: + """`(module, line, handler, limit)` for every listing call a read-only + route reaches -- the same module-local, depth-4 name-following the route + audit above already uses, and the same honest bound: a call reached + through a callable handed in from another module is invisible to it. + """ + found: list[tuple[str, int, str, object]] = [] + for path in ROUTE_MODULES: + tree = ast.parse(read(path)) + funcs: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + funcs.setdefault(node.name, node) + consts = _module_int_constants(path) + for handler in funcs.values(): + methods: list[str] = [] + for dec in handler.decorator_list: + methods += _route_methods(dec) or [] + if not any(m in {"GET", "HEAD"} for m in methods): + continue + reached = [handler] + seen: set[str] = set() + frontier = _called_names(handler) + for _ in range(4): + nxt: set[str] = set() + for name in frontier - seen: + seen.add(name) + helper = funcs.get(name) + if helper is not None and helper is not handler: + reached.append(helper) + nxt |= _called_names(helper) + frontier = nxt - seen + if not frontier: + break + for fn in reached: + for node in ast.walk(fn): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in _BOUNDED_READS + ): + found.append( + (path.name, node.lineno, handler.name, _limit_passed(node, consts)) + ) + return found + + def test_row_osv1_015() -> None: - """Core 10 VIOLATION pin: the L1 view's unbounded query, and the dead - uncapped helper beside it. + """Core 10 CONFORMS: every adapter listing call a read-only route reaches + passes an explicit, finite limit -- MEASURED here, not asserted from a + remembered line number. + + Retargeted from the pin on `webbrowse.py`'s `limit=0` when + work_item_pipeline-8vv landed. The pin named ONE call; this audits the + population, so the next unbounded view query fails here too rather than + slipping in beside a fixed one. + + `limit=0` is bd's own "unlimited" (`adapter.Beads.list`'s docstring says + so outright) and an omitted `limit` leaves bd's default in place + implicitly -- the clause asks for an EXPLICIT bound, so both are failures. """ - assert contains(WEBBROWSE, "bd.list(status=status_filter, include_resolved=True, limit=0)"), ( - "OSV1-015 (Core 10) PIN BROKE THE RIGHT WAY: webbrowse.py's `limit=0` call is " - "gone. If the L1 view now passes a bound that actually bounds (e.g. via " - "`adapter.list_bounded`), flip OSV1-015 to CONFORMS and retarget this probe IN " - "THE SAME CHANGE (work_item_pipeline-8vv). A passing pin is not conformance." - ) - app = read(WEBAPP) - assert app.count("_oldest_ready_item") == 1, ( - f"OSV1-015 (Core 10): `_oldest_ready_item` now has " - f"{app.count('_oldest_ready_item') - 1} caller(s). It calls `bd.list(...)` with " - f"NO limit at all -- dead code was the only reason it did not violate this " - f"clause. Give it a bound or delete it." + calls = view_listing_calls() + assert any(m == "webbrowse.py" and h == "project_view" for m, _, h, _ in calls), ( + f"OSV1-015 (Core 10): the traversal no longer reaches `project_view`'s item " + f"listing -- an audit that matches nothing passes forever while proving " + f"nothing. Re-derive this row. Found: {calls}" + ) + offenders = [c for c in calls if c[3] is None or (isinstance(c[3], int) and c[3] <= 0)] + assert not offenders, ( + "OSV1-015 (Core 10) REGRESSION -- a view-reached adapter read no longer " + "passes an explicit, finite limit (`None` = no `limit=` at all, so the call " + "silently inherits the seam's default; `0` = bd's own \"unlimited\"):\n " + + "\n ".join(f"{m}:{ln} in {h}() -> limit={lim!r}" for m, ln, h, lim in offenders) + + "\n This surface re-renders every 20 seconds; an unbounded read here runs " + "three times a minute per open tab." + ) + name, module = _UNREACHED_UNCAPPED + app = read(module) + assert app.count(name) == 1, ( + f"OSV1-015 (Core 10): `{name}` now has {app.count(name) - 1} caller(s). It " + f"calls `bd.list(...)` with NO limit at all -- being reached by nothing was " + f"the only reason it did not violate this clause. Give it a bound or delete it." ) assert contains(WEBAPP, 'items = bd.list(lane=A.LANE_WORK, status="open")'), ( "OSV1-015 (Core 10): the uncapped `bd.list` in `_oldest_ready_item` is gone -- " - "welcome, and the row's second pinned fact just changed. Re-derive." + "welcome, and the row's exemption just changed. Re-derive." ) @@ -906,51 +1208,98 @@ def test_row_osv1_015() -> None: def test_row_osv1_016() -> None: - """Core 10 VIOLATION pin: the theme choice persists nowhere. - - Pinned narrowly, on theme only. At seed the density preference raised a - genuine reading question (localStorage survives a refresh but IS 'only in - the browser') that the reconcile returned to the root rather than deciding, - so this probe deliberately said nothing about it. - - SETTLED 2026-09-04 by the owner-ratified DRAFT true-up #1: the machine - check now reads "no view holds state that does not survive a refresh - (state persisted in `localStorage` or on the server survives; ...)". Under - that wording density is CONFORMANT and theme is still a violation, so the - probe now asserts BOTH -- the pin on theme, and the persistence that makes - density conformant, since a row whose notes rule on density must notice if - density stops persisting. + """Core 10 CONFORMS: a chosen theme survives a refresh -- and so does the + other preference on this surface. + + Retargeted from the pin on `wtSetTheme` persisting NOTHING, when + work_item_pipeline-dg3 landed. Three facts hold it up, and all three are + load-bearing: + + * the choice is WRITTEN (`wtSetTheme` -> `localStorage`); + * it is READ BACK AT FIRST PAINT, from ``, before `` is + parsed -- a body-end read applies it one paint too late, which is a + flash, not a fix; + * writer and reader name the SAME key, taken from ONE declaration. Two + spellings would fail silently and look exactly like "the toggle does + nothing". + + The server's `data-theme="dark"` default is asserted UNCHANGED: the + resolver only ever replaces that attribute, never removes it, so PR #55's + fix (a light-OS browser silently winning the token cascade because + `` carried no `data-theme` at all) stays fixed. + + Density is checked alongside, unchanged in meaning from the pin: under + the 2026-09-04 aligned wording its `localStorage` persistence is exactly + why it is conformant, and a row whose notes rule on density must notice + if density stops persisting. """ app = read(WEBAPP) - setter = app[app.index("function wtSetTheme(t){") :][:340] - assert "setAttribute('data-theme', t)" in setter, ( - "OSV1-016 (Core 10): `wtSetTheme` no longer sets the theme attribute -- the " - "mechanism this row measures moved." - ) - for persistence in ("localStorage", "sessionStorage", "document.cookie", "fetch("): - assert persistence not in setter, ( - f"OSV1-016 (Core 10) PIN BROKE THE RIGHT WAY: `wtSetTheme` now uses " - f"{persistence!r}. If the theme choice survives a refresh, flip OSV1-016 to " - f"CONFORMS and retarget this probe IN THE SAME CHANGE " - f"(work_item_pipeline-dg3). A passing pin is not conformance." - ) + theme_src = read(WEBTHEME) + + key_decl = 'THEME_STORAGE_KEY = "wt-theme"' + assert key_decl in theme_src, ( + "OSV1-016 (Core 10): `webtheme.THEME_STORAGE_KEY` is gone or renamed. It is " + "the ONE declaration the writer and the first-paint reader both take their " + "key from -- re-derive this row before letting them drift apart." + ) + + assert "function wtSetTheme(t){" in app, ( + "OSV1-016 (Core 10): `wtSetTheme` is gone or renamed -- the mechanism this row " + "measures moved. Re-derive." + ) + setter = app[app.index("function wtSetTheme(t){") :][:200] + assert "localStorage.setItem(" in setter, ( + "OSV1-016 (Core 10) REGRESSION: `wtSetTheme` no longer persists the choice. " + "A theme held only in page memory dies on the next load -- that is this " + "clause's anti-goal in its own words (work_item_pipeline-dg3)." + ) + + assert "def theme_boot_js()" in theme_src, ( + "OSV1-016 (Core 10) REGRESSION: `webtheme.theme_boot_js` is gone. Without a " + "first-paint resolver a stored theme is written and never read back." + ) + boot = theme_src[theme_src.index("def theme_boot_js()") :] + boot = boot[: boot.index("\ndef ")] + assert "localStorage.getItem(" in boot and "setAttribute('data-theme'" in boot, ( + "OSV1-016 (Core 10) REGRESSION: the first-paint resolver no longer reads the " + "stored theme and applies it. Written-but-never-read is not persistence." + ) + assert "removeAttribute" not in boot, ( + "OSV1-016 (Core 10): the resolver now REMOVES `data-theme` rather than only " + "replacing it -- that is how a light-OS browser silently wins the token " + "cascade again (PR #55). Re-derive." + ) + + page_src = theme_src[theme_src.index("def page(") :] + page_src = page_src[: page_src.index("\n# ---")] + assert "" in page_src, ( + "OSV1-016 (Core 10) REGRESSION: `page()` no longer inlines the first-paint " + "resolver at all, so a stored theme is never applied on load." + ) + head_at = page_src.index('') + script_at = page_src.index("") + body_at = page_src.index("` ahead of the body. Applying a stored theme after the body is " + "parsed is a flash of the wrong theme on every single load." + ) assert contains(WEBTHEME, ''), ( - 'OSV1-016 (Core 10): the server no longer hardcodes `data-theme="dark"` on ' - "every page. That hardcoding is the other half of why a chosen theme dies on " - "refresh -- re-derive the row." + 'OSV1-016 (Core 10): the server no longer renders `data-theme="dark"` as the ' + "first-paint default. That attribute is what the resolver REPLACES; without " + "it a light-OS browser wins the cascade before any script runs (PR #55)." ) + # The OTHER preference on this surface, and the one the true-up's aligned - # wording rules CONFORMANT: density persists, so it survives a refresh. Not - # a pin -- a live check on the fact this row's notes rule on. - theme_src = read(WEBTHEME) - assert theme_src.count("wt-density") == 2 and contains( + # wording rules CONFORMANT: density persists, so it survives a refresh. + assert "var KEY='wt-density';" in theme_src and contains( WEBTHEME, "localStorage.setItem(KEY, next ? 'compact' : 'comfortable')" ), ( "OSV1-016 (Core 10): the density preference no longer persists in " "`localStorage`. Under the 2026-09-04 aligned wording that persistence is " - "exactly why density is CONFORMANT while theme is not -- if it stopped, " - "density became a second violation of this clause and this row's ruling note " - "is stale. Re-derive." + "exactly why density is CONFORMANT alongside theme -- if it stopped, density " + "became a violation of this clause and this row's ruling note is stale. " + "Re-derive." ) @@ -998,12 +1347,32 @@ def test_row_osv1_017() -> None: def test_row_osv1_020() -> None: - """Conformance 1 pin: no browser kit, so no calm pixel sweep exists.""" - assert not _exists(TIER_B_KIT), ( - f"OSV1-020 (Conformance 1): {TIER_B_KIT} now exists. Freeze 4 requires the BAD " - f"half to fail against the defect it names, DEMONSTRATED BY RUNNING IT -- " - f"re-derive this row from that demonstration, never from the file's existence " - f"(work_item_pipeline-qgo)." + """Conformance 1 CONFORMS: the fixture exists AND its bad halves caught the + defects they name -- re-read, not taken on the kit's word. + + Deliberately not `_exists(TIER_B_KIT)` alone. Freeze 4 is about a bad half + that FAILS against its defect, demonstrated by running it, so this reads + the three demonstrations' own numbers back out of the run summary. + """ + assert _exists(TIER_B_KIT), ( + f"OSV1-020 (Conformance 1): {TIER_B_KIT} is gone -- the fixture this row " + f"records no longer exists." + ) + chip = tier_b("calm.zero_alarm_pixels", "bad-alarm-chip/L0/dark") + assert chip["alarm"] > 0, ( + f"OSV1-020 (Conformance 1): the injected `var(--alarm)` chip bad half " + f"reported {chip['alarm']} --alarm pixels. A bad half that no longer " + f"discriminates makes the good half vacuous." + ) + retired = tier_b("calm.zero_alarm_pixels", "bad-retired-palette/L0/dark") + assert retired["retired_amber"] > 0, ( + "OSV1-020 (Conformance 1): the contract's own named bad half -- the retired " + "#D9A253 palette region reinstated -- reported zero pixels of it." + ) + real = tier_b("calm.zero_alarm_pixels", "bad-alarm-fixture/L0/dark") + assert real["alarm"] > 0 or real["blocked"] > 0, ( + "OSV1-020 (Conformance 1): the genuinely-alarming fixture (one real blocked " + "item, nothing injected) painted no reserved status hue at all." ) assert contains( OPERATOR_CONTRACT_PATH, "the sweep reports zero pixels matching `--alarm` or `--blocked`" @@ -1011,33 +1380,100 @@ def test_row_osv1_020() -> None: def test_row_osv1_021() -> None: - """Conformance 2 pin: the Tier-A half landed; the Tier-B hue half has not, - and that absence is what keeps this row red.""" - assert not _exists(TIER_B_KIT), ( - f"OSV1-021 (Conformance 2): {TIER_B_KIT} now exists. This is the one fixture " - f"that spans BOTH tiers -- the Tier-A accessible-name half already landed " - f"(work_item_pipeline-c1a, measured in OSV1-004); re-derive this row from the " - f"Tier-B hue half's own demonstrated pair (work_item_pipeline-qgo)." + """Conformance 2 CONFORMS: BOTH tiers' halves landed and both discriminate. + + RE-DERIVED 2026-09-05 at the wave-2 integration. This is the only + Conformance fixture that spans both tiers, so it was pinned twice -- once + by each lane, each pinning the OTHER half's absence. Both halves are now + present, so neither pin is true any more and the row is derived from what + the two kits actually measure: + + TIER-A half (work_item_pipeline-c1a) `state.not_colour_only` exists in + the Tier-A kit, ships a bad half, and is NOT deferred behind an xfail -- + a deferred good half is a check that does not currently pass. + TIER-B half (work_item_pipeline-qgo) the alarm region paints a RESERVED + hue, and every status-bearing element on L0/L1/L2 carries a word after + CSS has had its say. + + Each half keeps its own lane's assertions: this row goes red if EITHER + stops holding, which is the only way a two-tier fixture can be honest. + """ + # --- the Tier-A half: present, ships its bad half, and not deferred ----- + assert _exists(TIER_A_KIT), ( + f"OSV1-021 (Conformance 2) REGRESSION: {TIER_A_KIT} is gone. This row went " + f"CONFORMS on BOTH halves landing (work_item_pipeline-c1a); losing the " + f"Tier-A half puts it back to red." ) kit = _kit_source() assert "check_state_not_colour_only" in _kit_defs(kit), ( - "OSV1-021 (Conformance 2): the Tier-A half is gone from the kit. This row is " - "red on the Tier-B half ONLY -- losing the Tier-A half is a second, new defect." + "OSV1-021 (Conformance 2) REGRESSION: the Tier-A accessible-name half is gone from the kit." ) assert _kit_bad_halves(kit, "test_state_not_colour_only"), ( - "OSV1-021 (Conformance 2): the Tier-A half no longer ships a bad half. A " - "fixture whose bad half nobody runs is a claim, not a fixture (Freeze 4)." - ) + "OSV1-021 (Conformance 2) REGRESSION: the Tier-A half no longer ships a bad " + "half. A fixture whose bad half nobody runs is a claim, not a fixture " + "(Freeze 4)." + ) + deferred = _kit_deferred_rows(kit, "test_state_not_colour_only") + assert not deferred, ( + f"OSV1-021 (Conformance 2) REGRESSION: the Tier-A good half is deferred " + f"behind an xfail naming {sorted(deferred)}. A green Conformance row whose " + f"good half does not currently pass is a claim." + ) + # --- the Tier-B half: re-read from the committed run summary ------------ + assert _exists(TIER_B_KIT), ( + f"OSV1-021 (Conformance 2) REGRESSION: {TIER_B_KIT} is gone -- the hue half " + f"(work_item_pipeline-qgo) is what took this row green alongside Tier A." + ) + hue = tier_b("alarm.reserved_hue", "alarm/L1/dark") + assert hue["alarm"] > 0 or hue["blocked"] > 0, ( + "OSV1-021 (Conformance 2): the Tier-B hue half stopped measuring -- the " + "alarm render painted neither --alarm nor --blocked." + ) + for level in ("L0", "L1", "L2"): + words = tier_b("state.not_colour_only", f"alarm/{level}/dark") + assert words["status_elements"] > 0, ( + f"OSV1-021 (Conformance 2): no status-bearing element was found on " + f"{level} in the render -- the check cannot pass vacuously." + ) + assert words["wordless"] == 0, ( + f"OSV1-021 (Conformance 2): {words['wordless']} status-bearing element(s) " + f"on {level} now carry a class and no word. Core 3's rendered half was " + f"CLEAN at this measurement -- a regression (OSV1-004)." + ) assert contains( OPERATOR_CONTRACT_PATH, "fixture whose status chips carry only a status class" ), "OSV1-021: Conformance 2's bad half moved in the contract -- re-review the row." def test_row_osv1_022() -> None: - """Conformance 3 pin: no browser kit, so body-swap survival is untested.""" - assert not _exists(TIER_B_KIT), ( - f"OSV1-022 (Conformance 3): {TIER_B_KIT} now exists -- re-derive from the " - f"post-swap DOM snapshot and the demonstrated bad half (work_item_pipeline-qgo)." + """Conformance 3 CONFORMS: both bad halves ran and both caught their defect. + + Two of them, because the contract's literal bad half does not discriminate + on scroll here (a synchronous whole-body replacement preserves the offset + by itself on chromium 148); the reflow variant does. Both are re-read. + """ + assert _exists(TIER_B_KIT), ( + f"OSV1-022 (Conformance 3): {TIER_B_KIT} is gone -- the fixture this row " + f"records no longer exists." + ) + naive = tier_b("swap.survives", "bad-naive-replacement/L0/dark") + assert not naive["open_details_preserved"], ( + "OSV1-022 (Conformance 3): the contract's literal bad half -- a naive " + "whole-body innerHTML replacement -- no longer loses the open `
`, " + "so it discriminates against nothing." + ) + reflow = tier_b("swap.survives", "bad-naive-replacement-reflow/L0/dark") + assert not reflow["scroll_preserved"], ( + "OSV1-022 (Conformance 3): the reflow bad half no longer loses the scroll " + "offset. It exists precisely because the literal bad half cannot " + "discriminate on scroll -- without it, the good half's scroll assertion is " + "unproven." + ) + good = tier_b("swap.survives", "calm/L0/dark") + assert good["scroll_preserved"] and not good["open_details_preserved"], ( + "OSV1-022 (Conformance 3): the good half's own outcome moved (scroll " + f"{good['scroll_preserved']}, disclosures {good['open_details_preserved']}) " + f"-- re-derive this row and OSV1-008 together." ) assert contains(WEBTHEME, "document.body.innerHTML = doc.body.innerHTML"), ( "OSV1-022 (Conformance 3): the whole-body innerHTML swap is gone. That IS the " @@ -1047,12 +1483,55 @@ def test_row_osv1_022() -> None: def test_row_osv1_023() -> None: - """Conformance 4 pin: no viewport sweep, and the breakpoints it must sweep.""" - assert not _exists(TIER_B_KIT), ( - f"OSV1-023 (Conformance 4): {TIER_B_KIT} now exists -- re-derive from the " - f"emitted scrollWidth/clientWidth, bounding-box and contrast numbers " - f"(work_item_pipeline-qgo)." - ) + """Conformance 4 CONFORMS: the sweep covers what the clause names, and its + bad halves ran. + + The contrast bad half carries its own control -- a literal below-floor pair + must come back under 4.5:1 and a literal above-floor pair at or over it, in + BOTH themes -- which is what separates "the probe measures" from "the probe + always says no". + """ + assert _exists(TIER_B_KIT), ( + f"OSV1-023 (Conformance 4): {TIER_B_KIT} is gone -- the fixture this row " + f"records no longer exists." + ) + summary = tier_b_summary()["checks"]["perception.floors"] + swept = {s for s in summary if s.startswith("calm/")} + expected = { + f"calm/{level}/{width}/{theme}" + for level in ("L0", "L1", "L2") + for width in (430, 900, 1280) + for theme in ("dark", "light") + } + assert swept == expected, ( + f"OSV1-023 (Conformance 4): the sweep no longer covers L0/L1/L2 at 430, 900 " + f"and 1280px in both themes. Missing: {sorted(expected - swept)}; " + f"unexpected: {sorted(swept - expected)}." + ) + wide = summary.get("bad-wide-element/L0/430/dark") + assert wide and wide["elements_beyond_viewport_moved"], ( + "OSV1-023 (Conformance 4): the overflow bad half -- a 900px fixed-width " + "element at a 430px viewport -- no longer moves the element-level reading." + ) + assert wide["scroll_width_moved"] is False, ( + "OSV1-023 (Conformance 4) PIN BROKE THE RIGHT WAY: the injected wide element " + "now DOES move `scrollWidth`. That means `overflow-x: clip` is gone from the " + "surface, the clause's literal metric has become discriminating, and this " + "row's note about it should be re-derived." + ) + for theme in ("light", "dark"): + measured = summary.get(f"bad-low-contrast/L0/{theme}") + assert measured, f"OSV1-023 (Conformance 4): the contrast bad half did not run in {theme}." + assert measured["min_ratio"] < 4.5 <= measured["control_ratio"], ( + f"OSV1-023 (Conformance 4): in {theme} the injected literal pair measured " + f"{measured['min_ratio']}:1 and its control {measured['control_ratio']}:1. " + f"The bad half needs BOTH -- the below-floor pair under 4.5:1 AND the " + f"control at or above it -- or it is not demonstrating a measurement, it " + f"is a probe that always says no. RE-DERIVED 2026-09-05: the pair is now " + f"LITERAL (#9aa3b2 on #eef2fb, 2.27:1) in both themes, because the " + f"live-token version stopped naming a defect when the contrast lane fixed " + f"`--ink-quiet` (OSV1-009, OSV1-030)." + ) theme = read(WEBTHEME) for width in ("1280px", "900px", "430px"): assert f"max-width:{width}" in theme or f"min-width:{width}" in theme, ( @@ -1162,18 +1641,22 @@ def test_row_osv1_027() -> None: on the word "conformance": the Makefile and ci.yml already say "conformance ledger" about Tier 4, and a check a pre-existing comment satisfies asserts nothing. + + NARROWED 2026-09-05 (work_item_pipeline-qgo's narrowing, kept through the + wave-2 integration and turned round to the CONFORMS direction): the Tier-B + browser kit is now wired too (Makefile `test-conformance-b`, CI "Tier 7"), + so bare containment of `tests/conformance` would be satisfied by wiring + that has nothing to do with Freeze 1. The Tier-B path is stripped from + each file BEFORE the check, so what remains can only be the Tier-A wiring + this row is about. """ assert _exists(TIER_A_KIT), f"OSV1-027 (Freeze 1) REGRESSION: {TIER_A_KIT} is gone." make = read(MAKEFILE) - assert "tests/conformance" in make and "test-conformance-a:" in make, ( - "OSV1-027 (Freeze 1) REGRESSION: the Makefile no longer carries a target " - "covering tests/conformance. Existing is not the same as running." + assert "test-conformance-a:" in make, ( + "OSV1-027 (Freeze 1) REGRESSION: the Makefile no longer carries the " + "`test-conformance-a` target. Existing is not the same as running." ) ci = read(CI_WORKFLOW) - assert "tests/conformance/operator_surface" in ci, ( - "OSV1-027 (Freeze 1) REGRESSION: ci.yml no longer runs the Tier-A kit -- " - "Freeze 1's second half is 'runs on every pull request'." - ) assert "pull_request" in ci, ( "OSV1-027 (Freeze 1) REGRESSION: the workflow that runs the kit no longer " "triggers on pull_request." @@ -1187,55 +1670,271 @@ def test_row_osv1_027() -> None: "Without them it can quietly cover less of the contract than it claims, which " "is the failure Freeze 1 and Freeze 4 exist to prevent." ) - - -def test_row_osv1_028() -> None: - """Freeze 2 pin: no browser kit path, and no browser driver anywhere.""" - assert not _exists(TIER_B_KIT), ( - f"OSV1-028 (Freeze 2): {TIER_B_KIT} now exists. Freeze 2 has THREE further " - f"conditions -- a PINNED chromium, ISOLATED fixture data, and its OWN CI tier. " - f"A partial build satisfies none of them (work_item_pipeline-qgo)." - ) - for driver in BROWSER_DRIVERS: - assert not _repo_mentions(driver), ( - f"OSV1-028 (Freeze 2): {driver!r} now appears in the repo -- a browser is " - f"being driven. Re-derive this row and every row that depends on it." + tier_b_dir = TIER_B_KIT.rsplit("/", 1)[0] + for path, where in ((MAKEFILE, "the Makefile"), (CI_WORKFLOW, "ci.yml")): + remainder = read(path).replace(TIER_B_KIT, "").replace(tier_b_dir, "") + assert TIER_A_KIT.rsplit("/", 1)[0] in remainder, ( + f"OSV1-027 (Freeze 1) REGRESSION: {where} no longer wires the TIER-A kit " + f"path (the Tier-B browser wiring is stripped before this check, so it " + f"cannot stand in for it). Freeze 1's second half is 'runs on every pull " + f"request'." ) -def test_row_osv1_029() -> None: - """Freeze 3 pin: there is no Tier-B artifact for the orchestrator to re-check. +def test_row_osv1_028() -> None: + """Freeze 2 CONFORMS: all four sub-conditions, checked independently. - Pinned even though it is vacuously unmet today, because this is the - condition most likely to be quietly skipped once a browser tier exists and - its own assertions look green (PROTOCOL.md pillar 2). + The seed said a partial build satisfies none of them, so none of them is + inferred from another: the path, the PIN, the isolation and the OWN TIER + are four separate assertions. """ - assert not _exists(TIER_B_KIT), ( - f"OSV1-029 (Freeze 3): {TIER_B_KIT} now exists. This row is NOT satisfied by " - f"the kit passing -- it is satisfied when the kit EMITS artifacts (contrast " - f"numbers, pixel counts, bounding boxes, DOM snapshots) that ledger probes " - f"here re-read for themselves. Until the ledger re-checks them, a green " - f"Tier-B tier is a self-report (work_item_pipeline-qgo)." + assert _exists(TIER_B_KIT), f"OSV1-028 (Freeze 2): {TIER_B_KIT} is gone." + + pinned = "playwright==" + manifest = read(PYPROJECT) + assert pinned in manifest, ( + "OSV1-028 (Freeze 2): `pyproject.toml` no longer pins playwright to an EXACT " + "version. A playwright release pins exactly one chromium build, and that pin " + "is the whole of 'a pinned chromium' -- an unpinned browser makes every " + "recorded contrast ratio and pixel count unreproducible." + ) + version = manifest.split(pinned, 1)[1].split('"', 1)[0].strip() + recorded = tier_b_summary()["browser"] + assert recorded.get("playwright") == version, ( + f"OSV1-028 (Freeze 2): the manifest pins playwright {version!r} but the " + f"Tier-B run summary was produced by {recorded.get('playwright')!r}. The " + f"recorded numbers came from a different browser build than the one the repo " + f"now pins -- re-run `make test-conformance-b`." + ) + assert recorded.get("name") == "chromium" and recorded.get("version"), ( + f"OSV1-028 (Freeze 2): the run summary names no chromium build ({recorded})." + ) + + makefile = read(MAKEFILE) + assert "test-conformance-b:" in makefile and "playwright-install" in makefile, ( + "OSV1-028 (Freeze 2): the Makefile no longer carries `test-conformance-b` " + "(and its `playwright-install` dependency). Existing is not the same as " + "running." ) - assert not _exists("tests/conformance/operator_surface/browser/artifacts"), ( - "OSV1-029 (Freeze 3): an artifact directory appeared -- wire the ledger probes " - "to re-read it, then flip this row." + ci = read(CI_WORKFLOW) + assert "Tier 7 -- operator-surface conformance (Tier B, browser)" in ci, ( + "OSV1-028 (Freeze 2): CI no longer runs the browser tier as its OWN step. A " + "browser tier folded into another tier is not the tier Freeze 2 asks for." + ) + assert "-m tier_b" in ci and "playwright install" in ci, ( + "OSV1-028 (Freeze 2): the CI step no longer selects the `tier_b` marker or no " + "longer installs chromium -- either way the tier runs nothing." ) + kit = read(REPO_ROOT / "tests/conformance/operator_surface/browser/conftest.py") + assert "isolated_dolt_server" in kit and "port=0" in kit, ( + "OSV1-028 (Freeze 2): the kit no longer states its isolation -- the inherited " + "isolated dolt server, or the ephemeral `port=0` bind. The live service must " + "be unreachable from this tier by construction, not by luck." + ) + + +def test_row_osv1_029() -> None: + """Freeze 3 CONFORMS: artifacts are emitted, AND this ledger re-reads them. + + The row's own closing condition, written at seed: "a stable on-disk + artifact path with a documented JSON shape, and ledger probes here that + RE-READ those numbers rather than trusting the browser test's own + pass/fail." This probe checks both halves -- the summary parses and names + its schema and browser, and the kit is structurally incapable of asserting + on a screenshot. + """ + data = tier_b_summary() + assert data.get("recorded_at") and data.get("browser"), ( + "OSV1-029 (Freeze 3): the Tier-B run summary carries no provenance " + "(recorded_at / browser). A number with no named engine behind it is not " + "reproducible, which is the whole reason Freeze 2 pins the browser." + ) + for check in ("calm.zero_alarm_pixels", "swap.survives", "perception.floors"): + assert data["checks"].get(check), ( + f"OSV1-029 (Freeze 3): the run summary carries no `{check}` numbers. " + f"Every Tier-B check must emit artifacts the orchestrator re-checks." + ) + + kit = read(REPO_ROOT / TIER_B_KIT) + writes, reads = kit.count("artifacts.write("), kit.count("_artifacts.read(") + assert writes > 0 and reads >= writes, ( + f"OSV1-029 (Freeze 3): the Tier-B kit performs {writes} artifact write(s) but " + f"only {reads} read-back(s). Measure -> write -> read back -> assert is the " + f"discipline this row records: a check that writes an artifact and then " + f"asserts on the value still in its own local variable is a self-report, and " + f"the artifact it left behind is decoration." + ) + assert "save_screenshot" in kit and "assert" not in kit.split("save_screenshot")[0][-200:], ( + "OSV1-029 (Freeze 3): a screenshot appears to be feeding an assertion. " + "Screenshots are evidence for a human's Freeze 8 look, never an input to a " + "pass -- 'no check reports a rendered impression as a pass'." + ) + artifacts_module = read(REPO_ROOT / "tests/conformance/operator_surface/browser/_artifacts.py") + assert "carries an empty measurement" in artifacts_module, ( + "OSV1-029 (Freeze 3): the artifact reader no longer refuses an empty " + "measurement. A check that wrote `{}` and asserted `.get(..., 0) == 0` would " + "pass forever while measuring nothing." + ) + + +#: What Conformance 4's contrast bad half injects: a LITERAL below-floor pair +#: and a LITERAL above-floor control, neither of them reachable from the token +#: set. It injected the LIVE tokens (`--ink-quiet` on `--ground`) until the +#: contrast lane closed OSV1-009 by moving `--ink-quiet` off the recorded +#: colour and the "defect" measured 5.36:1 -- above the floor. A fixture whose +#: bad input is the product's own current state stops naming a defect the +#: moment the product is fixed, which is the failure this row exists to catch. +_LITERAL_BELOW_FLOOR_INJECTION = "color:#9aa3b2;background:#eef2fb" +_LITERAL_CONTROL_INJECTION = "color:#1b2430;background:#eef2fb" +_LIVE_TOKEN_PAIR_INJECTION = "color:var(--ink-quiet);background:var(--ground)" + +#: Freeze 4's population, enumerated: every Conformance fixture arm that has +#: to have been RUN and seen to bite. Nine Tier-B scenarios read back out of +#: the committed run summary, four Tier-A good halves whose bad halves are +#: read out of the kit's own source, and the kit's own tripwire that no check +#: may ship without one. Fourteen arms; a fixture that quietly drops one fails +#: here rather than narrowing the claim in silence. +_TIER_B_DEMONSTRATIONS = ( + ("calm.zero_alarm_pixels", "bad-alarm-chip/L0/dark"), + ("calm.zero_alarm_pixels", "bad-retired-palette/L0/dark"), + ("calm.zero_alarm_pixels", "bad-alarm-fixture/L0/dark"), + ("state.not_colour_only", "bad-wordless-chips/L1/dark"), + ("swap.survives", "bad-naive-replacement/L0/dark"), + ("swap.survives", "bad-naive-replacement-reflow/L0/dark"), + ("perception.floors", "bad-wide-element/L0/430/dark"), + ("perception.floors", "bad-low-contrast/L0/light"), + ("perception.floors", "bad-low-contrast/L0/dark"), +) + +_TIER_A_DEMONSTRATIONS = ( + ("Conformance 2", "test_state_not_colour_only"), + ("Conformance 5", "test_hero_velocity_and_counts"), + ("Conformance 6", "test_visual_single_source"), + ("Conformance 7", "test_calm_keeps_slot"), +) def test_row_osv1_030() -> None: - """Freeze 4 pin: the Tier-A fixtures are demonstrated; no browser kit - exists, so the three Tier-B fixtures are not.""" - assert not _exists(TIER_B_KIT), ( - "OSV1-030 (Freeze 4): the Tier-B kit path appeared. 'Demonstrated by running " - "it' is the whole clause -- record WHICH revert produced WHICH failure, the " - "way CCV1-023 did for the custody family, before flipping this row." - ) - assert _exists(TIER_A_KIT) and _kit_bad_halves(_kit_source(), "test_calm_keeps_slot"), ( - "OSV1-030 (Freeze 4): the Tier-A fixtures' demonstrated bad halves are gone. " - "This row is red on the TIER-B half only -- losing the Tier-A half is a " - "second, new defect." + """Freeze 4 CONFORMS: all fourteen Conformance fixture arms demonstrate on + THIS tree, re-read from a run rather than taken on either kit's word. + + RE-DERIVED 2026-09-05 (second wave-2 integration pass), VIOLATION -> + CONFORMS. The pin this row carried was Conformance 4's contrast bad half: + it injected the LIVE `--ink-quiet`/`--ground` pair, which WAS the recorded + 4.27:1 specimen until the contrast lane closed OSV1-009 by moving the + token -- after which the same injection measured 5.36:1 in light, above + the floor, and the bad half named no defect at all. Two Tier-A bad halves + had lost their specimens the same way in the same wave (the `limit=0` call + and the non-persisting theme setter, both fixed by the core10 lane). + + All three were rebuilt to own their bad input -- literal colours here, a + fabricated view module and a fabricated script module in the Tier-A kit -- + and the kit was re-run: `make test-conformance-b` on this tree is 52 + passed / 35 xfailed / 0 failed, `make test-conformance-a` 38 passed / + 4 xfailed / 0 failed. The pin is retargeted onto the invariant it was + waiting for: every arm ran, and each one still bites. + """ + # --- the Tier-A half ----------------------------------------------------- + assert _exists(TIER_A_KIT), ( + f"OSV1-030 (Freeze 4): {TIER_A_KIT} is gone -- the Conformance 5/6/7 (and " + f"Core 3) demonstrations went with it." ) + kit = _kit_source() + for fixture, good in _TIER_A_DEMONSTRATIONS: + assert _kit_bad_halves(kit, good), ( + f"OSV1-030 (Freeze 4): {fixture}'s bad half is gone from the Tier-A " + f"kit. A bad half that has never been executed is a claim." + ) + assert "test_every_check_ships_a_bad_half" in _kit_defs(kit), ( + "OSV1-030 (Freeze 4): the Tier-A kit dropped the tripwire that every check " + "ships a bad half -- without it the demonstration can narrow silently, " + "which is the failure this clause exists to prevent." + ) + # A bad half must own its bad INPUT. Both of these fabricate their + # specimen; before the 2026-09-05 rebuild they borrowed the shipped + # `webbrowse.py` call and the shipped theme setter, and both stopped + # discriminating the moment those were fixed. + for owned in ("_UNBOUNDED_VIEW_SPECIMEN", "_UNPERSISTED_STATE_SPECIMEN"): + assert owned in kit, ( + f"OSV1-030 (Freeze 4): the Tier-A kit no longer fabricates `{owned}`. " + f"Core 10's bad halves used the SHIPPED defects as their specimens and " + f"went red when the product was fixed -- a fixture that depends on the " + f"product staying broken demonstrates nothing." + ) + + # --- the Tier-B half: nine scenarios, all read back from the run --------- + assert _exists(TIER_B_KIT), ( + f"OSV1-030 (Freeze 4): {TIER_B_KIT} is gone -- Conformance 1-4's " + f"demonstrations went with it." + ) + checks = tier_b_summary()["checks"] + missing = sorted(f"{c}/{s}" for c, s in _TIER_B_DEMONSTRATIONS if s not in checks.get(c, {})) + assert not missing, ( + f"OSV1-030 (Freeze 4): {len(missing)} Conformance 1-4 bad half/halves were " + f"not run in the recorded Tier-B run: {missing}. A bad half that has never " + f"been executed is a claim." + ) + stripped = checks["state.not_colour_only"]["bad-wordless-chips/L1/dark"] + assert stripped["stripped"] > 0 and stripped["wordless"] > 0, ( + "OSV1-030 (Freeze 4): the Conformance 2 bad half stopped discriminating -- " + f"it stripped {stripped['stripped']} chips and still reported " + f"{stripped['wordless']} wordless." + ) + chip = checks["calm.zero_alarm_pixels"]["bad-alarm-chip/L0/dark"] + retired = checks["calm.zero_alarm_pixels"]["bad-retired-palette/L0/dark"] + real = checks["calm.zero_alarm_pixels"]["bad-alarm-fixture/L0/dark"] + assert chip["alarm"] > 0 and retired["retired_amber"] > 0, ( + f"OSV1-030 (Freeze 4): a Conformance 1 injection stopped painting its hue " + f"({chip['alarm']} --alarm px for the chip, {retired['retired_amber']} px " + f"for the reinstated retired palette)." + ) + assert real["alarm"] > 0 or real["blocked"] > 0, ( + "OSV1-030 (Freeze 4): the genuinely-alarming fixture painted no reserved " + "status hue -- the strongest of Conformance 1's three arms, and the only " + "one with nothing injected." + ) + naive = checks["swap.survives"]["bad-naive-replacement/L0/dark"] + reflow = checks["swap.survives"]["bad-naive-replacement-reflow/L0/dark"] + assert not naive["open_details_preserved"] and not reflow["scroll_preserved"], ( + "OSV1-030 (Freeze 4): a Conformance 3 bad half stopped losing what it " + "exists to lose (the naive replacement's open `
`, the reflow " + "variant's scroll offset)." + ) + assert checks["perception.floors"]["bad-wide-element/L0/430/dark"][ + "elements_beyond_viewport_moved" + ], ( + "OSV1-030 (Freeze 4): Conformance 4's overflow bad half stopped moving the " + "element-level reading -- the only reading `overflow-x: clip` cannot hide." + ) + + # --- the arm this row was red for: it must own its specimen now --------- + probe_lib = read(REPO_ROOT / TIER_B_PROBE_LIB) + assert _LIVE_TOKEN_PAIR_INJECTION not in probe_lib, ( + "OSV1-030 (Freeze 4) REGRESSION: Conformance 4's contrast bad half injects " + "the LIVE token pair again. That is precisely how this row went red at the " + "wave-2 integration: `--ink-quiet` moved and the bad half's defect moved " + "with it, leaving a fixture that measures whatever the product currently " + "does. Inject a literal below-floor pair." + ) + assert _LITERAL_BELOW_FLOOR_INJECTION in probe_lib, ( + f"OSV1-030 (Freeze 4): the contrast bad half no longer injects the literal " + f"below-floor pair ({_LITERAL_BELOW_FLOOR_INJECTION}). Its specimen must be " + f"its own, not the token set's." + ) + assert _LITERAL_CONTROL_INJECTION in probe_lib, ( + f"OSV1-030 (Freeze 4): the contrast bad half lost its literal control " + f"({_LITERAL_CONTROL_INJECTION}). Without a pair that must come back ABOVE " + f"the floor, a probe that always says no would satisfy the bad half." + ) + for theme in ("light", "dark"): + measured = checks["perception.floors"][f"bad-low-contrast/L0/{theme}"] + assert measured["min_ratio"] < 4.5 <= measured["control_ratio"], ( + f"OSV1-030 (Freeze 4): in {theme} the recorded run measures the contrast " + f"bad half at {measured['min_ratio']}:1 with its control at " + f"{measured['control_ratio']}:1. Freeze 4 asks for a bad half that FAILS " + f"the check it names, demonstrated by running it -- re-derive this row " + f"and OSV1-023 together." + ) def test_row_osv1_031() -> None: @@ -1263,8 +1962,8 @@ def test_row_osv1_031() -> None: "to CONFORMS and retarget this probe to assert no Core row is red " "(work_item_pipeline-umm)." ) - assert len(red) == 7, ( - f"OSV1-031 (Freeze 5): pinned 7 red Core-carrying rows, observed {len(red)}: " + assert len(red) == 5, ( + f"OSV1-031 (Freeze 5): pinned 5 red Core-carrying rows, observed {len(red)}: " f"{red}. Movement in either direction means this gate's tally changed -- update " f"the pin and the row's notes in the same change. (Was 10 at seed; OSV1-009 " f"went green 2026-09-04, work_item_pipeline-sxh.)" diff --git a/ledger/rows.yaml b/ledger/rows.yaml index 5cd1722..fbcabe5 100644 --- a/ledger/rows.yaml +++ b/ledger/rows.yaml @@ -1037,14 +1037,14 @@ (OSV1-003). - id: OSV1-003 - title: a calm screen paints zero alarm and zero blocked pixels -- unmeasured, no kit + title: a calm L1 paints 97 --blocked pixels with nothing blocked contract: file: contracts/operator-surface.v1.md clause: Core 2 quote: | On a calm screen — nothing held past TTL, nothing blocked — zero `--alarm` and zero `--blocked` pixels are painted; that absence is what makes the alarm pop. - disposition: GAP + disposition: VIOLATION work: work_item_pipeline-qgo assertion: kind: probe @@ -1053,33 +1053,52 @@ PINNING ROW -- the probe asserts the CURRENT, KNOWN-WRONG shape on purpose; a passing probe here is NOT conformance. Flip direction VIOLATION-MOVEMENT. - GAP, REASON "KIT NOT BUILT" -- never NOT-ASSERTABLE. The contract names the - check (`calm.zero_alarm_pixels`), names its tier (B) and names its file - (`tests/conformance/operator_surface/browser/test_tier_b.py`, Conformance - 1). It IS assertable; it simply is not asserted yet. - - MEASURED 2026-09-04 against 4aaee50: `tests/conformance` does not exist at - all (`tests/` holds only cli/, integration/, unit/, conftest.py, - _dolt_isolation.py, _util.py), and the repo contains ZERO occurrences of - playwright, selenium, screenshot or axe-core -- so no pixel of this surface - has ever been asserted by any test. - - KNOWN SPECIMEN THE SWEEP WOULD HAVE TO CATCH, measured here so the kit has - a real target rather than a synthetic one: webpwa.py:121-122 renders the - offline body with the RETIRED palette inline - (`background:#0D0D0C;color:#F2EEE6`), and webtrust.py:258-259 declares - `--ground:#0D0D0C; --ink:#F2EEE6; --amber:#D9A253` in its own `\n" diff --git a/tests/conformance/__init__.py b/tests/conformance/__init__.py index bdaa4c8..ec3edcc 100644 --- a/tests/conformance/__init__.py +++ b/tests/conformance/__init__.py @@ -5,4 +5,8 @@ DISCRIMINATING PAIR: a good half asserted against the real artifact, and a bad half asserted against a deliberately-wrong one that the same check must report. A check nobody has watched fail is a check that might assert nothing. + +A kit may span tiers: `operator_surface/` holds the in-process Tier-A checks, +and its real-browser Tier-B half lives beside them in +`operator_surface/browser/` as its own `tier_b`-marked tier. """ diff --git a/tests/conformance/operator_surface/__init__.py b/tests/conformance/operator_surface/__init__.py index 8a23fba..9e0907c 100644 --- a/tests/conformance/operator_surface/__init__.py +++ b/tests/conformance/operator_surface/__init__.py @@ -3,5 +3,6 @@ Tier A is everything assertable in-process: a static pass over `src/`, a token computation, or an assertion over rendered HTML. Tier B (a real browser, pixel sweeps, computed contrast, post-swap DOM snapshots) lives beside this package -in `browser/` and is a different lane's work. +in `browser/`, is marked `tier_b`, and runs as its own tier via +`make test-conformance-b`. """ diff --git a/tests/conformance/operator_surface/browser/LAST_RUN.json b/tests/conformance/operator_surface/browser/LAST_RUN.json new file mode 100644 index 0000000..490d2f3 --- /dev/null +++ b/tests/conformance/operator_surface/browser/LAST_RUN.json @@ -0,0 +1,397 @@ +{ + "browser": { + "name": "chromium", + "playwright": "1.60.0", + "version": "148.0.7778.0" + }, + "checks": { + "alarm.reserved_hue": { + "alarm/L1/dark": { + "alarm": 2669, + "blocked": 734, + "pixels_swept": 3868160, + "retired_amber": 0, + "watch": 1161 + } + }, + "calm.zero_alarm_pixels": { + "bad-alarm-chip/L0/dark": { + "alarm": 10531, + "blocked": 0, + "pixels_swept": 2813440, + "retired_amber": 0, + "watch": 0 + }, + "bad-alarm-chip/L0/light": { + "alarm": 10540, + "blocked": 0, + "pixels_swept": 2813440, + "retired_amber": 0, + "watch": 5318 + }, + "bad-alarm-fixture/L0/dark": { + "alarm": 0, + "blocked": 264, + "pixels_swept": 2813440, + "retired_amber": 0, + "watch": 0 + }, + "bad-retired-palette/L0/dark": { + "alarm": 0, + "blocked": 0, + "pixels_swept": 2813440, + "retired_amber": 16681, + "watch": 0 + }, + "bad-retired-palette/L0/light": { + "alarm": 16, + "blocked": 0, + "pixels_swept": 2813440, + "retired_amber": 16681, + "watch": 5116 + }, + "calm/L0/dark": { + "alarm": 0, + "blocked": 0, + "pixels_swept": 2813440, + "retired_amber": 0, + "watch": 0 + }, + "calm/L0/light": { + "alarm": 0, + "blocked": 0, + "pixels_swept": 2813440, + "retired_amber": 0, + "watch": 5259 + }, + "calm/L1/dark": { + "alarm": 0, + "blocked": 97, + "pixels_swept": 3792640, + "retired_amber": 0, + "watch": 1161 + }, + "calm/L1/light": { + "alarm": 0, + "blocked": 97, + "pixels_swept": 3792640, + "retired_amber": 0, + "watch": 6449 + } + }, + "kit.pinned_browser": { + "run/environment": { + "name": "chromium", + "playwright": "1.60.0", + "version": "148.0.7778.0" + } + }, + "perception.floors": { + "bad-low-contrast/L0/dark": { + "control_ratio": 13.96, + "min_ratio": 2.27 + }, + "bad-low-contrast/L0/light": { + "control_ratio": 13.96, + "min_ratio": 2.27 + }, + "bad-wide-element/L0/430/dark": { + "elements_beyond_viewport_moved": true, + "overflow_x_style": "clip", + "scroll_width_moved": false + }, + "calm/L0/1280/dark": { + "client_width": 1280, + "controls": 34, + "controls_below_44px": 26, + "elements_beyond_viewport": 0, + "non_text_below_floor": 16, + "non_text_measured": 79, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 1280, + "text_below_floor": 0, + "text_scored": 136 + }, + "calm/L0/1280/light": { + "client_width": 1280, + "controls": 34, + "controls_below_44px": 26, + "elements_beyond_viewport": 0, + "non_text_below_floor": 16, + "non_text_measured": 79, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 1280, + "text_below_floor": 0, + "text_scored": 136 + }, + "calm/L0/430/dark": { + "client_width": 430, + "controls": 34, + "controls_below_44px": 16, + "elements_beyond_viewport": 0, + "non_text_below_floor": 16, + "non_text_measured": 77, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 430, + "text_below_floor": 0, + "text_scored": 127 + }, + "calm/L0/430/light": { + "client_width": 430, + "controls": 34, + "controls_below_44px": 16, + "elements_beyond_viewport": 0, + "non_text_below_floor": 16, + "non_text_measured": 77, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 430, + "text_below_floor": 0, + "text_scored": 127 + }, + "calm/L0/900/dark": { + "client_width": 900, + "controls": 34, + "controls_below_44px": 26, + "elements_beyond_viewport": 0, + "non_text_below_floor": 16, + "non_text_measured": 79, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 900, + "text_below_floor": 0, + "text_scored": 136 + }, + "calm/L0/900/light": { + "client_width": 900, + "controls": 34, + "controls_below_44px": 26, + "elements_beyond_viewport": 0, + "non_text_below_floor": 16, + "non_text_measured": 79, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 900, + "text_below_floor": 0, + "text_scored": 136 + }, + "calm/L1/1280/dark": { + "client_width": 1280, + "controls": 41, + "controls_below_44px": 22, + "elements_beyond_viewport": 0, + "non_text_below_floor": 23, + "non_text_measured": 73, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 1280, + "text_below_floor": 3, + "text_scored": 183 + }, + "calm/L1/1280/light": { + "client_width": 1280, + "controls": 41, + "controls_below_44px": 22, + "elements_beyond_viewport": 0, + "non_text_below_floor": 23, + "non_text_measured": 73, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 1280, + "text_below_floor": 4, + "text_scored": 183 + }, + "calm/L1/430/dark": { + "client_width": 430, + "controls": 40, + "controls_below_44px": 21, + "elements_beyond_viewport": 0, + "non_text_below_floor": 23, + "non_text_measured": 52, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 430, + "text_below_floor": 3, + "text_scored": 177 + }, + "calm/L1/430/light": { + "client_width": 430, + "controls": 40, + "controls_below_44px": 21, + "elements_beyond_viewport": 0, + "non_text_below_floor": 23, + "non_text_measured": 52, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 430, + "text_below_floor": 4, + "text_scored": 178 + }, + "calm/L1/900/dark": { + "client_width": 900, + "controls": 41, + "controls_below_44px": 22, + "elements_beyond_viewport": 0, + "non_text_below_floor": 23, + "non_text_measured": 73, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 900, + "text_below_floor": 3, + "text_scored": 183 + }, + "calm/L1/900/light": { + "client_width": 900, + "controls": 41, + "controls_below_44px": 22, + "elements_beyond_viewport": 0, + "non_text_below_floor": 23, + "non_text_measured": 73, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 900, + "text_below_floor": 4, + "text_scored": 183 + }, + "calm/L2/1280/dark": { + "client_width": 1280, + "controls": 20, + "controls_below_44px": 11, + "elements_beyond_viewport": 0, + "non_text_below_floor": 11, + "non_text_measured": 33, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 1280, + "text_below_floor": 1, + "text_scored": 58 + }, + "calm/L2/1280/light": { + "client_width": 1280, + "controls": 20, + "controls_below_44px": 11, + "elements_beyond_viewport": 0, + "non_text_below_floor": 11, + "non_text_measured": 33, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 1280, + "text_below_floor": 2, + "text_scored": 58 + }, + "calm/L2/430/dark": { + "client_width": 430, + "controls": 20, + "controls_below_44px": 11, + "elements_beyond_viewport": 0, + "non_text_below_floor": 11, + "non_text_measured": 33, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 430, + "text_below_floor": 1, + "text_scored": 58 + }, + "calm/L2/430/light": { + "client_width": 430, + "controls": 20, + "controls_below_44px": 11, + "elements_beyond_viewport": 0, + "non_text_below_floor": 11, + "non_text_measured": 33, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 430, + "text_below_floor": 2, + "text_scored": 58 + }, + "calm/L2/900/dark": { + "client_width": 900, + "controls": 20, + "controls_below_44px": 11, + "elements_beyond_viewport": 0, + "non_text_below_floor": 11, + "non_text_measured": 33, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 900, + "text_below_floor": 1, + "text_scored": 58 + }, + "calm/L2/900/light": { + "client_width": 900, + "controls": 20, + "controls_below_44px": 11, + "elements_beyond_viewport": 0, + "non_text_below_floor": 11, + "non_text_measured": 33, + "overflow_x_style": "clip", + "running_animations_under_reduced_motion": 0, + "scroll_width": 900, + "text_below_floor": 2, + "text_scored": 58 + } + }, + "state.not_colour_only": { + "alarm/L0/dark": { + "status_elements": 1, + "wordless": 0 + }, + "alarm/L1/dark": { + "status_elements": 19, + "wordless": 0 + }, + "alarm/L2/dark": { + "status_elements": 1, + "wordless": 0 + }, + "bad-wordless-chips/L1/dark": { + "stripped": 19, + "wordless": 11 + } + }, + "swap.survives": { + "bad-naive-replacement-reflow/L0/dark": { + "details_with_id": 0, + "live_regions_before": 1, + "marked_live_regions_after": 0, + "open_details_preserved": false, + "pause_control_preserved": false, + "pause_flag_preserved": true, + "scroll_preserved": false + }, + "bad-naive-replacement/L0/dark": { + "details_with_id": 0, + "live_regions_before": 1, + "marked_live_regions_after": 0, + "open_details_preserved": false, + "pause_control_preserved": false, + "pause_flag_preserved": true, + "scroll_preserved": true + }, + "calm/L0/dark": { + "details_with_id": 0, + "live_regions_before": 1, + "marked_live_regions_after": 0, + "open_details_preserved": false, + "pause_control_preserved": false, + "pause_flag_preserved": true, + "scroll_preserved": true + }, + "calm/L1/dark": { + "details_with_id": 0, + "live_regions_before": 0, + "marked_live_regions_after": 0, + "open_details_preserved": false, + "pause_control_preserved": false, + "pause_flag_preserved": true, + "scroll_preserved": true + } + } + }, + "recorded_at": "2026-09-05T10:35:48Z", + "schema": "operator-surface-tier-b/1" +} diff --git a/tests/conformance/operator_surface/browser/__init__.py b/tests/conformance/operator_surface/browser/__init__.py new file mode 100644 index 0000000..a47c9b6 --- /dev/null +++ b/tests/conformance/operator_surface/browser/__init__.py @@ -0,0 +1,17 @@ +"""Tier-B (real-browser) conformance kit for `contracts/operator-surface.v1.md`. + +The contract names this package's `test_tier_b.py` by path in Conformance 1, +2, 3 and 4 and in Freeze 2. Everything else here exists to serve it: + + `_png.py` a dependency-free PNG decoder + colour histogram, so a pixel + sweep is a NUMBER this repo computed, not an eyeball verdict. + `_probe.py` the in-page JavaScript that measures contrast, target boxes, + overflow, motion and live regions, plus the pure-Python + re-checks that read those numbers back. + `_artifacts.py` the on-disk artifact contract (Freeze 3). + `conftest.py` the app-on-an-ephemeral-port + pinned-chromium fixtures. + +Read `test_tier_b.py`'s module docstring for the rules this kit runs under -- +in particular: no assertion in this package may rest on a screenshot being +LOOKED at. A screenshot is evidence; the artifact is the measurement. +""" diff --git a/tests/conformance/operator_surface/browser/_artifacts.py b/tests/conformance/operator_surface/browser/_artifacts.py new file mode 100644 index 0000000..cb0eb35 --- /dev/null +++ b/tests/conformance/operator_surface/browser/_artifacts.py @@ -0,0 +1,191 @@ +"""The on-disk artifact contract for the Tier-B kit (Freeze 3). + + **Freeze 3:** every Tier-B check emits artifacts the orchestrator + re-checks itself; no check reports a rendered impression as a pass. + +The rule that follows from that, and that every check in `test_tier_b.py` +obeys: **measure -> write -> read back -> assert**. A check computes numbers +in the browser, writes them to a JSON file here, and then asserts against the +file it just read back. Nothing asserts on a value that only ever lived in a +local variable, and nothing asserts on a screenshot. + +Screenshots ARE saved, next to the JSON, and they are evidence only -- a +human artefact for a Freeze 8 look. No assertion anywhere in this package +opens one. + +Artifact layout +--------------- + + _artifacts// + index.json every artifact written by the run + ..json one measurement record + ..png the screenshot it was measured from + +`` is `-`, so concurrent runs never overwrite each +other and a failed run's numbers survive for inspection. The directory is +gitignored: an artifact is a RUN's evidence, not repo content, and a committed +one would be a number nobody re-measured. + +Every record carries the same envelope: + + { + "check": "calm.zero_alarm_pixels", # the contract's own check name + "clause": "Core 2", # what it is evidence for + "scenario": "calm/L0/dark", # which fixture and render + "recorded_at": "2026-09-04T18:11:02Z", + "browser": {"name": "chromium", "version": "148.0.7778.0", + "playwright": "1.60.0"}, + "measurement": { ... check-specific numbers ... } + } + +`browser` is not decoration: a contrast ratio or a pixel count is only +reproducible against a named engine build, which is why Freeze 2 asks for a +PINNED chromium in the first place. + +The durable summary +------------------- +The per-run directory is gitignored and disappears with the checkout, so the +conformance ledger -- in-process, sub-second, browserless by design -- could +never read a number out of it. `LAST_RUN.json`, beside this module and +COMMITTED, is the bridge: every check contributes a short `headline` (the two +or three numbers its verdict actually rests on) and the summary is rewritten +at the end of every run. + +That is what lets `ledger/checks/test_operator_rows.py` re-read Tier-B +numbers for itself instead of trusting the browser tier's own green +(OSV1-029 / Freeze 3). It is a RECORDED measurement, not a live one, and the +ledger treats it as such: it pins the browser build the numbers came from, so +moving the playwright pin fails the ledger until the kit is re-run. + +Re-running the kit rewrites `LAST_RUN.json`; the diff IS the new measurement. +""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_ENVELOPE_KEYS = ("check", "clause", "scenario", "recorded_at", "browser", "measurement") + +#: The committed summary the conformance ledger reads. Beside this module, not +#: under the gitignored per-run directory. +SUMMARY_PATH = Path(__file__).parent / "LAST_RUN.json" + +SUMMARY_SCHEMA = "operator-surface-tier-b/1" + + +@dataclass +class RunArtifacts: + """One run's artifact directory, and the index of what it wrote.""" + + root: Path + written: list[str] = field(default_factory=list) + #: `{check: {scenario: headline}}` -- what `LAST_RUN.json` carries. + summary: dict[str, dict[str, Any]] = field(default_factory=dict) + browser: dict[str, str] = field(default_factory=dict) + + @classmethod + def for_this_run(cls, base: Path) -> RunArtifacts: + run_id = f"{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}-{os.getpid()}" + root = base / run_id + root.mkdir(parents=True, exist_ok=True) + return cls(root=root) + + def write( + self, + *, + check: str, + clause: str, + scenario: str, + browser: dict[str, str], + measurement: dict[str, Any], + headline: dict[str, Any] | None = None, + ) -> Path: + """Write one measurement record and return its path. + + The caller is expected to read it straight back (`read`) and assert + against THAT -- see this module's docstring. Returning the path rather + than the dict is deliberate: it makes the read-back the natural next + line instead of an easily-skipped extra step. + """ + record = { + "check": check, + "clause": clause, + "scenario": scenario, + "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "browser": browser, + "measurement": measurement, + } + path = self.root / f"{check}.{scenario.replace('/', '_')}.json" + path.write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8") + self.written.append(path.name) + self.browser = dict(browser) + if headline is not None: + self.summary.setdefault(check, {})[scenario] = headline + return path + + def save_screenshot(self, *, check: str, scenario: str, png: bytes) -> Path: + """Save a screenshot as EVIDENCE. Nothing asserts on the bytes here.""" + path = self.root / f"{check}.{scenario.replace('/', '_')}.png" + path.write_bytes(png) + self.written.append(path.name) + return path + + def write_index(self) -> Path: + path = self.root / "index.json" + path.write_text( + json.dumps( + {"run_dir": str(self.root), "artifacts": sorted(self.written)}, + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) + return path + + def write_summary(self) -> Path: + """Rewrite the committed `LAST_RUN.json` the conformance ledger reads. + + Only written when the run actually produced headlines: a partial run + (`-k something`) that measured three scenarios must not overwrite a + full run's record with a near-empty one, because the ledger would then + read "no such scenario" as an absence of evidence. + """ + if not self.summary: + return SUMMARY_PATH + SUMMARY_PATH.write_text( + json.dumps( + { + "schema": SUMMARY_SCHEMA, + "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "browser": self.browser, + "checks": self.summary, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return SUMMARY_PATH + + +def read(path: Path) -> dict[str, Any]: + """Read a record back and prove it is a well-formed artifact. + + The envelope check is not ceremony. A check that wrote `{}` and then + asserted `record.get("measurement", {}).get("alarm_pixels", 0) == 0` would + pass forever while measuring nothing at all -- the exact hollow green + Freeze 3 names. Reading through this function makes that impossible. + """ + record = json.loads(path.read_text(encoding="utf-8")) + missing = [k for k in _ENVELOPE_KEYS if k not in record] + if missing: + raise AssertionError(f"{path.name} is not a Tier-B artifact -- missing keys {missing}") + if not isinstance(record["measurement"], dict) or not record["measurement"]: + raise AssertionError(f"{path.name} carries an empty measurement -- nothing was measured") + return record diff --git a/tests/conformance/operator_surface/browser/_png.py b/tests/conformance/operator_surface/browser/_png.py new file mode 100644 index 0000000..683d88e --- /dev/null +++ b/tests/conformance/operator_surface/browser/_png.py @@ -0,0 +1,180 @@ +"""A dependency-free PNG decoder and colour histogram. + +Why this exists rather than a dependency +---------------------------------------- +`calm.zero_alarm_pixels` (Core 2 / Conformance 1) is specified as a PIXEL +SWEEP: "a rendered calm fixture, swept pixel-wise in both themes, contains no +`--alarm` or `--blocked` colour". A sweep needs the actual pixels, and +Playwright hands them over as PNG bytes. Pillow would decode them, but this +kit already carries one heavyweight test-only dependency (playwright, pinned +for reproducibility) and a second one buys nothing here: Chromium's own +encoder emits a narrow, well-specified subset -- 8-bit RGB or RGBA, +non-interlaced -- and that subset is ~80 lines of stdlib `zlib`. + +Fails LOUD on anything outside that subset (`UnsupportedPng`). A decoder that +silently returned an empty pixel buffer for an unexpected colour type would +make every sweep report "zero alarm pixels" and pass forever, which is +precisely the hollow-green failure Freeze 3 exists to prevent. + +The histogram, not a per-pixel scan +----------------------------------- +`histogram()` reduces an image to `{(r, g, b): count}` using strided `bytes` +slices and `collections.Counter`, so the hot loop runs in C rather than in +Python. Tolerance matching is then applied over the DISTINCT colours (a few +thousand on a real page) instead of over the millions of pixels, which is what +makes a full-page sweep at three viewports affordable in a test tier. +""" + +from __future__ import annotations + +import struct +import zlib +from collections import Counter +from dataclasses import dataclass + +_SIGNATURE = b"\x89PNG\r\n\x1a\n" + +#: bytes-per-pixel by PNG colour type, for the 8-bit subset we accept. +_CHANNELS = {2: 3, 6: 4} + + +class UnsupportedPng(Exception): + """The PNG is outside the subset this decoder accepts. + + Never downgraded to "decode what we can": a partially-decoded image + produces a partially-swept page, and a sweep that missed half the pixels + reporting zero is indistinguishable from a calm page. + """ + + +@dataclass(frozen=True) +class Image: + """Raw, unfiltered 8-bit pixel data plus the geometry to walk it.""" + + width: int + height: int + channels: int # 3 (RGB) or 4 (RGBA) + pixels: bytes # width * height * channels, row-major, no filter bytes + + @property + def pixel_count(self) -> int: + return self.width * self.height + + +def _paeth(a: int, b: int, c: int) -> int: + p = a + b - c + pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + if pa <= pb and pa <= pc: + return a + if pb <= pc: + return b + return c + + +def decode(data: bytes) -> Image: + """Decode an 8-bit, non-interlaced RGB/RGBA PNG into raw pixels.""" + if not data.startswith(_SIGNATURE): + raise UnsupportedPng("not a PNG: signature missing") + + idat = bytearray() + header: tuple[int, int, int, int, int, int, int] | None = None + pos = len(_SIGNATURE) + while pos < len(data): + (length,) = struct.unpack(">I", data[pos : pos + 4]) + ctype = data[pos + 4 : pos + 8] + body = data[pos + 8 : pos + 8 + length] + pos += 12 + length # 4 len + 4 type + length + 4 crc + if ctype == b"IHDR": + header = struct.unpack(">IIBBBBB", body) + elif ctype == b"IDAT": + idat += body + elif ctype == b"IEND": + break + + if header is None: + raise UnsupportedPng("no IHDR chunk") + width, height, depth, colour_type, compression, filter_method, interlace = header + if depth != 8: + raise UnsupportedPng(f"bit depth {depth} (only 8 is supported)") + if colour_type not in _CHANNELS: + raise UnsupportedPng(f"colour type {colour_type} (only 2=RGB and 6=RGBA are supported)") + if compression != 0 or filter_method != 0: + raise UnsupportedPng(f"compression={compression} filter_method={filter_method}") + if interlace != 0: + raise UnsupportedPng("interlaced PNG (Adam7) is not supported") + + channels = _CHANNELS[colour_type] + raw = zlib.decompress(bytes(idat)) + stride = width * channels + expected = (stride + 1) * height + if len(raw) != expected: + raise UnsupportedPng(f"decompressed {len(raw)} bytes, expected {expected}") + + out = bytearray(stride * height) + prev = bytes(stride) + src = 0 + for y in range(height): + ftype = raw[src] + src += 1 + line = bytearray(raw[src : src + stride]) + src += stride + if ftype == 0: + pass + elif ftype == 1: # Sub + for i in range(channels, stride): + line[i] = (line[i] + line[i - channels]) & 0xFF + elif ftype == 2: # Up + for i in range(stride): + line[i] = (line[i] + prev[i]) & 0xFF + elif ftype == 3: # Average + for i in range(stride): + left = line[i - channels] if i >= channels else 0 + line[i] = (line[i] + ((left + prev[i]) >> 1)) & 0xFF + elif ftype == 4: # Paeth + for i in range(stride): + left = line[i - channels] if i >= channels else 0 + upper_left = prev[i - channels] if i >= channels else 0 + line[i] = (line[i] + _paeth(left, prev[i], upper_left)) & 0xFF + else: + raise UnsupportedPng(f"unknown filter type {ftype} on row {y}") + out[y * stride : (y + 1) * stride] = line + prev = bytes(line) + + return Image(width=width, height=height, channels=channels, pixels=bytes(out)) + + +def histogram(image: Image) -> Counter[tuple[int, int, int]]: + """`{(r, g, b): pixel_count}` for the whole image, alpha ignored. + + Alpha is dropped rather than composited because a page screenshot is + already composited: Chromium hands back opaque pixels over the page's own + background, which is exactly the surface an operator's eye receives. + """ + px = image.pixels + step = image.channels + return Counter(zip(px[0::step], px[1::step], px[2::step], strict=True)) + + +def count_near( + hist: Counter[tuple[int, int, int]], target: tuple[int, int, int], *, tolerance: int +) -> int: + """How many pixels sit within `tolerance` of `target` on every channel. + + Per-channel Chebyshev distance rather than Euclidean: a hue is "this + colour, antialiased" when no channel has moved far, and that reading is + the one that stays stable across a rendering-engine bump. + """ + tr, tg, tb = target + return sum( + n + for (r, g, b), n in hist.items() + if abs(r - tr) <= tolerance and abs(g - tg) <= tolerance and abs(b - tb) <= tolerance + ) + + +def parse_hex(value: str) -> tuple[int, int, int]: + """`"#f59e0b"` -> `(245, 158, 11)`.""" + text = value.strip().lstrip("#") + if len(text) != 6: + raise ValueError(f"expected a 6-digit hex colour, got {value!r}") + return (int(text[0:2], 16), int(text[2:4], 16), int(text[4:6], 16)) diff --git a/tests/conformance/operator_surface/browser/_probe.py b/tests/conformance/operator_surface/browser/_probe.py new file mode 100644 index 0000000..e0d4c3e --- /dev/null +++ b/tests/conformance/operator_surface/browser/_probe.py @@ -0,0 +1,710 @@ +"""In-page measurement probes, and the pure-Python re-checks that read them. + +Every constant here is JavaScript evaluated inside the real page; every +function here is a re-check the test runs over the JSON that came back. The +split is the whole design: the browser MEASURES, the artifact RECORDS, and +Python DECIDES. No probe returns a verdict, and no re-check trusts one. + +What each probe measures, and the honest limits of each +------------------------------------------------------ + +`TEXT_CONTRAST_JS` + Every visible text-bearing element's computed foreground against its + effective background, resolved by walking ancestors until an opaque + background-color is found. An element whose background is a gradient or + an image cannot be reduced to one colour, so it is returned with + `resolved: false` and COUNTED SEPARATELY rather than being scored against + a guess -- an invented background would manufacture whichever verdict the + guess happened to produce. + +`TARGET_SIZE_JS` + The border box of every interactive element. Each is classified `inline` + (an `` with `display: inline` sitting inside flowing text) or + `control`. The 44px floor is asserted over CONTROLS only, which is WCAG + 2.5.8's own inline-link exception -- and the inline population is still + emitted, so the exemption is visible in the artifact rather than hidden in + the code. + +`OVERFLOW_JS` + `document.scrollingElement.scrollWidth` vs `clientWidth`, plus the widest + offending elements when they differ, so a failure names a culprit instead + of only a number. + +`MOTION_JS` + Every animation `document.getAnimations()` reports, with its effective + duration, its target's path and the property or keyframe name driving it. + Under `prefers-reduced-motion: reduce` the surface's kernel rule collapses + durations to `.001ms` rather than removing animations, so the honest + question is "does anything actually RUN", i.e. is any effective duration + longer than `MOTION_EPSILON_MS`. + + WHEN it is asked matters as much as what it asks: a transition created + before the preference was applied keeps the duration it was created with, + so a sample taken at the instant of the change catches the page finishing + what it had already started. `test_tier_b.py::_settled_motion` polls this + probe until the page is quiescent and records the instant-of-change + reading separately; neither reading is dropped. + +`NON_TEXT_JS` + Border colours against their own element's background, and SVG + stroke/fill against theirs. This is a NAMED SUBSET of "non-text + contrast", not all of it: a decorative gradient edge or an icon drawn as + a background image is not reachable this way. The subset measured is + recorded in the artifact so nobody reads the number as a stronger claim + than it is. + +`SWAP_*` + The Core 6 body-swap instrumentation. See `test_tier_b.py`'s + `swap.survives` section for why the poll guard is lifted for exactly one + synchronous call and restored before the swap lands. +""" + +from __future__ import annotations + +from typing import Any + +#: An animation whose effective duration is at or under this is not running. +#: The surface's reduced-motion rule sets `animation-duration:.001ms`, so the +#: floor has to be a small positive number rather than exactly zero. +MOTION_EPSILON_MS = 1.0 + +#: WCAG floors, quoted from Core 7: "Text contrast is at least 4.5:1 and +#: non-text contrast at least 3:1 ... interactive targets are at least 44px". +TEXT_CONTRAST_FLOOR = 4.5 +NON_TEXT_CONTRAST_FLOOR = 3.0 +TARGET_SIZE_FLOOR_PX = 44.0 + +# --------------------------------------------------------------- shared JS + +_COLOUR_HELPERS = r""" +function parseColour(value){ + if(!value) return null; + var m = value.match(/^rgba?\(([^)]+)\)$/); + if(!m) return null; + var parts = m[1].split(',').map(function(p){ return parseFloat(p.trim()); }); + if(parts.length < 3) return null; + var a = parts.length > 3 ? parts[3] : 1; + return {r: parts[0], g: parts[1], b: parts[2], a: a}; +} +function channel(c){ + var s = c / 255; + return s <= 0.03928 ? s / 12.92 : Math.pow((s + 0.055) / 1.055, 2.4); +} +function luminance(c){ + return 0.2126 * channel(c.r) + 0.7152 * channel(c.g) + 0.0722 * channel(c.b); +} +function ratio(fg, bg){ + var l1 = luminance(fg), l2 = luminance(bg); + var hi = Math.max(l1, l2), lo = Math.min(l1, l2); + return (hi + 0.05) / (lo + 0.05); +} +function composite(fg, bg){ + if(fg.a >= 1) return {r: fg.r, g: fg.g, b: fg.b, a: 1}; + var a = fg.a; + return {r: fg.r * a + bg.r * (1 - a), + g: fg.g * a + bg.g * (1 - a), + b: fg.b * a + bg.b * (1 - a), a: 1}; +} +function hex(c){ + function h(v){ var s = Math.round(v).toString(16); return s.length < 2 ? '0' + s : s; } + return '#' + h(c.r) + h(c.g) + h(c.b); +} +/* The effective background BEHIND `el`: the first ancestor with an opaque + background-color, compositing any translucent layers passed on the way. + Returns resolved:false when a gradient/image intervenes -- see the module + docstring for why that is not silently treated as transparent. */ +function effectiveBackground(el){ + var stack = []; + var node = el; + while(node && node.nodeType === 1){ + var cs = getComputedStyle(node); + if(cs.backgroundImage && cs.backgroundImage !== 'none'){ + return {resolved: false, reason: 'background-image', colour: null}; + } + var c = parseColour(cs.backgroundColor); + if(c && c.a > 0){ + if(c.a >= 1){ + var out = {r: c.r, g: c.g, b: c.b, a: 1}; + for(var i = stack.length - 1; i >= 0; i--) out = composite(stack[i], out); + return {resolved: true, reason: '', colour: out}; + } + stack.push(c); + } + node = node.parentElement; + } + return {resolved: false, reason: 'no opaque ancestor background', colour: null}; +} +function isVisible(el){ + var cs = getComputedStyle(el); + if(cs.visibility === 'hidden' || cs.display === 'none') return false; + if(parseFloat(cs.opacity) === 0) return false; + var r = el.getBoundingClientRect(); + return r.width > 0 && r.height > 0; +} +function pathOf(el){ + var bits = []; + var node = el; + while(node && node.nodeType === 1 && bits.length < 4){ + var b = node.tagName.toLowerCase(); + if(node.id) { b += '#' + node.id; bits.unshift(b); break; } + if(node.className && typeof node.className === 'string'){ + var cls = node.className.trim().split(/\s+/).slice(0, 2).join('.'); + if(cls) b += '.' + cls; + } + bits.unshift(b); + node = node.parentElement; + } + return bits.join('>'); +} +""" + + +def _with_helpers(body: str) -> str: + """Wrap a probe body so the colour helpers are in scope. + + `page.evaluate` treats its argument as a single EXPRESSION, so a string + that opens with `function parseColour(...)` is a syntax error, not a + program. Measured the hard way: every contrast, target and motion number + in this kit's first run was lost to `SyntaxError: Unexpected token + 'function'`, and the tests around them failed for a reason that had + nothing to do with the surface. Wrapping the helpers plus the body in one + arrow-IIFE makes the whole thing an expression again. + """ + # `return (` on ONE line, deliberately: JavaScript's automatic semicolon + # insertion turns a bare `return` followed by a newline into `return;`. + # Measured -- an earlier form here put the body on the next line and every + # probe using these helpers silently returned `undefined`, which reached + # Python as `None` and surfaced as a TypeError three frames away. + return f"(() => {{\n{_COLOUR_HELPERS}\nreturn ({body});\n}})()" + + +TEXT_CONTRAST_JS = _with_helpers( + r""" +(() => { + var out = [], unresolved = []; + var all = document.querySelectorAll('body *'); + for(var i = 0; i < all.length; i++){ + var el = all[i]; + var own = ''; + for(var k = 0; k < el.childNodes.length; k++){ + var n = el.childNodes[k]; + if(n.nodeType === 3) own += n.textContent; + } + own = own.replace(/\s+/g, ' ').trim(); + if(!own) continue; + if(!isVisible(el)) continue; + var cs = getComputedStyle(el); + var fg = parseColour(cs.color); + if(!fg) continue; + var bg = effectiveBackground(el); + var size = parseFloat(cs.fontSize); + var weight = parseInt(cs.fontWeight, 10) || 400; + var entry = { + path: pathOf(el), + text: own.slice(0, 60), + colour: cs.color, + font_px: size, + font_weight: weight, + large: size >= 24 || (size >= 18.66 && weight >= 700) + }; + if(!bg.resolved){ entry.reason = bg.reason; unresolved.push(entry); continue; } + var solid = composite(fg, bg.colour); + entry.foreground_hex = hex(solid); + entry.background_hex = hex(bg.colour); + entry.ratio = Math.round(ratio(solid, bg.colour) * 100) / 100; + out.push(entry); + } + return {scored: out, unresolved: unresolved}; +})() +""" +) + +TARGET_SIZE_JS = r""" +(() => { + var sel = 'a[href], button, input:not([type=hidden]), select, textarea, summary,' + + '[role=button], [role=link], [role=tab], [onclick]'; + var out = []; + var els = document.querySelectorAll(sel); + for(var i = 0; i < els.length; i++){ + var el = els[i]; + var cs = getComputedStyle(el); + if(cs.visibility === 'hidden' || cs.display === 'none') continue; + var r = el.getBoundingClientRect(); + if(r.width === 0 && r.height === 0) continue; + /* WCAG 2.5.8's inline exception: a link rendered inline inside flowing + text is sized by the sentence, not by the author. */ + var inline = el.tagName === 'A' && cs.display === 'inline'; + var label = (el.getAttribute('aria-label') || el.textContent || '').replace(/\s+/g,' ').trim(); + out.push({ + path: (el.tagName.toLowerCase() + (el.id ? '#' + el.id : '')), + label: label.slice(0, 40), + kind: inline ? 'inline' : 'control', + width: Math.round(r.width * 100) / 100, + height: Math.round(r.height * 100) / 100 + }); + } + return out; +})() +""" + +#: Horizontal overflow, measured TWO ways on purpose. +#: +#: Conformance 4's literal metric is `scrollWidth == clientWidth`. Measured on +#: this surface, that metric is VACUOUS: `webtheme.py` sets `overflow-x: clip` +#: on `html`/`body`, so content that runs past the viewport is clipped rather +#: than scrolled and `scrollWidth` never grows -- the contract's own bad half +#: ("a fixed-width element wider than 430px emits scrollWidth > clientWidth") +#: does not fire against the shipped page at all. So the probe ALSO reports +#: `elements_beyond_viewport`: elements whose border box extends past +#: `clientWidth`, which clipping cannot hide. Both numbers, plus the computed +#: `overflow-x` that explains the difference, go into the artifact -- naming +#: the vacuity instead of quietly passing on it. +OVERFLOW_JS = r""" +(() => { + var se = document.scrollingElement || document.documentElement; + var limit = se.clientWidth; + var beyond = []; + var all = document.querySelectorAll('body *'); + for(var i = 0; i < all.length; i++){ + var el = all[i]; + var cs = getComputedStyle(el); + if(cs.visibility === 'hidden' || cs.display === 'none') continue; + if(cs.position === 'fixed') continue; + var r = el.getBoundingClientRect(); + if(r.width === 0 || r.height === 0) continue; + if(r.right > limit + 1){ + beyond.push({ + path: (el.tagName.toLowerCase() + + (el.id ? '#' + el.id : '') + + (el.className && typeof el.className === 'string' + ? '.' + el.className.trim().split(/\s+/)[0] : '')), + right: Math.round(r.right * 100) / 100, + width: Math.round(r.width * 100) / 100 + }); + } + } + beyond.sort(function(a, b){ return b.right - a.right; }); + return { + scroll_width: se.scrollWidth, + client_width: limit, + overflow_px: se.scrollWidth - limit, + overflow_x_style: getComputedStyle(document.body).overflowX, + root_overflow_x_style: getComputedStyle(document.documentElement).overflowX, + elements_beyond_viewport: beyond.length, + widest_beyond: beyond.slice(0, 8) + }; +})() +""" + +#: The status hues AS THE PAGE RESOLVES THEM for the current theme. Read from +#: the live token block rather than hardcoded in Python, because the light +#: theme redefines all three (`--alarm:#92400e`, `--blocked:#991b1b`, +#: `--watch:#3a4468` -- webtheme.py:391). The kit's first run hardcoded the +#: dark values and consequently swept the light renders for colours that +#: cannot appear there, reporting a clean zero for a page it had not actually +#: examined. +RESOLVE_STATUS_TOKENS_JS = r""" +(() => { + var cs = getComputedStyle(document.documentElement); + return { + theme: document.documentElement.getAttribute('data-theme'), + alarm: cs.getPropertyValue('--alarm').trim(), + blocked: cs.getPropertyValue('--blocked').trim(), + watch: cs.getPropertyValue('--watch').trim() + }; +})() +""" + +MOTION_JS = _with_helpers( + r""" +(() => { + var running = []; + var anims = typeof document.getAnimations === 'function' ? document.getAnimations() : []; + var now = performance.now(); + for(var i = 0; i < anims.length; i++){ + var a = anims[i]; + var timing = a.effect && a.effect.getComputedTiming ? a.effect.getComputedTiming() : {}; + var duration = typeof timing.duration === 'number' ? timing.duration : 0; + var target = a.effect && a.effect.target ? a.effect.target : null; + /* NAMED, not just counted: a row that has to say WHICH animations still + run under the preference cannot be written from a bare integer, and an + artifact that only carries the integer forces the note to hand-wave. */ + running.push({ + state: a.playState, + duration_ms: Math.round(duration * 1000) / 1000, + target: target ? target.tagName.toLowerCase() : '', + path: target ? pathOf(target) : '', + label: target ? (target.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 40) : '', + kind: (a.constructor && a.constructor.name) || '', + property: a.transitionProperty || a.animationName || '', + current_time_ms: Math.round((a.currentTime || 0) * 1000) / 1000, + measured_at_ms: Math.round(now * 1000) / 1000 + }); + } + var sampled = []; + var els = document.querySelectorAll('body *'); + for(var j = 0; j < els.length && sampled.length < 400; j++){ + var cs = getComputedStyle(els[j]); + var ad = cs.animationDuration || '0s', td = cs.transitionDuration || '0s'; + if(ad !== '0s' || td !== '0s'){ + sampled.push({animation: ad, transition: td}); + } + } + return { + reduced_motion_matches: window.matchMedia('(prefers-reduced-motion: reduce)').matches, + animations: running, + non_zero_declared: sampled + }; +})() +""" +) + +#: Non-text contrast over a NAMED, DEFENSIBLE population: the visual boundary +#: of an interactive control, and SVG icon strokes. WCAG 1.4.11 covers "user +#: interface components and graphical objects", not decorative rules -- an +#: earlier form here measured EVERY border on the page and reported 36-65 +#: failures per view, most of them panel hairlines the guideline exempts. A +#: number that large stops being a finding and starts being noise, so the +#: population is narrowed here and NAMED in the artifact (`population`), where +#: it can be argued with. +NON_TEXT_JS = _with_helpers( + r""" +(() => { + var interactive = 'a[href], button, input:not([type=hidden]), select, textarea, summary,' + + '[role=button], [role=link], [role=tab], [onclick]'; + var out = [], unresolved = 0, skipped_decorative = 0; + var els = document.querySelectorAll('body *'); + for(var i = 0; i < els.length; i++){ + var el = els[i]; + if(!isVisible(el)) continue; + var isControl = el.matches(interactive); + var isIcon = el.tagName.toLowerCase() === 'svg' || !!el.ownerSVGElement; + if(!isControl && !isIcon){ skipped_decorative++; continue; } + var cs = getComputedStyle(el); + var bg = effectiveBackground(el.parentElement || el); + if(!bg.resolved){ unresolved++; continue; } + var record = function(kind, value, widthPx){ + var c = parseColour(value); + if(!c || c.a === 0) return; + var solid = composite(c, bg.colour); + out.push({ + path: pathOf(el), kind: kind, + colour: hex(solid), background_hex: hex(bg.colour), + thickness_px: widthPx, + ratio: Math.round(ratio(solid, bg.colour) * 100) / 100 + }); + }; + var bw = parseFloat(cs.borderTopWidth) || 0; + if(isControl && bw >= 1 && cs.borderTopStyle !== 'none'){ + record('control-border', cs.borderTopColor, bw); + } + if(isIcon && cs.stroke && cs.stroke !== 'none'){ + record('icon-stroke', cs.stroke, parseFloat(cs.strokeWidth) || 1); + } + } + return { + population: 'interactive control borders + svg icon strokes (WCAG 1.4.11 scope)', + measured: out, + unresolved_backgrounds: unresolved, + skipped_decorative: skipped_decorative + }; +})() +""" +) + + +# ------------------------------------------------------------------ Core 6 + +#: Installed with `page.add_init_script`, so it runs BEFORE the surface's own +#: inline scripts. It captures the poller's `setInterval(tick, 20000)` +#: registration instead of scheduling it: the only body-swap in a Core 6 run +#: is then the one the test forces, which is what makes the measurement +#: deterministic rather than a race against a 20-second timer. +SWAP_CAPTURE_INIT_JS = r""" +(() => { + var real = window.setInterval.bind(window); + window.__wtCapturedIntervals = []; + window.setInterval = function(fn, ms){ + window.__wtCapturedIntervals.push({fn: fn, ms: ms}); + return -1; + }; + window.__wtRealSetInterval = real; +})(); +""" + +#: Drop a sentinel into the body before a swap. A whole-body `innerHTML` +#: replacement destroys it, so its ABSENCE afterwards is the proof the swap +#: actually happened. Without this the good half passes vacuously whenever the +#: forced fetch quietly fails -- the poller swallows every error by design +#: (`.catch(function(){ /* silent */ })`), so "nothing changed" and "nothing +#: was swapped" look identical from the outside. +SWAP_SENTINEL_JS = r""" +(() => { + var el = document.createElement('div'); + el.id = 'wt-swap-sentinel'; + el.setAttribute('hidden', 'hidden'); + document.body.appendChild(el); + return true; +})() +""" + +SWAP_SENTINEL_PRESENT_JS = "!!document.getElementById('wt-swap-sentinel')" + +#: Tag every live region present BEFORE the swap. A whole-body innerHTML +#: replacement destroys the tagged nodes, so counting survivors afterwards is +#: a direct, numeric answer to "was a pending announcement destroyed?". +SWAP_MARK_LIVE_REGIONS_JS = r""" +(() => { + var sel = '[aria-live], [role=status], [role=alert], [role=log]'; + var found = document.querySelectorAll(sel); + var marked = []; + for(var i = 0; i < found.length; i++){ + found[i].setAttribute('data-wt-preswap', String(i)); + marked.push({ + index: i, + role: found[i].getAttribute('role') || '', + aria_live: found[i].getAttribute('aria-live') || '', + text: (found[i].textContent || '').replace(/\s+/g, ' ').trim().slice(0, 80) + }); + } + return marked; +})() +""" + +#: The post/pre-swap DOM snapshot Conformance 3 asks for. +#: +#: `
` are keyed by ORDINAL + class signature, not by id: measured on +#: the shipped surface, NO `
` on L0, L1 or L2 carries an id at all +#: (help popover, activity feed, actions drawer -- all id-less), and +#: `restoreState` only ever re-opens `details[id]`. Keying this snapshot by id +#: would therefore compare two empty lists and report survival for a mechanism +#: that has no targets. `open_details_by_id` is kept alongside so the artifact +#: shows both readings. +SWAP_STATE_JS = r""" +(() => { + var btn = document.getElementById('refreshToggle'); + var open = [], byId = []; + var all = document.querySelectorAll('details'); + for(var i = 0; i < all.length; i++){ + var d = all[i]; + var sig = i + ':' + (typeof d.className === 'string' ? d.className.trim() : ''); + if(d.open){ open.push(sig); if(d.id) byId.push(d.id); } + } + var live = document.querySelectorAll('[aria-live], [role=status], [role=alert], [role=log]'); + return { + scroll_y: Math.round(window.scrollY), + details_total: all.length, + details_with_id: document.querySelectorAll('details[id]').length, + open_details: open.sort(), + open_details_by_id: byId.sort(), + pause_flag: !!window.__wtRefreshPaused, + pause_control_pressed: btn ? btn.getAttribute('aria-pressed') : null, + live_region_count: live.length, + surviving_marked_live_regions: document.querySelectorAll('[data-wt-preswap]').length + }; +})() +""" + +#: Force ONE tick of the surface's own poller. +#: +#: The pause guard is lifted for exactly the synchronous entry of `tick()` and +#: restored on the very next statement -- before the fetch it starts can +#: resolve, and therefore before the swap it performs. What is bypassed is the +#: SCHEDULING guard ("a paused page should not poll"); what is measured is the +#: page's state at the moment the swap lands, which is still `paused`. Doing +#: it any other way would mean either never swapping while paused (measuring +#: nothing) or clearing the very flag under test (measuring a lie). +SWAP_FORCE_TICK_JS = r""" +(() => { + var ticks = window.__wtCapturedIntervals || []; + var chosen = null; + for(var i = 0; i < ticks.length; i++){ + if(ticks[i].ms >= 1000){ chosen = ticks[i]; break; } + } + var q = document.getElementById('q'); + var guards = { + document_hidden: document.hidden, + visibility_state: document.visibilityState, + active_element: document.activeElement ? document.activeElement.tagName : null, + q_value: q ? q.value : null, + paused_flag: !!window.__wtRefreshPaused + }; + if(!chosen) return {forced: false, reason: 'no poller interval was registered', + guards: guards, + captured: ticks.map(function(t){ return t.ms; })}; + var wasPaused = window.__wtRefreshPaused; + window.__wtRefreshPaused = false; + try { chosen.fn(); } finally { window.__wtRefreshPaused = wasPaused; } + return {forced: true, interval_ms: chosen.ms, guard_restored_to: !!wasPaused, + guards: guards}; +})() +""" + +#: The BAD half of Conformance 3: "a whole-body innerHTML replacement that +#: recreates the region". A naive swap with no capture/restore at all -- which +#: is what the surface would do if `captureState`/`restoreState` were deleted. +SWAP_NAIVE_REPLACEMENT_JS = r""" +(async () => { + var u = new URL(location.href); + var res = await fetch(u.pathname + (u.search || ''), + {credentials: 'same-origin', + headers: {'X-Requested-With': 'wt-auto-refresh'}}); + var html = await res.text(); + var doc = new DOMParser().parseFromString(html, 'text/html'); + document.body.innerHTML = doc.body.innerHTML; + return true; +})() +""" + +#: The same naive replacement, but with a forced layout flush between +#: emptying the body and refilling it. +#: +#: Why a second variant exists, measured rather than assumed: on chromium 148 +#: a SYNCHRONOUS `body.innerHTML = html` preserves `window.scrollY` all by +#: itself -- the document never gets a chance to collapse, so the browser +#: never clamps the offset. The contract's literal bad half therefore does not +#: discriminate on the scroll half of Core 6 at all. Clearing the body, +#: reading `offsetHeight` to force layout, and only then refilling is the same +#: whole-body replacement written the other common way, and it DOES lose the +#: offset -- which is what proves the good half's scroll assertion can see a +#: loss when there is one. +SWAP_NAIVE_REPLACEMENT_WITH_REFLOW_JS = r""" +(async () => { + var u = new URL(location.href); + var res = await fetch(u.pathname + (u.search || ''), + {credentials: 'same-origin', + headers: {'X-Requested-With': 'wt-auto-refresh'}}); + var html = await res.text(); + var doc = new DOMParser().parseFromString(html, 'text/html'); + document.body.innerHTML = ''; + void document.body.offsetHeight; /* force layout: the document collapses */ + document.body.innerHTML = doc.body.innerHTML; + return true; +})() +""" + +#: The BAD half of Conformance 1: "the retired-palette region reinstated -- a +#: hardcoded amber outside the token set". `#D9A253` is the real specimen +#: still in the tree at `webtrust.py`'s page-local `