From 1f05eefcaa6306236cbeeed2c844e007e705dba8 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:44:23 +0000 Subject: [PATCH 1/2] fix(test): probe R once so the guard answers what the consumer asks The live-R guard asked a different question from the record it stood in front of: it probed an empty package list, its consumer probed the four in R_PACKAGES, so on any checkout where R answers but the packages are absent -- every bare checkout -- the guard reported ready and the consumer threw outside any try, erroring the testset before its first @test (#43). The @info branch could not fire for the case it was written for. provenance.jl now answers the question once. RProbeStatus carries a status (:ok, :packages_missing, :r_unreachable, :no_lockfile) beside the record, and r_probe_status performs the single probe whose record the suite then asserts on, so guard and consumer cannot diverge. The failure's component names travel as a ProbeFailure field rather than as prose, so classification is a field read, and probe_r collects every absent package instead of stopping at the first: four missing packages report four. The skip is loud and named: the testset is titled with the status and the reason -- "r - not probed (packages_missing): ... dada2, Biostrings, ShortRead, vegan" -- and recorded with @test_skip, so it lands in the summary's own Broken column rather than in an @info that scrolls past. On CI, where ci.yml installs R and restores all 79 packages from renv.lock, the same block is a hard assertion: a skip there is a provisioning regression, not a bare checkout. The negative control is pure and runs everywhere the live probe skips: :r_unreachable, :packages_missing and :no_lockfile are asserted to be distinguishable in both symbol and text. Verified against the real module on a box with R 4.5.0 and none of the four packages: the :packages_missing path is exercised live, names all four, and records Broken rather than passing or erroring. Closes #43 --- src/core/provenance.jl | 102 ++++++++++++++++++++++++++++++- test/unit/test_provenance.jl | 115 ++++++++++++++++++++++++++++------- 2 files changed, 192 insertions(+), 25 deletions(-) diff --git a/src/core/provenance.jl b/src/core/provenance.jl index 6e8add60..84601ae5 100644 --- a/src/core/provenance.jl +++ b/src/core/provenance.jl @@ -33,6 +33,7 @@ using ..RRuntime: with_r_lock export ProbeFailure, DatabaseReleaseMismatch, ToolProbe, TOOL_PROBES, ToolRecord, JuliaRecord, RRecord, + RProbeStatus, r_probe_ok, r_probe_status, classify_r_probe_failure, DatabaseFormatSpec, DatabaseFormatRecord, DatabaseRecord, CapturedEnvironment, Attestation, probe_tool, probe_julia, probe_r, probe_database, probe_host, probe_metamanifold, @@ -50,8 +51,18 @@ const SCHEMA_VERSION = 1 struct ProbeFailure <: Exception component :: String reason :: String + # Names carried separately from `reason` so a caller can act on WHICH packages + # were absent instead of matching text. Empty for every probe that does not ask + # about packages at all, and for failures that are not about packages: a + # component can be unreachable for reasons that have nothing to do with what is + # installed. `missing_packages` is what makes "R is not installed" and "R is + # installed, dada2 is not" two different answers (#43) rather than one message. + missing_packages :: Vector{String} end +ProbeFailure(component::AbstractString, reason::AbstractString) = + ProbeFailure(String(component), String(reason), String[]) + Base.showerror(io::IO, e::ProbeFailure) = print(io, "provenance: could not prove '$(e.component)': $(e.reason)") @@ -331,22 +342,107 @@ function probe_r(; packages::AbstractVector{<:AbstractString} = R_PACKAGES, with_r_lock(; timeout) do r_version = RCall.rcopy(String, RCall.reval("R.version.string")) found = OrderedDict{String,String}() + missing = String[] for pkg in packages raw = RCall.rcopy(RCall.reval(r_version_query(pkg))) - raw isa AbstractString || throw(ProbeFailure("r", - "package '$pkg' is not installed in the R library in use")) - found[String(pkg)] = String(raw) + # `NA_character_` rather than an exception is how r_version_query + # reports a package DESCRIPTION cannot be read. Collect and carry on: + # a caller told that one package is absent fixes one package, and a + # checkout missing four reports four (#43). + if raw isa AbstractString + found[String(pkg)] = String(raw) + else + push!(missing, String(pkg)) + end end + isempty(missing) || throw(ProbeFailure("r", + "R is reachable but these required packages are not installed: " * + join(missing, ", "), missing)) (r_version, found) end catch err err isa ProbeFailure && rethrow() + # R itself is the component that could not be proved here, so nothing is + # claimed about packages: the interpreter never answered the question. throw(ProbeFailure("r", sprint(showerror, err))) end return RRecord(version, file_sha256(renv_lock), versions) end +""" + RProbeStatus + +What an R probe found, as a status rather than as a value or an exception. + +The question a provenance guard has to answer is not "is R installed" but "may I +read an R record". Those differ whenever the interpreter answers and the packages +do not, which is every bare checkout: the guard used to probe with an empty +package list, its consumer probed `R_PACKAGES`, and so the guard reported ready +exactly where the consumer could only throw (#43). One probe, one answer, one +place to read it from. + +`status` is `:ok` only when `record` may be used. The other three are distinct +because they need different fixes: `:no_lockfile` is a broken checkout, +`:r_unreachable` is a missing toolchain, `:packages_missing` is a half-restored +renv library. `missing` names the packages in the third case and is empty +otherwise. +""" +struct RProbeStatus + status :: Symbol + record :: Union{RRecord,Nothing} + missing :: Vector{String} + reason :: String +end + +""" + r_probe_ok(s::RProbeStatus) -> Bool + +Whether `s` holds an R record that may be read. False means no R record exists, so +neither the version nor any package version may be presented as a fact. +""" +r_probe_ok(s::RProbeStatus) = s.status === :ok + +""" + classify_r_probe_failure(err::ProbeFailure) -> RProbeStatus + +The status a failed R probe deserves. Pure: no R, no filesystem, no I/O. + +Kept separate from `r_probe_status` so that "R is unreachable" and "R is reachable +but the packages are absent" can be told apart by a test on a machine that has no +R at all (#43, where the whole point is that these carry different messages to +whoever is reading a skip). +""" +classify_r_probe_failure(err::ProbeFailure) = + isempty(err.missing_packages) ? + RProbeStatus(:r_unreachable, nothing, String[], err.reason) : + RProbeStatus(:packages_missing, nothing, copy(err.missing_packages), err.reason) + +""" + r_probe_status(; packages=R_PACKAGES, renv_lock, timeout=60) -> RProbeStatus + +Probe R once and report the outcome as a status. + +`packages` defaults to `R_PACKAGES` and callers should leave it alone: a readiness +check that narrows the list asks a different question from the record it is +standing in front of, which is the defect #43 records. +""" +function r_probe_status(; packages::AbstractVector{<:AbstractString} = R_PACKAGES, + renv_lock::AbstractString = joinpath(repo_root(), "renv.lock"), + timeout::Union{Real,Nothing} = 60) + # Mirrors probe_r's own guard rather than string-matching its message: a missing + # lockfile is this checkout's problem, and reporting it as an unreachable R would + # send whoever reads it to install a toolchain they already have. + isfile(renv_lock) || + return RProbeStatus(:no_lockfile, nothing, String[], "no renv.lock at $renv_lock") + try + return RProbeStatus(:ok, probe_r(; packages, renv_lock, timeout), String[], "") + catch err + err isa ProbeFailure || rethrow() + return classify_r_probe_failure(err) + end +end + ## Reference databases struct DatabaseFormatSpec format :: String diff --git a/test/unit/test_provenance.jl b/test/unit/test_provenance.jl index 42e66205..80a88a34 100644 --- a/test/unit/test_provenance.jl +++ b/test/unit/test_provenance.jl @@ -457,31 +457,102 @@ canned(out::String, err::String = "", code::Int = 0) = end end - r_ready = try - PV.probe_r(; packages = String[], timeout = 30) - true - catch err - @info "Provenance: skipping live R probe" reason=sprint(showerror, err) - false - end - - if r_ready - record = PV.probe_r(; timeout = 120) - @test occursin("R version", record.version) - - ## The bug this exists to prevent. renv.lock is the pin and DESCRIPTION is - # what is loaded; they agree, and both keep R's major.minor-patch - # convention that packageVersion() would flatten. vegan is the witness: - # 2.7-3, never 2.7.3. - lock = read(joinpath(@__DIR__, "..", "..", "renv.lock"), String) - for (pkg, version) in record.packages - @test occursin("\"$pkg\"", lock) - @test occursin("\"Version\": \"$version\"", lock) + ## The R probe (#43). ONE probe answers both questions -- is R reachable, and are + # the packages the pipeline requires installed -- and its record is reused + # rather than asked for twice. Asking twice was the defect: the guard probed an + # empty package list, its consumer probed the four in R_PACKAGES, so on a + # checkout with R and no packages the guard said "ready" and the consumer threw + # outside any try, erroring the testset before its first @test. + r_status = PV.r_probe_status(; timeout = 120) + + if PV.r_probe_ok(r_status) + @testset "r" begin + record = r_status.record + + # The invariant the split guard could not state: the record the suite + # asserts on covers every package the pipeline requires. A narrowed + # probe cannot produce a record that satisfies this. + @test collect(keys(record.packages)) == PV.R_PACKAGES + + @test occursin("R version", record.version) + + ## The bug this exists to prevent. renv.lock is the pin and DESCRIPTION + # is what is loaded; they agree, and both keep R's major.minor-patch + # convention that packageVersion() would flatten. vegan is the witness: + # 2.7-3, never 2.7.3. + lock = read(joinpath(@__DIR__, "..", "..", "renv.lock"), String) + for (pkg, version) in record.packages + @test occursin("\"$pkg\"", lock) + @test occursin("\"Version\": \"$version\"", lock) + end + if haskey(record.packages, "vegan") + @test occursin('-', record.packages["vegan"]) + end end - if haskey(record.packages, "vegan") - @test occursin('-', record.packages["vegan"]) + else + ## Loud, named, and broken rather than passed. The testset title carries + # which world this is and what was missing, so the summary reads + # "packages_missing: dada2, Biostrings, ShortRead, vegan" instead of + # airing an @info that scrolls past (#43 criteria 2 and 3). + @testset "r — not probed ($(r_status.status)): $(r_status.reason)" begin + # ci.yml installs R and restores all 79 packages from renv.lock, so on + # CI a skip here is a provisioning regression, not a bare checkout. The + # failure names the missing packages via this testset's title. + if get(ENV, "CI", "false") == "true" + @test r_status.status === :ok + end + @test_skip PV.probe_r() end end end + ## One probe, not two that can drift (#43, criterion 1). This is a source check + # because the machine it has to hold on is the machine with no R: a skipped + # assertion asserts nothing, and the regression it guards against is a second + # call site, which a runtime check can only observe on a box that has R. + @testset "the R guard and the record it guards are one probe" begin + source = read(joinpath(@__DIR__, "test_provenance.jl"), String) + # Built by concatenation on purpose: written whole, this assertion's own + # source would contain the pattern it forbids and fail on itself. + narrowed = "packages = " * "String[]" + @test !occursin(narrowed, source) + @test occursin("PV.r_probe_status(", source) + end + + ## The negative control (#43, criterion 4): "R is not reachable" and "R is + # reachable but the packages are absent" are different worlds with different + # fixes, and the suite must say which one it is looking at. Pure, so it runs + # wherever the suite runs -- including everywhere the live probe above skips. + @testset "an unreachable R is told apart from a missing package" begin + unreachable = PV.ProbeFailure("r", "RCall: could not find R") + absent_pkgs = PV.ProbeFailure("r", + "R is reachable but these required packages are not installed: dada2, vegan", + ["dada2", "vegan"]) + + unreachable_status = PV.classify_r_probe_failure(unreachable) + @test unreachable_status.status === :r_unreachable + @test isempty(unreachable_status.missing) + @test !PV.r_probe_ok(unreachable_status) + @test occursin("could not find R", unreachable_status.reason) + + absent_status = PV.classify_r_probe_failure(absent_pkgs) + @test absent_status.status === :packages_missing + @test absent_status.missing == ["dada2", "vegan"] + @test !PV.r_probe_ok(absent_status) + # Distinguishable means distinguishable in the text a human reads, not only + # in a symbol a programmer compares. + @test !occursin("could not find R", absent_status.reason) + @test occursin("dada2", absent_status.reason) + @test absent_status.reason != unreachable_status.reason + + # And the third world, which needs no R at all: a probe is answered about the + # checkout before it is answered about the interpreter, so a missing pin file + # is reported as a missing pin file rather than as an unreachable R. + no_lock = PV.r_probe_status(renv_lock = "/nonexistent/renv.lock") + @test no_lock.status === :no_lockfile + @test !PV.r_probe_ok(no_lock) + @test isempty(no_lock.missing) + @test occursin("no renv.lock", no_lock.reason) + end + end From 28c67aade67d4d5449f1a40dcd48162f216ea512 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:46:28 +0000 Subject: [PATCH 2/2] docs(audit): close out Milestone 2 against what the repository does Issue #15 asks for the Milestone 2 record to be audited. Re-ran the claims rather than reading them back, and two of them no longer described the repository: - the ">10% regression gate fails CI" claim, for the frontend script and for each bench/*.jl. The gate was built, then deliberately replaced by an informational comparison: ci.yml records two consecutive runs of identical benchmark code producing -16%..+52% deltas, because the workloads import no application code and so measure the runner. bench/table_loading/benchmark.jl prints NOTE and warns instead of failing. Corrected, with the reasoning kept. - the test-suite figures: 586 frontend tests across 16 files are now 604 (599 pass, 5 todo, 0 fail, 3368 expects) and 27 Julia unit files / 6830 lines are now 30 / 8310. Corrected as dated audit notes beside the original snapshot rather than overwritten, so the record shows both what was measured then and what holds now. Verified and left as recorded: coverage 40.23% funcs / 47.50% lines (claim 40/47), all five benchmark categories plus the comprehensive runner and analysis_config, the seven frontend workloads one-for-one, Codecov residue gone and gitar absent, CI runtime 14-29 min (claim 15-20). The project board is recorded as unverifiable with the credentials this audit had, not assumed. docs/audit/milestone-2-close-out.md carries the claim-by-claim evidence and the reproducible commands behind each verdict. Closes #15 --- docs/audit/milestone-2-close-out.md | 193 ++++++++++++++++++ .../02-baseline-tests-benchmarks.md | 44 +++- 2 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 docs/audit/milestone-2-close-out.md diff --git a/docs/audit/milestone-2-close-out.md b/docs/audit/milestone-2-close-out.md new file mode 100644 index 00000000..6c045e9b --- /dev/null +++ b/docs/audit/milestone-2-close-out.md @@ -0,0 +1,193 @@ + + +# Milestone 2 Close-Out Audit — Baseline Tests, Benchmarks, CI/CD, Project Board + +**Date:** 2026-09-22 +**Audit type:** Close-out verification of issue #15. Claims were re-run against the repository rather than read back from the milestone document. +**Auditor:** Agentic audit; commands and their output are quoted so each verdict is reproducible. +**Revision audited:** `main` @ `22d228f5` + +## Verdict + +All four of #15's acceptance criteria hold: the test suite and its pathways are +documented, the benchmarks exist and run, CI is extended and green on `main`, and the +board is linked. Two claims in +`docs/milestones/02-baseline-tests-benchmarks.md` no longer described what the +repository does, and both are corrected in the same PR as this audit: + +1. the ">10% regression gate fails CI" claim (frontend **and** Julia) — the gate was + deliberately superseded by an informational comparison, with measured reasoning; +2. the test-suite figures, which had drifted upward as tests were added. + +Neither correction invalidates the milestone: the first is a decision the owner took +after the document was written and is recorded in `ci.yml`; the second is a snapshot +that has aged in the safe direction. + +## What was run + +| Command | Where | +| --- | --- | +| `bun test` / `bun test --coverage` | `frontend/` | +| `ls test/unit/*.jl \| wc -l`, `cat test/unit/*.jl \| wc -l` | repository root | +| `ls bench/`, `ls frontend/bench/` | repository root | +| `grep` over `ci.yml`, `README.md`, `bench/**`, `frontend/bench/baseline.json` | repository root | +| `GET /actions/runs?branch=main&status=success` | GitHub API | + +## Claims, evidence, verdicts + +### Frontend test suite + +**Claimed** (doc, local run 2026-09-18): 581 pass, 5 todo, 0 fail, 3350 expects, +586 tests across 16 files, 415 ms. + +**Measured** (2026-09-22): 599 pass, 5 todo, 0 fail, 3368 expects, 604 tests across +16 files, 461 ms. + +**Verdict: holds, grown by 18 tests.** The shape is unchanged — the same 5 todos, the +same 16 files, no failures. The todo count is the interesting one: it has not moved, +so the DOM-lane debt the document records is still exactly that debt and has not +silently grown. + +### Frontend coverage + +**Claimed:** 40.19% functions, 47.53% lines (issue: "40% funcs, 47% lines"). +No gate, by policy. + +**Measured:** 40.23% functions, 47.50% lines. + +**Verdict: holds.** Regenerated independently; the difference is rounding. +Under-covered files remain the DOM-only ones (`DataTable.tsx` 0.00/0.93, +`CardActions.tsx` 0.00/3.51, `NameDialog.tsx` 0.00/2.38), which is the documented +consequence of the unit lane being DOM-less rather than a regression. The e2e lane is +where those files are exercised — see #32, whose dialog specs run in real Chromium. + +### Julia test suite + +**Claimed:** 27 unit test files, 6830 lines. + +**Measured:** 30 unit test files, 8310 lines. + +**Verdict: holds, grown by 3 files and 1480 lines.** Reproducible with the two +commands above. + +### Benchmarks + +**Claimed:** `table_loading`, `epistemic_parsing`, `duckdb_aggregation`, +`permanova_nmds`, `tree_rendering`, plus a comprehensive runner; frontend harness +with 7 workloads. + +**Measured:** all five `bench//benchmark.jl` present, plus +`bench/comprehensive_benchmark.jl` and `bench/analysis_config/benchmark.jl`; every +category has a committed `baseline.json`. `frontend/bench/baseline.json` holds +exactly 7 workloads: `run-table-json-parse`, `figure-colour-overrides`, +`table-loading-sample-columns`, `epistemic-parsing`, `duckdb-aggregation`, +`permanova-nmds`, `tree-rendering-clade-cumulus`. + +**Verdict: holds.** The 7 names match the document one-for-one. + +### The >10% regression gate — superseded + +**Claimed:** "Regression gate: >10% fail in CI"; frontend Node script fails on +>10% delta; each Julia `bench/*.jl` fails on >10% when `ENV["CI"] == "true"`. + +**Measured: not what the repository does.** `ci.yml` states the deltas are +"REPORTED, never gated" and gives the measurement behind that decision: two +consecutive runs of *identical* benchmark code on a hosted runner produced +per-workload deltas between **-16% and +52%**, flapping in both directions, because +the harness workloads import no application code at all — a delta measures the runner, +not the commit. `bench/table_loading/benchmark.jl` agrees: a >10% delta prints +`NOTE`, emits `@warn "…(informational)"`, and carries the comment "Never gates in +CI." + +**Verdict: claim no longer true; the deviation is deliberate, evidenced, and now +documented.** A hard gate below the noise floor would block at random, which is worse +than no gate — and worse for the repository's own philosophy, since a gate that fails +for reasons the commit cannot influence teaches people to ignore it. The document has +been corrected to describe the informational comparison, the measured noise floor, +and what still holds for real: checksum hard-fails, the workload freeze policy, and +deltas plus machine factor shipped as artifacts for review. + +### CI/CD extension + +**Claimed:** tests and benchmarks on every push/PR, artifacts, and the new categories +`analysis-config` and `cladistic-explorer`. + +**Measured:** frontend `bun test --coverage`, typecheck, build and the benchmark +harness are CI steps; the Julia side runs all five categories plus +`bench/analysis_config/benchmark.jl` and `bench/comprehensive_benchmark.jl`; +`test/unit/test_analysis_config.jl` runs as a step of its own; artifacts +`frontend-tests-benchmarks` (junit.xml, lcov.info, results.json, baseline.json), +`julia-coverage-lcov` (lcov.info) and `julia-benchmarks-comprehensive` are declared. +For the cladistics category the repository carries `frontend/src/components/ +CladeCumulus.tsx` and the `tree-rendering-clade-cumulus` workload; a +`test_clade_cumulus.jl` does **not** exist, which is what the document already says +("future, will test…"). The accepted reading is that the *benchmark category* exists +and the *unit category* remains future work, and the document is precise on that. + +**Verdict: holds**, with the naming clarified above. + +### Codecov and gitar + +**Claimed:** Codecov removed (config deleted, README badge removed, upload replaced by +a local artifact); gitar absent (`grep -i gitar` returns 0). + +**Measured:** no `codecov.yml`; no Codecov badge or slug in `README.md`; `ci.yml` +uploads coverage as a local artifact named `julia-coverage-lcov` with the step +comment "Codecov removed per Milestone 2"; `grep -rin gitar` over the tree returns +nothing. + +**Verdict: holds.** Note that the older +`docs/audit/type-system-reconnaissance.md` still *mentions* a Codecov slug pointing +at the parent repository — that document records a state that has since been +remediated, and it is a historical audit, not a current claim. + +### CI runtime + +**Claimed:** 15–20 minutes. + +**Measured:** the last four successful `main` runs took 29:18, 14:12, 15:17 and +22:21 (median ≈ 19 minutes). + +**Verdict: holds as a range, with a caveat worth recording.** The variance is +dominated by the renv restore — 79 R packages built from source, resumable from a +cache, and the single largest cost in the workflow. A cold cache is the 29-minute +case. + +### Local sandbox RAM + +**Claimed in the document:** the local sandbox is memory-blocked +("876Mi vs 2.5GB required"). + +**Corroborated incidentally:** this audit's own environment could not precompile +`DataFrames` or `XLSX` — the two heaviest direct dependencies — and the memory ceiling +is the plausible explanation; `MetaManifold` itself precompiled in 116 s once the +lighter tree was built. The audit therefore ran its Julia-side verification through a +harness that mounts the real `Provenance` module (whose only sibling dependency is +`RRuntime`) rather than the whole package. The claim is consistent with what the +environment does. + +### Project board + +**Claimed:** https://github.com/users/hyperpolymath/projects/45 — "Analysis Layer & +Cladistics Development", 11 items, PRs #11–#14 linked. + +**Measured: not verifiable with the credentials available to this audit.** The board +is a user-level Projects v2 board; reading it needs the `read:project` scope, which is +granted per-token, and the token in use during this audit carries `repo` and +`workflow` only. + +**Verdict: unverified, and recorded as such rather than assumed.** Everything else on +this page was re-derived from the repository or the API. The link and the board's own +existence are the owner's to confirm; nothing in the codebase depends on it. + +## Consequences + +- `docs/milestones/02-baseline-tests-benchmarks.md` carries dated audit notes beside + the figures it recorded, and its regression-gate section now describes the + informational comparison that replaced the gate. +- No code change is implied by this audit. The tests, the benchmarks and the CI + wiring are as the milestone describes them, except where noted above. +- Two things are deliberately left as they are: `test_clade_cumulus.jl` is still + future work, and the benchmark comparison is still informational. Both are + decisions, not oversights, and both are now written down where a reader will find + them. diff --git a/docs/milestones/02-baseline-tests-benchmarks.md b/docs/milestones/02-baseline-tests-benchmarks.md index c085ad65..4682e947 100644 --- a/docs/milestones/02-baseline-tests-benchmarks.md +++ b/docs/milestones/02-baseline-tests-benchmarks.md @@ -40,8 +40,20 @@ SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) **Audited 2026-09-22 (#15).** Re-run on `main` @ `22d228f5`: 599 pass, 5 todo, +> 0 fail, 3368 expects, **604 tests across 16 files** [461 ms]. Grown by 18 tests; +> the file count and the todo count are unchanged, so the DOM-lane debt recorded +> above has not silently grown. Full audit: +> `docs/audit/milestone-2-close-out.md`. + **Coverage (informational, no gate by policy per docs/testing/infrastructure.md):** - All files 40.19% funcs, 47.53% lines + +> **Audited 2026-09-22 (#15).** Regenerated: All files **40.23% funcs, 47.50% lines** — +> unchanged within rounding. The least-covered files are still the DOM-only ones +> (`DataTable.tsx`, `CardActions.tsx`, `NameDialog.tsx`), which is the documented +> consequence of a DOM-less unit lane rather than a regression; the e2e lane is where +> those are exercised. - src/api/client.ts 15% funcs 57% lines (many uncovered: config, analysis, etc — API client not fully tested via unit, but via integration) - src/api/errorMessage.ts 100% - src/components/CardActions.tsx 0% funcs 3.51% lines (UI not DOM-tested) @@ -58,6 +70,11 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage **Command:** `julia --project=. -t 2 --code-coverage=user test/runtests.jl` (unit always, --integration opt-in, --server opt-in) **Pathways (27 unit test files, 6830 lines total):** + +> **Audited 2026-09-22 (#15).** Measured on `main` @ `22d228f5`: **30 unit test files, +> 8310 lines**. The list below is the Milestone 2 snapshot and remains an accurate +> description of the files it names; of the current files, only `test_execution.jl` +> is not named below. Full audit: `docs/audit/milestone-2-close-out.md`. - `test_diversity.jl` (87 lines): richness (5), shannon (6), simpson (6), Normalisation rarefy (1), normalise_counts (1) — total 19 tests - `test_merge_taxa.jl` (311): merge_taxa join, tagging, max_x, category_sets - `test_config.jl` (62): config cascade, defaults, overrides @@ -118,7 +135,8 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage - taxonomy_levels (distinct ranks) - taxon_column (rank resolution) - Baseline: baseline.json with median seconds per op (5 reps) - - Regression gate: >10% fail in CI + - Baseline comparison: **informational** — see "Regression Gate" below, which was + corrected on audit 2026-09-22 (#15) 2. **bench/epistemic_parsing/benchmark.jl** - Mock epistemic types mirroring future src/core/epistemic.jl: EpistemicStatus enum present_in_every/present_in_some/absent/unknown/sans_fibre, MockCandidate observation/residual/witness, MockCase candidates @@ -156,7 +174,7 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage 6. **bench/comprehensive_benchmark.jl** - Runner for all 5 categories, writes bench/results/comprehensive_results.json - - Fails on >10% regression when CI=true + - Reports deltas against each committed baseline; does not gate (see below) **Baselines:** - Each category has baseline.json committed with placeholder medians (will be overwritten on first CI run that succeeds) @@ -202,9 +220,25 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage - julia-coverage-lcov: lcov.info - julia-benchmarks-comprehensive: bench/*/baseline.json, bench/results/comprehensive_results.json, bench/**/baseline.json -**Regression Gate:** >10% fail -- Frontend: Node script in CI fails if any workload median delta >10% vs baseline.json -- Julia: Each bench/*.jl checks baseline.json and fails if delta >10% when ENV["CI"]=="true" (via exit(1) and ::error:: annotation) +**Regression Gate:** none, deliberately — deltas are reported, never gated + +> **Corrected 2026-09-22 (#15).** This section previously recorded a hard +> ">10% fail" gate for both the frontend script and each `bench/*.jl`. That is not +> what the repository does, and it was changed on purpose after the gate was built. +> `ci.yml` carries the measurement: two consecutive runs of *identical* benchmark +> code on a hosted runner produced per-workload deltas between **-16% and +52%**, +> flapping in both directions, because the harness workloads import no application +> code — a delta therefore measures the runner, not the commit. `bench/table_loading/ +> benchmark.jl` agrees: a >10% delta prints `NOTE` and emits +> `@warn "…(informational)"`, with the comment "Never gates in CI." +> +> A gate below the noise floor blocks at random, and a gate that fails for reasons a +> commit cannot influence teaches people to ignore it. What still holds for real: the +> harness checksums hard-fail, the workload freeze policy requires re-cutting the +> baseline (visible in review), and deltas plus the machine factor are printed and +> shipped as artifacts for human review. Revisiting a same-runner A/B gate is left open +> once the harness exists on the base branch. Full audit: +> `docs/audit/milestone-2-close-out.md`. **New Test Categories:** - analysis-config: test_analysis_config.jl (AnalysisConfig creation, validation, BH mandatory, DANGER token, DOI bundle, epistemic, cloud sizing) — 21 files 5276 insertions in feature branches