From 09d4d3450ba24760c107d6ff69e3f69b5f0a23ab Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 8 Jul 2026 02:07:37 -0400 Subject: [PATCH] =?UTF-8?q?test(callaway):=20machine-precision=20golden=20?= =?UTF-8?q?tolerances=20=E2=80=94=20SE-audit=20C6=20closure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The no-covariate DR golden asserted ATT only, within 0.02, and never asserted the SE; measured agreement with R did on the fixture is ~6e-11 ATT / ~2e-11 relative SE (no-covariate DR is deterministic algebra — no propensity IRLS enters), so both are pinned at 1e-8 with the previously-missing SE assertion added. The reg (0.02) and ipw (0.05) ATT bands — predating the DRDID estimation-effect IF terms — tighten to 1e-8 on measured ~3e-11/~4e-11. The covariate-DR scenario keeps its documented 2e-3 band (~1e-3 DR small-sample numerics via the propensity nuisance). TODO SE-audit row updated: C6 closed; C7/C8 definitions need re-derivation from a fresh loose-tolerance inventory. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 8 ++++++++ TODO.md | 2 +- tests/test_csdid_ported.py | 32 ++++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b8b3e7..a56ec7ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `TestHonestFLCIParityR`); the M=0 result and all existing behaviour are unchanged. ### Testing +- **Callaway-Sant'Anna golden tolerances tightened to machine precision (SE-audit C6 + closure).** The no-covariate DR golden asserted ATT only, within 0.02, and never + asserted the SE; measured agreement with R `did` is ~6e-11 ATT / ~2e-11 relative SE + (no-covariate DR is deterministic algebra — no propensity IRLS enters), so both are + now pinned at 1e-8 with the SE assertion added. The reg (0.02) and ipw (0.05) ATT + bands — predating the DRDID estimation-effect IF terms — are tightened to 1e-8 on + measured ~3e-11/~4e-11 agreement. The covariate-DR scenario keeps its documented 2e-3 + band (~1e-3 DR small-sample numerics via the propensity nuisance). - **fixest hetero + cluster SE machine-precision locks on an unbalanced, heteroskedastic DGP (SE-audit G2 completion).** The committed `fixest_did_twfe_golden.json` gains two appended scenarios (error sd varying by arm/period, ~15% rows dropped; the original diff --git a/TODO.md b/TODO.md index a47cacd3..24f726cd 100644 --- a/TODO.md +++ b/TODO.md @@ -60,7 +60,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| -| SE-audit CI-lock — remaining "fiddly bits" after the second coverage batch landed (that batch pinned C2 `dof_hc2_bm`/`dof_per_coef` via CI-inversion, C3 LOO `df`, C4 estimatr HC1/CR1 intercept SE, C5 Yatchew `p`/`sigma2_lin`/`sigma2_diff`, and the G2 fixest cluster-SE band). Still deferred, each needing a golden regeneration, new computation, or a documented-deviation call — **(a) G2 hetero/cluster is DONE (2026-07-07)**: the unbalanced/heteroskedastic-DGP regen landed, the DiD-path hetero AND cluster CR1 SEs are machine-precision-locked (the plain-OLS CR1 matches fixest exactly — the DOF-convention deviation is absorbed-FE-only), and the TWFE cluster band-pin is retained for the documented non-nested-FE ssc deviation (tracked under "Needs external reference"); TWFE has no public unclustered-hetero surface (auto-cluster convention). Remaining: **(b) PlaceboTests `boundary_gap`** — a permutation randomization-inference margin NOT computed anywhere in code (a new feature + result field, not a coverage lock); **(c) StackedDiD intercept SEs** (`se_cr1/cr2_intercept`, C1) — MEASURED to diverge ~0.3% from R: a nuisance-parameter reference-cell/parameterization gap, NOT machine-precision lockable (the event-study interaction SEs already match ~2e-13; surfacing it would add an unasserted, R-divergent public field); **(d) estimatr `classical` intercept SE** — same documented `O(1/n)` projection/DOF deviation as the slope (reference-only, excluded from parity). Plus the tolerance-tightenings C6-C8. | `benchmarks/R/generate_fixest_did_twfe_golden.R`, `tests/test_fixest_did_twfe_parity.py`, `tests/test_methodology_stacked_did.py`, `tests/test_methodology_placebo.py` | SE-audit | Mid | Low | +| SE-audit CI-lock — remaining "fiddly bits" after the second coverage batch landed (that batch pinned C2 `dof_hc2_bm`/`dof_per_coef` via CI-inversion, C3 LOO `df`, C4 estimatr HC1/CR1 intercept SE, C5 Yatchew `p`/`sigma2_lin`/`sigma2_diff`, and the G2 fixest cluster-SE band). Still deferred, each needing a golden regeneration, new computation, or a documented-deviation call — **(a) G2 hetero/cluster is DONE (2026-07-07)**: the unbalanced/heteroskedastic-DGP regen landed, the DiD-path hetero AND cluster CR1 SEs are machine-precision-locked (the plain-OLS CR1 matches fixest exactly — the DOF-convention deviation is absorbed-FE-only), and the TWFE cluster band-pin is retained for the documented non-nested-FE ssc deviation (tracked under "Needs external reference"); TWFE has no public unclustered-hetero surface (auto-cluster convention). Remaining: **(b) PlaceboTests `boundary_gap`** — a permutation randomization-inference margin NOT computed anywhere in code (a new feature + result field, not a coverage lock); **(c) StackedDiD intercept SEs** (`se_cr1/cr2_intercept`, C1) — MEASURED to diverge ~0.3% from R: a nuisance-parameter reference-cell/parameterization gap, NOT machine-precision lockable (the event-study interaction SEs already match ~2e-13; surfacing it would add an unasserted, R-divergent public field); **(d) estimatr `classical` intercept SE** — same documented `O(1/n)` projection/DOF deviation as the slope (reference-only, excluded from parity). C6 CLOSED 2026-07-08 (dr remainder measured ~6e-11 ATT / ~2e-11 rel SE on the no-cov fixture and tightened to 1e-8 with the previously-missing SE assertion added; reg/ipw ATT bands 0.02/0.05 also tightened to 1e-8; the covariate-DR 2e-3 band stays as documented small-sample numerics). C7/C8 definitions were lost with the audit session scratchpad — re-derive from a fresh loose-tolerance inventory before acting. | `benchmarks/R/generate_fixest_did_twfe_golden.R`, `tests/test_fixest_did_twfe_parity.py`, `tests/test_methodology_stacked_did.py`, `tests/test_methodology_placebo.py` | SE-audit | Mid | Low | | Render `docs/methodology/REPORTING.md` and `REGISTRY.md` as in-site Sphinx pages so cross-refs can use `:doc:` instead of off-site `blob/main` URLs (stable-docs readers can otherwise land on a different revision than their package version). Two paths: (a) add `myst-parser` to `conf.py` + docs extras and link with `:doc:`, or (b) convert both to `.rst`. **Note:** REGISTRY.md is ~4.5k lines of LaTeX-heavy markdown — high risk under the `-W` (warnings-as-errors) Sphinx build; budget multiple rounds. | `docs/conf.py`, `docs/api/business_report.rst`, `docs/api/diagnostic_report.rst`, tutorials 18 & 19 | follow-up | Mid | Low | | `ImputationDiD` covariate-path variance lacks a dedicated parity anchor — only the no-covariate staggered panel is R-parity'd, though the covariate path shares the same validated projection code. Add a small dense-design **hand-calc** for the covariate projection (no external tooling), or a covariate (time-varying X) R `didimputation` golden asserting overall/ES SE parity (the golden variant needs local R). | `tests/test_methodology_imputation.py`, `benchmarks/R/generate_didimputation_golden.R` | imputation-validation | Mid | Low | | Add true half-sample BRR replicate-weight regressions per estimator family (current tests use Fay-like 0.5/1.5 perturbations; `test_survey_phase6.py` covers true BRR at the helper level). | `tests/test_replicate_weight_expansion.py` | #253 | Mid | Low | diff --git a/tests/test_csdid_ported.py b/tests/test_csdid_ported.py index ff6f473b..43baa853 100644 --- a/tests/test_csdid_ported.py +++ b/tests/test_csdid_ported.py @@ -948,7 +948,17 @@ def _run_scenario( return results, scenario["results"] def test_golden_default_dgp_dr(self, golden_values): - """ATT(g,t) from default no-covariate DGP match R within 1e-2.""" + """No-covariate DR matches R - ATT AND per-cell SE, machine precision. + + SE-audit C6 (dr remainder): measured agreement on this fixture is + ~6e-11 ATT / ~2e-11 relative SE (no-covariate DR is deterministic + algebra — no propensity IRLS enters), so 1e-8 is pure platform + headroom. The previous 0.02 ATT band predates the DRDID + estimation-effect IF terms; the SE was entirely unasserted here. + (The covariate DR scenario keeps its documented 2e-3 band — the + ~1e-3 gap is DR small-sample numerics via the propensity nuisance, + see test_golden_default_dgp_dr_covariates.) + """ results, expected = self._run_scenario( golden_values, "default_no_covariates_dr", @@ -959,11 +969,14 @@ def test_golden_default_dgp_dr(self, golden_values): for i, (g, t) in enumerate(zip(r_gt["group"], r_gt["time"])): g, t = int(g), int(t) if (g, t) in results.group_time_effects: - py_att = results.group_time_effects[(g, t)]["effect"] + eff = results.group_time_effects[(g, t)] r_att = r_gt["att"][i] assert ( - abs(py_att - r_att) < 0.02 - ), f"ATT(g={g}, t={t}): Python={py_att:.6f}, R={r_att:.6f}" + abs(eff["effect"] - r_att) < 1e-8 + ), f"DR ATT(g={g}, t={t}): Python={eff['effect']:.10f}, R={r_att:.10f}" + assert ( + abs(eff["se"] - r_gt["se"][i]) < 1e-8 + ), f"DR SE(g={g}, t={t}): Py={eff['se']:.10f}, R={r_gt['se'][i]:.10f}" def test_golden_default_dgp_reg(self, golden_values): """Regression method with covariates matches R - ATT AND per-cell SE. @@ -987,9 +1000,10 @@ def test_golden_default_dgp_reg(self, golden_values): if (g, t) in results.group_time_effects: eff = results.group_time_effects[(g, t)] r_att = r_gt["att"][i] + # Measured ~3e-11 (deterministic OLS); 1e-8 is platform headroom. assert ( - abs(eff["effect"] - r_att) < 0.02 - ), f"REG ATT(g={g}, t={t}): Py={eff['effect']:.6f}, R={r_att:.6f}" + abs(eff["effect"] - r_att) < 1e-8 + ), f"REG ATT(g={g}, t={t}): Py={eff['effect']:.10f}, R={r_att:.10f}" assert ( abs(eff["se"] - r_gt["se"][i]) < 1e-8 ), f"REG SE(g={g}, t={t}): Py={eff['se']:.10f}, R={r_gt['se'][i]:.10f}" @@ -1018,9 +1032,11 @@ def test_golden_default_dgp_ipw(self, golden_values): if (g, t) in results.group_time_effects: eff = results.group_time_effects[(g, t)] r_att = r_gt["att"][i] + # Measured ~4e-11; 1e-8 leaves the same IRLS-vs-glm headroom + # rationale as the SE band below (scaled to the tighter base). assert ( - abs(eff["effect"] - r_att) < 0.05 - ), f"IPW ATT(g={g}, t={t}): Py={eff['effect']:.6f}, R={r_att:.6f}" + abs(eff["effect"] - r_att) < 1e-8 + ), f"IPW ATT(g={g}, t={t}): Py={eff['effect']:.10f}, R={r_att:.10f}" assert ( abs(eff["se"] - r_gt["se"][i]) < 1e-6 ), f"IPW SE(g={g}, t={t}): Py={eff['se']:.10f}, R={r_gt['se'][i]:.10f}"