From dcbde46f95b1256f7066154e5f818dbbcc049a5b Mon Sep 17 00:00:00 2001 From: igerber Date: Thu, 2 Jul 2026 09:26:54 -0400 Subject: [PATCH] feat(synthetic_control): split cv-infeasible from failed in placebo/LOO diagnostics Under v_method="cv", an excluded in_space_placebo() / leave_one_out() refit whose (pseudo-treated / reduced) donor pool is indistinguishable in a re-aggregated CV window is now tallied as a structural "infeasible" exclusion, distinct from a solver "failed" non-convergence -- mirroring the split in_time_placebo already reports. - _outer_solve_V_cv returns a 6th structural-infeasible flag; _placebo_fit_unit returns a (result, status) tuple with status in {ran, infeasible, failed}. - in_space adds a public n_infeasible field; leave_one_out adds _loo_n_infeasible. Statuses gain all_placebos_infeasible / all_placebos_unusable (resp. all_refits_*); the per-row status column carries "infeasible"; DiagnosticReport surfaces reason_code + n_failed / n_infeasible for both surfaces. - placebo_p_value / n_placebos are unchanged (both causes excluded from the rank / ATT range identically) -- only the diagnostic attribution is refined. - __setstate__ backfills the new counters to 0 on legacy unpickle so summary() / to_dict() never AttributeError on a result pickled by an older version. REGISTRY + CHANGELOG document the split; the TODO row is resolved. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 16 ++ TODO.md | 1 - diff_diff/diagnostic_report.py | 44 ++- diff_diff/synthetic_control.py | 76 ++++-- diff_diff/synthetic_control_results.py | 287 +++++++++++++++----- docs/methodology/REGISTRY.md | 3 +- tests/test_methodology_synthetic_control.py | 218 ++++++++++++++- 7 files changed, 534 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f4da379..1478d7f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `cluster_name`. The mass-point path and the event-study (Phase 2b) path are unchanged (Phase 2b still defers cluster with a warning). +### Changed +- **`SyntheticControl` in-space / leave-one-out placebo diagnostics now distinguish structural `cv` + infeasibility from solver non-convergence.** Under `v_method="cv"`, an excluded `in_space_placebo()` + / `leave_one_out()` refit whose pseudo-treated (in-space) or reduced (leave-one-out) donor pool is + indistinguishable in a re-aggregated CV window — a structural identification failure, not an + under-optimized solve — is now tallied in a new `n_infeasible` field (in-space) / `_loo_n_infeasible` + (leave-one-out) with a `status="infeasible"` row, mirroring the split `in_time_placebo` already + reported; `n_failed` now counts only genuine solver non-convergences. `_placebo_status` / + `_loo_status` gain `all_placebos_infeasible` / `all_placebos_unusable` (resp. + `all_refits_infeasible` / `all_refits_unusable`) codes, and `DiagnosticReport` surfaces a + machine-readable `reason_code` alongside `n_failed` / `n_infeasible`. The permutation + `placebo_p_value` / `n_placebos` are UNCHANGED — both causes are excluded from the rank / ATT range + identically, so only the diagnostic attribution is refined. `n_infeasible` is 0 for the non-`cv` + `v_method`s (no structural-identification gate). Internal: `_placebo_fit_unit` now returns a + `(result, status)` tuple and `_outer_solve_V_cv` a structural-infeasible flag. + ## [3.6.1] - 2026-07-01 ### Added diff --git a/TODO.md b/TODO.md index 8f91dfcb..1b20054c 100644 --- a/TODO.md +++ b/TODO.md @@ -28,7 +28,6 @@ The `Origin` column (Actionable tables) and the `PR` column (Deferred tables) bo | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| -| `SyntheticControl` cv: thread an `"infeasible"` reason-code from `_outer_solve_V_cv()` / `_placebo_fit_unit()` so `in_space_placebo()` / `leave_one_out()` distinguish a structural cv-refit exclusion (donor-indistinguishable re-aggregated window) from a genuine inner-solver non-convergence — mirror the split `in_time_placebo()` already emits. Warnings already distinguish the two causes; only the machine-readable status/count is missing. | `synthetic_control.py`, `synthetic_control_results.py` | follow-up | Mid | Low | | Survey-design resolution / collapse patterns are inconsistent across panel estimators — `ContinuousDiD` rebuilds unit-level design in SE code, `EfficientDiD` builds once in `fit()`, `StackedDiD` re-resolves on stacked data. Extract shared helpers for panel-to-unit collapse, post-filter re-resolution, and metadata recomputation. | `continuous_did.py`, `efficient_did.py`, `stacked_did.py` | #226 | Mid | Low | | `SyntheticControl` remaining ADH-2015 §4 items: the regression-weight `W^reg = X_0'(X_0 X_0')^{-1} X_1` extrapolation diagnostic (flag implied OLS weights outside `[0,1]`) and sparse-SC subset search (`l < J`, holding `V` fixed). LOO, in-time placebo, CV `V`-selection, and inverse-variance `V` have landed; these two are the deferred tail. | `synthetic_control.py`, `synthetic_control_results.py` | ADH-2015 | Mid | Low | | `SyntheticControl` conformal (CWZ 2021) extensions: (a) one-sided / signed-`t` variants (§7); (b) covariates in the conformal proxy (`X_jt`, eqs 4/6 — current proxy is outcomes-only); (c) AR / innovation-permutation path (Lemmas 5-7) for time-series proxies. The joint test, pointwise CIs, and average-effect CI have landed. | `conformal.py`, `synthetic_control_results.py` | CWZ-2021 | Heavy | Low | diff --git a/diff_diff/diagnostic_report.py b/diff_diff/diagnostic_report.py index 6e49ba0c..b6ce2b17 100644 --- a/diff_diff/diagnostic_report.py +++ b/diff_diff/diagnostic_report.py @@ -2287,6 +2287,7 @@ def _scm_native(self, r: Any) -> Dict[str, Any]: "rmspe_ratio": _to_python_float(getattr(r, "rmspe_ratio", None)), "n_placebos": _to_python_scalar(n_placebos), "n_failed": _to_python_scalar(getattr(r, "n_failed", None)), + "n_infeasible": _to_python_scalar(getattr(r, "n_infeasible", None)), } # Distinguish a valid run from an attempted-but-infeasible one so BR/DR # consumers see an explicit status/reason rather than a bare NaN p-value. @@ -2312,15 +2313,35 @@ def _scm_native(self, r: Any) -> Dict[str, Any]: "all_placebos_failed": ( "in_space_placebo() was run but every donor refit failed to " "converge, so no placebo entered the reference set; " - "placebo_p_value is NaN." + "placebo_p_value is NaN. Raise n_starts or loosen the " + "optimizer tolerances." + ), + "all_placebos_infeasible": ( + "in_space_placebo() was run but every donor refit was " + "structurally infeasible under v_method='cv' (the " + "pseudo-treated donor pool is indistinguishable in a " + "re-aggregated CV window), so no placebo entered the reference " + "set; placebo_p_value is NaN. Adjust the predictors, v_cv_t0, " + "or the donor pool." + ), + "all_placebos_unusable": ( + "in_space_placebo() was run but no donor refit was usable: " + "some failed to converge AND some were structurally infeasible " + "(see n_failed / n_infeasible); placebo_p_value is NaN." ), } block["status"] = "infeasible" + # Machine-readable code distinguishing a solver convergence failure + # ("all_placebos_failed") from structural infeasibility + # ("all_placebos_infeasible" / "too_few_donors") or a mix + # ("all_placebos_unusable"), without parsing `reason`. + block["reason_code"] = placebo_status block["reason"] = _reasons.get( placebo_status, "in_space_placebo() was run but produced no valid reference set " "(fewer than 2 donors, a non-converged treated fit, or all donor " - "refits failed); placebo_p_value is NaN.", + "refits failed / were structurally infeasible); placebo_p_value is " + "NaN.", ) out["in_space_placebo"] = block else: @@ -2353,6 +2374,7 @@ def _scm_native(self, r: Any) -> Dict[str, Any]: else None ), "n_failed": _to_python_scalar(getattr(r, "_loo_n_failed", None)), + "n_infeasible": _to_python_scalar(getattr(r, "_loo_n_infeasible", None)), } else: _loo_reasons = { @@ -2371,13 +2393,29 @@ def _scm_native(self, r: Any) -> Dict[str, Any]: "(see the status='failed' rows); raise n_starts or loosen the " "optimizer tolerances." ), + "all_refits_infeasible": ( + "leave_one_out() was run but every donor-drop refit was " + "structurally infeasible under v_method='cv' (the reduced donor " + "pool is indistinguishable in a re-aggregated CV window; see the " + "status='infeasible' rows); adjust the predictors, v_cv_t0, or " + "the donor pool." + ), + "all_refits_unusable": ( + "leave_one_out() was run but no donor-drop refit was usable: " + "some failed to converge AND some were structurally infeasible " + "(see n_failed / n_infeasible)." + ), } out["leave_one_out"] = { "status": "infeasible", # Machine-readable code so consumers can distinguish a numerical # convergence failure ("all_refits_failed") from structural - # infeasibility ("too_few_donors") without parsing `reason`. + # infeasibility ("all_refits_infeasible" / "too_few_donors") or a + # mix ("all_refits_unusable") without parsing `reason`. The n_failed + # / n_infeasible counts give the exact breakdown. "reason_code": loo_status, + "n_failed": _to_python_scalar(getattr(r, "_loo_n_failed", None)), + "n_infeasible": _to_python_scalar(getattr(r, "_loo_n_infeasible", None)), "reason": _loo_reasons.get( loo_status, "leave_one_out() produced no valid refits." ), diff --git a/diff_diff/synthetic_control.py b/diff_diff/synthetic_control.py index 7b396047..1cfc13f2 100644 --- a/diff_diff/synthetic_control.py +++ b/diff_diff/synthetic_control.py @@ -539,7 +539,10 @@ def fit( # the predictor precondition were resolved/validated up front (above), so # resolved_v_cv_t0 is a valid int here. assert resolved_v_cv_t0 is not None - v, w, converged, mspe_v, outer_converged = _outer_solve_V_cv( + # The 6th return (structural-infeasible flag) is for the placebo/LOO path; + # fit() enforces the fully-spanning + donor-variation precondition up front + # (raising ValueError), so the headline solve never returns it -> ignore here. + v, w, converged, mspe_v, outer_converged, _ = _outer_solve_V_cv( pivots, specs, treated_id, @@ -1510,7 +1513,7 @@ def _outer_solve_V_cv( inner_max_iter: int, inner_min_decrease: float, standardize: str, -) -> Tuple[np.ndarray, np.ndarray, bool, float, bool]: +) -> Tuple[np.ndarray, np.ndarray, bool, float, bool, Optional[str]]: """Out-of-sample cross-validation V selection (ADH 2015 §; Abadie 2021 Eq. 9). Faithful per-window re-aggregation. The pre-period is split positionally at ``t0`` @@ -1541,9 +1544,16 @@ def _outer_solve_V_cv( every re-aggregated spec is non-empty; this guards defensively and fails closed (non-converged sentinel) otherwise. - Returns ``(v_star, w_star, inner_converged, mspe_v, outer_converged)`` where - ``mspe_v`` is the step-3 validation-MSPE selection criterion at V* (the validation - MSPE of the TRAINING-window fit). + Returns ``(v_star, w_star, inner_converged, mspe_v, outer_converged, infeasible)`` + where ``mspe_v`` is the step-3 validation-MSPE selection criterion at V* (the + validation MSPE of the TRAINING-window fit), and ``infeasible`` is ``"infeasible"`` + when the failure is STRUCTURAL (a non-spanning spec set or a re-aggregated window + with no cross-donor variation — the weights are unidentified, not merely + under-optimized) and ``None`` otherwise. The caller (``_placebo_fit_unit``) uses it + to report a dropped placebo as ``status="infeasible"`` (remedied by adjusting the + predictors / ``v_cv_t0`` / donor pool) vs ``status="failed"`` (a solver + non-convergence, remedied by ``inner_max_iter`` / ``n_starts``) — mirroring the + split ``in_time_placebo`` already emits. """ k = len(specs) train_specs = _truncate_specs_to_window(specs, set(pre_periods[:t0])) @@ -1557,7 +1567,7 @@ def _outer_solve_V_cv( # this guards the path defensively, returning a non-converged sentinel so the caller # drops/raises rather than crashing on an empty predictor build. if any(s is None for s in train_specs) or any(s is None for s in val_specs): - return np.ones(k) / k, np.zeros(len(donor_ids)), False, float("nan"), False + return np.ones(k) / k, np.zeros(len(donor_ids)), False, float("nan"), False, "infeasible" train_specs_c = cast(List["_PredictorSpec"], train_specs) val_specs_c = cast(List["_PredictorSpec"], val_specs) @@ -1574,7 +1584,7 @@ def _outer_solve_V_cv( # placebos reassign the treated role and shrink the donor pool, so donor-distinguishability # must be re-checked for each pseudo-treated unit, not just the headline.) if not _window_has_donor_variation(X0_tr) or not _window_has_donor_variation(X0_va): - return np.ones(k) / k, np.zeros(len(donor_ids)), False, float("nan"), False + return np.ones(k) / k, np.zeros(len(donor_ids)), False, float("nan"), False, "infeasible" X1s_tr, X0s_tr, _ = _standardize(X1_tr, X0_tr, standardize) X1s_va, X0s_va, _ = _standardize(X1_va, X0_va, standardize) @@ -1589,7 +1599,9 @@ def _outer_solve_V_cv( # non-converged) so downstream placebo/LOO diagnostics do not run off an invalid CV # criterion (there is no outer search here, so conv_tr IS the "search" convergence). mspe_v = float(np.mean((Z1_va - Z0_va @ w_tr) ** 2)) if conv_tr else float("nan") - return v, w_star, converged, mspe_v, conv_tr + # conv_tr=False here is a SOLVER truncation (training fit did not converge), not a + # structural infeasibility -> infeasible=None so the caller reports "failed". + return v, w_star, converged, mspe_v, conv_tr, None _st = {"total": 0, "nonconv": 0} # Penalty bound on the VALIDATION window (the objective's window), so a truncated @@ -1729,7 +1741,9 @@ def _is_better( outer_converged = False # Step 4 reported weights: refit V* on the VALIDATION-window re-aggregated predictors. w_star, converged = _inner_solve_W(X1s_va, X0s_va, v_star, inner_max_iter, inner_min_decrease) - return v_star, w_star, converged, mspe_v, outer_converged + # A non-converged outer search / truncated step-4 refit is a SOLVER failure (the + # windows spanned + varied, else we returned "infeasible" above) -> infeasible=None. + return v_star, w_star, converged, mspe_v, outer_converged, None def _compute_gap_path( @@ -2133,23 +2147,39 @@ def _placebo_fit_unit( unit: Any, donor_pool: List[Any], n_starts: int, -) -> Optional[Tuple[Dict[Any, float], float]]: +) -> Tuple[Optional[Tuple[Dict[Any, float], float]], str]: """Refit a synthetic control for one (pseudo-)treated ``unit`` vs ``donor_pool``. Reuses the exact predictor build / standardization / weight solve / gap-path path as ``fit()`` — none of those helpers reads estimator (``self``) state, so the refit is driven entirely by the snapshot. The real treated unit is never in - ``donor_pool`` (the caller passes the other ``J−1`` donors). Returns - ``(gap_path, rmspe_ratio)``, or ``None`` when the weight solve does not converge - (the caller excludes such placebos from the permutation reference set). - Per-placebo ``UserWarning``s (poor fit, zero-variance row, non-convergence) are - suppressed here; the caller surfaces an aggregate count. + ``donor_pool`` (the caller passes the other ``J−1`` donors). + + Returns ``(result, status)`` where ``status`` is one of: + + - ``"ran"`` — ``result`` is ``(gap_path, rmspe_ratio)`` (a valid optimum). + - ``"infeasible"`` — ``result`` is ``None``; the refit is STRUCTURALLY + unidentified (non-finite cells, or — under ``v_method="cv"`` — a re-aggregated + window with no cross-donor variation for this pseudo-treated pool). Remedied by + adjusting the predictors / ``v_cv_t0`` / donor pool, NOT the optimizer budget. + - ``"failed"`` — ``result`` is ``None``; the refit reached no valid optimum (a + non-converged inner weight solve or outer V search). Remedied by a larger + ``inner_max_iter`` / ``n_starts``. + + The caller excludes both non-``"ran"`` outcomes from the permutation reference set + but counts them separately (``n_failed`` vs ``n_infeasible``), mirroring the split + ``in_time_placebo`` already emits. Per-placebo ``UserWarning``s (poor fit, + zero-variance row, non-convergence) are suppressed here; the caller surfaces an + aggregate count. """ X1, X0, _ = _build_predictor_matrix(snap.pivots, snap.specs, unit, donor_pool) # Belt-and-suspenders: fit() already gated non-finite outcomes over the full # treated+donor panel, so a donor reassigned as pseudo-treated has finite cells. if not (np.all(np.isfinite(X1)) and np.all(np.isfinite(X0))): - return None + # Non-finite predictor cells make the pool structurally unfittable (a data + # problem, not a solver hiccup) -> "infeasible". (Defensive: fit() already gated + # finiteness over the full treated+donor panel.) + return None, "infeasible" # inverse_variance applies 1/Var to the RAW predictors; cv re-standardizes each window # separately inside _outer_solve_V_cv (mirrors fit()). Both skip the full-pre # _standardize pass (unused on those paths) and its irrelevant zero-variance warning. @@ -2161,8 +2191,11 @@ def _placebo_fit_unit( Z1 = Y.loc[snap.pre_periods, unit].to_numpy(dtype=float) Z0 = Y.loc[snap.pre_periods, donor_pool].to_numpy(dtype=float) # ``outer_converged`` is trivially True when there is no outer V search (custom - # V or a single-donor degenerate pool). + # V or a single-donor degenerate pool). ``infeasible_reason`` is set to + # ``"infeasible"`` only by the cv path's structural sentinel (below); every other + # path's non-convergence is a solver failure. outer_converged = True + infeasible_reason: Optional[str] = None with warnings.catch_warnings(): warnings.simplefilter("ignore") if snap.v_method == "custom": @@ -2193,7 +2226,7 @@ def _placebo_fit_unit( # with an out-of-range pinned t0 nulled to the default). n_pre = len(snap.pre_periods) t0 = snap.v_cv_t0 if snap.v_cv_t0 is not None else n_pre // 2 - _, w, converged, _, outer_converged = _outer_solve_V_cv( + _, w, converged, _, outer_converged, infeasible_reason = _outer_solve_V_cv( snap.pivots, snap.specs, unit, @@ -2226,14 +2259,15 @@ def _placebo_fit_unit( # Exclude a placebo whose fit is not a valid optimum — a truncated inner W OR an # under-optimized outer V search. An under-optimized placebo V fits the pre-period # worse, shrinking its RMSPE ratio and biasing the permutation p-value - # anti-conservatively, so such placebos must not silently enter the rank. + # anti-conservatively, so such placebos must not silently enter the rank. Attribute + # a cv structural sentinel to "infeasible"; every other non-convergence to "failed". if not (converged and outer_converged): - return None + return None, ("infeasible" if infeasible_reason == "infeasible" else "failed") gap_path = _compute_gap_path(Y, w, unit, donor_pool, snap.all_periods) pre_gaps = np.array([gap_path[p] for p in snap.pre_periods], dtype=float) post_gaps = np.array([gap_path[p] for p in snap.post_periods], dtype=float) scale = float(np.max(np.abs(Z1))) if Z1.size else 0.0 - return gap_path, _rmspe_ratio(pre_gaps, post_gaps, scale) + return (gap_path, _rmspe_ratio(pre_gaps, post_gaps, scale)), "ran" # ============================================================================= diff --git a/diff_diff/synthetic_control_results.py b/diff_diff/synthetic_control_results.py index b8049775..0b8ba829 100644 --- a/diff_diff/synthetic_control_results.py +++ b/diff_diff/synthetic_control_results.py @@ -199,9 +199,14 @@ class SyntheticControlResults: In-space placebo permutation p-value (``rank / (n_placebos + 1)``), NaN until :meth:`in_space_placebo` is run. SEPARATE from the (always-NaN) analytical ``p_value``; ``is_significant`` stays bound to ``p_value``. - n_placebos, n_failed : int + n_placebos, n_failed, n_infeasible : int Donor placebos that entered the permutation reference set / were excluded - for non-convergence. Both 0 until :meth:`in_space_placebo` is run. + for solver non-convergence / were excluded as structurally infeasible (under + ``v_method="cv"``, a re-aggregated window with no cross-donor variation once + that donor is pseudo-treated). All 0 until :meth:`in_space_placebo` is run. + ``n_infeasible`` mirrors the split :meth:`in_time_placebo` already reports; the + permutation ``placebo_p_value`` uses only the ``n_placebos`` that entered the + rank, so it is unaffected by how the excluded remainder is attributed. survey_metadata : Any, optional Reserved; always None in this release. @@ -245,6 +250,13 @@ class SyntheticControlResults: rmspe_ratio: float = np.nan n_placebos: int = 0 n_failed: int = 0 + # Donor placebos excluded as STRUCTURALLY infeasible (distinct from n_failed's solver + # non-convergence): under v_method="cv", pseudo-treating a donor can leave a + # re-aggregated CV window with no cross-donor variation, so the weights are + # unidentified. 0 until in_space_placebo() runs. Mirrors the split in_time_placebo + # reports via _in_time_n_infeasible. Excluded from the permutation rank just like + # n_failed, so placebo_p_value is unaffected by the attribution. + n_infeasible: int = 0 # Confidence set for the treatment-effect path by test inversion (Firpo & Possebom # 2018, "Synthetic Control Method: Inference, Sensitivity Analysis and Confidence # Sets," J. Causal Inference 6(2), §4), populated by ``confidence_set()``. A small @@ -276,7 +288,10 @@ def __post_init__(self) -> None: # tell a non-converged treated fit ("treated_fit_nonconverged", n_failed=0) # apart from too few donors ("too_few_donors", also n_failed=0). Values: # None (not run), "ran", "treated_fit_nonconverged", "too_few_donors", - # "all_placebos_failed". A small string, so it survives pickling. + # "all_placebos_failed" (every excluded donor was a solver non-convergence), + # "all_placebos_infeasible" (every excluded donor was structurally infeasible), + # "all_placebos_unusable" (a MIX of failed + infeasible with none usable) — + # mirrors the in_time_placebo split. A small string, so it survives pickling. self._placebo_status: Optional[str] = None # Per-unit floored pre-period denominators (treated + each converged placebo), # captured by in_space_placebo() so the sharp-null test inversion @@ -295,7 +310,10 @@ def __post_init__(self) -> None: self._loo_df: Optional[pd.DataFrame] = None self._loo_gaps: Optional[Dict[Any, Dict[Any, float]]] = None # Reason a leave-one-out run was infeasible/absent. Values: None (not run), - # "ran", "treated_fit_nonconverged", "too_few_donors", "all_refits_failed". + # "ran", "treated_fit_nonconverged", "too_few_donors", "all_refits_failed" + # (all excluded drops were solver non-convergences), "all_refits_infeasible" + # (all excluded drops were structurally infeasible), "all_refits_unusable" (a + # MIX with none usable) — mirrors the in_time_placebo split. self._loo_status: Optional[str] = None # (min, max) ATT across the successful leave-one-out refits (the absolute # spread of counterfactual ATTs); None until run. @@ -306,6 +324,10 @@ def __post_init__(self) -> None: # would be. None until run. self._loo_max_abs_delta_att: Optional[float] = None self._loo_n_failed: int = 0 + # Leave-one-out drops excluded as STRUCTURALLY infeasible (cv donor-pool + # indistinguishability), distinct from _loo_n_failed's solver non-convergence. + # Mirrors _in_time_n_infeasible. 0 until leave_one_out() runs. + self._loo_n_infeasible: int = 0 self._in_time_df: Optional[pd.DataFrame] = None self._in_time_gaps: Optional[Dict[Any, Dict[Any, float]]] = None # Reason an in-time placebo run was infeasible/absent. Values: None (not run), @@ -343,10 +365,10 @@ def __getstate__(self) -> Dict[str, Any]: ``_fit_snapshot`` retains the full treated+donor panel and ``_placebo_gaps`` the per-unit gap paths — both panel-derived, a privacy/size hazard if the pickle is sent elsewhere. The scalar placebo fields (``placebo_p_value``, - ``rmspe_ratio``, ``n_placebos``, ``n_failed``) and the small ``_placebo_df`` - aggregate table survive. An unpickled result keeps all public fields; a - diagnostic call that needs the snapshot (``in_space_placebo``) then raises a - ValueError directing the user to re-fit. Mirrors ``SyntheticDiDResults``. + ``rmspe_ratio``, ``n_placebos``, ``n_failed``, ``n_infeasible``) and the small + ``_placebo_df`` aggregate table survive. An unpickled result keeps all public + fields; a diagnostic call that needs the snapshot (``in_space_placebo``) then + raises a ValueError directing the user to re-fit. Mirrors ``SyntheticDiDResults``. """ state = self.__dict__.copy() state["_fit_snapshot"] = None @@ -359,6 +381,20 @@ def __getstate__(self) -> Dict[str, Any]: state["_in_time_gaps"] = None return state + def __setstate__(self, state: Dict[str, Any]) -> None: + """Restore pickled state, backfilling scalar diagnostic fields added later. + + Unpickling bypasses ``__init__`` / ``__post_init__``, so a pickle written by an + OLDER version (before ``n_infeasible`` / ``_loo_n_infeasible`` existed) would + otherwise leave those attributes unset and make ``summary()`` / ``to_dict()`` / + ``DiagnosticReport`` raise ``AttributeError``. Default any missing counter to 0 + (the "no infeasible refits recorded" state) so a legacy result reports cleanly. + """ + self.__dict__.update(state) + for _attr, _default in (("n_infeasible", 0), ("_loo_n_infeasible", 0)): + if not hasattr(self, _attr): + setattr(self, _attr, _default) + def __repr__(self) -> str: """Concise string representation.""" return ( @@ -507,6 +543,18 @@ def summary(self, alpha: Optional[float] = None) -> str: if placebo_attempted and np.isfinite(self.placebo_p_value): # The classic analytical fields above stay n/a (no SE); this is the # permutation p-value of the post/pre RMSPE ratio, p = rank/(n_placebos+1). + # Excluded donors split into solver failures + structural cv infeasibilities; + # show the breakdown when any donor was infeasible so the two are not conflated. + n_excluded = self.n_failed + self.n_infeasible + if n_excluded and self.n_infeasible: + excluded_suffix = ( + f" ({n_excluded} excluded: {self.n_failed} failed, " + f"{self.n_infeasible} infeasible)" + ) + elif n_excluded: + excluded_suffix = f" ({n_excluded} excluded)" + else: + excluded_suffix = "" lines.extend( [ "In-space placebo permutation inference " @@ -514,7 +562,7 @@ def summary(self, alpha: Optional[float] = None) -> str: f"{' RMSPE ratio (post/pre):':<34} {self.rmspe_ratio:>10.4f}", f"{' Permutation p-value:':<34} {self.placebo_p_value:>10.4f}", f"{' Placebos in reference set:':<34} {self.n_placebos:>10d}" - + (f" ({self.n_failed} excluded)" if self.n_failed else ""), + + excluded_suffix, "", "(Analytical SE is still undefined for classic SCM; the " "p-value above is permutation-based.)", @@ -542,6 +590,23 @@ def summary(self, alpha: Optional[float] = None) -> str: "placebo is fit against the other donors); too few were", "available. placebo_p_value is undefined. Inspect " "get_placebo_df().", ] + elif status == "all_placebos_infeasible": + reason = [ + "In-space placebo permutation inference was attempted but every " + "donor refit was structurally infeasible", + f"({self.n_infeasible} of {self.n_donors}; under v_method='cv' the " + "pseudo-treated donor pool is indistinguishable in a", + "re-aggregated CV window). placebo_p_value is undefined — adjust the " + "predictors / v_cv_t0 / donor pool. Inspect get_placebo_df().", + ] + elif status == "all_placebos_unusable": + reason = [ + "In-space placebo permutation inference was attempted but no donor " + "refit was usable", + f"({self.n_failed} failed to converge, {self.n_infeasible} " + "structurally infeasible under v_method='cv').", + "placebo_p_value is undefined. Inspect get_placebo_df().", + ] else: # "all_placebos_failed" (or a legacy unpickle without the status) reason = [ "In-space placebo permutation inference was attempted but " @@ -595,12 +660,13 @@ def to_dict(self) -> Dict[str, Any]: "v_cv_t0": self.v_cv_t0, "standardize": self.standardize, # In-space placebo permutation inference. rmspe_ratio is set at fit; - # placebo_p_value / n_placebos / n_failed stay at their no-inference - # defaults (NaN / 0) until in_space_placebo() runs. + # placebo_p_value / n_placebos / n_failed / n_infeasible stay at their + # no-inference defaults (NaN / 0) until in_space_placebo() runs. "rmspe_ratio": self.rmspe_ratio, "placebo_p_value": self.placebo_p_value, "n_placebos": self.n_placebos, "n_failed": self.n_failed, + "n_infeasible": self.n_infeasible, } # Test-inversion confidence set (Firpo & Possebom 2018), flattened to scalars so # to_dataframe() stays a single row of scalars; all None until confidence_set() @@ -719,9 +785,9 @@ def in_space_placebo( Reassigns the treatment to each donor in turn, re-estimates a synthetic control for that pseudo-treated donor against the OTHER donors, and ranks the real treated unit's post/pre RMSPE ratio among all units. Populates - ``placebo_p_value``, ``n_placebos`` and ``n_failed`` on this object - (``rmspe_ratio`` — the treated unit's own ratio — is set at fit time) and - returns the placebo distribution via :meth:`get_placebo_df`. + ``placebo_p_value``, ``n_placebos``, ``n_failed`` and ``n_infeasible`` on + this object (``rmspe_ratio`` — the treated unit's own ratio — is set at fit + time) and returns the placebo distribution via :meth:`get_placebo_df`. The real treated unit is **excluded from every placebo's donor pool**: its post-period outcome is treatment-contaminated, so allowing a placebo to @@ -737,12 +803,19 @@ def in_space_placebo( ``p_value`` (which stays NaN — classic SCM has no analytical SE) and from ``is_significant`` (which also stays bound to the NaN ``p_value``). - A placebo is **excluded** from the reference set (counted in ``n_failed``) - when its fit is not a valid optimum — EITHER its inner Frank-Wolfe weight - solve did not converge (a truncated ``W`` is unusable) OR its outer ``V`` - search did not converge (an under-optimized ``V`` fits the pre-period worse, - shrinking its RMSPE ratio and biasing the permutation p-value - anti-conservatively). Each placebo refit **inherits the original fit's + A placebo is **excluded** from the reference set for one of two reasons, + counted separately. A **solver non-convergence** (counted in ``n_failed``, + ``status="failed"``) is EITHER an inner Frank-Wolfe weight solve that did not + converge (a truncated ``W`` is unusable) OR an outer ``V`` search that did not + converge (an under-optimized ``V`` fits the pre-period worse, shrinking its + RMSPE ratio and biasing the permutation p-value anti-conservatively). A + **structural cv infeasibility** (counted in ``n_infeasible``, + ``status="infeasible"``; ``v_method="cv"`` only) is a pseudo-treated donor + pool that is indistinguishable in a re-aggregated CV window, so the weights are + unidentified — remedied by adjusting the predictors / ``v_cv_t0`` / donor pool, + NOT the optimizer budget. Both are excluded from the rank identically, so + ``placebo_p_value`` is unaffected by the attribution. Each placebo refit + **inherits the original fit's ``optimizer_options`` / ``n_starts``**, so valid inference requires settings adequate for the outer ``V`` search to converge: production defaults do; with cheap settings, raise ``n_starts`` here or re-fit with a larger @@ -833,6 +906,7 @@ def in_space_placebo( self.placebo_p_value = np.nan self.n_placebos = 0 self.n_failed = 0 + self.n_infeasible = 0 self._placebo_gaps = {} self._placebo_pre_denoms = {} self._placebo_status = "treated_fit_nonconverged" @@ -850,6 +924,7 @@ def in_space_placebo( self.placebo_p_value = np.nan self.n_placebos = 0 self.n_failed = 0 + self.n_infeasible = 0 self._placebo_gaps = {} self._placebo_pre_denoms = {} self._placebo_status = "too_few_donors" @@ -869,17 +944,22 @@ def in_space_placebo( placebo_gaps: Dict[Any, Dict[Any, float]] = {} ranked_ratios: List[float] = [] n_failed = 0 + n_infeasible = 0 for j in donors: pool = [d for d in donors if d != j] - fitted = _placebo_fit_unit(snap, j, pool, n_starts_eff) + fitted, fit_status = _placebo_fit_unit(snap, j, pool, n_starts_eff) if fitted is None: - # Non-converged inner Frank-Wolfe weight solve (a truncated W is - # unusable for ranking): exclude from BOTH the numerator and the - # denominator (never penalize a truncated solve into the rank). - # Still record the donor with NaN metrics so get_placebo_df() - # returns the full treated + every-donor unit set. - n_failed += 1 + # Excluded from BOTH the numerator and the denominator (never rank a + # non-optimal fit). "failed" (a truncated inner W / outer V search) and + # "infeasible" (a structural cv donor-indistinguishability for this + # pseudo-treated pool) are dropped alike but COUNTED separately, mirroring + # the split in_time_placebo reports. Still record the donor with NaN + # metrics so get_placebo_df() returns the full treated + every-donor set. + if fit_status == "infeasible": + n_infeasible += 1 + else: + n_failed += 1 rows.append( { "unit": j, @@ -887,7 +967,7 @@ def in_space_placebo( "post_mspe": np.nan, "rmspe_ratio": np.nan, "is_treated": False, - "status": "failed", + "status": fit_status, } ) continue @@ -910,9 +990,9 @@ def in_space_placebo( n_placebos = len(ranked_ratios) if n_placebos == 0: warnings.warn( - "No in-space placebo entered the reference set (all donors " - f"failed to converge or were filtered out of {n_donors}); " - "placebo_p_value is NaN.", + "No in-space placebo entered the reference set (all donors failed to " + "converge, were structurally infeasible, or were filtered out of " + f"{n_donors}); placebo_p_value is NaN.", UserWarning, stacklevel=2, ) @@ -924,20 +1004,26 @@ def in_space_placebo( rank = 1 + sum(1 for r in ranked_ratios if r >= treated_ratio) p_value = rank / (n_placebos + 1) - if n_failed > 0: - cv_note = ( - " Under v_method='cv' an excluded refit may instead be STRUCTURALLY " - "infeasible (the pseudo-treated unit's donor pool is indistinguishable in a " - "re-aggregated CV window) — remedied by adjusting the predictors, v_cv_t0, " - "or the donor pool, NOT inner_max_iter / n_starts." - if snap.v_method == "cv" - else "" + # Two distinct exclusion causes, warned separately (mirrors in_time_placebo) so a + # structural cv exclusion is not mis-attributed to a solver budget the user could + # raise. Both remain out of the permutation rank; placebo_p_value uses n_placebos. + if n_infeasible > 0: + warnings.warn( + f"{n_infeasible} of {n_donors} in-space placebos were STRUCTURALLY " + "infeasible under v_method='cv' (the pseudo-treated donor pool is " + "indistinguishable in a re-aggregated CV window, so the weights are " + "unidentified) and were excluded with status='infeasible'; remedy by " + "adjusting the predictors, v_cv_t0, or the donor pool (NOT inner_max_iter " + f"/ n_starts). placebo_p_value uses the remaining {n_placebos}.", + UserWarning, + stacklevel=2, ) + if n_failed > 0: warnings.warn( - f"{n_failed} of {n_donors} in-space placebos were excluded from the " - "permutation distribution (the refit did not reach a valid optimum — a " - "non-converged inner weight solve or outer V search); " - f"placebo_p_value uses the remaining {n_placebos}.{cv_note}", + f"{n_failed} of {n_donors} in-space placebos failed to reach a valid " + "optimum (a non-converged inner weight solve or outer V search) and were " + "excluded with status='failed'; raise n_starts or loosen the optimizer " + f"tolerances. placebo_p_value uses the remaining {n_placebos}.", UserWarning, stacklevel=2, ) @@ -959,8 +1045,22 @@ def in_space_placebo( self.placebo_p_value = float(p_value) self.n_placebos = int(n_placebos) self.n_failed = int(n_failed) + self.n_infeasible = int(n_infeasible) self._placebo_gaps = placebo_gaps - self._placebo_status = "ran" if n_placebos > 0 else "all_placebos_failed" + # Classify a no-reference-set run by cause (mirrors in_time_placebo): a pure + # solver failure ("all_placebos_failed", actionable via n_starts / tolerances) and + # pure structural infeasibility ("all_placebos_infeasible", actionable via + # predictors / v_cv_t0 / donor pool) are distinct; a MIX gets "all_placebos_unusable" + # (both counters surfaced). By this point too-few-donors / non-converged-treated-fit + # have already returned, so >=1 donor was attempted. + if n_placebos > 0: + self._placebo_status = "ran" + elif n_failed > 0 and n_infeasible > 0: + self._placebo_status = "all_placebos_unusable" + elif n_infeasible > 0: + self._placebo_status = "all_placebos_infeasible" + else: + self._placebo_status = "all_placebos_failed" self._placebo_df = pd.DataFrame(rows, columns=self._PLACEBO_COLS) return self._placebo_df.copy() @@ -1008,9 +1108,12 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: ------- pandas.DataFrame One ``status="baseline"`` row (the full fit, ``delta_att=0``) followed by - one row per dropped donor (``status="loo"``, or ``"failed"`` with NaN - metrics when its refit did not converge), sorted by ``|delta_att|`` - descending (failed rows last). Columns: ``dropped_unit``, ``att``, + one row per dropped donor: ``status="loo"``, or — with NaN metrics — an + excluded drop that is ``"failed"`` (its refit did not converge) or + ``"infeasible"`` (under ``v_method="cv"`` the reduced donor pool is + indistinguishable in a re-aggregated CV window). Rows are sorted by + ``|delta_att|`` descending, with the excluded (``"failed"`` / + ``"infeasible"``) rows last. Columns: ``dropped_unit``, ``att``, ``pre_rmspe``, ``post_rmspe``, ``rmspe_ratio``, ``delta_att`` (``att_loo - full_att``), ``status``. @@ -1067,6 +1170,7 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: self._loo_status = "treated_fit_nonconverged" self._loo_att_range = None self._loo_n_failed = 0 + self._loo_n_infeasible = 0 self._loo_gaps = {} self._loo_df = pd.DataFrame([baseline_row], columns=self._LOO_COLS) return self._loo_df.copy() @@ -1083,6 +1187,7 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: self._loo_status = "too_few_donors" self._loo_att_range = None self._loo_n_failed = 0 + self._loo_n_infeasible = 0 self._loo_gaps = {} self._loo_df = pd.DataFrame([baseline_row], columns=self._LOO_COLS) return self._loo_df.copy() @@ -1096,12 +1201,19 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: loo_rows: List[Dict[str, Any]] = [] atts: List[float] = [] n_failed = 0 + n_infeasible = 0 for d in pos_donors: pool = [x for x in snap.donor_ids if x != d] - fitted = _placebo_fit_unit(snap, snap.treated_id, pool, n_starts_eff) + fitted, fit_status = _placebo_fit_unit(snap, snap.treated_id, pool, n_starts_eff) if fitted is None: - n_failed += 1 + # "infeasible" (structural cv donor-indistinguishability of the reduced + # pool) vs "failed" (solver non-convergence): counted separately, both + # excluded from the ATT range. Mirrors the in_time_placebo split. + if fit_status == "infeasible": + n_infeasible += 1 + else: + n_failed += 1 loo_rows.append( { "dropped_unit": d, @@ -1110,7 +1222,7 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: "post_rmspe": np.nan, "rmspe_ratio": np.nan, "delta_att": np.nan, - "status": "failed", + "status": fit_status, } ) continue @@ -1131,45 +1243,58 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: ) # Sort successful drops by |delta_att| desc (most influential donor first); - # non-converged drops sort last. + # excluded drops (failed OR infeasible) sort last. finite_rows = sorted( (r for r in loo_rows if r["status"] == "loo"), key=lambda r: abs(r["delta_att"]), reverse=True, ) - failed_rows = [r for r in loo_rows if r["status"] == "failed"] - ordered = [baseline_row] + finite_rows + failed_rows + excluded_rows = [r for r in loo_rows if r["status"] != "loo"] + ordered = [baseline_row] + finite_rows + excluded_rows - if n_failed > 0: - cv_note = ( - " Under v_method='cv' a 'failed' drop may instead be STRUCTURALLY " - "infeasible (the reduced donor pool is indistinguishable in a re-aggregated " - "CV window) — remedied by adjusting the predictors, v_cv_t0, or the donor " - "pool, NOT inner_max_iter / n_starts." - if snap.v_method == "cv" - else "" + # Two exclusion causes, warned separately (mirrors in_time_placebo) so a structural + # cv exclusion is not mis-attributed to a solver budget. Both drop out of the ATT range. + if n_infeasible > 0: + warnings.warn( + f"{n_infeasible} of {len(pos_donors)} leave-one-out refits were STRUCTURALLY " + "infeasible under v_method='cv' (the reduced donor pool is indistinguishable " + "in a re-aggregated CV window) and were excluded with status='infeasible'; " + "remedy by adjusting the predictors, v_cv_t0, or the donor pool (NOT " + "inner_max_iter / n_starts); the ATT range uses the remaining refits.", + UserWarning, + stacklevel=2, ) + if n_failed > 0: warnings.warn( f"{n_failed} of {len(pos_donors)} leave-one-out refits were excluded with " "NaN metrics (status='failed'; the refit did not reach a valid optimum — a " "non-converged inner weight solve or outer V search); the ATT range uses " - f"the remaining refits.{cv_note}", + "the remaining refits.", UserWarning, stacklevel=2, ) self._loo_gaps = loo_gaps self._loo_n_failed = int(n_failed) + self._loo_n_infeasible = int(n_infeasible) self._loo_att_range = (min(atts), max(atts)) if atts else None # Baseline-relative headline: the largest swing of any single donor-drop from # the full-fit ATT (max |delta_att|). Robust to a uniform shift that a raw # att_range would understate. self._loo_max_abs_delta_att = max(abs(a - float(self.att)) for a in atts) if atts else None - # Distinguish a real run from "every donor-drop refit failed to converge" - # (no valid leave-one-out estimate produced) so DR/BR do not report an empty - # diagnostic as completed. (pos_donors empty — a converged fit always has >=1 - # positive weight — falls through to "ran": baseline-only, benign.) - self._loo_status = "all_refits_failed" if (pos_donors and not atts) else "ran" + # Distinguish a real run from "no valid leave-one-out estimate produced" (so DR/BR + # do not report an empty diagnostic as completed) AND classify the no-success cause + # by solver-failure vs structural-infeasibility vs a mix (mirrors in_time_placebo). + # (pos_donors empty — a converged fit always has >=1 positive weight — falls through + # to "ran": baseline-only, benign.) + if atts or not pos_donors: + self._loo_status = "ran" + elif n_failed > 0 and n_infeasible > 0: + self._loo_status = "all_refits_unusable" + elif n_infeasible > 0: + self._loo_status = "all_refits_infeasible" + else: + self._loo_status = "all_refits_failed" self._loo_df = pd.DataFrame(ordered, columns=self._LOO_COLS) return self._loo_df.copy() @@ -1417,9 +1542,18 @@ def in_time_placebo( } ) continue - fitted = _placebo_fit_unit(snap_mod, snap.treated_id, snap.donor_ids, n_starts_eff) + fitted, fit_status = _placebo_fit_unit( + snap_mod, snap.treated_id, snap.donor_ids, n_starts_eff + ) if fitted is None: - n_failed += 1 + # _truncate_snapshot_in_time already applied the cv structural checks to the + # truncated snapshot, so a None here is normally a solver non-convergence + # ("failed"); defensively honor an "infeasible" status if the solve still + # reports one (counts it alongside the truncation-level n_infeasible). + if fit_status == "infeasible": + n_infeasible += 1 + else: + n_failed += 1 rows.append( { "placebo_period": t_f, @@ -1429,7 +1563,7 @@ def in_time_placebo( "n_pre_fake": n_pre_fake, "n_post_fake": n_post_fake, "n_dropped_specs": len(dropped), - "status": "failed", + "status": fit_status, } ) continue @@ -1569,8 +1703,8 @@ def _require_placebo_reference(self, n_starts: Optional[int]) -> None: (every sharp null re-ranks the SAME gaps), so honouring ``n_starts`` would mean an expensive O(J) re-fit that the caller did not ask for. Raises ValueError when no valid reference set could be produced (fewer than 2 donors, a non-converged treated - fit, or all donor refits failed) — there is then no permutation distribution to - invert. + fit, or all donor refits failed / were structurally infeasible) — there is then no + permutation distribution to invert. """ if self._placebo_gaps is None: # Builds the reference set; raises ValueError if the snapshot is unavailable. @@ -1598,6 +1732,15 @@ def _require_placebo_reference(self, n_starts: Optional[int]) -> None: "every donor refit failed to converge, so no placebo entered the " "reference set" ), + "all_placebos_infeasible": ( + "every donor refit was structurally infeasible (under v_method='cv' " + "the pseudo-treated donor pool is indistinguishable in a re-aggregated " + "CV window), so no placebo entered the reference set" + ), + "all_placebos_unusable": ( + "no donor refit was usable — some failed to converge and some were " + "structurally infeasible — so no placebo entered the reference set" + ), } default_reason = "no valid in-space placebo reference set was produced" status = self._placebo_status diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 33a7c618..5a8be777 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -2187,7 +2187,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **`V` selection (`v_method`):** `"nested"` chooses diagonal PSD `V` minimizing the pre-period **outcome** MSPE `mean((Z1 − Z0·W*(V))²)` over all pre periods; `"cv"` chooses `V` by **out-of-sample cross-validation** (ADH 2015 §; Abadie 2021 Eq. 9 — see the per-window re-aggregation Note); `"inverse_variance"` uses the closed-form `v_h = 1/Var(X_{h·})` (Abadie 2021 §3.2(a); no search); `"custom"` skips the search and uses a user-supplied `custom_v` (trace-normalized). `mspe_v` reports the selected `V`'s objective value — the pre-period MSPE under `"nested"`, the held-out validation-window MSPE under `"cv"`, and `None` for the search-free `"custom"`/`"inverse_variance"` paths (not comparable across methods). - **Effect:** gap path `α̂_1t = Y_1t − Σ_j w_j·Y_jt`; `att` = mean post-period gap; `pre_rmspe` = pre-period fit diagnostic. -**Inference:** **No analytical standard error** (Section 2.4) — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. Significance comes from **in-space placebo permutation inference** via `SyntheticControlResults.in_space_placebo()`: reassign treatment to each donor, refit a synthetic control for it, and rank the treated unit's post/pre RMSPE ratio (`rmspe_ratio` = `RMSPE_post / RMSPE_pre` = `sqrt(MSPE_post / MSPE_pre)`) among all units; `placebo_p_value = rank / (n_placebos + 1)`, where `rank = 1 + #{placebos with ratio ≥ treated ratio}` — an **upper-tail rank test on the (unsigned) RMSPE-ratio statistic**, ties counted conservatively via `≥`. Because the ratio squares the gaps it is direction-agnostic: a large ratio signals an effect of *either* sign, so this is NOT a signed/one-directional ("one-sided") hypothesis test on the treatment-effect direction. ADH 2010 §3.4 reports the *MSPE* ratio (the square of `rmspe_ratio`); the two are monotone-equivalent, so the rank and p-value are identical — only the reported statistic's scale differs. `rmspe_ratio` (the treated statistic) is computed at fit time; `placebo_p_value` / `n_placebos` / `n_failed` are populated by the opt-in `in_space_placebo()` call. `get_placebo_df()` returns the per-unit ratio table used for the rank (the per-period placebo gap paths for a "spaghetti" plot are retained internally, not in this summary). +**Inference:** **No analytical standard error** (Section 2.4) — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. Significance comes from **in-space placebo permutation inference** via `SyntheticControlResults.in_space_placebo()`: reassign treatment to each donor, refit a synthetic control for it, and rank the treated unit's post/pre RMSPE ratio (`rmspe_ratio` = `RMSPE_post / RMSPE_pre` = `sqrt(MSPE_post / MSPE_pre)`) among all units; `placebo_p_value = rank / (n_placebos + 1)`, where `rank = 1 + #{placebos with ratio ≥ treated ratio}` — an **upper-tail rank test on the (unsigned) RMSPE-ratio statistic**, ties counted conservatively via `≥`. Because the ratio squares the gaps it is direction-agnostic: a large ratio signals an effect of *either* sign, so this is NOT a signed/one-directional ("one-sided") hypothesis test on the treatment-effect direction. ADH 2010 §3.4 reports the *MSPE* ratio (the square of `rmspe_ratio`); the two are monotone-equivalent, so the rank and p-value are identical — only the reported statistic's scale differs. `rmspe_ratio` (the treated statistic) is computed at fit time; `placebo_p_value` / `n_placebos` / `n_failed` / `n_infeasible` are populated by the opt-in `in_space_placebo()` call (`n_failed` = solver non-convergences, `n_infeasible` = structural `cv` exclusions — see the split Note below). `get_placebo_df()` returns the per-unit ratio table used for the rank (the per-period placebo gap paths for a "spaghetti" plot are retained internally, not in this summary). **ADH-2015 §4 robustness diagnostics (opt-in):** Two further diagnostics from Abadie-Diamond-Hainmueller (2015, §4), each a thin re-run of the validated solver — populated only when called and surfaced under `estimator_native_diagnostics` (the analytical inference contract is unchanged; `se`/`t_stat`/`p_value`/`conf_int`/`is_significant` stay bound to the NaN analytical `p_value`): - **Leave-one-out donor robustness** (`leave_one_out()`): drops each **reportably-weighted** donor and re-fits the treated unit against the reduced pool, returning a per-drop ATT / `delta_att` table (a `status="baseline"` row first, then one row per dropped donor sorted by `|delta_att|`). A large `delta_att` flags single-donor dependence (a single *dominant* donor is still dropped — the others absorb its mass — and its large `delta_att` is the intended signal, not a failure). The reporting stack's headline donor-sensitivity number is `max_abs_delta_att` = `max |delta_att|` over the drops (baseline-relative, so a uniform shift of every drop away from the full-fit ATT is not masked the way a raw ATT range would be). `get_leave_one_out_gaps()` returns the per-drop trajectories for the overlay plot. Fails closed on a non-converged treated fit or `< 2` donors. @@ -2220,6 +2220,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Deviation from R:** predictor/outcome **aggregation fails closed on any non-finite (NaN/inf) cell**, whereas R `Synth::dataprep` hardwires `na.rm=TRUE` (aggregating over the observed cells of a partially-missing window). The fail-closed contract is deliberate: na-dropping silently aggregates different period subsets across units, yielding incomparable predictors with no warning. The analyst must restrict `predictor_window` / `special_predictors` / `pre_period_outcomes` periods (and the outcome panel) to where each variable is observed; both partially- and fully-missing windows raise `ValueError`. Only the row *ordering* matches `dataprep`, not the missing-data handling. - **Note (in-space placebo donor pool):** the real treated unit is **excluded from every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J−1` donors, never the real treated unit (whose post-period is treatment-contaminated). The ranking set is still the `J+1` units {treated} ∪ {J placebos}. ADH 2010 §2.4 does not spell out placebo donor-pool composition; this matches the standard `SCtools::generate.placebos` construction (rotate the pseudo-treated identity through the donor pool; the original treated unit is never re-added as a donor). - **Note (placebo failure handling):** a placebo is **excluded from both the numerator and the denominator** of the rank (never penalized into it) and tallied in `n_failed` when its fit is not a valid optimum — EITHER its **inner Frank-Wolfe weight solve** did not converge (a truncated `W` is unusable) OR its **outer `V` search** did not converge (an under-optimized `V` fits the pre-period worse, shrinking the RMSPE ratio and biasing the p-value anti-conservatively, so it must not silently enter the rank). The reported p-value uses the **effective** count `rank / (n_placebos + 1)`, where `n_placebos` is the number of placebos that entered the reference set. Failed donors still appear in `get_placebo_df()` (`status="failed"`, NaN metrics), so once a reference set is produced the table is the full treated + every-donor unit set (`n_donors + 1` rows). In the fail-closed cases the placebo loop does not run and only the treated row is returned: `J < 2` → `placebo_p_value` is NaN with a warning (no placebo distribution; `J == 2` warns the distribution is coarse), and a treated fit whose own **inner OR outer** search did not converge also fails closed (ranking a truncated / under-optimized treated statistic would not be a valid permutation). **Caveat:** each placebo refit inherits the original fit's `optimizer_options` / `n_starts`, so valid inference requires settings adequate for the outer `V` search to converge to a comparable-quality synthetic (production defaults do; cheap settings under-optimize placebo `V` and those placebos are dropped as failed — raise `n_starts` on `in_space_placebo()` or re-fit with a larger `optimizer_options['maxiter']`). +- **Note (in-space / leave-one-out infeasible vs failed split):** an excluded `in_space_placebo()` / `leave_one_out()` refit is attributed to one of two causes, mirroring the split `in_time_placebo` already reports. A **solver non-convergence** (a truncated inner `W` or outer `V` search) is tallied in `n_failed` (`_loo_n_failed`) with a `status="failed"` row; a **structural cv infeasibility** — under `v_method="cv"`, the pseudo-treated donor pool (in-space) or reduced pool (leave-one-out) is indistinguishable in a re-aggregated CV window, so the weights are unidentified (the window-information gate above; `_outer_solve_V_cv` returns a structural sentinel that `_placebo_fit_unit` threads out) — is tallied in `n_infeasible` (`_loo_n_infeasible`) with a `status="infeasible"` row. Both are excluded from the rank / ATT range identically, so `placebo_p_value` / `n_placebos` are **unchanged** by the attribution — only the diagnostic accounting is refined. `_placebo_status` / `_loo_status` gain `all_placebos_infeasible` / `all_placebos_unusable` (resp. `all_refits_infeasible` / `all_refits_unusable`) codes for a no-usable-refit run that is purely structural or a mix, and `DiagnosticReport` surfaces the machine-readable `reason_code` alongside `n_failed` / `n_infeasible`. `n_infeasible` is 0 for the non-`cv` `v_method`s (no structural-identification gate). The remedy differs by cause: a structural infeasibility needs different predictors / `v_cv_t0` / donor pool, NOT a larger `n_starts` / `inner_max_iter`. - **Note (RMSPE-ratio floor):** the reported `rmspe_ratio = sqrt(MSPE_post / MSPE_pre)` floors the pre-period MSPE denominator at a scale-aware `1e-8 · max(|pre-outcomes|, 1)²` (before the square root) so a (near-)perfect pre-fit (`pre-MSPE → 0`) yields a large-but-FINITE ratio rather than `inf`/`nan` (which would corrupt the rank). Ties (`ratio_j ≥ treated_ratio`) are counted, making the p-value conservative. Mirrors the `_fit_tol` poor-fit guard. - **Note (placebo p-value is non-analytical):** `placebo_p_value` is deliberately a SEPARATE field from `p_value` (which stays NaN) — it is a permutation p-value with no SE / t-stat, so it does not flow through `safe_inference`. `is_significant` likewise stays bound to the (NaN) `p_value`, NOT `placebo_p_value`; a tool gating on `is_significant` will see `False` even when `placebo_p_value` is small. The reporting stack surfaces the placebo p-value through `estimator_native_diagnostics`, never the analytical headline. - **Note (confidence set is non-analytical — `conf_int` stays NaN):** the Firpo & Possebom test-inversion `effect_confidence_set` is a permutation set at level `1−γ`, kept DELIBERATELY SEPARATE from the analytical `conf_int` (which stays `(NaN, NaN)` — classic SCM has no Wald interval). It is parametrized by `γ` (not the estimator's `alpha`, and granular in `1/(J+1)`), can be a *set* rather than an interval (the linear family) and can be unbounded or non-contiguous, so it cannot be coerced into the `(lo, hi)` `conf_int` tuple without breaking the `safe_inference` NaN-consistency contract. `se`/`t_stat`/`p_value`/`conf_int`/`is_significant` all stay at their NaN state; the set is surfaced only via `confidence_set()` / `get_confidence_set_df()` / `effect_confidence_set` (and `estimator_native_diagnostics`) — mirroring how `placebo_p_value` is kept off `p_value`. diff --git a/tests/test_methodology_synthetic_control.py b/tests/test_methodology_synthetic_control.py index 3b2ee97b..42706e8f 100644 --- a/tests/test_methodology_synthetic_control.py +++ b/tests/test_methodology_synthetic_control.py @@ -1031,6 +1031,7 @@ def test_in_space_placebo_pickle_drops_snapshot_keeps_scalars(): assert restored.placebo_p_value == res.placebo_p_value assert restored.rmspe_ratio == res.rmspe_ratio assert restored.n_placebos == res.n_placebos and restored.n_failed == res.n_failed + assert restored.n_infeasible == res.n_infeasible assert restored._fit_snapshot is None and restored._placebo_gaps is None # The small aggregate table survives, so get_placebo_df still works... assert len(restored.get_placebo_df()) == len(res.get_placebo_df()) @@ -1039,6 +1040,30 @@ def test_in_space_placebo_pickle_drops_snapshot_keeps_scalars(): restored.in_space_placebo() +def test_legacy_pickle_missing_infeasible_fields_backfills_to_zero(): + # A result pickled by a version predating n_infeasible / _loo_n_infeasible unpickles + # (bypassing __init__ / __post_init__) WITHOUT those attributes. __setstate__ must + # backfill them to 0 so the reporting paths that dereference them directly + # (summary() / to_dict()) do not raise AttributeError on a legacy result. + res = _fit_for_placebo(n_donors=4) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.in_space_placebo() + res.leave_one_out() + legacy_state = res.__getstate__() # what pickle would persist ... + legacy_state.pop("n_infeasible", None) # ... minus the fields an older version lacked + legacy_state.pop("_loo_n_infeasible", None) + restored = object.__new__(type(res)) # bypasses __init__/__post_init__, as pickle does + restored.__setstate__(legacy_state) + assert restored.n_infeasible == 0 and restored._loo_n_infeasible == 0 + # Reporting paths that dereference the new fields directly must not raise. + assert isinstance(restored.summary(), str) + assert restored.to_dict()["n_infeasible"] == 0 + native = DiagnosticReport(restored).to_dict()["estimator_native_diagnostics"] + assert native["in_space_placebo"]["n_infeasible"] == 0 + assert native["leave_one_out"]["n_infeasible"] == 0 + + def test_in_space_placebo_custom_v_path(): df, _, _ = _make_panel(n_donors=4) # Default predictors = all pre-period outcomes -> k = number of pre periods (T0). @@ -1173,8 +1198,8 @@ def test_get_placebo_df_includes_failed_donors(monkeypatch): def flaky_fit_unit(snap, unit, donor_pool, n_starts): calls["n"] += 1 - if calls["n"] <= 2: # first two donor refits "fail" - return None + if calls["n"] <= 2: # first two donor refits "fail" (solver non-convergence) + return None, "failed" return real_fit_unit(snap, unit, donor_pool, n_starts) monkeypatch.setattr(sc, "_placebo_fit_unit", flaky_fit_unit) @@ -1539,8 +1564,8 @@ def test_leave_one_out_refit_failure_tallied(monkeypatch): def flaky_fit_unit(snap, unit, donor_pool, n_starts): calls["n"] += 1 - if calls["n"] == 1: # first leave-one-out refit "fails" - return None + if calls["n"] == 1: # first leave-one-out refit "fails" (solver non-convergence) + return None, "failed" return real_fit_unit(snap, unit, donor_pool, n_starts) monkeypatch.setattr(sc, "_placebo_fit_unit", flaky_fit_unit) @@ -1894,7 +1919,7 @@ def test_leave_one_out_uniform_shift_surfaced_by_delta_not_range(monkeypatch): def uniform_shift(snap_arg, unit, pool, n_starts): gp = {p: 0.0 for p in snap.pre_periods} gp.update({p: baseline + shift for p in snap.post_periods}) - return gp, 1.0 + return (gp, 1.0), "ran" monkeypatch.setattr(sc, "_placebo_fit_unit", uniform_shift) with warnings.catch_warnings(): @@ -2169,7 +2194,8 @@ def test_leave_one_out_all_refits_failed_status(monkeypatch): sc = importlib.import_module("diff_diff.synthetic_control") res = _fit_for_placebo(n_donors=4) - monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: None) # every drop fails + # every drop fails to converge (solver, not structural) + monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: (None, "failed")) with pytest.warns(UserWarning, match="did not reach a valid optimum"): loo = res.leave_one_out() # Distinct status (NOT "ran"); att_range is None; baseline + only failed rows. @@ -2190,7 +2216,8 @@ def test_in_time_placebo_all_dates_failed_status(monkeypatch): sc = importlib.import_module("diff_diff.synthetic_control") res = _fit_for_placebo(n_donors=4) - monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: None) # every refit fails + # every refit fails to converge (solver, not structural) + monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: (None, "failed")) with pytest.warns(UserWarning, match="failed to converge"): itp = res.in_time_placebo() # Convergence failure must NOT be mislabeled as dimensional infeasibility. @@ -2212,7 +2239,7 @@ def test_in_time_placebo_mixed_failed_and_infeasible_status(monkeypatch): sc = importlib.import_module("diff_diff.synthetic_control") res = _fit_for_placebo(n_donors=4) # Feasible dates "fail" to converge; 2001 (1 pre-fake) is dimensionally infeasible. - monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: None) + monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: (None, "failed")) with warnings.catch_warnings(): warnings.simplefilter("ignore") itp = res.in_time_placebo([2001, 2003]) # 2001 infeasible, 2003 fails @@ -2224,6 +2251,162 @@ def test_in_time_placebo_mixed_failed_and_infeasible_status(monkeypatch): assert block["n_failed"] == 1 and block["n_infeasible"] == 1 +def test_in_space_placebo_all_infeasible_status(monkeypatch): + # Every donor refit is STRUCTURALLY infeasible (cv donor-indistinguishability) -> + # "all_placebos_infeasible", distinct from the solver "all_placebos_failed", with a + # machine-readable reason_code + n_infeasible surfaced on DiagnosticReport. + import importlib + + sc = importlib.import_module("diff_diff.synthetic_control") + res = _fit_for_placebo(n_donors=4) + monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: (None, "infeasible")) + with pytest.warns(UserWarning, match="STRUCTURALLY infeasible"): + pdf = res.in_space_placebo() + assert res._placebo_status == "all_placebos_infeasible" + assert res.n_placebos == 0 and res.n_failed == 0 and res.n_infeasible == res.n_donors + assert (pdf["status"] == "infeasible").sum() == res.n_donors # every donor row + block = DiagnosticReport(res).to_dict()["estimator_native_diagnostics"]["in_space_placebo"] + assert block["status"] == "infeasible" + assert block["reason_code"] == "all_placebos_infeasible" + assert block["n_infeasible"] == res.n_donors and block["n_failed"] == 0 + assert "structurally infeasible" in block["reason"] + + +def test_in_space_placebo_unusable_status(monkeypatch): + # A mix of solver failures AND structural infeasibilities with no usable placebo -> + # "all_placebos_unusable" (both counters surfaced), not mislabeled as exclusively one. + import importlib + + sc = importlib.import_module("diff_diff.synthetic_control") + res = _fit_for_placebo(n_donors=4) + calls = {"n": 0} + + def mixed(*a, **k): + calls["n"] += 1 + return (None, "infeasible") if calls["n"] % 2 else (None, "failed") + + monkeypatch.setattr(sc, "_placebo_fit_unit", mixed) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.in_space_placebo() + assert res._placebo_status == "all_placebos_unusable" + assert res.n_placebos == 0 and res.n_failed > 0 and res.n_infeasible > 0 + block = DiagnosticReport(res).to_dict()["estimator_native_diagnostics"]["in_space_placebo"] + assert block["reason_code"] == "all_placebos_unusable" + assert block["n_failed"] == res.n_failed and block["n_infeasible"] == res.n_infeasible + + +def test_confidence_set_reason_names_all_placebos_infeasible(monkeypatch): + # The test-inversion entrypoint (_require_placebo_reference, shared by confidence_set / + # test_sharp_null) must NAME the new no-reference statuses: an all-infeasible in-space + # run raises with the STRUCTURAL reason, not the generic "no valid reference set" fallback. + import importlib + + sc = importlib.import_module("diff_diff.synthetic_control") + res = _fit_for_placebo(n_donors=4) + monkeypatch.setattr(sc, "_placebo_fit_unit", lambda *a, **k: (None, "infeasible")) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.in_space_placebo() + assert res._placebo_status == "all_placebos_infeasible" + with pytest.raises(ValueError, match="structurally infeasible"): + res.confidence_set(family="constant", gamma=0.25) + + +def test_cv_leave_one_out_infeasible_drop_counted_separately(monkeypatch): + # A single structurally-infeasible drop is routed to _loo_n_infeasible (NOT + # _loo_n_failed); the run still "ran" via the other drops, and the excluded row + # carries status="infeasible". + import importlib + + sc = importlib.import_module("diff_diff.synthetic_control") + res = _fit_for_placebo(n_donors=4) + real_fit_unit = sc._placebo_fit_unit + calls = {"n": 0} + + def flaky(snap, unit, pool, n_starts): + calls["n"] += 1 + if calls["n"] == 1: # first drop is structurally infeasible + return None, "infeasible" + return real_fit_unit(snap, unit, pool, n_starts) + + monkeypatch.setattr(sc, "_placebo_fit_unit", flaky) + with pytest.warns(UserWarning, match="STRUCTURALLY infeasible"): + loo = res.leave_one_out() + assert res._loo_status == "ran" + assert res._loo_n_infeasible == 1 and res._loo_n_failed == 0 + assert (loo["status"] == "infeasible").sum() == 1 + native = DiagnosticReport(res).to_dict()["estimator_native_diagnostics"]["leave_one_out"] + assert native["status"] == "ran" and native["n_infeasible"] == 1 + + +def test_cv_leave_one_out_flat_reduced_pool_infeasible(): + # LOO real-mechanism test (solve-level cv sentinel, mirroring the in-space flat-refit + # test): the treated unit equals the linchpin donor d0 (=8) in the flat validation + # window {2003,2004,2005}, so the cv fit loads all weight on d0 (the only donor at 8). + # Dropping d0 — the sole reportably-weighted donor — leaves {d1,d2,d3} identical (=10) + # there, so the reduced pool is donor-indistinguishable: a STRUCTURAL cv infeasibility + # routed to _loo_n_infeasible (NOT the solver _loo_n_failed), with status="infeasible". + rng = np.random.default_rng(3) + years = list(range(2000, 2008)) + rows = [] + for j in range(4): + for yr in years: + if yr in (2003, 2004, 2005): + y = 8.0 if j == 0 else 10.0 # d0 distinguishes the pool in the flat window + elif yr <= 2005: + y = 5.0 + j + rng.normal(0, 0.2) + else: + y = 10.0 + rng.normal(0, 0.2) + rows.append({"unit": f"d{j}", "year": yr, "y": y, "treated": 0}) + for yr in years: + # treated == d0 in the flat window -> the synthetic must weight d0 (only donor at 8) + y = 8.0 if yr in (2003, 2004, 2005) else (5.5 + rng.normal(0, 0.2) if yr <= 2005 else 13.0) + rows.append({"unit": "treated", "year": yr, "y": y, "treated": int(yr >= 2006)}) + df = pd.DataFrame(rows) + res = _fit_cv(df, seed=0) + assert np.isfinite(res.att) # headline well-posed (full donor set distinguishable) + assert list(res._fit_snapshot.weighted_donor_ids) == ["d0"] # d0 is the sole weighted donor + with pytest.warns(UserWarning, match="STRUCTURALLY infeasible"): + loo = res.leave_one_out() + # Real cv sentinel: the d0 drop is structural infeasibility, not solver failure. + assert res._loo_status == "all_refits_infeasible" + assert res._loo_n_infeasible == 1 and res._loo_n_failed == 0 + assert res._loo_att_range is None + dropped = loo[loo["dropped_unit"] == "d0"] + assert len(dropped) == 1 and dropped.iloc[0]["status"] == "infeasible" + native = DiagnosticReport(res).to_dict()["estimator_native_diagnostics"]["leave_one_out"] + assert native["status"] != "ran" + assert native["reason_code"] == "all_refits_infeasible" + assert native["n_infeasible"] == 1 and native["n_failed"] == 0 + + +def test_leave_one_out_unusable_status(monkeypatch): + # A mix of failed + infeasible drops with none usable -> "all_refits_unusable". + import importlib + + sc = importlib.import_module("diff_diff.synthetic_control") + res = _fit_for_placebo(n_donors=4) + assert len(res._fit_snapshot.weighted_donor_ids) >= 2 # need >=2 drops for a mix + calls = {"n": 0} + + def mixed(*a, **k): + calls["n"] += 1 + return (None, "infeasible") if calls["n"] == 1 else (None, "failed") + + monkeypatch.setattr(sc, "_placebo_fit_unit", mixed) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.leave_one_out() + assert res._loo_status == "all_refits_unusable" + assert res._loo_n_failed > 0 and res._loo_n_infeasible > 0 + native = DiagnosticReport(res).to_dict()["estimator_native_diagnostics"]["leave_one_out"] + assert native["reason_code"] == "all_refits_unusable" + assert ( + native["n_failed"] == res._loo_n_failed and native["n_infeasible"] == res._loo_n_infeasible + ) + + # =========================================================================== # V-selection menu: v_method="inverse_variance" and v_method="cv" # (ADH 2015 § / Abadie 2021 Eq. 9; §3.2(a) inverse-variance). The CV per-window @@ -2949,13 +3132,22 @@ def test_cv_in_space_placebo_excludes_donor_flat_refits(): df = pd.DataFrame(rows) res = _fit_cv(df, seed=0) assert np.isfinite(res.att) # headline well-posed: the full donor set is distinguishable - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - res.in_space_placebo() - # The d0 placebo (pool {d1,d2,d3} identical in val) is dropped; the others enter. + # The d0 placebo (pool {d1,d2,d3} identical in val) is dropped as STRUCTURALLY + # infeasible (cv donor-indistinguishability), NOT a solver "failed": the threading + # from _outer_solve_V_cv's structural sentinel through _placebo_fit_unit routes it to + # n_infeasible. The others keep d0 in the pool, so they remain identified and enter. + with pytest.warns(UserWarning, match="STRUCTURALLY infeasible"): + pdf = res.in_space_placebo() assert res._placebo_status == "ran" assert 1 <= res.n_placebos < res.n_donors - assert res.n_failed >= 1 + assert res.n_infeasible >= 1 and res.n_failed == 0 # structural, not solver + # The excluded donor row carries status="infeasible" (not "failed"). + excluded = pdf[pdf["rmspe_ratio"].isna()] + assert len(excluded) == res.n_infeasible + assert (excluded["status"] == "infeasible").all() + # DiagnosticReport surfaces the split count on the ran block. + block = DiagnosticReport(res).to_dict()["estimator_native_diagnostics"]["in_space_placebo"] + assert block["status"] == "ran" and block["n_infeasible"] == res.n_infeasible @pytest.mark.parametrize("specs", [_CV_SPANNING, [("y", [2000, 2002, 2004], "mean")]])