diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 638ae3f..f80db19 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,4 +11,4 @@ jobs: cache: pip - run: pip install -e ".[dev]" - run: python -m unittest discover -s tests -v - - run: ruff check engine tests run.py run_cross_sectional.py + - run: ruff check engine tests run.py run_cross_sectional.py run_overfitting_study.py diff --git a/README.md b/README.md index 0319aa0..9ff367c 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,22 @@ explicitly first. Outputs are saved under `data/`, including every ledger, the cost-sensitivity table, and an equity chart. +## Strategy-selection bias research + +Version 1.1 adds a controlled multiple-testing experiment and an implementation +of combinatorially symmetric cross-validation. It demonstrates how searching +more zero-alpha strategy variants can manufacture an impressive in-sample +winner whose holdout performance does not persist. + +```bash +python run_overfitting_study.py +``` + +See [the selection-bias methodology](docs/SELECTION_BIAS.md) and +[recorded results](docs/SELECTION_BIAS_RESULTS.md). This diagnostic requires the +complete strategy/configuration set; applying it only to surviving backtests +would hide the selection process it is intended to measure. + ## Minimal network-free use ```python diff --git a/docs/SELECTION_BIAS.md b/docs/SELECTION_BIAS.md new file mode 100644 index 0000000..78c3a72 --- /dev/null +++ b/docs/SELECTION_BIAS.md @@ -0,0 +1,42 @@ +# Strategy-selection bias study + +## Research question + +If every tested strategy has zero true alpha, how does trying more alternatives +change the best reported Sharpe ratio and its out-of-sample performance? + +## Controlled experiment + +The runner generates correlated monthly returns under a global zero-alpha null. +It repeats the experiment for 5, 20, and 100 candidate strategies. Every +candidate has the same volatility and no expected excess return; a common shock +creates correlation between strategies. + +For each simulated dataset the study reports: + +- the best full-sample Sharpe selected with hindsight; +- the Sharpe of the first-half winner in the first half; +- that selected strategy's Sharpe in the untouched second half; +- probability of backtest overfitting estimated with combinatorially symmetric + cross-validation (CSCV). + +## CSCV and PBO + +The time series is divided into eight contiguous slices. Every choice of four +slices forms an in-sample set, and the complement forms its out-of-sample set, +creating 70 symmetric splits. For each split, the highest in-sample Sharpe is +selected and ranked against all configurations out of sample. PBO is the fraction +of selected winners whose out-of-sample relative rank is at or below the median. + +This follows the framework in Bailey et al., *The Probability of Backtest +Overfitting*: https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2326253 + +## What the experiment can establish + +Because the data-generating process sets every alpha to zero, any impressive +winner is known to be selection noise. The experiment isolates the multiple- +testing mechanism and tests whether holdout and CSCV diagnostics reveal it. + +It does not estimate the PBO of every repository strategy. Applying CSCV to real +research requires recording the complete set of configurations attempted, not +only the configurations retained after looking at results. diff --git a/docs/SELECTION_BIAS_RESULTS.md b/docs/SELECTION_BIAS_RESULTS.md new file mode 100644 index 0000000..f7f16c2 --- /dev/null +++ b/docs/SELECTION_BIAS_RESULTS.md @@ -0,0 +1,40 @@ +# Selection-bias experiment results + +## Recorded run + +Each row summarizes 40 independently simulated 20-year monthly datasets. Every +candidate strategy has zero true alpha and 4% monthly volatility, with a 0.25 +common correlation. Intervals are normal-approximation 95% intervals for the +mean across repetitions. + +| Candidates tried | Best full-sample Sharpe | Selected holdout Sharpe | Mean PBO | +| ---: | ---: | ---: | ---: | +| 5 | 0.210 [0.168, 0.252] | 0.000 [-0.101, 0.102] | 0.584 [0.511, 0.657] | +| 20 | 0.389 [0.341, 0.436] | 0.011 [-0.094, 0.117] | 0.504 [0.448, 0.560] | +| 100 | 0.481 [0.441, 0.520] | 0.009 [-0.078, 0.095] | 0.514 [0.457, 0.571] | + +![Selection bias under a zero-alpha null](assets/selection_bias.png) + +## Interpretation + +Expanding the search from 5 to 100 configurations more than doubles the mean +best full-sample Sharpe, from 0.210 to 0.481, even though the simulation assigns +zero alpha to every candidate. The non-overlapping intervals make the direction +of this selection effect clear in the recorded design. + +The first-half winner's mean holdout Sharpe remains approximately zero for every +candidate count. Its intervals include zero comfortably. The apparent winner +therefore does not preserve its advantage on untouched observations. + +Mean PBO remains close to 0.5. Under the global null, the in-sample winner has +essentially random out-of-sample rank, so it falls below the median about half +the time. PBO is a diagnostic of ranking failure; it need not increase +monotonically with the number of candidates in this experiment. + +## What this does not prove + +The result does not show that all optimized strategies are false or that a +particular live strategy has zero alpha. It demonstrates a controlled mechanism: +searching a larger configuration set increases the best observed statistic even +when no candidate has genuine predictive value. A real application must retain +the complete research path, including failed and discarded configurations. diff --git a/docs/assets/selection_bias.png b/docs/assets/selection_bias.png new file mode 100644 index 0000000..9ad4f6e Binary files /dev/null and b/docs/assets/selection_bias.png differ diff --git a/engine/selection_bias.py b/engine/selection_bias.py new file mode 100644 index 0000000..6153d63 --- /dev/null +++ b/engine/selection_bias.py @@ -0,0 +1,194 @@ +"""Combinatorially symmetric cross-validation for strategy-selection bias.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from itertools import combinations + +import numpy as np +import pandas as pd + + +@dataclass(frozen=True) +class PBOResult: + probability_of_backtest_overfitting: float + median_logit_rank: float + mean_selected_in_sample_sharpe: float + mean_selected_out_of_sample_sharpe: float + splits: int + split_results: pd.DataFrame + + +def annualized_sharpe(returns: np.ndarray, periods_per_year: int = 12) -> np.ndarray: + """Calculate column-wise annualized Sharpe ratios with a zero cash rate.""" + values = np.asarray(returns, dtype=float) + if values.ndim == 1: + values = values[:, None] + if values.shape[0] < 2: + raise ValueError("at least two return observations are required") + means = values.mean(axis=0) + volatility = values.std(axis=0, ddof=1) + return np.divide( + means * math.sqrt(periods_per_year), + volatility, + out=np.zeros_like(means), + where=volatility > 0, + ) + + +def _relative_rank(value: float, population: np.ndarray) -> float: + """Return a tie-aware rank strictly inside zero and one.""" + less = float(np.sum(population < value)) + equal = float(np.sum(population == value)) + return (less + 0.5 * equal) / len(population) + + +def combinatorially_symmetric_cv( + returns: pd.DataFrame, + n_slices: int = 8, + periods_per_year: int = 12, +) -> PBOResult: + """Estimate the probability that the in-sample winner ranks below median OOS. + + Time is divided into contiguous slices. Every combination of half the slices + forms an in-sample set; the complement is its out-of-sample set. For each + split, the best in-sample strategy is ranked among all strategies out of + sample. PBO is the fraction of selected strategies whose OOS rank is at or + below the median. + """ + if not isinstance(returns, pd.DataFrame) or returns.empty: + raise ValueError("returns must be a non-empty DataFrame") + if returns.shape[1] < 2: + raise ValueError("at least two strategy configurations are required") + if returns.isna().any().any() or not np.isfinite(returns.to_numpy()).all(): + raise ValueError("returns must contain only finite values") + if n_slices < 4 or n_slices % 2: + raise ValueError("n_slices must be an even integer of at least four") + if len(returns) < n_slices * 2: + raise ValueError("not enough observations for the requested slices") + + blocks = tuple(np.array_split(np.arange(len(returns)), n_slices)) + values = returns.to_numpy(dtype=float) + rows: list[dict[str, float | int | str]] = [] + half = n_slices // 2 + + for split_number, selected_blocks in enumerate(combinations(range(n_slices), half)): + selected = set(selected_blocks) + in_indices = np.concatenate([blocks[index] for index in selected_blocks]) + out_indices = np.concatenate([blocks[index] for index in range(n_slices) if index not in selected]) + in_scores = annualized_sharpe(values[in_indices], periods_per_year) + out_scores = annualized_sharpe(values[out_indices], periods_per_year) + winner_index = int(np.argmax(in_scores)) + relative_rank = _relative_rank(out_scores[winner_index], out_scores) + clipped_rank = min(1.0 - 1e-12, max(1e-12, relative_rank)) + logit_rank = math.log(clipped_rank / (1.0 - clipped_rank)) + rows.append( + { + "split": split_number, + "selected_strategy": str(returns.columns[winner_index]), + "in_sample_sharpe": float(in_scores[winner_index]), + "out_of_sample_sharpe": float(out_scores[winner_index]), + "out_of_sample_relative_rank": relative_rank, + "logit_rank": logit_rank, + "overfit": int(relative_rank <= 0.5), + } + ) + + frame = pd.DataFrame(rows) + return PBOResult( + probability_of_backtest_overfitting=float(frame["overfit"].mean()), + median_logit_rank=float(frame["logit_rank"].median()), + mean_selected_in_sample_sharpe=float(frame["in_sample_sharpe"].mean()), + mean_selected_out_of_sample_sharpe=float(frame["out_of_sample_sharpe"].mean()), + splits=len(frame), + split_results=frame, + ) + + +def simulate_null_strategies( + observations: int, + strategies: int, + rng: np.random.Generator, + common_correlation: float = 0.25, + monthly_volatility: float = 0.04, +) -> pd.DataFrame: + """Generate correlated zero-alpha strategy returns under a global null.""" + if observations < 16 or strategies < 2: + raise ValueError("simulation requires at least 16 observations and 2 strategies") + if not 0 <= common_correlation < 1 or monthly_volatility <= 0: + raise ValueError("invalid correlation or volatility") + common = rng.normal(size=(observations, 1)) + independent = rng.normal(size=(observations, strategies)) + standardized = math.sqrt(common_correlation) * common + math.sqrt(1.0 - common_correlation) * independent + return pd.DataFrame( + monthly_volatility * standardized, + columns=[f"strategy_{index:03d}" for index in range(strategies)], + ) + + +def run_selection_bias_experiment( + candidate_counts: tuple[int, ...] = (5, 20, 100), + repetitions: int = 40, + observations: int = 240, + n_slices: int = 8, + seed: int = 17, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Measure how trying more zero-alpha strategies manufactures winners.""" + if repetitions < 2: + raise ValueError("at least two repetitions are required") + if not candidate_counts or any(count < 2 for count in candidate_counts): + raise ValueError("candidate counts must contain values of at least two") + rng = np.random.default_rng(seed) + rows: list[dict[str, float | int]] = [] + + for candidate_count in candidate_counts: + for repetition in range(repetitions): + returns = simulate_null_strategies(observations, candidate_count, rng) + full_scores = annualized_sharpe(returns.to_numpy()) + winner = int(np.argmax(full_scores)) + midpoint = observations // 2 + first_half_scores = annualized_sharpe(returns.iloc[:midpoint].to_numpy()) + selected = int(np.argmax(first_half_scores)) + holdout_scores = annualized_sharpe(returns.iloc[midpoint:].to_numpy()) + pbo = combinatorially_symmetric_cv(returns, n_slices=n_slices) + rows.append( + { + "candidate_count": candidate_count, + "repetition": repetition, + "naive_best_full_sample_sharpe": float(full_scores[winner]), + "first_half_selected_sharpe": float(first_half_scores[selected]), + "selected_holdout_sharpe": float(holdout_scores[selected]), + "pbo": pbo.probability_of_backtest_overfitting, + } + ) + + raw = pd.DataFrame(rows) + summaries: list[dict[str, float | int]] = [] + for candidate_count in candidate_counts: + selected = raw[raw["candidate_count"] == candidate_count] + row: dict[str, float | int] = { + "candidate_count": candidate_count, + "repetitions": len(selected), + } + for output_name, column in ( + ("naive_best_sharpe", "naive_best_full_sample_sharpe"), + ("first_half_selected_sharpe", "first_half_selected_sharpe"), + ("selected_holdout_sharpe", "selected_holdout_sharpe"), + ("pbo", "pbo"), + ): + mean, low, high = _sample_mean_interval(selected[column].to_numpy()) + row[f"mean_{output_name}"] = mean + row[f"{output_name}_ci95_low"] = low + row[f"{output_name}_ci95_high"] = high + summaries.append(row) + summary = pd.DataFrame(summaries) + return raw, summary + + +def _sample_mean_interval(values: np.ndarray) -> tuple[float, float, float]: + if len(values) < 2: + raise ValueError("at least two values are required") + mean = float(np.mean(values)) + margin = 1.96 * float(np.std(values, ddof=1)) / math.sqrt(len(values)) + return mean, mean - margin, mean + margin diff --git a/pyproject.toml b/pyproject.toml index 2c1f310..e88c46b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "backtest-engine-asoracca" -version = "1.0.0" +version = "1.1.0" description = "A small event-driven backtesting engine with next-bar execution and auditable ledgers" requires-python = ">=3.10" dependencies = ["numpy>=1.26", "pandas>=2.1", "yfinance>=0.2.40", "matplotlib>=3.8"] diff --git a/run_overfitting_study.py b/run_overfitting_study.py new file mode 100644 index 0000000..f41f5b7 --- /dev/null +++ b/run_overfitting_study.py @@ -0,0 +1,77 @@ +"""Demonstrate strategy-selection bias under a controlled zero-alpha null.""" + +from pathlib import Path + +import matplotlib.pyplot as plt + +from engine.selection_bias import run_selection_bias_experiment + + +def main(): + output = Path("data/selection_bias") + output.mkdir(parents=True, exist_ok=True) + raw, summary = run_selection_bias_experiment() + raw.to_csv(output / "simulation_runs.csv", index=False) + summary.to_csv(output / "summary.csv", index=False) + + figure, axes = plt.subplots(1, 2, figsize=(11, 4.8)) + axes[0].errorbar( + summary["candidate_count"], + summary["mean_naive_best_sharpe"], + yerr=[ + summary["mean_naive_best_sharpe"] - summary["naive_best_sharpe_ci95_low"], + summary["naive_best_sharpe_ci95_high"] - summary["mean_naive_best_sharpe"], + ], + marker="o", + capsize=4, + label="Best full-sample Sharpe", + ) + axes[0].errorbar( + summary["candidate_count"], + summary["mean_selected_holdout_sharpe"], + yerr=[ + summary["mean_selected_holdout_sharpe"] - summary["selected_holdout_sharpe_ci95_low"], + summary["selected_holdout_sharpe_ci95_high"] - summary["mean_selected_holdout_sharpe"], + ], + marker="s", + capsize=4, + label="Selected strategy holdout Sharpe", + ) + axes[0].axhline(0, color="black", linewidth=0.8) + axes[0].set_xscale("log") + axes[0].set_xlabel("Number of strategies tried") + axes[0].set_ylabel("Mean annualized Sharpe") + axes[0].set_title("Selection manufactures an in-sample winner") + axes[0].legend() + + axes[1].errorbar( + summary["candidate_count"], + summary["mean_pbo"], + yerr=[ + summary["mean_pbo"] - summary["pbo_ci95_low"], + summary["pbo_ci95_high"] - summary["mean_pbo"], + ], + marker="o", + capsize=4, + color="#d9472b", + ) + axes[1].axhline(0.5, color="black", linewidth=0.8, linestyle="--") + axes[1].set_xscale("log") + axes[1].set_ylim(0, 1) + axes[1].set_xlabel("Number of strategies tried") + axes[1].set_ylabel("Mean probability of backtest overfitting") + axes[1].set_title("CSCV out-of-sample rank failure") + figure.tight_layout() + figure.savefig(output / "selection_bias.png", dpi=180) + documentation = Path("docs/assets/selection_bias.png") + documentation.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(documentation, dpi=180) + plt.close(figure) + + print("ZERO-ALPHA STRATEGY-SELECTION EXPERIMENT") + print(summary.to_string(index=False, float_format=lambda value: f"{value:.3f}")) + print(f"\nSaved results to {output}/") + + +if __name__ == "__main__": + main() diff --git a/tests/test_selection_bias.py b/tests/test_selection_bias.py new file mode 100644 index 0000000..4c6d543 --- /dev/null +++ b/tests/test_selection_bias.py @@ -0,0 +1,64 @@ +import unittest + +import numpy as np +import pandas as pd + +from engine.selection_bias import ( + annualized_sharpe, + combinatorially_symmetric_cv, + run_selection_bias_experiment, + simulate_null_strategies, +) + + +class SelectionBiasTests(unittest.TestCase): + def test_annualized_sharpe_is_columnwise(self): + returns = np.array([[0.01, -0.01], [0.02, 0.01], [0.00, 0.00]]) + scores = annualized_sharpe(returns) + self.assertEqual(scores.shape, (2,)) + self.assertGreater(scores[0], scores[1]) + + def test_cscv_has_all_symmetric_splits(self): + rng = np.random.default_rng(4) + returns = pd.DataFrame(rng.normal(size=(40, 3))) + result = combinatorially_symmetric_cv(returns, n_slices=4) + self.assertEqual(result.splits, 6) + self.assertTrue(0 <= result.probability_of_backtest_overfitting <= 1) + self.assertEqual(len(result.split_results), 6) + + def test_stable_signal_has_low_overfitting_probability(self): + alternating = np.tile([0.009, 0.011], 40) + returns = pd.DataFrame( + { + "stable": alternating, + "unstable_a": np.r_[np.full(40, 0.03), np.full(40, -0.03)], + "unstable_b": np.r_[np.full(40, -0.02), np.full(40, 0.02)], + } + ) + result = combinatorially_symmetric_cv(returns, n_slices=8) + self.assertLess(result.probability_of_backtest_overfitting, 0.25) + + def test_null_simulation_is_reproducible(self): + first = simulate_null_strategies(20, 4, np.random.default_rng(9)) + second = simulate_null_strategies(20, 4, np.random.default_rng(9)) + pd.testing.assert_frame_equal(first, second) + + def test_experiment_is_reproducible(self): + first = run_selection_bias_experiment( + candidate_counts=(3, 6), repetitions=3, observations=40, n_slices=4 + ) + second = run_selection_bias_experiment( + candidate_counts=(3, 6), repetitions=3, observations=40, n_slices=4 + ) + pd.testing.assert_frame_equal(first[0], second[0]) + pd.testing.assert_frame_equal(first[1], second[1]) + + def test_invalid_inputs_are_rejected(self): + with self.assertRaises(ValueError): + combinatorially_symmetric_cv(pd.DataFrame({"only": [0.0] * 20})) + with self.assertRaises(ValueError): + combinatorially_symmetric_cv(pd.DataFrame({"a": [0.0] * 20, "b": [0.0] * 20}), n_slices=3) + + +if __name__ == "__main__": + unittest.main()