From 49dd9567b84c164eb9478ca487494a1e9c69025b Mon Sep 17 00:00:00 2001 From: ShivianNaidoo Date: Mon, 18 May 2026 00:44:00 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(phase10):=20conviction-weighted=20inst?= =?UTF-8?q?itutional=20flow=20=E2=80=94=20primary=20rejected,=20ultra-T1?= =?UTF-8?q?=20variant=20under=20research=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests Δpct_portfolio of T1+T2 active managers (conviction-tier-weighted) as a second alpha alongside fundamental_margin_compression. Primary construction (9 T1 funds @ weight 1.0 + 4 T2 funds @ weight 0.4) is null at every horizon (best raw t=+0.94 at 63d on N=54; HLZ M=400 threshold |t|≈3.78). Registered status='rejected' in signal_registry. Restricting to the 5 ultra-concentrated T1 funds only (Lone Pine, Viking, Tiger, Coatue, Point72 — all avg holdings ≤ 62 of a 140-ticker universe) roughly doubles the full-window 21d t-stat to +1.74 and produces 21d/63d late-third t-stats of +3.02 / +3.33 on n=18-20. Still fails HLZ; late-third is in-sample by construction. Registered as a named variant 'institutional_conviction_flow_ultra_t1' with status='research' and an explicit dated review gate (2026-08-15, promotion rule: full-window 21d t > 2.0 on the Q2-2026-extended sample). NOT wired into aggregator; paper trader unchanged from Phase 10 Session 1. Migration 0012 adds fund_strategy(fund_id, conviction_tier, exclude_from_flow, strategy_note) with a 22-row seed: 9 T1 + 4 T2 + 6 T3 passive + 3 T3 bank 13Fs flagged exclude_from_flow (Goldman/MS/JPM — custody and facilitation flows are not a conviction signal). Factor is point-in-time gated on ownership_edges.available_as_of (the 13F filing date, median +44d after quarter_end). 9 TDD tests cover tier-3 exclusion, bank-exclusion, point-in-time integrity, new-position and liquidation deltas, and size-invariance of the Δpct normalization. --- PROGRESS.md | 2 + migrations/versions/0012_fund_strategy.py | 120 +++++++++ nexus/signals/backtest.py | 81 +++++- nexus/signals/factors/institutional.py | 239 +++++++++++++++++ .../register_institutional_conviction_flow.py | 158 +++++++++++ scripts/run_institutional_ic.py | 132 +++++++++ scripts/run_institutional_ic_subtests.py | 129 +++++++++ tests/test_institutional_factors.py | 252 ++++++++++++++++++ 8 files changed, 1112 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/0012_fund_strategy.py create mode 100644 nexus/signals/factors/institutional.py create mode 100644 scripts/register_institutional_conviction_flow.py create mode 100644 scripts/run_institutional_ic.py create mode 100644 scripts/run_institutional_ic_subtests.py create mode 100644 tests/test_institutional_factors.py diff --git a/PROGRESS.md b/PROGRESS.md index 8475bbf..ac0cd96 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -39,3 +39,5 @@ Phase docs live in `docs/progress/` (untracked, local only). | 8 | Factor backtest Session 3 | COMPLETE — multi-horizon IC backtest on 140-ticker universe (21d/63d/126d); Phase 5 t-stats shown to be N=30 artifacts (momentum_12_1 126d: 4.149→1.506); graph_delta_eigenvector sign-flipped (t=−3.194 at 63d) → wired as centrality penalty overlay in portfolio.py; graph_customer_momentum definitively null (CLOSED); rolling_registry rebuilt (406 rows); **Phase 8 paper trader CAGR +5.97%, Sharpe 0.374, Max DD −36.68%** — degradation vs Phase 7A reflects accurate N=140 factor gates + extended cash periods | `docs/progress/phase_8.md` | | 9 | EDGAR XBRL fundamentals Session 1 | COMPLETE — migration 0011 adds form_type/accession_number/UNIQUE; `form_xbrl.py` parser (filed≥end integrity invariant, 5 metrics, 4 revenue aliases); 74,662 rows / 137 of 140 tickers covered; 4 factors; **`fundamental_margin_compression` t=+4.834 at 126d — first NEXUS factor to PASS HLZ M=400 Bonferroni** (composer sign-flipped vs literature: compression = buy; CALM-regime t=+6.35; sub-window late-third t=+5.13 — strengthening, not decaying); registered status='approved' in signal_registry; rolling_registry refreshed (464 rows); ROA decayed (late-third t=-0.06) → NOT registered; rd_intensity / asset_growth null; **paper trader CAGR +8.72%, Sharpe 0.488, Max DD -32.68%** (vs Phase 8 baseline +5.97% / 0.374 / -36.68%) | `docs/progress/phase_9.md` | | 9 | HGT retraining Session 2 | COMPLETE — retrained HGT on 140-ticker graph (no code changes; metadata extracted dynamically); val AUC **0.9807** at epoch 280 (vs 0.9803 / e=240 on the prior 30-ticker run); 7m 03s wall-clock; `MODEL_VERSION` bumped `hgt_link_pred_v1` → **`hgt_link_pred_v2`**; node_embeddings re-backfilled at 58 monthly snapshots → **8,120 rows** (140 × 58, dim=64); embedding validation passed (cos(NVDA,AMD)=0.98 > cos(NVDA,ARW)=0.63; per-component std median 0.05); **`graph_gnn_embedding_drift` IC backtest NULL at all horizons** (t=+0.382 @ 21d / +0.524 @ 63d / +0.368 @ 126d on N=52..57; HLZ fail by 10×); registered `status='rejected'` in signal_registry with full evidence record; paper trader unchanged from Phase 9 Session 1 | `docs/progress/phase_9.md` | +| 10 | Regime-aware aggregator Session 1 | COMPLETE — `_apply_regime_gate` helper added to aggregator; `fsi_value` plumbed through `load_factor_records` + paper trader; 5 unit tests pass. Tested `non_calm_action='zero'` on `fundamental_margin_compression` (hypothesis: NON-CALM 126d t=−2.31 drags returns). Paper trader: CAGR +8.72%→**+7.68%**, Sharpe 0.488→**0.450**, Max DD −32.68%→**−35.39%**. Hypothesis refuted (factor made money in 4/6 NON-CALM forward months at monthly horizon); gate flag rolled back from `signal_registry`. Infrastructure retained for future opt-in. | `docs/progress/phase_10.md` | +| 10 | Conviction-weighted institutional flow Session 2 | COMPLETE — migration 0012 (`fund_strategy`, 22 rows: 9 T1 + 4 T2 + 6 T3 + 3 excluded banks); `compose_conviction_flow` pure helper (Δpct_portfolio, point-in-time gated on `available_as_of`); 9 TDD tests; institutional panel wired opt-in into backtest. **Primary `institutional_conviction_flow` NULL** (best raw t=+0.94 at 63d, HLZ fail by 4×) → `status='rejected'`. Ultra-T1 sub-test (5 funds: Lone Pine/Viking/Tiger/Coatue/Point72) full-window 21d t=+1.74; late-third 21d t=+3.02, 63d t=+3.33 — material but in-sample, fails HLZ M=400 (|t|≥3.78). Registered **`institutional_conviction_flow_ultra_t1` as `status='research'`** with dated review gate (2026-08-15, promote if full-window 21d t > 2.0 on extended Q2-2026 sample). NOT wired into aggregator. Paper trader unchanged. | `docs/progress/phase_10.md` | diff --git a/migrations/versions/0012_fund_strategy.py b/migrations/versions/0012_fund_strategy.py new file mode 100644 index 0000000..01006e3 --- /dev/null +++ b/migrations/versions/0012_fund_strategy.py @@ -0,0 +1,120 @@ +"""fund_strategy classification table for conviction-weighted institutional flow. + +Revision ID: 0012 +Revises: 0011 +Create Date: 2026-05-17 + +Phase 10 Session 2: classifies the 22 funds in companies (node_type='fund') +into three conviction tiers used by institutional_conviction_flow: + + Tier 1 (weight 1.0): active high-conviction managers + Tier 2 (weight 0.4): active but diversified + Tier 3 (weight 0.0): passive / index + +exclude_from_flow flags funds whose 13F filings are dominated by custody or +client-facilitation activity (the three bank holding companies) and cannot +be interpreted as a conviction signal; the factor loader drops these rows +before any weighting step. + +Tier counts: 9 T1 + 4 T2 + 6 T3 (passive) + 3 T3 (excluded) = 22 rows. +""" +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0012" +down_revision: Union[str, Sequence[str], None] = "0011" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Seed rows: (fund_cik, conviction_tier, exclude_from_flow, strategy_note). +# CIK-keyed (not id-keyed) so the migration is portable across environments +# where companies.id autoincrement may have been re-sequenced. +_SEED: list[tuple[str, int, bool, str]] = [ + # Tier 1 — active high-conviction (weight 1.0) + ("1061165", 1, False, "Lone Pine Capital — concentrated long/short, avg 5.5 holdings"), + ("1103804", 1, False, "Viking Global — concentrated long/short, avg 11.6 holdings"), + ("1167483", 1, False, "Tiger Global — concentrated growth equity, avg 13.2 holdings"), + ("1135730", 1, False, "Coatue — concentrated TMT specialist, avg 16.6 holdings"), + ("1603466", 1, False, "Point72 — active discretionary equity"), + ("1273087", 1, False, "Millennium — multi-manager active equity"), + ("1037389", 1, False, "Renaissance Technologies — systematic active long-bias"), + ("1179392", 1, False, "Two Sigma — quant active"), + ("1423053", 1, False, "Citadel — multi-strat active research-driven sleeves"), + # Tier 2 — active diversified (weight 0.4) + ("1697748", 2, False, "ARK — active thematic, theme-driven rotation"), + ("1088875", 2, False, "Baillie Gifford — active growth, house-style"), + ("1422848", 2, False, "Capital Research Global Investors — American Funds family"), + ("315066", 2, False, "FMR (Fidelity) — active mutual fund family"), + # Tier 3 — passive / index (weight 0.0) + ("102909", 3, False, "Vanguard — index-dominant"), + ("1364742", 3, False, "BlackRock — iShares-dominant"), + ("93751", 3, False, "State Street — SPDR-dominant"), + ("1214717", 3, False, "Geode — Fidelity index sub-advisor"), + ("914208", 3, False, "Invesco — diversified, index-heavy (128 names)"), + ("1374170", 3, False, "Norges Bank — sovereign wealth, near-index"), + # Tier 3 + exclude_from_flow=TRUE — bank 13F mixes prop + custody + facilitation + ("886982", 3, True, "Goldman Sachs — bank 13F, not conviction-classifiable"), + ("895421", 3, True, "Morgan Stanley — bank 13F, not conviction-classifiable"), + ("19617", 3, True, "JPMorgan Chase — bank 13F, not conviction-classifiable"), +] + + +def upgrade() -> None: + op.create_table( + "fund_strategy", + sa.Column( + "fund_id", + sa.BigInteger, + sa.ForeignKey("companies.id"), + primary_key=True, + ), + sa.Column("conviction_tier", sa.SmallInteger, nullable=False), + sa.Column( + "exclude_from_flow", + sa.Boolean, + nullable=False, + server_default=sa.text("FALSE"), + ), + sa.Column("strategy_note", sa.Text, nullable=True), + sa.CheckConstraint( + "conviction_tier IN (1, 2, 3)", + name="ck_fund_strategy_tier_range", + ), + ) + op.create_index( + "ix_fund_strategy_tier", + "fund_strategy", + ["conviction_tier", "exclude_from_flow"], + ) + + bind = op.get_bind() + for cik, tier, excl, note in _SEED: + bind.execute( + sa.text( + """ + INSERT INTO fund_strategy + (fund_id, conviction_tier, exclude_from_flow, strategy_note) + SELECT id, :tier, :excl, :note + FROM companies + WHERE cik = :cik AND node_type = 'fund' + """ + ), + {"cik": cik, "tier": tier, "excl": excl, "note": note}, + ) + + seeded = bind.execute(sa.text("SELECT COUNT(*) FROM fund_strategy")).scalar() + if seeded != len(_SEED): + raise RuntimeError( + f"fund_strategy seed integrity check failed: " + f"expected {len(_SEED)} rows, inserted {seeded}. " + f"A fund CIK in _SEED is missing from companies(node_type='fund')." + ) + + +def downgrade() -> None: + op.drop_index("ix_fund_strategy_tier", table_name="fund_strategy") + op.drop_table("fund_strategy") diff --git a/nexus/signals/backtest.py b/nexus/signals/backtest.py index afab7e2..2481505 100644 --- a/nexus/signals/backtest.py +++ b/nexus/signals/backtest.py @@ -51,6 +51,7 @@ prior_panel as _fund_prior_panel, ) from nexus.signals.factors.graph_based import _compose_signal +from nexus.signals.factors.institutional import compose_conviction_flow from nexus.signals.hlz import format_table, hlz_correct, update_registry_hlz FORWARD_TRADING_DAYS = 21 @@ -459,6 +460,75 @@ def _xs_fundamental_roa( return _df_to_dict(compose_roa(curr)) +# --------------------------------------------------------------------------- +# Institutional conviction flow panel (Phase 10 Session 2) +# --------------------------------------------------------------------------- + +@dataclass +class _InstitutionalPanel: + """All qualifying ownership_edges rows + the T1/T2 fund weight map. + + ``ownership`` columns: fund_id, ticker, quarter_end, available_as_of, + pct_portfolio (float). The cross-section helper filters by + ``available_as_of <= as_of`` BEFORE picking (curr, prev) per fund. + + ``fund_weights`` comes from ``fund_strategy`` and contains only funds + with ``exclude_from_flow=FALSE AND conviction_tier IN (1, 2)``. T3 and + excluded banks are absent — they cannot contribute at all. + """ + ownership: pd.DataFrame + fund_weights: dict[int, float] + + +_TIER_WEIGHT_LOOKUP: dict[int, float] = {1: 1.0, 2: 0.4} + + +def _load_institutional_panel(engine) -> _InstitutionalPanel: + own_df = pd.read_sql( + text( + """ + SELECT oe.fund_id, c.ticker, oe.quarter_end, oe.available_as_of, + CAST(oe.pct_portfolio AS FLOAT) AS pct_portfolio + FROM ownership_edges oe + JOIN companies c ON c.id = oe.company_id + WHERE oe.pct_portfolio IS NOT NULL + """ + ), + engine, + ) + if not own_df.empty: + own_df["quarter_end"] = pd.to_datetime(own_df["quarter_end"]).dt.date + own_df["available_as_of"] = pd.to_datetime(own_df["available_as_of"]).dt.date + + weights_df = pd.read_sql( + text( + """ + SELECT fund_id, conviction_tier + FROM fund_strategy + WHERE exclude_from_flow = FALSE + AND conviction_tier IN (1, 2) + """ + ), + engine, + ) + fund_weights = { + int(row.fund_id): _TIER_WEIGHT_LOOKUP[int(row.conviction_tier)] + for row in weights_df.itertuples() + } + return _InstitutionalPanel(ownership=own_df, fund_weights=fund_weights) + + +def _xs_institutional_conviction_flow( + as_of: date, panel: _InstitutionalPanel +) -> dict[str, float]: + if panel.ownership.empty or not panel.fund_weights: + return {} + composed = compose_conviction_flow(panel.ownership, as_of, panel.fund_weights) + if composed.empty: + return {} + return dict(zip(composed["ticker"], composed["signal"])) + + @dataclass(frozen=True) class _SupplyEdgeRow: filing_date: date @@ -572,6 +642,7 @@ def _xs_gnn_embedding_drift(as_of: date, panel: _EmbeddingPanel) -> dict[str, fl def _factor_xs_dispatch( supply_panel: _SupplyEdgePanel | None = None, fundamentals_panel: _FundamentalsPanel | None = None, + institutional_panel: _InstitutionalPanel | None = None, ) -> dict[str, callable]: """Return name → cross-section closure for each Tier A factor. @@ -611,6 +682,11 @@ def _factor_xs_dispatch( base["fundamental_roa"] = ( lambda snap, pp, cp, ep: _xs_fundamental_roa(snap, fp) ) + if institutional_panel is not None: + ip = institutional_panel + base["institutional_conviction_flow"] = ( + lambda snap, pp, cp, ep: _xs_institutional_conviction_flow(snap, ip) + ) return base @@ -656,6 +732,7 @@ def compute_factor_ics( period_filter: set[date] | None = None, supply_panel: _SupplyEdgePanel | None = None, fundamentals_panel: _FundamentalsPanel | None = None, + institutional_panel: _InstitutionalPanel | None = None, ) -> tuple[list[float], list[int], list[date]]: """Compute per-period ICs for one factor across a set of snapshots. @@ -668,7 +745,9 @@ def compute_factor_ics( Returns (ics, cross_section_sizes, periods). """ dispatch = _factor_xs_dispatch( - supply_panel=supply_panel, fundamentals_panel=fundamentals_panel, + supply_panel=supply_panel, + fundamentals_panel=fundamentals_panel, + institutional_panel=institutional_panel, ) xs_func = dispatch[name] ics: list[float] = [] diff --git a/nexus/signals/factors/institutional.py b/nexus/signals/factors/institutional.py new file mode 100644 index 0000000..e701077 --- /dev/null +++ b/nexus/signals/factors/institutional.py @@ -0,0 +1,239 @@ +"""Institutional conviction flow factor — Phase 10 Session 2. + +Conviction-weighted quarter-over-quarter change in 13F portfolio weights. + +Hypothesis (Grinblatt-Titman 1989, Kacperczyk-Sialm-Zheng 2005, Wermers 2000): +position changes by *active high-conviction* managers carry forward-return +information; position changes by passive index funds reflect AUM flow and +rebalancing noise. The signal is therefore a tier-weighted sum of fund-level +portfolio-weight deltas: + + signal_i = Σ_f conviction_weight_f × Δpct_portfolio_{f, i} + +where: + - conviction_weight_f is taken from the ``fund_strategy`` table seeded in + migration 0012: Tier 1 → 1.0, Tier 2 → 0.4. Tier 3 funds and funds with + ``exclude_from_flow = TRUE`` are absent from the weight map and dropped + before any aggregation. + - Δpct_portfolio_{f, i} = pct_portfolio_curr - pct_portfolio_prev, with + new positions treated as +pct_curr and liquidations as -pct_prev. This + normalization is point-in-time native (no external shares-outstanding + source needed) and matches the Wermers / Grinblatt-Titman definition of + institutional flow as portfolio-weight change. + +Sign convention: HIGH signal → expected HIGHER forward return (buying by +conviction funds is bullish). + +Point-in-time integrity +----------------------- +All factor queries filter ``ownership_edges`` rows by +``available_as_of <= as_of`` BEFORE picking the (curr, prev) quarter pair. +``available_as_of`` is the 13F filing date (median 44 days after +quarter_end); using ``quarter_end`` for the gate would leak ~6 weeks of +forward information. + +Pure helper vs DB-backed wrapper +-------------------------------- +``compose_conviction_flow`` is pure: pandas in, pandas out, no IO. Tests +bind to it directly. + +``compute_conviction_flow`` is a thin DB-backed wrapper that loads +``ownership_edges`` once, fetches the fund-tier weight map from +``fund_strategy``, and calls the composer. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date + +import numpy as np +import pandas as pd +from sqlalchemy import create_engine, text + +from nexus.config import settings +from nexus.signals.factors.price_based import _factor_frame, register_signal + +# Production tier weights — must match the seed in migration 0012. +_TIER_WEIGHT: dict[int, float] = {1: 1.0, 2: 0.4, 3: 0.0} + + +@dataclass(frozen=True) +class FactorMeta: + name: str + signal_type: str + tier: str + description: str + + +CONVICTION_FLOW = FactorMeta( + name="institutional_conviction_flow", + signal_type="institutional", + tier="A", + description=( + "Conviction-weighted QoQ change in 13F portfolio weights. T1 funds " + "(weight 1.0) and T2 funds (weight 0.4) contribute; T3 and excluded " + "bank 13F rows are filtered. High = active buying by conviction funds." + ), +) + + +# --------------------------------------------------------------------------- +# Pure composer +# --------------------------------------------------------------------------- + +def _pct_for_quarter( + fund_ticker_rows: pd.DataFrame, quarter_end: date +) -> float | None: + """Restatement-aware lookup of pct_portfolio for one fund-ticker at one + quarter_end. Returns the value from the row with the latest + ``available_as_of`` (so an amended filing supersedes the original).""" + matching = fund_ticker_rows[fund_ticker_rows["quarter_end"] == quarter_end] + if matching.empty: + return None + latest = matching.sort_values("available_as_of", ascending=False).iloc[0] + return float(latest["pct_portfolio"]) + + +def compose_conviction_flow( + ownership_df: pd.DataFrame, + as_of: date, + fund_weights: dict[int, float], +) -> pd.DataFrame: + """Pure composer: ownership rows → per-ticker z-scored conviction flow. + + Parameters + ---------- + ownership_df + Long-format DataFrame with columns: + ``[fund_id, ticker, quarter_end, available_as_of, pct_portfolio]``. + May contain rows from any fund — funds not in ``fund_weights`` are + dropped before aggregation. + as_of + Decision date. Rows with ``available_as_of > as_of`` are excluded. + fund_weights + Mapping ``fund_id -> conviction_weight``. Funds absent from this + map contribute nothing. + + Returns + ------- + DataFrame with columns ``[ticker, signal]`` where ``signal`` is the + cross-sectional z-score over qualifying tickers. Tickers with no + qualifying contributor are absent from the output (not NaN-filled). + """ + if ownership_df.empty: + return pd.DataFrame(columns=["ticker", "signal"]) + + # 1. Drop funds outside the weight map. This handles T3 and excluded + # banks in one step — they are simply absent from fund_weights. + qualifying = ownership_df[ownership_df["fund_id"].isin(fund_weights.keys())] + if qualifying.empty: + return pd.DataFrame(columns=["ticker", "signal"]) + + # 2. Point-in-time filter — must come BEFORE picking (curr, prev). + visible = qualifying[qualifying["available_as_of"] <= as_of] + if visible.empty: + return pd.DataFrame(columns=["ticker", "signal"]) + + # 3. For each fund, find its latest visible quarter and the prior one. + # Liquidations (in prior, absent in latest) and new positions + # (absent in prior, in latest) are detected via fund-level quarters, + # NOT per-ticker — otherwise a one-row prior-only fixture is + # indistinguishable from a held position. + contributions: dict[str, float] = {} + for fund_id, fund_rows in visible.groupby("fund_id"): + fund_quarters = sorted(fund_rows["quarter_end"].unique(), reverse=True) + if not fund_quarters: + continue + q_curr = fund_quarters[0] + q_prev = fund_quarters[1] if len(fund_quarters) >= 2 else None + tickers = fund_rows["ticker"].unique() + weight = fund_weights[fund_id] + for ticker in tickers: + grp = fund_rows[fund_rows["ticker"] == ticker] + pct_curr = _pct_for_quarter(grp, q_curr) + pct_prev = _pct_for_quarter(grp, q_prev) if q_prev is not None else None + if pct_curr is None and pct_prev is None: + continue + if pct_curr is not None and pct_prev is not None: + delta = pct_curr - pct_prev + elif pct_curr is not None: # new position this quarter + delta = pct_curr + else: # liquidated: held prior, absent in latest fund quarter + delta = -pct_prev # type: ignore[operator] + contributions[ticker] = contributions.get(ticker, 0.0) + weight * delta + + if not contributions: + return pd.DataFrame(columns=["ticker", "signal"]) + + raw = pd.Series(contributions, name="raw") + # 4. Cross-sectional z-score. Constant-cross-section degenerate case + # (std == 0 or only one ticker) returns 0.0 for every surviving ticker — + # NaN would be misleading because the data IS present, just informationless. + std = raw.std(ddof=0) + if std == 0 or len(raw) < 2 or not np.isfinite(std): + z = pd.Series(0.0, index=raw.index) + else: + z = (raw - raw.mean()) / std + return pd.DataFrame({"ticker": z.index, "signal": z.values}) + + +# --------------------------------------------------------------------------- +# DB-backed wrapper +# --------------------------------------------------------------------------- + +_OWNERSHIP_QUERY = text( + """ + SELECT oe.fund_id, + c.ticker, + oe.quarter_end, + oe.available_as_of, + CAST(oe.pct_portfolio AS FLOAT) AS pct_portfolio + FROM ownership_edges oe + JOIN companies c ON c.id = oe.company_id + WHERE oe.pct_portfolio IS NOT NULL + """ +) + +_FUND_WEIGHTS_QUERY = text( + """ + SELECT fund_id, conviction_tier + FROM fund_strategy + WHERE exclude_from_flow = FALSE + AND conviction_tier IN (1, 2) + """ +) + + +def load_ownership() -> pd.DataFrame: + """Load all qualifying ownership_edges rows. Single roundtrip.""" + engine = create_engine(settings.database_url_sync) + with engine.connect() as c: + df = pd.read_sql(_OWNERSHIP_QUERY, c) + engine.dispose() + df["quarter_end"] = pd.to_datetime(df["quarter_end"]).dt.date + df["available_as_of"] = pd.to_datetime(df["available_as_of"]).dt.date + return df + + +def load_fund_weights() -> dict[int, float]: + """Load tier → weight mapping for T1/T2 funds, excluded banks omitted.""" + engine = create_engine(settings.database_url_sync) + with engine.connect() as c: + rows = c.execute(_FUND_WEIGHTS_QUERY).fetchall() + engine.dispose() + return {int(fund_id): _TIER_WEIGHT[int(tier)] for fund_id, tier in rows} + + +def compute_conviction_flow( + as_of_date: date, register: bool = False +) -> pd.DataFrame: + """Public API: factor frame ready for the aggregator.""" + ownership = load_ownership() + fund_weights = load_fund_weights() + composed = compose_conviction_flow(ownership, as_of_date, fund_weights) + if register: + register_signal( + CONVICTION_FLOW.name, CONVICTION_FLOW.signal_type, CONVICTION_FLOW.tier + ) + values = dict(zip(composed["ticker"], composed["signal"])) + return _factor_frame(values, CONVICTION_FLOW.name) diff --git a/scripts/register_institutional_conviction_flow.py b/scripts/register_institutional_conviction_flow.py new file mode 100644 index 0000000..2733daa --- /dev/null +++ b/scripts/register_institutional_conviction_flow.py @@ -0,0 +1,158 @@ +"""Phase 10 Session 2 — register both institutional conviction flow variants. + +Two rows are upserted into signal_registry. Neither is wired into the +aggregator; this script records verdicts only. + +1) institutional_conviction_flow — status='rejected' + Original construction (T1 weight 1.0, T2 weight 0.4, 13 funds). + Null at every horizon (|t| < 1 overall: +0.81 / +0.94 / -0.01 at + 21d/63d/126d on N=51..56). HLZ Bonferroni fails by an order of + magnitude. Cleanly null. + +2) institutional_conviction_flow_ultra_t1 — status='research' + Restricted construction: 5 ultra-concentrated T1 funds only + (Lone Pine, Viking, Tiger, Coatue, Point72), all weight 1.0. + Full-window 21d t=+1.74 (below HLZ); late-third (n=20) shows + 21d t=+3.02 and 63d t=+3.33 — material but in-sample by + construction. Recorded as research-only with an explicit + review gate: re-evaluate after Q2 2026 13F filings land + (~mid-August 2026). If the full-window 21d t-stat exceeds 2.0 + on the extended sample, promote to approved and wire into + the aggregator. + +Idempotent: INSERT ... ON CONFLICT (name) DO UPDATE. +""" +import json + +from sqlalchemy import create_engine, text + +from nexus.config import settings + + +_REJECTED_NAME = "institutional_conviction_flow" +_RESEARCH_NAME = "institutional_conviction_flow_ultra_t1" + + +_REJECTED_PROFILE = { + "role": "institutional_flow", + "verdict": "null_at_full_power", + "phase": "10.2", + "construction": ( + "Δpct_portfolio weighted by conviction tier: T1=1.0 (9 funds), " + "T2=0.4 (4 funds). T3 and bank-13F funds excluded entirely." + ), + "universe_size": 140, + "horizons": { + "21d": {"n": 56, "mean_ic": +0.0117, "t_stat": +0.812, "hlz_passes": False}, + "63d": {"n": 54, "mean_ic": +0.0130, "t_stat": +0.938, "hlz_passes": False}, + "126d": {"n": 51, "mean_ic": -0.0002, "t_stat": -0.012, "hlz_passes": False}, + }, + "regime_split_63d": { + "calm": {"n": 46, "mean_ic": +0.0204, "t_stat": +1.571}, + "non_calm": {"n": 8, "mean_ic": -0.0298, "t_stat": -0.523}, + }, + "evidence": ( + "Best raw |t| = 0.94 at 63d. HLZ M=400 threshold |t| ≈ 3.78. " + "Even raw α=0.01 (|t| ≥ 2.57) is not met. CALM-only 63d t = " + "+1.571 — directionally correct but underpowered. Tier-2 + " + "diversified T1 dilution suspected; see ultra_t1 variant." + ), +} + + +_RESEARCH_PROFILE = { + "role": "institutional_flow_concentrated", + "verdict": "research_only_pending_review_gate", + "phase": "10.2", + "construction": ( + "Δpct_portfolio weighted equally across 5 ultra-concentrated T1 " + "funds only: Lone Pine (5.5 avg holdings), Viking (11.6), Tiger " + "(13.2), Coatue (16.6), Point72 (62.0). All weight 1.0. T2 and " + "all other funds excluded." + ), + "fund_ciks": ["1061165", "1103804", "1167483", "1135730", "1603466"], + "universe_size": 140, + "horizons_full_window": { + "21d": {"n": 56, "mean_ic": +0.0303, "t_stat": +1.740, "avg_xs": 85.5}, + "63d": {"n": 54, "mean_ic": +0.0181, "t_stat": +0.954, "avg_xs": 85.2}, + "126d": {"n": 51, "mean_ic": +0.0023, "t_stat": +0.127, "avg_xs": 84.9}, + }, + "horizons_late_third": { + "window": "2024-08-31 -> 2026-03-31", + "21d": {"n": 20, "mean_ic": +0.0624, "t_stat": +3.023, "avg_xs": 88.6}, + "63d": {"n": 18, "mean_ic": +0.0519, "t_stat": +3.330, "avg_xs": 88.0}, + "126d": {"n": 15, "mean_ic": +0.0032, "t_stat": +0.170, "avg_xs": 87.6}, + }, + "hlz_passes": False, + "hlz_note": ( + "HLZ M=400 Bonferroni threshold |t| ≈ 3.78. Best late-third " + "result is 63d t=3.330 — clears raw α=0.001 but not HLZ. " + "Late-third was selected from the same data and is in-sample." + ), + "review_gate": { + "trigger_date": "2026-08-15", + "trigger_event": "Q2 2026 13F filings available", + "promotion_rule": ( + "If full-window 21d t-stat exceeds 2.0 on the extended sample " + "(adds ~2 quarters of out-of-sample snapshots), promote " + "status from 'research' to 'approved' and wire into the " + "aggregator. Otherwise downgrade to 'rejected'." + ), + "baseline_t_21d_full_window": 1.740, + }, + "wired_into_aggregator": False, +} + + +def _upsert(conn, name: str, type_: str, tier: str, t_stat: float, + hlz_passes: bool, status: str, profile: dict) -> dict: + result = conn.execute( + text( + """ + INSERT INTO signal_registry + (name, type, tier, t_stat, hlz_passes, status, regime_profile) + VALUES + (:name, :type, :tier, :t_stat, :hlz_passes, :status, + CAST(:regime_profile AS JSONB)) + ON CONFLICT (name) DO UPDATE + SET t_stat = EXCLUDED.t_stat, + hlz_passes = EXCLUDED.hlz_passes, + status = EXCLUDED.status, + regime_profile = EXCLUDED.regime_profile + RETURNING signal_id, name, status, t_stat, hlz_passes + """ + ), + { + "name": name, "type": type_, "tier": tier, + "t_stat": t_stat, "hlz_passes": hlz_passes, + "status": status, "regime_profile": json.dumps(profile), + }, + ) + return result.fetchone()._asdict() + + +def main() -> None: + engine = create_engine(settings.database_url_sync) + with engine.begin() as conn: + rejected = _upsert( + conn, _REJECTED_NAME, "institutional", "A", + t_stat=0.938, # highest |t| across the three tested horizons (63d) + hlz_passes=False, status="rejected", profile=_REJECTED_PROFILE, + ) + research = _upsert( + conn, _RESEARCH_NAME, "institutional", "A", + t_stat=3.330, # highest |t| in the late-third sub-window (63d) + hlz_passes=False, status="research", profile=_RESEARCH_PROFILE, + ) + engine.dispose() + + for row in (rejected, research): + print( + f"[+] signal_registry: signal_id={row['signal_id']} " + f"name={row['name']} status={row['status']} " + f"t_stat={row['t_stat']} hlz_passes={row['hlz_passes']}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_institutional_ic.py b/scripts/run_institutional_ic.py new file mode 100644 index 0000000..300693b --- /dev/null +++ b/scripts/run_institutional_ic.py @@ -0,0 +1,132 @@ +"""Phase 10 Session 2 — IC backtest for institutional_conviction_flow. + +Diagnostic-A format: 21d / 63d / 126d forward-return horizons, plus CALM vs +NON-CALM FSI regime split, plus a late-third sub-window decay check. Mirrors +the Phase 9 Session 1 margin-compression diagnostic. + +Read-only — does NOT touch signal_registry or rolling_registry. Step 4 of +the Phase 10 plan is gated on user approval of these numbers. +""" +from __future__ import annotations + +import numpy as np +from sqlalchemy import create_engine + +from nexus.config import settings +from nexus.signals.backtest import ( + CALM_FSI_THRESHOLD, + FORWARD_TRADING_DAYS, + _aggregate, + _compute_forward_returns, + _load_fsi_by_date, + _load_institutional_panel, + compute_factor_ics, + load_all_panels, +) + + +FACTOR_NAME = "institutional_conviction_flow" + + +def _hlz_verdict(t: float, n: int) -> str: + """One-line HLZ M=400 Bonferroni verdict for a t-stat.""" + if not np.isfinite(t) or n < 10: + return "INSUFFICIENT_N" + # Two-sided Bonferroni threshold for M=400 tests at α=0.05. + # From the project's hlz module: critical |t| ≈ 3.78 (N~50, df~49). + if abs(t) >= 3.78: + return "PASS" + if abs(t) >= 2.57: # raw α=0.01 two-sided, ignoring HLZ + return "BORDERLINE" + return "FAIL" + + +def _print_horizon_row(label: str, ics, sizes, periods, t: float) -> None: + n = len(ics) + mean_ic = float(np.mean(ics)) if ics else float("nan") + std_ic = float(np.std(ics, ddof=1)) if n > 1 else float("nan") + icir = mean_ic / std_ic * np.sqrt(n) if (n > 1 and std_ic > 0) else float("nan") + hit = float(np.mean([1.0 if ic > 0 else 0.0 for ic in ics])) if ics else float("nan") + print( + f" {label:24s} N={n:>3d} mean_IC={mean_ic:>+.4f} " + f"t={t:>+.3f} IC-IR={icir:>+.3f} hit={hit:>.2%} " + f"avg_xs={float(np.mean(sizes)) if sizes else float('nan'):>5.1f} " + f"verdict={_hlz_verdict(t, n)}" + ) + + +def main() -> None: + engine = create_engine(settings.database_url_sync) + print("[*] Loading panels (price, centrality, embedding, institutional)...", flush=True) + panels = load_all_panels(engine) + price_panel, centrality_panel, _embed = panels + institutional_panel = _load_institutional_panel(engine) + snap_dates = centrality_panel.sorted_dates + fsi_by_date = _load_fsi_by_date(snap_dates, engine) + engine.dispose() + + n_funds = len(institutional_panel.fund_weights) + n_own_rows = len(institutional_panel.ownership) + print(f"[*] Snapshots: {len(snap_dates)} ({snap_dates[0]} -> {snap_dates[-1]})") + print(f"[*] Conviction funds (T1+T2): {n_funds} ownership rows: {n_own_rows:,}") + + # -------------------------------------------------------------------- # + # A. Multi-horizon sweep + # -------------------------------------------------------------------- # + print(f"\n=== Diagnostic-A — institutional_conviction_flow ===") + horizon_results: dict[int, tuple[list[float], int]] = {} + for h in (FORWARD_TRADING_DAYS, 63, 126): + fwd = _compute_forward_returns(snap_dates, price_panel, forward_days=h) + ics, sizes, periods = compute_factor_ics( + FACTOR_NAME, snap_dates, fwd, *panels, + institutional_panel=institutional_panel, + ) + agg = _aggregate(FACTOR_NAME, ics, sizes, periods) + horizon_results[h] = (ics, agg.n_periods) + _print_horizon_row(f"horizon={h}d overall", ics, sizes, periods, agg.t_stat) + + # -------------------------------------------------------------------- # + # B. Regime split at 21d, 63d, 126d + # -------------------------------------------------------------------- # + print(f"\n=== Regime split (CALM = FSI < {CALM_FSI_THRESHOLD}, NON-CALM = FSI >= {CALM_FSI_THRESHOLD}) ===") + calm = {d for d in snap_dates if fsi_by_date.get(d) is not None and fsi_by_date[d] < CALM_FSI_THRESHOLD} + noncalm = {d for d in snap_dates if fsi_by_date.get(d) is not None and fsi_by_date[d] >= CALM_FSI_THRESHOLD} + print(f" CALM periods: {len(calm)}") + print(f" NON-CALM periods: {len(noncalm)}") + for h in (FORWARD_TRADING_DAYS, 63, 126): + fwd = _compute_forward_returns(snap_dates, price_panel, forward_days=h) + for label, period_filter in (("CALM", calm), ("NON-CALM", noncalm)): + ics, sizes, periods = compute_factor_ics( + FACTOR_NAME, snap_dates, fwd, *panels, + period_filter=period_filter, + institutional_panel=institutional_panel, + ) + agg = _aggregate(FACTOR_NAME, ics, sizes, periods) + _print_horizon_row(f"{h}d {label}", ics, sizes, periods, agg.t_stat) + + # -------------------------------------------------------------------- # + # C. Late-third sub-window (decay check) + # -------------------------------------------------------------------- # + print(f"\n=== Late-third sub-window decay check ===") + n = len(snap_dates) + late_cutoff = snap_dates[(2 * n) // 3] + late_set = {d for d in snap_dates if d >= late_cutoff} + print(f" Late-third cutoff date: {late_cutoff} (n={len(late_set)} periods)") + for h in (FORWARD_TRADING_DAYS, 63, 126): + fwd = _compute_forward_returns(snap_dates, price_panel, forward_days=h) + ics, sizes, periods = compute_factor_ics( + FACTOR_NAME, snap_dates, fwd, *panels, + period_filter=late_set, + institutional_panel=institutional_panel, + ) + agg = _aggregate(FACTOR_NAME, ics, sizes, periods) + _print_horizon_row(f"{h}d late-third", ics, sizes, periods, agg.t_stat) + + print( + "\n[*] No registry writes performed. Awaiting user approval before " + "Step 4 (registry + rolling registry + paper trader)." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_institutional_ic_subtests.py b/scripts/run_institutional_ic_subtests.py new file mode 100644 index 0000000..37fce71 --- /dev/null +++ b/scripts/run_institutional_ic_subtests.py @@ -0,0 +1,129 @@ +"""Phase 10 Session 2 — sub-tests requested after the full-T1+T2 IC failed HLZ. + +Two restrictions on fund_weights, both read-only (no registry writes): + + 1. Ultra-concentrated T1 only — 5 funds (Lone Pine, Viking, Tiger, Coatue, + Point72), all weight 1.0. Re-run IC at 21d/63d/126d. + 2. Same 5 funds, restricted to the late-third snapshot window + (2024-08-31 -> 2026-03-31). + +Pass criterion: t > 2.0 at any horizon -> report and pause; otherwise both +fail and the session closes with status='rejected'. +""" +from __future__ import annotations + +import numpy as np +from sqlalchemy import create_engine + +from nexus.config import settings +from nexus.signals.backtest import ( + FORWARD_TRADING_DAYS, + _aggregate, + _compute_forward_returns, + _load_institutional_panel, + compute_factor_ics, + load_all_panels, +) + + +FACTOR_NAME = "institutional_conviction_flow" + +# Resolve fund_ids from companies.cik (canonical, env-stable). +_ULTRA_T1_CIKS = { + "1061165", # Lone Pine + "1103804", # Viking Global + "1167483", # Tiger Global + "1135730", # Coatue + "1603466", # Point72 +} + + +def _resolve_fund_ids(engine, ciks: set[str]) -> dict[int, float]: + rows = engine.execute( + # SQLAlchemy 2.x: use text via the engine.connect context; this is + # a small one-shot script so the simpler pattern is fine. + __import__("sqlalchemy").text( + "SELECT id, cik FROM companies WHERE cik = ANY(:ciks) AND node_type='fund'" + ), + {"ciks": list(ciks)}, + ).fetchall() + return {int(r.id): 1.0 for r in rows} + + +def _print_row(label: str, ics, sizes, t: float) -> None: + n = len(ics) + mean_ic = float(np.mean(ics)) if ics else float("nan") + print( + f" {label:32s} N={n:>3d} mean_IC={mean_ic:>+.4f} " + f"t={t:>+.3f} avg_xs={float(np.mean(sizes)) if sizes else float('nan'):>5.1f}" + ) + + +def main() -> None: + engine = create_engine(settings.database_url_sync) + print("[*] Loading panels...", flush=True) + panels = load_all_panels(engine) + price_panel, centrality_panel, _embed = panels + institutional_panel = _load_institutional_panel(engine) + snap_dates = centrality_panel.sorted_dates + + # Resolve the 5 ultra-concentrated fund_ids. + with engine.connect() as conn: + from sqlalchemy import text + rows = conn.execute( + text( + "SELECT id, cik, name FROM companies " + "WHERE cik = ANY(:ciks) AND node_type='fund'" + ), + {"ciks": list(_ULTRA_T1_CIKS)}, + ).fetchall() + engine.dispose() + + ultra_weights = {int(r.id): 1.0 for r in rows} + print(f"[*] Ultra-T1 funds resolved ({len(ultra_weights)}):") + for r in rows: + print(f" fund_id={r.id:>3d} CIK={r.cik:>8s} {r.name}") + assert len(ultra_weights) == 5, f"expected 5 funds, got {len(ultra_weights)}" + + # Override the panel's weights map. The factor composer drops any + # fund_id absent from the map, so this is the right restriction point. + institutional_panel.fund_weights = ultra_weights + + # ------------------------------------------------------------------ + # Sub-test 1 — full window, ultra-T1 only + # ------------------------------------------------------------------ + print("\n=== Sub-test 1: Ultra-T1 only (5 funds, weight 1.0), full window ===") + for h in (FORWARD_TRADING_DAYS, 63, 126): + fwd = _compute_forward_returns(snap_dates, price_panel, forward_days=h) + ics, sizes, periods = compute_factor_ics( + FACTOR_NAME, snap_dates, fwd, *panels, + institutional_panel=institutional_panel, + ) + agg = _aggregate(FACTOR_NAME, ics, sizes, periods) + _print_row(f"horizon={h}d", ics, sizes, agg.t_stat) + + # ------------------------------------------------------------------ + # Sub-test 2 — late-third window, ultra-T1 only + # ------------------------------------------------------------------ + n = len(snap_dates) + late_cutoff = snap_dates[(2 * n) // 3] + late_set = {d for d in snap_dates if d >= late_cutoff} + print( + f"\n=== Sub-test 2: Ultra-T1 only, late-third window " + f"({late_cutoff} -> {snap_dates[-1]}, n={len(late_set)}) ===" + ) + for h in (FORWARD_TRADING_DAYS, 63, 126): + fwd = _compute_forward_returns(snap_dates, price_panel, forward_days=h) + ics, sizes, periods = compute_factor_ics( + FACTOR_NAME, snap_dates, fwd, *panels, + period_filter=late_set, + institutional_panel=institutional_panel, + ) + agg = _aggregate(FACTOR_NAME, ics, sizes, periods) + _print_row(f"horizon={h}d late-third", ics, sizes, agg.t_stat) + + print("\n[*] No registry writes performed.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_institutional_factors.py b/tests/test_institutional_factors.py new file mode 100644 index 0000000..df012ff --- /dev/null +++ b/tests/test_institutional_factors.py @@ -0,0 +1,252 @@ +"""Unit tests for Phase 10 Session 2 institutional conviction flow factor. + +Pure-helper tests against ``compose_conviction_flow``. No DB, no network. +Fixture frames carry the same schema the DB-backed loader produces: + + columns: [fund_id, ticker, quarter_end, available_as_of, pct_portfolio] + +Tier weights used throughout the fixtures match the production seed in +migration 0012 (T1=1.0, T2=0.4, T3 absent from the weights map). +""" +from __future__ import annotations + +import math +from datetime import date + +import pandas as pd +import pytest + +from nexus.signals.factors.institutional import compose_conviction_flow + +# Fund-id labels chosen to match real seed rows so the tests double as a +# sanity check on the integration. Goldman (39) is in the "excluded" set +# and is intentionally NOT in any test's fund_weights map. +_T1_RENAISSANCE = 44 +_T1_LONE_PINE = 51 +_T2_FMR = 35 +_T3_VANGUARD = 31 # weight 0.0 → omitted from fund_weights +_EXCL_GOLDMAN = 39 # exclude_from_flow → omitted from fund_weights + +# Reference snapshot date used across tests. +_AS_OF = date(2024, 3, 15) +_Q_PRIOR = date(2023, 9, 30) +_Q_CURR = date(2023, 12, 31) +_FILED_PRIOR = date(2023, 11, 14) +_FILED_CURR = date(2024, 2, 14) + + +def _row(fund_id, ticker, quarter_end, available_as_of, pct_portfolio): + return { + "fund_id": fund_id, + "ticker": ticker, + "quarter_end": quarter_end, + "available_as_of": available_as_of, + "pct_portfolio": float(pct_portfolio), + } + + +def _signal(df: pd.DataFrame, ticker: str) -> float: + """Return the signal for `ticker`, or NaN if absent.""" + row = df[df["ticker"] == ticker] + if row.empty: + return float("nan") + return float(row["signal"].iloc[0]) + + +# --------------------------------------------------------------------------- +# Tier-3 / excluded handling +# --------------------------------------------------------------------------- + +def test_tier3_funds_contribute_zero_weight(): + """T3 funds are absent from fund_weights — their position changes do + not move the signal. With no T1/T2 contributors, the universe is empty + and the result is empty (no NaN rows manufactured).""" + facts = pd.DataFrame([ + _row(_T3_VANGUARD, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T3_VANGUARD, "NVDA", _Q_CURR, _FILED_CURR, 0.030), # +2pp + ]) + out = compose_conviction_flow( + facts, as_of=_AS_OF, + fund_weights={_T1_RENAISSANCE: 1.0, _T2_FMR: 0.4}, + ) + assert out.empty or _signal(out, "NVDA") != _signal(out, "NVDA") # NaN + + +def test_excluded_banks_filtered_before_weighting(): + """Bank rows are absent from fund_weights and must not affect the + signal even when they show large position changes.""" + base = pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.020), + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.030), + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "AMD", _Q_CURR, _FILED_CURR, 0.012), + ]) + with_goldman = pd.concat([base, pd.DataFrame([ + _row(_EXCL_GOLDMAN, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.005), + _row(_EXCL_GOLDMAN, "NVDA", _Q_CURR, _FILED_CURR, 0.080), # huge buy + ])], ignore_index=True) + + weights = {_T1_RENAISSANCE: 1.0, _T2_FMR: 0.4} + out_base = compose_conviction_flow(base, as_of=_AS_OF, fund_weights=weights) + out_with = compose_conviction_flow(with_goldman, as_of=_AS_OF, fund_weights=weights) + + # Signals must be identical (Goldman ignored). + for t in ("NVDA", "AMD"): + assert math.isclose(_signal(out_base, t), _signal(out_with, t), abs_tol=1e-12) + + +# --------------------------------------------------------------------------- +# Directional behaviour +# --------------------------------------------------------------------------- + +def test_tier1_fund_buying_produces_positive_signal(): + """Renaissance increases NVDA portfolio weight; AMD flat. NVDA should + z-score above zero and AMD below zero.""" + facts = pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.040), # +3pp + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "AMD", _Q_CURR, _FILED_CURR, 0.010), # 0 + ]) + out = compose_conviction_flow( + facts, as_of=_AS_OF, fund_weights={_T1_RENAISSANCE: 1.0}, + ) + assert _signal(out, "NVDA") > 0 + assert _signal(out, "AMD") < 0 + + +def test_new_position_treated_as_positive_delta(): + """Fund opens an NVDA position this quarter; no prior row. Δ = +current + pct_portfolio, NOT NaN.""" + facts = pd.DataFrame([ + # NVDA only in current quarter + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.020), + # AMD baseline so cross-section has ≥2 tickers for z-score + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "AMD", _Q_CURR, _FILED_CURR, 0.010), + ]) + out = compose_conviction_flow( + facts, as_of=_AS_OF, fund_weights={_T1_RENAISSANCE: 1.0}, + ) + # NVDA must be present and positive (new T1 buy = bullish signal). + sig = _signal(out, "NVDA") + assert not math.isnan(sig) + assert sig > _signal(out, "AMD") + + +def test_liquidation_treated_as_negative_delta(): + """Fund held AMD prior quarter, no current row → Δ = -prior.""" + facts = pd.DataFrame([ + # AMD only in prior quarter (liquidated) + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.020), + # NVDA baseline (flat) + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.010), + ]) + out = compose_conviction_flow( + facts, as_of=_AS_OF, fund_weights={_T1_RENAISSANCE: 1.0}, + ) + assert _signal(out, "AMD") < _signal(out, "NVDA") + + +# --------------------------------------------------------------------------- +# Point-in-time integrity +# --------------------------------------------------------------------------- + +def test_point_in_time_excludes_future_filings(): + """A 13F filed after as_of must be invisible to the factor. + + Reference frame contains only the prior quarter (visible). + Test frame adds a current-quarter row filed AFTER as_of — output must + match the reference (NVDA flat, no Δ → NaN cross-section). + """ + reference = pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.020), + _row(_T1_RENAISSANCE, "AMD", _Q_CURR, _FILED_CURR, 0.025), + ]) + invisible_filing = date(2024, 6, 15) # after _AS_OF + later = pd.concat([reference, pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", date(2024, 3, 31), invisible_filing, 0.080), + ])], ignore_index=True) + weights = {_T1_RENAISSANCE: 1.0} + ref_out = compose_conviction_flow(reference, as_of=_AS_OF, fund_weights=weights) + test_out = compose_conviction_flow(later, as_of=_AS_OF, fund_weights=weights) + pd.testing.assert_frame_equal( + ref_out.sort_values("ticker").reset_index(drop=True), + test_out.sort_values("ticker").reset_index(drop=True), + ) + + +def test_no_ownership_data_returns_nan(): + """Ticker with no T1/T2 rows at all is absent from the output (the + cross-section is built over tickers with at least one qualifying + contributor).""" + facts = pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.030), + # MRVL only owned by Vanguard (T3): Vanguard absent from weights, + # MRVL has no T1/T2 rows. + _row(_T3_VANGUARD, "MRVL", _Q_CURR, _FILED_CURR, 0.005), + ]) + out = compose_conviction_flow( + facts, as_of=_AS_OF, + fund_weights={_T1_RENAISSANCE: 1.0, _T2_FMR: 0.4}, + ) + assert "MRVL" not in set(out["ticker"]) + assert "NVDA" in set(out["ticker"]) + + +# --------------------------------------------------------------------------- +# Normalization invariance +# --------------------------------------------------------------------------- + +def test_pct_portfolio_normalization_size_invariant(): + """A 1pp portfolio shift in NVDA (large-cap) and in COHU (tiny-cap) + produce equal pre-z fund-weighted contributions. + + Test by constructing a fully symmetric frame: same Δpct, same weighting, + only ticker label differs. The z-scores must be equal-and-opposite or + equal — never asymmetric due to a hidden size dependency. + """ + facts = pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.020), + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.030), # +1pp + _row(_T1_RENAISSANCE, "COHU", _Q_PRIOR, _FILED_PRIOR, 0.001), + _row(_T1_RENAISSANCE, "COHU", _Q_CURR, _FILED_CURR, 0.011), # +1pp + # Third anchor so the cross-section is non-trivial. + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "AMD", _Q_CURR, _FILED_CURR, 0.010), + ]) + out = compose_conviction_flow( + facts, as_of=_AS_OF, fund_weights={_T1_RENAISSANCE: 1.0}, + ) + assert math.isclose(_signal(out, "NVDA"), _signal(out, "COHU"), abs_tol=1e-9) + + +def test_tier_weight_scales_contribution(): + """A T2 fund (weight 0.4) should produce a smaller signal than the + same flow from a T1 fund (weight 1.0), with the sign preserved.""" + # T1 case: Renaissance +3pp NVDA + t1_facts = pd.DataFrame([ + _row(_T1_RENAISSANCE, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "NVDA", _Q_CURR, _FILED_CURR, 0.040), + _row(_T1_RENAISSANCE, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T1_RENAISSANCE, "AMD", _Q_CURR, _FILED_CURR, 0.010), + ]) + # T2 case: FMR +3pp NVDA (identical flow, lower-tier source) + t2_facts = pd.DataFrame([ + _row(_T2_FMR, "NVDA", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T2_FMR, "NVDA", _Q_CURR, _FILED_CURR, 0.040), + _row(_T2_FMR, "AMD", _Q_PRIOR, _FILED_PRIOR, 0.010), + _row(_T2_FMR, "AMD", _Q_CURR, _FILED_CURR, 0.010), + ]) + weights = {_T1_RENAISSANCE: 1.0, _T2_FMR: 0.4} + t1_out = compose_conviction_flow(t1_facts, as_of=_AS_OF, fund_weights=weights) + t2_out = compose_conviction_flow(t2_facts, as_of=_AS_OF, fund_weights=weights) + # Raw weighted contribution scales by 0.4, but cross-sectional z-score + # normalizes that away. So the structural test here is: both signs + # match (NVDA > AMD), and the post-z magnitudes are equal because + # standardization is scale-invariant on a 2-point cross-section. + assert _signal(t1_out, "NVDA") > _signal(t1_out, "AMD") + assert _signal(t2_out, "NVDA") > _signal(t2_out, "AMD") From 6f1d0ab4e6c120956a614fcec6c5b596fb492b20 Mon Sep 17 00:00:00 2001 From: ShivianNaidoo Date: Mon, 18 May 2026 01:23:43 +0200 Subject: [PATCH 2/2] fix(phase10): make 0012 fund_strategy seed environment-safe for CI The 22 fund rows in companies(node_type='fund') are inserted as a side effect of scripts/run_ownership_extraction.py (EDGAR 13F ingestion), not by any migration. CI runs alembic upgrade head on a fresh database with no ingested data, so the seed found 0 funds to resolve and the integrity check raised. Differentiate two states in upgrade(): - seeded == 0 -> warn and continue (fresh DB / pre-bootstrap) - 0 < seeded < 22 -> still raise (partial state is a real data bug) Add scripts/bootstrap_fund_strategy.py as the explicit post-ingestion recovery path. It imports _SEED from the migration module (single source of truth) and runs the same INSERT ... ON CONFLICT logic, raising if any fund CIK is still missing from companies. Idempotent. --- migrations/versions/0012_fund_strategy.py | 17 +++++- scripts/bootstrap_fund_strategy.py | 73 +++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 scripts/bootstrap_fund_strategy.py diff --git a/migrations/versions/0012_fund_strategy.py b/migrations/versions/0012_fund_strategy.py index 01006e3..d51aa59 100644 --- a/migrations/versions/0012_fund_strategy.py +++ b/migrations/versions/0012_fund_strategy.py @@ -107,7 +107,22 @@ def upgrade() -> None: ) seeded = bind.execute(sa.text("SELECT COUNT(*) FROM fund_strategy")).scalar() - if seeded != len(_SEED): + if seeded == 0: + # Fresh DB / pre-data-bootstrap environment (CI, new dev box). The + # 22 fund rows in `companies` are inserted as a side effect of + # scripts/run_ownership_extraction.py (EDGAR 13F ingestion), which + # cannot run during alembic upgrade. Defer the seed to the + # companion script `scripts/bootstrap_fund_strategy.py`, to be + # invoked AFTER 13F ingestion completes. + import warnings + warnings.warn( + "fund_strategy table created but seed skipped: no rows in " + "companies(node_type='fund') yet. Run " + "scripts/bootstrap_fund_strategy.py after EDGAR 13F ingestion " + "to populate.", + stacklevel=2, + ) + elif seeded != len(_SEED): raise RuntimeError( f"fund_strategy seed integrity check failed: " f"expected {len(_SEED)} rows, inserted {seeded}. " diff --git a/scripts/bootstrap_fund_strategy.py b/scripts/bootstrap_fund_strategy.py new file mode 100644 index 0000000..2af464f --- /dev/null +++ b/scripts/bootstrap_fund_strategy.py @@ -0,0 +1,73 @@ +"""Seed fund_strategy after EDGAR 13F ingestion has populated companies. + +Migration 0012 creates the fund_strategy table but skips the row seed when +companies(node_type='fund') is empty (e.g. fresh CI database, new dev box). +This script applies the seed once the 22 fund rows exist. Idempotent — safe +to re-run; the ON CONFLICT clause updates the tier/note in place. + +Usage: + .venv/Scripts/python.exe scripts/bootstrap_fund_strategy.py + +Must be run AFTER scripts/run_ownership_extraction.py has inserted the +fund rows into companies. Raises if any expected fund CIK is missing. +""" +import importlib.util +from pathlib import Path + +import sqlalchemy as sa +from sqlalchemy import create_engine + +from nexus.config import settings + + +_MIGRATION_PATH = ( + Path(__file__).resolve().parent.parent + / "migrations" / "versions" / "0012_fund_strategy.py" +) + + +def _load_seed() -> list[tuple[str, int, bool, str]]: + """Import _SEED from the migration module — single source of truth.""" + spec = importlib.util.spec_from_file_location("_m0012", _MIGRATION_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module._SEED + + +def main() -> None: + seed = _load_seed() + engine = create_engine(settings.database_url_sync) + with engine.begin() as conn: + for cik, tier, excl, note in seed: + conn.execute( + sa.text( + """ + INSERT INTO fund_strategy + (fund_id, conviction_tier, exclude_from_flow, strategy_note) + SELECT id, :tier, :excl, :note + FROM companies + WHERE cik = :cik AND node_type = 'fund' + ON CONFLICT (fund_id) DO UPDATE + SET conviction_tier = EXCLUDED.conviction_tier, + exclude_from_flow = EXCLUDED.exclude_from_flow, + strategy_note = EXCLUDED.strategy_note + """ + ), + {"cik": cik, "tier": tier, "excl": excl, "note": note}, + ) + seeded = conn.execute( + sa.text("SELECT COUNT(*) FROM fund_strategy") + ).scalar() + engine.dispose() + + if seeded != len(seed): + raise RuntimeError( + f"fund_strategy bootstrap failed: expected {len(seed)} rows, " + f"have {seeded}. Run scripts/run_ownership_extraction.py first " + f"to populate companies(node_type='fund') from EDGAR 13F data." + ) + print(f"[+] fund_strategy seeded: {seeded}/{len(seed)} rows") + + +if __name__ == "__main__": + main()