Fix/wo 07 umap precompute

CONTRIBUTING.md

@@ -50,8 +50,12 @@ python scripts/generate_keys.py
 # 6. Local database (Postgres + pgvector via Docker)
 docker compose up -d
 
-# 7. Seed the 3 demo authors
-python scripts/seed_corpus.py
+# 7. Seed the 3 demo authors (raw text + embeddings + style profiles)
+python scripts/seed_corpus.py --with-embeddings --with-profiles
+# ⚠️  This step is slow on first run. It downloads two large ML models:
+#     • all-mpnet-base-v2  (~420 MB, sentence-transformers)
+#     • en_core_web_lg     (~560 MB, spaCy)
+# Omit both flags for a fast seed (raw text only, no profiles/embeddings).
 ```
 
 Editor extensions (highly recommended — auto-format on save): **Ruff**, **Black Formatter**, **Prettier**, **Python**. See §8.4.

Makefile

@@ -9,7 +9,7 @@
 # Usage: `make help` to list targets.
 
 .DEFAULT_GOAL := help
-.PHONY: help install install-py install-front keys db-up db-down seed seed-dry demo \
+.PHONY: help install install-py install-front keys db-up db-down seed seed-full seed-dry demo \
         dev back front lint format test clean
 
 # ---- Meta -------------------------------------------------------------------
@@ -58,10 +58,14 @@ db-down: ## Stop the local database
 	$(call need_file,docker-compose.yml,Sprint 1)
 	docker compose down
 
-seed: ## Seed the DB with the 3 preloaded authors (Austen, Dickens, Poe)
+seed: ## Seed the DB with raw text only — fast, no ML models required
 	$(call need_file,scripts/seed_corpus.py,Sprint 2)
 	python scripts/seed_corpus.py
 
+seed-full: ## Seed the DB + compute embeddings and style profiles (downloads ~980 MB of ML models on first run)
+	$(call need_file,scripts/seed_corpus.py,Sprint 2)
+	python scripts/seed_corpus.py --with-embeddings --with-profiles
+
 seed-dry: ## Validate corpus files and print manifest without touching the DB
 	$(call need_file,scripts/seed_corpus.py,Sprint 2)
 	python scripts/seed_corpus.py --dry-run

README.md

@@ -170,8 +170,12 @@ make keys
 # 5. Start local Postgres + pgvector
 make db-up
 
-# 6. Seed the database with the 3 preloaded authors
-make seed
+# 6. Seed the database: raw text + embeddings + style profiles
+make seed-full
+# ⚠️  This step is slow on first run. It downloads two large ML models:
+#     • all-mpnet-base-v2  (~420 MB, sentence-transformers)
+#     • en_core_web_lg     (~560 MB, spaCy)
+# If you only need raw text without profiles/embeddings, use `make seed` instead.
 
 # 7. Start backend + frontend (parallel)
 make dev

ai_pipeline/autoria_ai/extractor/style_profile.py

@@ -271,7 +271,9 @@ def compute_style_profile(
         "stylistic": stylistic,
         "distinctive_vocab": distinctive,
         "semantic_centroid": centroid,
-        # Placeholder — scripts/precompute_umap.py owns real 2-D coords.
+        # Pre-projection placeholder — scripts/precompute_umap.py reads the
+        # umap_coords table (created by 0004_umap_coords.sql) and overwrites
+        # this field with real 2-D centroid + spread after UMAP fitting.
         "embedding_umap_2d": {"centroid": [0.0, 0.0], "spread": 0.0},
     }
 

ai_pipeline/tests/test_style_profile_compute.py

@@ -6,7 +6,8 @@
 
 from __future__ import annotations
 
-from unittest.mock import patch
+import json
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -60,6 +61,9 @@ def test_compute_style_profile_shape(mock_embeddings) -> None:
     assert profile["corpus_stats"]["n_tokens"] > 0
     assert 0.0 < profile["lexical"]["avg_word_length"] < 20.0
     assert len(profile["semantic_centroid"]) == 768
+    # embedding_umap_2d starts as the pre-projection placeholder; the real
+    # centroid/spread is written later by scripts/precompute_umap.py once
+    # umap_coords rows exist (WO-07 / 0004_umap_coords.sql).
     assert profile["embedding_umap_2d"] == {"centroid": [0.0, 0.0], "spread": 0.0}
     assert set(profile["stylistic"]["punct_distribution"].keys()) >= {",", ".", '"'}
 
@@ -144,3 +148,134 @@ def test_lemmatize_corpus_drops_proper_nouns() -> None:
     # lemma string): the common nouns of the same sentences must survive.
     kept = sorted(noun for noun in _COMMON_NOUNS if noun in lemmas)
     assert kept == sorted(_COMMON_NOUNS), f"common nouns were dropped too: {kept}"
+
+
+# ---------------------------------------------------------------------------
+# UMAP projection back-fill (WO-07)
+# ---------------------------------------------------------------------------
+#
+# update_style_profiles() in scripts/precompute_umap.py aggregates per-chunk
+# UMAP coords into a { "centroid": [x, y], "spread": float } dict and writes
+# it to style_profiles.json_data.embedding_umap_2d via a parameterised UPDATE.
+#
+# The test uses a synthetic (author_id, coords) fixture so it runs without a
+# real database or UMAP installation.  psycopg2 is mocked at the connection
+# level: we capture the SQL and parameters passed to cur.execute and verify
+# they encode the correct centroid / spread values.
+
+_AUTHOR_A = "aaaaaaaa-0000-0000-0000-000000000001"
+_AUTHOR_B = "bbbbbbbb-0000-0000-0000-000000000002"
+
+
+def _make_coords() -> np.ndarray:
+    """Synthetic 2-D coords: 4 points for author A, 3 for author B."""
+    return np.array(
+        [
+            # author A — centroid should be (1.0, 2.0)
+            [0.0, 1.0],
+            [1.0, 2.0],
+            [2.0, 3.0],
+            [1.0, 2.0],
+            # author B — centroid should be (10.0, 20.0)
+            [9.0, 18.0],
+            [10.0, 20.0],
+            [11.0, 22.0],
+        ],
+        dtype=np.float64,
+    )
+
+
+def _make_author_ids() -> list[str]:
+    return [_AUTHOR_A] * 4 + [_AUTHOR_B] * 3
+
+
+def test_update_style_profiles_sql_payload() -> None:
+    """update_style_profiles writes correct centroid/spread JSON via UPDATE."""
+    # Import lazily: the script lives outside the package; add scripts/ to sys.path
+    import os
+    import sys
+
+    scripts_dir = os.path.join(os.path.dirname(__file__), "..", "..", "scripts")
+    scripts_dir = os.path.normpath(scripts_dir)
+    if scripts_dir not in sys.path:
+        sys.path.insert(0, scripts_dir)
+
+    # psycopg2 may not be installed in every CI image — skip gracefully.
+    pytest.importorskip("psycopg2")
+
+    # Patch psycopg2.connect so the script never touches a real DB.
+    mock_conn = MagicMock()
+    mock_cur = MagicMock()
+    mock_conn.cursor.return_value.__enter__ = lambda s: mock_cur
+    mock_conn.cursor.return_value.__exit__ = MagicMock(return_value=False)
+
+    # Import after sys.path is set up.
+    import precompute_umap
+
+    author_ids = _make_author_ids()
+    coords = _make_coords()
+
+    precompute_umap.update_style_profiles(mock_conn, author_ids, coords)
+
+    # One UPDATE call per author, then one commit.
+    assert mock_conn.commit.called
+    update_calls = list(mock_cur.execute.call_args_list)
+    assert len(update_calls) == 2, f"Expected 2 UPDATE calls, got {len(update_calls)}"
+
+    # Collect (payload_dict, author_id) from the two calls.
+    results: dict[str, dict] = {}
+    for c in update_calls:
+        args = c.args  # (sql, (payload_json, author_id))
+        payload_json, aid = args[1]
+        results[aid] = json.loads(payload_json)
+
+    # Author A: centroid = mean([0,1,2,1], [1,2,3,2]) = [1.0, 2.0]
+    centroid_a = results[_AUTHOR_A]["centroid"]
+    assert abs(centroid_a[0] - 1.0) < 1e-9, centroid_a
+    assert abs(centroid_a[1] - 2.0) < 1e-9, centroid_a
+    assert results[_AUTHOR_A]["spread"] >= 0.0
+
+    # Author B: centroid = mean([9,10,11], [18,20,22]) = [10.0, 20.0]
+    centroid_b = results[_AUTHOR_B]["centroid"]
+    assert abs(centroid_b[0] - 10.0) < 1e-9, centroid_b
+    assert abs(centroid_b[1] - 20.0) < 1e-9, centroid_b
+    assert results[_AUTHOR_B]["spread"] >= 0.0
+
+    # The two authors must have different centroids.
+    assert centroid_a != centroid_b
+
+
+def test_update_style_profiles_spread_formula() -> None:
+    """spread = mean Euclidean distance of each chunk from its author centroid."""
+    import os
+    import sys
+
+    scripts_dir = os.path.join(os.path.dirname(__file__), "..", "..", "scripts")
+    scripts_dir = os.path.normpath(scripts_dir)
+    if scripts_dir not in sys.path:
+        sys.path.insert(0, scripts_dir)
+
+    pytest.importorskip("psycopg2")
+
+    mock_conn = MagicMock()
+    mock_cur = MagicMock()
+    mock_conn.cursor.return_value.__enter__ = lambda s: mock_cur
+    mock_conn.cursor.return_value.__exit__ = MagicMock(return_value=False)
+
+    import precompute_umap
+
+    # Four points equidistant from centroid (0,0) at radius=1.
+    coords = np.array([[1.0, 0.0], [-1.0, 0.0], [0.0, 1.0], [0.0, -1.0]], dtype=np.float64)
+    author_ids = [_AUTHOR_A] * 4
+
+    precompute_umap.update_style_profiles(mock_conn, author_ids, coords)
+
+    update_calls = mock_cur.execute.call_args_list
+    assert len(update_calls) == 1
+    payload_json, _ = update_calls[0].args[1]
+    result = json.loads(payload_json)
+
+    # centroid = (0, 0); each point is distance 1 from centre → spread = 1.0
+    assert abs(result["centroid"][0]) < 1e-9
+    assert abs(result["centroid"][1]) < 1e-9
+    assert abs(result["spread"] - 1.0) < 1e-9, f"spread={result['spread']}"

docs/erd.md

@@ -1,7 +1,7 @@
 # AutorIA — Database ERD
 
-> **Status**: draft for Sprint 0 · **Owner**: P3 · **Last updated**: 2026-06-29
-> **Source of truth for the schema**: [`infra/supabase/migrations/0001_init.sql`](../infra/supabase/migrations/0001_init.sql)
+> **Status**: draft for Sprint 0 · **Owner**: P3 · **Last updated**: 2026-07-30
+> **Source of truth for the schema**: [`infra/supabase/migrations/0001_init.sql`](../infra/supabase/migrations/0001_init.sql) · [`infra/supabase/migrations/0004_umap_coords.sql`](../infra/supabase/migrations/0004_umap_coords.sql)
 > **Higher-level data model**: [`docs/MVP.md`](MVP.md) §6 (Database)
 
 This document is the human-readable companion to the SQL migration. It explains
@@ -22,8 +22,10 @@ The data flows in one direction during ingest:
 
 ```
 author ─► documents ─► chunks (+ embeddings)
-   │
-   ├─► style_profiles   (computed from the whole corpus)
+   │                       │
+   │                       └─► umap_coords  (precomputed by precompute_umap.py)
+   │                                │
+   ├─► style_profiles   ◄───────────┘  (embedding_umap_2d projected back)
    └─► passports        (emitted per conditioned generation)
 ```
 
@@ -37,6 +39,7 @@ erDiagram
     AUTHORS ||--o{ STYLE_PROFILES  : "has"
     AUTHORS ||--o{ PASSPORTS       : "has"
     DOCUMENTS ||--o{ CHUNKS        : "split into"
+    AUTHORS ||--o{ UMAP_COORDS     : "has chunks in"
 
     AUTHORS {
         uuid        id PK
@@ -83,6 +86,13 @@ erDiagram
         text        jws_token "compact JWS, ES256"
         timestamptz created_at
     }
+
+    UMAP_COORDS {
+        serial      id PK
+        uuid        author_id "denormalised; no FK (volatile cache)"
+        float8      x "UMAP dim 1"
+        float8      y "UMAP dim 2"
+    }
 ```
 
 ---
@@ -172,6 +182,28 @@ payload and its signed token so verification is fully reproducible offline.
 | `jws_token` | `text` | Compact JWS signed with **ES256** (ECDSA P-256). |
 | `created_at` | `timestamptz` | |
 
+### `umap_coords`
+Volatile cache holding the per-chunk 2-D UMAP coordinates produced by
+[`scripts/precompute_umap.py`](../scripts/precompute_umap.py). One row per
+embedded chunk. The table is **truncated and repopulated** on every precompute
+run; `scripts/precompute_umap.py` then aggregates these rows per author into a
+`{ "centroid": [x, y], "spread": float }` value which it writes back to
+`style_profiles.json_data.embedding_umap_2d` — the field the Style DNA scatter
+plot reads.
+
+| Column | Type | Notes |
+|---|---|---|
+| `id` | `serial` PK | Surrogate key; reset on every repopulation. |
+| `author_id` | `uuid` (no FK) | Denormalised author UUID. No FK constraint: table is a volatile cache; the FK would add latency to the TRUNCATE + re-insert without referential-integrity value. |
+| `x` | `double precision` | 1st UMAP dimension. |
+| `y` | `double precision` | 2nd UMAP dimension. |
+
+> **No FK on `author_id`**: the table is populated in a single bulk load from
+> the precompute script and is never modified row-by-row. A FK would require
+> either DEFERRABLE INITIALLY DEFERRED gymnastics on every reload or a
+> per-row existence check, with no safety gain (the data is always regenerated
+> from live `style_profiles` rows anyway).
+
 ---
 
 ## 4. Relationships & cascade behaviour
@@ -198,6 +230,7 @@ passports). This keeps "remove an author" a single, clean operation.
 | `chunks_embedding_hnsw_idx` | `chunks(embedding)` | **HNSW** ANN search, `vector_cosine_ops`. The core RAG index. |
 | `style_profiles_author_id_computed_at_idx` | `style_profiles(author_id, computed_at desc)` | Get the latest profile per author. |
 | `passports_author_id_idx` | `passports(author_id)` | List passports by author. |
+| `umap_coords_author_id_idx` | `umap_coords(author_id)` | Fast per-author aggregation by `scripts/precompute_umap.py`. B-tree (not HNSW — aggregation, not ANN). |
 
 ### Vector search
 

infra/supabase/migrations/0004_umap_coords.sql

@@ -0,0 +1,71 @@
+-- =============================================================================
+-- Migration : 0004_umap_coords.sql
+-- Project   : AutorIA
+-- Purpose   : Create the public.umap_coords table and its author_id index.
+--             This table holds one row per chunk after scripts/precompute_umap.py
+--             runs UMAP on all chunk embeddings.  It is the intermediate store
+--             that precompute_umap.py aggregates into
+--             style_profiles.json_data.embedding_umap_2d (WO-07).
+--
+-- Background
+-- ----------
+-- Prior to this migration the table was created at runtime by
+-- scripts/precompute_umap.py using CREATE TABLE IF NOT EXISTS.  Moving DDL
+-- into a proper migration gives it:
+--   * a canonical location in the migration sequence,
+--   * consistent comments and index naming, and
+--   * an idempotent path that is safe to replay on any environment.
+-- The runtime DDL has been removed from precompute_umap.py in the same commit.
+--
+-- Table design
+-- ------------
+-- * One row per embedded chunk, not per author.  After UMAP fits on all chunks
+--   combined, each chunk gets a 2-D (x, y) coordinate in the shared space.
+--   precompute_umap.py then aggregates those per-author into centroid + spread
+--   and writes the result back to style_profiles.json_data.embedding_umap_2d.
+-- * SERIAL primary key (not UUID) because these rows have no external identity
+--   and are truncated + repopulated on every precompute run.
+-- * author_id is NOT NULL and indexed for fast per-author aggregation in the
+--   precompute script.  There is intentionally no FK to public.authors: the
+--   table is a volatile cache and the FK would add latency to the TRUNCATE +
+--   re-insert without providing referential-integrity value (the data is always
+--   regenerated from scratch on every run).
+--
+-- Source of truth: docs/erd.md · scripts/precompute_umap.py
+--
+-- Apply manually:
+--   psql "$DATABASE_URL" -f infra/supabase/migrations/0004_umap_coords.sql
+--
+-- Or via Supabase CLI:
+--   supabase db push
+--
+-- Idempotent: both statements use IF NOT EXISTS, so re-running against an
+-- already-migrated database is a safe no-op.
+-- =============================================================================
+
+begin;
+
+-- ---------------------------------------------------------------------------
+-- Table: umap_coords
+-- ---------------------------------------------------------------------------
+
+create table if not exists public.umap_coords (
+    id        serial           primary key,
+    author_id uuid             not null,
+    x         double precision not null,
+    y         double precision not null
+);
+
+comment on table  public.umap_coords           is 'Per-chunk 2-D UMAP coordinates, produced by scripts/precompute_umap.py. One row per embedded chunk. Truncated and repopulated on every precompute run. Aggregated into style_profiles.json_data.embedding_umap_2d (centroid + spread) to power the Style DNA scatter plot.';
+comment on column public.umap_coords.author_id is 'UUID of the author who owns the chunk. Denormalised here (the canonical author_id lives on documents via chunks → documents → authors) for fast per-author aggregation without a join. No FK constraint: this table is a volatile cache.';
+comment on column public.umap_coords.x         is '1st UMAP dimension. Produced by umap.UMAP(n_components=2, n_neighbors=15, min_dist=0.1, metric=''cosine'', random_state=42).';
+comment on column public.umap_coords.y         is '2nd UMAP dimension. Same reducer as x.';
+
+-- ---------------------------------------------------------------------------
+-- Index: fast per-author lookup / aggregation
+-- ---------------------------------------------------------------------------
+
+create index if not exists umap_coords_author_id_idx
+    on public.umap_coords (author_id);
+
+commit;