fix(ml): sample the whole corpus and drop PROPN from distinctive_vocab (WO-18)

ai_pipeline/autoria_ai/extractor/style_profile.py

@@ -38,6 +38,10 @@
 # Soft cap so spaCy does not OOM on full Victorian novels in one Doc.
 # Features are still computed over many chunks via nlp.pipe; this only
 # limits how much raw text we keep when building the lemma string for TF-IDF.
+#
+# The cap is a memory bound, not a corpus definition: it is spent on chunks
+# drawn from across the whole corpus (``_spread_order``), never on a prefix.
+# See docs/style_features.md §4.1 "Preprocessing" and issue #100.
 _MAX_LEMMA_CHARS: int = 800_000
 
 # Cap centroid embedding cost on huge corpora (evenly subsampled).
@@ -69,17 +73,65 @@ def _weighted_mean(dicts: list[dict[str, Any]], weights: list[float]) -> dict[st
     return out
 
 
+def _spread_order(n: int) -> list[int]:
+    """Permutation of ``range(n)`` whose every prefix is spread over the range.
+
+    Bisection (van der Corput) order: ``0, n/2, n/4, 3n/4, n/8, …``.  Taking
+    the first *k* elements therefore samples the *whole* sequence rather than
+    its opening — the property :func:`_lemmas_from_docs` needs so that a
+    truncated lemma budget is not spent entirely on the first document of the
+    corpus (issue #100: the 800k-char cap fell at 21% of Dickens' chunks, so
+    ``distinctive_vocab`` was the vocabulary of *Great Expectations* alone).
+
+    Cheap and deterministic: no extra spaCy work, no extra memory, and the
+    same chunks are selected on every run for a given corpus.
+    """
+    if n <= 0:
+        return []
+    order: list[int] = [0]
+    seen: set[int] = {0}
+    denom = 2
+    while len(order) < n and denom <= 2 * n:
+        for num in range(1, denom, 2):
+            idx = (num * n) // denom
+            if idx < n and idx not in seen:
+                seen.add(idx)
+                order.append(idx)
+        denom *= 2
+    order.extend(i for i in range(n) if i not in seen)
+    return order
+
+
+def _in_spread_order(items: list[Any]) -> list[Any]:
+    """*items* reordered by :func:`_spread_order`."""
+    return [items[i] for i in _spread_order(len(items))]
+
+
 def _lemmas_from_docs(docs: Iterable[Any], max_chars: int = _MAX_LEMMA_CHARS) -> str:
-    """Space-joined lower lemmas (alpha only) for TF-IDF input.
+    """Space-joined lower lemmas (alpha, non-proper, len>=3) for TF-IDF input.
 
     *docs* may be a lazy iterable (e.g. the generator returned by
     ``nlp.pipe``): the function stops consuming it as soon as *max_chars*
     is reached, so the cost is bounded by the cap, not by corpus size.
+    Callers are responsible for handing the docs over in spread order
+    (:func:`_in_spread_order`) so that an early stop still samples the whole
+    corpus.
+
+    ``PROPN`` tokens are dropped.  Character and place names are genuinely
+    concentrated in one author's corpus, so TF-IDF ranks them first — but they
+    identify the *novel*, not the author's hand, and ``distinctive_vocab`` is
+    the one feature a non-technical juror reads directly
+    (docs/style_features.md §4.1; decision 2026-07-28 in docs/decision_log.md).
+    The filter lives here, where spaCy's POS tag is already computed, so it is
+    free and stays correct for authors added later — unlike a hand-maintained
+    word blacklist applied after the TF-IDF.
     """
     parts: list[str] = []
     size = 0
     for doc in docs:
         for tok in doc:
+            if tok.pos_ == "PROPN":
+                continue
             if tok.is_alpha and len(tok.lemma_) >= 3:
                 piece = tok.lemma_.lower()
                 parts.append(piece)
@@ -103,19 +155,23 @@ def lemmatize_corpus(
     authors combined.  Callers that own more than one author's corpus (the
     seeding script) use this to build the ``comparison_lemmas`` mapping that
     :func:`compute_style_profile` expects, without having to reimplement the
-    lemma rules — alpha-only, lowercase, ``len >= 3`` — or the
+    lemma rules — alpha-only, lowercase, ``len >= 3``, no ``PROPN`` — or the
     ``_MAX_LEMMA_CHARS`` cap that bounds peak memory per author.
 
     The spaCy pass streams and stops at *max_chars*, so lemmatizing a
-    comparison corpus costs a fraction of a full feature pass over it.
+    comparison corpus costs a fraction of a full feature pass over it.  The
+    chunks are fed in :func:`_spread_order`, so the chunks that fit inside the
+    cap are drawn from every document of the corpus instead of from whichever
+    novel happens to come first (issue #100).
 
     Returns an empty string when *documents* produce no chunks.
     """
     full_text = "\n\n".join(documents)
     chunks = chunk_texts if chunk_texts is not None else chunk_text(full_text)
     if not chunks:
         return ""
-    return _lemmas_from_docs(nlp.pipe(chunks, batch_size=64), max_chars=max_chars)
+    sampled = _in_spread_order(chunks)
+    return _lemmas_from_docs(nlp.pipe(sampled, batch_size=64), max_chars=max_chars)
 
 
 def _subsample(items: list[str], max_n: int) -> list[str]:
@@ -170,7 +226,10 @@ def compute_style_profile(
     syntactic = _weighted_mean([compute_syntactic(d) for d in docs], weights)
     stylistic = _weighted_mean([compute_stylistic(d) for d in docs], weights)
 
-    author_lemmas = _lemmas_from_docs(docs)
+    # Spread order, same as lemmatize_corpus: this author's own TF-IDF
+    # "document" must be built the same way as the comparison corpora it is
+    # scored against, and the cap must not fall inside the first novel (#100).
+    author_lemmas = _lemmas_from_docs(_in_spread_order(docs))
     corpora = dict(comparison_lemmas or {})
     corpora[author_slug] = author_lemmas
     try:

ai_pipeline/tests/test_style_profile_compute.py

@@ -15,6 +15,7 @@
 
 from autoria_ai.extractor.style_profile import (  # noqa: E402
     compute_style_profile,
+    lemmatize_corpus,
     profile_hash,
 )
 
@@ -79,3 +80,67 @@ def test_profile_hash_stable(mock_embeddings) -> None:
 def test_compute_style_profile_rejects_empty() -> None:
     with pytest.raises(ValueError):
         compute_style_profile(author_slug="x", documents=[], nlp=_NLP)
+
+
+# --- Corpus sampling (issue #100 / WO-18) ---------------------------------
+#
+# docs/style_features.md 4.1: "each author's full corpus is one document".
+# The max_chars cap bounds peak memory, but it must not degenerate into a
+# prefix of the corpus -- that made distinctive_vocab the vocabulary of
+# whichever novel happened to be first in the manifest.
+
+_MARKERS = ("kitchen", "mountain", "elephant")
+
+
+def _marked_document(marker: str) -> str:
+    """~5k tokens of neutral prose whose only distinctive noun is *marker*."""
+    return (
+        f"The {marker} was quiet that morning and the pale light fell across "
+        f"the table where the {marker} rested beside the open window. "
+    ) * 200
+
+
+def test_lemmatize_corpus_samples_every_document() -> None:
+    """A capped lemma string must draw on more than one document per author."""
+    documents = [_marked_document(m) for m in _MARKERS]
+    # Cap far below the corpus size: enough for several chunks, nowhere near
+    # all of them. Prefix truncation would spend the whole budget on doc #1.
+    lemmas = lemmatize_corpus(documents=documents, nlp=_NLP, max_chars=12_000)
+
+    sampled = {marker for marker in _MARKERS if marker in lemmas.split()}
+    assert len(sampled) > 1, f"capped corpus sampled only {sampled or 'nothing'}"
+    assert sampled == set(_MARKERS), f"documents missing from the sample: {sampled}"
+
+
+# --- Proper-noun filter (issue #100 / WO-18) ------------------------------
+#
+# docs/style_features.md 4.1 requires proper nouns to be dropped *inside* the
+# lemmatization pass, so character and place names never reach the TF-IDF.
+# Without the filter these names are the top-scoring terms of an author's
+# distinctive_vocab -- they identify the novel, not the author's hand.
+# This test fails if the `tok.pos_ == "PROPN"` guard in _lemmas_from_docs is
+# removed: the names below are the only PROPN tokens in the sample.
+
+_PROPER_NOUNS = ("havisham", "wemmick", "pemberley")
+_COMMON_NOUNS = ("parlour", "candle", "housekeeper", "lantern", "garden")
+
+_NAMED_SENTENCE = (
+    "Havisham sat alone in the parlour while Wemmick counted the candles "
+    "and the housekeeper carried a heavy lantern toward the garden gate at "
+    "Pemberley before the evening meal was served. "
+)
+
+
+def test_lemmatize_corpus_drops_proper_nouns() -> None:
+    """Character and place names must not survive into the TF-IDF input."""
+    lemmas = set(
+        lemmatize_corpus(documents=[_NAMED_SENTENCE * 200], nlp=_NLP, max_chars=60_000).split()
+    )
+
+    leaked = sorted(name for name in _PROPER_NOUNS if name in lemmas)
+    assert not leaked, f"proper nouns reached the TF-IDF input: {leaked}"
+
+    # Guard against the test passing for the wrong reason (empty/degenerate
+    # lemma string): the common nouns of the same sentences must survive.
+    kept = sorted(noun for noun in _COMMON_NOUNS if noun in lemmas)
+    assert kept == sorted(_COMMON_NOUNS), f"common nouns were dropped too: {kept}"

docs/decision_log.md

@@ -46,4 +46,6 @@ Every decision that affects the product, the process, or the team lives here. Ap
 | 2026-07-21 | **Frontend screens consolidated into one studio screen** (#19): the Style DNA panel and the generation studio now live on a single `/author/[id]` route instead of the two-route split (`/author/[id]` + a separate `/author/[id]/generate`). `GenerateStudio` moved from `app/author/[id]/generate/` into `components/`; the standalone `generate` route and its now-dead i18n keys (`authorDetail.generateCta`, `generate.pageTitle`, `generate.backToProfile`) are removed. The parent server component owns the page header and `data-voice`; `GenerateStudio` starts in an idle visual state before the first generation. No api_contract or backend change | P1 (frontend)  | design-system §7 defines a single "studio" screen and §8.5 (Sprint 3 cluster-landing) requires Style DNA and generation visible together; the separate "Generate in this voice" link was easy to miss and added a redundant navigation hop. One screen also removes a second route transition that made the flow feel slower                                                                                                                                              |
 | 2026-07-21 | **#28 (Sprint 2) + #41-remainder (Sprint 1) implemented with NO api_contract change.** (a) The vanilla-vs-AutorIA comparative metrics table is **measured client-side** from each generation's text (`lib/textMetrics.ts`: avg sentence length, type–token ratio, word count, top words) because `GenerationOutput` exposes only `text`/`fit_score`/`latency_ms`; it renders only when both branches succeed. (b) The "Generate Passport" button is added as an **affordance only** — enabled iff `GenerateResponse.passport` is present, disabled with helper copy otherwise; the actual formatted-JSON download stays in #42/#44 and the verify view in #29. (c) `StyleProfile.distinctive_vocab` is now surfaced as a top-10 TF-IDF table in the Style DNA panel (its type JSDoc previously said "not rendered"), with a graceful empty state until ML #14 lands real data | P1 (frontend)  | `docs/api_contract.yaml` is LOCKED (2026-06-24) — measuring descriptive statistics from the real generated text satisfies #28's "comparative metrics" AC without a contract amendment and without faking the contrast (design-system §8, forensic honesty). Scoping the passport button to an affordance keeps #28 from overlapping #42/#44/#29. The vocab table completes #41's only outstanding AC using data the panel already fetches, so it ships now instead of blocking on ML #14 |
 | 2026-07-21 | **Frontend testing harness = Vitest** (dev-only, first test runner for `frontend/`). Added `vitest` devDependency, `test` (`vitest run`) + `test:watch` scripts, `frontend/vitest.config.ts` (node environment, `@`→`src` alias, no jsdom), and a `Unit tests` step in the `lint-frontend` CI job. First coverage: `src/lib/textMetrics.ts` (26 pure-function cases). Convention: **pure-function unit tests only** for now — no jsdom / `@testing-library` / component tests until they are actually warranted | P1 (frontend)  | Frontend had no test runner (CI ran only ESLint + `tsc`); the cross-cutting "happy-path tests" responsibility (2026-06-24) and the new client-side metrics logic (#28) needed a regression guard. Vitest matches the Vite/TS toolchain, runs pure TS with zero extra transform config, and stays lightweight by excluding a DOM environment until component tests earn their keep |
+| 2026-07-28 | **`distinctive_vocab` excludes proper nouns, and its corpus is sampled instead of truncated** (#100, WO-18). (a) The 800k-lemma-char memory cap in `style_profile.py` no longer keeps a *prefix* of the corpus: chunks are fed to spaCy in a deterministic bisection order, so the capped sample spans every document of every author. Before the fix the cap fell at 21% of Dickens' chunks (34% Austen, 89% Poe), so "each author's full corpus is one document" (`docs/style_features.md` §4.1) was really *Great Expectations* alone. (b) Tokens tagged `PROPN` by spaCy are dropped **inside** the lemmatization pass (`_lemmas_from_docs`), not by a blacklist after the TF-IDF. `docs/style_features.md` §4.1 updated to match. Re-seeding is required (#86): every existing `style_profiles` row is invalidated. | Sergi          | Character names are statistically distinctive but they are plot, not style — they identify the novel, not the author's hand. §4.1's own purpose is that "the jury can read and feel the difference", and its own example (`countenance`, `physiognomy`, `presently`) is a style marker, not a cast list; the demo is judged by what a non-technical human perceives. Filtering at the POS level is free (the tag is already computed) and stays correct for authors added later, which a hand-maintained word blacklist would not. |
+| 2026-07-28 | **Exception accepted: #100 ships with a top-10 `distinctive_vocab` overlap of 5, not the ≤3 its Definition of Done requires.** The DoD of #100 states verbatim: *"top-10 overlap across the three authors stays ≤3 (#89 already brought it down from 6 to 3; it must not get worse)"*. **Measured after the fix: 5** — `know`, `little`, `make`, `say`, `time` are common to all three top-10 lists (pairwise: austen∩dickens 8, austen∩poe 6, dickens∩poe 6). The criterion is **not met and is worse than what #89 left**. The full measured top-10 per author is recorded in `docs/style_features.md` §4.1 ("Measured output"), together with the honest reading: the feature returns ordinary high-frequency verbs and nouns, not signature vocabulary, and none of `countenance` / `physiognomy` / `presently` reaches any author's top-30. The `PROPN` filter itself is confirmed working (no proper noun survives in any top-30) and is now covered by a regression test (`ai_pipeline/tests/test_style_profile_compute.py::test_lemmatize_corpus_drops_proper_nouns`, verified to fail when the filter is removed). Ratification: entered by the executor of #100 on measured evidence and accepted by the project owner; **still needs the 2/3 vote** the decision policy of 2026-06-24 requires | Sergi (project owner) — pending 2/3 ratification | With only three documents in the collection, every term present in all three has `df = 3/3` and therefore identical idf (1.000), so TF-IDF degenerates into raw frequency: `countenance` (29 occurrences in Dickens) and `say` (1 866) are idf-tied and the common word always wins. Proper nouns were the only terms with a discriminating `df`, so the overlap of 3 that #89 measured was an artefact of the cast lists being counted as style. Dropping them (the correct call — character names are plot, not the author's hand) necessarily raised the overlap to 5. Two alternatives were considered and rejected for this issue: (a) redefining the scoring (log-odds, or idf against a general-English reference corpus) would change the algorithm `docs/style_features.md` §4.1 declares as closed, which is a scope decision and not a fix to #100; (b) reverting the `PROPN` filter would restore the ≤3 number by putting `pip`, `havisham` and `wemmick` back in front of the jury as "style", i.e. buying the metric with the defect the issue exists to remove. A dull-but-honest list is preferred to a varied-but-wrong one; the gap is documented rather than hidden |
 | 2026-07-28 | **`api_contract.yaml` descriptions corrected for `dialogue_ratio` and `first_person_ratio` (prose only — NO contract amendment).** Both were described as *"Fraction of sentences…"*. Neither is. `dialogue_ratio` is the fraction of non-punctuation **tokens** inside straight ASCII double quotes (`docs/style_features.md` §3.3, `dialogue_tokens / total_tokens`; implemented that way in `ai_pipeline/autoria_ai/extractor/stylistic.py`). `first_person_ratio` is first-person singular pronouns **per 1,000 tokens** (§3.4), so it is **not bounded by 1** — Poe's expected range is 18–30. The declared shape (`type: number`) is unchanged in both cases, so no client or server contract breaks; only the prose changed | Sergi (repo owner) | Found while implementing #92: three fixture `first_person_ratio` values were written as fractions ([0,1]) because the contract described them that way, making them wrong by a factor of ~45 against the ranges in §7. The LOCKED policy (2026-06-24) requires a log entry for any change to this file; recorded here as a documentation correction, not a scope change — the contract never matched the implementation or the feature spec, and a reader building a UI against it would size the widget for 0–1 |