diff --git a/ai_pipeline/autoria_ai/extractor/style_profile.py b/ai_pipeline/autoria_ai/extractor/style_profile.py index 8d08e44..76655ca 100644 --- a/ai_pipeline/autoria_ai/extractor/style_profile.py +++ b/ai_pipeline/autoria_ai/extractor/style_profile.py @@ -38,6 +38,10 @@ # Soft cap so spaCy does not OOM on full Victorian novels in one Doc. # Features are still computed over many chunks via nlp.pipe; this only # limits how much raw text we keep when building the lemma string for TF-IDF. +# +# The cap is a memory bound, not a corpus definition: it is spent on chunks +# drawn from across the whole corpus (``_spread_order``), never on a prefix. +# See docs/style_features.md §4.1 "Preprocessing" and issue #100. _MAX_LEMMA_CHARS: int = 800_000 # Cap centroid embedding cost on huge corpora (evenly subsampled). @@ -69,17 +73,65 @@ def _weighted_mean(dicts: list[dict[str, Any]], weights: list[float]) -> dict[st return out +def _spread_order(n: int) -> list[int]: + """Permutation of ``range(n)`` whose every prefix is spread over the range. + + Bisection (van der Corput) order: ``0, n/2, n/4, 3n/4, n/8, …``. Taking + the first *k* elements therefore samples the *whole* sequence rather than + its opening — the property :func:`_lemmas_from_docs` needs so that a + truncated lemma budget is not spent entirely on the first document of the + corpus (issue #100: the 800k-char cap fell at 21% of Dickens' chunks, so + ``distinctive_vocab`` was the vocabulary of *Great Expectations* alone). + + Cheap and deterministic: no extra spaCy work, no extra memory, and the + same chunks are selected on every run for a given corpus. + """ + if n <= 0: + return [] + order: list[int] = [0] + seen: set[int] = {0} + denom = 2 + while len(order) < n and denom <= 2 * n: + for num in range(1, denom, 2): + idx = (num * n) // denom + if idx < n and idx not in seen: + seen.add(idx) + order.append(idx) + denom *= 2 + order.extend(i for i in range(n) if i not in seen) + return order + + +def _in_spread_order(items: list[Any]) -> list[Any]: + """*items* reordered by :func:`_spread_order`.""" + return [items[i] for i in _spread_order(len(items))] + + def _lemmas_from_docs(docs: Iterable[Any], max_chars: int = _MAX_LEMMA_CHARS) -> str: - """Space-joined lower lemmas (alpha only) for TF-IDF input. + """Space-joined lower lemmas (alpha, non-proper, len>=3) for TF-IDF input. *docs* may be a lazy iterable (e.g. the generator returned by ``nlp.pipe``): the function stops consuming it as soon as *max_chars* is reached, so the cost is bounded by the cap, not by corpus size. + Callers are responsible for handing the docs over in spread order + (:func:`_in_spread_order`) so that an early stop still samples the whole + corpus. + + ``PROPN`` tokens are dropped. Character and place names are genuinely + concentrated in one author's corpus, so TF-IDF ranks them first — but they + identify the *novel*, not the author's hand, and ``distinctive_vocab`` is + the one feature a non-technical juror reads directly + (docs/style_features.md §4.1; decision 2026-07-28 in docs/decision_log.md). + The filter lives here, where spaCy's POS tag is already computed, so it is + free and stays correct for authors added later — unlike a hand-maintained + word blacklist applied after the TF-IDF. """ parts: list[str] = [] size = 0 for doc in docs: for tok in doc: + if tok.pos_ == "PROPN": + continue if tok.is_alpha and len(tok.lemma_) >= 3: piece = tok.lemma_.lower() parts.append(piece) @@ -103,11 +155,14 @@ def lemmatize_corpus( authors combined. Callers that own more than one author's corpus (the seeding script) use this to build the ``comparison_lemmas`` mapping that :func:`compute_style_profile` expects, without having to reimplement the - lemma rules — alpha-only, lowercase, ``len >= 3`` — or the + lemma rules — alpha-only, lowercase, ``len >= 3``, no ``PROPN`` — or the ``_MAX_LEMMA_CHARS`` cap that bounds peak memory per author. The spaCy pass streams and stops at *max_chars*, so lemmatizing a - comparison corpus costs a fraction of a full feature pass over it. + comparison corpus costs a fraction of a full feature pass over it. The + chunks are fed in :func:`_spread_order`, so the chunks that fit inside the + cap are drawn from every document of the corpus instead of from whichever + novel happens to come first (issue #100). Returns an empty string when *documents* produce no chunks. """ @@ -115,7 +170,8 @@ def lemmatize_corpus( chunks = chunk_texts if chunk_texts is not None else chunk_text(full_text) if not chunks: return "" - return _lemmas_from_docs(nlp.pipe(chunks, batch_size=64), max_chars=max_chars) + sampled = _in_spread_order(chunks) + return _lemmas_from_docs(nlp.pipe(sampled, batch_size=64), max_chars=max_chars) def _subsample(items: list[str], max_n: int) -> list[str]: @@ -170,7 +226,10 @@ def compute_style_profile( syntactic = _weighted_mean([compute_syntactic(d) for d in docs], weights) stylistic = _weighted_mean([compute_stylistic(d) for d in docs], weights) - author_lemmas = _lemmas_from_docs(docs) + # Spread order, same as lemmatize_corpus: this author's own TF-IDF + # "document" must be built the same way as the comparison corpora it is + # scored against, and the cap must not fall inside the first novel (#100). + author_lemmas = _lemmas_from_docs(_in_spread_order(docs)) corpora = dict(comparison_lemmas or {}) corpora[author_slug] = author_lemmas try: diff --git a/ai_pipeline/tests/test_style_profile_compute.py b/ai_pipeline/tests/test_style_profile_compute.py index 75a7543..4cf0077 100644 --- a/ai_pipeline/tests/test_style_profile_compute.py +++ b/ai_pipeline/tests/test_style_profile_compute.py @@ -15,6 +15,7 @@ from autoria_ai.extractor.style_profile import ( # noqa: E402 compute_style_profile, + lemmatize_corpus, profile_hash, ) @@ -79,3 +80,67 @@ def test_profile_hash_stable(mock_embeddings) -> None: def test_compute_style_profile_rejects_empty() -> None: with pytest.raises(ValueError): compute_style_profile(author_slug="x", documents=[], nlp=_NLP) + + +# --- Corpus sampling (issue #100 / WO-18) --------------------------------- +# +# docs/style_features.md 4.1: "each author's full corpus is one document". +# The max_chars cap bounds peak memory, but it must not degenerate into a +# prefix of the corpus -- that made distinctive_vocab the vocabulary of +# whichever novel happened to be first in the manifest. + +_MARKERS = ("kitchen", "mountain", "elephant") + + +def _marked_document(marker: str) -> str: + """~5k tokens of neutral prose whose only distinctive noun is *marker*.""" + return ( + f"The {marker} was quiet that morning and the pale light fell across " + f"the table where the {marker} rested beside the open window. " + ) * 200 + + +def test_lemmatize_corpus_samples_every_document() -> None: + """A capped lemma string must draw on more than one document per author.""" + documents = [_marked_document(m) for m in _MARKERS] + # Cap far below the corpus size: enough for several chunks, nowhere near + # all of them. Prefix truncation would spend the whole budget on doc #1. + lemmas = lemmatize_corpus(documents=documents, nlp=_NLP, max_chars=12_000) + + sampled = {marker for marker in _MARKERS if marker in lemmas.split()} + assert len(sampled) > 1, f"capped corpus sampled only {sampled or 'nothing'}" + assert sampled == set(_MARKERS), f"documents missing from the sample: {sampled}" + + +# --- Proper-noun filter (issue #100 / WO-18) ------------------------------ +# +# docs/style_features.md 4.1 requires proper nouns to be dropped *inside* the +# lemmatization pass, so character and place names never reach the TF-IDF. +# Without the filter these names are the top-scoring terms of an author's +# distinctive_vocab -- they identify the novel, not the author's hand. +# This test fails if the `tok.pos_ == "PROPN"` guard in _lemmas_from_docs is +# removed: the names below are the only PROPN tokens in the sample. + +_PROPER_NOUNS = ("havisham", "wemmick", "pemberley") +_COMMON_NOUNS = ("parlour", "candle", "housekeeper", "lantern", "garden") + +_NAMED_SENTENCE = ( + "Havisham sat alone in the parlour while Wemmick counted the candles " + "and the housekeeper carried a heavy lantern toward the garden gate at " + "Pemberley before the evening meal was served. " +) + + +def test_lemmatize_corpus_drops_proper_nouns() -> None: + """Character and place names must not survive into the TF-IDF input.""" + lemmas = set( + lemmatize_corpus(documents=[_NAMED_SENTENCE * 200], nlp=_NLP, max_chars=60_000).split() + ) + + leaked = sorted(name for name in _PROPER_NOUNS if name in lemmas) + assert not leaked, f"proper nouns reached the TF-IDF input: {leaked}" + + # Guard against the test passing for the wrong reason (empty/degenerate + # lemma string): the common nouns of the same sentences must survive. + kept = sorted(noun for noun in _COMMON_NOUNS if noun in lemmas) + assert kept == sorted(_COMMON_NOUNS), f"common nouns were dropped too: {kept}" diff --git a/docs/decision_log.md b/docs/decision_log.md index 89f5ece..2f4844e 100644 --- a/docs/decision_log.md +++ b/docs/decision_log.md @@ -46,4 +46,6 @@ Every decision that affects the product, the process, or the team lives here. Ap | 2026-07-21 | **Frontend screens consolidated into one studio screen** (#19): the Style DNA panel and the generation studio now live on a single `/author/[id]` route instead of the two-route split (`/author/[id]` + a separate `/author/[id]/generate`). `GenerateStudio` moved from `app/author/[id]/generate/` into `components/`; the standalone `generate` route and its now-dead i18n keys (`authorDetail.generateCta`, `generate.pageTitle`, `generate.backToProfile`) are removed. The parent server component owns the page header and `data-voice`; `GenerateStudio` starts in an idle visual state before the first generation. No api_contract or backend change | P1 (frontend) | design-system §7 defines a single "studio" screen and §8.5 (Sprint 3 cluster-landing) requires Style DNA and generation visible together; the separate "Generate in this voice" link was easy to miss and added a redundant navigation hop. One screen also removes a second route transition that made the flow feel slower | | 2026-07-21 | **#28 (Sprint 2) + #41-remainder (Sprint 1) implemented with NO api_contract change.** (a) The vanilla-vs-AutorIA comparative metrics table is **measured client-side** from each generation's text (`lib/textMetrics.ts`: avg sentence length, type–token ratio, word count, top words) because `GenerationOutput` exposes only `text`/`fit_score`/`latency_ms`; it renders only when both branches succeed. (b) The "Generate Passport" button is added as an **affordance only** — enabled iff `GenerateResponse.passport` is present, disabled with helper copy otherwise; the actual formatted-JSON download stays in #42/#44 and the verify view in #29. (c) `StyleProfile.distinctive_vocab` is now surfaced as a top-10 TF-IDF table in the Style DNA panel (its type JSDoc previously said "not rendered"), with a graceful empty state until ML #14 lands real data | P1 (frontend) | `docs/api_contract.yaml` is LOCKED (2026-06-24) — measuring descriptive statistics from the real generated text satisfies #28's "comparative metrics" AC without a contract amendment and without faking the contrast (design-system §8, forensic honesty). Scoping the passport button to an affordance keeps #28 from overlapping #42/#44/#29. The vocab table completes #41's only outstanding AC using data the panel already fetches, so it ships now instead of blocking on ML #14 | | 2026-07-21 | **Frontend testing harness = Vitest** (dev-only, first test runner for `frontend/`). Added `vitest` devDependency, `test` (`vitest run`) + `test:watch` scripts, `frontend/vitest.config.ts` (node environment, `@`→`src` alias, no jsdom), and a `Unit tests` step in the `lint-frontend` CI job. First coverage: `src/lib/textMetrics.ts` (26 pure-function cases). Convention: **pure-function unit tests only** for now — no jsdom / `@testing-library` / component tests until they are actually warranted | P1 (frontend) | Frontend had no test runner (CI ran only ESLint + `tsc`); the cross-cutting "happy-path tests" responsibility (2026-06-24) and the new client-side metrics logic (#28) needed a regression guard. Vitest matches the Vite/TS toolchain, runs pure TS with zero extra transform config, and stays lightweight by excluding a DOM environment until component tests earn their keep | +| 2026-07-28 | **`distinctive_vocab` excludes proper nouns, and its corpus is sampled instead of truncated** (#100, WO-18). (a) The 800k-lemma-char memory cap in `style_profile.py` no longer keeps a *prefix* of the corpus: chunks are fed to spaCy in a deterministic bisection order, so the capped sample spans every document of every author. Before the fix the cap fell at 21% of Dickens' chunks (34% Austen, 89% Poe), so "each author's full corpus is one document" (`docs/style_features.md` §4.1) was really *Great Expectations* alone. (b) Tokens tagged `PROPN` by spaCy are dropped **inside** the lemmatization pass (`_lemmas_from_docs`), not by a blacklist after the TF-IDF. `docs/style_features.md` §4.1 updated to match. Re-seeding is required (#86): every existing `style_profiles` row is invalidated. | Sergi | Character names are statistically distinctive but they are plot, not style — they identify the novel, not the author's hand. §4.1's own purpose is that "the jury can read and feel the difference", and its own example (`countenance`, `physiognomy`, `presently`) is a style marker, not a cast list; the demo is judged by what a non-technical human perceives. Filtering at the POS level is free (the tag is already computed) and stays correct for authors added later, which a hand-maintained word blacklist would not. | +| 2026-07-28 | **Exception accepted: #100 ships with a top-10 `distinctive_vocab` overlap of 5, not the ≤3 its Definition of Done requires.** The DoD of #100 states verbatim: *"top-10 overlap across the three authors stays ≤3 (#89 already brought it down from 6 to 3; it must not get worse)"*. **Measured after the fix: 5** — `know`, `little`, `make`, `say`, `time` are common to all three top-10 lists (pairwise: austen∩dickens 8, austen∩poe 6, dickens∩poe 6). The criterion is **not met and is worse than what #89 left**. The full measured top-10 per author is recorded in `docs/style_features.md` §4.1 ("Measured output"), together with the honest reading: the feature returns ordinary high-frequency verbs and nouns, not signature vocabulary, and none of `countenance` / `physiognomy` / `presently` reaches any author's top-30. The `PROPN` filter itself is confirmed working (no proper noun survives in any top-30) and is now covered by a regression test (`ai_pipeline/tests/test_style_profile_compute.py::test_lemmatize_corpus_drops_proper_nouns`, verified to fail when the filter is removed). Ratification: entered by the executor of #100 on measured evidence and accepted by the project owner; **still needs the 2/3 vote** the decision policy of 2026-06-24 requires | Sergi (project owner) — pending 2/3 ratification | With only three documents in the collection, every term present in all three has `df = 3/3` and therefore identical idf (1.000), so TF-IDF degenerates into raw frequency: `countenance` (29 occurrences in Dickens) and `say` (1 866) are idf-tied and the common word always wins. Proper nouns were the only terms with a discriminating `df`, so the overlap of 3 that #89 measured was an artefact of the cast lists being counted as style. Dropping them (the correct call — character names are plot, not the author's hand) necessarily raised the overlap to 5. Two alternatives were considered and rejected for this issue: (a) redefining the scoring (log-odds, or idf against a general-English reference corpus) would change the algorithm `docs/style_features.md` §4.1 declares as closed, which is a scope decision and not a fix to #100; (b) reverting the `PROPN` filter would restore the ≤3 number by putting `pip`, `havisham` and `wemmick` back in front of the jury as "style", i.e. buying the metric with the defect the issue exists to remove. A dull-but-honest list is preferred to a varied-but-wrong one; the gap is documented rather than hidden | | 2026-07-28 | **`api_contract.yaml` descriptions corrected for `dialogue_ratio` and `first_person_ratio` (prose only — NO contract amendment).** Both were described as *"Fraction of sentences…"*. Neither is. `dialogue_ratio` is the fraction of non-punctuation **tokens** inside straight ASCII double quotes (`docs/style_features.md` §3.3, `dialogue_tokens / total_tokens`; implemented that way in `ai_pipeline/autoria_ai/extractor/stylistic.py`). `first_person_ratio` is first-person singular pronouns **per 1,000 tokens** (§3.4), so it is **not bounded by 1** — Poe's expected range is 18–30. The declared shape (`type: number`) is unchanged in both cases, so no client or server contract breaks; only the prose changed | Sergi (repo owner) | Found while implementing #92: three fixture `first_person_ratio` values were written as fractions ([0,1]) because the contract described them that way, making them wrong by a factor of ~45 against the ranges in §7. The LOCKED policy (2026-06-24) requires a log entry for any change to this file; recorded here as a documentation correction, not a scope change — the contract never matched the implementation or the feature spec, and a reader building a UI against it would size the widget for 0–1 | diff --git a/docs/style_features.md b/docs/style_features.md index 21d56b6..22bfa50 100644 --- a/docs/style_features.md +++ b/docs/style_features.md @@ -302,13 +302,15 @@ first_person_ratio = (fp_count / len(doc)) * 1000 # per 1k tokens **What it measures**: the words that are **most characteristic of one author relative to the others** — not just frequent words, but words that are unusually concentrated in that author's corpus. -**Why it matters**: this is the feature the audience *sees* in the demo. When the AutorIA output contains "countenance", "physiognomy", and "presently" for Dickens, the jury can read and feel the difference without understanding a single number. +**Why it matters**: this is the feature the audience *sees* in the demo — it is rendered directly in the Style DNA panel, so whatever it ranks first is what a non-technical juror reads. That makes it the one feature where the gap between what the section *wants* and what the algorithm *returns* has to be stated plainly. See **Measured output** below: on the current three-author corpus it returns **common English verbs and nouns**, not rare period diction. **How it is computed**: standard TF-IDF where each author's full corpus is one "document" and the collection is all three authors combined. ```python from sklearn.feature_extraction.text import TfidfVectorizer +# Each value is the author's corpus already lemmatized and PROPN-filtered +# (see Preprocessing below), sampled across every document of the corpus. corpora = { "austen": "", "dickens": "", @@ -329,7 +331,30 @@ tfidf_matrix = vectorizer.fit_transform(corpora.values()) **Stored as**: a list of `{ "term": str, "score": float }` objects, sorted by score descending. Top 30 terms per author. -**Preprocessing**: lemmatize before TF-IDF, exclude stopwords, exclude tokens shorter than 3 characters. +**Preprocessing**: lemmatize before TF-IDF, exclude stopwords, exclude tokens shorter than 3 characters, and **exclude proper nouns** (spaCy `token.pos_ == "PROPN"`). + +**Why proper nouns are excluded** — decision 2026-07-28, see `docs/decision_log.md`. Character and place names *are* statistically distinctive: `pip`, `havisham`, `wemmick` are concentrated in Dickens and nowhere else, so TF-IDF ranks them first. But they describe the **plot** of one novel, not the author's style, and this feature is labelled to a juror as the author's *style*, which a cast list is not. Removing them did **not** promote rare period diction into their place — it promoted common verbs and nouns (see **Measured output**), and it raised the three-way top-10 overlap from 3 to 5. That trade was accepted knowingly: a wrong-but-varied list is worse than a dull-but-honest one. The filter is applied **inside the lemmatization pass** (`lemmatize_corpus` / `_lemmas_from_docs` in `ai_pipeline/autoria_ai/extractor/style_profile.py`), where spaCy's POS tag is already computed and the rule lives in one place. A stop-word blacklist applied after the TF-IDF is explicitly **not** how this is done: it would have to be maintained by hand for every new author. + +**How "full corpus" is realised** — `_MAX_LEMMA_CHARS` (800 000 lemma characters per author) bounds the seed's peak memory to roughly 1.7 GB. That budget is spent on chunks drawn from **across the whole corpus**, in a deterministic bisection order, so every document of every author is represented. It must never be spent as a *prefix*: doing so meant Dickens' `distinctive_vocab` was computed from the first 21% of his chunks — effectively *Great Expectations* alone — which is why the top terms were its cast list (issue #100). + +**Measured output** — run 2026-07-28 on the full `corpus/` (10 files, 8.68 MB) through the seed's own code path (`build_comparison_lemmas` + `compute_distinctive_vocab`, `top_n=30`, `_MAX_LEMMA_CHARS = 800 000`). This is what the feature returns today; it is recorded here because the section above must not be read as a promise of anything else. + +| # | austen | | dickens | | poe | | +|---|---|---|---|---|---|---| +| 1 | say | 0.3786 | say | 0.5737 | say | 0.2716 | +| 2 | know | 0.2380 | know | 0.2290 | make | 0.1974 | +| 3 | think | 0.2358 | look | 0.2063 | great | 0.1401 | +| 4 | make | 0.1919 | come | 0.2044 | time | 0.1340 | +| 5 | come | 0.1491 | make | 0.1626 | long | 0.1320 | +| 6 | time | 0.1446 | man | 0.1534 | man | 0.1294 | +| 7 | good | 0.1435 | time | 0.1442 | know | 0.1233 | +| 8 | great | 0.1306 | little | 0.1417 | day | 0.1156 | +| 9 | look | 0.1251 | think | 0.1374 | eye | 0.1095 | +| 10 | little | 0.1243 | hand | 0.1294 | little | 0.1054 | + +**Read this honestly.** These are ordinary high-frequency English words, not signature vocabulary. `countenance`, `physiognomy` and `presently` — the words earlier drafts of this section advertised — appear in **neither the top-10 nor the top-30 of any author**. The three top-10 lists intersect in **5 terms** (`know`, `little`, `make`, `say`, `time`); pairwise, austen∩dickens = 8, austen∩poe = 6, dickens∩poe = 6. What the feature discriminates is *how hard each author leans on the same common words*, not which unusual words each author owns. No proper noun survives in any top-30, so the `PROPN` filter is working as specified. + +**Why it cannot do better as specified.** The collection is three documents, so any term appearing in all three has `df = 3/3` and therefore the **same** idf (1.000 with sklearn's smoothing). `countenance` (29 occurrences in Dickens) and `say` (1 866) are idf-tied, so TF-IDF collapses to raw frequency and the common word always wins. Proper nouns were the only terms with a discriminating df, which is exactly why filtering them raised the overlap. Fixing this means changing the scoring (log-odds, or idf against a general-English reference corpus) — a change to the algorithm this section declares, not a tuning knob. It was considered and deferred; see the 2026-07-28 exception entry in `docs/decision_log.md`. --- diff --git a/scripts/seed_corpus.py b/scripts/seed_corpus.py index e373a97..cf3d221 100644 --- a/scripts/seed_corpus.py +++ b/scripts/seed_corpus.py @@ -624,6 +624,11 @@ def build_comparison_lemmas(nlp: Any, authors: list[str] | None = None) -> dict[ ``style_profile._MAX_LEMMA_CHARS`` (800k chars). The pass streams ``nlp.pipe`` and stops at the cap, so it is a fraction of a full feature pass; peak extra memory is 3 x 800 KB of lemma text. + + The cap is spent on chunks sampled across the *whole* corpus, and PROPN + tokens are excluded -- see docs/style_features.md 4.1 "Preprocessing" and + issue #100. Both rules live in ``_lemmas_from_docs``, so this function + inherits them. """ from autoria_ai.extractor.style_profile import lemmatize_corpus