feat(ml): Jeffreys log-odds vocab, better prompt, and fit_score calibration

README.md

@@ -85,7 +85,7 @@ The `StyleProfile v1.0` captures an author's stylistic DNA across four orthogona
 | **Lexical**                | Type-Token Ratio, MATTR-500, hapax ratio, avg word length         | `ai_pipeline/autoria_ai/extractor/lexical.py`    |
 | **Syntactic**              | Sentence length distribution, subordination ratio, dep-tree depth | `ai_pipeline/autoria_ai/extractor/syntactic.py`  |
 | **Stylistic**              | Punctuation & POS distribution, discourse markers                 | `ai_pipeline/autoria_ai/extractor/stylistic.py`  |
-| **Distinctive Vocabulary** | Top-50 TF-IDF terms vs reference corpus                           | `ai_pipeline/autoria_ai/extractor/vocabulary.py` |
+| **Distinctive Vocabulary** | Top-30 terms vs the other authors, ranked by log-odds-ratio        | `ai_pipeline/autoria_ai/extractor/vocabulary.py` |
 | **Semantic**               | Author centroid (768-dim) + UMAP 2D projection                    | `ai_pipeline/autoria_ai/embedder.py`             |
 
 Full feature spec → **[docs/style_features.md](docs/style_features.md)**.

ai_pipeline/autoria_ai/conditioner.py

@@ -61,12 +61,25 @@
 # ---------------------------------------------------------------------------
 
 _TEMPLATE = (
-    "Write in the style of author {author_id}. "
-    "Your writing must have: average sentence length ~{avg_sentence_length} tokens "
-    "with high variation, {subordination_rule}, and vocabulary including terms like "
-    "{vocab_list}. "
-    "Here are example passages: {chunks}. "
-    "Write only in that style; do not explain."
+    "You are writing as {author_id}. Every sentence must be indistinguishable "
+    "from their published prose. Obey ALL of the following constraints — they are "
+    "non-negotiable:\n"
+    "1. SENTENCE LENGTH: Target an average of exactly {avg_sentence_length} words per sentence. "
+    "Do not write sentences that are significantly longer than this average. "
+    "Match the rhythm and length of the provided passages.\n"
+    "2. SYNTACTIC COMPLEXITY: {subordination_rule}.\n"
+    "3. NARRATIVE MODE: {dialogue_rule}\n"
+    "4. SIGNATURE VOCABULARY — these words are statistically unique to {author_id}'s "
+    "writing. Weave as many as naturally possible into your prose: {vocab_list}.\n"
+    "5. TONE AND REGISTER: Match the emotional register, irony level, and narrative "
+    "distance demonstrated in the example passages below.\n\n"
+    "AUTHENTIC EXAMPLE PASSAGES from {author_id} "
+    "(study rhythm, diction and voice — do not copy verbatim):\n"
+    "---\n"
+    "{chunks}\n"
+    "---\n\n"
+    "Write ONLY the requested text in {author_id}'s voice. "
+    "No preamble, no meta-commentary, no explanations."
 )
 
 
@@ -121,35 +134,13 @@ def _fit_chunks_to_token_budget(chunks: list[str], budget_tokens: int) -> list[s
 
 
 def build_system_prompt(style_profile: dict, rag_chunks: list[str]) -> str:
-    """Compose the conditioned system prompt for the Watsonx LLM.
-
-    Parameters
-    ----------
-    style_profile:
-        A StyleProfile v1.0 dict (see ``autoria_ai/schemas/style_profile.json``).
-        Missing keys are handled with safe fallbacks so the function never raises
-        on a partial profile.
-    rag_chunks:
-        Retrieved example passages (top-k by cosine similarity from pgvector).
-        At most ``_MAX_CHUNKS`` (5) are considered; the result is then packed
-        into the remaining token budget (see module docstring), so the
-        returned prompt never exceeds ``_MAX_PROMPT_TOKENS`` tokens
-        regardless of how long the individual chunks are.
-
-    Returns
-    -------
-    str
-        A fully-rendered system prompt string ready to be passed as the
-        ``system`` parameter of a Watsonx chat-completion call, guaranteed to
-        be at most ``_MAX_PROMPT_TOKENS`` tokens under cl100k_base.
-    """
+    """Compose the conditioned system prompt for the Watsonx LLM."""
     # -- author id -------------------------------------------------------------
     author_id: str = style_profile.get("author_id", "unknown")
 
     # -- avg sentence length ---------------------------------------------------
     syntactic: dict = style_profile.get("syntactic", {})
     avg_sentence_length: float = syntactic.get("avg_sentence_length_tokens", 20.0)
-    # Format as integer-like when it's a whole number, otherwise one decimal.
     avg_sl_str = (
         str(int(avg_sentence_length))
         if avg_sentence_length == int(avg_sentence_length)
@@ -159,11 +150,20 @@ def build_system_prompt(style_profile: dict, rag_chunks: list[str]) -> str:
     # -- subordination rule (natural language translation) ---------------------
     subordination_ratio: float = syntactic.get("subordination_ratio", 0.0)
     if subordination_ratio >= 0.3:
-        subordination_rule = "heavy use of subordinate clauses"
+        subordination_rule = "heavy use of subordinate clauses. However, you MUST use periods (.) frequently to end sentences and prevent the average sentence length from exceeding the target"
     elif subordination_ratio >= 0.15:
         subordination_rule = "moderate use of subordinate clauses"
     else:
         subordination_rule = "straightforward clause structure with few subordinate clauses"
+    # -- dialogue rule (dialogue_ratio lives under stylistic, not syntactic) ---
+    stylistic: dict = style_profile.get("stylistic", {})
+    dialogue_ratio: float = stylistic.get("dialogue_ratio", 0.0)
+    if dialogue_ratio >= 0.15:
+        dialogue_rule = "Integrate conversational dialogue frequently, mirroring the author's formatting and pacing."
+    elif dialogue_ratio >= 0.05:
+        dialogue_rule = "Use dialogue sparingly and only when appropriate to the scene."
+    else:
+        dialogue_rule = "Focus heavily on prose and internal exposition; avoid dialogue unless strictly necessary."
 
     #  -- distinctive vocab (top 10-15 terms to avoid token bloat) -------------
     raw_vocab: list[dict] = style_profile.get("distinctive_vocab", [])
@@ -175,13 +175,12 @@ def build_system_prompt(style_profile: dict, rag_chunks: list[str]) -> str:
     # -- rag chunks: count cap first (existing safeguard), then token budget --
     count_capped_chunks: list[str] = rag_chunks[:_MAX_CHUNKS]
 
-    # Everything except `chunks` is fixed at this point, so the true chunk
-    # budget is whatever tokens are left after rendering the rest of the
-    # template with the "no passages" fallback in place of the real chunks.
     fixed_prompt = _TEMPLATE.format(
         author_id=author_id,
         avg_sentence_length=avg_sl_str,
+        max_sentence_length=str(int(avg_sentence_length * 1.5)),
         subordination_rule=subordination_rule,
+        dialogue_rule=dialogue_rule,
         vocab_list=vocab_list,
         chunks=_NO_CHUNKS_FALLBACK,
     )
@@ -194,22 +193,22 @@ def build_system_prompt(style_profile: dict, rag_chunks: list[str]) -> str:
     prompt = _TEMPLATE.format(
         author_id=author_id,
         avg_sentence_length=avg_sl_str,
+        max_sentence_length=str(int(avg_sentence_length * 1.5)),
         subordination_rule=subordination_rule,
+        dialogue_rule=dialogue_rule,
         vocab_list=vocab_list,
         chunks=chunks_text,
     )
 
-    # Defense in depth: BPE merges across the chunk/template boundary can
-    # shift the count by a token or two relative to the estimate above. If
-    # that ever pushes the total over budget, drop the last chunk and retry
-    # rather than ship a prompt over the contract.
     while _token_count(prompt) > _MAX_PROMPT_TOKENS and safe_chunks:
         safe_chunks = safe_chunks[:-1]
         chunks_text = _CHUNK_SEPARATOR.join(safe_chunks) if safe_chunks else _NO_CHUNKS_FALLBACK
         prompt = _TEMPLATE.format(
             author_id=author_id,
             avg_sentence_length=avg_sl_str,
+            max_sentence_length=str(int(avg_sentence_length * 1.5)),
             subordination_rule=subordination_rule,
+            dialogue_rule=dialogue_rule,
             vocab_list=vocab_list,
             chunks=chunks_text,
         )

ai_pipeline/autoria_ai/extractor/cleaner.py

@@ -14,6 +14,23 @@
     re.IGNORECASE,
 )
 
+# Illustrated Gutenberg editions (e.g. Pride and Prejudice, Sense and
+# Sensibility, Great Expectations in this corpus) embed bracketed image
+# placeholders — bare "[Illustration]" or "[Illustration: caption text]" —
+# inline wherever a plate appeared in the printed book. These are page-layout
+# artifacts, not prose, but were surviving into the lemmatized corpus used for
+# distinctive_vocab: 193 occurrences of the word "illustration" put it in
+# Austen's TF-IDF/log-odds top terms, which is not a style signal by any
+# definition (docs/decision_log.md, 2026-07-30 distinctive_vocab entries).
+# The caption can itself contain one bracketed sub-run (Gutenberg's own
+# "[_Copyright 1894 by George Allen._]" credit line nested inside the
+# illustration block), so the pattern allows exactly one level of nested
+# brackets rather than stopping at the first "]" it finds.
+_PG_ILLUSTRATION = re.compile(
+    r"\[Illustration(?:[^\[\]]|\[[^\[\]]*\])*\]",
+    re.IGNORECASE,
+)
+
 # Two or more consecutive blank lines (any mix of spaces/tabs between newlines).
 _MULTI_BLANK = re.compile(r"\n{3,}")
 
@@ -43,7 +60,10 @@ def clean_text(text: str) -> str:
     # the punct_distribution em-dash bucket (§3.1) captures them.
     text = text.replace("--", "\u2014")  # — (U+2014 EM DASH)
 
-    # ── 5. Collapse multiple consecutive blank lines → single blank line ──────
+    # ── 5. Strip "[Illustration]" / "[Illustration: caption]" blocks ─────────
+    text = _PG_ILLUSTRATION.sub("", text)
+
+    # ── 6. Collapse multiple consecutive blank lines → single blank line ──────
     text = _MULTI_BLANK.sub("\n\n", text)
 
     return text.strip()

ai_pipeline/autoria_ai/extractor/style_profile.py

@@ -14,7 +14,7 @@
 Pipeline (docs/style_features.md §8)
 ------------------------------------
 cleaned documents → chunk (500 / 50) → spaCy ``nlp.pipe`` on chunks
-→ aggregate linguistic features → TF-IDF distinctive vocab →
+→ aggregate linguistic features → Jeffreys log-odds-ratio distinctive vocab →
 sentence-transformers centroid over chunks.
 """
 
@@ -37,7 +37,8 @@
 
 # Soft cap so spaCy does not OOM on full Victorian novels in one Doc.
 # Features are still computed over many chunks via nlp.pipe; this only
-# limits how much raw text we keep when building the lemma string for TF-IDF.
+# limits how much raw text we keep when building the lemma string for
+# distinctive_vocab (Jeffreys log-odds-ratio, formerly TF-IDF / Monroe z-score).
 #
 # The cap is a memory bound, not a corpus definition: it is spent on chunks
 # drawn from across the whole corpus (``_spread_order``), never on a prefix.
@@ -108,7 +109,7 @@ def _in_spread_order(items: list[Any]) -> list[Any]:
 
 
 def _lemmas_from_docs(docs: Iterable[Any], max_chars: int = _MAX_LEMMA_CHARS) -> str:
-    """Space-joined lower lemmas (alpha, non-proper, len>=3) for TF-IDF input.
+    """Space-joined lower lemmas (NOUN/ADJ/ADV, alpha, len>=3) for log-odds input.
 
     *docs* may be a lazy iterable (e.g. the generator returned by
     ``nlp.pipe``): the function stops consuming it as soon as *max_chars*
@@ -117,23 +118,53 @@ def _lemmas_from_docs(docs: Iterable[Any], max_chars: int = _MAX_LEMMA_CHARS) ->
     (:func:`_in_spread_order`) so that an early stop still samples the whole
     corpus.
 
-    ``PROPN`` tokens are dropped.  Character and place names are genuinely
-    concentrated in one author's corpus, so TF-IDF ranks them first — but they
-    identify the *novel*, not the author's hand, and ``distinctive_vocab`` is
-    the one feature a non-technical juror reads directly
+    Only ``NOUN``, ``ADJ``, and ``ADV`` tokens are kept.  These three POS
+    categories carry the strongest stylistic fingerprint in computational
+    stylistics: authors differ most in descriptive adjectives (Poe:
+    *ghastly/sepulchral*, Austen: *amiable/sensible*), thematic nouns, and
+    manner adverbs.  Common narrative verbs (say, know, think, make) appear
+    at high frequency in **all** literary prose and add noise to the ranking
+    rather than signal.  ``PROPN`` tokens are also excluded — character and
+    place names identify the *novel*, not the author's hand
     (docs/style_features.md §4.1; decision 2026-07-28 in docs/decision_log.md).
-    The filter lives here, where spaCy's POS tag is already computed, so it is
-    free and stays correct for authors added later — unlike a hand-maintained
-    word blacklist applied after the TF-IDF.
+
+    A small set of corpus-metadata lemmas (``_CORPUS_META_STOPS``) are also
+    filtered out: these are editorial / Project Gutenberg structural words
+    (e.g. *copyright*, *chapter*, *edition*) that survive the header/footer
+    strip in some editions and rank spuriously high.
     """
+    _KEEP_POS: frozenset[str] = frozenset({"NOUN", "ADJ", "ADV"})
+    # Editorial / Gutenberg structural words that are NOT style features.
+    # Kept small and specific — only words observed to pollute the ranking.
+    _CORPUS_META_STOPS: frozenset[str] = frozenset(
+        {
+            "copyright",
+            "gutenberg",
+            "project",
+            "ebook",
+            "produce",
+            "transcribe",
+            "edition",
+            "chapter",
+            "volume",
+            "illustration",
+            "preface",
+            "appendix",
+            "footnote",
+            "translator",
+            "publisher",
+        }
+    )
     parts: list[str] = []
     size = 0
     for doc in docs:
         for tok in doc:
-            if tok.pos_ == "PROPN":
+            if tok.pos_ not in _KEEP_POS:
                 continue
             if tok.is_alpha and len(tok.lemma_) >= 3:
                 piece = tok.lemma_.lower()
+                if piece in _CORPUS_META_STOPS:
+                    continue
                 parts.append(piece)
                 size += len(piece) + 1
                 if size >= max_chars:
@@ -148,14 +179,15 @@ def lemmatize_corpus(
     chunk_texts: list[str] | None = None,
     max_chars: int = _MAX_LEMMA_CHARS,
 ) -> str:
-    """Lemmatize *documents* into the single TF-IDF "document" of §4.1.
-
-    docs/style_features.md §4.1 defines ``distinctive_vocab`` as TF-IDF where
-    each author's full corpus is one document and the collection is all three
-    authors combined.  Callers that own more than one author's corpus (the
-    seeding script) use this to build the ``comparison_lemmas`` mapping that
-    :func:`compute_style_profile` expects, without having to reimplement the
-    lemma rules — alpha-only, lowercase, ``len >= 3``, no ``PROPN`` — or the
+    """Lemmatize *documents* into the single log-odds bag of §4.1.
+
+    docs/style_features.md §4.1 defines ``distinctive_vocab`` as Jeffreys
+    log-odds-ratio where each author's full corpus is one bag and the
+    background is the pooled lemmas of the other authors.  Callers that own
+    more than one author's corpus (the seeding script) use this to build the
+    ``comparison_lemmas`` mapping that :func:`compute_style_profile` expects,
+    without having to reimplement the lemma rules — NOUN/ADJ/ADV only,
+    alpha-only, lowercase, ``len >= 3``, no corpus-metadata stops — or the
     ``_MAX_LEMMA_CHARS`` cap that bounds peak memory per author.
 
     The spaCy pass streams and stops at *max_chars*, so lemmatizing a
@@ -203,7 +235,7 @@ def compute_style_profile(
     nlp:
         Loaded spaCy model (``en_core_web_lg``).  Caller owns lifecycle.
     comparison_lemmas:
-        Optional ``{slug: lemmatized_corpus}`` for TF-IDF.  When omitted or
+        Optional ``{slug: lemmatized_corpus}`` for cross-author log-odds.  When omitted or
         containing only this author, distinctive_vocab may be empty / weak.
     chunk_texts:
         Precomputed ~500-token chunks.  When ``None``, documents are chunked
@@ -226,7 +258,7 @@ def compute_style_profile(
     syntactic = _weighted_mean([compute_syntactic(d) for d in docs], weights)
     stylistic = _weighted_mean([compute_stylistic(d) for d in docs], weights)
 
-    # Spread order, same as lemmatize_corpus: this author's own TF-IDF
+    # Spread order, same as lemmatize_corpus: this author's own log-odds
     # "document" must be built the same way as the comparison corpora it is
     # scored against, and the cap must not fall inside the first novel (#100).
     author_lemmas = _lemmas_from_docs(_in_spread_order(docs))