From f2cafcd804a406aa5839e9a2c9c5e9f7948130ac Mon Sep 17 00:00:00 2001 From: Som Tripathi Date: Tue, 8 Sep 2026 00:49:25 -0500 Subject: [PATCH 1/2] Add an opt-in literal-diff gate for technical memory The NLI signal finds contradiction in logical structure: negation, antonyms, numeric conflict, named-entity swaps. A same-slot swap whose only change is an opaque code token has none of that, so NLI reads it as two compatible statements and the stale record survives. Measured on 16 coding-guidance pairs at the shipped 0.62/0.50 thresholds: 14/16 with 0/8 false positives. Both misses were pure code-token swaps. cos 0.945, contra 0.013 `cargo test --workspace` -> `cargo nextest run --workspace` cos 0.839, contra 0.011 "the planning folder" -> "docs/planning" `use_literal_gate=True` extracts code literals (backtick spans, paths, flags) and treats a same-slot pair whose literal sets differ as a supersession. 15/16, still 0/8 false positives. Both-sidedness is load-bearing: a one-sided rule fires on elaborations that merely mention a path. The discriminating negative is a restatement at cos 0.963 whose literals are identical, which correctly stays quiet - so this is not "high cosine means supersede". The second miss stays missed, because one side carries no literal at all and guessing there would cost the zero-false-positive property. Off by default. It was validated only on coding guidance, and the drift and LongMemEval numbers are prose, where the gate is inert because neither side carries a literal. Existing behaviour is unchanged with the flag off. Four tests added, covering the recovered case and the three negatives that keep the gate honest. Full suite green. --- README.md | 11 +++++++ memory/sidecar.py | 43 ++++++++++++++++++++++++- tests/test_sidecar.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94122cf..6e8dd57 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,17 @@ few-millisecond local op instead of a ~400 ms LLM call. no logical conflict falls back to retrieval recency. That small model also over-fires on parallel cross-topic structure, so a same-topic cosine gate (tuned on a dev split, validated held-out) runs in front of it. +- On **technical memory** the NLI signal has a second blind spot, measured 2026-09-08 on 16 + coding-guidance pairs at the shipped thresholds: 14/16 with 0/8 false positives, but both misses were + same-slot swaps whose only change was an opaque code token — `cargo test --workspace` → + `cargo nextest run --workspace` scores cosine 0.945 and contradiction 0.013. Two near-identical + sentences differing in an identifier are not a logical contradiction, and NLI is right about that; + they are still a supersession. Every pair the gate caught carried negation, an antonym, a numeric + conflict, or a named-entity swap. The opt-in `use_literal_gate=True` adds a token-free literal-diff + signal for this case (15/16, still 0/8 false positives). It is **off by default**: it was validated + only on coding guidance, and the drift and LongMemEval numbers above are prose, where it is inert + because neither side carries a literal. The remaining miss — a prose location superseded by a path, + literals on one side only — is unfixed by design rather than guessed at. ## Honest positioning: what is ours, what is not diff --git a/memory/sidecar.py b/memory/sidecar.py index 8eeffd6..cb1a13c 100644 --- a/memory/sidecar.py +++ b/memory/sidecar.py @@ -14,6 +14,7 @@ """ from __future__ import annotations +import re import sqlite3 from collections import deque from dataclasses import dataclass @@ -29,6 +30,22 @@ EMBED_MODEL = "BAAI/bge-small-en-v1.5" DAY_S = 86_400.0 +# Code literals the NLI model cannot reason about: backtick spans, path-like tokens, flags. +_BACKTICK = re.compile(r"`([^`]+)`") +_PATHY = re.compile(r"(? frozenset[str]: + """Extract code-ish tokens from a memory record. + + Used only by the optional literal-diff gate (`use_literal_gate`). Returns an empty + set for ordinary prose, which is what makes the gate inert on natural-language + memory: no literals on either side means the gate can never fire. + """ + found = {m.strip() for m in _BACKTICK.findall(text)} + found |= {m.strip() for m in _PATHY.findall(text)} + return frozenset(f for f in found if f) + @lru_cache(maxsize=4) def _get_embedder(model: str, device: str) -> SentenceTransformer: @@ -70,6 +87,7 @@ def __init__( nli_model: str = "cross-encoder/nli-deberta-v3-xsmall", nli_contra_threshold: float = 0.5, nli_candidate_sim: float = 0.62, + use_literal_gate: bool = False, embedder: SentenceTransformer | None = None, adaptive_window: int = 15, adaptive_k: float = 0.5, @@ -91,6 +109,7 @@ def __init__( self.nli_model_name = nli_model self.nli_contra_threshold = nli_contra_threshold self.nli_candidate_sim = nli_candidate_sim # loose cosine pre-filter when NLI decides + self.use_literal_gate = use_literal_gate # opt-in third signal for technical memory self._nli = None # lazy NLI cross-encoder (loaded on first contradiction check) self.embedder = embedder or _get_embedder(embed_model, device) @@ -242,13 +261,35 @@ def _contradicted_records(self, emb: np.ndarray, text: str) -> list[ObservationR and unrelated facts (contra~0.01) are kept; only a real contradiction (>= nli_contra_threshold) counts. Shared by admission-override AND forgetting so NLI runs once per write. Called BEFORE insert, so the new record is not a candidate. HONEST LIMITATION: a same-slot swap with low - lexical overlap AND no logical conflict falls back to retrieval recency.""" + lexical overlap AND no logical conflict falls back to retrieval recency. + + A second limitation, measured 2026-09-08 on technical memory: a swap with HIGH lexical + overlap and no logical conflict also falls through, because the change lives in an opaque + code token rather than in sentence structure. `use_literal_gate=True` adds a token-free + third signal for that case; it is opt-in and inert on prose (see the gate comment below).""" hits = [] + new_lits = code_literals(text) if self.use_literal_gate else frozenset() for r in self._active_records(): if float(np.dot(emb, r.embedding)) < self.nli_candidate_sim: continue if self._nli_contradiction(r.text, text) >= self.nli_contra_threshold: hits.append(r) + elif new_lits: + # LITERAL-DIFF GATE (opt-in, technical memory). NLI finds contradiction in + # logical STRUCTURE - negation, antonyms, numeric conflict. A same-slot swap + # whose only change is an opaque code token has none of that, so NLI reads it + # as two compatible statements and the stale record survives forever. + # Measured on 16 coding-guidance pairs (0.62/0.50 defaults unchanged): + # cos 0.945, contra 0.013 `cargo test --workspace` -> `cargo nextest run --workspace` + # Same-slot (cosine already passed) + BOTH sides carrying literals + those + # literal sets differing is a supersession. Both-sided is load-bearing: a + # one-sided rule fires on elaborations that merely mention a path. The + # discriminating negative is a restatement at cos 0.963 whose literals are + # IDENTICAL - it correctly stays quiet, so this is not "high cosine means + # supersede". Zero tokens, zero model calls, same thesis as the other two. + old_lits = code_literals(r.text) + if old_lits and old_lits != new_lits: + hits.append(r) return hits def _decay(self, new: ObservationRecord, contradicted: list[ObservationRecord]) -> None: diff --git a/tests/test_sidecar.py b/tests/test_sidecar.py index 2f55954..ba50d33 100644 --- a/tests/test_sidecar.py +++ b/tests/test_sidecar.py @@ -101,3 +101,78 @@ def test_admission_via_contradiction(): assert update is not None, "a contradicting knowledge-update must be admitted even when surprise is shut" assert {r.id: r for r in s._active_records()}[base.id].superseded_by == update.id, \ "the admitted update must also supersede the stale fact it corrects" + + +# --------------------------------------------------------------- literal-diff gate +# The NLI signal finds contradiction in logical structure: negation, antonyms, numeric +# conflict. A same-slot swap whose only change is an opaque code token has none of that, +# so NLI scores it ~0.01 and the stale record survives. Measured on 16 coding-guidance +# pairs at the shipped 0.62/0.50 defaults: 14/16 with 0/8 false positives, and BOTH +# misses were pure code-token swaps. The opt-in gate recovers one of them; the other +# (prose location -> path, literals on one side only) stays missed on purpose. + + +def test_code_literals_extracts_only_code_ish_tokens(): + from memory.sidecar import code_literals + + assert code_literals("The tests run with `cargo test --workspace`.") == frozenset( + {"cargo test --workspace", "--workspace"} + ) + assert code_literals("Docs live under docs/planning today.") == frozenset({"docs/planning"}) + # Ordinary prose has none, which is what keeps the gate inert on natural-language memory. + assert code_literals("I moved from Ames to Chicago.") == frozenset() + assert code_literals("My dog is named Rex.") == frozenset() + + +def test_literal_gate_is_off_by_default(): + s = MemorySidecar("lit-default", tau=0.0) + assert s.use_literal_gate is False, "the gate must stay opt-in; prose benchmarks assume it off" + + +def test_literal_gate_supersedes_a_command_swap_that_nli_misses(tmp_path): + """The motivating case: cos 0.945, NLI contradiction 0.013, stale record survives forever.""" + s = MemorySidecar("lit-on", db_path=str(tmp_path / "m.db"), tau=0.0, + use_nli=True, use_literal_gate=True) + try: + _ = s._nli_contradiction("a", "b") # force-load; skip if offline + except Exception as e: + pytest.skip(f"NLI model unavailable: {e}") + + old = s.write("The tests for this project run with `cargo test --workspace`.") + assert old is not None + new = s.write("The tests for this project run with `cargo nextest run --workspace`.") + assert new is not None, "a command swap is an update and must be admitted" + assert {r.id: r for r in s._active_records()}[old.id].superseded_by == new.id, "the new command must supersede the stale one" + + +def test_literal_gate_keeps_a_restatement_with_identical_literals(tmp_path): + """The discriminating negative: cos 0.963, but the literals match, so this is not an update. + + Without this the gate would degenerate into 'high cosine means supersede'. + """ + s = MemorySidecar("lit-dup", db_path=str(tmp_path / "m.db"), tau=0.0, + use_nli=True, use_literal_gate=True) + try: + _ = s._nli_contradiction("a", "b") + except Exception as e: + pytest.skip(f"NLI model unavailable: {e}") + + old = s.write("The tests run with `cargo test --workspace`.") + assert old is not None + s.write("Use `cargo test --workspace` to run the tests.") + assert {r.id: r for r in s._active_records()}[old.id].superseded_by is None, "a restatement carrying the same literals must not supersede anything" + + +def test_literal_gate_keeps_an_elaboration_that_mentions_one_literal(tmp_path): + """Both sides must carry literals. A one-sided rule would forget elaborations.""" + s = MemorySidecar("lit-elab", db_path=str(tmp_path / "m.db"), tau=0.0, + use_nli=True, use_literal_gate=True) + try: + _ = s._nli_contradiction("a", "b") + except Exception as e: + pytest.skip(f"NLI model unavailable: {e}") + + old = s.write("The tests for this project run with `cargo test --workspace`.") + assert old is not None + s.write("The test suite currently has 148 passing tests.") + assert {r.id: r for r in s._active_records()}[old.id].superseded_by is None, "an elaboration with no literal of its own must not supersede the original" From a7726cc2ce7bf1fe6ed274027d52e7f2206c4a06 Mon Sep 17 00:00:00 2001 From: Som Tripathi Date: Tue, 8 Sep 2026 01:10:25 -0500 Subject: [PATCH 2/2] Drop the literal gate; a held-out fixture says it earns nothing The gate in the previous commit was designed on a 16-pair probe where two command swaps scored NLI contradiction 0.013, and it recovered one of them. This commit adds benchmarks/technical_memory.py: 36 fresh pairs of coding guidance, written in one pass before any policy was run against them, sharing no pair with that probe. For the gate it is a genuine held-out set. policy recall precision FP FN cosine-only 0.94 0.77 5 1 nli 0.94 1.00 0 1 nli+literal 0.94 1.00 0 1 The gate is exactly inert. On this set the NLI model fires correctly on every command swap, contradiction 0.841 to 0.990, so the failure the gate targets does not generalise. It was a property of two specific sentence pairs, not of code-token swaps as a class. So memory/sidecar.py, its tests, and the README go back to what they were; the policy survives only inside the benchmark, as a documented negative result nobody has to re-derive. What the benchmark does establish is worth more than the gate was. The shipped NLI path transfers to technical memory at PERFECT PRECISION: it never once forgot approved guidance. All five cosine-only false positives are restatements ("Use `pytest -q` to run the unit tests" against "The unit tests run with `pytest -q`", cosine 0.974), and NLI scores every one at contradiction <= 0.002. That is a clean domain-transferred demonstration of what the contradiction signal buys over a density gate. The one shared false negative is an antonym pair with no code literal at all, cosine 0.846 and contradiction 0.021. Cosine-only catches it and both NLI policies miss it. That is the real remaining gap. --- README.md | 11 -- benchmarks/technical_memory.py | 276 +++++++++++++++++++++++++++++++++ memory/sidecar.py | 43 +---- tests/test_sidecar.py | 75 --------- 4 files changed, 277 insertions(+), 128 deletions(-) create mode 100644 benchmarks/technical_memory.py diff --git a/README.md b/README.md index 6e8dd57..94122cf 100644 --- a/README.md +++ b/README.md @@ -109,17 +109,6 @@ few-millisecond local op instead of a ~400 ms LLM call. no logical conflict falls back to retrieval recency. That small model also over-fires on parallel cross-topic structure, so a same-topic cosine gate (tuned on a dev split, validated held-out) runs in front of it. -- On **technical memory** the NLI signal has a second blind spot, measured 2026-09-08 on 16 - coding-guidance pairs at the shipped thresholds: 14/16 with 0/8 false positives, but both misses were - same-slot swaps whose only change was an opaque code token — `cargo test --workspace` → - `cargo nextest run --workspace` scores cosine 0.945 and contradiction 0.013. Two near-identical - sentences differing in an identifier are not a logical contradiction, and NLI is right about that; - they are still a supersession. Every pair the gate caught carried negation, an antonym, a numeric - conflict, or a named-entity swap. The opt-in `use_literal_gate=True` adds a token-free literal-diff - signal for this case (15/16, still 0/8 false positives). It is **off by default**: it was validated - only on coding guidance, and the drift and LongMemEval numbers above are prose, where it is inert - because neither side carries a literal. The remaining miss — a prose location superseded by a path, - literals on one side only — is unfixed by design rather than guessed at. ## Honest positioning: what is ours, what is not diff --git a/benchmarks/technical_memory.py b/benchmarks/technical_memory.py new file mode 100644 index 0000000..f37bc9c --- /dev/null +++ b/benchmarks/technical_memory.py @@ -0,0 +1,276 @@ +"""Technical-memory supersession benchmark: does the write gate work on CODING guidance? + +The drift suite measures personal-fact slot updates ("I moved to Chicago"). Agent memory is +increasingly used for a different content type: standing engineering guidance, where the +facts are build commands, paths, flags, versions and conventions. This benchmark measures +the write-and-forget decision on that domain. + +WHY IT IS A SEPARATE BENCHMARK. The NLI signal finds contradiction in logical STRUCTURE: +negation, antonyms, numeric conflict, entity swaps. A large share of real technical +supersessions have none of that. "The tests run with `pytest -q`" and "The tests run with +`pytest -n auto`" are not logically contradictory sentences; they are two compatible-looking +statements that happen to disagree about an opaque token. That shape is under-represented in +prose benchmarks and over-represented in engineering memory. + +THREE POLICIES, one embedder, one fixture, so any gap is the policy's doing: + + cosine-only supersede_sim 0.75, the coarse path used when use_nli is False + nli cosine 0.62 candidate gate + NLI contradiction >= 0.50 (the shipped default) + nli+literal the same, plus the opt-in use_literal_gate literal-diff signal + +The asymmetry that matters: a FALSE POSITIVE forgets guidance the user deliberately approved, +while a FALSE NEGATIVE leaves one stale line in the store. Precision is worth more than recall +here, so the table reports both rather than a single accuracy number. + +RESULT (36 pairs, 18 positive / 18 negative): + + policy recall precision FP FN + cosine-only 0.94 0.77 5 1 + nli 0.94 1.00 0 1 + nli+literal 0.94 1.00 0 1 + +Two things worth reading off that table. + +The shipped NLI path transfers to technical memory at PERFECT PRECISION. It never once +forgot approved guidance. All five cosine-only false positives are restatements ("Use +`pytest -q` to run the unit tests" against "The unit tests run with `pytest -q`", cosine +0.974), and NLI scores every one of them at contradiction <= 0.002. That is a clean, +domain-transferred demonstration of exactly what the contradiction signal buys over a +density gate. + +The literal-diff policy earns NOTHING here and is therefore NOT shipped in `memory/`. It was +designed on an earlier 16-pair probe where two command swaps scored contradiction 0.013, and +it recovered one of them. On this held-out set the NLI model fires correctly on every command +swap (contradiction 0.841 to 0.990), so the failure it targets does not generalise: it was a +property of two specific sentence pairs, not of code-token swaps as a class. The policy stays +in this file as a documented negative result, so nobody re-derives it. + +The single shared false negative is an antonym pair with no code literal at all ("generated +at build time" against "checked in and never generated", cosine 0.846, contradiction 0.021). +Cosine-only catches it; both NLI policies miss it. That is the real remaining gap. + +FIXTURE PROVENANCE, stated because it bounds every number above. 36 pairs, written in one +pass before any policy was run against them, and sharing no pair with the 16-pair probe that +motivated the literal gate. For that gate this is therefore a genuine held-out set. Still +self-authored by one person, so it measures transfer to one author's idea of technical +memory, not to a sampled population of real stores. + +Run: python -m benchmarks.technical_memory +""" + +from __future__ import annotations + +import re + +import numpy as np +from sentence_transformers import CrossEncoder, SentenceTransformer + +from memory.sidecar import EMBED_MODEL + +# Code literals an NLI model cannot reason about: backtick spans, path-like tokens, flags. +# Local to this benchmark on purpose. The literal-diff policy below is a CANDIDATE that this +# fixture rejects, so nothing in `memory/` depends on it. +_BACKTICK = re.compile(r"`([^`]+)`") +_PATHY = re.compile(r"(? frozenset[str]: + found = {m.strip() for m in _BACKTICK.findall(text)} + found |= {m.strip() for m in _PATHY.findall(text)} + return frozenset(f for f in found if f) + +NLI_MODEL = "cross-encoder/nli-deberta-v3-xsmall" +CANDIDATE_SIM = 0.62 # MemorySidecar.nli_candidate_sim default +CONTRA_THRESHOLD = 0.50 # MemorySidecar.nli_contra_threshold default +SUPERSEDE_SIM = 0.75 # MemorySidecar.supersede_sim default (cosine-only path) + +# (category, should_supersede, stored_record, new_observation) +PAIRS: list[tuple[str, bool, str, str]] = [ + # ---- supersession: the change lives in an opaque code token (the hard class) ---- + ("swap-command", True, + "The unit tests run with `pytest -q`.", + "The unit tests run with `pytest -n auto`."), + ("swap-command", True, + "Build the release with `make release`.", + "Build the release with `just release`."), + ("swap-command", True, + "Format the codebase using `black .`.", + "Format the codebase using `ruff format .`."), + ("swap-command", True, + "Start the dev server with `npm run dev`.", + "Start the dev server with `pnpm dev`."), + ("swap-path", True, + "Configuration lives in config/settings.yaml.", + "Configuration lives in etc/app/settings.yaml."), + ("swap-path", True, + "Migrations are stored under db/migrate.", + "Migrations are stored under alembic/versions."), + ("swap-flag", True, + "Run the linter with the --strict flag.", + "Run the linter with the --pedantic flag."), + ("swap-version", True, + "The project targets Python 3.11.", + "The project targets Python 3.13."), + ("swap-port", True, + "The API listens on port 8080.", + "The API listens on port 9090."), + + # ---- supersession: carries logical structure NLI can see ---- + ("negation", True, + "The deploy script pushes tags automatically.", + "The deploy script does not push tags automatically."), + ("negation", True, + "Secrets may be committed to the private repo.", + "Secrets must never be committed to any repo."), + ("antonym", True, + "The staging database is writable by developers.", + "The staging database is read-only for developers."), + ("antonym", True, + "Test fixtures are generated at build time.", + "Test fixtures are checked in and never generated."), + ("entity", True, + "The CI provider for this project is Travis.", + "The CI provider for this project is GitHub Actions."), + ("entity", True, + "The team's package manager is Yarn.", + "The team's package manager is pnpm."), + ("numeric", True, + "Pull requests need two approving reviews.", + "Pull requests need one approving review."), + ("numeric", True, + "The request timeout is 30 seconds.", + "The request timeout is 5 seconds."), + ("policy", True, + "Release branches are cut monthly.", + "Release branches are cut every two weeks."), + + # ---- elaboration: same subject, compatible, must NOT supersede ---- + ("elaboration", False, + "The unit tests run with `pytest -q`.", + "The suite takes about four minutes on CI."), + ("elaboration", False, + "Configuration lives in config/settings.yaml.", + "Every key in that file has a documented default."), + ("elaboration", False, + "The API listens on port 8080.", + "Health checks are served from the same process."), + ("elaboration", False, + "The CI provider for this project is GitHub Actions.", + "The workflow runs on ubuntu-latest and macos-latest."), + ("elaboration", False, + "Secrets must never be committed to any repo.", + "The team uses a managed secret store for deploy credentials."), + ("elaboration", False, + "The project targets Python 3.13.", + "Type hints are checked in strict mode."), + ("elaboration", False, + "Build the release with `just release`.", + "The release artifact is a single static binary."), + + # ---- restatement: same fact reworded, must NOT supersede ---- + ("restatement", False, + "The unit tests run with `pytest -q`.", + "Use `pytest -q` to run the unit tests."), + ("restatement", False, + "Run the linter with the --strict flag.", + "The linter should be invoked using --strict."), + ("restatement", False, + "Pull requests need two approving reviews.", + "Two approving reviews are required on a pull request."), + ("restatement", False, + "Migrations are stored under db/migrate.", + "You will find the migrations in db/migrate."), + ("restatement", False, + "The deploy script pushes tags automatically.", + "Tags are pushed automatically by the deploy script."), + + # ---- unrelated: different subject, must NOT supersede ---- + ("unrelated", False, + "The unit tests run with `pytest -q`.", + "The changelog follows Keep a Changelog format."), + ("unrelated", False, + "Configuration lives in config/settings.yaml.", + "Code review turnaround is expected within one working day."), + ("unrelated", False, + "The API listens on port 8080.", + "Documentation is built with `mkdocs build`."), + ("unrelated", False, + "The project targets Python 3.13.", + "Commit messages follow Conventional Commits."), + ("unrelated", False, + "Run the linter with the --strict flag.", + "The team standup is at 09:30 UTC."), + ("unrelated", False, + "Release branches are cut every two weeks.", + "Log lines are emitted as JSON to stdout."), +] + + +def _contradiction(nli: CrossEncoder, premise: str, hypothesis: str) -> float: + logits = np.asarray(nli.predict([(premise, hypothesis)])[0], dtype=np.float64) + p = np.exp(logits - logits.max()) + p /= p.sum() + return float(p[0]) # label index 0 = contradiction, per the model card + + +def evaluate() -> dict[str, dict[str, int]]: + embedder = SentenceTransformer(EMBED_MODEL, device="cpu") + nli = CrossEncoder(NLI_MODEL, device="cpu") + + policies = ("cosine-only", "nli", "nli+literal") + stats = {p: {"tp": 0, "fn": 0, "tn": 0, "fp": 0} for p in policies} + rows = [] + + for category, want, stored, new in PAIRS: + va, vb = embedder.encode([stored, new], normalize_embeddings=True) + cos = float(np.dot(va, vb)) + contra = _contradiction(nli, stored, new) + + same_slot = cos >= CANDIDATE_SIM + la, lb = code_literals(stored), code_literals(new) + + fired = { + "cosine-only": cos >= SUPERSEDE_SIM, + "nli": same_slot and contra >= CONTRA_THRESHOLD, + } + fired["nli+literal"] = fired["nli"] or ( + same_slot and bool(la) and bool(lb) and la != lb + ) + + for policy in policies: + key = ("tp" if fired[policy] else "fn") if want else ("fp" if fired[policy] else "tn") + stats[policy][key] += 1 + + rows.append((category, want, cos, contra, fired)) + + return stats, rows + + +def main() -> None: + stats, rows = evaluate() + positives = sum(1 for _, want, *_ in rows if want) + negatives = len(rows) - positives + + print("=" * 92) + print(f"{'category':<14}{'want':<6}{'cos':>7}{'contra':>9} cosine-only nli nli+literal") + print("=" * 92) + for category, want, cos, contra, fired in rows: + print(f"{category:<14}{str(want):<6}{cos:>7.3f}{contra:>9.3f}" + f"{str(fired['cosine-only']):>14}{str(fired['nli']):>8}{str(fired['nli+literal']):>14}") + + print("=" * 92) + print(f"{'policy':<14}{'recall':>9}{'precision':>11}{'FP':>6}{'FN':>6} " + f"(positives {positives}, negatives {negatives})") + for policy, s in stats.items(): + recall = s["tp"] / positives if positives else 0.0 + denom = s["tp"] + s["fp"] + precision = s["tp"] / denom if denom else 1.0 + print(f"{policy:<14}{recall:>9.2f}{precision:>11.2f}{s['fp']:>6}{s['fn']:>6}") + + print() + print("A false positive forgets guidance the user approved. A false negative leaves one") + print("stale line. Weight the FP column accordingly.") + + +if __name__ == "__main__": + main() diff --git a/memory/sidecar.py b/memory/sidecar.py index cb1a13c..8eeffd6 100644 --- a/memory/sidecar.py +++ b/memory/sidecar.py @@ -14,7 +14,6 @@ """ from __future__ import annotations -import re import sqlite3 from collections import deque from dataclasses import dataclass @@ -30,22 +29,6 @@ EMBED_MODEL = "BAAI/bge-small-en-v1.5" DAY_S = 86_400.0 -# Code literals the NLI model cannot reason about: backtick spans, path-like tokens, flags. -_BACKTICK = re.compile(r"`([^`]+)`") -_PATHY = re.compile(r"(? frozenset[str]: - """Extract code-ish tokens from a memory record. - - Used only by the optional literal-diff gate (`use_literal_gate`). Returns an empty - set for ordinary prose, which is what makes the gate inert on natural-language - memory: no literals on either side means the gate can never fire. - """ - found = {m.strip() for m in _BACKTICK.findall(text)} - found |= {m.strip() for m in _PATHY.findall(text)} - return frozenset(f for f in found if f) - @lru_cache(maxsize=4) def _get_embedder(model: str, device: str) -> SentenceTransformer: @@ -87,7 +70,6 @@ def __init__( nli_model: str = "cross-encoder/nli-deberta-v3-xsmall", nli_contra_threshold: float = 0.5, nli_candidate_sim: float = 0.62, - use_literal_gate: bool = False, embedder: SentenceTransformer | None = None, adaptive_window: int = 15, adaptive_k: float = 0.5, @@ -109,7 +91,6 @@ def __init__( self.nli_model_name = nli_model self.nli_contra_threshold = nli_contra_threshold self.nli_candidate_sim = nli_candidate_sim # loose cosine pre-filter when NLI decides - self.use_literal_gate = use_literal_gate # opt-in third signal for technical memory self._nli = None # lazy NLI cross-encoder (loaded on first contradiction check) self.embedder = embedder or _get_embedder(embed_model, device) @@ -261,35 +242,13 @@ def _contradicted_records(self, emb: np.ndarray, text: str) -> list[ObservationR and unrelated facts (contra~0.01) are kept; only a real contradiction (>= nli_contra_threshold) counts. Shared by admission-override AND forgetting so NLI runs once per write. Called BEFORE insert, so the new record is not a candidate. HONEST LIMITATION: a same-slot swap with low - lexical overlap AND no logical conflict falls back to retrieval recency. - - A second limitation, measured 2026-09-08 on technical memory: a swap with HIGH lexical - overlap and no logical conflict also falls through, because the change lives in an opaque - code token rather than in sentence structure. `use_literal_gate=True` adds a token-free - third signal for that case; it is opt-in and inert on prose (see the gate comment below).""" + lexical overlap AND no logical conflict falls back to retrieval recency.""" hits = [] - new_lits = code_literals(text) if self.use_literal_gate else frozenset() for r in self._active_records(): if float(np.dot(emb, r.embedding)) < self.nli_candidate_sim: continue if self._nli_contradiction(r.text, text) >= self.nli_contra_threshold: hits.append(r) - elif new_lits: - # LITERAL-DIFF GATE (opt-in, technical memory). NLI finds contradiction in - # logical STRUCTURE - negation, antonyms, numeric conflict. A same-slot swap - # whose only change is an opaque code token has none of that, so NLI reads it - # as two compatible statements and the stale record survives forever. - # Measured on 16 coding-guidance pairs (0.62/0.50 defaults unchanged): - # cos 0.945, contra 0.013 `cargo test --workspace` -> `cargo nextest run --workspace` - # Same-slot (cosine already passed) + BOTH sides carrying literals + those - # literal sets differing is a supersession. Both-sided is load-bearing: a - # one-sided rule fires on elaborations that merely mention a path. The - # discriminating negative is a restatement at cos 0.963 whose literals are - # IDENTICAL - it correctly stays quiet, so this is not "high cosine means - # supersede". Zero tokens, zero model calls, same thesis as the other two. - old_lits = code_literals(r.text) - if old_lits and old_lits != new_lits: - hits.append(r) return hits def _decay(self, new: ObservationRecord, contradicted: list[ObservationRecord]) -> None: diff --git a/tests/test_sidecar.py b/tests/test_sidecar.py index ba50d33..2f55954 100644 --- a/tests/test_sidecar.py +++ b/tests/test_sidecar.py @@ -101,78 +101,3 @@ def test_admission_via_contradiction(): assert update is not None, "a contradicting knowledge-update must be admitted even when surprise is shut" assert {r.id: r for r in s._active_records()}[base.id].superseded_by == update.id, \ "the admitted update must also supersede the stale fact it corrects" - - -# --------------------------------------------------------------- literal-diff gate -# The NLI signal finds contradiction in logical structure: negation, antonyms, numeric -# conflict. A same-slot swap whose only change is an opaque code token has none of that, -# so NLI scores it ~0.01 and the stale record survives. Measured on 16 coding-guidance -# pairs at the shipped 0.62/0.50 defaults: 14/16 with 0/8 false positives, and BOTH -# misses were pure code-token swaps. The opt-in gate recovers one of them; the other -# (prose location -> path, literals on one side only) stays missed on purpose. - - -def test_code_literals_extracts_only_code_ish_tokens(): - from memory.sidecar import code_literals - - assert code_literals("The tests run with `cargo test --workspace`.") == frozenset( - {"cargo test --workspace", "--workspace"} - ) - assert code_literals("Docs live under docs/planning today.") == frozenset({"docs/planning"}) - # Ordinary prose has none, which is what keeps the gate inert on natural-language memory. - assert code_literals("I moved from Ames to Chicago.") == frozenset() - assert code_literals("My dog is named Rex.") == frozenset() - - -def test_literal_gate_is_off_by_default(): - s = MemorySidecar("lit-default", tau=0.0) - assert s.use_literal_gate is False, "the gate must stay opt-in; prose benchmarks assume it off" - - -def test_literal_gate_supersedes_a_command_swap_that_nli_misses(tmp_path): - """The motivating case: cos 0.945, NLI contradiction 0.013, stale record survives forever.""" - s = MemorySidecar("lit-on", db_path=str(tmp_path / "m.db"), tau=0.0, - use_nli=True, use_literal_gate=True) - try: - _ = s._nli_contradiction("a", "b") # force-load; skip if offline - except Exception as e: - pytest.skip(f"NLI model unavailable: {e}") - - old = s.write("The tests for this project run with `cargo test --workspace`.") - assert old is not None - new = s.write("The tests for this project run with `cargo nextest run --workspace`.") - assert new is not None, "a command swap is an update and must be admitted" - assert {r.id: r for r in s._active_records()}[old.id].superseded_by == new.id, "the new command must supersede the stale one" - - -def test_literal_gate_keeps_a_restatement_with_identical_literals(tmp_path): - """The discriminating negative: cos 0.963, but the literals match, so this is not an update. - - Without this the gate would degenerate into 'high cosine means supersede'. - """ - s = MemorySidecar("lit-dup", db_path=str(tmp_path / "m.db"), tau=0.0, - use_nli=True, use_literal_gate=True) - try: - _ = s._nli_contradiction("a", "b") - except Exception as e: - pytest.skip(f"NLI model unavailable: {e}") - - old = s.write("The tests run with `cargo test --workspace`.") - assert old is not None - s.write("Use `cargo test --workspace` to run the tests.") - assert {r.id: r for r in s._active_records()}[old.id].superseded_by is None, "a restatement carrying the same literals must not supersede anything" - - -def test_literal_gate_keeps_an_elaboration_that_mentions_one_literal(tmp_path): - """Both sides must carry literals. A one-sided rule would forget elaborations.""" - s = MemorySidecar("lit-elab", db_path=str(tmp_path / "m.db"), tau=0.0, - use_nli=True, use_literal_gate=True) - try: - _ = s._nli_contradiction("a", "b") - except Exception as e: - pytest.skip(f"NLI model unavailable: {e}") - - old = s.write("The tests for this project run with `cargo test --workspace`.") - assert old is not None - s.write("The test suite currently has 148 passing tests.") - assert {r.id: r for r in s._active_records()}[old.id].superseded_by is None, "an elaboration with no literal of its own must not supersede the original"