diff --git a/benchmarks/technical_memory.py b/benchmarks/technical_memory.py new file mode 100644 index 0000000..f37bc9c --- /dev/null +++ b/benchmarks/technical_memory.py @@ -0,0 +1,276 @@ +"""Technical-memory supersession benchmark: does the write gate work on CODING guidance? + +The drift suite measures personal-fact slot updates ("I moved to Chicago"). Agent memory is +increasingly used for a different content type: standing engineering guidance, where the +facts are build commands, paths, flags, versions and conventions. This benchmark measures +the write-and-forget decision on that domain. + +WHY IT IS A SEPARATE BENCHMARK. The NLI signal finds contradiction in logical STRUCTURE: +negation, antonyms, numeric conflict, entity swaps. A large share of real technical +supersessions have none of that. "The tests run with `pytest -q`" and "The tests run with +`pytest -n auto`" are not logically contradictory sentences; they are two compatible-looking +statements that happen to disagree about an opaque token. That shape is under-represented in +prose benchmarks and over-represented in engineering memory. + +THREE POLICIES, one embedder, one fixture, so any gap is the policy's doing: + + cosine-only supersede_sim 0.75, the coarse path used when use_nli is False + nli cosine 0.62 candidate gate + NLI contradiction >= 0.50 (the shipped default) + nli+literal the same, plus the opt-in use_literal_gate literal-diff signal + +The asymmetry that matters: a FALSE POSITIVE forgets guidance the user deliberately approved, +while a FALSE NEGATIVE leaves one stale line in the store. Precision is worth more than recall +here, so the table reports both rather than a single accuracy number. + +RESULT (36 pairs, 18 positive / 18 negative): + + policy recall precision FP FN + cosine-only 0.94 0.77 5 1 + nli 0.94 1.00 0 1 + nli+literal 0.94 1.00 0 1 + +Two things worth reading off that table. + +The shipped NLI path transfers to technical memory at PERFECT PRECISION. It never once +forgot approved guidance. All five cosine-only false positives are restatements ("Use +`pytest -q` to run the unit tests" against "The unit tests run with `pytest -q`", cosine +0.974), and NLI scores every one of them at contradiction <= 0.002. That is a clean, +domain-transferred demonstration of exactly what the contradiction signal buys over a +density gate. + +The literal-diff policy earns NOTHING here and is therefore NOT shipped in `memory/`. It was +designed on an earlier 16-pair probe where two command swaps scored contradiction 0.013, and +it recovered one of them. On this held-out set the NLI model fires correctly on every command +swap (contradiction 0.841 to 0.990), so the failure it targets does not generalise: it was a +property of two specific sentence pairs, not of code-token swaps as a class. The policy stays +in this file as a documented negative result, so nobody re-derives it. + +The single shared false negative is an antonym pair with no code literal at all ("generated +at build time" against "checked in and never generated", cosine 0.846, contradiction 0.021). +Cosine-only catches it; both NLI policies miss it. That is the real remaining gap. + +FIXTURE PROVENANCE, stated because it bounds every number above. 36 pairs, written in one +pass before any policy was run against them, and sharing no pair with the 16-pair probe that +motivated the literal gate. For that gate this is therefore a genuine held-out set. Still +self-authored by one person, so it measures transfer to one author's idea of technical +memory, not to a sampled population of real stores. + +Run: python -m benchmarks.technical_memory +""" + +from __future__ import annotations + +import re + +import numpy as np +from sentence_transformers import CrossEncoder, SentenceTransformer + +from memory.sidecar import EMBED_MODEL + +# Code literals an NLI model cannot reason about: backtick spans, path-like tokens, flags. +# Local to this benchmark on purpose. The literal-diff policy below is a CANDIDATE that this +# fixture rejects, so nothing in `memory/` depends on it. +_BACKTICK = re.compile(r"`([^`]+)`") +_PATHY = re.compile(r"(? frozenset[str]: + found = {m.strip() for m in _BACKTICK.findall(text)} + found |= {m.strip() for m in _PATHY.findall(text)} + return frozenset(f for f in found if f) + +NLI_MODEL = "cross-encoder/nli-deberta-v3-xsmall" +CANDIDATE_SIM = 0.62 # MemorySidecar.nli_candidate_sim default +CONTRA_THRESHOLD = 0.50 # MemorySidecar.nli_contra_threshold default +SUPERSEDE_SIM = 0.75 # MemorySidecar.supersede_sim default (cosine-only path) + +# (category, should_supersede, stored_record, new_observation) +PAIRS: list[tuple[str, bool, str, str]] = [ + # ---- supersession: the change lives in an opaque code token (the hard class) ---- + ("swap-command", True, + "The unit tests run with `pytest -q`.", + "The unit tests run with `pytest -n auto`."), + ("swap-command", True, + "Build the release with `make release`.", + "Build the release with `just release`."), + ("swap-command", True, + "Format the codebase using `black .`.", + "Format the codebase using `ruff format .`."), + ("swap-command", True, + "Start the dev server with `npm run dev`.", + "Start the dev server with `pnpm dev`."), + ("swap-path", True, + "Configuration lives in config/settings.yaml.", + "Configuration lives in etc/app/settings.yaml."), + ("swap-path", True, + "Migrations are stored under db/migrate.", + "Migrations are stored under alembic/versions."), + ("swap-flag", True, + "Run the linter with the --strict flag.", + "Run the linter with the --pedantic flag."), + ("swap-version", True, + "The project targets Python 3.11.", + "The project targets Python 3.13."), + ("swap-port", True, + "The API listens on port 8080.", + "The API listens on port 9090."), + + # ---- supersession: carries logical structure NLI can see ---- + ("negation", True, + "The deploy script pushes tags automatically.", + "The deploy script does not push tags automatically."), + ("negation", True, + "Secrets may be committed to the private repo.", + "Secrets must never be committed to any repo."), + ("antonym", True, + "The staging database is writable by developers.", + "The staging database is read-only for developers."), + ("antonym", True, + "Test fixtures are generated at build time.", + "Test fixtures are checked in and never generated."), + ("entity", True, + "The CI provider for this project is Travis.", + "The CI provider for this project is GitHub Actions."), + ("entity", True, + "The team's package manager is Yarn.", + "The team's package manager is pnpm."), + ("numeric", True, + "Pull requests need two approving reviews.", + "Pull requests need one approving review."), + ("numeric", True, + "The request timeout is 30 seconds.", + "The request timeout is 5 seconds."), + ("policy", True, + "Release branches are cut monthly.", + "Release branches are cut every two weeks."), + + # ---- elaboration: same subject, compatible, must NOT supersede ---- + ("elaboration", False, + "The unit tests run with `pytest -q`.", + "The suite takes about four minutes on CI."), + ("elaboration", False, + "Configuration lives in config/settings.yaml.", + "Every key in that file has a documented default."), + ("elaboration", False, + "The API listens on port 8080.", + "Health checks are served from the same process."), + ("elaboration", False, + "The CI provider for this project is GitHub Actions.", + "The workflow runs on ubuntu-latest and macos-latest."), + ("elaboration", False, + "Secrets must never be committed to any repo.", + "The team uses a managed secret store for deploy credentials."), + ("elaboration", False, + "The project targets Python 3.13.", + "Type hints are checked in strict mode."), + ("elaboration", False, + "Build the release with `just release`.", + "The release artifact is a single static binary."), + + # ---- restatement: same fact reworded, must NOT supersede ---- + ("restatement", False, + "The unit tests run with `pytest -q`.", + "Use `pytest -q` to run the unit tests."), + ("restatement", False, + "Run the linter with the --strict flag.", + "The linter should be invoked using --strict."), + ("restatement", False, + "Pull requests need two approving reviews.", + "Two approving reviews are required on a pull request."), + ("restatement", False, + "Migrations are stored under db/migrate.", + "You will find the migrations in db/migrate."), + ("restatement", False, + "The deploy script pushes tags automatically.", + "Tags are pushed automatically by the deploy script."), + + # ---- unrelated: different subject, must NOT supersede ---- + ("unrelated", False, + "The unit tests run with `pytest -q`.", + "The changelog follows Keep a Changelog format."), + ("unrelated", False, + "Configuration lives in config/settings.yaml.", + "Code review turnaround is expected within one working day."), + ("unrelated", False, + "The API listens on port 8080.", + "Documentation is built with `mkdocs build`."), + ("unrelated", False, + "The project targets Python 3.13.", + "Commit messages follow Conventional Commits."), + ("unrelated", False, + "Run the linter with the --strict flag.", + "The team standup is at 09:30 UTC."), + ("unrelated", False, + "Release branches are cut every two weeks.", + "Log lines are emitted as JSON to stdout."), +] + + +def _contradiction(nli: CrossEncoder, premise: str, hypothesis: str) -> float: + logits = np.asarray(nli.predict([(premise, hypothesis)])[0], dtype=np.float64) + p = np.exp(logits - logits.max()) + p /= p.sum() + return float(p[0]) # label index 0 = contradiction, per the model card + + +def evaluate() -> dict[str, dict[str, int]]: + embedder = SentenceTransformer(EMBED_MODEL, device="cpu") + nli = CrossEncoder(NLI_MODEL, device="cpu") + + policies = ("cosine-only", "nli", "nli+literal") + stats = {p: {"tp": 0, "fn": 0, "tn": 0, "fp": 0} for p in policies} + rows = [] + + for category, want, stored, new in PAIRS: + va, vb = embedder.encode([stored, new], normalize_embeddings=True) + cos = float(np.dot(va, vb)) + contra = _contradiction(nli, stored, new) + + same_slot = cos >= CANDIDATE_SIM + la, lb = code_literals(stored), code_literals(new) + + fired = { + "cosine-only": cos >= SUPERSEDE_SIM, + "nli": same_slot and contra >= CONTRA_THRESHOLD, + } + fired["nli+literal"] = fired["nli"] or ( + same_slot and bool(la) and bool(lb) and la != lb + ) + + for policy in policies: + key = ("tp" if fired[policy] else "fn") if want else ("fp" if fired[policy] else "tn") + stats[policy][key] += 1 + + rows.append((category, want, cos, contra, fired)) + + return stats, rows + + +def main() -> None: + stats, rows = evaluate() + positives = sum(1 for _, want, *_ in rows if want) + negatives = len(rows) - positives + + print("=" * 92) + print(f"{'category':<14}{'want':<6}{'cos':>7}{'contra':>9} cosine-only nli nli+literal") + print("=" * 92) + for category, want, cos, contra, fired in rows: + print(f"{category:<14}{str(want):<6}{cos:>7.3f}{contra:>9.3f}" + f"{str(fired['cosine-only']):>14}{str(fired['nli']):>8}{str(fired['nli+literal']):>14}") + + print("=" * 92) + print(f"{'policy':<14}{'recall':>9}{'precision':>11}{'FP':>6}{'FN':>6} " + f"(positives {positives}, negatives {negatives})") + for policy, s in stats.items(): + recall = s["tp"] / positives if positives else 0.0 + denom = s["tp"] + s["fp"] + precision = s["tp"] / denom if denom else 1.0 + print(f"{policy:<14}{recall:>9.2f}{precision:>11.2f}{s['fp']:>6}{s['fn']:>6}") + + print() + print("A false positive forgets guidance the user approved. A false negative leaves one") + print("stale line. Weight the FP column accordingly.") + + +if __name__ == "__main__": + main()