diff --git a/.gitignore b/.gitignore index a3c72310..3b3253a6 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,9 @@ id_rsa id_ed25519 # Local design references Link Console Handoff.html + +# Local editor/agent tooling +.claude/ + +# vhs demo render workspace +docs/media/.aha-demo/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecc61e9..bf997b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,50 @@ Release sections use `MAJOR.MINOR.PATCH` versions that match `link-mcp` on PyPI ## [Unreleased] +## [1.6.0] - 2026-07-09 + +- Added an animated "aha" demo to the Getting Started page: a self-contained SVG (`docs/assets/link-aha.svg`, plain text, no external runtime, animates in any browser) showing the two moments Link is built for — recall that matches by meaning rather than keywords, and memory injected into a new agent session automatically. The README shows the matching recorded GIF (`docs/assets/link-aha.gif`), rendered from real `lnk` commands via a checked-in charmbracelet vhs tape (`docs/media/link-aha.tape`) so it is reproducible, not synthetic. + +- Fixed first-ten-minutes friction found by walking Link cold as a brand-new user: + - `lnk onboard` now surfaces the automatic-memory path: it explains `--hooks` and prints the ready-to-run `--agent --hooks --write` command, and each hook-capable agent preview offers "Make memory automatic (recommended)". Previously the flagship 1.6 feature was invisible in the guided setup. + - A recall that finds nothing while memories exist now tells the user paraphrase matching (semantic recall) is off by default and how to turn it on, instead of a bare "No matching memories found". The README's paraphrase example is reframed as opt-in so it never reads like a broken default, and the landing hero calls hybrid recall optional. + - Generated commands in source-checkout mode use a friendly `python3 link.py` instead of the raw interpreter path (e.g. `python@3.14`); Homebrew users still see plain `lnk`. + - `lnk proof` now says its workspace is a throwaway demo and points to `lnk onboard` for real memory, with a plain "what this means for you" line. + - `scripts/prepare_release.py` reminds maintainers to bump the Homebrew tap so `brew install` never serves an older Link than the docs describe. + +- Completed 1.6 coverage across the second-tier docs and shipped skills: the official CLI skills now teach the hooks-installed rule, the consolidation pass, semantic match labels, and the `lnk semantic` status check; the memory contract documents the hooked loop and honest recall signals; concepts covers hybrid retrieval and the automatic lifecycle; troubleshooting gains "hooks not firing" and "semantic recall not working" sections; and the scale page links the measured benchmarks. + +### Added + +- Added `lnk connect --hooks` to install agent session hooks alongside MCP config for Claude Code, Codex, and Cursor: every new session starts with a bounded Link memory brief injected automatically, and session end stores proposal-only session notes with memory candidates, so the memory loop no longer depends on the agent remembering to call Link. Codex has no session-end hook event, so it gets the session-start brief only; Cursor uses its flat `hooks.json` schema and JSON `additional_context` envelope. +- Added `lnk consolidate` and MCP `review(action="consolidate")` for a read-only backlog plan: pending capture counts, memories needing review, duplicate-capture groups, and paste-safe accept/discard/review commands — nothing is merged, deleted, or saved without the user approving each action. +- Added an automatic backlog nudge to the injected session-start brief: when pending captures or review items cross a threshold, the brief tells the agent to offer the user a short consolidation pass instead of letting the inbox silently grow. +- Added session-end capture noise controls: sessions with no memory-worthy proposal candidates are skipped entirely, and duplicate end events for the same conversation content are deduplicated with a local fingerprint, so automatic hooks cannot flood the capture inbox. +- Added optional hybrid semantic recall (`pip install "link-mcp[semantic]"` + `lnk semantic --setup`): a small local static-embedding model retrieves close paraphrases that token matching misses, across CLI recall, memory briefs, MCP recall, and smart query packets. Lexical recall stays the default and the fallback. +- Kept the local-first guarantee for semantic recall: the model loads offline-only so a query can never trigger a download (only the explicit `--setup` may fetch the model once), embeddings live in plain JSON under `.link-cache/`, similarity is computed in-process with no vector database or service, and `LINK_SEMANTIC=off` disables the layer. +- Added standout-based semantic scoring: candidates are selected by how much they stand out from the rest of the corpus for the query (not by raw cosine thresholds, which are not comparable across queries for static models), and semantic-only matches never outrank exact lexical hits. +- Added honest labeling for semantic recall: recalled memories now carry `match` (`lexical`, `semantic`, or `hybrid`) and `semantic_similarity`, and a match with no lexical evidence is capped at moderate confidence so agents verify paraphrase matches before acting on them. +- Added `lnk semantic` for the layer's status (provider, model, index state, mode) with explicit setup/rebuild actions and next-step guidance. +- Added a publication-grade recall benchmark: `scripts/recall_dataset.py` (62-memory corpus with distractors, 294 authored queries plus deterministic phrasing variants for 1,176 total cases, every query auto-classified by measured token overlap so the paraphrase group provably shares no significant stemmed token with its target) and `scripts/eval_recall_quality.py` (hit@1/3/5, MRR@5, per-domain breakdown, recall latency, JSON output, and a regression gate that fails if hybrid ever scores below lexical). CI runs the gate with a deterministic no-model embedder. +- Published measured results in `benchmarks/RESULTS.md` with methodology, hardware, model-size ablation, honest limitations, and reproduction steps: hybrid recall lifts token-overlap hit@1 0.589 → 0.703 and doubles-to-triples zero-overlap paraphrase hit@3/hit@5, at ~2.8 ms per recall in-process. +- Added `python3 -m link_mcp --semantic-setup` so MCP-only installs (no `lnk` CLI) can run the explicit one-time semantic model fetch and index build; the MCP server itself still never touches the network. +- Added a second semantic tier: `pip install "link-mcp[semantic-quality]"` uses a local contextual ONNX model (all-MiniLM-L6-v2 via fastembed) and is preferred automatically when installed; the static-model fast tier remains for instant-load CLI and hook use, and `LINK_SEMANTIC_PROVIDER` picks explicitly. On the bundled benchmark the quality tier roughly quadruples pure-paraphrase hit@3/hit@5 over lexical recall. Ablations that did not survive measurement (retrieval-tuned static models, multi-view embeddings) are documented in `benchmarks/RESULTS.md`. +- Added a third-party benchmark track: `scripts/eval_locomo.py` scores Link recall on the LoCoMo long-term conversational memory dataset (turns as memories, evidence-annotated questions as queries; retrieval stage only, no LLM anywhere). Hybrid recall lifts any-evidence hit@10 from 0.578 to 0.685 and evidence recall@10 from 0.517 to 0.608 over 1,536 third-party queries. The dataset (CC BY-NC 4.0, Snap Inc.) is downloaded by the user, never redistributed; the script contains no network code. +- Rewrote the public "Why Link?" positioning around the four architectural commitments competitors cannot bolt on — readable Markdown memory, review-gated writes, no LLM in the memory layer, CI-enforced zero network — with named comparisons against Mem0/OpenMemory, Zep/Graphiti, and Letta, and the benchmark as supporting evidence. +- Added `lnk onboard --hooks` so the guided first-run path can install session hooks alongside MCP wiring, and made `connect`/`onboard --hooks --write` refresh workspace runtimes that predate session hooks (preview warns first), preventing broken hooks after upgrades. +- Made the memory-backlog consolidation nudge part of the core brief payload so CLI `start`, MCP briefs, skills, and session hooks all surface it consistently. +- Improved `lnk semantic` diagnostics: status names the Python interpreter being checked, and when the Link MCP Python differs, errors print the exact venv-side setup command; quality-tier setup states the ~5s short-lived-CLI load tradeoff explicitly. +- Made the injected session-start brief compact for empty workspaces (two actionable lines instead of an empty statistics skeleton) and gave every missing-wiki CLI error a concrete next step instead of a dead end. +- Titled automatic session captures with their project, clustered near-duplicate captures in consolidation plans by token overlap instead of exact text, and documented session hooks, semantic recall, and consolidation across the PyPI README, LINK.md, installed agent instructions, MCP instructions resource, and the docs site. +- Added `lnk hook session-start` to print the bounded session-start memory brief (readiness, relevant memories with confidence, pending review and capture state, and retrieval guidance) for agent hook runtimes; it scopes the brief to the hook's working directory project and never fails the agent session. +- Added `lnk hook session-end` to turn an agent transcript into review-gated memory: it extracts bounded user/assistant text (skipping tool calls and outputs), skips trivial sessions, and stores proposal-only session notes through the same duplicate/conflict-safe capture path as `lnk session-end`. +- Added idempotent, non-destructive session-hook writing to `~/.claude/settings.json` that preserves existing user hooks and settings, replaces only Link's own hook entries on rerun, and skips re-injection on session resume. + +### Fixed + +- `python -m link_mcp --help` now prints usage and the MCP config snippet instead of silently starting the stdio server (which hung in a terminal with no output). The parser still ignores unknown arguments so an agent launch config can never crash the server. +- Automatic session-end capture now mines memory proposals from the user's own turns only, not the assistant's replies. Dogfooding showed the assistant's prose (e.g. a summary line like "you prefer small commits") was being extracted and proposed as the user's own preference. The raw capture still keeps the full transcript for review context; only the proposal candidates are restricted to what the user actually said (`extract_transcript_text(..., roles=("user",))`). + ## [1.5.0] - 2026-07-03 ### Added diff --git a/LINK.md b/LINK.md index d3d1b584..d91360e6 100644 --- a/LINK.md +++ b/LINK.md @@ -623,3 +623,9 @@ If the wiki is empty, start here: If the wiki already exists, read `wiki/index.md` and `wiki/log.md` first to understand current state before doing anything. To verify MCP access, run `python3 link.py verify-mcp .` when `link.py` is available. It checks whether `link_mcp` imports in the configured Python and prints the MCP client config for the current wiki. + +## Memory Maintenance + +- **Session hooks.** Agents with hook support (Claude Code, Codex, Cursor) can install Link session hooks (`python3 link.py connect . --hooks --write`): the memory brief is injected automatically at session start and proposal-only session notes are stored at session end. Durable memory always requires review. +- **Consolidation.** When briefs report a memory backlog (pending captures or reviews above threshold), run `python3 link.py consolidate .` (or MCP `review(action="consolidate")`) for a read-only plan with accept/discard/review commands. Apply actions only after the user approves each one. +- **Semantic recall (optional, local).** With `link-mcp[semantic]` or `link-mcp[semantic-quality]` installed and a one-time `python3 link.py semantic . --setup`, recall also finds paraphrases. Recalled memories then carry `match: lexical|semantic|hybrid`; treat semantic-only matches as hints to verify, not facts. diff --git a/README.md b/README.md index a965dd4d..6d40eadb 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,11 @@ The wiki is the storage layer. The product is durable memory that stays on your machine, remains readable in plain files, and can be shared across multiple agents instead of locked inside one vendor profile. +

+ lnk recall finds a memory saved in completely different words — matched by meaning, not keywords +

+

Ask in your own words; Link matches by meaning, not keywords. All local, all plain files.

+ ## How It Works Link gives agents four simple moves: @@ -73,6 +78,29 @@ Link follows Andrej Karpathy's keep knowledge outside the chat window, make claims inspectable, and let context compound over time. +## Why Link Is Different + +Every other agent-memory system stores memory as embeddings in a vector +database or as an LLM-extracted graph. Link made four architectural +commitments those designs cannot bolt on: + +1. **Memory you can read.** Every memory is a plain Markdown file — open it, + grep it, git-diff it. If Link disappeared tomorrow, your memory is still yours. +2. **Review-gated writes.** Agents propose; you approve. Even the automatic + session hooks capture proposals, never facts. +3. **No LLM in the memory layer.** Ingestion and recall are deterministic — + nothing can hallucinate a fact into your memory, because there is no model + in the write path. +4. **Provably local.** CI blocks outbound network code in the runtime, and the + optional semantic models load offline-only after one explicit setup. + +And the claims are measured, not asserted: a reproducible 1,176-case recall +benchmark plus a third-party LoCoMo retrieval track, with published miss rates +and a CI gate against regressions — +[benchmarks/RESULTS.md](benchmarks/RESULTS.md). Named comparisons against +Mem0/OpenMemory, Zep/Graphiti, and Letta: +[Why Link?](https://gowtham0992.github.io/link/why-link.html) + ## Quick Start Start with the memory proof. It creates a clean local workspace, writes one @@ -320,6 +348,53 @@ lnk connect kiro ~/link --write lnk verify-mcp ~/link ``` +For agents with session-hook support — Claude Code, Codex, and Cursor — add +`--hooks` (works with `lnk onboard` too) to make the memory loop automatic: +the brief is injected at session start and proposal-only notes are captured at +session end, so memory no longer depends on the agent remembering to call +Link. Empty sessions and duplicate end events are skipped, and when the +backlog builds up the brief nudges the agent to offer a read-only +`lnk consolidate` pass. Durable memory still requires your approval. Codex and +Cursor hook support is new (wired to their documented schemas — report +issues). + +```bash +lnk connect claude-code ~/link --hooks --write +lnk connect codex ~/link --hooks --write # session-start brief (Codex has no session-end event) +lnk connect cursor ~/link --hooks --write +lnk consolidate ~/link # read-only backlog plan, apply only with approval +``` + +### Optional: hybrid semantic recall (still fully local) + +Lexical recall is always the default and the fallback. Paraphrase matching is +opt-in: after the two setup commands below, "how should I structure my pull +requests" finds a memory saved about commit style. Until then, recall matches +on shared words, and a miss tells you how to turn paraphrase matching on. +Installing the optional semantic extra adds a small local static-embedding +model. Recall never touches the +network: the model loads offline-only after a one-time explicit setup, +embeddings live in plain JSON under `.link-cache/`, similarity runs in-process +with no vector database, and semantic-only matches carry capped confidence +labels so agents verify before trusting them. + +```bash +pip install "link-mcp[semantic]" # fast tier: tiny static model, instant load +pip install "link-mcp[semantic-quality]" # quality tier: contextual model, best recall +lnk semantic ~/link --setup # one-time model fetch, with your approval +lnk semantic ~/link # status: lexical only vs hybrid, active tier +python3 -m link_mcp --semantic-setup --wiki ~/link/wiki # MCP-only installs +``` + +Measured, not asserted: on the bundled 1,176-case benchmark, the quality +tier lifts token-overlap hit@1 from 0.589 to 0.749 and pure-paraphrase +(zero token overlap) hit@3/hit@5 by ~4×, at ~10 ms per recall with no +service or vector database. On the third-party LoCoMo retrieval track +(1,536 evidence-annotated questions over 5,882 conversation turns), hybrid +recall lifts any-evidence hit@10 from 0.578 to 0.685. Full methodology, +honest limitations, and reproduction steps: +[benchmarks/RESULTS.md](benchmarks/RESULTS.md). +
MCP-only install @@ -400,15 +475,18 @@ model-facing tools. CLI and skill workflows call the same core behavior through next actions. - `recall`: the one read path for startup briefs, answer-ready query packets, wiki search, graph context, token budgets, and follow-up actions. Every - recalled memory carries a `confidence` label (`strong`, `moderate`, `weak`), - so agents verify weak lexical matches with the user instead of trusting them. + recalled memory carries a `confidence` label (`strong`, `moderate`, `weak`) + and a `match` field (`lexical`, `semantic`, `hybrid` when the optional local + semantic tier is installed), so agents verify weak or paraphrase matches with + the user instead of trusting them. - `remember`: durable local memory only after explicit user approval, with duplicate/conflict checks, provenance, review state, visibility, optional `review_after`, and optional `expires_at`. - `ingest`: exact next steps for raw files, source safety, stale ingest detection, validation, and rebuild checks. - `review`: memory inbox, profile, audit, log, explain, archive, restore, - forget, and lifecycle review workflows. + forget, and lifecycle review workflows — plus `review(action="consolidate")`, + a read-only backlog plan applied only with per-action user approval. - `admin`: the escape hatch for backup, migrate, validate, graph export, pages, captures, rebuilds, compatibility actions, and advanced updates. @@ -508,6 +586,11 @@ Link itself is local-first: checks. `lnk validate` and `lnk doctor` also fail if secret-looking values are found inside wiki pages before they can be served through the local UI or returned through agent context. +- Optional semantic recall stays local: models load offline-only at recall + time (only the explicit `lnk semantic --setup` may fetch a model, once), and + embeddings live in plain JSON under `.link-cache/`. +- Automatic session hooks store proposal-only notes; transcript extraction + skips tool calls and outputs, and no durable memory is written without review. - The local web server binds to `127.0.0.1` and is not meant to be exposed to the internet without additional auth. diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 00000000..e700337e --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,139 @@ +# Link recall quality benchmark + +Link's recall is measured, not asserted. This document holds the current +numbers, exactly how they were produced, and how to reproduce them on your +own machine. There are two tracks: + +1. **Link recall benchmark** — our own deterministic, fully auditable + dataset (checked into this repo; no LLM, no network, no randomness). +2. **LoCoMo third-party track** — the retrieval stage of the long-term + conversational memory benchmark the hosted-memory industry quotes + (Maharana et al., ACL 2024, Snap Research), using only its third-party + questions and evidence annotations. + +## Semantic tiers + +Lexical recall is always the default and the fallback (zero dependencies). +Two optional local semantic tiers upgrade it — both load offline-only at +recall time, keep embeddings in plain JSON under `.link-cache/`, and use no +vector database or service: + +| tier | install | model | load time | best for | +|---|---|---|---|---| +| fast | `pip install "link-mcp[semantic]"` | model2vec potion-base-8M (~30 MB) | ~0.1 s | CLI, session-start hooks | +| quality | `pip install "link-mcp[semantic-quality]"` | all-MiniLM-L6-v2 ONNX (~90 MB) | ~5 s | MCP server, long-lived agents | + +The quality tier is preferred automatically when installed +(`LINK_SEMANTIC_PROVIDER` overrides). + +## Track 1: Link recall benchmark + +Dataset (`scripts/recall_dataset.py`): 62 memories across six domains +including 20 distractors; 1,176 cases (294 authored queries + deterministic +phrasing variants). Queries are grouped by *measured* overlap: a case counts +as `zero-overlap` only if it provably shares no significant stemmed token +with its target memory — pure paraphrases that token matching cannot reach. + +Full suite, Apple M4, macOS 26.5.1, Python 3.14, run 2026-07-08, Link +`develop` (post-1.5.0). + +### Token-overlap queries (800 cases) + +| metric | lexical | fast tier | quality tier | +|---|---|---|---| +| hit@1 | 0.589 | 0.703 | **0.749** | +| hit@3 | 0.729 | 0.833 | **0.886** | +| hit@5 | 0.815 | 0.880 | **0.926** | +| MRR@5 | 0.668 | 0.769 | **0.818** | + +### Zero-overlap queries — pure paraphrases (376 cases) + +| metric | lexical | fast tier | quality tier | +|---|---|---|---| +| hit@1 | 0.048 | 0.074 | **0.120** (2.5×) | +| hit@3 | 0.064 | 0.136 | **0.255** (4.0×) | +| hit@5 | 0.082 | 0.202 | **0.338** (4.1×) | +| MRR@5 | 0.058 | 0.115 | **0.191** (3.3×) | + +### Latency (per recall, 62-memory corpus, model load excluded) + +| mode | p50 | mean | +|---|---|---| +| lexical | 1.3 ms | 1.3 ms | +| fast tier | 2.8 ms | 2.8 ms | +| quality tier | 9.3 ms | 10.0 ms | + +### Ablations we ran and rejected + +- **potion-retrieval-32M** (retrieval-tuned static model) and **multi-view + embeddings** (title/tldr/body embedded separately, max-similarity): both + improved token-overlap slightly but did not move zero-overlap paraphrases. + The zero-overlap ceiling is the static-embedding paradigm itself, which is + why the quality tier uses a contextual model instead of a bigger static one. +- **potion-base-32M**: marginal over 8M; not worth 4× the size as a default. +- **Token-level late interaction (MaxSim over static token vectors)**: worse + than blob embeddings on both groups (zero-overlap hit@5 0.160 vs 0.202) — + static per-token vectors are too noisy for ColBERT-style matching. +- **Corpus-mined PMI query expansion** (learning the user's vocabulary from + their own wiki): cannot help zero-overlap queries by construction (there is + no shared token to expand from) and slightly hurt token-overlap hit@1 by + pulling in competing memories. Rejected. + +## Track 2: LoCoMo third-party retrieval + +Every dialog turn of a LoCoMo conversation becomes one memory record; every +evidence-annotated question (adversarial category excluded) becomes a recall +query; we measure whether Link ranks the annotated evidence turns highly. +10 conversations, 5,882 turn-memories (~590 per conversation), 1,536 +third-party queries. No LLM anywhere: this isolates the retrieval stage with +third-party queries and third-party gold labels. + +| metric | lexical | hybrid (quality tier) | +|---|---|---| +| any-evidence hit@1 | 0.290 | **0.309** | +| any-evidence hit@5 | 0.496 | **0.540** | +| any-evidence hit@10 | 0.578 | **0.685** | +| evidence recall@10 | 0.517 | **0.608** | +| latency p50 / mean | 16 ms | 45 ms / 61 ms | + +**Not comparable to published LoCoMo QA scores** (mem0, Zep, etc. report +end-to-end LLM answer quality with server-side pipelines). This track scores +deterministic local ranking only — no answer generation, no LLM judging, no +network. The dataset is CC BY-NC 4.0 © Snap Inc. and is not redistributed +here; the script prints the download command. + +## Honest limitations + +- **Pure paraphrases are much better, not solved.** The quality tier + quadruples zero-overlap hit@3/hit@5 over lexical, yet roughly two thirds + of pure paraphrases still miss the top 5 on our corpus. Link labels every + semantic-only match (`match: semantic`, capped confidence) so agents + verify before trusting — we publish the miss rate rather than hiding it. +- **Track 1 is self-authored.** It is deterministic, auditable, and gated + against regressions in CI, but it was written by the Link project. + Track 2 exists precisely to complement it with third-party data; + adversarial case contributions to Track 1 are welcome (five lines per + intent in `scripts/recall_dataset.py`). +- **The quality tier costs a ~5 s model load**, so short-lived CLI calls + and session-start hooks default to the fast tier unless you opt in. + +## Reproduce + +```bash +git clone https://github.com/gowtham0992/link && cd link + +# Track 1 (lexical baseline needs nothing): +python3 scripts/eval_recall_quality.py --suite full --mode off +python3 -m venv /tmp/linkbench +/tmp/linkbench/bin/pip install model2vec # fast tier +/tmp/linkbench/bin/pip install fastembed # quality tier (preferred when present) +/tmp/linkbench/bin/python scripts/eval_recall_quality.py --suite full --mode real --allow-download + +# Track 2 (download the dataset yourself; CC BY-NC 4.0 © Snap Inc.): +curl -L -o /tmp/locomo10.json https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json +python3 scripts/eval_locomo.py /tmp/locomo10.json --mode off +/tmp/linkbench/bin/python scripts/eval_locomo.py /tmp/locomo10.json --mode real +``` + +`--mode fake` runs a deterministic no-model embedder; CI uses it with a +regression gate: hybrid may never score below lexical on any group metric. diff --git a/docs/assets/link-aha.gif b/docs/assets/link-aha.gif new file mode 100644 index 00000000..d14e3d44 Binary files /dev/null and b/docs/assets/link-aha.gif differ diff --git a/docs/assets/link-aha.svg b/docs/assets/link-aha.svg new file mode 100644 index 00000000..83b73855 --- /dev/null +++ b/docs/assets/link-aha.svg @@ -0,0 +1,89 @@ + + + + + + + + lnk · local agent memory + $ lnk remember "feat/short-topic branch names" + ✓ saved to local memory + $ lnk recall "how should I name my git branches" + ✓ feat/short-topic branch names + → matched by meaning, not keywords + — you open a new agent session — + Link memory · injected automatically + • prefers feat/short-topic branch names + • keep PR descriptions short + no one asked. it just remembered. + + diff --git a/docs/cli.html b/docs/cli.html index cd6cf06e..eb1579df 100644 --- a/docs/cli.html +++ b/docs/cli.html @@ -129,7 +129,11 @@

Maintenance

lnk connect codex ~/link
 lnk connect codex ~/link --write
 lnk connect kiro ~/link --write
+lnk connect claude-code ~/link --hooks --write
 lnk verify-mcp ~/link
+

Add --hooks (Claude Code, Codex, Cursor) to also install session hooks: every new session then starts with a bounded Link memory brief injected automatically, and session end stores proposal-only notes with memory candidates for later review — no durable memory is written without approval. Codex has no session-end event, so it gets the session-start brief only. Sessions without memory-worthy content are skipped and duplicate end events are deduplicated, so the capture inbox does not fill with noise. The hooks run lnk hook session-start and lnk hook session-end, which you can also invoke directly to inspect what they inject or capture. Codex and Cursor hook support is new and follows those vendors' documented hook schemas — if a hook misbehaves there, please open an issue. If the workspace runtime predates session hooks, --write refreshes it automatically.

+

Use lnk consolidate when the capture or review backlog builds up. It is read-only: it counts pending captures and memories needing review, groups duplicate captures, and prints paste-safe accept/discard/review commands to run with the user. When the backlog crosses a threshold, the injected session-start brief nudges the agent to offer a consolidation pass, and MCP agents can request the same plan through review(action="consolidate").

+

Use lnk semantic to inspect or enable optional hybrid recall. Lexical recall is always the default and the fallback; installing pip install "link-mcp[semantic]" and running lnk semantic --setup once (or python3 -m link_mcp --semantic-setup for MCP-only installs) adds a small local static-embedding model so paraphrased queries also find memories phrased differently. Recall itself never touches the network — the model loads offline-only, embeddings live in plain JSON under .link-cache/, and semantic-only matches are labeled with capped confidence so agents verify before trusting them. Measured results and methodology live in benchmarks/RESULTS.md.

From a source checkout, use the synthetic large-wiki smoke when you want local scale evidence without touching your real wiki. The script prints the exact lnk serve command and graph URL for the generated fixture.

python3 scripts/smoke_large_wiki.py --pages 10000
@@ -138,7 +142,7 @@

All Commands

lnk version lnk init [dir] lnk serve [dir] [--port 3000] -lnk onboard [dir] [--agent codex] [--write] [--first-memory "..."] [--seed-project .] +lnk onboard [dir] [--agent codex] [--write] [--hooks] [--first-memory "..."] [--seed-project .] lnk seed [project-dir] [dir] [--project-name name] [--overwrite] [--dry-run] lnk try [dir] [--force] [--serve] [--port 3000] lnk proof [dir] [--force] [--serve] [--port 3000] @@ -169,6 +173,8 @@

All Commands

lnk start [dir] [--task "task"] [--project slug] lnk brief "task" [--project slug] lnk memory-audit [--project slug] +lnk consolidate [dir] [--project slug] [--limit N] +lnk semantic [dir] [--setup] [--rebuild] lnk recall "query" [--project slug] lnk profile [--project slug] lnk wins [--project slug] @@ -188,7 +194,8 @@

All Commands

lnk rebuild-index lnk rebuild-backlinks lnk verify-mcp [--json] -lnk connect <agent> [dir] [--write] [--config path] [--python python] +lnk connect <agent> [dir] [--write] [--config path] [--python python] [--hooks] +lnk hook session-start|session-end [dir] [--limit N] [--project slug] lnk backup [--list] [--include-raw] lnk restore-backup <backup.tar.gz> [--include-raw] --confirm python3 link.py demo diff --git a/docs/concepts.html b/docs/concepts.html index 98002034..dd1a66fc 100644 --- a/docs/concepts.html +++ b/docs/concepts.html @@ -87,6 +87,7 @@

Three User Moves

Raw files do not silently personalize future agents. Ingest creates source-backed wiki knowledge. Explicit remember creates durable user or project memory.

Memory Lifecycle

+

The lifecycle can run automatically: session hooks inject the memory brief at session start and store proposal-only notes at session end, and when the review backlog grows, briefs nudge the agent to offer a read-only lnk consolidate pass. Every write still requires explicit user approval — automation changes when memory is proposed, never who decides.

A memory is a Markdown page with status, scope, visibility, source, review state, optional review_after and expires_at dates, graph links, and local log entries. It can be proposed, remembered, reviewed, updated, archived, restored, explained, or forgotten.

Propose

Generate candidate memories from chat notes or raw captures without writing durable memory.

@@ -97,6 +98,7 @@

Memory Lifecycle

Smart Query Packets

+

Retrieval is lexical by default (token matching, stemming, SQLite FTS) and optionally hybrid: two local embedding tiers (a tiny static model for instant CLI use, a contextual model for the MCP server) add paraphrase recall while staying fully offline at query time. Semantic matches are labeled and confidence-capped so agents verify before trusting them; measured results live in benchmarks/RESULTS.md.

recall is designed for agents. It returns a compact packet with a recall_capsule, relevant memory, ranked wiki pages, graph context, provenance, budget reports, estimated size, and follow-up actions.

Budget tiers keep context predictable:

    diff --git a/docs/getting-started.html b/docs/getting-started.html index e685f0ff..8bea1ecf 100644 --- a/docs/getting-started.html +++ b/docs/getting-started.html @@ -55,8 +55,14 @@

    Prove that your agent can remember.

    Save one memory Ask the agent to ingest Verify the loop + Make it automatic + Semantic recall
    +
    + Animated demo: lnk recall finds a memory phrased in completely different words, and a new agent session is greeted with your memory automatically. +
    The two moments Link is built for: recall that matches by meaning, and memory that shows up on its own.
    +

    1. Prove The Memory Loop

    Start with lnk proof. It creates a clean local workspace, writes one reviewed memory, then recalls it through the same bounded path used by CLI, skills, and MCP. This proves the core product before you configure an agent or open the web viewer.

    macOS with Homebrew:

    @@ -104,7 +110,8 @@

    2. Onboard A Real Workspace

    lnk onboard --first-memory "I prefer concise release notes" lnk onboard --seed-project . lnk onboard --agent codex -lnk onboard --agent codex --write +lnk onboard --agent codex --write +lnk onboard --agent claude-code --hooks --write

    From source, use python3 link.py onboard on macOS/Linux or py link.py onboard on Windows. The command is safe to re-run: it preserves existing wiki data and only applies safe structural repairs. Add --seed-project . from inside a repo when you want onboarding to create the first source-backed project page. If the local viewer is running, http://127.0.0.1:3000/onboard shows the same setup loop with copy buttons.

    3. Seed Project Context

    @@ -190,6 +197,20 @@

    8. Verify The Loop

    lnk verify-mcp should report Result: ready when you use MCP. Then ask your agent:

    query Link for first Link memory

    If the answer comes from Link, local agent memory is working.

    +

    9. Make The Loop Automatic (Session Hooks)

    +

    Agents with session-hook support — Claude Code, Codex, and Cursor — can run the memory loop without being asked. --hooks installs hooks that inject a bounded memory brief at the start of every new session and store proposal-only session notes at session end. Empty sessions are skipped, duplicate end events are deduplicated, and durable memory still requires your approval. When the review backlog grows, the injected brief nudges the agent to offer a read-only lnk consolidate pass.

    + 
    lnk connect claude-code ~/link --hooks --write
+lnk connect codex ~/link --hooks --write    # session-start brief (Codex has no session-end event)
+lnk connect cursor ~/link --hooks --write
    +

    Codex and Cursor hook support is new and follows those vendors' documented hook schemas; if a hook misbehaves there, please open an issue.

    + +

    10. Optional: Hybrid Semantic Recall

    +

    Lexical recall is always the default and the fallback. Two optional local tiers add paraphrase recall — "how should I structure my pull requests" finds a memory about commit style. The models load offline-only at recall time (a query can never trigger a download), embeddings are plain JSON under .link-cache/, and there is no vector database or service.

    + 
    pip install "link-mcp[semantic]"          # fast tier: tiny static model, instant load
+pip install "link-mcp[semantic-quality]"  # quality tier: contextual model, best recall
+lnk semantic ~/link --setup               # explicit one-time model fetch
+python3 -m link_mcp --semantic-setup --wiki ~/link/wiki   # MCP-only installs
    +

    Recall quality is measured, not asserted: see benchmarks/RESULTS.md for the full methodology, numbers, and honest limitations.

    diff --git a/docs/index.html b/docs/index.html index ee4a8ca0..24944ba4 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,7 +3,7 @@ Link — Local memory for AI agents - + @@ -184,7 +184,7 @@ diff --git a/docs/mcp.html b/docs/mcp.html index f8d78ca8..fcb279ed 100644 --- a/docs/mcp.html +++ b/docs/mcp.html @@ -103,7 +103,11 @@

    Agent Installers

    MCP Only

    python3 -m pip install --upgrade link-mcp
-python3 -m link_mcp --version
    +python3 -m link_mcp --version +# optional local semantic recall (fast or quality tier): +python3 -m pip install "link-mcp[semantic-quality]" +python3 -m link_mcp --semantic-setup --wiki ~/link/wiki +

    The semantic extras stay fully local: the embedding model is fetched once by the explicit --semantic-setup command, and the serving path loads it offline-only — a recall can never download anything.

    {
   "mcpServers": {
     "link": {
@@ -128,6 +132,7 @@ 
    MCP Only
    
 }

    Predictable Agent Workflow

    +

    If Link session hooks are installed for the agent (lnk connect <agent> --hooks --write), the startup brief is injected automatically — agents should skip the manual brief call and go straight to bounded task recall. When a brief reports a memory backlog, review(action="consolidate") returns a read-only consolidation plan to walk through with the user.

    New MCP configs use the slim surface by default so agents see one obvious read tool and one obvious write tool instead of a long menu of overlapping helpers. The full compatibility surface remains available with --surface full.

    Slim agents should use Link in this order:

      diff --git a/docs/media/README.md b/docs/media/README.md new file mode 100644 index 00000000..17f9c5a9 --- /dev/null +++ b/docs/media/README.md @@ -0,0 +1,13 @@ +# Docs media + +- `link-aha.svg` (in `../assets/`) — the animated "aha" demo used on the + Getting Started page. Self-contained SVG (plain text, no external runtime), + animates in any modern browser. Regenerate by editing the generator snippet + in the 1.6 changelog history or hand-editing the SVG. +- `link-aha.tape` — charmbracelet [vhs](https://github.com/charmbracelet/vhs) + script that renders the README GIF (`../assets/link-aha.gif`) from real `lnk` + commands. See the header of the tape for the one-time render command; the GIF + is not checked in until rendered. + +Other GIFs/screenshots under `../assets/` are real product captures, verified +(not generated) by `scripts/generate_docs_media.py`. diff --git a/docs/media/link-aha.tape b/docs/media/link-aha.tape new file mode 100644 index 00000000..ade6fbff --- /dev/null +++ b/docs/media/link-aha.tape @@ -0,0 +1,44 @@ +# Link "aha" demo — the paraphrase-recall moment, for the GitHub README. +# +# Renders a crisp, deterministic GIF (no synthetic frames) with charmbracelet vhs. +# +# One-time render (needs ffmpeg + ttyd, pulled in by `brew install vhs`): +# brew install vhs +# pip install model2vec # the fast local semantic tier +# cd docs/media && vhs link-aha.tape +# Then wire it into README.md: +# Link recall finds a memory phrased in different words +# and add "link-aha.gif" to REQUIRED_ASSETS in scripts/generate_docs_media.py. + +Output ../assets/link-aha.gif + +Require lnk + +Set Shell bash +Set FontSize 20 +Set Width 1280 +Set Height 500 +Set Padding 44 +Set Theme { "background": "#221c12", "foreground": "#f3ece0", "cursor": "#e0955f", "black": "#221c12", "green": "#86c79a", "brightBlack": "#8a8174", "white": "#f3ece0", "blue": "#e0955f", "brightBlue": "#e0955f", "cyan": "#d9b48c", "brightCyan": "#d9b48c" } + +# ── prep the demo workspace off-screen; make it the working directory ── +Hide +Type "O=$PWD; L=$O/.aha-demo; rm -rf $L; mkdir -p $L; cd $L" Enter +Type "lnk init . >/dev/null 2>&1" Enter +Type "lnk remember 'Name branches feat/short-topic, never long' . --type preference >/dev/null 2>&1" Enter +Type "lnk semantic . --setup >/dev/null 2>&1" Enter +Type "clear" Enter +Show + +# ── the moment: ask in totally different words, it finds it ── +Sleep 900ms +Type@85ms "lnk recall 'how should I name my git branches'" +Sleep 600ms +Enter +Sleep 3s +Type@85ms "# different words, same memory - matched by meaning, not keywords" +Sleep 2400ms + +# ── cleanup off-screen ── +Hide +Type "cd $O; rm -rf $L" Enter diff --git a/docs/memory-contract.html b/docs/memory-contract.html index 2e5b1ddb..27f2fc93 100644 --- a/docs/memory-contract.html +++ b/docs/memory-contract.html @@ -65,6 +65,7 @@

      Contract Promise

      Recommended Agent Loop

      +

      With session hooks installed (lnk connect <agent> --hooks --write), step one happens automatically: the bounded memory brief is injected at session start and proposal-only notes are captured at session end. Hooked agents skip the manual brief call and go straight to bounded task recall.

      1. Call status. If schema or validation needs attention, follow the safe action it returns.
      2. Call recall with an empty query once at the first substantive turn of a session.
        3. @@ -95,6 +96,8 @@

        Core Tool Groups

        +

        Recalled memories carry honest signals agents must respect: a confidence label (strong/moderate/weak) and, when the optional local semantic tier is installed, a match field (lexical/hybrid/semantic). Semantic-only matches are capped below strong confidence — verify them with the user before acting. When a brief reports a memory backlog, review(action="consolidate") (or lnk consolidate) returns a read-only plan; apply its actions only with per-item user approval.

        +

        Write Rules

        Agents should treat Link memory as durable state, not scratch space.

          diff --git a/docs/scale.html b/docs/scale.html index 4251f71b..91a8aefe 100644 --- a/docs/scale.html +++ b/docs/scale.html @@ -75,6 +75,7 @@

          Bounded Surfaces

          Measure Locally

          +

          Recall quality is measured too, not just speed: the repo ships a 1,176-case recall benchmark and a third-party LoCoMo retrieval track with reproduction commands — see benchmarks/RESULTS.md.

          Use lnk benchmark on your real wiki. It reports cache time, persistent-cache reuse, search backend, search/query timing, graph payload shape, value evidence, and recommendations. The value section compares broad wiki body text with the bounded query packet so you can see whether Link is reducing context-budget waste.

          lnk benchmark "agent memory"
 lnk health
diff --git a/docs/security.html b/docs/security.html
index ac3c56bc..d3c74cae 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -63,6 +63,8 @@ 
          Privacy Model
          
           
        • No external API calls from serve.py or link-mcp.
          • 
           
        • Raw sources and generated wiki pages are ignored by git by default.
          • 
           
        • SQLite search, when available, is an in-memory derived index. Markdown remains the source of truth.
          • 
+          
        • Optional semantic recall stays local: embedding models load offline-only at recall time (only the explicit lnk semantic --setup may fetch a model, once), embeddings live in plain JSON under .link-cache/, and CI blocks outbound network code in the runtime.
          • 
+          
        • Automatic session hooks store proposal-only notes; no durable memory is written without explicit review, and transcript extraction skips tool calls and tool outputs.
          •

        The public GitHub Pages documentation may use lightweight analytics to understand install interest. It does not run inside Link, read local wiki data, or capture source/memory content.

        diff --git a/docs/skills.html b/docs/skills.html index ddbd80cb..f15433ac 100644 --- a/docs/skills.html +++ b/docs/skills.html @@ -91,6 +91,7 @@

        Use Them

        remember that I prefer short release notes

        Rules For Agents

        +

        Two additions with 1.6: if Link session hooks are installed, the startup brief arrives automatically — skip the manual brief and go straight to bounded task recall. And when a brief reports a memory backlog, offer the user a read-only lnk consolidate pass instead of letting captures pile up.

        • Prefer lnk health when readiness is unclear, especially after an install, upgrade, restore, or broad wiki edit.
        • Start with lnk start for readiness plus memory context, and use lnk session-end to capture proposal-only memory candidates at the end of meaningful work.
          • diff --git a/docs/troubleshooting.html b/docs/troubleshooting.html index a04149f7..97dd15f6 100644 --- a/docs/troubleshooting.html +++ b/docs/troubleshooting.html @@ -53,6 +53,8 @@

          Start with status, then repair deliberately.

          Interrupted writes Graph is stale Demo looks stale + Hooks not firing + Semantic recall The wiki feels slow pip is blocked @@ -94,6 +96,13 @@

          Demo Looks Stale

          python3 link.py query "why does Link help agents?" link-demo --budget small

          The current generated demo should include three raw sources, source-backed wiki pages, four starter memories (three reviewed, one pending review), one exploration, current backlinks, and schema v1.

          +

          Session Hooks Are Not Firing

          +

          Check the agent's settings file (for Claude Code, ~/.claude/settings.json) for the Link entries under SessionStart/SessionEnd; rerunning lnk connect <agent> --hooks --write is idempotent and repairs a workspace runtime that predates hooks. If sessions start with the brief but nothing is captured at session end, that is usually correct behavior — trivial sessions, duplicates, and restatements of existing memory are skipped by design. See exactly why with:

          + 
          lnk hook session-end ~/link --explain
          + +

          Semantic Recall Is Not Working

          +

          lnk semantic ~/link shows the active provider and tier. Common causes: the extra was installed into a different Python than the one running Link (the status output names the interpreter it checked, and the error prints the exact venv-side command such as python3 -m link_mcp --semantic-setup); the one-time model fetch has not run (lnk semantic ~/link --setup); or LINK_SEMANTIC=off is set. Recall itself never downloads anything — a missing model degrades silently to lexical recall by design.

          +

          The Wiki Feels Slow

          lnk benchmark "agent memory"
 lnk graph-summary "agent memory" --limit 40 --depth 1
          diff --git a/docs/why-link.html b/docs/why-link.html index 625a6ac7..8583b219 100644 --- a/docs/why-link.html +++ b/docs/why-link.html @@ -47,6 +47,7 @@

          Link is not a notes app. It is local memory for agents.

          +

          The Architecture Is the Product

          +

          Every other agent-memory system — Mem0/OpenMemory, Zep/Graphiti, Letta — stores memory as embeddings in a vector database or as an LLM-extracted graph. You cannot read your own memory, and a model sits inside the write path. Link made four architectural commitments that cannot be bolted onto those designs:

          +
          +

          Memory you can read

          Every memory is a plain Markdown file. Open it, grep it, git-diff it, back it up. If Link disappeared tomorrow, your memory is still yours.

          +

          Review-gated writes

          No durable memory is created without your approval. Agents propose; you decide. Automatic session hooks capture proposals — never facts.

          +

          No LLM in the memory layer

          Ingestion and recall are deterministic. An extraction model can hallucinate facts into a knowledge graph; Link's memory layer cannot, because there is no model in the write path.

          +

          Provably local

          CI blocks outbound network code in the runtime. The optional semantic models load offline-only after one explicit setup. "Local-first" here is enforced, not promised.

          +
          +

          Recall quality is measured, not asserted: a 1,176-case reproducible benchmark plus a third-party LoCoMo retrieval track, with published miss rates and a CI gate against regressions. See benchmarks/RESULTS.md.

          +

          Best Fit

          Use Link when you want one local memory layer that multiple agents can share. It is strongest for developer and power-user workflows where privacy, provenance, and inspectable files matter.

          @@ -89,11 +100,11 @@

          Compared With Alternatives

          AlternativeWhere Link winsWhere they win
          ObsidianAgent-ready memory lifecycle, MCP/CLI retrieval, validation, source-backed query packets.Human-first note editing, mobile sync, plugins, and a mature visual graph.
          -
          Mem0No hosted account, local Markdown storage, cross-agent desktop use, inspectable provenance.Managed cloud APIs, hosted dashboards, and team/app integration primitives.
          +
          Mem0 / OpenMemoryReadable Markdown storage instead of a vector database, review-gated writes instead of silent extraction, no LLM in the memory layer, hooks-guaranteed session loop, published benchmark with miss rates.Managed cloud APIs, hosted dashboards, larger community, and team/app integration primitives.
          LettaWorks beside existing agents instead of becoming the agent runtime; simpler local file model.Full stateful-agent runtime, managed execution loop, and hosted deployment options.
          -
          GraphitiPersonal/project memory with reviewable Markdown and simple local operations.Temporal knowledge graphs, automatic extraction, and enterprise graph use cases.
          +
          Zep / GraphitiDeterministic memory with no LLM extraction cost or hallucination risk, reviewable Markdown, millisecond local recall, review/expiry lifecycle for time-sensitive memory.Bi-temporal knowledge graphs, automatic entity extraction, multi-user business data, and enterprise graph use cases.
          Built-in agent memoryOne memory layer shared across Codex, Claude, Cursor, Kiro, VS Code, Antigravity, and local agents.Zero setup inside one vendor's product.
          -
          Plain RAG or vector searchReviewable memory, source files, graph context, lifecycle controls, and bounded agent packets.Semantic retrieval quality, embedding connectors, and application-specific pipelines.
          +
          Plain RAG or vector searchReviewable memory, source files, graph context, lifecycle controls, bounded agent packets — plus optional local hybrid semantic recall with measured quality and honest confidence labels.Cloud-scale embedding models, connectors, and application-specific pipelines over huge corpora.

          Trust Model

          diff --git a/integrations/_shared/link-instructions-project.md b/integrations/_shared/link-instructions-project.md index 4bf258a4..0995a48d 100644 --- a/integrations/_shared/link-instructions-project.md +++ b/integrations/_shared/link-instructions-project.md @@ -33,6 +33,9 @@ After ingesting raw sources or making substantial wiki edits, use MCP `ingest` a When the user explicitly asks Link to remember something, use MCP `remember` when available. For uncertain or long-session memory, use MCP `admin` action `propose_memories` or `capture_session` first, then MCP `review` to inspect/approve. Use MCP `review` for memory inbox, profile, audit, log, explain, archive, restore, and forget workflows. Use MCP `admin` only for less-common maintenance and compatibility actions. +If a memory brief reports a memory backlog (pending captures or reviews above threshold), offer the user a short consolidation pass: use MCP `review` with action `consolidate` when available, or run `python3 link.py consolidate`. The plan is read-only; apply its accept/discard/review commands only after the user approves each action. + +If Link session hooks are installed for this agent, the session-start memory brief is injected automatically — do not run a second startup recall; go straight to bounded task recall. Recalled memories carry a `match` field: treat `semantic` matches (paraphrase similarity with capped confidence) as hints to verify with the user, not facts to act on. When the user says **"remember"**, **"recall"**, **"ingest"**, **"query"**, **"lint"**, or **"research"**, read `LINK.md` for instructions and follow the protocol. diff --git a/integrations/_shared/link-instructions.md b/integrations/_shared/link-instructions.md index 341a5765..9f2781d9 100644 --- a/integrations/_shared/link-instructions.md +++ b/integrations/_shared/link-instructions.md @@ -27,6 +27,9 @@ When the user explicitly asks Link to remember something, use MCP `remember` whe At the end of a meaningful work session, propose memory instead of silently saving it. Use MCP `admin` action `session_end` with concise session notes when available, or run `lnk session-end `. Show the returned proposals to the user and save durable memory only after approval. Use MCP `review` for memory inbox, profile, audit, log, explain, archive, restore, and forget workflows. Use MCP `admin` only for less-common maintenance and compatibility actions. +If a memory brief reports a memory backlog (pending captures or reviews above threshold), offer the user a short consolidation pass: use MCP `review` with action `consolidate` when available, or run `lnk consolidate`. The plan is read-only; apply its accept/discard/review commands only after the user approves each action. + +If Link session hooks are installed for this agent, the session-start memory brief is injected automatically — do not run a second startup recall; go straight to bounded task recall. Recalled memories carry a `match` field: treat `semantic` matches (paraphrase similarity with capped confidence) as hints to verify with the user, not facts to act on. When the user says **"remember"**, **"recall"**, **"ingest"**, **"query"**, **"lint"**, or **"research"**, read `~/link/LINK.md` for instructions and follow the protocol. Use terminal commands to access `~/link/` since it's outside the workspace. diff --git a/link.py b/link.py index 592b2847..c6d88f29 100644 --- a/link.py +++ b/link.py @@ -52,6 +52,7 @@ """ from __future__ import annotations +import hashlib import json import os import shutil @@ -255,12 +256,31 @@ check_link_mcp_import as _core_check_link_mcp_import, display_command as _core_display_command, render_mcp_verify_text as _core_render_mcp_verify_text, + resolve_mcp_python as _core_resolve_mcp_python, set_link_command_override as _core_set_link_command_override, ) from link_core.mcp_connect import ( build_mcp_connect_payload as _core_build_mcp_connect_payload, supported_agents as _core_supported_agents, ) +from link_core.agent_hooks import ( + build_agent_hooks_payload as _core_build_agent_hooks_payload, + extract_transcript_text as _core_extract_transcript_text, + hook_supported_agents as _core_hook_supported_agents, + supports_agent_hooks as _core_supports_agent_hooks, +) +from link_core.consolidate import ( + build_consolidation_plan as _core_build_consolidation_plan, + render_consolidate_text as _core_render_consolidate_text, +) +from link_core.semantic import ( + build_semantic_status as _core_build_semantic_status, + load_embedder as _core_load_semantic_embedder, + refresh_memory_index as _core_refresh_semantic_index, + render_semantic_status_text as _core_render_semantic_status_text, + semantic_memory_scores as _core_semantic_memory_scores, + semantic_provider as _core_semantic_provider, +) from link_core.obsidian import ( import_obsidian_vault as _core_import_obsidian_vault, render_import_obsidian_text as _core_render_import_obsidian_text, @@ -289,9 +309,11 @@ render_query_text as _core_render_query_text, ) from link_core.cli_runtime import ( + render_agent_hooks_text as _core_render_agent_hooks_text, render_demo_text as _core_render_demo_text, render_init_text as _core_render_init_text, render_mcp_connect_text as _core_render_mcp_connect_text, + render_session_start_hook_text as _core_render_session_start_hook_text, render_onboard_text as _core_render_onboard_text, render_proof_text as _core_render_proof_text, render_start_text as _core_render_start_text, @@ -341,6 +363,18 @@ def _wiki_pages(wiki_dir: Path) -> list[Path]: ) +def _missing_wiki_error(wiki_dir: Path) -> int: + """Explain a missing wiki with a next step instead of a dead end.""" + print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) + print( + "Point Link at your workspace (for example: " + f"{_display_command(['lnk', 'status', str(Path.home() / 'link')])}) " + f"or create one here with {_display_command(['lnk', 'init', '.'])}.", + file=sys.stderr, + ) + return 1 + + def _resolve_wiki_dir(target: Path) -> Path: target = target.expanduser().resolve() if target.name == "wiki" and (target / "index.md").exists(): @@ -440,13 +474,15 @@ def _memory_profile(wiki_dir: Path, limit: int = 10, project: str | None = None) def _memory_brief(wiki_dir: Path, query: str = "", limit: int = 6, project: str | None = None) -> dict[str, object]: + records = _memory_records(wiki_dir) return _core_memory_brief( - _memory_records(wiki_dir), + records, query=query, limit=limit, review_command="review-memory", project=project, command_target=wiki_dir.parent, + semantic_scores=_core_semantic_memory_scores(wiki_dir.parent, query, records), ) @@ -473,12 +509,14 @@ def _recall_memories( include_archived: bool = False, project: str | None = None, ) -> list[dict[str, object]]: + records = _memory_records(wiki_dir) return _core_recall_memories( - _memory_records(wiki_dir), + records, query, limit=limit, include_archived=include_archived, project=project, + semantic_scores=_core_semantic_memory_scores(wiki_dir.parent, query, records), ) @@ -933,8 +971,7 @@ def team_sync(target: Path, remote: str | None = None, json_output: bool = False def share(target: Path, identifier: str, port: int = 3000, host: str = "127.0.0.1", json_output: bool = False) -> int: wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) payload = _core_share_page_payload(wiki_dir, identifier, host=host, port=port) return _emit_json_or_text(payload, json_output, _core_render_share_text, json_code=0 if payload.get("found") else 1) @@ -1007,8 +1044,7 @@ def import_obsidian( def rebuild_backlinks(target: Path) -> int: wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) try: backlinks = _build_backlinks(wiki_dir) except OSError as exc: @@ -1030,8 +1066,7 @@ def rebuild_backlinks(target: Path) -> int: def rebuild_index(target: Path) -> int: wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) try: result = _core_rebuild_index(wiki_dir) except OSError as exc: @@ -1116,8 +1151,7 @@ def propose_memories( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) text, source = _read_proposal_input(target, source_input) if not text.strip(): print("Memory proposal input is required", file=sys.stderr) @@ -1152,8 +1186,7 @@ def capture_session( root = _resolve_link_root(target) wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) text, source = _read_proposal_input(root, source_input) if not text.strip(): @@ -1215,14 +1248,14 @@ def session_end( title: str | None = None, limit: int = 3, project: str | None = None, + proposal_text: str | None = None, json_output: bool = False, ) -> int: target = target.expanduser().resolve() root = _resolve_link_root(target) wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) text, source = _read_proposal_input(root, source_input) if not text.strip(): @@ -1240,9 +1273,12 @@ def session_end( path_source=True, ) rel_path = str(capture_record["path"]) + # The raw capture keeps the full session for review context, but memory + # proposals are mined from proposal_text when given (the user's turns only) + # so the assistant's prose is never proposed as the user's preference. result = _propose_memories_from_text( wiki_dir, - text, + proposal_text if proposal_text is not None else text, source=rel_path, limit=max(1, min(limit, 10)), project=project_name, @@ -1298,8 +1334,7 @@ def capture_inbox( root = _resolve_link_root(target) wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) payload = _core_capture_inbox( root, limit=limit, @@ -1346,8 +1381,7 @@ def accept_capture( root = _resolve_link_root(target) wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) try: selection = _core_capture_proposal_selection( root, @@ -1431,8 +1465,7 @@ def redact_capture( root = _resolve_link_root(target) wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) try: payload = _core_redact_capture_file( root, @@ -1473,8 +1506,7 @@ def delete_capture( root = _resolve_link_root(target) wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) try: payload = _core_delete_capture_file(root, capture, confirm=confirm) except ValueError: @@ -1566,8 +1598,7 @@ def recall( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) project_name = project or _default_project(target) results = _recall_memories( wiki_dir, @@ -1587,12 +1618,32 @@ def recall( }, indent=2)) return 0 + miss_hint = "" + if not results: + records = _memory_records(wiki_dir) + if records: + if _core_semantic_provider() is None: + miss_hint = ( + "These memories exist but your words did not match any. Paraphrase matching " + "(semantic recall) is off by default. Turn it on to find memories phrased " + "differently:\n" + " pip install \"link-mcp[semantic]\"\n" + f" {_display_command(['link', 'semantic', str(target), '--setup'])}" + ) + elif _core_load_semantic_embedder() is None: + miss_hint = ( + "These memories exist but your words did not match any. Finish enabling " + "paraphrase matching (semantic recall):\n" + f" {_display_command(['link', 'semantic', str(target), '--setup'])}" + ) + code, text = _core_render_recall_text( query=query, results=results, include_archived=include_archived, project=project_name, target=target, + miss_hint=miss_hint, ) _print_text(text) return code @@ -1630,8 +1681,7 @@ def forget_memory(target: Path, identifier: str, confirm: bool = False, json_out target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) def rebuild_memory_backlinks() -> bool: backlinks = _build_backlinks(wiki_dir) @@ -1675,8 +1725,7 @@ def memory_inbox( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) inbox = _memory_inbox(wiki_dir, limit=limit, include_archived=include_archived, project=project) return _emit_json_or_text( @@ -1694,8 +1743,7 @@ def memory_log(target: Path, limit: int = 50, include_captures: bool = True, jso target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) payload = _core_memory_log_payload(wiki_dir, limit=limit, include_captures=include_captures) return _emit_json_or_text( payload, @@ -1708,8 +1756,7 @@ def memory_wins(target: Path, limit: int = 6, project: str | None = None, json_o target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) payload = _core_memory_wins_payload(wiki_dir, limit=limit, project=project) return _emit_json_or_text( payload, @@ -1732,8 +1779,7 @@ def explain_memory(target: Path, identifier: str, json_output: bool = False) -> target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) try: explanation = _memory_explanation(wiki_dir, identifier) except ValueError as exc: @@ -1759,8 +1805,7 @@ def query( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) query_text = _clean_text_input(query_text, max_len=500) project_name = project or _default_project(target) payload = _query_link(wiki_dir, query_text, budget=budget, project=project_name) @@ -1783,8 +1828,7 @@ def graph_summary( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) topic = _clean_text_input(topic, max_len=500) cache = _core_build_wiki_cache(wiki_dir) payload = _core_graph_summary( @@ -1814,8 +1858,7 @@ def benchmark( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) query_text = _clean_text_input(query_text, max_len=500) project_name = project or _default_project(target) payload = _core_build_benchmark_payload( @@ -1844,8 +1887,7 @@ def brief( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) query = _clean_text_input(query, max_len=500) project_name = project or _default_project(target) payload = _memory_brief(wiki_dir, query=query, limit=limit, project=project_name) @@ -1873,8 +1915,7 @@ def start( target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) task = _clean_text_input(task, max_len=500) project_name = project or _default_project(target) status_payload = _core_link_status(wiki_dir, version=LINK_VERSION, include_validation=True) @@ -1882,6 +1923,7 @@ def start( brief_payload = _core_add_capture_review_to_brief( brief_payload, _capture_review_summary(target, project=project_name), + command_target=_resolve_link_root(target), ) query_text = task or "your current task" relevant_count = int(brief_payload.get("relevant_count") or len(brief_payload.get("relevant_memories") or [])) @@ -1945,12 +1987,262 @@ def start( return code +def _read_hook_stdin() -> dict[str, object]: + """Read the agent hook event JSON from stdin, if one was piped in.""" + if sys.stdin is None or sys.stdin.isatty(): + return {} + try: + raw = sys.stdin.read() + except OSError: + return {} + if not raw.strip(): + return {} + try: + payload = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return {} + return payload if isinstance(payload, dict) else {} + + +def semantic(target: Path, setup: bool = False, rebuild: bool = False, json_output: bool = False) -> int: + """Show, set up, or rebuild the optional local semantic recall layer.""" + target = target.expanduser().resolve() + root = _resolve_link_root(target) + wiki_dir = _resolve_wiki_dir(target) + if not wiki_dir.exists(): + return _missing_wiki_error(wiki_dir) + records = _memory_records(wiki_dir) + active_count = len([ + record for record in records + if str(record.get("status") or "active").lower() == "active" + ]) + action_error = "" + action_result = "" + if setup or rebuild: + if setup and not json_output: + _print_text( + "Setting up semantic recall: this may download the local embedding model once. " + "Recall itself never uses the network." + ) + embedder = _core_load_semantic_embedder(allow_download=setup) + if embedder is None: + install_hint = f'{sys.executable} -m pip install "link-mcp[semantic]"' + action_error = ( + f"Semantic provider unavailable for {sys.executable}. Install it first: {install_hint}" + if setup + else "Semantic model not available offline. Run: lnk semantic --setup" + ) + mcp_python = _core_resolve_mcp_python(target, wiki_dir, None, default_python=sys.executable) + if mcp_python != sys.executable: + action_error += ( + f"\nYour Link MCP Python is {mcp_python}. If you installed the extra there, " + f"set it up through the MCP runtime instead: " + f"{mcp_python} -m link_mcp --semantic-setup --wiki {wiki_dir}" + ) + else: + index = _core_refresh_semantic_index(root, records, embedder=embedder) + items = index.get("items") if isinstance(index.get("items"), dict) else {} + action_result = f"Indexed {len(items)} memories." + if setup and _core_semantic_provider() == "fastembed": + action_result += ( + " Quality tier active: expect a ~5s model load per short-lived CLI command; " + "the MCP server loads it once and stays fast. Prefer instant CLI recall? " + "Set LINK_SEMANTIC_PROVIDER=model2vec (fast tier)." + ) + payload = _core_build_semantic_status( + root, memory_count=active_count, command_target=root, python_cmd=sys.executable + ) + if action_result: + payload["action_result"] = action_result + if action_error: + payload["action_error"] = action_error + if json_output: + print(json.dumps(payload, indent=2)) + return 1 if action_error else 0 + code, text = _core_render_semantic_status_text(payload) + if action_result: + _print_text(action_result) + if action_error: + print(action_error, file=sys.stderr) + code = 1 + _print_text(text) + return code + + +def consolidate(target: Path, limit: int = 50, project: str | None = None, json_output: bool = False) -> int: + """Print a read-only consolidation plan for capture and review backlogs.""" + target = target.expanduser().resolve() + root = _resolve_link_root(target) + wiki_dir = _resolve_wiki_dir(target) + if not wiki_dir.exists(): + return _missing_wiki_error(wiki_dir) + captures_payload = _core_capture_inbox( + root, + limit=max(1, min(limit, 50)), + project=project, + commands_for=lambda rel_path: _core_cli_capture_commands(rel_path, root), + ) + inbox_payload = _memory_inbox(wiki_dir, limit=max(1, min(limit, 50)), project=project) + payload = _core_build_consolidation_plan( + captures_payload=captures_payload, + inbox_payload=inbox_payload, + command_target=root, + project=project, + ) + return _emit_json_or_text(payload, json_output, _core_render_consolidate_text) + + +def _hook_project_dir(hook_event: dict[str, object]) -> str: + """Return the project directory the hook fired in, across agent schemas.""" + hook_cwd = str(hook_event.get("cwd") or "").strip() + if hook_cwd: + return hook_cwd + roots = hook_event.get("workspace_roots") + if isinstance(roots, list) and roots and isinstance(roots[0], str) and roots[0].strip(): + return roots[0].strip() + return "" + + +def _emit_session_start(text: str, emit: str) -> None: + if emit == "cursor": + print(json.dumps({"additional_context": text})) + return + print(text) + + +def _hook_session_start( + target: Path, hook_event: dict[str, object], limit: int, project: str | None, emit: str +) -> int: + wiki_dir = _resolve_wiki_dir(target) + if not wiki_dir.exists(): + _emit_session_start( + f"Link: wiki missing at {wiki_dir}; run {_display_command(['lnk', 'init', str(target)])} to restore it.", + emit, + ) + return 0 + project_name = project + if not project_name: + project_dir = _hook_project_dir(hook_event) + if project_dir: + project_name = _default_project(Path(project_dir)) + if not project_name: + project_name = _default_project(target) + status_payload = _core_link_status(wiki_dir, version=LINK_VERSION, include_validation=False) + brief_payload = _memory_brief(wiki_dir, query="", limit=limit, project=project_name) + brief_payload = _core_add_capture_review_to_brief( + brief_payload, + _capture_review_summary(target, project=project_name), + command_target=_resolve_link_root(target), + ) + relevant_count = int(brief_payload.get("relevant_count") or len(brief_payload.get("relevant_memories") or [])) + project_seed_recommended = bool(status_payload.get("ready")) and not relevant_count and not int( + status_payload.get("content_page_count") or 0 + ) + _, brief_text = _core_render_brief_text(brief_payload, query="", project=project_name) + captures_payload = brief_payload.get("captures") if isinstance(brief_payload.get("captures"), dict) else {} + _, text = _core_render_session_start_hook_text({ + "target": str(target), + "project": project_name, + "status": status_payload, + "brief_text": brief_text, + "capture_count": int(captures_payload.get("count") or 0), + "project_seed_recommended": project_seed_recommended, + "backlog": brief_payload.get("backlog") or {}, + }) + _emit_session_start(text, emit) + return 0 + + +def _session_end_hook_state_path(target: Path) -> Path: + return _resolve_link_root(target) / ".link-cache" / "session-end-hook.hash" + + +def _session_notes_fingerprint(notes: str) -> str: + normalized = " ".join(notes.split()).lower() + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def _hook_session_end(target: Path, hook_event: dict[str, object], limit: int, project: str | None) -> int: + transcript_value = str(hook_event.get("transcript_path") or "").strip() + if not transcript_value: + return 0 + transcript_path = Path(transcript_value).expanduser() + notes = _core_extract_transcript_text(transcript_path) + if len(notes.strip()) < 200: + return 0 + # Memory proposals come from the user's own turns only. The assistant's + # prose is help, not the user's preferences; mining it would attribute the + # assistant's words to the user (found in dogfooding). The raw capture below + # still keeps the full transcript for review context. + user_notes = _core_extract_transcript_text(transcript_path, roles=("user",)) + # Skip duplicate firings for the same conversation content (e.g. /clear + # immediately followed by exit, or repeated end events). + state_path = _session_end_hook_state_path(target) + fingerprint = _session_notes_fingerprint(notes) + try: + if state_path.exists() and state_path.read_text(encoding="utf-8").strip() == fingerprint: + return 0 + except OSError: + pass + project_name = project + if not project_name: + project_dir = _hook_project_dir(hook_event) + if project_dir: + project_name = _default_project(Path(project_dir)) + # Only store a capture when the user's turns produced memory-worthy + # candidates; otherwise every session would add review-inbox noise. + wiki_dir = _resolve_wiki_dir(target) + root = _resolve_link_root(target) + proposal_limit = max(1, min(limit, 10)) + preview = _propose_memories_from_text( + wiki_dir, + user_notes, + source="agent-session-hook", + limit=proposal_limit, + project=project_name, + command_target=root, + ) + if not int(preview.get("count") or 0): + return 0 + code = session_end( + target, + notes, + title="Agent session notes" + (f" — {project_name}" if project_name else ""), + limit=proposal_limit, + project=project_name, + proposal_text=user_notes, + ) + if code == 0: + try: + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text(fingerprint, encoding="utf-8") + except OSError: + pass + return code + + +def run_agent_hook( + target: Path, event: str, limit: int = 5, project: str | None = None, emit: str = "text" +) -> int: + """Run an installed agent session hook; never fail the agent session.""" + target = target.expanduser().resolve() + hook_event = _read_hook_stdin() + try: + if event == "session-start": + return _hook_session_start(target, hook_event, limit, project, emit) + if event == "session-end": + return _hook_session_end(target, hook_event, limit, project) + print(f"Unknown hook event: {event}", file=sys.stderr) + except Exception as exc: + print(f"Link {event} hook failed: {exc}", file=sys.stderr) + return 0 + + def profile(target: Path, limit: int = 10, project: str | None = None, json_output: bool = False) -> int: target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) project_name = project or _default_project(target) profile_data = _memory_profile(wiki_dir, limit=limit, project=project_name) @@ -1984,8 +2276,7 @@ def memory_audit(target: Path, limit: int = 10, project: str | None = None, json target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) if not wiki_dir.exists(): - print(f"Missing wiki directory: {wiki_dir}", file=sys.stderr) - return 1 + return _missing_wiki_error(wiki_dir) payload = _memory_audit_payload(target, wiki_dir, limit=limit, project=project) if json_output: @@ -2005,7 +2296,10 @@ def _configure_link_command_display() -> None: if os.environ.get("LINK_CLI_COMMAND"): _core_set_link_command_override(None) else: - _core_set_link_command_override([sys.executable, str(ROOT / "link.py")]) + # Source-checkout runs: show a friendly `python3 link.py` in generated + # commands, not the raw interpreter path (e.g. python@3.14). The + # absolute link.py path stays for paste-safety from any directory. + _core_set_link_command_override(["python3", str(ROOT / "link.py")]) def verify_mcp( @@ -2042,10 +2336,15 @@ def connect_mcp( write: bool = False, config_path: str | None = None, python_cmd: str | None = None, + hooks: bool = False, json_output: bool = False, ) -> int: target = target.expanduser().resolve() wiki_dir = _resolve_wiki_dir(target) + if hooks and not _core_supports_agent_hooks(agent): + supported = ", ".join(_core_hook_supported_agents()) + print(f"--hooks is not supported for {agent}. Session hooks are available for: {supported}", file=sys.stderr) + return 1 payload = _core_build_mcp_connect_payload( target=target, wiki_dir=wiki_dir, @@ -2057,14 +2356,52 @@ def connect_mcp( config_path=config_path, write=write, ) + hooks_payload: dict[str, object] | None = None + if hooks: + runtime_script = target / "link.py" + runtime_note = "" + if runtime_script.exists(): + # A workspace runtime copied before session hooks existed would + # make every installed hook fail with an argparse error. + if not (target / "link_core" / "agent_hooks.py").exists(): + if write: + _copy_runtime_files(target) + runtime_note = f"Refreshed the Link runtime at {target}: it predated session hooks." + else: + runtime_note = ( + f"The Link runtime at {target} predates session hooks; " + "--write will refresh it automatically (or run " + f"{_display_command(['lnk', 'init', str(target)])} first)." + ) + else: + runtime_script = ROOT / "link.py" + hooks_payload = _core_build_agent_hooks_payload( + target=target, + agent=agent, + runtime_script=runtime_script, + python_cmd=sys.executable, + write=write, + ) + if runtime_note: + hooks_payload["runtime_note"] = runtime_note + payload["session_hooks"] = hooks_payload if json_output: print(json.dumps(payload, indent=2)) write_status = payload.get("write") if isinstance(payload.get("write"), dict) else {} - return 0 if not write or bool(write_status.get("ok")) else 1 + ok = not write or bool(write_status.get("ok")) + if write and hooks_payload is not None: + hooks_write = hooks_payload.get("write") if isinstance(hooks_payload.get("write"), dict) else {} + ok = ok and bool(hooks_write.get("ok")) + return 0 if ok else 1 code, text = _core_render_mcp_connect_text(payload) _print_text(text) + if hooks_payload is not None: + hooks_code, hooks_text = _core_render_agent_hooks_text(hooks_payload) + print() + _print_text(hooks_text) + code = code or hooks_code return code @@ -2095,6 +2432,7 @@ def onboard( agents: list[str] | None = None, all_agents: bool = False, write: bool = False, + hooks: bool = False, first_memory: str | None = None, seed_project: str | None = None, project: str | None = None, @@ -2180,7 +2518,7 @@ def onboard( connections: list[dict[str, object]] = [] for agent in _onboard_agent_names(agents, all_agents): try: - connections.append(_core_build_mcp_connect_payload( + connection = _core_build_mcp_connect_payload( target=target, wiki_dir=wiki_dir, agent=agent, @@ -2188,7 +2526,22 @@ def onboard( init_command=[sys.executable, str(ROOT / "link.py"), "init", str(target)], default_python=sys.executable, write=write, - )) + ) + if hooks and _core_supports_agent_hooks(agent): + connection["session_hooks"] = _core_build_agent_hooks_payload( + target=target, + agent=agent, + runtime_script=(target / "link.py") if (target / "link.py").exists() else ROOT / "link.py", + python_cmd=sys.executable, + write=write, + ) + elif hooks: + connection["session_hooks"] = { + "agent": agent, + "write": {"requested": False, "ok": False, + "message": "session hooks are not available for this agent yet"}, + } + connections.append(connection) except ValueError as exc: connections.append({ "agent": agent, @@ -2198,6 +2551,19 @@ def onboard( "next_actions": [], }) + # Surface the automatic-memory (hooks) path: without this, users who follow + # the guided onboarding never discover the flagship 1.6 feature. + for connection in connections: + agent_name = str(connection.get("agent") or "") + if agent_name and _core_supports_agent_hooks(agent_name) and "session_hooks" not in connection: + connection["hooks_command"] = _display_command( + ["link", "onboard", str(target), "--agent", agent_name, "--hooks", "--write"] + ) + hooks_agents = [ + agent for agent in _onboard_agent_names(agents, all_agents) + if _core_supports_agent_hooks(agent) + ] + status_payload = _core_link_status(wiki_dir, version=LINK_VERSION, include_validation=True) starter_payload = _core_starter_prompt_payload(target, project=project) prompts = starter_payload.get("prompts", []) @@ -2234,6 +2600,12 @@ def onboard( _display_command(["link", "onboard", str(target), "--agent", agent]) for agent in ("codex", "claude-code", "cursor") ], + "hooks_hint": ( + "Make memory automatic — add --hooks (Claude Code, Codex, Cursor): the memory brief " + "is injected at session start and proposals are captured at session end, so no agent " + "has to remember to call Link. Example:\n" + f" {_display_command(['link', 'onboard', str(target), '--agent', 'claude-code', '--hooks', '--write'])}" + ) if (hooks_agents or not connections) and not hooks else "", "url": f"http://127.0.0.1:{port}", } @@ -2659,6 +3031,9 @@ def main(argv: list[str] | None = None) -> int: "benchmark": benchmark, "brief": brief, "start": start, + "hook": run_agent_hook, + "consolidate": consolidate, + "semantic": semantic, "profile": profile, "wins": memory_wins, "memory-audit": memory_audit, diff --git a/mcp_package/README.md b/mcp_package/README.md index bbf86621..b236b808 100644 --- a/mcp_package/README.md +++ b/mcp_package/README.md @@ -112,7 +112,8 @@ New MCP configs should expose Link through six model-facing tools: 3. `remember(text, ...)` writes only explicit user-approved durable memories. 4. `ingest(action?, strict?)` checks or validates raw-source ingest work. 5. `review(action?, ...)` handles memory inbox, profile, audit, log, explain, - archive, restore, forget, and visibility review workflows. + archive, restore, forget, visibility, and read-only `consolidate` (backlog + plan) review workflows. 6. `admin(action, arguments?)` is the escape hatch for backup, migrate, validate, graph export, pages, captures, rebuilds, and advanced updates. @@ -139,7 +140,8 @@ Slim agents should call: 6. `remember(...)` only when the user explicitly approves saving durable memory. 7. `admin(action="session_end", arguments="{...}")` at session end to propose memory without silently saving it. 8. `review(action="inbox"|"audit"|"profile"|"explain"|...)` for memory lifecycle review. -9. `admin(action, arguments)` for backup, migrate, validate, graph export, captures, rebuilds, and compatibility actions. +9. `review(action="consolidate")` when a brief reports a memory backlog: it returns a read-only plan; apply its actions only after the user approves each one. +10. `admin(action, arguments)` for backup, migrate, validate, graph export, captures, rebuilds, and compatibility actions. Add `review_after` for memories that should return to the review inbox after a date, or `expires_at` for temporary context that should leave default recall @@ -152,6 +154,33 @@ terminal text. In the local web proposal picker, unreadable raw files are surfaced as `Fix access` instead of being loaded as empty proposal text. +## Automatic Session Hooks + +Agents with session-hook support (Claude Code, Codex, Cursor) can run the +memory loop automatically: `lnk connect --hooks --write` (from a Link +checkout or installer) installs hooks that inject a bounded memory brief into +every new session and store proposal-only session notes at session end. +Sessions with nothing memory-worthy are skipped, duplicate end events are +deduplicated, and durable memory still requires explicit review. If hooks are +installed, agents should skip the manual startup brief call. + +## Optional Semantic Recall (still fully local) + +Lexical recall is the default and the fallback. Two optional local tiers add +paraphrase recall: + +```bash +pip install "link-mcp[semantic]" # fast tier: tiny static model +pip install "link-mcp[semantic-quality]" # quality tier: contextual ONNX model +python3 -m link_mcp --semantic-setup --wiki ~/link/wiki # explicit one-time model fetch +``` + +The models load offline-only at recall time (a query can never trigger a +download), embeddings live in plain JSON under `.link-cache/`, and there is no +vector database or service. Semantic-only matches are labeled +(`match: semantic`, capped confidence) so agents verify before trusting them. +Measured results: . + ## Privacy and Scale - Local-first: `link-mcp` reads the wiki path you configure and does not call diff --git a/mcp_package/link_core/agent_hooks.py b/mcp_package/link_core/agent_hooks.py new file mode 100644 index 00000000..2fd63254 --- /dev/null +++ b/mcp_package/link_core/agent_hooks.py @@ -0,0 +1,384 @@ +"""Agent session-hook configuration helpers for Link. + +Hooks let supported agents run the Link memory loop automatically: +a session-start hook injects a bounded memory brief into new sessions, +and a session-end hook stores proposal-only session notes for review. + +Supported agents differ in mechanism, so each config records its schema: +- Claude Code: nested hook groups inside `~/.claude/settings.json`; + session-start stdout becomes model context; SessionEnd gets a transcript. +- Codex: the same nested hook schema in `~/.codex/hooks.json`; stdout becomes + model context; there is no session-end event (Stop fires per turn, which + would be too noisy for capture), so only session-start is installed. +- Cursor: a flat `~/.cursor/hooks.json` with `version: 1`; session-start must + print a JSON envelope with `additional_context`; sessionEnd is fire-and-forget + and only captures when Cursor provides a readable transcript path. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .files import atomic_write_json +from .mcp_verify import display_command + +SESSION_START_TIMEOUT_SECONDS = 30 +SESSION_END_TIMEOUT_SECONDS = 60 + +_HOOK_SCRIPT_MARKER = "link.py" + + +@dataclass(frozen=True) +class AgentHookConfig: + name: str + display_name: str + aliases: tuple[str, ...] + default_settings: str + schema: str = "nested" # "nested" (Claude Code, Codex) or "flat" (Cursor) + start_event: str = "SessionStart" + end_event: str | None = "SessionEnd" + # Skip "resume": the resumed context already carries the earlier brief. + start_matcher: str | None = "startup|clear|compact" + start_emit: str = "text" # "text" stdout-to-context, or "cursor" JSON envelope + restart_hint: str = "Restart the agent; new sessions will start with the Link memory brief." + + +HOOK_AGENT_CONFIGS: tuple[AgentHookConfig, ...] = ( + AgentHookConfig( + name="claude-code", + display_name="Claude Code", + aliases=("claude-code", "claude", "claude-code-cli"), + default_settings="~/.claude/settings.json", + ), + AgentHookConfig( + name="codex", + display_name="Codex", + aliases=("codex",), + default_settings="~/.codex/hooks.json", + end_event=None, + restart_hint=( + "Restart Codex and approve the hook when Codex asks you to trust it; " + "new sessions will then start with the Link memory brief." + ), + ), + AgentHookConfig( + name="cursor", + display_name="Cursor", + aliases=("cursor",), + default_settings="~/.cursor/hooks.json", + schema="flat", + start_event="sessionStart", + end_event="sessionEnd", + start_matcher=None, + start_emit="cursor", + ), +) + + +def hook_supported_agents() -> tuple[str, ...]: + """Return canonical agent names that support `lnk connect --hooks`.""" + return tuple(config.name for config in HOOK_AGENT_CONFIGS) + + +def _find_hook_agent(agent: str) -> AgentHookConfig | None: + normalized = agent.strip().lower().replace("_", "-") + for config in HOOK_AGENT_CONFIGS: + if normalized == config.name or normalized in config.aliases: + return config + return None + + +def supports_agent_hooks(agent: str) -> bool: + return _find_hook_agent(agent) is not None + + +def _hook_agent_by_name(agent: str) -> AgentHookConfig: + config = _find_hook_agent(agent) + if config is not None: + return config + choices = ", ".join(hook_supported_agents()) + raise ValueError(f"session hooks are not supported for agent: {agent}. Try one of: {choices}") + + +def _settings_path(default_settings: str, override: str | None) -> Path: + path = Path(override or default_settings).expanduser() + if not path.is_absolute(): + path = (Path.cwd() / path).resolve() + return path + + +def _hook_command( + python_cmd: str, + runtime_script: Path, + event: str, + target: Path, + emit: str = "text", +) -> str: + parts = [python_cmd, str(runtime_script), "hook", event, str(target)] + if emit != "text": + parts.extend(["--emit", emit]) + return display_command(parts) + + +def _nested_entry(command: str, timeout: int) -> dict[str, object]: + return {"type": "command", "command": command, "timeout": timeout} + + +def _flat_entry(command: str, timeout: int) -> dict[str, object]: + return {"command": command, "timeout": timeout} + + +def _is_link_hook_command(command: object, event: str) -> bool: + if not isinstance(command, str): + return False + return _HOOK_SCRIPT_MARKER in command and f" hook {event}" in command + + +def _merge_nested_event( + settings: dict[str, Any], + event_name: str, + event: str, + entry: dict[str, object], + matcher: str | None = None, +) -> None: + hooks = settings.get("hooks") + if not isinstance(hooks, dict): + hooks = {} + settings["hooks"] = hooks + groups = hooks.get(event_name) + if not isinstance(groups, list): + groups = [] + replaced = False + for group in groups: + if not isinstance(group, dict): + continue + group_hooks = group.get("hooks") + if not isinstance(group_hooks, list): + continue + for index, existing in enumerate(group_hooks): + if isinstance(existing, dict) and _is_link_hook_command(existing.get("command"), event): + group_hooks[index] = dict(entry) + replaced = True + if not replaced: + group: dict[str, object] = {"hooks": [dict(entry)]} + if matcher: + group["matcher"] = matcher + groups.append(group) + hooks[event_name] = groups + + +def _merge_flat_event( + settings: dict[str, Any], + event_name: str, + event: str, + entry: dict[str, object], +) -> None: + settings.setdefault("version", 1) + hooks = settings.get("hooks") + if not isinstance(hooks, dict): + hooks = {} + settings["hooks"] = hooks + entries = hooks.get(event_name) + if not isinstance(entries, list): + entries = [] + replaced = False + for index, existing in enumerate(entries): + if isinstance(existing, dict) and _is_link_hook_command(existing.get("command"), event): + entries[index] = dict(entry) + replaced = True + if not replaced: + entries.append(dict(entry)) + hooks[event_name] = entries + + +def _event_plan(config: AgentHookConfig, python_cmd: str, runtime_script: Path, target: Path) -> list[dict[str, object]]: + """Return the ordered event entries this agent should install.""" + make_entry = _nested_entry if config.schema == "nested" else _flat_entry + plan: list[dict[str, object]] = [ + { + "event_name": config.start_event, + "event": "session-start", + "matcher": config.start_matcher, + "entry": make_entry( + _hook_command(python_cmd, runtime_script, "session-start", target, emit=config.start_emit), + SESSION_START_TIMEOUT_SECONDS, + ), + } + ] + if config.end_event: + plan.append({ + "event_name": config.end_event, + "event": "session-end", + "matcher": None, + "entry": make_entry( + _hook_command(python_cmd, runtime_script, "session-end", target), + SESSION_END_TIMEOUT_SECONDS, + ), + }) + return plan + + +def _hooks_snippet(config: AgentHookConfig, plan: list[dict[str, object]]) -> str: + hooks: dict[str, object] = {} + for item in plan: + entry = item["entry"] + if config.schema == "nested": + group: dict[str, object] = {"hooks": [entry]} + if item["matcher"]: + group["matcher"] = item["matcher"] + hooks[str(item["event_name"])] = [group] + else: + hooks[str(item["event_name"])] = [entry] + payload: dict[str, object] = {"hooks": hooks} + if config.schema == "flat": + payload = {"version": 1, "hooks": hooks} + return json.dumps(payload, indent=2) + + +def _write_hooks(path: Path, config: AgentHookConfig, plan: list[dict[str, object]]) -> None: + settings: dict[str, Any] = {} + if path.exists() and path.read_text(encoding="utf-8", errors="replace").strip(): + settings = json.loads(path.read_text(encoding="utf-8", errors="replace")) + if not isinstance(settings, dict): + raise ValueError(f"{path} must contain a JSON object") + for item in plan: + entry = item["entry"] + assert isinstance(entry, dict) + if config.schema == "nested": + _merge_nested_event( + settings, + str(item["event_name"]), + str(item["event"]), + entry, + matcher=item["matcher"] if isinstance(item["matcher"], str) else None, + ) + else: + _merge_flat_event(settings, str(item["event_name"]), str(item["event"]), entry) + atomic_write_json(path, settings) + + +def build_agent_hooks_payload( + *, + target: Path, + agent: str, + runtime_script: Path, + python_cmd: str, + settings_path: str | None = None, + write: bool = False, +) -> dict[str, object]: + """Build or write session-hook configuration for a supported local agent.""" + config = _hook_agent_by_name(agent) + path = _settings_path(config.default_settings, settings_path) + plan = _event_plan(config, python_cmd, runtime_script, target) + write_status: dict[str, object] = {"requested": write, "ok": False, "message": "preview only"} + if write: + try: + _write_hooks(path, config, plan) + write_status = {"requested": True, "ok": True, "message": f"updated {path}"} + except Exception as exc: + write_status = {"requested": True, "ok": False, "message": str(exc)} + + entry_commands = { + str(item["event_name"]): str(item["entry"]["command"]) # type: ignore[index] + for item in plan + } + behavior = [ + f"{config.start_event}: injects a bounded Link memory brief into new agent sessions.", + ] + if config.end_event: + behavior.append( + f"{config.end_event}: stores proposal-only session notes locally; durable memory still requires review." + ) + else: + behavior.append( + f"{config.display_name} has no session-end hook event; end sessions with `lnk session-end` " + "or the MCP session_end action to capture memory proposals." + ) + if config.name in {"codex", "cursor"}: + behavior.append( + f"New: {config.display_name} hook support follows the vendor's documented schema; " + "if a hook misbehaves, please open an issue." + ) + + return { + "agent": config.name, + "display_name": config.display_name, + "target": str(target), + "settings_path": str(path), + "events": entry_commands, + "snippet": _hooks_snippet(config, plan), + "write": write_status, + "behavior": behavior, + "restart_hint": config.restart_hint, + } + + +def _content_text(content: object) -> str: + if isinstance(content, str): + return content.strip() + parts: list[str] = [] + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text") + if isinstance(text, str) and text.strip(): + parts.append(text.strip()) + return "\n".join(parts) + + +def extract_transcript_text( + transcript_path: Path, + *, + max_chars: int = 6000, + max_message_chars: int = 800, + roles: tuple[str, ...] = ("user", "assistant"), +) -> str: + """Extract bounded conversation text from an agent transcript JSONL file. + + Keeps text blocks for the given `roles` (default user + assistant), skips + tool calls/results and meta entries, and returns the most recent messages + within `max_chars`. Pass roles=("user",) to mine only what the user said — + memory proposals should come from the user's own words, not the assistant's + prose, which would otherwise be mis-attributed as user preferences. + """ + role_set = set(roles) + try: + raw = transcript_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + lines: list[str] = [] + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if not isinstance(entry, dict) or entry.get("isMeta"): + continue + if entry.get("type") not in role_set: + continue + message = entry.get("message") + if not isinstance(message, dict): + continue + text = _content_text(message.get("content")) + if not text: + continue + if len(text) > max_message_chars: + text = text[: max_message_chars].rstrip() + " …" + role = "User" if entry.get("type") == "user" else "Assistant" + lines.append(f"{role}: {text}") + if not lines: + return "" + kept: list[str] = [] + total = 0 + for line in reversed(lines): + cost = len(line) + 2 + if kept and total + cost > max_chars: + break + kept.append(line) + total += cost + return "\n\n".join(reversed(kept)) diff --git a/mcp_package/link_core/cli_memory.py b/mcp_package/link_core/cli_memory.py index 875400de..687023f7 100644 --- a/mcp_package/link_core/cli_memory.py +++ b/mcp_package/link_core/cli_memory.py @@ -207,6 +207,7 @@ def render_recall_text( include_archived: bool = False, project: str | None = None, target: object = ".", + miss_hint: str = "", ) -> tuple[int, str]: lines = [f"Link memory recall: {query}"] if project: @@ -215,8 +216,10 @@ def render_recall_text( lines.append("Including archived/stale memories") lines.append("") if not results: + lines.append("No matching memories found.") + if miss_hint: + lines.extend(["", miss_hint]) lines.extend([ - "No matching memories found.", "", "Next:", f" Add one: {_shell_words('python3', 'link.py', 'remember', 'Memory to keep', target)}", diff --git a/mcp_package/link_core/cli_parser.py b/mcp_package/link_core/cli_parser.py index 3fc97273..36e7dae5 100644 --- a/mcp_package/link_core/cli_parser.py +++ b/mcp_package/link_core/cli_parser.py @@ -56,6 +56,11 @@ def build_cli_parser( onboard_cmd.add_argument("--agent", action="append", default=[], help="agent config to preview or write; repeatable") onboard_cmd.add_argument("--all-agents", action="store_true", help="preview or write all supported agent configs") onboard_cmd.add_argument("--write", action="store_true", help="update selected agent config files") + onboard_cmd.add_argument( + "--hooks", + action="store_true", + help="also configure session hooks for selected agents that support them (Claude Code, Codex, Cursor)", + ) onboard_cmd.add_argument("--first-memory", default=None, help="seed one explicit memory for review") onboard_cmd.add_argument( "--seed-project", @@ -304,6 +309,30 @@ def build_cli_parser( start_cmd.add_argument("--project", default=None, help="include user/global memories plus this project's memories") start_cmd.add_argument("--json", action="store_true", help="print machine-readable startup packet") + hook_cmd = sub.add_parser("hook", help="run an agent session hook (invoked by installed agent hooks)") + hook_cmd.add_argument("event", choices=["session-start", "session-end"], help="agent session lifecycle event") + hook_cmd.add_argument("target", nargs="?", default=".") + hook_cmd.add_argument("--limit", type=int, default=5, help="maximum memories in the session-start brief") + hook_cmd.add_argument("--project", default=None, help="include user/global memories plus this project's memories") + hook_cmd.add_argument( + "--emit", + choices=["text", "cursor"], + default="text", + help="session-start output envelope: plain text (Claude Code, Codex) or Cursor additional_context JSON", + ) + + semantic_cmd = sub.add_parser("semantic", help="show or set up optional local semantic recall") + semantic_cmd.add_argument("target", nargs="?", default=".") + semantic_cmd.add_argument("--setup", action="store_true", help="fetch the local embedding model once and build the index") + semantic_cmd.add_argument("--rebuild", action="store_true", help="rebuild the semantic index offline") + semantic_cmd.add_argument("--json", action="store_true", help="print machine-readable semantic status") + + consolidate_cmd = sub.add_parser("consolidate", help="print a read-only plan for the capture and review backlog") + consolidate_cmd.add_argument("target", nargs="?", default=".") + consolidate_cmd.add_argument("--limit", type=int, default=50, help="maximum captures and review items to include") + consolidate_cmd.add_argument("--project", default=None, help="restrict the plan to one project's captures and memories") + consolidate_cmd.add_argument("--json", action="store_true", help="print machine-readable consolidation plan") + profile_cmd = sub.add_parser("profile", help="show what Link remembers") profile_cmd.add_argument("target", nargs="?", default=".") profile_cmd.add_argument("--limit", type=int, default=10) @@ -380,6 +409,11 @@ def build_cli_parser( connect_cmd.add_argument("--write", action="store_true", help="update the detected agent config file") connect_cmd.add_argument("--config", default=None, help="override the agent config file path") connect_cmd.add_argument("--python", default=None, help="Python executable for the MCP server") + connect_cmd.add_argument( + "--hooks", + action="store_true", + help="also configure session hooks so new sessions start with the Link brief (Claude Code)", + ) connect_cmd.add_argument("--json", action="store_true", help="print machine-readable connection plan") return parser @@ -418,6 +452,7 @@ def dispatch_cli_command(args: Any, handlers: Mapping[str, CliHandler]) -> int: agents=args.agent, all_agents=args.all_agents, write=args.write, + hooks=args.hooks, first_memory=args.first_memory, seed_project=args.seed_project, project=args.project, @@ -655,6 +690,18 @@ def dispatch_cli_command(args: Any, handlers: Mapping[str, CliHandler]) -> int: project=args.project, json_output=args.json, ) + if command == "hook": + return handlers["hook"]( + Path(args.target), + args.event, + limit=args.limit, + project=args.project, + emit=args.emit, + ) + if command == "consolidate": + return handlers["consolidate"](Path(args.target), limit=args.limit, project=args.project, json_output=args.json) + if command == "semantic": + return handlers["semantic"](Path(args.target), setup=args.setup, rebuild=args.rebuild, json_output=args.json) if command == "profile": return handlers["profile"](Path(args.target), limit=args.limit, project=args.project, json_output=args.json) if command == "wins": @@ -699,6 +746,7 @@ def dispatch_cli_command(args: Any, handlers: Mapping[str, CliHandler]) -> int: write=args.write, config_path=args.config, python_cmd=args.python, + hooks=args.hooks, json_output=args.json, ) raise ValueError(f"unknown command: {command}") diff --git a/mcp_package/link_core/cli_runtime.py b/mcp_package/link_core/cli_runtime.py index a48392f3..c9864fb8 100644 --- a/mcp_package/link_core/cli_runtime.py +++ b/mcp_package/link_core/cli_runtime.py @@ -122,6 +122,14 @@ def render_start_text(payload: Mapping[str, object]) -> tuple[int, str]: lines.append(f"- Need more context: {commands['query']}") if isinstance(commands, Mapping) and commands.get("review"): lines.append(f"- Review pending memory: {commands['review']}") + brief = payload.get("brief") if isinstance(payload.get("brief"), Mapping) else {} + backlog = brief.get("backlog") if isinstance(brief.get("backlog"), Mapping) else {} + if backlog.get("backlog"): + lines.append( + f"- Memory backlog ({backlog.get('pending_captures', 0)} captures · " + f"{backlog.get('needs_review_memories', 0)} reviews): offer a consolidation pass — " + f"{backlog.get('command')}" + ) lines.append("- Save memory only after explicit user approval.") return 0 if status.get("ready") else 1, "\n".join(lines) @@ -267,10 +275,16 @@ def render_proof_text(payload: Mapping[str, object]) -> tuple[int, str]: "Cross-agent memory continuity works" if ready else "Cross-agent memory proof needs attention", "", "What happened", - f"1. Workspace: {'created' if created else 'reused'} local Markdown wiki.", + f"1. Workspace: {'created' if created else 'reused'} a throwaway demo wiki (not your real memory).", f"2. Memory: {memory_status}: {title}", f"3. Recall: {recall_status} through the same bounded recall path used by CLI, skills, and MCP.", "", + "What this means for you", + "- Save something once; any of your agents can recall it later, from plain local files.", + "- Ready for real use? Create your durable workspace and wire an agent:", + f" {display_command(['lnk', 'onboard'])}", + " (this proof workspace is a demo — your memory will live at ~/link)", + "", "Try it with two agents", f"Agent A: {prompts.get('agent_a', 'remember that this project uses Link')}", f"Agent B: {prompts.get('agent_b', 'start with Link before we continue')}", @@ -301,9 +315,19 @@ def _first_mapping_items(value: object, limit: int) -> list[Mapping[str, object] def _connection_state(connection: Mapping[str, object]) -> str: write_status = connection.get("write") if isinstance(connection.get("write"), Mapping) else {} + state = "preview" if write_status.get("requested"): - return "updated" if write_status.get("ok") else "failed" - return "preview" + state = "updated" if write_status.get("ok") else "failed" + session_hooks = connection.get("session_hooks") + if isinstance(session_hooks, Mapping): + hooks_write = session_hooks.get("write") if isinstance(session_hooks.get("write"), Mapping) else {} + if hooks_write.get("requested"): + state += " · hooks " + ("updated" if hooks_write.get("ok") else "failed") + elif hooks_write.get("message"): + state += f" · hooks: {hooks_write.get('message')}" + else: + state += " · hooks preview" + return state def render_onboard_text(payload: Mapping[str, object]) -> tuple[int, str]: @@ -384,6 +408,8 @@ def render_onboard_text(payload: Mapping[str, object]) -> tuple[int, str]: if action.get("label") == "write config": lines.append(f" Write when ready: {action.get('command_text')}") break + if connection.get("hooks_command"): + lines.append(f" Make memory automatic (recommended): {connection.get('hooks_command')}") if restart_hint: lines.append(f" After writing: {restart_hint}") elif state == "updated": @@ -399,6 +425,9 @@ def render_onboard_text(payload: Mapping[str, object]) -> tuple[int, str]: lines.append("- not connected yet. Preview an agent config with:") for command in payload.get("agent_examples", []): lines.append(f" {command}") + hooks_hint = str(payload.get("hooks_hint") or "").strip() + if hooks_hint: + lines.extend(["", *hooks_hint.splitlines()]) prompts = _first_mapping_items(payload.get("prompts"), 4) lines.extend(["", "Ask your agent"]) @@ -464,3 +493,99 @@ def render_mcp_connect_text(payload: Mapping[str, object]) -> tuple[int, str]: if restart_hint: lines.append(f" {restart_hint}") return code, "\n".join(lines) + + +def render_agent_hooks_text(payload: Mapping[str, object]) -> tuple[int, str]: + """Render a session-hook configuration plan for a supported local agent.""" + write_status = payload.get("write") if isinstance(payload.get("write"), Mapping) else {} + requested = bool(write_status.get("requested")) + ok = bool(write_status.get("ok")) + code = 0 if not requested or ok else 1 + lines = [ + f"Link session hooks: {payload.get('display_name')}", + "", + f"Settings: {payload.get('settings_path')}", + ] + behavior = payload.get("behavior") + if isinstance(behavior, Sequence) and not isinstance(behavior, (str, bytes)): + lines.append("") + lines.extend(f" {item}" for item in behavior) + runtime_note = str(payload.get("runtime_note") or "").strip() + if runtime_note: + lines.extend(["", f" {runtime_note}"]) + lines.append("") + if requested: + lines.append(f"Write: {'updated' if ok else 'failed'}") + message = write_status.get("message") + if message: + lines.append(f" {message}") + else: + lines.append("Preview only. Rerun with --write to update the settings file.") + lines.extend(["", "Hooks snippet:"]) + snippet = str(payload.get("snippet") or "") + lines.extend(f" {line}" if line else "" for line in snippet.splitlines()) + restart_hint = payload.get("restart_hint") + if restart_hint: + lines.extend(["", f" {restart_hint}"]) + return code, "\n".join(lines) + + +def render_session_start_hook_text(payload: Mapping[str, object]) -> tuple[int, str]: + """Render the bounded memory-brief context block injected by session-start hooks.""" + status = payload.get("status") if isinstance(payload.get("status"), Mapping) else {} + target = str(payload.get("target") or "") + project = str(payload.get("project") or "").strip() + lines = [ + "Link memory (local, source-backed)" + + (f" · project {project}" if project else ""), + ] + if not status.get("ready"): + lines.extend([ + "Link is not ready; skipping the memory brief.", + f"Check with: {display_command(['lnk', 'health', target])}", + ]) + return 0, "\n".join(lines) + + # Empty workspace: inject two useful lines, not a skeleton of zeros. + if ( + not int(status.get("active_memory_count") or 0) + and not int(status.get("content_page_count") or 0) + and not int(payload.get("capture_count") or 0) + ): + lines[0] += " — empty workspace, nothing to recall yet." + lines.extend([ + "To give day-one recall real project context, seed allowlisted repo docs: " + f"{display_command(['lnk', 'seed', '.', target])} (source-backed, no durable memory).", + "Save durable memory only after the user explicitly approves it.", + ]) + return 0, "\n".join(lines) + + brief_text = str(payload.get("brief_text") or "").strip() + if brief_text: + lines.extend(["", brief_text]) + + seed_recommended = bool(payload.get("project_seed_recommended")) + if seed_recommended: + lines.extend([ + "", + "No project context or relevant memory yet. To seed source-backed project context " + f"from this repo's docs, suggest: {display_command(['lnk', 'seed', '.', target])}", + ]) + backlog = payload.get("backlog") if isinstance(payload.get("backlog"), Mapping) else {} + if backlog.get("backlog"): + lines.extend([ + "", + ( + f"Memory backlog: {backlog.get('pending_captures', 0)} pending captures · " + f"{backlog.get('needs_review_memories', 0)} memories need review. " + "Offer the user a short consolidation pass this session; " + f"{backlog.get('command')} prints a read-only plan with approve/discard commands." + ), + ]) + lines.extend([ + "", + "Use this brief before asking the user to repeat durable context. " + f"For task-specific context: {display_command(['lnk', 'query', '', target, '--budget', 'micro'])} " + "or the Link MCP recall tool. Save durable memory only after explicit user approval.", + ]) + return 0, "\n".join(lines) diff --git a/mcp_package/link_core/consolidate.py b/mcp_package/link_core/consolidate.py new file mode 100644 index 00000000..2262b866 --- /dev/null +++ b/mcp_package/link_core/consolidate.py @@ -0,0 +1,217 @@ +"""Read-only memory consolidation planning for Link. + +Consolidation never writes: it detects backlog (pending raw captures and +memories that need review), groups duplicate captures, and prints the exact +review-gated commands to resolve each item with the user. Automatic session +hooks use the same backlog summary to nudge agents to offer consolidation. +""" +from __future__ import annotations + +import re +from pathlib import Path + +from .mcp_verify import display_command + +BACKLOG_CAPTURE_THRESHOLD = 5 +BACKLOG_REVIEW_THRESHOLD = 8 + + +def consolidate_command(command_target: str | Path = ".") -> str: + return display_command(["lnk", "consolidate", str(command_target)]) + + +def memory_backlog_summary( + *, + capture_count: int, + needs_review_count: int, + command_target: str | Path = ".", +) -> dict[str, object]: + """Return the shared backlog signal used by hooks, briefs, and status views.""" + backlog = capture_count >= BACKLOG_CAPTURE_THRESHOLD or needs_review_count >= BACKLOG_REVIEW_THRESHOLD + return { + "pending_captures": capture_count, + "needs_review_memories": needs_review_count, + "backlog": backlog, + "capture_threshold": BACKLOG_CAPTURE_THRESHOLD, + "review_threshold": BACKLOG_REVIEW_THRESHOLD, + "command": consolidate_command(command_target), + } + + +DUPLICATE_JACCARD = 0.8 + + +def _snippet_tokens(capture: dict[str, object]) -> set[str]: + snippet = str(capture.get("snippet") or "").lower() + return set(re.findall(r"[a-z0-9]{3,}", snippet)) + + +def _duplicate_capture_groups(captures: list[dict[str, object]]) -> list[dict[str, object]]: + """Cluster near-duplicate captures by snippet token overlap; newest is kept. + + Exact duplicates have Jaccard 1.0, so one similarity clustering covers + both identical and lightly reworded captures of the same session content. + """ + clusters: list[dict[str, object]] = [] + for capture in captures: # capture_records sorts newest first + tokens = _snippet_tokens(capture) + if not tokens: + continue + for cluster in clusters: + keep_tokens: set[str] = cluster["tokens"] # type: ignore[assignment] + union = tokens | keep_tokens + if union and len(tokens & keep_tokens) / len(union) >= DUPLICATE_JACCARD: + cluster["members"].append(capture) # type: ignore[union-attr] + break + else: + clusters.append({"tokens": tokens, "keep": capture, "members": []}) + groups: list[dict[str, object]] = [] + for cluster in clusters: + members = cluster["members"] + if not members: + continue + keep = cluster["keep"] + groups.append({ + "keep": {"path": keep.get("path"), "title": keep.get("title")}, + "duplicates": [ + { + "path": item.get("path"), + "title": item.get("title"), + "delete_command": (item.get("commands") or {}).get("delete", "") + if isinstance(item.get("commands"), dict) + else "", + } + for item in members + ], + }) + return groups + + +def build_consolidation_plan( + *, + captures_payload: dict[str, object], + inbox_payload: dict[str, object], + command_target: str | Path = ".", + project: str | None = None, +) -> dict[str, object]: + """Build a read-only consolidation plan from capture and review backlogs.""" + captures = captures_payload.get("captures") if isinstance(captures_payload.get("captures"), list) else [] + capture_count = int(captures_payload.get("count") or len(captures)) + review_items = inbox_payload.get("items") if isinstance(inbox_payload.get("items"), list) else [] + needs_review_count = int(inbox_payload.get("review_count") or len(review_items)) + duplicate_groups = _duplicate_capture_groups([c for c in captures if isinstance(c, dict)]) + duplicate_count = sum(len(group["duplicates"]) for group in duplicate_groups) + + capture_plan = [] + duplicate_paths = { + str(item["path"]) + for group in duplicate_groups + for item in group["duplicates"] + } + for capture in captures: + if not isinstance(capture, dict): + continue + commands = capture.get("commands") if isinstance(capture.get("commands"), dict) else {} + capture_plan.append({ + "path": capture.get("path"), + "title": capture.get("title"), + "project": capture.get("project"), + "snippet": capture.get("snippet"), + "secret_warning_count": capture.get("warning_count", 0), + "duplicate": str(capture.get("path")) in duplicate_paths, + "accept_command": commands.get("accept", ""), + "delete_command": commands.get("delete", ""), + }) + + review_plan = [] + for item in review_items[:10]: + if not isinstance(item, dict): + continue + primary = item.get("primary_action") if isinstance(item.get("primary_action"), dict) else {} + review_plan.append({ + "title": item.get("title"), + "severity": item.get("highest_severity"), + "command": primary.get("command_text") or primary.get("command") or "", + }) + + backlog = memory_backlog_summary( + capture_count=capture_count, + needs_review_count=needs_review_count, + command_target=command_target, + ) + return { + "project": project or "", + "backlog": backlog, + "pending_captures": capture_count, + "needs_review_memories": needs_review_count, + "duplicate_groups": duplicate_groups, + "duplicate_capture_count": duplicate_count, + "captures": capture_plan, + "review_queue": review_plan, + "safety": ( + "Read-only plan. Nothing was merged, deleted, or saved. " + "Run the listed commands only after the user approves each action." + ), + } + + +def render_consolidate_text(payload: dict[str, object]) -> tuple[int, str]: + """Render the consolidation plan for terminal and agent use.""" + backlog = payload.get("backlog") if isinstance(payload.get("backlog"), dict) else {} + lines = [ + "Link consolidation plan (read-only)", + "", + ( + f"Pending captures: {payload.get('pending_captures', 0)} · " + f"Memories needing review: {payload.get('needs_review_memories', 0)}" + ), + ] + if backlog.get("backlog"): + lines.append("Backlog is above threshold; a review session with the user is recommended.") + else: + lines.append("Backlog is small; consolidation is optional right now.") + + duplicate_groups = payload.get("duplicate_groups") if isinstance(payload.get("duplicate_groups"), list) else [] + if duplicate_groups: + lines.extend(["", f"Duplicate captures ({payload.get('duplicate_capture_count', 0)} safe to delete after review):"]) + for group in duplicate_groups: + if not isinstance(group, dict): + continue + keep = group.get("keep") if isinstance(group.get("keep"), dict) else {} + lines.append(f"- Keep: {keep.get('path')}") + for item in group.get("duplicates", []): + if isinstance(item, dict): + lines.append(f" Duplicate: {item.get('path')}") + if item.get("delete_command"): + lines.append(f" {item.get('delete_command')}") + + captures = payload.get("captures") if isinstance(payload.get("captures"), list) else [] + unique_captures = [c for c in captures if isinstance(c, dict) and not c.get("duplicate")] + if unique_captures: + lines.extend(["", "Captures to review with the user:"]) + for capture in unique_captures[:10]: + title = str(capture.get("title") or capture.get("path")) + lines.append(f"- {title} ({capture.get('path')})") + snippet = str(capture.get("snippet") or "").strip() + if snippet: + lines.append(f" {snippet[:140]}") + if capture.get("accept_command"): + lines.append(f" Accept: {capture.get('accept_command')}") + if capture.get("delete_command"): + lines.append(f" Discard: {capture.get('delete_command')}") + + review_queue = payload.get("review_queue") if isinstance(payload.get("review_queue"), list) else [] + if review_queue: + lines.extend(["", "Memories needing review:"]) + for item in review_queue: + if not isinstance(item, dict): + continue + lines.append(f"- [{item.get('severity', '?')}] {item.get('title')}") + if item.get("command"): + lines.append(f" {item.get('command')}") + + if not duplicate_groups and not unique_captures and not review_queue: + lines.extend(["", "Nothing to consolidate. Memory state is clean."]) + + lines.extend(["", str(payload.get("safety") or "")]) + return 0, "\n".join(line for line in lines if line is not None) diff --git a/mcp_package/link_core/memory.py b/mcp_package/link_core/memory.py index 149f472b..19aaaf23 100644 --- a/mcp_package/link_core/memory.py +++ b/mcp_package/link_core/memory.py @@ -7,7 +7,9 @@ from datetime import date, datetime, timezone from pathlib import Path +from .consolidate import memory_backlog_summary from .files import atomic_write_text +from .semantic import semantic_confidence_cap, semantic_match_points from .frontmatter import ( csv_values, frontmatter_int, @@ -1775,8 +1777,9 @@ def memory_audit_next_actions( def add_capture_review_to_brief( payload: Mapping[str, object], captures: Mapping[str, object], + command_target: str | Path = ".", ) -> dict[str, object]: - """Attach raw-capture review state and guidance to a memory brief.""" + """Attach raw-capture review state, backlog signal, and guidance to a brief.""" result = dict(payload) capture_payload = dict(captures) guidance = [str(item) for item in result.get("agent_guidance", [])] @@ -1793,6 +1796,18 @@ def add_capture_review_to_brief( guidance.append("Redact raw captures with secret warnings before sharing snippets or using their contents.") if read_warning_count: guidance.append("Fix unreadable raw captures before deciding whether capture memory should be accepted or deleted.") + review = result.get("review") if isinstance(result.get("review"), Mapping) else {} + backlog = memory_backlog_summary( + capture_count=capture_count, + needs_review_count=int(review.get("count") or 0), + command_target=command_target, + ) + result["backlog"] = backlog + if backlog.get("backlog"): + guidance.append( + "The memory backlog is above threshold; offer the user a short consolidation pass " + f"({backlog.get('command')} prints a read-only plan with approve/discard commands)." + ) result["agent_guidance"] = guidance return result @@ -1804,6 +1819,7 @@ def memory_brief( review_command: str = "review-memory", project: str | None = None, command_target: str | Path = ".", + semantic_scores: Mapping[str, float] | None = None, ) -> dict[str, object]: """Return the compact memory payload an agent should read before work.""" limit = max(1, min(limit, 20)) @@ -1823,7 +1839,9 @@ def memory_brief( ) if q: - relevant = recall_memories(record_list, q, limit=limit, project=project_name) + relevant = recall_memories( + record_list, q, limit=limit, project=project_name, semantic_scores=semantic_scores + ) selection = "query" else: relevant = [] @@ -2007,6 +2025,7 @@ def recall_memories( limit: int = 10, include_archived: bool = False, project: str | None = None, + semantic_scores: Mapping[str, Mapping[str, float]] | None = None, ) -> list[dict[str, object]]: q = query.strip() if not q: @@ -2019,14 +2038,28 @@ def recall_memories( continue if not include_archived and not is_active_memory(record): continue - score = score_memory(record, q) + lexical_score = score_memory(record, q) + semantic_match = None + if semantic_scores: + semantic_match = semantic_scores.get(str(record.get("name") or "")) + score = lexical_score + semantic_match_points(semantic_match) if score >= MEMORY_RECALL_MIN_SCORE: + lexical_hit = lexical_score >= MEMORY_RECALL_MIN_SCORE rank_score = memory_rank_score(record, score, project=project_name) issues = memory_review_issues(record) slim = slim_memory(record) slim["score"] = score slim["rank_score"] = rank_score - slim["confidence"] = memory_recall_confidence(record, q) + slim["match"] = ( + "hybrid" if (lexical_hit and semantic_match) else ("semantic" if semantic_match else "lexical") + ) + if semantic_match: + slim["semantic_similarity"] = float(semantic_match.get("cosine") or 0.0) + # A match with no lexical evidence is honest about its basis: a + # close paraphrase is at most moderate confidence, never strong. + slim["confidence"] = ( + memory_recall_confidence(record, q) if lexical_hit else semantic_confidence_cap(semantic_match) + ) slim["recall"] = recall_state(record, issues) slim["review_issue_count"] = len(issues) slim["highest_review_severity"] = ( diff --git a/mcp_package/link_core/query.py b/mcp_package/link_core/query.py index 4f43da46..6c896b8a 100644 --- a/mcp_package/link_core/query.py +++ b/mcp_package/link_core/query.py @@ -16,6 +16,7 @@ normalize_project, recall_memories, ) +from .semantic import semantic_memory_scores from .wiki import context_for_topic, search_pages @@ -415,11 +416,13 @@ def query_link( "context_packet": [], } + semantic_scores = semantic_memory_scores(wiki_dir.parent, q, record_list) raw_memories = recall_memories( record_list, q, limit=limits["memories"] + 1, project=project_name, + semantic_scores=semantic_scores, ) memory_has_more = len(raw_memories) > limits["memories"] memories = [_compact_memory(memory) for memory in raw_memories[: limits["memories"]]] @@ -429,6 +432,7 @@ def query_link( limit=limits["memories"], review_command=review_command, project=project_name, + semantic_scores=semantic_scores, ) raw_search_results = search_pages(q, cache, limit=limits["search_results"] + 1) search_has_more = len(raw_search_results) > limits["search_results"] diff --git a/mcp_package/link_core/semantic.py b/mcp_package/link_core/semantic.py new file mode 100644 index 00000000..f5e65427 --- /dev/null +++ b/mcp_package/link_core/semantic.py @@ -0,0 +1,438 @@ +"""Optional local semantic recall for Link. + +Lexical recall stays the default and the fallback. When the optional local +embedding provider is installed (`pip install "link-mcp[semantic]"`) and its +small static-embedding model has been fetched once through the explicit +`lnk semantic --setup` command, memory recall additionally retrieves close +paraphrases ("how do I like my PRs structured" finding a memory phrased +around "commit style") that token matching misses. + +Local-first guarantees preserved: +- No network at recall time, ever: model loading is forced offline + (`HF_HUB_OFFLINE=1`) everywhere except the explicit setup command, so a + query can never trigger a download. +- No services, no vector database: embeddings live in a plain JSON cache + under `.link-cache/semantic/`, similarity is brute-force cosine in pure + Python — personal wikis have hundreds of memories, not millions. +- Deterministic degradation: if the provider, model, or cache is missing or + broken, every entry point returns empty results and recall behaves exactly + as before. +""" +from __future__ import annotations + +import hashlib +import json +import math +import os +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path + +from .files import atomic_write_text + +DEFAULT_SEMANTIC_MODEL = "minishlab/potion-base-8M" +SEMANTIC_MODEL_ENV = "LINK_SEMANTIC_MODEL" +SEMANTIC_DISABLE_ENV = "LINK_SEMANTIC" +SEMANTIC_INDEX_VERSION = 1 + +# Absolute cosine values from small static-embedding models are not +# comparable across queries (a correct match can score 0.25 on one query and +# 0.55 on another), so candidate selection is *standout-based*: a memory +# counts as a semantic match when its similarity stands out from the rest of +# the corpus for this query (z-score), with a small absolute floor to reject +# noise-on-noise. `strength` in [0, 1] expresses how much it stands out. +SEMANTIC_NOISE_FLOOR = 0.15 +SEMANTIC_STANDOUT_Z = 1.0 +SEMANTIC_MAX_CANDIDATES = 5 +SEMANTIC_MODERATE_STRENGTH = 0.5 +# Small corpora make standout statistics unstable; fall back to absolute. +SEMANTIC_MIN_CORPUS_FOR_STANDOUT = 5 +SEMANTIC_MIN_COSINE = 0.35 + +Embedder = Callable[[list[str]], list[list[float]]] + +_MODEL_CACHE: dict[str, object] = {} + +# Two provider tiers, both fully local: +# - "fastembed" (quality): contextual ONNX sentence embeddings. Best recall; +# ~5 s one-time model load, so it shines in long-lived processes like the +# MCP server. Preferred automatically when installed. +# - "model2vec" (fast): tiny static embeddings. ~100 ms load, ideal for +# short-lived CLI calls and session-start hooks. +SEMANTIC_PROVIDER_ENV = "LINK_SEMANTIC_PROVIDER" +DEFAULT_FASTEMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2" + + +def _provider_override() -> str: + return os.environ.get(SEMANTIC_PROVIDER_ENV, "").strip().lower() + + +def _fastembed_installed() -> bool: + try: + import fastembed # noqa: F401 + except Exception: + return False + return True + + +def _model2vec_installed() -> bool: + try: + import model2vec # noqa: F401 + except Exception: + return False + return True + + +def semantic_provider() -> str | None: + """Return the active provider name, or None when nothing is installed.""" + override = _provider_override() + if override == "fastembed": + return "fastembed" if _fastembed_installed() else None + if override == "model2vec": + return "model2vec" if _model2vec_installed() else None + if _fastembed_installed(): + return "fastembed" + if _model2vec_installed(): + return "model2vec" + return None + + +def semantic_model_name() -> str: + override = os.environ.get(SEMANTIC_MODEL_ENV, "").strip() + if override: + return override + if semantic_provider() == "fastembed": + return DEFAULT_FASTEMBED_MODEL + return DEFAULT_SEMANTIC_MODEL + + +def semantic_model_key() -> str: + """Provider-qualified model id; changing provider or model rebuilds the index.""" + return f"{semantic_provider() or 'none'}:{semantic_model_name()}" + + +def semantic_disabled() -> bool: + return os.environ.get(SEMANTIC_DISABLE_ENV, "").strip().lower() in {"0", "off", "false", "no"} + + +def provider_installed() -> bool: + return semantic_provider() is not None + + +def _set_offline_guard(allow_download: bool) -> None: + if not allow_download: + # Force offline so recall can never silently reach the network. + os.environ["HF_HUB_OFFLINE"] = "1" + else: + os.environ.pop("HF_HUB_OFFLINE", None) + + +def _load_model(allow_download: bool = False): + """Load the embedding model; offline unless setup explicitly allows.""" + provider = semantic_provider() + model_name = semantic_model_name() + cache_key = f"{provider}:{model_name}" + cached = _MODEL_CACHE.get(cache_key) + if cached is not None: + return cached + _set_offline_guard(allow_download) + if provider == "fastembed": + from fastembed import TextEmbedding + + model = TextEmbedding(model_name) + else: + from model2vec import StaticModel + + model = StaticModel.from_pretrained(model_name) + _MODEL_CACHE[cache_key] = model + return model + + +def load_embedder(allow_download: bool = False) -> Embedder | None: + """Return a batch embedding callable, or None when unavailable.""" + provider = semantic_provider() + if semantic_disabled() or provider is None: + return None + try: + model = _load_model(allow_download=allow_download) + except Exception: + return None + + if provider == "fastembed": + def _embed(texts: list[str]) -> list[list[float]]: + return [[float(value) for value in vector] for vector in model.embed(texts)] + else: + def _embed(texts: list[str]) -> list[list[float]]: + return [[float(value) for value in vector] for vector in model.encode(texts)] + + return _embed + + +def model_available() -> bool: + """True when the model is loadable fully offline.""" + return load_embedder(allow_download=False) is not None + + +def _normalize(vector: list[float]) -> list[float]: + norm = math.sqrt(sum(value * value for value in vector)) + if norm <= 0: + return vector + return [value / norm for value in vector] + + +def _cosine(a: list[float], b: list[float]) -> float: + # Vectors are stored normalized, so cosine is a plain dot product. + return sum(x * y for x, y in zip(a, b)) + + +def _content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()[:16] + + +def memory_embedding_text(record: Mapping[str, object]) -> str: + """The bounded text that represents one memory in the semantic index.""" + tags = " ".join(str(tag) for tag in record.get("tags", []) if str(tag).strip()) + parts = [ + str(record.get("title") or ""), + str(record.get("tldr") or ""), + tags, + str(record.get("body") or "")[:1000], + ] + return "\n".join(part for part in parts if part.strip()) + + +def semantic_index_path(root: Path) -> Path: + return root.expanduser().resolve() / ".link-cache" / "semantic" / "memories.json" + + +def _load_index(path: Path) -> dict[str, object]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, ValueError): + return {} + if not isinstance(payload, dict) or payload.get("version") != SEMANTIC_INDEX_VERSION: + return {} + return payload + + +def _save_index(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + atomic_write_text(path, json.dumps(payload)) + + +def refresh_memory_index( + root: Path, + records: Iterable[Mapping[str, object]], + *, + embedder: Embedder, + model_name: str | None = None, +) -> dict[str, object]: + """Embed new or changed memories; prune deleted ones. Returns the index.""" + model = model_name or semantic_model_key() + path = semantic_index_path(root) + index = _load_index(path) + items = index.get("items") if isinstance(index.get("items"), dict) else {} + if index.get("model") != model: + items = {} + + wanted: dict[str, str] = {} + texts_by_name: dict[str, str] = {} + for record in records: + name = str(record.get("name") or "").strip() + if not name: + continue + text = memory_embedding_text(record) + if not text.strip(): + continue + wanted[name] = _content_hash(text) + texts_by_name[name] = text + + stale = [ + name for name, digest in wanted.items() + if not isinstance(items.get(name), dict) or items[name].get("hash") != digest + ] + removed = [name for name in list(items) if name not in wanted] + if stale: + vectors = embedder([texts_by_name[name] for name in stale]) + for name, vector in zip(stale, vectors): + items[name] = { + "hash": wanted[name], + "vec": [round(value, 5) for value in _normalize(vector)], + } + for name in removed: + items.pop(name, None) + + payload = {"version": SEMANTIC_INDEX_VERSION, "model": model, "items": items} + if stale or removed or not path.exists(): + _save_index(path, payload) + return payload + + +def _candidate_strengths(cosines: dict[str, float]) -> dict[str, dict[str, float]]: + """Select standout candidates and grade each with a strength in [0, 1].""" + if not cosines: + return {} + values = list(cosines.values()) + if len(values) < SEMANTIC_MIN_CORPUS_FOR_STANDOUT: + # Too few memories for standout statistics: absolute fallback. + return { + name: { + "cosine": round(value, 4), + "strength": round(min(1.0, max(0.0, (value - SEMANTIC_MIN_COSINE) / 0.3)), 4), + } + for name, value in cosines.items() + if value >= SEMANTIC_MIN_COSINE + } + mean = sum(values) / len(values) + variance = sum((value - mean) ** 2 for value in values) / len(values) + std = math.sqrt(variance) or 1e-6 + ranked = sorted(cosines.items(), key=lambda item: item[1], reverse=True) + candidates: dict[str, dict[str, float]] = {} + for name, value in ranked[:SEMANTIC_MAX_CANDIDATES]: + if value < SEMANTIC_NOISE_FLOOR: + continue + z = (value - mean) / std + if z < SEMANTIC_STANDOUT_Z: + continue + strength = min(1.0, max(0.0, (z - SEMANTIC_STANDOUT_Z) / 2.5)) + if strength <= 0: + continue + candidates[name] = {"cosine": round(value, 4), "strength": round(strength, 4)} + return candidates + + +def semantic_memory_scores( + root: Path, + query: str, + records: Iterable[Mapping[str, object]], + *, + embedder: Embedder | None = None, +) -> dict[str, dict[str, float]]: + """Return {memory name: {cosine, strength}} for the query, or {}. + + Never raises and never touches the network: any missing provider, model, + cache, or unexpected error degrades to lexical-only recall. + """ + q = query.strip() + if not q: + return {} + try: + active_embedder = embedder or load_embedder(allow_download=False) + if active_embedder is None: + return {} + index = refresh_memory_index(root, records, embedder=active_embedder) + items = index.get("items") + if not isinstance(items, dict) or not items: + return {} + query_vector = _normalize(active_embedder([q])[0]) + cosines: dict[str, float] = {} + for name, entry in items.items(): + vector = entry.get("vec") if isinstance(entry, dict) else None + if not isinstance(vector, list): + continue + cosines[name] = _cosine(query_vector, vector) + return _candidate_strengths(cosines) + except Exception: + return {} + + +def semantic_match_points(match: Mapping[str, float] | None) -> int: + """Map a semantic match's strength onto the lexical match-score scale. + + A barely-standout candidate contributes little; a clear standout can + clear the recall floor on its own but never dominates an exact lexical + hit (max 10 points vs 20+ for a verbatim title match). + """ + if not match: + return 0 + strength = float(match.get("strength") or 0.0) + return max(0, round(strength * 10)) + + +def semantic_confidence_cap(match: Mapping[str, float] | None) -> str: + """Honest confidence for a match with no lexical evidence.""" + strength = float(match.get("strength") or 0.0) if match else 0.0 + return "moderate" if strength >= SEMANTIC_MODERATE_STRENGTH else "weak" + + +def build_semantic_status( + root: Path, + *, + memory_count: int, + command_target: str | Path = ".", + python_cmd: str | None = None, +) -> dict[str, object]: + """Readiness report for the optional semantic recall layer.""" + provider = semantic_provider() + installed = provider is not None + disabled = semantic_disabled() + ready = False + index_items = 0 + index = _load_index(semantic_index_path(root)) + items = index.get("items") + if isinstance(items, dict): + index_items = len(items) + if installed and not disabled: + ready = model_available() + + next_actions: list[str] = [] + if disabled: + next_actions.append(f"unset {SEMANTIC_DISABLE_ENV} to re-enable semantic recall") + elif not installed: + next_actions.append('pip install "link-mcp[semantic]" # fast tier (tiny static model)') + next_actions.append('pip install "link-mcp[semantic-quality]" # quality tier (contextual model)') + next_actions.append(f"lnk semantic {command_target} --setup") + elif not ready: + next_actions.append(f"lnk semantic {command_target} --setup") + elif index_items < memory_count: + next_actions.append(f"lnk semantic {command_target} --rebuild") + if installed and provider == "model2vec" and not _fastembed_installed(): + next_actions.append( + 'optional quality upgrade: pip install "link-mcp[semantic-quality]" then rerun --setup' + ) + + tier = None + if provider == "fastembed": + tier = "quality (contextual embeddings; ~5s load, best for the MCP server)" + elif provider == "model2vec": + tier = "fast (static embeddings; instant load, best for CLI and hooks)" + + return { + "enabled": ready, + "disabled_by_env": disabled, + "provider": provider, + "tier": tier, + "python": python_cmd, + "model": semantic_model_name(), + "model_available_offline": ready, + "index_path": str(semantic_index_path(root)), + "indexed_memories": index_items, + "memory_count": memory_count, + "mode": "hybrid (lexical + semantic)" if ready else "lexical only", + "network_policy": ( + "Recall never downloads anything: the model loads offline-only. " + "Only `lnk semantic --setup` may fetch the model, once, with your approval." + ), + "next_actions": next_actions, + } + + +def render_semantic_status_text(payload: Mapping[str, object]) -> tuple[int, str]: + lines = [ + "Link semantic recall", + "", + f"Mode: {payload.get('mode')}", + f"Provider: {payload.get('provider') or 'not installed'}" + + (f" · {payload.get('tier')}" if payload.get("tier") else ""), + *( [f"Python: {payload.get('python')}"] if payload.get("python") else [] ), + f"Model: {payload.get('model')}", + f"Indexed memories: {payload.get('indexed_memories')} of {payload.get('memory_count')}", + f"Index: {payload.get('index_path')}", + ] + if payload.get("disabled_by_env"): + lines.append(f"Disabled via {SEMANTIC_DISABLE_ENV} environment variable.") + actions = payload.get("next_actions") + if isinstance(actions, list) and actions: + lines.extend(["", "Next:"]) + lines.extend(f" {action}" for action in actions) + lines.extend(["", str(payload.get("network_policy") or "")]) + return 0, "\n".join(lines) diff --git a/mcp_package/link_core/version.py b/mcp_package/link_core/version.py index df830011..35c92fba 100644 --- a/mcp_package/link_core/version.py +++ b/mcp_package/link_core/version.py @@ -1,4 +1,4 @@ """Shared Link release version.""" from __future__ import annotations -LINK_VERSION = "1.5.0" +LINK_VERSION = "1.6.0" diff --git a/mcp_package/link_mcp/__init__.py b/mcp_package/link_mcp/__init__.py index 90c25129..8a1e3b35 100644 --- a/mcp_package/link_mcp/__init__.py +++ b/mcp_package/link_mcp/__init__.py @@ -1,2 +1,2 @@ """Link MCP Server — personal knowledge wiki as MCP tools.""" -__version__ = "1.5.0" +__version__ = "1.6.0" diff --git a/mcp_package/link_mcp/server.py b/mcp_package/link_mcp/server.py index 3231053f..25b5b524 100644 --- a/mcp_package/link_mcp/server.py +++ b/mcp_package/link_mcp/server.py @@ -35,10 +35,27 @@ from link_core.version import LINK_VERSION # ── Resolve wiki directory ──────────────────────────────────────────── +# The parser keeps add_help=False and parse_known_args so an agent launch +# config with unexpected args can never crash the server. Handle --help +# explicitly first: without this, `python -m link_mcp --help` would start +# the stdio server and hang silently waiting for MCP messages. +if "-h" in sys.argv[1:] or "--help" in sys.argv[1:]: + print(__doc__.strip()) + print( + "\nOptions:\n" + " --wiki PATH wiki directory (default: ~/link/wiki)\n" + " --surface SURFACE tool surface: slim (recommended) or full\n" + " --version print the link-mcp version and exit\n" + " --semantic-setup one-time semantic model fetch + index build\n" + " -h, --help show this help and exit" + ) + sys.exit(0) + parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--wiki", default=None) parser.add_argument("--surface", choices=("full", "slim"), default=None) parser.add_argument("--version", action="store_true") +parser.add_argument("--semantic-setup", action="store_true") args, _ = parser.parse_known_args() if args.version: @@ -50,6 +67,37 @@ else: WIKI_DIR = Path.home() / "link" / "wiki" +if args.semantic_setup: + # One-time explicit opt-in for MCP-only installs (no `lnk` CLI): fetch + # the local embedding model and build the semantic index, then exit. + # This is the only link-mcp entry point allowed to touch the network. + from link_core.memory import memory_records as _setup_memory_records + from link_core.semantic import ( + load_embedder as _setup_load_embedder, + refresh_memory_index as _setup_refresh_index, + semantic_model_name as _setup_model_name, + ) + + if not WIKI_DIR.exists(): + print(f"[link-mcp] Wiki not found at {WIKI_DIR}; pass --wiki /path/to/wiki.", file=sys.stderr) + sys.exit(2) + print( + f"[link-mcp] Setting up semantic recall: this may download {_setup_model_name()} " + "once. Recall itself never uses the network." + ) + setup_embedder = _setup_load_embedder(allow_download=True) + if setup_embedder is None: + print( + "[link-mcp] Semantic provider unavailable. Install it first: " + "pip install \"link-mcp[semantic]\"", + file=sys.stderr, + ) + sys.exit(2) + setup_index = _setup_refresh_index(WIKI_DIR.parent, _setup_memory_records(WIKI_DIR), embedder=setup_embedder) + setup_items = setup_index.get("items") if isinstance(setup_index.get("items"), dict) else {} + print(f"[link-mcp] Semantic recall ready: indexed {len(setup_items)} memories.") + sys.exit(0) + MCP_SURFACE = (args.surface or os.environ.get("LINK_MCP_SURFACE") or "slim").strip().lower() if MCP_SURFACE not in {"full", "slim"}: print( @@ -205,6 +253,12 @@ def _slim_tool(): redact_capture_file as _core_redact_capture_file, write_session_capture as _core_write_session_capture, ) +from link_core.consolidate import ( + build_consolidation_plan as _core_build_consolidation_plan, +) +from link_core.semantic import ( + semantic_memory_scores as _core_semantic_memory_scores, +) from link_core.files import ( atomic_write_json as _core_atomic_write_json, ) @@ -431,12 +485,17 @@ def _memory_profile(limit: int = 10, project: str = "") -> dict[str, object]: def _memory_brief(query: str = "", limit: int = 6, project: str = "") -> dict[str, object]: project_name = _resolve_project(project) + clean_query = _clean_text_input(query, max_len=500) + records = _memory_records() payload = _core_memory_brief( - _memory_records(), query=_clean_text_input(query, max_len=500), + records, query=clean_query, limit=limit, review_command="review_memory", project=project_name, command_target=WIKI_DIR.parent, + semantic_scores=_core_semantic_memory_scores(WIKI_DIR.parent, clean_query, records), + ) + return _core_add_capture_review_to_brief( + payload, _capture_review_summary(project=project_name), command_target=WIKI_DIR.parent ) - return _core_add_capture_review_to_brief(payload, _capture_review_summary(project=project_name)) def _query_link(query: str, budget: str = "medium", project: str = "") -> dict[str, object]: @@ -513,12 +572,14 @@ def _recall_memories( project: str = "", ) -> list[dict[str, object]]: query = _clean_text_input(query) + records = _memory_records() return _core_recall_memories( - _memory_records(), + records, query, limit=limit, include_archived=include_archived, project=_resolve_project(project), + semantic_scores=_core_semantic_memory_scores(WIKI_DIR.parent, query, records), ) @@ -882,7 +943,14 @@ def link_instructions_resource() -> str: "6. At session end, use `admin(action=\"session_end\", arguments=\"{...}\")` or `capture_session` " "to save proposal-only notes for user review.\n" "7. Use `review` for inbox, explain, archive, restore, forget, profile, audit, and log workflows.\n" - "8. Use `admin` only for maintenance, graph/context expansion, pages, backups, migrations, and captures.\n\n" + "8. If a brief reports a memory backlog, offer the user a short consolidation pass: " + "`review(action=\"consolidate\")` returns a read-only plan; apply its accept/discard actions only " + "after the user approves each one.\n" + "9. Use `admin` only for maintenance, graph/context expansion, pages, backups, migrations, and captures.\n\n" + "If Link session hooks are installed for this agent, the startup brief is injected automatically — " + "skip step 2 and go straight to bounded task recall.\n" + "Recalled memories carry a `match` field: treat `semantic` matches (paraphrase similarity, capped " + "confidence) as hints to verify, not facts to act on.\n\n" "Never silently save durable memory. Prefer reviewed memories and source-backed wiki pages, and cite " "provenance when explaining why Link knows something.\n" ) @@ -1218,8 +1286,10 @@ def review( """Review, explain, and manage local memory lifecycle. Supported actions: inbox, audit, profile, log, wins, explain, reviewed, - archive, restore, forget. Prefer archive over forget unless the user asks - for permanent deletion. + archive, restore, forget, consolidate. Prefer archive over forget unless + the user asks for permanent deletion. Use consolidate for a read-only plan + when the capture or review backlog builds up; apply its actions only after + the user approves each one. """ clean_action = (_clean_text_input(action, max_len=80) or "inbox").lower().replace("-", "_") parsed_limit = _parse_limit(limit, default=20, max_limit=50) @@ -1245,12 +1315,19 @@ def review( payload = _set_memory_status(identifier, "active") elif clean_action == "forget": payload = _forget_memory(identifier, confirm=confirm) + elif clean_action == "consolidate": + payload = _core_build_consolidation_plan( + captures_payload=_capture_inbox(limit=parsed_limit, project=clean_project), + inbox_payload=_memory_inbox(limit=parsed_limit, project=clean_project), + command_target=WIKI_DIR.parent, + project=clean_project, + ) else: return json.dumps({ "surface": "slim", "tool": "review", "error": f"unsupported action: {clean_action}", - "supported_actions": ["inbox", "audit", "profile", "log", "wins", "explain", "reviewed", "archive", "restore", "forget"], + "supported_actions": ["inbox", "audit", "profile", "log", "wins", "explain", "reviewed", "archive", "restore", "forget", "consolidate"], }) except ValueError as exc: return json.dumps({"surface": "slim", "tool": "review", "updated": False, "error": str(exc)}) diff --git a/mcp_package/pyproject.toml b/mcp_package/pyproject.toml index 8468ab09..864f2ea1 100644 --- a/mcp_package/pyproject.toml +++ b/mcp_package/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "link-mcp" -version = "1.5.0" +version = "1.6.0" description = "MCP server for Link local agent memory — remember, recall, search, context, and graph traversal" readme = "README.md" license = { text = "MIT" } @@ -24,6 +24,10 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] +[project.optional-dependencies] +semantic = ["model2vec>=0.3"] +semantic-quality = ["fastembed>=0.5"] + [project.urls] Homepage = "https://github.com/gowtham0992/link" Repository = "https://github.com/gowtham0992/link" diff --git a/mcp_package/server.json b/mcp_package/server.json index 40470da3..95aff742 100644 --- a/mcp_package/server.json +++ b/mcp_package/server.json @@ -6,12 +6,12 @@ "url": "https://github.com/gowtham0992/link", "source": "github" }, - "version": "1.5.0", + "version": "1.6.0", "packages": [ { "registryType": "pypi", "identifier": "link-mcp", - "version": "1.5.0", + "version": "1.6.0", "transport": { "type": "stdio" } diff --git a/scripts/check_tool_contract.py b/scripts/check_tool_contract.py index d37d74e0..05b9360b 100644 --- a/scripts/check_tool_contract.py +++ b/scripts/check_tool_contract.py @@ -17,6 +17,7 @@ "capture-inbox", "capture-session", "connect", + "consolidate", "compliance-export", "delete-capture", "demo", @@ -26,6 +27,7 @@ "forget-memory", "graph-summary", "health", + "hook", "import-obsidian", "ingest-status", "init", @@ -52,6 +54,7 @@ "restore-memory", "review-memory", "seed", + "semantic", "serve", "set-memory-visibility", "session-end", diff --git a/scripts/eval_locomo.py b/scripts/eval_locomo.py new file mode 100644 index 00000000..cef72da5 --- /dev/null +++ b/scripts/eval_locomo.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Third-party retrieval track: LoCoMo evidence retrieval with Link recall. + +LoCoMo (Maharana et al., ACL 2024, Snap Research) is the long-term +conversational memory benchmark the hosted-memory industry quotes. This track +uses only its third-party ground truth — no LLM, no judging, no generation: + +- every dialog turn of a conversation becomes one Link memory record; +- every evidence-annotated question becomes a recall query; +- we measure whether Link's ranking returns the annotated evidence turns + (any-evidence hit@k and evidence recall@k), lexical vs hybrid. + +This is NOT the LoCoMo QA task (no answers are generated or scored), so the +numbers are not comparable to end-to-end LLM QA scores quoted elsewhere; it +isolates the retrieval stage with third-party queries and third-party gold +labels over third-party conversations. + +Dataset: locomo10.json, CC BY-NC 4.0, (c) Snap Inc. Not redistributed here — +download it yourself first (this script contains no network code): + + curl -L -o /tmp/locomo10.json \ + https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json + +Run: + python3 scripts/eval_locomo.py /tmp/locomo10.json --mode off # lexical + python3 scripts/eval_locomo.py /tmp/locomo10.json --mode real # hybrid +""" +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "mcp_package")) + +from link_core.memory import recall_memories # noqa: E402 +from link_core.semantic import load_embedder, semantic_memory_scores # noqa: E402 + +ADVERSARIAL_CATEGORY = 5 + + +def _turn_records(sample: dict) -> list[dict[str, object]]: + records: list[dict[str, object]] = [] + conversation = sample["conversation"] + session = 1 + while f"session_{session}" in conversation: + date = str(conversation.get(f"session_{session}_date_time") or "") + for turn in conversation[f"session_{session}"] or []: + text = str(turn.get("text") or "").strip() or str(turn.get("blip_caption") or "").strip() + if not text: + continue + records.append({ + "name": str(turn.get("dia_id")), + "title": f"{turn.get('speaker')} (session {session})", + "tldr": date, + "tags": [], + "body": text, + "status": "active", + "scope": "user", + "memory_type": "fact", + "review_status": "reviewed", + }) + session += 1 + return records + + +def _queries(sample: dict) -> list[dict[str, object]]: + queries = [] + for qa in sample.get("qa", []): + if int(qa.get("category") or 0) == ADVERSARIAL_CATEGORY: + continue + evidence = qa.get("evidence") or [] + if not isinstance(evidence, list) or not evidence: + continue + queries.append({"question": str(qa.get("question") or ""), "evidence": [str(e) for e in evidence]}) + return queries + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dataset", help="path to locomo10.json (see module docstring for the download command)") + parser.add_argument("--mode", choices=["off", "real"], default="off") + parser.add_argument("--k", type=int, default=10) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + dataset_path = Path(args.dataset).expanduser() + if not dataset_path.exists(): + print(f"Dataset not found: {dataset_path}", file=sys.stderr) + print("Download it first (CC BY-NC 4.0, (c) Snap Inc.):", file=sys.stderr) + print( + " curl -L -o /tmp/locomo10.json " + "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json", + file=sys.stderr, + ) + return 2 + + embedder = None + if args.mode == "real": + embedder = load_embedder(allow_download=False) + if embedder is None: + print( + "Semantic model unavailable offline. Install a provider and run " + "`lnk semantic --setup` (or `python3 -m link_mcp --semantic-setup`) first.", + file=sys.stderr, + ) + return 2 + + samples = json.loads(dataset_path.read_text(encoding="utf-8")) + k = max(1, args.k) + total_queries = 0 + total_turns = 0 + any_hits = {1: 0, 5: 0, k: 0} + evidence_recall: list[float] = [] + latencies: list[float] = [] + + with tempfile.TemporaryDirectory() as temp: + for index, sample in enumerate(samples): + records = _turn_records(sample) + queries = _queries(sample) + total_turns += len(records) + root = Path(temp) / f"conv-{index}" + for query in queries: + started = time.perf_counter() + scores = ( + semantic_memory_scores(root, query["question"], records, embedder=embedder) + if embedder is not None + else None + ) + results = recall_memories(records, query["question"], limit=k, semantic_scores=scores) + latencies.append((time.perf_counter() - started) * 1000) + names = [str(item["name"]) for item in results] + gold = set(query["evidence"]) + for cutoff in any_hits: + if gold & set(names[:cutoff]): + any_hits[cutoff] += 1 + evidence_recall.append(len(gold & set(names[:k])) / len(gold)) + total_queries += 1 + + report = { + "dataset": "LoCoMo locomo10.json (CC BY-NC 4.0, Snap Inc.) — retrieval stage only", + "mode": args.mode, + "conversations": len(samples), + "turn_memories": total_turns, + "queries": total_queries, + "any_evidence_hit@1": round(any_hits[1] / total_queries, 4), + "any_evidence_hit@5": round(any_hits[5] / total_queries, 4), + f"any_evidence_hit@{k}": round(any_hits[k] / total_queries, 4), + f"evidence_recall@{k}": round(statistics.fmean(evidence_recall), 4), + "latency_ms_p50": round(statistics.median(latencies), 2), + "latency_ms_mean": round(statistics.fmean(latencies), 2), + } + if args.json: + print(json.dumps(report, indent=2)) + else: + for key, value in report.items(): + print(f"{key}: {value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval_recall_quality.py b/scripts/eval_recall_quality.py new file mode 100644 index 00000000..9c535274 --- /dev/null +++ b/scripts/eval_recall_quality.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Benchmark Link memory recall quality: lexical vs hybrid (semantic) recall. + +Dataset: scripts/recall_dataset.py — fully authored, deterministic, auditable +(no LLM, no network, no randomness). Queries are classified by *measured* +token overlap with their target memory, not by how they were authored: + +- token-overlap: the query shares at least one significant stemmed token + with its target memory (lexical recall has a fighting chance) +- zero-overlap: the query provably shares no significant stemmed token with + its target (pure paraphrase; token matching cannot find it directly) + +Metrics per group and mode: hit@1, hit@3, hit@5, MRR@5, plus recall latency. + +Modes: +- --mode off lexical-only baseline +- --mode fake deterministic synonym-axis embedder (CI-safe, no model) +- --mode real the actual local model (pip install "link-mcp[semantic]"; + pass --allow-download to fetch it here explicitly) + +Exit code is non-zero if hybrid recall scores below lexical recall on any +group metric (hybrid must never regress lexical behavior). + +Reproduce the published numbers: + python3 -m venv /tmp/linkbench && /tmp/linkbench/bin/pip install model2vec + /tmp/linkbench/bin/python scripts/eval_recall_quality.py \ + --suite full --mode real --allow-download +""" +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "mcp_package")) +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) + +from link_core.memory import ( # noqa: E402 + memory_tokens, + recall_memories, + significant_memory_tokens, + stemmed_memory_tokens, +) +from link_core.semantic import load_embedder, semantic_memory_scores # noqa: E402 +from recall_dataset import build_cases, build_corpus # noqa: E402 +from test_semantic_core import fake_embedder # noqa: E402 + +RANK_LIMIT = 5 + + +def _target_tokens(memory: dict[str, object]) -> set[str]: + text = " ".join([ + str(memory.get("title") or ""), + str(memory.get("tldr") or ""), + " ".join(str(tag) for tag in memory.get("tags", [])), + str(memory.get("body") or ""), + ]) + return stemmed_memory_tokens(memory_tokens(text)) + + +def classify_cases(cases: list[dict[str, str]], corpus: list[dict[str, object]]) -> None: + """Annotate each case with its measured overlap group.""" + tokens_by_name = {str(memory["name"]): _target_tokens(memory) for memory in corpus} + for case in cases: + query_tokens = stemmed_memory_tokens(significant_memory_tokens(case["query"])) + overlap = query_tokens & tokens_by_name[case["target"]] + case["group"] = "token-overlap" if overlap else "zero-overlap" + + +def _blank_stats() -> dict[str, float]: + return {"hit@1": 0.0, "hit@3": 0.0, "hit@5": 0.0, "mrr@5": 0.0, "cases": 0} + + +def run_suite( + cases: list[dict[str, str]], + corpus: list[dict[str, object]], + embedder, + root: Path, +) -> dict[str, object]: + groups: dict[str, dict[str, float]] = {} + domains: dict[str, dict[str, float]] = {} + latencies: list[float] = [] + for case in cases: + started = time.perf_counter() + scores = ( + semantic_memory_scores(root, case["query"], corpus, embedder=embedder) + if embedder is not None + else None + ) + results = recall_memories(corpus, case["query"], limit=RANK_LIMIT, semantic_scores=scores) + latencies.append((time.perf_counter() - started) * 1000) + names = [str(item["name"]) for item in results] + rank = names.index(case["target"]) + 1 if case["target"] in names else 0 + for bucket in (groups.setdefault(case["group"], _blank_stats()), + domains.setdefault(case["domain"], _blank_stats())): + bucket["cases"] += 1 + if rank == 1: + bucket["hit@1"] += 1 + if 1 <= rank <= 3: + bucket["hit@3"] += 1 + if 1 <= rank <= 5: + bucket["hit@5"] += 1 + if rank: + bucket["mrr@5"] += 1.0 / rank + for bucket_map in (groups, domains): + for stats in bucket_map.values(): + count = stats["cases"] or 1 + for metric in ("hit@1", "hit@3", "hit@5", "mrr@5"): + stats[metric] = round(stats[metric] / count, 4) + return { + "groups": groups, + "domains": domains, + "latency_ms": { + "p50": round(statistics.median(latencies), 2), + "p95": round(sorted(latencies)[int(len(latencies) * 0.95) - 1], 2), + "mean": round(statistics.fmean(latencies), 2), + }, + } + + +def _print_block(label: str, block: dict[str, object], show_domains: bool) -> None: + print(f"\n{label}:") + for group in sorted(block["groups"]): + stats = block["groups"][group] + print( + f" {group:14s} hit@1 {stats['hit@1']:.3f} hit@3 {stats['hit@3']:.3f}" + f" hit@5 {stats['hit@5']:.3f} mrr@5 {stats['mrr@5']:.3f} ({int(stats['cases'])} cases)" + ) + latency = block["latency_ms"] + print(f" latency/query p50 {latency['p50']}ms p95 {latency['p95']}ms mean {latency['mean']}ms") + if show_domains: + for domain in sorted(block["domains"]): + stats = block["domains"][domain] + print( + f" {domain:12s} hit@1 {stats['hit@1']:.3f} hit@3 {stats['hit@3']:.3f}" + f" ({int(stats['cases'])} cases)" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["off", "fake", "real"], default="fake") + parser.add_argument("--suite", choices=["small", "full"], default="small", + help="small: authored queries only; full: plus deterministic phrasing variants") + parser.add_argument("--allow-download", action="store_true", help="allow the real model to be fetched once") + parser.add_argument("--domains", action="store_true", help="show per-domain breakdown") + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + embedder = None + if args.mode == "fake": + embedder = fake_embedder + elif args.mode == "real": + embedder = load_embedder(allow_download=args.allow_download) + if embedder is None: + print( + "Real model unavailable. Install with: pip install \"link-mcp[semantic]\" " + "and cache the model via `lnk semantic --setup` (or pass --allow-download).", + file=sys.stderr, + ) + return 2 + + corpus = build_corpus() + cases = build_cases(expand=(args.suite == "full")) + classify_cases(cases, corpus) + authored = sum(1 for case in cases if case["authored"] == "yes") + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + report: dict[str, object] = { + "suite": args.suite, + "mode": args.mode, + "corpus_memories": len(corpus), + "total_cases": len(cases), + "authored_cases": authored, + "wrapped_variant_cases": len(cases) - authored, + "lexical_baseline": run_suite(cases, corpus, None, root), + "hybrid": run_suite(cases, corpus, embedder, root) if embedder is not None else None, + } + + if args.json: + print(json.dumps(report, indent=2)) + else: + print( + f"Link recall benchmark — suite: {args.suite}, mode: {args.mode}, " + f"corpus: {report['corpus_memories']} memories, cases: {report['total_cases']} " + f"({authored} authored + {report['wrapped_variant_cases']} phrasing variants)" + ) + _print_block("lexical-only", report["lexical_baseline"], args.domains) + if report["hybrid"] is not None: + _print_block("hybrid", report["hybrid"], args.domains) + + if report["hybrid"] is not None: + baseline_groups = report["lexical_baseline"]["groups"] + hybrid_groups = report["hybrid"]["groups"] + for group, baseline in baseline_groups.items(): + for metric in ("hit@1", "hit@3", "hit@5", "mrr@5"): + if hybrid_groups[group][metric] < baseline[metric]: + print(f"REGRESSION: hybrid {group} {metric} below lexical baseline", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_docs_media.py b/scripts/generate_docs_media.py index 66c5acb7..96f97cae 100644 --- a/scripts/generate_docs_media.py +++ b/scripts/generate_docs_media.py @@ -27,6 +27,8 @@ "link-cli.png", "link-mcp.png", "link-memory-flow.svg", + "link-aha.svg", + "link-aha.gif", "link-ui-tour.gif", "link-cli-tour.gif", "link-mcp-agent-chat.gif", diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index cd7b3c0f..2d7ed236 100644 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -230,6 +230,12 @@ def main() -> int: print("After the PR merges and CI passes, publish with:") for command in release_commands(args.version): print(command) + print("") + print( + "Then bump the Homebrew tap (gowtham0992/homebrew-link) to " + f"{normalize_version(args.version)} so `brew install` serves this " + "version — otherwise new users get an older Link than the docs describe." + ) return 0 diff --git a/scripts/recall_dataset.py b/scripts/recall_dataset.py new file mode 100644 index 00000000..1cfe0694 --- /dev/null +++ b/scripts/recall_dataset.py @@ -0,0 +1,376 @@ +"""Benchmark dataset for Link memory recall quality. + +Deterministic and fully auditable: every memory and query is authored text in +this file — no LLM, no network, no random generation. Each intent contributes +one memory plus several queries. Queries are NOT trusted to be "lexical" or +"paraphrase" by authorship; the benchmark runner classifies each query by its +actual significant-token overlap with the target memory, so the reported +paraphrase group provably shares no significant stemmed token with its target. + +Distractor memories (no queries of their own) grow the corpus so ranking is +measured against realistic competition rather than a handful of candidates. +""" +from __future__ import annotations + +# (name, domain, title, tldr, body, queries) +# Write queries in natural developer voice; mix token-overlapping phrasings +# and zero-overlap paraphrases. ~7 queries per intent. +INTENTS: list[tuple[str, str, str, str, str, list[str]]] = [ + # ── tooling ────────────────────────────────────────────────────────── + ("ruff-linting", "tooling", "Ruff is the Python linter", + "Python linting uses Ruff; flake8 and pylint were retired.", + "All Python linting goes through Ruff with the repo config. flake8 and pylint were removed in the cleanup.", + ["ruff linting", "which linter do we use", "ruff config", + "what checks my python code style", "the tool that flags style problems in our code", + "did we keep flake8", "what runs over the codebase to catch style issues"]), + ("uv-package-manager", "tooling", "uv manages Python packages", + "Use uv instead of pip for installing and locking dependencies.", + "Dependency management uses uv: uv add for new packages, uv lock for the lockfile. Plain pip is only for one-off experiments.", + ["uv package manager", "how do we install dependencies", "uv lock workflow", + "what do I use to pull in a new library", "adding a third party module to the project", + "is plain pip allowed", "how are our requirements pinned"]), + ("pytest-over-unittest", "tooling", "pytest for new tests", + "New tests use pytest style, not unittest classes.", + "The team prefers pytest for new tests: plain functions, fixtures, and parametrize instead of unittest.TestCase classes.", + ["pytest over unittest", "which test framework", "pytest fixtures preference", + "how should I write new test cases", "the style for checking behavior in code we add", + "are TestCase classes ok", "what does the team want for verifying new features"]), + ("prettier-formatting", "tooling", "Prettier formats the frontend", + "All JS/TS formatting is Prettier with the checked-in config.", + "Frontend code is formatted by Prettier using the repo .prettierrc; never hand-format or argue style in review.", + ["prettier formatting", "how is javascript formatted", "prettier config location", + "what keeps the frontend code style consistent", "tool that rewrites my typescript layout", + "should style comments go in code review", "who decides whitespace in the web code"]), + ("make-targets", "tooling", "Make targets drive common tasks", + "Use make test, make lint, make dev instead of raw commands.", + "Common workflows are wrapped in Makefile targets: make test, make lint, make dev. Raw commands drift from CI.", + ["make targets", "makefile commands", "make test lint dev", + "the shortcuts for running everyday project chores", "how do I start the local development loop", + "what wraps our common shell invocations", "is there a single entry point for routine jobs"]), + ("node-version", "tooling", "Node 22 LTS is required", + "The frontend builds on Node 22 LTS; older majors fail.", + "Builds require Node 22 LTS. Engines are pinned in package.json and CI fails on older majors.", + ["node version", "which node do we build with", "node 22 requirement", + "what javascript runtime release must be installed", "my frontend build fails on an old runtime", + "minimum engine for the web build", "runtime prerequisite for compiling the ui"]), + ("docker-compose-dev", "tooling", "docker compose runs local services", + "Local Postgres and Redis come from docker compose up.", + "Local development services (Postgres, Redis) run via docker compose up; never install them directly on the laptop.", + ["docker compose dev", "how do I run local services", "compose up postgres redis", + "getting the database running on my machine", "spin up the supporting backends for hacking locally", + "should I brew install the datastore", "local copies of the storage services"]), + ("precommit-hooks", "tooling", "pre-commit runs before every commit", + "Install pre-commit; it runs lint and format on staged files.", + "The repo uses pre-commit hooks that lint and format staged files; install them with make setup once per clone.", + ["pre-commit hooks", "what runs before a commit", "pre-commit install", + "the thing that cleans files as I check them in", "automatic checks when saving work to git", + "why did my commit get rewritten", "setup step after cloning the repository"]), + # ── git/process ───────────────────────────────────────────────────── + ("commit-style", "process", "Commit and PR structure", + "Small commits; PR description starts with a one-paragraph summary.", + "The user prefers small, focused commits and pull requests whose description opens with a one-paragraph summary followed by bullets.", + ["commit style", "pr description format", "small commits preference", + "how should I structure my pull requests", "the shape reviewers expect for proposed changes", + "what goes at the top when I send work for review", "how granular should my checkpoints be"]), + ("deploy-from-main", "process", "Deploy only from main", + "Production releases ship only from the main branch after CI.", + "Releases ship only from the main branch after CI passes; never deploy from feature branches.", + ["deploy from main", "which branch do we release from", "main branch deploys", + "which branch do we ship production builds from", "where do live rollouts originate", + "can I push my feature straight to prod", "the source of truth for what customers run"]), + ("release-branch-naming", "process", "Release branches are release/x.y.z", + "Cut release branches named release/x.y.z from main.", + "Release preparation happens on branches named release/x.y.z cut from main; tags are created there.", + ["release branch naming", "release/x.y.z convention", "how are release branches named", + "what do I call the branch when cutting a version", "the naming scheme for shipping a new build", + "branch label before tagging", "convention for version preparation work"]), + ("review-before-merge", "process", "Review required before merge", + "Every PR needs at least one approving review.", + "Every pull request needs at least one review pass with approval before merging to the default branch.", + ["review before merge", "pr approval required", "how many reviews per pr", + "can I land this change without another set of eyes", "merging work nobody else looked at", + "who signs off before code goes in", "is a second person needed to accept my patch"]), + ("squash-merge", "process", "Squash-merge pull requests", + "PRs are squash-merged so main stays linear.", + "Pull requests are squash-merged; main history stays linear with one commit per PR.", + ["squash merge", "merge strategy for prs", "linear history main", + "what happens to my many little commits when the pr lands", "how does the trunk history stay tidy", + "do merge commits appear on the default branch", "the collapse policy when accepting changes"]), + ("conventional-commits", "process", "Conventional commit messages", + "Commit subjects follow feat:/fix:/docs: prefixes.", + "Commit messages follow Conventional Commits: feat:, fix:, docs:, chore: prefixes with imperative subjects.", + ["conventional commits", "commit message prefixes", "feat fix docs chore", + "the labeling scheme at the start of change descriptions", "how do I word the subject line of a checkpoint", + "grammar for messages in version history", "standard for describing what a change does"]), + ("no-force-push", "process", "Never force-push shared branches", + "Force-pushing main or develop is forbidden.", + "Never force-push shared branches (main, develop); rewriting published history breaks everyone's clones.", + ["no force push", "force push policy", "rewriting shared branches", + "can I overwrite the remote history others pull from", "rules about rewriting what teammates already fetched", + "why did my history rewrite get reverted", "is amending published work allowed"]), + ("issue-first", "process", "Open an issue before big changes", + "Significant work starts with a tracking issue and discussion.", + "Significant changes start with a tracking issue describing the problem and approach before any code is written.", + ["issue first workflow", "tracking issue before pr", "open issue for big changes", + "what comes before writing code for a large feature", "where do we debate an approach before building it", + "paperwork prior to a major refactor", "the step before investing days of work"]), + # ── infra/deploy ──────────────────────────────────────────────────── + ("staging-env", "infra", "Staging mirrors production", + "Every change bakes on staging before production rollout.", + "Changes deploy to the staging environment first and bake for a day before production rollout.", + ["staging environment", "staging before production", "bake time on staging", + "where does code sit before customers see it", "the rehearsal copy of our live system", + "how long does a change wait before going live", "the environment between my laptop and prod"]), + ("rollback-procedure", "infra", "Rollback via redeploy of last tag", + "Roll back by redeploying the previous tagged release.", + "Rollbacks redeploy the previous tagged release; never hotfix directly on production hosts.", + ["rollback procedure", "how to roll back a release", "redeploy previous tag", + "undoing a bad ship to customers", "the escape hatch when a rollout goes wrong", + "can I ssh into prod and patch it", "recovering after a broken deployment"]), + ("secrets-in-vault", "infra", "Secrets live in Vault", + "API keys and credentials come from Vault, never env files.", + "All credentials and API keys live in Vault and are injected at deploy time; committed .env files are forbidden.", + ["secrets in vault", "where are api keys stored", "vault credentials", + "the place passwords and tokens are kept", "how does the app get its private keys at runtime", + "can I commit an env file with credentials", "storage for sensitive configuration values"]), + ("terraform-infra", "infra", "Infrastructure is Terraform", + "All cloud resources are managed in the terraform/ directory.", + "Cloud infrastructure is defined in Terraform under terraform/; console changes get reverted by the next apply.", + ["terraform infrastructure", "infra as code", "terraform directory", + "how are our cloud resources defined", "editing servers by clicking the web console", + "the declarative description of our hosting", "where compute and networking are specified"]), + ("oncall-rotation", "infra", "Weekly on-call rotation", + "On-call rotates weekly; handoff notes go in the runbook.", + "On-call rotates weekly on Mondays; the outgoing person writes handoff notes in the runbook.", + ["oncall rotation", "who is on call", "weekly oncall handoff", + "the schedule for who answers pages", "when does incident duty switch people", + "notes passed between shifts of production duty", "how often does alert ownership change"]), + ("logs-in-grafana", "infra", "Logs and dashboards in Grafana", + "Production logs and metrics are viewed through Grafana.", + "Production observability lives in Grafana: logs via Loki, metrics via Prometheus dashboards.", + ["grafana logs", "where are production logs", "grafana dashboards metrics", + "how do I see what the live system is doing", "the place to look when something misbehaves in prod", + "viewer for runtime output of the service", "charts of system health over time"]), + # ── data/storage ──────────────────────────────────────────────────── + ("sqlite-storage", "data", "SQLite for local storage", + "Local data lives in SQLite with FTS; no external DB services.", + "The project stores local data in SQLite with FTS enabled; no external database services.", + ["sqlite storage", "local database sqlite", "sqlite fts", + "what do we use to persist data on disk", "the file-based store holding app state", + "do we run a database server locally", "where do records live on the user's machine"]), + ("postgres-production", "data", "Postgres 16 in production", + "Production data lives in Postgres 16 on RDS.", + "Production uses Postgres 16 on RDS with pgbouncer in front; schema changes go through migrations only.", + ["postgres production", "production database", "postgres 16 rds", + "what holds customer records in the live system", "the relational store behind the deployed app", + "which engine keeps our persistent server-side state", "backend that answers our sql"]), + ("migrations-alembic", "data", "Schema changes via Alembic", + "Every schema change is an Alembic migration; no manual DDL.", + "Database schema changes are Alembic migrations checked into the repo; manual DDL against any environment is forbidden.", + ["alembic migrations", "schema change process", "database migrations", + "how do I add a column safely", "evolving the table layout without breaking things", + "can I run alter statements by hand", "the versioned path for structural data changes"]), + ("no-pii-logs", "data", "Never log PII", + "Emails, names, and tokens must never appear in logs.", + "Logs must never contain PII: no emails, names, addresses, or tokens. Use the redaction helpers before logging request data.", + ["no pii in logs", "pii logging policy", "redact personal data logs", + "what personal details are banned from our output streams", "can user emails show up in diagnostics", + "privacy rules for what the service writes about requests", "scrubbing sensitive fields before recording"]), + ("backups-nightly", "data", "Nightly encrypted backups", + "Databases back up nightly, encrypted, with 30-day retention.", + "Databases are backed up nightly with encryption at rest and 30-day retention; restores are tested monthly.", + ["nightly backups", "backup retention 30 days", "encrypted database backups", + "how often do we copy the data somewhere safe", "recovering data if the store is lost", + "the safety net for catastrophic data loss", "snapshot cadence for our records"]), + ("no-cloud-sync", "data", "Memory stays local", + "Agent memory stays in local Markdown; no cloud sync.", + "The project decided agent memory stays in local Markdown files with no cloud synchronization.", + ["no cloud sync", "local markdown memory", "memory stays local", + "does anything leave this machine", "is my information uploaded anywhere", + "where does remembered context physically live", "the privacy stance on syncing notes off device"]), + # ── preferences ───────────────────────────────────────────────────── + ("short-answers", "preference", "Short answers with sources", + "Keep answers short and cite the wiki pages they came from.", + "The user prefers short, direct answers that cite the wiki pages they came from.", + ["short answers", "answer style preference", "cite sources in answers", + "how verbose should my replies be", "the length people want when I respond", + "do I need to point at where facts came from", "tone and size expected in written responses"]), + ("release-notes-short", "preference", "Release notes stay short", + "Release notes are a few bullets, user-facing language only.", + "The user prefers release notes kept to a few bullets in user-facing language; no internal jargon or commit lists.", + ["short release notes", "release notes style", "release notes bullets", + "how much detail goes in the changelog customers read", "writing up what shipped for end users", + "should the announcement list every commit", "the voice for describing a new version publicly"]), + ("morning-syncs", "preference", "User prefers morning meetings", + "Schedule syncs before noon in the user's timezone.", + "The user prefers meetings scheduled in the morning, before noon local time; afternoons are deep-work blocks.", + ["morning meetings", "meeting time preference", "syncs before noon", + "when should I put things on the calendar", "the part of the day kept free for focus", + "best hour to book a discussion", "scheduling around the user's energy"]), + ("dark-theme", "preference", "Dark theme everywhere", + "The user runs dark mode in every tool and expects demos to match.", + "The user uses dark theme in every tool; screenshots and demos should be captured in dark mode.", + ["dark theme preference", "dark mode", "screenshots dark mode", + "which appearance do their tools use", "how should captured ui images look", + "light or dark for the demo recording", "the visual scheme on their machine"]), + ("tabs-vs-spaces", "preference", "Four-space indentation", + "Python and config files indent with four spaces, never tabs.", + "Indentation is four spaces everywhere; tab characters are rejected by the linter.", + ["four space indentation", "tabs vs spaces", "indent width", + "how far do nested blocks step in", "the whitespace convention inside files", + "will tab characters pass the checks", "layout rule for code depth"]), + ("typed-python", "preference", "Type hints required", + "New Python code carries full type annotations.", + "New Python functions carry full type annotations; untyped code is flagged in review.", + ["type hints required", "typed python", "annotations policy", + "do I have to declare what functions accept and return", "static typing expectations for new modules", + "will unannotated helpers pass review", "how strict are we about signatures"]), + ("english-docs", "preference", "Docs are written in English", + "All documentation and comments are in English.", + "Documentation, comments, and commit messages are written in English even though the team is multilingual.", + ["docs in english", "documentation language", "english comments", + "which tongue do we write manuals in", "language for explaining code to others", + "can I comment in my native language", "the lingua franca of the repository"]), + # ── project facts ─────────────────────────────────────────────────── + ("python-versions", "project", "Supported Python versions", + "Support Python 3.10 through 3.14, all tested in CI.", + "The project supports Python 3.10 through 3.14 and tests all of them in CI.", + ["supported python versions", "python 3.10 to 3.14", "which python versions", + "which interpreter releases must keep working", "the oldest runtime we still promise to run on", + "compatibility window for the language runtime", "can I use syntax from the newest interpreter"]), + ("api-port-8080", "project", "API listens on 8080", + "The backend API serves on port 8080 locally.", + "The backend API listens on port 8080 in local development; the frontend proxies /api there.", + ["api port 8080", "which port backend", "local api port", + "where does the server accept requests on my machine", "the number after localhost for the backend", + "what does the web ui proxy its calls to", "socket the service binds during development"]), + ("license-mit", "project", "MIT licensed", + "The project is MIT licensed; dependencies must be compatible.", + "The project is MIT licensed; new dependencies must carry MIT-compatible licenses (no GPL).", + ["mit license", "project license", "license compatibility", + "the legal terms our code ships under", "can I add a copyleft dependency", + "what usage rights do downstream users get", "restrictions when vendoring third party code"]), + ("weekly-release", "project", "Releases ship weekly", + "A release train leaves every Thursday.", + "Releases ship weekly on Thursdays; anything not merged by Wednesday noon waits for the next train.", + ["weekly release thursday", "release cadence", "when do releases ship", + "how often does a new version go out", "the cutoff for making this week's ship", + "rhythm of delivering to customers", "if I merge friday when do users get it"]), + ("meeting-notes-obsidian", "project", "Meeting notes live in Obsidian", + "Meeting notes are kept in the Obsidian vault and imported to Link.", + "The user keeps meeting notes in an Obsidian vault and imports the relevant ones into Link.", + ["meeting notes obsidian", "obsidian vault notes", "where are meeting notes", + "where are the writeups from our sync calls", "the store of what was said in discussions", + "records of past conversations with the team", "place to find decisions from last week's call"]), + ("customer-sla", "project", "24-hour support SLA", + "Paid customers get first response within 24 hours.", + "Paid customers have a 24-hour first-response SLA on support tickets, business days only.", + ["support sla 24 hours", "customer sla", "first response time", + "how fast must we get back to paying users", "the promise on ticket turnaround", + "deadline for acknowledging a complaint", "response guarantee in the contract"]), + ("feature-flags", "project", "Features launch behind flags", + "New features roll out behind feature flags, off by default.", + "New features launch behind feature flags, default off, and are enabled progressively per cohort.", + ["feature flags", "launch behind flag", "flags default off", + "how do risky capabilities reach users gradually", "the switch controlling who sees new behavior", + "shipping something without turning it on for everyone", "progressive rollout mechanism"]), +] + +# Distractor memories: realistic corpus filler with no benchmark queries. +_DISTRACTOR_TOPICS = [ + ("adr-records", "process", "Architecture decisions in ADRs", + "Significant architecture choices are recorded as ADR markdown files under docs/adr."), + ("browser-support", "project", "Browser support matrix", + "The web app supports the last two versions of Chrome, Firefox, Safari, and Edge."), + ("css-tailwind", "tooling", "Tailwind for styling", + "Frontend styling uses Tailwind utility classes; bespoke CSS files need a review exception."), + ("error-tracking-sentry", "infra", "Errors go to Sentry", + "Unhandled exceptions report to Sentry with release tagging and user-scrubbed context."), + ("i18n-later", "project", "Internationalization deferred", + "Internationalization is out of scope until the enterprise tier ships."), + ("jira-tickets", "process", "Work is tracked in Jira", + "All planned work is tracked as Jira tickets linked from pull requests."), + ("kafka-events", "data", "Events stream through Kafka", + "Cross-service events flow through Kafka topics with schema-registry enforced Avro."), + ("load-testing-k6", "infra", "Load tests use k6", + "Load testing runs k6 scripts from the perf/ directory before each major release."), + ("mobile-react-native", "project", "Mobile app is React Native", + "The mobile app is React Native with a shared TypeScript core."), + ("nginx-ingress", "infra", "NGINX terminates TLS", + "NGINX ingress terminates TLS and forwards plain HTTP to the app pods."), + ("openapi-spec", "project", "API described by OpenAPI", + "The public API is described by an OpenAPI 3.1 spec that generates the client SDKs."), + ("pagerduty-alerts", "infra", "Alerts page through PagerDuty", + "Critical alerts page the on-call engineer through PagerDuty; Slack is best-effort only."), + ("redis-caching", "data", "Redis caches hot reads", + "Hot read paths cache in Redis with 5-minute TTLs and explicit invalidation on writes."), + ("storybook-components", "tooling", "Components documented in Storybook", + "Shared UI components are documented and visually tested in Storybook."), + ("vpn-required", "infra", "VPN required for internal tools", + "Internal dashboards and admin tools are reachable only over the company VPN."), + ("weekly-demo", "process", "Friday demo ritual", + "Every Friday the team demos shipped work in a 30-minute open call."), + ("design-figma", "tooling", "Designs live in Figma", + "Product designs and prototypes live in Figma; engineers comment there, not in screenshots."), + ("analytics-posthog", "data", "Product analytics in PostHog", + "Product analytics events flow to self-hosted PostHog with anonymized user ids."), + ("code-owners", "process", "CODEOWNERS gates sensitive paths", + "Changes under auth/ and billing/ require approval from the owners listed in CODEOWNERS."), + ("changelog-keepachangelog", "process", "Changelog follows Keep a Changelog", + "CHANGELOG.md follows the Keep a Changelog format with an Unreleased section."), +] + + +def _memory(name: str, domain: str, title: str, tldr: str, body: str, memory_type: str = "preference") -> dict[str, object]: + return { + "name": name, + "title": title, + "tldr": tldr, + "tags": [domain], + "body": body, + "status": "active", + "scope": "user", + "memory_type": memory_type, + "review_status": "reviewed", + "domain": domain, + } + + +# Deterministic phrasing wrappers applied to authored queries to grow the +# suite with surface variation (word order and framing changes only; they are +# counted separately from authored queries in reporting). +_WRAPPERS = [ + "{q}", + "quick question: {q}", + "remind me: {q}", + "for this project, {q}", +] + + +def build_corpus() -> list[dict[str, object]]: + memories = [ + _memory(name, domain, title, tldr, body) + for name, domain, title, tldr, body, _queries in INTENTS + ] + memories.extend( + _memory(name, domain, title, body[:80], body) + for name, domain, title, body in _DISTRACTOR_TOPICS + ) + return memories + + +def build_cases(expand: bool = True) -> list[dict[str, str]]: + """Return benchmark cases: {query, target, domain, authored}.""" + cases: list[dict[str, str]] = [] + for name, domain, _title, _tldr, _body, queries in INTENTS: + for query in queries: + cases.append({"query": query, "target": name, "domain": domain, "authored": "yes"}) + if expand: + for wrapper in _WRAPPERS[1:]: + cases.append({ + "query": wrapper.format(q=query), + "target": name, + "domain": domain, + "authored": "wrapped", + }) + return cases diff --git a/skills/link-health/SKILL.md b/skills/link-health/SKILL.md index 2bfb25b1..ba0bddef 100644 --- a/skills/link-health/SKILL.md +++ b/skills/link-health/SKILL.md @@ -32,3 +32,9 @@ Use the `lnk` CLI. Load this skill before trusting a new or changed Link wiki, a ``` If the user asks whether MCP is ready, run `lnk verify-mcp [link-root]`. Do not start `lnk serve` for MCP or CLI work. + +To check whether optional local semantic recall is active (lexical is always the fallback): +```bash +lnk semantic [link-root] +``` +It reports the provider tier, model, and index state, and prints the exact setup command when the layer is available but not yet enabled. diff --git a/skills/link-memory/SKILL.md b/skills/link-memory/SKILL.md index 93a13e59..d2b37924 100644 --- a/skills/link-memory/SKILL.md +++ b/skills/link-memory/SKILL.md @@ -7,6 +7,8 @@ description: Use after important user-approved decisions, when durable context s Use this skill after important user-approved decisions, preference changes, project conventions, or long work sessions that may deserve durable context. In a source checkout, replace `lnk` with `python3 link.py`. Do not silently save durable memory; propose first unless the user directly asks to remember, approves a proposal, or explicitly confirms an important decision should become durable memory. +If Link session hooks are installed for this agent, the memory brief is injected automatically at session start — skip step 1 and go straight to task-specific recall. + 1. Prime before work: ```bash lnk brief "" [link-root] @@ -25,6 +27,12 @@ Use this skill after important user-approved decisions, preference changes, proj lnk remember "" [link-root] --type note --scope user ``` Use `--project ` for project-scoped memory, `--visibility private|project|team` for sharing intent, `--review-after YYYY-MM-DD` for stale-risk memories, and `--expires-at YYYY-MM-DD` for temporary context. +When a brief or recall reports a memory backlog (pending captures or reviews above threshold), offer the user a short consolidation pass: + ```bash + lnk consolidate [link-root] + ``` + The plan is read-only: it groups duplicates and recurring themes and prints accept/discard/review commands. Apply an action only after the user approves it. + 5. Review and explain before trusting uncertain memory: ```bash lnk memory-inbox [link-root] diff --git a/skills/link-retrieve/SKILL.md b/skills/link-retrieve/SKILL.md index 85a52123..003627ed 100644 --- a/skills/link-retrieve/SKILL.md +++ b/skills/link-retrieve/SKILL.md @@ -35,3 +35,5 @@ Use bounded CLI commands so the agent does not dump the whole wiki into context. ``` Do not enumerate every page, grep raw files, or request the full graph unless the user explicitly asks for an export or exhaustive audit, or the compact packet is insufficient and tells you which follow-up to use. + +Recalled memories carry `confidence` labels and, when the optional local semantic tier is installed, a `match` field: `lexical`, `hybrid`, or `semantic`. Treat `semantic` matches (paraphrase similarity, capped confidence) and `weak` matches as hints to verify with the user, not facts to act on. diff --git a/tests/test_agent_hooks_core.py b/tests/test_agent_hooks_core.py new file mode 100644 index 00000000..acb8eee8 --- /dev/null +++ b/tests/test_agent_hooks_core.py @@ -0,0 +1,274 @@ +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "mcp_package")) + +from link_core.agent_hooks import ( # noqa: E402 + build_agent_hooks_payload, + extract_transcript_text, + hook_supported_agents, + supports_agent_hooks, +) + + +def _transcript_line(role: str, content: object) -> str: + return json.dumps({"type": role, "message": {"role": role, "content": content}}) + + +class AgentHooksCoreTests(unittest.TestCase): + def test_hook_supported_agents_include_hook_capable_agents(self): + agents = hook_supported_agents() + for agent in ("claude-code", "codex", "cursor"): + self.assertIn(agent, agents) + + def test_supports_agent_hooks_accepts_aliases_and_rejects_others(self): + self.assertTrue(supports_agent_hooks("claude-code")) + self.assertTrue(supports_agent_hooks("claude")) + self.assertTrue(supports_agent_hooks("codex")) + self.assertTrue(supports_agent_hooks("cursor")) + self.assertFalse(supports_agent_hooks("kiro")) + self.assertFalse(supports_agent_hooks("vscode")) + + def test_build_payload_rejects_unsupported_agent(self): + with self.assertRaises(ValueError): + build_agent_hooks_payload( + target=Path("/tmp/link"), + agent="kiro", + runtime_script=Path("/tmp/link/link.py"), + python_cmd="python3", + ) + + def test_codex_gets_session_start_only(self): + payload = build_agent_hooks_payload( + target=Path("/tmp/link"), + agent="codex", + runtime_script=Path("/tmp/link/link.py"), + python_cmd="python3", + ) + + self.assertIn("SessionStart", payload["events"]) + self.assertNotIn("SessionEnd", payload["events"]) + self.assertIn("hooks.json", str(payload["settings_path"])) + self.assertIn(".codex", str(payload["settings_path"])) + snippet = json.loads(str(payload["snippet"])) + self.assertEqual(list(snippet["hooks"].keys()), ["SessionStart"]) + self.assertTrue(any("no session-end hook event" in item for item in payload["behavior"])) + + def test_cursor_uses_flat_schema_and_cursor_emit(self): + payload = build_agent_hooks_payload( + target=Path("/tmp/link"), + agent="cursor", + runtime_script=Path("/tmp/link/link.py"), + python_cmd="python3", + ) + + self.assertIn("--emit cursor", str(payload["events"]["sessionStart"])) + self.assertNotIn("--emit", str(payload["events"]["sessionEnd"])) + snippet = json.loads(str(payload["snippet"])) + self.assertEqual(snippet["version"], 1) + # Flat schema: entries directly in the event array, no matcher groups. + self.assertIn("command", snippet["hooks"]["sessionStart"][0]) + self.assertNotIn("hooks", snippet["hooks"]["sessionStart"][0]) + + def test_cursor_write_preserves_version_and_foreign_entries(self): + with tempfile.TemporaryDirectory() as temp: + settings = Path(temp) / "hooks.json" + settings.write_text( + json.dumps({ + "version": 1, + "hooks": { + "sessionStart": [{"command": "./my-hook.sh"}], + "stop": [{"command": "./on-stop.sh"}], + }, + }), + encoding="utf-8", + ) + + for _ in range(2): + payload = build_agent_hooks_payload( + target=Path(temp), + agent="cursor", + runtime_script=Path(temp) / "link.py", + python_cmd="python3", + settings_path=str(settings), + write=True, + ) + self.assertTrue(payload["write"]["ok"], payload["write"]) + + data = json.loads(settings.read_text(encoding="utf-8")) + self.assertEqual(data["version"], 1) + self.assertEqual(data["hooks"]["stop"], [{"command": "./on-stop.sh"}]) + starts = data["hooks"]["sessionStart"] + self.assertEqual(starts[0], {"command": "./my-hook.sh"}) + link_entries = [e for e in starts if "hook session-start" in e.get("command", "")] + self.assertEqual(len(link_entries), 1) + self.assertEqual(len(data["hooks"]["sessionEnd"]), 1) + + def test_build_preview_includes_both_events_and_commands(self): + payload = build_agent_hooks_payload( + target=Path("/tmp/my link"), + agent="claude-code", + runtime_script=Path("/tmp/my link/link.py"), + python_cmd="/usr/bin/python3", + ) + + self.assertEqual(payload["agent"], "claude-code") + self.assertFalse(payload["write"]["ok"]) + events = payload["events"] + self.assertIn(" hook session-start ", str(events["SessionStart"])) + self.assertIn(" hook session-end ", str(events["SessionEnd"])) + # Paths with spaces must stay shell-safe in the written command: + # shlex single quotes on POSIX, list2cmdline double quotes on Windows. + script = str(Path("/tmp/my link/link.py")) + quoted = f'"{script}"' if os.name == "nt" else f"'{script}'" + self.assertIn(quoted, str(events["SessionStart"])) + snippet = json.loads(str(payload["snippet"])) + self.assertIn("SessionStart", snippet["hooks"]) + self.assertIn("SessionEnd", snippet["hooks"]) + self.assertEqual(snippet["hooks"]["SessionStart"][0]["matcher"], "startup|clear|compact") + + def test_write_preserves_existing_settings_and_hooks(self): + with tempfile.TemporaryDirectory() as temp: + settings = Path(temp) / "settings.json" + settings.write_text( + json.dumps({ + "model": "opus", + "hooks": { + "PreToolUse": [{"matcher": "Bash", "hooks": [{"type": "command", "command": "my-guard"}]}], + "SessionStart": [{"hooks": [{"type": "command", "command": "echo user-hook"}]}], + }, + }), + encoding="utf-8", + ) + + payload = build_agent_hooks_payload( + target=Path(temp), + agent="claude-code", + runtime_script=Path(temp) / "link.py", + python_cmd="python3", + settings_path=str(settings), + write=True, + ) + + self.assertTrue(payload["write"]["ok"], payload["write"]) + data = json.loads(settings.read_text(encoding="utf-8")) + self.assertEqual(data["model"], "opus") + self.assertEqual(data["hooks"]["PreToolUse"][0]["hooks"][0]["command"], "my-guard") + self.assertEqual(data["hooks"]["SessionStart"][0]["hooks"][0]["command"], "echo user-hook") + start_groups = data["hooks"]["SessionStart"] + self.assertEqual(len(start_groups), 2) + self.assertEqual(start_groups[1]["matcher"], "startup|clear|compact") + self.assertEqual(len(data["hooks"]["SessionEnd"]), 1) + + def test_rewrite_is_idempotent(self): + with tempfile.TemporaryDirectory() as temp: + settings = Path(temp) / "settings.json" + for _ in range(2): + payload = build_agent_hooks_payload( + target=Path(temp), + agent="claude-code", + runtime_script=Path(temp) / "link.py", + python_cmd="python3", + settings_path=str(settings), + write=True, + ) + self.assertTrue(payload["write"]["ok"], payload["write"]) + + data = json.loads(settings.read_text(encoding="utf-8")) + self.assertEqual(len(data["hooks"]["SessionStart"]), 1) + self.assertEqual(len(data["hooks"]["SessionStart"][0]["hooks"]), 1) + self.assertEqual(len(data["hooks"]["SessionEnd"]), 1) + self.assertEqual(len(data["hooks"]["SessionEnd"][0]["hooks"]), 1) + + def test_write_refuses_non_object_settings_file(self): + with tempfile.TemporaryDirectory() as temp: + settings = Path(temp) / "settings.json" + settings.write_text("[]", encoding="utf-8") + + payload = build_agent_hooks_payload( + target=Path(temp), + agent="claude-code", + runtime_script=Path(temp) / "link.py", + python_cmd="python3", + settings_path=str(settings), + write=True, + ) + + self.assertFalse(payload["write"]["ok"]) + self.assertEqual(settings.read_text(encoding="utf-8"), "[]") + + def test_extract_transcript_keeps_text_and_skips_tool_blocks(self): + with tempfile.TemporaryDirectory() as temp: + transcript = Path(temp) / "transcript.jsonl" + transcript.write_text( + "\n".join([ + _transcript_line("user", "We decided to use SQLite FTS."), + _transcript_line("assistant", [ + {"type": "text", "text": "Noted the SQLite FTS decision."}, + {"type": "tool_use", "id": "x", "name": "Bash", "input": {"command": "secret-tool-call"}}, + ]), + _transcript_line("user", [ + {"type": "tool_result", "tool_use_id": "x", "content": "tool output noise"}, + ]), + json.dumps({"type": "summary", "summary": "meta line"}), + "not json at all", + ]), + encoding="utf-8", + ) + + text = extract_transcript_text(transcript) + + self.assertIn("User: We decided to use SQLite FTS.", text) + self.assertIn("Assistant: Noted the SQLite FTS decision.", text) + self.assertNotIn("secret-tool-call", text) + self.assertNotIn("tool output noise", text) + self.assertNotIn("meta line", text) + + def test_extract_transcript_bounds_output_to_most_recent_messages(self): + with tempfile.TemporaryDirectory() as temp: + transcript = Path(temp) / "transcript.jsonl" + lines = [_transcript_line("user", f"message {index}: " + ("x" * 400)) for index in range(50)] + transcript.write_text("\n".join(lines), encoding="utf-8") + + text = extract_transcript_text(transcript, max_chars=2000) + + self.assertLessEqual(len(text), 2200) + self.assertIn("message 49", text) + self.assertNotIn("message 0:", text) + + def test_extract_transcript_can_keep_user_turns_only(self): + # Memory proposals must come from the user's words, not the assistant's + # prose (which dogfooding showed gets mis-attributed as user preferences). + with tempfile.TemporaryDirectory() as temp: + transcript = Path(temp) / "transcript.jsonl" + transcript.write_text( + "\n".join([ + _transcript_line("user", "ok go ahead"), + _transcript_line("assistant", [{"type": "text", + "text": "Tests pass on broken things; eyes don't. I prefer small commits."}]), + _transcript_line("user", "We decided to require signed commits on every branch."), + ]), + encoding="utf-8", + ) + + both = extract_transcript_text(transcript) + user_only = extract_transcript_text(transcript, roles=("user",)) + + self.assertIn("Tests pass on broken things", both) + self.assertNotIn("Tests pass on broken things", user_only) + self.assertNotIn("I prefer small commits", user_only) + self.assertIn("signed commits", user_only) + self.assertIn("ok go ahead", user_only) + + def test_extract_transcript_handles_missing_file(self): + self.assertEqual(extract_transcript_text(Path("/nonexistent/transcript.jsonl")), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_runtime_core.py b/tests/test_cli_runtime_core.py index 01a433c7..c67c805e 100644 --- a/tests/test_cli_runtime_core.py +++ b/tests/test_cli_runtime_core.py @@ -240,7 +240,8 @@ def test_render_proof_text(self): self.assertEqual(code, 0) self.assertIn("Cross-agent memory continuity works", text) - self.assertIn("Workspace: created local Markdown wiki", text) + self.assertIn("throwaway demo wiki", text) + self.assertIn("What this means for you", text) self.assertIn("Memory: created and reviewed", text) self.assertIn("same bounded recall path used by CLI, skills, and MCP", text) self.assertIn("Try it with two agents", text) diff --git a/tests/test_docs_site.py b/tests/test_docs_site.py index 66d96b48..500ba094 100644 --- a/tests/test_docs_site.py +++ b/tests/test_docs_site.py @@ -118,3 +118,44 @@ def test_github_pages_analytics_is_docs_only_and_manual(self): if __name__ == "__main__": unittest.main() + +class FoundingIdentityTests(unittest.TestCase): + """New releases layer onto Link's founding story; they must never bury it. + + These are the identity claims that made Link through 1.5.0. If a landing + or README rewrite drops one, this test fails and the author must decide + deliberately — not by accident of enthusiasm for the newest feature. + """ + + PILLARS = { + "source-backed": "provenance: memory that can say why it is known", + "Markdown": "inspectable plain-file storage", + "approve": "review-gated writes: agents propose, the user decides", + "your machine": "local-first: no hosted profile", + "every agent": "one memory shared across agents", + "proof": "the first-run proof loop (lnk proof)", + } + + @staticmethod + def _flat(path): + # Markdown and templates wrap lines; claims are judged on prose, + # not line breaks. + return " ".join((ROOT / path).read_text(encoding="utf-8").split()) + + def test_landing_keeps_the_founding_claims(self): + text = self._flat("docs/index.html") + for phrase, meaning in self.PILLARS.items(): + self.assertIn(phrase, text, f"landing lost founding claim: {meaning}") + + def test_readme_keeps_the_founding_claims(self): + text = self._flat("README.md") + for phrase, meaning in self.PILLARS.items(): + variants = { + "approve": ("approve", "approval"), + "every agent": ("every agent", "across multiple agents", "different agents", "across agents"), + }.get(phrase, (phrase,)) + self.assertTrue( + any(variant in text for variant in variants), + f"README lost founding claim: {meaning}", + ) + diff --git a/tests/test_link_cli.py b/tests/test_link_cli.py index 17efecc8..de6ab68b 100644 --- a/tests/test_link_cli.py +++ b/tests/test_link_cli.py @@ -2775,5 +2775,376 @@ def test_doctor_fails_on_service_account_filename(self): self.assertIn("service-account-prod.json", out.getvalue()) +class AgentHookCliTests(unittest.TestCase): + def _hook_stdin(self, payload: dict) -> StringIO: + return StringIO(json.dumps(payload)) + + def test_hook_session_start_prints_memory_brief(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"cwd": str(tmp), "source": "startup"})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-start") + + self.assertEqual(code, 0) + text = out.getvalue() + self.assertIn("Link memory (local, source-backed)", text) + self.assertIn("Relevant memories", text) + self.assertIn("Save durable memory only after explicit user approval.", text) + + def test_hook_session_start_empty_workspace_is_compact(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "empty" + with redirect_stdout(StringIO()): + link_cli.init_wiki(target) + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"cwd": str(tmp)})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-start") + + self.assertEqual(code, 0) + text = out.getvalue() + self.assertIn("empty workspace, nothing to recall yet", text) + self.assertNotIn("Relevant memories", text) + self.assertLess(len(text.splitlines()), 6) + + def test_missing_wiki_error_points_to_next_step(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + + err = StringIO() + with redirect_stderr(err): + code = link_cli.recall(tmp / "nowhere", "anything") + + self.assertEqual(code, 1) + self.assertIn("Missing wiki directory", err.getvalue()) + self.assertIn("init", err.getvalue()) + + def test_hook_session_start_missing_wiki_exits_zero_with_guidance(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "missing" + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"source": "startup"})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-start") + + self.assertEqual(code, 0) + self.assertIn("wiki missing", out.getvalue()) + + def test_hook_session_end_captures_proposal_only_notes(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + transcript.write_text( + "\n".join( + json.dumps({ + "type": "user", + "message": { + "role": "user", + "content": f"We decided {index}: deploy the staging site only from the tagged release branch.", + }, + }) + for index in range(6) + ), + encoding="utf-8", + ) + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"cwd": str(tmp), "transcript_path": str(transcript)})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-end") + + self.assertEqual(code, 0) + captures = list((target / "raw/memory-captures").glob("*agent-session-notes*.md")) + self.assertEqual(len(captures), 1) + self.assertIn("proposal-only", out.getvalue()) + + def test_hook_session_start_cursor_emit_wraps_additional_context(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"workspace_roots": [str(tmp)]})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-start", emit="cursor") + + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertIn("Link memory (local, source-backed)", payload["additional_context"]) + + def test_hook_session_end_skips_duplicate_transcript_content(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + transcript.write_text( + "\n".join( + json.dumps({ + "type": "user", + "message": { + "role": "user", + "content": f"We decided {index}: deploy the staging site only from the tagged release branch.", + }, + }) + for index in range(6) + ), + encoding="utf-8", + ) + + for _ in range(2): + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"transcript_path": str(transcript)})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-end") + self.assertEqual(code, 0) + + captures = list((target / "raw/memory-captures").glob("*agent-session-notes*.md")) + self.assertEqual(len(captures), 1) + + def test_hook_session_end_skips_sessions_without_memory_proposals(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + transcript.write_text( + "\n".join( + json.dumps({ + "type": "assistant", + "message": { + "role": "assistant", + "content": [{ + "type": "text", + "text": f"I looked at file number {index} and it seems fine to me over there.", + }], + }, + }) + for index in range(8) + ), + encoding="utf-8", + ) + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"transcript_path": str(transcript)})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-end") + + self.assertEqual(code, 0) + captures = list((target / "raw/memory-captures").glob("*agent-session-notes*.md")) + self.assertEqual(captures, []) + + def test_consolidate_prints_read_only_plan(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + transcript.write_text( + "\n".join( + json.dumps({ + "type": "user", + "message": { + "role": "user", + "content": f"We decided {index}: deploy the staging site only from the tagged release branch.", + }, + }) + for index in range(6) + ), + encoding="utf-8", + ) + with patch("sys.stdin", self._hook_stdin({"transcript_path": str(transcript)})): + with redirect_stdout(StringIO()): + link_cli.run_agent_hook(target, "session-end") + + out = StringIO() + with redirect_stdout(out): + code = link_cli.consolidate(target, json_output=True) + + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["pending_captures"], 1) + self.assertIn("Read-only plan", payload["safety"]) + self.assertTrue(payload["captures"][0]["accept_command"]) + self.assertTrue(payload["captures"][0]["delete_command"]) + + def test_hook_session_end_ignores_assistant_prose(self): + tmp = Path(tempfile.mkdtemp(prefix="link-assistant-prose-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + # Only the assistant states preference-shaped sentences; the user just + # acknowledges. Nothing should be captured. + transcript.write_text( + "\n".join([ + json.dumps({"type": "user", "message": {"role": "user", "content": "ok go ahead"}}), + json.dumps({"type": "assistant", "message": {"role": "assistant", "content": [{ + "type": "text", + "text": "Tests pass on broken things; eyes don't. These are shell commands. " + "I prefer small commits and short PR descriptions for this project always."}]}}), + json.dumps({"type": "user", "message": {"role": "user", "content": "makes sense, nice"}}), + ]), + encoding="utf-8", + ) + + with patch("sys.stdin", self._hook_stdin({"transcript_path": str(transcript)})): + with redirect_stdout(StringIO()): + code = link_cli.run_agent_hook(target, "session-end") + + self.assertEqual(code, 0) + captures = list((target / "raw/memory-captures").glob("*agent-session-notes*.md")) + self.assertEqual(captures, [], "assistant prose must not become a capture") + + def test_hook_session_end_captures_user_stated_decision(self): + tmp = Path(tempfile.mkdtemp(prefix="link-user-decision-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + transcript.write_text( + "\n".join([ + json.dumps({"type": "assistant", "message": {"role": "assistant", "content": [{ + "type": "text", "text": "Here are some options for the release branch."}]}}), + json.dumps({"type": "user", "message": {"role": "user", "content": + "For this project we decided to always cut releases from the develop branch, " + "never straight to main, and to keep every commit without co-author trailers. " + "Please treat that as the standing release convention from now on so we stay " + "consistent across the whole team and every future release we ship together."}}), + ]), + encoding="utf-8", + ) + + with patch("sys.stdin", self._hook_stdin({"transcript_path": str(transcript)})): + with redirect_stdout(StringIO()): + code = link_cli.run_agent_hook(target, "session-end") + + self.assertEqual(code, 0) + captures = list((target / "raw/memory-captures").glob("*agent-session-notes*.md")) + self.assertEqual(len(captures), 1, "a user-stated decision should be captured") + + def test_hook_session_end_skips_trivial_sessions(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + transcript = tmp / "transcript.jsonl" + transcript.write_text( + json.dumps({"type": "user", "message": {"role": "user", "content": "hi"}}), + encoding="utf-8", + ) + + out = StringIO() + with patch("sys.stdin", self._hook_stdin({"transcript_path": str(transcript)})): + with redirect_stdout(out): + code = link_cli.run_agent_hook(target, "session-end") + + self.assertEqual(code, 0) + captures = list((target / "raw/memory-captures").glob("*agent-session-notes*.md")) + self.assertEqual(captures, []) + + def test_hook_session_end_without_stdin_payload_is_noop(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + + with patch("sys.stdin", StringIO("")): + code = link_cli.run_agent_hook(target, "session-end") + + self.assertEqual(code, 0) + + def test_connect_hooks_rejects_unsupported_agent(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + + err = StringIO() + with redirect_stderr(err): + code = link_cli.connect_mcp(target, "kiro", hooks=True) + + self.assertEqual(code, 1) + self.assertIn("--hooks is not supported for kiro", err.getvalue()) + + def test_connect_hooks_preview_includes_session_hooks_payload(self): + tmp = Path(tempfile.mkdtemp(prefix="link-hook-test-")) + target = tmp / "demo" + create_demo_quiet(target) + + out = StringIO() + with redirect_stdout(out): + code = link_cli.connect_mcp(target, "claude-code", hooks=True, json_output=True) + + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + session_hooks = payload["session_hooks"] + self.assertEqual(session_hooks["agent"], "claude-code") + self.assertFalse(session_hooks["write"]["ok"]) + self.assertIn(" hook session-start ", session_hooks["events"]["SessionStart"]) + # The command must point at the demo's own runtime script. Compare the + # stable path tail: on Windows the temp dir in the command is resolved + # to its long form (runneradmin) while mkdtemp returns the 8.3 short + # form (RUNNER~1), so the absolute prefix differs. + self.assertIn(str(Path(target.name) / "link.py"), session_hooks["events"]["SessionStart"]) + + +class NewUserFrictionTests(unittest.TestCase): + def test_recall_miss_hints_at_semantic_when_memories_exist(self): + tmp = Path(tempfile.mkdtemp(prefix="link-miss-hint-")) + target = tmp / "wiki-root" + with redirect_stdout(StringIO()): + link_cli.init_wiki(target) + link_cli.remember(target, "I prefer short PR descriptions with a one-line summary first", + memory_type="preference") + + out = StringIO() + with redirect_stdout(out): + code = link_cli.recall(target, "how do I like my pull requests written") + + self.assertEqual(code, 0) + text = out.getvalue() + self.assertIn("No matching memories found", text) + # The whole point: a paraphrase miss must point the user at semantic. + self.assertIn("semantic recall", text.lower()) + self.assertIn("--setup", text) + + def test_recall_miss_on_empty_wiki_gives_no_semantic_hint(self): + tmp = Path(tempfile.mkdtemp(prefix="link-miss-empty-")) + target = tmp / "wiki-root" + with redirect_stdout(StringIO()): + link_cli.init_wiki(target) + + out = StringIO() + with redirect_stdout(out): + code = link_cli.recall(target, "anything at all") + + self.assertEqual(code, 0) + # No memories yet: don't nag about semantic, just say add one. + self.assertNotIn("semantic recall", out.getvalue().lower()) + + def test_onboard_surfaces_the_hooks_path(self): + tmp = Path(tempfile.mkdtemp(prefix="link-onboard-hooks-")) + target = tmp / "link" + + out = StringIO() + with redirect_stdout(out): + code = link_cli.onboard(target) + + self.assertEqual(code, 0) + text = out.getvalue() + self.assertIn("Make memory automatic", text) + self.assertIn("--hooks", text) + + def test_onboard_agent_preview_offers_hooks(self): + tmp = Path(tempfile.mkdtemp(prefix="link-onboard-agent-hooks-")) + target = tmp / "link" + + out = StringIO() + with redirect_stdout(out): + code = link_cli.onboard(target, agents=["claude-code"]) + + self.assertEqual(code, 0) + self.assertIn("Make memory automatic (recommended)", out.getvalue()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mcp_contract.py b/tests/test_mcp_contract.py index 61e977e6..804b5065 100644 --- a/tests/test_mcp_contract.py +++ b/tests/test_mcp_contract.py @@ -403,6 +403,35 @@ def test_missing_wiki_message_points_to_current_setup_paths(self): finally: sys.argv = previous_argv + def test_help_flag_prints_usage_instead_of_starting_the_server(self): + # Without explicit handling, --help is swallowed by parse_known_args + # and the stdio server starts, hanging silently in a terminal — the + # first exploratory command a pip user runs must not dead-end. + previous_argv = sys.argv[:] + missing = Path(tempfile.mkdtemp(prefix="link-mcp-help-")) / "missing" / "wiki" + module_name = f"link_mcp_server_help_{id(missing)}" + try: + sys.argv = ["link_mcp.server", "--wiki", str(missing), "--help"] + spec = importlib.util.spec_from_file_location(module_name, ROOT / "mcp_package/link_mcp/server.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + out = StringIO() + err = StringIO() + with redirect_stdout(out), redirect_stderr(err), self.assertRaises(SystemExit) as cm: + spec.loader.exec_module(module) + + self.assertEqual(cm.exception.code, 0) + text = out.getvalue() + self.assertIn("Usage:", text) + self.assertIn("--wiki", text) + self.assertIn("--surface", text) + self.assertIn("--semantic-setup", text) + self.assertIn("mcpServers", text) + self.assertEqual(err.getvalue(), "") + finally: + sys.modules.pop(module_name, None) + sys.argv = previous_argv + def test_version_flag_does_not_require_wiki_or_mcp_sdk(self): previous_argv = sys.argv[:] missing = Path(tempfile.mkdtemp(prefix="link-mcp-version-")) / "missing" / "wiki" diff --git a/tests/test_recall_benchmark.py b/tests/test_recall_benchmark.py new file mode 100644 index 00000000..636960e2 --- /dev/null +++ b/tests/test_recall_benchmark.py @@ -0,0 +1,40 @@ +import json +import subprocess +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class RecallBenchmarkTests(unittest.TestCase): + def test_small_fake_suite_passes_regression_gate(self): + completed = subprocess.run( + [sys.executable, str(ROOT / "scripts/eval_recall_quality.py"), + "--suite", "small", "--mode", "fake", "--json"], + capture_output=True, text=True, timeout=300, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr or completed.stdout) + report = json.loads(completed.stdout) + self.assertGreaterEqual(report["corpus_memories"], 50) + self.assertGreaterEqual(report["authored_cases"], 250) + groups = report["lexical_baseline"]["groups"] + self.assertIn("token-overlap", groups) + self.assertIn("zero-overlap", groups) + # The paraphrase group must stay genuinely hard for lexical recall; + # if this rises, queries have drifted into token overlap. + self.assertLess(groups["zero-overlap"]["hit@1"], 0.2) + self.assertGreaterEqual(groups["zero-overlap"]["cases"], 50) + + def test_full_suite_reaches_one_thousand_cases(self): + sys.path.insert(0, str(ROOT / "scripts")) + from recall_dataset import build_cases, build_corpus + + self.assertGreaterEqual(len(build_cases(expand=True)), 1000) + self.assertGreaterEqual(len(build_corpus()), 50) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_semantic_core.py b/tests/test_semantic_core.py new file mode 100644 index 00000000..d22910cd --- /dev/null +++ b/tests/test_semantic_core.py @@ -0,0 +1,229 @@ +import json +import math +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "mcp_package")) + +from link_core.memory import recall_memories # noqa: E402 +from link_core.semantic import ( # noqa: E402 + SEMANTIC_MIN_COSINE, + build_semantic_status, + memory_embedding_text, + refresh_memory_index, + semantic_confidence_cap, + semantic_index_path, + semantic_match_points, + semantic_memory_scores, +) + +# Tiny deterministic embedder: maps known concepts onto fixed axes so +# paraphrases ("structure my pull requests" / "commit style") land close +# together without any model. It ABSTAINS on text it does not recognize +# (zero vector), like an honest weak model: it can add signal only where it +# has knowledge and can never inject ranking noise elsewhere. CI uses it to +# exercise the full hybrid pipeline with a hard no-regression gate. +_CONCEPTS = { + 0: {"commit", "commits", "committing", "pr", "prs", "pull", "requests", "structure", "structured", "style"}, + 1: {"deploy", "deploys", "release", "releases", "ship", "shipping"}, + 2: {"database", "sqlite", "postgres", "storage", "persist", "disk", "data"}, +} +_FAKE_STOPWORDS = { + "the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "from", "with", + "how", "what", "which", "where", "do", "does", "we", "my", "our", "i", "should", + "can", "must", "are", "is", "be", "this", "that", "it", "user", "prefers", +} +_DIM = 16 + + +def fake_embedder(texts: list[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + vector = [0.0] * _DIM + for token in text.lower().split(): + token = "".join(ch for ch in token if ch.isalnum()) + if not token or token in _FAKE_STOPWORDS: + continue + for axis, concepts in _CONCEPTS.items(): + if token in concepts: + vector[axis] += 1.0 + break + vectors.append(vector) + return vectors + + +def _memory(name: str, title: str, body: str, **extra) -> dict[str, object]: + record = { + "name": name, + "title": title, + "tldr": "", + "tags": [], + "body": body, + "status": "active", + "scope": "user", + "memory_type": "preference", + "review_status": "reviewed", + } + record.update(extra) + return record + + +COMMIT_MEMORY = _memory( + "commit-style", + "Commit style", + "The user prefers small commits and PRs structured with a summary first.", +) +DEPLOY_MEMORY = _memory( + "deploy-from-main", + "Deploy from main", + "Releases ship only from the main branch after CI passes.", +) + + +class SemanticCoreTests(unittest.TestCase): + def test_refresh_index_embeds_and_reuses_unchanged(self): + calls: list[int] = [] + + def counting_embedder(texts: list[str]) -> list[list[float]]: + calls.append(len(texts)) + return fake_embedder(texts) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + records = [COMMIT_MEMORY, DEPLOY_MEMORY] + index = refresh_memory_index(root, records, embedder=counting_embedder) + self.assertEqual(len(index["items"]), 2) + self.assertEqual(sum(calls), 2) + + refresh_memory_index(root, records, embedder=counting_embedder) + self.assertEqual(sum(calls), 2) # unchanged: no re-embedding + + changed = dict(COMMIT_MEMORY) + changed["body"] = "The user now prefers a single squash commit per PR." + index = refresh_memory_index(root, [changed], embedder=counting_embedder) + self.assertEqual(sum(calls), 3) # one changed record re-embedded + self.assertEqual(list(index["items"]), ["commit-style"]) # deploy pruned + + def test_index_file_is_plain_json(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + refresh_memory_index(root, [COMMIT_MEMORY], embedder=fake_embedder) + payload = json.loads(semantic_index_path(root).read_text(encoding="utf-8")) + self.assertIn("commit-style", payload["items"]) + vector = payload["items"]["commit-style"]["vec"] + self.assertAlmostEqual(math.sqrt(sum(v * v for v in vector)), 1.0, places=3) + + def test_semantic_scores_find_paraphrase(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + scores = semantic_memory_scores( + root, + "how should I structure my pull requests", + [COMMIT_MEMORY, DEPLOY_MEMORY], + embedder=fake_embedder, + ) + + self.assertIn("commit-style", scores) + self.assertGreaterEqual(scores["commit-style"]["cosine"], SEMANTIC_MIN_COSINE) + self.assertGreater(scores["commit-style"]["strength"], 0.0) + self.assertNotIn("deploy-from-main", scores) + + def test_semantic_scores_empty_query_or_failure_degrade_to_empty(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + self.assertEqual(semantic_memory_scores(root, "", [COMMIT_MEMORY], embedder=fake_embedder), {}) + + def broken_embedder(texts: list[str]) -> list[list[float]]: + raise RuntimeError("boom") + + self.assertEqual( + semantic_memory_scores(root, "commit style", [COMMIT_MEMORY], embedder=broken_embedder), + {}, + ) + + def test_recall_rescues_paraphrase_with_capped_confidence(self): + # "structure my pull requests" shares no significant lexical token + # with the deploy memory and few with commit-style's exact tokens. + query = "how should I structure my pull requests" + lexical_only = recall_memories([COMMIT_MEMORY, DEPLOY_MEMORY], query) + with tempfile.TemporaryDirectory() as temp: + scores = semantic_memory_scores( + Path(temp), query, [COMMIT_MEMORY, DEPLOY_MEMORY], embedder=fake_embedder + ) + hybrid = recall_memories([COMMIT_MEMORY, DEPLOY_MEMORY], query, semantic_scores=scores) + + hybrid_names = [str(item["name"]) for item in hybrid] + self.assertIn("commit-style", hybrid_names) + recalled = next(item for item in hybrid if item["name"] == "commit-style") + self.assertIn(recalled["match"], {"semantic", "hybrid"}) + self.assertIn("semantic_similarity", recalled) + if recalled["match"] == "semantic": + # No lexical evidence: confidence must be capped below strong. + self.assertIn(recalled["confidence"], {"weak", "moderate"}) + # Hybrid recall is a superset of lexical recall here. + for item in lexical_only: + self.assertIn(item["name"], hybrid_names) + + def test_lexical_match_keeps_lexical_confidence(self): + results = recall_memories( + [COMMIT_MEMORY], + "commit style", + semantic_scores={"commit-style": {"cosine": 0.9, "strength": 0.9}}, + ) + self.assertEqual(results[0]["match"], "hybrid") + self.assertEqual(results[0]["confidence"], "strong") + + def test_match_points_scale(self): + self.assertEqual(semantic_match_points(None), 0) + self.assertEqual(semantic_match_points({"strength": 0.0}), 0) + self.assertGreaterEqual(semantic_match_points({"strength": 0.5}), 4) + self.assertLessEqual(semantic_match_points({"strength": 1.0}), 10) + + def test_confidence_cap(self): + self.assertEqual(semantic_confidence_cap({"strength": 0.3}), "weak") + self.assertEqual(semantic_confidence_cap({"strength": 0.7}), "moderate") + self.assertEqual(semantic_confidence_cap(None), "weak") + + def test_provider_override_requires_installed_package(self): + import os + from link_core import semantic + + # Neither provider package is installed in CI: overrides must not + # invent a provider, and detection must return None. + for override in ("fastembed", "model2vec"): + os.environ[semantic.SEMANTIC_PROVIDER_ENV] = override + try: + installed = ( + semantic._fastembed_installed() if override == "fastembed" + else semantic._model2vec_installed() + ) + if not installed: + self.assertIsNone(semantic.semantic_provider()) + finally: + os.environ.pop(semantic.SEMANTIC_PROVIDER_ENV, None) + + def test_model_key_is_provider_qualified(self): + from link_core import semantic + + key = semantic.semantic_model_key() + self.assertIn(":", key) + self.assertTrue(key.startswith(("none:", "fastembed:", "model2vec:"))) + + def test_status_without_provider_reports_lexical_only(self): + with tempfile.TemporaryDirectory() as temp: + payload = build_semantic_status(Path(temp), memory_count=3, command_target=temp) + self.assertEqual(payload["mode"], "lexical only") + self.assertFalse(payload["enabled"]) + self.assertTrue(any("--setup" in action for action in payload["next_actions"])) + + def test_memory_embedding_text_is_bounded(self): + record = _memory("big", "Big memory", "x" * 10000) + self.assertLess(len(memory_embedding_text(record)), 1200) + + +if __name__ == "__main__": + unittest.main()