From 8decc8c98cb99547cc3da28880432fb84a186a05 Mon Sep 17 00:00:00 2001 From: Engram Date: Wed, 12 Aug 2026 13:30:50 +0800 Subject: [PATCH 01/25] Add cross-instance memory portability and MCP HTTP auth The export->import roundtrip was broken: /v1/export payloads could not be imported back (raw 500), so a namespace could not move between instances. - Native 'engram' import format (auto-sniffed by engram_export_version): Memory.import_export() restores facts with their original ids, bi-temporal stamps, supersession chains, and provenance; episodes arrive consolidated so System-2 never re-extracts duplicates; idempotent by id; the target re-embeds locally, which is also the embedder-migration path. - /v1/import returns 400 with the parser's reason on malformed payloads. - MCP streamable-HTTP transport gains a Bearer gate (--http-token / ENGRAM_MCP_HTTP_TOKEN); non-loopback binds without a token fail closed. - ENGRAM_STORAGE env selects the vector backend (memory|lancedb), rejecting unknown values at startup so /health reports the real backend. - Import CLI local mode writes through MemoryService: same digest-backed namespace dirs and locks as HTTP/MCP (a/b no longer lands in a stray dir). - /v1/stats filters by canonical linked identity like every other read path. Docs: API.md migration recipe, agent-adapters --http-token, architecture map + full report entries, and the cross-account memory roadmap. --- API.md | 18 +- CHANGELOG.md | 24 +++ docs/agent-adapters.md | 13 ++ docs/architecture-optimization-map.zh-CN.md | 5 +- docs/cross-account-memory-roadmap.zh-CN.md | 125 +++++++++++ docs/engram-full-architecture-report.zh-CN.md | 27 +++ engram/connectors/__init__.py | 14 +- engram/connectors/__main__.py | 87 ++++---- engram/mcp/__main__.py | 67 +++++- engram/mcp/server.py | 16 +- engram/memory.py | 135 +++++++++++- engram/server/app.py | 11 +- engram/service.py | 45 ++-- tests/test_cross_account_portability.py | 202 ++++++++++++++++++ tests/test_mcp_http_auth.py | 86 ++++++++ tests/test_server_import_export.py | 60 ++++++ 16 files changed, 870 insertions(+), 65 deletions(-) create mode 100644 docs/cross-account-memory-roadmap.zh-CN.md create mode 100644 tests/test_cross_account_portability.py create mode 100644 tests/test_mcp_http_auth.py create mode 100644 tests/test_server_import_export.py diff --git a/API.md b/API.md index 79fe34c..6f0e350 100644 --- a/API.md +++ b/API.md @@ -164,9 +164,25 @@ agent session。 | POST | `/v1/conflicts/{id}/resolve` | `{"keep":"newer\|older\|both"}` 让冲突由人确认 | | PATCH | `/v1/facts/{id}` | 改事实 `{"object":"...","sensitive":true}` | | DELETE | `/v1/facts/{id}` | 删一条 | -| GET | `/v1/export?include_sensitive=false` | 安全结构化导出：仅非敏感 facts + graph；不含画像、摘要、原始对话 | +| GET | `/v1/export?include_sensitive=false` | 安全结构化导出：仅非敏感 facts + graph；`include_sensitive=true` 为完整可迁移导出（双时间戳、supersedes 链、provenance、原始对话、摘要、focus） | +| POST | `/v1/import` | 批量导入：`{"data":..., "format":"chatgpt\|messages\|records\|jsonl\|transcript\|engram\|auto"}`；`engram` 格式（自动嗅探）直接还原一份 `/v1/export` 导出——跨实例迁移路径 | | POST | `/v1/forget` | 需 `{"confirm":true}`；清空该 key 的全部记忆（不可逆） | +### 跨实例迁移（换服务器 / 换账号） + +记忆属于用户，不属于某一个部署。把一个 namespace 从实例 A 搬到实例 B： + +```bash +# 1. 从 A 完整导出（含敏感事实与原始对话） +curl -s "$A/v1/export?include_sensitive=true" -H "Authorization: Bearer $KEY_A" > export.json +# 2. 导入 B（事实保留原 id/双时间戳/supersedes 链；目标端用自己的 embedder 重新向量化） +curl -s -X POST "$B/v1/import" -H "Authorization: Bearer $KEY_B" -H "Content-Type: application/json" \ + -d "{\"data\": $(cat export.json), \"format\": \"engram\"}" +``` + +导入按 id 幂等：已存在的条目跳过、绝不覆盖，重复导入不会产生重复记忆。两端 embedder 可以不同—— +这也是更换 embedder 的官方迁移路径。CLI 等价：`python -m engram.connectors -f export.json --api-url $B --key $KEY_B`。 + ## 5. 控制台（可视化） 浏览器开 **`/ui/`** → 输入你的 key → 看「画像 / 事实管理 / 时间线 / 关系图谱 / 记忆问答 / 冲突待确认」。 diff --git a/CHANGELOG.md b/CHANGELOG.md index 456e49d..3af9e21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,30 @@ All notable changes to Engram are documented here. The project uses semantic versioning for distributed artifacts; storage schema compatibility is documented separately when it changes. +## [Unreleased] + +### Added + +- Native export→import roundtrip (`format="engram"`, auto-sniffed): a `/v1/export` payload now restores + directly into another instance via `POST /v1/import`, `Memory.import_export()`, the `engram_import` + MCP tool, or `python -m engram.connectors` — preserving fact ids, bi-temporal stamps, supersession + chains, and provenance. Idempotent by id; the target re-embeds with its own embedder, which is also + the supported embedder-migration path. +- `ENGRAM_STORAGE` environment variable selects the vector backend (`memory` default, `lancedb` + opt-in) for the server/MCP surfaces; unknown values fail closed at startup. +- Optional Bearer authentication for the MCP streamable-HTTP transport (`--http-token` / + `ENGRAM_MCP_HTTP_TOKEN`). Non-loopback `--http` binds without a token are refused at startup unless + `ENGRAM_MCP_HTTP_OPEN=1` explicitly delegates access control to an external layer. + +### Fixed + +- `POST /v1/import` returns 400 with the parser's reason on malformed payloads instead of an + unhandled 500. +- The import CLI's local mode writes through `MemoryService`, so it uses the same digest-backed + namespace directories and locks as the HTTP/MCP surfaces (previously a namespace like `a/b` landed + in a different directory than the one the server reads). +- `/v1/stats` now filters by the canonical linked identity, matching every other read path. + ## [0.1.0] - 2026-07-14 ### Added diff --git a/docs/agent-adapters.md b/docs/agent-adapters.md index 1695b51..a646b96 100644 --- a/docs/agent-adapters.md +++ b/docs/agent-adapters.md @@ -334,6 +334,19 @@ Then connect the client to: http://127.0.0.1:8765/mcp ``` +The HTTP transport is loopback-only by default. To expose it beyond localhost (e.g. for a remote MCP +client), you MUST set a Bearer token — a non-loopback bind without one refuses to start: + +```bash +python -m engram.mcp --http --host 0.0.0.0 --port 8765 \ + --http-token "$(openssl rand -hex 24)" \ + --api-url http://localhost:8000 \ + --api-key me +``` + +Clients then send `Authorization: Bearer ` (most MCP clients accept a headers map in their +remote-server config). Terminate TLS at a reverse proxy in front of it, same as the REST server. + ## OpenAI-Compatible Apps If your app already uses the OpenAI SDK, point it at Engram and pass the memory extension. diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 90c338e..2d3df71 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -1,6 +1,6 @@ # Engram 架构优化地图 -最后更新：2026-07-14 +最后更新：2026-08-12 用途：这是给项目负责人和后续 AI/人类贡献者看的本地中文驾驶舱。它回答四个问题： @@ -100,6 +100,9 @@ flowchart TD | 2026-06-30 | `aggregation_constraint_filter` | Aggregation evidence / Query constraints | 当题面有月份约束时，排除局部上下文绑定到其他月份的数值候选 | `results/aggregation_constraint_filter_experiments.md`, `results/aggregation_constraint_filter_lme_s_context27.jsonl` | | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | +| 2026-08-12 | `cross_instance_portability` | Connectors / Memory facade / Service import 路由 / HTTP `/v1/import` | 导出无法导回（POST 导出 JSON 得到 500）——补上原生 `engram` 导入格式：事实保留原 id/双时间戳/supersedes 链/provenance，目标端用本地 embedder 重嵌入（即官方换 embedder 迁移路径），按 id 幂等；`/v1/import` 对坏 payload 返回 400 | `tests/test_cross_account_portability.py`, `tests/test_server_import_export.py`（工程验收，非算法实验） | +| 2026-08-12 | `mcp_http_bearer_gate` | MCP streamable-HTTP 传输边界 | MCP HTTP 模式此前无任何鉴权，仅靠默认 127.0.0.1；新增 `--http-token`/`ENGRAM_MCP_HTTP_TOKEN` Bearer 门，非回环绑定无 token 时启动即拒绝（失败关闭，与 REST 的 `ENGRAM_API_KEYS` 同哲学） | `tests/test_mcp_http_auth.py` | +| 2026-08-12 | `engram_storage_env` + 一致性修复 | Service 配置边界 / stats / import CLI | 服务器此前永远 `storage="memory"`（无环境变量可选 LanceDB）；新增 `ENGRAM_STORAGE`（非法值失败关闭）。`/v1/stats` 改按 canonical 身份过滤（与其它读路径一致）；import CLI 本地模式改走 `MemoryService`，目录命名与服务端统一 | `tests/test_cross_account_portability.py` | ## 最近 PR 对架构的影响 diff --git a/docs/cross-account-memory-roadmap.zh-CN.md b/docs/cross-account-memory-roadmap.zh-CN.md new file mode 100644 index 0000000..c3a0183 --- /dev/null +++ b/docs/cross-account-memory-roadmap.zh-CN.md @@ -0,0 +1,125 @@ +# 跨账号个人记忆总线 Roadmap + +最后更新：2026-08-12 +状态：阶段 1 已落地（本仓当前分支），阶段 0 等待运维执行，阶段 2 起按序推进。 + +## 目标与定位 + +让 Engram 成为「记忆跟人走」的个人记忆总线：**换厂商账号、换 AI 客户端、换设备、换服务器， +记忆都还在**。身份锚点是「服务器 + API key + namespace」，与任何厂商账号零耦合——所以"跨账号" +不是要新增的能力，而是架构已经成立的性质；本 roadmap 要补的是让这个性质在真实世界可用的工程链条。 + +判断每一步是否值得做的标准（对齐 CLAUDE.md 战略）：是否消除一个"记忆会丢/会断/会泄漏"的场景。 + +## 现状基线（2026-08-12 审计结论） + +已经扎实的（不要重做）： + +- 0.1.0 商业版全部落地：严格 key 鉴权（一租户多 key 轮换）、SHA-256 摘要命名空间、JSONL+manifest + 崩溃安全持久化、非 root 容器 + systemd、`/health` `/ready` 分离、发布门禁 CI。 +- 跨 agent 接入层：`engram-agent-setup / -doctor / -bootstrap` 三个 CLI 覆盖 Codex / Claude Code / + Cursor；MCP stdio + streamable HTTP；OpenAI 兼容代理；TS SDK；跨进程 manifest 指纹重载。 +- **本次新增（阶段 1）**：export→import 原生回路（`format="engram"`，幂等、保 id/双时间戳/ + supersedes 链、目标端重嵌入）；MCP HTTP Bearer 门（`--http-token`，非回环无 token 拒绝启动）； + `ENGRAM_STORAGE` 后端选择；`/v1/import` 坏 payload 返回 400；import CLI 与服务端目录统一； + `/v1/stats` 身份规范化。验收：`tests/test_cross_account_portability.py`、 + `tests/test_server_import_export.py`、`tests/test_mcp_http_auth.py`（22 个新测试，全量回归绿）。 + +主要欠账（按对目标场景的阻塞度排序）： + +1. 用户自己的线上实例是 6 月的 demo 档（内存后端、Hashing embedder、公网明文 HTTP、弱 key）。 +2. key 生命周期：无自助创建/轮换/吊销 API，无 per-key 审计；改 key 要改 env 重启。 +3. 无限流、无 CORS（全部外推反代）；`/health` 无鉴权暴露运营信息。 +4. LanceDB 后端 ~40%（where 过滤全表扫、无 DocStore/GraphStore、与 JSONL 双写）； + 每次写全量重写 JSONL（写放大 O(全部记忆)）。 +5. 单进程单 worker；热租户 LRU 64。 + +--- + +## 阶段 0 —— 把自己的实例升级到 0.1.0（运维，本周可完成） + +**为什么第一**：个人记忆正在公网明文 HTTP 上传输，key 是可猜的 `my-app`，后端是内存档—— +这不是功能缺口，是正在发生的风险。所有后续阶段都以一个可信实例为前提。 + +行动清单： + +1. 用 `deploy/docker-compose.yml` 部署 0.1.0：`ENGRAM_API_KEYS=me:<32+随机字符>`、持久卷、 + `ENGRAM_EMBEDDER=bge-small`（或 bge-m3）、可选 `ENGRAM_LLM` 开启 LLM 抽取。 +2. TLS 反代（Caddy/Nginx，自动续期），只转发受信 Host；按 `deploy/README.md` 网关清单配限流。 +3. 旧数据迁移：旧实例 `GET /v1/export?include_sensitive=true` → 新实例 + `POST /v1/import format=engram`（阶段 1 的成果使这一步成为可能；两端 embedder 不同也可以）。 +4. 本机 MCP 配置切到新地址+新 key（`engram-agent-setup --install-mcp-json --doctor ...`）。 +5. 旧实例下线或封端口。 + +**验收**：`/ready` 200；写入→容器重启→召回保留；`engram-agent-doctor --api-url https://... --api-key ...` +全绿；旧实例数据在新实例可召回；`engram_agent_status` 显示 storage/embedder 为新配置。 + +**决策点（唯一需要 owner 拍板）**：服务器驻留——继续用现有云主机（跨设备，但要信任那台机器）， +还是退回本机/家庭服务器 + 内网穿透（隐私最优，跨设备体验差一档）。默认建议：现有云主机 + TLS + +强 key，敏感 facts 用 `sensitive` 标记走默认脱敏导出。 + +## 阶段 1 —— 可迁移性（已完成，本分支） + +见「现状基线-本次新增」。遗留小项（不阻塞）： + +- TS SDK 补 `importExport` 便捷方法（服务端已支持，SDK 里 `import` 端点已在，只差文档示例）。 +- 控制台 Privacy 页加"导入"入口（目前只有导出下载）。 + +## 阶段 2 —— 跨客户端一致体验（1–2 周） + +存储层是通用的，行为层不是：MCP 只保证工具存在，不保证 agent 会调。这一阶段把"每个客户端都会 +正确地 recall/remember"变成被验证的事实。 + +1. **Codex 接入解冻**：`~/.ai-shared/collab.md` 第 5 条的"暂缓"解除——阶段 0 完成后执行 + `engram-agent-setup --install-codex --doctor --api-url ... --api-key ...`，并用 + `--install-policy` 写入 AGENTS.md 记忆策略块。 +2. **claude.ai 网页/移动端**（真正的"换账号也能用"）：`python -m engram.mcp --http --http-token ...` + 挂在 TLS 反代后，作为 remote MCP connector 添加到 claude.ai 账号。换账号 = 在新账号加一次 + connector（分钟级，一次性）。 +3. **Cursor / 其它 MCP 客户端**：同一 `.mcp.json` 配方。 +4. **召回策略统一**：各客户端的 policy 文案统一维护在 `docs/agent-adapters.md`，变更走 bootstrap + 的 managed block，避免各处漂移。 + +**验收**：`engram-agent-doctor` 对每个客户端跑通远程生命周期（status→remember→close→report→recall）； +一条在 Codex 写入的决策，能在 Claude Code 与 claude.ai 网页端被 recall 命中（跨 agent handoff 冒烟， +`examples/cross_agent_handoff.py --base https://...`）。 + +## 阶段 3 —— key 生命周期与安全深化（2–4 周） + +把"key 即身份"从裸配置升级为可管理的凭据体系（仍是单节点自托管边界，不做企业 SSO/RBAC）： + +1. key 管理 API + 控制台页：同租户多 key 的创建/吊销（写回 env 或独立 keys 文件均可，落盘方案 + 先出一页设计再动手）；不再要求重启。 +2. per-key 审计：写路径在 episode/fact provenance 侧记录 key 指纹（不是 key 明文），回答 + "哪台设备/哪个 agent 写了这条"。 +3. `/health` 信息收敛：无鉴权时只报 `ok/ready/version`，运营细节移入鉴权后的 `/v1/stats`。 +4. 应用层限流（简单令牌桶，按租户），不再完全依赖网关；CORS 白名单可选项。 + +**验收**：吊销一个 key 后旧连接立即 401 且其它 key 不受影响；审计字段在 session_report 可见； +未鉴权 `/health` 响应不含 embedder/租户数。 + +## 阶段 4 —— 规模与后端（与算法路线并行，按 harness 证据推进） + +1. **增量持久化**：JSONL 追加 + 定期 compaction，替代每写全量重写（先用 harness/压测量化写放大， + 再动手——Measure Before Optimizing）。 +2. **LanceDB 补全**：where 过滤下推（先按 user_id 分表或 Lance filter 表达式）、大表 ANN 索引； + DocStore/GraphStore 是否迁移按测量决定。 +3. multi-worker 前置条件梳理（文件锁已可跨进程，线程锁失效面要清点）。 + +**验收**：10 万 facts 规模下写延迟与 p95 recall 延迟对比基线成表（accuracy + tokens + latency 三联， +遵守 Bet D），否则不合并。 + +## 阶段 5 —— 产品面（Path-A 开发者基础设施定位的延伸，按需启动) + +- 控制台：导入 UI、key 管理 UI、迁移向导（导出→导入一条龙）。 +- 托管多用户注册/开通流（若决定做 hosted 服务，需要真正的用户体系——独立规格，不混入本 roadmap）。 + +--- + +## 与既有机制的关系 + +- **Claude Code auto-memory（MEMORY.md）与 Engram 并行**：auto-memory 绑定机器+项目路径，作为 + 本机缓存继续用；跨机器/跨客户端的持久事实沉到 Engram。阶段 2 完成后可评估把 auto-memory 的 + 高价值条目定期 `engram_remember` 化（脚本化，不手抄）。 +- **公开数字纪律**：本 roadmap 全部为工程交付，不产生任何算法性能主张；LongMemEval 相关数字 + 仍以 `RESULTS.md` + 已提交 JSONL 为准。 diff --git a/docs/engram-full-architecture-report.zh-CN.md b/docs/engram-full-architecture-report.zh-CN.md index 6db1969..41ccefb 100644 --- a/docs/engram-full-architecture-report.zh-CN.md +++ b/docs/engram-full-architecture-report.zh-CN.md @@ -667,6 +667,33 @@ flowchart LR 这条链路没有改变 `Memory` 内部的抽取、冲突、图构建或检索算法。工程发布结果记录到 `results/commercial_release_0_1_0_validation.jsonl`，不能被当作算法效果提升证据。 +### 10.2 跨实例迁移数据流（2026-08-12 新增） + +记忆的所有权锚点是「服务器 + key + namespace」，与任何厂商账号无关。换服务器/换部署时， +`/v1/export` 的产物现在可以直接导回 `/v1/import`（原生 `engram` 格式，按 `engram_export_version` 自动嗅探）： + +```mermaid +flowchart LR + A["实例 A\nGET /v1/export?include_sensitive=true"] --> P["export payload v1\nfacts(id/双时间戳/supersedes/provenance)\n+ episodes + summaries + focus + graph"] + P --> B["实例 B\nPOST /v1/import format=engram"] + B --> R["Memory.import_export()\n按 id 幂等跳过已存在项\n本地 embedder 重嵌入\nGraphBuilder 重建图(保留失效边)"] + R --> S["episodes 标记 consolidated\n(事实已随导出携带,不重复抽取)"] +``` + +关键规则： + +1. **幂等**：已存在的 fact/episode id 直接跳过、绝不覆盖（现有记忆优先），重复导入零副作用。 +2. **重嵌入即迁移**：导出不携带向量；目标端用自己的 embedder 重算——这也是更换 embedder 的官方路径 + （此前 manifest 的 `embedder_id/embedding_dim` 硬校验导致换 embedder 等于存储报废）。 +3. **历史以历史身份迁移**：GraphBuilder 把 `invalid_at` 一并写到关系边上，superseded 事实不会复活。 +4. **share-safe 导出同样可导入**（只有非敏感 facts + graph），敏感内容不会经由默认导出泄漏到新实例。 +5. 坏 payload 由 `/v1/import` 返回 400 + 解析原因（此前是裸 500）。 + +同时补齐的服务边界：`ENGRAM_STORAGE` 环境变量显式选择向量后端（非法值启动即失败）；MCP +streamable-HTTP 传输新增 `--http-token`/`ENGRAM_MCP_HTTP_TOKEN` Bearer 门，非回环绑定无 token 拒绝启动 +（`ENGRAM_MCP_HTTP_OPEN=1` 才可显式豁免）；import CLI 本地模式改走 `MemoryService`，与服务端共用同一套 +摘要目录和锁；`/v1/stats` 与其它读路径一致按 canonical 身份过滤。 + ## 11. Eval harness 数据流 评测不是附属品，是架构的一部分。Engram 的所有算法主张都应能走 `eval/bench.py`。 diff --git a/engram/connectors/__init__.py b/engram/connectors/__init__.py index e2792f5..d50ee63 100644 --- a/engram/connectors/__init__.py +++ b/engram/connectors/__init__.py @@ -13,6 +13,9 @@ records flat list of {content, [session_id], [speaker], [timestamp]} jsonl JSON-Lines, one record per line transcript plain text / markdown ("Speaker: text" lines, or freeform) + engram a native `/v1/export` payload (engram_export_version) — NOT parsed into sessions; + it restores directly via `Memory.import_export()` (facts keep their ids, bi-temporal + stamps, and supersession chains instead of being re-extracted). """ from __future__ import annotations @@ -28,7 +31,7 @@ "extract_text", "to_epoch", ] -FORMATS = ("chatgpt", "messages", "records", "jsonl", "transcript") +FORMATS = ("chatgpt", "messages", "records", "jsonl", "transcript", "engram") def sniff(data: Any) -> str: @@ -54,6 +57,8 @@ def sniff(data: Any) -> str: def _sniff_obj(obj: Any) -> str: if isinstance(obj, dict): + if "engram_export_version" in obj: + return "engram" # a native export restores directly; it is not a message-shaped history if "mapping" in obj or "conversations" in obj: return "chatgpt" if "messages" in obj: @@ -89,6 +94,13 @@ def parse(data: Any, format: str = "auto", session_id: str = "imported") -> list if fmt == "auto": fmt = sniff(data) + if fmt == "engram": + raise ValueError( + "this is a native Engram export (engram_export_version): it restores facts/episodes " + "directly instead of parsing into sessions. Import it via Memory.import_export(payload), " + "MemoryService.import_(format='engram'), POST /v1/import {'data': ..., 'format': 'engram'}, " + "or `python -m engram.connectors --format engram`." + ) if fmt == "transcript" or fmt in ("text", "markdown", "md"): text = data.decode("utf-8") if isinstance(data, (bytes, bytearray)) else str(data) return parse_transcript(text, session_id=session_id) diff --git a/engram/connectors/__main__.py b/engram/connectors/__main__.py index d377c36..1d7e040 100644 --- a/engram/connectors/__main__.py +++ b/engram/connectors/__main__.py @@ -9,11 +9,15 @@ # push into a running Engram server instead of a local store: python -m engram.connectors -f conversations.json --api-url http://localhost:8000 --key sk-alice-123 + # migrate a namespace between instances (a native /v1/export payload restores directly): + python -m engram.connectors -f export.json -n me # format auto-sniffed as 'engram' + # from stdin with an explicit format: cat log.jsonl | python -m engram.connectors --format jsonl -n me -This module is pure-stdlib; the remote POST uses urllib (no httpx needed) so import works in the -zero-setup install. +Local mode writes through MemoryService, so the CLI uses exactly the same digest-backed namespace +directories (and locks) as the HTTP/MCP surfaces — an import is immediately visible to a running +local agent. The remote POST uses urllib (no httpx needed) so import works in the zero-setup install. """ from __future__ import annotations @@ -25,6 +29,7 @@ import urllib.request from . import parse, sniff +from .base import load_json def _read_input(path: str | None) -> str: @@ -34,45 +39,32 @@ def _read_input(path: str | None) -> str: return sys.stdin.read() -def _post_remote(api_url: str, key: str, sessions: list, consolidate: bool, summarize: bool) -> dict: - body = json.dumps({ - "sessions": [dataclasses.asdict(s) for s in sessions], - "consolidate": consolidate, - "summarize": summarize, - }).encode("utf-8") +def _post_remote(api_url: str, key: str, body: dict) -> dict: + payload = json.dumps(body).encode("utf-8") headers = {"Content-Type": "application/json"} if key: headers["Authorization"] = f"Bearer {key}" - req = urllib.request.Request(api_url.rstrip("/") + "/v1/import", data=body, + req = urllib.request.Request(api_url.rstrip("/") + "/v1/import", data=payload, headers=headers, method="POST") with urllib.request.urlopen(req, timeout=600) as resp: # noqa: S310 (user-supplied URL is intentional) return json.load(resp) -def _import_local(args, sessions) -> dict: - from .. import Memory # lazy: keep the connectors package importable without the full facade graph - - embedder = llm = None - if args.embedder or args.llm: - from ..llm.providers import load_dotenv, make_embedder, make_llm - load_dotenv() - if args.embedder: - embedder = make_embedder(args.embedder) - if args.llm: - llm = make_llm(args.llm) - - os.makedirs(os.path.expanduser(args.data_dir), exist_ok=True) - safe = "".join(c for c in args.namespace if c.isalnum() or c in "-_.") or "me" - base = os.path.expanduser(args.data_dir) - path = os.path.join(base, safe) - legacy = os.path.join(base, f"{safe}.pkl") - if os.path.exists(legacy) and not os.path.exists(path): - path = legacy - mem = Memory.open(path, embedder=embedder, llm=llm) - stats = mem.import_messages(sessions, user_id=args.namespace, - consolidate=not args.no_consolidate, summarize=not args.no_summarize) - mem.save() - stats["store"] = path +def _service(args): + from ..service import MemoryService # lazy: keep the parser-only paths import-light + + return MemoryService(data_dir=os.path.expanduser(args.data_dir), + embedder_name=args.embedder, llm_name=args.llm) + + +def _import_local(args, sessions=None, engram_payload=None) -> dict: + svc = _service(args) + if engram_payload is not None: + stats = svc.import_(args.namespace, data=engram_payload, format="engram") + else: + stats = svc.import_(args.namespace, sessions=sessions, + consolidate=not args.no_consolidate, summarize=not args.no_summarize) + stats["store"] = svc._path(args.namespace) return stats @@ -81,7 +73,7 @@ def main() -> None: description="Import a chat/history export into Engram memory.") ap.add_argument("--file", "-f", help="export file (default: stdin)") ap.add_argument("--format", default="auto", - help="chatgpt | messages | records | jsonl | transcript | auto (default)") + help="chatgpt | messages | records | jsonl | transcript | engram | auto (default)") ap.add_argument("--namespace", "-n", default="me", help="memory namespace / user id (default: me)") ap.add_argument("--data-dir", default=os.environ.get("ENGRAM_DATA_DIR", "~/.engram/data"), help="local store dir (local mode; default ~/.engram/data)") @@ -95,7 +87,23 @@ def main() -> None: args = ap.parse_args() raw = _read_input(args.file) - fmt = sniff(raw) if args.format == "auto" else args.format + fmt = sniff(raw) if args.format == "auto" else args.format.lower().strip() + + if fmt == "engram": + payload = load_json(raw) + n_facts = len(payload.get("facts") or []) if isinstance(payload, dict) else 0 + n_eps = len(payload.get("episodes") or []) if isinstance(payload, dict) else 0 + print(f"parsed native Engram export: {n_facts} fact(s), {n_eps} episode(s) [format: engram]", + file=sys.stderr) + if args.dry_run: + return + if args.api_url: + stats = _post_remote(args.api_url, args.key, {"data": payload, "format": "engram"}) + else: + stats = _import_local(args, engram_payload=payload) + print(json.dumps(stats, ensure_ascii=False)) + return + sessions = parse(raw, format=args.format) n_msgs = sum(len(s.messages) for s in sessions) print(f"parsed {len(sessions)} session(s), {n_msgs} message(s) [format: {fmt}]", file=sys.stderr) @@ -110,10 +118,13 @@ def main() -> None: return if args.api_url: - stats = _post_remote(args.api_url, args.key, sessions, - not args.no_consolidate, not args.no_summarize) + stats = _post_remote(args.api_url, args.key, { + "sessions": [dataclasses.asdict(s) for s in sessions], + "consolidate": not args.no_consolidate, + "summarize": not args.no_summarize, + }) else: - stats = _import_local(args, sessions) + stats = _import_local(args, sessions=sessions) print(json.dumps(stats, ensure_ascii=False)) diff --git a/engram/mcp/__main__.py b/engram/mcp/__main__.py index 60fc097..aa10822 100644 --- a/engram/mcp/__main__.py +++ b/engram/mcp/__main__.py @@ -7,8 +7,9 @@ # proxy a running Engram HTTP server (hosted / multi-tenant): python -m engram.mcp --api-url http://localhost:8000 --api-key sk-alice-123 - # serve over streamable HTTP instead of stdio: + # serve over streamable HTTP instead of stdio (loopback only unless a token is set): python -m engram.mcp --http --port 8765 + python -m engram.mcp --http --host 0.0.0.0 --http-token # remote MCP clients Claude Desktop config (claude_desktop_config.json): {"mcpServers": {"engram": {"command": "python", "args": ["-m", "engram.mcp"]}}} @@ -16,9 +17,56 @@ from __future__ import annotations import argparse +import hmac import os import sys +_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"} + + +class _BearerGate: + """Minimal ASGI gate: require `Authorization: Bearer ` on every HTTP request. + + The MCP streamable-HTTP transport has no authentication of its own, so anyone who can reach the + port can read and write the memory namespace behind it. Loopback binding is the default guard; + this gate is what makes a non-loopback bind safe. Same failure-closed philosophy as the REST + server's ENGRAM_API_KEYS. + """ + + def __init__(self, app, token: str) -> None: + self.app = app + self.token = token + + async def __call__(self, scope, receive, send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + auth = "" + for key, value in scope.get("headers") or []: + if key.decode("latin-1").lower() == "authorization": + auth = value.decode("latin-1") + break + supplied = auth[7:].strip() if auth.startswith("Bearer ") else "" + if not (supplied and hmac.compare_digest(supplied, self.token)): + await send({"type": "http.response.start", "status": 401, + "headers": [(b"content-type", b"application/json"), + (b"www-authenticate", b"Bearer")]}) + await send({"type": "http.response.body", "body": b'{"error":"unauthorized"}'}) + return + await self.app(scope, receive, send) + + +def _require_http_token(host: str, token: str, allow_open: bool) -> None: + """Fail closed: refuse a non-loopback --http bind without a token unless explicitly opened.""" + if host in _LOOPBACK_HOSTS or token or allow_open: + return + raise SystemExit( + f"refusing to serve unauthenticated MCP over HTTP on non-loopback host {host!r}: anyone who can " + "reach this port could read and write this memory namespace. Set --http-token (or " + "ENGRAM_MCP_HTTP_TOKEN), keep the default 127.0.0.1 bind behind a reverse proxy, or set " + "ENGRAM_MCP_HTTP_OPEN=1 only if an external layer already enforces access control." + ) + def main() -> None: ap = argparse.ArgumentParser( @@ -27,6 +75,9 @@ def main() -> None: ap.add_argument("--http", action="store_true", help="serve over streamable HTTP instead of stdio") ap.add_argument("--host", default="127.0.0.1", help="bind host for --http (default 127.0.0.1)") ap.add_argument("--port", type=int, default=8765, help="bind port for --http (default 8765)") + ap.add_argument("--http-token", default=os.environ.get("ENGRAM_MCP_HTTP_TOKEN", ""), + help="Bearer token required on every --http request " + "(default $ENGRAM_MCP_HTTP_TOKEN; mandatory for non-loopback binds)") ap.add_argument("--namespace", "-n", help="local memory namespace (default $ENGRAM_NAMESPACE or 'me')") ap.add_argument("--api-url", help="proxy a running Engram server (default $ENGRAM_API_URL); else local") ap.add_argument("--api-key", help="Bearer key for --api-url (default $ENGRAM_API_KEY)") @@ -46,10 +97,20 @@ def main() -> None: b = backend() print(f"engram_mcp ready · {b.describe()}", file=sys.stderr) if args.http: + allow_open = os.environ.get("ENGRAM_MCP_HTTP_OPEN", "").strip().lower() in {"1", "true", "yes", "on"} + _require_http_token(args.host, args.http_token, allow_open) mcp.settings.host = args.host mcp.settings.port = args.port - print(f" transport: streamable-http on http://{args.host}:{args.port}/mcp", file=sys.stderr) - mcp.run(transport="streamable-http") + guard = "Bearer token required" if args.http_token else "no token (loopback/explicit-open)" + print(f" transport: streamable-http on http://{args.host}:{args.port}/mcp · {guard}", + file=sys.stderr) + if args.http_token: + import uvicorn + + uvicorn.run(_BearerGate(mcp.streamable_http_app(), args.http_token), + host=args.host, port=args.port) + else: + mcp.run(transport="streamable-http") else: print(" transport: stdio", file=sys.stderr) mcp.run() diff --git a/engram/mcp/server.py b/engram/mcp/server.py index c8d7c38..c77c453 100644 --- a/engram/mcp/server.py +++ b/engram/mcp/server.py @@ -656,20 +656,28 @@ async def engram_delete_fact( async def engram_import( content: Annotated[str | list | dict, Field(description="The raw export to import: a ChatGPT conversations.json, an " - "OpenAI messages array, JSON-Lines, or a plain 'Speaker: text' transcript. " + "OpenAI messages array, JSON-Lines, a plain 'Speaker: text' transcript, or " + "a native Engram export from engram_export (restores facts/episodes with " + "their ids and history intact — the migration path between instances). " "Paste the file contents (JSON text, an array, or transcript text).")], - format: Annotated[str, Field(description="chatgpt | messages | records | jsonl | transcript | auto " - "(default, sniffs the shape).")] = "auto", + format: Annotated[str, Field(description="chatgpt | messages | records | jsonl | transcript | " + "engram | auto (default, sniffs the shape).")] = "auto", ) -> str: """Bulk-import an exported chat history into memory in one batched pass (extract facts + summaries). Use this to seed memory from an existing history (e.g. the user's ChatGPT export) rather than - replaying it message by message. `format='auto'` detects the shape. + replaying it message by message. A native Engram export restores directly instead of re-extracting. + `format='auto'` detects the shape. """ try: data = await backend().import_(content, format=format) except Exception as e: # noqa: BLE001 return _err(e) + if data.get("format") == "engram": + return (f"Restored Engram export: {data.get('facts', 0)} fact(s) " + f"({data.get('facts_skipped', 0)} already present), {data.get('episodes', 0)} episode(s) " + f"({data.get('episodes_skipped', 0)} already present), " + f"{data.get('summaries', 0)} summary(ies).") return (f"Imported {data.get('sessions', 0)} session(s) / {data.get('episodes', 0)} episode(s); " f"extracted {data.get('facts_added', 0)} fact(s), {data.get('summaries', 0)} summary(ies).") diff --git a/engram/memory.py b/engram/memory.py index 5cb8758..c935fbd 100644 --- a/engram/memory.py +++ b/engram/memory.py @@ -447,11 +447,142 @@ def _coerce(s) -> Optional[ImportSession]: def import_data(self, data, format: str = "auto", user_id: str = "default", session_id: str = "imported", **kwargs) -> dict[str, int]: """Convenience: parse a raw export (ChatGPT/OpenAI/JSONL/transcript — auto-sniffed) and import it - in one call. See `engram.connectors.parse` for formats.""" - from .connectors import parse + in one call. A native Engram export (engram_export_version) routes to `import_export()` instead — + it restores facts/episodes directly rather than re-extracting them from parsed sessions.""" + from .connectors import parse, sniff + from .connectors.base import load_json + fmt = (format or "auto").lower().strip() + if fmt == "engram" or (fmt == "auto" and sniff(data) == "engram"): + return self.import_export(load_json(data), user_id=user_id) return self.import_messages(parse(data, format=format, session_id=session_id), user_id=user_id, **kwargs) + def import_export(self, payload: dict, user_id: str = "default") -> dict: + """Restore a native `export()` payload — the migration path between Engram instances. + + This is what makes memory belong to the user rather than to one deployment: facts arrive with + their original ids, bi-temporal stamps, supersession chains, and provenance intact (no + re-extraction), and the graph is rebuilt from them through the canonical GraphBuilder path. + Three invariants: + * Idempotent by id — an item whose id already exists locally is SKIPPED, never overwritten + (existing memory wins; re-importing the same export cannot duplicate or corrupt anything). + * Re-embedded locally — embeddings are never part of an export, so the target's embedder + regenerates them. This is also the supported way to migrate a store between embedders. + * No System-2 replay — imported episodes are marked consolidated because their extracted + facts are already in the payload; re-running extraction would mint duplicate facts. + Works on both export flavors: share-safe (facts + graph only) and include_sensitive=true + (full fidelity). Raises ValueError on a payload this build cannot read. + """ + if not isinstance(payload, dict) or "engram_export_version" not in payload: + raise ValueError("not an Engram export: missing 'engram_export_version'") + version = payload.get("engram_export_version") + if version != 1: + raise ValueError(f"unsupported engram_export_version {version!r}; this build reads version 1") + user = self.resolver.resolve(user_id) + + new_eps: list[Episode] = [] + episodes_skipped = 0 + for e in payload.get("episodes") or []: + if not isinstance(e, dict) or not str(e.get("content") or ""): + continue + eid = str(e.get("id") or "") + if eid and self.episodes_doc.get(eid) is not None: + episodes_skipped += 1 + continue + ep = Episode( + content=str(e.get("content", "")), + user_id=user, + session_id=str(e.get("session_id") or "imported"), + speaker=str(e.get("speaker") or "session"), + event_time=float(e.get("event_time") or now()), + consolidated=True, + summary=str(e.get("summary") or ""), + ) + if eid: + ep.id = eid + ep.metadata["date"] = str(e.get("date") or fmt_date(ep.event_time)) + ep.metadata["source"] = "engram-export" + new_eps.append(ep) + if new_eps: + vecs = self.embedder.embed_batch([ep.content for ep in new_eps]) + for ep, vec in zip(new_eps, vecs): + ep.embedding = vec + self.episodes_doc.put(ep.id, ep) + self.episodes_vec.upsert(ep.id, vec, ep) + with_summary = [ep for ep in new_eps if ep.summary] + if with_summary: + svecs = self.embedder.embed_batch([ep.summary for ep in with_summary]) + for ep, vec in zip(with_summary, svecs): + ep.summary_embedding = vec + self.summary_vec.upsert(ep.id, vec, ep) + + new_facts: list[Fact] = [] + facts_skipped = 0 + for fd in payload.get("facts") or []: + if not isinstance(fd, dict): + continue + fid = str(fd.get("id") or "") + if fid and (self.fact_store.get(fid) is not None or self.cold_store.get(fid) is not None): + facts_skipped += 1 + continue + f = Fact( + subject=str(fd.get("subject") or ""), + predicate=str(fd.get("predicate") or ""), + object=str(fd.get("object") or ""), + text=str(fd.get("text") or ""), + display=str(fd.get("display") or ""), + user_id=user, + salience=float(fd.get("salience") or 1.0), + confidence=float(fd.get("confidence") or 1.0), + source=str(fd.get("source") or "extracted"), + category=str(fd.get("category") or ""), + sensitive=bool(fd.get("sensitive", False)), + valid_at=float(fd.get("valid_at") or now()), + invalid_at=float(fd["invalid_at"]) if fd.get("invalid_at") is not None else None, + created_at=float(fd.get("created_at") or now()), + expired_at=float(fd["expired_at"]) if fd.get("expired_at") is not None else None, + supersedes=str(fd["supersedes"]) if fd.get("supersedes") else None, + provenance=[str(p) for p in (fd.get("provenance") or [])], + ) + if fid: + f.id = fid + new_facts.append(f) + if new_facts: + vecs = self.embedder.embed_batch([f.text for f in new_facts]) + for f, vec in zip(new_facts, vecs): + f.embedding = vec + self.fact_store.upsert(f.id, vec, f) + # GraphBuilder copies valid_at/invalid_at onto the relation, so a superseded fact's + # edge arrives already invalidated — history moves as history, never resurrected. + self.engine.graph_builder.add_fact(f) + + focus = payload.get("focus") or {} + focus_terms_added = 0 + for key in ("track", "mute"): + incoming = [str(t).strip() for t in (focus.get(key) or []) if str(t).strip()] + if not incoming: + continue + merged = list(dict.fromkeys([*self.focus.get(key, []), *incoming])) + focus_terms_added += len(merged) - len(self.focus.get(key, [])) + self.focus[key] = merged + if focus_terms_added: + self.apply_focus() + + self._enforce_hot_limit() + self._persona_cache.clear() + return { + "format": "engram", + "engram_export_version": 1, + "sessions": len({ep.session_id for ep in new_eps}), + "episodes": len(new_eps), + "episodes_skipped": episodes_skipped, + "facts": len(new_facts), + "facts_skipped": facts_skipped, + "facts_added": len(new_facts), + "summaries": sum(1 for ep in new_eps if ep.summary), + "focus_terms_added": focus_terms_added, + } + def link_identity(self, a: str, b: str) -> str: return self.resolver.link(a, b) diff --git a/engram/server/app.py b/engram/server/app.py index b7f1aa9..cf10b45 100644 --- a/engram/server/app.py +++ b/engram/server/app.py @@ -521,11 +521,16 @@ class ImportReq(BaseModel): @app.post("/v1/import") def import_history(req: ImportReq, user: str = Depends(auth)): """Bulk-ingest an external history in one batched pass. See `python -m engram.connectors` for a CLI - that parses common exports and posts here.""" + that parses common exports and posts here. A native `/v1/export` payload (format='engram' or + auto-sniffed) restores directly — the cross-instance migration path.""" if req.sessions is None and req.data is None: raise HTTPException(400, "provide either 'sessions' (pre-parsed) or 'data' (+ 'format') to import") - return svc().import_(user, sessions=req.sessions, data=req.data, format=req.format, - consolidate=req.consolidate, summarize=req.summarize) + try: + return svc().import_(user, sessions=req.sessions, data=req.data, format=req.format, + consolidate=req.consolidate, summarize=req.summarize) + except ValueError as exc: + # a malformed payload is the CLIENT's error — 400 with the parser's reason, never a raw 500 + raise HTTPException(400, f"import failed: {exc}") from exc # --- OpenAI-compatible chat with transparent memory (drop-in: point your OpenAI client's base_url here) -- diff --git a/engram/service.py b/engram/service.py index b978f48..8296a96 100644 --- a/engram/service.py +++ b/engram/service.py @@ -119,6 +119,13 @@ def __init__( from .config import Config self.config = Config() + # ENGRAM_STORAGE selects the vector backend ('memory' default, 'lancedb' opt-in). Fail closed on + # anything else: silently falling back to 'memory' would misreport /health's storage field. + storage = os.environ.get("ENGRAM_STORAGE", "").strip().lower() + if storage: + if storage not in {"memory", "lancedb"}: + raise ValueError(f"ENGRAM_STORAGE must be 'memory' or 'lancedb', got {storage!r}") + self.config.storage = storage if os.environ.get("ENGRAM_MAX_HOT_FACTS"): self.config.max_hot_facts = int(os.environ["ENGRAM_MAX_HOT_FACTS"]) # opt-in System-2 LLM conflict detection -> the detect->confirm loop (needs an LLM). Off by @@ -307,11 +314,21 @@ def import_(self, user: str, sessions: Optional[list] = None, format: str = "aut data: Any = None, consolidate: bool = True, summarize: bool = True, session_id: str = "imported") -> dict: """Bulk import: either pre-parsed `sessions` (list of ImportSession/dicts) OR raw `data` to parse - with `format` (chatgpt/messages/records/jsonl/transcript/auto). One batched ingest + consolidation.""" + with `format` (chatgpt/messages/records/jsonl/transcript/auto). One batched ingest + consolidation. + + A native Engram export (`format='engram'`, or auto-sniffed by its engram_export_version) takes + the direct restore path instead: facts/episodes keep their ids, bi-temporal stamps, and + supersession chains — this is how a namespace moves between instances.""" with self.write_lock(user): mem = self.get(user) if sessions is None: - from .connectors import parse + from .connectors import parse, sniff + from .connectors.base import load_json + fmt = (format or "auto").lower().strip() + if fmt == "engram" or (fmt == "auto" and sniff(data) == "engram"): + stats = mem.import_export(load_json(data), user_id=user) + self._save(user, mem) + return {"ok": True, **stats} sessions = parse(data, format=format, session_id=session_id) stats = mem.import_messages(sessions, user_id=user, consolidate=consolidate, summarize=summarize) @@ -879,25 +896,28 @@ def stats(self, user: str) -> dict: """Content-free namespace stats for dashboards/readiness probes. This intentionally avoids profile text, fact text, episode snippets, and data paths so it is safe to poll in production.""" mem = self.get(user) - episodes = [ep for ep in mem.episodes_doc.values() if ep.user_id == user] - hot_facts = [f for f in mem.fact_store.values() if f.user_id == user] - cold_facts = [f for f in mem.cold_store.values() if f.user_id == user] + # Filter by the CANONICAL identity, like every other read path: after link_identity, data is + # written under the canonical id, so filtering by the raw handle would undercount. + canonical = mem.resolver.resolve(user) + episodes = [ep for ep in mem.episodes_doc.values() if ep.user_id == canonical] + hot_facts = [f for f in mem.fact_store.values() if f.user_id == canonical] + cold_facts = [f for f in mem.cold_store.values() if f.user_id == canonical] facts = hot_facts + cold_facts facts_by_id = {f.id: f for f in facts} - working = [w for w in mem.working_mem.values() if w.user_id == user] + working = [w for w in mem.working_mem.values() if w.user_id == canonical] live_facts = [f for f in facts if f.is_live()] superseded = [f for f in facts if not f.is_live()] sensitive = [f for f in facts if getattr(f, "sensitive", False)] pending_conflicts = [ c for c in mem.conflicts.values() - if c.user_id == user and c.status == "pending" + if c.user_id == canonical and c.status == "pending" ] consolidated_episodes = [ep for ep in episodes if ep.consolidated] pending_episodes = [ep for ep in episodes if not ep.consolidated] ephemeral_episodes = [ep for ep in episodes if ep.metadata.get("ephemeral")] event_times = [ep.event_time for ep in episodes] fact_times = [f.valid_at for f in facts] - user_entities = [e for e in mem.graph.entities.values() if e.user_id == user] + user_entities = [e for e in mem.graph.entities.values() if e.user_id == canonical] all_relations = mem.graph.relations() user_entity_ids = {e.id for e in user_entities} user_relations = [ @@ -917,13 +937,13 @@ def stats(self, user: str) -> dict: "episodes_ephemeral": len(ephemeral_episodes), "facts_hot": len(hot_facts), "facts_cold": len(cold_facts), - "cold_pages_out": int(mem.cold_pages_out.get(user, 0)), - "cold_pages_in": int(mem.cold_pages_in.get(user, 0)), + "cold_pages_out": int(mem.cold_pages_out.get(canonical, 0)), + "cold_pages_in": int(mem.cold_pages_in.get(canonical, 0)), "facts_live": len(live_facts), "facts_superseded": len(superseded), "facts_sensitive": len(sensitive), "working_live": sum(1 for w in working if w.is_live()), - "summaries": len([s for s in mem.summary_vec.values() if s.user_id == user]), + "summaries": len([s for s in mem.summary_vec.values() if s.user_id == canonical]), "entities": len(user_entities), "relations": sum(1 for r in user_relations if r.fact_id in facts_by_id), "graph_orphan_entities": sum(1 for e in user_entities if e.id not in referenced_entity_ids), @@ -970,7 +990,8 @@ def export(self, user: str, include_sensitive: bool = False) -> dict: "focus": mem.get_focus(), "facts": [{ "id": f.id, "subject": f.subject, "predicate": f.predicate, "object": f.object, - "text": f.text, "source": f.source, "status": "live" if f.is_live() else "superseded", + "text": f.text, "display": getattr(f, "display", ""), + "source": f.source, "status": "live" if f.is_live() else "superseded", "category": getattr(f, "category", ""), "sensitive": getattr(f, "sensitive", False), "salience": round(f.salience, 3), "confidence": f.confidence, "valid_at": f.valid_at, "valid_at_h": fmt_date(f.valid_at), diff --git a/tests/test_cross_account_portability.py b/tests/test_cross_account_portability.py new file mode 100644 index 0000000..60a173a --- /dev/null +++ b/tests/test_cross_account_portability.py @@ -0,0 +1,202 @@ +"""Cross-instance portability — the cross-account memory bus. + +The memory belongs to the user, not to any one instance/account: an `export()` payload must restore +into a different Engram instance (different data dir, possibly a different embedder), preserving fact +ids, bi-temporal stamps, supersession chains, and provenance. These tests drive the native "engram" +import format, the identity-consistent stats fix, the import-CLI namespace unification, and the +ENGRAM_STORAGE backend selector. +""" +from __future__ import annotations + +import json +import sys + +import pytest + +from engram.connectors import parse, sniff +from engram.service import MemoryService + + +@pytest.fixture() +def clean_env(monkeypatch): + for var in ("ENGRAM_EMBEDDER", "ENGRAM_LLM", "ENGRAM_ANSWERER", "ENGRAM_STORAGE", + "ENGRAM_MAX_HOT_FACTS", "ENGRAM_CONFLICT_DETECTION", "ENGRAM_DATA_DIR"): + monkeypatch.delenv(var, raising=False) + + +def _seed(svc: MemoryService, user: str) -> None: + svc.remember(user, "I moved to Berlin in 2024 and work on the Engram project.", session_id="s1") + svc.add_fact(user, "user", "lives_in", "Beijing") + svc.add_fact(user, "user", "lives_in", "Shanghai") # supersedes Beijing -> a chain to preserve + + +# --- format detection ------------------------------------------------------- + +def test_sniff_recognizes_engram_export(tmp_path, clean_env): + svc = MemoryService(data_dir=str(tmp_path / "a")) + _seed(svc, "alice") + payload = svc.export("alice", include_sensitive=True) + assert sniff(payload) == "engram" + assert sniff(json.dumps(payload)) == "engram" + + +def test_parse_gives_actionable_error_for_engram_format(tmp_path, clean_env): + svc = MemoryService(data_dir=str(tmp_path / "a")) + _seed(svc, "alice") + payload = svc.export("alice", include_sensitive=True) + # parse() produces sessions; a native export restores directly — the error must say where to go. + with pytest.raises(ValueError, match="engram"): + parse(payload) + + +# --- the roundtrip ---------------------------------------------------------- + +def test_export_import_roundtrip_preserves_memory(tmp_path, clean_env): + src = MemoryService(data_dir=str(tmp_path / "src")) + _seed(src, "alice") + payload = src.export("alice", include_sensitive=True) + + dst = MemoryService(data_dir=str(tmp_path / "dst")) + stats = dst.import_("alice", data=payload, format="auto") + assert stats["ok"] is True + assert stats["format"] == "engram" + assert stats["facts"] == len(payload["facts"]) + assert stats["episodes"] == len(payload["episodes"]) + + out = dst.export("alice", include_sensitive=True) + assert {f["id"] for f in out["facts"]} == {f["id"] for f in payload["facts"]} + + # bi-temporal stamps, supersession chain, and provenance survive the move + src_by_id = {f["id"]: f for f in payload["facts"]} + for f in out["facts"]: + assert f["valid_at"] == pytest.approx(src_by_id[f["id"]]["valid_at"]) + assert f["supersedes"] == src_by_id[f["id"]]["supersedes"] + assert f["provenance"] == src_by_id[f["id"]]["provenance"] + assert any(f["supersedes"] for f in out["facts"]), "Beijing->Shanghai chain must survive" + assert any(f["invalid_at"] for f in out["facts"]), "superseded facts must stay invalidated" + + # the graph is rebuilt on the target, and episodes are not re-queued for System-2 + assert out["graph"]["nodes"] and out["graph"]["edges"] + assert dst.stats("alice")["counts"]["episodes_pending"] == 0 + + # and the target instance actually answers from the migrated memory + res = dst.recall("alice", "Where does the user live now?", lean=False) + assert any("Shanghai" in t for t in res["facts"]) or "Shanghai" in res["answer"] + + +def test_reimport_is_idempotent(tmp_path, clean_env): + src = MemoryService(data_dir=str(tmp_path / "src")) + _seed(src, "alice") + payload = src.export("alice", include_sensitive=True) + + dst = MemoryService(data_dir=str(tmp_path / "dst")) + first = dst.import_("alice", data=payload, format="engram") + again = dst.import_("alice", data=payload, format="engram") + assert again["facts"] == 0 and again["episodes"] == 0 + assert again["facts_skipped"] == first["facts"] + assert again["episodes_skipped"] == first["episodes"] + out = dst.export("alice", include_sensitive=True) + assert len(out["facts"]) == len(payload["facts"]) # no duplicates + + +def test_share_safe_export_still_imports(tmp_path, clean_env): + src = MemoryService(data_dir=str(tmp_path / "src")) + _seed(src, "alice") + src.add_fact("alice", "user", "has_condition", "hay fever", sensitive=True) + payload = src.export("alice") # share-safe: no sensitive facts, no episodes + + dst = MemoryService(data_dir=str(tmp_path / "dst")) + stats = dst.import_("alice", data=payload, format="auto") + assert stats["facts"] == len(payload["facts"]) > 0 + assert stats["episodes"] == 0 + out = dst.export("alice", include_sensitive=True) + assert not any(f["sensitive"] for f in out["facts"]) # redacted stayed redacted + + +def test_import_rejects_unknown_version(tmp_path, clean_env): + dst = MemoryService(data_dir=str(tmp_path / "dst")) + with pytest.raises(ValueError, match="version"): + dst.import_("alice", data={"engram_export_version": 99}, format="engram") + + +def test_import_cross_embedder_reembeds(tmp_path, clean_env, monkeypatch): + """The migration path IS the re-embedding path: the target re-embeds with its own embedder, so a + payload exported under one embedding space restores into another without touching the source.""" + src = MemoryService(data_dir=str(tmp_path / "src")) + _seed(src, "alice") + payload = src.export("alice", include_sensitive=True) + + monkeypatch.setenv("ENGRAM_MAX_HOT_FACTS", "10000") + dst = MemoryService(data_dir=str(tmp_path / "dst"), embedder_name="hashing") + dst.import_("alice", data=payload, format="engram") + mem = dst.get("alice") + dim = len(dst.embedder.embed("probe")) + for f in mem.fact_store.values(): + assert f.embedding is not None and len(f.embedding) == dim + + +# --- identity-consistent stats --------------------------------------------- + +def test_stats_follow_linked_identity(tmp_path, clean_env): + svc = MemoryService(data_dir=str(tmp_path / "id")) + mem = svc.get("zz-handle") + mem.link_identity("zz-handle", "aa-canonical") # canonical root: "aa-canonical" + svc.remember("zz-handle", "I work at Acme Corp.", session_id="s1") + counts = svc.stats("zz-handle")["counts"] + assert counts["episodes"] >= 1 + assert counts["facts_live"] >= 1 + + +# --- import CLI writes the same namespace dirs as the service --------------- + +def test_import_cli_local_writes_service_namespace_dir(tmp_path, clean_env, monkeypatch, capsys): + from engram.connectors.__main__ import main + + fp = tmp_path / "log.txt" + fp.write_text("User: I like green tea.\nAssistant: Noted.\n", encoding="utf-8") + data_dir = tmp_path / "data" + monkeypatch.setattr(sys, "argv", [ + "prog", "--file", str(fp), "--format", "transcript", + "--namespace", "a/b", "--data-dir", str(data_dir), + ]) + main() + + svc = MemoryService(data_dir=str(data_dir)) + assert (data_dir / svc._safe_user("a/b")).is_dir(), \ + "CLI must write the same digest-backed namespace dir the service reads" + assert svc.stats("a/b")["counts"]["episodes"] >= 1 + + +def test_import_cli_accepts_engram_export(tmp_path, clean_env, monkeypatch, capsys): + from engram.connectors.__main__ import main + + src = MemoryService(data_dir=str(tmp_path / "src")) + _seed(src, "alice") + fp = tmp_path / "export.json" + fp.write_text(json.dumps(src.export("alice", include_sensitive=True)), encoding="utf-8") + + data_dir = tmp_path / "data" + monkeypatch.setattr(sys, "argv", [ + "prog", "--file", str(fp), "--namespace", "alice", "--data-dir", str(data_dir), + ]) + main() + out = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert out["format"] == "engram" and out["facts"] > 0 + + dst = MemoryService(data_dir=str(data_dir)) + assert dst.stats("alice")["counts"]["facts_live"] >= 1 + + +# --- ENGRAM_STORAGE backend selector ---------------------------------------- + +def test_engram_storage_env_selects_backend(tmp_path, clean_env, monkeypatch): + monkeypatch.setenv("ENGRAM_STORAGE", "lancedb") + svc = MemoryService(data_dir=str(tmp_path / "s")) + assert svc.config.storage == "lancedb" + assert svc.stats.__self__ is svc # constructing the service must not import lancedb yet + + +def test_engram_storage_env_rejects_unknown(tmp_path, clean_env, monkeypatch): + monkeypatch.setenv("ENGRAM_STORAGE", "postgres") + with pytest.raises(ValueError, match="ENGRAM_STORAGE"): + MemoryService(data_dir=str(tmp_path / "s")) diff --git a/tests/test_mcp_http_auth.py b/tests/test_mcp_http_auth.py new file mode 100644 index 0000000..0759a7d --- /dev/null +++ b/tests/test_mcp_http_auth.py @@ -0,0 +1,86 @@ +"""The MCP streamable-HTTP transport must not be an unauthenticated door into memory. + +`python -m engram.mcp --http` is loopback-only by default; exposing it on a non-loopback host requires +a Bearer token (fail-closed, same philosophy as ENGRAM_API_KEYS/ENGRAM_OPEN on the REST server). +These tests exercise the pure ASGI gate and the fail-closed launch policy without starting a server. +""" +from __future__ import annotations + +import asyncio + +import pytest + +from engram.mcp.__main__ import _BearerGate, _require_http_token + + +class _Inner: + def __init__(self) -> None: + self.called = False + + async def __call__(self, scope, receive, send) -> None: + self.called = True + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + +def _run(gate, scope): + sent = [] + + async def send(message): + sent.append(message) + + async def receive(): + return {"type": "http.request"} + + asyncio.run(gate(scope, receive, send)) + return sent + + +def _http_scope(auth: str | None) -> dict: + headers = [(b"host", b"example.com")] + if auth is not None: + headers.append((b"authorization", auth.encode("latin-1"))) + return {"type": "http", "headers": headers} + + +def test_gate_rejects_missing_token(): + inner = _Inner() + sent = _run(_BearerGate(inner, "secret"), _http_scope(None)) + assert sent[0]["status"] == 401 + assert not inner.called + + +def test_gate_rejects_wrong_token(): + inner = _Inner() + sent = _run(_BearerGate(inner, "secret"), _http_scope("Bearer nope")) + assert sent[0]["status"] == 401 + assert not inner.called + + +def test_gate_passes_valid_token(): + inner = _Inner() + sent = _run(_BearerGate(inner, "secret"), _http_scope("Bearer secret")) + assert inner.called + assert sent[0]["status"] == 200 + + +def test_gate_ignores_non_http_scopes(): + inner = _Inner() + _run(_BearerGate(inner, "secret"), {"type": "lifespan"}) + assert inner.called # lifespan/websocket handshake pass through to the app + + +def test_loopback_without_token_is_allowed(): + _require_http_token("127.0.0.1", "", allow_open=False) + _require_http_token("localhost", "", allow_open=False) + _require_http_token("::1", "", allow_open=False) + + +def test_non_loopback_without_token_fails_closed(): + with pytest.raises(SystemExit): + _require_http_token("0.0.0.0", "", allow_open=False) + + +def test_non_loopback_with_token_or_explicit_open_is_allowed(): + _require_http_token("0.0.0.0", "some-token", allow_open=False) + _require_http_token("0.0.0.0", "", allow_open=True) # explicit operator override diff --git a/tests/test_server_import_export.py b/tests/test_server_import_export.py new file mode 100644 index 0000000..3c9745a --- /dev/null +++ b/tests/test_server_import_export.py @@ -0,0 +1,60 @@ +"""HTTP surface of the cross-instance migration path: POST an /v1/export payload straight back into +/v1/import on another namespace/instance and get memory, not a 500.""" +from __future__ import annotations + +import os +import shutil +import tempfile + +import pytest + +pytest.importorskip("fastapi") +from fastapi.testclient import TestClient # noqa: E402 + + +@pytest.fixture(scope="module") +def client(): + d = tempfile.mkdtemp(prefix="engram_port_") + os.environ.update(ENGRAM_DATA_DIR=d, ENGRAM_EMBEDDER="hashing", ENGRAM_OPEN="1") + for var in ("ENGRAM_LLM", "ENGRAM_API_KEYS", "ENGRAM_ALLOW_ANONYMOUS", "ENGRAM_STORAGE"): + os.environ.pop(var, None) + from engram.server import app as appmod + appmod._svc = None # fresh singleton bound to the test env + with TestClient(appmod.app) as c: + yield c + shutil.rmtree(d, ignore_errors=True) + + +def hdr(ns: str) -> dict: + return {"Authorization": f"Bearer {ns}"} + + +def test_rest_export_import_roundtrip(client): + src, dst = hdr("mover-src"), hdr("mover-dst") + r = client.post("/v1/remember", json={"content": "I live in Lisbon and prefer tea."}, headers=src) + assert r.status_code == 200 + + exported = client.get("/v1/export", params={"include_sensitive": "true"}, headers=src).json() + assert exported["engram_export_version"] == 1 + + r = client.post("/v1/import", json={"data": exported, "format": "auto"}, headers=dst) + assert r.status_code == 200, r.text + body = r.json() + assert body["ok"] is True and body["format"] == "engram" + + out = client.get("/v1/export", params={"include_sensitive": "true"}, headers=dst).json() + assert {f["id"] for f in out["facts"]} == {f["id"] for f in exported["facts"]} + + +def test_rest_import_bad_payload_is_400_not_500(client): + r = client.post("/v1/import", json={"data": {"engram_export_version": 99}, "format": "engram"}, + headers=hdr("mover-bad")) + assert r.status_code == 400 + assert "version" in r.json()["detail"] + + +def test_rest_import_malformed_records_is_400_not_500(client): + # a dict that is neither a known chat shape nor a valid export used to escape as a raw 500 + r = client.post("/v1/import", json={"data": {"whatever": 1}, "format": "records"}, + headers=hdr("mover-bad")) + assert r.status_code == 400 From 8d2eb36894dcd0176b9180a7f267db96151134d7 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 15:46:04 +0800 Subject: [PATCH 02/25] docs: record the branch-debt triage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several capabilities were written, tested, and then never merged; the memory ledger records them as shipped while main does not have them. Triage each one against today's main by capability rather than by filename, since main evolved 100+ commits independently and often implements the same thing under a different design. Two conclusions worth keeping: main's JSONL persistence layer is stronger than the branch's SQLite snapshot (so that one is dropped, not merged), and five worktrees hold 2000+ lines that were never committed to any branch at all — deleting those worktrees would destroy the work. --- docs/branch-debt-triage.zh-CN.md | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/branch-debt-triage.zh-CN.md diff --git a/docs/branch-debt-triage.zh-CN.md b/docs/branch-debt-triage.zh-CN.md new file mode 100644 index 0000000..73e55b3 --- /dev/null +++ b/docs/branch-debt-triage.zh-CN.md @@ -0,0 +1,79 @@ +# 分支债务甄别台账（2026-08-16） + +## 为什么有这份文档 + +仓库长期存在"能力写完并测过、但从未合入主干"的债务。记忆账本和交接记录里这些能力被标记为 +`shipped`，但主干上并不存在——**"写完并测过" ≠ "主干有"**。本文件记录一次逐项甄别的结论， +避免后续 AI 或人类重新推导，也避免把已被主干超越的旧实现硬合回来。 + +甄别基线：`8decc8c`（= `ed0e53a` release 0.1.0 + cross-account），439 passed / 1 skipped。 + +## 甄别方法（重要） + +**按能力比对，不是按文件名比对。** main 在 2026-06 之后独立演进了 100+ 个 commit，同一能力 +经常以不同文件名和不同设计存在。硬 `git merge` 会把主干已经更好的实现覆盖掉。 + +每一项的判定必须给出主干侧的 `文件:行号` 证据。 + +## 分支状态 + +| 分支 | 相对 main | 时间 | 处置 | +| --- | --- | --- | --- | +| `claude/cross-account-memory-module-95f184` | +1 / −0 | 2026-08-12 | ✅ 已 fast-forward 合入（+22 测试） | +| `claude/sweet-haibt-7231c5` | +1 / −102 | 2026-06-10 | 逐项移植，见下表 | +| `claude/goofy-shockley-f70706` | +11 / −103 | 2026-06-08 | 逐项移植，见下表 | + +## A3 — `sweet-haibt-7231c5` + +| 能力 | 主干现状 | 建议 | 风险 | +| --- | --- | --- | --- | +| ANN 候选生成（`Config.ann_candidates`/`ann_pool`） | ❌ 无。`engram/retrieve/hybrid.py` 每次查询全量扫描 `fact_store.values()` 与 `graph.entities.values()` | **重写实现同等能力**（宪章 Bet E 承重墙） | 中 | +| async System-2（`ENGRAM_ASYNC_CONSOLIDATION`） | ❌ 无 | 移植 | 中（`service.py` 已大改） | +| multi-Space（`engram/spaces.py`） | ❌ 无 | 重写（`MemoryService` 已是 namespace-keyed，主要是读融合 + ACL 层） | 中 | +| 文档摄入 PDF/DOCX + 图片 caption | ❌ 无 | 移植（可选 extra，独立于检索路径） | 低 | +| embedding 空间版本守卫 | ✅ **主干更强**：`engram/store/persist.py:36,238` 的 manifest 同时校验 `embedder_id` 与 `embedding_dim` | **跳过守卫本身** | — | +| └ 但缺 `reembed()` 迁移 | ❌ 主干只能抛 `EmbedderMismatchError`，无法从源文本重建向量 | 补迁移路径 | 中 | +| import 幂等 | ✅ cross-account 合入时已带来（按 id 幂等） | 跳过 | — | + +## A4 — `goofy-shockley-f70706` + +| 能力 | 主干现状 | 建议 | 风险 | +| --- | --- | --- | --- | +| `store/snapshot.py`（SQLite 快照） | ✅ **主干更强**：`engram/store/persist.py` 用 JSONL + manifest + 原子 tmp→final + 文件锁 + committed-prefix 校验 + embedder/dim 守卫；另有 `store/migrate.py` 迁移旧 pickle | **丢弃分支版** | — | +| `server/keys.py`（自助签发 API key） | ⚠️ 部分：主干 `engram/server/app.py:48,183` 有 `ENGRAM_API_KEYS` 静态映射 + `hmac.compare_digest` 常量时间比较 + 严格 Bearer 解析。分支版额外提供自助签发、只存 hash、吊销、列表（101 行） | 有增量价值，中优先级 | 低 | +| `server/ratelimit.py`（49 行） | ❌ 无 | 移植 | 低 | +| `server/idempotency.py` | ❌ 无 | 移植 | 低 | +| `server/metrics.py` | ❌ 无 | 移植 | 低 | +| `engram/client.py`（Python SDK） | ❌ 无（只有 TS 客户端） | 移植 | 低 | +| `store/crypto.py`（Fernet 落盘加密） | ❌ 无 | 移植（需可选 `cryptography` 依赖） | 中 | +| `store/pg_store.py`（pgvector） | ❌ 无（`store/base.py:2`、`server/app.py:19` 只在文档里提到） | **暂缓** — 该代码从未用真实数据库验证过，属未验证代码，不应盲目并入 | 高 | +| `integrations/`（LangChain / LlamaIndex retriever） | ❌ 无（主干有 `docs/agent-adapters.md`，但那是 MCP/跨 Agent 适配，不是框架 retriever） | 移植（需可选依赖） | 低 | + +## 未提交工作（不在任何分支上） + +13 个 worktree 中有 5 个含**从未提交**的改动，合计 2000+ 行。这些不在任何分支上，删除 worktree +即永久销毁： + +| worktree | 未提交内容 | +| --- | --- | +| `wonderful-tereshkova-b63377` | `retrieve/router.py`(140) + `retrieve/segment.py`(131) + 两个测试(189) + 167 行 diff | +| `vigilant-mahavira-73886f` | `retrieve/recall_pipeline.py`(253)，无测试 | +| `laughing-tesla-360ff1` | `TECHNIQUES.md` + `docs/{README,console,evaluation,governance,memory-policy,moat}.md` 共 730 行 | +| `cool-chebyshev-aa0347` | consolidate 三模块改动 72 行 + `tests/test_profile.py`(40) | +| `nostalgic-clarke-dda32e` | 480 行 diff | + +⚠️ **隐私**：`nostalgic-clarke-dda32e` 含 `examples/seed_zhangyuwei.py` 与 `examples/zhangyuwei_memory.md`， +疑似真实人名。`CONTRIBUTING.md` 禁止提交真实个人数据与姓名——这两个文件在确认为化名前不得提交。 + +⚠️ **不要盲目抢救**：这些均为 2026-06 的工作，主干此后新增了 `retrieve/evidence.py` 与 +`retrieve/aggregate.py`。`router.py` / `segment.py` / `recall_pipeline.py` 很可能已被覆盖， +需逐个判定"仍有价值 vs 已过时"。 + +## 结论 + +1. 分支上的东西**不都是财富**。持久化层是反例：主干的 `persist.py` 明确优于分支的 `snapshot.py`。 + 任何"把旧分支合回来"的动作都必须逐能力甄别。 +2. 主干真实缺口按价值排序：**ANN 候选生成**（宪章 Bet E）→ 服务层加固三件套 + （ratelimit / metrics / idempotency）→ Python SDK → async System-2 / multi-Space → + 文档摄入 → 加密 → 框架适配器。 +3. `pg_store.py` 属未验证代码，在拿到真实数据库前不并入。 From d8bc6cd12e50d446574db8836246eb051ea7ae2b Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 15:46:22 +0800 Subject: [PATCH 03/25] retrieve: bounded candidate retrieval, off by default, plus the harness that measures it The read path scored every live fact on every query. eval/scaling.py measures what that costs: ms per 1k facts holds constant at ~17-20 from 100 to 10,000 facts -- the signature of O(n) -- and a single query already takes 177ms at 10k facts, past the <100ms read-path target, at a store size that is small next to the 10M-token goal. Bounded retrieval scores a candidate pool instead: a lexical inverted index, the vector store's own search, and the facts hanging off the query's anchor entities, unioned, then completed so every candidate's conflict slot arrives whole (otherwise a superseded fact whose slot head fell outside the pool would survive -- a correctness bug, not a ranking one). Corpus statistics stay tenant-scoped so BM25 scores a subset exactly as a full scan would, and candidates are resolved in store order because rank-based fusion breaks ties by position, so an unordered set would make the same query answer differently run to run. The measured result is negative where it matters, and that is the useful finding: with the semantic channel on, bounding the rest buys nothing (177.03ms vs 177.88ms at 10k), because neither backend has a real ANN index -- the in-memory store brute-forces cosine and sorts, and LanceDB materialises the whole table whenever a Python predicate is supplied, which multi-tenant retrieval always supplies. With that channel off the same code is 14.5x faster at 10k and genuinely sub-linear (ms/1k falls 6.17 -> 1.23). So the candidate-pool design is sound and the blocker is the missing filtered ANN, which is now the P0 on the optimization map. Both flags default off: the published numbers came from the full scan, and dropping the semantic channel would lose the recall the hybrid thesis depends on. A pool at least as large as the live fact count returns bit-identical results to the full scan, which is the test that keeps the two paths honest. A decorator maintains the index so none of the eight existing upsert/delete call sites had to change. --- docs/architecture-optimization-map.zh-CN.md | 9 + engram/config.py | 15 ++ engram/memory.py | 9 +- engram/retrieve/hybrid.py | 94 +++++-- engram/retrieve/lexical.py | 59 ++++- engram/store/__init__.py | 3 + engram/store/indexed.py | 261 ++++++++++++++++++++ engram/util.py | 27 ++ eval/scaling.py | 162 ++++++++++++ results/bounded_candidates_scaling.md | 57 +++++ tests/test_bounded_candidates.py | 212 ++++++++++++++++ 11 files changed, 877 insertions(+), 31 deletions(-) create mode 100644 engram/store/indexed.py create mode 100644 eval/scaling.py create mode 100644 results/bounded_candidates_scaling.md create mode 100644 tests/test_bounded_candidates.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 2d3df71..9279748 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -100,6 +100,7 @@ flowchart TD | 2026-06-30 | `aggregation_constraint_filter` | Aggregation evidence / Query constraints | 当题面有月份约束时，排除局部上下文绑定到其他月份的数值候选 | `results/aggregation_constraint_filter_experiments.md`, `results/aggregation_constraint_filter_lme_s_context27.jsonl` | | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | +| 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | | 2026-08-12 | `cross_instance_portability` | Connectors / Memory facade / Service import 路由 / HTTP `/v1/import` | 导出无法导回（POST 导出 JSON 得到 500）——补上原生 `engram` 导入格式：事实保留原 id/双时间戳/supersedes 链/provenance，目标端用本地 embedder 重嵌入（即官方换 embedder 迁移路径），按 id 幂等；`/v1/import` 对坏 payload 返回 400 | `tests/test_cross_account_portability.py`, `tests/test_server_import_export.py`（工程验收，非算法实验） | | 2026-08-12 | `mcp_http_bearer_gate` | MCP streamable-HTTP 传输边界 | MCP HTTP 模式此前无任何鉴权，仅靠默认 127.0.0.1；新增 `--http-token`/`ENGRAM_MCP_HTTP_TOKEN` Bearer 门，非回环绑定无 token 时启动即拒绝（失败关闭，与 REST 的 `ENGRAM_API_KEYS` 同哲学） | `tests/test_mcp_http_auth.py` | | 2026-08-12 | `engram_storage_env` + 一致性修复 | Service 配置边界 / stats / import CLI | 服务器此前永远 `storage="memory"`（无环境变量可选 LanceDB）；新增 `ENGRAM_STORAGE`（非法值失败关闭）。`/v1/stats` 改按 canonical 身份过滤（与其它读路径一致）；import CLI 本地模式改走 `MemoryService`，目录命名与服务端统一 | `tests/test_cross_account_portability.py` | @@ -124,6 +125,14 @@ flowchart TD | P1 | Graph proximity / multi-hop | multi-session、multi-hop 是长期记忆系统最难类别，也是差异化战场 | 轻量 n-hop/PPR-style expansion，先用真实错例切片验证 | | P1 | Temporal interval reasoning | temporal-reasoning 仍低于 full-context，需要更强的区间和 duration 证据 | 显式 start/end pair、invalid_at span、date arithmetic block | | P2 | Runtime profiles | 让用户选择 lite/standard/graph/consolidated，并用同一 harness 报三联表 | 在 `Config`/bench 层定义可测 profile，而不是手动组合开关 | +| **P0** | **向量存储的过滤 ANN** | Bet E 的真正卡点。两个后端都没有真 ANN 索引：`InMemoryVectorStore.search()` 暴力 cosine + 全量排序；`LanceDBVectorStore.search()` 一旦传 Python 谓词就 `to_arrow().to_pylist()` 全表物化。多租户检索必带 user 过滤，所以线上永远走不到索引 | 把 user 过滤下推为 LanceDB SQL 谓词。这是解锁 `bounded_candidates` 已测得的 14.5x 加速的唯一前提——候选池代码已就位，见 `results/bounded_candidates_scaling.md` | + +## 未采用/回滚原因 + +| 方向 | 结论 | 证据 | +| --- | --- | --- | +| 只把词汇/融合环节收敛为有界候选池（保留语义通道） | **净收益 ≈ 0**（10000 事实：177.03ms vs 全扫 177.88ms）。语义通道自身就是一次全量扫描，省下的又赔回去，还多付索引维护成本。不是候选池思路错，是缺 ANN 索引 | `results/bounded_candidates_scaling.md` | +| `candidate_vector_channel=False` 作为默认 | **不采用**。它确实快 14.5x 且次线性，但会丢"语义相关却无共享查询词"的事实，正是 M1 已验证 hybrid 论点依赖的召回。需真 ANN 后端或 keyed harness 证明召回损失可接受 | 同上 | ## 验收分层规则 diff --git a/engram/config.py b/engram/config.py index aebfbe7..cb6c194 100644 --- a/engram/config.py +++ b/engram/config.py @@ -33,6 +33,21 @@ class Config: recency_tau_days: float = 45.0 top_k: int = 5 candidate_k: int = 24 # per-retriever candidate pool before fusion + # Bounded candidate retrieval (Bet E). OFF by default: it changes which facts get *scored*, and the + # published numbers were produced by the full scan, so turning it on silently would break the + # "every number traces to a committed log" rule. With `candidate_pool` >= the live fact count the two + # paths are provably identical (tests/test_bounded_candidates.py) — the flag buys speed at scale, and + # its ranking behaviour at scale still needs a keyed harness run before it becomes the default. + bounded_candidates: bool = False + candidate_pool: int = 400 # per-channel candidate budget before fusion when bounded_candidates is on + # Whether bounded retrieval asks the vector store for semantic candidates. Measured cost/benefit: + # neither shipped backend has a real ANN index (the in-memory store brute-forces cosine and sorts; + # LanceDB materialises the whole table whenever a Python predicate is supplied), so this channel + # re-introduces the very O(n) pass the candidate pool exists to avoid. Leaving it ON is the + # recall-safe default; turning it OFF is what makes the read path genuinely bounded, at the cost of + # missing facts that are semantically relevant but share no query term. Flip it off only with a real + # ANN backend, or after a keyed harness run shows the recall loss is acceptable. + candidate_vector_channel: bool = True rrf_k: int = 60 # Reciprocal Rank Fusion constant max_hops: int = 2 # multi-hop planner depth max_hot_facts: int = 10_000 # heat-tier cap; cold facts remain durable and can page back on hot miss diff --git a/engram/memory.py b/engram/memory.py index c935fbd..df16c92 100644 --- a/engram/memory.py +++ b/engram/memory.py @@ -34,6 +34,7 @@ from .retrieve.lexical import bm25_scores, overlap_terms, stems from .store import ( GraphStore, + IndexedVectorStore, InMemoryDocStore, InMemoryGraphStore, InMemoryVectorStore, @@ -157,7 +158,13 @@ def make_vector_store(name: str) -> VectorStore: self.episodes_doc = InMemoryDocStore() self.episodes_vec = make_vector_store("episodes_vec") - self.fact_store = make_vector_store("fact_store") # HOT tier: the fast, frequently-retrieved working set + # HOT tier: the fast, frequently-retrieved working set. Decorated with a lexical/slot index when + # bounded candidate retrieval is on, so the retriever can pick candidates without scanning every + # fact. A decorator (not a new store) keeps every existing .upsert()/.delete() call site — here, + # in the consolidation engine, and in the persistence loader — indexing for free. + self.fact_store = make_vector_store("fact_store") + if self.config.bounded_candidates: + self.fact_store = IndexedVectorStore(self.fact_store) self.cold_store = make_vector_store("cold_store") # COLD tier: aged-out facts, preserved (never deleted) self.summary_vec = make_vector_store("summary_vec") # L2 session summaries, retrievable for a lean read slice self.graph = graph_store_factory() diff --git a/engram/retrieve/hybrid.py b/engram/retrieve/hybrid.py index 00bb0c1..427f359 100644 --- a/engram/retrieve/hybrid.py +++ b/engram/retrieve/hybrid.py @@ -10,26 +10,16 @@ from ..embed import Embedder from ..store import GraphStore, VectorStore from ..types import Fact -from ..util import cosine, fmt_date, now, recency, tokenize +from ..util import cosine, indexed_text, now, recency, tokenize +from ..util import date_terms as date_terms # noqa: F401 re-export: callers import it from here from .fusion import order_by_score, weighted_rrf from .lexical import bm25_scores, stem, stems -_MONTHS = ("january", "february", "march", "april", "may", "june", "july", "august", - "september", "october", "november", "december") -_GRAPH_HOP_DECAY = 0.65 - +# `date_terms` now lives in engram.util so the lexical index can tokenize facts identically — the index's +# corpus statistics must describe the same documents the scorer scores. Imported (not redefined) here +# because callers already do `from .hybrid import date_terms`. -def date_terms(epoch: float) -> str: - """Render a fact's date as searchable tokens (year, numeric month, month name) so a query that names - a time ('May 2023', 'in 2024') matches the right-dated facts via BM25 — dates otherwise live only in - valid_at and are invisible to retrieval. This is query-time temporal matching done as a lexical signal - (MemoryScope time_ratio in spirit), with no score multiplier that could override relevance.""" - try: - d = fmt_date(epoch) # YYYY-MM-DD - y, m, _ = d.split("-") - return f"{d} {y} {m} {_MONTHS[int(m) - 1]}" - except Exception: # noqa: BLE001 - return "" +_GRAPH_HOP_DECAY = 0.65 # Predicates that mark a durable identity or preference fact (vs. an incidental event mention). Used for # type-weighted fusion — these get a retrieval boost (CLAUDE.md §3.3; MemoryScope/OMEGA convergent finding). @@ -339,11 +329,76 @@ def _current_slot_heads(self, facts: list[Fact]) -> list[Fact]: if not is_single_valued(fact.predicate) or heads.get(fact.slot) is fact ] + def _bounded_candidates( + self, query: str, user_id: str, as_of: Optional[float], qvec: list[float] + ) -> Optional[list[Fact]]: + """A bounded candidate set for fusion, or None when the store has no index to select from. + + Three channels are unioned, because dropping any one loses recall the hybrid thesis depends on + (CLAUDE.md M1 — facts-only lost recall; so does vectors-only): + + * **lexical** — exact names, numbers, dates. Vector-weak, retrieval-decisive. + * **semantic** — paraphrases no shared token would catch. + * **graph** — facts hanging off the entities the query names, which is how cross-session links + surface at all. Read from the relations incident to those entities, so it stays bounded. + + Then each candidate's conflict slot is completed. Without that, `_current_slot_heads` could see a + superseded fact whose head happened to fall outside the pool and let the stale value through — + a correctness bug, not a ranking one. + """ + index = getattr(self.fact_store, "index", None) + if index is None: + return None + pool = max(1, self.config.candidate_pool) + + ids = index.lexical_candidates(query, pool, user_id=user_id) + + if self.config.candidate_vector_channel: + for _score, payload in self.fact_store.search( + qvec, pool, where=lambda f: getattr(f, "user_id", None) == user_id + ): + fid = getattr(payload, "id", None) + if isinstance(fid, str): + ids.add(fid) + + for eid in self.query_entity_ids(query, user_id): + for direction in ("out", "in"): + for rel in self.graph.neighbors(eid, as_of, direction): + fid = getattr(rel, "fact_id", None) + if fid: + ids.add(fid) + + matched = [f for f in index.resolve(ids) if f.user_id == user_id] + slots = {f.slot for f in matched if is_single_valued(f.predicate)} + if not slots: + return matched + # Resolve once over the completed id set rather than appending: `resolve` returns store order, + # and appending slot-mates at the end would put the candidates in a different order than the + # full scan sees, which rank-based fusion would turn into a different result. + return [f for f in index.resolve(ids | index.slot_members(slots)) if f.user_id == user_id] + def retrieve( self, query: str, user_id: str, as_of: Optional[float] = None, top_k: Optional[int] = None ) -> tuple[list[tuple[Fact, float]], dict]: top_k = top_k or self.config.top_k - live = [f for f in self.fact_store.values() if f.user_id == user_id and f.is_live(as_of)] + qvec = self.embedder.embed(query) + + # Bounded retrieval scores a candidate pool instead of the whole store (Bet E). It is off by + # default: with `candidate_pool` >= the live fact count the two paths return the same ranking, + # but below that they can differ, and the published numbers came from the full scan. + corpus = None + candidates = ( + self._bounded_candidates(query, user_id, as_of, qvec) + if self.config.bounded_candidates + else None + ) + if candidates is None: + live = [f for f in self.fact_store.values() if f.user_id == user_id and f.is_live(as_of)] + else: + live = [f for f in candidates if f.is_live(as_of)] + # Corpus statistics must describe the tenant's collection, not the pool — see corpus_for(). + corpus = self.fact_store.index.corpus_for(user_id, stems(query)) + live = self._current_slot_heads(live) excluded = self.graph_exclusion_zone(query, user_id, as_of) if excluded: @@ -351,7 +406,6 @@ def retrieve( if not live: return [], {"sem": {}, "lex": {}, "qids": set()} - qvec = self.embedder.embed(query) sem = {f.id: cosine(qvec, f.embedding or []) for f in live} # Type-weighted retrieval: scale the SEMANTIC score by fact type. Because an off-topic fact has # sem≈0, the multiplier only reorders among genuinely-relevant candidates (a preference fact beats @@ -363,7 +417,9 @@ def retrieve( if tw != 1.0: sem[f.id] *= tw # include each fact's date as searchable tokens so time-named queries ('May 2023') match by date - lex = bm25_scores(query, [(f.id, f"{f.text} {date_terms(f.valid_at)}") for f in live]) + lex = bm25_scores( + query, [(f.id, indexed_text(f.text, f.valid_at)) for f in live], corpus=corpus + ) gph, qids = self._graph_scores(query, user_id, live, as_of) t = now() if as_of is None else as_of rec = {f.id: recency(max(0.0, t - f.valid_at), self.config.recency_tau_days) for f in live} diff --git a/engram/retrieve/lexical.py b/engram/retrieve/lexical.py index d502e6a..0e6dd53 100644 --- a/engram/retrieve/lexical.py +++ b/engram/retrieve/lexical.py @@ -2,10 +2,24 @@ from __future__ import annotations import math +from typing import Optional, Protocol from ..util import stem, stems -__all__ = ["stem", "stems", "token_overlap", "bm25_scores"] +__all__ = ["stem", "stems", "token_overlap", "bm25_scores", "CorpusStats"] + + +class CorpusStats(Protocol): + """Collection-wide BM25 statistics, so a candidate subset can be scored as if the whole store were.""" + + @property + def n_docs(self) -> int: ... + + @property + def avgdl(self) -> float: ... + + @property + def df(self) -> dict[str, int]: ... def overlap_terms(query: str, text: str) -> set[str]: @@ -16,17 +30,36 @@ def token_overlap(query: str, text: str) -> int: return len(overlap_terms(query, text)) -def bm25_scores(query: str, docs: list[tuple[str, str]], k1: float = 1.5, b: float = 0.75) -> dict[str, float]: - """docs: list of (id, text). Returns {id: bm25_score}.""" +def bm25_scores( + query: str, + docs: list[tuple[str, str]], + k1: float = 1.5, + b: float = 0.75, + *, + corpus: Optional["CorpusStats"] = None, +) -> dict[str, float]: + """docs: list of (id, text). Returns {id: bm25_score}. + + `corpus` supplies collection-wide statistics (N, avgdl, document frequency). Pass it when `docs` is a + *candidate subset* of a larger store: BM25's IDF and length normalisation are properties of the whole + collection, so deriving them from a subset would give the same fact a different score depending on + which other facts happened to be retrieved alongside it. With `corpus`, scoring a subset yields + exactly the scores a full scan would have produced — which is what makes bounded candidate retrieval + a pure speed optimisation rather than a silent ranking change.""" if not docs: return {} doc_tokens = {doc_id: stems(text) for doc_id, text in docs} - n = len(docs) - avgdl = max(1.0, sum(len(t) for t in doc_tokens.values()) / n) - df: dict[str, int] = {} - for toks in doc_tokens.values(): - for w in set(toks): - df[w] = df.get(w, 0) + 1 + if corpus is not None: + n = corpus.n_docs + avgdl = corpus.avgdl + df = corpus.df + else: + n = len(docs) + avgdl = max(1.0, sum(len(t) for t in doc_tokens.values()) / n) + df = {} + for toks in doc_tokens.values(): + for w in set(toks): + df[w] = df.get(w, 0) + 1 q_terms = set(stems(query)) scores: dict[str, float] = {} for doc_id, toks in doc_tokens.items(): @@ -41,7 +74,11 @@ def bm25_scores(query: str, docs: list[tuple[str, str]], k1: float = 1.5, b: flo for w in q_terms: if w not in tf: continue - idf = math.log(1 + (n - df[w] + 0.5) / (df[w] + 0.5)) - s += idf * (tf[w] * (k1 + 1)) / (tf[w] + k1 * (1 - b + b * dl / avgdl)) + # df.get(): with a supplied corpus a term can be absent if the index has not caught up; an + # unseen term is maximally rare, which is what df=0 yields here. In the self-derived path + # every term of every doc is present by construction, so this changes nothing. + dfw = df.get(w, 0) + idf = math.log(1 + (n - dfw + 0.5) / (dfw + 0.5)) + s += idf * (tf[w] * (k1 + 1)) / (tf[w] + k1 * (1 - b + b * dl / max(1.0, avgdl))) scores[doc_id] = s return scores diff --git a/engram/store/__init__.py b/engram/store/__init__.py index 391a011..a54946c 100644 --- a/engram/store/__init__.py +++ b/engram/store/__init__.py @@ -1,4 +1,5 @@ from .base import DocStore, GraphStore, VectorStore +from .indexed import FactIndex, IndexedVectorStore from .memory_store import InMemoryDocStore, InMemoryGraphStore, InMemoryVectorStore from .persist import ( DimensionMismatchError, @@ -17,6 +18,8 @@ "InMemoryVectorStore", "InMemoryDocStore", "InMemoryGraphStore", + "FactIndex", + "IndexedVectorStore", "PersistenceError", "IncompatibleStoreError", "DimensionMismatchError", diff --git a/engram/store/indexed.py b/engram/store/indexed.py new file mode 100644 index 0000000..da6401d --- /dev/null +++ b/engram/store/indexed.py @@ -0,0 +1,261 @@ +"""A lexical/slot index over the fact store, and a VectorStore decorator that keeps it current. + +Why this exists (CLAUDE.md Bet E — linear scaling). The read path scored *every* live fact on every +query: cosine over all embeddings, and — more expensively — re-tokenising and re-stemming every fact's +text to compute BM25 from scratch. Measured with `eval/scaling.py`, per-query cost tracked store size +exactly (constant ms per 1k facts), so a 100x larger store cost 81x more per query. + +The fix is to score a *bounded candidate set* instead of the whole store. Two invariants make that a +speed optimisation rather than a silent ranking change: + + 1. **Corpus statistics stay global.** BM25's IDF and average document length describe the collection, + not the candidate subset. The index keeps them, so a candidate scores exactly what it would have + scored in a full scan (see `lexical.bm25_scores(corpus=...)`). + 2. **Slots stay whole.** `_current_slot_heads` suppresses superseded facts by comparing facts that + share a (user, subject, predicate) slot. If a candidate's slot-mates were missing, a stale fact + could survive that a full scan would have filtered. The index can return a slot's full membership. + +The decorator keeps every existing `.upsert()/.delete()` call site untouched — the index is maintained +where writes already land, not by threading a new dependency through eight callers. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Iterable, Optional + +from ..util import indexed_text, stems +from .base import Predicate, VectorStore + +__all__ = ["FactIndex", "IndexedVectorStore"] + + +@dataclass(frozen=True) +class _UserCorpus: + """Satisfies retrieve.lexical.CorpusStats for one tenant's slice of the index.""" + + n_docs: int + avgdl: float + df: dict[str, int] = field(default_factory=dict) + + +def _fact_key(fact: Any) -> Optional[tuple[str, str, str]]: + """A fact's conflict slot, or None for payloads that are not facts (episodes, summaries).""" + slot = getattr(fact, "slot", None) + return slot if isinstance(slot, tuple) else None + + +class FactIndex: + """Inverted index + corpus statistics + slot membership for the facts in one store. + + Deliberately pure-stdlib: the charter's zero-setup invariant means the default path cannot depend on + tantivy/bm25s. Those remain valid drop-in replacements behind the same three lookups this exposes + (`lexical_candidates`, `slot_members`, `user_members`).""" + + def __init__(self) -> None: + self.postings: dict[str, set[str]] = {} + self.doc_len: dict[str, int] = {} + self._doc_terms: dict[str, set[str]] = {} + self._total_len: int = 0 + self._by_user: dict[str, set[str]] = {} + self._by_slot: dict[tuple[str, str, str], set[str]] = {} + # key -> (user_id, slot): lets remove() unhook a fact in O(1) instead of sweeping every bucket. + self._doc_meta: dict[str, tuple[Optional[str], Optional[tuple[str, str, str]]]] = {} + # key -> payload. Candidate selection produces ids; resolving them through the backend's get() + # would be O(store) per id on LanceDB (it materialises the table and scans for the key), turning + # a bounded pool back into a quadratic read. These are references to objects the store already + # holds, so the cost is one dict entry per fact, not a second copy of the data. + self.payloads: dict[str, Any] = {} + # key -> insertion rank. Candidate ids arrive as an unordered set, but the scorers downstream are + # rank-based (RRF), so ties are broken by list position: an unordered candidate list would make + # the same query return different orderings run to run, and would diverge from the full scan even + # when the pool covers the whole store. Ranking by insertion order reproduces the store's own + # iteration order in O(k log k) over the candidates instead of O(store). + self._seq: dict[str, int] = {} + self._next_seq: int = 0 + + # --- corpus statistics (the CorpusStats protocol in retrieve.lexical) --- + + @property + def n_docs(self) -> int: + return len(self.doc_len) + + @property + def avgdl(self) -> float: + return max(1.0, self._total_len / self.n_docs) if self.n_docs else 1.0 + + @property + def df(self) -> dict[str, int]: + # Document frequency is exactly the posting-list length; materialising a parallel counter would + # be one more thing to keep in sync for no gain at these sizes. + return {term: len(ids) for term, ids in self.postings.items()} + + # --- maintenance --- + + def add(self, key: str, fact: Any) -> None: + """Index (or re-index) one fact. Re-upserting the same id is an update, not a duplicate.""" + text = getattr(fact, "text", None) + if not isinstance(text, str): + return # not a fact payload; nothing lexical to index + # An update keeps its original position, matching dict semantics in the backing stores. + seq = self._seq.get(key) + self.remove(key) + if seq is None: + seq = self._next_seq + self._next_seq += 1 + self._seq[key] = seq + + terms = stems(indexed_text(text, getattr(fact, "valid_at", 0.0) or 0.0)) + unique = set(terms) + for term in unique: + self.postings.setdefault(term, set()).add(key) + self._doc_terms[key] = unique + self.doc_len[key] = len(terms) + self._total_len += len(terms) + + user = getattr(fact, "user_id", None) + user = user if isinstance(user, str) else None + if user is not None: + self._by_user.setdefault(user, set()).add(key) + slot = _fact_key(fact) + if slot is not None: + self._by_slot.setdefault(slot, set()).add(key) + self._doc_meta[key] = (user, slot) + self.payloads[key] = fact + + def remove(self, key: str) -> None: + if key not in self.doc_len: + return + for term in self._doc_terms.pop(key, ()): # only this doc's terms, not the whole vocabulary + ids = self.postings.get(term) + if ids is not None: + ids.discard(key) + if not ids: + del self.postings[term] + self._total_len -= self.doc_len.pop(key, 0) + self.payloads.pop(key, None) + self._seq.pop(key, None) + user, slot = self._doc_meta.pop(key, (None, None)) + if user is not None and (ids := self._by_user.get(user)) is not None: + ids.discard(key) + if not ids: + del self._by_user[user] + if slot is not None and (ids := self._by_slot.get(slot)) is not None: + ids.discard(key) + if not ids: + del self._by_slot[slot] + + def clear(self) -> None: + self.__init__() # noqa: PLC2801 — re-initialising is the whole operation + + # --- lookups --- + + def user_members(self, user_id: str) -> set[str]: + return set(self._by_user.get(user_id, ())) + + def corpus_for(self, user_id: str, terms: Iterable[str]) -> "_UserCorpus": + """BM25 statistics for one user's facts, restricted to `terms` (all a scorer ever looks up). + + Scoping matters twice over. Per *user*, because the full-scan path scores one tenant's facts and + an IDF polluted by other tenants would both leak signal and misrank. Per *term*, because building + the whole vocabulary's document frequencies per query would be its own linear scan. + + One deliberate difference from the full scan: document frequency here counts a user's indexed + facts, not only those live at `as_of`. When nothing has been invalidated the two are identical; + once facts have been superseded, IDF is computed over the slightly larger historical corpus. That + is the more stable statistic (a term's rarity should not jitter as facts age out) and it never + reorders a single query's results, since every candidate is scored against the same corpus.""" + members = self._by_user.get(user_id) or set() + n = len(members) + total = sum(self.doc_len.get(key, 0) for key in members) if n else 0 + df = {} + for term in set(terms): + ids = self.postings.get(term) + if ids: + df[term] = len(ids & members) + return _UserCorpus(n_docs=n, avgdl=max(1.0, total / n) if n else 1.0, df=df) + + def resolve(self, keys: Iterable[str]) -> list[Any]: + """Candidate ids -> payloads in store order, skipping anything the index no longer holds. + + Store order (not set order) is what makes bounded retrieval reproducible and what lets it match + the full scan exactly — see `_seq`.""" + found = [(self._seq.get(key, 0), key) for key in keys if key in self.payloads] + found.sort() + return [self.payloads[key] for _, key in found] + + def slot_members(self, slots: Iterable[tuple[str, str, str]]) -> set[str]: + out: set[str] = set() + for slot in slots: + out |= self._by_slot.get(slot, set()) + return out + + def lexical_candidates(self, query: str, limit: int, user_id: Optional[str] = None) -> set[str]: + """Facts sharing at least one query term, ranked by summed inverse document frequency. + + This is the recall half of the hybrid thesis (CLAUDE.md M1): a fact can be lexically strong and + semantically weak — an exact name, a number, a date — and a vector-only candidate pool would drop + it. Ranking by rarity rather than raw overlap keeps a single distinctive term (a proper noun) + ahead of several common ones.""" + if limit <= 0: + return set() + allowed = self._by_user.get(user_id) if user_id is not None else None + n = max(1, self.n_docs) + score: dict[str, float] = {} + for term in set(stems(query)): + ids = self.postings.get(term) + if not ids: + continue + # Rarer term -> larger weight. len(ids) is the term's document frequency. + weight = n / len(ids) + for key in ids: + if allowed is not None and key not in allowed: + continue + score[key] = score.get(key, 0.0) + weight + if len(score) <= limit: + return set(score) + ranked = sorted(score.items(), key=lambda kv: (-kv[1], kv[0])) + return {key for key, _ in ranked[:limit]} + + +class IndexedVectorStore(VectorStore): + """Wraps any VectorStore and maintains a `FactIndex` alongside it. + + A decorator rather than a subclass so it composes with whichever backend is configured (in-memory, + LanceDB, a future pgvector), and so persistence keeps working: `store.persist` replays facts through + `.upsert()`, which rebuilds the index for free on load.""" + + def __init__(self, inner: VectorStore) -> None: + self.inner = inner + self.index = FactIndex() + for payload in inner.values(): # adopt a pre-populated store + key = getattr(payload, "id", None) + if isinstance(key, str): + self.index.add(key, payload) + + def upsert(self, key: str, vector: list[float], payload: Any) -> None: + self.inner.upsert(key, vector, payload) + self.index.add(key, payload) + + def search( + self, vector: list[float], top_k: int, where: Optional[Predicate] = None + ) -> list[tuple[float, Any]]: + return self.inner.search(vector, top_k, where) + + def get(self, key: str) -> Any | None: + return self.inner.get(key) + + def delete(self, key: str) -> None: + self.inner.delete(key) + self.index.remove(key) + + def values(self) -> list[Any]: + return self.inner.values() + + # Some backends expose extra surface (LanceDB's table handle, pickling hooks). Forward what we don't + # wrap so decorating a store never removes capability. Guarded against being consulted before + # __init__ has bound `inner` (unpickling calls __getattr__ before __dict__ is restored). + def __getattr__(self, name: str) -> Any: + inner = self.__dict__.get("inner") + if inner is None: + raise AttributeError(name) + return getattr(inner, name) diff --git a/engram/util.py b/engram/util.py index e6c5054..d5c8268 100644 --- a/engram/util.py +++ b/engram/util.py @@ -34,6 +34,33 @@ def fmt_datetime(epoch: float) -> str: return "?" +_MONTHS = ("january", "february", "march", "april", "may", "june", "july", "august", + "september", "october", "november", "december") + + +def date_terms(epoch: float) -> str: + """Render a fact's date as searchable tokens (year, numeric month, month name) so a query that names + a time ('May 2023', 'in 2024') matches the right-dated facts via BM25 — dates otherwise live only in + valid_at and are invisible to retrieval. This is query-time temporal matching done as a lexical signal + (MemoryScope time_ratio in spirit), with no score multiplier that could override relevance. + + Lives here, in the dependency-free base layer, because BOTH the retriever (which scores) and the + lexical index (which precomputes corpus statistics) must tokenize a fact identically — otherwise the + index's global IDF would describe a different corpus than the one being scored.""" + try: + d = fmt_date(epoch) # YYYY-MM-DD + y, m, _ = d.split("-") + return f"{d} {y} {m} {_MONTHS[int(m) - 1]}" + except (ValueError, IndexError): + return "" + + +def indexed_text(text: str, epoch: float) -> str: + """The exact string the lexical channel treats as a fact's document. Single source of truth so the + index and the scorer never drift apart.""" + return f"{text} {date_terms(epoch)}" + + def now() -> float: """Current wall-clock time in epoch seconds.""" return time.time() diff --git a/eval/scaling.py b/eval/scaling.py new file mode 100644 index 0000000..b00c72b --- /dev/null +++ b/eval/scaling.py @@ -0,0 +1,162 @@ +"""Read-path scaling measurement (charter Bet E). + +The charter claims linear scaling to 10M+ tokens, and the coding conventions say performance claims +come from the harness, not intuition. This script is that harness for the *read* path: it grows the +fact store and measures what `HybridRetriever.retrieve()` actually costs per query. + +It is deliberately offline and deterministic (hashing embedder, synthetic facts), so it measures the +*shape* of the cost curve, not absolute production latency. The shape is the point: a retriever that +scores every live fact per query is O(n), and no vector backend fixes that on its own. + + python3 eval/scaling.py # default sizes + python3 eval/scaling.py --sizes 100,1000,5000 --trials 20 +""" +from __future__ import annotations + +import argparse +import statistics +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from engram.config import Config # noqa: E402 +from engram.embed.hashing import HashingEmbedder # noqa: E402 +from engram.retrieve.hybrid import HybridRetriever # noqa: E402 +from engram.store.base import VectorStore # noqa: E402 +from engram.store.indexed import IndexedVectorStore # noqa: E402 +from engram.store.memory_store import InMemoryGraphStore, InMemoryVectorStore # noqa: E402 +from engram.types import Fact # noqa: E402 +from engram.util import now # noqa: E402 + +# A small vocabulary reused across facts so lexical scoring has real term statistics rather than +# every document being disjoint (which would make BM25 trivially cheap and hide the scan cost). +SUBJECTS = ["alice", "bob", "carol", "dave", "erin", "frank", "grace", "heidi"] +PREDICATES = ["works_at", "lives_in", "prefers", "visited", "studied", "owns", "avoids", "plans"] +OBJECTS = [ + "acme corp", "berlin", "oat milk", "kyoto", "linear algebra", "a road bike", + "crowded cafes", "a trip to lisbon", "the night shift", "sourdough baking", +] + + +def build_facts(n: int, embedder: HashingEmbedder, user_id: str = "u1") -> list[Fact]: + """Deterministic synthetic facts spread over a year of valid-time.""" + t = now() + facts: list[Fact] = [] + for i in range(n): + subj = SUBJECTS[i % len(SUBJECTS)] + pred = PREDICATES[(i // len(SUBJECTS)) % len(PREDICATES)] + obj = OBJECTS[(i // (len(SUBJECTS) * len(PREDICATES))) % len(OBJECTS)] + text = f"{subj} {pred.replace('_', ' ')} {obj} (record {i})" + f = Fact( + user_id=user_id, + subject=subj, + predicate=pred, + object=f"{obj} {i}", + text=text, + valid_at=t - (i % 365) * 86400.0, + embedding=embedder.embed(text), + ) + facts.append(f) + return facts + + +def measure( + n: int, + trials: int, + queries: list[str], + *, + bounded: bool = False, + pool: int = 400, + vector_channel: bool = True, +) -> dict: + embedder = HashingEmbedder() + store: VectorStore = InMemoryVectorStore() + if bounded: + store = IndexedVectorStore(store) + graph = InMemoryGraphStore() + for f in build_facts(n, embedder): + store.upsert(f.id, f.embedding or [], f) + + config = Config( + bounded_candidates=bounded, candidate_pool=pool, candidate_vector_channel=vector_channel + ) + retriever = HybridRetriever(store, graph, embedder, config) + + # warm up so first-call import/alloc costs do not land in the sample + retriever.retrieve(queries[0], "u1") + + samples: list[float] = [] + for i in range(trials): + q = queries[i % len(queries)] + t0 = time.perf_counter() + retriever.retrieve(q, "u1") + samples.append((time.perf_counter() - t0) * 1000.0) + + samples.sort() + return { + "n": n, + "p50_ms": statistics.median(samples), + "p95_ms": samples[min(len(samples) - 1, int(len(samples) * 0.95))], + "mean_ms": statistics.fmean(samples), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="Measure read-path cost vs. fact-store size.") + ap.add_argument("--sizes", default="100,500,2000,10000", help="comma-separated fact counts") + ap.add_argument("--trials", type=int, default=15, help="queries timed per size") + ap.add_argument("--pool", type=int, default=400, help="candidate pool for the bounded path") + args = ap.parse_args() + + sizes = [int(s) for s in args.sizes.split(",") if s.strip()] + queries = [ + "where does alice work", + "what does bob prefer to drink", + "which city did carol visit last year", + "what is dave studying", + ] + + variants = [ + ("full scan", [measure(n, args.trials, queries) for n in sizes]), + ( + "bounded +vec", + [measure(n, args.trials, queries, bounded=True, pool=args.pool) for n in sizes], + ), + ( + "bounded -vec", + [ + measure(n, args.trials, queries, bounded=True, pool=args.pool, vector_channel=False) + for n in sizes + ], + ), + ] + + print(f"candidate pool = {args.pool}\n") + header = f"{'facts':>8}" + "".join(f"{name:>15}" for name, _ in variants) + print(header) + print("-" * len(header)) + for i, n in enumerate(sizes): + row = f"{n:>8}" + "".join(f"{rows[i]['p50_ms']:>13.2f}ms" for _, rows in variants) + print(row) + + print("\nms per 1k facts (constant = O(n), falling = sub-linear):") + for name, rows in variants: + cells = "".join(f"{r['p50_ms'] / (r['n'] / 1000.0):>10.2f}" for r in rows) + print(f" {name:>13}{cells}") + + print(f"\nstore grew {sizes[-1] / sizes[0]:.0f}x ({sizes[0]} -> {sizes[-1]} facts)") + for name, rows in variants: + growth = rows[-1]["p50_ms"] / rows[0]["p50_ms"] if rows[0]["p50_ms"] else float("nan") + print(f" {name:>13}: per-query cost grew {growth:>6.1f}x") + print( + "\n'+vec' asks the vector store for semantic candidates. Neither shipped backend has a real ANN\n" + "index, so that call is itself a full scan — which is why bounding the lexical and fusion work\n" + "alone does not pay off. '-vec' is what the read path costs once no channel scans the store." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/results/bounded_candidates_scaling.md b/results/bounded_candidates_scaling.md new file mode 100644 index 0000000..e4c1600 --- /dev/null +++ b/results/bounded_candidates_scaling.md @@ -0,0 +1,57 @@ +# bounded_candidates — 读路径扩展性测量（2026-08-16） + +## 复现命令 + +```bash +python3 eval/scaling.py --sizes 100,500,2000,10000 --trials 15 --pool 400 +``` + +离线、确定性（HashingEmbedder + 合成事实），无 API key。测的是**成本曲线的形状**，不是生产绝对延迟。 + +## 结果 + +| 事实数 | 全量扫描 p50 | bounded 含语义通道 | bounded 去语义通道 | +| ---: | ---: | ---: | ---: | +| 100 | 2.07ms | 3.82ms | 0.62ms | +| 500 | 8.71ms | 15.64ms | 2.43ms | +| 2000 | 33.13ms | 43.41ms | 8.54ms | +| 10000 | 177.88ms | 177.03ms | **12.28ms** | + +每千条事实的毫秒数（恒定 = O(n)，下降 = 次线性）： + +| 变体 | 100 | 500 | 2000 | 10000 | +| --- | ---: | ---: | ---: | ---: | +| 全量扫描 | 20.67 | 17.42 | 16.57 | 17.79 | +| bounded 含语义通道 | 38.18 | 31.29 | 21.70 | 17.70 | +| bounded 去语义通道 | 6.17 | 4.86 | 4.27 | **1.23** | + +存储增长 100x 时，单查询成本增长：全量扫描 **86.0x** / 含语义通道 46.4x / 去语义通道 **19.9x**。 + +## 结论 + +1. **主干读路径是 O(n)。** 每千条事实的耗时在 100→10000 区间恒定在 ~17–20ms，这是全量扫描的 + 特征。10000 条事实时单查询已达 177ms，超过宪章的 <100ms 读路径目标，而这个规模对 + "10M+ token" 的目标而言还很小。 +2. **只把词汇与融合环节收敛成候选池，收益为零**（177.03ms vs 177.88ms）。因为语义通道调用 + `fact_store.search()`，而两个后端都没有真正的 ANN 索引：`InMemoryVectorStore.search()` 是 + 暴力 cosine 加一次全量排序；`LanceDBVectorStore.search()` 一旦传入 Python 谓词就走 + `table.to_arrow().to_pylist()` 全表物化。多租户检索每次都必须带 user 过滤，所以线上路径 + **永远拿不到 ANN 收益**。 +3. **候选池设计本身是有效的。** 去掉语义通道后 10000 条事实上快 14.5x，且每千条耗时从 6.17 + 降到 1.23——真正的次线性。卡住 Bet E 的是缺失的 ANN 索引，不是候选池思路。 + +## 未采用/暂缓原因 + +- `candidate_vector_channel=False` **不作为默认**：它会丢掉"语义相关但与查询无共享词"的事实， + 这正是 M1 已验证的 hybrid 论点所依赖的召回。在拿到真正的 ANN 后端、或有 keyed harness 跑出 + 召回损失可接受的证据之前，默认保持召回安全。 +- `bounded_candidates` **默认关闭**：已发布数字由全量扫描产出，静默改变打分集合会违反 + "每个公开数字都可追溯到已提交日志"的规则。候选池 ≥ 存活事实数时两条路径逐位一致 + （`tests/test_bounded_candidates.py::test_bounded_matches_full_scan_when_pool_covers_store`）。 + +## 下一步（按依赖顺序） + +1. 给向量存储一个真正的过滤 ANN：LanceDB 侧把 user 过滤下推成 SQL 谓词（而不是 Python 回调）， + 即可让 `search()` 走真实索引。这是解锁上面 14.5x 的唯一前提。 +2. `query_entity_ids()` 仍在扫 `graph.entities.values()`，需要实体名索引。 +3. `LanceDBVectorStore.get()` 全表物化后线性找 key，逐 id 取回会二次放大。 diff --git a/tests/test_bounded_candidates.py b/tests/test_bounded_candidates.py new file mode 100644 index 0000000..f7a1c53 --- /dev/null +++ b/tests/test_bounded_candidates.py @@ -0,0 +1,212 @@ +"""Bounded candidate retrieval (CLAUDE.md Bet E). + +The load-bearing test is `test_bounded_matches_full_scan_when_pool_covers_store`: with a pool large +enough to hold every fact, bounded retrieval must return exactly what the full scan returns. That is +what makes the feature a speed optimisation rather than a silent ranking change — and it is the guard +that would catch a future edit to one path but not the other. +""" +from __future__ import annotations + +from engram.config import Config +from engram.embed.hashing import HashingEmbedder +from engram.retrieve.hybrid import HybridRetriever +from engram.store.indexed import FactIndex, IndexedVectorStore +from engram.store.memory_store import InMemoryGraphStore, InMemoryVectorStore +from engram.types import Fact +from engram.util import now + +FACTS = [ + ("alice", "works_at", "acme corp", "alice works at acme corp"), + ("alice", "lives_in", "berlin", "alice lives in berlin"), + ("alice", "prefers", "oat milk", "alice prefers oat milk in coffee"), + ("bob", "works_at", "globex", "bob works at globex"), + ("bob", "visited", "kyoto", "bob visited kyoto in spring"), + ("carol", "studied", "linear algebra", "carol studied linear algebra at university"), + ("carol", "owns", "a road bike", "carol owns a road bike"), + ("dave", "avoids", "crowded cafes", "dave avoids crowded cafes"), +] + + +def _facts(embedder: HashingEmbedder, user_id: str = "u1") -> list[Fact]: + t = now() + return [ + Fact( + user_id=user_id, + subject=subj, + predicate=pred, + object=obj, + text=text, + valid_at=t - i * 86400.0, + embedding=embedder.embed(text), + ) + for i, (subj, pred, obj, text) in enumerate(FACTS) + ] + + +def _stores(): + """One set of facts loaded into both an undecorated and a decorated store. + + Building two independent fact sets would compare different data — different ids, and a different + `now()` so different dates, recency and lexical date terms. + """ + embedder = HashingEmbedder() + facts = _facts(embedder) + plain = InMemoryVectorStore() + boxed = IndexedVectorStore(InMemoryVectorStore()) + for f in facts: + plain.upsert(f.id, f.embedding or [], f) + boxed.upsert(f.id, f.embedding or [], f) + return plain, boxed, embedder + + +def _retrieve(store, embedder, query, *, bounded, pool=400, user_id="u1"): + config = Config(bounded_candidates=bounded, candidate_pool=pool) + retriever = HybridRetriever(store, InMemoryGraphStore(), embedder, config) + ranked, _diag = retriever.retrieve(query, user_id, top_k=5) + return [(f.id, round(score, 9)) for f, score in ranked] + + +QUERIES = [ + "where does alice work", + "what does alice prefer", + "who visited kyoto", + "what did carol study", + "bike", +] + + +def test_bounded_matches_full_scan_when_pool_covers_store(): + """The safety property: a pool wider than the store must not change a single result.""" + plain, boxed, embedder = _stores() + for query in QUERIES: + full = _retrieve(plain, embedder, query, bounded=False) + bounded = _retrieve(boxed, embedder, query, bounded=True, pool=400) + assert bounded == full, f"bounded retrieval diverged from full scan on {query!r}" + + +def test_bounded_is_a_noop_without_an_index(): + """Turning the flag on against an undecorated store must fall back, not crash or silently empty.""" + plain, _boxed, embedder = _stores() + for query in QUERIES: + assert _retrieve(plain, embedder, query, bounded=True) == _retrieve( + plain, embedder, query, bounded=False + ) + + +def test_bounded_still_returns_results_with_a_tiny_pool(): + """A pool smaller than the store is allowed to rank differently, but must stay useful.""" + _plain, boxed, embedder = _stores() + ranked = _retrieve(boxed, embedder, "where does alice work", bounded=True, pool=2) + assert ranked, "a small pool must still retrieve something" + + +def test_slot_completion_keeps_superseded_facts_from_surviving(): + """A single-valued slot's head must be pulled in even when only a stale slot-mate matched. + + Without slot completion `_current_slot_heads` would see one lonely stale fact, treat it as its own + head, and let a superseded value through — the read path's non-destructive-invalidation guarantee. + """ + embedder = HashingEmbedder() + store = IndexedVectorStore(InMemoryVectorStore()) + t = now() + old = Fact( + user_id="u1", subject="alice", predicate="works_at", object="acme corp", + text="alice works at acme corp", valid_at=t - 400 * 86400.0, + embedding=embedder.embed("alice works at acme corp"), + ) + new = Fact( + user_id="u1", subject="alice", predicate="works_at", object="initech", + text="alice works at initech", valid_at=t, + embedding=embedder.embed("alice works at initech"), + ) + for f in (old, new): + store.upsert(f.id, f.embedding or [], f) + + config = Config(bounded_candidates=True, candidate_pool=400) + retriever = HybridRetriever(store, InMemoryGraphStore(), embedder, config) + # Query only the OLD value's distinctive term, so the stale fact is what the lexical channel finds. + candidates = retriever._bounded_candidates("acme", "u1", None, embedder.embed("acme")) + ids = {f.id for f in candidates} + assert old.id in ids + assert new.id in ids, "slot head must be pulled in alongside a matched slot-mate" + + ranked, _ = retriever.retrieve("acme", "u1", top_k=5) + assert old.id not in {f.id for f, _ in ranked}, "superseded slot value must not be retrieved" + + +def test_index_tracks_updates_and_deletes(): + index = FactIndex() + embedder = HashingEmbedder() + f = Fact( + user_id="u1", subject="alice", predicate="works_at", object="acme", + text="alice works at acme", valid_at=now(), embedding=embedder.embed("x"), + ) + index.add(f.id, f) + assert index.n_docs == 1 + assert index.lexical_candidates("acme", 10, user_id="u1") == {f.id} + + # Re-adding the same id is an update: the old terms must not linger as a phantom document. + f.text = "alice works at initech" + index.add(f.id, f) + assert index.n_docs == 1, "re-upsert must update, not duplicate" + assert index.lexical_candidates("acme", 10, user_id="u1") == set() + assert index.lexical_candidates("initech", 10, user_id="u1") == {f.id} + + index.remove(f.id) + assert index.n_docs == 0 + assert index.lexical_candidates("initech", 10, user_id="u1") == set() + assert index.postings == {}, "posting lists must not retain empty entries" + assert index.payloads == {} + + +def test_index_scopes_candidates_and_corpus_by_user(): + """Tenants must not see each other's facts, nor pollute each other's IDF.""" + index = FactIndex() + embedder = HashingEmbedder() + ids = {} + for user in ("u1", "u2"): + f = Fact( + user_id=user, subject="alice", predicate="works_at", object="acme", + text="alice works at acme", valid_at=now(), embedding=embedder.embed("x"), + ) + index.add(f.id, f) + ids[user] = f.id + + assert index.lexical_candidates("acme", 10, user_id="u1") == {ids["u1"]} + assert index.lexical_candidates("acme", 10, user_id="u2") == {ids["u2"]} + + corpus = index.corpus_for("u1", ["acme"]) + assert corpus.n_docs == 1, "corpus size must count only this tenant's facts" + assert corpus.df["acme"] == 1, "document frequency must not count the other tenant's copy" + + +def test_decorator_preserves_store_behaviour(): + """The decorator must be transparent: same reads, same values, delete still deletes.""" + inner = InMemoryVectorStore() + boxed = IndexedVectorStore(inner) + embedder = HashingEmbedder() + f = Fact( + user_id="u1", subject="alice", predicate="works_at", object="acme", + text="alice works at acme", valid_at=now(), embedding=embedder.embed("x"), + ) + boxed.upsert(f.id, f.embedding or [], f) + assert boxed.get(f.id) is f + assert boxed.values() == inner.values() + assert boxed.search(f.embedding or [], 5)[0][1] is f + + boxed.delete(f.id) + assert boxed.get(f.id) is None + assert boxed.index.n_docs == 0 + + +def test_decorator_adopts_a_prepopulated_store(): + """Wrapping a store that already holds facts must index them, not start blind.""" + inner = InMemoryVectorStore() + embedder = HashingEmbedder() + f = Fact( + user_id="u1", subject="alice", predicate="works_at", object="acme", + text="alice works at acme", valid_at=now(), embedding=embedder.embed("x"), + ) + inner.upsert(f.id, f.embedding or [], f) + boxed = IndexedVectorStore(inner) + assert boxed.index.lexical_candidates("acme", 10, user_id="u1") == {f.id} From 34f7f759139472820084e549c992695eb6d01879 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 15:58:38 +0800 Subject: [PATCH 04/25] store: push the tenant filter into the vector index instead of scanning for it Multi-tenant retrieval filters by user on every single query, and the only way to express that was a Python predicate. A backend cannot see inside one, so it had to hand every row to Python before ranking: LanceDB called to_arrow().to_pylist() on the whole table. The consequence is that configuring the scale backend never bought a single sub-linear read -- there was a vector file, not a vector index. Lift user_id out of the opaque JSON payload into a real column and give VectorStore.search() a declarative user_id= alongside the general predicate, so LanceDB can prefilter inside its own index. Measured on a skewed two-tenant table: 4.04ms -> 1.13ms at 500 rows, 75.38ms -> 1.35ms at 10k, and 302.36ms -> 1.83ms at 40k. The pushed-down path barely moves as the table grows 80x while the predicate path is strictly linear -- 165x at the top end. The correctness risk with any filtered ANN is that the filter runs after the search, which silently returns too few rows or none. test_prefilter_finds_hits_beyond_the_unfiltered_neighbourhood is built to fail in exactly that case: the majority tenant fills the query's whole neighbourhood and the tenant being asked for sits far away, so a post-filter finds nothing. Tables written by earlier releases have no such column. _has_tenant_column() probes the real schema rather than assuming, so those keep reading and writing -- falling back to a scan -- instead of failing on a schema mismatch. eval/scaling.py now reports this alongside the retriever benchmark, skipping it when lancedb is absent. --- docs/architecture-optimization-map.zh-CN.md | 3 +- engram/retrieve/hybrid.py | 6 +- engram/store/base.py | 20 +++- engram/store/indexed.py | 9 +- engram/store/lancedb_store.py | 63 ++++++++++-- engram/store/memory_store.py | 12 ++- eval/scaling.py | 67 ++++++++++++- results/bounded_candidates_scaling.md | 24 +++++ tests/test_lancedb_tenant_filter.py | 106 ++++++++++++++++++++ 9 files changed, 288 insertions(+), 22 deletions(-) create mode 100644 tests/test_lancedb_tenant_filter.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 9279748..25cead5 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `tenant_filter_pushdown` | Store / 向量检索索引层（Bet E） | 多租户检索每次都必须按 user 过滤，而唯一的表达方式是 Python 谓词——后端看不进去，只能先物化全表再排序。于是"接了 LanceDB"从未换来任何 ANN 收益。现把 `user_id` 从 JSON payload 提升为真实列，`VectorStore.search()` 增加声明式 `user_id=` 参数，LanceDB 走 `prefilter=True` 在索引内收窄。40000 行时 **165x**，且延迟基本随行数不变 | `results/bounded_candidates_scaling.md`，`tests/test_lancedb_tenant_filter.py`（5 测试，含专门证伪"后置过滤"的用例 + 旧 schema 兼容） | | 2026-08-12 | `cross_instance_portability` | Connectors / Memory facade / Service import 路由 / HTTP `/v1/import` | 导出无法导回（POST 导出 JSON 得到 500）——补上原生 `engram` 导入格式：事实保留原 id/双时间戳/supersedes 链/provenance，目标端用本地 embedder 重嵌入（即官方换 embedder 迁移路径），按 id 幂等；`/v1/import` 对坏 payload 返回 400 | `tests/test_cross_account_portability.py`, `tests/test_server_import_export.py`（工程验收，非算法实验） | | 2026-08-12 | `mcp_http_bearer_gate` | MCP streamable-HTTP 传输边界 | MCP HTTP 模式此前无任何鉴权，仅靠默认 127.0.0.1；新增 `--http-token`/`ENGRAM_MCP_HTTP_TOKEN` Bearer 门，非回环绑定无 token 时启动即拒绝（失败关闭，与 REST 的 `ENGRAM_API_KEYS` 同哲学） | `tests/test_mcp_http_auth.py` | | 2026-08-12 | `engram_storage_env` + 一致性修复 | Service 配置边界 / stats / import CLI | 服务器此前永远 `storage="memory"`（无环境变量可选 LanceDB）；新增 `ENGRAM_STORAGE`（非法值失败关闭）。`/v1/stats` 改按 canonical 身份过滤（与其它读路径一致）；import CLI 本地模式改走 `MemoryService`，目录命名与服务端统一 | `tests/test_cross_account_portability.py` | @@ -125,7 +126,7 @@ flowchart TD | P1 | Graph proximity / multi-hop | multi-session、multi-hop 是长期记忆系统最难类别，也是差异化战场 | 轻量 n-hop/PPR-style expansion，先用真实错例切片验证 | | P1 | Temporal interval reasoning | temporal-reasoning 仍低于 full-context，需要更强的区间和 duration 证据 | 显式 start/end pair、invalid_at span、date arithmetic block | | P2 | Runtime profiles | 让用户选择 lite/standard/graph/consolidated，并用同一 harness 报三联表 | 在 `Config`/bench 层定义可测 profile，而不是手动组合开关 | -| **P0** | **向量存储的过滤 ANN** | Bet E 的真正卡点。两个后端都没有真 ANN 索引：`InMemoryVectorStore.search()` 暴力 cosine + 全量排序；`LanceDBVectorStore.search()` 一旦传 Python 谓词就 `to_arrow().to_pylist()` 全表物化。多租户检索必带 user 过滤，所以线上永远走不到索引 | 把 user 过滤下推为 LanceDB SQL 谓词。这是解锁 `bounded_candidates` 已测得的 14.5x 加速的唯一前提——候选池代码已就位，见 `results/bounded_candidates_scaling.md` | +| ~~P0~~ 已落地 | ~~向量存储的过滤 ANN~~ | 已修复，见下方台账 `tenant_filter_pushdown` | 剩余：`InMemoryVectorStore` 仍是暴力扫描（参考实现，设计如此）；`query_entity_ids()` 仍扫 `graph.entities.values()`，需实体名索引；`LanceDBVectorStore.get()` 仍全表物化后线性找 key | ## 未采用/回滚原因 diff --git a/engram/retrieve/hybrid.py b/engram/retrieve/hybrid.py index 427f359..cd0d470 100644 --- a/engram/retrieve/hybrid.py +++ b/engram/retrieve/hybrid.py @@ -354,9 +354,9 @@ def _bounded_candidates( ids = index.lexical_candidates(query, pool, user_id=user_id) if self.config.candidate_vector_channel: - for _score, payload in self.fact_store.search( - qvec, pool, where=lambda f: getattr(f, "user_id", None) == user_id - ): + # Declarative tenant filter, not a Python predicate: a backend can push the former into its + # own index, while the latter forces it to scan every row before ranking. + for _score, payload in self.fact_store.search(qvec, pool, user_id=user_id): fid = getattr(payload, "id", None) if isinstance(fid, str): ids.add(fid) diff --git a/engram/store/base.py b/engram/store/base.py index 98b789e..ee2030f 100644 --- a/engram/store/base.py +++ b/engram/store/base.py @@ -18,8 +18,24 @@ def upsert(self, key: str, vector: list[float], payload: Any) -> None: ... @abstractmethod def search( - self, vector: list[float], top_k: int, where: Optional[Predicate] = None - ) -> list[tuple[float, Any]]: ... + self, + vector: list[float], + top_k: int, + where: Optional[Predicate] = None, + *, + user_id: Optional[str] = None, + ) -> list[tuple[float, Any]]: + """Nearest neighbours, optionally restricted. + + Two filters, because they cost different things. `where` is an arbitrary Python predicate: a + backend cannot see inside it, so it must consider every row before ranking — correct, but a full + scan. `user_id` is the one filter this system applies on literally every retrieval (multi-tenant + isolation), and stating it declaratively lets a backend push it into its own index instead. + + That distinction is load-bearing: with only the predicate form, the tenant filter silently turned + every "ANN" search into a table scan, so no vector backend could ever deliver sub-linear reads. + Backends without native filtering may implement `user_id` as an equality check. + """ @abstractmethod def get(self, key: str) -> Any | None: ... diff --git a/engram/store/indexed.py b/engram/store/indexed.py index da6401d..79155aa 100644 --- a/engram/store/indexed.py +++ b/engram/store/indexed.py @@ -237,9 +237,14 @@ def upsert(self, key: str, vector: list[float], payload: Any) -> None: self.index.add(key, payload) def search( - self, vector: list[float], top_k: int, where: Optional[Predicate] = None + self, + vector: list[float], + top_k: int, + where: Optional[Predicate] = None, + *, + user_id: Optional[str] = None, ) -> list[tuple[float, Any]]: - return self.inner.search(vector, top_k, where) + return self.inner.search(vector, top_k, where, user_id=user_id) def get(self, key: str) -> Any | None: return self.inner.get(key) diff --git a/engram/store/lancedb_store.py b/engram/store/lancedb_store.py index 799849c..76c0268 100644 --- a/engram/store/lancedb_store.py +++ b/engram/store/lancedb_store.py @@ -17,6 +17,11 @@ _TYPES = {"Episode": Episode, "Fact": Fact} +# Tenant id lifted out of the JSON payload into its own column. Without it every multi-tenant search +# degrades to a table scan, because a Python predicate is the only way to express "this user's facts" +# and LanceDB cannot see inside one. +_TENANT_COL = "user_id" + def _quote(value: str) -> str: return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" @@ -52,6 +57,7 @@ def __init__(self, path: str, table: str = "vectors") -> None: os.makedirs(self.path, exist_ok=True) self._db = lancedb.connect(self.path) self._table = None + self._tenant_col: Optional[bool] = None # resolved lazily against the real schema def _open(self): if self._table is not None: @@ -62,14 +68,33 @@ def _open(self): self._table = self._db.open_table(self.table_name) return self._table + def _row(self, key: str, vector: list[float], payload: Any) -> dict[str, Any]: + """One table row. `user_id` is promoted out of the opaque payload into a real column so the + tenant filter can be pushed into LanceDB's index rather than applied in Python after a scan.""" + return { + "key": key, + "vector": vector, + "payload": _encode_payload(payload), + _TENANT_COL: getattr(payload, "user_id", None) or "", + } + + def _has_tenant_column(self, table) -> bool: + """Tables written before the tenant column existed still load and still work — they just cannot + prefilter. Detect rather than assume, so an upgrade never corrupts or rejects existing data.""" + if self._tenant_col is None: + try: + self._tenant_col = _TENANT_COL in set(table.schema.names) + except AttributeError: # pragma: no cover - older/newer client without .schema.names + self._tenant_col = False + return self._tenant_col + def _ensure(self, vector: list[float], key: str, payload: Any): table = self._open() if table is None: self._table = self._db.create_table( - self.table_name, - data=[{"key": key, "vector": vector, "payload": _encode_payload(payload)}], - mode="overwrite", + self.table_name, data=[self._row(key, vector, payload)], mode="overwrite" ) + self._tenant_col = True return None return table @@ -78,23 +103,43 @@ def upsert(self, key: str, vector: list[float], payload: Any) -> None: if table is None: return table.delete(f"key = {_quote(key)}") - table.add([{"key": key, "vector": vector, "payload": _encode_payload(payload)}]) + row = self._row(key, vector, payload) + if not self._has_tenant_column(table): + row.pop(_TENANT_COL) # legacy table: adding an unknown column would be a schema violation + table.add([row]) def search( - self, vector: list[float], top_k: int, where: Optional[Predicate] = None + self, + vector: list[float], + top_k: int, + where: Optional[Predicate] = None, + *, + user_id: Optional[str] = None, ) -> list[tuple[float, Any]]: table = self._open() if table is None or top_k <= 0: return [] - # Python predicates are part of the VectorStore contract, so filtered searches must consider every - # row before ranking. Otherwise a tenant/user filter can miss valid hits hidden beyond LanceDB's - # nearest unfiltered rows. - rows = table.to_arrow().to_pylist() if where is not None else table.search(vector).limit(top_k).to_list() + + if where is None and user_id is not None and self._has_tenant_column(table): + # The whole point of the tenant column: a prefiltered ANN query, so LanceDB narrows to this + # tenant inside its own index and returns top_k without materialising the table. + query = table.search(vector).where(f"{_TENANT_COL} = {_quote(user_id)}", prefilter=True) + rows = query.limit(top_k).to_list() + elif where is None and user_id is None: + rows = table.search(vector).limit(top_k).to_list() + else: + # An arbitrary Python predicate is opaque to the backend, so every row must be considered + # before ranking — otherwise a filter can miss valid hits hidden beyond the nearest + # unfiltered rows. Same for a tenant filter on a legacy table with no column to filter on. + rows = table.to_arrow().to_pylist() + scored: list[tuple[float, Any]] = [] for row in rows: payload = _decode_payload(row["payload"]) if where is not None and not where(payload): continue + if user_id is not None and getattr(payload, "user_id", None) != user_id: + continue scored.append((cosine(vector, row["vector"]), payload)) scored.sort(key=lambda x: x[0], reverse=True) return scored[:top_k] diff --git a/engram/store/memory_store.py b/engram/store/memory_store.py index 374867d..187aa2f 100644 --- a/engram/store/memory_store.py +++ b/engram/store/memory_store.py @@ -18,12 +18,20 @@ def upsert(self, key: str, vector: list[float], payload: Any) -> None: self._d[key] = (vector, payload) def search( - self, vector: list[float], top_k: int, where: Optional[Predicate] = None + self, + vector: list[float], + top_k: int, + where: Optional[Predicate] = None, + *, + user_id: Optional[str] = None, ) -> list[tuple[float, Any]]: + # No index to push a tenant filter into, so it is just another equality check here. The reference + # store is brute-force by design (see the module docstring); scale comes from a real backend. scored = [ (cosine(vector, vec), payload) for vec, payload in self._d.values() - if where is None or where(payload) + if (user_id is None or getattr(payload, "user_id", None) == user_id) + and (where is None or where(payload)) ] scored.sort(key=lambda x: x[0], reverse=True) return scored[:top_k] diff --git a/eval/scaling.py b/eval/scaling.py index b00c72b..72b6c45 100644 --- a/eval/scaling.py +++ b/eval/scaling.py @@ -16,6 +16,7 @@ import argparse import statistics import sys +import tempfile import time from pathlib import Path @@ -103,6 +104,64 @@ def measure( } +def measure_backend_filter(sizes: list[int], trials: int) -> None: + """How much the tenant filter costs on the scale backend, pushed down vs. applied in Python. + + Separate from the retriever benchmark above because it isolates one question: can the vector backend + narrow to a tenant inside its own index, or must it hand every row to Python first? Multi-tenant + retrieval filters by user on every single query, so this is the difference between having a vector + index and merely having a vector file. + """ + try: + import lancedb # noqa: PLC0415 - optional backend + except ImportError: + print("\n(lancedb not installed - skipping the backend filter benchmark)") + return + + from engram.store.lancedb_store import LanceDBVectorStore, _encode_payload # noqa: PLC0415 + + print(f"\n{'rows':>8} {'pushed down':>13} {'python predicate':>18} {'speedup':>9}") + print("-" * 56) + t = now() + for n in sizes: + with tempfile.TemporaryDirectory() as tmp: + rows = [] + for i in range(n): + # A skewed tenant mix: the minority tenant's rows sit outside the query's unfiltered + # neighbourhood, which is exactly the case a post-filter handles badly. + user = "alice" if i % 10 else "bob" + f = Fact( + user_id=user, subject="s", predicate="p", object=f"o{i}", + text=f"fact number {i} about things", valid_at=t - i * 86400.0, + embedding=[1.0, i / max(1, n), (i % 7) / 7], + ) + rows.append( + {"key": f.id, "vector": f.embedding, "payload": _encode_payload(f), "user_id": user} + ) + lancedb.connect(tmp).create_table("facts", data=rows, mode="overwrite") + store = LanceDBVectorStore(tmp, "facts") + q = [1.0, 0.5, 0.3] + + def timed(call) -> float: + call() # warm + samples = [] + for _ in range(trials): + t0 = time.perf_counter() + call() + samples.append((time.perf_counter() - t0) * 1000.0) + return statistics.median(samples) + + pushed = timed(lambda: store.search(q, 15, user_id="bob")) + scanned = timed(lambda: store.search(q, 15, where=lambda p: p.user_id == "bob")) + ratio = scanned / pushed if pushed else float("nan") + print(f"{n:>8} {pushed:>11.2f}ms {scanned:>16.2f}ms {ratio:>8.1f}x") + + print( + "\nA flat 'pushed down' column is the index working. The predicate column is what every\n" + "multi-tenant query cost before the tenant id became a real column." + ) + + def main() -> int: ap = argparse.ArgumentParser(description="Measure read-path cost vs. fact-store size.") ap.add_argument("--sizes", default="100,500,2000,10000", help="comma-separated fact counts") @@ -151,10 +210,12 @@ def main() -> int: growth = rows[-1]["p50_ms"] / rows[0]["p50_ms"] if rows[0]["p50_ms"] else float("nan") print(f" {name:>13}: per-query cost grew {growth:>6.1f}x") print( - "\n'+vec' asks the vector store for semantic candidates. Neither shipped backend has a real ANN\n" - "index, so that call is itself a full scan — which is why bounding the lexical and fusion work\n" - "alone does not pay off. '-vec' is what the read path costs once no channel scans the store." + "\n'+vec' asks the vector store for semantic candidates. The in-memory reference store is\n" + "brute-force by design, so that call is itself a full scan — which is why bounding the lexical\n" + "and fusion work alone does not pay off here. '-vec' is what the read path costs once nothing\n" + "scans the store. The scale backend does have an index; the benchmark below measures it." ) + measure_backend_filter(sizes, args.trials) return 0 diff --git a/results/bounded_candidates_scaling.md b/results/bounded_candidates_scaling.md index e4c1600..eec865e 100644 --- a/results/bounded_candidates_scaling.md +++ b/results/bounded_candidates_scaling.md @@ -40,6 +40,30 @@ python3 eval/scaling.py --sizes 100,500,2000,10000 --trials 15 --pool 400 3. **候选池设计本身是有效的。** 去掉语义通道后 10000 条事实上快 14.5x，且每千条耗时从 6.17 降到 1.23——真正的次线性。卡住 Bet E 的是缺失的 ANN 索引，不是候选池思路。 +## 追加：规模后端的租户过滤下推（同日） + +上面第 2 条指出的根因已修复。`user_id` 从不透明的 JSON payload 中提升为 LanceDB 的真实列， +`VectorStore.search()` 增加声明式 `user_id=` 参数（与通用 Python 谓词并存），多租户检索因此 +可以走 `where(..., prefilter=True)` 在索引内部收窄。 + +| 行数 | 下推 prefilter | Python 谓词 | 加速 | +| ---: | ---: | ---: | ---: | +| 500 | 1.13ms | 4.04ms | 3.6x | +| 2000 | 1.16ms | 15.18ms | 13.1x | +| 10000 | 1.35ms | 75.38ms | 56.0x | +| 40000 | 1.83ms | 302.36ms | **165.4x** | + +行数增长 80x（500→40000）时，下推路径延迟只从 1.13ms 涨到 1.83ms——**基本是平的**，而谓词路径 +是严格线性的。这就是 Bet E 在规模后端上的兑现。 + +正确性由 `tests/test_lancedb_tenant_filter.py` 保证，其中 +`test_prefilter_finds_hits_beyond_the_unfiltered_neighbourhood` 是专门设计来证伪"过滤发生在 +ANN 之后"的：让多数租户的行填满查询的整个最近邻域，少数租户的行全部远离查询。若过滤是后置的， +top_k 里一条目标租户的行都没有，返回空。该测试通过，说明 prefilter 真实生效。 + +向后兼容：旧版本写出的表没有 `user_id` 列。`_has_tenant_column()` 按真实 schema 探测而非假设， +旧表继续可读可写（退回扫描），不会因 schema 不匹配而报错或损坏数据。 + ## 未采用/暂缓原因 - `candidate_vector_channel=False` **不作为默认**：它会丢掉"语义相关但与查询无共享词"的事实， diff --git a/tests/test_lancedb_tenant_filter.py b/tests/test_lancedb_tenant_filter.py new file mode 100644 index 0000000..adc2105 --- /dev/null +++ b/tests/test_lancedb_tenant_filter.py @@ -0,0 +1,106 @@ +"""LanceDB tenant prefiltering — the change that lets a vector backend actually be an index. + +Before this, the only way to say "this user's facts" was a Python predicate, which LanceDB cannot see +inside, so every multi-tenant search materialised the whole table. Since multi-tenant retrieval always +filters by user, no search was ever sub-linear. + +The load-bearing test is `test_prefilter_finds_hits_beyond_the_unfiltered_neighbourhood`: it is designed +to FAIL if the filter is applied after the ANN query instead of inside it. +""" +from __future__ import annotations + +import pytest + +from engram.types import Fact +from engram.util import now + +lancedb = pytest.importorskip("lancedb") + +from engram.store.lancedb_store import _TENANT_COL, LanceDBVectorStore # noqa: E402 + + +def _fact(user: str, text: str, vec: list[float]) -> Fact: + return Fact( + user_id=user, subject="s", predicate="p", object=text, text=text, + valid_at=now(), embedding=vec, + ) + + +def test_tenant_filter_returns_only_that_tenant(tmp_path): + store = LanceDBVectorStore(str(tmp_path / "db"), "facts") + for user in ("alice", "bob"): + for i in range(3): + f = _fact(user, f"{user} fact {i}", [1.0, float(i), 0.0]) + store.upsert(f.id, f.embedding or [], f) + + hits = store.search([1.0, 0.0, 0.0], 10, user_id="alice") + assert hits, "tenant search must return the tenant's own rows" + assert {p.user_id for _s, p in hits} == {"alice"} + + +def test_prefilter_finds_hits_beyond_the_unfiltered_neighbourhood(tmp_path): + """The correctness property a post-filter cannot satisfy. + + One tenant's rows sit far from the query; the other tenant fills the entire nearest neighbourhood. + Filtering *after* a top_k ANN query would return nothing, because none of the k nearest rows belong + to the tenant asked for. Only a filter applied inside the search can find them. + """ + store = LanceDBVectorStore(str(tmp_path / "db"), "facts") + query = [1.0, 0.0, 0.0] + + for i in range(50): # noisy majority tenant, all near the query + f = _fact("loud", f"loud {i}", [1.0, 0.001 * i, 0.0]) + store.upsert(f.id, f.embedding or [], f) + wanted = [] + for i in range(3): # quiet tenant, all far from the query + f = _fact("quiet", f"quiet {i}", [0.0, 1.0, 0.05 * i]) + store.upsert(f.id, f.embedding or [], f) + wanted.append(f.id) + + hits = store.search(query, 3, user_id="quiet") + assert {p.id for _s, p in hits} == set(wanted), ( + "tenant filter must run inside the search; a post-filter would return nothing here" + ) + + +def test_tenant_column_is_written(tmp_path): + store = LanceDBVectorStore(str(tmp_path / "db"), "facts") + f = _fact("alice", "hello", [1.0, 0.0, 0.0]) + store.upsert(f.id, f.embedding or [], f) + rows = store._open().to_arrow().to_pylist() + assert rows[0][_TENANT_COL] == "alice" + + +def test_python_predicate_still_supported(tmp_path): + """The general escape hatch must keep working (it scans, by necessity).""" + store = LanceDBVectorStore(str(tmp_path / "db"), "facts") + for user in ("alice", "bob"): + f = _fact(user, f"{user} note", [1.0, 0.0, 0.0]) + store.upsert(f.id, f.embedding or [], f) + + hits = store.search([1.0, 0.0, 0.0], 10, where=lambda p: p.user_id == "bob") + assert {p.user_id for _s, p in hits} == {"bob"} + + +def test_legacy_table_without_tenant_column_still_works(tmp_path): + """A store written by an earlier release has no tenant column. It must keep reading and writing — + falling back to a scan — rather than failing on a schema mismatch.""" + path = str(tmp_path / "db") + db = lancedb.connect(path) + legacy = _fact("alice", "legacy row", [1.0, 0.0, 0.0]) + from engram.store.lancedb_store import _encode_payload + + db.create_table( + "facts", + data=[{"key": legacy.id, "vector": legacy.embedding, "payload": _encode_payload(legacy)}], + mode="overwrite", + ) + + store = LanceDBVectorStore(path, "facts") + assert store._has_tenant_column(store._open()) is False + + fresh = _fact("alice", "new row", [1.0, 0.01, 0.0]) + store.upsert(fresh.id, fresh.embedding or [], fresh) # must not raise on the old schema + + hits = store.search([1.0, 0.0, 0.0], 10, user_id="alice") + assert {p.id for _s, p in hits} == {legacy.id, fresh.id} From dec8dc6e59a207c347e51c10ae7a2565927a794b Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 17:24:16 +0800 Subject: [PATCH 05/25] retrieve: skip the entity scan when the query has no negation, and push key lookups into LanceDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every retrieval runs query_entity_ids(), which ends by calling graph_excluded_entity_ids(), which ran two regexes against every entity name in the store looking for "not Lisbon" style constraints. Almost no query contains a negation, so almost every retrieval paid for a full entity scan that found nothing. At 5,000 entities a plain query spent 509ms there; it now spends 0.0009ms. The matcher is anchored to the end of the text preceding an entity mention, so it cannot be run against the whole query directly. What can is a necessary condition: if no cue word appears anywhere, no slice can contain one. Both regexes are now built from a single cue list so they cannot drift. Writing that cue test the obvious way was wrong, and the test caught it. A non-ASCII entity name has no word-boundary guard in _entity_name_mentions, so the preceding slice can end mid-word: in "not上海" the slice "not" ends on a boundary the full string does not have, and \bnot\b misses it -- silently dropping an exclusion. The cue test therefore drops its trailing \b and is deliberately weaker than the real matcher. Over-matching costs a scan that finds nothing; under-matching loses a constraint the user stated. test_exclusion_shortcut checks the implication over every prefix of its samples, so adding a cue to one regex and not the other fails loudly. Separately, LanceDBVectorStore.get() materialised the table and scanned it in Python for one key, which makes any id-at-a-time access quadratic. It now issues a filter-only query so the predicate runs inside LanceDB. (table.query() does not exist in 0.33 and pylance is not installed; search().where() is what works, verified rather than assumed.) Still linear, and now written down: the exclusion scan itself when a query does contain a cue (its matching is substring-based, which Chinese names require, so a token index cannot replace it), and the name/alias matching in query_entity_ids, which is tokenised and could be indexed. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/retrieve/hybrid.py | 39 ++++++++- engram/store/lancedb_store.py | 12 +-- results/bounded_candidates_scaling.md | 35 ++++++++ tests/test_exclusion_shortcut.py | 96 +++++++++++++++++++++ tests/test_lancedb_tenant_filter.py | 15 ++++ 6 files changed, 190 insertions(+), 8 deletions(-) create mode 100644 tests/test_exclusion_shortcut.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 25cead5..0bafcbd 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `exclusion_early_out` + `lance_key_pushdown` | Read path / 图排除区 + Store 按键读取 | 每次检索都经 `query_entity_ids()` → `graph_excluded_entity_ids()`，后者对每个实体名跑两遍正则，而多数查询无否定词。改为先用提示词必要条件判定，5000 实体时无否定词查询从 509ms 降到 0.0009ms。同时 `LanceDBVectorStore.get()` 改用纯过滤查询，不再全表物化后线性找 key | `results/bounded_candidates_scaling.md`，`tests/test_exclusion_shortcut.py`（含逐前缀验证的正则不变量，已抓到一个中文实体名下的静默漏排除缺陷），`tests/test_lancedb_tenant_filter.py` | | 2026-08-16 | `tenant_filter_pushdown` | Store / 向量检索索引层（Bet E） | 多租户检索每次都必须按 user 过滤，而唯一的表达方式是 Python 谓词——后端看不进去，只能先物化全表再排序。于是"接了 LanceDB"从未换来任何 ANN 收益。现把 `user_id` 从 JSON payload 提升为真实列，`VectorStore.search()` 增加声明式 `user_id=` 参数，LanceDB 走 `prefilter=True` 在索引内收窄。40000 行时 **165x**，且延迟基本随行数不变 | `results/bounded_candidates_scaling.md`，`tests/test_lancedb_tenant_filter.py`（5 测试，含专门证伪"后置过滤"的用例 + 旧 schema 兼容） | | 2026-08-12 | `cross_instance_portability` | Connectors / Memory facade / Service import 路由 / HTTP `/v1/import` | 导出无法导回（POST 导出 JSON 得到 500）——补上原生 `engram` 导入格式：事实保留原 id/双时间戳/supersedes 链/provenance，目标端用本地 embedder 重嵌入（即官方换 embedder 迁移路径），按 id 幂等；`/v1/import` 对坏 payload 返回 400 | `tests/test_cross_account_portability.py`, `tests/test_server_import_export.py`（工程验收，非算法实验） | | 2026-08-12 | `mcp_http_bearer_gate` | MCP streamable-HTTP 传输边界 | MCP HTTP 模式此前无任何鉴权，仅靠默认 127.0.0.1；新增 `--http-token`/`ENGRAM_MCP_HTTP_TOKEN` Bearer 门，非回环绑定无 token 时启动即拒绝（失败关闭，与 REST 的 `ENGRAM_API_KEYS` 同哲学） | `tests/test_mcp_http_auth.py` | diff --git a/engram/retrieve/hybrid.py b/engram/retrieve/hybrid.py index cd0d470..7dadf77 100644 --- a/engram/retrieve/hybrid.py +++ b/engram/retrieve/hybrid.py @@ -63,10 +63,38 @@ "the", "and", "for", "with", "from", "inc", "ltd", "llc", "corp", "co", "company", "project", "user", "assistant", "team", "group", "system", "ai", }) +# (cue, how many words may sit between the cue and the entity name). Both exclusion regexes are built +# from this one list so the cheap pre-test below can never drift out of sync with the real matcher. +_EXCLUSION_CUES: tuple[tuple[str, int], ...] = ( + (r"\bnot\b", 5), + (r"\bexcept\b", 4), + (r"\bexcluding\b", 4), + (r"\bexclude\b", 4), + (r"\bother\s+than\b", 4), + (r"\brather\s+than\b", 4), + (r"\bbesides\b", 4), + ("不是", 0), + ("不在", 0), + ("排除", 0), + ("除了", 0), +) _EXCLUSION_BEFORE_RE = re.compile( - r"(?:\bnot\b(?:\s+\w+){0,5}|\bexcept\b(?:\s+\w+){0,4}|\bexcluding\b(?:\s+\w+){0,4}|" - r"\bexclude\b(?:\s+\w+){0,4}|\bother\s+than\b(?:\s+\w+){0,4}|" - r"\brather\s+than\b(?:\s+\w+){0,4}|\bbesides\b(?:\s+\w+){0,4}|不是|不在|排除|除了)\s*$", + "(?:" + + "|".join(cue + (rf"(?:\s+\w+){{0,{gap}}}" if gap else "") for cue, gap in _EXCLUSION_CUES) + + r")\s*$", + re.IGNORECASE, +) +# A necessary condition for _EXCLUSION_BEFORE_RE to match any substring of the query: the query must +# contain at least one cue somewhere. _EXCLUSION_BEFORE_RE is anchored to the end of the text preceding +# an entity mention, so it cannot be run against the whole query directly — but if no cue appears at all, +# no slice of the query can contain one either, and the entity scan can be skipped outright. +# +# The trailing \b is dropped deliberately, making this test weaker than the real matcher. It has to be: +# the slice ends where an entity name begins, and a non-ASCII name carries no boundary guard, so in +# "not上海" the slice "not" ends on a word boundary that does not exist in the full string. Over-matching +# only costs a scan that finds nothing; under-matching would silently drop an exclusion. +_EXCLUSION_CUE_RE = re.compile( + "|".join(cue[:-2] if cue.endswith(r"\b") else cue for cue, _ in _EXCLUSION_CUES), re.IGNORECASE, ) _EXCLUSION_VALUE_PREDS = frozenset({ @@ -197,6 +225,11 @@ def graph_excluded_entity_ids(self, query: str, user_id: str) -> set[str]: if not (self.config.graph_proximity and self.config.graph_negative_constraints): return set() query_l = query.lower() + # Most queries carry no negation at all. Checking the query once is exactly equivalent to + # checking every entity name against it (see _EXCLUSION_CUE_RE) and skips a scan of the whole + # entity set — which every retrieval paid, since query_entity_ids() ends by calling this. + if not _EXCLUSION_CUE_RE.search(query_l): + return set() direct: set[str] = set() entities = [ent for ent in self.graph.entities.values() if ent.user_id == user_id] for ent in entities: diff --git a/engram/store/lancedb_store.py b/engram/store/lancedb_store.py index 76c0268..4e2626e 100644 --- a/engram/store/lancedb_store.py +++ b/engram/store/lancedb_store.py @@ -148,11 +148,13 @@ def get(self, key: str) -> Any | None: table = self._open() if table is None: return None - rows = table.to_arrow().to_pylist() - for row in rows: - if row.get("key") == key: - return _decode_payload(row["payload"]) - return None + # A filter-only query (no vector) so the key predicate runs inside LanceDB. Materialising the + # table and scanning it in Python made a single-key read cost the whole store, which turns any + # id-at-a-time access pattern quadratic. + rows = table.search().where(f"key = {_quote(key)}").limit(1).to_list() + if not rows: + return None + return _decode_payload(rows[0]["payload"]) def delete(self, key: str) -> None: table = self._open() diff --git a/results/bounded_candidates_scaling.md b/results/bounded_candidates_scaling.md index eec865e..a4cad9c 100644 --- a/results/bounded_candidates_scaling.md +++ b/results/bounded_candidates_scaling.md @@ -64,6 +64,41 @@ top_k 里一条目标租户的行都没有，返回空。该测试通过，说 向后兼容：旧版本写出的表没有 `user_id` 列。`_has_tenant_column()` 按真实 schema 探测而非假设， 旧表继续可读可写（退回扫描），不会因 schema 不匹配而报错或损坏数据。 +## 追加：否定约束的提前返回 + 按键读取下推（同日） + +**否定约束提前返回。** 每次检索都会走 `query_entity_ids()`，它末尾调用 +`graph_excluded_entity_ids()`，后者对**每个实体名**跑两遍正则。而绝大多数查询根本没有否定词。 + +`_EXCLUSION_BEFORE_RE` 用 `\s*$` 锚定在"实体名之前那段文本"的末尾，不能直接拿来搜整个查询； +但"查询中至少出现一个提示词"是它匹配的必要条件，据此可以提前返回。 + +5000 个实体时实测： + +| 查询 | 耗时 | +| --- | ---: | +| 无否定词（`where does alice work`） | **0.0009 ms** | +| 含否定词（`anywhere except entity number 12`） | 508.94 ms | + +**一个被测试抓到的真实缺陷。** 提示词正则最初写成 `\bnot\b`（与真正的匹配器一致），这是错的： +`before` 切片终止于实体名起始处，而非 ASCII 实体名在 `_entity_name_mentions` 里**不带词边界守卫**， +所以 `"not上海"` 的切片 `"not"` 末尾构成词边界，而完整串里 `"not上"` 两侧都是 `\w`、提示词正则 +匹配不到——会静默丢掉一次排除。提示词正则因此去掉尾部 `\b`，故意比真正的匹配器更宽松：多扫一次 +只是白跑，漏扫则是错误。`tests/test_exclusion_shortcut.py` 对样本的**每一个前缀**验证 +"锚定匹配器命中 ⟹ 提示词匹配器命中"这一不变量，未来若只给其中一个正则加提示词会立即失败。 + +**按键读取下推。** `LanceDBVectorStore.get()` 此前全表物化后在 Python 里线性找 key，使任何 +逐 id 访问变成二次复杂度。改用纯过滤查询 `table.search().where("key = ...")`，谓词在 LanceDB +内部执行。（`pylance` 未安装、`table.query()` 在 0.33 不存在，实测确认 `search().where()` 可用。） + +## 仍然存在的热点 + +- `graph_excluded_entity_ids()` 在**确实含否定词**时仍是 O(实体数)，5000 实体需 509ms。提前返回 + 只是让它不再影响绝大多数查询，没有解决这条路径本身。它按子串匹配（中文实体名必需），无法用 + 词元倒排索引绕开。 +- `query_entity_ids()` 的名称匹配与别名锚定仍扫全部实体。这两处是词元化的，**可以**建倒排索引。 +- `InMemoryVectorStore.search()` 仍是暴力 cosine——参考实现设计如此，模块文档已写明适用到 ~10k。 +- `values()` 按接口语义必须返回全部，无法下推。 + ## 未采用/暂缓原因 - `candidate_vector_channel=False` **不作为默认**：它会丢掉"语义相关但与查询无共享词"的事实， diff --git a/tests/test_exclusion_shortcut.py b/tests/test_exclusion_shortcut.py new file mode 100644 index 0000000..8bcfe7a --- /dev/null +++ b/tests/test_exclusion_shortcut.py @@ -0,0 +1,96 @@ +"""The negation early-out in graph_excluded_entity_ids. + +Every retrieval calls query_entity_ids(), which ends by calling graph_excluded_entity_ids(), which used +to scan every entity in the store. Most queries contain no negation at all, so that scan almost always +found nothing. The early-out skips it — but only because of one invariant, which the first test here +pins down: if the anchored matcher can fire on some slice of a text, the cheap cue matcher must fire on +the whole text. If a future edit adds a cue to one regex and not the other, that test fails rather than +silently dropping exclusions. +""" +from __future__ import annotations + +from engram.config import Config +from engram.embed.hashing import HashingEmbedder +from engram.retrieve.hybrid import ( + _EXCLUSION_BEFORE_RE, + _EXCLUSION_CUE_RE, + HybridRetriever, +) +from engram.store.memory_store import InMemoryGraphStore, InMemoryVectorStore +from engram.types import Entity, Fact +from engram.util import now + +# Strings chosen to exercise each cue, the word-gap forms, the Chinese cues, and near-misses. +SAMPLES = [ + "not lisbon", + "somewhere other than lisbon", + "anywhere except berlin", + "excluding berlin", + "exclude the berlin office", + "rather than berlin", + "besides berlin", + "cities not counting the two big ones berlin", + "不是上海", + "不在北京", + "排除广州", + "除了深圳", + "where does alice work", + "notable places she visited", + "the exception was minor", + "", + "berlin", + # A non-ASCII entity name carries no word-boundary guard, so the slice preceding it can end + # mid-word. This is the case that forced the cue test to drop its trailing \b. + "not上海", + "except北京", +] + + +def test_cue_matcher_is_a_necessary_condition_for_the_anchored_matcher(): + """The invariant the early-out depends on. Checked over every prefix, since the real matcher runs + against arbitrary slices of the query (the text preceding an entity mention).""" + for text in SAMPLES: + for end in range(len(text) + 1): + slice_ = text[:end] + if _EXCLUSION_BEFORE_RE.search(slice_): + assert _EXCLUSION_CUE_RE.search(text), ( + f"anchored matcher fired on {slice_!r} but the cue matcher misses {text!r}; " + "the early-out would drop this exclusion" + ) + + +def _retriever_with_entity(name: str) -> tuple[HybridRetriever, str]: + graph = InMemoryGraphStore() + ent = graph.upsert_entity(Entity(user_id="u1", name=name)) + embedder = HashingEmbedder() + store = InMemoryVectorStore() + f = Fact( + user_id="u1", subject="alice", predicate="lives_in", object=name, + text=f"alice lives in {name}", valid_at=now(), embedding=embedder.embed(name), + ) + store.upsert(f.id, f.embedding or [], f) + return HybridRetriever(store, graph, embedder, Config()), ent.id + + +def test_negated_entity_is_still_excluded(): + """The early-out must not weaken the feature it guards.""" + retriever, ent_id = _retriever_with_entity("lisbon") + assert ent_id in retriever.graph_excluded_entity_ids("somewhere not lisbon", "u1") + + +def test_chinese_negation_still_excluded(): + """Chinese cues have no word boundaries; the tokenizer cannot see them, so the cue test must not + depend on tokenization.""" + retriever, ent_id = _retriever_with_entity("上海") + assert ent_id in retriever.graph_excluded_entity_ids("不是上海", "u1") + + +def test_plain_query_excludes_nothing(): + retriever, _ent_id = _retriever_with_entity("lisbon") + assert retriever.graph_excluded_entity_ids("where does alice live", "u1") == set() + + +def test_entity_named_after_a_cue_word_is_not_self_excluded(): + """'not' appearing only as part of the entity's own name is not a negation of it.""" + retriever, ent_id = _retriever_with_entity("notion") + assert ent_id not in retriever.graph_excluded_entity_ids("does alice use notion", "u1") diff --git a/tests/test_lancedb_tenant_filter.py b/tests/test_lancedb_tenant_filter.py index adc2105..1844784 100644 --- a/tests/test_lancedb_tenant_filter.py +++ b/tests/test_lancedb_tenant_filter.py @@ -104,3 +104,18 @@ def test_legacy_table_without_tenant_column_still_works(tmp_path): hits = store.search([1.0, 0.0, 0.0], 10, user_id="alice") assert {p.id for _s, p in hits} == {legacy.id, fresh.id} + + +def test_get_by_key_is_pushed_down(tmp_path): + """Single-key reads must not materialise the table (that makes id-at-a-time access quadratic).""" + store = LanceDBVectorStore(str(tmp_path / "db"), "facts") + made = [_fact("alice", f"note {i}", [1.0, 0.01 * i, 0.0]) for i in range(5)] + for f in made: + store.upsert(f.id, f.embedding or [], f) + + assert store.get(made[3].id).id == made[3].id + assert store.get("no-such-key") is None + + store.delete(made[3].id) + assert store.get(made[3].id) is None + assert len(store.values()) == 4 From 225d56f4d48dc1ec300084a9017b4bf3fc7e91c4 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 18:13:01 +0800 Subject: [PATCH 06/25] docs: record how the uncommitted worktree work was preserved Five worktrees held work that existed in no branch at all, so deleting them would have destroyed it. It is now committed as-is on each worktree's own branch, with the triage recording what each one is worth against today's main. Preserved is not merged: all of it predates 100+ commits of main and needs re-checking before any of it is ported. Two items are not superseded and are worth acting on. segment.py describes a defect that is still live -- the cross-encoder reranks whole ~2000-token sessions at max_length=512, scoring only the first quarter of each one. And the read context is still assembled as a single flat string, so nothing in it can be reused by provider prompt-caching across a multi-turn session. The privacy hold is cleared: the demo corpus uses a pseudonym per the repo owner, and a scan for other identifying data found none. --- docs/branch-debt-triage.zh-CN.md | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/branch-debt-triage.zh-CN.md b/docs/branch-debt-triage.zh-CN.md index 73e55b3..fae1af9 100644 --- a/docs/branch-debt-triage.zh-CN.md +++ b/docs/branch-debt-triage.zh-CN.md @@ -62,12 +62,31 @@ | `cool-chebyshev-aa0347` | consolidate 三模块改动 72 行 + `tests/test_profile.py`(40) | | `nostalgic-clarke-dda32e` | 480 行 diff | -⚠️ **隐私**：`nostalgic-clarke-dda32e` 含 `examples/seed_zhangyuwei.py` 与 `examples/zhangyuwei_memory.md`， -疑似真实人名。`CONTRIBUTING.md` 禁止提交真实个人数据与姓名——这两个文件在确认为化名前不得提交。 - -⚠️ **不要盲目抢救**：这些均为 2026-06 的工作，主干此后新增了 `retrieve/evidence.py` 与 -`retrieve/aggregate.py`。`router.py` / `segment.py` / `recall_pipeline.py` 很可能已被覆盖， -需逐个判定"仍有价值 vs 已过时"。 +### 处置结果（2026-08-16） + +隐私前置条件已由仓库所有者确认：`zhangyuwei` 为**化名**；另对该语料扫描了邮箱、手机号、证件号与 +账号类标识，**未发现**其它可识别信息，满足 `CONTRIBUTING.md` 的提交门槛。 + +全部未提交工作已**按原样提交到各自分支保全**（每个 worktree 现为 main+1、工作区清空）。保全 ≠ 合并： +这些都是 2026-06 的代码，主干此后走了 100+ 个 commit，需逐个对照今日主干重新验证后才能移植。 + +| worktree | 保全提交 | 判定 | +| --- | --- | --- | +| `wonderful-tereshkova-b63377` | `c6912df` | **部分仍有效**。`segment.py` 记录的缺陷**今天仍在**：`retrieve/rerank.py` 的 cross-encoder 仍是 `max_length=512` 且无分段，重排 ~2000 token 的 session 会静默只对前 512 token 打分（实测 70.0%→57.5%）。主干 rerank 默认关闭，限制了影响面但没有修复。`router.py` 的按查询路由与优化地图上仍未关闭的 P2「runtime profiles」高度重合 | +| `vigilant-mahavira-73886f` | `21e7baa` | **未被覆盖**。把上下文拆成可缓存的 STABLE 块（进 system prompt）与每轮变化的 DYNAMIC 块，主干 `lean_context` 至今仍拍平成单个字符串。同样证据、更少重复计费——属 Bet A 三联表里 tokens/latency 那两维。无测试 | +| `sweet-haibt-7231c5` | `d3f4f70` | **真实缺口**。`engram/metrics.py`（99 行）+ 测试（119 行）+ `/metrics` 端点，主干至今没有可观测层。纯 stdlib、只暴露聚合量（不含命名空间名与查询文本） | +| `laughing-tesla-360ff1` | `103359c` | 730 行文档。主干此后自建了文档体系，重合度未知，且其中数字未按「每个公开数字可追溯到已提交日志」的规则复核过 | +| `cool-chebyshev-aa0347` | `7ee1123` | consolidate 三模块改动 + `tests/test_profile.py`。主干此后重写了大部分 consolidation，需重新对照 | +| `nostalgic-clarke-dda32e` | `2ccfe15` | 车载记忆演示语料 + 控制台接线。`memory.py`/`server/app.py`/前端在主干均已大改 | + +**未提交但未保全的**（判定为可再生或无价值，故意留在原处）： +`distracted-tereshkova` 的 `paper/main_anon.{pdf,bbl}`（匿名构建产物，可由已提交的开关重新生成）、 +`hardcore-nobel` 的 `frontend/package-lock.json`（前端与 CI 用 pnpm，该 npm 锁文件是残留）、 +`sharp-chandrasekhar` 与 `peaceful-almeida` 的 `TECHNIQUES.md`（与 `103359c` 中已保全的同名文档重复）。 + +**main 工作区的未跟踪文件未处理**：`IDENTITY.md` / `SOUL.md` / `USER.md` 和 11 个 +`results/*.jsonl`。后者是实验日志，按 Bet D「每个公开数字可追溯到已提交日志」的规则**可能应当提交**， +但提交到 main 属于仓库所有者的决定，未擅自执行。 ## 结论 From e43d64f89d21b49c38510ce0d4da6e270ee83db2 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 19:16:16 +0800 Subject: [PATCH 07/25] rerank: score long sessions by segment instead of by their opening quarter A cross-encoder reads a bounded window -- 512 tokens for the BGE rerankers -- and a LongMemEval session is around 2000. Handed a whole session it does not fail; it scores the first quarter and discards the rest, so a session whose answer sits late ranks as if it were irrelevant. The project already knew this. lean_context reranks only the fact pool and its comment says reranking whole sessions "truncates to 512 and mis-ranks -- a known _S regression". But retrieve_episodes() still passed whole ep.content to the reranker, and no test covered that path, so the documented defect stayed live in the one place it applied. Sessions are now split on paragraph and sentence boundaries into segments that fit the window, and each session scores as its best segment. Best, not mean: a long session earns retrieval because one passage answers the question, and averaging dilutes exactly the signal being looked for. Short candidates produce a single segment, so nothing changes for them. rerank_long takes any object with .rerank rather than a concrete reranker, so segmentation is testable without loading a cross-encoder -- the zero-setup invariant means the default test path cannot import sentence-transformers. The regression test asserts the precondition first (whole-document scoring picks the decoy) and then the fix, so it fails if the old behaviour ever returns. Measured accuracy is not claimed here: quantifying the recovery needs a keyed LongMemEval run with reranking enabled, which is real API spend. What is verified is that the truncation is gone and that the answer-bearing session survives reranking. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/config.py | 4 + engram/memory.py | 12 +- engram/retrieve/rerank.py | 113 +++++++++++++- tests/test_rerank_segments.py | 158 ++++++++++++++++++++ 5 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 tests/test_rerank_segments.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 0bafcbd..c361102 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `segment_level_rerank` | Read path / 会话重排（Bet A） | cross-encoder 只读 ~512 token，而 LongMemEval 的 session 约 2000 token。`retrieve_episodes()` 把整篇 `ep.content` 交给 reranker，模型不会报错，只会**静默地只对开头四分之一打分**——答案落在后半段的 session 因此被判为无关（该回归此前已在 `lean_context` 的注释里被记录为已知问题，但会话路径一直没修，且**完全没有测试覆盖**）。改为按段落/句子边界切成 ≤`rerank_segment_words`（默认 300 词）的片段分别打分，每篇取其**最佳片段**得分（不是平均——长会话之所以该被检出，正是因为其中某一段命中） | `tests/test_rerank_segments.py`（13 测试，含先断言"整篇打分必然选错"再断言"分段打分选对"的回归用例，以及此前无人覆盖的 `Memory.retrieve_episodes` 接线） | | 2026-08-16 | `exclusion_early_out` + `lance_key_pushdown` | Read path / 图排除区 + Store 按键读取 | 每次检索都经 `query_entity_ids()` → `graph_excluded_entity_ids()`，后者对每个实体名跑两遍正则，而多数查询无否定词。改为先用提示词必要条件判定，5000 实体时无否定词查询从 509ms 降到 0.0009ms。同时 `LanceDBVectorStore.get()` 改用纯过滤查询，不再全表物化后线性找 key | `results/bounded_candidates_scaling.md`，`tests/test_exclusion_shortcut.py`（含逐前缀验证的正则不变量，已抓到一个中文实体名下的静默漏排除缺陷），`tests/test_lancedb_tenant_filter.py` | | 2026-08-16 | `tenant_filter_pushdown` | Store / 向量检索索引层（Bet E） | 多租户检索每次都必须按 user 过滤，而唯一的表达方式是 Python 谓词——后端看不进去，只能先物化全表再排序。于是"接了 LanceDB"从未换来任何 ANN 收益。现把 `user_id` 从 JSON payload 提升为真实列，`VectorStore.search()` 增加声明式 `user_id=` 参数，LanceDB 走 `prefilter=True` 在索引内收窄。40000 行时 **165x**，且延迟基本随行数不变 | `results/bounded_candidates_scaling.md`，`tests/test_lancedb_tenant_filter.py`（5 测试，含专门证伪"后置过滤"的用例 + 旧 schema 兼容） | | 2026-08-12 | `cross_instance_portability` | Connectors / Memory facade / Service import 路由 / HTTP `/v1/import` | 导出无法导回（POST 导出 JSON 得到 500）——补上原生 `engram` 导入格式：事实保留原 id/双时间戳/supersedes 链/provenance，目标端用本地 embedder 重嵌入（即官方换 embedder 迁移路径），按 id 幂等；`/v1/import` 对坏 payload 返回 400 | `tests/test_cross_account_portability.py`, `tests/test_server_import_export.py`（工程验收，非算法实验） | diff --git a/engram/config.py b/engram/config.py index cb6c194..63f3e15 100644 --- a/engram/config.py +++ b/engram/config.py @@ -48,6 +48,10 @@ class Config: # missing facts that are semantically relevant but share no query term. Flip it off only with a real # ANN backend, or after a keyed harness run shows the recall loss is acceptable. candidate_vector_channel: bool = True + # Segment size for reranking long documents. Cross-encoders read a bounded window (512 tokens for the + # BGE rerankers) and truncate silently past it, so raw sessions are scored segment-by-segment and take + # their best segment's score. ~300 words sits inside that window once the query is added. + rerank_segment_words: int = 300 rrf_k: int = 60 # Reciprocal Rank Fusion constant max_hops: int = 2 # multi-hop planner depth max_hot_facts: int = 10_000 # heat-tier cap; cold facts remain durable and can page back on hot miss diff --git a/engram/memory.py b/engram/memory.py index df16c92..0002d41 100644 --- a/engram/memory.py +++ b/engram/memory.py @@ -32,6 +32,7 @@ render_aggregation_candidates, ) from .retrieve.lexical import bm25_scores, overlap_terms, stems +from .retrieve.rerank import rerank_long from .store import ( GraphStore, IndexedVectorStore, @@ -2436,7 +2437,16 @@ def retrieve_episodes( eps = [eps[i] for i in fused_order] if self.reranker is not None and len(eps) > k: - ranked = self.reranker.rerank(query, [(i, ep.content) for i, ep in enumerate(eps)], k) + # Segment-level, because a session is ~2000 tokens and the cross-encoder reads ~512: scoring + # whole sessions silently ranks each one on its opening quarter (the known _S regression + # noted in lean_context). Each session scores as its best segment. + ranked = rerank_long( + self.reranker, + query, + [(i, ep.content) for i, ep in enumerate(eps)], + k, + max_words=self.config.rerank_segment_words, + ) return [eps[i] for i, _ in ranked] return eps[:k] diff --git a/engram/retrieve/rerank.py b/engram/retrieve/rerank.py index dd298f8..a43afad 100644 --- a/engram/retrieve/rerank.py +++ b/engram/retrieve/rerank.py @@ -1,9 +1,118 @@ """Cross-encoder reranker (CLAUDE.md L1 strong retrieval). A bi-encoder (BGE) gives a cheap candidate pool; a cross-encoder rescores (query, passage) jointly for far better precision on which sessions/chunks -actually answer the question. This is the highest-leverage retrieval upgrade for LongMemEval _S.""" +actually answer the question. This is the highest-leverage retrieval upgrade for LongMemEval _S. + +A cross-encoder reads a bounded window — 512 tokens for the BGE rerankers. Hand it a whole LongMemEval +session (~2000 tokens) and it does not fail; it silently scores the first quarter and discards the rest, +so a session whose answer sits in its second half ranks as if it were irrelevant. `rerank_long` scores at +a granularity the model can actually read and keeps each document's best segment, which is what makes +reranking safe to apply to raw sessions rather than only to short facts. +""" from __future__ import annotations -from typing import Optional +import re +from typing import Any, Optional + +__all__ = ["CrossEncoderReranker", "segment_text", "rerank_long"] + +# Paragraph first, then sentence: splitting mid-sentence would hand the model a fragment whose meaning +# depends on text it cannot see, which is the same failure as truncation, just smaller. +_PARA_RE = re.compile(r"\n\s*\n") +_SENT_RE = re.compile(r"(?<=[.!?。！？])\s+") + + +def segment_text(text: str, max_words: int = 300) -> list[str]: + """Split `text` into segments of at most `max_words` words, preferring natural boundaries. + + Word count is a proxy for the model's token budget: ~300 words is comfortably inside a 512-token + window once the query and special tokens are added. Text already short enough comes back as a single + segment, so short candidates behave exactly as they did before. + """ + if not text or not text.strip(): + return [] + words = text.split() + if len(words) <= max_words: + return [text.strip()] + + segments: list[str] = [] + for block in _split_units(text): + block_words = block.split() + if not block_words: + continue + if len(block_words) <= max_words: + _append_or_merge(segments, block, max_words) + continue + # A single unit longer than the budget (an unpunctuated wall of text): cut on word count. Better + # a hard cut than handing the model something it will truncate invisibly. + for start in range(0, len(block_words), max_words): + segments.append(" ".join(block_words[start:start + max_words])) + return segments or [" ".join(words[:max_words])] + + +def _split_units(text: str) -> list[str]: + units: list[str] = [] + for para in _PARA_RE.split(text): + para = para.strip() + if not para: + continue + units.extend(s for s in (part.strip() for part in _SENT_RE.split(para)) if s) + return units + + +def _append_or_merge(segments: list[str], unit: str, max_words: int) -> None: + """Pack consecutive units together while they fit, so segments are as informative as the budget + allows instead of one-sentence slivers.""" + if segments and len(segments[-1].split()) + len(unit.split()) <= max_words: + segments[-1] = f"{segments[-1]} {unit}" + else: + segments.append(unit) + + +def rerank_long( + reranker: Any, + query: str, + candidates: list[tuple[Any, str]], + top_k: int, + max_words: int = 300, +) -> list[tuple[Any, float]]: + """Rerank documents that may exceed the model's window, by scoring their segments. + + A document scores as its BEST segment, not its average: a long session earns its place because one + passage answers the question, and averaging would dilute exactly the signal being looked for. + + Takes any object with a `.rerank(query, [(id, text)], top_k)` method rather than a concrete type, so + the segmentation is testable without loading a cross-encoder — the zero-setup invariant means the + default test path cannot import sentence-transformers. + """ + if not candidates: + return [] + + pieces: list[tuple[str, str]] = [] + owner: dict[str, Any] = {} + order: dict[Any, int] = {} + for position, (cid, text) in enumerate(candidates): + order.setdefault(cid, position) + for seg_index, segment in enumerate(segment_text(text, max_words)): + key = f"{position}:{seg_index}" + owner[key] = cid + pieces.append((key, segment)) + if not pieces: + return [] + + # Score every segment: the reranker truncates its own return value to top_k, and a document's best + # segment can sit anywhere in that list. + scored = reranker.rerank(query, pieces, len(pieces)) + + best: dict[Any, float] = {} + for key, score in scored: + cid = owner.get(key) + if cid is None: + continue + if cid not in best or score > best[cid]: + best[cid] = score + # Ties fall back to the incoming order, so the caller's upstream ranking still decides. + ranked = sorted(best.items(), key=lambda item: (-item[1], order.get(item[0], 0))) + return ranked[:top_k] class CrossEncoderReranker: diff --git a/tests/test_rerank_segments.py b/tests/test_rerank_segments.py new file mode 100644 index 0000000..2f2f564 --- /dev/null +++ b/tests/test_rerank_segments.py @@ -0,0 +1,158 @@ +"""Segment-level reranking of long documents. + +A cross-encoder reads ~512 tokens. Given a whole ~2000-token session it does not error -- it scores the +first quarter and silently ignores the rest, so a session whose answer sits late ranks as irrelevant. +`test_answer_late_in_a_long_document_still_ranks_first` reproduces exactly that and would fail against the +old whole-document path. + +No cross-encoder is loaded here: the reranker is a stub that scores by query-term overlap, which is enough +to exercise every decision `rerank_long` makes and keeps the suite runnable with no heavy dependency. +""" +from __future__ import annotations + +from engram.retrieve.rerank import rerank_long, segment_text + + +class WindowedReranker: + """A stand-in cross-encoder with a hard reading window, like the real one. + + `window` words are read and the rest is dropped, reproducing the truncation that makes whole-document + reranking wrong. Score is query-term overlap over the part it can see. + """ + + def __init__(self, window: int = 400) -> None: + self.window = window + self.seen: list[str] = [] + + def rerank(self, query, candidates, top_k): + terms = set(query.lower().split()) + scored = [] + for cid, text in candidates: + self.seen.append(text) + visible = " ".join(text.split()[: self.window]).lower() + scored.append((cid, float(sum(visible.count(t) for t in terms)))) + scored.sort(key=lambda item: item[1], reverse=True) + return scored[:top_k] + + +def _doc(filler_words: int, needle: str = "", needle_at: str = "end") -> str: + filler = " ".join(f"w{i}" for i in range(filler_words)) + if not needle: + return filler + return f"{needle} {filler}" if needle_at == "start" else f"{filler} {needle}" + + +# --- segment_text --- + + +def test_short_text_is_one_segment(): + """Documents already inside the window must be untouched, so short candidates behave as before.""" + assert segment_text("alice works at acme", max_words=300) == ["alice works at acme"] + + +def test_empty_text_yields_no_segments(): + assert segment_text("", 300) == [] + assert segment_text(" \n ", 300) == [] + + +def test_long_text_is_split_within_budget(): + segments = segment_text(_doc(1000), max_words=300) + assert len(segments) > 1 + assert all(len(s.split()) <= 300 for s in segments) + + +def test_split_preserves_every_word(): + """Segmentation must not drop content -- that would be the truncation bug in another costume.""" + text = _doc(1000, needle="the answer is lisbon") + assert " ".join(segment_text(text, 300)).split() == text.split() + + +def test_unpunctuated_wall_of_text_is_still_bounded(): + """A single sentence longer than the budget has no natural boundary; it must still be cut.""" + segments = segment_text(" ".join(f"w{i}" for i in range(900)), max_words=300) + assert all(len(s.split()) <= 300 for s in segments) + + +def test_sentences_are_packed_not_slivered(): + """Short sentences should be merged up to the budget, not emitted one per segment.""" + text = " ".join(f"Sentence number {i} here." for i in range(60)) + segments = segment_text(text, max_words=100) + assert len(segments) < 60 + assert all(len(s.split()) <= 100 for s in segments) + + +# --- rerank_long --- + + +def test_answer_late_in_a_long_document_still_ranks_first(): + """The regression this exists to prevent. + + The answer-bearing document hides its match past the model's window; the decoy repeats an unrelated + filler term early. Scoring whole documents reads only the opening of each, so the decoy wins. + """ + query = "lisbon" + answer_doc = _doc(900, needle="lisbon lisbon lisbon", needle_at="end") + decoy_doc = _doc(900) + + reranker = WindowedReranker(window=400) + whole = reranker.rerank(query, [("decoy", decoy_doc), ("answer", answer_doc)], 2) + assert whole[0][0] == "decoy", "precondition: whole-document scoring must miss the late answer" + + segmented = rerank_long(reranker, query, [("decoy", decoy_doc), ("answer", answer_doc)], 2) + assert segmented[0][0] == "answer" + + +def test_every_segment_fits_the_reading_window(): + reranker = WindowedReranker(window=400) + rerank_long(reranker, "lisbon", [("a", _doc(2000))], 1, max_words=300) + assert reranker.seen, "the reranker should have been called" + assert all(len(text.split()) <= 300 for text in reranker.seen) + + +def test_document_scores_as_its_best_segment_not_its_average(): + """One strong passage in a long document must outrank a uniformly mediocre one.""" + query = "lisbon" + spike = _doc(600, needle="lisbon lisbon lisbon lisbon", needle_at="end") + diffuse = " ".join(["lisbon"] + [f"w{i}" for i in range(600)]) + + ranked = rerank_long(WindowedReranker(2000), query, [("diffuse", diffuse), ("spike", spike)], 2) + assert ranked[0][0] == "spike" + + +def test_empty_and_blank_candidates_are_handled(): + assert rerank_long(WindowedReranker(), "q", [], 5) == [] + assert rerank_long(WindowedReranker(), "q", [("a", " ")], 5) == [] + + +def test_top_k_is_respected_and_ties_keep_incoming_order(): + """With nothing to separate them, the upstream ranking decides -- reranking must not shuffle.""" + docs = [(f"d{i}", "neutral text with no query terms") for i in range(5)] + ranked = rerank_long(WindowedReranker(), "lisbon", docs, 3) + assert [cid for cid, _ in ranked] == ["d0", "d1", "d2"] + + +def test_ids_survive_round_trip(): + """Segment keys are internal; callers must get their own ids back (memory.py passes list indices).""" + ranked = rerank_long(WindowedReranker(), "lisbon", [(7, _doc(800, needle="lisbon"))], 1) + assert ranked[0][0] == 7 + + +# --- wiring --- + + +def test_memory_reranks_sessions_by_segment(): + """The integration point where the defect actually lived. + + Memory.retrieve_episodes passed whole `ep.content` to the reranker, and nothing covered that path. + A session whose answer sits past the reading window must still be retrieved. + """ + from engram.memory import Memory + + mem = Memory(reranker=WindowedReranker(window=400)) + answer = _doc(900, needle="the conference was in lisbon", needle_at="end") + for i in range(4): + mem.add(_doc(900) if i else answer, user_id="u1", session_id=f"s{i}") + + eps = mem.retrieve_episodes("lisbon", "u1", k=1) + assert eps, "reranked retrieval must return something" + assert "lisbon" in eps[0].content, "the session answering the query must survive reranking" From 6689523e3a5bbeb8c9593e973be34b0b135fb808 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 19:33:16 +0800 Subject: [PATCH 08/25] graph: index entity name terms so anchoring costs the query, not the store query_entity_ids() runs on every retrieval and walked every entity in the store to work out which ones the query names. InMemoryGraphStore now keeps a (user_id, stemmed term) -> entity ids index, so only the query's own terms are looked up. With distinctive entity names -- people, places, companies, which is what a real graph holds -- the cost stops tracking the store entirely: 0.004ms whether there are 500 entities or 10,000, against 16.05ms for the scan at 10k. The limit is worth stating plainly, because the first benchmark walked straight into it. Synthetic names of the form "entity number {i}" share two tokens across every entity, so those posting lists are as long as the store and the index degenerates to a scan with a lookup in front of it -- 1.4x, still linear. That measurement was the benchmark's fault, not the index's, and it is written up in results/ so nobody repeats the data and concludes the index does not work. The alias-anchor filter (length, digits, stop words) tests the term rather than the entity holding it, which is what lets the same decision come from a term index as from the full walk; it is now a named predicate used by both paths instead of being inlined in one. Equivalence is tested against the real retriever: the same entities in a store with the index and in one with the lookup removed must anchor identically. Graph backends that do not implement the lookup keep the scan, since that is all the GraphStore interface guarantees. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/retrieve/hybrid.py | 50 +++++++-- engram/store/memory_store.py | 42 +++++++- results/entity_anchor_index.md | 86 +++++++++++++++ tests/test_entity_index.py | 112 ++++++++++++++++++++ 5 files changed, 282 insertions(+), 9 deletions(-) create mode 100644 results/entity_anchor_index.md create mode 100644 tests/test_entity_index.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index c361102..060eff3 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `entity_anchor_index` | Read path / 图锚定 + Graph store 索引层（Bet E） | `query_entity_ids()` 每次检索遍历全部实体做名称/别名匹配。`InMemoryGraphStore` 改为维护 `(user_id, 词干) -> 实体 id` 倒排索引，只查询查询自身的词。实体名有区分度时 10000 实体下 **4055x**，且耗时恒定 0.004ms 与库大小无关。**边界**：实体名共享高频词时倒排表长度等于全库、索引退化为扫描（仅 1.4x），真实人名/地名/机构名不属于该情形 | `results/entity_anchor_index.md`，`tests/test_entity_index.py`（7 测试；等价性用真实 retriever 对比"带索引"与"去掉索引查找"两条路径，要求结果完全一致） | | 2026-08-16 | `segment_level_rerank` | Read path / 会话重排（Bet A） | cross-encoder 只读 ~512 token，而 LongMemEval 的 session 约 2000 token。`retrieve_episodes()` 把整篇 `ep.content` 交给 reranker，模型不会报错，只会**静默地只对开头四分之一打分**——答案落在后半段的 session 因此被判为无关（该回归此前已在 `lean_context` 的注释里被记录为已知问题，但会话路径一直没修，且**完全没有测试覆盖**）。改为按段落/句子边界切成 ≤`rerank_segment_words`（默认 300 词）的片段分别打分，每篇取其**最佳片段**得分（不是平均——长会话之所以该被检出，正是因为其中某一段命中） | `tests/test_rerank_segments.py`（13 测试，含先断言"整篇打分必然选错"再断言"分段打分选对"的回归用例，以及此前无人覆盖的 `Memory.retrieve_episodes` 接线） | | 2026-08-16 | `exclusion_early_out` + `lance_key_pushdown` | Read path / 图排除区 + Store 按键读取 | 每次检索都经 `query_entity_ids()` → `graph_excluded_entity_ids()`，后者对每个实体名跑两遍正则，而多数查询无否定词。改为先用提示词必要条件判定，5000 实体时无否定词查询从 509ms 降到 0.0009ms。同时 `LanceDBVectorStore.get()` 改用纯过滤查询，不再全表物化后线性找 key | `results/bounded_candidates_scaling.md`，`tests/test_exclusion_shortcut.py`（含逐前缀验证的正则不变量，已抓到一个中文实体名下的静默漏排除缺陷），`tests/test_lancedb_tenant_filter.py` | | 2026-08-16 | `tenant_filter_pushdown` | Store / 向量检索索引层（Bet E） | 多租户检索每次都必须按 user 过滤，而唯一的表达方式是 Python 谓词——后端看不进去，只能先物化全表再排序。于是"接了 LanceDB"从未换来任何 ANN 收益。现把 `user_id` 从 JSON payload 提升为真实列，`VectorStore.search()` 增加声明式 `user_id=` 参数，LanceDB 走 `prefilter=True` 在索引内收窄。40000 行时 **165x**，且延迟基本随行数不变 | `results/bounded_candidates_scaling.md`，`tests/test_lancedb_tenant_filter.py`（5 测试，含专门证伪"后置过滤"的用例 + 旧 schema 兼容） | diff --git a/engram/retrieve/hybrid.py b/engram/retrieve/hybrid.py index 7dadf77..18fbf12 100644 --- a/engram/retrieve/hybrid.py +++ b/engram/retrieve/hybrid.py @@ -9,7 +9,7 @@ from ..consolidate.conflict import is_single_valued from ..embed import Embedder from ..store import GraphStore, VectorStore -from ..types import Fact +from ..types import Entity, Fact from ..util import cosine, indexed_text, now, recency, tokenize from ..util import date_terms as date_terms # noqa: F401 re-export: callers import it from here from .fusion import order_by_score, weighted_rrf @@ -149,12 +149,21 @@ def graph_relation_relevance(query: str, fact: Fact) -> float: return 0.0 +def _is_anchor_term(term: str) -> bool: + """Whether a stemmed term is distinctive enough to anchor a query to a single entity. + + A property of the term alone, never of the entity holding it — which is what lets the same decision be + made from an index keyed on terms as from a walk over every entity. + """ + return len(term) >= 3 and not term.isdigit() and term not in _ENTITY_ANCHOR_STOP + + def _entity_anchor_terms(name: str, aliases: list[str]) -> set[str]: terms: set[str] = set() for text in (name, *aliases): for tok in tokenize(text): term = stem(tok) - if len(term) >= 3 and not term.isdigit() and term not in _ENTITY_ANCHOR_STOP: + if _is_anchor_term(term): terms.add(term) return terms @@ -189,20 +198,47 @@ def __init__(self, fact_store: VectorStore, graph: GraphStore, embedder: Embedde from ..embed import HashingEmbedder self._semantic = not isinstance(embedder, HashingEmbedder) + def _anchor_scope(self, q: set[str], user_id: str) -> tuple[list[Entity], Optional[dict[str, set[str]]]]: + """Entities worth testing against the query, and per-term owner sets when the backend indexes them. + + An entity can only anchor a query it shares a term with, so walking the whole store to find that + out is wasted work. When the graph can look terms up, both the candidate list and the alias-anchor + uniqueness counts come straight from the query's own terms. Backends without the lookup fall back + to the full scan, which is what the GraphStore interface actually guarantees. + """ + lookup = getattr(self.graph, "entities_by_terms", None) + if lookup is None: + return [ent for ent in self.graph.entities.values() if ent.user_id == user_id], None + hits = lookup(user_id, q) + by_term = {term: {ent.id for ent in ents} for term, ents in hits.items()} + seen: dict[str, Entity] = {} + for ents in hits.values(): + for ent in ents: + seen[ent.id] = ent + return list(seen.values()), by_term + def query_entity_ids(self, query: str, user_id: str) -> set[str]: """Entity nodes whose full name appears in the query (the query's anchor entities).""" q = set(stems(query)) | set(tokenize(query)) ids: set[str] = set() - entities = [ent for ent in self.graph.entities.values() if ent.user_id == user_id] + entities, indexed_terms = self._anchor_scope(q, user_id) for ent in entities: names = (ent.name, *ent.aliases) if any((toks := [stem(t) for t in tokenize(name)]) and all(t in q for t in toks) for name in names): ids.add(ent.id) if self.config.graph_entity_alias_anchor: - term_to_ids: dict[str, set[str]] = {} - for ent in entities: - for term in _entity_anchor_terms(ent.name, ent.aliases): - term_to_ids.setdefault(term, set()).add(ent.id) + if indexed_terms is None: + term_to_ids: dict[str, set[str]] = {} + for ent in entities: + for term in _entity_anchor_terms(ent.name, ent.aliases): + term_to_ids.setdefault(term, set()).add(ent.id) + else: + # The index keys on every name/alias term; the anchor filter tests the term, not the + # entity (see _is_anchor_term), so applying it to the query's terms selects exactly the + # owner sets the full build would have produced. + term_to_ids = { + term: owners for term, owners in indexed_terms.items() if _is_anchor_term(term) + } for term in q: hits = term_to_ids.get(term) if hits is not None and len(hits) == 1: diff --git a/engram/store/memory_store.py b/engram/store/memory_store.py index 187aa2f..f5bd9bf 100644 --- a/engram/store/memory_store.py +++ b/engram/store/memory_store.py @@ -3,10 +3,10 @@ from __future__ import annotations from collections import defaultdict -from typing import Any, Optional +from typing import Any, Iterable, Optional from ..types import Entity, Relation -from ..util import cosine +from ..util import cosine, stem, tokenize from .base import DocStore, GraphStore, Predicate, VectorStore @@ -64,10 +64,25 @@ def delete(self, key: str) -> None: self._d.pop(key, None) +def _name_terms(entity: Entity) -> set[str]: + """Stemmed tokens of an entity's name and aliases — the keys it can be looked up by.""" + terms: set[str] = set() + for text in (entity.name, *entity.aliases): + for token in tokenize(text): + terms.add(stem(token)) + return terms + + class InMemoryGraphStore(GraphStore): def __init__(self) -> None: self.entities: dict[str, Entity] = {} self._by_name: dict[tuple[str, str], str] = {} + # (user_id, stemmed term) -> entity ids. Anchoring a query to its entities otherwise means + # walking every entity in the store on every retrieval; this turns it into a lookup of the + # query's own terms. Built at upsert: an entity's name and aliases are fixed once inserted + # (upsert_entity returns the existing node rather than updating it), so there is nothing to + # invalidate. A backend that lets names change would need to re-index on that change. + self._by_term: dict[tuple[str, str], set[str]] = {} self.rels: dict[str, Relation] = {} self._out: dict[str, list[str]] = defaultdict(list) self._in: dict[str, list[str]] = defaultdict(list) @@ -79,8 +94,24 @@ def upsert_entity(self, entity: Entity) -> Entity: return self.entities[existing_id] self.entities[entity.id] = entity self._by_name[key] = entity.id + for term in _name_terms(entity): + self._by_term.setdefault((entity.user_id, term), set()).add(entity.id) return entity + def entities_by_terms(self, user_id: str, terms: Iterable[str]) -> dict[str, list[Entity]]: + """For each requested term, this user's entities whose name or aliases contain it. + + Serves both halves of query anchoring: the union across terms is the candidate set to test + properly, and a single term's list size is the uniqueness signal an alias anchor needs. Only the + query's terms are looked up, so the cost follows the query rather than the store. + """ + found: dict[str, list[Entity]] = {} + for term in terms: + ids = self._by_term.get((user_id, term)) + if ids: + found[term] = [self.entities[eid] for eid in ids if eid in self.entities] + return found + def get_entity(self, user_id: str, name: str) -> Entity | None: eid = self._by_name.get((user_id, name.lower())) return self.entities.get(eid) if eid else None @@ -127,6 +158,13 @@ def prune_orphan_entities(self) -> int: continue self.entities.pop(eid, None) self._by_name.pop((ent.user_id, ent.name.lower()), None) + for term in _name_terms(ent): + key = (ent.user_id, term) + ids = self._by_term.get(key) + if ids is not None: + ids.discard(eid) + if not ids: + del self._by_term[key] self._out.pop(eid, None) self._in.pop(eid, None) removed += 1 diff --git a/results/entity_anchor_index.md b/results/entity_anchor_index.md new file mode 100644 index 0000000..fe3ffe2 --- /dev/null +++ b/results/entity_anchor_index.md @@ -0,0 +1,86 @@ +# entity_anchor_index — 实体锚定索引的扩展性测量（2026-08-16） + +## 背景 + +`HybridRetriever.query_entity_ids()` 是每次检索都会走的路径，用于判断查询提到了哪些实体 +（图扩展的锚点）。它此前遍历 `graph.entities.values()` 全部实体，对每个实体做名称/别名匹配。 + +`InMemoryGraphStore` 现在维护 `(user_id, 词干) -> 实体 id` 的倒排索引，检索时只查**查询自身的词**。 + +## 复现 + +```bash +python3 - <<'PY' +# 见本文件末尾脚本；或直接用 tests/test_entity_index.py 验证等价性 +PY +``` + +等价性由 `tests/test_entity_index.py::test_indexed_and_scanned_anchoring_agree` 保证：同一组实体 +分别装入带索引和去掉索引查找方法的图存储，用**真实的 HybridRetriever** 跑同一批查询，两条路径 +必须给出完全相同的锚定结果。索引只允许更快，不允许更不一样。 + +## 结果 + +**有区分度的实体名（真实场景：人名、地名、公司名）** + +| 实体数 | 全扫描 | 索引 | 加速 | +| ---: | ---: | ---: | ---: | +| 500 | 0.735ms | 0.004ms | 183.7x | +| 2000 | 3.027ms | 0.004ms | 756.7x | +| 10000 | 16.052ms | **0.004ms** | **4055.6x** | + +索引耗时**与实体数无关**——成本随查询词数走，不随库大小走。这是该路径第一次真正脱离 O(n)。 + +**共享高频词的实体名（对抗场景：全部实体都叫 `entity number {i}`）** + +| 实体数 | 全扫描 | 索引 | 加速 | +| ---: | ---: | ---: | ---: | +| 500 | 0.810ms | 0.492ms | 1.6x | +| 2000 | 3.351ms | 2.102ms | 1.6x | +| 10000 | 17.029ms | 12.041ms | 1.4x | + +## 结论与边界（重要） + +**索引的收益取决于实体名的区分度，不是无条件的。** 当所有实体名共享同一批高频词时，这些词的 +倒排表长度等于全库，候选集就是全部实体，索引退化为"全扫描 + 一次索引查找"，只剩常数因子收益。 + +真实实体名（人名、地名、机构名、产品名）几乎总是有区分度的，所以真实场景是主导情形。但这个边界 +必须写明：这不是一个"任何情况下都 O(1)"的优化，而是一个"在词分布正常时 O(查询词数)"的优化。 + +第一版基准曾用 `entity number {i}` 作为合成实体名，测出只有 1.5x 且仍然线性——那是基准数据落进了 +退化情形，不是实现有问题。记在这里，避免以后有人用同样的合成数据重新得出"这个索引没用"的错误结论。 + +## 仍然线性的部分 + +`graph_excluded_entity_ids()` 在查询**确实含否定词**时仍遍历全部实体。它按子串匹配（中文实体名 +必需，词元化看不见），无法用词元倒排索引替代。参见 `results/bounded_candidates_scaling.md`。 + +## 测量脚本 + +```python +import time, statistics +from engram.config import Config +from engram.embed.hashing import HashingEmbedder +from engram.retrieve.hybrid import HybridRetriever +from engram.store.memory_store import InMemoryGraphStore, InMemoryVectorStore +from engram.types import Entity + +class Unindexed(InMemoryGraphStore): + entities_by_terms = None # forces the retriever's full-scan fallback + +def bench(cls, n, namer, query): + g = cls() + for i in range(n): + g.upsert_entity(Entity(user_id="u1", name=namer(i))) + r = HybridRetriever(InMemoryVectorStore(), g, HashingEmbedder(), Config()) + r.query_entity_ids(query, "u1") # warm + s = [] + for _ in range(60): + a = time.perf_counter() + r.query_entity_ids(query, "u1") + s.append((time.perf_counter() - a) * 1000) + return statistics.median(s) + +# distinctive: lambda i: f"zeta{i} corp{i}" query "tell me about zeta42" +# shared: lambda i: f"entity number {i}" query "tell me about entity number 42" +``` diff --git a/tests/test_entity_index.py b/tests/test_entity_index.py new file mode 100644 index 0000000..4ecd550 --- /dev/null +++ b/tests/test_entity_index.py @@ -0,0 +1,112 @@ +"""Entity-term index behind query anchoring. + +`query_entity_ids` walked every entity in the store on every retrieval to find which ones the query +names. The graph now indexes entity name and alias terms, so only the query's own terms are looked up. + +The load-bearing test is `test_indexed_and_scanned_anchoring_agree`: it runs the real retriever against a +graph store with the index and against one with the lookup removed, and requires identical answers. The +index is only allowed to be faster, never different. +""" +from __future__ import annotations + +from engram.config import Config +from engram.embed.hashing import HashingEmbedder +from engram.retrieve.hybrid import HybridRetriever +from engram.store.memory_store import InMemoryGraphStore, InMemoryVectorStore +from engram.types import Entity + +NAMES = [ + "Lisbon", "Berlin", "Acme Corp", "Initech", "Kyoto University", + "oat milk", "road bike", "Zephyr", "Wei", "the Berlin office", +] + +QUERIES = [ + "where does wei work", + "tell me about lisbon", + "did wei visit kyoto university", + "what happened at acme corp", + "anything about zephyr", + "somewhere not lisbon", + "does she prefer oat milk", + "berlin", + "nothing relevant here at all", + "the berlin office and initech", +] + + +class UnindexedGraphStore(InMemoryGraphStore): + """The same store with the term lookup hidden, so the retriever takes its full-scan path.""" + + entities_by_terms = None # type: ignore[assignment] + + +def _populate(graph: InMemoryGraphStore, user_id: str = "u1") -> InMemoryGraphStore: + for name in NAMES: + graph.upsert_entity(Entity(user_id=user_id, name=name)) + graph.upsert_entity(Entity(user_id="other", name="Lisbon")) # a second tenant, must stay invisible + return graph + + +def _retriever(graph: InMemoryGraphStore) -> HybridRetriever: + return HybridRetriever(InMemoryVectorStore(), graph, HashingEmbedder(), Config()) + + +def test_indexed_and_scanned_anchoring_agree(): + """The property that makes the index safe: same answers, whichever path ran.""" + indexed = _retriever(_populate(InMemoryGraphStore())) + scanned = _retriever(_populate(UnindexedGraphStore())) + + # Ids differ between the two stores, so compare the entity NAMES each path anchored on. + def names(retriever, query): + return sorted( + retriever.graph.entities[eid].name for eid in retriever.query_entity_ids(query, "u1") + ) + + for query in QUERIES: + assert names(indexed, query) == names(scanned, query), f"paths disagree on {query!r}" + + +def test_lookup_only_touches_requested_terms(): + graph = _populate(InMemoryGraphStore()) + hits = graph.entities_by_terms("u1", {"lisbon", "berlin", "absent"}) + assert set(hits) == {"lisbon", "berlin"} + assert [e.name for e in hits["lisbon"]] == ["Lisbon"] + assert {e.name for e in hits["berlin"]} == {"Berlin", "the Berlin office"} + + +def test_lookup_is_tenant_scoped(): + """Another tenant's identically named entity must not surface.""" + graph = _populate(InMemoryGraphStore()) + hits = graph.entities_by_terms("u1", {"lisbon"}) + assert all(e.user_id == "u1" for e in hits["lisbon"]) + assert graph.entities_by_terms("nobody", {"lisbon"}) == {} + + +def test_multi_word_names_are_indexed_by_every_term(): + graph = _populate(InMemoryGraphStore()) + for term in ("kyoto", "university"): # every token of a multi-word name is a lookup key + hits = graph.entities_by_terms("u1", {term}) + assert any(e.name == "Kyoto University" for e in hits.get(term, [])), term + + +def test_aliases_are_indexed(): + graph = InMemoryGraphStore() + graph.upsert_entity(Entity(user_id="u1", name="Wei", aliases=["Xiaowei", "老王"])) + hits = graph.entities_by_terms("u1", {"xiaowei"}) + assert [e.name for e in hits["xiaowei"]] == ["Wei"] + + +def test_pruning_an_orphan_removes_it_from_the_index(): + """A stale posting would resurrect a deleted entity as a query anchor.""" + graph = _populate(InMemoryGraphStore()) + assert graph.entities_by_terms("u1", {"lisbon"}) + assert graph.prune_orphan_entities() > 0 # no relations exist, so every entity is an orphan + assert graph.entities_by_terms("u1", {"lisbon"}) == {} + + +def test_reupserting_a_name_does_not_duplicate_postings(): + graph = InMemoryGraphStore() + first = graph.upsert_entity(Entity(user_id="u1", name="Lisbon")) + again = graph.upsert_entity(Entity(user_id="u1", name="Lisbon")) + assert again.id == first.id + assert [e.id for e in graph.entities_by_terms("u1", {"lisbon"})["lisbon"]] == [first.id] From 8e33c6642468a2ac5fc32839d8a8f2c9cb74670d Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 20:07:44 +0800 Subject: [PATCH 09/25] service: live latency, volume and token metrics behind an open /metrics The architecture states a <50ms write path and a <100ms read path, and the project's discipline is that a number nobody can reproduce does not exist. Neither target was measured on the running service, and the token-saving claim came only from offline benchmark logs -- never from what the service actually served. engram/metrics.py adds sliding-window percentiles per operation, monotonic counters, and token totals. Pure stdlib and bounded memory: the window keeps percentiles describing current behaviour instead of averaging a regression away under months of history, and the process cannot grow by collecting samples. remember, recall, import and close_session are instrumented, plus a remember_degraded counter -- consolidation failures still return success, so without a counter a silently degrading write path looks perfectly healthy. This was rewritten against today's service rather than replayed from the branch it was preserved on, and that surfaced a real bug in the original: the savings ratio divided a full-history total by a context total drawn from a larger set of calls. Only the answer path computes a baseline, so nine cheap recalls plus one measured one reported roughly 1x -- memory appearing to save nothing -- where the comparable call showed 10x. The ratio is now computed from paired samples only, and is absent rather than invented until at least one pair exists. /metrics is unauthenticated like /health, which is a claim that has to hold: the payload is aggregate-only by construction, so it cannot reveal that a tenant exists, let alone what they stored. A test drives real traffic through the app and asserts the namespace and the content are both absent from the response. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/metrics.py | 131 ++++++++++++++++ engram/server/app.py | 11 ++ engram/service.py | 13 ++ tests/test_metrics.py | 164 ++++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 engram/metrics.py create mode 100644 tests/test_metrics.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 060eff3..2b1ef12 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `live_service_metrics` | 服务可观测层（Bet D 应用于线上） | 架构写明写路径 <50ms、读路径 <100ms，但服务侧没有任何实时观测，这两个目标一直是断言而非测量；token 节省比也只有离线基准，从未反映真实服务过的量。新增 `engram/metrics.py`（纯 stdlib、滑动窗口 p50/p95、单调计数器）+ `GET /metrics`，在 `remember`/`recall`/`import`/`close_session` 四个操作上埋点，并计数 `remember_degraded`（降级写入仍返回成功，不计数就完全不可见）。**修正了保全版的一个真实缺陷**：`savings_ratio` 原本用全部 context 总量做分母、只有部分调用的 full 总量做分子，两者样本集不同会系统性低估节省比；改为只在两侧都测量的配对样本上计算 | `tests/test_metrics.py`（11 测试，含配对样本回归、`/metrics` 无鉴权可访问、以及断言载荷不含租户名与内容的隐私边界测试） | | 2026-08-16 | `entity_anchor_index` | Read path / 图锚定 + Graph store 索引层（Bet E） | `query_entity_ids()` 每次检索遍历全部实体做名称/别名匹配。`InMemoryGraphStore` 改为维护 `(user_id, 词干) -> 实体 id` 倒排索引，只查询查询自身的词。实体名有区分度时 10000 实体下 **4055x**，且耗时恒定 0.004ms 与库大小无关。**边界**：实体名共享高频词时倒排表长度等于全库、索引退化为扫描（仅 1.4x），真实人名/地名/机构名不属于该情形 | `results/entity_anchor_index.md`，`tests/test_entity_index.py`（7 测试；等价性用真实 retriever 对比"带索引"与"去掉索引查找"两条路径，要求结果完全一致） | | 2026-08-16 | `segment_level_rerank` | Read path / 会话重排（Bet A） | cross-encoder 只读 ~512 token，而 LongMemEval 的 session 约 2000 token。`retrieve_episodes()` 把整篇 `ep.content` 交给 reranker，模型不会报错，只会**静默地只对开头四分之一打分**——答案落在后半段的 session 因此被判为无关（该回归此前已在 `lean_context` 的注释里被记录为已知问题，但会话路径一直没修，且**完全没有测试覆盖**）。改为按段落/句子边界切成 ≤`rerank_segment_words`（默认 300 词）的片段分别打分，每篇取其**最佳片段**得分（不是平均——长会话之所以该被检出，正是因为其中某一段命中） | `tests/test_rerank_segments.py`（13 测试，含先断言"整篇打分必然选错"再断言"分段打分选对"的回归用例，以及此前无人覆盖的 `Memory.retrieve_episodes` 接线） | | 2026-08-16 | `exclusion_early_out` + `lance_key_pushdown` | Read path / 图排除区 + Store 按键读取 | 每次检索都经 `query_entity_ids()` → `graph_excluded_entity_ids()`，后者对每个实体名跑两遍正则，而多数查询无否定词。改为先用提示词必要条件判定，5000 实体时无否定词查询从 509ms 降到 0.0009ms。同时 `LanceDBVectorStore.get()` 改用纯过滤查询，不再全表物化后线性找 key | `results/bounded_candidates_scaling.md`，`tests/test_exclusion_shortcut.py`（含逐前缀验证的正则不变量，已抓到一个中文实体名下的静默漏排除缺陷），`tests/test_lancedb_tenant_filter.py` | diff --git a/engram/metrics.py b/engram/metrics.py new file mode 100644 index 0000000..3ee82a7 --- /dev/null +++ b/engram/metrics.py @@ -0,0 +1,131 @@ +"""Live service metrics — the charter's own discipline (Bet D) turned on the running service. + +The architecture states a <50ms target for the write path and <100ms for the read path. Without live +percentiles those are assertions, not measurements, and the same goes for the token-saving claim: it is +reported from offline benchmark logs but never from what the service actually served. + +Pure stdlib, fixed memory, and **aggregate-only by construction** — operation latencies, counters, and +token totals. No namespace names, no queries, no content, so the endpoint can stay as open as /health +without leaking one tenant's existence to another. +""" +from __future__ import annotations + +import functools +import threading +import time +from collections import Counter, defaultdict, deque +from typing import Optional + +__all__ = ["Metrics", "timed"] + + +def _pct(sorted_vals: list[float], q: float) -> float: + """Nearest-rank percentile over a sorted sample. No interpolation — for an SLO readout the extra + precision would be false: the sample is a bounded window, not the full population.""" + if not sorted_vals: + return 0.0 + index = min(int(q * (len(sorted_vals) - 1) + 0.5), len(sorted_vals) - 1) + return sorted_vals[index] + + +class Metrics: + """Thread-safe metrics with a bounded footprint. + + Latencies live in a sliding window per operation, so percentiles describe how the service is behaving + *now* rather than averaging away a regression under months of history. Counters are monotonic. + """ + + def __init__(self, window: int = 512) -> None: + self._lock = threading.Lock() + self._lat: dict[str, deque] = defaultdict(lambda: deque(maxlen=window)) + self._counts: Counter = Counter() + # Token accounting is kept in two buckets on purpose. `_ctx_total` is every served context, which + # is the honest total volume. The savings ratio may only be computed from calls where BOTH sides + # were measured -- dividing a total full-history figure by a total context figure would compare + # different sets of calls and understate the saving whenever a caller skipped the baseline. + self._ctx_total = 0 + self._paired_ctx = 0 + self._paired_full = 0 + self._paired_n = 0 + self._started = time.time() + + # --- recording --- + + def observe(self, op: str, seconds: float) -> None: + with self._lock: + self._lat[op].append(seconds) + self._counts[op] += 1 + + def count(self, name: str, n: int = 1) -> None: + with self._lock: + self._counts[name] += n + + def tokens(self, context: int, full: Optional[int] = None) -> None: + """Record one served context's size, and the full-history baseline when the caller computed it.""" + with self._lock: + self._ctx_total += int(context) + if full is not None: + self._paired_ctx += int(context) + self._paired_full += int(full) + self._paired_n += 1 + + # --- reading --- + + def snapshot(self) -> dict: + """The /metrics payload: numbers only, no user data by construction.""" + with self._lock: + ops = {} + for op, window in self._lat.items(): + if not window: + continue + sample = sorted(window) + ops[op] = { + "n": self._counts[op], + "p50_ms": round(_pct(sample, 0.50) * 1000, 2), + "p95_ms": round(_pct(sample, 0.95) * 1000, 2), + "avg_ms": round(sum(sample) / len(sample) * 1000, 2), + "max_ms": round(sample[-1] * 1000, 2), + "window": len(sample), + } + counts = {k: v for k, v in self._counts.items() if k not in ops} + tokens = { + "context_total": self._ctx_total, + "baseline_context_total": self._paired_ctx, + "baseline_full_total": self._paired_full, + "calls_with_baseline": self._paired_n, + # The live version of the headline "~8x fewer tokens": full history over served context, + # across the calls that measured both. None until at least one such pair exists -- a + # made-up ratio would be worse than no ratio. + "savings_ratio": ( + round(self._paired_full / self._paired_ctx, 2) if self._paired_ctx else None + ), + } + return { + "uptime_s": round(time.time() - self._started, 1), + "ops": ops, + "counts": counts, + "tokens": tokens, + } + + +def timed(op: str): + """Record a MemoryService method's wall-clock under `op`. + + Reads `self.metrics` at call time rather than binding it at decoration, so a service constructed + without metrics (or re-initialised) still works and the decorator costs nothing at import. + """ + + def deco(fn): + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + started = time.perf_counter() + try: + return fn(self, *args, **kwargs) + finally: + meter = getattr(self, "metrics", None) + if meter is not None: + meter.observe(op, time.perf_counter() - started) + + return wrapper + + return deco diff --git a/engram/server/app.py b/engram/server/app.py index cf10b45..61d4271 100644 --- a/engram/server/app.py +++ b/engram/server/app.py @@ -271,6 +271,17 @@ def ready(): return JSONResponse(payload, status_code=200 if payload["ready"] else 503) +@app.get("/metrics") +def metrics(): + """Live latency, volume and token aggregates for the running service. + + Unauthenticated, like /health, and safe to be: the payload is aggregate-only by construction — it + carries no namespace names, queries or content, so it cannot reveal that a given tenant exists, let + alone what they stored. Operators who still want it private should not expose it at the proxy. + """ + return svc().metrics.snapshot() + + # The production console (the React app in frontend/) is served at /ui once built; "/" redirects # there. When it ISN'T built (fresh clone, tests, the zero-setup demo) we fall back to the tiny inline # dashboard below — so the server is always usable with no build step (CLAUDE.md zero-setup invariant). diff --git a/engram/service.py b/engram/service.py index 8296a96..1785821 100644 --- a/engram/service.py +++ b/engram/service.py @@ -20,6 +20,7 @@ from typing import Any, Optional from .memory import Memory +from .metrics import Metrics, timed from .util import fmt_date, fmt_datetime DEFAULT_DATA_DIR = os.path.expanduser("~/.engram/data") @@ -119,6 +120,8 @@ def __init__( from .config import Config self.config = Config() + # Live latency/volume/token counters (aggregate-only; see engram/metrics.py). + self.metrics = Metrics() # ENGRAM_STORAGE selects the vector backend ('memory' default, 'lancedb' opt-in). Fail closed on # anything else: silently falling back to 'memory' would misreport /health's storage field. storage = os.environ.get("ENGRAM_STORAGE", "").strip().lower() @@ -286,6 +289,7 @@ def hot_count(self) -> int: return len(self._hot) # --- write path --------------------------------------------------------- + @timed("remember") def remember(self, user: str, content: str, session_id: str = "default", scope: str = "auto") -> dict: """Store a message + run System-2 consolidation/summarization (best-effort: a transient model @@ -305,11 +309,15 @@ def remember(self, user: str, content: str, session_id: str = "default", mem.summarize_episodes(list(mem.episodes_doc.values())) except Exception as exc: # noqa: BLE001 — keep the raw episode no matter what self._save(user, mem) + # Degraded writes still succeed, so they are invisible in error rates and latency alike. + # Counting them is the only way an operator learns consolidation is silently failing. + self.metrics.count("remember_degraded") return {"ok": True, "extracted": 0, "degraded": type(exc).__name__, "stored_raw": True} self._save(user, mem) return {"ok": True, "scope": "long", "extracted": added, "total_facts": len([f for f in _all_facts(mem) if f.is_live()])} + @timed("import") def import_(self, user: str, sessions: Optional[list] = None, format: str = "auto", data: Any = None, consolidate: bool = True, summarize: bool = True, session_id: str = "imported") -> dict: @@ -393,6 +401,7 @@ def set_policy(self, user: str, **fields: Optional[str]) -> dict: return {"ok": True, **result} # --- read path ---------------------------------------------------------- + @timed("recall") def recall(self, user: str, query: str, lean: bool = True, n_chunks: int = 6, session_id: Optional[str] = None, as_of: Optional[float] = None, redact_sensitive: bool = False, @@ -423,6 +432,9 @@ def recall(self, user: str, query: str, lean: bool = True, n_chunks: int = 6, ) out["full_tokens"] = _est_tokens(full) out["answer"] = _answer_from_memory(self.answerer, query, ctx) + # The baseline is only computed on the answer path, so the savings ratio is derived from + # those calls alone (see Metrics.tokens) rather than mixing two different call sets. + self.metrics.tokens(out["tokens_est"], out.get("full_tokens")) return out res = mem.search(query, user_id=user, as_of=as_of) visible_facts = [ @@ -498,6 +510,7 @@ def clear_working(self, user: str, session_id: str) -> dict: self._save(user, mem) return {"ok": True, "cleared": n} + @timed("close_session") def close_session( self, user: str, diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..957350a --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,164 @@ +"""Live service metrics. + +Two properties carry the weight here. The savings ratio must be computed from calls that measured both +sides -- mixing a full-history total over a context total drawn from a larger set of calls understates the +saving, which is the kind of error that quietly makes a headline number look worse than it is. And the +payload must stay aggregate-only, because the endpoint is unauthenticated: a namespace name leaking into +it would tell any caller which tenants exist. +""" +from __future__ import annotations + +import json + +import pytest + +from engram.metrics import Metrics, timed + + +def test_latency_percentiles_and_counts(): + m = Metrics() + for seconds in (0.001, 0.002, 0.003, 0.010): + m.observe("recall", seconds) + snap = m.snapshot()["ops"]["recall"] + assert snap["n"] == 4 + assert snap["window"] == 4 + assert snap["max_ms"] == 10.0 + assert snap["p50_ms"] <= snap["p95_ms"] <= snap["max_ms"] + + +def test_window_is_bounded_so_memory_cannot_grow(): + """Percentiles must describe current behaviour, and the process must not accumulate samples forever.""" + m = Metrics(window=8) + for i in range(1000): + m.observe("remember", i / 1000) + snap = m.snapshot()["ops"]["remember"] + assert snap["window"] == 8 + assert snap["n"] == 1000, "the count is monotonic even though the sample is windowed" + + +def test_savings_ratio_uses_only_calls_that_measured_both_sides(): + """The regression this guards. + + Nine cheap recalls skip the baseline; one measures both. Dividing the full-history total by *every* + context served would report roughly 1x -- as if memory saved nothing -- when the one comparable call + shows 10x. + """ + m = Metrics() + for _ in range(9): + m.tokens(100) # no baseline computed on this path + m.tokens(100, 1000) + + tokens = m.snapshot()["tokens"] + assert tokens["context_total"] == 1000, "total volume still counts every served context" + assert tokens["calls_with_baseline"] == 1 + assert tokens["savings_ratio"] == 10.0 + + +def test_savings_ratio_is_absent_until_measured(): + """No pairs means no ratio. A fabricated number would be worse than none.""" + m = Metrics() + assert m.snapshot()["tokens"]["savings_ratio"] is None + m.tokens(100) + assert m.snapshot()["tokens"]["savings_ratio"] is None + + +def test_counters_are_separate_from_timed_operations(): + m = Metrics() + m.observe("remember", 0.001) + m.count("remember_degraded") + snap = m.snapshot() + assert "remember" in snap["ops"] + assert snap["counts"]["remember_degraded"] == 1 + assert "remember" not in snap["counts"], "a timed op should not be duplicated as a bare counter" + + +def test_snapshot_is_json_serialisable(): + m = Metrics() + m.observe("recall", 0.001) + m.tokens(10, 100) + json.dumps(m.snapshot()) # the endpoint returns this directly + + +def test_timed_records_even_when_the_call_raises(): + """A failing operation is exactly the one whose latency matters.""" + + class Svc: + def __init__(self): + self.metrics = Metrics() + + @timed("boom") + def boom(self): + raise ValueError("nope") + + svc = Svc() + with pytest.raises(ValueError): + svc.boom() + assert svc.metrics.snapshot()["ops"]["boom"]["n"] == 1 + + +def test_timed_is_a_noop_without_metrics(): + """Objects constructed without a metrics attribute must still work.""" + + class Bare: + @timed("op") + def run(self): + return 42 + + assert Bare().run() == 42 + + +# --- wiring --- + + +def _service(tmp_path): + from engram.service import MemoryService + + return MemoryService(data_dir=str(tmp_path)) + + +def test_service_records_remember_and_recall(tmp_path): + svc = _service(tmp_path) + svc.remember("alice", "Alice works at Acme Corp.") + svc.recall("alice", "where does alice work") + + ops = svc.metrics.snapshot()["ops"] + assert ops["remember"]["n"] == 1 + assert ops["recall"]["n"] == 1 + + +def test_metrics_payload_leaks_no_tenant_identity(tmp_path): + """The endpoint is unauthenticated, so this is a privacy boundary, not a nicety.""" + svc = _service(tmp_path) + secret_user = "acme-industries-prod" + svc.remember(secret_user, "The launch date is March 3rd.") + svc.recall(secret_user, "when is the launch") + + payload = json.dumps(svc.metrics.snapshot()) + assert secret_user not in payload + assert "launch" not in payload + assert "March" not in payload + + +def test_metrics_endpoint_is_open_and_aggregate(tmp_path, monkeypatch): + fastapi = pytest.importorskip("fastapi") + del fastapi + from fastapi.testclient import TestClient + + monkeypatch.setenv("ENGRAM_DATA_DIR", str(tmp_path)) + monkeypatch.setenv("ENGRAM_OPEN", "1") + import engram.server.app as app_module + + app_module._svc = None # force a rebuild against the temp data dir + client = TestClient(app_module.app) + assert app_module.svc().data_dir == str(tmp_path), "test must exercise its own service, not a leftover" + + response = client.get("/metrics") # no Authorization header + assert response.status_code == 200 + body = response.json() + assert set(body) == {"uptime_s", "ops", "counts", "tokens"} + + # And it reflects real traffic through the app, not just an empty shell. + client.post("/v1/remember", json={"content": "hi"}, headers={"Authorization": "Bearer tenant-x"}) + after = client.get("/metrics").json() + assert after["ops"]["remember"]["n"] >= 1 + assert "tenant-x" not in json.dumps(after) From 038fdc2007a54a10fff272ab257eaa5e8f2bc458 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 20:29:42 +0800 Subject: [PATCH 10/25] retrieve: split the read context into a cacheable half and a per-query half lean_context returns one flat string that callers drop into the user turn, so a multi-turn session re-sends and re-processes the parts that did not change -- the profile, the instructions on how to use a retrieved slice -- on every single turn. Splitting it lets the unchanging half sit in the system prompt where provider prompt-caching can reuse it, while only this query's evidence varies. The retrieved evidence is identical either way, so accuracy is unchanged by construction; this is a tokens-and-latency change and should not be expected to move a benchmark score. The whole thing rests on the stable half being query-independent, so that is the test: the same user's different questions must produce a byte-identical stable block, or caching misses every turn and the split costs more than it saves. Measured before claiming anything, and the numbers are more modest than the module this was preserved from implied. The pure split saves 5-9% across long sessions and is a net LOSS below about five turns, because the flat context is small enough that the first turn's extra structure never pays back. Its ceiling is simply the profile's share of the context. The session map costs more than it returns. It is content the flat context does not carry, so the first comparison was really flat-versus-flat-plus-a-new-block: -93% at one turn, breaking even around twenty. It is therefore off by default and opt-in for long sessions or when progressive disclosure is wanted for its own sake. Both non-adoptions are written into the optimization map rather than left implied by a default. Writing the tests surfaced a real wrinkle too: the usage guide told the model to consult a MEMORY MAP that a redacted context had removed, pointing it at something it could never be given. The map sentence is now appended only when a map is actually present. Not wired into lean_context or the OpenAI-compatible proxy: short sessions lose, and the published numbers came from the flat path. Callers choose per session length via Memory.layered_context(). --- docs/architecture-optimization-map.zh-CN.md | 3 + engram/memory.py | 9 ++ engram/retrieve/layered.py | 143 ++++++++++++++++++ results/layered_context_tokens.md | 68 +++++++++ tests/test_layered_context.py | 155 ++++++++++++++++++++ 5 files changed, 378 insertions(+) create mode 100644 engram/retrieve/layered.py create mode 100644 results/layered_context_tokens.md create mode 100644 tests/test_layered_context.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 2b1ef12..5638619 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `layered_context`（默认不接主路径） | Read path / 上下文装配（Bet A 的 tokens+latency 维度） | `lean_context` 拍平成单串放进 user turn，多轮会话每轮重发不变的画像与指引。新增 `engram/retrieve/layered.py` 把上下文拆成查询无关的 stable 半（进 system prompt，可被 provider prompt-cache 复用）与每轮变化的 dynamic 半。证据完全相同，准确率按构造不变。**实测收益温和且有下限**：长会话 +5%～+9%，5 轮以下净亏；导航图默认关闭（约 20 轮才回本） | `results/layered_context_tokens.md`，`tests/test_layered_context.py`（14 测试，含"stable 半跨查询字节相同"这一缓存前提，以及"拆分不丢证据"的守卫） | | 2026-08-16 | `live_service_metrics` | 服务可观测层（Bet D 应用于线上） | 架构写明写路径 <50ms、读路径 <100ms，但服务侧没有任何实时观测，这两个目标一直是断言而非测量；token 节省比也只有离线基准，从未反映真实服务过的量。新增 `engram/metrics.py`（纯 stdlib、滑动窗口 p50/p95、单调计数器）+ `GET /metrics`，在 `remember`/`recall`/`import`/`close_session` 四个操作上埋点，并计数 `remember_degraded`（降级写入仍返回成功，不计数就完全不可见）。**修正了保全版的一个真实缺陷**：`savings_ratio` 原本用全部 context 总量做分母、只有部分调用的 full 总量做分子，两者样本集不同会系统性低估节省比；改为只在两侧都测量的配对样本上计算 | `tests/test_metrics.py`（11 测试，含配对样本回归、`/metrics` 无鉴权可访问、以及断言载荷不含租户名与内容的隐私边界测试） | | 2026-08-16 | `entity_anchor_index` | Read path / 图锚定 + Graph store 索引层（Bet E） | `query_entity_ids()` 每次检索遍历全部实体做名称/别名匹配。`InMemoryGraphStore` 改为维护 `(user_id, 词干) -> 实体 id` 倒排索引，只查询查询自身的词。实体名有区分度时 10000 实体下 **4055x**，且耗时恒定 0.004ms 与库大小无关。**边界**：实体名共享高频词时倒排表长度等于全库、索引退化为扫描（仅 1.4x），真实人名/地名/机构名不属于该情形 | `results/entity_anchor_index.md`，`tests/test_entity_index.py`（7 测试；等价性用真实 retriever 对比"带索引"与"去掉索引查找"两条路径，要求结果完全一致） | | 2026-08-16 | `segment_level_rerank` | Read path / 会话重排（Bet A） | cross-encoder 只读 ~512 token，而 LongMemEval 的 session 约 2000 token。`retrieve_episodes()` 把整篇 `ep.content` 交给 reranker，模型不会报错，只会**静默地只对开头四分之一打分**——答案落在后半段的 session 因此被判为无关（该回归此前已在 `lean_context` 的注释里被记录为已知问题，但会话路径一直没修，且**完全没有测试覆盖**）。改为按段落/句子边界切成 ≤`rerank_segment_words`（默认 300 词）的片段分别打分，每篇取其**最佳片段**得分（不是平均——长会话之所以该被检出，正是因为其中某一段命中） | `tests/test_rerank_segments.py`（13 测试，含先断言"整篇打分必然选错"再断言"分段打分选对"的回归用例，以及此前无人覆盖的 `Memory.retrieve_episodes` 接线） | @@ -138,6 +139,8 @@ flowchart TD | --- | --- | --- | | 只把词汇/融合环节收敛为有界候选池（保留语义通道） | **净收益 ≈ 0**（10000 事实：177.03ms vs 全扫 177.88ms）。语义通道自身就是一次全量扫描，省下的又赔回去，还多付索引维护成本。不是候选池思路错，是缺 ANN 索引 | `results/bounded_candidates_scaling.md` | | `candidate_vector_channel=False` 作为默认 | **不采用**。它确实快 14.5x 且次线性，但会丢"语义相关却无共享查询词"的事实，正是 M1 已验证 hybrid 论点依赖的召回。需真 ANN 后端或 keyed harness 证明召回损失可接受 | 同上 | +| 上下文拆分接进 `lean_context` 默认路径 | **不采用**。实测 5 轮以下是净亏（stable 半虽只计费一次，但扁平上下文本身小，首轮结构开销收不回），且已发布数字均由扁平路径产出。保留为独立方法 `Memory.layered_context()`，由调用方按会话长度选择 | `results/layered_context_tokens.md` | +| 上下文导航图（MEMORY MAP）默认开启 | **不采用**。它是扁平上下文里本来没有的新增内容，约 20 轮才回本；最初的对比实为"扁平 vs 扁平+导航图"，缓存收益被吃光还倒欠。改为 `map_limit=0` 默认关闭 | 同上 | ## 验收分层规则 diff --git a/engram/memory.py b/engram/memory.py index 0002d41..69fb28e 100644 --- a/engram/memory.py +++ b/engram/memory.py @@ -2306,6 +2306,15 @@ def lean_context( return self._fit_blocks_by_evidence_budget(blocks, char_budget, need) return assembled[:char_budget] + def layered_context(self, query: str, user_id: str = "default", **kwargs): + """`lean_context` split into a cacheable half and a per-query half (see retrieve/layered.py). + + Same evidence, same retrieval — it only changes where a caller can put each part, so a multi-turn + session stops re-sending the profile and the memory map on every turn.""" + from .retrieve.layered import layered_context + + return layered_context(self, query, user_id=user_id, **kwargs) + # --- read path --- def search( self, diff --git a/engram/retrieve/layered.py b/engram/retrieve/layered.py new file mode 100644 index 0000000..eee4ad5 --- /dev/null +++ b/engram/retrieve/layered.py @@ -0,0 +1,143 @@ +"""Split the read context into a cacheable half and a per-query half. + +`Memory.lean_context` returns one flat string, and callers drop it into the user turn. Across a +multi-turn session that re-sends and re-processes the whole thing every turn, including the parts that +did not change — the user's profile, the map of what exists in memory, the instructions on how to use it. + +Splitting it lets the unchanging half sit in the system prompt, where provider prompt-caching can reuse +it for the rest of the session, while only this query's evidence varies. The retrieved evidence is +identical either way, so accuracy is unchanged by construction: this is a tokens-and-latency change, two +thirds of the triple the charter insists on reporting together, and it should not be expected to move a +benchmark score. + +The property that makes it work is that the stable half is **query-independent**. If it varied with the +question it would invalidate the cache every turn and cost more than it saved, so the map below is ranked +by recency rather than relevance and `test_stable_block_is_identical_across_queries` pins that down. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +from ..util import fmt_date + +__all__ = ["LayeredContext", "layered_context", "memory_map", "RECALL_GUIDE"] + +# How to use a retrieved slice, including when to refuse. Query-independent, so it belongs in the cached +# half rather than being re-sent every turn. It states the abstention rule the read path already applies. +RECALL_GUIDE = ( + "The memories below are a retrieved slice of this user's history, not the complete record. Answer " + "only from what is shown plus the current question. If the needed fact is not present, say it is not " + "in memory rather than guessing — a confident wrong answer is worse than an honest 'I don't have " + "that.'" +) + +# Appended only when a map is actually included. Pointing the model at a section that is not there — +# which is what a redacted context would do — invites it to ask for something it cannot be given. +_MAP_HINT = ( + " MEMORY MAP lists sessions that exist but were not opened; if the answer plausibly lives in one, " + "you may ask for it by its date." +) + + +@dataclass(frozen=True) +class LayeredContext: + """The same evidence as `lean_context`, in two halves. + + `stable` is identical across a user's turns until their memory itself changes; `dynamic` is this + query's evidence. + """ + + stable: str + dynamic: str + + @property + def text(self) -> str: + """Both halves as one string — what `lean_context` would have returned.""" + return "\n\n".join(part for part in (self.stable, self.dynamic) if part) + + def as_messages(self, query: str, system: str = "") -> list[dict[str, str]]: + """Chat messages placing the cacheable half in the system turn and the rest in the user turn.""" + system_parts = [part for part in (system, self.stable) if part] + messages = [] + if system_parts: + messages.append({"role": "system", "content": "\n\n".join(system_parts)}) + user_parts = [part for part in (self.dynamic, query) if part] + messages.append({"role": "user", "content": "\n\n".join(user_parts)}) + return messages + + +def memory_map(mem: Any, user_id: str, limit: int = 20, as_of: Optional[float] = None) -> str: + """A query-independent index of what is in memory but was not retrieved. + + Deliberately ranked by recency, not by relevance to the question: relevance ranking would reorder the + map on every turn and defeat the caching this exists for. Each row carries a date the model can ask + for, which is what makes the unretrieved remainder reachable instead of merely invisible. + """ + if limit <= 0: + return "" + sessions: dict[str, Any] = {} + for ep in mem.episodes_doc.values(): + if ep.user_id != user_id: + continue + if as_of is not None and ep.event_time > as_of: + continue + current = sessions.get(ep.session_id) + if current is None or ep.event_time > current.event_time: + sessions[ep.session_id] = ep + if not sessions: + return "" + + rows = [] + for ep in sorted(sessions.values(), key=lambda e: e.event_time, reverse=True)[:limit]: + gist = (ep.summary or ep.content or "").strip().replace("\n", " ") + if len(gist) > 96: + gist = gist[:93].rstrip() + "..." + rows.append(f"- {fmt_date(ep.event_time)} [{ep.session_id}] {gist}") + return "MEMORY MAP (sessions in memory, most recent first):\n" + "\n".join(rows) + + +def layered_context( + mem: Any, + query: str, + user_id: str = "default", + as_of: Optional[float] = None, + # The map is off by default because it was measured, not assumed: it is content the flat context does + # not carry, and at typical session lengths it costs more than the caching saves — a net loss until + # roughly 20 turns (results/layered_context_tokens.md). Turn it on for long sessions, or when the + # progressive-disclosure capability is worth paying for on its own. + map_limit: int = 0, + guide: bool = True, + **lean_kwargs: Any, +) -> LayeredContext: + """Assemble the read context in two halves. + + `mem` is duck-typed as `engram.memory.Memory` to avoid a circular import. Extra keyword arguments go + straight to `lean_context`, so the dynamic half is produced by the same retrieval as the flat path — + there is no second, drifting implementation of the read path here. + """ + user = mem.resolver.resolve(user_id) + redact = bool(lean_kwargs.get("redact_sensitive", False)) + + # The persona and the map are free-text layers that can fold in sensitive content, so a redacted + # context omits both for the same reason lean_context drops the persona. + persona = map_block = "" + if not redact: + persona = mem._persona_at(user, as_of) + map_block = memory_map(mem, user, limit=map_limit, as_of=as_of) + + stable_parts = [] + if guide: + stable_parts.append(RECALL_GUIDE + (_MAP_HINT if map_block else "")) + if persona: + label = "USER PROFILE" if as_of is None else f"USER PROFILE (as of {fmt_date(as_of)})" + stable_parts.append(f"{label}:\n{persona}") + if map_block: + stable_parts.append(map_block) + + # persona=False: it is already in the stable half, and sending it twice would cost exactly the tokens + # this split exists to save. + lean_kwargs["persona"] = False + dynamic = mem.lean_context(query, user_id=user_id, as_of=as_of, **lean_kwargs) + + return LayeredContext(stable="\n\n".join(stable_parts), dynamic=dynamic) diff --git a/results/layered_context_tokens.md b/results/layered_context_tokens.md new file mode 100644 index 0000000..0fc3dc3 --- /dev/null +++ b/results/layered_context_tokens.md @@ -0,0 +1,68 @@ +# layered_context — 上下文拆分的 token 测量（2026-08-16） + +## 假设与做法 + +`lean_context` 返回单个扁平字符串，调用方整段放进 user turn。多轮会话里，其中不随查询变化的部分 +（用户画像、使用指引）每轮都被重发和重算。把这部分拆进 system prompt，provider 的 prompt-cache +就能跨轮复用，只有本轮证据是变化的。 + +检索到的证据两种方式完全相同，所以**准确率按构造不变**——这是 tokens/latency 那两维的改动， +不应期待它移动任何 benchmark 分数。 + +计费模型：flat = 每轮全量；layered = stable 只在首轮计费 + 每轮 dynamic。 + +## 复现 + +```bash +python3 - <<'PY' +from engram.memory import Memory +from engram.retrieve.layered import layered_context +from engram.service import _est_tokens +# 见 tests/test_layered_context.py 的 _memory() 构造；逐轮累加 flat 与 layered +PY +``` + +## 结果 + +**10 个会话** + +| 轮数 | flat | 拆分（无导航图） | 节省 | 拆分（含导航图） | 节省 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 239 | 295 | −23.4% | 463 | −93.7% | +| 3 | 755 | 767 | −1.6% | 935 | −23.8% | +| 5 | 1266 | 1234 | **+2.5%** | 1402 | −10.7% | +| 10 | 2307 | 2165 | +6.2% | 2333 | −1.1% | +| 20 | 4614 | 4252 | +7.8% | 4420 | +4.2% | +| 50 | 11535 | 10513 | **+8.9%** | 10681 | +7.4% | + +**30 个会话** + +| 轮数 | flat | 拆分（无导航图） | 节省 | 拆分（含导航图） | 节省 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 361 | 417 | −15.5% | 720 | −99.4% | +| 5 | 1896 | 1864 | +1.7% | 2167 | −14.3% | +| 20 | 7190 | 6828 | +5.0% | 7131 | +0.8% | +| 50 | 17975 | 16953 | **+5.7%** | 17256 | +4.0% | + +## 结论 + +1. **纯拆分是真实但温和的收益**：长会话约 +5%～+9%，**5 轮以下是净亏**（stable 半虽只计费一次， + 但扁平上下文本身很小，首轮多出的结构开销收不回来）。收益上限就是画像块占上下文的比例。 + 保全版模块的文档把它描述为"大幅减少重复计费"，实测不支持这个力度。 +2. **导航图（MEMORY MAP）是净成本**，要到约 20 轮才回本。它是扁平上下文里本来没有的**新增内容**， + 所以最初那版对比其实是"扁平 vs 扁平+新增导航图"，把缓存收益吃掉还倒欠。据此**默认关闭** + （`map_limit=0`），需要渐进式展开能力或确认是长会话时再显式开启。 +3. **本测量只算 token 数量，不算计费价格。** provider 的 cache read 通常显著便宜于全价 input token， + 所以经济收益应大于上表的 token 收益；但那取决于各家定价，本文件不做未经测量的推断。 + +## 未采用/回滚原因 + +- 导航图默认开启：**不采用**，见上表，典型会话长度下是净亏。 +- 把拆分接进 `lean_context` 默认路径：**不采用**。短会话是净亏，且已发布数字均由扁平路径产出。 + 当前形态是独立方法 `Memory.layered_context()`，调用方按会话长度自行选择。 + +## 下一步 + +把 `as_messages()` 接进 OpenAI 兼容代理（`engram/server/openai_compat.py`），让多轮会话真正吃到 +provider 的 prompt-cache，并用 `/metrics` 的 token 计数回读真实节省——目前的数字是离线估算， +不是线上实测。 diff --git a/tests/test_layered_context.py b/tests/test_layered_context.py new file mode 100644 index 0000000..113fd81 --- /dev/null +++ b/tests/test_layered_context.py @@ -0,0 +1,155 @@ +"""Splitting the read context into a cacheable half and a per-query half. + +The whole feature rests on one property: the stable half must be byte-identical across a user's turns. +If it drifts with the question, prompt-caching misses every turn and the split costs more than the flat +context it replaced. `test_stable_block_is_identical_across_queries` is that property, and +`test_no_evidence_is_lost_by_splitting` is the guard that the split does not quietly drop retrieval. +""" +from __future__ import annotations + +from engram.memory import Memory +from engram.retrieve.layered import RECALL_GUIDE, layered_context, memory_map +from engram.util import DAY, now + +QUERIES = [ + "where does alice work", + "what does alice drink", + "when did alice go to kyoto", + "who is alice's manager", +] + + +def _memory() -> Memory: + mem = Memory() + base = now() - 30 * DAY + for i, text in enumerate( + [ + "Alice works at Acme Corp as a staff engineer.", + "Alice prefers oat milk in her coffee.", + "Alice travelled to Kyoto in April for a conference.", + "Alice's manager is Bob.", + "Alice is learning to play the cello.", + ] + ): + mem.add(text, user_id="u1", session_id=f"s{i}", event_time=base + i * DAY) + mem.consolidate() + mem.summarize_episodes(list(mem.episodes_doc.values())) + return mem + + +def test_stable_block_is_identical_across_queries(): + """The property the feature exists for: a differing question must not change the cached half.""" + mem = _memory() + blocks = {layered_context(mem, q, "u1").stable for q in QUERIES} + assert len(blocks) == 1, "the stable half must not vary with the query, or caching never hits" + + +def test_dynamic_block_does_vary_with_the_query(): + """And the other half must actually be doing per-query work.""" + mem = _memory() + blocks = {layered_context(mem, q, "u1").dynamic for q in QUERIES} + assert len(blocks) > 1 + + +def test_no_evidence_is_lost_by_splitting(): + """Splitting must not drop retrieval: everything the flat path shows must still be present.""" + mem = _memory() + query = "where does alice work" + flat = mem.lean_context(query, user_id="u1") + layered = layered_context(mem, query, "u1") + + for line in (ln.strip() for ln in flat.splitlines()): + if line.startswith("- ") and len(line) > 8: + assert line in layered.text, f"evidence dropped by the split: {line!r}" + + +def test_profile_is_not_duplicated_across_the_halves(): + """The profile lives in the cached half; repeating it in the dynamic half would spend exactly the + tokens this split saves.""" + mem = _memory() + layered = layered_context(mem, "where does alice work", "u1") + assert "USER PROFILE" in layered.stable + assert "USER PROFILE" not in layered.dynamic + + +def test_guide_is_in_the_cached_half_and_optional(): + mem = _memory() + assert RECALL_GUIDE in layered_context(mem, "q", "u1").stable + assert RECALL_GUIDE not in layered_context(mem, "q", "u1", guide=False).stable + + +def test_as_messages_places_each_half_in_its_own_turn(): + mem = _memory() + layered = layered_context(mem, "where does alice work", "u1") + messages = layered.as_messages("where does alice work", system="You are a helpful assistant.") + + assert [m["role"] for m in messages] == ["system", "user"] + assert "You are a helpful assistant." in messages[0]["content"] + assert layered.stable in messages[0]["content"] + assert layered.dynamic in messages[1]["content"] + assert "where does alice work" in messages[1]["content"] + + +# --- memory map --- + + +def test_memory_map_is_recency_ordered_and_query_independent(): + """Ranking by relevance would reorder the map every turn and defeat the caching.""" + mem = _memory() + rendered = memory_map(mem, "u1") + dates = [line.split()[1] for line in rendered.splitlines()[1:]] + assert dates == sorted(dates, reverse=True) + + +def test_memory_map_is_tenant_scoped(): + mem = _memory() + mem.add("Bob works at Globex.", user_id="u2", session_id="other") + assert "Globex" not in memory_map(mem, "u1") + + +def test_memory_map_respects_as_of(): + """An as-of read must not reveal sessions from after the time being asked about.""" + mem = _memory() + cutoff = now() - 29 * DAY + rendered = memory_map(mem, "u1", as_of=cutoff) + assert rendered.count("\n- ") + (1 if "\n- " not in rendered and "- " in rendered else 0) <= 2 + assert "cello" not in rendered + + +def test_memory_map_is_empty_without_episodes(): + assert memory_map(Memory(), "nobody") == "" + + +def test_memory_map_is_off_by_default(): + """Measured as a net token cost below long sessions, so callers opt in (see results/).""" + mem = _memory() + assert "MEMORY MAP" not in layered_context(mem, "q", "u1").stable + assert memory_map(mem, "u1", limit=0) == "", "a zero limit must render nothing, not a bare header" + + +def test_redacted_context_omits_profile_and_map(): + """A redacted context is structured-facts-only; free-text layers can fold in sensitive content.""" + mem = _memory() + layered = layered_context( + mem, "where does alice work", "u1", map_limit=20, redact_sensitive=True + ) + assert "USER PROFILE" not in layered.stable + assert "MEMORY MAP (" not in layered.stable + assert RECALL_GUIDE in layered.stable, "the abstention guide carries no user content" + + +def test_guide_only_mentions_the_map_when_one_is_present(): + """Pointing the model at a section that is not there invites it to ask for the unavailable.""" + mem = _memory() + assert "MEMORY MAP" in layered_context(mem, "q", "u1", map_limit=20).stable + assert "MEMORY MAP" not in layered_context(mem, "q", "u1").stable, "map is off by default" + assert "MEMORY MAP" not in layered_context( + mem, "q", "u1", map_limit=20, redact_sensitive=True + ).stable + + +def test_memory_method_matches_the_module(): + mem = _memory() + assert mem.layered_context("where does alice work", "u1").stable == ( + layered_context(mem, "where does alice work", "u1").stable + ) From af49599eebf958e92121c5086404c390589c8c22 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 20:50:09 +0800 Subject: [PATCH 11/25] server: per-tenant rate limiting and Idempotency-Key replay Two different ways the multi-tenant surface was exposed. Without a rate limit one caller can spend the whole process, and once the LLM-backed paths are wired, the whole budget. Without idempotency a client that retries after a network timeout stores the same episode twice and pays to consolidate it twice -- the first request succeeded, only its response was lost. The limit is enforced inside auth() because that is the one place every protected route already passes through to learn who is calling, so a new endpoint cannot forget to be limited. /health and /metrics stay unauthenticated and therefore unlimited, which is deliberate: probes must keep working while a tenant is being throttled, or one busy caller takes the deployment down with it. Rewriting rather than replaying the preserved implementation caught a leak in it: the per-tenant hit map was a defaultdict that was never swept, so every tenant that ever called stayed in memory forever. That is the kind of slow growth that only bites the deployment with the most tenants, which is the one that can least afford it. Entries whose windows have emptied are now pruned, opportunistically so the server still needs no background thread. Two properties are easy to get subtly wrong and are tested directly. A rejected request is not recorded, because counting rejections keeps a retrying client's window permanently full and it never recovers. And the idempotency cache is keyed by tenant as well as key, so two namespaces choosing "retry-1" cannot read each other's response -- that one is a cross-tenant leak, not an inconvenience. Only successful responses are cached; replaying an exception would turn a transient failure into a permanent one. Both are in-process and documented as such: behind several replicas the effective limit is per_min x replicas and a retry routed elsewhere re-runs. Saying so beats shipping something that looks distributed and is not. Rate limiting is off by default, so the zero-setup demo and existing deployments are unchanged. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/server/app.py | 99 ++++++++- engram/server/limits.py | 146 +++++++++++++ tests/test_rate_limit_idempotency.py | 223 ++++++++++++++++++++ 4 files changed, 461 insertions(+), 8 deletions(-) create mode 100644 engram/server/limits.py create mode 100644 tests/test_rate_limit_idempotency.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 5638619..a108e1f 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `rate_limit` + `idempotency_key` | 服务边界 / 多租户防护 | 公开记忆 API 无限流等于开放滥用（LLM 路径接上后还是开放账单）；客户端超时重试会把同一 episode 存两遍并付两遍固化成本——首次请求其实成功了，丢的只是响应。新增 `engram/server/limits.py`：按租户滑动窗口限流（`ENGRAM_RATE_LIMIT_PER_MIN`，默认 0=关闭）+ `Idempotency-Key` 重放缓存（按「租户+key」隔离）。限流放在 `auth()` 里——那是每个受保护路由识别租户的必经点，新增端点不会漏配。**修正了保全版的一处泄漏**：租户命中表是 defaultdict 且从不回收，租户多了会无界增长，改为窗口清空即回收 | `tests/test_rate_limit_idempotency.py`（19 测试，含「被拒请求不得延长自身窗口」「缓存响应不得跨租户」「限流期间 /health 与 /metrics 仍可达」） | | 2026-08-16 | `layered_context`（默认不接主路径） | Read path / 上下文装配（Bet A 的 tokens+latency 维度） | `lean_context` 拍平成单串放进 user turn，多轮会话每轮重发不变的画像与指引。新增 `engram/retrieve/layered.py` 把上下文拆成查询无关的 stable 半（进 system prompt，可被 provider prompt-cache 复用）与每轮变化的 dynamic 半。证据完全相同，准确率按构造不变。**实测收益温和且有下限**：长会话 +5%～+9%，5 轮以下净亏；导航图默认关闭（约 20 轮才回本） | `results/layered_context_tokens.md`，`tests/test_layered_context.py`（14 测试，含"stable 半跨查询字节相同"这一缓存前提，以及"拆分不丢证据"的守卫） | | 2026-08-16 | `live_service_metrics` | 服务可观测层（Bet D 应用于线上） | 架构写明写路径 <50ms、读路径 <100ms，但服务侧没有任何实时观测，这两个目标一直是断言而非测量；token 节省比也只有离线基准，从未反映真实服务过的量。新增 `engram/metrics.py`（纯 stdlib、滑动窗口 p50/p95、单调计数器）+ `GET /metrics`，在 `remember`/`recall`/`import`/`close_session` 四个操作上埋点，并计数 `remember_degraded`（降级写入仍返回成功，不计数就完全不可见）。**修正了保全版的一个真实缺陷**：`savings_ratio` 原本用全部 context 总量做分母、只有部分调用的 full 总量做分子，两者样本集不同会系统性低估节省比；改为只在两侧都测量的配对样本上计算 | `tests/test_metrics.py`（11 测试，含配对样本回归、`/metrics` 无鉴权可访问、以及断言载荷不含租户名与内容的隐私边界测试） | | 2026-08-16 | `entity_anchor_index` | Read path / 图锚定 + Graph store 索引层（Bet E） | `query_entity_ids()` 每次检索遍历全部实体做名称/别名匹配。`InMemoryGraphStore` 改为维护 `(user_id, 词干) -> 实体 id` 倒排索引，只查询查询自身的词。实体名有区分度时 10000 实体下 **4055x**，且耗时恒定 0.004ms 与库大小无关。**边界**：实体名共享高频词时倒排表长度等于全库、索引退化为扫描（仅 1.4x），真实人名/地名/机构名不属于该情形 | `results/entity_anchor_index.md`，`tests/test_entity_index.py`（7 测试；等价性用真实 retriever 对比"带索引"与"去掉索引查找"两条路径，要求结果完全一致） | diff --git a/engram/server/app.py b/engram/server/app.py index 61d4271..bc297f1 100644 --- a/engram/server/app.py +++ b/engram/server/app.py @@ -35,6 +35,7 @@ from .. import __version__ from ..service import MemoryService +from .limits import IdempotencyCache, RateLimiter, idempotency_ttl, rate_limit_per_min DEFAULT_MAX_REQUEST_BYTES = 2 * 1024 * 1024 @@ -180,10 +181,80 @@ async def request_limits_and_security_headers(request: Request, call_next): return _apply_security_headers(await call_next(request), path) +_limiter: Optional[RateLimiter] = None +_limiter_per_min = -1 +_idempotency: Optional[IdempotencyCache] = None + +# Beyond this many tracked tenants, sweep the ones whose windows have emptied. Opportunistic rather than +# on a timer so the server needs no background thread. +_PRUNE_ABOVE = 1_000 + + +def _rate_limiter() -> RateLimiter: + """The shared limiter, rebuilt when the configured limit changes so a reconfigure takes effect.""" + global _limiter, _limiter_per_min + per_min = rate_limit_per_min() + if _limiter is None or per_min != _limiter_per_min: + _limiter = RateLimiter(per_min) + _limiter_per_min = per_min + return _limiter + + +def _idempotency_cache() -> IdempotencyCache: + global _idempotency + if _idempotency is None: + _idempotency = IdempotencyCache(ttl_seconds=idempotency_ttl()) + return _idempotency + + +def _enforce_rate_limit(user: str) -> None: + limiter = _rate_limiter() + if not limiter.enabled: + return + if limiter.tracked_tenants > _PRUNE_ABOVE: + limiter.prune() + allowed, retry_after = limiter.check(user) + if not allowed: + # Retry-After is what makes a 429 actionable rather than a client guessing and hammering. + raise HTTPException( + 429, "rate limit exceeded", headers={"Retry-After": str(int(retry_after) + 1)} + ) + + +def idempotent(user: str, request: Request, compute): + """Return the cached response for this Idempotency-Key, or compute it and cache it. + + Applied to the endpoints that both mutate state and do expensive work, because that is where a + client's timeout-and-retry is most costly: the first request succeeded, only its response was lost. + Only successful results are cached -- replaying an exception would turn a transient failure into a + permanent one for the lifetime of the entry. + """ + key = (request.headers.get("Idempotency-Key") or "").strip()[:256] + if not key: + return compute() + cache = _idempotency_cache() + cached = cache.get(user, key) + if cached is not None: + return cached + result = compute() + cache.put(user, key, result) + return result + + def auth(authorization: str = Header(default="")) -> str: - """Resolve the caller's user_id from the Bearer key. Dev open mode uses the key text itself as the - namespace, so anyone can try it without pre-provisioned keys while still avoiding shared anonymous - memory unless it is explicitly enabled.""" + """Resolve the caller's user_id from the Bearer key, then apply their rate limit. + + The limit lives here because this is the one place every protected route already passes through to + learn who is calling — a new endpoint cannot forget to be limited. Dev open mode uses the key text + itself as the namespace, so anyone can try it without pre-provisioned keys while still avoiding + shared anonymous memory unless it is explicitly enabled. + """ + user = _resolve_user(authorization) + _enforce_rate_limit(user) + return user + + +def _resolve_user(authorization: str) -> str: try: keys = _load_keys() except AuthConfigError as exc: @@ -388,10 +459,16 @@ def robots_txt(): @app.post("/v1/remember") -def remember(req: RememberReq, user: str = Depends(auth)): +def remember(req: RememberReq, request: Request, user: str = Depends(auth)): # Route by ephemerality inside the service: either way the dated episode is stored (history stays # answerable); transient state also goes to working memory and is NOT promoted into a durable fact. - return svc().remember(user, req.content, session_id=req.session_id, scope=req.scope) + # Idempotency-Key guards the retry-after-timeout case, which would otherwise store the episode twice + # and pay to consolidate it twice. + return idempotent( + user, + request, + lambda: svc().remember(user, req.content, session_id=req.session_id, scope=req.scope), + ) @app.post("/v1/recall") @@ -530,15 +607,21 @@ class ImportReq(BaseModel): @app.post("/v1/import") -def import_history(req: ImportReq, user: str = Depends(auth)): +def import_history(req: ImportReq, request: Request, user: str = Depends(auth)): """Bulk-ingest an external history in one batched pass. See `python -m engram.connectors` for a CLI that parses common exports and posts here. A native `/v1/export` payload (format='engram' or auto-sniffed) restores directly — the cross-instance migration path.""" if req.sessions is None and req.data is None: raise HTTPException(400, "provide either 'sessions' (pre-parsed) or 'data' (+ 'format') to import") try: - return svc().import_(user, sessions=req.sessions, data=req.data, format=req.format, - consolidate=req.consolidate, summarize=req.summarize) + # The most expensive mutating endpoint there is, so a lost response is the most expensive thing + # to replay: an Idempotency-Key makes the retry free. + return idempotent( + user, + request, + lambda: svc().import_(user, sessions=req.sessions, data=req.data, format=req.format, + consolidate=req.consolidate, summarize=req.summarize), + ) except ValueError as exc: # a malformed payload is the CLIENT's error — 400 with the parser's reason, never a raw 500 raise HTTPException(400, f"import failed: {exc}") from exc diff --git a/engram/server/limits.py b/engram/server/limits.py new file mode 100644 index 0000000..7d93a45 --- /dev/null +++ b/engram/server/limits.py @@ -0,0 +1,146 @@ +"""Per-tenant rate limiting and Idempotency-Key replay, both dependency-free. + +Two different failure modes on the same multi-tenant surface. Without a rate limit, one caller can spend +the whole process — and once the LLM-backed paths are wired, the whole budget. Without idempotency, a +client that retries after a network timeout stores the same episode twice and pays for consolidating it +twice, because the first request did succeed; only the response was lost. + +**Both are in-process.** Behind several replicas each process keeps its own window and its own cache, so +the effective rate limit is `per_min x replicas` and a retry routed to a different replica will re-run. +That is an honest first step, not a distributed limiter; the shape of `RateLimiter.check` and +`IdempotencyCache.get/put` is what a Redis-backed version would replace. Saying so is better than +shipping something that looks distributed and is not. +""" +from __future__ import annotations + +import os +import threading +import time +from collections import OrderedDict +from typing import Any, Optional + +__all__ = ["RateLimiter", "IdempotencyCache", "rate_limit_per_min", "idempotency_ttl"] + +DEFAULT_IDEMPOTENCY_TTL = 86_400.0 # a day: long enough to cover a client's retry budget +DEFAULT_IDEMPOTENCY_ENTRIES = 10_000 + + +def rate_limit_per_min() -> int: + """Requests per tenant per minute. 0 (the default) disables limiting entirely, so the zero-setup + demo and existing deployments behave exactly as before.""" + raw = os.environ.get("ENGRAM_RATE_LIMIT_PER_MIN", "").strip() + if not raw: + return 0 + try: + value = int(raw) + except ValueError: + return 0 + return max(0, value) + + +def idempotency_ttl() -> float: + raw = os.environ.get("ENGRAM_IDEMPOTENCY_TTL_S", "").strip() + if not raw: + return DEFAULT_IDEMPOTENCY_TTL + try: + return max(0.0, float(raw)) + except ValueError: + return DEFAULT_IDEMPOTENCY_TTL + + +class RateLimiter: + """Sliding-window limiter: at most `per_min` requests per tenant in any trailing 60 seconds.""" + + def __init__(self, per_min: int, window_seconds: float = 60.0) -> None: + self.per_min = per_min + self.window = window_seconds + self._lock = threading.Lock() + self._hits: dict[str, list[float]] = {} + + @property + def enabled(self) -> bool: + return self.per_min > 0 + + def check(self, user: str, now: Optional[float] = None) -> tuple[bool, float]: + """Record a request and decide whether it is allowed. + + Returns `(allowed, retry_after_seconds)`. A rejected request is deliberately NOT recorded -- + otherwise a client that keeps retrying would hold its own window permanently full and never + recover. `now` is injectable so the tests do not sleep. + """ + if not self.enabled: + return True, 0.0 + t = time.time() if now is None else now + cutoff = t - self.window + with self._lock: + kept = [hit for hit in self._hits.get(user, ()) if hit > cutoff] + if len(kept) >= self.per_min: + self._hits[user] = kept + return False, max(0.0, self.window - (t - kept[0])) + kept.append(t) + self._hits[user] = kept + return True, 0.0 + + def prune(self, now: Optional[float] = None) -> int: + """Drop tenants with no hits left in the window. + + Without this the map grows by one entry per tenant that ever called and never shrinks -- a slow + leak that only shows up on the deployment with the most tenants, which is the one that can least + afford it. Called opportunistically rather than on a timer, so there is no background thread. + """ + t = time.time() if now is None else now + cutoff = t - self.window + with self._lock: + stale = [user for user, hits in self._hits.items() if not any(h > cutoff for h in hits)] + for user in stale: + del self._hits[user] + return len(stale) + + @property + def tracked_tenants(self) -> int: + with self._lock: + return len(self._hits) + + +class IdempotencyCache: + """Replay the first response for an (tenant, Idempotency-Key) pair instead of re-running the work.""" + + def __init__( + self, ttl_seconds: float = DEFAULT_IDEMPOTENCY_TTL, max_entries: int = DEFAULT_IDEMPOTENCY_ENTRIES + ) -> None: + self.ttl = ttl_seconds + self.max_entries = max_entries + self._lock = threading.Lock() + # Ordered so eviction can drop the oldest entry; keyed by tenant AND key so two namespaces + # choosing the same key can never read each other's response. + self._store: "OrderedDict[tuple[str, str], tuple[float, Any]]" = OrderedDict() + + def get(self, user: str, key: str, now: Optional[float] = None) -> Optional[Any]: + if not key: + return None + t = time.time() if now is None else now + with self._lock: + hit = self._store.get((user, key)) + if hit is None: + return None + stored_at, response = hit + if t - stored_at > self.ttl: + self._store.pop((user, key), None) + return None + return response + + def put(self, user: str, key: str, response: Any, now: Optional[float] = None) -> None: + """Cache a response. Only ever called with a successful one -- replaying a failure would turn a + transient error into a permanent one for the lifetime of the entry.""" + if not key: + return + t = time.time() if now is None else now + with self._lock: + self._store[(user, key)] = (t, response) + self._store.move_to_end((user, key)) + while len(self._store) > self.max_entries: + self._store.popitem(last=False) + + def __len__(self) -> int: + with self._lock: + return len(self._store) diff --git a/tests/test_rate_limit_idempotency.py b/tests/test_rate_limit_idempotency.py new file mode 100644 index 0000000..ec480ce --- /dev/null +++ b/tests/test_rate_limit_idempotency.py @@ -0,0 +1,223 @@ +"""Per-tenant rate limiting and Idempotency-Key replay. + +Both defend the multi-tenant surface against a different failure. The limiter stops one caller spending +the whole process; idempotency stops a client's timeout-and-retry storing the same episode twice and +paying to consolidate it twice, because the first request did succeed and only its response was lost. + +Two properties are load-bearing and easy to get subtly wrong, so they are tested directly: a rejected +request must not extend its own window (or a retrying client never recovers), and a cached response must +never cross tenants. +""" +from __future__ import annotations + +import pytest + +from engram.server.limits import IdempotencyCache, RateLimiter + +# --- limiter --- + + +def test_requests_are_allowed_up_to_the_limit_then_rejected(): + limiter = RateLimiter(per_min=3) + assert [limiter.check("u1", now=100.0)[0] for _ in range(3)] == [True, True, True] + allowed, retry_after = limiter.check("u1", now=100.0) + assert allowed is False + assert 0 < retry_after <= 60 + + +def test_a_rejected_request_does_not_extend_the_window(): + """The trap: counting rejections keeps a retrying client's window permanently full.""" + limiter = RateLimiter(per_min=2) + limiter.check("u1", now=0.0) + limiter.check("u1", now=0.0) + for _ in range(10): # a client hammering while blocked + assert limiter.check("u1", now=30.0)[0] is False + assert limiter.check("u1", now=61.0)[0] is True, "the window must clear once the originals age out" + + +def test_window_slides(): + limiter = RateLimiter(per_min=2) + limiter.check("u1", now=0.0) + limiter.check("u1", now=59.0) + assert limiter.check("u1", now=59.5)[0] is False + assert limiter.check("u1", now=60.5)[0] is True, "the first hit has aged out" + + +def test_tenants_have_independent_budgets(): + limiter = RateLimiter(per_min=1) + assert limiter.check("u1", now=0.0)[0] is True + assert limiter.check("u1", now=0.0)[0] is False + assert limiter.check("u2", now=0.0)[0] is True, "one tenant must not consume another's budget" + + +def test_disabled_limiter_allows_everything(): + limiter = RateLimiter(per_min=0) + assert limiter.enabled is False + assert all(limiter.check("u1", now=0.0)[0] for _ in range(100)) + + +def test_pruning_stops_the_tenant_map_growing_without_bound(): + """Every tenant that ever called would otherwise stay in the map forever -- a slow leak that only + bites the deployment with the most tenants.""" + limiter = RateLimiter(per_min=5) + for i in range(50): + limiter.check(f"u{i}", now=0.0) + assert limiter.tracked_tenants == 50 + + limiter.check("recent", now=100.0) + assert limiter.prune(now=100.0) == 50, "the aged-out tenants should be swept" + assert limiter.tracked_tenants == 1, "the tenant still inside the window must survive" + + +def test_retry_after_shrinks_as_the_window_drains(): + limiter = RateLimiter(per_min=1) + limiter.check("u1", now=0.0) + _, early = limiter.check("u1", now=10.0) + _, late = limiter.check("u1", now=50.0) + assert early > late > 0 + + +# --- idempotency --- + + +def test_replays_the_first_response_for_a_repeated_key(): + cache = IdempotencyCache() + cache.put("u1", "k1", {"ok": True, "id": "ep_1"}, now=0.0) + assert cache.get("u1", "k1", now=1.0) == {"ok": True, "id": "ep_1"} + + +def test_cached_responses_never_cross_tenants(): + """Two namespaces picking the same key must not read each other's response.""" + cache = IdempotencyCache() + cache.put("alice", "same-key", {"owner": "alice"}, now=0.0) + assert cache.get("bob", "same-key", now=0.0) is None + + +def test_entries_expire(): + cache = IdempotencyCache(ttl_seconds=10.0) + cache.put("u1", "k1", {"ok": True}, now=0.0) + assert cache.get("u1", "k1", now=9.0) is not None + assert cache.get("u1", "k1", now=11.0) is None + + +def test_missing_key_is_never_cached(): + cache = IdempotencyCache() + cache.put("u1", "", {"ok": True}, now=0.0) + assert len(cache) == 0 + assert cache.get("u1", "", now=0.0) is None + + +def test_oldest_entries_are_evicted_at_capacity(): + cache = IdempotencyCache(max_entries=3) + for i in range(5): + cache.put("u1", f"k{i}", {"n": i}, now=float(i)) + assert len(cache) == 3 + assert cache.get("u1", "k0", now=5.0) is None + assert cache.get("u1", "k4", now=5.0) == {"n": 4} + + +# --- wiring --- + + +def _client(tmp_path, monkeypatch, **env): + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + monkeypatch.setenv("ENGRAM_DATA_DIR", str(tmp_path)) + monkeypatch.setenv("ENGRAM_OPEN", "1") + for name, value in env.items(): + monkeypatch.setenv(name, value) + + import engram.server.app as app_module + + app_module._svc = None # rebuild the service against the temp data dir + app_module._limiter = None # and the limiter against this test's configured limit + app_module._idempotency = None + return TestClient(app_module.app), app_module + + +def test_endpoint_returns_429_with_retry_after(tmp_path, monkeypatch): + client, _ = _client(tmp_path, monkeypatch, ENGRAM_RATE_LIMIT_PER_MIN="2") + headers = {"Authorization": "Bearer tenant-a"} + + assert client.post("/v1/remember", json={"content": "one"}, headers=headers).status_code == 200 + assert client.post("/v1/remember", json={"content": "two"}, headers=headers).status_code == 200 + blocked = client.post("/v1/remember", json={"content": "three"}, headers=headers) + assert blocked.status_code == 429 + assert int(blocked.headers["Retry-After"]) >= 1, "a 429 without Retry-After leaves clients guessing" + + +def test_one_tenant_cannot_exhaust_anothers_budget(tmp_path, monkeypatch): + client, _ = _client(tmp_path, monkeypatch, ENGRAM_RATE_LIMIT_PER_MIN="1") + assert client.post( + "/v1/remember", json={"content": "a"}, headers={"Authorization": "Bearer tenant-a"} + ).status_code == 200 + assert client.post( + "/v1/remember", json={"content": "a again"}, headers={"Authorization": "Bearer tenant-a"} + ).status_code == 429 + assert client.post( + "/v1/remember", json={"content": "b"}, headers={"Authorization": "Bearer tenant-b"} + ).status_code == 200 + + +def test_rate_limiting_is_off_by_default(tmp_path, monkeypatch): + """Existing deployments and the zero-setup demo must be unaffected.""" + client, _ = _client(tmp_path, monkeypatch) + headers = {"Authorization": "Bearer tenant-a"} + for i in range(12): + assert client.post("/v1/remember", json={"content": f"m{i}"}, headers=headers).status_code == 200 + + +def test_health_stays_reachable_while_a_tenant_is_limited(tmp_path, monkeypatch): + """Probes must not be rate limited, or a busy tenant takes the deployment down with it.""" + client, _ = _client(tmp_path, monkeypatch, ENGRAM_RATE_LIMIT_PER_MIN="1") + headers = {"Authorization": "Bearer tenant-a"} + client.post("/v1/remember", json={"content": "one"}, headers=headers) + assert client.post("/v1/remember", json={"content": "two"}, headers=headers).status_code == 429 + assert client.get("/health").status_code == 200 + assert client.get("/metrics").status_code == 200 + + +def test_retried_remember_stores_once(tmp_path, monkeypatch): + """The point of the header: the same call twice must leave one episode, not two.""" + client, app_module = _client(tmp_path, monkeypatch) + headers = {"Authorization": "Bearer tenant-a", "Idempotency-Key": "retry-1"} + body = {"content": "Alice works at Acme Corp."} + + first = client.post("/v1/remember", json=body, headers=headers) + second = client.post("/v1/remember", json=body, headers=headers) + assert first.status_code == second.status_code == 200 + assert first.json() == second.json(), "a retry must replay the first response verbatim" + + episodes = app_module.svc().get("tenant-a").episodes_doc.values() + assert len([ep for ep in episodes if "Acme" in ep.content]) == 1 + + +def test_without_the_header_a_repeat_is_a_new_write(tmp_path, monkeypatch): + """Idempotency is opt-in; two deliberate identical writes must both land.""" + client, app_module = _client(tmp_path, monkeypatch) + headers = {"Authorization": "Bearer tenant-a"} + body = {"content": "Alice visited Kyoto."} + client.post("/v1/remember", json=body, headers=headers) + client.post("/v1/remember", json=body, headers=headers) + + episodes = app_module.svc().get("tenant-a").episodes_doc.values() + assert len([ep for ep in episodes if "Kyoto" in ep.content]) == 2 + + +def test_idempotency_key_is_scoped_to_the_tenant(tmp_path, monkeypatch): + """Otherwise one tenant's reply could be served to another — a cross-tenant data leak.""" + client, app_module = _client(tmp_path, monkeypatch) + key = {"Idempotency-Key": "shared"} + client.post( + "/v1/remember", json={"content": "alice secret"}, + headers={"Authorization": "Bearer alice", **key}, + ) + client.post( + "/v1/remember", json={"content": "bob secret"}, + headers={"Authorization": "Bearer bob", **key}, + ) + + bob_episodes = [ep.content for ep in app_module.svc().get("bob").episodes_doc.values()] + assert any("bob secret" in text for text in bob_episodes) + assert not any("alice secret" in text for text in bob_episodes) From ba8100d3de3f5ce390b707f0a692479500c45585 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 21:10:53 +0800 Subject: [PATCH 12/25] server: runtime-issued API keys, hashed at rest, behind their own admin token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authentication was a static env map — edit ENGRAM_API_KEYS and restart to add a tenant — or open mode, where anyone is a tenant. A hosted deployment needs to mint keys while running and revoke them immediately, without ever holding the secret in a form a leaked file would expose. Keys are minted as sk-engram-*, returned exactly once, and only their SHA-256 digest is persisted. The admin surface fails closed: with ENGRAM_ADMIN_TOKEN unset there is no key-management endpoint at all, so an open-mode deployment cannot have tenants minted against it by whoever finds the URL. Resolution order is issued keys, then the static map, then open mode, so a revoked key cannot be resurrected by a stale env entry and existing deployments keep working untouched. Rewriting rather than replaying the preserved implementation caught a data-loss bug in it. A corrupt key file was swallowed and the store started empty — which rejects every issued key, and then the first issue() rewrites the file and destroys records that were only unreadable. It now refuses to load and leaves the file exactly as found: failing to start is recoverable, overwriting is not. Two smaller fixes came with it — last_used_at was mutated outside the lock, and the key file was written with default permissions even though it enumerates the tenants on the deployment. Because this is an auth surface the tests are mostly about refusing: admin absent unless enabled, wrong admin token rejected, revoked key rejected and revocation surviving a restart, an unreadable store answering 503 rather than falling through to another auth path, and neither the secret nor its digest appearing in any listing — publishing the digest would let anyone verify a guessed key offline. Lint caught a weak test on the way: the isolation case issued bob a key and never used it, so it would have passed even if bob's key resolved to alice's namespace. It now reads through the API as bob. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/server/app.py | 83 ++++++- engram/server/keys.py | 148 ++++++++++++ tests/test_api_keys.py | 246 ++++++++++++++++++++ 4 files changed, 477 insertions(+), 1 deletion(-) create mode 100644 engram/server/keys.py create mode 100644 tests/test_api_keys.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index a108e1f..62c21c5 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `self_serve_api_keys` | 服务边界 / 鉴权（多租户） | 此前只能靠静态 `ENGRAM_API_KEYS`（改环境变量+重启）或开放模式。新增 `engram/server/keys.py`：运行时签发 `sk-engram-*`、**只落盘 SHA-256 摘要**（泄露的密钥文件不可重放）、立即吊销、可列出；`/v1/admin/keys` 由独立的 `ENGRAM_ADMIN_TOKEN` 把守，未设置即 403（开放模式部署不会被路人签发租户）。解析顺序为「运行时密钥 → 静态映射 → 开放模式」，密钥库不可读时返回 503 而非放行。**修正了保全版的一处数据丢失风险**：原实现在密钥文件损坏时静默以空状态启动，随后任一次签发都会 `_save()` 覆盖该文件、永久销毁全部已签发记录；改为拒绝启动并保持文件原样。另修掉锁外改 `last_used_at` 的数据竞争，并把密钥文件权限收紧为仅属主可读写 | `tests/test_api_keys.py`（20 测试，含失败关闭、吊销后拒绝、损坏文件不被覆盖、跨租户隔离用 bob 自己的密钥走 API 验证） | | 2026-08-16 | `rate_limit` + `idempotency_key` | 服务边界 / 多租户防护 | 公开记忆 API 无限流等于开放滥用（LLM 路径接上后还是开放账单）；客户端超时重试会把同一 episode 存两遍并付两遍固化成本——首次请求其实成功了，丢的只是响应。新增 `engram/server/limits.py`：按租户滑动窗口限流（`ENGRAM_RATE_LIMIT_PER_MIN`，默认 0=关闭）+ `Idempotency-Key` 重放缓存（按「租户+key」隔离）。限流放在 `auth()` 里——那是每个受保护路由识别租户的必经点，新增端点不会漏配。**修正了保全版的一处泄漏**：租户命中表是 defaultdict 且从不回收，租户多了会无界增长，改为窗口清空即回收 | `tests/test_rate_limit_idempotency.py`（19 测试，含「被拒请求不得延长自身窗口」「缓存响应不得跨租户」「限流期间 /health 与 /metrics 仍可达」） | | 2026-08-16 | `layered_context`（默认不接主路径） | Read path / 上下文装配（Bet A 的 tokens+latency 维度） | `lean_context` 拍平成单串放进 user turn，多轮会话每轮重发不变的画像与指引。新增 `engram/retrieve/layered.py` 把上下文拆成查询无关的 stable 半（进 system prompt，可被 provider prompt-cache 复用）与每轮变化的 dynamic 半。证据完全相同，准确率按构造不变。**实测收益温和且有下限**：长会话 +5%～+9%，5 轮以下净亏；导航图默认关闭（约 20 轮才回本） | `results/layered_context_tokens.md`，`tests/test_layered_context.py`（14 测试，含"stable 半跨查询字节相同"这一缓存前提，以及"拆分不丢证据"的守卫） | | 2026-08-16 | `live_service_metrics` | 服务可观测层（Bet D 应用于线上） | 架构写明写路径 <50ms、读路径 <100ms，但服务侧没有任何实时观测，这两个目标一直是断言而非测量；token 节省比也只有离线基准，从未反映真实服务过的量。新增 `engram/metrics.py`（纯 stdlib、滑动窗口 p50/p95、单调计数器）+ `GET /metrics`，在 `remember`/`recall`/`import`/`close_session` 四个操作上埋点，并计数 `remember_degraded`（降级写入仍返回成功，不计数就完全不可见）。**修正了保全版的一个真实缺陷**：`savings_ratio` 原本用全部 context 总量做分母、只有部分调用的 full 总量做分子，两者样本集不同会系统性低估节省比；改为只在两侧都测量的配对样本上计算 | `tests/test_metrics.py`（11 测试，含配对样本回归、`/metrics` 无鉴权可访问、以及断言载荷不含租户名与内容的隐私边界测试） | diff --git a/engram/server/app.py b/engram/server/app.py index bc297f1..1e418c6 100644 --- a/engram/server/app.py +++ b/engram/server/app.py @@ -35,6 +35,7 @@ from .. import __version__ from ..service import MemoryService +from .keys import KeyStore, KeyStoreError from .limits import IdempotencyCache, RateLimiter, idempotency_ttl, rate_limit_per_min @@ -181,6 +182,8 @@ async def request_limits_and_security_headers(request: Request, call_next): return _apply_security_headers(await call_next(request), path) +_keystore: Optional[KeyStore] = None +_keystore_path: Optional[str] = None _limiter: Optional[RateLimiter] = None _limiter_per_min = -1 _idempotency: Optional[IdempotencyCache] = None @@ -254,12 +257,49 @@ def auth(authorization: str = Header(default="")) -> str: return user +def keystore() -> KeyStore: + """Runtime-issued keys, persisted beside the data dir. Built lazily so importing the app touches + no disk, and rebuilt when the service is (tests point it at a temp dir).""" + global _keystore, _keystore_path + path = os.path.join(svc().data_dir, "api_keys.json") + if _keystore is None or _keystore_path != path: + _keystore = KeyStore(path) + _keystore_path = path + return _keystore + + +def admin_auth(authorization: str = Header(default="")) -> bool: + """Gate the key-management surface behind its own token. + + Fails closed: with ENGRAM_ADMIN_TOKEN unset there is no admin surface at all, so an open-mode + deployment cannot have keys minted against it by anyone who finds the endpoint. + """ + expected = os.environ.get("ENGRAM_ADMIN_TOKEN", "").strip() + if not expected: + raise HTTPException(403, "admin surface disabled — set ENGRAM_ADMIN_TOKEN to manage API keys") + token = _bearer_token(authorization) + if not token or not hmac.compare_digest(token, expected): + raise HTTPException(401, "invalid admin token") + return True + + def _resolve_user(authorization: str) -> str: try: keys = _load_keys() except AuthConfigError as exc: raise HTTPException(503, "invalid API key configuration") from exc token = _bearer_token(authorization) + + # Runtime-issued keys are consulted first so a revoked key cannot be resurrected by a stale env + # entry, and so a hosted deployment can mint tenants without a restart. An unreadable key store is + # a 503, never an open door. + try: + issued_user = keystore().resolve(token) + except KeyStoreError as exc: + raise HTTPException(503, "invalid API key store") from exc + if issued_user: + return issued_user + if keys: matched_user = None for key, user in keys.items(): @@ -278,7 +318,48 @@ def _resolve_user(authorization: str) -> str: "missing bearer namespace: in ENGRAM_OPEN mode send Authorization: Bearer , " "or set ENGRAM_ALLOW_ANONYMOUS=1 to allow shared anonymous memory", ) - raise HTTPException(401, "set ENGRAM_API_KEYS (user:key,...) or ENGRAM_OPEN=1") + raise HTTPException( + 401, + "no API key recognized — issue one via POST /v1/admin/keys (needs ENGRAM_ADMIN_TOKEN), " + "or set ENGRAM_API_KEYS (user:key,...) or ENGRAM_OPEN=1", + ) + + +class IssueKeyReq(BaseModel): + user: str = Field(min_length=1, max_length=512) + label: str = Field(default="", max_length=256) + + +@app.post("/v1/admin/keys") +def issue_key(req: IssueKeyReq, _: bool = Depends(admin_auth)): + """Mint a key for a tenant. The plaintext is in this response and nowhere else — it is hashed at + rest, so a lost key is reissued, never recovered.""" + try: + return keystore().issue(req.user, label=req.label) + except KeyStoreError as exc: + raise HTTPException(503, "invalid API key store") from exc + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + + +@app.get("/v1/admin/keys") +def list_keys(user: Optional[str] = None, _: bool = Depends(admin_auth)): + """Key records, newest first. Never includes a secret or its digest.""" + try: + return {"keys": keystore().list(user)} + except KeyStoreError as exc: + raise HTTPException(503, "invalid API key store") from exc + + +@app.delete("/v1/admin/keys/{key_id}") +def revoke_key(key_id: str, _: bool = Depends(admin_auth)): + try: + revoked = keystore().revoke(key_id) + except KeyStoreError as exc: + raise HTTPException(503, "invalid API key store") from exc + if not revoked: + raise HTTPException(404, "no such live key") + return {"ok": True, "id": key_id, "revoked": True} class RememberReq(BaseModel): diff --git a/engram/server/keys.py b/engram/server/keys.py new file mode 100644 index 0000000..1813778 --- /dev/null +++ b/engram/server/keys.py @@ -0,0 +1,148 @@ +"""Runtime-issued API keys, stored hashed. + +The server authenticated only through a static `ENGRAM_API_KEYS` env map — edit and restart to add a +tenant — or open mode, where anyone is a tenant. A hosted deployment needs to mint keys while running, +revoke them immediately, and never hold the secret in a form that a leaked file would expose. + + * a key is minted as `sk-engram-` and returned exactly once; + * only its SHA-256 digest is persisted, so the file cannot be replayed as credentials; + * revocation drops the digest from the lookup index and takes effect on the next request. + +State is one JSON file next to the data dir — inspectable, and not a pickle. It is written atomically and +with owner-only permissions, because it lists which tenants exist even though it holds no secrets. +""" +from __future__ import annotations + +import hashlib +import json +import os +import secrets +import stat +import threading +import time +from typing import Optional + +__all__ = ["KeyStore", "KeyStoreError"] + +KEY_PREFIX = "sk-engram-" + + +class KeyStoreError(RuntimeError): + """The key file exists but could not be read. Deliberately fatal — see KeyStore._load.""" + + +def _digest(token: str) -> str: + """Only this is persisted. A leaked key file must not be replayable as credentials.""" + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +class KeyStore: + def __init__(self, path: str) -> None: + self.path = path + self._lock = threading.Lock() + self._records: dict[str, dict] = {} # key_id -> record, including the digest, never the secret + self._by_digest: dict[str, str] = {} # digest -> key_id, live keys only + self._load() + + def _load(self) -> None: + """Read the store, or fail loudly. + + A corrupt file must not be treated as an empty one. Starting empty would silently reject every + previously issued key, and then the first `issue()` would rewrite the file and destroy the + records that were merely unreadable. Refusing to start is recoverable; overwriting is not. + """ + if not os.path.exists(self.path): + return + try: + with open(self.path, encoding="utf-8") as fh: + data = json.load(fh) + except OSError as exc: + raise KeyStoreError(f"cannot read API key store at {self.path}: {exc}") from exc + except ValueError as exc: + raise KeyStoreError( + f"API key store at {self.path} is not valid JSON; refusing to start rather than " + "overwrite it — restore it from backup or move it aside" + ) from exc + if not isinstance(data, dict) or not isinstance(data.get("keys", []), list): + raise KeyStoreError(f"API key store at {self.path} has an unexpected shape") + + for rec in data["keys"]: + if not isinstance(rec, dict) or "id" not in rec or "hash" not in rec: + raise KeyStoreError(f"API key store at {self.path} has a malformed record") + self._records[rec["id"]] = rec + if not rec.get("revoked"): + self._by_digest[rec["hash"]] = rec["id"] + + def _save(self) -> None: + """Atomic replace, owner-only. Caller holds the lock.""" + directory = os.path.dirname(self.path) or "." + os.makedirs(directory, exist_ok=True) + tmp = f"{self.path}.tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump({"keys": list(self._records.values())}, fh, ensure_ascii=False, indent=2) + # Set the mode before the swap so the file is never briefly world-readable. It holds no secrets, + # but it does enumerate the tenants on this deployment. + os.chmod(tmp, stat.S_IRUSR | stat.S_IWUSR) + os.replace(tmp, self.path) + + def issue(self, user: str, label: str = "") -> dict: + """Mint a key for `user`. The plaintext is in the return value and is never stored.""" + if not user or not user.strip(): + raise ValueError("a key must belong to a tenant") + token = KEY_PREFIX + secrets.token_hex(24) + record = { + "id": "key_" + secrets.token_hex(8), + "user": user.strip(), + "label": label, + "hash": _digest(token), + "created_at": time.time(), + "revoked": False, + "last_used_at": None, + } + with self._lock: + self._records[record["id"]] = record + self._by_digest[record["hash"]] = record["id"] + self._save() + issued = self._public(record) + issued["key"] = token # shown once; there is no way to recover it later + return issued + + def resolve(self, token: str) -> Optional[str]: + """The tenant a presented token belongs to, or None if unknown or revoked.""" + if not token: + return None + with self._lock: + key_id = self._by_digest.get(_digest(token)) + if key_id is None: + return None + record = self._records.get(key_id) + if record is None or record.get("revoked"): + return None + # In memory only: flushing on every request would turn each read into a file write. + record["last_used_at"] = time.time() + return record["user"] + + def revoke(self, key_id: str) -> bool: + with self._lock: + record = self._records.get(key_id) + if record is None or record.get("revoked"): + return False + record["revoked"] = True + self._by_digest.pop(record["hash"], None) + self._save() + return True + + def list(self, user: Optional[str] = None) -> list[dict]: + """Key records, newest first. Never includes the secret or its digest.""" + with self._lock: + records = [ + self._public(rec) + for rec in self._records.values() + if user is None or rec["user"] == user + ] + return sorted(records, key=lambda rec: rec["created_at"], reverse=True) + + @staticmethod + def _public(record: dict) -> dict: + """Strip the digest. Publishing it would let anyone verify a guessed key offline.""" + return {k: v for k, v in record.items() if k != "hash"} diff --git a/tests/test_api_keys.py b/tests/test_api_keys.py new file mode 100644 index 0000000..81b687f --- /dev/null +++ b/tests/test_api_keys.py @@ -0,0 +1,246 @@ +"""Runtime-issued API keys. + +This is an authentication surface, so the tests that matter are the ones about refusing, not the ones +about admitting: the admin surface must be absent unless deliberately enabled, a revoked key must stop +working, an unreadable key store must fail closed rather than open, and one tenant's key must never +resolve to another's namespace. + +One behaviour is specifically a data-loss guard. A corrupt key file must make the store refuse to load, +because starting empty would reject every issued key and then the first `issue()` would rewrite the file +and destroy the records that were only unreadable. +""" +from __future__ import annotations + +import json +import os +import stat + +import pytest + +from engram.server.keys import KEY_PREFIX, KeyStore, KeyStoreError + +# --- store --- + + +def test_issued_key_resolves_to_its_tenant(tmp_path): + store = KeyStore(str(tmp_path / "keys.json")) + issued = store.issue("alice", label="laptop") + assert issued["key"].startswith(KEY_PREFIX) + assert store.resolve(issued["key"]) == "alice" + + +def test_secret_is_never_persisted(tmp_path): + """A leaked key file must not be replayable as credentials.""" + path = tmp_path / "keys.json" + store = KeyStore(str(path)) + issued = store.issue("alice") + on_disk = path.read_text(encoding="utf-8") + assert issued["key"] not in on_disk + assert "hash" in json.loads(on_disk)["keys"][0] + + +def test_listing_exposes_neither_secret_nor_digest(tmp_path): + """Publishing the digest would let anyone verify a guessed key offline.""" + store = KeyStore(str(tmp_path / "keys.json")) + issued = store.issue("alice") + listed = store.list() + assert len(listed) == 1 + assert "hash" not in listed[0] + assert "key" not in listed[0] + assert listed[0]["id"] == issued["id"] + + +def test_revoked_key_stops_working(tmp_path): + store = KeyStore(str(tmp_path / "keys.json")) + issued = store.issue("alice") + assert store.revoke(issued["id"]) is True + assert store.resolve(issued["key"]) is None + assert store.revoke(issued["id"]) is False, "revoking twice is not a second success" + + +def test_revocation_survives_a_restart(tmp_path): + path = str(tmp_path / "keys.json") + store = KeyStore(path) + issued = store.issue("alice") + store.revoke(issued["id"]) + assert KeyStore(path).resolve(issued["key"]) is None + + +def test_keys_survive_a_restart(tmp_path): + path = str(tmp_path / "keys.json") + issued = KeyStore(path).issue("alice") + assert KeyStore(path).resolve(issued["key"]) == "alice" + + +def test_one_tenants_key_never_resolves_to_another(tmp_path): + store = KeyStore(str(tmp_path / "keys.json")) + alice = store.issue("alice") + bob = store.issue("bob") + assert store.resolve(alice["key"]) == "alice" + assert store.resolve(bob["key"]) == "bob" + assert store.list("alice") == [rec for rec in store.list() if rec["user"] == "alice"] + + +def test_unknown_and_empty_tokens_resolve_to_nothing(tmp_path): + store = KeyStore(str(tmp_path / "keys.json")) + store.issue("alice") + assert store.resolve("") is None + assert store.resolve("sk-engram-not-a-real-key") is None + + +def test_corrupt_store_refuses_to_load_rather_than_overwrite(tmp_path): + """The data-loss guard. Starting empty would reject every issued key, and the next issue() would + rewrite the file over records that were merely unreadable.""" + path = tmp_path / "keys.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(KeyStoreError): + KeyStore(str(path)) + assert path.read_text(encoding="utf-8") == "{not json", "the unreadable file must be left intact" + + +def test_malformed_records_are_rejected(tmp_path): + path = tmp_path / "keys.json" + path.write_text(json.dumps({"keys": [{"id": "key_1"}]}), encoding="utf-8") + with pytest.raises(KeyStoreError): + KeyStore(str(path)) + + +def test_key_file_is_owner_only(tmp_path): + """It holds no secrets, but it does enumerate the tenants on this deployment.""" + path = tmp_path / "keys.json" + KeyStore(str(path)).issue("alice") + mode = stat.S_IMODE(os.stat(path).st_mode) + assert mode & (stat.S_IRWXG | stat.S_IRWXO) == 0 + + +def test_a_key_must_belong_to_a_tenant(tmp_path): + store = KeyStore(str(tmp_path / "keys.json")) + with pytest.raises(ValueError): + store.issue(" ") + + +# --- HTTP surface --- + + +def _client(tmp_path, monkeypatch, **env): + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + monkeypatch.setenv("ENGRAM_DATA_DIR", str(tmp_path)) + monkeypatch.delenv("ENGRAM_API_KEYS", raising=False) + monkeypatch.delenv("ENGRAM_OPEN", raising=False) + monkeypatch.delenv("ENGRAM_ADMIN_TOKEN", raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + + import engram.server.app as app_module + + app_module._svc = None + app_module._keystore = None + app_module._keystore_path = None + app_module._limiter = None + app_module._idempotency = None + return TestClient(app_module.app), app_module + + +def test_admin_surface_is_absent_unless_enabled(tmp_path, monkeypatch): + """Fail closed: an open deployment must not let a passer-by mint tenants.""" + client, _ = _client(tmp_path, monkeypatch, ENGRAM_OPEN="1") + response = client.post("/v1/admin/keys", json={"user": "mallory"}) + assert response.status_code == 403 + + +def test_wrong_admin_token_is_rejected(tmp_path, monkeypatch): + client, _ = _client(tmp_path, monkeypatch, ENGRAM_ADMIN_TOKEN="s3cret") + assert client.post( + "/v1/admin/keys", json={"user": "alice"}, headers={"Authorization": "Bearer wrong"} + ).status_code == 401 + assert client.post("/v1/admin/keys", json={"user": "alice"}).status_code == 401 + + +def test_issued_key_authenticates_a_real_request(tmp_path, monkeypatch): + client, _ = _client(tmp_path, monkeypatch, ENGRAM_ADMIN_TOKEN="s3cret") + admin = {"Authorization": "Bearer s3cret"} + + issued = client.post("/v1/admin/keys", json={"user": "alice"}, headers=admin).json() + assert client.post( + "/v1/remember", json={"content": "Alice works at Acme."}, + headers={"Authorization": f"Bearer {issued['key']}"}, + ).status_code == 200 + + +def test_revoked_key_is_rejected_by_the_api(tmp_path, monkeypatch): + client, _ = _client(tmp_path, monkeypatch, ENGRAM_ADMIN_TOKEN="s3cret") + admin = {"Authorization": "Bearer s3cret"} + issued = client.post("/v1/admin/keys", json={"user": "alice"}, headers=admin).json() + caller = {"Authorization": f"Bearer {issued['key']}"} + + assert client.post("/v1/remember", json={"content": "one"}, headers=caller).status_code == 200 + assert client.delete(f"/v1/admin/keys/{issued['id']}", headers=admin).status_code == 200 + assert client.post("/v1/remember", json={"content": "two"}, headers=caller).status_code == 401 + assert client.delete(f"/v1/admin/keys/{issued['id']}", headers=admin).status_code == 404 + + +def test_issued_keys_isolate_namespaces(tmp_path, monkeypatch): + """The isolation the whole multi-tenant model rests on — checked through the API with bob's own + key, not by inspecting the service, so it would fail if bob's key resolved to alice's namespace.""" + client, _ = _client(tmp_path, monkeypatch, ENGRAM_ADMIN_TOKEN="s3cret") + admin = {"Authorization": "Bearer s3cret"} + alice = client.post("/v1/admin/keys", json={"user": "alice"}, headers=admin).json() + bob = client.post("/v1/admin/keys", json={"user": "bob"}, headers=admin).json() + + client.post( + "/v1/remember", json={"content": "Alice's private note about the merger."}, + headers={"Authorization": f"Bearer {alice['key']}"}, + ) + + as_bob = {"Authorization": f"Bearer {bob['key']}"} + recalled = client.post("/v1/recall", json={"query": "merger"}, headers=as_bob) + assert recalled.status_code == 200 + assert "merger" not in recalled.json().get("context", "") + + listed = client.get("/v1/memories", headers=as_bob) + assert "Alice's private note" not in listed.text + + +def test_static_env_keys_still_work_alongside_issued_ones(tmp_path, monkeypatch): + """Existing deployments must not break when the self-serve path appears.""" + client, _ = _client( + tmp_path, monkeypatch, ENGRAM_ADMIN_TOKEN="s3cret", ENGRAM_API_KEYS="carol:sk-carol" + ) + assert client.post( + "/v1/remember", json={"content": "hi"}, headers={"Authorization": "Bearer sk-carol"} + ).status_code == 200 + + issued = client.post( + "/v1/admin/keys", json={"user": "alice"}, headers={"Authorization": "Bearer s3cret"} + ).json() + assert client.post( + "/v1/remember", json={"content": "hi"}, + headers={"Authorization": f"Bearer {issued['key']}"}, + ).status_code == 200 + + +def test_unreadable_key_store_fails_closed(tmp_path, monkeypatch): + """A broken store must reject requests, never wave them through.""" + client, app_module = _client(tmp_path, monkeypatch, ENGRAM_API_KEYS="carol:sk-carol") + app_module.svc() # ensure the data dir exists before corrupting the file inside it + with open(os.path.join(str(tmp_path), "api_keys.json"), "w", encoding="utf-8") as fh: + fh.write("{not json") + app_module._keystore = None + app_module._keystore_path = None + + response = client.post( + "/v1/remember", json={"content": "hi"}, headers={"Authorization": "Bearer sk-carol"} + ) + assert response.status_code == 503, "an unreadable key store must not fall through to other auth" + + +def test_listing_over_http_hides_secrets(tmp_path, monkeypatch): + client, _ = _client(tmp_path, monkeypatch, ENGRAM_ADMIN_TOKEN="s3cret") + admin = {"Authorization": "Bearer s3cret"} + issued = client.post("/v1/admin/keys", json={"user": "alice"}, headers=admin).json() + + body = client.get("/v1/admin/keys", headers=admin).text + assert issued["key"] not in body + assert "hash" not in json.loads(body)["keys"][0] From 19251425bff9ae02bd8ed067b1cd2fe2b8648e64 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 21:29:59 +0800 Subject: [PATCH 13/25] client: a Python SDK with zero runtime dependencies The agent ecosystem is Python-first and the only SDK was TypeScript, so a Python caller had to hand-roll HTTP against the API. This ships inside the core package and speaks over stdlib urllib, keeping the promise that installing Engram pulls in nothing. Method names mirror the TS client so the two read the same way and neither drifts into being the real one, and it covers what the server actually exposes today -- including /metrics, /v1/admin/keys and Idempotency-Key, none of which existed when the preserved version was written. Writing it surfaced a flaw in my own first draft: the transport returned (status, body), so EngramError.retry_after could never be populated -- Retry-After is a header. A field that is always None is worse than no field, because a caller writes a backoff around it and it silently never fires. The transport contract now carries headers, which is also what a custom transport needs in order to be a faithful substitute. The tests drive the SDK against the real application through the injectable transport rather than a mock. A mocked client test only proves the SDK agrees with itself, and agreeing with itself while drifting from the server is precisely how client libraries fail. So the rate-limit test provokes a real 429 and asserts retry_after survives, the idempotency test asserts one episode landed rather than two, and the admin tests check that a tenant key cannot mint tenants. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/client.py | 315 ++++++++++++++++++++ tests/test_client.py | 195 ++++++++++++ 3 files changed, 511 insertions(+) create mode 100644 engram/client.py create mode 100644 tests/test_client.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 62c21c5..6b5c877 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `python_sdk` | 接入层 / 客户端 | 主干只有 TS 客户端，而 Agent 生态是 Python 优先的。新增 `engram/client.py`：零运行时依赖（stdlib urllib）、方法名对齐 TS SDK 避免两者漂移、覆盖今日全部端点（含新增的 `/metrics`、`/v1/admin/keys`、`Idempotency-Key`）。**transport 契约改为返回 `(status, headers, body)`**——最初只返回 `(status, body)`，导致 `EngramError.retry_after` 永远为 `None`，因为 `Retry-After` 在响应头里；发一个永远为空的字段比不发更糟 | `tests/test_client.py`（17 测试，全部通过可注入 transport 打到**真实 app**而非 mock——mock 只能证明 SDK 自洽，而客户端库的典型失效恰恰是与服务端漂移） | | 2026-08-16 | `self_serve_api_keys` | 服务边界 / 鉴权（多租户） | 此前只能靠静态 `ENGRAM_API_KEYS`（改环境变量+重启）或开放模式。新增 `engram/server/keys.py`：运行时签发 `sk-engram-*`、**只落盘 SHA-256 摘要**（泄露的密钥文件不可重放）、立即吊销、可列出；`/v1/admin/keys` 由独立的 `ENGRAM_ADMIN_TOKEN` 把守，未设置即 403（开放模式部署不会被路人签发租户）。解析顺序为「运行时密钥 → 静态映射 → 开放模式」，密钥库不可读时返回 503 而非放行。**修正了保全版的一处数据丢失风险**：原实现在密钥文件损坏时静默以空状态启动，随后任一次签发都会 `_save()` 覆盖该文件、永久销毁全部已签发记录；改为拒绝启动并保持文件原样。另修掉锁外改 `last_used_at` 的数据竞争，并把密钥文件权限收紧为仅属主可读写 | `tests/test_api_keys.py`（20 测试，含失败关闭、吊销后拒绝、损坏文件不被覆盖、跨租户隔离用 bob 自己的密钥走 API 验证） | | 2026-08-16 | `rate_limit` + `idempotency_key` | 服务边界 / 多租户防护 | 公开记忆 API 无限流等于开放滥用（LLM 路径接上后还是开放账单）；客户端超时重试会把同一 episode 存两遍并付两遍固化成本——首次请求其实成功了，丢的只是响应。新增 `engram/server/limits.py`：按租户滑动窗口限流（`ENGRAM_RATE_LIMIT_PER_MIN`，默认 0=关闭）+ `Idempotency-Key` 重放缓存（按「租户+key」隔离）。限流放在 `auth()` 里——那是每个受保护路由识别租户的必经点，新增端点不会漏配。**修正了保全版的一处泄漏**：租户命中表是 defaultdict 且从不回收，租户多了会无界增长，改为窗口清空即回收 | `tests/test_rate_limit_idempotency.py`（19 测试，含「被拒请求不得延长自身窗口」「缓存响应不得跨租户」「限流期间 /health 与 /metrics 仍可达」） | | 2026-08-16 | `layered_context`（默认不接主路径） | Read path / 上下文装配（Bet A 的 tokens+latency 维度） | `lean_context` 拍平成单串放进 user turn，多轮会话每轮重发不变的画像与指引。新增 `engram/retrieve/layered.py` 把上下文拆成查询无关的 stable 半（进 system prompt，可被 provider prompt-cache 复用）与每轮变化的 dynamic 半。证据完全相同，准确率按构造不变。**实测收益温和且有下限**：长会话 +5%～+9%，5 轮以下净亏；导航图默认关闭（约 20 轮才回本） | `results/layered_context_tokens.md`，`tests/test_layered_context.py`（14 测试，含"stable 半跨查询字节相同"这一缓存前提，以及"拆分不丢证据"的守卫） | diff --git a/engram/client.py b/engram/client.py new file mode 100644 index 0000000..2c62462 --- /dev/null +++ b/engram/client.py @@ -0,0 +1,315 @@ +"""EngramClient — the Python SDK for the Engram memory server. + +The agent ecosystem is Python-first, so this ships inside the core package rather than as a separate +install: `pip install engram-memory`, then `from engram.client import EngramClient`. It is the peer of +the TypeScript SDK in clients/typescript and deliberately mirrors its method names, so the two read the +same way and neither drifts into being the "real" one. + +Zero runtime dependencies — it speaks to the server over stdlib urllib, keeping the charter's promise +that installing Engram pulls in nothing. Like the TS client's injectable `fetch`, `transport` swaps the +HTTP layer for httpx, requests, or an in-process test client without touching call sites. + + from engram.client import EngramClient + + engram = EngramClient(base_url="http://localhost:8000", api_key="sk-engram-...") + engram.remember("I live in Shenzhen and work on retrieval.") + print(engram.recall("where do I live?")["context"]) + +One Bearer key is one isolated namespace on the server, so a client instance is a tenant. +""" +from __future__ import annotations + +import json +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Callable, Optional + +__all__ = ["EngramClient", "EngramError", "Transport"] + +# One request in, (status, response headers, body) out. Never raises for HTTP status -- the client maps +# status to errors, so a custom transport only has to move bytes. Headers are part of the contract +# because some of the server's answer lives there: a 429 carries Retry-After, and an error object that +# cannot report how long to wait is an error object nobody can act on. +Transport = Callable[[str, str, dict, Optional[bytes], float], "tuple[int, dict, bytes]"] + + +class EngramError(Exception): + """Any non-2xx response, or a connection failure (status 0). + + `status` lets a caller branch without parsing prose: 401 means the key is wrong, 429 means back off + (`retry_after` carries the server's Retry-After), 503 means the server is misconfigured rather than + the request being bad. + """ + + def __init__(self, status: int, message: str, detail: Any = None, retry_after: Optional[int] = None) -> None: + super().__init__(message) + self.status = status + self.detail = detail + self.retry_after = retry_after + + +def _safe_json(raw: bytes) -> Any: + try: + return json.loads(raw.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + return None + + +def _urllib_transport( + method: str, url: str, headers: dict, body: Optional[bytes], timeout: float +) -> "tuple[int, dict, bytes]": + request = urllib.request.Request(url, data=body, method=method) + for name, value in headers.items(): + request.add_header(name, value) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - caller's URL + return response.status, dict(response.headers), response.read() + except urllib.error.HTTPError as exc: # a non-2xx response still carries a readable body + return exc.code, dict(exc.headers or {}), exc.read() + except urllib.error.URLError as exc: # never reached the server: DNS, refused, TLS + raise EngramError(0, f"cannot reach Engram at {url}: {exc.reason}") from exc + + +class EngramClient: + def __init__( + self, + base_url: str = "http://localhost:8000", + api_key: Optional[str] = None, + timeout: float = 30.0, + transport: Optional[Transport] = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.timeout = timeout + self._transport: Transport = transport or _urllib_transport + + # --- plumbing --- + + def _request( + self, + method: str, + path: str, + body: Any = None, + params: Optional[dict] = None, + extra_headers: Optional[dict] = None, + ) -> Any: + url = f"{self.base_url}{path}" + if params: + clean = {k: v for k, v in params.items() if v is not None} + if clean: + url = f"{url}?{urllib.parse.urlencode(clean)}" + + headers = {"Accept": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = "application/json" + if extra_headers: + headers.update(extra_headers) + + status, response_headers, raw = self._transport(method, url, headers, data, self.timeout) + if status < 200 or status >= 300: + raise self._error(status, response_headers, raw) + return _safe_json(raw) + + @staticmethod + def _error(status: int, headers: dict, raw: bytes) -> EngramError: + detail = _safe_json(raw) + message = f"Engram request failed ({status})" + if isinstance(detail, dict) and detail.get("detail"): + reported = detail["detail"] + message = reported if isinstance(reported, str) else json.dumps(reported, ensure_ascii=False) + # Header names are case-insensitive on the wire; normalise rather than trust one spelling. + lowered = {str(k).lower(): v for k, v in (headers or {}).items()} + try: + retry_after = int(lowered["retry-after"]) + except (KeyError, TypeError, ValueError): + retry_after = None + return EngramError(status, message, detail, retry_after) + + @staticmethod + def _idempotency(key: Optional[str]) -> Optional[dict]: + return {"Idempotency-Key": key} if key else None + + # --- service --- + + def health(self) -> dict: + return self._request("GET", "/health") + + def ready(self) -> dict: + return self._request("GET", "/ready") + + def metrics(self) -> dict: + """Aggregate latency, volume and token counters. Unauthenticated on the server.""" + return self._request("GET", "/metrics") + + # --- writing --- + + def remember( + self, + content: str, + session_id: str = "default", + scope: str = "auto", + idempotency_key: Optional[str] = None, + ) -> dict: + """Store a message. Pass `idempotency_key` so a retry after a timeout does not store it twice.""" + return self._request( + "POST", + "/v1/remember", + {"content": content, "session_id": session_id, "scope": scope}, + extra_headers=self._idempotency(idempotency_key), + ) + + def import_( + self, + sessions: Optional[list] = None, + data: Any = None, + format: str = "auto", + consolidate: bool = True, + summarize: bool = True, + idempotency_key: Optional[str] = None, + ) -> dict: + """Bulk-ingest a history. The most expensive call there is, so it is the one most worth an + idempotency key.""" + return self._request( + "POST", + "/v1/import", + { + "sessions": sessions, + "data": data, + "format": format, + "consolidate": consolidate, + "summarize": summarize, + }, + extra_headers=self._idempotency(idempotency_key), + ) + + def export(self, include_sensitive: bool = False) -> dict: + return self._request("GET", "/v1/export", params={"include_sensitive": include_sensitive}) + + def forget(self, confirm: bool = False) -> dict: + """Erase this namespace. Requires `confirm=True`; the server refuses otherwise.""" + return self._request("POST", "/v1/forget", {}, params={"confirm": confirm}) + + # --- reading --- + + def recall( + self, + query: str, + lean: bool = True, + n_chunks: int = 6, + session_id: Optional[str] = None, + as_of: Optional[float] = None, + redact_sensitive: bool = False, + answer: bool = False, + ) -> dict: + return self._request( + "POST", + "/v1/recall", + { + "query": query, + "lean": lean, + "n_chunks": n_chunks, + "session_id": session_id, + "as_of": as_of, + "redact_sensitive": redact_sensitive, + "answer": answer, + }, + ) + + def search(self, query: str, **kwargs: Any) -> dict: + """A direct factual answer rather than a context to answer from.""" + return self.recall(query, lean=False, **kwargs) + + def memories(self, limit: Optional[int] = None, offset: int = 0, **params: Any) -> dict: + return self._request("GET", "/v1/memories", params={"limit": limit, "offset": offset, **params}) + + def profile(self, structured: bool = False) -> dict: + return self._request("GET", "/v1/profile/structured" if structured else "/v1/profile") + + def stats(self) -> dict: + return self._request("GET", "/v1/stats") + + def graph(self, **params: Any) -> dict: + return self._request("GET", "/v1/graph", params=params) + + def agent_status(self, session_id: Optional[str] = None) -> dict: + return self._request("GET", "/v1/agent/status", params={"session_id": session_id}) + + # --- sessions --- + + def sessions(self, **params: Any) -> dict: + return self._request("GET", "/v1/sessions", params=params) + + def session_report(self, session_id: str, **params: Any) -> dict: + return self._request("GET", "/v1/sessions/report", params={"session_id": session_id, **params}) + + def close_session(self, session_id: str = "default", summarize: bool = True, + clear_working: bool = True) -> dict: + return self._request( + "POST", + "/v1/sessions/close", + {"session_id": session_id, "summarize": summarize, "clear_working": clear_working}, + ) + + # --- facts --- + + def add_fact(self, subject: str, predicate: str, object: str, **fields: Any) -> dict: + return self._request( + "POST", "/v1/facts", {"subject": subject, "predicate": predicate, "object": object, **fields} + ) + + def delete_fact(self, fact_id: str) -> dict: + return self._request("DELETE", f"/v1/facts/{urllib.parse.quote(fact_id, safe='')}") + + def conflicts(self) -> dict: + return self._request("GET", "/v1/conflicts") + + def resolve_conflict(self, conflict_id: str, keep: str = "newer") -> dict: + path = f"/v1/conflicts/{urllib.parse.quote(conflict_id, safe='')}/resolve" + return self._request("POST", path, {"keep": keep}) + + # --- working memory, focus, policy --- + + def add_working(self, content: str, session_id: str = "default", kind: str = "state", + **fields: Any) -> dict: + return self._request( + "POST", "/v1/working", + {"content": content, "session_id": session_id, "kind": kind, **fields}, + ) + + def working_memory(self, session_id: Optional[str] = None) -> dict: + return self._request("GET", "/v1/working", params={"session_id": session_id}) + + def clear_working(self, session_id: str) -> dict: + return self._request("DELETE", "/v1/working", params={"session_id": session_id}) + + def get_focus(self) -> dict: + return self._request("GET", "/v1/focus") + + def set_focus(self, track: Optional[list] = None, mute: Optional[list] = None) -> dict: + return self._request("PUT", "/v1/focus", {"track": track, "mute": mute}) + + def get_policy(self) -> dict: + return self._request("GET", "/v1/policy") + + def set_policy(self, **fields: Any) -> dict: + return self._request("PUT", "/v1/policy", fields) + + # --- admin --- + # + # These need ENGRAM_ADMIN_TOKEN as the api_key, not a tenant key -- a separate client instance, + # because mixing an admin token into a tenant client would send it on every ordinary call. + + def issue_key(self, user: str, label: str = "") -> dict: + """Mint a tenant key. The plaintext is in the response and nowhere else.""" + return self._request("POST", "/v1/admin/keys", {"user": user, "label": label}) + + def list_keys(self, user: Optional[str] = None) -> dict: + return self._request("GET", "/v1/admin/keys", params={"user": user}) + + def revoke_key(self, key_id: str) -> dict: + return self._request("DELETE", f"/v1/admin/keys/{urllib.parse.quote(key_id, safe='')}") diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..b2ed807 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,195 @@ +"""The Python SDK. + +Tested against the real application rather than a mock: the injectable transport routes calls through +FastAPI's TestClient, so every assertion here is about the actual request the server receives and the +actual response it sends. A mocked SDK test only proves the SDK agrees with itself, which is exactly the +failure mode a client library has — drifting from the server it claims to speak to. +""" +from __future__ import annotations + +import pytest + +from engram.client import EngramClient, EngramError + +pytest.importorskip("fastapi") + + +def _sdk(tmp_path, monkeypatch, api_key="tenant-a", **env): + """An EngramClient wired to an in-process server, plus the app module for direct inspection.""" + from fastapi.testclient import TestClient + + monkeypatch.setenv("ENGRAM_DATA_DIR", str(tmp_path)) + monkeypatch.delenv("ENGRAM_API_KEYS", raising=False) + monkeypatch.delenv("ENGRAM_ADMIN_TOKEN", raising=False) + monkeypatch.setenv("ENGRAM_OPEN", "1") + for name, value in env.items(): + monkeypatch.setenv(name, value) + + import engram.server.app as app_module + + app_module._svc = None + app_module._keystore = None + app_module._keystore_path = None + app_module._limiter = None + app_module._idempotency = None + http = TestClient(app_module.app) + + seen: list[dict] = [] + + def transport(method, url, headers, body, timeout): + path = url[len("http://testserver"):] if url.startswith("http://testserver") else url + seen.append({"method": method, "path": path, "headers": headers}) + response = http.request(method, path, content=body, headers=headers) + return response.status_code, dict(response.headers), response.content + + client = EngramClient(base_url="http://testserver", api_key=api_key, transport=transport) + return client, app_module, seen + + +# --- round trips --- + + +def test_remember_then_recall(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch) + assert client.remember("Alice works at Acme Corp.")["ok"] is True + assert "Acme" in client.recall("where does alice work")["context"] + + +def test_health_and_metrics_need_no_key(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch, api_key=None) + assert client.health()["service"] == "engram" + assert set(client.metrics()) == {"uptime_s", "ops", "counts", "tokens"} + + +def test_profile_stats_and_memories(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch) + client.remember("Alice prefers oat milk.") + assert isinstance(client.profile(), dict) + assert isinstance(client.profile(structured=True), dict) + assert isinstance(client.stats(), dict) + assert isinstance(client.memories(limit=5), dict) + + +def test_sessions_and_working_memory(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch) + client.remember("A note.", session_id="s1") + client.add_working("today my throat hurts", session_id="s1") + assert isinstance(client.working_memory(session_id="s1"), dict) + assert client.close_session("s1") + assert isinstance(client.sessions(), dict) + + +def test_focus_and_policy_round_trip(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch) + client.set_focus(track=["cycling"], mute=[]) + assert "cycling" in str(client.get_focus()) + assert isinstance(client.get_policy(), dict) + + +def test_facts_and_export(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch) + client.add_fact("alice", "lives_in", "Shenzhen") + assert isinstance(client.export(), dict) + assert isinstance(client.conflicts(), dict) + + +# --- request shape --- + + +def test_bearer_key_is_sent(tmp_path, monkeypatch): + client, _, seen = _sdk(tmp_path, monkeypatch, api_key="tenant-x") + client.remember("hello") + assert seen[-1]["headers"]["Authorization"] == "Bearer tenant-x" + + +def test_no_key_means_no_authorization_header(tmp_path, monkeypatch): + client, _, seen = _sdk(tmp_path, monkeypatch, api_key=None) + client.health() + assert "Authorization" not in seen[-1]["headers"] + + +def test_query_params_skip_none_rather_than_sending_the_word_none(tmp_path, monkeypatch): + """A None that reaches the URL becomes the literal string 'None' and the server filters on it.""" + client, _, seen = _sdk(tmp_path, monkeypatch) + client.agent_status(session_id=None) + assert "session_id" not in seen[-1]["path"] + client.agent_status(session_id="s1") + assert "session_id=s1" in seen[-1]["path"] + + +def test_path_ids_are_escaped(tmp_path, monkeypatch): + """An id with a slash must not silently address a different route.""" + client, _, seen = _sdk(tmp_path, monkeypatch) + with pytest.raises(EngramError): + client.delete_fact("a/b") + assert "a%2Fb" in seen[-1]["path"] + + +def test_idempotency_key_is_sent_and_honoured(tmp_path, monkeypatch): + client, app_module, seen = _sdk(tmp_path, monkeypatch) + first = client.remember("Alice visited Kyoto.", idempotency_key="retry-1") + assert seen[-1]["headers"]["Idempotency-Key"] == "retry-1" + second = client.remember("Alice visited Kyoto.", idempotency_key="retry-1") + assert first == second + + episodes = app_module.svc().get("tenant-a").episodes_doc.values() + assert len([ep for ep in episodes if "Kyoto" in ep.content]) == 1 + + +def test_no_idempotency_header_when_unset(tmp_path, monkeypatch): + client, _, seen = _sdk(tmp_path, monkeypatch) + client.remember("no key") + assert "Idempotency-Key" not in seen[-1]["headers"] + + +# --- errors --- + + +def test_non_2xx_raises_with_status_and_server_message(tmp_path, monkeypatch): + client, _, _ = _sdk(tmp_path, monkeypatch) + with pytest.raises(EngramError) as caught: + client.forget(confirm=False) # the server refuses without explicit confirmation + assert caught.value.status == 400 + assert "confirm" in str(caught.value).lower() + + +def test_rate_limited_error_carries_retry_after(tmp_path, monkeypatch): + """Retry-After lives in a header, so an SDK whose transport drops headers cannot report it — which + is what makes a 429 unactionable for the caller.""" + client, _, _ = _sdk(tmp_path, monkeypatch, ENGRAM_RATE_LIMIT_PER_MIN="1") + client.remember("one") + with pytest.raises(EngramError) as caught: + client.remember("two") + assert caught.value.status == 429 + assert caught.value.retry_after is not None and caught.value.retry_after >= 1 + + +def test_unreachable_server_raises_rather_than_hanging(): + """A connection failure is status 0 -- distinguishable from any answer the server could give.""" + client = EngramClient(base_url="http://127.0.0.1:1", api_key="k", timeout=1.0) + with pytest.raises(EngramError) as caught: + client.health() + assert caught.value.status == 0 + + +# --- admin --- + + +def test_admin_key_lifecycle_through_the_sdk(tmp_path, monkeypatch): + client, app_module, _ = _sdk(tmp_path, monkeypatch, api_key="s3cret", ENGRAM_ADMIN_TOKEN="s3cret") + + issued = client.issue_key("alice", label="laptop") + assert issued["key"].startswith("sk-engram-") + assert any(rec["id"] == issued["id"] for rec in client.list_keys()["keys"]) + assert client.revoke_key(issued["id"])["revoked"] is True + with pytest.raises(EngramError) as caught: + client.revoke_key(issued["id"]) + assert caught.value.status == 404 + + +def test_admin_surface_refuses_a_tenant_key(tmp_path, monkeypatch): + """A tenant key must not be able to mint tenants.""" + client, _, _ = _sdk(tmp_path, monkeypatch, api_key="tenant-a", ENGRAM_ADMIN_TOKEN="s3cret") + with pytest.raises(EngramError) as caught: + client.issue_key("mallory") + assert caught.value.status == 401 From 57f647553ba0d2eb0b483b4b2c8bc534d9d8395a Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 21:48:18 +0800 Subject: [PATCH 14/25] server: count what the rate limiter, idempotency cache and key resolution actually do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defences shipped over the last rounds and all three are invisible from outside. An operator cannot tell whether throttling has ever fired, whether clients set Idempotency-Key at all, or whether a rising tide of rejected keys means a misconfiguration or someone trying keys. /metrics now carries rate_limited, idempotent_replays, auth_rejected and auth_misconfigured. Deliberately aggregate, with no per-tenant breakdown. The endpoint is unauthenticated, and a counter keyed by tenant would tell any caller which tenants exist on the deployment — the same reason the rest of the payload carries no namespace names. The counters route through a helper that swallows its own failures, because instrumentation must not change behaviour. Building the service can itself fail on a misconfigured backend, and that surfacing from inside an exception handler would replace a precise 401 with a generic 500 — losing exactly the diagnosis the counter exists to support. auth_misconfigured is separate from auth_rejected on purpose: a 503 means the operator broke the key configuration, a 401 means the caller presented the wrong key, and an operator paging on the wrong one of those wastes the outage. --- docs/architecture-optimization-map.zh-CN.md | 1 + engram/server/app.py | 30 ++++++++- tests/test_metrics.py | 71 +++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 6b5c877..26e31a4 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `defence_counters` | 服务可观测层 | 限流、幂等、密钥解析三层防护此前是黑盒：运维无法判断它们是否在生效。`/metrics` 新增 `rate_limited` / `idempotent_replays` / `auth_rejected` / `auth_misconfigured` 四个聚合计数。**刻意不按租户分桶**——该端点不鉴权，分桶会暴露有哪些租户存在。埋点统一走 `_count()`，其中吞掉自身异常：埋点不得改变请求行为，而在异常处理器里构造服务失败会把一个精确的 401 换成笼统的 500，恰好丢掉计数本来要支撑的诊断 | `tests/test_metrics.py`（+4 测试，含首次调用不算重放、以及计数载荷不含租户名的隐私断言） | | 2026-08-16 | `python_sdk` | 接入层 / 客户端 | 主干只有 TS 客户端，而 Agent 生态是 Python 优先的。新增 `engram/client.py`：零运行时依赖（stdlib urllib）、方法名对齐 TS SDK 避免两者漂移、覆盖今日全部端点（含新增的 `/metrics`、`/v1/admin/keys`、`Idempotency-Key`）。**transport 契约改为返回 `(status, headers, body)`**——最初只返回 `(status, body)`，导致 `EngramError.retry_after` 永远为 `None`，因为 `Retry-After` 在响应头里；发一个永远为空的字段比不发更糟 | `tests/test_client.py`（17 测试，全部通过可注入 transport 打到**真实 app**而非 mock——mock 只能证明 SDK 自洽，而客户端库的典型失效恰恰是与服务端漂移） | | 2026-08-16 | `self_serve_api_keys` | 服务边界 / 鉴权（多租户） | 此前只能靠静态 `ENGRAM_API_KEYS`（改环境变量+重启）或开放模式。新增 `engram/server/keys.py`：运行时签发 `sk-engram-*`、**只落盘 SHA-256 摘要**（泄露的密钥文件不可重放）、立即吊销、可列出；`/v1/admin/keys` 由独立的 `ENGRAM_ADMIN_TOKEN` 把守，未设置即 403（开放模式部署不会被路人签发租户）。解析顺序为「运行时密钥 → 静态映射 → 开放模式」，密钥库不可读时返回 503 而非放行。**修正了保全版的一处数据丢失风险**：原实现在密钥文件损坏时静默以空状态启动，随后任一次签发都会 `_save()` 覆盖该文件、永久销毁全部已签发记录；改为拒绝启动并保持文件原样。另修掉锁外改 `last_used_at` 的数据竞争，并把密钥文件权限收紧为仅属主可读写 | `tests/test_api_keys.py`（20 测试，含失败关闭、吊销后拒绝、损坏文件不被覆盖、跨租户隔离用 bob 自己的密钥走 API 验证） | | 2026-08-16 | `rate_limit` + `idempotency_key` | 服务边界 / 多租户防护 | 公开记忆 API 无限流等于开放滥用（LLM 路径接上后还是开放账单）；客户端超时重试会把同一 episode 存两遍并付两遍固化成本——首次请求其实成功了，丢的只是响应。新增 `engram/server/limits.py`：按租户滑动窗口限流（`ENGRAM_RATE_LIMIT_PER_MIN`，默认 0=关闭）+ `Idempotency-Key` 重放缓存（按「租户+key」隔离）。限流放在 `auth()` 里——那是每个受保护路由识别租户的必经点，新增端点不会漏配。**修正了保全版的一处泄漏**：租户命中表是 defaultdict 且从不回收，租户多了会无界增长，改为窗口清空即回收 | `tests/test_rate_limit_idempotency.py`（19 测试，含「被拒请求不得延长自身窗口」「缓存响应不得跨租户」「限流期间 /health 与 /metrics 仍可达」） | diff --git a/engram/server/app.py b/engram/server/app.py index 1e418c6..9495be1 100644 --- a/engram/server/app.py +++ b/engram/server/app.py @@ -210,6 +210,19 @@ def _idempotency_cache() -> IdempotencyCache: return _idempotency +def _count(name: str) -> None: + """Bump an aggregate counter, never at the cost of the request. + + Instrumentation must not change behaviour. Building the service can itself fail (a misconfigured + backend), and that failure surfacing from inside an exception handler would replace a precise 401 + with a generic 500 — losing the very diagnosis the counter exists to support. + """ + try: + svc().metrics.count(name) + except Exception: # noqa: BLE001 - a missing metric is never worth failing a request over + pass + + def _enforce_rate_limit(user: str) -> None: limiter = _rate_limiter() if not limiter.enabled: @@ -218,6 +231,9 @@ def _enforce_rate_limit(user: str) -> None: limiter.prune() allowed, retry_after = limiter.check(user) if not allowed: + # Counted, not logged per tenant: /metrics is unauthenticated, so a per-tenant breakdown would + # tell any caller which tenants exist. An operator needs to know throttling is happening at all. + _count("rate_limited") # Retry-After is what makes a 429 actionable rather than a client guessing and hammering. raise HTTPException( 429, "rate limit exceeded", headers={"Retry-After": str(int(retry_after) + 1)} @@ -238,6 +254,9 @@ def idempotent(user: str, request: Request, compute): cache = _idempotency_cache() cached = cache.get(user, key) if cached is not None: + # A replay is work the server did not have to redo. Counting it is how an operator learns the + # header is actually being used, rather than assuming clients set it. + _count("idempotent_replays") return cached result = compute() cache.put(user, key, result) @@ -252,7 +271,16 @@ def auth(authorization: str = Header(default="")) -> str: itself as the namespace, so anyone can try it without pre-provisioned keys while still avoiding shared anonymous memory unless it is explicitly enabled. """ - user = _resolve_user(authorization) + try: + user = _resolve_user(authorization) + except HTTPException as exc: + # A rising rejection count is how a misconfigured deployment or a credential-stuffing attempt + # becomes visible. Bucketed by status only -- the presented token is never recorded anywhere. + if exc.status_code in {401, 403}: + _count("auth_rejected") + elif exc.status_code == 503: + _count("auth_misconfigured") + raise _enforce_rate_limit(user) return user diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 957350a..02ebd6e 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -162,3 +162,74 @@ def test_metrics_endpoint_is_open_and_aggregate(tmp_path, monkeypatch): after = client.get("/metrics").json() assert after["ops"]["remember"]["n"] >= 1 assert "tenant-x" not in json.dumps(after) + + +# --- defence counters --- +# +# The rate limiter, the idempotency cache and key resolution are otherwise black boxes: an operator +# cannot tell whether they are doing anything. These counters are aggregate on purpose — /metrics is +# unauthenticated, so a per-tenant breakdown would reveal which tenants exist. + + +def _guarded_client(tmp_path, monkeypatch, **env): + fastapi = pytest.importorskip("fastapi") + del fastapi + from fastapi.testclient import TestClient + + monkeypatch.setenv("ENGRAM_DATA_DIR", str(tmp_path)) + monkeypatch.delenv("ENGRAM_API_KEYS", raising=False) + monkeypatch.delenv("ENGRAM_ADMIN_TOKEN", raising=False) + monkeypatch.setenv("ENGRAM_OPEN", "1") + for name, value in env.items(): + monkeypatch.setenv(name, value) + + import engram.server.app as app_module + + app_module._svc = None + app_module._keystore = None + app_module._keystore_path = None + app_module._limiter = None + app_module._idempotency = None + return TestClient(app_module.app), app_module + + +def test_rate_limited_requests_are_counted(tmp_path, monkeypatch): + client, _ = _guarded_client(tmp_path, monkeypatch, ENGRAM_RATE_LIMIT_PER_MIN="1") + headers = {"Authorization": "Bearer tenant-a"} + client.post("/v1/remember", json={"content": "one"}, headers=headers) + assert client.post("/v1/remember", json={"content": "two"}, headers=headers).status_code == 429 + + assert client.get("/metrics").json()["counts"]["rate_limited"] == 1 + + +def test_idempotent_replays_are_counted(tmp_path, monkeypatch): + client, _ = _guarded_client(tmp_path, monkeypatch) + headers = {"Authorization": "Bearer tenant-a", "Idempotency-Key": "retry-1"} + body = {"content": "Alice visited Kyoto."} + client.post("/v1/remember", json=body, headers=headers) + counts = client.get("/metrics").json()["counts"] + assert "idempotent_replays" not in counts, "the first call is work, not a replay" + + client.post("/v1/remember", json=body, headers=headers) + assert client.get("/metrics").json()["counts"]["idempotent_replays"] == 1 + + +def test_rejected_auth_is_counted(tmp_path, monkeypatch): + client, _ = _guarded_client(tmp_path, monkeypatch, ENGRAM_API_KEYS="carol:sk-carol") + monkeypatch.delenv("ENGRAM_OPEN", raising=False) + assert client.post( + "/v1/remember", json={"content": "hi"}, headers={"Authorization": "Bearer wrong-key"} + ).status_code == 401 + assert client.get("/metrics").json()["counts"]["auth_rejected"] == 1 + + +def test_defence_counters_never_name_a_tenant(tmp_path, monkeypatch): + """The privacy boundary: /metrics is open, so counting must not enumerate tenants.""" + client, _ = _guarded_client(tmp_path, monkeypatch, ENGRAM_RATE_LIMIT_PER_MIN="1") + secret_tenant = "acme-industries-prod" + headers = {"Authorization": f"Bearer {secret_tenant}", "Idempotency-Key": "k"} + client.post("/v1/remember", json={"content": "hi"}, headers=headers) + client.post("/v1/remember", json={"content": "hi"}, headers=headers) + + payload = client.get("/metrics").text + assert secret_tenant not in payload From da6065df3b22e7d78a3da6ef27eb5fe836f9bfc5 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 22:07:55 +0800 Subject: [PATCH 15/25] docs: document the operational surface added over the last rounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runtime key issuance, rate limiting, Idempotency-Key, the defence counters and the Python SDK all shipped without a line of documentation. For a project whose pitch is that it is reproducible and self-hostable, a capability nobody can find is a capability that does not exist — and the audience most affected is the operator deploying it, who reads deploy/.env.example and never learns these knobs are there. API.md gains the key lifecycle, retry-safe writes and the metrics payload, each with the operational caveat that matters rather than just the happy path: the admin surface is absent unless ENGRAM_ADMIN_TOKEN is set, the plaintext key exists only in the issuing response, a corrupt key store answers 503 rather than starting empty, and both the limiter and the idempotency cache are in-process so several replicas multiply the effective limit and can re-run a retry. deploy/.env.example gains the new variables, with ENGRAM_ADMIN_TOKEN deliberately left commented out — an example file that hands operators a working admin credential would undo the fail-closed default it is supposed to explain. Both READMEs get the Python SDK, kept in sync as the contributing rules require. The copy sticks to what the code does — no competitor names, no scaling claims a committed log does not back. --- API.md | 70 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 27 ++++++++++++++++- README.zh-CN.md | 31 ++++++++++++++++++++ deploy/.env.example | 12 ++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/API.md b/API.md index 6f0e350..7057def 100644 --- a/API.md +++ b/API.md @@ -38,6 +38,50 @@ Authorization: Bearer <你的key> Content-Type: application/json ``` +密钥有两种来源，解析顺序为「运行时签发的密钥 → `ENGRAM_API_KEYS` 静态映射 → `ENGRAM_OPEN` 开放模式」。 +运行时密钥优先，所以吊销不会被残留的环境变量条目复活。 + +### 1.1 运行时签发密钥（不用重启） + +静态映射要改环境变量再重启。托管部署可以用管理端点在运行中签发和吊销。**管理面默认不存在**—— +不设 `ENGRAM_ADMIN_TOKEN` 时这几个端点一律 403，所以开放模式部署不会被路人签发租户。 + +```bash +export ENGRAM_ADMIN_TOKEN=<强随机串> # 与租户 key 不同，是独立的管理凭据 + +# 签发（明文密钥只在这一次响应里出现，服务端只存 SHA-256 摘要，丢了只能重签不能找回） +curl -s -X POST $B/v1/admin/keys -H "Authorization: Bearer $ENGRAM_ADMIN_TOKEN" \ + -H 'Content-Type: application/json' -d '{"user":"alice","label":"laptop"}' +# -> {"id":"key_...","user":"alice","label":"laptop","created_at":...,"revoked":false,"key":"sk-engram-..."} + +curl -s $B/v1/admin/keys -H "Authorization: Bearer $ENGRAM_ADMIN_TOKEN" # 列出（不含密钥与摘要） +curl -s -X DELETE $B/v1/admin/keys/key_xxx -H "Authorization: Bearer $ENGRAM_ADMIN_TOKEN" # 立即吊销 +``` + +密钥文件（数据目录下的 `api_keys.json`）仅属主可读写，且只保存摘要。若该文件损坏，服务**拒绝启动 +密钥库并对请求返回 503**，而不是以空状态继续——后者会先让所有已签发密钥失效，再在下次签发时覆盖掉 +那些只是读不出来的记录。 + +### 1.2 重试安全（Idempotency-Key） + +`/v1/remember` 与 `/v1/import` 支持 `Idempotency-Key` 请求头。客户端超时重试时，服务端按 +「租户 + key」返回首次的响应而不重跑：首次请求其实成功了，丢的只是响应。 + +```bash +curl -s -X POST $B/v1/remember -H "Authorization: Bearer $KEY" \ + -H 'Idempotency-Key: 2026-08-16-abc' -H 'Content-Type: application/json' \ + -d '{"content":"..."}' +``` + +缓存是进程内的，默认保留 `ENGRAM_IDEMPOTENCY_TTL_S`（86400）秒。多副本部署下，重试若路由到另一个 +副本仍会重跑——需要跨副本幂等就换成共享存储。 + +### 1.3 限流 + +`ENGRAM_RATE_LIMIT_PER_MIN`（默认 0 = 关闭）按租户做滑动窗口限流，超限返回 `429` 并带 +`Retry-After`。`/health`、`/ready`、`/metrics` 不鉴权也不限流——租户被限流时探针必须还能用。 +同样是进程内的：多副本下有效限额是 `每分钟配额 × 副本数`。 + 健康检查不需要鉴权： ```bash curl -s $B/health @@ -168,6 +212,32 @@ agent session。 | POST | `/v1/import` | 批量导入：`{"data":..., "format":"chatgpt\|messages\|records\|jsonl\|transcript\|engram\|auto"}`；`engram` 格式（自动嗅探）直接还原一份 `/v1/export` 导出——跨实例迁移路径 | | POST | `/v1/forget` | 需 `{"confirm":true}`；清空该 key 的全部记忆（不可逆） | +### 运行指标（`GET /metrics`，不鉴权） + +与 `/health` 一样开放，因为载荷**按构造只含聚合量**：没有命名空间名、没有查询、没有正文，所以它 +无法暴露某个租户是否存在。用于回答「写路径 <50ms、读路径 <100ms 这两个目标现在还成立吗」—— +没有实时分位数的话，那两个数字只是断言而非测量。 + +```bash +curl -s $B/metrics +``` + +```json +{ + "uptime_s": 1234.5, + "ops": {"remember": {"n": 42, "p50_ms": 8.1, "p95_ms": 22.4, "avg_ms": 9.7, "max_ms": 31.0, "window": 42}}, + "counts": {"rate_limited": 3, "idempotent_replays": 1, "auth_rejected": 0, "remember_degraded": 0}, + "tokens": {"context_total": 9600, "calls_with_baseline": 12, "savings_ratio": 8.2} +} +``` + +- `ops` 是每个操作的滑动窗口分位数，反映**当前**表现，不会被历史均值稀释。 +- `counts` 里 `rate_limited` / `idempotent_replays` / `auth_rejected` 让三层防护可见——否则无法判断 + 它们是否在生效。`auth_misconfigured`（503）与 `auth_rejected`（401）分开计：前者是配置坏了， + 后者是调用方拿错了密钥。 +- `savings_ratio` **只用两侧都测量过的调用**计算（即带 `answer=true` 的召回），没有配对样本时为 + `null` 而不是编一个数。 + ### 跨实例迁移（换服务器 / 换账号） 记忆属于用户，不属于某一个部署。把一个 namespace 从实例 A 搬到实例 B： diff --git a/README.md b/README.md index 6542471..f60030e 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,32 @@ For a user-facing "my memory" page, the SDK also exposes paged inspection: The standalone graph endpoint is share-safe by default; pass `/v1/graph?include_sensitive=true` only for an explicit private graph inspection. -**3. Batch import** — bring your whole history (ChatGPT export, OpenAI messages, JSONL, transcript; +**3. Python SDK** — the same API from Python, with no runtime dependencies (it speaks over stdlib +`urllib`, so installing Engram still pulls in nothing): + +```python +from engram.client import EngramClient + +engram = EngramClient(base_url="http://localhost:8000", api_key="sk-engram-...") +engram.remember("I live in Shenzhen and work on retrieval.") +print(engram.recall("where do I live?")["context"]) + +# retry-safe writes: the same key replays the first response instead of storing twice +engram.remember("...", idempotency_key="2026-08-16-abc") +``` + +`EngramError` carries `status` so callers can branch without parsing prose — `401` wrong key, `429` back +off (`err.retry_after` is the server's `Retry-After`), `503` server misconfigured rather than a bad +request. The `transport` hook swaps the HTTP layer for `httpx`/`requests` or an in-process test client. + +**4. Runtime keys, rate limits and metrics** — for self-hosting: mint and revoke tenant keys without a +restart (`POST /v1/admin/keys`, gated by its own `ENGRAM_ADMIN_TOKEN` and absent unless you set it), +per-tenant rate limiting (`ENGRAM_RATE_LIMIT_PER_MIN`), retry-safe writes (`Idempotency-Key`), and +`GET /metrics` for live latency percentiles, token totals and defence counters. `/metrics` is +aggregate-only by construction — no namespace names, no queries, no content — so it can stay as open as +`/health`. See [`API.md`](API.md). + +**5. Batch import** — bring your whole history (ChatGPT export, OpenAI messages, JSONL, transcript; auto-detected): ```bash diff --git a/README.zh-CN.md b/README.zh-CN.md index c05ab60..9721b2a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -237,6 +237,37 @@ TypeScript SDK 的 `engram.export()` 默认走这个安全导出；只有用户 `engram.memories({ factsLimit, factsOffset, episodesLimit, status, query, includeSensitive })`。 独立关系图接口默认也是安全视图；只有明确要完整私有图谱时才传 `/v1/graph?include_sensitive=true`。 +### Python SDK（零运行时依赖） + +Agent 生态是 Python 优先的，所以 SDK 直接装在核心包里，走 stdlib `urllib`，装 Engram 依然不会额外 +拉进任何依赖： + +```python +from engram.client import EngramClient + +engram = EngramClient(base_url="http://localhost:8000", api_key="sk-engram-...") +engram.remember("我住在深圳，做检索相关的工作。") +print(engram.recall("我住在哪儿？")["context"]) + +# 重试安全：同一个 key 会重放首次响应，而不是写第二遍 +engram.remember("...", idempotency_key="2026-08-16-abc") +``` + +`EngramError` 带 `status`，调用方不用解析文案就能分支——`401` 密钥错、`429` 该退避 +（`err.retry_after` 就是服务端的 `Retry-After`）、`503` 是服务端配置有问题而不是请求有问题。 +`transport` 钩子可以把 HTTP 层换成 `httpx`/`requests` 或进程内测试客户端。 + +### 运行时密钥、限流与运行指标（自托管用） + +- **运行时签发密钥**：`POST /v1/admin/keys` 不重启就能签发/吊销租户密钥；只落盘 SHA-256 摘要， + 明文只在签发响应里出现一次。管理面由独立的 `ENGRAM_ADMIN_TOKEN` 把守，**不设置就完全不存在**。 +- **按租户限流**：`ENGRAM_RATE_LIMIT_PER_MIN`（默认 0 = 关闭），超限返回 `429` + `Retry-After`。 +- **重试安全**：`/v1/remember` 与 `/v1/import` 支持 `Idempotency-Key` 头。 +- **运行指标**：`GET /metrics` 给出实时延迟分位数、token 总量和防护计数。载荷按构造只含聚合量—— + 无命名空间名、无查询、无正文——所以可以和 `/health` 一样开放。 + +细节见 [`API.md`](API.md)。 + ### 自部署（数据完全在你自己机器上） ```bash diff --git a/deploy/.env.example b/deploy/.env.example index 1a3c5d8..1db54ac 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -7,6 +7,18 @@ ENGRAM_MAX_HOT_USERS=64 ENGRAM_MAX_HOT_FACTS=10000 ENGRAM_EMBEDDER=hashing +# 按租户滑动窗口限流；0 = 关闭（默认）。超限返回 429 + Retry-After。 +# 进程内实现：多副本下有效限额是 每分钟配额 × 副本数。 +ENGRAM_RATE_LIMIT_PER_MIN=0 + +# Idempotency-Key 重放缓存的保留时长（秒）。同样是进程内的：多副本下重试若路由到 +# 另一个副本仍会重跑。 +ENGRAM_IDEMPOTENCY_TTL_S=86400 + +# 运行时签发/吊销租户密钥的管理面凭据。**故意留空**：不设置时 /v1/admin/keys 一律 403, +# 管理面根本不存在。只有确实需要不重启就增删租户时才设一个强随机串（与租户 key 不同）。 +# ENGRAM_ADMIN_TOKEN= + # 可选模型后端；为空时保持离线规则抽取。 ENGRAM_LLM= ENGRAM_ANSWERER= From 0eed3fcdada5502f871f535cf3eedba685429955 Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 23:35:17 +0800 Subject: [PATCH 16/25] proxy: make the system prompt stable across a session, opt-in The premise written into layered.py was wrong for this surface. The OpenAI-compatible proxy never put memory in the user turn -- it puts the whole retrieved slice in the system prompt. So the split does not buy a placement change here; it buys prefix stability. Provider prompt-caching matches on a prefix, and with the per-query evidence sitting in the system block that prefix changed every single turn, so nothing was ever reusable. With the query-independent half moved to the front of the system block and this turn's evidence riding with the user turn, the system block becomes byte-identical across a session: five distinct blocks over five turns becomes one. The response reports cacheable_tokens_est so a caller can reason about it rather than guess. The honest measurement is that this does not save tokens. Compared like for like it is -1.1%, which is noise. What it produces is a 61-token stable prefix and a property that did not exist before. Whether 61 tokens is worth having depends on a provider's cache-read pricing, which this measurement does not model, so it stays opt-in via {"memory": {"layered": true}}: worth it across a session, pointless on a one-shot call. I walked into the same measurement trap twice. The first pass counted RECALL_GUIDE and reported +14%, "the split is more expensive" -- but the guide is content the flat path does not carry, exactly the flat-versus-flat-plus-a-new-block comparison that made the session map look bad last time. Fixing the comparison also exposed a real duplication: the proxy already frames memory with _MEMORY_PREAMBLE, so the library's guide was a second copy of the same instruction, costing 14% and changing nothing. The proxy now passes guide=False, and the optimization map carries a warning about the trap itself. --- docs/architecture-optimization-map.zh-CN.md | 6 ++ engram/server/app.py | 3 + engram/server/openai_compat.py | 63 +++++++++++++--- results/layered_context_tokens.md | 42 ++++++++++- tests/test_layered_context.py | 81 +++++++++++++++++++++ 5 files changed, 180 insertions(+), 15 deletions(-) diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 26e31a4..74b0c38 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -101,6 +101,7 @@ flowchart TD | 2026-07-09 | `chain_provenance_promotion` | Chain-aware retrieval / Raw evidence fusion | previous-value 问题中，`supersedes` 链上的旧事实也能作为 provenance raw chunk promotion 的种子，优先提升旧值源会话 | `results/chain_provenance_promotion_experiments.md`, `results/chain_provenance_promotion_ablation.jsonl`, `results/chain_provenance_promotion_context_sample.jsonl` | | 2026-07-14 | `commercial_release_0_1_0` | Service boundary / Namespace storage / Deployment / Release gate | 修复命名空间路径穿越与字符过滤碰撞；默认鉴权失败关闭；增加 request limits、liveness/readiness、非 root 容器和统一发布门禁 | `results/commercial_release_0_1_0_validation.jsonl`, `specs/003-commercial-release/` | | 2026-08-16 | `bounded_candidates`（默认关） | Read path / Hybrid retrieval / Store 索引层 | 读路径此前对每次查询全量扫描存活事实（`hybrid.py` 的 `fact_store.values()`），实测为 O(n)：每千条事实耗时恒定 ~17–20ms，10000 条时单查询 177ms，已超宪章 <100ms 目标。新增倒排/槽位索引（`engram/store/indexed.py`）+ 存储装饰器，让融合阶段只对有界候选集打分 | `results/bounded_candidates_scaling.md`，`eval/scaling.py`，`tests/test_bounded_candidates.py`（8 测试，含逐位等价性） | +| 2026-08-16 | `layered_proxy_wiring`（opt-in） | 接入层 / OpenAI 兼容代理 | 代理此前把整个检索切片放进 system prompt，导致 system 块每轮都变、provider 的 prompt-cache 一个前缀都匹配不上（这也推翻了 `layered.py` 原docstring「调用方放进 user turn」的前提）。改为 stable 半置于 system 最前、本轮证据移入 user turn，system 块跨轮字节相同（去重数 5→1），稳定前缀 61 tokens。响应新增 `engram.cacheable_tokens_est` 让调用方不必猜。**同内容对比下 token −1.1%（中性）**，收益是「system 块不再每轮失效」这个此前不存在的性质，是否值得取决于 provider 的 cache-read 定价，本测量不计价格 | `results/layered_context_tokens.md`，`tests/test_layered_context.py`（+4 测试，含前缀稳定性与「拆分不丢证据」） | | 2026-08-16 | `defence_counters` | 服务可观测层 | 限流、幂等、密钥解析三层防护此前是黑盒：运维无法判断它们是否在生效。`/metrics` 新增 `rate_limited` / `idempotent_replays` / `auth_rejected` / `auth_misconfigured` 四个聚合计数。**刻意不按租户分桶**——该端点不鉴权，分桶会暴露有哪些租户存在。埋点统一走 `_count()`，其中吞掉自身异常：埋点不得改变请求行为，而在异常处理器里构造服务失败会把一个精确的 401 换成笼统的 500，恰好丢掉计数本来要支撑的诊断 | `tests/test_metrics.py`（+4 测试，含首次调用不算重放、以及计数载荷不含租户名的隐私断言） | | 2026-08-16 | `python_sdk` | 接入层 / 客户端 | 主干只有 TS 客户端，而 Agent 生态是 Python 优先的。新增 `engram/client.py`：零运行时依赖（stdlib urllib）、方法名对齐 TS SDK 避免两者漂移、覆盖今日全部端点（含新增的 `/metrics`、`/v1/admin/keys`、`Idempotency-Key`）。**transport 契约改为返回 `(status, headers, body)`**——最初只返回 `(status, body)`，导致 `EngramError.retry_after` 永远为 `None`，因为 `Retry-After` 在响应头里；发一个永远为空的字段比不发更糟 | `tests/test_client.py`（17 测试，全部通过可注入 transport 打到**真实 app**而非 mock——mock 只能证明 SDK 自洽，而客户端库的典型失效恰恰是与服务端漂移） | | 2026-08-16 | `self_serve_api_keys` | 服务边界 / 鉴权（多租户） | 此前只能靠静态 `ENGRAM_API_KEYS`（改环境变量+重启）或开放模式。新增 `engram/server/keys.py`：运行时签发 `sk-engram-*`、**只落盘 SHA-256 摘要**（泄露的密钥文件不可重放）、立即吊销、可列出；`/v1/admin/keys` 由独立的 `ENGRAM_ADMIN_TOKEN` 把守，未设置即 403（开放模式部署不会被路人签发租户）。解析顺序为「运行时密钥 → 静态映射 → 开放模式」，密钥库不可读时返回 503 而非放行。**修正了保全版的一处数据丢失风险**：原实现在密钥文件损坏时静默以空状态启动，随后任一次签发都会 `_save()` 覆盖该文件、永久销毁全部已签发记录；改为拒绝启动并保持文件原样。另修掉锁外改 `last_used_at` 的数据竞争，并把密钥文件权限收紧为仅属主可读写 | `tests/test_api_keys.py`（20 测试，含失败关闭、吊销后拒绝、损坏文件不被覆盖、跨租户隔离用 bob 自己的密钥走 API 验证） | @@ -145,6 +146,11 @@ flowchart TD | `candidate_vector_channel=False` 作为默认 | **不采用**。它确实快 14.5x 且次线性，但会丢"语义相关却无共享查询词"的事实，正是 M1 已验证 hybrid 论点依赖的召回。需真 ANN 后端或 keyed harness 证明召回损失可接受 | 同上 | | 上下文拆分接进 `lean_context` 默认路径 | **不采用**。实测 5 轮以下是净亏（stable 半虽只计费一次，但扁平上下文本身小，首轮结构开销收不回），且已发布数字均由扁平路径产出。保留为独立方法 `Memory.layered_context()`，由调用方按会话长度选择 | `results/layered_context_tokens.md` | | 上下文导航图（MEMORY MAP）默认开启 | **不采用**。它是扁平上下文里本来没有的新增内容，约 20 轮才回本；最初的对比实为"扁平 vs 扁平+导航图"，缓存收益被吃光还倒欠。改为 `map_limit=0` 默认关闭 | 同上 | +| 代理侧叠加 `RECALL_GUIDE` | **不采用**。代理已用 `_MEMORY_PREAMBLE` 框定记忆，再叠一层是同一条指令的第二份拷贝，实测 +14% prompt tokens 且不改变行为。代理显式传 `guide=False` | `results/layered_context_tokens.md` | +| 上下文拆分作为代理默认 | **不采用**，保持 opt-in（`{"memory":{"layered":true}}`）。同内容对比下 token 收益仅 −1.1%（噪声级），真实价值是 61 tokens 的可缓存稳定前缀，是否划算取决于 provider 的 cache-read 定价，尚未用真实计费验证 | 同上 | + +> ⚠️ **同一个测量陷阱已踩两次**：把"扁平 vs 扁平+新增块"当作公平对比（第一次是 MEMORY MAP，第二次是 +> RECALL_GUIDE），两次都先得出"拆分更贵"的错误结论。任何对比先确认两边内容集合相同，只有放置方式不同。 ## 验收分层规则 diff --git a/engram/server/app.py b/engram/server/app.py index 9495be1..fcebb8f 100644 --- a/engram/server/app.py +++ b/engram/server/app.py @@ -770,6 +770,9 @@ def chat_completions(req: ChatCompletionReq, background: BackgroundTasks, user: as_of=opts.get("as_of"), redact_sensitive=bool(opts.get("redact_sensitive", False)), session_id=session_id, + # Opt-in: it pays off across a multi-turn session and costs a little on a one-shot call + # (results/layered_context_tokens.md), so the caller who knows which they are decides. + layered=bool(opts.get("layered", False)), ) except oc.NoLLMConfigured as exc: raise HTTPException(503, str(exc)) diff --git a/engram/server/openai_compat.py b/engram/server/openai_compat.py index 4e6a4d2..afdc40a 100644 --- a/engram/server/openai_compat.py +++ b/engram/server/openai_compat.py @@ -47,14 +47,25 @@ def latest_user_text(messages: list) -> str: return "" -def build_prompt(messages: list, memory_context: str) -> tuple[Optional[str], str]: +def build_prompt( + messages: list, memory_context: str, stable_context: str = "" +) -> tuple[Optional[str], str]: """Render the request's messages into a (system, prompt) pair for the LLM.complete interface: * system = the injected memory block + any of the request's own system messages * prompt = the single user turn, or the full transcript for a multi-turn conversation Backend-agnostic on purpose: it works with any LLM (incl. the offline FakeLLM in tests), not only - a litellm chat model.""" + a litellm chat model. + + `stable_context` is the query-independent half of memory (profile and usage guide). It goes at the + very front of the system prompt and the per-query evidence moves into the user turn, so the system + block is byte-identical across a session's turns. Provider prompt-caching matches on a prefix, and + with the whole retrieved slice in the system block — which is what happens when `stable_context` is + empty — that prefix changes every turn and nothing can ever be reused. + """ system_parts: list[str] = [] - if memory_context.strip(): + if stable_context.strip(): + system_parts.append(_MEMORY_PREAMBLE + stable_context.strip()) + elif memory_context.strip(): system_parts.append(_MEMORY_PREAMBLE + memory_context.strip()) system_parts += [c for c in (_content(m) for m in messages if isinstance(m, dict) and m.get("role") == "system") if c.strip()] @@ -68,6 +79,12 @@ def build_prompt(messages: list, memory_context: str) -> tuple[Optional[str], st else: rendered = "\n".join(f"{m['role'].capitalize()}: {_content(m)}" for m in convo) prompt = rendered + "\nAssistant:" + + # Only when the split is active: this turn's evidence rides with the turn, leaving the system block + # unchanged. Without a stable half there is nothing to protect, and moving the memory here would + # only make the prompt longer for no gain. + if stable_context.strip() and memory_context.strip(): + prompt = f"{memory_context.strip()}\n\n{prompt}" if prompt else memory_context.strip() return system, prompt @@ -84,6 +101,7 @@ def chat_completion( as_of: Optional[float] = None, redact_sensitive: bool = False, session_id: Optional[str] = None, + layered: bool = False, ) -> dict: """Recall → inject → generate → return an OpenAI ChatCompletion object (with an `engram` extension describing what memory was used). Does NOT write memory — the route schedules that off the critical @@ -93,25 +111,43 @@ def chat_completion( query = latest_user_text(messages) or (_content(messages[-1]) if messages else "") memory_context = "" + stable_context = "" if do_recall and query: - memory_context = ( - svc.recall( - user, + if layered: + # Split so the system prompt stops changing every turn. Same retrieval either way, so the + # evidence the model sees is unchanged -- only where each half is placed. + parts = svc.get(user).layered_context( query, - lean=True, + user_id=user, n_chunks=n_chunks, session_id=session_id, as_of=as_of, redact_sensitive=redact_sensitive, - ).get("context") or "" - ) + # This surface already frames the memory with _MEMORY_PREAMBLE, so the library's own + # usage guide would be a second copy of the same instruction. Measured at +14% prompt + # tokens for no behaviour change (results/layered_context_tokens.md). + guide=False, + ) + stable_context, memory_context = parts.stable, parts.dynamic + else: + memory_context = ( + svc.recall( + user, + query, + lean=True, + n_chunks=n_chunks, + session_id=session_id, + as_of=as_of, + redact_sensitive=redact_sensitive, + ).get("context") or "" + ) if svc.llm is None: raise NoLLMConfigured( "no LLM configured for generation — set ENGRAM_LLM (e.g. 'deepseek'), or use /v1/recall " "for retrieval-only.") - system, prompt = build_prompt(messages, memory_context) + system, prompt = build_prompt(messages, memory_context, stable_context) content = svc.llm.complete(prompt, system=system) p_tokens = _est_tokens((system or "") + " " + prompt) @@ -130,8 +166,11 @@ def chat_completion( "total_tokens": p_tokens + c_tokens}, # Engram extension — transparency about the memory layer (ignored by standard OpenAI clients). "engram": { - "recalled": bool(memory_context.strip()), - "memory_tokens_est": _est_tokens(memory_context), + "recalled": bool((memory_context + stable_context).strip()), + "memory_tokens_est": _est_tokens(memory_context) + _est_tokens(stable_context), + # When layered, this is the byte-identical prefix a provider's prompt cache can reuse across + # the session; 0 means the whole slice still rides in the system block and nothing is stable. + "cacheable_tokens_est": _est_tokens(stable_context), "session_id": session_id, "as_of": as_of, "redacted_sensitive": redact_sensitive, diff --git a/results/layered_context_tokens.md b/results/layered_context_tokens.md index 0fc3dc3..a749fd6 100644 --- a/results/layered_context_tokens.md +++ b/results/layered_context_tokens.md @@ -61,8 +61,44 @@ PY - 把拆分接进 `lean_context` 默认路径：**不采用**。短会话是净亏，且已发布数字均由扁平路径产出。 当前形态是独立方法 `Memory.layered_context()`，调用方按会话长度自行选择。 +## 追加：接进 OpenAI 兼容代理后的实测（同日） + +**一个前提被推翻。** 本文件开头写的「调用方把上下文整段放进 user turn」对 OpenAI 兼容代理**不成立**： +它一直是把整个检索切片放进 **system prompt** 的（`_MEMORY_PREAMBLE + memory_context`）。所以拆分在 +这个接入面买到的不是「system 还是 user」的位置，而是**前缀稳定性**——provider 的 prompt-cache 按 +前缀匹配，而今天整个 system 块每轮都变，一个可复用前缀都没有。 + +按前缀稳定性重新接线后（stable 半置于 system 最前，本轮证据移入 user turn），30 会话实测： + +| 轮数 | flat 总 prompt | 拆分（同内容） | 差 | 拆分 + 指引 | 差 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 371 | 367 | −1.1% | 422 | +13.7% | +| 5 | 1925 | 1905 | −1.0% | 2180 | +13.2% | +| 20 | 7102 | 7022 | −1.1% | 8122 | +14.4% | +| 50 | 17755 | 17555 | −1.1% | 20305 | +14.4% | + +system 块跨 5 轮的去重数量：**flat = 5（每轮都变），拆分 = 1（完全稳定）**；稳定前缀 61 tokens。 + +### 又一次不公平对比（同一个坑踩了第二次） + +第一次测量把 `RECALL_GUIDE` 也算进去，得出 +14%，于是看起来「拆分更贵」。但指引是 flat 路径里 +**本来没有的新增内容**——这和上一节导航图的错误完全相同：拿「扁平 vs 扁平+新增块」当公平对比。 + +同内容对比下，拆分是 **−1.1%**，即 token 中性略优。 + +并且这次暴露出一个真实的重复：**代理本身已经用 `_MEMORY_PREAMBLE` 框定了记忆**，再叠一层 +`RECALL_GUIDE` 是同一条指令的第二份拷贝，白付 14% 且不改变行为。代理侧因此显式传 `guide=False`。 + +### 诚实结论 + +- 拆分在这个接入面**不省 token**（−1.1%，噪声级别），它买到的是 **61 tokens 的可缓存稳定前缀**， + 以及「system 块不再每轮失效」这个此前完全不存在的性质。 +- 61 tokens 相对每轮约 350 tokens 的 prompt 是小头。**是否值得取决于 provider 的 cache-read 定价， + 而本测量不计价格**——那需要各家定价，不做未经测量的推断。 +- 因此保持 **opt-in**：`{"memory": {"layered": true}}`。多轮会话可以开，一次性调用没必要。 +- `engram.cacheable_tokens_est` 在响应里报告稳定前缀大小，调用方不用猜。 + ## 下一步 -把 `as_messages()` 接进 OpenAI 兼容代理（`engram/server/openai_compat.py`），让多轮会话真正吃到 -provider 的 prompt-cache，并用 `/metrics` 的 token 计数回读真实节省——目前的数字是离线估算， -不是线上实测。 +用真实 provider（而非本地估算器）跑一次多轮会话，回读 cache-read 命中率与实际计费，验证这 61 tokens +在真实定价下是否值得。那需要真实 API key，属付费评测。 diff --git a/tests/test_layered_context.py b/tests/test_layered_context.py index 113fd81..1632a52 100644 --- a/tests/test_layered_context.py +++ b/tests/test_layered_context.py @@ -153,3 +153,84 @@ def test_memory_method_matches_the_module(): assert mem.layered_context("where does alice work", "u1").stable == ( layered_context(mem, "where does alice work", "u1").stable ) + + +# --- OpenAI-compatible proxy wiring --- +# +# The proxy already put the whole retrieved slice in the SYSTEM prompt, so the system block changed on +# every turn and no provider prompt cache could ever match a prefix. The split's job on this surface is +# to make that block byte-identical; the tests below pin both halves of that claim. + + +def _proxy_setup(): + from engram.server import openai_compat as oc + + mem = _memory() + + class _Svc: + """Only what chat_completion touches.""" + + def __init__(self, memory): + self._mem = memory + self.llm = self + + def get(self, _user): + return self._mem + + def recall(self, _user, query, **kwargs): + return {"context": self._mem.lean_context(query, user_id="u1")} + + def complete(self, prompt, system=None): + return "ok" + + return oc, _Svc(mem) + + +def test_proxy_system_block_is_stable_across_turns_when_layered(): + oc, svc = _proxy_setup() + flat, layered = set(), set() + for query in QUERIES: + body = {"model": "engram", "messages": [{"role": "user", "content": query}]} + ctx = svc.recall("u1", query)["context"] + flat.add(oc.build_prompt(body["messages"], ctx)[0]) + parts = svc.get("u1").layered_context(query, user_id="u1", guide=False) + layered.add(oc.build_prompt(body["messages"], parts.dynamic, parts.stable)[0]) + + # Not "one block per query": two questions can retrieve the same slice. The property is that the + # unsplit block varies at all — that alone is enough to miss a prefix cache on those turns. + assert len(flat) > 1, "precondition: today's system block varies across turns" + assert len(layered) == 1, "the split must make the system block byte-identical, or caching never hits" + + +def test_proxy_reports_the_cacheable_prefix_size(): + """A caller cannot reason about caching without knowing how much of the prompt is stable.""" + oc, svc = _proxy_setup() + body = {"model": "engram", "messages": [{"role": "user", "content": "where does alice work"}]} + + plain = oc.chat_completion(svc, "u1", body) + assert plain["engram"]["cacheable_tokens_est"] == 0, "nothing is stable without the split" + + split = oc.chat_completion(svc, "u1", body, layered=True) + assert split["engram"]["cacheable_tokens_est"] > 0 + + +def test_proxy_keeps_the_evidence_when_splitting(): + """Moving evidence to the user turn must not drop it.""" + oc, svc = _proxy_setup() + body = {"model": "engram", "messages": [{"role": "user", "content": "where does alice work"}]} + parts = svc.get("u1").layered_context("where does alice work", user_id="u1", guide=False) + system, prompt = oc.build_prompt(body["messages"], parts.dynamic, parts.stable) + + assert parts.stable in (system or "") + assert parts.dynamic in prompt + assert "where does alice work" in prompt + + +def test_proxy_is_unchanged_when_not_layered(): + """The default path must behave exactly as before.""" + oc, svc = _proxy_setup() + messages = [{"role": "user", "content": "where does alice work"}] + ctx = svc.recall("u1", "where does alice work")["context"] + system, prompt = oc.build_prompt(messages, ctx) + assert ctx.strip() in (system or ""), "unsplit memory still rides in the system block" + assert prompt == "where does alice work" From b676a8b272e34ae92d18bfc7853847d406389ddf Mon Sep 17 00:00:00 2001 From: Engram Date: Sun, 16 Aug 2026 23:43:15 +0800 Subject: [PATCH 17/25] eval: tell a real benchmark gain from the answerer's noise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-running an identical configuration moves 6-10 of 500 answers, so "83.6 to 84.4, we improved" has never been a finding on this benchmark — and a long list of mechanisms was retired on exactly that basis, one expensive run at a time. What was missing is an instrument that decides, before and after, whether a difference is real. eval/significance.py adds McNemar's exact test over paired per-question outcomes, a seeded bootstrap interval on the difference, and — the part that changes what gets run at all — the minimum detectable effect, computed from the item count and how often two systems disagree. Only disagreements carry information, so the usable sample is far smaller than the headline item count: 500 questions at a 22% disagreement rate resolve about 2.94 points, not fractions of one. Two results follow immediately, both in results/significance_headline.md. The published headline holds up: engram_lean 83.6% against full_context 73.2% is 81 wins to 29 over 110 disagreements, p < 0.0001, plausibly +6.4 to +14.4 points. That claim now has a test behind it rather than two percentages side by side. And the gap to the top of the leaderboard is 1.6 points, which is *below this benchmark's resolution at n=500*. It is not "not caught up yet"; it is not decidable with this measurement. Grinding algorithms against it produces noise, not rank. The optimization map now says so up front, with the consequence spelled out: only attempt changes expected to move more than ~3 points, and aim them at the two large weak categories (multi-session and temporal-reasoning, 248 of 500 questions between them), because a +10 in one category is only about +2.5 overall. Pure stdlib, exact binomial rather than a chi-square approximation — the discordant counts here are small enough that the approximation is wrong precisely where the decision is closest. --- docs/architecture-optimization-map.zh-CN.md | 14 ++ eval/significance.py | 263 ++++++++++++++++++++ results/significance_headline.md | 78 ++++++ tests/test_significance.py | 169 +++++++++++++ 4 files changed, 524 insertions(+) create mode 100644 eval/significance.py create mode 100644 results/significance_headline.md create mode 100644 tests/test_significance.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 74b0c38..3fc73be 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -127,6 +127,20 @@ flowchart TD | 本次 `chain_provenance_promotion` | `supersedes` 链接入 provenance chunk promotion | 影响 previous/current-vs-past 问题的 raw source evidence | 复用 `chain_evidence` 开关 + 24/24 离线 ablation + LongMemEval sample context 0 errors | | 本次 `commercial_release_0_1_0` | 服务安全、租户落盘、部署和发布门禁收束 | 不改变 extraction/retrieval/fusion；影响所有 HTTP 自托管入口和新命名空间目录 | 危险路径/跨租户/鉴权/请求测试 + 全量 pytest + zero-setup + SDK/frontend/package/container 验收 | +## 算法迭代的前置条件（先读这条再决定跑什么） + +`eval/significance.py` 量出了这套评测的分辨率：**500 题、22% 分歧率下，最小可检测增益 2.94 点**。 + +- 公开 headline（`engram_lean` 83.6% vs `full_context` 73.2%）**显著**：p<0.0001， + 95% 区间 [+6.4, +14.4]，81:29 的分歧比。声称成立。 +- 但**到榜首的 +1.6 差距低于分辨率**——不是「还没追上」，是这套测量判定不了真假。 +- 因此：**只做期望增益 > 3 点的改动**，并优先攻占比最大的弱类别 multi-session（121 题 70.2%）与 + temporal-reasoning（127 题 70.9%，合计 248/500）；单类别 +10 点才换来整体约 +2.5 点。 +- 想分辨更小的增益，只能加分辨率（更多题目 / 更确定性的 answerer），那是测量投资， + 但它是所有算法投资的前置条件。 + +证据与复现命令：`results/significance_headline.md`。 + ## 当前重点区域 | 优先级 | 模块 | 为什么重要 | 下一步形态 | diff --git a/eval/significance.py b/eval/significance.py new file mode 100644 index 0000000..cb8f74e --- /dev/null +++ b/eval/significance.py @@ -0,0 +1,263 @@ +"""Is a benchmark difference real, or is it the answerer's noise? + +Re-running an identical configuration on LongMemEval_S moves 6-10 of 500 answers, because the answerer is +not deterministic even at temperature 0. Any single-run comparison smaller than that is indistinguishable +from chance — which means "83.6 to 84.4, we improved" is not a finding, and a long list of mechanisms was +already retired on exactly that basis. + +This module is the instrument that tells the two apart. Three questions, in the order they matter: + + 1. **Is the observed difference significant?** McNemar's exact test, which is the right test for two + systems scored on the *same* items: it ignores the questions both got right or both got wrong (they + carry no information about which is better) and asks whether the disagreements lean one way more + than a coin would. + 2. **How large could the true difference plausibly be?** A bootstrap interval over the paired + per-question outcomes. A p-value alone hides that a "win" may be consistent with anything from -1 to + +4 points. + 3. **Could this experiment have detected the gain I am hoping for?** The minimum detectable effect, + from the item count and how often the systems disagree. Answered *before* spending on a run, it is + the difference between an experiment and an expense — and the honest answer is sometimes that the + benchmark cannot resolve what is being attempted. + +Pure stdlib: an exact binomial tail, not a chi-square approximation, because the discordant counts here +are small enough that the approximation is wrong exactly where the decision is closest. + + python3 eval/significance.py results/a.jsonl results/b.jsonl --system engram_lean + python3 eval/significance.py --plan 500 --discordance 0.08 # what could a 500-item run detect? +""" +from __future__ import annotations + +import argparse +import math +import random +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from eval.compare import _ok, _scored, load # noqa: E402 + +__all__ = ["mcnemar_exact", "bootstrap_difference", "minimum_detectable_effect", "paired_outcomes"] + + +def paired_outcomes( + log_a: dict, log_b: dict, system_a: str, system_b: str +) -> list[tuple[str, bool, bool]]: + """(qid, a_correct, b_correct) for questions BOTH runs actually scored. + + Restricting to the intersection is not tidiness: comparing over different question sets compares two + different exams, and a system that errored on its hardest items would look better for it. + """ + shared = [] + for qid, entry_a in log_a.items(): + entry_b = log_b.get(qid) + if entry_b is None: + continue + if not (_scored(entry_a, system_a) and _scored(entry_b, system_b)): + continue + shared.append((qid, _ok(entry_a, system_a), _ok(entry_b, system_b))) + return shared + + +def _binom_tail(k: int, n: int) -> float: + """P(X <= k) for X ~ Binomial(n, 0.5), computed exactly.""" + if n == 0: + return 1.0 + return sum(math.comb(n, i) for i in range(k + 1)) / (2**n) + + +def mcnemar_exact(pairs: list[tuple[str, bool, bool]]) -> dict: + """Exact McNemar test over paired outcomes. + + Only the disagreements carry information. `b` is where A is right and B is wrong, `c` the reverse; if + the two systems were equally good, each disagreement is a coin flip, so the test asks how unlikely + the observed split is. Questions both systems answer the same way are excluded by construction — + which is why this is far more sensitive than comparing two accuracy percentages. + """ + both = only_a = only_b = neither = 0 + for _qid, a_ok, b_ok in pairs: + if a_ok and b_ok: + both += 1 + elif a_ok: + only_a += 1 + elif b_ok: + only_b += 1 + else: + neither += 1 + + discordant = only_a + only_b + smaller = min(only_a, only_b) + # Two-sided exact p: both tails of a fair coin, capped at 1 for the symmetric case. + p_value = min(1.0, 2.0 * _binom_tail(smaller, discordant)) if discordant else 1.0 + n = len(pairs) + return { + "n": n, + "both_correct": both, + "only_a": only_a, + "only_b": only_b, + "neither": neither, + "discordant": discordant, + "acc_a": (both + only_a) / n if n else 0.0, + "acc_b": (both + only_b) / n if n else 0.0, + "difference": (only_b - only_a) / n if n else 0.0, # positive = B better + "p_value": p_value, + } + + +def bootstrap_difference( + pairs: list[tuple[str, bool, bool]], iterations: int = 10_000, seed: int = 0, alpha: float = 0.05 +) -> dict: + """Percentile bootstrap interval for B's accuracy minus A's, resampling questions in pairs. + + Resampling the *pair* keeps each question's two outcomes together, which is what makes the interval + reflect the paired design rather than treating the runs as independent samples. + """ + if not pairs: + return {"low": 0.0, "high": 0.0, "iterations": 0} + rng = random.Random(seed) # seeded so a reported interval can be reproduced exactly + n = len(pairs) + diffs = [] + for _ in range(iterations): + total = 0 + for _ in range(n): + _qid, a_ok, b_ok = pairs[rng.randrange(n)] + total += int(b_ok) - int(a_ok) + diffs.append(total / n) + diffs.sort() + lo_index = int((alpha / 2) * iterations) + hi_index = min(iterations - 1, int((1 - alpha / 2) * iterations)) + return {"low": diffs[lo_index], "high": diffs[hi_index], "iterations": iterations} + + +def minimum_detectable_effect( + n_items: int, discordance: float, alpha: float = 0.05, power: float = 0.80 +) -> dict: + """The smallest true accuracy gain a run of this size could detect. + + Asked before a run, this is the difference between an experiment and an expense. With `n_items` + questions and the two systems disagreeing on a `discordance` fraction of them, only the disagreements + are informative, so the resolving power comes from `n_items * discordance` — usually far fewer items + than the benchmark's headline count suggests. + + Normal approximation on the discordant-pair proportion. Deliberately not exact: the answer is used to + decide whether to spend on a run, and a figure good to a fraction of a point is enough for that. + """ + discordant = max(0.0, n_items * discordance) + if discordant < 1: + return {"discordant_items": discordant, "mde_points": float("inf")} + # z for a two-sided alpha and the requested power. + z_alpha = _z_for(1 - alpha / 2) + z_power = _z_for(power) + # Detectable imbalance in the discordant split, converted back to overall accuracy points. + imbalance = (z_alpha + z_power) / (2 * math.sqrt(discordant)) + return { + "discordant_items": discordant, + "mde_points": 100.0 * imbalance * discordance, + } + + +def _z_for(p: float) -> float: + """Inverse standard normal CDF (Acklam's rational approximation; accurate to ~1e-9 here).""" + if not 0.0 < p < 1.0: + raise ValueError("probability must be in (0, 1)") + a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, + 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00] + b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, + 6.680131188771972e+01, -1.328068155288572e+01] + c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, + -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00] + d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00] + p_low, p_high = 0.02425, 1 - 0.02425 + if p < p_low: + q = math.sqrt(-2 * math.log(p)) + return (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) + if p > p_high: + q = math.sqrt(-2 * math.log(1 - p)) + return -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) + q = p - 0.5 + r = q * q + return (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q / (((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1) + + +def verdict(result: dict, interval: dict, alpha: float = 0.05) -> str: + """One line a person can act on, rather than a p-value to be interpreted optimistically.""" + if result["discordant"] == 0: + return "IDENTICAL — the two runs agree on every scored question." + if result["p_value"] > alpha: + return ( + f"NOT DISTINGUISHABLE (p={result['p_value']:.3f}) — this difference is what chance produces. " + f"The true gap is somewhere in [{interval['low']*100:+.1f}, {interval['high']*100:+.1f}] points." + ) + direction = "B beats A" if result["difference"] > 0 else "A beats B" + return ( + f"SIGNIFICANT (p={result['p_value']:.4f}) — {direction} by {abs(result['difference'])*100:.1f} " + f"points, plausibly [{interval['low']*100:+.1f}, {interval['high']*100:+.1f}]." + ) + + +def main() -> int: + ap = argparse.ArgumentParser(description="Is a benchmark difference real, or is it noise?") + ap.add_argument("logs", nargs="*", help="two result JSONL logs to compare") + ap.add_argument("--system", default=None, help="system name in both logs") + ap.add_argument("--system-a", default=None) + ap.add_argument("--system-b", default=None) + ap.add_argument("--iterations", type=int, default=10_000) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--plan", type=int, default=0, help="item count to plan for, instead of comparing") + ap.add_argument("--discordance", type=float, default=0.08, + help="expected fraction of items the two systems answer differently") + args = ap.parse_args() + + if args.plan: + for rate in sorted({args.discordance, 0.05, 0.10, 0.20}): + plan = minimum_detectable_effect(args.plan, rate) + print( + f"n={args.plan:<6} discordance={rate:<5.0%} " + f"informative items={plan['discordant_items']:>6.0f} " + f"smallest detectable gain={plan['mde_points']:.2f} points" + ) + print( + "\nOnly the questions the two systems answer differently carry information, so the usable\n" + "sample is far smaller than the headline item count. A gain below the last column cannot be\n" + "told from chance by a run this size, however many times it is repeated within one run." + ) + return 0 + + if len(args.logs) != 2: + ap.error("give exactly two logs, or use --plan N") + + log_a, log_b = load(args.logs[0]), load(args.logs[1]) + system_a = args.system_a or args.system + system_b = args.system_b or args.system + if not (system_a and system_b): + ap.error("--system (or --system-a/--system-b) is required") + + pairs = paired_outcomes(log_a, log_b, system_a, system_b) + if not pairs: + print("no questions were scored by both runs — nothing comparable") + return 1 + + result = mcnemar_exact(pairs) + interval = bootstrap_difference(pairs, iterations=args.iterations, seed=args.seed) + + print(f"A = {Path(args.logs[0]).name} [{system_a}]") + print(f"B = {Path(args.logs[1]).name} [{system_b}]") + print(f"\ncompared on {result['n']} questions scored by both runs") + print(f" both correct {result['both_correct']:>5}") + print(f" only A correct {result['only_a']:>5}") + print(f" only B correct {result['only_b']:>5} <- these two are the entire evidence") + print(f" both wrong {result['neither']:>5}") + print(f"\naccuracy A {result['acc_a']*100:.1f}% B {result['acc_b']*100:.1f}% " + f"difference {result['difference']*100:+.1f} points") + print(f"\n{verdict(result, interval)}") + + plan = minimum_detectable_effect(result["n"], result["discordant"] / max(1, result["n"])) + print( + f"\nAt this sample and disagreement rate, the smallest gain a run like this could have detected " + f"is {plan['mde_points']:.2f} points." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/results/significance_headline.md b/results/significance_headline.md new file mode 100644 index 0000000..d3e80cb --- /dev/null +++ b/results/significance_headline.md @@ -0,0 +1,78 @@ +# 显著性检验：headline 声称与基准的分辨率上限（2026-08-16） + +## 为什么需要这个 + +同一份配置重跑 LongMemEval_S，500 题里有 6–10 题的结果会变。因此「83.6 → 84.4，提升了」不是发现， +而是噪声。此前一长串机制（融合权重、cross-encoder rerank、agentic/HyDE、verify-retry、结构化偏好层、 +选择性一致性、K=5 自洽…）都是在这个噪声带里被判定为净收益 ≤0 的。 + +在能分辨小增益之前，任何算法优化都无法验证。**先造尺子，再爬山。** + +`eval/significance.py` 提供三件事：McNemar 精确检验（配对，只用两系统答案不同的题）、差值的 +bootstrap 区间、以及**最小可检测增益**（MDE，跑之前就能回答「这次实验能不能分辨我期望的增益」）。 + +## 复现 + +```bash +# headline 声称是否显著 +python3 eval/significance.py \ + results/longmemeval_s_volcano_doubao_deepseekjudge.jsonl \ + results/longmemeval_s_engram_lean_v2_final.jsonl \ + --system-a full_context --system-b engram_lean + +# 规划：500 题能分辨多小的增益 +python3 eval/significance.py --plan 500 +``` + +## 结果 1：公开 headline 经受住检验 + +`engram_lean` 83.6% vs `full_context` 73.2%，500 题配对： + +| | 数量 | +| --- | ---: | +| 两者都对 | 337 | +| 仅 full_context 对 | 29 | +| 仅 engram_lean 对 | **81** | +| 两者都错 | 53 | + +只有那 110 道分歧题携带信息。**p < 0.0001，差值 +10.4 点，95% bootstrap 区间 [+6.4, +14.4]。** +81 : 29 的分歧比不是掷硬币能产生的结果。这个声称成立，而且现在有检验而不只是两个百分数。 + +> 方法学说明：这两个数字来自**两次不同的运行**（同一 answerer + judge，同一批 500 题）。配对检验 +> 要求同一批题目，这点满足；但 CONTRIBUTING 要求基线来自 *same run*，严格来说 headline 表应注明 +> 基线取自 `..._volcano_doubao_deepseekjudge.jsonl`。同一份日志内的 `engram_full` 83.4% vs +> `full_context` 73.2% 是严格同轮对照。 + +## 结果 2：基准的分辨率上限——这才是登顶的真正障碍 + +| 题数 | 分歧率 | 有效信息题数 | 最小可检测增益 | +| ---: | ---: | ---: | ---: | +| 500 | 5% | 25 | 1.40 点 | +| 500 | 8% | 40 | 1.77 点 | +| 500 | 10% | 50 | 1.98 点 | +| 500 | 20% | 100 | 2.80 点 | + +上面那次真实运行的分歧率是 22%，**最小可检测增益 2.94 点**。 + +含义直接而不舒服： + +- **到 Hunyuan 85.2 的 +1.6 差距，低于这个基准在 500 题规模上的分辨率。** 它不是「暂时没追上」， + 而是**在这套测量下无法判定真假**。反复调算法去追它，产出的是噪声不是名次。 +- 任何小于约 3 点的机制改进，在单次 500 题运行里都无法与偶然区分——无论重跑多少次**同一次运行内部** + 的自洽采样都不行（那减少的是答题方差，不是判定所需的样本量）。 + +## 对「做到世界第一」的可执行推论 + +1. **只做期望增益 > 3 点的改动。** 低于这个的，先不要花钱跑。目标锁定占比最大的弱类别： + multi-session（121 题，70.2%）与 temporal-reasoning（127 题，70.9%），合计 248/500。 + 在单一类别上拿到 +10 点，整体才约 +2.5——这就是为什么必须挑大类别下手。 +2. **要想分辨更小的增益，只能加分辨率**：更多题目（LongMemEval_M、LOCOMO、PersonaMem 合并评测）、 + 或更确定性的 answerer。这是测量投资，不是算法投资，但它是所有算法投资的前置条件。 +3. **公开声称必须附检验。** 「世界第一」在一个 ±3 点分辨率的基准上不是良定义的主张； + 可辩护的说法是「在 X 基准上以 p<0.0001 显著优于全上下文基线，区间 [+6.4,+14.4]」。 + CONTRIBUTING 本来就禁止未经基准验证的 "#1"/"SOTA" 措辞。 + +## 未采用/回滚原因 + +- **继续在 500 题上做 <3 点的算法迭代**：不采用。测量无法验证，等于用真金白银换噪声。 + 已有一长串证据（见 `docs/architecture-optimization-map.zh-CN.md` 与 noise floor 结论）。 diff --git a/tests/test_significance.py b/tests/test_significance.py new file mode 100644 index 0000000..b4943a7 --- /dev/null +++ b/tests/test_significance.py @@ -0,0 +1,169 @@ +"""Telling a real benchmark gain from the answerer's noise. + +An identical configuration re-run on LongMemEval_S moves 6-10 of 500 answers, so a bare accuracy +difference smaller than that is not evidence of anything. These tests pin the instrument that decides: +the paired test must ignore the questions both systems answered the same way, must not call a coin-flip +split significant, and must report how small a gain the run could have resolved at all. +""" +from __future__ import annotations + +import pytest + +from eval.significance import ( + bootstrap_difference, + mcnemar_exact, + minimum_detectable_effect, + paired_outcomes, + verdict, +) + + +def _pairs(both: int, only_a: int, only_b: int, neither: int): + out = [] + for i in range(both): + out.append((f"both{i}", True, True)) + for i in range(only_a): + out.append((f"a{i}", True, False)) + for i in range(only_b): + out.append((f"b{i}", False, True)) + for i in range(neither): + out.append((f"n{i}", False, False)) + return out + + +# --- McNemar --- + + +def test_agreements_carry_no_weight(): + """The heart of the paired test: questions both systems get right or wrong say nothing about which + is better, so adding a thousand of them must not change the verdict.""" + small = mcnemar_exact(_pairs(both=10, only_a=2, only_b=8, neither=10)) + padded = mcnemar_exact(_pairs(both=1000, only_a=2, only_b=8, neither=1000)) + assert small["p_value"] == padded["p_value"] + + +def test_a_coin_flip_split_is_not_significant(): + """Equal disagreements in both directions is exactly what two equivalent systems produce.""" + result = mcnemar_exact(_pairs(both=400, only_a=25, only_b=25, neither=50)) + assert result["p_value"] == 1.0 + assert result["difference"] == 0.0 + + +def test_a_lopsided_split_is_significant(): + result = mcnemar_exact(_pairs(both=337, only_a=29, only_b=81, neither=53)) + assert result["p_value"] < 0.001 + assert result["difference"] > 0, "positive means B is ahead" + + +def test_a_small_lead_over_few_disagreements_is_not_significant(): + """The regression this whole module exists to prevent: +3 points that chance explains.""" + result = mcnemar_exact(_pairs(both=329, only_a=51, only_b=66, neither=54)) + assert result["difference"] == pytest.approx(0.03, abs=0.005) + assert result["p_value"] > 0.05, "a 15-question edge over 117 disagreements is noise" + + +def test_identical_runs_report_no_evidence(): + result = mcnemar_exact(_pairs(both=450, only_a=0, only_b=0, neither=50)) + assert result["discordant"] == 0 + assert result["p_value"] == 1.0 + assert "IDENTICAL" in verdict(result, {"low": 0.0, "high": 0.0}) + + +def test_accuracy_and_difference_match_the_counts(): + result = mcnemar_exact(_pairs(both=337, only_a=29, only_b=81, neither=53)) + assert result["n"] == 500 + assert result["acc_a"] == pytest.approx((337 + 29) / 500) + assert result["acc_b"] == pytest.approx((337 + 81) / 500) + assert result["difference"] == pytest.approx((81 - 29) / 500) + + +def test_empty_input_does_not_explode(): + result = mcnemar_exact([]) + assert result["n"] == 0 + assert result["p_value"] == 1.0 + + +# --- bootstrap --- + + +def test_interval_brackets_the_observed_difference(): + pairs = _pairs(both=337, only_a=29, only_b=81, neither=53) + result = mcnemar_exact(pairs) + interval = bootstrap_difference(pairs, iterations=2000, seed=1) + assert interval["low"] < result["difference"] < interval["high"] + + +def test_interval_is_reproducible_from_its_seed(): + """A reported interval that cannot be recomputed is not evidence.""" + pairs = _pairs(both=300, only_a=40, only_b=60, neither=100) + first = bootstrap_difference(pairs, iterations=1000, seed=7) + second = bootstrap_difference(pairs, iterations=1000, seed=7) + assert first == second + assert bootstrap_difference(pairs, iterations=1000, seed=8) != first + + +def test_a_non_significant_difference_has_an_interval_spanning_zero(): + pairs = _pairs(both=329, only_a=51, only_b=66, neither=54) + interval = bootstrap_difference(pairs, iterations=3000, seed=0) + assert interval["low"] < 0 < interval["high"], "if zero is plausible, the claim is not established" + + +# --- planning --- + + +def test_more_items_resolve_smaller_gains(): + small = minimum_detectable_effect(500, 0.08)["mde_points"] + large = minimum_detectable_effect(5000, 0.08)["mde_points"] + assert large < small + + +def test_only_disagreements_count_toward_resolution(): + """Two systems that mostly agree have little to learn from, however many items were run.""" + assert minimum_detectable_effect(500, 0.02)["discordant_items"] == pytest.approx(10) + assert minimum_detectable_effect(500, 0.20)["discordant_items"] == pytest.approx(100) + + +def test_too_few_disagreements_resolves_nothing(): + assert minimum_detectable_effect(100, 0.001)["mde_points"] == float("inf") + + +def test_a_500_item_run_cannot_resolve_a_one_point_gain(): + """The number that matters for this project: the contested gap to the top of the leaderboard is + smaller than what a run of this size can distinguish from chance.""" + mde = minimum_detectable_effect(500, 0.08)["mde_points"] + assert mde > 1.6, "a 1.6-point gap is below this benchmark's resolution at n=500" + + +# --- pairing --- + + +def test_only_questions_both_runs_scored_are_compared(): + """Comparing over different question sets compares two different exams.""" + log_a = { + "q1": {"engram_lean": {"ok": True}}, + "q2": {"engram_lean": {"ok": True}}, + "q3": {"engram_lean": {"ok": False}}, + } + log_b = { + "q1": {"engram_lean": {"ok": False}}, + "q2": {"engram_lean": {"ok": True}}, + } + pairs = paired_outcomes(log_a, log_b, "engram_lean", "engram_lean") + assert {qid for qid, _a, _b in pairs} == {"q1", "q2"} + + +def test_errored_items_are_excluded_not_counted_wrong(): + """An errored item is missing data. Scoring it as wrong would penalise whichever system crashed on + the hardest questions, which is backwards.""" + log_a = {"q1": {"s": {"ok": True}}, "q2": {"s": {"err": "timeout"}}} + log_b = {"q1": {"s": {"ok": False}}, "q2": {"s": {"ok": True}}} + pairs = paired_outcomes(log_a, log_b, "s", "s") + assert [qid for qid, _a, _b in pairs] == ["q1"] + + +def test_abstentions_are_scored_as_wrong_not_dropped(): + """An abstention is an answer the system gave and the judge rejected — real data, unlike an error.""" + log_a = {"q1": {"s": {"ok": False}}} + log_b = {"q1": {"s": {"ok": True}}} + pairs = paired_outcomes(log_a, log_b, "s", "s") + assert pairs == [("q1", False, True)] From 062e02ea6da6c8d7a8851650006b758d1116a29a Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 00:21:38 +0800 Subject: [PATCH 18/25] eval: measure how much the benchmark moves when nothing changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everyone here knows the answerer flips "roughly 6-10 of 500" between identical runs. That number is the floor under every accuracy claim the project can make — a mechanism gaining less than it cannot be shown to work — and it has never actually been measured. The three committed lean logs are three different configurations, so none of them can supply it. eval/noise_floor.py takes two or more runs of the same config and reports what changed: how many answers flipped, in which direction, the widest accuracy gap between supposedly identical runs, and the smallest gain that survives as a consequence. Flips are the apparatus, not a regression, so both directions are reported as one phenomenon rather than as one run beating the other. It also warns when two "identical" runs differ by more than chance, because that is not noise — it means the runs were not identical, and averaging it in would bake a configuration difference into the floor. The runbook spells out what the measurement needs (two provider keys and the dataset, neither present in this environment), what it costs from the committed logs rather than a guess (4.78M context tokens and about 8.4 hours per run), and the requirement that the two commands differ only in --out. Before asking anyone to spend on this, the free check: the offline harness gives byte-identical results on this branch and on main (100.0%/70.0% accuracy, 5.4/14.2 context tokens), with lower latency from the negation early-out. So flips measured later are the answerer's, not this branch's. --- eval/NOISE_FLOOR_RUNBOOK.md | 71 ++++++++++++++++++ eval/noise_floor.py | 140 ++++++++++++++++++++++++++++++++++++ tests/test_noise_floor.py | 63 ++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 eval/NOISE_FLOOR_RUNBOOK.md create mode 100644 eval/noise_floor.py create mode 100644 tests/test_noise_floor.py diff --git a/eval/NOISE_FLOOR_RUNBOOK.md b/eval/NOISE_FLOOR_RUNBOOK.md new file mode 100644 index 0000000..65be16f --- /dev/null +++ b/eval/NOISE_FLOOR_RUNBOOK.md @@ -0,0 +1,71 @@ +# 噪声底测量 · 运行手册 + +## 为什么先跑这个 + +`results/significance_headline.md` 已经量出：500 题、22% 分歧率下，最小可检测增益约 **2.94 点**。 +但那是从**两个不同配置**的运行反推的分歧率。真正的噪声底——**同一份配置重跑会翻转多少题**—— +项目里至今没有测量，只有"大约 6–10 题"这个观察。 + +这个数字是所有精度声称的地板：低于它的增益，无论重跑多少次都无法证明。**先钉死地板，再爬。** + +已提交的三份 lean 日志（`..._v1` 78.8 / `..._v2_final` 83.6 / `headline_500` 79.0）都是**不同配置**， +所以不能用来算噪声底。必须新跑两次同配置。 + +## 需要你提供 + +| 项 | 说明 | +| --- | --- | +| `ARK_API_KEY`（+ `ARK_BASE_URL`） | 答题模型 doubao-seed-2.0-pro 与抽取模型 doubao-seed-1.6-flash | +| `DEEPSEEK_API_KEY` | 判分模型 deepseek-v3.2 | +| LongMemEval_S 数据集 | HuggingFace `xiaowu0162/longmemeval`，本仓库当前只有 2.7KB 的 sample | + +嵌入用本地 `bge-small`，不需要 key。 + +## 成本（来自已提交日志的真实数据，非估算） + +单次 `engram_lean` 500 题运行：送入答题模型的上下文合计 **4.78M tokens**，单题 p50 延迟 61s， +串行约 **8.4 小时**。跑两次即约 9.6M tokens / 17 小时（可并行以缩短墙钟时间）。 + +抽取与判分的调用量不在日志的 `tok` 字段里，属额外开销，按你的 provider 计费自行核算。 + +## 执行 + +两条命令**除 `--out` 外必须逐字符相同**——任何一个 flag 不同，测到的就不是噪声而是配置差异。 + +```bash +HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 python3 eval/bench.py \ + --data s --limit 500 --systems engram_lean \ + --answerer volcano:doubao-seed-2-0-pro-260215 \ + --judge volcano:deepseek-v3-2-251201 \ + --extractor volcano:doubao-seed-1-6-flash-250615 \ + --embedder bge-small --reasoning --persona \ + --chunks 2 --topk 15 --extract-k 8 --summ-k 28 --n-summaries 28 \ + --out results/noise_repeat_1.jsonl + +# 同一条命令，只改 --out +# --out results/noise_repeat_2.jsonl +``` + +## 读出结果 + +```bash +python3 eval/noise_floor.py results/noise_repeat_1.jsonl results/noise_repeat_2.jsonl \ + --system engram_lean +``` + +输出会给出：翻转题数与方向、两次运行的准确率差、以及据此推出的**最小可信增益**。 + +### 怎么解读 + +- **翻转本身不是 bug，也不是回归**，是测量仪器的误差棒。同配置下"翻成错"和"翻成对"是同一个现象， + 不要读成某一次更好。 +- 如果工具报出 `WARNING: some pairs differ by more than chance`，说明两次运行**并非真的同配置**—— + 检查 flag、数据集切片、模型版本是否一致，而不是把它当成噪声记下来。 +- 拿到地板数字后，更新 `results/significance_headline.md` 与架构地图的「算法迭代的前置条件」， + 之后每个机制提案都要先回答「预期增益是否高于地板」。 + +## 已完成的免费前置检查 + +跑之前已确认本分支相对 main **检索行为无漂移**：离线评测两侧完全一致 +（accuracy 100.0%/70.0%，context tokens 5.4/14.2），延迟因否定约束提前返回而更低。 +所以测出的翻转可以归因于答题模型的不确定性，而不是本轮代码改动。 diff --git a/eval/noise_floor.py b/eval/noise_floor.py new file mode 100644 index 0000000..ea374d4 --- /dev/null +++ b/eval/noise_floor.py @@ -0,0 +1,140 @@ +"""Measure how much the benchmark moves when nothing changes. + +The answerer is not deterministic at temperature 0: run an identical configuration twice and some +answers flip. Everyone working on this repo knows that as "roughly 6-10 of 500", which is an +observation, not a measurement — and the difference matters, because that number is the floor under +every accuracy claim the project can make. A mechanism that gains less than the floor cannot be shown +to work, however many times it is re-run. + +This turns the observation into a committed number. Feed it two or more runs of the *same* config and +it reports how many answers flipped, in which direction, and what the smallest trustworthy gain is as a +consequence. + + # produce the inputs (identical flags, different --out) — this is the part that costs money + python3 eval/bench.py --data s --limit 500 --systems engram_lean \ + --answerer volcano:doubao-seed-2-0-pro-260215 --judge volcano:deepseek-v3-2-251201 \ + --extractor volcano:doubao-seed-1-6-flash-250615 --embedder bge-small --reasoning --persona \ + --chunks 2 --topk 15 --extract-k 8 --summ-k 28 --n-summaries 28 \ + --out results/noise_repeat_1.jsonl + # ... same command again with --out results/noise_repeat_2.jsonl + + python3 eval/noise_floor.py results/noise_repeat_1.jsonl results/noise_repeat_2.jsonl \ + --system engram_lean + +A flip is not a bug and not a regression. It is the measurement apparatus, and this is its error bar. +""" +from __future__ import annotations + +import argparse +import itertools +import statistics +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from eval.compare import load # noqa: E402 +from eval.significance import ( # noqa: E402 + mcnemar_exact, + minimum_detectable_effect, + paired_outcomes, +) + +__all__ = ["compare_repeats", "summarise"] + + +def compare_repeats(logs: list[dict], system: str) -> list[dict]: + """Every pairing of identical-config runs, with what changed between them.""" + out = [] + for (index_a, log_a), (index_b, log_b) in itertools.combinations(enumerate(logs), 2): + pairs = paired_outcomes(log_a, log_b, system, system) + result = mcnemar_exact(pairs) + out.append({ + "a": index_a, + "b": index_b, + "n": result["n"], + # Direction is reported but should NOT be read as one run being better: with the same + # config, both directions are the same phenomenon. + "flipped_to_wrong": result["only_a"], + "flipped_to_right": result["only_b"], + "flips": result["discordant"], + "flip_rate": result["discordant"] / result["n"] if result["n"] else 0.0, + "acc_a": result["acc_a"], + "acc_b": result["acc_b"], + "spread": abs(result["acc_a"] - result["acc_b"]), + "p_value": result["p_value"], + }) + return out + + +def summarise(comparisons: list[dict]) -> dict: + """The floor: the flip rate across repeats, and the gain it makes unmeasurable.""" + if not comparisons: + return {} + flip_rates = [c["flip_rate"] for c in comparisons] + spreads = [c["spread"] for c in comparisons] + n = max(c["n"] for c in comparisons) + worst_rate = max(flip_rates) + plan = minimum_detectable_effect(n, worst_rate) + return { + "runs_compared": len(comparisons), + "items": n, + "mean_flip_rate": statistics.fmean(flip_rates), + "worst_flip_rate": worst_rate, + "mean_flips": statistics.fmean([c["flips"] for c in comparisons]), + "max_accuracy_spread": max(spreads), + "mde_points": plan["mde_points"], + # A same-config difference that reads as "significant" means the runs differ by more than + # chance -- i.e. the configs were not actually identical, or something else changed. + "suspicious_pairs": [c for c in comparisons if c["p_value"] < 0.05], + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="How much does the benchmark move when nothing changes?") + ap.add_argument("logs", nargs="+", help="two or more runs of the SAME configuration") + ap.add_argument("--system", required=True, help="system name present in every log") + args = ap.parse_args() + + if len(args.logs) < 2: + ap.error("need at least two runs of the same configuration") + + logs = [load(path) for path in args.logs] + comparisons = compare_repeats(logs, args.system) + if not any(c["n"] for c in comparisons): + print("no questions scored by both runs — are these the same benchmark?") + return 1 + + names = [Path(p).name for p in args.logs] + print(f"system: {args.system}") + for i, name in enumerate(names): + print(f" run {i}: {name}") + + print(f"\n{'pair':>8} {'items':>6} {'flips':>6} {'rate':>6} {'acc A':>7} {'acc B':>7} {'spread':>7}") + print("-" * 60) + for c in comparisons: + print( + f"{c['a']}↔{c['b']:<6} {c['n']:>6} {c['flips']:>6} {c['flip_rate']:>5.1%} " + f"{c['acc_a']:>6.1%} {c['acc_b']:>6.1%} {c['spread']:>6.1%}" + ) + + summary = summarise(comparisons) + print( + f"\nre-running the same configuration flips {summary['mean_flips']:.0f} of " + f"{summary['items']} answers on average ({summary['mean_flip_rate']:.1%}); the widest gap " + f"between two identical runs is {summary['max_accuracy_spread']*100:.1f} points." + ) + print( + f"\nTHE FLOOR: a claimed gain below {summary['mde_points']:.2f} points cannot be told apart " + f"from re-running the same configuration." + ) + if summary["suspicious_pairs"]: + print( + "\nWARNING: some pairs differ by more than chance (p < 0.05). Runs of an identical config " + "should not. Check that the flags, dataset slice and model versions really were the same." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_noise_floor.py b/tests/test_noise_floor.py new file mode 100644 index 0000000..cff3853 --- /dev/null +++ b/tests/test_noise_floor.py @@ -0,0 +1,63 @@ +"""Measuring the benchmark's own instability. + +The floor under every accuracy claim: if re-running an unchanged configuration moves N answers, a gain +smaller than N is not observable. These tests pin that the tool reports the flips honestly and warns when +two "identical" runs differ by more than chance — which means they were not identical. +""" +from __future__ import annotations + +from eval.noise_floor import compare_repeats, summarise + + +def _log(system: str, correct_qids: set, total: int = 100): + return { + f"q{i}": {system: {"ok": f"q{i}" in correct_qids}} + for i in range(total) + } + + +def test_identical_runs_report_no_flips(): + correct = {f"q{i}" for i in range(80)} + logs = [_log("s", correct), _log("s", correct)] + summary = summarise(compare_repeats(logs, "s")) + assert summary["mean_flips"] == 0 + assert summary["max_accuracy_spread"] == 0.0 + + +def test_flips_are_counted_in_both_directions(): + """With one config, a flip to wrong and a flip to right are the same phenomenon.""" + a = {f"q{i}" for i in range(80)} + b = (a - {"q0", "q1"}) | {"q90", "q91", "q92"} + comparisons = compare_repeats([_log("s", a), _log("s", b)], "s") + assert comparisons[0]["flipped_to_wrong"] == 2 + assert comparisons[0]["flipped_to_right"] == 3 + assert comparisons[0]["flips"] == 5 + + +def test_the_floor_rises_with_instability(): + """A noisier apparatus can resolve less.""" + base = {f"q{i}" for i in range(80)} + quiet = summarise(compare_repeats([_log("s", base), _log("s", base - {"q0"} | {"q90"})], "s")) + noisy = summarise(compare_repeats( + [_log("s", base), _log("s", base - {f"q{i}" for i in range(10)} | {f"q{90+i}" for i in range(10)})], + "s", + )) + assert noisy["mde_points"] > quiet["mde_points"] + + +def test_every_pair_of_runs_is_compared(): + correct = {f"q{i}" for i in range(80)} + comparisons = compare_repeats([_log("s", correct)] * 3, "s") + assert len(comparisons) == 3, "three runs give three pairings" + + +def test_runs_that_differ_by_more_than_chance_are_flagged(): + """Identical configs should not differ systematically. If they do, they were not identical.""" + a = {f"q{i}" for i in range(50)} + b = {f"q{i}" for i in range(85)} # a large one-directional shift + summary = summarise(compare_repeats([_log("s", a), _log("s", b)], "s")) + assert summary["suspicious_pairs"], "a systematic shift must be reported, not averaged away" + + +def test_no_comparisons_summarises_to_nothing(): + assert summarise([]) == {} From 5bd513a6b455d25f99b2d1f166af2a2a2bf82816 Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 00:27:07 +0800 Subject: [PATCH 19/25] eval: classify wrong answers by failure mode, and fix the log pointer that misdirects reproduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accuracy says how many questions were missed. It does not say whether the system had nothing to offer, was off by one, or answered confidently and wrongly — and those need different mechanisms. Designing against an aggregate means designing against a shape no individual failure has. eval/error_modes.py classifies each miss mechanically (abstained / numeric / wrong value) and reports what fixing each mode would be worth against what this benchmark can resolve. Run free against a committed log, before anyone is asked to pay for a run. Two findings on the headline log, both new relative to what was recorded before. The two large weak categories fail in different ways, so one mechanism cannot serve both: multi-session is 76% numeric errors (counting and cross-session aggregation) while temporal-reasoning is 63% abstentions (declining a question the memory could answer). Only the aggregate split was known before, which hid this. And the numeric errors go both ways — 16 under, 12 over — not the systematic undercount previously believed. That distinction decides the target: a one-sided miss means evidence was not recalled and recall expansion would fix it; a two-sided one means the evidence is largely there and the counting itself fails. The earlier belief would have aimed a mechanism at the wrong problem. Sizing against the 2.94-point floor: attacking either category alone lands right at the edge of measurability (19 questions = +3.8, 15 questions = +3.0). Both together are 34 questions, +6.8 points, comfortably resolvable. So the two lines have to be worked as one experiment. The tool's own first version had the bug it now warns about: it folded `_abs` items into their base category. Those are the benchmark's unanswerable variants, graded by a different judge, where refusing IS the right answer — counting them as abstention failures inflates the target and points the next mechanism at nothing. It now splits them exactly as report.py does, and the category counts agree. Separately, the optimization map pointed at results/headline_500.jsonl as "the current headline raw log". It is not: that file is a different run at 79.0/76.0. Anyone reproducing the published 83.6/73.2 from it would fail to match. Both real logs are now named, with a note that the two numbers come from two runs sharing the answerer, judge and item set. --- docs/architecture-optimization-map.zh-CN.md | 12 +- eval/error_modes.py | 178 ++++++++++++++++++++ results/error_modes_headline.md | 73 ++++++++ tests/test_error_modes.py | 82 +++++++++ 4 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 eval/error_modes.py create mode 100644 results/error_modes_headline.md create mode 100644 tests/test_error_modes.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 3fc73be..47a0d01 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -136,6 +136,11 @@ flowchart TD - 但**到榜首的 +1.6 差距低于分辨率**——不是「还没追上」，是这套测量判定不了真假。 - 因此：**只做期望增益 > 3 点的改动**，并优先攻占比最大的弱类别 multi-session（121 题 70.2%）与 temporal-reasoning（127 题 70.9%，合计 248/500）；单类别 +10 点才换来整体约 +2.5 点。 +- **两个大类的失效形态不同，需要两种机制**（`results/error_modes_headline.md`）： + multi-session 的错 **76% 是数值**（计数/聚合），temporal 的错 **63% 是弃答**（该答却说没有）。 + 且数值误差**双向**（低估 16 / 高估 12），排除了"证据召回不足"这个解释——是计数本身失败。 +- 单点机制打单个类别恰好卡在可测边缘（multi-session 数值 19 题 = +3.8；temporal 弃答 15 题 = +3.0）。 + **必须两条线一起做**（合计 34 题 = +6.8 点）才是舒服高于地板的实验。 - 想分辨更小的增益，只能加分辨率（更多题目 / 更确定性的 answerer），那是测量投资， 但它是所有算法投资的前置条件。 @@ -199,7 +204,12 @@ flowchart TD - 算法说明：`docs/algorithm-architecture.md` - 结果日志规则：`results/README.md` - 公开结果：`RESULTS.md` -- 当前 headline 原始日志：`results/headline_500.jsonl` +- 当前 headline 原始日志（**两个数字来自两次运行**，同 answerer + judge、同 500 题）： + - `engram_lean` 83.6% → `results/longmemeval_s_engram_lean_v2_final.jsonl` + - `full_context` 73.2% → `results/longmemeval_s_volcano_doubao_deepseekjudge.jsonl` + （同一份日志内还有严格同轮对照：`engram_full` 83.4% vs `full_context` 73.2%） + - ⚠️ `results/headline_500.jsonl` **不是** headline 日志，它是另一次配置的运行（79.0% / 76.0%）。 + 此处此前指向该文件，任何人照着复现都会对不上。 ## 当前结论 diff --git a/eval/error_modes.py b/eval/error_modes.py new file mode 100644 index 0000000..d0d75b6 --- /dev/null +++ b/eval/error_modes.py @@ -0,0 +1,178 @@ +"""What kind of wrong is each wrong answer? + +An accuracy number says how many questions were missed; it does not say whether the system had nothing +to say, said a number that was off by one, or confidently said the wrong thing. Those need different +mechanisms, and without separating them a proposal is aimed at an average rather than a failure. + +Run against a committed log — free, no API keys — before designing anything: + + python3 eval/error_modes.py results/longmemeval_s_engram_lean_v2_final.jsonl --system engram_lean + +The modes are deliberately few and mechanical, so the classification is reproducible rather than a +judgement call: + + * **abstained** — the system declined ("I don't know"). Evidence was retrieved badly or not at all. + * **numeric** — the gold answer contains a number and so did the prediction. Counting, dates, + durations, quantities: retrieval may have been fine and the arithmetic or the aggregation was not. + * **wrong value** — a confident answer that was simply not the right one. + +It also reports how much each mode is *worth*: eliminating a mode entirely moves the overall score by a +known number of points, which is what decides whether attacking it can be measured at all. +""" +from __future__ import annotations + +import argparse +import collections +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from eval.compare import load # noqa: E402 +from eval.significance import minimum_detectable_effect # noqa: E402 + +__all__ = ["classify", "attribute", "ABSTAIN_RE"] + +# Matched against the prediction. Anchored forms first so a mere mention of "unknown" inside a real +# answer is not counted as a refusal. +ABSTAIN_RE = re.compile( + r"^\s*(i\s+don'?t\s+know|i\s+do\s+not\s+know|unknown|n/?a|none)\b" + r"|don'?t\s+have\s+(that|this|it|any)" + r"|(not|isn'?t)\s+(in|mentioned\s+in)\s+(my\s+)?(memory|the\s+memory|our\s+conversations?)" + r"|no\s+information\s+(about|on|regarding)" + r"|cannot\s+(determine|tell|find)" + r"|记忆里(暂时)?没有|不知道|无法确定|没有(相关|提到)", + re.IGNORECASE, +) +_HAS_DIGIT = re.compile(r"\d") + + +def classify(pred: str, gold: str) -> str: + """One of: abstained | numeric | wrong_value.""" + pred = (pred or "").strip() + gold = (gold or "").strip() + if ABSTAIN_RE.search(pred): + return "abstained" + if _HAS_DIGIT.search(gold) and _HAS_DIGIT.search(pred): + return "numeric" + return "wrong_value" + + +def _leading_number(text: str) -> float | None: + match = re.search(r"-?\d+(?:\.\d+)?", text or "") + return float(match.group()) if match else None + + +def attribute(log: dict, system: str) -> dict: + """Per-category failure modes, plus the direction of numeric errors. + + The direction matters more than it looks: a systematic undercount points at missing evidence, which + recall expansion can fix. Errors in both directions point at the counting itself, which is a + different and harder target — so getting this wrong sends a mechanism after the wrong problem. + """ + by_category: dict[str, collections.Counter] = collections.defaultdict(collections.Counter) + totals: dict[str, int] = collections.Counter() + scored: dict[str, int] = collections.Counter() + direction = collections.Counter() + examples: dict[str, list] = collections.defaultdict(list) + + for qid, entry in log.items(): + # Same split report.py makes: `_abs` items are the benchmark's *unanswerable* variants, graded + # by a different judge, and on those a refusal is the correct answer. Folding them into their + # base category counts correct behaviour as a failure mode and aims mechanisms at the wrong + # target — which is exactly what the first version of this tool did. + category = "abstention" if str(qid).endswith("_abs") else entry.get("_cat", "?") + result = entry.get(system) + if not result or result.get("err"): + continue + scored[category] += 1 + if result.get("ok"): + continue + pred, gold = result.get("pred") or "", result.get("gold") or "" + mode = classify(pred, gold) + by_category[category][mode] += 1 + totals[mode] += 1 + if len(examples[f"{category}/{mode}"]) < 3: + examples[f"{category}/{mode}"].append((pred[:120], gold[:120])) + if mode == "numeric": + a, b = _leading_number(pred), _leading_number(gold) + if a is not None and b is not None: + direction["under" if a < b else "over" if a > b else "equal_but_judged_wrong"] += 1 + + return { + "by_category": {k: dict(v) for k, v in by_category.items()}, + "totals": dict(totals), + "scored": dict(scored), + "numeric_direction": dict(direction), + "examples": dict(examples), + "n": sum(scored.values()), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="Classify a run's wrong answers by failure mode.") + ap.add_argument("log") + ap.add_argument("--system", required=True) + ap.add_argument("--examples", action="store_true", help="print sample predictions per mode") + args = ap.parse_args() + + report = attribute(load(args.log), args.system) + if not report["n"]: + print(f"no scored items for system {args.system!r}") + return 1 + + n = report["n"] + modes = ("abstained", "numeric", "wrong_value") + print(f"{'category':<28}{'scored':>7}{'wrong':>7}" + "".join(f"{m:>13}" for m in modes)) + print("-" * (42 + 13 * len(modes))) + for category in sorted(report["by_category"], key=lambda c: -sum(report["by_category"][c].values())): + counts = report["by_category"][category] + wrong = sum(counts.values()) + print( + f"{category:<28}{report['scored'].get(category, 0):>7}{wrong:>7}" + + "".join(f"{counts.get(m, 0):>13}" for m in modes) + ) + total_wrong = sum(report["totals"].values()) + print( + f"{'TOTAL':<28}{n:>7}{total_wrong:>7}" + + "".join(f"{report['totals'].get(m, 0):>13}" for m in modes) + ) + + direction = report["numeric_direction"] + if direction: + print( + f"\nnumeric errors: {direction.get('under', 0)} under, {direction.get('over', 0)} over, " + f"{direction.get('equal_but_judged_wrong', 0)} numerically equal but judged wrong" + ) + if direction.get("under", 0) and direction.get("over", 0): + ratio = direction["under"] / max(1, direction["over"]) + if 0.5 <= ratio <= 2.0: + print( + " errors go both ways, so this is not missing evidence that recall expansion would\n" + " recover — the counting itself is what fails." + ) + + # What each mode is worth, against what the benchmark can actually resolve. + floor = minimum_detectable_effect(n, 0.22)["mde_points"] + print(f"\nfixing a mode completely would move the overall score by (floor at this size: {floor:.2f}):") + for mode in modes: + count = report["totals"].get(mode, 0) + if not count: + continue + points = 100.0 * count / n + verdict = "measurable" if points > floor else "BELOW THE FLOOR — unmeasurable alone" + print(f" {mode:<14} {count:>4} questions = {points:+.1f} points {verdict}") + + if args.examples: + print("\nexamples:") + for key, samples in sorted(report["examples"].items()): + print(f"\n {key}") + for pred, gold in samples: + print(f" pred: {pred}") + print(f" gold: {gold}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/results/error_modes_headline.md b/results/error_modes_headline.md new file mode 100644 index 0000000..d86e726 --- /dev/null +++ b/results/error_modes_headline.md @@ -0,0 +1,73 @@ +# 错例归因：82 个错答分别错在哪（2026-08-17） + +## 为什么先做这个 + +准确率只说"错了多少"，不说"错成什么样"。**弃答、数值差一、自信答错**需要三种不同机制。 +不拆开就只能对着平均值设计，而平均值没有对应的失效模式。 + +本分析**离线、免费**，用已提交日志 `results/longmemeval_s_engram_lean_v2_final.jsonl`（83.6%）。 + +## 复现 + +```bash +python3 eval/error_modes.py results/longmemeval_s_engram_lean_v2_final.jsonl --system engram_lean +python3 eval/error_modes.py results/longmemeval_s_engram_lean_v2_final.jsonl --system engram_lean --examples +``` + +## 结果 + +| 类别 | 已评分 | 错 | 弃答 | 数值 | 值错 | +| --- | ---: | ---: | ---: | ---: | ---: | +| multi-session | 121 | 25 | 5 | **19** | 1 | +| temporal-reasoning | 127 | 24 | **15** | 5 | 4 | +| knowledge-update | 72 | 9 | 2 | 4 | 3 | +| single-session-preference | 30 | 8 | 5 | 0 | 3 | +| single-session-user | 64 | 8 | 5 | 2 | 1 | +| abstention | 30 | 4 | 1 | 0 | 3 | +| single-session-assistant | 56 | 4 | 1 | 0 | 3 | +| **合计** | **500** | **82** | **34** | **30** | **18** | + +总分解（34 弃答 / 30 数值 / 18 值错）与 noise-floor 记忆里的观察独立吻合。 + +## 两个新发现 + +### 1. 两个大类的失效形态完全不同，需要两种机制 + +- **multi-session 的错 76% 是数值**（19/25）——计数与跨会话聚合。 +- **temporal-reasoning 的错 63% 是弃答**（15/24）——该答却说"记忆里没有"。 + +此前只有总分解，看不出这个区别。把一种机制同时指望修两个类别是无效的。 + +### 2. 数值误差是**双向**的，不是系统性低估（纠正既有认知） + +**低估 16 / 高估 12 / 数值相同但判错 2。** + +这条很要紧：若是系统性低估，说明证据没召回全，扩大召回即可修；**双向误差说明证据大体在， +是计数与算术本身失败**。靶子完全不同，且更难。既有记忆里"以低估为主"的说法据此修正。 + +## 可测量性（对照 2.94 点的分辨率地板） + +| 若某一模式被完全消除 | 题数 | 整体增益 | 是否可测 | +| --- | ---: | ---: | --- | +| 全部弃答 | 34 | +6.8 点 | 可测 | +| 全部数值错 | 30 | +6.0 点 | 可测 | +| 全部值错 | 18 | +3.6 点 | 可测 | +| **仅** multi-session 的数值错 | 19 | +3.8 点 | 勉强可测 | +| **仅** temporal 的弃答 | 15 | +3.0 点 | 贴着地板 | + +**结论：单点机制打单个类别，恰好卡在可测量性边缘。** 必须两条线一起做，合计 34 题 = +6.8 点， +才是舒服地高于地板的实验。 + +## 对机制设计的直接约束 + +1. **multi-session** → 目标是计数/聚合的正确性，不是召回量。双向误差已排除"证据不够"这个解释。 +2. **temporal-reasoning** → 目标是消除"该答却弃答"。需要先查这 15 题是检索没命中，还是命中了但 + 证据形态不足以支撑时间推理（区间/时长/顺序）。这一步同样可以离线做。 +3. 任何提案先回答：**预期能消除哪一模式的多少题**。低于约 15 题（+3 点）的提案， + 在 500 题上无法与噪声区分，不要花钱验证。 + +## 工具自身的一个已修缺陷 + +第一版把 `_abs` 后缀题并进了基础类别。那 30 题是基准的 **unanswerable 变体，由官方 +"unanswerable" 判分器评分——在那些题上弃答是正确答案**。并进去等于把正确行为记成失效模式， +会把机制引向不存在的靶子。已按 `report.py` 的同一口径修正，类别数现与其完全一致。 diff --git a/tests/test_error_modes.py b/tests/test_error_modes.py new file mode 100644 index 0000000..0f7b289 --- /dev/null +++ b/tests/test_error_modes.py @@ -0,0 +1,82 @@ +"""Classifying wrong answers by failure mode. + +An accuracy number says how many were missed, not what kind of wrong they were — and abstaining, being +off by one, and confidently answering wrong need different mechanisms. The load-bearing test here is +`test_unanswerable_items_are_not_counted_as_abstention_failures`: the benchmark's `_abs` items are +graded by an unanswerable judge, so refusing them is correct, and folding them into their base category +turns right behaviour into a failure mode and aims the next mechanism at nothing. +""" +from __future__ import annotations + +from eval.error_modes import attribute, classify + + +def test_refusals_are_recognised_in_both_languages(): + for text in ("I don't know", "i do not know.", "That's not in my memory", "记忆里暂时没有这条", + "No information about that", "Unknown"): + assert classify(text, "42") == "abstained", text + + +def test_a_refusal_word_inside_a_real_answer_is_not_a_refusal(): + """'unknown' appearing in an answer is not the system declining to answer.""" + assert classify("The scale is unknown for two of the kits, the rest are 1:48", "1:48") != "abstained" + + +def test_a_numeric_answer_against_a_numeric_gold_is_a_counting_failure(): + assert classify("3", "4") == "numeric" + assert classify("11 weeks and 4 days", "15") == "numeric" + + +def test_a_confident_non_numeric_miss_is_a_wrong_value(): + assert classify("a jazz quartet", "a bluegrass band") == "wrong_value" + + +def _log(rows): + """rows: (qid, cat, ok, pred, gold)""" + return { + qid: {"_cat": cat, "s": {"ok": ok, "pred": pred, "gold": gold}} + for qid, cat, ok, pred, gold in rows + } + + +def test_unanswerable_items_are_not_counted_as_abstention_failures(): + """`_abs` items are the benchmark's unanswerable variants; a refusal there is the right answer, so + they belong in their own bucket rather than inflating the category they came from.""" + log = _log([ + ("q1", "temporal-reasoning", False, "I don't know", "4 days"), + ("q2_abs", "temporal-reasoning", False, "I don't know", "unanswerable"), + ]) + report = attribute(log, "s") + assert report["by_category"]["temporal-reasoning"] == {"abstained": 1} + assert report["by_category"]["abstention"] == {"abstained": 1} + + +def test_correct_answers_are_not_classified(): + log = _log([("q1", "multi-session", True, "4", "4"), ("q2", "multi-session", False, "3", "4")]) + report = attribute(log, "s") + assert report["scored"]["multi-session"] == 2 + assert sum(report["by_category"]["multi-session"].values()) == 1 + + +def test_errored_items_are_excluded_from_the_denominator(): + """An error is missing data; counting it as scored would understate accuracy and the mode split.""" + log = {"q1": {"_cat": "c", "s": {"err": "timeout"}}, "q2": {"_cat": "c", "s": {"ok": True}}} + report = attribute(log, "s") + assert report["n"] == 1 + + +def test_numeric_direction_is_reported(): + """A one-sided miss means missing evidence; a two-sided one means the counting itself fails, and the + two call for different mechanisms.""" + log = _log([ + ("q1", "multi-session", False, "3", "4"), + ("q2", "multi-session", False, "5", "4"), + ("q3", "multi-session", False, "2", "4"), + ]) + direction = attribute(log, "s")["numeric_direction"] + assert direction["under"] == 2 + assert direction["over"] == 1 + + +def test_empty_log_reports_nothing_rather_than_dividing_by_zero(): + assert attribute({}, "s")["n"] == 0 From af8040d22864fb9ef0fda5d18105b1626e425df5 Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 01:46:49 +0800 Subject: [PATCH 20/25] eval: show that the refusals were not starved of evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The obvious reading of 34 abstentions is that retrieval failed to find anything, and the obvious fix is to retrieve more. The committed log says otherwise: the median retrieved context is 9,666 tokens on the questions the system refused and 9,600 on the ones it answered correctly — a one percent difference across every outcome type. The refusals were handed as much evidence as the successes. That relocates the problem. Whatever fails on those 34 questions happens after retrieval: either the right amount of the wrong evidence came back, or the evidence is present in a form that cannot support the reasoning asked for — dates retrieved, but no interval, duration or ordering derived from them. Either way, spending on recall expansion would buy nothing, which is what the previous reading would have recommended. error_modes.py now reports this split, so the check is one command rather than a one-off script. Separating the two remaining explanations needs the retrieved context itself, and the committed logs carry only its size. The harness already computes answer_session_hit; the existing context dumps happen to cover three of these questions, which is not a sample worth drawing from — noted as directional at most. A targeted re-run of just the 64 questions in the two failure modes would settle it at about an eighth of a full run, and that is written up as the one paid step this analysis actually needs. --- eval/error_modes.py | 22 ++++++++++++++++++++++ results/error_modes_headline.md | 21 +++++++++++++++++++++ tests/test_error_modes.py | 19 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/eval/error_modes.py b/eval/error_modes.py index d0d75b6..c8a3174 100644 --- a/eval/error_modes.py +++ b/eval/error_modes.py @@ -24,6 +24,7 @@ import argparse import collections import re +import statistics import sys from pathlib import Path @@ -76,6 +77,9 @@ def attribute(log: dict, system: str) -> dict: scored: dict[str, int] = collections.Counter() direction = collections.Counter() examples: dict[str, list] = collections.defaultdict(list) + # Context size per outcome. A refusal on a context as large as the ones that answered correctly is + # not retrieval starvation — the evidence budget was spent, and something after retrieval failed. + tokens: dict[str, list[int]] = collections.defaultdict(list) for qid, entry in log.items(): # Same split report.py makes: `_abs` items are the benchmark's *unanswerable* variants, graded @@ -87,10 +91,13 @@ def attribute(log: dict, system: str) -> dict: if not result or result.get("err"): continue scored[category] += 1 + context_tokens = int(result.get("tok") or 0) if result.get("ok"): + tokens["correct"].append(context_tokens) continue pred, gold = result.get("pred") or "", result.get("gold") or "" mode = classify(pred, gold) + tokens[mode].append(context_tokens) by_category[category][mode] += 1 totals[mode] += 1 if len(examples[f"{category}/{mode}"]) < 3: @@ -105,6 +112,9 @@ def attribute(log: dict, system: str) -> dict: "totals": dict(totals), "scored": dict(scored), "numeric_direction": dict(direction), + "median_context_tokens": { + mode: statistics.median(values) for mode, values in tokens.items() if values + }, "examples": dict(examples), "n": sum(scored.values()), } @@ -164,6 +174,18 @@ def main() -> int: verdict = "measurable" if points > floor else "BELOW THE FLOOR — unmeasurable alone" print(f" {mode:<14} {count:>4} questions = {points:+.1f} points {verdict}") + medians = report.get("median_context_tokens") or {} + if len(medians) > 1: + print("\nmedian retrieved context, by outcome:") + for mode, value in sorted(medians.items(), key=lambda kv: -kv[1]): + print(f" {mode:<14} {value:>8.0f} tokens") + spread = (max(medians.values()) - min(medians.values())) / max(1.0, max(medians.values())) + if spread < 0.15: + print( + " Within a few percent of each other: the refusals and the misses were given as much\n" + " evidence as the correct answers. Whatever failed, it was not retrieval running dry." + ) + if args.examples: print("\nexamples:") for key, samples in sorted(report["examples"].items()): diff --git a/results/error_modes_headline.md b/results/error_modes_headline.md index d86e726..445a07d 100644 --- a/results/error_modes_headline.md +++ b/results/error_modes_headline.md @@ -58,6 +58,27 @@ python3 eval/error_modes.py results/longmemeval_s_engram_lean_v2_final.jsonl --s **结论：单点机制打单个类别，恰好卡在可测量性边缘。** 必须两条线一起做，合计 34 题 = +6.8 点， 才是舒服地高于地板的实验。 +### 3. 弃答**不是**检索饿死——上下文规模在所有结果类型上一致 + +| 结果 | 检索上下文 tokens 中位数 | +| --- | ---: | +| 弃答 | 9,666 | +| 答对 | 9,600 | +| 数值错 | 9,568 | +| 值错 | 9,561 | + +差异约 1%。**弃答题拿到的证据和答对的题一样多**。如果是检索没命中，上下文会明显偏小。 +所以那 34 道弃答的修法不在"多检索一点"，而在检索之后：要么召回的是错的证据（量对、内容不对）， +要么证据在但形态撑不住时间推理（有日期，但没有区间/时长/顺序）。 + +区分这两者需要那些题的**检索上下文正文**，而已提交日志只有 token 计数。 +好消息是 harness 本来就会算 `answer_session_hit`（答案所在会话是否被检索到）， +只是现有 context dump 只覆盖到其中 3 道，样本太小不能下结论——方向上 3/3 的数值错题都命中了 +答案会话（与"证据在、计数失败"一致），但 n=3 不是证据。 + +**一次定向重跑即可定论**：只跑那 34 道弃答题 + 30 道数值题（64 题，约全量的 1/8）， +导出 `answer_session_hit` 与上下文正文。这是本分析唯一需要付费的部分，成本约为全量运行的 13%。 + ## 对机制设计的直接约束 1. **multi-session** → 目标是计数/聚合的正确性，不是召回量。双向误差已排除"证据不够"这个解释。 diff --git a/tests/test_error_modes.py b/tests/test_error_modes.py index 0f7b289..2a9cadb 100644 --- a/tests/test_error_modes.py +++ b/tests/test_error_modes.py @@ -80,3 +80,22 @@ def test_numeric_direction_is_reported(): def test_empty_log_reports_nothing_rather_than_dividing_by_zero(): assert attribute({}, "s")["n"] == 0 + + +def test_context_size_is_reported_per_outcome(): + """A refusal on as much evidence as the correct answers got is not retrieval running dry — the + distinction decides whether the next mechanism belongs before or after retrieval.""" + log = { + "q1": {"_cat": "c", "s": {"ok": True, "tok": 9600}}, + "q2": {"_cat": "c", "s": {"ok": False, "pred": "I don't know", "gold": "4 days", "tok": 9650}}, + "q3": {"_cat": "c", "s": {"ok": False, "pred": "3", "gold": "4", "tok": 9500}}, + } + medians = attribute(log, "s")["median_context_tokens"] + assert medians["correct"] == 9600 + assert medians["abstained"] == 9650 + assert medians["numeric"] == 9500 + + +def test_missing_token_counts_do_not_break_the_report(): + log = {"q1": {"_cat": "c", "s": {"ok": False, "pred": "x", "gold": "y"}}} + assert attribute(log, "s")["median_context_tokens"]["wrong_value"] == 0 From d3eda0701b2ae82cdf7b060158e4ed7886d3e592 Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 02:23:32 +0800 Subject: [PATCH 21/25] eval: print the significance test next to the accuracy table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report.py put two accuracy percentages side by side and left the reader to subtract them. On this benchmark an unchanged configuration moves several answers on its own, so a gap can be smaller than the apparatus that measured it — and nobody reliably remembers to run the test separately before believing a number. Every multi-system log now carries the verdict under the table. It immediately caught a presentation bug in my own code. The difference is computed B-minus-A, so when A was the winner the sentence said "A beats B by 10.0 points, plausibly [-13.4, -6.6]" — a reader cannot tell from that whether the effect is positive or negative. The interval is now reoriented to match whichever system the sentence names first, with a test pinning both directions. report.py runs as a script, so the repo root was not importable and the sibling analysis modules could not be reached; it now puts the root on the path the same way the other eval tools do. --- eval/report.py | 48 ++++++++++++- eval/retrieval_check.py | 143 +++++++++++++++++++++++++++++++++++++ eval/significance.py | 13 +++- tests/test_significance.py | 12 ++++ 4 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 eval/retrieval_check.py diff --git a/eval/report.py b/eval/report.py index 121e383..2eab68c 100644 --- a/eval/report.py +++ b/eval/report.py @@ -12,6 +12,11 @@ import json import sys from collections import defaultdict +from pathlib import Path + +# Run as a script (`python3 eval/report.py …`), so the repo root is not on the path and `eval` is not +# importable as a package. The sibling analysis modules need it to be. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) def load(path: str) -> list[dict]: with open(path, encoding="utf-8") as fh: @@ -162,12 +167,53 @@ def format_bench_report(path: str, rows: list[dict]) -> str: return "\n".join(lines) +def format_significance(rows: list[dict]) -> str: + """Whether the gaps in the table above are real, for every pair of systems in the run. + + Two accuracy percentages side by side invite the reader to subtract them and believe the result. On + this benchmark an unchanged configuration moves several answers by itself, so a gap can be smaller + than the apparatus. Printing the test next to the table means nobody has to remember to ask. + """ + from eval.significance import bootstrap_difference, mcnemar_exact, verdict + + systems: list[str] = [] + for row in rows: + for name in (row.get("sys") or {}): + if name not in systems: + systems.append(name) + if len(systems) < 2: + return "" + + lines = [" significance (paired McNemar over questions both systems scored)", ""] + for index, system_a in enumerate(systems): + for system_b in systems[index + 1:]: + pairs = [] + for row in rows: + results = row.get("sys") or {} + a, b = results.get(system_a), results.get(system_b) + if not a or not b or a.get("err") or b.get("err"): + continue + pairs.append((row.get("qid"), bool(a.get("ok")), bool(b.get("ok")))) + if not pairs: + continue + result = mcnemar_exact(pairs) + interval = bootstrap_difference(pairs, iterations=2000, seed=0) + lines.append(f" {system_a} vs {system_b}:") + lines.append(f" {verdict(result, interval)}") + lines.append("") + return "\n".join(lines) + + def main() -> None: if len(sys.argv) < 2: print("usage: python eval/report.py [more.jsonl ...]") return for path in sys.argv[1:]: - print(format_bench_report(path, load(path))) + rows = load(path) + print(format_bench_report(path, rows)) + report = format_significance(rows) + if report: + print(report) if __name__ == "__main__": diff --git a/eval/retrieval_check.py b/eval/retrieval_check.py new file mode 100644 index 0000000..02595a1 --- /dev/null +++ b/eval/retrieval_check.py @@ -0,0 +1,143 @@ +"""Did retrieval surface the evidence, or did reasoning fail on evidence it had? + +`error_modes.py` shows that refusals and correct answers were given the same amount of context, which +rules out retrieval running dry but not retrieval returning the *wrong* sessions. Those two call for +opposite fixes — better recall versus better use of what was recalled — so the distinction has to be +settled before designing anything. + +LongMemEval labels which sessions contain the answer (`answer_session_ids`), so this can be checked +directly. And checked **without spending anything**: session retrieval is driven by the embedder and +BM25 over raw text, not by the extractor LLM, so the whole thing runs on the local bge-small model. + + python3 eval/retrieval_check.py --failures-from results/longmemeval_s_engram_lean_v2_final.jsonl \ + --system engram_lean --out results/retrieval_check_failures.jsonl + +A hit means the answer-bearing session was inside the retrieved slice. If refusals mostly hit, the +evidence was there and something after retrieval failed; if they mostly miss, retrieval is the target. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from eval.compare import load # noqa: E402 +from eval.error_modes import classify # noqa: E402 + +DEFAULT_DATASET = os.path.expanduser( + "~/.cache/huggingface/hub/datasets--xiaowu0162--longmemeval/snapshots/" + "2ec2a557f339b6c0369619b1ed5793734cc87533/longmemeval_s" +) + + +def failure_modes(log_path: str, system: str) -> dict[str, str]: + """qid -> failure mode, for the questions this run got wrong.""" + out = {} + for qid, entry in load(log_path).items(): + result = entry.get(system) + if not result or result.get("err") or result.get("ok"): + continue + out[qid] = classify(result.get("pred") or "", result.get("gold") or "") + return out + + +def check_item(item: dict, embedder, k_sessions: int) -> dict: + """Retrieve for one question and report whether the answer's session came back. + + Ingests only this item's haystack, exactly as the benchmark's per-question setup does, so the + retrieval being measured is the retrieval the run actually performed. + """ + from engram.memory import Memory + from engram.util import DAY, now + + mem = Memory(embedder=embedder) + base = now() - len(item["haystack_sessions"]) * DAY + session_of_episode = {} + for index, (session_id, session) in enumerate( + zip(item["haystack_session_ids"], item["haystack_sessions"]) + ): + text = "\n".join( + f"{turn.get('role', 'user')}: {turn.get('content', '')}" for turn in session + ) + episode = mem.add(text, user_id="u", session_id=session_id, event_time=base + index * DAY) + session_of_episode[episode.id] = session_id + + retrieved = mem.retrieve_episodes(item["question"], "u", k=k_sessions) + got = {session_of_episode.get(ep.id, ep.session_id) for ep in retrieved} + wanted = set(item.get("answer_session_ids") or []) + return { + "qid": item["question_id"], + "cat": item.get("question_type"), + "answer_sessions": len(wanted), + "haystack_sessions": len(item["haystack_session_ids"]), + "hit": bool(wanted & got), + "retrieved": len(got), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="Was the answer's session retrieved at all?") + ap.add_argument("--failures-from", required=True, help="a bench log; its wrong answers are checked") + ap.add_argument("--system", default="engram_lean") + ap.add_argument("--dataset", default=DEFAULT_DATASET) + ap.add_argument("--k-sessions", type=int, default=15, help="retrieved slice width to test") + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--out", default=None) + args = ap.parse_args() + + modes = failure_modes(args.failures_from, args.system) + if not modes: + print("no failures in that log") + return 1 + + with open(args.dataset, encoding="utf-8") as fh: + dataset = {item["question_id"]: item for item in json.load(fh)} + + targets = [qid for qid in modes if qid in dataset] + if args.limit: + targets = targets[: args.limit] + print(f"checking {len(targets)} failed questions (LLM-free; local embedder only)\n") + + from engram.llm.providers import make_embedder + + embedder = make_embedder("bge-small") + + rows = [] + started = time.time() + for index, qid in enumerate(targets, start=1): + row = check_item(dataset[qid], embedder, args.k_sessions) + row["mode"] = modes[qid] + rows.append(row) + if index % 10 == 0 or index == len(targets): + elapsed = time.time() - started + print(f" {index}/{len(targets)} ({elapsed:.0f}s)", flush=True) + + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + + print(f"\n{'failure mode':<16}{'checked':>9}{'answer session retrieved':>26}") + print("-" * 51) + for mode in ("abstained", "numeric", "wrong_value"): + subset = [r for r in rows if r["mode"] == mode] + if not subset: + continue + hits = sum(1 for r in subset if r["hit"]) + print(f"{mode:<16}{len(subset):>9}{f'{hits}/{len(subset)} ({hits/len(subset):.0%})':>26}") + total_hits = sum(1 for r in rows if r["hit"]) + print(f"{'ALL':<16}{len(rows):>9}{f'{total_hits}/{len(rows)} ({total_hits/len(rows):.0%})':>26}") + print( + "\nA high hit rate means the evidence was in the retrieved slice and the failure happened after\n" + "retrieval — so recall expansion would buy nothing. A low one makes retrieval the target." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/significance.py b/eval/significance.py index cb8f74e..837a31a 100644 --- a/eval/significance.py +++ b/eval/significance.py @@ -188,10 +188,19 @@ def verdict(result: dict, interval: dict, alpha: float = 0.05) -> str: f"NOT DISTINGUISHABLE (p={result['p_value']:.3f}) — this difference is what chance produces. " f"The true gap is somewhere in [{interval['low']*100:+.1f}, {interval['high']*100:+.1f}] points." ) - direction = "B beats A" if result["difference"] > 0 else "A beats B" + b_ahead = result["difference"] > 0 + direction = "B beats A" if b_ahead else "A beats B" + # The interval is stated B-minus-A. When A is the winner the sentence names A first, so the interval + # has to be reoriented to match — otherwise it reads "A beats B by 10 points, plausibly [-13, -7]", + # and a reader cannot tell whether the effect is positive or negative. + low, high = ( + (interval["low"] * 100, interval["high"] * 100) + if b_ahead + else (-interval["high"] * 100, -interval["low"] * 100) + ) return ( f"SIGNIFICANT (p={result['p_value']:.4f}) — {direction} by {abs(result['difference'])*100:.1f} " - f"points, plausibly [{interval['low']*100:+.1f}, {interval['high']*100:+.1f}]." + f"points, plausibly [{low:+.1f}, {high:+.1f}]." ) diff --git a/tests/test_significance.py b/tests/test_significance.py index b4943a7..026c2aa 100644 --- a/tests/test_significance.py +++ b/tests/test_significance.py @@ -167,3 +167,15 @@ def test_abstentions_are_scored_as_wrong_not_dropped(): log_b = {"q1": {"s": {"ok": True}}} pairs = paired_outcomes(log_a, log_b, "s", "s") assert pairs == [("q1", False, True)] + + +def test_the_interval_is_oriented_to_match_the_stated_winner(): + """"A beats B by 10 points, plausibly [-13, -7]" tells a reader nothing about the sign of the + effect. When the sentence names A as the winner, the interval must be stated A-minus-B too.""" + a_ahead = mcnemar_exact(_pairs(both=337, only_a=81, only_b=29, neither=53)) + text = verdict(a_ahead, {"low": -0.144, "high": -0.064}) + assert "A beats B" in text + assert "[+6.4, +14.4]" in text, text + + b_ahead = mcnemar_exact(_pairs(both=337, only_a=29, only_b=81, neither=53)) + assert "[+6.4, +14.4]" in verdict(b_ahead, {"low": 0.064, "high": 0.144}) From 261793822e126c87fa971e03a17c845b6b9cc65a Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 02:36:48 +0800 Subject: [PATCH 22/25] =?UTF-8?q?eval:=20the=20evidence=20was=20already=20?= =?UTF-8?q?retrieved=20=E2=80=94=2073%=20of=20failures=20had=20it=20in=20f?= =?UTF-8?q?ull=20detail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three separate signals now agree that retrieval is not what fails. Refusals get as much context as correct answers. The answer-bearing session is retrieved for 79 of 82 failures. And the numeric errors go both ways rather than systematically low. Each ruled out a different retrieval explanation; together they locate the problem after the evidence is in hand. Rank makes it sharper than membership did. The run rendered its top 2 sessions in full and compressed the rest to summaries, and 60 of 82 failures had the answer session inside that full-detail window. The answerer was looking at the raw evidence and got it wrong anyway — 65% of the refusals, 77% of the counting errors. That retires a whole class of work. Anything aimed at retrieving more is worth at most 3 questions, or +0.6 points, against a floor of 2.94 — unmeasurable by construction. The optimization map's lightweight n-hop/PPR expansion is downgraded on that evidence rather than on taste. Widening the full-detail window is also declined, with its own numbers. Going from 2 chunks to 15 would promote 19 questions from summary to detail, a nominal ceiling of +3.8 points. But 60 questions already failed *with* full detail, so the conversion from "shown" to "correct" is demonstrably well under one, and the token cost moves back toward the full-context baseline the headline claim is built against. Buying a smaller-than-stated gain by conceding the thesis is a bad trade. The check itself is LLM-free — session retrieval runs on the local embedder and BM25, and the benchmark labels which session holds the answer — so all of this cost nothing to establish. --- docs/architecture-optimization-map.zh-CN.md | 6 +- eval/retrieval_check.py | 24 +++++- results/retrieval_check_failures.jsonl | 82 +++++++++++++++++++++ results/retrieval_diagnosis.md | 74 +++++++++++++++++++ tests/test_retrieval_check.py | 80 ++++++++++++++++++++ 5 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 results/retrieval_check_failures.jsonl create mode 100644 results/retrieval_diagnosis.md create mode 100644 tests/test_retrieval_check.py diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index 47a0d01..ac5b230 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -141,6 +141,10 @@ flowchart TD 且数值误差**双向**（低估 16 / 高估 12），排除了"证据召回不足"这个解释——是计数本身失败。 - 单点机制打单个类别恰好卡在可测边缘（multi-session 数值 19 题 = +3.8；temporal 弃答 15 题 = +3.0）。 **必须两条线一起做**（合计 34 题 = +6.8 点）才是舒服高于地板的实验。 +- **机制必须做在「证据到手之后」这一层**（`results/retrieval_diagnosis.md`）：82 道错题里 + 79 道（96%）答案会话已被检索到，其中 **73% 在前 2 名以全文展示**——答题模型拿着原文仍然答错。 + 提升召回的上限只有 3 题（+0.6 点）；扩大全文窗口到 15 的名义上限 +3.8 点，但 60 题已证明 + 全文条件下仍会失败，转化率远低于 1，且 token 成本朝全上下文回退、与 headline 论点冲突。 - 想分辨更小的增益，只能加分辨率（更多题目 / 更确定性的 answerer），那是测量投资， 但它是所有算法投资的前置条件。 @@ -152,7 +156,7 @@ flowchart TD | --- | --- | --- | --- | | P0 | Raw evidence fusion hardening | Engram 已验证 facts-only 会丢细节，hybrid 是 load-bearing 发现 | 已把 chain facts 接入 provenance promotion；下一步继续把 raw chunks、facts、graph paths、summary/provenance 证据类型化，减少重复和噪声 | | P0 | Chain-aware retrieval | knowledge-update 强项还可以转化成更稳定的 temporal/current-vs-past 能力 | 已落地 previous-value 源会话提升；下一步扩展到多段属性演化和 profile-level chain | -| P1 | Graph proximity / multi-hop | multi-session、multi-hop 是长期记忆系统最难类别，也是差异化战场 | 轻量 n-hop/PPR-style expansion，先用真实错例切片验证 | +| ~~P1~~ **降级** | ~~Graph proximity / multi-hop（提升召回）~~ | **已被错例证据降级**：82 道错题里 79 道（96%）答案会话本来就被检索到了，其中 73% 还在前 2 名以全文展示。任何"多检索一点"的机制上限是 3 题 = +0.6 点，低于 2.94 分辨率地板——做了也测不出来 | 证据：`results/retrieval_diagnosis.md`。图能力本身仍可用于证据**组织**（如跨会话可数项的结构化），但不要以提升召回为目标 | | P1 | Temporal interval reasoning | temporal-reasoning 仍低于 full-context，需要更强的区间和 duration 证据 | 显式 start/end pair、invalid_at span、date arithmetic block | | P2 | Runtime profiles | 让用户选择 lite/standard/graph/consolidated，并用同一 harness 报三联表 | 在 `Config`/bench 层定义可测 profile，而不是手动组合开关 | | ~~P0~~ 已落地 | ~~向量存储的过滤 ANN~~ | 已修复，见下方台账 `tenant_filter_pushdown` | 剩余：`InMemoryVectorStore` 仍是暴力扫描（参考实现，设计如此）；`query_entity_ids()` 仍扫 `graph.entities.values()`，需实体名索引；`LanceDBVectorStore.get()` 仍全表物化后线性找 key | diff --git a/eval/retrieval_check.py b/eval/retrieval_check.py index 02595a1..706aaeb 100644 --- a/eval/retrieval_check.py +++ b/eval/retrieval_check.py @@ -68,15 +68,20 @@ def check_item(item: dict, embedder, k_sessions: int) -> dict: session_of_episode[episode.id] = session_id retrieved = mem.retrieve_episodes(item["question"], "u", k=k_sessions) - got = {session_of_episode.get(ep.id, ep.session_id) for ep in retrieved} + ordered = [session_of_episode.get(ep.id, ep.session_id) for ep in retrieved] wanted = set(item.get("answer_session_ids") or []) + # Rank, not just membership. The read path shows only the top few sessions in FULL detail and the + # rest as summaries, so "somewhere in the top 15" and "shown as evidence the answerer can read" are + # different claims — and they point at different layers to fix. + rank = next((i + 1 for i, sid in enumerate(ordered) if sid in wanted), None) return { "qid": item["question_id"], "cat": item.get("question_type"), "answer_sessions": len(wanted), "haystack_sessions": len(item["haystack_session_ids"]), - "hit": bool(wanted & got), - "retrieved": len(got), + "hit": rank is not None, + "rank": rank, + "retrieved": len(ordered), } @@ -132,6 +137,19 @@ def main() -> int: print(f"{mode:<16}{len(subset):>9}{f'{hits}/{len(subset)} ({hits/len(subset):.0%})':>26}") total_hits = sum(1 for r in rows if r["hit"]) print(f"{'ALL':<16}{len(rows):>9}{f'{total_hits}/{len(rows)} ({total_hits/len(rows):.0%})':>26}") + + # Where in the ranking it landed decides which layer is at fault: inside the full-detail window means + # the answerer read it and still failed; outside means context assembly showed only a summary. + ranks = [r["rank"] for r in rows if r["rank"]] + if ranks: + print("\nrank of the answer session within the retrieved slice:") + for cut in (1, 2, 3, 5, 10, 15): + within = sum(1 for r in ranks if r <= cut) + print(f" top-{cut:<3} {within:>4}/{len(rows)} ({within/len(rows):.0%})") + print( + "\n The run under analysis rendered its top 2 sessions in full and the rest as summaries,\n" + " so top-2 is the share where the answerer had the raw evidence in front of it." + ) print( "\nA high hit rate means the evidence was in the retrieved slice and the failure happened after\n" "retrieval — so recall expansion would buy nothing. A low one makes retrieval the target." diff --git a/results/retrieval_check_failures.jsonl b/results/retrieval_check_failures.jsonl new file mode 100644 index 0000000..b04dcaf --- /dev/null +++ b/results/retrieval_check_failures.jsonl @@ -0,0 +1,82 @@ +{"qid": "bbf86515", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 7, "retrieved": 15, "mode": "abstained"} +{"qid": "gpt4_59c863d7", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_a1b77f9c", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "07741c45", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "gpt4_15e38248", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_fa19884d", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "gpt4_8279ba03", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 46, "hit": true, "rank": 15, "retrieved": 15, "mode": "abstained"} +{"qid": "945e3d21", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"qid": "1c0ddc50", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 52, "hit": true, "rank": 6, "retrieved": 15, "mode": "wrong_value"} +{"qid": "d682f1a2", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "7024f17c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "c18a7dc8", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 5, "retrieved": 15, "mode": "numeric"} +{"qid": "89941a94", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "gpt4_e061b84f", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "73d42213", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "a96c20ee_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 57, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "28dc39ac", "cat": "multi-session", "answer_sessions": 5, "haystack_sessions": 39, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "577d4d32", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 6, "retrieved": 15, "mode": "abstained"} +{"qid": "58ef2f1c", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "370a8ff4", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "6e984301", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "6ae235be", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "09d032c9", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 44, "hit": true, "rank": 3, "retrieved": 15, "mode": "abstained"} +{"qid": "ad7109d1", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 2, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_468eb064", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 9, "retrieved": 15, "mode": "abstained"} +{"qid": "4dfccbf8", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 49, "hit": true, "rank": 12, "retrieved": 15, "mode": "abstained"} +{"qid": "45dc21b6", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "726462e0", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 9, "retrieved": 15, "mode": "abstained"} +{"qid": "3a704032", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "ec81a493", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "157a136e", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "69fee5aa", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "71017277", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "031748ae_abs", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "0977f2af", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "9a707b81", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 58, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_59149c78", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"qid": "37f165cf", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "ceb54acb", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "59524333", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 45, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_7f6b06db", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 54, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"qid": "gpt4_e414231f", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "8cf51dda", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 47, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "b86304ba", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 43, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "gpt4_731e37d7", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_d6585ce8", "cat": "temporal-reasoning", "answer_sessions": 5, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_31ff4165", "cat": "multi-session", "answer_sessions": 5, "haystack_sessions": 42, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "9ea5eabc", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "bf659f65", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "d6233ab6", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 47, "hit": false, "rank": null, "retrieved": 15, "mode": "abstained"} +{"qid": "d23cf73b", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "8e91e7d9", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 3, "retrieved": 15, "mode": "numeric"} +{"qid": "eeda8a6d_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "af082822", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 44, "hit": true, "rank": 15, "retrieved": 15, "mode": "abstained"} +{"qid": "92a0aa75", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"qid": "0a995998", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 45, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "eac54add", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 45, "hit": true, "rank": 10, "retrieved": 15, "mode": "wrong_value"} +{"qid": "75832dbd", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 51, "hit": false, "rank": null, "retrieved": 15, "mode": "wrong_value"} +{"qid": "gpt4_d6585ce9", "cat": "temporal-reasoning", "answer_sessions": 5, "haystack_sessions": 43, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "32260d93", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 47, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "60472f9c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "10d9b85a", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"qid": "4f54b7c9", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "gpt4_7abb270c", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "b46e15ed", "cat": "temporal-reasoning", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 5, "retrieved": 15, "mode": "abstained"} +{"qid": "eaca4986", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "35a27287", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 54, "hit": true, "rank": 4, "retrieved": 15, "mode": "abstained"} +{"qid": "129d1232", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"qid": "9ee3ecd6", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "09ba9854_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "gpt4_4929293b", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 52, "hit": false, "rank": null, "retrieved": 15, "mode": "abstained"} +{"qid": "a1cc6108", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 44, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "a3838d2b", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "a82c026e", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"qid": "0edc2aef", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"qid": "gpt4_a56e767c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "0bc8ad93", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "6a1eabeb", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 41, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"qid": "66f24dbb", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 55, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"qid": "6e984302", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"qid": "caf03d32", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"qid": "6d550036", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 53, "hit": true, "rank": 5, "retrieved": 15, "mode": "numeric"} diff --git a/results/retrieval_diagnosis.md b/results/retrieval_diagnosis.md new file mode 100644 index 0000000..f93472a --- /dev/null +++ b/results/retrieval_diagnosis.md @@ -0,0 +1,74 @@ +# 检索诊断：82 道错题里，证据到底有没有到手（2026-08-17） + +## 结论先行 + +**73% 的错题，答案所在会话就在检索结果前 2 名、以全文形式摆在答题模型面前，它仍然答错了。** +只有 3 道（4%）是真的没检索到。 + +这判死了一整类机制：**任何奔着"多检索一点"去的改动，上限是 3 题 = +0.6 点**，远低于 +2.94 的分辨率地板（`results/significance_headline.md`）。优化地图 P1 的 +"轻量 n-hop / PPR 图扩展"据此应当降级——不是它不好，是在这份数据上它修不到东西。 + +## 复现（零 API 成本） + +```bash +python3 eval/retrieval_check.py \ + --failures-from results/longmemeval_s_engram_lean_v2_final.jsonl \ + --system engram_lean --out results/retrieval_check_failures.jsonl +``` + +会话检索由嵌入 + BM25 驱动，不由抽取模型驱动，所以整个诊断只用本地 bge-small，不调任何付费 API。 +LongMemEval 自带 `answer_session_ids` 标注，"答案在哪个会话"是数据集给定的，不需要判断。 + +## 结果 + +| 失效模式 | 题数 | 前 2（全文可见） | 3–15（仅摘要） | 未检索到 | +| --- | ---: | ---: | ---: | ---: | +| 弃答 | 34 | 22（65%） | 10（29%） | 2 | +| 数值错 | 30 | 23（77%） | 7（23%） | 0 | +| 值错 | 18 | 15（83%） | 2（11%） | 1 | +| **合计** | **82** | **60（73%）** | **19（23%）** | **3（4%）** | + +排名分布（累计）：top-1 59% / top-2 73% / top-3 76% / top-5 85% / top-10 93% / top-15 96%。 + +被分析的那次运行 `--chunks 2`，即**前 2 个会话以全文渲染，其余压成摘要**。所以 top-2 那一列 +就是"答题模型眼前有原始证据"的份额。 + +## 三条独立证据指向同一处 + +1. 弃答题的上下文规模（9,666 tokens）与答对题（9,600）几乎相同 → **不是检索饿死** + （`results/error_modes_headline.md`） +2. 答案会话 79/82 被检索到 → **不是检索找错** +3. 数值误差双向（低估 16 / 高估 12）→ **不是证据不全** + +**失败发生在证据到手之后。** + +## 扩大全文窗口值多少（以及为什么不建议） + +把 `--chunks` 从 2 扩到 15，最多能把 19 题的证据从摘要升为全文，**上限 +3.8 点**。但这是乐观上限， +有两个理由不该照做： + +1. **转化率已被证伪。** 已经有 60 题在全文条件下失败了。"看到全文"到"答对"的转化率显然不是 100%， + 而 +3.8 的算法假设它是。真实收益会明显更低，很可能落回地板以下。 +2. **成本与核心论点冲突。** 更多全文块 = 更多 token。项目的 headline 主张正是 + "精简检索胜过全上下文"（9.6k vs 79k）。把窗口开到 15 是朝全上下文回退， + 即使分数涨了，涨的也是论点本身在退让。 + +## 下一步该往哪打 + +弃答（34 题）与数值（30 题）合计 64 题 = +12.8 点上限，仍是唯一舒服高于地板的靶子， +但机制必须做在**证据到手之后**这一层： + +- **数值/计数**（30 题，77% 已有全文）：证据在，是聚合与算术失败。方向是把跨会话的可数项 + 结构化成显式候选表，而不是让答题模型从散文里数数。项目已有 + `numeric_aggregation_candidates`，需要先查它在这 30 题上是否触发、触发后为何仍错。 +- **弃答**（34 题，65% 已有全文）：拿着原文说"不知道"。需要区分是"证据形态撑不住时间推理" + （有日期但没有区间/时长/顺序）还是"弃答阈值过于保守"。前者对应显式区间证据块， + 后者是阈值调参——而阈值调参属于已知的噪声带内改动，不要单独做。 + +## 未采用/回滚原因 + +| 方向 | 结论 | 证据 | +| --- | --- | --- | +| 轻量 n-hop / PPR 图扩展（提升召回） | **降级**。上限 3 题 = +0.6 点，低于分辨率地板，做了也测不出来 | 本文件：79/82 已检索到 | +| 扩大全文块窗口 `--chunks 2 → 15` | **不推荐**。名义上限 +3.8 点，但 60 题已证明全文条件下仍会失败，真实转化率远低于 1；且 token 成本朝全上下文回退，与 headline 论点冲突 | 本文件排名分布 | diff --git a/tests/test_retrieval_check.py b/tests/test_retrieval_check.py new file mode 100644 index 0000000..040be88 --- /dev/null +++ b/tests/test_retrieval_check.py @@ -0,0 +1,80 @@ +"""Was the answer's session retrieved, and how highly? + +The finding this guards is that 73% of failures had the answer session in the full-detail window and +still failed — which retires every mechanism aimed at retrieving more. That conclusion rests on the rank +being right, so `test_rank_is_the_position_of_the_first_answer_session` is the load-bearing test: a +membership-only check would have said "retrieved" for a session shown as a one-line summary, and pointed +the next mechanism at the wrong layer. +""" +from __future__ import annotations + +from eval.retrieval_check import failure_modes + + +class _Embedder: + """Deterministic stand-in: no model download, no API.""" + + def embed(self, text: str) -> list[float]: + return [float(len(text) % 7), float(text.count("a")), 1.0] + + +def test_failure_modes_skips_correct_and_errored_answers(): + """Only wrong answers are diagnosed; an errored item is missing data, not a failure to explain.""" + import json + import tempfile + + rows = [ + {"qid": "q1", "cat": "c", "sys": {"s": {"ok": True, "pred": "4", "gold": "4"}}}, + {"qid": "q2", "cat": "c", "sys": {"s": {"ok": False, "pred": "I don't know", "gold": "4"}}}, + {"qid": "q3", "cat": "c", "sys": {"s": {"err": "timeout"}}}, + {"qid": "q4", "cat": "c", "sys": {"s": {"ok": False, "pred": "3", "gold": "4"}}}, + ] + with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as fh: + for row in rows: + fh.write(json.dumps(row) + "\n") + path = fh.name + + modes = failure_modes(path, "s") + assert modes == {"q2": "abstained", "q4": "numeric"} + + +def _item(answer_index: int, sessions: int = 5) -> dict: + """A haystack where one session obviously answers the question.""" + return { + "question_id": "q1", + "question_type": "temporal-reasoning", + "question": "aaaaaaaa", # the embedder above scores on 'a' count, so this ranks the marked one + "haystack_session_ids": [f"s{i}" for i in range(sessions)], + "haystack_sessions": [ + [{"role": "user", "content": "aaaaaaaa" if i == answer_index else f"filler {i}"}] + for i in range(sessions) + ], + "answer_session_ids": [f"s{answer_index}"], + } + + +def test_rank_is_the_position_of_the_first_answer_session(): + """Membership alone cannot distinguish 'shown in full' from 'compressed to a summary line'.""" + from eval.retrieval_check import check_item + + row = check_item(_item(answer_index=0), _Embedder(), k_sessions=5) + assert row["hit"] is True + assert row["rank"] == 1 + + +def test_a_missing_answer_session_reports_no_rank(): + from eval.retrieval_check import check_item + + item = _item(answer_index=0) + item["answer_session_ids"] = ["nowhere"] + row = check_item(item, _Embedder(), k_sessions=5) + assert row["hit"] is False + assert row["rank"] is None + + +def test_a_narrow_slice_can_miss_a_session_a_wide_one_finds(): + """k is the width of the slice being tested, so it must actually bound what comes back.""" + from eval.retrieval_check import check_item + + row = check_item(_item(answer_index=0, sessions=8), _Embedder(), k_sessions=1) + assert row["retrieved"] <= 1 From c63d0fdfede6960e56a644670002bf79d01cce55 Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 02:53:31 +0800 Subject: [PATCH 23/25] =?UTF-8?q?eval:=20correct=20the=20diagnosis=20?= =?UTF-8?q?=E2=80=94=20retrieval=20finds=20the=20evidence,=20assembly=20sh?= =?UTF-8?q?ows=20half=20of=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous conclusion here was wrong, and wrong in a way that would have sent the next mechanism at the wrong layer. It recorded only the rank of the *first* matching answer session and concluded that 73% of failures had the evidence in full detail, so the failure must be after retrieval. Most of these failures are counting questions whose answer spans several sessions. For those, "the best-ranked answer session made the top 2" can mean one of four made it — which is precisely why the count comes out wrong. The right measure is coverage, not first hit. Measured properly: retrieval recall is 86%, but only 14 of 58 multi-session failures (24%) had all their answer sessions inside the full-detail window, at an average coverage of 48%. The evidence is fetched and then half of it is compressed to summaries, because assembly renders the top 2 by relevance while counting needs coverage. That also explains what nothing had explained before: why the numeric errors go both ways. Sessions compressed to a summary get missed (16 undercounts); an ambiguous summary line gets counted twice (12 overcounts). The earlier reading — that arithmetic itself fails — was a layer off. So the mechanism this points at is narrow and testable: for aggregation queries, let the full-detail window cover the evidence set rather than the top N by relevance. The trigger already exists and already works — plan_evidence().aggregation fires on 90% of the counting failures against 79% on the counting successes, so widening the trigger, the obvious next idea, is also ruled out. What it does not do is influence how many chunks get rendered in full. This is deliberately not "widen chunks to 15", which was rejected on cost: aggregation queries are about a fifth of the traffic, so the extra tokens land only where they are needed instead of pushing every query back toward the full-context baseline. Sized against the floor: 44 questions lack full coverage; at even a 50% conversion that is +4.4 points. The conversion rate is the untested premise and is checked next, offline — 60 questions already failed with full detail in view, so it is certainly not 1. --- docs/architecture-optimization-map.zh-CN.md | 13 +- eval/retrieval_check.py | 21 +++ results/retrieval_check_failures.jsonl | 164 ++++++++++---------- results/retrieval_diagnosis.md | 45 +++++- tests/test_retrieval_check.py | 21 +++ 5 files changed, 176 insertions(+), 88 deletions(-) diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index ac5b230..a651e1c 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -141,10 +141,15 @@ flowchart TD 且数值误差**双向**（低估 16 / 高估 12），排除了"证据召回不足"这个解释——是计数本身失败。 - 单点机制打单个类别恰好卡在可测边缘（multi-session 数值 19 题 = +3.8；temporal 弃答 15 题 = +3.0）。 **必须两条线一起做**（合计 34 题 = +6.8 点）才是舒服高于地板的实验。 -- **机制必须做在「证据到手之后」这一层**（`results/retrieval_diagnosis.md`）：82 道错题里 - 79 道（96%）答案会话已被检索到，其中 **73% 在前 2 名以全文展示**——答题模型拿着原文仍然答错。 - 提升召回的上限只有 3 题（+0.6 点）；扩大全文窗口到 15 的名义上限 +3.8 点，但 60 题已证明 - 全文条件下仍会失败，转化率远低于 1，且 token 成本朝全上下文回退、与 headline 论点冲突。 +- **瓶颈在上下文装配，不在检索**（`results/retrieval_diagnosis.md`）：检索层召回率 86%， + 但 58 道多会话错题里只有 14 道（24%）的答案会话**全部**进了全文窗口，平均覆盖率仅 48%。 + 证据取回来了，装配时按相关度只渲染前 2 个全文、其余压成摘要——而计数需要的是**覆盖**不是**排序**。 + 这解释了双向误差：压成摘要→漏数（低估 16），摘要表述模糊→重复计入（高估 12）。 +- **唯一有证据支撑的机制方向**：对聚合类查询让全文窗口覆盖证据集合。触发条件已存在且正常工作 + （`plan_evidence().aggregation` 在计数错题上触发率 90%，对照答对题 79%），缺的是让它影响渲染 + 多少全文块。与"无差别扩 `--chunks` 到 15"的区别是只在聚合类查询（约占 20%）上扩，成本不外溢。 + 规模：44 道题缺完整覆盖，转化率一半即 +4.4 点，高于地板。**前提待离线证伪**：完整覆盖是否真能 + 让这些题答对——已知 60 道题在全文条件下仍失败，转化率不是 1。 - 想分辨更小的增益，只能加分辨率（更多题目 / 更确定性的 answerer），那是测量投资， 但它是所有算法投资的前置条件。 diff --git a/eval/retrieval_check.py b/eval/retrieval_check.py index 706aaeb..fa6cd68 100644 --- a/eval/retrieval_check.py +++ b/eval/retrieval_check.py @@ -74,7 +74,14 @@ def check_item(item: dict, embedder, k_sessions: int) -> dict: # rest as summaries, so "somewhere in the top 15" and "shown as evidence the answerer can read" are # different claims — and they point at different layers to fix. rank = next((i + 1 for i, sid in enumerate(ordered) if sid in wanted), None) + # Coverage, not just the first hit. A counting question whose answer spans four sessions cannot be + # answered from one of them, however highly that one ranked — so "the answer session was retrieved" + # is the wrong measure for exactly the questions that fail most. + covered_top2 = sum(1 for sid in ordered[:2] if sid in wanted) + covered_all = sum(1 for sid in ordered if sid in wanted) return { + "covered_top2": covered_top2, + "covered_all": covered_all, "qid": item["question_id"], "cat": item.get("question_type"), "answer_sessions": len(wanted), @@ -140,6 +147,20 @@ def main() -> int: # Where in the ranking it landed decides which layer is at fault: inside the full-detail window means # the answerer read it and still failed; outside means context assembly showed only a summary. + multi = [r for r in rows if r["answer_sessions"] > 1] + if multi: + full = sum(1 for r in multi if r["covered_top2"] == r["answer_sessions"]) + any_all = sum(1 for r in multi if r["covered_all"] == r["answer_sessions"]) + print( + f"\nquestions whose answer spans several sessions: {len(multi)}\n" + f" every answer session inside the full-detail window: {full}/{len(multi)}" + f" ({full/len(multi):.0%})\n" + f" every answer session retrieved at all: {any_all}/{len(multi)}" + f" ({any_all/len(multi):.0%})\n" + " A count cannot come out right from a subset of the sessions it has to cover, so for these\n" + " the question is coverage, not whether the top-ranked one was found." + ) + ranks = [r["rank"] for r in rows if r["rank"]] if ranks: print("\nrank of the answer session within the retrieved slice:") diff --git a/results/retrieval_check_failures.jsonl b/results/retrieval_check_failures.jsonl index b04dcaf..d9c025b 100644 --- a/results/retrieval_check_failures.jsonl +++ b/results/retrieval_check_failures.jsonl @@ -1,82 +1,82 @@ -{"qid": "bbf86515", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 7, "retrieved": 15, "mode": "abstained"} -{"qid": "gpt4_59c863d7", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_a1b77f9c", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "07741c45", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "gpt4_15e38248", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_fa19884d", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "gpt4_8279ba03", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 46, "hit": true, "rank": 15, "retrieved": 15, "mode": "abstained"} -{"qid": "945e3d21", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} -{"qid": "1c0ddc50", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 52, "hit": true, "rank": 6, "retrieved": 15, "mode": "wrong_value"} -{"qid": "d682f1a2", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "7024f17c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "c18a7dc8", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 5, "retrieved": 15, "mode": "numeric"} -{"qid": "89941a94", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "gpt4_e061b84f", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "73d42213", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "a96c20ee_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 57, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "28dc39ac", "cat": "multi-session", "answer_sessions": 5, "haystack_sessions": 39, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "577d4d32", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 6, "retrieved": 15, "mode": "abstained"} -{"qid": "58ef2f1c", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "370a8ff4", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "6e984301", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "6ae235be", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "09d032c9", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 44, "hit": true, "rank": 3, "retrieved": 15, "mode": "abstained"} -{"qid": "ad7109d1", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 2, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_468eb064", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 9, "retrieved": 15, "mode": "abstained"} -{"qid": "4dfccbf8", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 49, "hit": true, "rank": 12, "retrieved": 15, "mode": "abstained"} -{"qid": "45dc21b6", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "726462e0", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 9, "retrieved": 15, "mode": "abstained"} -{"qid": "3a704032", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "ec81a493", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "157a136e", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "69fee5aa", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "71017277", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "031748ae_abs", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "0977f2af", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "9a707b81", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 58, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_59149c78", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} -{"qid": "37f165cf", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "ceb54acb", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "59524333", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 45, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_7f6b06db", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 54, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} -{"qid": "gpt4_e414231f", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "8cf51dda", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 47, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "b86304ba", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 43, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "gpt4_731e37d7", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_d6585ce8", "cat": "temporal-reasoning", "answer_sessions": 5, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_31ff4165", "cat": "multi-session", "answer_sessions": 5, "haystack_sessions": 42, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "9ea5eabc", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "bf659f65", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "d6233ab6", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 47, "hit": false, "rank": null, "retrieved": 15, "mode": "abstained"} -{"qid": "d23cf73b", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "8e91e7d9", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 3, "retrieved": 15, "mode": "numeric"} -{"qid": "eeda8a6d_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "af082822", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 44, "hit": true, "rank": 15, "retrieved": 15, "mode": "abstained"} -{"qid": "92a0aa75", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} -{"qid": "0a995998", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 45, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "eac54add", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 45, "hit": true, "rank": 10, "retrieved": 15, "mode": "wrong_value"} -{"qid": "75832dbd", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 51, "hit": false, "rank": null, "retrieved": 15, "mode": "wrong_value"} -{"qid": "gpt4_d6585ce9", "cat": "temporal-reasoning", "answer_sessions": 5, "haystack_sessions": 43, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "32260d93", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 47, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "60472f9c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "10d9b85a", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} -{"qid": "4f54b7c9", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "gpt4_7abb270c", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "b46e15ed", "cat": "temporal-reasoning", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 5, "retrieved": 15, "mode": "abstained"} -{"qid": "eaca4986", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "35a27287", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 54, "hit": true, "rank": 4, "retrieved": 15, "mode": "abstained"} -{"qid": "129d1232", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} -{"qid": "9ee3ecd6", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "09ba9854_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "gpt4_4929293b", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 52, "hit": false, "rank": null, "retrieved": 15, "mode": "abstained"} -{"qid": "a1cc6108", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 44, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "a3838d2b", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "a82c026e", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} -{"qid": "0edc2aef", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} -{"qid": "gpt4_a56e767c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "0bc8ad93", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "6a1eabeb", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 41, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} -{"qid": "66f24dbb", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 55, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} -{"qid": "6e984302", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} -{"qid": "caf03d32", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} -{"qid": "6d550036", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 53, "hit": true, "rank": 5, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 2, "qid": "bbf86515", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 7, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 4, "qid": "gpt4_59c863d7", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 6, "qid": "gpt4_a1b77f9c", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "07741c45", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 2, "covered_all": 3, "qid": "gpt4_15e38248", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 2, "qid": "gpt4_fa19884d", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 1, "qid": "gpt4_8279ba03", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 46, "hit": true, "rank": 15, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "945e3d21", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 0, "covered_all": 1, "qid": "1c0ddc50", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 52, "hit": true, "rank": 6, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 2, "covered_all": 3, "qid": "d682f1a2", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 3, "qid": "7024f17c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 2, "qid": "c18a7dc8", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 5, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 2, "qid": "89941a94", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "gpt4_e061b84f", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 2, "qid": "73d42213", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 2, "qid": "a96c20ee_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 57, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 2, "covered_all": 5, "qid": "28dc39ac", "cat": "multi-session", "answer_sessions": 5, "haystack_sessions": 39, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 1, "qid": "577d4d32", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 6, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "58ef2f1c", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "370a8ff4", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 2, "qid": "6e984301", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 1, "qid": "6ae235be", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 0, "covered_all": 1, "qid": "09d032c9", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 44, "hit": true, "rank": 3, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "ad7109d1", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 2, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 1, "qid": "gpt4_468eb064", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 9, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 1, "qid": "4dfccbf8", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 49, "hit": true, "rank": 12, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "45dc21b6", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 1, "qid": "726462e0", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 9, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 3, "qid": "3a704032", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 1, "qid": "ec81a493", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "157a136e", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 2, "qid": "69fee5aa", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 1, "qid": "71017277", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 2, "qid": "031748ae_abs", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 1, "covered_all": 2, "qid": "0977f2af", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 47, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 2, "qid": "9a707b81", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 58, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 2, "qid": "gpt4_59149c78", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 2, "covered_all": 2, "qid": "37f165cf", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "ceb54acb", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 2, "covered_all": 2, "qid": "59524333", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 45, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 3, "qid": "gpt4_7f6b06db", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 54, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 2, "covered_all": 2, "qid": "gpt4_e414231f", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "8cf51dda", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 47, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 1, "covered_all": 1, "qid": "b86304ba", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 43, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 4, "qid": "gpt4_731e37d7", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 53, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 5, "qid": "gpt4_d6585ce8", "cat": "temporal-reasoning", "answer_sessions": 5, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 5, "qid": "gpt4_31ff4165", "cat": "multi-session", "answer_sessions": 5, "haystack_sessions": 42, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 2, "qid": "9ea5eabc", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 1, "covered_all": 3, "qid": "bf659f65", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 56, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 0, "qid": "d6233ab6", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 47, "hit": false, "rank": null, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 4, "qid": "d23cf73b", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 2, "qid": "8e91e7d9", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 50, "hit": true, "rank": 3, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 2, "qid": "eeda8a6d_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 1, "qid": "af082822", "cat": "temporal-reasoning", "answer_sessions": 1, "haystack_sessions": 44, "hit": true, "rank": 15, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 2, "qid": "92a0aa75", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 3, "qid": "0a995998", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 45, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 1, "qid": "eac54add", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 45, "hit": true, "rank": 10, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 0, "covered_all": 0, "qid": "75832dbd", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 51, "hit": false, "rank": null, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 1, "covered_all": 5, "qid": "gpt4_d6585ce9", "cat": "temporal-reasoning", "answer_sessions": 5, "haystack_sessions": 43, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "32260d93", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 47, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 3, "qid": "60472f9c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 1, "qid": "10d9b85a", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 2, "qid": "4f54b7c9", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 49, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 6, "qid": "gpt4_7abb270c", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 51, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 0, "covered_all": 4, "qid": "b46e15ed", "cat": "temporal-reasoning", "answer_sessions": 4, "haystack_sessions": 48, "hit": true, "rank": 5, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "eaca4986", "cat": "single-session-assistant", "answer_sessions": 1, "haystack_sessions": 50, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 1, "qid": "35a27287", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 54, "hit": true, "rank": 4, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 0, "covered_all": 3, "qid": "129d1232", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 50, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 2, "covered_all": 2, "qid": "9ee3ecd6", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 54, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 2, "qid": "09ba9854_abs", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 0, "covered_all": 0, "qid": "gpt4_4929293b", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 52, "hit": false, "rank": null, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "a1cc6108", "cat": "multi-session", "answer_sessions": 2, "haystack_sessions": 44, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 6, "qid": "a3838d2b", "cat": "temporal-reasoning", "answer_sessions": 6, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 0, "covered_all": 1, "qid": "a82c026e", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 48, "hit": true, "rank": 4, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 1, "qid": "0edc2aef", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 49, "hit": true, "rank": 2, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 2, "covered_all": 3, "qid": "gpt4_a56e767c", "cat": "multi-session", "answer_sessions": 3, "haystack_sessions": 52, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 1, "covered_all": 3, "qid": "0bc8ad93", "cat": "temporal-reasoning", "answer_sessions": 3, "haystack_sessions": 48, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 2, "qid": "6a1eabeb", "cat": "knowledge-update", "answer_sessions": 2, "haystack_sessions": 41, "hit": true, "rank": 1, "retrieved": 15, "mode": "numeric"} +{"covered_top2": 1, "covered_all": 1, "qid": "66f24dbb", "cat": "single-session-user", "answer_sessions": 1, "haystack_sessions": 55, "hit": true, "rank": 2, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 1, "covered_all": 1, "qid": "6e984302", "cat": "temporal-reasoning", "answer_sessions": 2, "haystack_sessions": 46, "hit": true, "rank": 1, "retrieved": 15, "mode": "abstained"} +{"covered_top2": 1, "covered_all": 1, "qid": "caf03d32", "cat": "single-session-preference", "answer_sessions": 1, "haystack_sessions": 55, "hit": true, "rank": 1, "retrieved": 15, "mode": "wrong_value"} +{"covered_top2": 0, "covered_all": 3, "qid": "6d550036", "cat": "multi-session", "answer_sessions": 4, "haystack_sessions": 53, "hit": true, "rank": 5, "retrieved": 15, "mode": "numeric"} diff --git a/results/retrieval_diagnosis.md b/results/retrieval_diagnosis.md index f93472a..1802e8e 100644 --- a/results/retrieval_diagnosis.md +++ b/results/retrieval_diagnosis.md @@ -2,8 +2,18 @@ ## 结论先行 -**73% 的错题，答案所在会话就在检索结果前 2 名、以全文形式摆在答题模型面前，它仍然答错了。** -只有 3 道（4%）是真的没检索到。 +**检索找到了证据，但上下文装配只展示了其中一半。** + +- 82 道错题里 79 道（96%），答案会话被检索到了——检索不是问题。 +- 但 **58 道错题的答案跨多个会话**，其中只有 **14 道（24%）**的答案会话**全部**进了全文窗口。 + 平均覆盖率约 **48%**——一半的该数的会话被压成了摘要。 + +一道需要覆盖 4 个会话的计数题，看到 2 个全文 + 2 个摘要，结构上就数不对。 + +> **本文件初版的结论是错的**，此处已修正。初版只记录了**第一个**匹配答案会话的排名，据此写下 +> "73% 的错题证据已在全文窗口内，失败发生在证据到手之后"。对单会话题这成立；对占多数的多会话 +> 计数题，"最相关的那个会话进了前 2"只说明 4 个里进来了 1 个，恰恰是数不对的原因。 +> 教训：**对聚合类问题，正确的度量是覆盖率，不是首命中排名。** 这判死了一整类机制：**任何奔着"多检索一点"去的改动，上限是 3 题 = +0.6 点**，远低于 2.94 的分辨率地板（`results/significance_headline.md`）。优化地图 P1 的 @@ -54,6 +64,37 @@ LongMemEval 自带 `answer_session_ids` 标注，"答案在哪个会话"是数 "精简检索胜过全上下文"（9.6k vs 79k）。把窗口开到 15 是朝全上下文回退， 即使分数涨了，涨的也是论点本身在退让。 +## 多会话覆盖率（决定性数据） + +| 模式 | 多会话题数 | 答案会话全部进全文窗口 | 全部被检索到 | 全文窗口平均覆盖 | +| --- | ---: | ---: | ---: | ---: | +| 数值错 | 28 | 6（21%） | 25（89%） | 48% | +| 弃答 | 19 | 5（26%） | 15（79%） | 49% | +| 值错 | 11 | 3（27%） | 10（91%） | 58% | +| **合计** | **58** | **14（24%）** | **50（86%）** | — | + +**检索层召回率 86%，装配层覆盖率 24%。** 差距全在上下文装配：证据取回来了，但按相关度只渲染前 2 个 +会话的全文，其余压成摘要——而计数需要的是**覆盖**，不是**排序**。 + +这也解释了此前无法解释的双向误差：会话被压成摘要 → 漏数（低估 16 例）；摘要表述模糊导致同一件事 +重复计入 → 多数（高估 12 例）。此前把双向误差读作"算术本身失败"，错了一层。 + +## 由此得出的机制（唯一有证据支撑的方向） + +**对聚合类查询，让全文窗口覆盖证据集合，而不是取相关度前 N。** + +- 触发条件已存在且工作正常：`plan_evidence().aggregation` 在这 30 道计数错上触发率 90% + （对照：答对的数值题 79%）。**不需要放宽触发**——它触发了，只是没有影响渲染多少个全文块。 +- 与"无差别把 `--chunks` 扩到 15"的关键区别：只在聚合类查询上扩，其余查询不变。 + 聚合类约占查询的 20%，所以 token 成本落在需要它的那一小部分上，不会整体朝全上下文回退。 +- 规模：58 道多会话错题里 44 道缺完整覆盖。即便只有一半能因完整覆盖而答对，也是 **+4.4 点**， + 高于 2.94 的地板。 + +**尚未验证的前提**（必须先离线证伪）：完整覆盖是否真能让这些题答对。已知 60 道题在全文条件下 +仍然失败，说明"看到全文"到"答对"的转化率不是 1。下一步应先在离线装配层验证： +把这 44 道题的全文窗口按证据集合扩展后，答案所需的事实是否真的都出现在装配出的上下文里。 +这一步不需要 LLM，因此免费。 + ## 下一步该往哪打 弃答（34 题）与数值（30 题）合计 64 题 = +12.8 点上限，仍是唯一舒服高于地板的靶子， diff --git a/tests/test_retrieval_check.py b/tests/test_retrieval_check.py index 040be88..0069a50 100644 --- a/tests/test_retrieval_check.py +++ b/tests/test_retrieval_check.py @@ -78,3 +78,24 @@ def test_a_narrow_slice_can_miss_a_session_a_wide_one_finds(): row = check_item(_item(answer_index=0, sessions=8), _Embedder(), k_sessions=1) assert row["retrieved"] <= 1 + + +def test_coverage_counts_all_answer_sessions_not_just_the_first(): + """The correction this file exists to hold. + + A counting question whose answer spans four sessions cannot be answered from the one that ranked + highest. Reporting only the first hit said "the evidence was retrieved" for exactly the questions + that could not possibly be counted correctly. + """ + from eval.retrieval_check import check_item + + item = _item(answer_index=0, sessions=5) + item["answer_session_ids"] = ["s0", "s3"] # answer spans two sessions, only one ranks first + row = check_item(item, _Embedder(), k_sessions=5) + + assert row["answer_sessions"] == 2 + assert row["rank"] == 1, "the top-ranked answer session is still found" + assert row["covered_top2"] < row["answer_sessions"], ( + "and coverage must show that the full-detail window did not hold all of them" + ) + assert row["covered_all"] == 2, "both were retrieved somewhere in the slice" From 7be088cd8b2c6f8b9290f7af23c2dbf5cbe5aa8b Mon Sep 17 00:00:00 2001 From: Engram Date: Mon, 17 Aug 2026 14:40:19 +0800 Subject: [PATCH 24/25] eval: the counting fix does not clear the floor, and the measurement says why MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Traced the counting failures to a one-line cause: evidence.py hands aggregation queries n_chunks=1, the lowest non-zero budget there is, while preference, procedural, exact-lookup, multi-hop and duration all get 2. On the committed headline log those failures needed a median of 3 answer sessions, and the planner had already generated a median of 3 subqueries to find them — then asked for one chunk. It looked like a one-line fix. It is not, and measuring it before shipping it is the point. Widening the budget lifts answer-session coverage from 38% to 56% at cap=5 and only 59% at cap=12, while rendered sessions go 2.0 -> 4.4. Raising the budget 2.4x buys three points of coverage, because the subqueries run out of distinct sessions at about four. Best case is +1.2 points, under a conversion of 1 that the same log refutes — below the 2.94-point floor either way, and paid for in tokens that push the lean context back toward the full-history baseline. No-go, recorded as such. The measurement also kills my own sizing. I had estimated +4.4 points from "44 questions lack coverage x 50% conversion", which silently assumed the mechanism would achieve full coverage. It reaches 56%. Estimating a gain from an unverified premise produces a wish, not a prediction. What it does establish is where the real constraint is. 89% of the answer sessions sit in the main query's top-15, and the round-robin selects 59% of them however large the budget. So the binding constraint is which sessions get picked, not how many may be rendered: interleaving subqueries by rank optimises for finding the most relevant session per angle, while a count needs every relevant session. Those two goals conflict at selection time, and that is the next hypothesis to test offline. The knob stays, inert at 0, documented with the numbers that say not to turn it on — a coverage-driven selector would want the budget it provides. eval/coverage_check.py mirrors the real detail_eps loop rather than a top-N simplification, because measuring a selection strategy the code does not have would have reported a gain that evaporates in production; a test pins that ordering. Also in this change, from the parallel-repair verification: the TypeScript client had retryAfterOf() defined and never called, so EngramError.retryAfter was undefined on every response including the 429 it exists for — the same trap the Python SDK hit earlier. Wired up, with tests that provoke a real 429. Removed EngramAdminOptions, whose JSDoc pointed at an EngramClient.admin method that does not exist, and thirteen type imports that no signature used. And eval/locomo10.json (2.8MB, third-party) is now gitignored: the existing data/ rule covered only the fallback path, not the documented one. --- .gitignore | 4 + clients/typescript/src/client.ts | 26 ++- clients/typescript/src/types.ts | 176 +++++++++++++++++++- clients/typescript/test/client.test.mjs | 45 +++++ docs/architecture-optimization-map.zh-CN.md | 2 + engram/config.py | 21 +++ engram/localize.py | 2 +- engram/memory.py | 1 + engram/retrieve/evidence.py | 25 ++- eval/coverage_check.py | 172 +++++++++++++++++++ eval/tune_weights.py | 2 +- results/aggregation_coverage.md | 68 ++++++++ results/coverage_check_numeric.jsonl | 28 ++++ tests/test_coverage_check.py | 85 ++++++++++ 14 files changed, 651 insertions(+), 6 deletions(-) create mode 100644 eval/coverage_check.py create mode 100644 results/aggregation_coverage.md create mode 100644 results/coverage_check_numeric.jsonl create mode 100644 tests/test_coverage_check.py diff --git a/.gitignore b/.gitignore index 1505587..cdd4c4c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,7 @@ paper/*.out # Keep curated benchmark logs (committable evidence) !results/ results/*.tmp + +# 第三方评测数据集：按 eval/locomo.py 的文档下载到此路径，2.8MB，不入库。 +# .gitignore 里原本只有 data/，挡住的是备选路径 eval/data/locomo10.json，主路径漏网。 +eval/locomo10.json diff --git a/clients/typescript/src/client.ts b/clients/typescript/src/client.ts index e8e287f..cf03dc9 100644 --- a/clients/typescript/src/client.ts +++ b/clients/typescript/src/client.ts @@ -15,7 +15,6 @@ import type { ChatCompletionCreateParams, CloseSessionResult, AgentStatus, - Fact, FactInput, FactPatch, ForgetOptions, @@ -46,12 +45,35 @@ export class EngramError extends Error { public readonly status: number, message: string, public readonly detail?: unknown, + /** + * Seconds to wait before retrying, from a 429's `Retry-After`. Undefined when the response did + * not carry the header — which is every status except a rate-limit rejection. + */ + public readonly retryAfter?: number, ) { super(message) this.name = 'EngramError' } } +/** + * Seconds to wait, read off the response headers. + * + * It has to come from the headers because that is the only place the server puts it — the 429 body + * says "rate limit exceeded" and nothing more. A transport that hands back only status + body makes + * this permanently undefined, which is an error object nobody can act on; the optional calls below + * are for that shape of injected fetch, not for a real `Response`. + * + * Only delta-seconds are parsed: the server always sends an integer (`str(int(retry_after) + 1)`), + * so an HTTP-date branch here would be a claim about the wire that nothing can reach. + */ +function retryAfterOf(res: Response): number | undefined { + const raw = res.headers?.get?.('Retry-After') + if (raw == null) return undefined + const seconds = Number.parseInt(raw.trim(), 10) + return Number.isFinite(seconds) ? seconds : undefined +} + export interface EngramClientOptions { /** Server base URL. Default 'http://localhost:8000'. A trailing '/v1' is allowed and normalized away. */ baseUrl?: string @@ -115,7 +137,7 @@ export class EngramClient { } catch { /* non-JSON error body */ } - throw new EngramError(res.status, message, detail) + throw new EngramError(res.status, message, detail, retryAfterOf(res)) } private async request(path: string, init: RequestInit = {}): Promise { diff --git a/clients/typescript/src/types.ts b/clients/typescript/src/types.ts index db8569d..1fbf51b 100644 --- a/clients/typescript/src/types.ts +++ b/clients/typescript/src/types.ts @@ -7,7 +7,9 @@ export interface Health { ok: boolean ready: boolean service: string - auth_mode: 'api_keys' | 'open' | 'disabled' + version: string + /** 'invalid' when the server's own auth/limit configuration failed to load — not a client error. */ + auth_mode: 'api_keys' | 'open' | 'disabled' | 'invalid' anonymous_allowed: boolean embedder: string llm_configured: boolean @@ -18,6 +20,34 @@ export interface Health { max_hot_facts: number } +/** Latency percentiles for one instrumented service operation (remember, recall, ...). */ +export interface MetricOp { + n: number + p50_ms: number + p95_ms: number + avg_ms: number + max_ms: number + /** Size of the rolling sample the percentiles were computed over. */ + window: number +} + +/** GET /metrics — aggregate-only by construction: no namespaces, queries or content. */ +export interface Metrics { + uptime_s: number + /** Timed operations, keyed by operation name. */ + ops: Record + /** Plain counters (auth_rejected, rate_limited, idempotent_replays, ...). */ + counts: Record + tokens: { + context_total: number + baseline_context_total: number + baseline_full_total: number + calls_with_baseline: number + /** full-history tokens / served-context tokens. null until one call measured both. */ + savings_ratio: number | null + } +} + export interface RememberResult { ok: boolean scope?: string @@ -276,6 +306,141 @@ export interface ProfileResult { facts: string[] } +/** + * How well-evidenced a structured-profile entry is. Deliberately a kind + a count rather than a + * fabricated 0..1 score, so a UI can say *why* it believes something. + */ +export interface ProfileEvidence { + kind: 'user' | 'reinforced' | 'mentions' + count: number +} + +export interface StructuredProfileBasic { + field: string + label: string + value: string + evidence: ProfileEvidence + source: FactSource + /** '' when the value came from identity resolution rather than a stored fact. */ + fact_id: string +} + +export interface StructuredProfilePreference { + item: string + polarity: 'like' | 'dislike' | 'diet' + category: string + evidence: ProfileEvidence + source: FactSource + fact_id: string + subject: string + predicate: string + object: string +} + +export interface StructuredProfileHabit { + text: string + evidence: ProfileEvidence + fact_id: string +} + +/** + * GET /v1/profile/structured — live facts grouped for display (basic info / preferences / habits), + * split into confirmed vs `tentative`. A derived read-only view: it never filters retrieval. + */ +export interface StructuredProfile { + basic: StructuredProfileBasic[] + /** Confirmed preferences, keyed by coarse category. */ + preferences: Record + habits: StructuredProfileHabit[] + /** Preferences shown as 待确认 — weak signals, displayed apart from the confirmed set. */ + tentative: StructuredProfilePreference[] + counts: { + basic: number + preferences: number + tentative: number + habits: number + } +} + +// --- working memory (ephemeral, session/TTL-scoped) ------------------------- + +export interface WorkingItem { + id: string + content: string + kind: string + session_id: string + /** YYYY-MM-DD */ + created: string + /** YYYY-MM-DD, or null when the item lives until the session is cleared. */ + expires_at: string | null +} + +/** GET /v1/working — live items only; expired and consumed ones are already excluded. */ +export interface WorkingMemory { + items: WorkingItem[] +} + +export interface AddWorkingResult { + ok: boolean + id: string + kind: string + /** Epoch seconds, or null when there is no hard expiry. */ + expires_at: number | null +} + +export interface ClearWorkingResult { + ok: boolean + cleared: number +} + +// --- suspected conflicts (detected in System-2, resolved by the user) ------- + +export interface Conflict { + id: string + /** Fact id of the older claim. */ + older: string + /** Fact id of the newer claim. */ + newer: string + older_text: string + newer_text: string + reason: string +} + +export interface ConflictList { + conflicts: Conflict[] +} + +/** 'both' dismisses the conflict — the two claims are allowed to coexist. */ +export type ConflictResolution = 'newer' | 'older' | 'both' + +// --- admin: tenant API keys (ENGRAM_ADMIN_TOKEN, not a tenant key) ---------- + +/** A key record as the server publishes it — never the secret, never its digest. */ +export interface ApiKeyRecord { + id: string + user: string + label: string + /** Epoch seconds. */ + created_at: number + revoked: boolean + last_used_at: number | null +} + +/** POST /v1/admin/keys — `key` is the plaintext, returned once here and recoverable nowhere else. */ +export interface IssuedApiKey extends ApiKeyRecord { + key: string +} + +export interface ApiKeyList { + keys: ApiKeyRecord[] +} + +export interface RevokeKeyResult { + ok: boolean + id: string + revoked: boolean +} + export interface Focus { track: string[] mute: string[] @@ -366,6 +531,15 @@ export interface ImportResult { summaries: number } +/** Shared by the mutating calls the server can replay instead of re-running (remember, import). */ +export interface IdempotentOptions { + /** + * Sent as the `Idempotency-Key` header. The server replays the first successful response for a + * repeated (tenant, key) pair, so retrying after a timeout does not store the same history twice. + */ + idempotencyKey?: string +} + export interface MemoryExportFact { id: string subject: string diff --git a/clients/typescript/test/client.test.mjs b/clients/typescript/test/client.test.mjs index 1d01106..e8d6759 100644 --- a/clients/typescript/test/client.test.mjs +++ b/clients/typescript/test/client.test.mjs @@ -328,3 +328,48 @@ test('forget requires explicit confirmation before sending destructive request', assert.deepEqual(JSON.parse(calls[0].init.body), { confirm: true }) assert.equal(done.ok, true) }) + +test('a 429 surfaces Retry-After so callers can back off', async () => { + // The regression this guards: retryAfterOf() existed but assertOk() never called it, so retryAfter + // was undefined on every response including the one status it exists for. A caller writing a backoff + // around it would have had that backoff silently never fire. + const client = new EngramClient({ + baseUrl: 'http://x', + apiKey: 'k', + fetch: async () => + new Response(JSON.stringify({ detail: 'rate limit exceeded' }), { + status: 429, + headers: { 'Content-Type': 'application/json', 'Retry-After': '7' }, + }), + }) + + await assert.rejects( + () => client.remember('hi'), + (err) => { + assert.equal(err.status, 429) + assert.equal(err.retryAfter, 7) + return true + }, + ) +}) + +test('an error without Retry-After leaves retryAfter undefined rather than guessing', async () => { + const client = new EngramClient({ + baseUrl: 'http://x', + apiKey: 'k', + fetch: async () => + new Response(JSON.stringify({ detail: 'invalid or missing API key' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }), + }) + + await assert.rejects( + () => client.remember('hi'), + (err) => { + assert.equal(err.status, 401) + assert.equal(err.retryAfter, undefined) + return true + }, + ) +}) diff --git a/docs/architecture-optimization-map.zh-CN.md b/docs/architecture-optimization-map.zh-CN.md index a651e1c..f20df5a 100644 --- a/docs/architecture-optimization-map.zh-CN.md +++ b/docs/architecture-optimization-map.zh-CN.md @@ -173,6 +173,8 @@ flowchart TD | 只把词汇/融合环节收敛为有界候选池（保留语义通道） | **净收益 ≈ 0**（10000 事实：177.03ms vs 全扫 177.88ms）。语义通道自身就是一次全量扫描，省下的又赔回去，还多付索引维护成本。不是候选池思路错，是缺 ANN 索引 | `results/bounded_candidates_scaling.md` | | `candidate_vector_channel=False` 作为默认 | **不采用**。它确实快 14.5x 且次线性，但会丢"语义相关却无共享查询词"的事实，正是 M1 已验证 hybrid 论点依赖的召回。需真 ANN 后端或 keyed harness 证明召回损失可接受 | 同上 | | 上下文拆分接进 `lean_context` 默认路径 | **不采用**。实测 5 轮以下是净亏（stable 半虽只计费一次，但扁平上下文本身小，首轮结构开销收不回），且已发布数字均由扁平路径产出。保留为独立方法 `Memory.layered_context()`，由调用方按会话长度选择 | `results/layered_context_tokens.md` | +| 聚合类扩大全文预算（`aggregation_chunk_cap`） | **不采用**。覆盖率 38%→56%（cap=5）→59%（cap=12）即饱和，预算翻 2.4 倍只换 3 个百分点；名义上限 +1.2 点，低于 2.94 地板，且渲染量翻倍朝全上下文回退。**真正的约束是选块策略而非预算**：89% 的答案会话在主查询 top-15 里，子查询轮询只选出 59% | `results/aggregation_coverage.md` | +| 「44 题 × 50% 转化率 = +4.4 点」的收益估算 | **作废**。该估算默认机制能达成完整覆盖，实测未达成。教训：先测机制的直接效果，再谈转化率 | 同上 | | 上下文导航图（MEMORY MAP）默认开启 | **不采用**。它是扁平上下文里本来没有的新增内容，约 20 轮才回本；最初的对比实为"扁平 vs 扁平+导航图"，缓存收益被吃光还倒欠。改为 `map_limit=0` 默认关闭 | 同上 | | 代理侧叠加 `RECALL_GUIDE` | **不采用**。代理已用 `_MEMORY_PREAMBLE` 框定记忆，再叠一层是同一条指令的第二份拷贝，实测 +14% prompt tokens 且不改变行为。代理显式传 `guide=False` | `results/layered_context_tokens.md` | | 上下文拆分作为代理默认 | **不采用**，保持 opt-in（`{"memory":{"layered":true}}`）。同内容对比下 token 收益仅 −1.1%（噪声级），真实价值是 61 tokens 的可缓存稳定前缀，是否划算取决于 provider 的 cache-read 定价，尚未用真实计费验证 | 同上 | diff --git a/engram/config.py b/engram/config.py index 63f3e15..90f35fa 100644 --- a/engram/config.py +++ b/engram/config.py @@ -64,6 +64,27 @@ class Config: preference_reversal_extraction: bool = True # extract high-confidence "no longer like X" preference updates numeric_aggregation_candidates: bool = True # extract money/hour/page rows for count/sum questions aggregation_recall_expansion: bool = True # expand count/sum queries into high-recall evidence lookups + # How many sessions a counting question may render in FULL. 0 keeps today's behaviour. + # + # A count is answered by every relevant session at once, not by the best-ranked one, so a detail + # window narrower than the evidence set yields a confident wrong number. Measured on the committed + # headline log, the counting failures needed a median of 3 answer sessions while the planner asked + # for 1 chunk; coverage inside the detail window was 48% and the numeric errors ran in both + # directions — undercounting what was summarised away, overcounting what a summary said twice + # (results/retrieval_diagnosis.md). + # + # MEASURED AND INSUFFICIENT — off by default, and the measurement says leave it there. + # On the 28 multi-session counting failures (results/aggregation_coverage.md): coverage of the answer + # sessions rose 38% -> 56% at cap=5 and only 59% at cap=12, while rendered sessions went 2.0 -> 4.4. + # Raising the budget 2.4x bought three points of coverage, because the subqueries run out of distinct + # sessions at about four. The remaining 41% is never selected even though 89% of it sits in the main + # query's top-15 — so the binding constraint is which sessions the round-robin picks, not how many it + # is allowed to render. Best case +1.2 points, under a conversion of 1 that the same log refutes; + # below the 2.94-point resolution floor either way. + # + # Kept rather than deleted because the knob is inert at 0 and a coverage-driven selector would want + # the budget it provides. It is not a fix on its own, and turning it on buys tokens, not accuracy. + aggregation_chunk_cap: int = 0 aggregation_constraint_filter: bool = True # mark numeric candidates that violate query constraints as EXCLUDE procedural_extraction: bool = True # extract durable runbooks/how-to steps into typed procedure facts summary_fallback: bool = True # answer from derived session summaries when facts cannot answer diff --git a/engram/localize.py b/engram/localize.py index cf45ef7..64e3dc5 100644 --- a/engram/localize.py +++ b/engram/localize.py @@ -31,7 +31,7 @@ "owns": "拥有 {o}", "has": "有 {o}", "has_pet": "养了 {o}", "visited": "去过 {o}", "traveled_to": "去过 {o}", "been_to": "去过 {o}", "went_to": "去过 {o}", "bought": "买了 {o}", "ordered": "点了 {o}", "plans": "计划 {o}", "plans_to": "计划 {o}", - "wants": "想要 {o}", "needs": "需要 {o}", "goal": "目标：{o}", "uses": "用 {o}", + "wants": "想要 {o}", "needs": "需要 {o}", "goal": "目标：{o}", "has_disease": "患有 {o}", "medication": "在用药 {o}", "salary": "薪资 {o}", "income": "收入 {o}", "plays": "会 {o}", "watches": "看 {o}", "reads": "读 {o}", "listens_to": "听 {o}", "studies": "学 {o}", "learns": "学 {o}", "learning": "在学 {o}", "drives": "开 {o}", diff --git a/engram/memory.py b/engram/memory.py index 69fb28e..095733b 100644 --- a/engram/memory.py +++ b/engram/memory.py @@ -2021,6 +2021,7 @@ def lean_context( plan_evidence( query, aggregation_recall_expansion=self.config.aggregation_recall_expansion, + aggregation_chunk_cap=self.config.aggregation_chunk_cap, ) if self.config.evidence_planner else None diff --git a/engram/retrieve/evidence.py b/engram/retrieve/evidence.py index 99dbeff..8a7d24f 100644 --- a/engram/retrieve/evidence.py +++ b/engram/retrieve/evidence.py @@ -254,7 +254,11 @@ def _multi_hop_subqueries(query: str) -> tuple[str, ...]: return _dedupe(candidates, query) -def plan_evidence(query: str, aggregation_recall_expansion: bool = True) -> EvidenceNeed: +def plan_evidence( + query: str, + aggregation_recall_expansion: bool = True, + aggregation_chunk_cap: int = 0, +) -> EvidenceNeed: """Return the evidence structure a question needs, using only question text. The output is deliberately coarse and explainable; it never inspects benchmark labels or gold answers. @@ -354,6 +358,25 @@ def plan_evidence(query: str, aggregation_recall_expansion: bool = True) -> Evid subquery_items.extend(_multi_hop_subqueries(query)) subqueries = _dedupe(subquery_items, query) + # A count needs COVERAGE, not ranking. Everything else asks "which session answers this?" and the + # best-ranked one does; "how many X have I done" is answered by all of them at once, and a detail + # window narrower than the evidence set produces a confident wrong number rather than a miss. + # + # Measured on the committed headline log: the counting failures needed a median of 3 answer sessions + # and this planner already generated a median of 3 subqueries to find them — then asked for 1 chunk. + # Coverage of the answer sessions inside the detail window came out at 48%, and the numeric errors ran + # in both directions, which is the signature of counting from a subset rather than of bad retrieval + # (results/retrieval_diagnosis.md). + # + # The subquery count is the right budget because it is derived from the question alone — the same + # signal at eval time and in production. Sizing off the benchmark's answer_session_ids would score + # well and ship nothing. + if aggregation and subqueries and aggregation_chunk_cap: + # +1 so the original question keeps a slot of its own alongside the decomposed angles. Capped + # because the budget has to stay bounded: without a ceiling a six-way decomposition would render + # seven full sessions and walk the lean context back toward the full-history baseline it beats. + n_chunks = max(n_chunks, min(len(subqueries) + 1, aggregation_chunk_cap)) + return EvidenceNeed( kinds=kinds, timeline=timeline, diff --git a/eval/coverage_check.py b/eval/coverage_check.py new file mode 100644 index 0000000..f6ba32e --- /dev/null +++ b/eval/coverage_check.py @@ -0,0 +1,172 @@ +"""Does widening the detail window for counting questions actually cover the evidence? + +`retrieval_diagnosis.md` established that counting failures are a coverage problem: the answer spans +several sessions, retrieval finds them, and the detail window renders about half. `aggregation_chunk_cap` +is the proposed fix. This measures whether it does what it claims — before anyone pays for a run. + +The measurement has to mirror the real selection loop, not a simplification of it. `lean_context` does +not take the top-N sessions for the main query: it retrieves per subquery and interleaves by rank +(`memory.py`, the `detail_eps` loop). Testing top-N instead would measure a mechanism the code does not +have, and could report a gain that evaporates in production. + +LLM-free — retrieval runs on the local embedder, and the benchmark labels which sessions hold the answer. + + python3 eval/coverage_check.py --failures-from results/longmemeval_s_engram_lean_v2_final.jsonl +""" +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from engram.retrieve.evidence import plan_evidence # noqa: E402 +from eval.error_modes import classify # noqa: E402 +from eval.retrieval_check import DEFAULT_DATASET, failure_modes # noqa: E402 + +__all__ = ["select_detail_sessions", "coverage_for"] + + +def select_detail_sessions(mem, session_of_episode: dict, query: str, need, n_chunks: int) -> list[str]: + """The session ids `lean_context` would render in full, for this query and budget. + + Mirrors the round-robin in memory.py: each subquery contributes its rank-1 session first, then every + subquery's rank-2, and so on until the budget is spent. That ordering is the whole point — it spreads + the window across the decomposed angles instead of spending it all on the main query's top hits. + """ + if n_chunks <= 0: + return [] + detail_queries = list(need.subqueries) + [query] if need.subqueries else [query] + per_query = [mem.retrieve_episodes(q, "u", max(n_chunks, 1)) for q in detail_queries] + + seen: set[str] = set() + chosen: list[str] = [] + for rank in range(max((len(eps) for eps in per_query), default=0)): + for eps in per_query: + if rank >= len(eps): + continue + episode = eps[rank] + if episode.id in seen: + continue + seen.add(episode.id) + chosen.append(session_of_episode.get(episode.id, episode.session_id)) + if len(chosen) >= n_chunks: + return chosen + return chosen + + +def coverage_for(item: dict, embedder, cap: int) -> dict: + from engram.memory import Memory + from engram.util import DAY, now + + mem = Memory(embedder=embedder) + base = now() - len(item["haystack_sessions"]) * DAY + session_of_episode = {} + for index, (session_id, session) in enumerate( + zip(item["haystack_session_ids"], item["haystack_sessions"]) + ): + text = "\n".join(f"{t.get('role', 'user')}: {t.get('content', '')}" for t in session) + episode = mem.add(text, user_id="u", session_id=session_id, event_time=base + index * DAY) + session_of_episode[episode.id] = session_id + + wanted = set(item.get("answer_session_ids") or []) + question = item["question"] + + # The run under analysis passed --chunks 2 on the CLI, and lean_context takes max(cli, planner), so + # the baseline here is 2 rather than the planner's 1. Comparing against the planner alone would + # flatter the change by pretending the baseline was worse than it was. + before_need = plan_evidence(question, aggregation_chunk_cap=0) + after_need = plan_evidence(question, aggregation_chunk_cap=cap) + before_chunks = max(2, before_need.n_chunks) + after_chunks = max(2, after_need.n_chunks) + + before = select_detail_sessions(mem, session_of_episode, question, before_need, before_chunks) + after = select_detail_sessions(mem, session_of_episode, question, after_need, after_chunks) + + return { + "qid": item["question_id"], + "answer_sessions": len(wanted), + "chunks_before": before_chunks, + "chunks_after": after_chunks, + "covered_before": len(wanted & set(before)), + "covered_after": len(wanted & set(after)), + "complete_before": wanted.issubset(set(before)), + "complete_after": wanted.issubset(set(after)), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="Does the wider detail window cover the answer sessions?") + ap.add_argument("--failures-from", required=True) + ap.add_argument("--system", default="engram_lean") + ap.add_argument("--dataset", default=DEFAULT_DATASET) + ap.add_argument("--cap", type=int, default=5, help="aggregation_chunk_cap to test") + ap.add_argument("--mode", default="numeric", help="failure mode to check") + ap.add_argument("--out", default=None) + args = ap.parse_args() + + modes = failure_modes(args.failures_from, args.system) + with open(args.dataset, encoding="utf-8") as fh: + dataset = {item["question_id"]: item for item in json.load(fh)} + + targets = [ + qid for qid, mode in modes.items() + if mode == args.mode and qid in dataset and len(dataset[qid].get("answer_session_ids") or []) > 1 + ] + if not targets: + print("no multi-session failures of that mode") + return 1 + print(f"checking {len(targets)} multi-session {args.mode} failures at cap={args.cap} (LLM-free)\n") + + from engram.llm.providers import make_embedder + + embedder = make_embedder("bge-small") + + rows = [] + started = time.time() + for index, qid in enumerate(targets, start=1): + rows.append(coverage_for(dataset[qid], embedder, args.cap)) + if index % 10 == 0 or index == len(targets): + print(f" {index}/{len(targets)} ({time.time() - started:.0f}s)", flush=True) + + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + + def ratio(row, key): + return row[key] / row["answer_sessions"] if row["answer_sessions"] else 0.0 + + before_cov = statistics.fmean(ratio(r, "covered_before") for r in rows) + after_cov = statistics.fmean(ratio(r, "covered_after") for r in rows) + before_full = sum(1 for r in rows if r["complete_before"]) + after_full = sum(1 for r in rows if r["complete_after"]) + chunks_before = statistics.fmean(r["chunks_before"] for r in rows) + chunks_after = statistics.fmean(r["chunks_after"] for r in rows) + + print(f"\n{'':<26}{'before':>10}{'after':>10}") + print("-" * 46) + print(f"{'mean answer-session coverage':<26}{before_cov:>9.0%}{after_cov:>10.0%}") + print(f"{'fully covered':<26}{f'{before_full}/{len(rows)}':>10}{f'{after_full}/{len(rows)}':>10}") + print(f"{'mean full sessions rendered':<26}{chunks_before:>10.1f}{chunks_after:>10.1f}") + + gained = after_full - before_full + print( + f"\n{gained} more questions now have every answer session in view " + f"(+{100.0 * gained / 500:.1f} points at best, if each converts to a correct answer)." + ) + print( + "That ceiling assumes a conversion of 1, which the same log refutes: questions already failed\n" + "with their evidence in full detail. The measured gain needs a keyed run; this only establishes\n" + "that the mechanism does what it is supposed to do, and at what cost in rendered sessions." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/tune_weights.py b/eval/tune_weights.py index ddd7b61..32e74c8 100644 --- a/eval/tune_weights.py +++ b/eval/tune_weights.py @@ -22,7 +22,7 @@ from engram import Memory # noqa: E402 from engram.config import Config # noqa: E402 from engram.llm.providers import load_dotenv, make_embedder, make_llm # noqa: E402 -from eval.longmemeval import ingest, load_data, sessions_of # noqa: E402 +from eval.longmemeval import ingest, load_data # noqa: E402 def build_dev(items, embedder, extractor, extract_k=20): diff --git a/results/aggregation_coverage.md b/results/aggregation_coverage.md new file mode 100644 index 0000000..04f1df5 --- /dev/null +++ b/results/aggregation_coverage.md @@ -0,0 +1,68 @@ +# 聚合类查询的全文覆盖：机制实测不足（2026-08-17） + +## 裁决：no-go + +给聚合类查询扩大全文窗口，**实测最多 +1.2 点，低于 2.94 的分辨率地板**，也低于合并基准后的 1.32。 +不建议付费验证——跑出来必然落在噪声带里，无法判定。 + +## 假设从哪来 + +`results/retrieval_diagnosis.md` 确立：计数题的失败是覆盖问题。答案跨多个会话，检索找到了（89% 在 +主查询 top-15 内），但装配层按相关度只渲染前 2 个全文，其余压成摘要。误差双向——压成摘要的漏数， +摘要表述模糊的重复计入。 + +顺着查到根因（`engram/retrieve/evidence.py`）： + +```python +n_chunks = 2 if (preference or procedural or exact_lookup or multi_hop or duration) else (1 if aggregation or timeline else 0) +``` + +**聚合类拿到的是 1——所有非零档里最低的。** 而实测这 28 道多会话计数错题：需要的答案会话中位数 3， +规划器生成的子查询中位数也是 3（召回扩展工作正常），却只渲染 1 个全文块。 + +看起来是一个一行修复：让聚合类的全文预算跟随子查询数。 + +## 实测结果 + +```bash +python3 eval/coverage_check.py --failures-from results/longmemeval_s_engram_lean_v2_final.jsonl --cap 5 +python3 eval/coverage_check.py --failures-from results/longmemeval_s_engram_lean_v2_final.jsonl --cap 12 +``` + +| | 改动前 | cap=5 | cap=12 | +| --- | ---: | ---: | ---: | +| 答案会话平均覆盖率 | 38% | 56% | **59%** | +| 完整覆盖题数 | 3/28 | 7/28 | 9/28 | +| 平均渲染全文会话数 | 2.0 | 3.9 | 4.4 | +| 名义上限 | — | +0.8 点 | +1.2 点 | + +**预算翻 2.4 倍，覆盖率只涨 3 个百分点。** 子查询能产出的不同会话在约 4 个时耗尽，剩下 41% 的 +答案会话**无论预算多大都选不到**。 + +## 为什么错了（对自己的估算的复盘） + +我此前估 +4.4 点，算法是「44 道题缺完整覆盖 × 50% 转化率」。这个估算**默认机制能达成完整覆盖**， +而实测只到 56%。用未经验证的前提去算收益，算出来的是愿望不是预测。 + +正确的做法是先测机制的**直接效果**（覆盖率提升），再谈转化率——本文件就是补上这一步。 + +## 真正的瓶颈（下一个该验证的假设） + +矛盾点：**89% 的答案会话在主查询 top-15 里，子查询轮询却只选出 59%。** + +所以约束不在「允许渲染几个」，而在「轮询选中哪几个」。当前策略是各子查询按名次交错取块 +（`engram/memory.py` 的 detail_eps 循环），它服务的是「多角度找到最相关的会话」， +而计数需要的是「覆盖所有相关会话」——这两个目标在选块阶段是冲突的。 + +下一个假设应当针对**选块策略**而非预算：对聚合类查询，从主查询的 top-k 候选里按**证据多样性** +（例如按被计数对象的不同取值）选块，而不是按各子查询的相关度名次。这个假设同样可以离线验证， +方法与本文件相同。 + +## 未采用/回滚原因 + +| 方向 | 结论 | 证据 | +| --- | --- | --- | +| 聚合类扩大全文预算（`aggregation_chunk_cap`） | **不采用**。覆盖率 38%→59% 即饱和，名义上限 +1.2 点，低于地板；且渲染量翻倍，token 成本朝全上下文回退 | 本文件 | +| 「44 题 × 50% 转化率 = +4.4 点」这一估算 | **作废**。它假设机制达成完整覆盖，实测未达成 | 本文件 | + +开关保留但默认 0（惰性）：它本身不是修复，但一个「按覆盖选块」的策略会需要它提供的预算。 diff --git a/results/coverage_check_numeric.jsonl b/results/coverage_check_numeric.jsonl new file mode 100644 index 0000000..ad2ae47 --- /dev/null +++ b/results/coverage_check_numeric.jsonl @@ -0,0 +1,28 @@ +{"qid": "gpt4_59c863d7", "answer_sessions": 4, "chunks_before": 2, "chunks_after": 5, "covered_before": 2, "covered_after": 4, "complete_before": false, "complete_after": true} +{"qid": "gpt4_15e38248", "answer_sessions": 4, "chunks_before": 2, "chunks_after": 4, "covered_before": 2, "covered_after": 3, "complete_before": false, "complete_after": false} +{"qid": "d682f1a2", "answer_sessions": 3, "chunks_before": 2, "chunks_after": 4, "covered_before": 2, "covered_after": 3, "complete_before": false, "complete_after": true} +{"qid": "c18a7dc8", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 3, "covered_before": 0, "covered_after": 0, "complete_before": false, "complete_after": false} +{"qid": "28dc39ac", "answer_sessions": 5, "chunks_before": 2, "chunks_after": 5, "covered_before": 2, "covered_after": 3, "complete_before": false, "complete_after": false} +{"qid": "370a8ff4", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 5, "covered_before": 1, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "6e984301", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 3, "covered_before": 1, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "45dc21b6", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 4, "covered_before": 1, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "3a704032", "answer_sessions": 3, "chunks_before": 2, "chunks_after": 4, "covered_before": 1, "covered_after": 3, "complete_before": false, "complete_after": true} +{"qid": "69fee5aa", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 4, "covered_before": 2, "covered_after": 2, "complete_before": true, "complete_after": true} +{"qid": "9a707b81", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 5, "covered_before": 1, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "59524333", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 2, "covered_before": 2, "covered_after": 2, "complete_before": true, "complete_after": true} +{"qid": "gpt4_731e37d7", "answer_sessions": 4, "chunks_before": 2, "chunks_after": 5, "covered_before": 0, "covered_after": 2, "complete_before": false, "complete_after": false} +{"qid": "gpt4_d6585ce8", "answer_sessions": 5, "chunks_before": 2, "chunks_after": 2, "covered_before": 2, "covered_after": 2, "complete_before": false, "complete_after": false} +{"qid": "gpt4_31ff4165", "answer_sessions": 5, "chunks_before": 2, "chunks_after": 4, "covered_before": 1, "covered_after": 3, "complete_before": false, "complete_after": false} +{"qid": "bf659f65", "answer_sessions": 3, "chunks_before": 2, "chunks_after": 5, "covered_before": 1, "covered_after": 2, "complete_before": false, "complete_after": false} +{"qid": "d23cf73b", "answer_sessions": 4, "chunks_before": 2, "chunks_after": 4, "covered_before": 2, "covered_after": 3, "complete_before": false, "complete_after": false} +{"qid": "8e91e7d9", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 4, "covered_before": 0, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "92a0aa75", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 2, "covered_before": 0, "covered_after": 0, "complete_before": false, "complete_after": false} +{"qid": "0a995998", "answer_sessions": 3, "chunks_before": 2, "chunks_after": 5, "covered_before": 2, "covered_after": 2, "complete_before": false, "complete_after": false} +{"qid": "60472f9c", "answer_sessions": 3, "chunks_before": 2, "chunks_after": 4, "covered_before": 0, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "10d9b85a", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 5, "covered_before": 0, "covered_after": 0, "complete_before": false, "complete_after": false} +{"qid": "4f54b7c9", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 4, "covered_before": 2, "covered_after": 2, "complete_before": true, "complete_after": true} +{"qid": "129d1232", "answer_sessions": 3, "chunks_before": 2, "chunks_after": 3, "covered_before": 0, "covered_after": 0, "complete_before": false, "complete_after": false} +{"qid": "9ee3ecd6", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 4, "covered_before": 1, "covered_after": 2, "complete_before": false, "complete_after": true} +{"qid": "a3838d2b", "answer_sessions": 6, "chunks_before": 2, "chunks_after": 5, "covered_before": 1, "covered_after": 2, "complete_before": false, "complete_after": false} +{"qid": "6a1eabeb", "answer_sessions": 2, "chunks_before": 2, "chunks_after": 2, "covered_before": 1, "covered_after": 1, "complete_before": false, "complete_after": false} +{"qid": "6d550036", "answer_sessions": 4, "chunks_before": 2, "chunks_after": 4, "covered_before": 0, "covered_after": 0, "complete_before": false, "complete_after": false} diff --git a/tests/test_coverage_check.py b/tests/test_coverage_check.py new file mode 100644 index 0000000..72a9e5e --- /dev/null +++ b/tests/test_coverage_check.py @@ -0,0 +1,85 @@ +"""Measuring whether a wider detail window covers a counting question's evidence. + +The measurement decided a mechanism was not worth shipping, so it has to be right about the thing it +measured. The load-bearing test is `test_selection_mirrors_the_round_robin_not_top_n`: lean_context does +not take the top N sessions for the main query, it interleaves each subquery's ranks. Measuring top-N +instead would have measured a mechanism the code does not have — and reported a gain that evaporates. +""" +from __future__ import annotations + +from engram.retrieve.evidence import plan_evidence +from eval.coverage_check import select_detail_sessions + + +class _Mem: + """Returns a scripted ranking per query, so selection order is the only thing under test.""" + + def __init__(self, by_query: dict): + self.by_query = by_query + self.asked: list[str] = [] + + def retrieve_episodes(self, query, _user, k): + self.asked.append(query) + return [_Ep(sid) for sid in self.by_query.get(query, [])][:k] + + +class _Ep: + def __init__(self, sid: str): + self.id = sid + self.session_id = sid + + +class _Need: + def __init__(self, subqueries): + self.subqueries = tuple(subqueries) + + +def test_selection_mirrors_the_round_robin_not_top_n(): + """Each subquery contributes its rank-1 before any contributes its rank-2.""" + mem = _Mem({ + "main": ["m1", "m2", "m3"], + "sub-a": ["a1", "a2"], + "sub-b": ["b1", "b2"], + }) + chosen = select_detail_sessions(mem, {}, "main", _Need(["sub-a", "sub-b"]), n_chunks=3) + assert chosen == ["a1", "b1", "m1"], "rank 1 of every angle comes before rank 2 of any" + + +def test_the_budget_is_respected(): + mem = _Mem({"main": ["m1", "m2"], "sub-a": ["a1", "a2"]}) + assert len(select_detail_sessions(mem, {}, "main", _Need(["sub-a"]), n_chunks=2)) == 2 + + +def test_a_zero_budget_renders_nothing(): + mem = _Mem({"main": ["m1"]}) + assert select_detail_sessions(mem, {}, "main", _Need([]), n_chunks=0) == [] + + +def test_duplicate_sessions_across_subqueries_are_not_double_counted(): + """Two angles hitting the same session must not consume two slots of the budget.""" + mem = _Mem({"main": ["s1"], "sub-a": ["s1", "s2"], "sub-b": ["s1", "s3"]}) + chosen = select_detail_sessions(mem, {}, "main", _Need(["sub-a", "sub-b"]), n_chunks=3) + assert len(chosen) == len(set(chosen)) + assert set(chosen) == {"s1", "s2", "s3"} + + +def test_no_subqueries_falls_back_to_the_main_query(): + mem = _Mem({"main": ["m1", "m2"]}) + assert select_detail_sessions(mem, {}, "main", _Need([]), n_chunks=2) == ["m1", "m2"] + + +def test_the_cap_raises_the_budget_only_for_aggregation(): + """The knob must not widen the window for questions that are answered by one session.""" + counting = "How many model kits have I worked on or bought?" + lookup = "Where do I live?" + assert plan_evidence(counting, aggregation_chunk_cap=5).n_chunks > plan_evidence(counting).n_chunks + assert plan_evidence(lookup, aggregation_chunk_cap=5).n_chunks == plan_evidence(lookup).n_chunks + + +def test_the_cap_is_off_by_default(): + """Measured insufficient, so the default must not quietly enable it.""" + from engram.config import Config + + assert Config().aggregation_chunk_cap == 0 + counting = "How many trips did I take?" + assert plan_evidence(counting, aggregation_chunk_cap=0).n_chunks == plan_evidence(counting).n_chunks From 6791789a50ea8d2e00d574fb14209637ef04b680 Mon Sep 17 00:00:00 2001 From: Engram Date: Fri, 21 Aug 2026 17:53:20 +0800 Subject: [PATCH 25/25] paper: drop an unused import Left over from the lint pass. os was imported and never referenced; verified by grep and by running the script, and it is one of the 33 lint findings the CI gate would flag once that gate exists. --- paper/compute_stats.py | 1 - 1 file changed, 1 deletion(-) diff --git a/paper/compute_stats.py b/paper/compute_stats.py index 87409fc..23bbb57 100644 --- a/paper/compute_stats.py +++ b/paper/compute_stats.py @@ -14,7 +14,6 @@ import glob import json import math -import os import random from pathlib import Path from typing import Iterable