diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..860d7c9a0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +# ── Force LF for Unix-format files regardless of host's core.autocrlf ── +# +# Why: a developer with `git config core.autocrlf=true` on Windows will +# silently get CRLF on every text file at checkout, including shell +# scripts. Real-world hit on 2026-05-08: running scripts/restart-daemon.cmd +# (which calls `npm install` / `npm run build` / `npm link --force`) +# triggered git to refresh `bin/imcodes-launch.sh`, autocrlf converted it +# to CRLF, `git status` showed it modified, and a careless `git add .` +# would have committed CRLF into a #!/usr/bin/env bash script — bash on +# Linux/macOS treats the trailing `\r` as part of the next token and +# fails on every line ("$'\r': command not found", broken `if` tests, +# malformed variable assignments, etc.). +# +# `text eol=lf` means: treat as text in the index, force LF on checkout +# everywhere — overriding any local core.autocrlf=true setting. +*.sh text eol=lf +*.bash text eol=lf +*.mjs text eol=lf +*.cjs text eol=lf +*.mts text eol=lf +*.cts text eol=lf +# Files that have a Unix shebang interpreter — bin entries in npm +# packages and the husky hook scripts. Their extensions vary or are +# missing, so list them explicitly. +bin/* text eol=lf +.husky/_/husky.sh text eol=lf + +# ── Force CRLF on Windows-only batch / VBS files regardless of host ── +# +# Why: cmd.exe parses LF inconsistently (especially inside `if (...)` blocks) +# and a `\r`-less line tail can swallow the next line as part of the previous +# command. Without this rule, git on macOS/Linux checks the file out with LF +# (matching the blob), and any test that reads the bytes directly trips a +# "bare LF — needs CRLF" assertion (see test/util/restart-daemon-cmd.test.ts). +# +# `text eol=crlf` means: treat as text in the index, force CRLF on checkout +# everywhere. The blob can still be LF — git converts on the way out. +*.cmd text eol=crlf +*.bat text eol=crlf +*.vbs text eol=crlf +*.ps1 text eol=crlf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75c825622..1d1010ec2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,18 @@ jobs: - run: npm run build - run: npm run test:unit + preview-dist-smoke: + name: Preview Dist Smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION_PRIMARY }} + cache: 'npm' + - run: ./scripts/ci-npm-ci.sh . + - run: npm run test:preview-dist + embedding-real-tests: name: Embedding Integration Tests runs-on: ubuntu-latest @@ -132,7 +144,11 @@ jobs: - run: bash ./scripts/ci-npm-ci.sh . - run: npm run build - name: Run Windows-specific unit tests - run: npx vitest run test/agent/wezterm.test.ts test/daemon/hook-send.test.ts test/daemon/env-injection.test.ts test/cli/send.test.ts test/util/windows-daemon.test.ts test/util/windows-upgrade-script.test.ts test/util/windows-launch-artifacts.test.ts test/util/windows-launch-artifacts.cmd-parse.test.ts test/util/windows-stale-watchdog-cleanup.test.ts test/util/postinstall-sharp-repair.test.ts + run: npx vitest run test/agent/wezterm.test.ts test/daemon/hook-send.test.ts test/daemon/env-injection.test.ts test/cli/send.test.ts test/util/windows-daemon.test.ts test/util/windows-upgrade-script.test.ts test/util/windows-upgrade-runner.test.ts test/util/windows-launch-artifacts.test.ts test/util/windows-launch-artifacts.cmd-parse.test.ts test/util/postinstall-sharp-repair.test.ts test/util/sharp-repair-script.test.ts test/util/restart-daemon-cmd.test.ts + env: + IMCODES_MUX: wezterm + - name: Run Windows process-cleanup regression tests + run: npx vitest run test/util/windows-stale-watchdog-cleanup.test.ts env: IMCODES_MUX: wezterm @@ -148,7 +164,9 @@ jobs: - run: bash ./scripts/ci-npm-ci.sh . - run: npm run build - name: Run Windows ConPTY / startup regression tests - run: npx vitest run test/agent/conpty.test.ts test/agent/drivers/drivers.test.ts test/util/windows-daemon.test.ts test/util/windows-upgrade-script.test.ts test/util/windows-launch-artifacts.test.ts test/util/windows-launch-artifacts.cmd-parse.test.ts test/util/windows-stale-watchdog-cleanup.test.ts + run: npx vitest run test/agent/conpty.test.ts test/agent/drivers/drivers.test.ts test/util/windows-daemon.test.ts test/util/windows-upgrade-script.test.ts test/util/windows-upgrade-runner.test.ts test/util/windows-launch-artifacts.test.ts test/util/windows-launch-artifacts.cmd-parse.test.ts test/util/sharp-repair-script.test.ts test/util/restart-daemon-cmd.test.ts + - name: Run Windows ConPTY process-cleanup regression tests + run: npx vitest run test/util/windows-stale-watchdog-cleanup.test.ts # ── Web frontend tests ──────────────────────────────────────────────────── web-tests-unit: @@ -291,10 +309,17 @@ jobs: with: node-version: ${{ env.NODE_VERSION_PRIMARY }} cache: 'npm' - - name: Install tmux - run: sudo apt-get install -y tmux - - name: Prime tmux server - run: tmux new-session -d -s init && tmux kill-session -t init + # tmux is no longer needed here: `test:coverage` skips the e2e project + # (which spawns real tmux + agent processes), saving ~1 min per run. + # + # `npm run build` IS still needed: although most tests resolve from + # `src/` via vitest's tsx transform, two suites assert against the + # built output and FAIL without `dist/`: + # - test/packaging.test.ts (verifies bin/main/files paths) + # - test/util/postinstall-sharp-repair.test.ts (executes dist/.../postinstall-sharp-repair.js) + # Both run in the `daemon` project, which is included in coverage. They + # pass in the regular Unit Tests jobs because those run `npm run build` + # first; coverage must do the same. - run: ./scripts/ci-npm-ci.sh . - name: Install web deps (needed for tsx component tests) run: ./scripts/ci-npm-ci.sh web diff --git a/CLAUDE.md b/CLAUDE.md index 1a98d64ea..d6fd0a06b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,7 +88,7 @@ The web project uses `i18next` with `react-i18next` for internationalization. - Main sessions and sub-sessions are the same session model. Treat them as equally important in behavior, queueing, timeline semantics, edit/undo, and lifecycle handling. Differences should come only from parent/attachment relationship and presentation constraints, not from weaker semantics for sub-sessions. - Agent types: Process = `'claude-code' | 'codex' | 'gemini' | 'opencode' | 'shell' | 'script'`, Transport = `'openclaw' | 'qwen'` — the `AgentType` union in `src/agent/detect.ts`. - **Pod-sticky routing (MANDATORY for daemon-dependent requests)**: The server runs multiple replicas. Each daemon connects to ONE pod via WebSocket. The ingress uses `:serverId` in the URL path to route requests to the pod holding that daemon's WS. Any endpoint that depends on the daemon (file transfer, session commands, Watch API) **MUST** include `:serverId` in the URL path (e.g., `/api/server/:serverId/...`). In-memory state (download tokens, WsBridge instances, terminal streams) is per-pod — requests without serverId routing will hit a random pod and fail. -- **MANDATORY — Transport command liveness contract:** Daemon command receipt and urgent-control delivery MUST preserve current dev behavior. The daemon MUST NOT intercept `/compact`; `/compact` is an ordinary SDK-native message and is forwarded unchanged. Ordinary `session.send` ack is a daemon-receipt ack and MUST NOT wait for recall, live context bootstrap, memory lookup/enrichment, embedding, transport lock, pending relaunch, provider send-start, provider settlement, telemetry, or any background memory work. `/stop` and approval/feedback/control responses MUST use the priority path and MUST NOT be routed through or blocked by the ordinary send queue/locks. +- **MANDATORY — Transport command liveness contract:** Daemon command receipt and urgent-control delivery MUST preserve current dev behavior. The daemon MUST NOT intercept `/compact`; `/compact` is an ordinary SDK-native message and is forwarded unchanged to the transport provider. Provider adapters that expose a native compact RPC (for example Codex app-server `thread/compact/start`) MUST translate the raw `/compact` command at the SDK boundary instead of sending it as model text. Ordinary `session.send` ack is a daemon-receipt ack and MUST NOT wait for recall, live context bootstrap, memory lookup/enrichment, embedding, transport lock, pending relaunch, provider send-start, provider settlement, telemetry, or any background memory work. `/stop` and approval/feedback/control responses MUST use the priority path and MUST NOT be routed through or blocked by the ordinary send queue/locks. - Server secrets (`JWT_SIGNING_KEY`) are set via environment variables, never committed. - E2E tests require tmux. They are auto-skipped when `SKIP_TMUX_TESTS=1` or inside a Claude Code session (`CLAUDECODE` env var set). - **MANDATORY — Test session hygiene:** Any e2e/integration test that creates tmux sessions, main sessions, sub-sessions, or temporary projects/cwds **MUST** use naming/path patterns covered by `shared/test-session-guard.ts`. If a new test introduces a new naming family, you **MUST** update `shared/test-session-guard.ts` and its tests in the same change. Leaked test sessions must never persist to `~/.imcodes/sessions.json`, must never be written to the server DB, and must be cleaned from live terminal backends on daemon startup. diff --git a/bin/imcodes-launch.sh b/bin/imcodes-launch.sh new file mode 100755 index 000000000..e54c931ef --- /dev/null +++ b/bin/imcodes-launch.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# imcodes-launch — self-healing daemon supervisor. +# +# Entry point for systemd ExecStart / launchctl ProgramArguments. Sits +# in front of the real Node entry and pre-flight-checks the global +# install for half-finished upgrades: +# +# - if `imcodes upgrade` (or any `npm install -g imcodes@…`) gets +# killed mid-write — power loss, OOM-kill, ssh-disconnect, etc — +# npm leaves CRITICAL_DEPS as empty placeholder directories +# (e.g. `node_modules/commander/` exists but has no package.json). +# The next daemon start hits ERR_MODULE_NOT_FOUND on the FIRST +# `import 'commander'`, exits 1, systemd Restart=always thrashes +# forever. +# +# - this launcher detects that signature, re-installs the SAME +# pinned version (read from the surviving package.json — never +# rolls forward without explicit user intent), then execs the +# real daemon. systemd / launchctl never has to know. +# +# Pure bash by design — node_modules being broken is exactly when +# Node-side guard rails go missing too. No tool we use here lives +# under node_modules. +# +# Idempotent: if node_modules is healthy, this is just a thin wrapper +# that exec's the real entry with no overhead beyond a directory stat. +set -u + +# Resolve our own real path so we can find the package root regardless +# of how npm symlinked the bin (`$PREFIX/bin/imcodes-launch -> +# ../lib/node_modules/imcodes/bin/imcodes-launch.sh`). `readlink -f` +# is GNU on Linux; on macOS we fall back to `python3 os.path.realpath`, +# and finally `$0` raw if neither is around (works when invoked via +# absolute path, which systemd always does). +resolve_self() { + if command -v readlink >/dev/null 2>&1 && readlink -f "$0" >/dev/null 2>&1; then + readlink -f "$0" + elif command -v python3 >/dev/null 2>&1; then + python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$0" + else + echo "$0" + fi +} + +SELF_REAL="$(resolve_self)" +PKG_ROOT="$(cd "$(dirname "$SELF_REAL")/.." 2>/dev/null && pwd)" +ENTRY="$PKG_ROOT/dist/src/index.js" +NODE="${IMCODES_NODE_BIN:-$(command -v node 2>/dev/null || echo /usr/bin/node)}" +NPM="${IMCODES_NPM_BIN:-$(command -v npm 2>/dev/null || true)}" +HOME_DIR="${IMCODES_HOME:-$HOME}" +REPAIR_LOG="${IMCODES_LAUNCH_REPAIR_LOG:-$HOME_DIR/.imcodes/launch-repair.log}" + +log() { + echo "[imcodes-launch $(date '+%Y-%m-%d %H:%M:%S')] $*" >&2 +} + +# Critical deps — daemon CANNOT start without these. Each is a top-level +# dependency directly required by `dist/src/index.js` or its synchronous +# transitive imports. If npm install was killed mid-tarball-extract, +# these dirs exist (npm pre-creates them before fetching) but have no +# `package.json` inside. +# +# Keep this list short and stable — over-eager checks cost startup +# latency on every healthy boot. +CRITICAL_DEPS=(commander ws cors body-parser hono "@huggingface/transformers") + +# Returns 0 if a dep dir exists but lacks package.json (the half-install +# signature); 1 otherwise. A missing dep dir entirely is also fine — +# npm dedupes some packages to higher levels — only the EMPTY-DIR case +# is the smoking gun. +is_half_installed() { + local dep_dir="$1" + [ -d "$dep_dir" ] && [ ! -f "$dep_dir/package.json" ] +} + +# Clear stale `upgrade.lock.d/` left behind by killed `imcodes upgrade` +# runs. The lock isn't a daemon-blocker on Linux/macOS (it gates only +# the next upgrade attempt — see step 0.5 of the bash upgrade script) +# but a stuck lock surprises operators ("why does my upgrade say +# 'another upgrade is in progress'?"). The bash upgrade script has a +# 1800 s stale watchdog that fires when a NEW upgrade is initiated; +# this is the same logic, but at daemon-start time, so the lock gets +# cleared even if no human-or-cron-triggered upgrade ever arrives. +# +# Cheap: a stat per startup. Idempotent: never touches a lock that's +# fresh (a real upgrade really does block daemon restarts during its +# 5–60 s install window). +LOCK_DIR="$HOME_DIR/.imcodes/upgrade.lock.d" +LOCK_STALE_AFTER_SEC="${IMCODES_LAUNCH_LOCK_STALE_AFTER_SEC:-1800}" +if [ -d "$LOCK_DIR" ]; then + started="" + source="none" + if [ -f "$LOCK_DIR/started" ]; then + started="$(cat "$LOCK_DIR/started" 2>/dev/null || true)" + source="started-file" + fi + # Fall back to dir mtime if `started` was never written. `stat -c %Y` + # is GNU; `stat -f %m` is BSD. Try both — empty (= "stat failed") is + # treated as "unknown age", NOT zero, so we don't accidentally + # classify a fresh lock as stale because of a probe error. + if [ -z "$started" ]; then + started="$(stat -c %Y "$LOCK_DIR" 2>/dev/null || stat -f %m "$LOCK_DIR" 2>/dev/null || true)" + source="dir-mtime" + fi + # Reject blanks / non-numerics — anything else means we couldn't + # determine the age and SHOULD NOT decide to delete. + case "$started" in + ''|*[!0-9]*) started="" ;; + esac + if [ -n "$started" ]; then + now="$(date +%s)" + age=$(( now - started )) + if [ "$age" -gt "$LOCK_STALE_AFTER_SEC" ]; then + log "clearing stale upgrade.lock.d (age ${age}s, source=${source}, threshold ${LOCK_STALE_AFTER_SEC}s)" + rm -rf "$LOCK_DIR" 2>/dev/null || true + fi + fi +fi + +needs_repair=0 +missing_summary="" +if [ -d "$PKG_ROOT/node_modules" ]; then + for dep in "${CRITICAL_DEPS[@]}"; do + d="$PKG_ROOT/node_modules/$dep" + if is_half_installed "$d"; then + needs_repair=1 + missing_summary="$missing_summary $dep" + fi + done +fi + +# `dist/src/index.js` itself missing → the package files were also +# wiped (rare; tarball extraction is per-package atomic in npm v9+, +# but a corrupted disk or an accidentally-rm'd dist/ can produce +# this). Treat as needs_repair so the launcher doesn't just exec a +# missing file and lose the diagnostic. +if [ ! -f "$ENTRY" ]; then + needs_repair=1 + missing_summary="$missing_summary dist/src/index.js" +fi + +if [ "$needs_repair" = "1" ]; then + if [ -z "$NPM" ]; then + log "node_modules half-installed (missing:$missing_summary) but npm not on PATH — cannot self-repair" + elif [ ! -f "$PKG_ROOT/package.json" ]; then + log "node_modules half-installed but $PKG_ROOT/package.json missing — cannot read pinned version" + else + pinned="$("$NODE" -e "console.log(require('$PKG_ROOT/package.json').version)" 2>/dev/null || true)" + if [ -z "$pinned" ]; then + log "node_modules half-installed but pinned version unreadable — proceeding without repair" + else + log "node_modules half-installed (missing:$missing_summary) — reinstalling imcodes@$pinned" + mkdir -p "$(dirname "$REPAIR_LOG")" + # Clear the leftovers npm leaves when its rename step fails: + # 1. `.imcodes-XXXXX` — npm's atomic-rename tempdir from the + # interrupted install. + # 2. `~/.imcodes/upgrade.lock.d/` — daemon's own coordination + # lock from the killed `imcodes upgrade` flow. + # Both make the next `npm install` fail with ENOTEMPTY/EBUSY. + GLOBAL_LIB="$(dirname "$PKG_ROOT")" + { + echo "==== $(date '+%Y-%m-%d %H:%M:%S') self-repair start (target imcodes@$pinned) ====" + echo "GLOBAL_LIB=$GLOBAL_LIB" + echo "PKG_ROOT=$PKG_ROOT" + echo "missing:$missing_summary" + } >>"$REPAIR_LOG" 2>&1 || true + rm -rf "$GLOBAL_LIB"/.imcodes-* "$HOME_DIR/.imcodes/upgrade.lock.d" >>"$REPAIR_LOG" 2>&1 || true + if "$NPM" install -g --ignore-scripts --prefer-online "imcodes@$pinned" >>"$REPAIR_LOG" 2>&1; then + log "self-repair OK" + else + log "self-repair FAILED — see $REPAIR_LOG" + # Fall through and let exec fail; systemd will keep retrying + # and the next attempt may succeed (e.g. transient network). + fi + fi + fi +fi + +# Hand off to the real daemon. `exec` replaces this shell so +# systemd/launchctl tracks the node PID directly — no extra hop. +exec "$NODE" "$ENTRY" "$@" diff --git a/openspec/changes/daemon-file-preview-worker/.openspec.yaml b/openspec/changes/daemon-file-preview-worker/.openspec.yaml new file mode 100644 index 000000000..8d87be18e --- /dev/null +++ b/openspec/changes/daemon-file-preview-worker/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-05-07 diff --git a/openspec/changes/daemon-file-preview-worker/design.md b/openspec/changes/daemon-file-preview-worker/design.md new file mode 100644 index 000000000..e8476121b --- /dev/null +++ b/openspec/changes/daemon-file-preview-worker/design.md @@ -0,0 +1,424 @@ +## Context + +`fs.read` is a daemon protocol message, not a FileBrowser-only message. Web callers use `ws.fsReadFile(path)`, which sends `{ type: "fs.read", path, requestId }`. FileBrowser uses this path for preview open, auto-refresh, and fresh download-handle recovery. ChatView also uses it to obtain a `downloadId` before calling download logic. + +Today `handleFsRead` performs all read/preview work in the main daemon process: path expansion, `realpath`, sensitive-directory policy, `stat`, MIME/video classification, download-handle registration, size cap checks, cache lookup, content read, binary detection, base64 conversion, and response sending. Existing `fsReadCache`, `fsReadInflight`, and `fsReadGenerations` are main-process state. + +Binding constraints: + +- Filesystem policy is not an allow-root model. The daemon runs as the user, permits broad user-readable filesystem access, and deny-lists sensitive home directories such as `.ssh`, `.gnupg`, and `.pki` after canonical `realpath`. +- Download handles are short-lived path handles, not immutable content snapshots. +- Public `fs.read_response` shape is observable: binary preview failure uses `binary_file`, text success omits `encoding`, image/office base64 uses `encoding: "base64"`, and video stream mode omits inline content. +- Server bridge holds `fs.read` pending entries for 20 seconds and single-casts by external `requestId`. Daemon terminal responses must arrive before that pending entry expires. + +## Goals / Non-Goals + +Goals: + +- Move uncached `fs.read` preflight and snapshot work into daemon-local worker threads. +- Preserve the external `fs.read` / `fs.read_response` protocol and server bridge routing. +- Preserve current broad filesystem-access semantics while fixing sensitive-directory case-comparison gaps on Windows and default macOS. +- Preserve text, binary, image, office, video stream-mode, too-large, `downloadId`, and `mtime` behavior. +- Preserve existing wire error values and public response fields. +- Prevent frontend-visible raw filesystem/worker errors. +- Keep cache, inflight, generation, fan-out, deadlines, and response assembly in the main daemon coordinator. +- Provide bounded dev-version parallelism without unbounded worker creation. +- Make deadline, queue, fan-out, worker identity, restart, shutdown, packaging, and fallback behavior deterministic and testable. + +Non-goals: + +- New public preview protocol, endpoint, or `fs.read_cancel` in v1. +- Runtime auto-scaling or worker-count hot reload. +- `UV_THREADPOOL_SIZE` tuning. +- Moving non-`fs.read` filesystem/git operations into this worker pool. +- Broad non-`fs.read` error sanitization, except local file-transfer download errors. +- Immutable content snapshots for `downloadId`. +- Inline streaming of text/image/office previews. + +## Quick Reference + +- D1: bounded static worker pool +- D2: two-phase canonical keying +- D3: public protocol compatibility +- D4: coordinator entry and no uncached FS I/O in `handleFsRead` +- D5: main-owned cache/inflight/generation +- D6: restrictive worker IPC and identity +- D7: strict/lenient filesystem policy helper +- D8: validated download handles and local download sanitization +- D9: shared fs-read error codes +- D10: admission deadline and deterministic fail-fast formula +- D11: startup fallback and runtime failure rules +- D12: freshness validation and fan-out semantics +- D13: memory bounds and payload handling +- D14: build/dist worker bootstrap +- D15: shutdown drain +- D16: worker recycle observability +- D17: coordinator module boundaries +- D18: scope controls for non-`fs.read` callers +- D19: late-result cache guard and terminal cleanup +- D20: minimal generic fs error codes for `fs.write` + +## Decisions + +### D1: Use a bounded static daemon worker pool in v1 + +Use `node:worker_threads` with a bounded daemon-local worker pool. The coordinator lazily starts exactly `workersTarget` workers on first uncached `fs.read` work and keeps those workers alive until shutdown, restart, or optional recycle. + +Default v1 pool settings: + +- `workersTarget`: 2 +- accepted range: `1..4` +- hard maximum: 4 +- active jobs per worker: 1 +- auto-scaling: out of scope + +Configuration is read when the coordinator is created. Values below 1 clamp to 1; values above 4 clamp to 4 and produce a warning/metric. Tests must use constructor overrides instead of mutating process-global environment unless the parser itself is under test. + +Rationale: + +- The user explicitly wants multi-worker behavior in the dev version. +- A static pool makes admission and tests deterministic. +- Four is a hard v1 cap because 100 MB base64 payloads can multiply memory use quickly. +- Worker threads share the process libuv filesystem pool; more workers do not guarantee more filesystem throughput. + +### D2: Use two-phase worker jobs for canonical freshness keying + +All uncached `fs.read` work uses two worker phases: + +1. **Preflight job**: expands the raw path, performs strict canonical `realpath`, applies the filesystem policy, runs `stat`, computes `startSignature`, and classifies size/MIME/video/too-large metadata. It does not read inline content or create a `downloadId`. +2. **Snapshot job**: runs for one canonical freshness key, reads content when needed, performs binary detection, prepares text/base64 or stream-mode metadata, and returns `startSignature` plus `endSignature`. + +The coordinator stores each external request separately and uses the preflight result to build the canonical snapshot key: + +```text +realPath::startSignature::resourceGeneration +``` + +Requests that arrive with different raw paths but resolve to the same canonical freshness attach to the same snapshot job. Public response assembly still uses each external request's original raw `path` field. + +Rationale: + +- The spec requires `handleFsRead` not to perform uncached `realpath` or `stat`. +- The cache/fan-out model requires canonical freshness reuse. +- Two-phase worker execution is the only v1 design that satisfies both constraints without weakening symlink/canonical behavior. + +### D3: Preserve the public fs transport contract + +The browser continues to send `fs.read` with `requestId` and `path`; daemon responses remain `fs.read_response`. The server bridge remains requestId-based and unaware of worker phases. + +Public response compatibility rules: + +- Text success responses MUST omit `encoding`. +- Image/office inline payloads MUST expose `encoding: "base64"`. +- Video stream-mode responses MUST expose `previewMode: "stream"` and MUST NOT include inline base64 content or `content`. +- Binary preview failure MUST keep `error: "binary_file"` and `previewReason: "binary"`. +- Existing public values `forbidden_path` and `file_too_large` MUST remain unchanged. + +### D4: Route all valid daemon `fs.read` through the coordinator + +All valid protocol-level `fs.read` requests enter `PreviewReadCoordinator`. + +`handleFsRead` may: + +- validate request shape and response addressability, +- return/suppress invalid requests according to D9, +- perform no-FS-I/O cache-hit checks if the coordinator exposes one, +- call the coordinator, +- assemble and send already validated responses if that responsibility remains in `command-handler.ts`. + +`handleFsRead` MUST NOT perform uncached path expansion, `realpath`, `stat`, preview classification, content read, binary detection, or base64 conversion. + +Production startup direct-read fallback is not part of v1. Worker startup failure in v1 must return the configured stable worker-unavailable terminal response instead of invoking a direct-loader path. + +### D5: Keep cache, inflight fan-out, and generations in the main daemon + +The main coordinator owns: + +- `fsReadCache`, +- `fsReadInflight`, +- `fsReadGenerations`, +- external request records, +- preflight and snapshot job records, +- per-request deadline timers, +- fan-out maps, +- stale-completion suppression. + +Workers are stateless per job. Workers must not own durable read cache, generation maps, external request IDs, server links, or download registry state. + +### D6: Worker IPC schema is explicit and restrictive + +Add `src/daemon/file-preview-read-types.ts`. + +Worker request envelope fields: + +- `phase: "preflight" | "snapshot"` +- `workerRequestId` +- `workerSlotId` +- `workerGeneration` + +Preflight payload fields: + +- `rawPath` + +Snapshot payload fields: + +- validated canonical `realPath` +- `startSignature` +- `size` +- classification metadata needed to avoid duplicate MIME/video decisions + +Worker result fields: + +- `phase` +- `workerRequestId` +- `workerSlotId` +- `workerGeneration` +- `kind: "success" | "error"` +- success metadata appropriate for the phase +- stable `FsReadErrorCode` on error +- optional `previewReason` using shared constants + +The request/result schema MUST NOT include external `requestId`, `serverLink`, browser sockets, attachment IDs, download registry objects, `downloadId`, raw `Error.message`, stack traces, errno detail, or frontend-visible absolute path diagnostics. + +The main coordinator MUST verify `workerSlotId` and `workerGeneration` on every result before routing or cache writeback. + +`policyVersion` is not part of v1 IPC. Runtime policy hot reload is out of scope. + +### D7: Use strict and lenient canonical path helper modes + +Add `src/daemon/file-preview-path-policy.ts`. + +The canonical helper exposes two modes: + +```ts +type CanonicalMode = "strict" | "lenient"; + +async function resolveCanonical( + rawPath: string, + mode: CanonicalMode, +): Promise<{ realPath: string; usedFallback: boolean } | null>; +``` + +Strict mode: + +- used by worker-backed `fs.read` and download-handle creation, +- MUST call `fs.realpath`, +- MUST fail closed on `realpath` rejection, +- MUST NOT use fallback to non-canonical paths, +- always returns `usedFallback: false` on success. + +Lenient mode: + +- used by `fs.ls includeMetadata` where best-effort UX is required, +- MUST call `fs.realpath` first, +- MUST NOT be used by ordinary `fs.ls` calls without `includeMetadata`, +- MAY fall back to the resolved path only for Windows-specific reparse/junction/symlink-loop failures with explicit error-message evidence, +- MUST fail closed for generic Windows `EPERM` or `UNKNOWN` realpath failures that do not identify a reparse/junction/symlink-loop condition, +- MUST mark fallback results with `usedFallback: true`, +- fallback paths MUST NOT create download handles. + +Deny-list comparisons: + +- Windows MUST compare canonical real path and denied prefixes case-insensitively after `path.win32.normalize`. +- macOS SHOULD compare case-insensitively by default after POSIX normalization. +- Linux and other platforms MUST preserve current case-sensitive behavior. +- `os.homedir()` MUST be read at helper invocation time, not cached at module load. + +### D8: Main daemon validates download handles and sanitizes local downloads + +The worker never creates or returns `downloadId`. + +Download handle creation must use a validated canonical path boundary: + +- `ValidatedRealPath` or an equivalent branded/opaque type is created only by strict canonical policy helpers or explicit revalidation. +- `createProjectFileHandleFromValidatedPath` (or equivalent) registers trusted handles. +- `tryCreateProjectFileHandle` (or equivalent) is used by tolerant callers such as `fs.ls includeMetadata` and returns `null` on policy failure or fallback paths. + +All `source: "local"` download errors sent to the frontend MUST be sanitized to stable messages/codes such as `not_found`, `expired`, or `download_failed`. Raw paths, errno text, stack traces, and raw `Error.message` are logged only. + +Rationale: + +- The current registry has no reliable origin field, so limiting sanitization only to worker-backed `fs.read` handles is not enforceable. +- A worker bug must not be sufficient to register a denied path. + +### D9: Stable shared error codes preserve existing wire values + +Add `shared/fs-read-error-codes.ts`. + +Required values: + +- `binary_file` +- `forbidden_path` +- `file_too_large` +- `preview_worker_queue_full` +- `preview_worker_timeout` +- `preview_worker_unavailable` +- `preview_worker_crashed` +- `stale_read` +- `invalid_request` +- `internal_error` + +Invalid request behavior: + +- Missing external `requestId`: suppress because no response can be routed. +- Present `requestId` but missing/non-string/empty `path`: send exactly one `fs.read_response` with `status: "error"` and `error: "invalid_request"`; do not enqueue worker work. + +Production code outside the shared constants module must not define duplicate fs-read wire strings. Tests and specs may assert literal legacy values. + +### D10: Worker deadlines start at coordinator admission with deterministic admission control + +The daemon deadline is 18 seconds and starts when a valid external `fs.read` request enters the coordinator. It includes preflight queue wait, preflight execution, snapshot queue wait, snapshot execution, response assembly, and fan-out delay. + +The coordinator uses this deterministic admission formula: + +```text +projectedWaitMs = ((queueDepth + 1) * tEstimateMs) / workersTarget +reject if projectedWaitMs + tEstimateMs > deadlineMs - safetyMarginMs +``` + +Definitions: + +- `workersTarget`: configured worker instance count, default 2, clamped to `[1, 4]`. +- `queueDepth`: queued jobs only; active jobs are not counted here. +- `tEstimateMs`: rolling median of the last 16 completed worker jobs' active execution durations; seed 1500 ms. +- `deadlineMs`: 18000. +- `safetyMarginMs`: 2000. + +The bounded queue cap of 32 is an upper-bound safeguard; admission control may reject earlier. + +Constructor options must allow tests to override worker count, queue cap, deadline, safety margin, fake clock, and `tEstimateMs`. + +The admission deadline must be propagated to worker-pool scheduling as pool-local metadata, not as part of the worker IPC message. The worker pool must check the remaining deadline budget before enqueue, before dispatch from the queue, and when arming the active-job watchdog. Active watchdog duration must be `min(activeJobTimeoutMs, deadlineAt - now)` when a deadline exists. A job whose deadline has already expired must reject with timeout without entering the worker. Preflight and snapshot phases for a request share the same admission-time deadline. + +### D11: Startup fallback is explicit; runtime fallback is forbidden + +Startup fallback remains explicit and disabled in v1 production. If the real worker path cannot produce a terminal result, the request receives a stable worker error such as `preview_worker_unavailable`, `preview_worker_crashed`, or `preview_worker_timeout`; it does not synchronously re-enter the old main-process file-read path. + +The only direct in-process worker path in v1 is test-only and gated by Vitest-specific signals. A bare `NODE_ENV=test` is not sufficient to enable it, because dist/manual daemon runs may be misconfigured with that value. It is not a production startup fallback and must not be enabled by normal dist startup. + +Future rollout direct fallback, if ever required, must be proposed in a separate OpenSpec change with its own direct-loader module, performance budget, and tests. It is intentionally not implemented by this v1 change. + +Runtime timeout, worker crash, worker restart, stale result, and late completion MUST NOT synchronously fallback to direct read for the affected request. Those requests receive stable terminal errors and future requests may use restarted workers. + +### D12: Freshness is verified with start and end signatures + +Preflight returns a `startSignature`. Snapshot returns `startSignature` and `endSignature`. If signatures differ, the main coordinator MUST NOT cache the result and MUST return `stale_read` rather than a mixed success. + +Once the coordinator accepts a snapshot for fan-out, currently active attached requestIds receive that accepted snapshot unless their own deadline has already fired. Later invalidation affects future requests and cache writeback decisions, not already accepted fan-out sends. + +### D13: Fan-out and memory are bounded + +Default coordinator settings: + +- `workersTarget`: 2 +- `hardMaxWorkers`: 4 +- active worker jobs: one per worker instance +- queued worker jobs: 32 +- attached external requestIds per worker job: 32 +- daemon deadline: 18 seconds from coordinator admission + +Queue entries store metadata only, not preview content. Identical-freshness fan-out retains one accepted worker snapshot before serialization. Fan-out sends MUST be serialized or otherwise prove equivalent peak-memory bounds. + +Each external requestId has an independent timer armed at admission. If a timer fires before that request is sent, the coordinator MUST produce the terminal timeout response and MUST NOT wait for the fan-out queue to reach that request. + +The main-worker IPC for large payloads SHOULD prefer transferable `ArrayBuffer` or equivalent single-copy transfer. If v1 cannot avoid copies, memory behavior must be documented before increasing any default cap. + +Final v1 implementation keeps Node worker structured-clone payload transfer for strings and buffers instead of adding a transferable `ArrayBuffer` protocol. Peak memory is bounded by default two workers, hard maximum four workers, one active job per worker, queue metadata only, attached request cap thirty-two, serialized fan-out, video stream-mode avoiding base64, and default worker recycle after fifty completed jobs. A worst-case image/office/text preview can still duplicate a large payload across worker and main isolate during transfer and response serialization; the current 100 MB cap is therefore not raised in this change. Future cap increases or a higher worker maximum require measured RSS/heap evidence or a transferable-payload follow-up. + +### D14: Build uses the existing worker bootstrap pattern + +Add `src/daemon/file-preview-read-worker-bootstrap.mjs` and resolve it like `jsonl-parse-worker-bootstrap.mjs`. The existing postbuild copy script must copy this `.mjs` into `dist/src/daemon/`. + +Dist smoke must start the default pool size and dispatch at least two representative concurrent jobs, not merely prove a single worker bootstrap can load. + +Worker test fixtures must live under `test/` or be excluded from the bootstrap-copy path; `src/**/*.mjs` bootstrap copying must not accidentally package test fixtures. + +### D15: Graceful shutdown drains pending preview requests + +The coordinator exposes a shutdown/drain operation. During daemon graceful shutdown, active and queued preview reads should receive `preview_worker_unavailable` within a bounded shutdown budget, such as 1 second, instead of waiting for the server bridge pending timeout. Shutdown drain must not wait for slow preview reads to finish normally. + +Production `lifecycle.shutdown()` must call the default preview-read coordinator drain before `serverLink.disconnect()` so terminal unavailable responses still have a live daemon/server transport. + +### D16: Worker recycle is optional but lifecycle observability is required + +The coordinator SHOULD support job-count recycle: + +- default `WORKER_RECYCLE_JOB_COUNT`: 50, +- recycle after the current job completes, +- replacement starts before the next dispatch to that slot, +- recycle must not cancel or duplicate jobs on other workers. + +Regardless of whether automatic recycle is implemented in v1, worker lifecycle logs/metrics MUST include worker startup, shutdown, restart, crash, recycle if present, job count, queue full, timeout, stale read, and sanitized internal errors. Lifecycle logs must not include raw paths or raw filesystem errors. + +### D17: Coordinator module boundaries are explicit + +The preview-read coordinator should be implemented as composed modules instead of one large class: + +```text +file-preview-read-coordinator.ts entry/orchestrator +file-preview-read-pool.ts WorkerPool lifecycle, dispatch, restart, optional recycle +file-preview-read-admission.ts AdmissionQueue formula and queue cap +file-preview-read-fanout.ts FanOutDispatcher timers and sequential send +file-preview-read-cache-facade.ts ReadCacheFacade for cache/inflight/generation +file-preview-read-shutdown.ts DrainController for graceful shutdown +``` + +Dependency direction: + +- coordinator mediates all submodules, +- submodules do not import each other directly, +- only `WorkerPool` talks to worker threads, +- each submodule is testable with fake clock and fake collaborators. + +### D18: Scope controls for non-fs.read callers + +Non-`fs.read` callers may reuse extracted policy helpers only when behavior stays compatible. This change must not quietly alter `fs.write`, `fs.git_status`, `fs.git_diff`, or `fs.mkdir` public error behavior. If those call sites are touched for helper reuse, focused regression tests must prove existing public behavior remains intact. + +`fs.ls includeMetadata` is the intended non-`fs.read` caller affected by handle hardening. Allowed normal files must still receive `downloadId`; denied or fallback paths must omit `downloadId` without failing the entire directory listing. + +### D19: Late completions and terminal records are cleaned up + +The coordinator must treat request terminal state as the source of truth for cache writeback eligibility. A snapshot result whose attached requestIds have all timed out or otherwise reached terminal state must not be written into active `fs.read` cache. When at least one attached request remains eligible at completion time, the coordinator may write one active cache entry and fan out only to still-eligible requestIds. + +Terminal fan-out records and external request records are deleted after their terminal transition. Late preflight/snapshot completions that reference deleted requestIds naturally skip response assembly and cannot send duplicate terminal responses. + +### D20: Generic filesystem error codes are separated from read-preview-specific codes + +`shared/fs-error-codes.ts` owns generic filesystem protocol codes used by more than one `fs.*` command: `forbidden_path`, `file_too_large`, `invalid_request`, and `internal_error`. `shared/fs-read-error-codes.ts` extends those generic values with preview/read-specific values such as `binary_file`, `preview_worker_timeout`, and `stale_read`. + +`fs.write` remains outside the preview worker pool, but its catch-all error responses must use generic stable codes rather than raw `Error.message` or read-preview-specific constant names. New-target writes must fail closed if the target appears as a symlink after the initial existence check and before exclusive creation; a full no-follow/open-by-fd rewrite is left to a dedicated filesystem-write hardening change. + +## Risks / Trade-offs + +- Two-phase preflight adds one worker round trip. This is accepted to keep uncached `realpath/stat` out of `handleFsRead` while retaining canonical freshness fan-out. +- Worker threads share libuv filesystem threads. This isolates JS event-loop and CPU/base64 work, not physical filesystem throughput. +- Four workers with 100 MB base64 payloads can still cause high memory pressure. Hard cap, admission control, fan-out serialization, metrics, and optional recycle mitigate the risk. +- Strict/lenient policy modes introduce two behaviors. Tests must prove `fs.read` stays strict and `fs.ls includeMetadata` retains best-effort behavior without creating fallback download handles. +- All local download error sanitization intentionally widens scope. This avoids unverifiable origin inference and is safer than partial sanitization. + +## Migration Plan + +1. Freeze shared public wire constants and response compatibility tests. +2. Extract strict/lenient policy helpers and classifier helpers. +3. Harden local handle creation with validated canonical paths and sanitized local download errors. +4. Add worker IPC types, two-phase worker, and bootstrap. +5. Add coordinator submodules: worker pool, admission, fan-out, cache facade, shutdown drain. +6. Integrate `handleFsRead` with coordinator and public response assembly. +7. Add tests and validation gates listed in `tasks.md`. + +Rollback: + +- Set `workersTarget=1`. +- Worker startup fallback is already disabled in v1; rollback must not enable direct-read fallback. +- Keep shared constants, policy helper fixes, and local handle hardening because they close existing safety gaps. +- No database or external protocol rollback is required. + +## Open Questions + +These are follow-ups, not v1 blockers: + +1. Should image/office base64 preview get a lower cap than the current general preview size limit after memory measurements? +2. Should future versions add `fs.read_cancel`? +3. Should future versions tune `UV_THREADPOOL_SIZE` for heavy preview deployments? +4. Should macOS case-insensitive deny-list behavior become a hard MUST after compatibility feedback? +5. Should worker recycle become mandatory after heap/RSS measurements? diff --git a/openspec/changes/daemon-file-preview-worker/proposal.md b/openspec/changes/daemon-file-preview-worker/proposal.md new file mode 100644 index 000000000..5d83ef7b7 --- /dev/null +++ b/openspec/changes/daemon-file-preview-worker/proposal.md @@ -0,0 +1,128 @@ +## Why + +Daemon `fs.read` requests currently run through the normal WebSocket command path and `handleFsRead` performs path expansion, canonical `realpath`, sensitive-path policy checks, metadata lookup, MIME classification, file reading, binary detection, base64/text preparation, video stream-mode classification, download-handle registration, cache handling, and response sending in the daemon main process. + +Slow disks, network mounts, large base64 previews, repeated preview refreshes, and preview-triggered download-handle recovery can therefore add visible latency to unrelated daemon work. Moving uncached preview/read snapshot work into daemon-local worker threads gives preview work a bounded execution lane while preserving the existing browser/server/daemon protocol. + +The previous draft left several implementation-critical ambiguities: how canonical freshness fan-out works when `realpath/stat` move out of the main path, whether the worker pool may auto-scale beyond the default, how local download errors are identified and sanitized, how worker restart generations are correlated, and how queue/admission deadlines are tested. This change closes those gaps before implementation. + +## What Changes + +- Route every valid protocol-level daemon `fs.read` request through a main-process preview-read coordinator. +- Use a two-phase worker execution model: + - preflight job: path expansion, strict canonical `realpath`, policy check, `stat`, signature, size/MIME/video/too-large classification. + - snapshot job: content read, binary detection, text/base64 preparation, video stream metadata, and start/end freshness verification. +- Keep uncached `realpath`, `stat`, content read, binary detection, and base64 conversion out of `handleFsRead`; main-process fast paths are limited to no-FS-I/O cache hits, already-validated response assembly, deadline/queue management, generation checks, and handle registration. +- Keep cache, inflight fan-out, resource generations, per-request deadlines, stale-completion suppression, and public response assembly in the main daemon coordinator. +- Use a bounded static worker pool in v1: default `workersTarget=2`, accepted range `1..4`, hard maximum `4`, one active worker job per worker, no auto-scaling, no more than thirty-two queued worker jobs, and no more than thirty-two attached external requestIds per worker job. +- Use deterministic admission control with an 18 second daemon deadline that starts at coordinator admission and remains below the server bridge 20 second `fs.read` pending timeout. +- Preserve the external `fs.read` / `fs.read_response` protocol; no browser/server migration, no new WebSocket message type, and no new endpoint are introduced. +- Preserve public wire error values and response shapes: + - `binary_file`, `forbidden_path`, and `file_too_large` remain unchanged. + - text success responses omit `encoding`. + - image/office inline responses use `encoding: "base64"`. + - video stream-mode responses omit inline `content`. +- Add shared fs-read error/preview-reason constants and prohibit production duplicate wire-string literals outside the shared module. +- Add stable worker failure codes for queue full, timeout, unavailable, crash, stale read, invalid request, and internal error. +- Sanitize frontend-visible `fs.read_response.error` values and all local file-transfer download errors so raw host paths, errno text, stack traces, and raw `Error.message` are not exposed. +- Extract a strict/lenient canonical path policy helper: + - strict mode is used by worker-backed `fs.read` and download-handle creation and fails closed on `realpath` failure. + - lenient mode preserves existing `fs.ls includeMetadata` best-effort behavior where appropriate, but fallback paths cannot create download handles. +- Harden local download-handle creation with a validated canonical path boundary while preserving `fs.ls includeMetadata` behavior for allowed files. +- Preserve short-lived path-handle semantics for `downloadId`; this change does not make download handles immutable content snapshots. +- Add explicit worker identity and generation fields so late completions from crashed/restarted workers cannot route to newer requests or update cache. +- Document v1 production fallback-disabled behavior and forbid runtime direct-read fallback for startup failure, timeout, crash, restart, and late completion paths. +- Fail fast with `preview_worker_unavailable` when the worker pool has no executable worker during startup/restart backoff and fallback is disabled. +- Prevent late snapshot completions from writing active cache entries after every attached external request has timed out or reached terminal state. +- Delete terminal coordinator/fan-out request records so long-running daemons do not retain completed preview metadata indefinitely. +- Wire production daemon shutdown into preview coordinator drain before the daemon disconnects from the server link. +- Reuse the shared lenient canonical helper for `fs.ls` metadata listings instead of ad hoc Windows realpath fallback. +- Add generic fs error constants, minimally sanitize `fs.write_response.error` catch-all failures, and fail closed when a new-file write target appears as a symlink before creation without moving `fs.write` into the preview worker pool. +- Add dev and compiled `dist/` worker-pool smoke coverage, not just a single-worker happy path. +- Add compiled `dist/` default daemon coordinator smoke coverage that runs outside Vitest/test-mode shims, starts real worker threads, proves success plus sanitized worker-visible errors reach the public `fs.read_response` send path, and locks non-preview command responsiveness while real preview workers are delayed. + +## Scope + +In scope: + +- Every daemon `fs.read` request received by `handleFsRead`, including FileBrowser preview open, FileBrowser auto-refresh, download-handle recovery, and ChatView download-trigger behavior. +- Minimal shared-code and sanitization hardening for `fs.write_response.error` catch-all failures so non-preview filesystem commands do not keep using read-preview-specific constant names. +- Minimal `fs.write` new-target hardening for symlink races detected before exclusive creation; broader no-follow/open-by-fd write semantics remain out of scope. +- The v1 coordinator model: two-phase preflight/snapshot worker jobs, static worker pool, one active job per worker, bounded queueing, bounded attached requestIds, deterministic admission control, and per-request admission deadlines. +- Shared fs-read error/preview constants used by daemon, web, and server consumers. +- Daemon-local worker lifecycle, worker identity/generation, restart/backoff, optional job-count recycle, shutdown drain, timeout, crash handling, stale suppression, and dist packaging. +- Strict/lenient canonical path policy extraction, including Windows case-insensitive deny-list comparisons and macOS default case-insensitive deny-list behavior. +- Defense-in-depth local download-handle registration using validated canonical paths, with `fs.ls includeMetadata` regression coverage. +- Sanitized local file-transfer download errors for all `source: "local"` handles. +- Tests that lock protocol compatibility, security, freshness, fan-out, queue/admission, timeout, worker failure, packaging, and FileBrowser/ChatView no-regression behavior. + +Out of scope: + +- Adding `fs.read_cancel`, `fs.preview_read`, or any new public preview protocol in v1. +- Adding any new HTTP or WebSocket endpoint, including worker health or queue endpoints. +- Auto-scaling worker instances at runtime; v1 reads `workersTarget` once at coordinator creation and clamps it to `1..4`. +- Raising the hard worker cap above four without a separate OpenSpec change and memory/concurrency evidence. +- Running more than one active preview job inside a single worker. +- Tuning `UV_THREADPOOL_SIZE`; worker threads still share the process libuv filesystem pool. +- Moving `fs.ls`, `fs.git_status`, `fs.git_diff`, `fs.write`, file upload transfer, or local web-preview relay into this worker pool. +- Broadly sanitizing non-`fs.read` filesystem command errors beyond the minimal `fs.write_response.error` catch-all hardening in this change; only local file-transfer download errors are intentionally widened because the current local handle registry cannot reliably distinguish worker-backed origins. +- Providing content-snapshot download guarantees for `downloadId`. +- Replacing FileBrowser rendering or adding streaming inline text/image/office previews. +- Runtime policy hot reload; `policyVersion` is not part of v1 IPC. + +## Capabilities + +### New Capabilities + +- `daemon-file-preview-worker`: Defines daemon-owned two-phase worker-pool execution for protocol-level `fs.read`, bounded static concurrency, deterministic admission, response routing, protocol compatibility, error-code, filesystem policy, download-handle, packaging, failure-mode, and validation requirements. + +### Modified Capabilities + +- `daemon-fs-cache`: `fs.read` freshness-safe cache, canonical alias fan-out, inflight reuse, per-request metadata, per-request deadlines, start/end signature validation, and memory-bound fan-out requirements must remain valid when preview execution moves behind daemon workers. + +## Impact + +- Daemon: + - `src/daemon/command-handler.ts` `handleFsRead` + - new `src/daemon/file-preview-read-coordinator.ts` + - new `src/daemon/file-preview-read-pool.ts` + - new `src/daemon/file-preview-read-admission.ts` + - new `src/daemon/file-preview-read-fanout.ts` + - new `src/daemon/file-preview-read-cache-facade.ts` + - new `src/daemon/file-preview-read-worker.ts` + - new `src/daemon/file-preview-read-worker-bootstrap.mjs` + - new `src/daemon/file-preview-read-types.ts` + - new `src/daemon/file-preview-path-policy.ts` + - new `src/daemon/file-preview-classifier.ts` + - `src/daemon/file-transfer-handler.ts` local handle validation and local download error sanitization +- Shared: + - new `shared/fs-read-error-codes.ts` as the single source for fs-read error and preview-reason wire constants. +- Web/server: + - `web/src/ws-client.ts` public `fsReadFile(path)` contract remains unchanged. + - FileBrowser and ChatView continue to consume existing `fs.read_response` fields. + - `server/src/ws/bridge.ts` requestId single-cast routing remains unchanged; tests cover timeout margin if constants are exported for verification. +- Tests: + - shared constants and grep-gate tests + - daemon policy, classifier, worker, coordinator, fan-out, cache, invalidation, file-transfer, fallback, and shutdown tests + - fake-worker/fake-clock queue and admission tests + - dist worker-pool smoke test + - FileBrowser/ChatView no-regression tests + - test-session hygiene tests if integration/e2e tests create sessions or projects + +## Concurrency Answer + +Multiple file-read workers are part of v1, but v1 is a static bounded pool, not an auto-scaling service. The coordinator lazily starts exactly `workersTarget` workers, defaulting to two. Configuration is clamped to the accepted range `1..4`; four is a hard v1 maximum. Each worker runs one active preflight or snapshot job at a time. + +Different canonical files or different freshness states can run concurrently up to `workersTarget`. Additional distinct-freshness jobs queue, fail fast under the deterministic admission formula, or time out before the server bridge drops its 20 second pending entry. Identical canonical file plus identical freshness requests attach to one snapshot job and fan out one accepted snapshot instead of duplicating reads. + +Worker threads share the daemon process libuv filesystem pool, so this improves JS event-loop and CPU/base64 isolation but does not guarantee unlimited filesystem throughput. The design intentionally keeps the default pool small and requires metrics before any future worker-cap increase. + +## Pod-sticky Compatibility + +This change introduces no daemon-dependent endpoint: + +- `fs.read` and `fs.read_response` continue to flow over the existing server-id routed daemon WebSocket bridge. +- Download HTTP for stream-mode video and binary/too-large handles continues to use existing server-id routed download paths. +- The preview-read worker is daemon-local via `node:worker_threads`; it owns no cross-pod state. + +Verification must confirm no new frontend fetch/WebSocket path bypasses `/api/server/:serverId/...` and no worker health/queue endpoint is added in v1. diff --git a/openspec/changes/daemon-file-preview-worker/specs/daemon-file-preview-worker/spec.md b/openspec/changes/daemon-file-preview-worker/specs/daemon-file-preview-worker/spec.md new file mode 100644 index 000000000..8184ee7a5 --- /dev/null +++ b/openspec/changes/daemon-file-preview-worker/specs/daemon-file-preview-worker/spec.md @@ -0,0 +1,410 @@ +## ADDED Requirements + +### Requirement: Daemon SHALL route valid fs.read requests through the preview-read coordinator +The daemon SHALL route every valid protocol-level `fs.read` request received by `handleFsRead` through a main-process preview-read coordinator backed by daemon-local workers. The external `fs.read` / `fs.read_response` protocol SHALL remain backward compatible. + +#### Scenario: valid fs.read enters coordinator +- **WHEN** the daemon receives `fs.read` with a non-empty string `path` and string `requestId` +- **THEN** the daemon MUST schedule the request through the preview-read coordinator +- **AND** it MUST NOT perform uncached path expansion, `realpath`, `stat`, preview classification, content read, binary detection, or base64 conversion directly in `handleFsRead` + +#### Scenario: request without requestId is suppressed +- **WHEN** an `fs.read` request has no string `requestId` +- **THEN** the daemon MUST NOT enqueue worker work +- **AND** it MAY suppress the request because no response can be routed + +#### Scenario: invalid path with requestId returns invalid_request +- **WHEN** an `fs.read` request has a string `requestId` +- **AND** `path` is missing, not a string, or an empty string +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` +- **AND** the response MUST use the shared `invalid_request` error code +- **AND** the request MUST NOT be enqueued to the preview worker + +#### Scenario: response contract remains compatible +- **WHEN** a worker-backed `fs.read` completes +- **THEN** the daemon MUST send `fs.read_response` with the original external `requestId` +- **AND** the response MUST use existing public fields only where those fields already apply +- **AND** the public `path` field MUST be the original raw path for that external request +- **AND** `resolvedPath` MUST be the canonical path from worker preflight or snapshot metadata + +#### Scenario: existing server bridge remains sufficient +- **WHEN** the server bridge receives a worker-backed `fs.read_response` +- **THEN** it MUST be able to single-cast the response using its existing `requestId` pending map +- **AND** this change MUST NOT require a new server routing protocol or new endpoint + +### Requirement: Worker-backed fs.read SHALL preserve public response compatibility +Worker-backed reads SHALL preserve current public `fs.read_response` wire values and field presence for existing success and error cases. + +#### Scenario: text preview omits encoding +- **WHEN** a supported text file is returned inline +- **THEN** the daemon MUST return `status: "ok"` with `content` +- **AND** the public response MUST NOT include an `encoding` field + +#### Scenario: image and office preview returns base64 +- **WHEN** the canonical path passes policy validation and the file is a supported image or office preview type within the size limit +- **THEN** the daemon MUST return `status: "ok"` with `encoding: "base64"`, `content`, `mimeType`, `downloadId`, and `mtime` + +#### Scenario: video preview remains stream-mode +- **WHEN** the canonical path passes policy validation and the file is a supported video preview type within the size limit +- **THEN** the daemon MUST return `status: "ok"` with `previewMode: "stream"`, `mimeType`, `size`, `downloadId`, and `mtime` +- **AND** it MUST NOT base64-encode the video content into the WebSocket response +- **AND** it MUST NOT include inline `content` + +#### Scenario: binary preview keeps existing public error code +- **WHEN** the canonical path passes policy validation but binary detection rejects inline text preview +- **THEN** the daemon MUST return `status: "error"` with the shared `binary_file` error code and `previewReason: "binary"` +- **AND** the response MUST include a `downloadId` governed by the existing file-transfer handle TTL + +#### Scenario: oversized preview keeps downloadable handle +- **WHEN** the canonical path passes policy validation and the file exceeds the preview read size limit +- **THEN** the daemon MUST return `status: "error"` with the shared `file_too_large` error code and `previewReason: "too_large"` +- **AND** the response MUST include a `downloadId` governed by the existing file-transfer handle TTL + +### Requirement: Worker-backed fs.read SHALL use two-phase worker execution +Uncached worker-backed reads SHALL use a worker preflight phase before any content snapshot phase so canonical freshness fan-out remains possible without doing uncached `realpath` or `stat` in `handleFsRead`. + +#### Scenario: preflight resolves canonical freshness +- **WHEN** a valid uncached `fs.read` enters the coordinator +- **THEN** the coordinator MUST schedule a preflight worker job +- **AND** the preflight job MUST perform path expansion, strict canonical `realpath`, filesystem policy check, `stat`, signature computation, and preview classification +- **AND** the preflight job MUST NOT read inline file content or create `downloadId` + +#### Scenario: snapshot job reads one canonical freshness +- **WHEN** preflight succeeds for a canonical path and freshness signature +- **THEN** the coordinator MUST key snapshot work by canonical path, freshness signature, and current resource generation +- **AND** it MUST schedule at most one active snapshot worker job for that key +- **AND** the snapshot job MUST perform content read, binary detection, text/base64 preparation, or video stream metadata as needed + +#### Scenario: raw aliases attach to one canonical snapshot +- **WHEN** multiple raw paths canonicalize to the same file with the same freshness signature and resource generation +- **THEN** the coordinator MUST attach those external requestIds to one snapshot job +- **AND** each final response MUST preserve that request's own raw `path` +- **AND** every final response MUST use the same canonical `resolvedPath` + +### Requirement: Worker-backed fs.read SHALL preserve filesystem policy and classification +Worker-backed reads SHALL preserve the current daemon filesystem policy: broad user-readable filesystem access with a sensitive home-directory deny-list after canonical `realpath`. This change SHALL NOT introduce a new allow-root model. + +#### Scenario: denied sensitive path is rejected +- **WHEN** a requested path canonicalizes under a denied sensitive directory +- **THEN** the daemon MUST return `fs.read_response` with `status: "error"` and the shared `forbidden_path` error code +- **AND** the daemon MUST NOT return file content +- **AND** the daemon MUST NOT create a `downloadId` + +#### Scenario: symlink into denied path is rejected +- **WHEN** a requested path is a symlink or indirect path that canonicalizes under a denied sensitive directory +- **THEN** the daemon MUST reject it according to the same `forbidden_path` policy +- **AND** it MUST NOT create a `downloadId` + +#### Scenario: Windows sensitive directory comparison is case-insensitive +- **WHEN** the policy helper evaluates a Windows canonical path under `.SSH`, `.GnuPG`, or `.PKI` with any casing +- **THEN** it MUST treat that path as under the corresponding denied directory +- **AND** it MUST reject the path + +#### Scenario: macOS sensitive directory comparison defaults to case-insensitive +- **WHEN** the policy helper evaluates a macOS canonical path under `.SSH`, `.GnuPG`, or `.PKI` with any casing +- **THEN** it SHOULD treat that path as under the corresponding denied directory +- **AND** default tests MUST cover the case-insensitive deny behavior + +#### Scenario: Linux sensitive directory comparison remains case-sensitive +- **WHEN** the policy helper evaluates a Linux path under `.SSH` +- **AND** that path is distinct from `.ssh` on a case-sensitive filesystem +- **THEN** the helper MUST preserve current behavior and not reject solely because of the uppercase spelling + +#### Scenario: home directory is read per invocation +- **WHEN** the home directory source changes between two policy helper calls in tests +- **THEN** the second call MUST observe the new home directory +- **AND** the helper MUST NOT cache `homedir()` or `process.env.HOME` at module load + +### Requirement: Canonical path helper SHALL expose strict and lenient modes +The extracted canonical helper SHALL expose strict and lenient modes so `fs.read` remains fail-closed while `fs.ls includeMetadata` can preserve existing best-effort behavior where appropriate. + +#### Scenario: strict mode rejects realpath failure +- **WHEN** worker-backed `fs.read` calls strict canonical helper +- **AND** `fs.realpath` rejects +- **THEN** the helper MUST return no canonical path +- **AND** the daemon MUST emit a sanitized terminal error +- **AND** the daemon MUST NOT create a `downloadId` + +#### Scenario: strict mode never uses fallback paths +- **WHEN** strict mode succeeds +- **THEN** it MUST return a canonical real path with `usedFallback: false` +- **AND** strict mode MUST NOT use platform-specific resolved-path fallback + +#### Scenario: lenient mode may preserve Windows fs.ls best effort +- **WHEN** `fs.ls includeMetadata` calls lenient mode on a Windows reparse path +- **AND** `fs.realpath` rejects with an expected reparse-point failure +- **THEN** the helper MAY return a resolved path with `usedFallback: true` +- **AND** that fallback path MUST NOT be accepted for download-handle creation + +#### Scenario: ordinary fs.ls remains strict +- **WHEN** `fs.ls` runs without `includeMetadata` +- **THEN** it MUST use strict canonical resolution +- **AND** it MUST NOT use Windows lenient fallback to list a non-canonical path + +#### Scenario: generic Windows errors do not trigger lenient fallback +- **WHEN** lenient mode receives a Windows realpath error with generic `EPERM` or `UNKNOWN` +- **AND** the error does not identify a reparse, junction, symlink, or symlink-loop condition +- **THEN** the helper MUST fail closed + +### Requirement: fs.read errors SHALL be stable, shared, and sanitized +Every frontend-visible `fs.read_response.error` SHALL use a stable code from a shared module. Daemon, web, and server production code SHALL NOT duplicate cross-boundary fs-read error string literals outside that shared module. + +#### Scenario: existing wire errors are preserved +- **WHEN** shared fs-read error constants are introduced +- **THEN** public wire values for existing errors MUST remain `binary_file`, `forbidden_path`, and `file_too_large` +- **AND** implementation-specific constant names MUST NOT change those wire values + +#### Scenario: raw filesystem error is sanitized +- **WHEN** `realpath`, `stat`, `readFile`, worker startup, or worker execution fails with an internal error containing an absolute path, errno, or stack trace +- **THEN** the daemon MUST log the detailed internal error locally +- **AND** the `fs.read_response.error` field sent to the browser MUST be a shared stable code +- **AND** the response MUST NOT include raw `Error.message`, stack trace, errno text, or absolute host paths in any frontend-visible field + +#### Scenario: worker operational errors use stable codes +- **WHEN** the coordinator rejects a request because the queue or fan-out cap is full +- **THEN** it MUST return `preview_worker_queue_full` +- **WHEN** the worker request times out +- **THEN** it MUST return `preview_worker_timeout` +- **WHEN** the worker cannot start and startup fallback is disabled +- **THEN** it MUST return `preview_worker_unavailable` +- **WHEN** the worker crashes while requests are pending +- **THEN** affected requests MUST receive `preview_worker_crashed` + +### Requirement: Main daemon SHALL own download-handle registration and revalidation +The worker SHALL never create or return a `downloadId`. The main daemon SHALL create download handles only from validated canonical paths. + +#### Scenario: worker success is revalidated before handle creation +- **WHEN** the worker returns a successful snapshot for a canonical path +- **THEN** the main daemon MUST revalidate that path or accept an equivalent `ValidatedRealPath` +- **AND** it MUST refuse handle creation if the path fails policy validation + +#### Scenario: direct handle creation cannot bypass policy +- **WHEN** daemon code attempts to create a local file handle for a denied sensitive path +- **THEN** handle creation MUST fail +- **AND** it MUST NOT register an attachment handle + +#### Scenario: tolerant fs.ls metadata caller preserves allowed handles +- **WHEN** `fs.ls` runs with `includeMetadata: true` for a directory containing an allowed normal file +- **THEN** handle hardening MUST NOT prevent that allowed file from receiving a metadata `downloadId` + +#### Scenario: denied or fallback metadata entry does not register handle +- **WHEN** `fs.ls includeMetadata` encounters a denied path or a lenient fallback path +- **THEN** the entry MAY remain visible in the listing +- **AND** the entry MUST omit `downloadId` +- **AND** the listing MUST NOT expose raw deny-list details or raw filesystem errors + +#### Scenario: download handle remains a short-lived path handle +- **WHEN** a response includes `downloadId` +- **THEN** that handle MUST use existing short-lived file-transfer TTL and cleanup behavior +- **AND** the system MUST NOT claim the handle represents an immutable content snapshot + +#### Scenario: local download errors are sanitized +- **WHEN** a download of any `source: "local"` handle fails with an internal filesystem error +- **THEN** the frontend-visible file-transfer error MUST be sanitized to a stable message or code +- **AND** it MUST NOT include raw host paths, stack traces, errno text, or raw `Error.message` + +### Requirement: Preview-read coordinator SHALL use bounded static worker-pool execution +The coordinator SHALL enforce a bounded static worker pool, one active job per worker, bounded queueing, bounded fan-out, deadline-based terminal responses, worker identity validation, and stale completion suppression. + +Default v1 bounds SHALL be: + +- `workersTarget`: two worker instances +- accepted worker range: one to four worker instances +- hard maximum: four worker instances +- active jobs: one active preflight or snapshot job per worker +- queued worker jobs: at most thirty-two +- attached external requestIds per snapshot job: at most thirty-two +- daemon deadline: eighteen seconds from coordinator admission + +#### Scenario: pool starts static target count +- **WHEN** the coordinator starts under default configuration +- **THEN** it MUST lazily start exactly two worker instances +- **AND** it MUST NOT create four worker instances unless configuration explicitly sets `workersTarget` to four + +#### Scenario: worker count is clamped +- **WHEN** configuration requests fewer than one worker +- **THEN** the coordinator MUST clamp to one worker and emit a diagnostic +- **WHEN** configuration requests more than four workers +- **THEN** the coordinator MUST clamp to four workers and emit a diagnostic + +#### Scenario: v1 dispatches concurrent jobs through bounded pool +- **WHEN** multiple valid uncached `fs.read` worker jobs are ready +- **THEN** the coordinator MUST dispatch jobs concurrently up to `workersTarget` +- **AND** it MUST NOT run more than one active read job on the same worker +- **AND** additional jobs MUST wait in the bounded queue or fail with a terminal error + +#### Scenario: deterministic projected wait fail-fast +- **WHEN** a new valid `fs.read` enters the coordinator +- **AND** `((queueDepth + 1) * tEstimateMs) / workersTarget + tEstimateMs > deadlineMs - safetyMarginMs` +- **THEN** the coordinator MUST send exactly one terminal `fs.read_response` for that requestId +- **AND** the response MUST use `preview_worker_queue_full` +- **AND** the request MUST NOT be enqueued + +#### Scenario: admission constants are test-overridable +- **WHEN** coordinator is constructed in tests +- **THEN** worker count, queue cap, attached cap, deadline, safety margin, fake clock, and `tEstimateMs` MUST be overridable +- **AND** tests MUST be able to deterministically trigger both admission and rejection + +#### Scenario: queue full returns terminal error +- **WHEN** the preview worker queue is full and another distinct-freshness job arrives +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` +- **AND** the response MUST use `preview_worker_queue_full` + +#### Scenario: attached request cap returns terminal error +- **WHEN** a snapshot job already has the maximum allowed attached external requestIds +- **AND** another identical-freshness request tries to attach +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` +- **AND** the response MUST use `preview_worker_queue_full` + +#### Scenario: timeout returns before bridge pending expiry +- **WHEN** a worker-backed `fs.read` exceeds the daemon deadline, including queue wait and active execution +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` +- **AND** the response MUST use `preview_worker_timeout` +- **AND** the configured daemon timeout MUST leave transfer margin before the server bridge pending expiry + +#### Scenario: worker active watchdog uses remaining admission budget +- **WHEN** a preflight or snapshot job waits in the worker queue before entering a worker +- **THEN** the active job watchdog MUST use the remaining admission deadline budget rather than a fresh full active timeout +- **AND** an expired job MUST NOT be posted to a worker +- **AND** the admission deadline metadata MUST NOT be included in the worker IPC payload + +#### Scenario: unrelated command dispatch remains responsive +- **WHEN** all configured fake workers are blocked +- **THEN** at least one non-`fs.read` daemon command path MUST complete without waiting for the preview read to finish +- **AND** the design MUST NOT claim filesystem throughput isolation from libuv worker-pool contention +- **AND** dist or real-daemon coverage MUST prove a non-preview command remains visible while real preview workers are delayed + +### Requirement: Worker startup and runtime failures SHALL have deterministic terminal behavior +The coordinator SHALL distinguish startup fallback from runtime worker failures. + +#### Scenario: production startup fallback is disabled +- **WHEN** the worker pool cannot start +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` +- **AND** the response MUST use `preview_worker_unavailable` +- **AND** it MUST NOT direct-read for that request + +#### Scenario: all workers unavailable during restart backoff fails fast +- **WHEN** every configured preview worker slot is dead or restarting and no worker can execute the request immediately +- **AND** startup/runtime direct-read fallback is disabled +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` +- **AND** the response MUST use `preview_worker_unavailable` +- **AND** it MUST NOT wait for the server bridge pending timeout + +#### Scenario: queued jobs drain when live worker capacity disappears +- **WHEN** jobs are queued behind active worker jobs +- **AND** all active workers time out or crash +- **AND** replacement workers cannot start +- **THEN** queued jobs MUST be rejected with `preview_worker_unavailable` +- **AND** they MUST NOT remain pending until daemon shutdown or bridge timeout + +#### Scenario: bare NODE_ENV test does not enable direct worker path +- **WHEN** the daemon runs with `NODE_ENV=test` +- **AND** Vitest-specific environment signals are absent +- **THEN** the default coordinator MUST use the real worker pool path +- **AND** it MUST NOT enable the in-process direct worker test shim + +#### Scenario: active worker job watchdog releases the slot +- **WHEN** an active preview worker job exceeds the daemon preview deadline +- **THEN** the daemon MUST send or preserve exactly one terminal timeout response for affected requestIds +- **AND** the active worker slot MUST be terminated or otherwise made unavailable for stale completion +- **AND** a replacement worker MAY be started with bounded restart/backoff for future requests + +#### Scenario: runtime failure does not direct-read fallback +- **WHEN** a request fails because of worker timeout, crash, restart, stale read, or late completion +- **THEN** the daemon MUST NOT synchronously fallback to direct read for that request +- **AND** urgent daemon command handling MUST remain independent of the failed preview read + +#### Scenario: worker crash completes pending requests +- **WHEN** a preview worker exits or throws while jobs are pending +- **THEN** the daemon MUST send exactly one terminal `fs.read_response` to every affected external requestId +- **AND** each response MUST use `preview_worker_crashed` +- **AND** the daemon MUST restart the worker with bounded backoff for future requests + +#### Scenario: graceful shutdown drains pending requests +- **WHEN** the daemon begins graceful shutdown while preview-read requests are active or queued +- **THEN** the coordinator MUST attempt to send `preview_worker_unavailable` terminal responses within a bounded shutdown budget +- **AND** it MUST NOT wait for slow preview reads to finish normally + +### Requirement: Coordinator SHALL suppress stale and ghost completions +The coordinator SHALL ensure stale worker completions cannot create duplicate responses, update active cache incorrectly, or route to a newer request. + +#### Scenario: each routable request receives at most one terminal response +- **WHEN** a routable external requestId enters the coordinator +- **THEN** the coordinator MUST schedule a deadline-bounded terminal response +- **AND** it MUST NOT send more than one terminal `fs.read_response` for that requestId + +#### Scenario: late completion after timeout is ignored +- **WHEN** a worker completes a job after the coordinator has timed out attached requests +- **THEN** the daemon MUST NOT send a second response to timed-out requestIds +- **AND** it MUST NOT write the late result into active cache + +#### Scenario: terminal preview records are not retained indefinitely +- **WHEN** a preview request reaches success, error, timeout, shutdown, or cancellation terminal state +- **THEN** fan-out and external request records for that requestId MUST be removed from the active coordinator maps +- **AND** late worker completions for that requestId MUST be ignored without re-creating terminal state + +#### Scenario: worker identity prevents restart misrouting +- **WHEN** a worker restart occurs while a request is pending +- **THEN** any completion whose `workerSlotId` or `workerGeneration` does not match the active pending job MUST be ignored +- **AND** it MUST NOT be routed to a newer request with the same `workerRequestId` + +#### Scenario: daemon restart does not emit ghost responses +- **WHEN** the daemon restarts after losing pending `fs.read` state +- **THEN** it MUST NOT emit late `fs.read_response` messages for stale pre-restart requestIds + +### Requirement: Worker packaging SHALL work in dev and dist +The preview worker SHALL use a plain `.mjs` bootstrap entry that works under both dev/tsx and compiled `dist/` execution. + +#### Scenario: postbuild output contains worker bootstrap +- **WHEN** `npm run build` completes +- **THEN** `dist/src/daemon/file-preview-read-worker-bootstrap.mjs` MUST exist +- **AND** the compiled worker implementation MUST exist in the expected `dist/src/daemon/` location + +#### Scenario: dist worker-pool smoke succeeds +- **WHEN** the dist worker-pool smoke test runs +- **THEN** it MUST start the default worker count +- **AND** it MUST dispatch at least two representative concurrent worker jobs +- **AND** the smoke test MUST run in CI after build rather than being silently skipped +- **AND** a required smoke mode MUST fail if dist artifacts are missing instead of skipping the suite + +### Requirement: Worker lifecycle SHALL be observable and bounded +The coordinator SHALL expose sufficient diagnostics for worker lifecycle and memory-risk investigation. Job-count recycle is recommended but not required as a v1 blocker. + +#### Scenario: worker lifecycle is logged without raw paths +- **WHEN** a worker starts, crashes, restarts, times out a job, is recycled, or terminates during shutdown +- **THEN** the coordinator MUST emit a structured log or metric with stable event labels +- **AND** that diagnostic MUST NOT include raw paths, errno text, or stack traces from preview jobs + +#### Scenario: optional job-count recycle does not disrupt other workers +- **WHEN** job-count recycle is implemented and a worker completes its configured recycle-count job +- **THEN** the coordinator SHOULD terminate that worker after the response is sent +- **AND** it SHOULD spawn a replacement before the next dispatch to that slot +- **AND** in-flight jobs on other workers MUST complete normally + +### Requirement: Pod-sticky compatibility SHALL be preserved +This change SHALL NOT introduce any daemon-dependent endpoint that bypasses server-id routed pod-sticky paths. + +#### Scenario: no new endpoint is required +- **WHEN** the preview-read worker is implemented +- **THEN** browser requests and daemon responses MUST continue to use the existing server-id routed WebSocket bridge +- **AND** the system MUST NOT add a worker health, queue, or preview read endpoint in v1 + +#### Scenario: existing download paths remain server-id routed +- **WHEN** a browser downloads via a `downloadId` produced by worker-backed `fs.read` +- **THEN** it MUST use the existing server-id routed download path +- **AND** the worker MUST NOT create cross-pod or server-side shared state + +### Requirement: fs.write errors SHALL use sanitized generic error codes +The daemon SHALL NOT include raw `Error.message`, stack traces, errno text, or absolute host paths in `fs.write_response.error`. `fs.write` SHALL use generic shared filesystem codes for `forbidden_path`, `file_too_large`, `invalid_request`, and unexpected `internal_error`, plus existing write-specific codes such as `file_exists` and `parent_not_found`. + +#### Scenario: unexpected fs.write error is sanitized +- **WHEN** `fs.write` fails with an unhandled internal filesystem error +- **THEN** the daemon MUST send `fs.write_response` with `status: "error"` +- **AND** `error` MUST be `internal_error` +- **AND** the response MUST NOT include raw host paths, errno text, stack traces, or raw `Error.message` + +#### Scenario: new fs.write target symlink is rejected +- **WHEN** `fs.write` initially observes that the target does not exist +- **AND** the target appears as a symlink before creation +- **THEN** the daemon MUST fail closed without writing through that symlink +- **AND** the frontend-visible response MUST remain sanitized diff --git a/openspec/changes/daemon-file-preview-worker/specs/daemon-fs-cache/spec.md b/openspec/changes/daemon-file-preview-worker/specs/daemon-fs-cache/spec.md new file mode 100644 index 000000000..d5f5ff9cd --- /dev/null +++ b/openspec/changes/daemon-file-preview-worker/specs/daemon-fs-cache/spec.md @@ -0,0 +1,141 @@ +## ADDED Requirements + +### Requirement: Main daemon SHALL own worker-backed fs.read freshness state +The daemon SHALL preserve freshness-safe `fs.read` cache and inflight reuse semantics when preview execution moves behind workers. The main daemon coordinator SHALL own cache, inflight state, resource generations, external request records, fan-out, per-request deadlines, and invalidation state. Workers SHALL be stateless per job and SHALL NOT own durable fs-read cache or generations. + +#### Scenario: worker module does not hold cache maps +- **WHEN** the worker implementation is inspected +- **THEN** it MUST NOT import, mutate, or persist `fsReadCache`, `fsReadInflight`, or `fsReadGenerations` +- **AND** cache writes MUST occur only in the main coordinator after freshness validation + +#### Scenario: successful write invalidates worker-backed read state +- **WHEN** a successful `fs.write` or other daemon mutation invalidates a file path +- **THEN** the main coordinator MUST bump the affected resource generation +- **AND** it MUST invalidate any worker-backed cached `fs.read` snapshot for that path +- **AND** it MUST detach, mark stale, or prevent cache writeback for older inflight worker read work for that path + +#### Scenario: late completion after invalidation is not cached +- **WHEN** older worker-backed `fs.read` work completes after the relevant file generation has changed +- **THEN** the main coordinator MUST NOT write that stale result into active `fs.read` cache +- **AND** future `fs.read` requests for the file MUST observe the newer freshness state + +#### Scenario: late completion is not cached when no eligible request remains +- **WHEN** a worker snapshot completes after all attached requestIds have reached terminal state or exceeded their per-request deadlines +- **THEN** the main coordinator MUST NOT write the result into active `fs.read` cache +- **AND** future `fs.read` requests for the file MUST perform fresh worker-backed work or use only a previously valid cache entry + +### Requirement: Two-phase worker keying SHALL preserve canonical freshness reuse +The coordinator SHALL use worker preflight results to key snapshot work by canonical path, freshness signature, and resource generation. + +#### Scenario: identical canonical freshness reuses one snapshot job +- **WHEN** two `fs.read` requests target the same canonical file and the file's current freshness state has not changed +- **THEN** the main coordinator MUST attach both external requestIds to one current snapshot worker job +- **AND** both requesters MUST receive compatible `fs.read_response` results with their own external `requestId` + +#### Scenario: raw aliases attach after preflight +- **WHEN** two different raw paths canonicalize to the same real path and freshness signature +- **THEN** the coordinator MUST attach them to the same canonical snapshot job +- **AND** the worker MUST NOT run duplicate snapshot content reads for those aliases + +#### Scenario: changed freshness starts a new snapshot job +- **WHEN** a later `fs.read` request targets the same canonical file after its freshness state has changed +- **THEN** the main coordinator MUST NOT attach that request to an older worker-backed inflight snapshot +- **AND** it MUST start fresh worker-backed snapshot work for the newer freshness state + +#### Scenario: preflight failure does not poison canonical cache +- **WHEN** a preflight job fails because of policy, invalid path, or sanitized filesystem error +- **THEN** the coordinator MUST send terminal errors only to the attached external requestIds +- **AND** it MUST NOT write any snapshot cache entry for that raw path or unresolved canonical path + +### Requirement: Per-request metadata SHALL be preserved under fan-out +The coordinator SHALL keep external request metadata separate from canonical worker job metadata. + +#### Scenario: attached request stores original raw path +- **WHEN** an external request attaches to preflight or snapshot work +- **THEN** the coordinator MUST store that request's external `requestId`, original raw `path`, admission time, deadline, and terminal state + +#### Scenario: fan-out response keeps each raw path +- **WHEN** one canonical snapshot fans out to multiple external requestIds from different raw paths +- **THEN** each `fs.read_response.path` MUST equal that requester's original raw path +- **AND** each `fs.read_response.resolvedPath` MUST equal the canonical worker path + +#### Scenario: timed-out request is skipped without affecting active siblings +- **WHEN** one attached requestId times out before a shared snapshot is ready +- **THEN** that requestId MUST receive or already have received its terminal timeout response +- **AND** other attached requestIds whose deadlines have not expired MUST remain eligible for the shared snapshot + +### Requirement: Worker-backed reads SHALL verify start and end freshness +Worker-backed reads SHALL protect against files changing while a worker reads them. Snapshot results SHALL include both the freshness signature observed before reading and the freshness signature observed after reading. + +#### Scenario: unchanged start and end signatures can be cached +- **WHEN** a worker snapshot result has matching `startSignature` and `endSignature` +- **AND** the main coordinator generation still matches the generation associated with the snapshot job +- **THEN** the main coordinator MUST treat the result as eligible for active `fs.read` cache storage + +#### Scenario: changed signature returns stale_read instead of mixed success +- **WHEN** a worker snapshot result has different `startSignature` and `endSignature` +- **THEN** the main coordinator MUST NOT store the result in active `fs.read` cache +- **AND** it MUST NOT return a successful response that combines content from one file state with `mtime` from another file state +- **AND** v1 MUST return a terminal `fs.read_response` with the shared `stale_read` error code + +#### Scenario: fan-out accepted before invalidation is deterministic +- **WHEN** the coordinator accepts a worker snapshot for fan-out to currently active requestIds +- **AND** a later invalidation occurs while those responses are being serialized +- **THEN** already accepted active requestIds MUST receive the same accepted snapshot unless their own deadline has already expired +- **AND** invalidation MUST affect future requests and cache writeback decisions + +### Requirement: Worker-backed fan-out SHALL be memory bounded +The main coordinator SHALL avoid duplicating large preview payloads while reusing inflight worker work. Bounded queue length alone SHALL NOT be the only memory-control mechanism. + +#### Scenario: queue stores metadata only +- **WHEN** a worker job is queued or waiting +- **THEN** the coordinator queue MUST store request and job metadata only +- **AND** it MUST NOT store preview content or base64 payloads in queued entries + +#### Scenario: fan-out shares one snapshot object before serialization +- **WHEN** multiple external requestIds are attached to the same snapshot job +- **THEN** the coordinator MUST retain one accepted worker snapshot object for fan-out before WebSocket serialization +- **AND** it MUST NOT create one retained base64 payload copy per attached requester + +#### Scenario: attached requestIds are bounded +- **WHEN** identical-freshness requests continue to arrive for an already inflight snapshot job +- **THEN** the coordinator MUST enforce a configured per-job attached requestId cap or global pending external request cap +- **AND** requests exceeding that cap MUST receive exactly one terminal error without attaching to the job + +#### Scenario: per-request deadlines are independent under fan-out +- **WHEN** several external requestIds are attached to one snapshot job +- **THEN** each external requestId MUST keep its own admission-time deadline +- **AND** a requestId that times out MUST NOT receive a later success response from that job +- **AND** other still-active attached requestIds MUST remain eligible for the worker result if their deadlines have not expired + +#### Scenario: fan-out timers are independent of send order +- **WHEN** fan-out sends are serialized to bound memory +- **THEN** each external requestId's deadline timer MUST still fire independently +- **AND** the coordinator MUST NOT wait for the serialized send queue to reach a request before timing it out + +#### Scenario: fan-out sends avoid peak memory multiplication +- **WHEN** one worker snapshot fans out to multiple attached requestIds +- **THEN** the coordinator MUST avoid retaining one serialized response copy per requester +- **AND** implementation MUST serialize fan-out responses sequentially or prove equivalent peak-memory bounds + +#### Scenario: video avoids base64 payloads +- **WHEN** a worker-backed read classifies a supported video file +- **THEN** the result MUST remain stream-mode metadata +- **AND** it MUST NOT create a base64 payload for the video content + +### Requirement: Worker-backed cache behavior SHALL remain observable and testable +The worker-backed cache and fan-out behavior SHALL be testable with fake workers, fake clocks, deterministic freshness signatures, and deterministic admission inputs. + +#### Scenario: fake workers can saturate the pool +- **WHEN** tests use fake workers that block every active read slot in the configured pool +- **THEN** coordinator tests MUST prove additional different-freshness jobs queue, identical-freshness jobs attach, and non-`fs.read` daemon dispatch remains responsive + +#### Scenario: queue and deadline constants are validated together +- **WHEN** coordinator tests configure worker duration, queue size, and daemon deadline +- **THEN** tests MUST prove requests either complete or receive terminal errors before the bridge pending timeout budget is exceeded +- **AND** no request MUST remain pending only because it was queued behind active work + +#### Scenario: admission formula is deterministic +- **WHEN** tests inject `workersTarget`, `queueDepth`, `tEstimateMs`, `deadlineMs`, and `safetyMarginMs` +- **THEN** the admission decision MUST match the documented formula exactly +- **AND** boundary tests MUST cover both admit and `preview_worker_queue_full` outcomes diff --git a/openspec/changes/daemon-file-preview-worker/tasks.md b/openspec/changes/daemon-file-preview-worker/tasks.md new file mode 100644 index 000000000..f8b941f0b --- /dev/null +++ b/openspec/changes/daemon-file-preview-worker/tasks.md @@ -0,0 +1,163 @@ +## 0. OpenSpec Consistency Gate + +- [x] 0.1 Verify `proposal.md`, `design.md`, `specs/daemon-file-preview-worker/spec.md`, `specs/daemon-fs-cache/spec.md`, and `tasks.md` all describe the same v1 model: two-phase worker preflight/snapshot, static worker pool, hard max four workers, no auto-scale, no runtime policy hot reload +- [x] 0.2 Run `openspec validate daemon-file-preview-worker --strict` before implementation begins + +## 1. Protocol Constants And Compatibility + +- [x] 1.1 Add `shared/fs-read-error-codes.ts` with `FS_READ_ERROR_CODES`, `FsReadErrorCode`, `FS_READ_PREVIEW_REASONS`, `FsReadPreviewReason`, and type guards +- [x] 1.2 Preserve existing public wire values in shared constants: `binary_file`, `forbidden_path`, and `file_too_large` +- [x] 1.3 Add stable worker/control codes: `preview_worker_queue_full`, `preview_worker_timeout`, `preview_worker_unavailable`, `preview_worker_crashed`, `stale_read`, `invalid_request`, and `internal_error` +- [x] 1.4 Replace production hardcoded fs-read error and preview-reason literals in `src/daemon/command-handler.ts`, `web/src/components/FileBrowser.tsx`, and any server/web consumers with shared imports +- [x] 1.5 Add shared tests proving exported values are stable, type-safe, and preserve legacy wire strings +- [x] 1.6 Add a grep gate that scans production sources only (`src/`, `web/src/`, `server/src/`) and excludes `shared/fs-read-error-codes.ts`, `**/*.test.ts`, `**/test/**`, `**/__fixtures__/**`, and `openspec/**`; production code outside the shared module MUST NOT define fs-read wire error literals +- [x] 1.7 Add a contract test proving UI/server consumers handle unknown shared worker error codes as generic failures without treating them as success + +## 2. Extract Preview Policy And Classification Primitives + +- [x] 2.1 Add `src/daemon/file-preview-path-policy.ts` with strict/lenient canonical helper modes and a branded/opaque `ValidatedRealPath` or equivalent type +- [x] 2.2 Implement strict mode for worker-backed `fs.read` and handle creation: call `fs.realpath`, fail closed on rejection, never use fallback paths +- [x] 2.3 Implement lenient mode for `fs.ls includeMetadata`: call `fs.realpath` first and allow only documented Windows reparse fallback paths with `usedFallback: true` +- [x] 2.4 Ensure fallback paths cannot create download handles +- [x] 2.5 Preserve current broad user-permission filesystem policy: no allow-root model; deny sensitive home directories `.ssh`, `.gnupg`, and `.pki` +- [x] 2.6 Fix deny-list comparison to be case-insensitive on Windows; default macOS behavior SHOULD also be case-insensitive; Linux and other platforms remain case-sensitive +- [x] 2.7 Ensure policy helpers read `homedir()` at call time and do not cache `homedir()` or `process.env.HOME` at module load +- [x] 2.8 Add policy unit tests for allowed paths, denied paths, symlink into denied paths, Windows mixed-case `.SSH/.GnuPG/.PKI`, default macOS mixed-case deny behavior, Linux case-sensitive `.SSH`, strict realpath failure, lenient fallback, and per-call home directory freshness +- [x] 2.9 Add `src/daemon/file-preview-classifier.ts` for MIME classification, video stream-mode detection, office/image detection, size limit constants, binary detection helpers, and file-signature helpers +- [x] 2.10 Add classifier tests for text, binary, image, office, video, too-large, unknown extension, and signature helper behavior +- [x] 2.11 If non-`fs.read` callers are touched while extracting helpers, add focused regression tests proving their public error behavior remains unchanged except for local download error sanitization + +## 3. Download Handle And File Transfer Hardening + +- [x] 3.1 Update `src/daemon/file-transfer-handler.ts` so local file handles are created only from `ValidatedRealPath` or an equivalent validated canonical boundary +- [x] 3.2 Add `createProjectFileHandleFromValidatedPath` or equivalent throwing helper for already revalidated paths +- [x] 3.3 Add `tryCreateProjectFileHandle` or equivalent non-throwing helper for tolerant callers such as `fs.ls includeMetadata` +- [x] 3.4 Preserve existing short-lived path-handle semantics; document that `downloadId` does not represent an immutable content snapshot +- [x] 3.5 Sanitize all frontend-visible `file.download_error` responses for `source: "local"` handles to stable values such as `not_found`, `expired`, or `download_failed` +- [x] 3.6 Ensure raw paths, errno text, stack traces, and raw `Error.message` appear only in logs, not frontend-visible download errors +- [x] 3.7 Add tests proving denied canonical paths cannot register handles, fallback paths cannot register handles, direct denied handle creation fails without registry entry, too-large/binary validated paths keep downloadable handles, and local download errors are sanitized +- [x] 3.8 Add `fs.ls includeMetadata=true` regression tests proving allowed normal files still receive `downloadId` and denied/fallback entries omit `downloadId` without failing the full listing + +## 4. Worker IPC And Bootstrap + +- [x] 4.1 Add `src/daemon/file-preview-read-types.ts` with strict request/result unions for preflight and snapshot phases +- [x] 4.2 Include `phase`, `workerRequestId`, `workerSlotId`, and `workerGeneration` in worker envelopes and results +- [x] 4.3 Ensure worker IPC types explicitly forbid `serverLink`, browser sockets, external requestIds, download registry objects, `downloadId`, raw `Error.message`, stack traces, and errno details +- [x] 4.4 Do not include `policyVersion` in v1 IPC; runtime policy hot reload remains out of scope +- [x] 4.5 Add `src/daemon/file-preview-read-worker.ts` implementing stateless preflight and snapshot job handlers using the extracted policy/classifier helpers +- [x] 4.6 Preflight job MUST perform path expansion, strict canonical `realpath`, policy check, `stat`, signature, size/MIME/video/too-large classification, and return no content/downloadId +- [x] 4.7 Snapshot job MUST perform content read, binary detection, text/base64 preparation or stream metadata, and start/end signature reporting +- [x] 4.8 Prefer transferable `ArrayBuffer` or equivalent single-copy payload strategy for large content IPC; if not implemented in v1, document measured copy/memory behavior +- [x] 4.9 Add `src/daemon/file-preview-read-worker-bootstrap.mjs` using the same dev/dist bootstrap pattern as `src/daemon/jsonl-parse-worker-bootstrap.mjs` +- [x] 4.10 Verify `scripts/copy-worker-bootstraps.mjs` copies the new bootstrap into `dist/src/daemon/` after `npm run build` +- [x] 4.11 Ensure fake-worker fixtures live under `test/` or are excluded from production `.mjs` bootstrap copying +- [x] 4.12 Add worker tests covering preflight success/error, snapshot text/image/office/video/too-large/binary/stale, strict policy rejection, sanitized errors, and no forbidden IPC fields +- [x] 4.13 Add a dist worker-pool smoke test that runs after build, starts the default worker count, and completes at least two representative concurrent jobs + +## 5. Coordinator Submodules And State Model + +- [x] 5.1 Add `src/daemon/file-preview-read-pool.ts` with `WorkerPool` lifecycle, dispatch, slot identity, generation validation, restart/backoff, shutdown, and optional job-count recycle hooks +- [x] 5.2 Add `src/daemon/file-preview-read-admission.ts` with deterministic admission formula, queue cap, `workersTarget`, `tEstimateMs` rolling median, deadline, safety margin, and fake-clock/test overrides +- [x] 5.3 Add `src/daemon/file-preview-read-fanout.ts` with per-request timers, exactly-once terminal responses, sequential send or equivalent memory bounds, and timeout-before-send behavior +- [x] 5.4 Add `src/daemon/file-preview-read-cache-facade.ts` owning `fsReadCache`, `fsReadInflight`, `fsReadGenerations`, cache keys, invalidation, and cache writeback eligibility +- [x] 5.5 Add `src/daemon/file-preview-read-shutdown.ts` or equivalent drain controller for bounded graceful shutdown responses +- [x] 5.6 Compose the submodules in `src/daemon/file-preview-read-coordinator.ts`; submodules MUST NOT import each other directly, and only `WorkerPool` talks to worker threads +- [x] 5.7 Define `ExternalRequestRecord` with external requestId, rawPath, admittedAt, deadlineAt, terminal state, and attachment state +- [x] 5.8 Define preflight job records keyed by raw admission groups and snapshot job records keyed by `realPath::signature::resourceGeneration` +- [x] 5.9 Define worker slot state with slotId, generation, state (`idle`, `busy`, `restarting`, `dead`), current job, and job count +- [x] 5.10 Add tests for each submodule with fake clocks and fake collaborators before coordinator integration tests + +## 6. Two-Phase Coordinator Behavior + +- [x] 6.1 Implement request admission: missing requestId suppresses; invalid path with requestId returns exactly one `invalid_request`; valid requests receive an admission deadline immediately +- [x] 6.2 Implement preflight queueing/dispatch through `WorkerPool`; queued preflight time counts against the external request deadline +- [x] 6.3 Implement canonical snapshot key migration after preflight; raw aliases that canonicalize to identical freshness attach to one snapshot job +- [x] 6.4 Ensure each final public response uses the request's original raw `path` and the worker canonical `resolvedPath` +- [x] 6.5 Implement changed-freshness behavior: changed signature or resource generation starts a new snapshot job rather than attaching to old work +- [x] 6.6 Implement start/end signature validation; changed signatures return `stale_read` and never cache mixed results +- [x] 6.7 Implement generation-aware cache writeback and invalidation for successful `fs.write` or other daemon mutations +- [x] 6.8 Implement queue full, fan-out cap, deterministic admission fail-fast, timeout, unavailable, crashed, stale, invalid, and internal terminal responses using shared codes +- [x] 6.9 Implement worker restart generation suppression: stale `workerSlotId`/`workerGeneration` results are ignored and cannot route or update cache +- [x] 6.10 Implement shutdown drain that sends `preview_worker_unavailable` for active/queued requestIds within a bounded budget +- [x] 6.11 Add fake-worker/fake-clock integration tests for canonical aliases, same freshness fan-out, changed freshness, invalidation during preflight, invalidation during snapshot, queue cap, deterministic admission boundary, timeout from admission, worker crash, worker restart generation, late completion, fan-out cap, shutdown drain, exactly-once responses, and raw path preservation +- [x] 6.12 Add responsiveness tests proving at least one non-`fs.read` daemon dispatch path completes while all fake worker slots are blocked; document that filesystem throughput isolation is not guaranteed because workers share libuv + +## 7. Command Handler And Public Response Assembly + +- [x] 7.1 Update `handleFsRead` in `src/daemon/command-handler.ts` to validate request shape and delegate valid protocol-level `fs.read` requests to the coordinator +- [x] 7.2 Remove uncached filesystem I/O and preview classification from the main `handleFsRead` path; allow only validation, no-FS-I/O cache hits if exposed by the coordinator, deadline/queue orchestration, result revalidation, handle creation, and response assembly +- [x] 7.3 Add `src/daemon/file-preview-read-response.ts` or equivalent response assembler if needed to keep `command-handler.ts` small +- [x] 7.4 Assemble final public `fs.read_response` with current fields preserved for text, image, office, video stream-mode, too-large, binary, stale, invalid, and generic worker errors +- [x] 7.5 Add response assembly tests proving text success omits `encoding`, image/office include only `encoding: "base64"`, video stream-mode omits inline `content`, binary uses `binary_file` with `previewReason: "binary"`, too-large includes `downloadId`, stale uses `stale_read`, and invalid uses `invalid_request` +- [x] 7.6 Ensure late responses for timed-out external requestIds are ignored and do not update UI-visible state or active cache + +## 8. Startup Fallback And Runtime Failure Behavior + +- [x] 8.1 Document v1 production fallback-disabled mode; no production direct-loader module is enabled for rollout +- [x] 8.2 Keep the direct in-process worker path test-only and gated to Vitest-specific environment signals, reusing the same policy/classifier/worker helpers as the worker path +- [x] 8.3 Return stable worker terminal errors for disabled fallback and runtime failure paths instead of re-entering main-process direct reads +- [x] 8.4 Ensure runtime timeout, crash, restart, stale completion, and late completion never direct-read fallback for the affected request +- [x] 8.5 Add tests for timeout, crash, stale/late completion, shutdown unavailable, test-only worker behavior, and dist real-worker default coordinator behavior + +## 9. Worker Lifecycle Observability And Optional Recycle + +- [x] 9.1 Add structured logs and metrics/counters for worker startup, shutdown, unavailable, queue full, timeout, crash, restart, stale read, shutdown drain, optional recycle, and sanitized internal errors +- [x] 9.2 Ensure diagnostics do not include raw paths, errno text, stack traces, or raw worker exception messages from preview jobs +- [x] 9.3 If job-count recycle is implemented, default `WORKER_RECYCLE_JOB_COUNT` to 50 and recycle only after the current job response is settled +- [x] 9.4 Add tests proving optional recycle does not cancel or duplicate jobs on other workers and replacement workers get a new generation + +## 10. Web And Server No-Regression Coverage + +- [x] 10.1 Keep `web/src/ws-client.ts` `fsReadFile(path)` public signature and wire format unchanged +- [x] 10.2 Update FileBrowser tests to use shared fs-read constants and prove existing UI states still handle text, image, office, video stream-mode, too-large, binary, stale read, invalid request, and generic worker errors +- [x] 10.3 Add or update FileBrowser stale-cycle tests proving late `fs.read_response` messages after rapid file switching are ignored +- [x] 10.4 Add ChatView download-trigger coverage proving `ws.fsReadFile(path)` can still obtain a `downloadId` and call the existing server-id routed download path +- [x] 10.5 Keep server bridge routing unchanged unless implementation changes duplicate-requestId or timeout behavior; if changed, add `server/test/bridge.test.ts` coverage +- [x] 10.6 Add bridge-margin coverage proving daemon timeout responses, including queued/preflight/snapshot timeouts, arrive before the server bridge 20 second pending deletion +- [x] 10.7 Verify no new daemon-dependent frontend fetch/WebSocket path bypasses `/api/server/:serverId/...` + +## 11. Test Hygiene And Integration + +- [x] 11.1 If worker integration or e2e tests create temporary projects/cwds or tmux sessions, add `imcodes-test-preview-*` and `deck_test_preview_*` coverage to `shared/test-session-guard.ts` +- [x] 11.2 Add `test/shared/test-session-guard.test.ts` coverage for any new preview-worker test prefixes +- [x] 11.3 Add real-worker integration tests under `test/daemon/` using guarded temporary paths and cleaning all fixtures +- [x] 11.4 Add a CI hygiene assertion or documented validation that `~/.imcodes/sessions.json` contains no `deck_test_preview_*` entries after tests + +## 12. Validation And Rollout + +- [x] 12.1 Run `openspec validate daemon-file-preview-worker --strict` +- [x] 12.2 Run daemon focused tests for fs-read constants, policy, classifier, worker IPC, worker implementation, coordinator submodules, cache freshness, write invalidation, public response assembly, file-transfer handle hardening, startup fallback, shutdown drain, and lifecycle observability +- [x] 12.3 Run web FileBrowser and ChatView focused tests +- [x] 12.4 Run `npx tsc --noEmit` +- [x] 12.5 Run `npx tsc -p server/tsconfig.json --noEmit` +- [x] 12.6 Run `cd web && npx tsc --noEmit` +- [x] 12.7 Run `npm run build` and verify dist worker bootstrap and implementation artifacts exist +- [x] 12.8 Run the dist worker-pool smoke test in CI after build without skipping +- [x] 12.9 Document final `workersTarget`, hard max, queue cap, attached cap, deadline, safety margin, t-estimate seed, fallback mode, local-download sanitization behavior, and non-blocking future decisions around base64 caps, `fs.read_cancel`, worker recycle, and `UV_THREADPOOL_SIZE` + +## 13. Post-Audit Conformance Fixes + +- [x] 13.1 Update OpenSpec artifacts for late snapshot cache suppression, terminal record cleanup, startup fail-fast unavailable, active job watchdog, production shutdown drain, fs.ls lenient canonical reuse, and minimal fs.write generic-code sanitization +- [x] 13.2 Add `shared/fs-error-codes.ts` with generic fs error codes and make `FS_READ_ERROR_CODES` extend the generic set without changing public wire values +- [x] 13.3 Ensure `fs.write` uses generic fs error codes for invalid, too-large, forbidden, and unexpected internal errors, and never returns raw `Error.message` in catch-all failures +- [x] 13.4 Ensure `fs.ls includeMetadata=true` uses bounded lenient canonical fallback, ordinary `fs.ls` remains strict, broad Windows fallback logging is removed, and fallback paths are non-downloadable +- [x] 13.5 Delete terminal fan-out and coordinator external request records while preserving exactly-once terminal response behavior +- [x] 13.6 Prevent snapshot results from writing active cache when no attached requestId remains eligible +- [x] 13.7 Add pool active-job watchdog and fail-fast unavailable behavior when no worker slot can execute requests +- [x] 13.8 Connect daemon lifecycle shutdown to default preview-read coordinator drain before server link disconnect +- [x] 13.9 Fix test-only direct worker canonicalization so denied paths map through the same worker policy branch as real worker execution +- [x] 13.10 Expand focused tests for late snapshot cache suppression, terminal record cleanup, startup unavailable, active worker timeout/restart, fs.write sanitization, production shutdown hook ordering, and production-source static error-code gate + +## 14. Final Implementation Audit Closure + +- [x] 14.1 Remove production direct-loader fallback requirements from proposal/design/specs and document v1 fallback-disabled behavior +- [x] 14.2 Gate the in-process direct worker shim on Vitest-specific signals only; bare `NODE_ENV=test` must use the real worker pool +- [x] 14.3 Restrict `fs.ls` lenient canonical fallback to `includeMetadata=true`; ordinary listings must remain strict +- [x] 14.4 Fail closed for generic Windows `EPERM`/`UNKNOWN` realpath errors unless the message identifies reparse/junction/symlink-loop fallback evidence +- [x] 14.5 Propagate admission deadline to worker-pool scheduling as pool-local metadata without adding it to worker IPC +- [x] 14.6 Enforce remaining deadline budget before enqueue, before worker post, and in active watchdog timers for both preflight and snapshot jobs +- [x] 14.7 Drain queued worker jobs with unavailable when all live worker capacity disappears and replacement workers cannot start +- [x] 14.8 Add required dist smoke mode, package script, and CI job so missing dist artifacts fail instead of silently skipping +- [x] 14.9 Add real dist/daemon responsiveness smoke coverage showing non-preview commands remain visible while preview workers are delayed +- [x] 14.10 Add minimal `fs.write` new-target symlink hardening and sanitized regression coverage +- [x] 14.11 Run final strict OpenSpec validation, focused tests, build, dist smoke, and full test suite diff --git a/openspec/changes/memory-system-post-1-1-integration/.openspec.yaml b/openspec/changes/memory-system-post-1-1-integration/.openspec.yaml new file mode 100644 index 000000000..12e66c27b --- /dev/null +++ b/openspec/changes/memory-system-post-1-1-integration/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-04-30 diff --git a/openspec/changes/memory-system-post-1-1-integration/design.md b/openspec/changes/memory-system-post-1-1-integration/design.md new file mode 100644 index 000000000..e7e89468c --- /dev/null +++ b/openspec/changes/memory-system-post-1-1-integration/design.md @@ -0,0 +1,360 @@ +## Context + +`memory-system-1.1-foundations` established the memory pipeline baseline: durable archive/source provenance, tokenizer budgeting, bounded materialization, redaction, scope-aware read tools, SDK-native `/compact`, immediate daemon-receipt send ack, `/stop` plus approval/feedback priority, bounded fail-open recall/bootstrap, provider send-start watchdogs, and local materialization repair. + +Post-1.1 work in `docs/plan/mem1.1.md` is broad and interdependent. Quick search and citations depend on stable projection identity, scope filtering, render policy, and replay-safe citation identity. Preferences, markdown ingest, and skills depend on origin metadata, feature flags, telemetry, and startup-budget rules. Self-learning dedup must preserve scope and provenance. Authorization scope registry, namespace registry, and typed observations are schema work, but schema migration is not a deferral reason on dev. Every feature must preserve foundations send/stop/compact liveness. + +## Goals / Non-Goals + +**Goals:** + +- Make `memory-system-post-1-1-integration` the single implementation contract for post-1.1 memory waves. +- Land operational primitives before feature work: fingerprints, origins, namespace registry, typed observations, flags, telemetry, budgets, render policy, and repair/backoff/idempotency gates. +- Preserve existing scope semantics and promote scope extensions into a shared policy registry: `user_private`, `personal`, `project_shared`, `workspace_shared`, and `org_shared`. Session-tree membership is a context/namespace binding, not a separate authorization scope. Enterprise-wide shared standards use `org_shared`, not a new global namespace/scope. +- Preserve foundations liveness and safety invariants in every wave. +- Promote authorization-scope registry, cite-count ranking, namespace/observation storage, enterprise org-shared authored standards, and skill auto-creation into current Wave 1-5 scope with concrete migration/test requirements. +- Make every requirement traceable to tasks, code areas, and tests. +- Keep new behavior disabled/fail-closed until feature-specific acceptance passes. + +**Non-Goals:** + +- Do not create separate implicit changes for Phase 1.5/1.6/1.7/1.8/1.9/1.7-O. +- Do not make later Phase 2/3 candidates blockers for Wave 1-5 completion. +- Do not reintroduce daemon-side `/compact` interception. +- Codex SDK provider dispatch has a final injected-context hard cap: daemon-added system/preference/memory/skill/shared-context text is capped to **32,000 characters** by default (`IMCODES_CODEX_SDK_CONTEXT_MAX_CHARS`, clamped 4,000-128,000). The current user turn text is not truncated by this guard; oversized user-provided content remains the user's responsibility. +- Do not make ordinary send ack wait for memory lookup, skill load, MD ingest, classification, telemetry, relaunch, transport lock, bootstrap, recall, embedding, provider send-start, or provider settlement. +- Do not introduce ad hoc authorization strings, a parallel namespace-tier taxonomy, or a separate session-tree authorization scope outside `shared/memory-scope.ts`; every actual scope must have an explicit policy, migration, auth filter, UI/admin behavior, and tests. +- Do not emit or implement quick-search cache origins in this milestone; cache origins are reserved until a future change defines TTL, invalidation, auth binding, and side-channel behavior. +- Do not run skill auto-creation/self-improvement in the ordinary send ack path and do not spawn a new foreground agent/session for it. Built-in skill content harvest, autonomous prefetch/LRU, and Hermes RL/model fine-tuning remain outside the current milestone. + +## Capability and Artifact Ownership + +- `proposal.md` defines why this is one change and where the completion boundary sits. +- `design.md` defines architecture, sequencing, defaults, migration/rollback, security, performance budgets, and plan mapping. +- `specs/daemon-memory-post-foundations/spec.md` defines runtime behavior for all current post-1.1 waves and hard foundations regression requirements. +- `tasks.md` defines executable work items with prerequisites, traceability, failure handling, tests, and acceptance gates. +- `specs/daemon-memory-pipeline/spec.md` is an archive-time migration target. Once `memory-system-1.1-foundations` is archived and `daemon-memory-pipeline` exists in cumulative OpenSpec specs, foundations-touching requirements from this change MUST move into that capability as `## MODIFIED Requirements` before this change is archived. This is artifact migration only; current runtime requirements remain binding here. + +## Wave Model + +1. **Wave 1 — Operational foundation and hardening gates.** Stable fingerprints, origin metadata, authorization scope policy registry, first-class namespace registry, multi-class observation store, feature flags, telemetry, startup budget, named-stage selection, typed render policy, sync semantics, and G1-G6 gates. +2. **Wave 2 — Self-learning memory.** Scope-bound classification/dedup/durable extraction and cold/warm/resumed startup-state selection. +3. **Wave 3 — Quick search, citations, and cite-count.** Authorized search, citation identity, drift indication, replay-safe cite-count, source lookup safety, ranking integration, and web integration. +4. **Wave 4 — MD ingest, preferences, and unified bootstrap.** Bounded notes ingest, user-only `@pref:` trust boundary, and unified startup context. +5. **Wave 5 — Enterprise authored standards and skills.** Enterprise org-shared authored standards, safe skill storage/import/render/admin foundations, layer precedence, project association, admin authorization, sanitization, packaging, safe rendering, and post-response skill auto-creation/self-improvement via the existing isolated compression/materialization path. + +Later candidates remain backlog notes only until promoted with requirements/tasks/tests. + +## Plan Mapping + +| Source plan area | Current disposition | Notes | +| --- | --- | --- | +| Phase 1.9 operational foundation | Included in Wave 1 | Fingerprints, origins, authorization scope registry, namespace registry, multi-class observation store, flags, telemetry, startup budgets, render policy, hardening gates. | +| Phase 1.5 self-learning | Included in Wave 2 | Uses existing isolated compression/materialization path; failures fail open for user delivery. | +| Phase 1.6 quick search + cite | Included in Wave 3 | Search, citation insertion, drift badge, same-shape unauthorized/missing lookup. | +| Phase 1.6 cite-count | Included in Wave 3 | Storage, increment triggers, replay/idempotency, ranking input, auth constraints, migrations, and tests are current scope. | +| Phase 1.6 autonomous prefetch / LRU | Deferred | Plan already marks no prefetch/no LRU for current wave. | +| Phase 1.7 MD ingest/preferences/bootstrap | Included in Wave 4 | No fs.watch; trusted triggers only; `@pref:` user-origin only. | +| Phase 1.8 skills storage/import/render/admin | Included in Wave 5 | Safe storage/import/render/admin foundations. | +| Phase 1.8 skill auto-creation/self-improvement | Included in Wave 5 | Runs only after response delivery through the existing isolated compression/materialization background path; it must not block send ack, provider delivery, `/stop`, feedback, or shutdown, and must not spawn a new foreground agent/session. | +| Built-in skill content harvest | Deferred | Wave 5 ships loader-ready empty manifest only. | +| Authorization scope extensions / namespace extensions / typed observations | Included in Wave 1 | Implement `shared/memory-scope.ts` scope policies, first-class namespace registry, and scope-bound `context_observations`/server equivalent. Current scope set is `user_private`, `personal`, `project_shared`, `workspace_shared`, and `org_shared`; session tree is represented by namespace/context binding (`root_session_id` / `session_tree_id`) rather than a new scope; no ad hoc scope strings outside the registry. | +| Enterprise-wide shared standards | Included in Wave 5 shared-context foundations | Use `org_shared` authored context bindings for enterprise-global coding standards/playbooks. Do not introduce `global`, `namespace_tier=global`, or unscoped cross-enterprise memory. | +| Drift recompaction / prompt caching / LLM redaction | Deferred | Deferred for behavioral/rollout complexity, not because of migration. Drift recompaction may be promoted after cite-count/drift signals are stable. | +| Quick-search result cache | Deferred | Deferred for cache safety semantics, not because of migration. No `quick_search_cache` origin may be emitted in this milestone because cache TTL/invalidation/auth semantics are not in scope. | +| Transport send stability | Included as cross-wave regression gate | Locks current dev ack/priority behavior. | + +## Cross-Wave Vocabularies and Shared Constants + +Implementation MUST add or reuse shared constants rather than duplicating literals. Expected shared files: + +- Project identity source of truth + - Durable project-scoped memory MUST key by canonical repository identity, not by device, cwd, session name, or local path. + - The canonical key is `canonicalRepoId` produced by the existing repository identity service from normalized git remote (`host/owner/repo`), with repository aliases for SSH/HTTPS equivalence and explicit migrations. + - Same signed-in user + same `canonicalRepoId` across laptop/desktop MUST resolve to the same project context for `personal` project-bound memory and enrolled shared project memory. + - `machine_id` is provenance/conflict metadata only; it MUST NOT be part of authorization or project identity when a canonical remote exists. + - Repositories without a usable remote may use local fallback identity, but that fallback is not cross-device project identity until the user enrolls/aliases it to a canonical remote. +- `shared/memory-scope.ts` + - `MEMORY_SCOPES = ['user_private', 'personal', 'project_shared', 'workspace_shared', 'org_shared'] as const` + - Defines per-scope policy: owner fields, required/forbidden identity fields, replication target, visibility predicate, search request expansion, promotion targets, and whether raw source access is allowed. + - Exports narrow subtypes such as `OwnerPrivateMemoryScope`, `ReplicableSharedProjectionScope`, `AuthoredContextScope`, and `SearchRequestScope` so enrollment/admin/authored-context APIs cannot accidentally accept private scopes. + - Defines request vocabulary: `owner_private`, `shared`, `all_authorized`, and a single explicit scope. Session-tree inclusion is represented by a separate context binding (`root_session_id` / `session_tree_id`) and must not be encoded as a scope. + - `user_private` is owner-only cross-project memory for preferences, user-level skills, persona/user facts, and private observations. Server sync, when enabled, MUST use a dedicated owner-private route/table guarded by `mem.feature.user_private_sync`; it MUST NOT reuse `shared_context_projections` or project/workspace/org membership filters. +- `shared/memory-origin.ts` + - `MEMORY_ORIGINS = ['chat_compacted', 'user_note', 'skill_import', 'manual_pin', 'agent_learned', 'md_ingest'] as const` + - `quick_search_cache` and other cache origins are reserved and MUST NOT be emitted in this milestone. + - New origin values require an OpenSpec delta and migration. +- `shared/send-origin.ts` + - `SEND_ORIGINS = ['user_keyboard', 'user_voice', 'user_resend', 'agent_output', 'tool_output', 'system_inject'] as const` + - Missing `session.send.origin` defaults to `system_inject`, which is untrusted for preference writes and may only preserve legacy send/ack compatibility. + - `TRUSTED_PREF_WRITE_ORIGINS = ['user_keyboard', 'user_voice', 'user_resend'] as const`. +- `shared/memory-fingerprint.ts` + - Canonical API: `computeMemoryFingerprint({ kind, content, scopeKey?, version?: 'v1' }): string`. + - `FingerprintKind = 'summary' | 'preference' | 'skill' | 'decision' | 'note'`. + - Legacy helpers must be deprecated or marked internal and must not be used by new call sites. +- `shared/memory-namespace.ts` + - Defines canonical namespace key constructors and binds namespace records to `MemoryScope` policies from `shared/memory-scope.ts`; it MUST NOT introduce parallel authorization tiers. + - For project-bound namespaces, `project_id` MUST be the canonical remote-backed `canonicalRepoId`; session tree ids are only optional binding/provenance within that project. +- `shared/memory-observation.ts` + - Defines `ObservationClass = 'fact' | 'decision' | 'bugfix' | 'feature' | 'refactor' | 'discovery' | 'preference' | 'skill_candidate' | 'workflow' | 'code_pattern' | 'note'` and typed observation payload validation. + - `note` is the canonical class for markdown/manual note durable content; do not introduce a parallel `memory_note` spelling. +- `shared/feature-flags.ts` + - Defines the memory feature flag registry listed below, including dependencies and disabled behavior. +- `shared/memory-counters.ts` + - Defines the closed telemetry counter enum and label constraints. +- `shared/skill-envelope.ts` + - `SKILL_ENVELOPE_OPEN = '<<>>'` + - `SKILL_ENVELOPE_CLOSE = '<<>>'` + - `SKILL_ENVELOPE_COLLISION_PATTERN = /<< persisted local/server config > environment startup default > registry default. Daemon-side management UI toggles persist local overrides in the daemon config store and therefore beat environment startup defaults without requiring a restart; enabling a flag through this operator surface also request-enables its dependency closure, while dependency evaluation still reports requested-vs-effective state so a child flag does not partially run while a parent is later disabled. Flag read failure fails closed for new features. Runtime disablement MUST stop new work within the documented propagation target. + +| Flag | Default | Runtime source | Dependencies | Observed by | Disabled behavior | +| --- | --- | --- | --- | --- | --- | +| `mem.feature.scope_registry_extensions` | `false` | local/server config + env startup default | none | daemon/server/web scope validators, namespace registry | legacy scopes remain accepted; new `user_private` writes fail closed except migration/backfill reads. | +| `mem.feature.user_private_sync` | `false` | local/server config + env startup default | scope registry extensions, namespace registry, observation store | daemon replication runner, server owner-private sync API/table, startup/search selection | `user_private` remains daemon-local owner-only; no owner-private server writes, replication jobs, or server reads are attempted. | +| `mem.feature.self_learning` | `false` | local daemon config + env startup default | namespace registry, observation store | materialization/compression pipeline | classification/dedup/durable extraction skipped; projection still commits without classification. | +| `mem.feature.namespace_registry` | `false` | local/server config + env startup default | none | daemon/server storage | no new namespace records outside migration/backfill; legacy projection reads remain available. | +| `mem.feature.observation_store` | `false` | local/server config + env startup default | namespace registry | daemon/server storage, materialization, preferences, skills | no new observation rows; projections remain readable. | +| `mem.feature.quick_search` | `false` | server config | namespace registry | web search UI, server/daemon search RPC | palette hidden; search endpoint returns same disabled envelope without search jobs. | +| `mem.feature.citation` | `false` | server config | quick search | web composer/citation RPC | citation UI hidden and RPC rejects with same disabled envelope; no citation rows. | +| `mem.feature.cite_count` | `false` | server config | citation | citation store, ranking/search | no new count increments; existing counts ignored in ranking without deleting data. | +| `mem.feature.cite_drift_badge` | `false` | server config | citation | web citation renderer | drift badge hidden; citation identity still preserved if citations are enabled. | +| `mem.feature.md_ingest` | `false` | local daemon config + env startup default | namespace registry, observation store | session bootstrap/MD ingest worker | no MD reads, parses, or ingest jobs. | +| `mem.feature.preferences` | `false` | local daemon config + env startup default | namespace registry, observation store | daemon send handler, preference store | `@pref:` lines pass through as text and are not persisted, stripped, or rendered into provider preference context. | +| `mem.feature.skills` | `false` | local/server config + env startup default | namespace registry, observation store | skill loader/render policy/admin API | loader returns empty set; render policy skips skills; admin writes rejected or disabled. | +| `mem.feature.skill_auto_creation` | `false` | local daemon config + env startup default | skills, self_learning | background skill review worker | no skill-review jobs claimed or created; existing skills still load if `mem.feature.skills` is enabled. | +| `mem.feature.org_shared_authored_standards` | `false` | server config + env startup default | scope registry extensions, shared-context document/version/binding migrations | server shared-context routes, authored-context resolver, web diagnostics | org-wide authored standard creation/binding is rejected with the documented disabled envelope; runtime selection skips org-wide bindings without blocking send ack or leaking inventory; project/workspace authored context remains governed by its existing controls. | + +In-flight work MAY finish only if it cannot corrupt state, block shutdown/upgrade, leak data, or violate authorization. Disabled-feature user-facing responses MUST preserve safe/same-shape envelopes where feature existence or object existence could otherwise leak. + +Enterprise authored standards are server shared-context control-plane objects, not daemon self-learning observations. They are still a post-1.1 Wave 5 feature and therefore have the explicit `mem.feature.org_shared_authored_standards` kill switch above. Disabling that flag MUST stop new org-wide authored-standard mutation/selection without disabling unrelated project/workspace bindings that already exist under the shared-context control plane. + +## Telemetry Registry + +Telemetry MUST be non-blocking, bounded, and type-safe. Counters MUST come from `shared/memory-counters.ts`. Initial counter set: + +- `mem.startup.silent_failure`, `mem.startup.budget_exceeded`, `mem.startup.stage_dropped` +- `mem.search.empty_results`, `mem.search.scope_filter_hit`, `mem.search.unauthorized_lookup`, `mem.search.disabled` +- `mem.citation.created`, `mem.citation.drift_observed`, `mem.citation.count_incremented`, `mem.citation.count_deduped`, `mem.citation.count_rejected`, `mem.citation.count_rate_limited` +- `mem.ingest.skipped_unsafe`, `mem.ingest.size_capped`, `mem.ingest.section_count_capped` +- `mem.skill.sanitize_rejected`, `mem.skill.collision_escaped`, `mem.skill.layer_conflict_resolved`, `mem.skill.review_throttled`, `mem.skill.review_deduped`, `mem.skill.review_failed` +- `mem.classify.failed`, `mem.classify.dedup_merge` +- `mem.preferences.untrusted_origin`, `mem.preferences.persisted`, `mem.preferences.duplicate_ignored`, `mem.preferences.rejected_untrusted` +- `mem.observation.duplicate_ignored`, `mem.observation.unauthorized_promotion_attempt`, `mem.observation.backfill_repaired` +- `mem.bridge.unrouted_response`, `mem.management.unauthorized` +- `mem.materialization.repair_triggered`, `mem.telemetry.buffer_overflow` + +Allowed label values are closed enums such as `MemoryOrigin`, `SendOrigin`, `MemoryFeatureFlag`, `FingerprintKind`, `ObservationClass`, and `SkillReviewTrigger`. Free-form session ids, project ids, user ids, file paths, raw text, and secrets are forbidden as metric labels. + +## Enterprise Shared Standards Model + +Enterprise-global sharing is represented by `org_shared`, not by a new `global` scope or namespace tier. There are two distinct enterprise sharing surfaces: + +1. **Authored standards / policies / playbooks** use the existing shared-context document model (`shared_context_documents`, `shared_context_document_versions`, `shared_context_document_bindings`). An org-wide binding has `enterprise_id` set, `workspace_id = NULL`, `enrollment_id = NULL`, and derived scope `org_shared`. It is visible only to members of that enterprise. Owner/admin roles may create, update, activate, deactivate, or delete versions/bindings; members may read only the bindings selected for their session. +2. **Processed project experience** uses `shared_context_projections` with scope `project_shared`, `workspace_shared`, or `org_shared`. Even when scope is `org_shared`, each projection MUST retain canonical `project_id` / `canonicalRepoId` as provenance and ranking input; org-shared processed memory is not an unowned global pool. + +`org_shared` authored context MAY include optional filters: `applicability_repo_id`, `applicability_language`, and `applicability_path_pattern`. Filters only narrow applicability inside the enterprise; they MUST NOT widen visibility outside the enterprise. `binding_mode = required` means the context must be preserved in the compiled payload or dispatch fails with the existing required-authored-context error. `binding_mode = advisory` may be dropped by budget/render policy with telemetry/diagnostics. + +Runtime selection order for authored standards is: project binding, workspace binding, then org binding, with required bindings preserved before advisory bindings. If multiple org-shared standards match, stable ordering MUST be deterministic by active version/binding metadata. User-visible diagnostics must distinguish org/workspace/project authored layers without leaking documents to non-members. + +## Storage and Schema Invariants + +The exact migration numbers are assigned at implementation time, but the following invariants are mandatory on both daemon SQLite and server PostgreSQL equivalents where applicable. + +### Authorization scope registry + +- Shared module: `shared/memory-scope.ts`. +- Required scopes: + - `user_private`: owner user across projects/workspaces, visible only to that user, suitable for preferences, user-level skills, persona/user facts, and user-private observations. When `mem.feature.user_private_sync=true`, it replicates through a dedicated owner-private sync route/table; when false it remains daemon-local. It MUST NOT be stored in or queried through shared projection membership filters. + - `personal`: legacy/project-bound private memory for the owner user and current project; remains supported for compatibility. + - `project_shared`: enterprise project members. + - `workspace_shared`: enterprise workspace members. + - `org_shared`: enterprise/team members only. Requires `enterprise_id`; `workspace_id` and enrollment-specific project binding are null for enterprise-wide authored standards. It is not public/global and never crosses enterprise boundaries. +- Every scope policy MUST define required identity fields, nullable fields, replication target, authorization predicate, allowed promotion targets, and search/default-selection behavior. +- Scope policy migration MUST replace hard-coded scope unions/predicates across daemon/server/web with shared constants or generated validators. + +### Namespace registry + +- Table/model: `context_namespaces`. +- Required fields: `id`, `tenant_id` or local daemon tenant marker, `scope`, `user_id`, `root_session_id`/`session_tree_id`, `session_id`, `workspace_id`, `project_id`, `org_id`, `key`, `visibility`, `created_at`, `updated_at`. Per-scope policy determines which identity fields are required, optional-for-provenance, or forbidden. For `personal`, `project_shared`, `workspace_shared`, and `org_shared`, `project_id` MUST be the canonical remote-backed `canonicalRepoId` when a remote exists so the same user's same project is visible across devices. `ContextNamespace.projectId` MUST NOT be globally required for `user_private`; session-tree context uses `root_session_id` / `session_tree_id` as binding metadata rather than a scope. +- `scope` MUST be one of `user_private`, `personal`, `project_shared`, `workspace_shared`, `org_shared` and must validate against the per-scope policy. +- `key` MUST be built only through `shared/memory-namespace.ts` canonical constructors. +- Unique constraint/index MUST prevent duplicate canonical namespace keys within the same tenant/scope context. +- Namespace migration MUST bind each legacy projection to exactly one namespace/scope policy and MUST NOT widen visibility. Legacy `personal` rows remain project-bound `personal` keyed by canonical project identity; same owner + same canonical remote across devices may see them when personal sync is enabled, but other projects/users may not. Automatic backfill MUST NOT reclassify them to `user_private`; any `personal` -> `user_private` movement requires explicit audited user/admin action. + +### Observation store + +- Table/model: `context_observations`. +- Required fields: `id`, `namespace_id`, `scope`, `class`, `origin`, `fingerprint`, `content_json`, `text_hash`, `source_event_ids_json`, `projection_id`, `state`, `confidence`, `created_at`, `updated_at`, `promoted_at`. +- `class` MUST use `ObservationClass` from `shared/memory-observation.ts`. +- `state` MUST be a closed enum such as `candidate`, `active`, `superseded`, `rejected`, `promoted`. +- Unique/index constraints MUST make same-scope duplicate writes idempotent by at least `namespace_id`, `class`, `fingerprint`, and `text_hash`. +- Observation writes must be transactional with projection aggregate updates or written through an outbox/repair path that can reconcile projection/observation mismatch. + +### Owner-private sync store + +- Server shared projections MUST accept only `personal`, `project_shared`, `workspace_shared`, and `org_shared` and MUST have a database CHECK/validator preventing `user_private` from entering that path. Rows MUST be keyed by canonical `project_id`/`canonicalRepoId`, not device-local paths. +- Session-tree context is not replicated as a separate authorization scope; it is carried only as namespace/context provenance where needed. +- `user_private` server sync, when `mem.feature.user_private_sync=true`, uses a dedicated owner-private table/route with owner-user authorization predicates, same-shape disabled/unauthorized envelopes, idempotency keys, retention/repair, and tests for cross-project owner visibility and non-owner denial. +- If the sync flag is off or server sync is unavailable, `user_private` remains daemon-local and user delivery/startup MUST fail open without blocking ordinary send ack. + +### Citation and idempotency store + +- Citation rows MUST store projection id, namespace/scope, created_at, authoritative citing message identity, idempotency key, and actor/caller context needed for authorization auditing. +- Citation idempotency keys MUST be derived by the authoritative daemon/server store and MUST NOT be accepted from untrusted clients. +- If stable citing message identity exists, use `sha256("cite:v1:" + scope_namespace + ":" + projection_id + ":" + citing_message_id)`. +- If stable citing message identity is not available, implementation MUST first add it or block cite-count work until the identity property is satisfied. +- Idempotency rows are retained for at least `citationIdempotencyRetentionDays`; pruning must not allow normal retry/replay windows to inflate counts. +- Cite-count may be stored directly on projection rows or in an auxiliary counter table, but ranking must consume a bounded normalized signal after scope filtering. + +### Promotion audit + +- Table/model: `observation_promotion_audit`. +- Required fields: `id`, `observation_id`, `actor_id`, `action`, `from_scope`, `to_scope`, `reason`, `created_at`. +- Allowed promotion actions in this milestone: web UI Promote, CLI `imcodes mem promote`, admin API `POST /api/v1/mem/promote`. +- Background workers MUST NOT promote observations across scopes without one of those authorized actions. + +## Data Flow and Interfaces + +- Memory writes flow through projection APIs that attach `origin`, `summary_fingerprint` or kind-specific fingerprint, namespace/scope, source ids, observation class where applicable, and render kind. Projections may remain the render/search aggregate, but durable facts/decisions/preferences/skill candidates/notes MUST also have typed observation rows when `mem.feature.observation_store` is enabled. +- Startup context flow is `collect -> prioritize -> apply quotas -> trim to total budget -> dedup -> render`. Each stage is independently testable and may fail open by dropping that source with telemetry. +- Search/citation flow is `authorized caller -> shared scope filter -> ranked projection results -> render-policy-safe preview -> citation token -> authoritative cite idempotency key -> authorized same-shape source lookup`. +- MD/preferences flow is `trusted trigger -> bounded parser -> scope validation/fail-closed -> origin/fingerprint/provenance fingerprint -> projection-backed idempotent write -> linked observation -> startup/search selection`. Markdown sections classified as `preference` remain markdown-derived project/user memory and do not become trusted owner-private `@pref:` preferences unless a later explicit audited promotion path is added. Filesystem markdown must not silently downgrade `user_private`, workspace, or org namespaces into project scope; unsupported scopes are dropped with telemetry, while authorized workspace/org standards use authored-context bindings. +- Observation flow is `source event/projection -> classify -> typed observation row -> projection aggregate/update -> search/startup render`. Observation rows carry class, content JSON, source event ids, projection id, namespace id, scope, origin, and fingerprint. +- `@pref:` flow is `session.send(origin) -> trusted-origin check -> leading-line parser -> idempotent preference write + preference observation scheduled asynchronously -> strip trusted raw command lines from user-visible/provider-bound user text -> render same-turn preference records plus active persisted preferences through the shared preference render policy -> provider dispatch with a bounded session-level preference context preamble + remaining user text; the same rendered preference block MUST NOT be injected on every later turn, and MUST be re-sent only when the block changes or after SDK/provider compaction may have discarded prior context`. Ack remains daemon receipt and does not wait for preference persistence, preference lookup, bootstrap, recall, locks, relaunch, or provider send-start. +- Authored standards flow is `admin/owner writes document/version -> org/workspace/project binding -> member session resolves matching bindings by canonicalRepoId/language/path -> required/advisory render policy -> provider dispatch`; org-wide standards are `org_shared` bindings with enterprise-only visibility. +- Skills flow is `import/install/review/admin-sync -> lightweight skill registry/manifest -> precedence/enforcement resolution -> optional provider-visible registry hint -> on-demand resolver reads only selected skill bodies when relevant`. Ordinary startup/send must not scan or read the full skill corpus. Explicit full-body rendering must pass through the render-policy-safe skill envelope. Skill auto-creation/update is `completed non-hidden non-error tool-result evidence or manual review -> response delivered -> background compression/materialization review -> daemon-local production worker -> create/update deterministic user-level skill -> upsert registry -> repair/backoff/idempotency`, never ordinary send ack work. +- Telemetry flow is hot-path enqueue into a bounded async buffer; sink failure never changes user-visible memory behavior. + +## Citation Ranking and Drift Model + +- Citation insertion is by projection identity, not raw source snapshot. +- Each insertion creates a citation row with its own `created_at` and authoritative idempotency key. +- Same citing message retry/replay dedupes; a different citing message citing the same authorized projection increments cite-count once for that different message. +- Unauthorized or missing citation attempts must return the same user-facing envelope and must not increment or reveal counts. +- Cite-count ranking is enabled only when `mem.feature.cite_count=true`, after scope filtering, and as a bounded additive signal that does not replace existing semantic score or `hitCount` behavior. +- Drift detection MUST use a canonical persistent `content_hash` computed from normalized projection content. Daemon SQLite and server PostgreSQL projection write paths MUST persist this marker for content-changing writes; citation rows capture it at cite time. Routine maintenance/idempotent upserts that do not change normalized projection content MUST NOT change `content_hash` or create false drift. + +## Skill Auto-Creation Model + +Skill auto-creation/self-improvement is background memory work, not send work. + +- Closed triggers: `tool_iteration_count` and `manual_review` only. +- `tool_iteration_count` trigger fires only after a completed user turn when completed, visible, non-error tool-result evidence reaches `skillReviewToolIterationThreshold`; hidden raw tool events, failed tool results, and below-threshold evidence are filtered or marked not-eligible outside the ordinary send ack/provider-delivery path. The threshold is reset only after a review job is accepted. +- `manual_review` trigger requires an explicit user/admin action. +- The worker MUST coalesce duplicate pending reviews per user/workspace/project/session scope. +- The worker MUST enforce per-scope concurrency, min-interval, daily caps, retry/backoff, idempotency, and cancellation on shutdown/disable. +- The worker MUST prefer updating an existing matching user-level skill before creating a new user-level skill. +- The worker MUST never create a project/workspace/org shared skill without the explicit admin paths in the promotion/admin model. + +## Capacity and Performance Budgets + +Current defaults are authoritative for shipped behavior until changed by a future OpenSpec delta and mirrored in `shared/memory-defaults.ts`. + +```json5 +// design-defaults +{ + startupTotalTokens: 8000, + pinnedTokens: 1600, + durableTokens: 4000, + recentTokens: 2400, + skillTokens: 1000, + projectDocsTokens: 2000, + markdownMaxBytes: 51200, + markdownMaxSections: 30, + markdownMaxSectionBytes: 16384, + markdownParserBudgetMs: 5000, + skillMaxBytes: 4096, + featureFlagPropagationP99Ms: 60000, + skillReviewToolIterationThreshold: 10, + skillReviewMinIntervalMs: 600000, + skillReviewDailyLimit: 6, + skillReviewManualMinIntervalMs: 60000, + skillReviewManualDailyLimit: 50, + skillRegistryMaxBytes: 1048576, + skillRegistryMaxEntries: 1024, + citationIdempotencyRetentionDays: 180, + preferenceIdempotencyRetentionDays: 180 +} +``` + +Trim priority defaults to `recent`, then `project_docs`, then `durable`; pinned content has highest preservation priority. MD ingest has no `fs.watch` in this milestone and is wired as bounded bootstrap/manual-sync background work, but completed schedules must release their in-flight key so later session starts/manual sync can re-read changed files. Quick search, citation preview, skill load, MD ingest, classification, skill review, and telemetry must not delay ordinary send ack. + +## Post-1.1 Management UI + +The shared-context management panel is also the operator surface for local post-1.1 daemon memory features. It must not require users to edit SQLite rows or skill registry files by hand. The minimum UI/API contract is: + +- **Feature status:** query daemon-resolved post-1.1 memory feature flags and show enabled/disabled/unknown state before exposing mutation actions. The same panel also sends shared `memory.features.set` requests so operators can enable/disable daemon-managed memory flags from the UI; the daemon requires server-derived/local-daemon management context, persists the requested value, cascades enable requests to dependencies, recomputes effective state with dependencies, returns source/dependency metadata, and rejects invalid or failed writes with shared error codes. Requested-on/effective-off states render as a distinct dependency-blocked warning instead of looking like an ordinary disabled flag. Disabled features may still show existing local records for inspection, but management writes/mutations/read-body actions MUST fail closed with shared error codes and localized web messages. +- **Project selector and memory index:** the Memory tab MUST default browsing to **All projects** and MUST NOT auto-select the current/local-tool project as a browse filter. The shared project picker is sourced from active/recent daemon sessions, enterprise enrolled canonical project identities, and `projects` indexes returned by local daemon, personal cloud, enterprise/shared, and semantic memory views. Each index entry carries canonical project id plus record counters and last-updated metadata so projects with memory remain selectable even when no current session exposes a local directory. The picker shows both canonical `canonicalRepoId` and local `projectDir` when known, searches name/id/directory, keeps canonical-only options usable for memory filtering, and routes directory-only entries through a daemon resolver before local filesystem tools run. Raw project id/path fields are advanced fallback/debug controls only and are not the primary UX. +- **Protocol routing and trust:** memory-management WebSocket requests use a closed request/response type set from `shared/memory-ws.ts`, MUST carry a unique `requestId`, and daemon responses MUST be single-cast back only to the pending browser socket for that `requestId`; unrouted or duplicate-pending responses are dropped and counted, never broadcast. The server bridge injects a server-derived management context (`actorId`, `userId`, role, requestId, and bound project hints). The role is derived from server-side membership data (`team_members` reached directly by `enterpriseId`/`orgId`, or through `shared_context_workspaces` / `shared_project_enrollments` when only workspace/project hints are present); browser-supplied role fields are ignored. Browser project/workspace/org fields are request hints only: they MUST NOT enter `boundProjects` unless the server verifies membership/enrollment for that exact canonical repo, workspace, or org. Daemon handlers ignore client-supplied owner/actor identity for preference, observation, and processed-memory mutations; client identity fields are display/input hints only and are never authorization inputs. Record-level `ownerUserId` / `createdByUserId` / `updatedByUserId` metadata is server/daemon-derived at create/update time and is distinct from management role: private records remain owner-only; shared records may be mutated by an authorized admin or by the record creator/owner when the namespace is otherwise visible. Legacy/display fields such as `userId`, `createdBy`, `authorUserId`, and `updatedBy` MAY be shown for old records, but MUST NOT grant mutation authority. Admin actions MUST preserve the original creator metadata and only update `updatedByUserId` / audit metadata. +- **Preferences:** query active `@pref` observations for the server-derived current user, create and update trusted explicit user-scoped preferences for that same current user, store creator/owner metadata derived from the authenticated actor, and delete only preferences owned by that user unless a future admin context explicitly authorizes otherwise. The UI uses daemon WebSocket message constants from `shared/memory-ws.ts`; user-visible labels and management errors live in all web locales. Preference create/update/delete is blocked when `mem.feature.preferences=false`, and every mutation invalidates provider-visible preference context so stale preferences are not reused. +- **Skills:** query the maintained skill registry/manifest, rebuild it only on an explicit operator action, preview one selected skill body on demand, and delete managed user/project skill files with path-root checks. Startup and ordinary sends still see only registry hints and never scan/read every skill body. Preview MUST reject non-file/symlink registry entries, and management registry writes MUST invalidate runtime registry cache. Rebuild/preview/delete are blocked when `mem.feature.skills=false` or the selected project lacks a validated `{ projectDir, canonicalRepoId }` pair. +- **Markdown ingest:** run a bounded manual ingest only when the selected project has a validated project directory and canonical project identity. The daemon must reject invalid project directories and canonical project identity mismatches before reading project files. Unsupported `user_private`/workspace/org filesystem scope continues to fail closed and the UI exposes only supported manual-ingest scopes (`personal`, `project_shared`). The UI surfaces files-checked and observations-written counters. Run is blocked when `mem.feature.md_ingest=false`. +- **Processed local memory:** local processed memory records are manageable, not read-only: the UI can manually add a project-bound personal memory, edit an existing visible record, archive/restore/delete it, and pin it into the pinned-note store. The daemon must authorize create/update/pin/delete/archive/restore from the server-derived management context, require explicit canonical project identity plus an authorized bound project for manual create, update linked projection/observation rows transactionally, delete linked observations when a processed projection is permanently deleted, clear stale embeddings on edits, and invalidate runtime memory caches with a projection-typed event after successful projection mutations. Manual create/edit stores `ownerUserId`, `createdByUserId`, and `updatedByUserId` in record content metadata; management lists display these fields so creator ownership is not confused with enterprise admin role. Pinning uses origin `manual_pin` and must be idempotent for the same projection id so repeated clicks do not create unbounded duplicates. All processed-memory management mutations are governed by `mem.feature.observation_store`; when it is effectively disabled, create/update/archive/restore/delete/pin fail closed with shared error codes and do not touch projection, observation, pinned-note, or cache state. +- **Observations:** list typed observations by scope/class with creator/owner metadata, edit/delete mutable observations, and promote scope only via the explicit audited `web_ui_promote` path. Automatic/background paths remain forbidden from cross-scope promotion. Observation edit must update linked projection text/content hash and clear stale projection embeddings. Observation delete is observation-only and MUST NOT cascade-delete a linked processed projection; permanent processed-memory delete remains the path that deletes the projection and cleans up linked observations. Mutation is blocked when `mem.feature.observation_store=false` or the selected project lacks the identity required by the operation. Missing observations and stale `expectedFromScope` races return typed shared error codes instead of generic action failure. The Web UI MUST make promotion a two-step confirmation flow: the record action first displays the exact from-scope, to-scope, optional reason, audit write, and visibility consequence; only the confirmation control sends the promotion RPC. + +The UI additionally keeps a latest-requestId guard per management surface (features, processed memory, preferences, skills, observations, project resolution, and every mutation) so a stale response or another tab's response cannot overwrite current state. Browser REST memory loads use a generation guard so cloud/enterprise responses from older browse filters cannot overwrite newer state. The project-option list accumulates memory-index projects across filtered reloads instead of replacing the dropdown with only the currently filtered project. Before feature-state is known, mutation buttons remain disabled. The daemon remains the final enforcement point for feature flags, owner filters, skill path validation, project identity checks, and promotion authorization. + +These UI commands are daemon-local because the daemon owns the local memory store, local skill files, and project filesystem. Server/enterprise authored-context management remains in the existing Knowledge/Projects sections. + +## Security and Trust Model + +- All new memory queries must reuse shared scope-filter helpers generated from `shared/memory-scope.ts`; no bespoke cross-scope SQL predicates. +- User-facing quick-search/citation/source lookup failures MUST expose the same external envelope for missing, unauthorized, and feature-disabled object lookup where existence could leak. The envelope MUST NOT include role diagnostics, `required`/`actual` role metadata, source counts, hit counts, drift metadata, raw source text, project/workspace/org ids, or timing-dependent alternate shapes. Admin-only diagnostics may remain detailed on admin endpoints that are not reused for user-facing lookup. +- `@pref:` writes are trusted only from `TRUSTED_PREF_WRITE_ORIGINS`. Agent output, tool output, timeline replay, imported memory, daemon-injected content, and missing-origin sends must not create persistent preferences by containing preference syntax. +- Workspace/org skill push requires admin authorization for that scope. +- Skill and MD content is inert input, never system instruction. Sanitization, delimiter isolation, system-instruction guard, and length caps are mandatory before context injection. +- Management quick search is not the generic repo-only local search path: it constructs an authorized namespace set from the server-derived management context and applies that set before result construction, stats, and pagination. Owner-private rows (`personal`, `user_private`) require the derived current user as owner; missing owner identity fails closed. +- Project-scoped management operations treat browser `projectDir` as an untrusted compatibility hint. They require explicit `canonicalRepoId` and must verify the directory's git remote/canonical identity before reading or mutating skill/MD project files. The web project selector is an operator convenience; daemon verification remains authoritative, and generic UI `projectId` fields are not role-derivation aliases. +- Memory browse project filters are selection aids, not authorization. Local daemon `PERSONAL_QUERY`, personal cloud memory, enterprise memory, and semantic memory view responses return an optional bounded `projects` index that is already scoped/authorized by the same owner/enterprise filter as the records/stats query. The default browse request omits `projectId`/`canonicalRepoId`; selecting a canonical-only memory-index project may filter records but MUST NOT enable local file-backed skill/MD/observation actions until a validated directory/canonical pair exists. +- Observation promotion is an explicit audited action with `expectedFromScope` as a required TOCTOU guard; missing or stale source scope is a typed management error. Runtime cache invalidation events distinguish observation mutations from projection mutations so future consumers do not have to interpret projection ids as observation ids. +- Web-visible failure states must use i18n (`t()`) across `en`, `zh-CN`, `zh-TW`, `es`, `ru`, `ja`, and `ko`. Protocol/type/status strings shared across daemon/server/web must be shared constants. + +## Skill Model + +Ordinary layer precedence, highest to lowest: + +1. `/.imc/skills/` project escape hatch. +2. User-level skills under `~/.imcodes/skills/` that match current project metadata. +3. User-level default skills under `~/.imcodes/skills/`. +4. Workspace-shared mirrored skills. +5. Org-shared mirrored skills. +6. Built-in fallback from `dist/builtin-skills/manifest.json` (empty in Wave 5). + +Built-in fallback is always lowest precedence, is always considered only after higher layers, and MUST NOT override user-authored, project, workspace, org, or explicitly selected skills. Enforcement is a separate axis. Workspace/org skills with `enforcement: 'enforced'` are always selected and override or hide same-name lower-layer skills according to documented conflict rules. Workspace/org skills with `enforcement: 'additive'` do not shadow project/user skills; they coexist and must show loaded-layer diagnostics. Wave 5 implements safe storage/import/render/admin foundations, the empty built-in loader, and post-response skill auto-creation/self-improvement through the existing isolated compression/materialization background path. Runtime startup dispatch exposes at most a bounded skill registry hint (key/layer/safe descriptor/redacted path or `skill://` URI) sourced from a maintained registry, not by scanning/reading every `SKILL.md`; full-body rendering remains available only through explicit on-demand resolver paths using the skill envelope sanitizer. Auto-creation always writes user-level skill candidates or updates existing user skills; it must not run in the send ack path or create a new foreground agent/session. The automatic `tool_iteration_count` path requires real completed, visible, non-error tool-result evidence meeting `skillReviewToolIterationThreshold`; `manual_review` may bypass that threshold. Runtime dispatch must have an actual production loader for project/user skill references; shared selection/render helpers alone are not sufficient acceptance evidence. + +## Migration and Rollback Plan + +- Schema changes are additive but Wave 1-5 are expected to introduce real migrations in dev. Migration/backfill work is explicitly in scope and MUST NOT be used as the reason to defer a post-1.1 requirement. +- Migration filenames MUST use the next available number after the current repository head at implementation time; stale plan numbers are non-authoritative. +- Fingerprint/origin columns, scope registry fields, namespace registry tables, typed observation tables, citation/idempotency tables, cite-count storage, promotion audit tables, and preference idempotency support start nullable or safely defaulted where needed and are lazily backfilled. +- Eager backfill, if implemented, must be an explicit CLI/admin action using bounded restartable batches. +- Rollback path is feature-flag disablement, returning to pre-feature behavior without deleting stored data. +- Destructive rollback is out of scope unless a later task explicitly designs it. +- New background workers must define stale in-progress recovery, bounded retry/backoff, idempotent reprocessing, and retention/pruning behavior. Scope and observation migrations must preserve existing projections, must not widen visibility automatically, and must not cross-promote scopes automatically. +- Acceptance scripts must validate this change id directly; validating only `memory-system-1.1-foundations` is insufficient for post-1.1 readiness. + +## Risks / Trade-offs + +- **Large change surface** -> ordered waves, finite milestone, feature flags, and per-wave gates. +- **OpenSpec capability timing** -> hold foundations deltas here until `daemon-memory-pipeline` exists, then migrate before archive. +- **Ack/stop regression** -> foundations regression matrix mandatory for every wave. +- **Scope leak / side channel** -> shared scope filters plus identical user-facing missing/unauthorized/disabled envelopes. +- **Citation replay inflation** -> authoritative idempotency key, stable citing message identity requirement, retention, and replay tests. +- **Hot-row cite-count contention** -> bounded ranking signal and option for auxiliary counters/rollups if direct projection updates become contentious. +- **Prompt injection via skills/MD/preferences** -> trust markers, line stripping, fail-closed sanitizer, delimiter collision tests, and render-policy layer. +- **Migration drift across daemon/server** -> shared fingerprint/namespace/observation implementations and byte-identical fixtures. +- **Telemetry overload** -> bounded buffer, sampling, closed counter names, and closed label values. +- **Defaults drift** -> `design-defaults` block plus shared constants coverage test. diff --git a/openspec/changes/memory-system-post-1-1-integration/proposal.md b/openspec/changes/memory-system-post-1-1-integration/proposal.md new file mode 100644 index 000000000..2e8caaeb5 --- /dev/null +++ b/openspec/changes/memory-system-post-1-1-integration/proposal.md @@ -0,0 +1,60 @@ +## Why + +`memory-system-1.1-foundations` is the stability baseline for daemon memory: durable provenance, bounded materialization, redaction, immediate daemon-receipt send ack, SDK-native `/compact`, `/stop` and approval/feedback priority, fail-open recall/bootstrap, provider send-start watchdogs, and local repair. Post-foundations work must build on that baseline without reintroducing the instability previously seen in memory branches. + +`docs/plan/mem1.1.md` contains the original roadmap for Phase 1.5, 1.6, 1.7, 1.8, 1.9, 1.7-O, and later Phase 2/3 candidates. Keeping those as implicit fragments makes scope, sequencing, failure handling, security review, and acceptance ambiguous. This change is the single authoritative OpenSpec contract for post-1.1 memory work. + +## Completion Boundary + +The current completion milestone is **Wave 1 through Wave 5**: + +1. Wave 1 — operational foundations, authorization scope registry, and hardening gates. +2. Wave 2 — self-learning memory. +3. Wave 3 — quick search, citations, drift, and cite-count ranking. +4. Wave 4 — markdown ingest, preferences, and unified bootstrap. +5. Wave 5 — enterprise org-shared authored standards plus safe skill storage/import/render/admin foundations and post-response skill auto-creation/self-improvement through the existing background compression/materialization path. + +Later candidates are tracked for continuity but do **not** block this milestone until promoted by a future OpenSpec delta with concrete requirements, tasks, and tests. Deferred candidates include drift recompaction loops, prompt caching, autonomous prefetch/LRU, topic-focused compact/context-selection behavior that still must not daemon-intercept `/compact`, LLM redaction, built-in skill content harvest, and quick-search result caching. These are deferred for behavioral/product/security reasons only, not because they require migrations. No post-1.1 item may be deferred merely because it requires schema migration, data backfill, or server/daemon migration coordination. Authorization scope registry extensions, namespace registry extensions, the multi-class observation store, cite-count storage/ranking, preference storage/idempotency, skill storage, enterprise org-shared authored standards, and skill auto-creation are included in Wave 1-5 because dev can carry the required migrations and safety gates. Wave 1 must add concrete scope policies for `user_private`, existing `personal`, `project_shared`, `workspace_shared`, and `org_shared`; these are not deferred backlog. Enterprise-wide shared standards MUST use existing `org_shared` semantics, not a new `global` or `namespace_tier=global`: `org_shared` is visible only inside the current enterprise/team, requires `enterprise_id`, and never crosses enterprise boundaries. Main sessions and sub-sessions already belong to one project/session tree and MUST share the same project/session context through namespace/context binding, not through a new authorization scope. Same signed-in user on different devices MUST see the same project-scoped memory when the project resolves to the same canonical remote repository identity (`canonicalRepoId`, derived from normalized git remote/remote aliases); local path or machine id must not split that project. `user_private` means owner-only cross-project memory and, when sync is enabled, MUST use a dedicated owner-private sync path rather than the shared projection authorization path. Skill auto-creation/self-improvement is part of Wave 5 only as post-response background compression/materialization work, never as send-path work. + +## Capability Bridging + +This change has one change id and two capability surfaces: + +- **New capability:** `daemon-memory-post-foundations`, containing all current Wave 1-5 runtime requirements and acceptance gates. +- **Archive-time modified capability migration:** `daemon-memory-pipeline`. Some requirements preserve or tighten behavior originally described by `memory-system-1.1-foundations` / `daemon-memory-pipeline`, especially send ack timing, priority controls, startup selection, render-policy payloads, and citation-aware recall. Because `memory-system-1.1-foundations` is still represented as an active change in this workspace, these deltas remain documented here until foundations is archived. Before this change is archived, they MUST be migrated into `specs/daemon-memory-pipeline/spec.md` as `## MODIFIED Requirements` when the cumulative capability exists. + +## What Changes + +- Consolidate all post-1.1 memory work under `memory-system-post-1-1-integration` instead of leaving phase-specific implicit plans. +- Establish Wave 1 primitives before product surfaces: stable kind-aware fingerprints, closed origin metadata, explicit authorization scope policy registry, first-class namespace registry, multi-class observation store, org-shared authored standards semantics, runtime feature flags, async telemetry, startup budget policy, named-stage selection, typed render policy, migration/backfill discipline, and cross-wave repair/backoff/idempotency gates. +- Implement Wave 2-5 in dependency order and keep every new surface disabled/fail-closed until its acceptance gates pass. +- Lock foundations regressions for every wave: ordinary `send` ack remains daemon receipt and never waits for memory/provider work; `/compact` stays SDK-native pass-through; `/stop` and approval/feedback remain priority-lane controls; recall/bootstrap failures still dispatch the original user message; redaction, scope filtering, source provenance, and materialization repair do not regress. +- Promote authorization-scope registry migration, cite-count ranking, namespace/observation migrations, enterprise org-shared authored standards, and skill auto-creation into current scope with concrete storage, identity, authorization, idempotency, backoff, and test gates instead of deferring them because they require migrations. +- Close the post-1.1 management UI/control-plane surface: server bridge single-casts management responses by `requestId`, daemon handlers authorize from server-derived context, Web mutation controls are disabled until feature state is known, daemon-managed feature flags can be enabled/disabled from the UI through persisted management RPCs, skill/MD management inputs are treated as untrusted, project browse defaults to all projects/no filter, project filter choices are populated from daemon/cloud/shared memory indexes plus known sessions/enrollments, and all management errors use shared codes plus localized UI strings. +- Replace ambiguous roadmap language with explicit requirements, failure modes, task ownership, and test anchors. + +## Capabilities + +### New Capabilities + +- `daemon-memory-post-foundations`: Runtime contract for post-1.1 memory integration, including operational foundations, self-learning compression, quick search/citation/cite-count, MD/preference ingest, skills, safety gates, and future-candidate tracking. + +### Modified Capabilities + +- `daemon-memory-pipeline`: Archive-time migration target. Until `memory-system-1.1-foundations` is archived and the cumulative capability exists, foundations-touching behavior is captured as hard regression requirements in `daemon-memory-post-foundations` and in `tasks.md` archive gates. This is not a runtime deferral and does not weaken the current send/stop/compact contract. + +## Acceptance Summary + +The change is ready for implementation only when: + +- `openspec validate memory-system-post-1-1-integration` passes. +- Every current-scope requirement has a stable ID, scenarios, implementation tasks, and test anchors; each test anchor is either an existing test path or an explicit task to create that path. +- Wave 1-5 tasks are present and later candidates are non-checkbox backlog items. +- Foundations regression tests for send ack, `/compact`, `/stop`, feedback/approval, recall/bootstrap failure, provider send-start, materialization repair, redaction, and scope/source safety are mandatory gates. +- Authorization-scope registry, org-shared authored standards, cite-count, namespace/observation, preference, and skill auto-creation behavior has explicit migration, idempotency, auth, backoff, disabled-feature, and replay tests. +- Management UI acceptance covers a searchable project selector/dropdown that defaults memory browsing to all projects, shows canonical ID plus directory when available, also lists canonical-only projects discovered from memory indexes, separates browse filtering from local file-backed action project selection, performs daemon-backed project resolution, and covers processed-memory manual add/edit/delete/archive/restore/pin, preference create/update/delete, skills, manual MD ingest, typed observation edit/delete/promotion with explicit from/to/effect confirmation before mutation, feature-state guards plus feature enable/disable controls, stale requestId rejection, bridge no-broadcast routing, record creator/owner metadata separate from management role, owner/scope authorization, symlink-safe skill preview, registry caps, and canonical project identity rejection. +- `docs/plan/mem1.1.md` remains historical rationale; these OpenSpec artifacts are the implementation authority. + +## Impact + +Future implementation will affect daemon memory modules (`src/context/*`, `src/store/context-store.ts`, `src/daemon/*`), shared utilities (`shared/*`), server migrations/search/scope surfaces (`server/src/*`), web quick-search/citation/skill UI (`web/src/*`), tests, and acceptance scripts. No breaking behavior is allowed for existing foundations flows. diff --git a/openspec/changes/memory-system-post-1-1-integration/specs/daemon-memory-pipeline/spec.md b/openspec/changes/memory-system-post-1-1-integration/specs/daemon-memory-pipeline/spec.md new file mode 100644 index 000000000..f04b13996 --- /dev/null +++ b/openspec/changes/memory-system-post-1-1-integration/specs/daemon-memory-pipeline/spec.md @@ -0,0 +1,61 @@ +## MODIFIED Requirements + +### Requirement: Transport dispatch SHALL bound memory-context pre-dispatch work and fail open +Transport-runtime sends SHALL treat live context bootstrap, per-message semantic memory recall, feature-flag reads, MD ingest, skill loading, quick-search/citation lookup, telemetry enqueue/sink work, classification, and skill-review scheduling as best-effort asynchronous or bounded enrichment. Ordinary non-P2P `session.send` ack is a daemon-receipt acknowledgement, not proof that memory recall succeeded or that the provider has started or completed the turn. Once the daemon validates ownership of a non-duplicate commandId, it MUST emit `command.ack accepted` before the first asynchronous delivery boundary in the send handler. + +The daemon MUST NOT wait for P2P preference reads, pending session relaunches, per-session transport locks, live context bootstrap, semantic recall, embedding generation, candidate scoring, feature-flag polling, MD ingest, skill loading, quick-search/citation lookup, telemetry sinks, skill review, provider send-start, provider settlement, or any background memory work before acking an accepted ordinary send. Downstream recall/bootstrap/enrichment success, failure, or timeout MUST NOT affect ack timing; the message MUST still be dispatched to the SDK/provider with memory context when available and without failed memory payloads otherwise. Daemon-handled controls whose ack intentionally reports command validation/result (`/model`, `/thinking`/`/effort`, `/clear`) MAY keep result/error ack semantics. `/compact` is not daemon-handled and MUST use the ordinary immediate-receipt ack plus SDK-forwarding path. + +Transport `/stop` and transport approval/feedback responses are priority-lane commands. `/stop` MUST emit receipt ack and clear queued resend work before P2P preference reads, pending relaunch waits, per-session send locks, context bootstrap, recall, embedding, provider cancel awaits, telemetry, or memory work. Provider cancellation MUST run in the background and surface failures via timeline/session state. Transport approval/feedback responses, including `transport.approval_response`, MUST be forwarded directly to the live runtime and MUST NOT be serialized behind normal send, relaunch, context, recall, telemetry, or memory work. + +#### Scenario: ordinary send ack is not delayed by post-1.1 memory features +- **WHEN** the daemon receives an ordinary non-P2P `session.send` with a fresh commandId +- **AND** post-1.1 features such as feature flags, MD ingest, skill loading, quick search, citation lookup, telemetry, classification, or skill review are slow, disabled, or failing +- **THEN** the daemon MUST emit `command.ack accepted` immediately after accepting command ownership and before the first async delivery boundary +- **AND** provider dispatch MUST still proceed later with available context or without failed context + +#### Scenario: stop and feedback remain priority-lane controls +- **WHEN** a transport session has a held send-control lock, pending relaunch, slow memory work, or pending provider send-start +- **AND** the user sends `/stop` or responds to an approval/feedback request +- **THEN** `/stop` MUST emit `command.ack accepted` and invoke provider cancellation without waiting for those blockers +- **AND** approval/feedback MUST reach the runtime approval handler without waiting for those blockers +- **AND** neither path MAY run memory recall, context bootstrap, feature reads, telemetry sinks, or skill work before reaching the transport runtime + +### Requirement: Manual `/compact` SHALL remain SDK-native pass-through +The daemon SHALL forward the literal `/compact` command unchanged through the normal transport send path for transport-runtime sessions. The daemon MUST NOT intercept `/compact` to replay history, call daemon compression/materialization helpers, relaunch the transport conversation, synthesize a compacted summary, emit a daemon-owned `compaction.result` event, or implement topic-focused daemon compaction in this milestone. If manual compaction appears broken, the implementation SHALL debug transport forwarding, SDK session state, provider health, lifecycle/admission races, or provider-side compact behavior rather than replacing SDK-native behavior. + +All transport providers SHALL receive slash control commands as raw provider-control payloads, not as memory-enriched user prompts. For such controls the transport runtime MUST skip daemon-added startup memory, per-turn recall, preference context preambles, authored context selection, and extra per-turn system prompt. This applies uniformly to Codex SDK, Claude Code SDK, Gemini ACP, Qwen, Cursor headless, Copilot SDK, OpenClaw, and future transport providers; provider-specific adapters may then translate the raw token to a native control API when one exists. + +SDK/provider adapters that expose a native compact RPC SHALL treat the send as accepted only after the native request is accepted, and SHALL then settle the transport runtime from native compact completion signals. The adapter MUST accept known upstream notification shape drift (for example `threadId`/`turnId` and `thread_id`/`turn_id`), MUST not leave the session busy when a native compact request is accepted but emits no asynchronous completion signal, and MUST fail with a bounded retryable provider error if an active compact never completes. + +#### Scenario: `/compact` is forwarded unchanged in post-1.1 builds +- **WHEN** a user sends `/compact` to a transport-runtime session +- **THEN** the active transport runtime MUST receive the exact string `/compact` +- **AND** daemon memory compression, materialization, topic selection, and summarization helpers MUST NOT be invoked for that command +- **AND** no provider-visible startup memory, recall block, preference block, authored-context block, or extra per-turn system prompt MAY be attached to the slash-control payload +- **AND** no daemon-owned compaction result event MUST be emitted +- **AND** a Codex SDK transport MUST call `thread/compact/start` for the active thread and later clear runtime busy state on `thread/compacted`, `contextCompaction` item completion, `turn/completed`, status-idle, or the bounded accepted/no-signal fallback + +### Requirement: Startup and recall memory rendering SHALL use explicit typed payloads and safe degradation +Transport startup memory and per-message recall SHALL preserve the existing fail-open dispatch behavior while using typed post-1.1 render payloads. Startup selection SHALL assemble memory through collect, prioritize, quota, trim, deduplicate, and render stages. Rendered items MUST carry explicit render kind (`summary`, `preference`, `note`, `skill`, `pinned`, or `citation_preview`) and MUST honor authorization and per-kind truncation before injection. + +Any stage failure for non-required memory sources MUST omit that source, emit bounded telemetry, and continue user delivery. Required authored context remains governed by the existing required-authored-context dispatch contract; advisory memory and post-1.1 enrichment MUST NOT block ordinary send ack. + +#### Scenario: startup stage failure degrades without blocking send ack +- **WHEN** one startup memory source, render stage, skill load, preference load, or citation preview fails +- **THEN** ordinary send ack MUST remain daemon receipt +- **AND** provider dispatch MUST continue with the remaining authorized context +- **AND** the failed source MUST be omitted rather than injecting raw or unauthorized data + +### Requirement: Citation-aware recall SHALL preserve authorization and replay-safe identity +Quick search, citation preview, citation insertion, drift metadata, and cite-count ranking MUST run after shared scope filtering. Citation insertion SHALL use projection identity, authoritative citing-message identity, and store-derived idempotency keys. Missing, unauthorized, and disabled source/projection lookups MUST return the same external response envelope wherever object existence could otherwise leak. Cite-count ranking, when enabled, MUST use bounded count signal only after scope filtering and MUST NOT reveal or increment counts for missing or unauthorized citation attempts. + +#### Scenario: inaccessible citation lookup does not leak inventory +- **WHEN** a caller requests a missing, unauthorized, or feature-disabled projection/source id +- **THEN** the response shape MUST be the same for all cases that would otherwise reveal existence +- **AND** it MUST NOT include raw source text, role diagnostics, source counts, hit counts, drift markers, cross-scope ids, or cite-count state + +#### Scenario: citation replay cannot inflate ranking count +- **WHEN** an authorized citation insertion is retried or replayed for the same citing message and projection +- **THEN** the authoritative idempotency key MUST dedupe the write +- **AND** cite count MUST increment at most once for that idempotency key +- **AND** ranking MUST consume cite count only after authorization filtering diff --git a/openspec/changes/memory-system-post-1-1-integration/specs/daemon-memory-post-foundations/spec.md b/openspec/changes/memory-system-post-1-1-integration/specs/daemon-memory-post-foundations/spec.md new file mode 100644 index 000000000..7f3fb37b9 --- /dev/null +++ b/openspec/changes/memory-system-post-1-1-integration/specs/daemon-memory-post-foundations/spec.md @@ -0,0 +1,592 @@ +## ADDED Requirements + +### Requirement: POST11-R1 Foundations liveness invariants MUST remain hard gates +Post-foundations memory features MUST NOT change daemon receipt semantics for ordinary sends or urgent controls. Ordinary `session.send` ack MUST remain daemon receipt for accepted non-duplicate sends and MUST be emitted before memory work, relaunch waits, transport locks, bootstrap, recall, embedding, provider send-start, provider settlement, telemetry sinks, MD ingest, skill load, quick-search/citation lookup, feature-flag polling, or skill review completes. `/compact` MUST remain SDK-native pass-through. `/stop` and approval/feedback controls MUST remain priority-lane controls. + +- **State variables:** command id ownership, duplicate-command status, ack status, transport lock state, relaunch state, bootstrap/recall/embedding/provider state, priority-control lane. +- **Failure modes:** pending relaunch, held transport lock, bootstrap hang, recall/embedding failure, provider send-start never settles, feature-flag read failure, telemetry timeout, duplicate command id. +- **Implemented by tasks:** 1.1, 1.6, 1.7, 8.1-8.8, 16.1-16.4. +- **Test anchors:** `server/test/ack-reliability.test.ts`, `test/ack-reliability-e2e.test.ts`, `test/daemon/command-handler-transport-queue.test.ts`, `test/daemon/transport-session-runtime.test.ts`, `test/agent/runtime-context-bootstrap.test.ts`, `test/agent/codex-sdk-provider.test.ts`, `test/daemon/transport-relay.test.ts`, `web/test/use-timeline-optimistic.test.ts`. + +#### Scenario: accepted ordinary send enters asynchronous memory work +- **WHEN** a normal user send has a non-duplicate command id accepted by the daemon +- **THEN** the daemon MUST emit a success receipt ack before feature-flag reads, named-stage startup selection, MD ingest, skill loading, quick-search/citation lookup, recall, embedding, bootstrap, telemetry, provider send-start, provider settlement, or skill review +- **AND** the success receipt ack MAY be `accepted` or `accepted_legacy` according to the existing client/command-id path +- **AND** duplicate non-retry command ids MAY emit the existing duplicate/error ack instead of success + +#### Scenario: downstream memory work fails after ack +- **WHEN** recall, bootstrap, embedding, MD ingest, skill load, search, citation lookup, classification, or skill review fails or times out after daemon receipt +- **THEN** the original user message MUST still be dispatched to the SDK/provider +- **AND** failed memory context MUST be omitted from the payload instead of blocking or spinning the send +- **AND** the failure MUST be reported through bounded telemetry/status where applicable + +#### Scenario: send is received while relaunch or transport lock is pending +- **WHEN** a normal send arrives while session relaunch, transport lock, bootstrap, or provider start is pending +- **THEN** daemon receipt ack MUST be emitted before waiting for that downstream condition +- **AND** later SDK/provider delivery MAY proceed after the condition clears or degrades + +#### Scenario: compact and urgent controls keep foundations behavior +- **WHEN** the user sends `/compact` +- **THEN** the daemon MUST forward it through the ordinary send path to the SDK/provider without daemon-side synthetic compaction or interception +- **AND** the transport runtime MUST treat slash controls as provider-control payloads for every transport provider, suppressing daemon-added startup memory, per-turn recall, preference preambles, authored context, and extra per-turn system prompt so the provider receives the raw control token +- **AND** provider adapters with a native compact API, such as Codex app-server `thread/compact/start`, MUST translate the raw `/compact` token at the SDK boundary and MUST NOT send `/compact` as ordinary model text +- **AND** the provider adapter MUST settle the transport runtime from native compact lifecycle signals (`thread/compacted`, `contextCompaction` item completion, turn completion, or equivalent thread-status idle), accepting both camelCase and snake_case thread/turn identifiers when the upstream SDK shape varies +- **AND** an accepted native compact request that produces no asynchronous completion signal MUST resolve through a bounded no-op/accepted fallback, while a compact request or active compaction that exceeds the hard timeout MUST clear the busy state and emit a retryable provider error instead of leaving the UI in `Agent working...` +- **AND** receipt ack timing MUST remain daemon receipt +- **WHEN** the user sends `/stop` or an approval/feedback response +- **THEN** the command MUST use the priority path and MUST NOT wait behind normal send locks, memory work, relaunch, provider cancel completion, or telemetry + +#### Scenario: SDK tool-side sender identity is a runtime guarantee +- **WHEN** a local SDK transport session is created with daemon-provided IM.codes session environment +- **THEN** the SDK provider integration MUST preserve `IMCODES_SESSION` and `IMCODES_SESSION_LABEL` as runtime/tool-side inputs or an equivalent non-prompt adapter +- **AND** prompt text alone MUST NOT be the only mechanism for `imcodes send` sender/reply identity + +#### Scenario: Codex SDK ctx usage is current-window and model-stable +- **WHEN** Codex app-server emits `thread/tokenUsage/updated` with both `last` and `total` token usage +- **THEN** the IM.codes ctx meter MUST represent the current live prompt/window from `tokenUsage.last.inputTokens`, falling back to `tokenUsage.total.inputTokens` only for older payloads that omit `last` +- **AND** cumulative `tokenUsage.total` values MAY be retained only as diagnostics and MUST NOT drive the visible ctx percentage when `last` is present +- **AND** because Codex/OpenAI `cachedInputTokens` is a subset of `inputTokens`, the timeline MUST normalize it as `inputTokens - cachedInputTokens` plus `cacheTokens`, so the visible total still equals the selected current-window input token count +- **AND** the provider-reported `modelContextWindow`, when present, MUST be propagated as the timeline context-window value with a provider-source marker unless it is a known stale/mismatched provider fallback for the selected model +- **AND** if a usage event omits `model`, the daemon MUST resolve the effective model from the persisted session metadata (`activeModel`, `requestedModel`, `modelDisplay`, or provider-specific stored model) before resolving the context window or forwarding usage to Web +- **AND** GPT-5.5 MUST resolve to the locked 922k model window for ctx display even when Codex SDK/native Codex reports stale fallback windows such as 258400 or 1000000 +- **AND** Web context UI MUST prefer a provider-marked explicit context window over model-family inference, while known stale/mismatched provider values and older unmarked/stale explicit context-window values MAY still be overridden by model-family inference + +### Requirement: POST11-R2 Stable memory fingerprints MUST be deterministic, kind-aware, and scope-safe +The system MUST compute stable fingerprints for post-foundations memory content using one shared implementation. Fingerprints MUST be deterministic across daemon SQLite and server PostgreSQL contexts and MUST NOT deduplicate across namespace/scope boundaries. + +- **State variables:** fingerprint kind, fingerprint version, normalized content, scope key, namespace, source ids. +- **Failure modes:** missing fingerprint, legacy helper misuse, normalization mismatch, cross-scope merge, backfill interruption. +- **Implemented by tasks:** 2.1-2.7. +- **Test anchors:** `test/context/memory-fingerprint-v1.test.ts`, `test/fixtures/fingerprint-v1/**`, daemon/server fixture parity tests. + +#### Scenario: equivalent scoped content is fingerprinted +- **WHEN** two memory entries of the same fingerprint kind normalize to the same content within the same namespace/scope +- **THEN** they MUST compute the same `v1` fingerprint through `computeMemoryFingerprint({ kind, content, scopeKey, version: 'v1' })` +- **AND** deduplication MAY merge them while preserving all source ids + +#### Scenario: identical content is in different scopes +- **WHEN** two entries have identical normalized content but different scopes or namespaces +- **THEN** they MUST NOT be merged into one logical memory +- **AND** citation, hit, drift, and ranking signals MUST remain scope-local + +#### Scenario: fingerprint backfill runs +- **WHEN** existing rows lack fingerprints +- **THEN** lazy backfill MUST NOT block daemon startup or ordinary send ack +- **AND** eager backfill, if provided, MUST run in bounded restartable batches + +### Requirement: POST11-R3 Origin metadata MUST be explicit and closed for the current milestone +Every post-foundations projection, preference, pinned note mirror, MD import, skill import, and self-learning output MUST carry explicit origin metadata from the shared `MEMORY_ORIGINS` enum: `chat_compacted`, `user_note`, `skill_import`, `manual_pin`, `agent_learned`, and `md_ingest`. `quick_search_cache` and other cache origins are reserved and MUST NOT be emitted in this milestone. New origin values require a later OpenSpec delta and migration. + +- **State variables:** origin, scope, writer kind, migration boundary, feature flag. +- **Failure modes:** missing origin, invalid origin, fallback default outside migration, cache origin emitted without cache contract, origin used to bypass authorization. +- **Implemented by tasks:** 3.1-3.6. +- **Test anchors:** origin migration/write tests, search/UI origin tests, reserved-origin rejection tests. + +#### Scenario: a new memory row is written +- **WHEN** post-foundations code writes or updates a projection, preference, pinned note mirror, MD import, skill import, or self-learning output +- **THEN** it MUST set origin metadata explicitly +- **AND** missing or invalid origin MUST be rejected outside a documented migration/backfill boundary + +#### Scenario: origin is used for UI, pruning, or feature flags +- **WHEN** memory is rendered, searched, pruned, or controlled by a feature flag +- **THEN** origin metadata MUST be available without parsing free-form summary text +- **AND** origin MUST NOT override scope authorization + +### Requirement: POST11-R4 Feature flags MUST fail closed and stop new background work when disabled +Every new post-foundations feature MUST have a concrete feature flag or kill switch before it can be enabled. Disabled features MUST return pre-feature behavior, enqueue no new background work, and perform no persistent writes for that feature. Runtime disablement MUST stop new work within the documented propagation target. The current registry MUST include `mem.feature.scope_registry_extensions`, `mem.feature.user_private_sync`, `mem.feature.self_learning`, `mem.feature.namespace_registry`, `mem.feature.observation_store`, `mem.feature.quick_search`, `mem.feature.citation`, `mem.feature.cite_count`, `mem.feature.cite_drift_badge`, `mem.feature.md_ingest`, `mem.feature.preferences`, `mem.feature.skills`, `mem.feature.skill_auto_creation`, and `mem.feature.org_shared_authored_standards`. + +- **State variables:** flag name, default, source of truth, dependency, propagation state, observer components, in-flight job state. +- **Failure modes:** flag read failure, missing registry entry, partial disablement, dependency enabled while parent disabled, UI disabled while workers run, server disabled while daemon writes, stale config. +- **Implemented by tasks:** 4.1-4.10. +- **Test anchors:** `test/context/memory-feature-flags.test.ts`, server/web feature-disable tests, dependency/default coverage tests. + +#### Scenario: a feature is disabled +- **WHEN** a disabled feature path is invoked +- **THEN** it MUST skip new reads, writes, RPCs, and background jobs for that feature +- **AND** it MUST preserve previous user-visible behavior or the documented same-shape disabled envelope +- **AND** ordinary send ack MUST still follow POST11-R1 timing + +#### Scenario: runtime kill switch changes +- **WHEN** an operator disables a memory feature at runtime +- **THEN** new work for that feature MUST stop within the documented propagation target +- **AND** in-flight work MAY finish only if it cannot corrupt state, block shutdown/upgrade, or leak data +- **AND** flag read failure MUST fail closed for new features + +#### Scenario: operator changes a daemon memory feature from the management UI +- **WHEN** the management UI sends a shared `memory.features.set` request for a closed registry flag +- **THEN** the daemon MUST require a server-derived or local-daemon management context before mutating config +- **AND** it MUST persist the requested override above environment startup defaults +- **AND** enabling a feature from this operator surface MUST also request-enable its dependency closure so the action can produce an effective enabled state when prerequisites are available +- **AND** the daemon MUST return the recomputed requested/effective records, value source, dependencies, blocked dependencies, and disabled behavior in a shared response +- **AND** invalid flags, malformed payloads, and config-write failures MUST fail closed with shared error codes and without changing feature state + +#### Scenario: dependent flag is enabled without its parent or prerequisite +- **WHEN** a dependent flag such as `mem.feature.cite_count`, `mem.feature.user_private_sync`, `mem.feature.skill_auto_creation`, or `mem.feature.org_shared_authored_standards` is enabled while its required parent flag is disabled or required registry/migration prerequisite is unavailable +- **THEN** the dependent feature MUST remain effectively disabled +- **AND** the system MUST emit bounded telemetry rather than partially running the dependent feature + +### Requirement: POST11-R5 Telemetry MUST be asynchronous, bounded, and low-cardinality +Post-foundations metrics and audit events MUST be emitted through a bounded asynchronous path. Telemetry sink failure MUST NOT block sends, memory reads, materialization, skill loading, MD ingest, search, citation, skill review, or shutdown. Counter names and labels MUST use shared closed enums. + +- **State variables:** telemetry buffer size, counter name, labels, sink state, sampling state. +- **Failure modes:** sink timeout, sink rejection, buffer overflow, unbounded label cardinality, secret/raw-content logging. +- **Implemented by tasks:** 5.1-5.6. +- **Test anchors:** telemetry sink timeout/reject tests, memory counter registry tests. + +#### Scenario: telemetry sink is unavailable +- **WHEN** the telemetry sink rejects, times out, or is unreachable +- **THEN** memory feature behavior MUST continue according to normal success/failure semantics +- **AND** high-frequency metric labels MUST NOT include unbounded identifiers, user content, file paths, session ids, project ids, user ids, or secrets + +#### Scenario: soft failure is swallowed intentionally +- **WHEN** a memory path degrades by returning empty/no-op instead of throwing +- **THEN** it MUST emit a rate-limited structured warning and a bounded counter from `MEMORY_COUNTERS` +- **AND** the warning MUST avoid secrets or raw private content + +### Requirement: POST11-R6 Startup context MUST use named-stage selection and a total budget +Startup memory assembly MUST be staged as collect, prioritize, apply quotas, trim to total budget, deduplicate, and render. The total rendered startup memory payload MUST stay under the configured token budget defined in `design.md` defaults unless changed by a later OpenSpec delta. + +- **State variables:** total budget, per-kind cap, trim priority, stage outputs, render kind, telemetry. +- **Failure modes:** over-budget payload, stage failure, render failure, duplicate content, unbounded project docs/skills. +- **Implemented by tasks:** 6.1-6.6. +- **Test anchors:** `test/context/startup-memory.test.ts`, startup over-budget fixture tests, `test/spec/design-defaults-coverage.test.ts`. + +#### Scenario: startup candidates exceed the budget +- **WHEN** collected startup memory exceeds the total budget +- **THEN** the system MUST trim using configured trim priority and per-kind caps +- **AND** final rendered output MUST be at or below the total budget +- **AND** pinned content MUST receive the highest preservation priority + +#### Scenario: a selection stage fails +- **WHEN** a collect, prioritize, dedup, or render stage fails for a non-critical source +- **THEN** startup assembly MUST degrade by omitting that source and recording telemetry +- **AND** ordinary send ack MUST NOT wait for recovery + +### Requirement: POST11-R7 Render policy MUST type memory before context injection +Every memory item injected into startup or provider context MUST be rendered through an explicit render kind such as `summary`, `preference`, `note`, `skill`, `pinned`, or `citation_preview`. Render policy MUST enforce per-kind truncation, delimiter, authorization, and safety rules. + +- **State variables:** render kind, source authorization, envelope, length cap, delimiter collision state. +- **Failure modes:** ad-hoc formatting, skill as system instruction, unauthorized raw source preview, delimiter collision. +- **Implemented by tasks:** 7.1-7.5. +- **Test anchors:** render policy tests, `test/context/skill-envelope.test.ts`. + +#### Scenario: skill content is rendered +- **WHEN** a skill is selected for context injection +- **THEN** it MUST be wrapped by `SKILL_ENVELOPE_OPEN` and `SKILL_ENVELOPE_CLOSE` +- **AND** it MUST respect `SKILL_MAX_BYTES` +- **AND** delimiter collisions MUST be rejected or escaped according to `SKILL_ENVELOPE_COLLISION_PATTERN` +- **AND** skill content MUST NOT be rendered as a system instruction outside the skill envelope + +#### Scenario: citation preview is rendered +- **WHEN** citation preview content is rendered +- **THEN** it MUST pass source authorization first +- **AND** unauthorized raw source content MUST NOT be present in the preview + +### Requirement: POST11-R8 Self-learning memory MUST be scope-bound and fail open for delivery +Classification, dedup-decision, durable-signal extraction, and cold/warm/resumed startup-state tagging MUST operate within the source namespace/scope. Failure in self-learning phases MUST NOT block ordinary send, urgent controls, materialization retry safety, or source provenance. + +- **State variables:** classifier output, dedup decision, source ids, origin, fingerprint, scope, retry state, startup state tag. +- **Failure modes:** classifier timeout, dedup error, cross-scope merge, local-fallback pollution, retry storm. +- **Implemented by tasks:** 9.1-9.6. +- **Test anchors:** classification/dedup tests, materialization repair tests. + +#### Scenario: classification succeeds +- **WHEN** a materialized summary is classified +- **THEN** classifier output MUST be stored with provenance, origin `agent_learned` where applicable, fingerprint, namespace, and scope +- **AND** dedup decisions MUST preserve all source event ids + +#### Scenario: classification fails +- **WHEN** classification, dedup-decision, or durable extraction fails +- **THEN** original user message delivery MUST continue +- **AND** the system MUST NOT persist local-fallback/raw-transcript pollution as active memory +- **AND** retry/backoff MUST remain bounded + +### Requirement: POST11-R9 Quick search MUST be authorized, scoped, and side-channel resistant +Quick search, palette search, and fast-path memory reads MUST use shared scope filtering and render-policy-safe previews. Missing, unauthorized, and disabled-feature projection/source lookups MUST return the same external response envelope where object existence could otherwise leak and MUST NOT leak existence through status shape, role diagnostics, counts, drift metadata, timing-dependent alternate shapes, or raw source fields. + +- **State variables:** caller scope, authorized scope set, search query, projection id, source id, response envelope, feature flag state. +- **Failure modes:** bespoke SQL scope bug, 403 role detail leak, count leak, drift leak, raw source leak, timing-dependent alternate shape, disabled-feature shape leak. +- **Implemented by tasks:** 10.1-10.8, 1.8 security matrix. +- **Test anchors:** `server/test/memory-search-auth.test.ts`, `test/context/memory-search-semantic.test.ts`, web quick-search tests. + +#### Scenario: user searches memory +- **WHEN** a caller invokes quick search +- **THEN** results MUST be restricted to the caller's authorized namespace/scope +- **AND** result previews MUST be rendered through approved render policy +- **AND** raw source content MUST NOT be returned through search results + +#### Scenario: caller requests inaccessible source +- **WHEN** a caller requests a missing, unauthorized, or feature-disabled projection/source id +- **THEN** the response MUST use the documented same-shape not-found/disabled envelope for all cases that would otherwise reveal object existence +- **AND** the response MUST NOT include role diagnostics, source counts, hit counts, drift markers, raw source content, or cross-scope identifiers + +### Requirement: POST11-R10 Citations MUST use projection identity, explicit drift semantics, and replay-safe cite-count +Citation insertion MUST use projection identity for the current wave. Each citation insertion MUST create a new citation record with its own `created_at` and authoritative idempotency key. Citation display MUST indicate drift using a content-stable projection marker, without exposing unauthorized source rows. Cite-count storage, idempotent incrementing, authorized ranking use, replay protection, migration/backfill, and tests are in current Wave 3 scope. + +- **State variables:** projection id, cite id, cite created_at, projection content marker, authorization state, drift flag, cite_count, citation idempotency key, citing message id, replay state. +- **Failure modes:** raw source snapshot, per-projection cite reuse, no-op update drift false positive, unauthorized drift/source leak, cite-count replay inflation, cross-scope count leak, repeated composer replay, missing citing message identity, hot-row contention. +- **Implemented by tasks:** 10.3-10.14. +- **Test anchors:** `test/context/memory-citation-drift.test.ts`, `test/context/memory-cite-count.test.ts`, web citation tests, source-lookup auth tests. + +#### Scenario: citation is inserted +- **WHEN** the user inserts a memory citation from authorized search results +- **THEN** the citation MUST store projection identity and a new citation `created_at` timestamp for that insertion +- **AND** it MUST NOT snapshot raw source content in the current wave +- **AND** it MUST include an authoritative idempotency key so composer retries, websocket replays, or timeline replays do not inflate cite counts +- **AND** the implementation MUST NOT trust a client-supplied citation idempotency key + +#### Scenario: cited projection content changes +- **WHEN** a cited projection's normalized content changes after citation creation +- **THEN** drift MUST evaluate using canonical persistent `content_hash` captured at citation time and stored/recomputed from current normalized projection content +- **AND** daemon/server projection writes MUST persist `content_hash`, and routine maintenance writes or idempotent upserts that do not change normalized content MUST NOT change `content_hash` or create false drift +- **AND** the drift indicator MUST NOT bypass source authorization + +#### Scenario: cite-count ranking signal is updated +- **WHEN** an authorized citation insertion is accepted exactly once for an idempotency key +- **THEN** the cited projection's `cite_count` MUST increment at most once for that idempotency key +- **AND** the same citing message replay MUST dedupe while a different citing message citing the same authorized projection MUST increment once for that different message +- **AND** the count MUST remain scoped to the authorized projection namespace/scope +- **AND** quick-search ranking MUST include a bounded `cite_count` signal when `mem.feature.cite_count=true`, only after scope filtering, and without replacing existing semantic score or `hitCount` behavior +- **AND** missing or unauthorized citation attempts MUST NOT reveal or increment counts + +#### Scenario: citation identity cannot be derived +- **WHEN** the system cannot derive a stable authoritative citing message identity +- **THEN** cite-count increment MUST fail closed for that citation attempt without blocking send ack or citation display +- **AND** implementation MUST emit bounded telemetry and preserve replay safety + +### Requirement: POST11-R11 Markdown ingest MUST be bounded, idempotent, and origin-aware +Markdown memory/preference ingest MUST run only from trusted triggers, enforce resource bounds, compute stable fingerprints, and store origin metadata. It MUST NOT silently promote or downgrade project content to cross-project, `user_private`, `workspace_shared`, `org_shared`, or enterprise-wide authored standards. Filesystem markdown is project-bound: unsupported `user_private`, workspace, and org bootstrap namespaces MUST fail closed without writing and MUST emit a bounded scope-dropped counter; authorized workspace/org standards must use the authored-context binding flow, not filesystem markdown scope promotion. + +- **State variables:** trigger kind, path, size, section count, per-section byte cap, parser budget, origin, fingerprint, provenance fingerprint, partial commit state. +- **Failure modes:** oversized file, unreadable file, disallowed symlink, invalid encoding, malformed section, prompt-injection-like section, partial write failure. +- **Implemented by tasks:** 11.1-11.7, 11.13. +- **Test anchors:** MD ingest tests, startup budget compatibility tests. + +#### Scenario: markdown file is ingested +- **WHEN** session start or manual sync triggers MD ingest +- **THEN** the parser MUST enforce size, section-count, per-section byte, and time bounds from the design defaults +- **AND** stored rows MUST be idempotent by stable fingerprint and origin `md_ingest`, through a production worker wired to session bootstrap/manual sync without entering ordinary send ack +- **AND** each accepted markdown section MUST update the projection/search/startup surface and the linked typed observation in the same write path or a repairable outbox path +- **AND** projection and observation idempotency MUST preserve per-file provenance: identical section text in two different supported files MUST NOT overwrite the other file's `path` or source ids +- **AND** malformed sections MUST NOT corrupt valid already-written rows + +#### Scenario: unsafe markdown input is encountered +- **WHEN** a file is oversized, unreadable, symlink-disallowed, invalidly encoded, or contains prompt-injection-like instructions +- **THEN** ingest MUST fail closed for unsafe sections and emit telemetry +- **AND** ordinary send ack MUST NOT wait for ingest result + +### Requirement: POST11-R12 Preferences MUST enforce a user-authored trust boundary +Persistent preference writes, including `@pref:` shortcuts, MUST be accepted only from trusted `SendOrigin` values. Agent text, assistant output, tool output, timeline replay, imported memory content, daemon-injected content, and missing-origin sends MUST NOT create persistent preferences by merely containing preference syntax. When `mem.feature.preferences=true`, trusted leading `@pref:` lines MUST persist idempotently, and their preference content MUST be rendered into the provider-visible preference context for the same turn and as stable session context on the first later eligible turn without exposing raw `@pref:` syntax. Identical rendered preference context MUST NOT be repeated on every ordinary send; it MUST be re-injected only when the rendered block changes, after `/compact` or provider-reported compaction, or after a fresh `/clear` conversation. + +- **State variables:** send origin, trusted origin set, preference line position, user-visible text, provider-visible preference context, preference fingerprint, origin, command/message id. +- **Failure modes:** missing origin, agent-authored preference syntax, raw preference command forwarded as prompt text, preference persisted but not rendered to the provider, duplicate preference, persistence failure, resend/replay duplicate. +- **Implemented by tasks:** 11.4-11.9. +- **Test anchors:** `test/context/preferences-trust-origin.test.ts`, send ack tests. + +#### Scenario: trusted user creates a preference +- **WHEN** an authenticated user sends leading `@pref:` lines through a trusted composer/command origin and `mem.feature.preferences=true` +- **THEN** the system MUST persist the preference with origin `user_note`, fingerprint, namespace, and scope +- **AND** duplicate submissions or retries with the same command/message identity MUST be idempotent and emit `mem.preferences.duplicate_ignored` +- **AND** the trusted raw `@pref:` command lines MUST be stripped from user-visible text and from the provider-bound user message +- **AND** the trusted preference content MUST be included in a controlled provider-visible preference context for that same turn, before persistence completes +- **AND** the first later eligible ordinary send with the preferences feature enabled MUST include active persisted preferences for that user/scope in the provider-visible preference context as stable session context +- **AND** subsequent sends with an unchanged rendered preference block MUST NOT repeat that preference context until `/compact`, provider-reported compaction, `/clear`, or a changed preference block resets the injection gate +- **AND** raw `@pref:` syntax MUST NOT appear in provider-visible context or committed timeline user messages + +#### Scenario: Codex SDK injected context has a final hard cap +- **WHEN** daemon-rendered system context, preferences, startup memory, skill hints, authored standards, or recall preambles would make a Codex SDK turn carry more than 32,000 characters of injected context by default +- **THEN** the Codex SDK adapter MUST truncate daemon-injected context before `turn/start` +- **AND** the adapter MUST preserve the current user turn text rather than truncating user-authored content +- **AND** the cap MAY be overridden only by the bounded `IMCODES_CODEX_SDK_CONTEXT_MAX_CHARS` runtime setting +- **AND** daemon receipt ack MUST NOT wait for preference persistence + +#### Scenario: untrusted output contains preference syntax +- **WHEN** assistant output, tool output, timeline replay, imported memory, daemon-injected content, or a missing-origin send contains text resembling `@pref:` +- **THEN** the system MUST NOT persist it as a user preference +- **AND** it MUST emit a bounded `mem.preferences.untrusted_origin` or `mem.preferences.rejected_untrusted` counter where applicable + +#### Scenario: preferences feature is disabled +- **WHEN** a trusted user sends leading `@pref:` lines while `mem.feature.preferences=false` +- **THEN** the text MUST pass through without persistence, stripping, or provider-visible preference context injection +- **AND** ordinary send ack MUST remain daemon receipt + +### Requirement: POST11-R13 Skills MUST follow safe storage, precedence, packaging, rendering, and background review rules +The skills subsystem MUST support user-level skills by default, optional project association by metadata, an explicit project escape hatch, workspace/org shared mirrors, a loader-ready empty built-in layer, and post-response skill auto-creation/self-improvement through the existing isolated compression/materialization background path. Skill resolution MUST follow documented ordinary precedence plus separate enforced policy semantics. Runtime startup context MUST NOT scan or read every skill markdown body. It MAY expose only a provider-visible registry hint containing bounded metadata and redacted/opaque readable paths sourced from an import/install/review/admin-sync maintained skill registry; full skill bodies MUST be read only on demand when a related request, explicit skill key, classifier match, or enforced-policy resolver requires it. The shared skill envelope/render policy remains the required sanitizer for any path that explicitly renders full skill content. Wave 5 MUST NOT ship built-in skill content. + +- **State variables:** skill layer, enforcement mode, project metadata, package manifest, loaded-layer diagnostics, skill registry entry, registry hint path/URI, render envelope, review trigger evidence, review job state. +- **Failure modes:** unsafe skill, malformed front matter, delimiter collision, over-cap content, missing built-in manifest, startup full-corpus scan/read, full skill body injected eagerly, stale registry path, ordinary shared skill shadowing project/user unexpectedly, auto-creation blocking send/provider delivery, duplicate skill creation, unbounded skill-review retry, hidden/error tool-result evidence pollution, trigger spam or below-threshold trigger spam. +- **Implemented by tasks:** 12.1-12.10. +- **Test anchors:** `test/context/skill-precedence.test.ts`, `test/context/skill-envelope.test.ts`, package/manifest tests, skill auto-creation background tests. + +#### Scenario: user skill is loaded +- **WHEN** a user skill under `~/.imcodes/skills/` is selected +- **THEN** the loader MUST record loaded layer and origin `skill_import` +- **AND** metadata/path parsing MUST be bounded and unsafe or invalid skills MUST fail closed without blocking ordinary send ack +- **AND** import/install/review/admin-sync code MUST update a lightweight skill registry/manifest; ordinary startup and ordinary send MUST NOT construct the registry by scanning or reading all skill markdown bodies +- **AND** the transport startup memory artifact MAY include a bounded registry hint with layer, key, redacted readable path or `skill://` URI, and safe descriptor when `mem.feature.skills=true` +- **AND** polluted, absolute, traversal, NUL-containing, or otherwise provider-unsafe registry display paths MUST be replaced by an opaque `skill://` URI before rendering startup hints +- **AND** unrelated turns MUST NOT read skill bodies; related turns or explicit skill requests MUST read only selected skill bodies through a bounded resolver and the shared skill envelope sanitizer + +#### Scenario: ordinary skill layers conflict +- **WHEN** project, user, workspace, org, and built-in layers provide matching skill names +- **THEN** ordinary precedence MUST be project escape hatch, project-scoped user metadata, user default, workspace shared, org shared, then built-in fallback +- **AND** built-in fallback MUST remain lowest precedence and MUST NOT override user-authored, project, workspace, org, or explicitly selected skills +- **AND** loaded-layer diagnostics MUST show which layers were considered + +#### Scenario: enforced workspace or org policy applies +- **WHEN** a workspace/org skill has `enforcement: 'enforced'` +- **THEN** it MUST be selected according to policy and MUST NOT be bypassed by user/project skills +- **AND** the registry hint or resolver diagnostics MUST show that the skill is enforced +- **AND** enforced policy MUST NOT require ordinary send ack to wait for skill body reads; any proactive read is bounded, post-ack, and priority-control safe + +#### Scenario: skill auto-creation runs after response delivery +- **WHEN** a closed skill-review trigger (`tool_iteration_count` or `manual_review`) fires for a completed user turn and `mem.feature.skill_auto_creation=true` +- **THEN** `tool_iteration_count` MUST require real completed, non-hidden, non-error tool-result evidence meeting the configured threshold before enqueue; `manual_review` MAY bypass that automatic threshold +- **AND** skill review MUST run only after the agent response has been delivered through the existing isolated compression/materialization background path +- **AND** it MUST NOT delay ordinary send ack, provider delivery, `/stop`, approval/feedback controls, or shutdown +- **AND** the daemon production worker/scheduler MUST coalesce duplicate pending reviews per scope/session, enforce configured tool-iteration threshold, concurrency/min-interval/daily caps, write only user-level skills, update the skill registry after successful writes, and emit `mem.skill.review_throttled` only for true throttles +- **AND** daily caps MUST be keyed by scope plus the current day/window, and automatic tool-iteration evidence MUST be cleared after each completed-turn scheduling decision so unrelated below-threshold turns cannot accumulate into a later trigger +- **AND** it MUST prefer updating an existing matching user-level skill before creating a new one +- **AND** duplicate, below-threshold, unsafe, over-cap, hidden/error evidence, or failed reviews MUST be handled with bounded retry/backoff and idempotency; below-threshold/non-eligible decisions MUST be distinguishable from throttling telemetry + +### Requirement: POST11-R14 Skill administration MUST enforce authorization and injection defenses +Workspace/org skill push MUST require admin authorization. Skill content MUST be checked for adversarial phrases, delimiter collision, system-instruction escape, and length cap before being accepted for context rendering. + +- **State variables:** caller role, target scope, skill content, sanitizer result, rejection envelope. +- **Failure modes:** non-admin push, inventory leak, sanitizer bypass, delimiter spoof, over-cap content. +- **Implemented by tasks:** 12.4-12.9. +- **Test anchors:** server/admin skill auth tests, sanitizer fixtures. + +#### Scenario: non-admin pushes workspace skill +- **WHEN** a non-admin attempts to push a workspace or org skill +- **THEN** the request MUST be rejected without creating or updating skill memory +- **AND** the rejection MUST NOT leak unrelated skill inventory + +#### Scenario: skill content attempts delimiter collision +- **WHEN** skill content attempts to close or spoof the skill delimiter envelope +- **THEN** sanitization MUST reject or escape the content according to the documented policy +- **AND** a negative fixture MUST cover the collision case + +### Requirement: POST11-R15 Web-visible post-foundations UI MUST obey i18n and shared-constant rules +User-visible strings introduced for search empty states, citation drift, MD ingest degradation, skill sanitization failures, feature-disabled states, preference rejection, preference management, skill registry management, manual MD ingest, project selection, feature-status display, management error states, and observation promotion MUST use the web i18n system and update all supported locales. Protocol/type/status strings MUST use shared constants. The memory management panel MUST provide the minimum operator surface for every runtime-affecting post-foundations feature: show daemon-resolved feature flag state, allow operator enable/disable for daemon-controlled memory flags through shared management RPCs, provide a searchable project selector/dropdown that defaults browse to all projects and shows both canonical ID and directory when available, list/create/delete trusted user preferences, list/rebuild/preview/delete skill registry entries without eager body reads, run bounded manual markdown ingest with explicit scope/project inputs, inspect typed observations, and promote observations only through the audited explicit UI action. + +- **State variables:** translation key, supported locale list, shared protocol constant, UI feature flag state, daemon WebSocket availability, browse project filter, local-action project option, memory-index project option, project resolution status, canonical repo id, project directory, preference user id, skill registry entry, MD project scope, observation class/scope, promotion target/reason. +- **Failure modes:** hardcoded string, missing locale key, duplicated protocol literal, inaccessible/a11y palette state, disabled feature still mutates persistent state, feature status can only display disabled without an operator toggle path, feature toggle persists nowhere or is lost on restart, dependency-blocked flags appear enabled, daemon error surfaced as raw unlocalized text, preference saved but not visible, skill file created but not visible in registry, management registry write leaves runtime skill cache stale, symlink/polluted registry preview reads outside managed skill roots, UI preview causing startup-style full-corpus skill reads, manual MD ingest reads files before canonical project identity is present, unsupported MD scope silently downgraded, cross-scope observation promotion without audit, ambiguous one-click observation promotion without from/to/effect disclosure, stale project-resolve response overwrites the selected project, stale REST memory response overwrites the active browse filter, hand-typed project IDs become the primary path, browse defaults to the current project instead of all projects, memory-index projects disappear after selecting a filter, canonical-only projects incorrectly enable local file-backed tools, local tools run against an unvalidated directory/ID pair. +- **Implemented by tasks:** 10.6, 11.10-11.12, 12.8, 12.17-12.19, 14.4, 14.7-14.9, 15.1-15.15. +- **Test anchors:** `web/test/i18n-coverage.test.ts`, `web/test/i18n-memory-post11.test.ts`, `web/test/components/SharedContextManagementPanel.test.tsx`, `server/test/bridge-memory-management.test.ts`, `server/test/shared-context-processed-remote.test.ts`, `test/daemon/command-handler-memory-context.test.ts`, `test/daemon/command-handler-transport-queue.test.ts`, `test/context/skill-registry-resolver.test.ts`, `test/context/context-observation-store.test.ts`, `test/context/memory-feature-flags.test.ts`. + +#### Scenario: web UI exposes a new memory state +- **WHEN** a post-foundations feature adds a user-visible web string +- **THEN** the implementation MUST use translation keys +- **AND** every locale in `SUPPORTED_LOCALES` (`en`, `zh-CN`, `zh-TW`, `es`, `ru`, `ja`, `ko`) MUST have the key +- **AND** protocol/status strings shared across daemon/server/web MUST be defined in shared code rather than duplicated literals + +#### Scenario: operator manages post-1.1 runtime memory surfaces +- **WHEN** the daemon is connected and the user opens memory management +- **THEN** the UI MUST query local feature states, preferences, skill registry entries, and typed observations through shared WebSocket message constants +- **AND** the feature-state area MUST expose enable/disable controls for daemon-managed memory flags, persist changes through daemon-side config, show requested-vs-effective dependency-blocked state as a distinct non-enabled warning state, and refresh downstream management panes after a change +- **AND** it MUST allow trusted preference creation/deletion, skill registry rebuild/preview/delete, bounded manual MD ingest, and audited observation promotion without requiring direct filesystem/database edits +- **AND** observation promotion in the Web UI MUST be a two-step action: the first click only opens an explicit confirmation showing source scope, target scope, and visibility/audit consequences; only the confirmation action may send the shared promotion RPC +- **AND** feature-disabled management mutations MUST be rejected by the daemon with shared error codes and localized web messages +- **AND** skill management MUST show registry metadata first and read a full skill body only for an explicit preview/read action +- **AND** skill preview MUST reject symlink/non-file polluted registry entries and management registry writes MUST invalidate runtime skill cache +- **AND** the memory page MUST offer a project selector/list that defaults to all projects for browsing, shows canonical project ID and directory when available, sources active/recent session directories, enterprise canonical projects, and authorized memory-index project summaries returned by local/cloud/shared memory queries, and does not require hand-typed IDs as the primary path +- **AND** the initial browse query MUST omit `projectId`/`canonicalRepoId` until the user explicitly selects a project filter +- **AND** the UI MUST keep browse filtering separate from local file-backed action project selection, so choosing or auto-resolving a local-action project does not silently filter memory browsing +- **AND** canonical-only memory-index projects MAY filter memory views but MUST NOT enable local skill/MD/observation file actions until a validated directory/canonical pair exists +- **AND** directory-only project choices MUST resolve through the daemon before local skill/MD/observation management actions can run +- **AND** MD ingest controls MUST require a selected validated project directory and canonical project identity before running +- **AND** the daemon MUST reject missing canonical project identity before reading project files +- **AND** UI mutation controls MUST remain disabled while feature state is unknown or disabled +- **AND** UI responses MUST be accepted only when their `requestId` matches the latest request for that management surface + +### Requirement: POST11-R16 New background memory workers MUST be repairable, idempotent, and bounded +Any new post-foundations background worker, including classification, ingest, search indexing, skill sync, skill auto-creation, or telemetry audit persistence, MUST define stale-state repair, bounded retry/backoff, idempotent reprocessing, retention/pruning, and feature-disable behavior. + +- **State variables:** job status, attempt count, next retry, stale threshold, feature flag, retention policy, repair marker. +- **Failure modes:** stuck running jobs, retry storm, duplicate writes, poisoned fallback projections, disabled feature continues writing, unbounded audit growth. +- **Implemented by tasks:** 1.6, 5.1-5.6, 8.2, 8.6, 9.4, 11.5, 12.6, 12.10. +- **Test anchors:** materialization repair tests, worker backoff/idempotency tests, skill auto-creation background tests. + +#### Scenario: worker is interrupted mid-run +- **WHEN** a post-foundations worker is interrupted after marking work in progress +- **THEN** startup or scheduled repair MUST detect stale in-progress state and return it to a retryable or failed state without blocking daemon startup +- **AND** retry MUST be bounded and observable + +#### Scenario: feature is disabled with pending jobs +- **WHEN** a feature flag disables a worker while jobs are pending +- **THEN** the worker MUST stop claiming new jobs for that feature +- **AND** existing data MUST remain readable or safely ignored according to the disabled feature contract + +### Requirement: POST11-R17 Namespace registry and multi-class observations MUST be first-class and scope-bound +Post-foundations memory MUST include a first-class namespace registry and multi-class observation store in the current Wave 1 milestone. Namespace records MUST bind to `MemoryScope` policies from `shared/memory-scope.ts` and MUST NOT use ad hoc scope strings outside that registry. Observation rows MUST represent typed durable memory facts, decisions, preferences, skill candidates, notes, and other closed classes while projections remain the aggregate/search/render surface. + +- **State variables:** namespace id/key, memory scope policy, observation class, content JSON, projection id, source event ids, origin, fingerprint, promotion state, audit action. +- **Failure modes:** cross-scope promotion, duplicate observation writes, class enum drift, projection/observation mismatch, migration backfill interruption, unauthorized namespace access, unauthorized promotion. +- **Implemented by tasks:** 3.7-3.19, 9.1-9.6, 11.5, 12.10. +- **Test anchors:** namespace migration tests, observation write/backfill tests, classification-to-observation tests, scope authorization tests, promotion audit tests. + +#### Scenario: namespace registry is migrated +- **WHEN** existing projection or memory rows are migrated into first-class namespace records +- **THEN** every namespace MUST bind to exactly one registered `MemoryScope` policy through canonical namespace constructors +- **AND** migration MUST NOT widen visibility beyond the scope policy +- **AND** old rows MUST remain readable during lazy backfill + +#### Scenario: typed observation is written +- **WHEN** classification, preference ingest, markdown ingest, or skill review writes durable structured memory +- **THEN** it MUST write an observation with a class from `ObservationClass`, content JSON, source event ids, origin, fingerprint, namespace id, and scope +- **AND** the associated projection aggregate MUST be updated transactionally or through a repairable outbox path +- **AND** markdown-ingested observations MUST NOT remain observation-only; they MUST become visible to authorized startup/search/provider paths through the projection aggregate +- **AND** duplicate observations MUST be idempotently merged or ignored within the same scope + +#### Scenario: observation promotion is requested +- **WHEN** an observation would move from a private scope (`user_private` or `personal`) to `project_shared`, `workspace_shared`, or `org_shared` +- **THEN** the promotion MUST require one explicit authorized action: web UI Promote, CLI `imcodes mem promote`, or admin API `POST /api/v1/mem/promote` +- **AND** the request MUST carry `expectedFromScope` and the promotion transaction MUST reject if the stored source scope differs or the expected scope is missing +- **AND** the promotion MUST write `observation_promotion_audit` +- **AND** the Web UI promotion path MUST disclose the from-scope, to-scope, and audit/visibility consequence before sending the mutation +- **AND** automatic classification or background skill review MUST NOT promote across scopes + + +### Requirement: POST11-R18 Authorization scope policy registry MUST be current-scope work +Post-foundations memory MUST promote authorization scope extensions into the current Wave 1 milestone. The system MUST define `MemoryScope = 'user_private' | 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'` in shared code and MUST migrate daemon, server, and web validation/filtering to that registry. `user_private` is a current-scope addition, not later backlog. Session tree is not a `MemoryScope`; main sessions and sub-sessions share project/session context through namespace/context binding. The registry MUST also expose narrow subtype unions and a `SearchRequestScope` vocabulary (`owner_private`, `shared`, `all_authorized`, or an explicit single `MemoryScope`) so request handling cannot confuse owner-private, legacy personal, and shared scopes. + +- **State variables:** scope name, owner identity fields, canonical repository identity (`canonicalRepoId`), repository alias mapping, project/workspace/org fields, optional namespace/context binding such as root session tree id, replication policy, raw-source access policy, search inclusion/request expansion policy, promotion target policy, feature flag state. +- **Failure modes:** hard-coded old enum, scope silently widened, user-private memory shown to project/workspace/org users, same remote project split by device/local path, unrelated projects merged by unsafe alias, session-tree binding mistaken for a scope, missing migration/backfill, old clients sending legacy `personal`. +- **Implemented by tasks:** 3.7, 3.20-3.25, 4.1-4.4, 8.7, 10.2, 14.2-14.6. +- **Test anchors:** memory scope policy tests, daemon/server scope migration tests, search authorization tests, web/admin scope validation tests. + +#### Scenario: session tree context is evaluated +- **WHEN** memory lookup/startup/bootstrap needs session/sub-session context +- **THEN** the main session and all sub-sessions under the same root session tree MUST share the project/session context available to that tree +- **AND** this sharing MUST be implemented through namespace/context binding such as `root_session_id` / `session_tree_id`, not by adding a new authorization scope +- **AND** sessions outside that root tree MUST NOT receive tree-bound context unless it is also available through existing project/user/shared scopes +- **AND** the binding MUST NOT create server shared projection rows by itself + +#### Scenario: same project is used on multiple devices +- **WHEN** the same signed-in user opens the same git project on two devices +- **AND** both working copies resolve to the same canonical remote repository identity (`canonicalRepoId`, normalized as `host/owner/repo` or through an authorized repository alias) +- **THEN** project-scoped `personal` memory and enrolled shared project memory MUST use that canonical project identity and be visible on both devices when the relevant sync/shared feature is enabled +- **AND** local cwd, session name, sub-session id, and `machine_id` MUST NOT split the project into separate authorization scopes +- **AND** if no usable remote identity exists, local fallback identity MAY remain device-local until explicitly aliased/enrolled to a canonical remote + +#### Scenario: user-private memory is written +- **WHEN** a preference, user-level skill, persona/user fact, or cross-project private observation is created with scope `user_private` +- **THEN** it MUST be visible only to the owning user across projects/workspaces +- **AND** when `mem.feature.user_private_sync=false`, it MUST remain daemon-local and no server write/read job may run +- **AND** when `mem.feature.user_private_sync=true`, it MUST sync only through a dedicated owner-private server route/table with owner-user authorization and idempotency +- **AND** it MUST NOT be inserted into or queried through `shared_context_projections` / project/workspace/org membership filters +- **AND** project/workspace/org/shared search MUST include it only for that same owner when the request explicitly includes `owner_private` or `all_authorized` + +#### Scenario: legacy personal memory is migrated +- **WHEN** existing `personal` rows are migrated into the scope registry +- **THEN** they MUST remain owner-only and project-bound `personal`, keyed by canonical `project_id` / `canonicalRepoId` when a remote exists +- **AND** the same owner using the same canonical project on another device MAY see them when personal sync is enabled +- **AND** automatic migration/backfill MUST NOT reclassify them as `user_private` or widen visibility to other projects/users +- **AND** any later `personal` -> `user_private` movement requires an explicit audited user/admin reclassification path and rollback story + +#### Scenario: search request scope is expanded +- **WHEN** quick search, citation lookup, source lookup, startup selection, MCP read tools, or web/admin validation query memory +- **THEN** authorization MUST be derived from `shared/memory-scope.ts` policy helpers and the request vocabulary (`owner_private`, `shared`, `all_authorized`, or an explicit single scope) +- **AND** `shared` MUST expand only to `personal`, `project_shared`, `workspace_shared`, and `org_shared` according to caller membership; it MUST NOT include `user_private`; `org_shared` requires enterprise membership and is not public/global +- **AND** `all_authorized` MAY include `user_private` only when the caller satisfies the owner policy +- **AND** session-tree inclusion, when needed, MUST be a separate namespace/context binding filter and not a scope expansion +- **AND** project matching MUST use canonical remote-backed project identity and repository aliases, not cwd or machine id +- **AND** bespoke SQL enum lists or duplicated scope literals MUST fail tests + +### Requirement: POST11-R19 Enterprise-wide authored standards MUST use `org_shared` +Enterprise-global coding standards, architecture guidelines, repo playbooks, and reusable policy documents MUST be modeled as `org_shared` authored context bindings inside one enterprise/team. The system MUST NOT introduce a separate `global` scope, `namespace_tier=global`, or any unscoped cross-enterprise memory surface for this purpose. + +- **State variables:** enterprise id, caller enterprise role, document id/version id, binding id, binding mode, derived scope, optional repo/language/path filters, active/superseded state, feature flag state. +- **Failure modes:** cross-enterprise visibility, non-admin mutation, required binding dropped silently, filters widening visibility, org document mistaken for public global memory, processed projection losing project provenance, disabled-feature inventory leak. +- **Implemented by tasks:** 4.1-4.4, 12.11-12.14, 14.3-14.6. +- **Test anchors:** `server/test/shared-context-org-authored-context.test.ts`, shared-context disabled-feature tests, shared-context control-plane tests, runtime authored-context selection tests, web/i18n diagnostics tests. + +#### Scenario: org-wide standard is created +- **WHEN** an enterprise owner/admin creates a coding standard or playbook intended for the whole enterprise +- **THEN** the document version MUST be bound with `enterprise_id` set, `workspace_id = NULL`, `enrollment_id = NULL`, and derived scope `org_shared` +- **AND** only members of that enterprise may receive it at runtime +- **AND** non-members or other enterprises MUST receive the same external not-found/unauthorized shape without inventory leakage + +#### Scenario: org-wide standard is selected for a session +- **WHEN** a member starts or sends in a session whose canonical project, language, and file path match an active org-shared binding +- **THEN** the runtime authored-context resolver MUST include that org-shared binding after more specific project/workspace bindings +- **AND** `required` bindings MUST be preserved or dispatch must fail with the existing required-authored-context error +- **AND** `advisory` bindings MAY be budget-trimmed only with diagnostics/telemetry +- **AND** optional repo/language/path filters MUST only narrow applicability within the caller enterprise + +#### Scenario: org-wide authored standards are disabled +- **WHEN** `mem.feature.org_shared_authored_standards=false` +- **THEN** creating, updating, activating, or binding an org-wide authored standard MUST fail closed with the documented disabled envelope +- **AND** runtime selection MUST skip org-wide authored standards without blocking ordinary send ack +- **AND** the disabled response MUST NOT reveal whether any org-wide standard exists + +#### Scenario: org-shared processed memory exists +- **WHEN** processed project experience is promoted or written with scope `org_shared` +- **THEN** it MUST retain canonical `project_id` / `canonicalRepoId`, source ids, origin, fingerprint, and authorization metadata +- **AND** it MUST remain visible only inside the enterprise +- **AND** it MUST NOT become an unowned global pool or lose project provenance + +### Requirement: POST11-R20 Memory management RPCs MUST be single-cast and server-authorized +Post-1.1 memory management WebSocket requests and responses MUST use the closed request/response vocabulary in `shared/memory-ws.ts`, including project-identity resolution used by the management UI. A management request MUST include a unique `requestId`; the server bridge MUST track that pending request and inject a server-derived management context before forwarding to the daemon. Daemon handlers MUST authorize using that context rather than trusting client-supplied `actorId`, `userId`, project, workspace, or org identity; missing/invalid management context MUST fail closed for all enabled management operations. Browser project/workspace/org fields are request hints only and MUST NOT enter daemon `boundProjects` unless the server verifies membership/enrollment for the exact canonical repo, workspace, or org. Management responses MUST be routed only to the pending requester for the matching `requestId`; unrouted responses MUST be dropped and counted, never broadcast to all browser clients. Personal-memory browse responses MUST include an authorized, bounded `projects` index so the UI can populate project filters from actual memory without requiring manual IDs or full table scans. + +- **State variables:** request type, response type, requestId, pending socket, management actor/user/role, record creator/owner/updater metadata, bound project hints, project index summary, project resolution status, feature state, owner id, observation scope, skill path, canonical project identity, processed-memory mutation state, pinned-note id. +- **Failure modes:** cross-tab/body leak, stale response overwrites current UI state, duplicate requestId hijack, missing context fallback, bridge context-construction failure leaving a stuck pending request, client-forged actor/user identity, client-provided project hints promoted into authorization bindings, preference owner mismatch, legacy display metadata granting shared mutation authority, record creator confused with admin role, personal-memory owner/scope leakage, unauthorized manual memory create/edit/pin/delete, unauthorized private/shared observation query, unauthorized observation edit/delete/promotion, observation delete accidentally cascading to a processed projection, stale linked projection embeddings after observation edit, raw-source search leak, symlink or oversize skill registry path, invalid project directory, canonical project mismatch, disabled feature mutation, arbitrary browser-supplied directory accepted as a memory project, all-project memory stats non-zero but project dropdown empty because project summaries are absent, project summary leakage across owner/enterprise authorization boundaries. +- **Implemented by tasks:** 11.10-11.13, 12.17-12.20, 15.1-15.16, 16.1-16.2, 17.1-17.11. +- **Test anchors:** `server/test/bridge-memory-management.test.ts`, `server/test/shared-context-processed-remote.test.ts`, `test/daemon/command-handler-memory-context.test.ts`, `test/daemon/command-handler-transport-queue.test.ts`, `web/test/components/SharedContextManagementPanel.test.tsx`, `web/test/i18n-memory-post11.test.ts`, `test/context/skill-registry-resolver.test.ts`, `test/context/context-observation-store.test.ts`, `test/context/memory-feature-flags.test.ts`. + +#### Scenario: management response would otherwise broadcast +- **WHEN** browser A sends a management request and browser B is connected to the same bridge +- **THEN** the daemon response for A's `requestId` MUST be delivered only to browser A +- **AND** browser B MUST NOT receive the response body or metadata +- **AND** a response with no pending `requestId` MUST be dropped with `mem.bridge.unrouted_response` + +#### Scenario: browser forges management identity +- **WHEN** a management request carries client-supplied `actorId`, `userId`, role, owner fields, `_memoryManagementContext`, or legacy `managementContext` that differ from the authenticated browser context +- **THEN** the bridge/daemon MUST derive actor and owner from the server-injected management context +- **AND** elevated management roles MUST come only from server-side membership records for the requested enterprise/workspace/project binding +- **AND** the bridge MUST NOT add a canonical repo, workspace, or org to `boundProjects` unless that same server membership/enrollment check succeeds; unverified browser hints remain in the request payload only as hints and do not authorize daemon shared-scope access +- **AND** generic `projectId` MUST NOT be silently treated as canonical repo identity for role derivation; project-scoped management MUST use explicit `canonicalRepoId` plus a verified project directory binding before filesystem access +- **AND** preference create/update/delete, observation query/update/delete/promotion, and processed-memory manual create/update/pin/archive/restore/delete MUST fail closed or filter records when the derived context is not authorized; record-level `ownerUserId` / `createdByUserId` MUST be derived from the authenticated context at creation and MUST NOT be accepted from browser payloads +- **AND** legacy/display metadata fields such as `userId`, `createdBy`, `authorUserId`, and `updatedBy` MUST NOT grant preference, observation, or shared processed-memory mutation authority +- **AND** management search, archive, restore, delete, update, pin, skill preview/delete/rebuild, and manual MD ingest MUST apply the same derived-context authorization before returning data or mutating state +- **AND** management quick search and personal-memory management queries MUST NOT expose raw source text through `includeRaw`, MUST compute stats/pagination only after authorization, and MUST NOT return another user's `personal` / `user_private` rows from the same project +- **AND** personal-memory management queries MUST filter records, stats, pending records, and semantic results by the server-derived owner id plus `scope='personal'`; local daemon storage MUST maintain indexed namespace filter columns for processed projections, staged events, dirty targets, and jobs so these owner/project filters are applied in SQL before result construction rather than by unbounded full-table scans; missing daemon-side management context MUST return the same `PERSONAL_RESPONSE` shape with empty records/stats and a shared error code +- **AND** manual processed-memory creation MUST require non-empty text plus explicit canonical project identity and an authorized canonical project binding, write origin `user_note`, write creator/owner metadata, and create/update linked observation/projection state consistently +- **AND** processed-memory edit MUST update projection summary/content hash, linked observation text/fingerprint, `updatedByUserId`, and clear stale embeddings; permanent delete MUST remove linked observations; archive/restore/delete/update/pin MUST invalidate runtime memory cache with projection-typed invalidation; pin MUST create or update a deterministic `manual_pin` pinned note for the projection rather than appending duplicates +- **AND** observation edit MUST update linked projection text/content hash and clear stale embeddings; observation delete MUST delete only the observation row and MUST NOT cascade to the linked processed projection +- **AND** missing observations and stale `expectedFromScope` checks MUST return typed shared error codes instead of generic action failure +- **AND** private records remain mutable only by their owner; shared records may be mutated by an authorized admin or by the record creator/owner when the namespace is otherwise visible; admin mutations MUST preserve original creator metadata +- **AND** missing/unauthorized results MUST preserve the same safe envelope + +#### Scenario: bridge cannot derive management context after registering a request +- **WHEN** the bridge accepts a memory-management request and context construction or role derivation fails before daemon forwarding +- **THEN** the bridge MUST clear the pending request, send an error only to the requesting browser, and MUST NOT forward a partially authorized request or broadcast the error + +#### Scenario: management feature state is unknown or disabled +- **WHEN** the UI has not yet received daemon-resolved feature state, or the relevant feature is effectively disabled by dependency folding +- **THEN** mutation buttons MUST remain disabled in the UI +- **AND** forced daemon mutation/read-body requests MUST fail closed with shared error codes and no persistent writes/background work +- **AND** processed-memory management create/update/archive/restore/delete/pin MUST fail closed when `mem.feature.observation_store=false`, because those mutations create or update projection/observation consistency state + +#### Scenario: memory project selector resolves a directory +- **WHEN** the web Memory tab has a directory-only project option from an active/recent daemon session +- **THEN** it MUST send a `memory.project.resolve` request with a unique `requestId` +- **AND** the daemon MUST accept only daemon-known project directories, verify the path is a directory, derive `canonicalRepoId` from the repository remote identity, and reject mismatches before the UI enables local filesystem tools +- **AND** the web UI MUST ignore stale project-resolve responses whose `requestId` is no longer current +- **AND** the picker MUST show both canonical ID and directory for resolved projects and explain canonical-only projects as cloud/shared filtering only until a local directory is resolved + +#### Scenario: memory project selector is populated from memory indexes +- **WHEN** local daemon, personal cloud, enterprise/shared, or semantic memory responses contain authorized project summaries +- **THEN** the response MUST include a bounded `projects` array with canonical `projectId`, display name when available, record counters, pending count when available, and `updatedAt` metadata +- **AND** project summaries MUST be computed after owner/scope/enterprise authorization and must not reveal unauthorized project ids, counts, source text, or raw paths +- **AND** the web UI MUST merge those summaries into the project selector without replacing the full option set with only the currently filtered project +- **AND** selecting one of those projects MUST filter memory views by canonical id while preserving the all-project option + +#### Scenario: skill and markdown management inputs are untrusted +- **WHEN** a skill registry entry points outside managed roots, through a symlink directory, to a non-file, or over the configured byte cap +- **THEN** management preview/runtime resolver MUST fail closed with shared error/counter behavior and MUST NOT read the file +- **AND** registry files over the configured byte or entry limit MUST be refused before parsing unbounded content +- **AND** project-scoped skill registry query/rebuild/preview/delete MUST require explicit `canonicalRepoId`, a project directory, and verified repository identity before reading or mutating project skill files +- **WHEN** manual markdown ingest provides an invalid project directory, missing canonical project identity, mismatched canonical repository identity, or unsupported filesystem scope +- **THEN** the daemon MUST reject before reading project files and MUST NOT silently downgrade scope diff --git a/openspec/changes/memory-system-post-1-1-integration/tasks.md b/openspec/changes/memory-system-post-1-1-integration/tasks.md new file mode 100644 index 000000000..135ae0688 --- /dev/null +++ b/openspec/changes/memory-system-post-1-1-integration/tasks.md @@ -0,0 +1,297 @@ +## 1. Scope, traceability, and cross-wave foundations gates + +- [x] 1.1 Confirm the current completion milestone is Wave 1-5; keep Wave 6+ candidates as non-checkbox backlog until promoted by spec/task update. +- [x] 1.2 Keep `docs/plan/mem1.1.md` synchronized as historical rationale and point implementation to this OpenSpec change as the authoritative contract. +- [x] 1.3 Maintain the traceability matrix below; every `POST11-R*` requirement MUST have at least one implementation task and one test/validation anchor before implementation starts. +- [x] 1.4 Document foundations deltas in `design.md` while `memory-system-1.1-foundations` is active and cumulative `openspec/specs/daemon-memory-pipeline/spec.md` is unavailable. +- [x] 1.5 Archive gate: before archiving this change, re-check cumulative OpenSpec state. If `daemon-memory-pipeline` exists, create `specs/daemon-memory-pipeline/spec.md` with `## MODIFIED Requirements` for send ack, priority controls, startup selection, render payloads, and citation-aware recall deltas, then rerun `openspec validate memory-system-post-1-1-integration`. +- [x] 1.6 Run the foundations regression matrix for every wave PR: daemon-receipt send ack, `/compact` SDK-native pass-through, `/stop` and approval/feedback priority, fail-open recall/bootstrap, provider send-start watchdog, materialization repair, redaction, scope filtering, source lookup authorization, and same-shape missing/unauthorized/disabled lookup responses. +- [x] 1.7 Add shared constants inventory tasks to the first implementation PR: `shared/memory-scope.ts`, `shared/memory-origin.ts`, `shared/memory-namespace.ts`, `shared/memory-observation.ts`, `shared/send-origin.ts`, `shared/feature-flags.ts`, `shared/memory-counters.ts`, `shared/skill-envelope.ts`, `shared/skill-review-triggers.ts`, `shared/builtin-skill-manifest.ts`, `shared/memory-defaults.ts`, and `web/src/i18n/locales/index.ts`; `shared/memory-scope.ts` MUST export narrow scope subtypes and `SearchRequestScope`. +- [x] 1.8 Split security validation into atomic gates: redaction, scope filtering, source lookup authorization, missing-vs-unauthorized-vs-disabled response shape, metadata suppression, count suppression, drift suppression, and raw-source suppression. +- [x] 1.9 Migration/backfill rule: no current post-1.1 requirement may be deferred because it requires daemon SQLite migration, server PostgreSQL migration, backfill, migration-number coordination, or rollback/repair work; instead add the migration, rollback, repair, and tests to the same wave. +- [x] 1.10 Test-anchor rule: each path below is either an existing test to update or a new test file to create; implementation PRs must not claim completion against phantom paths. +- [x] 1.11 Acceptance harness rule: update the canonical acceptance wrapper so it validates `memory-system-post-1-1-integration` directly, not only `memory-system-1.1-foundations`. + +### Traceability matrix + +| Requirement | Implementation tasks | Expected code areas | Test anchors / validation | +| --- | --- | --- | --- | +| POST11-R1 foundations liveness | 1.6, 8.1-8.8, 14.2 | `src/daemon/*`, `src/agent/*`, `src/context/*`, server bridge where relevant | `server/test/ack-reliability.test.ts`, `test/ack-reliability-e2e.test.ts`, `test/daemon/command-handler-transport-queue.test.ts`, `test/daemon/transport-session-runtime.test.ts`, `test/agent/runtime-context-bootstrap.test.ts`, `web/test/use-timeline-optimistic.test.ts` | +| POST11-R2 fingerprints | 2.1-2.7 | `shared/memory-fingerprint.ts`, daemon/server write paths, migrations | `test/context/memory-fingerprint-v1.test.ts`, `test/fixtures/fingerprint-v1/**` | +| POST11-R3 origins | 3.1-3.6 | `shared/memory-origin.ts`, daemon SQLite, server migrations, write APIs | origin migration/write tests, reserved-origin rejection tests, search/UI origin tests | +| POST11-R4 feature flags | 4.1-4.9 | `shared/feature-flags.ts`, config propagation, daemon/server/web observers | `test/context/memory-feature-flags.test.ts`, server/web disabled-feature tests, dependency/default coverage tests | +| POST11-R17 namespace/observations | 3.7-3.19, 9.1-9.6, 11.5, 12.10 | `shared/memory-namespace.ts`, `shared/memory-observation.ts`, daemon SQLite migrations, server migrations, projection/observation write APIs | namespace migration tests, observation write/backfill tests, classification-to-observation tests, scope authorization tests, promotion audit tests | +| POST11-R18 authorization scope registry | 3.7, 3.20-3.25, 4.1-4.4, 8.7, 10.2 | `shared/memory-scope.ts`, shared validators, daemon/server/web scope filters, migrations | memory scope policy tests, daemon/server scope migration tests, search authorization tests, web/admin scope validation tests | +| POST11-R19 org-shared authored standards | 4.1-4.4, 12.11-12.14, 14.3-14.6 | `shared/feature-flags.ts`, `server/src/routes/shared-context.ts`, `server/src/routes/server.ts`, shared-context document/version/binding migrations, runtime authored-context resolver, web diagnostics | `server/test/shared-context-org-authored-context.test.ts`, shared-context disabled-feature tests, shared-context control-plane tests, runtime authored-context selection tests, web/i18n diagnostics tests | +| POST11-R20 memory management RPC auth/routing | 11.10-11.13, 12.17-12.20, 15.1-15.16 | `shared/memory-ws.ts`, `server/src/ws/bridge.ts`, `src/daemon/command-handler.ts`, `src/store/context-store.ts`, `shared/context-types.ts`, `src/context/memory-search.ts`, server/shared memory routes, management UI | `server/test/bridge-memory-management.test.ts`, `server/test/shared-context-processed-remote.test.ts`, `test/daemon/command-handler-memory-context.test.ts`, `test/daemon/command-handler-transport-queue.test.ts`, `test/context/memory-search.test.ts`, `web/test/components/SharedContextManagementPanel.test.tsx`, skill registry/feature flag tests | +| POST11-R5 telemetry | 5.1-5.7 | `shared/memory-counters.ts`, telemetry enqueue/sink | telemetry sink timeout/reject tests, counter registry tests | +| POST11-R6 startup budget | 6.1-6.6 | startup selection/render modules, `shared/memory-defaults.ts` | `test/context/startup-memory.test.ts`, startup over-budget fixture tests, `test/spec/design-defaults-coverage.test.ts` | +| POST11-R7 render policy | 7.1-7.5 | render policy module, skill/citation renderers | render policy tests, `test/context/skill-envelope.test.ts` | +| POST11-R8 self-learning | 9.1-9.6 | compression/materialization pipeline | classification/dedup tests, materialization repair tests | +| POST11-R9 quick search security | 10.1-10.8, 1.8 | server/daemon search, scope filters, web palette | `server/test/memory-search-auth.test.ts`, `test/context/memory-search-semantic.test.ts`, web quick-search tests | +| POST11-R10 citations/drift/cite-count | 10.3-10.14 | citation storage/API, idempotency store, cite-count columns or counter table, ranking, web citation renderer | `test/context/memory-citation-drift.test.ts`, `test/context/memory-cite-count.test.ts`, citation web tests, source lookup auth tests | +| POST11-R11 MD ingest | 11.1-11.7 | MD parser/ingest worker, startup bootstrap | MD ingest tests, startup compatibility tests | +| POST11-R12 preferences trust | 11.4-11.9 | send command schema, daemon preference parser, web/CLI send origin, preference idempotency | `test/context/preferences-trust-origin.test.ts`, ack tests | +| POST11-R13 skills storage/render/review | 12.1-12.10 | skill loader/store, manifest, render policy, background skill review | `test/context/skill-precedence.test.ts`, `test/context/skill-envelope.test.ts`, package manifest tests, skill auto-creation background tests | +| POST11-R14 skill admin | 12.4-12.9 | server/admin API, auth checks, sanitizer | admin skill auth tests, sanitizer fixtures | +| POST11-R15 web i18n/constants | 10.6, 12.8, 14.4, 14.9, 15.13, 15.16 | `web/src/i18n/*`, shared constants, web UI, `shared/context-types.ts`, `shared/memory-project-options.ts` | `web/test/i18n-coverage.test.ts`, `web/test/components/SharedContextManagementPanel.test.tsx`, web feature tests | +| POST11-R16 worker repair/backoff | 5.1-5.7, 8.2, 8.6, 9.4, 11.5, 12.6, 12.10 | worker/job tables, repair hooks, retention sweepers | materialization repair tests, worker backoff/idempotency tests, skill auto-creation background tests | + +## 2. Wave 1 — stable fingerprint foundation + +**Prerequisites:** foundations archive/source identity remains green. +**Satisfies:** POST11-R2. + +- [x] 2.1 Define canonical `shared/memory-fingerprint.ts` API: `computeMemoryFingerprint({ kind, content, scopeKey?, version?: 'v1' }): string` with `FingerprintKind = 'summary' | 'preference' | 'skill' | 'decision' | 'note'`. +- [x] 2.2 Mark older summary-only helpers as deprecated/internal and ensure new call sites use the canonical API. +- [x] 2.3 Add kind-specific normalization rules: summary, preference, skill front matter stripping, decision, and note handling. +- [x] 2.4 Migration: add nullable/backfillable fingerprint columns/indexes to daemon SQLite and server PostgreSQL surfaces that store projections/preferences/skills, using the next available migration number at implementation time. +- [x] 2.5 Failure handling: lazy backfill must not block daemon startup or send ack; eager backfill, if provided, must be explicit, bounded, and restartable. +- [x] 2.6 Tests: add byte-identical daemon/server fingerprint fixtures covering CJK, emoji, RTL, whitespace, front matter, punctuation, and scope separation. +- [x] 2.7 Acceptance: same-scope identical normalized content dedups; different scopes never merge. + +## 3. Wave 1 — origin metadata, namespace registry, and observation foundation + +**Prerequisites:** 2.x fingerprint design. +**Satisfies:** POST11-R3, POST11-R17, POST11-R18. + +- [x] 3.1 Define closed `MEMORY_ORIGINS` in `shared/memory-origin.ts`: `chat_compacted`, `user_note`, `skill_import`, `manual_pin`, `agent_learned`, `md_ingest`. Reserve but do not emit `quick_search_cache` until a future cache contract defines TTL/invalidation/auth semantics. +- [x] 3.2 Migration: add origin metadata to daemon processed local rows, server shared projections, pinned note mirrors, MD imports, preferences, and skills as applicable. +- [x] 3.3 Implementation: require explicit origin in new write APIs; only migration/backfill code may apply defaults. +- [x] 3.4 Failure handling: reject or no-op writes that cannot determine origin outside migration boundaries. +- [x] 3.5 Tests: cover backfill, explicit write paths, invalid origin rejection, reserved cache-origin rejection, and UI/search access to origin without parsing summary text. +- [x] 3.6 Split already-existing daemon-local baseline from new post-1.1/server parity work to avoid duplicate daemon migrations. +- [x] 3.7 Add `shared/memory-scope.ts` with `MemoryScope = 'user_private' | 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'`, narrow subtypes (`OwnerPrivateMemoryScope`, `ReplicableSharedProjectionScope`, `AuthoredContextScope`), `SearchRequestScope = 'owner_private' | 'shared' | 'all_authorized' | MemoryScope`, and per-scope policy metadata: required/forbidden identity fields, replication behavior, request expansion, raw-source access, and promotion targets. +- [x] 3.8 Add `shared/memory-namespace.ts` and define canonical namespace constructors that bind namespace keys to `MemoryScope` policies; project-bound namespaces MUST use canonical remote-backed `canonicalRepoId`/`project_id`; include `root_session_id`/`session_tree_id` only for session-tree context binding; do not require `projectId` globally for `user_private`; do not introduce ad hoc scope strings or parallel namespace tiers. +- [x] 3.9 Add `shared/memory-observation.ts` with `ObservationClass = 'fact' | 'decision' | 'bugfix' | 'feature' | 'refactor' | 'discovery' | 'preference' | 'skill_candidate' | 'workflow' | 'code_pattern' | 'note'` and typed content JSON validation. `note` is canonical; do not introduce `memory_note`. +- [x] 3.10 Migration: add daemon SQLite namespace and observation tables, plus matching server PostgreSQL tables/migrations using the next available migration numbers at implementation time. +- [x] 3.11 Namespace schema minimum: implement `context_namespaces(id, tenant_id/local_tenant, scope, user_id, root_session_id/session_tree_id, session_id, workspace_id, project_id, org_id, key, visibility, created_at, updated_at)` plus unique/index constraints preventing duplicate canonical namespace keys in the same tenant/scope context; for project-bound scopes `project_id` is canonical remote identity, not cwd/machine/session id. +- [x] 3.12 Observation schema minimum: implement `context_observations(id, namespace_id, scope, class, origin, fingerprint, content_json, text_hash, source_event_ids_json, projection_id, state, confidence, created_at, updated_at, promoted_at)` plus idempotency indexes over namespace/class/fingerprint/text hash. +- [x] 3.13 Projection/observation write semantics: new durable memory writes must write typed observations transactionally with projection aggregate updates or through a repairable outbox path, preserving source event ids, origin, fingerprint, namespace id, and scope. +- [x] 3.14 Backfill: create namespace records for existing projections and lazily backfill observation rows where class/source information is available; old projections must remain readable during backfill. +- [x] 3.15 Scope safety: automatic classification may preserve source scope but must not promote observations from private scopes (`user_private`, `personal`) to shared scopes without explicit authorized user/admin action. +- [x] 3.16 Promotion audit: implement `observation_promotion_audit(id, observation_id, actor_id, action, from_scope, to_scope, reason, created_at)` and allow only web UI Promote, CLI `imcodes mem promote`, and admin API `POST /api/v1/mem/promote` for cross-scope promotion. +- [x] 3.17 Failure handling: interrupted migration/backfill must be restartable; duplicate observations must be idempotently merged or ignored within the same scope. +- [x] 3.18 Tests: namespace migration, observation write/backfill, projection/observation consistency, class validation, idempotency, cross-scope promotion rejection, and promotion audit. +- [x] 3.19 Repair: add a consistency check/repair path for projection rows whose observation outbox/transaction failed midway. +- [x] 3.20 Scope migration: migrate daemon/server/web validators and storage schemas from hard-coded old scope unions to `shared/memory-scope.ts`, preserving legacy `personal` behavior. +- [x] 3.21 Lock project/session context binding: main session and all sub-sessions under the same root share the same project/session context without introducing a new `MemoryScope`; same signed-in user on another device sees the same project-bound memory when canonical `canonicalRepoId` matches and sync/shared policy allows it; sessions outside the root do not receive tree-bound context unless it is also available through existing project/user/shared scopes. +- [x] 3.22 Add `user_private` support: user-bound cross-project private observations/preferences/skills; daemon-local when `mem.feature.user_private_sync=false`; dedicated owner-private server sync route/table with owner-only auth/idempotency when true; owner-only search/startup selection across projects; no writes to `shared_context_projections`. +- [x] 3.23 Legacy backfill: existing `personal` rows stay owner-only and project-bound; automatic migration/backfill MUST NOT classify them as `user_private`; any explicit reclassification requires audited user/admin action and rollback. +- [x] 3.24 Scope filter helpers: quick search, citation lookup, source lookup, startup selection, MCP read tools, web/admin validation, and server SQL must use shared scope policy helpers and `SearchRequestScope` expansion rather than duplicated string lists. +- [x] 3.25 Scope tests: `(NEW) test/context/memory-scope-policy.test.ts`, `(NEW) test/context/session-tree-context-binding.test.ts`, `(NEW) test/context/project-remote-identity-sync.test.ts`, `(NEW) test/context/user-private-scope.test.ts`, `(NEW) test/context/scope-migration.test.ts`, `(NEW) server/test/memory-scope-replication-check.test.ts`, and `(NEW) server/test/memory-scope-authorization.test.ts` covering policy registry, legacy personal compatibility, same-root session tree context binding, same-user same-remote cross-device project visibility, remote alias equivalence, dedicated user-private sync path, owner-only cross-project search, shared-scope membership filtering, promotion target validation, and no hard-coded old enum literals in new code. + +## 4. Wave 1 — feature flags and kill switches + +**Prerequisites:** origin/fingerprint/scope/namespace design for feature-scoped data. +**Satisfies:** POST11-R4. + +- [x] 4.1 Add `shared/feature-flags.ts` with `mem.feature.scope_registry_extensions`, `mem.feature.user_private_sync`, `mem.feature.self_learning`, `mem.feature.namespace_registry`, `mem.feature.observation_store`, `mem.feature.quick_search`, `mem.feature.citation`, `mem.feature.cite_count`, `mem.feature.cite_drift_badge`, `mem.feature.md_ingest`, `mem.feature.preferences`, `mem.feature.skills`, `mem.feature.skill_auto_creation`, and `mem.feature.org_shared_authored_standards`. +- [x] 4.2 Implement or document runtime source-of-truth precedence: runtime config override > persisted local/server config > environment startup default > registry default. +- [x] 4.3 Encode dependencies: `observation_store` requires `namespace_registry`; `citation` requires `quick_search`; `cite_count` and `cite_drift_badge` require `citation`; `skill_auto_creation` requires `skills` and `self_learning`; `org_shared_authored_standards` requires scope registry extensions and shared-context document/version/binding migrations; `namespace_registry` observes scope policies; `scope_registry_extensions` gates new `user_private` writes while preserving legacy scopes; `user_private_sync` requires `scope_registry_extensions`, `namespace_registry`, and `observation_store`. +- [x] 4.4 Wire feature observers so disabled means no background work, no persistent writes, no new reads/RPCs for that feature, and pre-feature or same-shape disabled user-visible behavior. +- [x] 4.5 Failure handling: flag read failure fails closed for new features and never blocks ordinary send ack. +- [x] 4.6 Gate cite-count with `mem.feature.cite_count`; disabled mode stores no new count increments and ignores existing counts in ranking without dropping data. +- [x] 4.7 Gate skill review with `mem.feature.skill_auto_creation`; disabled mode claims no review jobs and creates/updates no skills. +- [x] 4.8 Tests: disabled feature paths skip writes/jobs; runtime disable stops new work within propagation target; dependency-disabled children remain effectively disabled. +- [x] 4.9 Ensure flags are shared constants, not duplicated daemon/server/web literals. +- [x] 4.10 Add daemon-persisted management overrides for feature flags: `memory.features.set` requires management context, validates closed registry names, cascades enable requests to dependencies, persists requested values above env startup defaults, returns requested/effective/source/dependency metadata, fails closed on missing context, malformed requests, or config write failures, and covers persistence plus dependency-blocked semantics in daemon tests. + +## 5. Wave 1 — telemetry and silent-failure tracking + +**Prerequisites:** feature flags for rollout safety. +**Satisfies:** POST11-R5, POST11-R16. + +- [x] 5.1 Add `shared/memory-counters.ts` with the closed counter registry from `design.md`, including citation count, preference duplicate/reject, skill review throttle/dedupe/failure, and observation promotion counters. +- [x] 5.2 Design async bounded telemetry buffer, sampling, retention, and PII/secrets boundaries. +- [x] 5.3 Implement non-blocking metric/audit enqueue path; sink failure must not affect memory behavior. +- [x] 5.4 Instrument intentional soft-fail paths in startup memory, search, citation, cite-count, MD ingest, skills, skill review, preferences, materialization, observations, and classification. +- [x] 5.5 Failure handling: buffer overflow drops/samples predictably without throwing in hot paths. +- [x] 5.6 Tests: telemetry sink timeout/reject does not block send, materialization, search, citation, skill load, skill review, MD ingest, or shutdown; labels reject free-form identifiers. +- [x] 5.7 Retention: define and test retention/pruning for persistent audit/idempotency tables introduced by this change. + +## 6. Wave 1 — startup budget and named-stage selection + +**Prerequisites:** telemetry for overrun visibility; render policy draft. +**Satisfies:** POST11-R6. + +- [x] 6.1 Add `shared/memory-defaults.ts` mirroring the `design.md` `design-defaults` JSON5 block. +- [x] 6.2 Add `test/spec/design-defaults-coverage.test.ts` to fail when design defaults drift from shared constants. +- [x] 6.3 Refactor startup selection into collect, prioritize, apply quotas, trim, dedup, render stages. +- [x] 6.4 Failure handling: stage failure omits that source and emits telemetry; ordinary send ack remains independent. +- [x] 6.5 Tests: over-budget fixtures trim in priority order and final output stays within budget. +- [x] 6.6 Acceptance: existing startup memory behavior remains compatible when new sources are disabled. + +## 7. Wave 1 — typed render policy + +**Prerequisites:** startup stage API. +**Satisfies:** POST11-R7. + +- [x] 7.1 Define render kinds `summary`, `preference`, `note`, `skill`, `pinned`, and `citation_preview`. +- [x] 7.2 Centralize per-kind render functions and prohibit ad-hoc formatting in feature code. +- [x] 7.3 Add `shared/skill-envelope.ts` constants and delimiter collision policy. +- [x] 7.4 Failure handling: render failure for one item drops that item with telemetry, not the whole send/startup path. +- [x] 7.5 Tests: pinned remains verbatim, skill is enveloped/capped, delimiter collisions are escaped/rejected, citation preview omits unauthorized raw source, and shared constants are used. + +## 8. Wave 1 — sync semantics and hardening gates G1-G6 + +**Prerequisites:** feature flags and telemetry. +**Satisfies:** POST11-R1, POST11-R16, operational hardening. + +- [x] 8.1 Send ack matrix: test ack before pending relaunch, transport lock, bootstrap, recall, embedding, feature-flag read, MD ingest, skill load, quick-search/citation lookup, telemetry, skill review, and provider send-start. +- [x] 8.2 Recall/bootstrap degrade: timeout/failure still sends original user message to SDK/provider without failed memory payload and without spinning. +- [x] 8.3 `/compact`: remains SDK-native pass-through; no daemon-side synthetic compaction or interception; every transport receives slash controls as raw provider-control payloads without daemon-added startup memory, per-turn recall, preference preambles, authored context, or extra per-turn system prompt; Codex SDK maps the raw command to app-server `thread/compact/start` instead of sending it as model text; Codex SDK settles runtime busy state from `thread/compacted`, `contextCompaction` completion, `turn/completed`, status-idle, or a bounded accepted/no-signal fallback, accepts camelCase/snake_case thread/turn identifiers, and emits a bounded retryable error instead of leaving `Agent working...` forever. +- [x] 8.4 `/stop` and approval/feedback: priority path bypasses normal send locks, memory work, and provider cancel waits. +- [x] 8.5 Materialization/worker repair: stale jobs reset, dirty pending refs clear, active recall contains no local-fallback/raw-transcript pollution. +- [x] 8.6 Persistent audit/telemetry/idempotency retention sweeper exists for any persistent audit/idempotency table introduced by this change. +- [x] 8.7 G1: add concurrent-write retry or optimistic concurrency tests for new write paths that update projections/preferences/skills/cite-counts/observations. +- [x] 8.8 Add a Codex SDK final injected-context cap: default 32,000 chars for daemon-added context, bounded env override, preserve user turn text, and cover with provider regression tests so memory/preference/skill/MD context cannot silently trigger repeated SDK auto-compaction. +- [x] 8.8 G3/G6: per-feature sanitizer and kill-switch wiring must land in the same PR as each feature or earlier. + +## 9. Wave 2 — self-learning memory + +**Prerequisites:** 2.x, 3.x, 4.x, 5.x, 7.x, 8.x. +**Satisfies:** POST11-R8. + +- [x] 9.1 Define classification and dedup-decision output enums, storage fields, startup-state tags, and scope constraints. +- [x] 9.2 Add classify/dedup/durable-signal phases to the existing isolated compression/materialization pipeline; do not create a new foreground agent/session. +- [x] 9.3 Add cold/warm/resumed startup-state switching using named-stage startup policy and budget caps; render policy remains owned by 7.x. +- [x] 9.4 Failure handling: classification/dedup failures must not block ordinary send, write fallback pollution, or delete retryable staged events incorrectly. +- [x] 9.5 Tests: scope-bounded classification, dedup source-id union, redaction/pinned preservation, failure degrade, startup state switching. +- [x] 9.6 Ensure feature flag disablement stops new classification/dedup work. + +## 10. Wave 3 — quick search, citations, cite-count, and fast-path reads + +**Prerequisites:** fingerprint, origin, namespace/observation, render policy, feature flags, scope helpers. +**Satisfies:** POST11-R9, POST11-R10, POST11-R15. + +- [x] 10.1 Define quick-search result shape, ranking inputs, rate/latency budget, authorized preview format, and same-shape disabled envelope. +- [x] 10.2 Use existing/shared scope filtering helpers for all server/daemon memory search queries; do not write bespoke cross-scope predicates. +- [x] 10.3 Define same-shape user-facing missing/unauthorized/disabled lookup envelope and forbid role diagnostics, source counts, hit counts, drift metadata, raw source text, and cross-scope ids unless authorized. +- [x] 10.4 Add citation insertion by projection identity and per-insertion `created_at`; no raw source snapshot in current wave. +- [x] 10.5 Add citation identity/idempotency storage. Authoritative store derives the key; untrusted clients must not provide it. Required properties: same citing message retry/replay dedupes; different citing message for same authorized projection increments once. +- [x] 10.6 If stable citing message identity is available, use `sha256("cite:v1:" + scope_namespace + ":" + projection_id + ":" + citing_message_id)`; otherwise add a preliminary stable `citing_message_id` task before cite-count can be enabled. +- [x] 10.7 Add drift badge using canonical persistent `content_hash` captured at citation time and recomputed from normalized projection content; daemon/server projection write paths must persist the marker, and maintenance writes/idempotent upserts that do not change normalized content must not change the hash or create false drift. +- [x] 10.8 Web gate: all user-visible strings use `t()` and every locale in `SUPPORTED_LOCALES`; shared protocol/status strings use shared constants. +- [x] 10.9 Tests: search scope isolation, full JSON shape equality for unauthorized/missing/disabled, citation insertion, drift badge, no raw source in preview, web i18n/a11y. +- [x] 10.10 Cite-count migration: add daemon SQLite and server PostgreSQL `cite_count` storage or an auxiliary citation counter table using next available migration numbers, plus lazy backfill/defaults where existing projections lack counts. +- [x] 10.11 Cite-count behavior: increment at most once per citation idempotency key; retries/replays must not inflate counts; unauthorized/missing citation attempts must not reveal or increment counts; ranking must use cite_count only after scope filtering. +- [x] 10.12 Ranking integration: when `mem.feature.cite_count=true`, quick-search ranking must include a bounded cite-count signal without replacing semantic score or existing `hitCount`; when disabled, existing counts are ignored without data loss. +- [x] 10.13 Abuse/concurrency: rate-limit citation count pumping, handle concurrent increments safely, and prevent cross-scope count leakage. +- [x] 10.14 Cite-count tests: storage migration, idempotent increment, replay dedup, different citing message increments, feature flag disabled behavior, cross-scope non-leakage, unauthorized no-increment, hot-row/concurrency, and ranking after auth filtering. + +## 11. Wave 4 — MD ingest, preferences, and unified bootstrap + +**Prerequisites:** fingerprint, origin, namespace/observation, feature flags, telemetry, startup policy, render policy. +**Satisfies:** POST11-R11, POST11-R12. + +- [x] 11.1 Define supported MD paths/triggers, parser section classes, resource caps, partial-commit semantics, and no-fs-watch rule. +- [x] 11.2 Add bounded MD ingest with stable fingerprint, origin `md_ingest`, idempotent projection-backed writes plus linked observations, feature flag, fail-closed scope validation for unsupported `user_private`/workspace/org filesystem ingest, and production bootstrap/manual-sync worker wiring that stays out of the ordinary send ack path and permits later re-ingest after prior jobs finish. +- [x] 11.3 Unify startup memory, preferences, project/user context, and future skills through named-stage bootstrap. +- [x] 11.4 Add `shared/send-origin.ts` and `session.send.origin` contract; missing origin defaults to `system_inject`, which is untrusted for preference writes. +- [x] 11.5 Accept persistent `@pref:` only from trusted user origins; leading trusted raw `@pref:` command lines persist idempotently, are stripped from user-visible/provider-bound user text, and their preference content is rendered into controlled provider-visible preference context for the same turn and as session-level stable context on the first later eligible turn, but identical rendered preference context MUST NOT be repeated on every send; compact/clear boundaries reset the injection gate; ack does not wait for persistence or preference context work. +- [x] 11.6 Preference idempotency: dedupe trusted resends/retries by command/message identity plus user/scope/fingerprint; emit `mem.preferences.persisted` only after actual persistence succeeds, `mem.preferences.duplicate_ignored` for replayed writes, `mem.preferences.persistence_failed` on write failure, and `mem.preferences.rejected_untrusted`/`mem.preferences.untrusted_origin` for untrusted origins. +- [x] 11.7 Failure handling: oversize, symlink-disallowed, unreadable, invalid encoding, malformed section, and prompt-injection-like content fail closed per section and emit telemetry. +- [x] 11.8 Tests: idempotent ingest, caps, partial valid section commit, projection/observation linkage, no cross-project/user-private/workspace/org promotion or silent downgrade, per-file provenance preservation for identical section text, repeated schedule re-ingest, agent-emitted `@pref:` rejected, missing-origin fail-closed for preference persistence, trusted raw-command strip plus provider-visible preference context injection, persisted preference reuse as one-shot session context rather than per-turn prompt growth, compact reset/re-injection, queued-send preamble preservation, disabled pass-through, resend idempotency, startup budget compatibility. +- [x] 11.9 Ensure `mem.feature.preferences` disabled path passes text through without persistence/strip. +- [x] 11.10 Add web/daemon management UI for trusted preference records: list active persisted preferences, create an explicit user-scoped preference, delete stale preferences, and keep all messages/constants/i18n shared. +- [x] 11.11 Add web/daemon manual MD ingest control with explicit project directory, canonical project id, scope, result counters, and no silent scope downgrade. +- [x] 11.12 Add daemon/Web management feature-state and fail-closed mutation guards: feature-disabled preference writes/deletes and manual MD ingest runs are rejected with shared error codes and localized UI messages; manual MD ingest rejects missing canonical project identity before file reads. +- [x] 11.13 Audit closure: MD parser production defaults derive from `shared/memory-defaults.ts`, including `markdownMaxBytes`, `markdownMaxSections`, `markdownMaxSectionBytes`, and `markdownParserBudgetMs`; parser-default tests cover oversize, section-count, and parser-budget failure behavior. + +## 12. Wave 5 — enterprise authored standards, skills subsystem, and background skill review + +**Prerequisites:** fingerprint, origin, namespace/observation, scope registry, feature flags, telemetry, render policy, shared-context document/version/binding migrations, G3 sanitizer. +**Satisfies:** POST11-R13, POST11-R14, POST11-R15, POST11-R16, POST11-R19. + +- [x] 12.1 Define skill metadata/front matter, project association, escape hatch `/.imc/skills/`, workspace/org shared mirrors, and empty built-in manifest schema. +- [x] 12.2 Add user-level skill storage under `~/.imcodes/skills/{category}/{skill-name}.md`. +- [x] 12.3 Implement ordinary precedence: project escape hatch, project-scoped user metadata, user default, workspace shared, org shared, built-in fallback. Built-in fallback is lowest precedence and must not override any user/project/workspace/org skill. +- [x] 12.4 Implement enforced policy as a separate workspace/org override axis; default Wave 5 admin-pushed skills are additive unless explicitly enforced. +- [x] 12.5 Add admin-only workspace/org skill push and reject unauthorized pushes without inventory leakage. +- [x] 12.6 Expose selected skills through a provider-visible registry hint containing bounded metadata and redacted readable paths/`skill://` URIs sourced from a maintained skill registry; ordinary startup/send must not scan or read every skill markdown body, and any full-body read must be on-demand through the resolver plus `shared/skill-envelope.ts`, system-instruction guard, and 4KB cap. +- [x] 12.7 Packaging: add `shared/builtin-skill-manifest.ts`, ship empty `dist/builtin-skills/manifest.json`, and ensure npm/Docker package includes the empty built-in layer. +- [x] 12.8 Web/i18n gate: skill failure states, disabled states, and layer diagnostics use `t()` and all supported locales. +- [x] 12.9 Tests: precedence conflicts, enforced/additive semantics, project association, sanitizer fixture set, delimiter collision negative fixture, empty manifest loads zero skills without error, admin authorization, i18n/shared constants. +- [x] 12.10 Skill auto-creation/self-improvement: run only after response delivery through the existing isolated compression/materialization background path; add `shared/skill-review-triggers.ts` with closed triggers `tool_iteration_count` and `manual_review`; require completed visible non-error tool-result evidence meeting `skillReviewToolIterationThreshold` before automatic `tool_iteration_count` enqueue while allowing explicit `manual_review`; provide a daemon-local production worker/scheduler that creates or updates deterministic user-level skills using matching skill keys before creating new files and updates the skill registry immediately after successful writes; never block send ack, provider delivery, `/stop`, approval/feedback, or shutdown; enforce coalescing, per-scope concurrency, min-intervals, daily caps, bounded retry/backoff, idempotency, disabled-feature behavior, and repair tests. + +- [x] 12.11 Enterprise authored standards: model enterprise-wide coding standards/playbooks as `org_shared` authored context bindings (`enterprise_id` set, `workspace_id = NULL`, `enrollment_id = NULL`) behind `mem.feature.org_shared_authored_standards`, never as `global` / `namespace_tier=global` / unscoped memory. Disabling the flag must stop new org-wide mutation/selection without affecting unrelated project/workspace shared-context bindings. +- [x] 12.12 Authorization: only enterprise owner/admin may create/update/activate/deactivate org-shared documents, versions, and bindings; members may read only matching active bindings; non-members and other enterprises receive same-shape not-found/unauthorized responses without inventory leakage. +- [x] 12.13 Runtime selection: project bindings override/precede workspace bindings, workspace bindings override/precede org bindings; required org-shared bindings must be preserved or dispatch fails, advisory org-shared bindings may be trimmed only with diagnostics/telemetry; optional repo/language/path filters narrow applicability only. +- [x] 12.14 Tests: add `server/test/shared-context-org-authored-context.test.ts` plus runtime resolver/web diagnostics coverage for org-wide standard creation, admin-only mutation, member-only runtime selection, project/workspace/org precedence, required/advisory behavior, filter narrowing, and cross-enterprise non-leakage. + +- [x] 12.15 Add skill registry/on-demand regression tests: startup registry hint works without existing skill body files, unrelated turns do not read skill bodies, explicit/matching resolver reads only the selected skill, stale/unauthorized resolver paths fail closed, and provider-visible hints never expose absolute home paths. +- [x] 12.16 Split skill-review telemetry so below-threshold/non-eligible evidence is distinguishable from true throttling; hidden/error tool results must not contribute to automatic `tool_iteration_count` evidence. +- [x] 12.17 Add web/daemon skill registry management UI: list registry metadata, rebuild registry only on explicit operator action, preview selected skill body on demand, delete managed skill files safely, and preserve startup manifest-only behavior. +- [x] 12.18 Add web/daemon observation-store management UI: list typed observations with scope/class filters and promote observations only through explicit audited UI actions. +- [x] 12.19 Harden skill management UI/API: skill preview rejects symlink/non-file or polluted registry paths, feature-disabled skill mutations/read-body actions fail closed, and registry management writes invalidate runtime registry cache. +- [x] 12.20 Audit closure: skill registry reads fail closed on entry-count overflow, registry display paths are sanitized to redacted paths or `skill://` URIs before provider-visible startup hints, and skill auto-review counters/evidence are scoped to the current day/completed turn rather than daemon lifetime or accumulated unrelated turns. + +## 13. Later candidates retained but not current blockers + +The following are backlog notes only. They are not checkboxes and do not block Wave 1-5 completion until promoted by a future OpenSpec delta: + +- Drift recompaction loops, prompt caching, topic-focused compact/context-selection behavior that still must not daemon-intercept `/compact`, LLM redaction, built-in skill content harvest, autonomous prefetch/LRU, and quick-search result caching. Authorization-scope registry work, including `user_private`, dedicated user-private sync, namespace registry, observation store, cite-count ranking, preferences, enterprise org-shared authored standards, and skill auto-creation are current Wave 1-5 scope, not backlog. +- Future MCP exposure beyond the read/search behavior explicitly scoped here. + +## 14. Final validation + +- [x] 14.1 Run `openspec validate memory-system-post-1-1-integration`. +- [x] 14.2 Run daemon typecheck/build and targeted daemon tests for changed memory modules. +- [x] 14.3 Run server typecheck/tests for migrations, embeddings, search, authorization, and scope filtering when touched. +- [x] 14.4 Run web typecheck/tests for quick search, citation UI, skills UI, i18n, locale coverage, and accessibility when touched. +- [x] 14.5 Update and run the canonical memory acceptance harness so it validates `memory-system-post-1-1-integration`; `bash scripts/run-acceptance-suite.sh` validates this change id and includes daemon/server/web tests plus integration coverage. +- [x] 14.6 Before marking Wave 1-5 complete, rerun the traceability matrix and confirm every requirement has passing test evidence. +- [x] 14.7 Validate post-1.1 management UI with web component coverage for preferences, skills, MD ingest controls, and observation promotion, daemon WebSocket handler coverage for management messages, plus daemon/web typechecks. +- [x] 14.8 Validate management UI hardening: feature-state display, localized shared error codes, disabled mutation guards, canonical-project-id MD ingest rejection, skill registry cache invalidation, and symlink-safe skill preview paths. +- [x] 14.9 Validate memory project-index synchronization: daemon personal-memory response includes project summaries, cloud/shared routes include authorized `projects` arrays, semantic memory view preserves project summaries after scoring, the Web memory tab defaults browse to all projects, memory-index options remain available after selecting/clearing a project filter, realpath project-directory aliases resolve successfully, and targeted daemon/server/web tests plus daemon/server/web typechecks pass. + +## 15. Management UI hardening closure + +**Prerequisites:** 11.x preference/MD management, 12.x skill/observation management, and bridge routing. +**Satisfies:** POST11-R15, POST11-R20. + +- [x] 15.1 Add a closed memory-management WebSocket request/response vocabulary in `shared/memory-ws.ts` and route management responses by pending `requestId` instead of the default browser broadcast path. +- [x] 15.2 Inject server-derived management context in `server/src/ws/bridge.ts`; daemon management handlers must use the injected actor/user/role/project context and ignore client-supplied owner/actor/role fields for authorization. Elevated roles are derived from server membership for the requested enterprise/workspace/project binding, never from browser payloads. +- [x] 15.3 Harden preference management: query/create/delete only the derived current user's preferences, reject non-owner delete with a shared error code, and use stable request/fingerprint idempotency rather than random retry identity for explicit creates. +- [x] 15.4 Harden observation management: filter private observations by derived owner, require explicit role authorization for private-to-shared promotion, verify `expectedFromScope` inside the promotion transaction, and publish cache invalidation after successful promotion. +- [x] 15.5 Harden manual MD ingest: require valid `projectDir`, canonical project identity, and matching repository identity before file reads; unsupported filesystem ingest scopes return a typed error instead of success+0 or silent downgrade. +- [x] 15.6 Harden skill management/runtime paths with a single managed-path helper, rejecting NUL, symlink directories, final symlinks/non-files, path escape, oversize previews, and oversized registry files/entry lists before unbounded parsing. +- [x] 15.7 Add runtime memory cache invalidation for preference, skill registry, MD ingest, and observation management mutations so subsequent startup/send context is not stale. +- [x] 15.8 Harden the Web management UI: latest-requestId guards per surface, mutation buttons disabled while feature state is unknown/disabled, supported MD scopes only, current-user preference create semantics, localized shared error codes in all supported locales, canonicalRepoId payload coverage for project-bound management actions, non-color feature-state accessibility labels, and regression coverage in `web/test/components/SharedContextManagementPanel.test.tsx`. +- [x] 15.9 Validation anchors added/run: `server/test/bridge-memory-management.test.ts`, `test/daemon/command-handler-memory-context.test.ts`, `test/daemon/command-handler-transport-queue.test.ts`, `test/daemon/context-store.test.ts`, `test/context/memory-search.test.ts`, `test/context/skill-registry-resolver.test.ts`, `test/context/context-observation-store.test.ts`, `test/context/memory-feature-flags.test.ts`, `web/test/components/SharedContextManagementPanel.test.tsx`, `web/test/i18n-coverage.test.ts`, and `web/test/i18n-memory-post11.test.ts`. +- [x] 15.10 Audit closure: management handlers fail closed when authenticated management context is absent, management personal/search/archive/restore/delete use the same authorization envelope as observation/preference handlers, raw search is not exposed through the management UI path, Web management requests carry project identity hints needed for server-injected bound-project authorization, and bridge context-construction failures clear pending requests with a requester-only error. +- [x] 15.11 Add daemon-backed memory project resolution: `memory.project.resolve` accepts only daemon-known project directories, derives canonical repo identity from the git remote, rejects invalid/mismatched/unauthorized directories, and returns a routed status response. +- [x] 15.12 Replace primary manual project ID/path entry in the memory UI with a searchable project selector sourced from active/recent sessions and enterprise canonical projects; wire old memory views plus skills/MD/observation actions to the selected identity, keep manual fields as advanced fallback only, add productized tabs/search controls, i18n keys, and regression coverage. +- [x] 15.13 Synchronize project browse indexes across local daemon, personal cloud, enterprise/shared, and semantic memory views: `ContextMemoryView.projects` / `ContextMemoryProjectView` provide authorized project summaries; daemon `PERSONAL_RESPONSE` includes `listMemoryProjectSummaries`; server memory routes and semantic memory views return project summaries after auth filters; the web project dropdown merges memory-index options, keeps all-project as the default/no-filter browse state, separates browse filtering from local file-backed action project selection, preserves options across filtered reloads, resolves directory aliases by realpath before local tools run, updates all locales for `memory_index`/local-action wording, and covers the behavior in daemon/server/web tests. +- [x] 15.14 Add management UI enable/disable controls for daemon memory feature flags: feature cards expose localized toggle buttons, send shared `memory.features.set` requests with requestId guards, render dependency-blocked requested-vs-effective state as a distinct warning rather than plain disabled, refresh downstream panes after a change, and cover the behavior in web component tests plus all locale files. +- [x] 15.15 Improve observation promotion usability: promotion buttons disclose the selected target scope, invalid from/to scope pairs are disabled before mutation, the first click opens an explicit confirmation showing source scope, target scope, optional reason, audit write, and visibility consequence, and only the confirmation action sends `memory.observation.promote`; cover the two-step flow with web component tests and all locale files. +- [x] 15.16 Add complete management CRUD for local memory records and preferences: processed memory supports manual project-bound create, edit, archive/restore/delete, and deterministic pinning with server-derived authorization, linked projection/observation updates, linked-observation cleanup on permanent delete, embedding invalidation, cache invalidation, shared WS constants, localized UI strings, and daemon/web regression tests; preferences support update in addition to existing create/delete, and observations support edit/delete in addition to audited promotion. Store and display record-level owner/creator/updater metadata separately from enterprise/workspace admin role; private records remain owner-only, and shared records are mutable by admins or the record creator/owner only after namespace authorization. + +## 16. Transport sender identity audit closure + +**Prerequisites:** foundations send ack/priority path and transport SDK session env construction. +**Satisfies:** POST11-R1, POST11-R20 operational diagnostics. + +- [x] 16.1 Transport session launch and restore construct per-session `SessionConfig.env` for every transport runtime using `IMCODES_SESSION` and `IMCODES_SESSION_LABEL`; local SDK/CLI providers that can pass tool/runtime environment MUST preserve that env, and any non-env-capable transport MUST provide an equivalent non-prompt adapter instead of relying only on prompt text. +- [x] 16.2 Add regression coverage proving transport sender identity is runtime-visible: Codex SDK app-server thread/turn requests carry per-session env, Claude SDK restored/launched transport sessions carry the same env into SDK query options, and CLI sender detection prefers `IMCODES_SESSION` over labels. +- [x] 16.3 Codex SDK context usage uses app-server `thread/tokenUsage/updated.tokenUsage.last` plus `modelContextWindow` for the UI ctx meter, falling back to `total` only when `last` is absent; it normalizes Codex/OpenAI cached tokens as a subset (`inputTokens - cachedInputTokens` new input plus `cacheTokens`) so the visible total equals the current-window input token count, and keeps cumulative totals only as diagnostics; regression coverage locks the provider and transport relay mappings so ctx does not inflate from accumulated billing/thread totals. +- [x] 16.4 Carry a provider-sourced context-window marker from Codex SDK/native Codex usage events through timeline extraction into Web ctx rendering, and lock the UI rule that provider-marked `modelContextWindow` wins over model-family inference except known stale/mismatched provider fallbacks; GPT-5.5 is a locked 922k model-window override for both too-low (`258400`) and too-high (`1000000`) Codex fallback values, while unmarked legacy/stale explicit windows keep the existing model-inference precedence. +- [x] 16.5 Resolve transport usage events against the persisted session model when provider usage omits `model`, so two sessions selected as GPT-5.5 cannot split between stale provider fallback windows (`258400` / `1000000`) and instead both render the locked 922k context limit; regression coverage locks no-model usage updates with stale and missing provider context-window values. + +## 17. P2P strict audit closure — management authorization follow-up + +**Prerequisites:** 15.x management UI hardening and P2P discussion `7b9def0b-86f`. +**Satisfies:** POST11-R17, POST11-R18, POST11-R20. + +- [x] 17.1 Management quick search and personal-memory management queries use an authorized namespace/scope+owner filter before result item construction, stats, pending-record counts, and pagination; caller-owned `personal` rows are included only for the derived current user, and other users' `personal` rows in the same project are excluded; daemon-local processed/staged/dirty/job tables maintain backfilled indexed scope/owner/project columns so these filters execute in SQL before JS result construction. +- [x] 17.2 Owner-private namespace authorization fails closed when `personal` / `user_private` owner identity is missing or does not match the derived management user. +- [x] 17.3 Project-scoped skill management requires explicit canonical repo identity plus project directory validation against the git remote before registry read/rebuild/preview/delete; generic `projectId` is not used as a role-derivation alias. +- [x] 17.4 Observation promotion requires `expectedFromScope` before promotion and returns a shared/localized error when omitted. +- [x] 17.5 Bridge regression coverage locks unauthenticated rejection, duplicate requestId rejection, pending-request cap, forged context stripping, and generic `projectId` non-elevation. +- [x] 17.6 Targeted tests cover management authorized search owner isolation, personal-memory list/search/pending owner isolation, authorized stats/pagination, same-user different-scope exclusion, daemon-local namespace filter index/backfill coverage, and expected-scope promotion rejection. +- [x] 17.7 Bridge authorization closure: browser-provided canonical repo/workspace/org hints enter `boundProjects` only after server membership/enrollment verification; unauthorized hints forward as request hints but authorize no shared daemon access. +- [x] 17.8 Metadata trust closure: record-level authorization uses trusted `ownerUserId` / `ownedByUserId` / `createdByUserId` only, while legacy/display fields (`userId`, `createdBy`, `authorUserId`, `updatedBy`) remain display-only and cannot grant shared mutation rights. +- [x] 17.9 Store consistency closure: observation delete is observation-only, processed-memory delete remains the projection+linked-observation cleanup path, and observation edits clear linked projection embeddings just like processed-memory edits. +- [x] 17.10 Feature/caching closure: processed-memory create/update/archive/restore/delete/pin fail closed when `mem.feature.observation_store=false`, and runtime memory cache invalidation distinguishes projection mutations from observation mutations. +- [x] 17.11 Validation anchors added/run: `server/test/bridge-memory-management.test.ts`, `test/daemon/command-handler-memory-context.test.ts`, and `test/context/context-observation-store.test.ts` cover verified bridge bindings, legacy metadata forgery rejection, processed mutation feature-disabled guards, observation-only delete, typed promotion errors, and linked-embedding invalidation. diff --git a/package-lock.json b/package-lock.json index cbefe7f5a..1abbc3994 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,9 @@ "zod": "^4.3.6" }, "bin": { - "imcodes": "dist/src/index.js" + "imcodes": "dist/src/index.js", + "imcodes-launch": "bin/imcodes-launch.sh", + "imcodes-launch-preflight": "dist/src/util/windows-launch-preflight.mjs" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/package.json b/package.json index 00f53730c..7a0fb9889 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,16 @@ "url": "https://github.com/im4codes/imcodes" }, "bin": { - "imcodes": "./dist/src/index.js" + "imcodes": "./dist/src/index.js", + "imcodes-launch": "./bin/imcodes-launch.sh", + "imcodes-launch-preflight": "./dist/src/util/windows-launch-preflight.mjs" }, "type": "module", "main": "./dist/src/index.js", "files": [ "dist/", - "config/" + "config/", + "bin/" ], "engines": { "node": ">=22" @@ -32,7 +35,8 @@ "test:web": "vitest run --project web", "test:e2e": "vitest run --project e2e", "test:integration": "vitest run --workspace vitest.integration.config.ts", - "test:coverage": "vitest run --coverage --no-file-parallelism --maxWorkers 1 --testTimeout 60000 --hookTimeout 60000 && node scripts/write-coverage-summary.mjs", + "test:preview-dist": "node scripts/run-preview-dist-smoke.mjs", + "test:coverage": "vitest run --coverage --project daemon --project web --project server && node scripts/write-coverage-summary.mjs", "test:watch": "vitest", "lint": "eslint src/", "typecheck": "tsc --noEmit", diff --git a/scripts/copy-worker-bootstraps.mjs b/scripts/copy-worker-bootstraps.mjs index 5b37dfbb1..c5bd3c4e0 100644 --- a/scripts/copy-worker-bootstraps.mjs +++ b/scripts/copy-worker-bootstraps.mjs @@ -11,7 +11,7 @@ * `dist/src/`, preserving directory structure. */ -import { cpSync, existsSync, mkdirSync, readdirSync, statSync } from 'node:fs'; +import { cpSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -49,4 +49,13 @@ function walk(dir) { } walk(srcRoot); -console.log(`copy-worker-bootstraps: copied ${copied} .mjs file(s) to dist/src/`); + +const builtinSkillManifestDir = join(repoRoot, 'dist', 'builtin-skills'); +mkdirSync(builtinSkillManifestDir, { recursive: true }); +writeFileSync( + join(builtinSkillManifestDir, 'manifest.json'), + `${JSON.stringify({ version: 1, skills: [] }, null, 2)}\n`, + 'utf8', +); + +console.log(`copy-worker-bootstraps: copied ${copied} .mjs file(s) to dist/src/ and wrote dist/builtin-skills/manifest.json`); diff --git a/scripts/mark-bin-executable.mjs b/scripts/mark-bin-executable.mjs index c0f22bcf3..57d4ebb79 100644 --- a/scripts/mark-bin-executable.mjs +++ b/scripts/mark-bin-executable.mjs @@ -5,15 +5,26 @@ import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const scriptDir = dirname(fileURLToPath(import.meta.url)); -const binPath = resolve(scriptDir, '../dist/src/index.js'); -if (!existsSync(binPath)) { - console.warn(`mark-bin-executable: skipped, missing ${binPath}`); - process.exit(0); -} +// Anything declared in `package.json#bin` must be +x or `npm install -g` +// silently creates an unusable symlink (no error, just EACCES at exec +// time). Keep this list in lockstep with package.json#bin. +const targets = [ + '../dist/src/index.js', + '../bin/imcodes-launch.sh', + '../dist/src/util/windows-launch-preflight.mjs', + '../dist/src/util/preinstall-cleanup.mjs', +]; -try { - chmodSync(binPath, 0o755); -} catch (error) { - console.warn(`mark-bin-executable: failed to chmod ${binPath}: ${error instanceof Error ? error.message : String(error)}`); +for (const rel of targets) { + const binPath = resolve(scriptDir, rel); + if (!existsSync(binPath)) { + console.warn(`mark-bin-executable: skipped, missing ${binPath}`); + continue; + } + try { + chmodSync(binPath, 0o755); + } catch (error) { + console.warn(`mark-bin-executable: failed to chmod ${binPath}: ${error instanceof Error ? error.message : String(error)}`); + } } diff --git a/scripts/restart-daemon.cmd b/scripts/restart-daemon.cmd new file mode 100644 index 000000000..a8e3d4f67 --- /dev/null +++ b/scripts/restart-daemon.cmd @@ -0,0 +1,110 @@ +@echo off +rem Rebuild, relink, and restart the local imcodes daemon service (dev only). +rem Windows counterpart of scripts/restart-daemon.sh. +rem +rem CRITICAL: this file MUST be pure ASCII (no Unicode em-dashes, box-drawing +rem chars, smart quotes, etc.) and use REM (not the ::-as-comment hack). +rem cmd.exe parses .cmd files using the current OEM codepage BEFORE the chcp +rem 65001 line takes effect, so any multi-byte UTF-8 sequence in the comments +rem at the top of the file gets reinterpreted as OEM bytes and can break the +rem parser (we lost a full restart trying to debug U+2014 in a comment). +rem +rem Steps: +rem 1. npm install / build / link --force (foreground - daemon stays alive +rem because Node modules were already loaded into V8; npm just overwrites +rem .js files on disk for the next launch). +rem 2. Spawn the actual "imcodes restart" fully detached via +rem wscript -- VBS -- CMD. This matters when the calling shell is itself +rem running inside a transport session managed by the daemon (Claude Code +rem via imcodes, etc.). A synchronous restart from such a session would +rem kill the daemon, which kills the session, which kills this script +rem before the new daemon can come up. wscript runs hidden in its own +rem process group, so the restart always completes. +rem +rem NOTE: We use "imcodes restart" (the standalone command), NOT +rem "imcodes service restart --no-build" like the .sh does. The latter +rem explicitly rejects win32 with "Unsupported platform" -- it only +rem knows launchctl/systemd. The standalone "imcodes restart" routes to +rem ensureDaemonRunning() in src/util/windows-daemon.ts which does the +rem proper Windows pidfile/watchdog dance. +rem +rem For a heavier "install from local clone (npm install -g .) + bounce" +rem that mirrors the production daemon.upgrade flow more closely (with +rem upgrade.lock to pause the watchdog, NODE_OPTIONS save/restore, etc.), +rem see ~/.imcodes/tmp/manual-upgrade.mjs. +rem +rem Usage: scripts\restart-daemon.cmd +rem Logs: %TEMP%\imcodes-restart-daemon.log +chcp 65001 >nul 2>&1 +setlocal EnableDelayedExpansion + +cd /d "%~dp0\.." + +echo [restart-daemon] npm install +call npm install +if errorlevel 1 ( + echo [restart-daemon] npm install FAILED + exit /b 1 +) + +echo [restart-daemon] npm run build +call npm run build +if errorlevel 1 ( + echo [restart-daemon] build FAILED + exit /b 1 +) + +echo [restart-daemon] npm link --force +call npm link --force +if errorlevel 1 ( + echo [restart-daemon] npm link FAILED + exit /b 1 +) + +rem Build the detached restart artifacts. +rem Per-run tmp dir so concurrent restarts don't trample each other. +set "STAMP=%RANDOM%-%RANDOM%" +set "TMP_DIR=%TEMP%\imcodes-restart-daemon-%STAMP%" +mkdir "%TMP_DIR%" >nul 2>&1 +set "RESTART_CMD=%TMP_DIR%\restart.cmd" +set "RESTART_VBS=%TMP_DIR%\restart.vbs" +set "LOG_FILE=%TEMP%\imcodes-restart-daemon.log" + +rem Write the inner CMD line by line. Avoids the ()-block escaping pain. +rem Notes on escaping: +rem - %% in the outer file becomes a literal % in the written file, then +rem the inner cmd expands %date% / %time% at run time. +rem - ^>, ^&, ^| are escaped so they pass through outer parsing and land +rem literally in the file (the inner cmd then interprets them as +rem redirect/pipe). +rem - %LOG_FILE% / %TMP_DIR% expand at outer parse time so the inner cmd +rem gets the absolute paths baked in (it has its own setlocal). +rem - Sleeps use ping, not "timeout /t", because under wscript->cmd there +rem is no console for stdin and timeout aborts immediately with +rem "Input redirection is not supported". +> "%RESTART_CMD%" echo @echo off +>> "%RESTART_CMD%" echo chcp 65001 ^>nul 2^>^&1 +>> "%RESTART_CMD%" echo setlocal +>> "%RESTART_CMD%" echo echo === detached restart at %%date%% %%time%% ^>^> "%LOG_FILE%" +>> "%RESTART_CMD%" echo ping -n 4 127.0.0.1 ^>nul 2^>^&1 +>> "%RESTART_CMD%" echo call imcodes restart ^>^> "%LOG_FILE%" 2^>^&1 +>> "%RESTART_CMD%" echo echo === detached restart done at %%date%% %%time%% ^>^> "%LOG_FILE%" +>> "%RESTART_CMD%" echo ping -n 61 127.0.0.1 ^>nul 2^>^&1 +>> "%RESTART_CMD%" echo rmdir /s /q "%TMP_DIR%" ^>nul 2^>^&1 +>> "%RESTART_CMD%" echo endlocal +>> "%RESTART_CMD%" echo exit /b 0 + +rem VBS wrapper: WshShell.Run with mode 0 (hidden) + False (no wait). +rem """ inside an echoed line passes through as literal """, which VBS then +rem parses as one " inside a string literal -- i.e. the file ends up with +rem WshShell.Run "", 0, False +> "%RESTART_VBS%" echo On Error Resume Next +>> "%RESTART_VBS%" echo Set WshShell = CreateObject("WScript.Shell") +>> "%RESTART_VBS%" echo WshShell.Run """%RESTART_CMD%""", 0, False + +echo [restart-daemon] Detaching restart; logs: %LOG_FILE% +start "" /b wscript "%RESTART_VBS%" +echo [restart-daemon] Restart dispatched. Daemon will come back on its own. + +endlocal +exit /b 0 diff --git a/scripts/run-acceptance-suite.sh b/scripts/run-acceptance-suite.sh index 1102cb418..2eabb9785 100755 --- a/scripts/run-acceptance-suite.sh +++ b/scripts/run-acceptance-suite.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -if [ -d openspec/changes/memory-system-1.1-foundations ]; then +if [ -d openspec/changes/memory-system-post-1-1-integration ]; then + openspec validate memory-system-post-1-1-integration +elif [ -d openspec/changes/memory-system-1.1-foundations ]; then openspec validate memory-system-1.1-foundations else openspec validate daemon-memory-pipeline --type spec @@ -15,6 +17,7 @@ npx tsc -p server/tsconfig.json --noEmit npm run test:unit npm run test:server npm run test:web +npm run test:integration npx vitest run --project e2e test/e2e/memory-pipeline.e2e.test.ts scripts/check-scope-filter.sh diff --git a/scripts/run-preview-dist-smoke.mjs b/scripts/run-preview-dist-smoke.mjs new file mode 100644 index 000000000..f53735615 --- /dev/null +++ b/scripts/run-preview-dist-smoke.mjs @@ -0,0 +1,29 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; + +const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'; +const npxCmd = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + stdio: 'inherit', + shell: false, + ...options, + }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +run(npmCmd, ['run', 'build']); +run(npxCmd, [ + 'vitest', + 'run', + 'test/daemon/file-preview-read-dist-smoke.test.ts', + 'test/daemon/file-preview-read-dist-daemon-smoke.test.ts', +], { + env: { + ...process.env, + PREVIEW_DIST_REQUIRED: '1', + }, +}); diff --git a/scripts/strip-onnxruntime-gpu.mjs b/scripts/strip-onnxruntime-gpu.mjs index 5e43d4167..c55498833 100644 --- a/scripts/strip-onnxruntime-gpu.mjs +++ b/scripts/strip-onnxruntime-gpu.mjs @@ -344,21 +344,31 @@ if (existsSync(imcodesPkgPath)) { const imcodesPkg = JSON.parse(readFileSync(imcodesPkgPath, 'utf8')); imcodesPkg.scripts = imcodesPkg.scripts ?? {}; const NOOP = 'echo "imcodes: published tarball, lifecycle scripts disabled"'; - // Path is relative to the imcodes package root at install time, which is - // also the postinstall script's cwd. Forward slashes work on Windows for - // `node` invocation (Node accepts both separators on the CLI). + // Paths are relative to the imcodes package root at install time, which + // is also the lifecycle scripts' cwd. Forward slashes work on Windows + // for `node` invocation (Node accepts both separators on the CLI). const POSTINSTALL_CMD = 'node dist/src/util/postinstall-sharp-repair.js'; + // Preinstall handles the "previous install was killed mid-way" residue + // (`.imcodes-XXXXX` siblings, stale upgrade.lock.d/) and aborts with a + // clear message when a parallel daemon-triggered upgrade is detected + // — preventing the confusing ENOTEMPTY collision that 215 hit on + // 2026-05-08. Pure Node built-ins so it runs even before our deps are + // extracted. + const PREINSTALL_CMD = 'node dist/src/util/preinstall-cleanup.mjs || true'; let neutralized = 0; for (const key of Object.keys(imcodesPkg.scripts)) { - if (key === 'postinstall') continue; // handled below + if (key === 'postinstall' || key === 'preinstall') continue; // handled below if (imcodesPkg.scripts[key] !== NOOP) { imcodesPkg.scripts[key] = NOOP; neutralized += 1; } } - // Force-write the published-only postinstall. The source-tree package.json - // doesn't define one (we have a `prepare` hook for husky during dev), so - // this is purely additive at pack time and gets reverted by postpack. + // Force-write the published-only preinstall + postinstall. The source-tree + // package.json doesn't define either (dev uses `prepare` for husky), so + // these are purely additive at pack time and get reverted by postpack. + if (imcodesPkg.scripts.preinstall !== PREINSTALL_CMD) { + imcodesPkg.scripts.preinstall = PREINSTALL_CMD; + } if (imcodesPkg.scripts.postinstall !== POSTINSTALL_CMD) { imcodesPkg.scripts.postinstall = POSTINSTALL_CMD; } diff --git a/server/src/db/migrations/044_memory_scope_search_citations_org.sql b/server/src/db/migrations/044_memory_scope_search_citations_org.sql new file mode 100644 index 000000000..8554d1446 --- /dev/null +++ b/server/src/db/migrations/044_memory_scope_search_citations_org.sql @@ -0,0 +1,182 @@ +-- Post-1.1 memory scope/search/citation/org-authored server foundations. + +CREATE TABLE IF NOT EXISTS owner_private_memories ( + id TEXT PRIMARY KEY, + owner_user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + scope TEXT NOT NULL DEFAULT 'user_private', + kind TEXT NOT NULL, + origin TEXT NOT NULL, + fingerprint TEXT NOT NULL, + text TEXT NOT NULL, + content_json JSONB NOT NULL DEFAULT '{}'::jsonb, + idempotency_key TEXT NOT NULL, + source_server_id TEXT REFERENCES servers(id) ON DELETE SET NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + replicated_at BIGINT NOT NULL, + CONSTRAINT owner_private_memories_scope_check CHECK (scope = 'user_private') +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_owner_private_memories_idempotency + ON owner_private_memories(owner_user_id, idempotency_key); + +CREATE INDEX IF NOT EXISTS idx_owner_private_memories_owner_updated + ON owner_private_memories(owner_user_id, updated_at DESC); + +CREATE TABLE IF NOT EXISTS shared_context_citations ( + id TEXT PRIMARY KEY, + projection_id TEXT NOT NULL REFERENCES shared_context_projections(id) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + citing_message_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL UNIQUE, + projection_content_hash TEXT NOT NULL, + created_at BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_shared_context_citations_projection + ON shared_context_citations(projection_id, created_at DESC); + +CREATE TABLE IF NOT EXISTS shared_context_projection_cite_counts ( + projection_id TEXT PRIMARY KEY REFERENCES shared_context_projections(id) ON DELETE CASCADE, + cite_count INTEGER NOT NULL DEFAULT 0, + updated_at BIGINT NOT NULL +); + +ALTER TABLE shared_context_projections + ADD COLUMN IF NOT EXISTS content_hash TEXT; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'shared_context_projections_scope_no_user_private' + ) THEN + ALTER TABLE shared_context_projections + ADD CONSTRAINT shared_context_projections_scope_no_user_private + CHECK (scope IN ('personal', 'project_shared', 'workspace_shared', 'org_shared')) NOT VALID; + END IF; +END +$$; + +CREATE INDEX IF NOT EXISTS idx_shared_context_document_bindings_runtime_specificity + ON shared_context_document_bindings( + enterprise_id, + status, + (CASE WHEN enrollment_id IS NOT NULL THEN 1 WHEN workspace_id IS NOT NULL THEN 2 ELSE 3 END), + binding_mode, + id + ); + +-- Nullable/backfillable metadata for post-1.1 fingerprint/origin parity. +ALTER TABLE shared_context_projections + ADD COLUMN IF NOT EXISTS summary_fingerprint TEXT; + +ALTER TABLE shared_context_projections + ADD COLUMN IF NOT EXISTS origin TEXT; + +ALTER TABLE shared_context_records + ADD COLUMN IF NOT EXISTS summary_fingerprint TEXT; + +ALTER TABLE shared_context_records + ADD COLUMN IF NOT EXISTS origin TEXT; + +CREATE INDEX IF NOT EXISTS idx_shared_context_projections_fingerprint + ON shared_context_projections(scope, project_id, projection_class, summary_fingerprint) + WHERE summary_fingerprint IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_shared_context_records_fingerprint + ON shared_context_records(scope, project_id, record_class, summary_fingerprint) + WHERE summary_fingerprint IS NOT NULL; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'shared_context_projections_origin_check' + ) THEN + ALTER TABLE shared_context_projections + ADD CONSTRAINT shared_context_projections_origin_check + CHECK (origin IS NULL OR origin IN ('chat_compacted', 'user_note', 'skill_import', 'manual_pin', 'agent_learned', 'md_ingest')) NOT VALID; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'shared_context_records_origin_check' + ) THEN + ALTER TABLE shared_context_records + ADD CONSTRAINT shared_context_records_origin_check + CHECK (origin IS NULL OR origin IN ('chat_compacted', 'user_note', 'skill_import', 'manual_pin', 'agent_learned', 'md_ingest')) NOT VALID; + END IF; +END +$$; + +-- Server-side typed namespace/observation parity with daemon SQLite tables. +CREATE TABLE IF NOT EXISTS memory_context_namespaces ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + scope TEXT NOT NULL, + user_id TEXT REFERENCES users(id) ON DELETE SET NULL, + root_session_id TEXT, + session_tree_id TEXT, + session_id TEXT, + workspace_id TEXT REFERENCES shared_context_workspaces(id) ON DELETE SET NULL, + project_id TEXT, + org_id TEXT REFERENCES teams(id) ON DELETE CASCADE, + key TEXT NOT NULL, + visibility TEXT NOT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + CONSTRAINT memory_context_namespaces_scope_check CHECK (scope IN ('user_private', 'personal', 'project_shared', 'workspace_shared', 'org_shared')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_memory_context_namespaces_tenant_scope_key + ON memory_context_namespaces(tenant_id, scope, key); + +CREATE INDEX IF NOT EXISTS idx_memory_context_namespaces_lookup + ON memory_context_namespaces(tenant_id, scope, user_id, project_id, workspace_id, org_id); + +CREATE INDEX IF NOT EXISTS idx_memory_context_namespaces_session_tree + ON memory_context_namespaces(root_session_id, session_tree_id, session_id); + +CREATE TABLE IF NOT EXISTS memory_context_observations ( + id TEXT PRIMARY KEY, + namespace_id TEXT NOT NULL REFERENCES memory_context_namespaces(id) ON DELETE CASCADE, + scope TEXT NOT NULL, + class TEXT NOT NULL, + origin TEXT NOT NULL, + fingerprint TEXT NOT NULL, + content_json JSONB NOT NULL, + text_hash TEXT NOT NULL, + source_event_ids_json JSONB NOT NULL, + projection_id TEXT REFERENCES shared_context_projections(id) ON DELETE SET NULL, + state TEXT NOT NULL, + confidence DOUBLE PRECISION, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + promoted_at BIGINT, + CONSTRAINT memory_context_observations_scope_check CHECK (scope IN ('user_private', 'personal', 'project_shared', 'workspace_shared', 'org_shared')), + CONSTRAINT memory_context_observations_class_check CHECK (class IN ('fact', 'decision', 'bugfix', 'feature', 'refactor', 'discovery', 'preference', 'skill_candidate', 'workflow', 'code_pattern', 'note')), + CONSTRAINT memory_context_observations_origin_check CHECK (origin IN ('chat_compacted', 'user_note', 'skill_import', 'manual_pin', 'agent_learned', 'md_ingest')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_memory_context_observations_idempotency + ON memory_context_observations(namespace_id, class, fingerprint, text_hash); + +CREATE INDEX IF NOT EXISTS idx_memory_context_observations_projection + ON memory_context_observations(projection_id); + +CREATE INDEX IF NOT EXISTS idx_memory_context_observations_scope_state + ON memory_context_observations(scope, state, updated_at DESC); + +CREATE TABLE IF NOT EXISTS memory_observation_promotion_audit ( + id TEXT PRIMARY KEY, + observation_id TEXT NOT NULL REFERENCES memory_context_observations(id) ON DELETE CASCADE, + actor_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + action TEXT NOT NULL, + from_scope TEXT NOT NULL, + to_scope TEXT NOT NULL, + reason TEXT, + created_at BIGINT NOT NULL, + CONSTRAINT memory_observation_promotion_audit_action_check CHECK (action IN ('web_ui_promote', 'cli_mem_promote', 'admin_api_promote')), + CONSTRAINT memory_observation_promotion_audit_from_scope_check CHECK (from_scope IN ('user_private', 'personal', 'project_shared', 'workspace_shared', 'org_shared')), + CONSTRAINT memory_observation_promotion_audit_to_scope_check CHECK (to_scope IN ('user_private', 'personal', 'project_shared', 'workspace_shared', 'org_shared')) +); + +CREATE INDEX IF NOT EXISTS idx_memory_observation_promotion_audit_observation + ON memory_observation_promotion_audit(observation_id, created_at); diff --git a/server/src/db/migrations/045_memory_post11_hardening.sql b/server/src/db/migrations/045_memory_post11_hardening.sql new file mode 100644 index 000000000..8f61bbc26 --- /dev/null +++ b/server/src/db/migrations/045_memory_post11_hardening.sql @@ -0,0 +1,62 @@ +-- Post-1.1 implementation hardening: close owner-private contracts, +-- prevent shared-table owner-private pollution, and backfill persistent +-- projection content_hash for citation drift. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +DELETE FROM shared_context_records WHERE scope = 'user_private'; +DELETE FROM shared_context_projections WHERE scope = 'user_private'; + +UPDATE shared_context_projections +SET content_hash = encode( + digest('projection-content:v1:' || btrim(summary) || E'\n' || COALESCE(content_json::text, 'null'), 'sha256'), + 'hex' +) +WHERE content_hash IS NULL OR content_hash = ''; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'owner_private_memories_kind_check' + ) THEN + ALTER TABLE owner_private_memories + ADD CONSTRAINT owner_private_memories_kind_check + CHECK (kind IN ('fact', 'decision', 'bugfix', 'feature', 'refactor', 'discovery', 'preference', 'skill_candidate', 'workflow', 'code_pattern', 'note')) NOT VALID; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'owner_private_memories_origin_check' + ) THEN + ALTER TABLE owner_private_memories + ADD CONSTRAINT owner_private_memories_origin_check + CHECK (origin IN ('chat_compacted', 'user_note', 'skill_import', 'manual_pin', 'agent_learned', 'md_ingest')) NOT VALID; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'owner_private_memories_size_check' + ) THEN + ALTER TABLE owner_private_memories + ADD CONSTRAINT owner_private_memories_size_check + CHECK (octet_length(text) <= 32768 AND octet_length(content_json::text) <= 131072) NOT VALID; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'shared_context_records_scope_no_user_private' + ) THEN + ALTER TABLE shared_context_records + ADD CONSTRAINT shared_context_records_scope_no_user_private + CHECK (scope IN ('personal', 'project_shared', 'workspace_shared', 'org_shared')) NOT VALID; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'shared_context_projections_personal_identity_check' + ) THEN + ALTER TABLE shared_context_projections + ADD CONSTRAINT shared_context_projections_personal_identity_check + CHECK ( + scope <> 'personal' + OR (user_id IS NOT NULL AND enterprise_id IS NULL AND workspace_id IS NULL) + ) NOT VALID; + END IF; +END +$$; diff --git a/server/src/memory/authored-context-runtime.ts b/server/src/memory/authored-context-runtime.ts new file mode 100644 index 000000000..59eae3f4d --- /dev/null +++ b/server/src/memory/authored-context-runtime.ts @@ -0,0 +1,59 @@ +import type { AuthoredContextScope } from '../../../shared/memory-scope.js'; + +export interface RuntimeAuthoredContextBindingLike { + bindingId: string; + mode: 'required' | 'advisory'; + scope: AuthoredContextScope; + content: string; +} + +export interface RuntimeAuthoredContextBudgetDiagnostic { + bindingId: string; + mode: 'required' | 'advisory'; + reason: 'advisory_trimmed' | 'required_over_budget'; + bytes: number; +} + +export type RuntimeAuthoredContextBudgetResult = + | { ok: true; bindings: T[]; diagnostics: RuntimeAuthoredContextBudgetDiagnostic[] } + | { ok: false; error: 'required_context_over_budget'; bindings: T[]; diagnostics: RuntimeAuthoredContextBudgetDiagnostic[] }; + +function utf8Bytes(text: string): number { + return new TextEncoder().encode(text).byteLength; +} + +/** + * Apply runtime authored-context budget after project/workspace/org ordering. + * Required bindings are preserved or dispatch fails; advisory bindings may be + * omitted only with explicit diagnostics. + */ +export function applyRuntimeAuthoredContextBudget( + bindings: readonly T[], + maxBytes: number | null | undefined, +): RuntimeAuthoredContextBudgetResult { + if (!Number.isFinite(maxBytes) || maxBytes === undefined || maxBytes === null || maxBytes <= 0) { + return { ok: true, bindings: [...bindings], diagnostics: [] }; + } + const diagnostics: RuntimeAuthoredContextBudgetDiagnostic[] = []; + const selected: T[] = []; + let used = 0; + for (const binding of bindings) { + const bytes = utf8Bytes(binding.content); + if (binding.mode === 'required') { + if (used + bytes > maxBytes) { + diagnostics.push({ bindingId: binding.bindingId, mode: binding.mode, reason: 'required_over_budget', bytes }); + return { ok: false, error: 'required_context_over_budget', bindings: selected, diagnostics }; + } + selected.push(binding); + used += bytes; + continue; + } + if (used + bytes > maxBytes) { + diagnostics.push({ bindingId: binding.bindingId, mode: binding.mode, reason: 'advisory_trimmed', bytes }); + continue; + } + selected.push(binding); + used += bytes; + } + return { ok: true, bindings: selected, diagnostics }; +} diff --git a/server/src/memory/citation.ts b/server/src/memory/citation.ts new file mode 100644 index 000000000..326a3452d --- /dev/null +++ b/server/src/memory/citation.ts @@ -0,0 +1,82 @@ +import type { Env } from '../env.js'; +import { sha256Text } from '../../../shared/memory-content-hash.js'; +export { + computeProjectionContentHash, + sha256Text, + stableJson, +} from '../../../shared/memory-content-hash.js'; + +const DEFAULT_CITATION_COUNT_RATE_LIMIT = 30; +const DEFAULT_CITATION_COUNT_RATE_LIMIT_WINDOW_MS = 60_000; +const CITATION_COUNT_RATE_LIMIT_ENV = 'IMCODES_MEM_CITATION_COUNT_RATE_LIMIT'; +const CITATION_COUNT_RATE_LIMIT_WINDOW_ENV = 'IMCODES_MEM_CITATION_COUNT_RATE_LIMIT_WINDOW_MS'; + +type CitationCountBucket = { + windowStartedAt: number; + count: number; +}; + +const citationCountBuckets = new Map(); + +export function deriveCitationIdempotencyKey(input: { + scopeNamespace: string; + projectionId: string; + citingMessageId: string; +}): string { + return sha256Text(`cite:v1:${input.scopeNamespace}:${input.projectionId}:${input.citingMessageId}`); +} + +function readPositiveIntegerEnv(env: Env | undefined, key: string, fallback: number): number { + const raw = (env as unknown as Record | undefined)?.[key] ?? process.env[key]; + if (raw == null || raw.trim() === '') return fallback; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +export function getCitationCountRateLimit(env?: Env): { + maxCount: number; + windowMs: number; +} { + return { + maxCount: readPositiveIntegerEnv(env, CITATION_COUNT_RATE_LIMIT_ENV, DEFAULT_CITATION_COUNT_RATE_LIMIT), + windowMs: readPositiveIntegerEnv(env, CITATION_COUNT_RATE_LIMIT_WINDOW_ENV, DEFAULT_CITATION_COUNT_RATE_LIMIT_WINDOW_MS), + }; +} + +export function consumeCitationCountRateLimit(input: { + env?: Env; + userId: string; + projectionId: string; + now: number; +}): { allowed: boolean; remaining: number; resetAt: number } { + const { maxCount, windowMs } = getCitationCountRateLimit(input.env); + const bucketKey = `${input.userId}\u0000${input.projectionId}`; + const existing = citationCountBuckets.get(bucketKey); + const bucket = existing && input.now - existing.windowStartedAt < windowMs + ? existing + : { windowStartedAt: input.now, count: 0 }; + if (bucket.count >= maxCount) { + citationCountBuckets.set(bucketKey, bucket); + return { + allowed: false, + remaining: 0, + resetAt: bucket.windowStartedAt + windowMs, + }; + } + bucket.count += 1; + citationCountBuckets.set(bucketKey, bucket); + + for (const [key, value] of citationCountBuckets.entries()) { + if (input.now - value.windowStartedAt >= windowMs * 2) citationCountBuckets.delete(key); + } + + return { + allowed: true, + remaining: Math.max(0, maxCount - bucket.count), + resetAt: bucket.windowStartedAt + windowMs, + }; +} + +export function resetCitationCountRateLimiterForTests(): void { + citationCountBuckets.clear(); +} diff --git a/server/src/memory/scope-policy.ts b/server/src/memory/scope-policy.ts new file mode 100644 index 000000000..2f2dff411 --- /dev/null +++ b/server/src/memory/scope-policy.ts @@ -0,0 +1,131 @@ +import type { Context } from 'hono'; +import type { Env } from '../env.js'; +import type { RuntimeAuthoredContextBinding } from '../../../shared/context-types.js'; +import { + AUTHORED_CONTEXT_SCOPES, + expandSearchRequestScope as expandSharedSearchRequestScope, + isAuthoredContextScope as isSharedAuthoredContextScope, + isMemoryScope, + isSharedContextProjectionScope, + SYNCED_PROJECTION_MEMORY_SCOPES, + type AuthoredContextScope, + type MemoryScope, + type SearchRequestScope, + type SharedContextProjectionScope, +} from '../../../shared/memory-scope.js'; +import { + MEMORY_FEATURE_FLAGS_BY_NAME, + resolveMemoryFeatureFlagValue, + type MemoryFeatureFlag, + type MemoryFeatureFlagValues, +} from '../../../shared/feature-flags.js'; +export type { AuthoredContextScope, MemoryScope, SearchRequestScope } from '../../../shared/memory-scope.js'; + +export type OwnerPrivateMemoryScope = 'user_private'; +export type SharedProjectionScope = SharedContextProjectionScope; + +export const MEMORY_FEATURES = { + quickSearch: MEMORY_FEATURE_FLAGS_BY_NAME.quickSearch, + citation: MEMORY_FEATURE_FLAGS_BY_NAME.citation, + citeCount: MEMORY_FEATURE_FLAGS_BY_NAME.citeCount, + citeDriftBadge: MEMORY_FEATURE_FLAGS_BY_NAME.citeDriftBadge, + userPrivateSync: MEMORY_FEATURE_FLAGS_BY_NAME.userPrivateSync, + orgSharedAuthoredStandards: MEMORY_FEATURE_FLAGS_BY_NAME.orgSharedAuthoredStandards, +} as const; + +export const SHARED_PROJECTION_SCOPES: readonly SharedProjectionScope[] = SYNCED_PROJECTION_MEMORY_SCOPES; +export { AUTHORED_CONTEXT_SCOPES }; + +export function isSearchRequestScope(value: unknown): value is SearchRequestScope { + return value === 'owner_private' || value === 'shared' || value === 'all_authorized' || isMemoryScope(value); +} + +export function isSharedProjectionScope(value: unknown): value is SharedProjectionScope { + return isSharedContextProjectionScope(value); +} + +export function isAuthoredContextScope(value: unknown): value is AuthoredContextScope { + return isSharedAuthoredContextScope(value); +} + +export function authoredContextScopeForBinding(input: { + workspaceId?: string | null; + enrollmentId?: string | null; +}): AuthoredContextScope { + if (input.enrollmentId) return 'project_shared'; + if (input.workspaceId) return 'workspace_shared'; + return 'org_shared'; +} + +export function expandSearchRequestScope( + requested: SearchRequestScope | undefined, + options: { includeOwnerPrivate: boolean }, +): MemoryScope[] { + const scopes = expandSharedSearchRequestScope(requested ?? 'all_authorized'); + return scopes.filter((scope) => scope !== 'user_private' || options.includeOwnerPrivate); +} + +export function sameShapeMemoryLookupEnvelope(): { + ok: false; + result: null; + citation: null; + error: 'not_found'; +} { + return { ok: false, result: null, citation: null, error: 'not_found' }; +} + +export function sameShapeSearchEnvelope(): { results: []; nextCursor: null } { + return { results: [], nextCursor: null }; +} + +type Feature = MemoryFeatureFlag; + +function envKeyForFeature(feature: Feature): string { + return `IMCODES_${feature.toUpperCase().replace(/[^A-Z0-9]+/g, '_')}`; +} + +export function isMemoryFeatureEnabled( + env: Env | undefined, + feature: Feature, + userConfig?: MemoryFeatureFlagValues, +): boolean { + const key = envKeyForFeature(feature); + const raw = (env as unknown as Record | undefined)?.[key] ?? process.env[key]; + return resolveMemoryFeatureFlagValue(feature, { + persistedConfig: userConfig, + environmentStartupDefault: raw == null ? undefined : { [feature]: raw === 'true' || raw === '1' }, + }); +} + +export async function jsonSameShapeNotFound( + c: Context<{ Bindings: Env }>, +): Promise { + return c.json(sameShapeMemoryLookupEnvelope(), 404); +} + +export function matchesAuthoredContextPathPattern(pattern: string, filePath: string): boolean { + const normalizedPattern = pattern.replace(/\\/g, '/'); + const normalizedPath = filePath.replace(/\\/g, '/'); + if (normalizedPattern.endsWith('/**')) { + return normalizedPath.startsWith(normalizedPattern.slice(0, -3)); + } + if (normalizedPattern.includes('*')) { + const escaped = normalizedPattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*'); + return new RegExp(`^${escaped}$`).test(normalizedPath); + } + return normalizedPattern === normalizedPath; +} + +export function compareRuntimeAuthoredContextBindings( + a: Pick, + b: Pick, +): number { + const rank: Record = { + project_shared: 1, + workspace_shared: 2, + org_shared: 3, + }; + if (rank[a.scope] !== rank[b.scope]) return rank[a.scope] - rank[b.scope]; + if (a.mode !== b.mode) return a.mode === 'required' ? -1 : 1; + return a.bindingId.localeCompare(b.bindingId); +} diff --git a/server/src/routes/server.ts b/server/src/routes/server.ts index 19b9ec585..3458ab87c 100644 --- a/server/src/routes/server.ts +++ b/server/src/routes/server.ts @@ -17,8 +17,10 @@ import { sha256Hex, randomHex } from '../security/crypto.js'; import { requireAuth } from '../security/authorization.js'; import { z } from 'zod'; import type { + ContextMemoryProjectView, ContextMemoryRecordView, ContextMemoryStatsView, + ContextMemoryView, ProcessedContextReplicationBody, RuntimeAuthoredContextBinding, SharedContextNamespaceResolution, @@ -33,20 +35,54 @@ import { import { searchSemanticMemoryView } from '../util/semantic-memory-view.js'; import { deletePersonalMemoryProjection } from '../util/memory-delete.js'; import { isMemoryNoiseSummary } from '../../../shared/memory-noise-patterns.js'; +import { MEMORY_ORIGINS } from '../../../shared/memory-origin.js'; +import { OBSERVATION_CLASSES } from '../../../shared/memory-observation.js'; +import { DAEMON_COMMAND_TYPES } from '../../../shared/daemon-command-types.js'; +import { + SYNCED_PROJECTION_MEMORY_SCOPES, + type AuthoredContextScope, + type SharedContextProjectionScope, +} from '../../../shared/memory-scope.js'; +import { computeProjectionContentHash } from '../memory/citation.js'; import { SUPERVISION_USER_DEFAULT_PREF_KEY } from '../../../shared/supervision-config.js'; +import { + MEMORY_FEATURE_CONFIG_PREF_KEY, + parseMemoryFeatureFlagValuesJson, +} from '../../../shared/feature-flags.js'; +import { + authoredContextScopeForBinding, + expandSearchRequestScope, + compareRuntimeAuthoredContextBindings, + isMemoryFeatureEnabled, + isSearchRequestScope, + matchesAuthoredContextPathPattern, + MEMORY_FEATURES, + sameShapeMemoryLookupEnvelope, + sameShapeSearchEnvelope, +} from '../memory/scope-policy.js'; export const serverRoutes = new Hono<{ Bindings: Env; Variables: { userId: string; role: string } }>(); +const OWNER_PRIVATE_MAX_RECORDS = 100; +const OWNER_PRIVATE_MAX_TEXT_BYTES = 32 * 1024; +const OWNER_PRIVATE_MAX_CONTENT_BYTES = 128 * 1024; +const OWNER_PRIVATE_MAX_QUERY_CHARS = 512; + +function utf8Bytes(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + const processedProjectionSchema = z.object({ id: z.string().min(1), namespace: z.object({ - scope: z.enum(['personal', 'project_shared', 'workspace_shared', 'org_shared']), + scope: z.enum(SYNCED_PROJECTION_MEMORY_SCOPES), projectId: z.string().min(1), userId: z.string().optional(), workspaceId: z.string().optional(), enterpriseId: z.string().optional(), }), class: z.enum(['recent_summary', 'durable_memory_candidate']), + origin: z.enum(MEMORY_ORIGINS), sourceEventIds: z.array(z.string()), summary: z.string(), content: z.record(z.string(), z.unknown()), @@ -59,6 +95,42 @@ const processedReplicationSchema = z.object({ projections: z.array(processedProjectionSchema).min(1), }); +const ownerPrivateRecordSchema = z.object({ + id: z.string().min(1).optional(), + kind: z.enum(OBSERVATION_CLASSES), + origin: z.enum(MEMORY_ORIGINS), + fingerprint: z.string().min(1).max(256), + text: z.string().min(1).refine((value) => utf8Bytes(value) <= OWNER_PRIVATE_MAX_TEXT_BYTES, { + message: 'text_too_large', + }), + content: z.record(z.string(), z.unknown()).optional().default({}), + idempotencyKey: z.string().min(1).max(256).optional(), + createdAt: z.number().finite().optional(), + updatedAt: z.number().finite().optional(), +}).superRefine((record, ctx) => { + if (utf8Bytes(JSON.stringify(record.content)) > OWNER_PRIVATE_MAX_CONTENT_BYTES) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['content'], + message: 'content_too_large', + }); + } +}); + +const ownerPrivateReplicationSchema = z.object({ + namespace: z.object({ + scope: z.literal('user_private'), + userId: z.string().optional(), + }), + records: z.array(ownerPrivateRecordSchema).min(1).max(OWNER_PRIVATE_MAX_RECORDS), +}); + +const ownerPrivateSearchSchema = z.object({ + query: z.string().trim().max(OWNER_PRIVATE_MAX_QUERY_CHARS).optional().default(''), + scope: z.unknown().optional(), + limit: z.number().finite().min(1).max(100).optional().default(20), +}); + const authoredContextQuerySchema = z.object({ namespace: processedProjectionSchema.shape.namespace.refine((namespace) => namespace.scope !== 'personal', { message: 'shared_scope_required', @@ -119,19 +191,44 @@ type MemoryStatsRow = { project_count?: number | null; }; +type MemoryProjectStatsRow = { + project_id: string; + total_records?: number | null; + recent_summary_count?: number | null; + durable_candidate_count?: number | null; + updated_at?: number | null; +}; + type MemoryRecordRow = { id: string; - scope: 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: SharedContextProjectionScope; project_id: string; projection_class: 'recent_summary' | 'durable_memory_candidate'; source_event_ids_json: string | string[]; summary: string; + content_json?: string | Record | null; updated_at: number; hit_count?: number | null; last_used_at?: number | null; status?: 'active' | 'archived' | null; }; +function parseRecordContent(raw: string | Record | null | undefined): Record { + if (!raw) return {}; + if (typeof raw === 'object' && !Array.isArray(raw)) return raw; + if (typeof raw !== 'string') return {}; + try { + const parsed = JSON.parse(raw) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as Record : {}; + } catch { + return {}; + } +} + +function metadataUserId(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + function buildMemoryStatsView( row: MemoryStatsRow | null | undefined, matchedRecords: number, @@ -149,26 +246,73 @@ function buildMemoryStatsView( } function mapMemoryRecordRows(rows: MemoryRecordRow[]): ContextMemoryRecordView[] { - return rows.map((row) => ({ - id: row.id, - scope: row.scope, - projectId: row.project_id, - summary: row.summary, - projectionClass: row.projection_class, - sourceEventCount: Array.isArray(row.source_event_ids_json) - ? row.source_event_ids_json.length - : JSON.parse(row.source_event_ids_json || '[]').length, - updatedAt: row.updated_at, - hitCount: row.hit_count ?? 0, - lastUsedAt: row.last_used_at ?? undefined, - status: row.status ?? 'active', - })); + return rows.map((row) => { + const content = parseRecordContent(row.content_json); + const ownerUserId = metadataUserId(content.ownerUserId) ?? metadataUserId(content.ownedByUserId) ?? metadataUserId(content.userId); + const createdByUserId = metadataUserId(content.createdByUserId) ?? metadataUserId(content.authorUserId) ?? ownerUserId; + return { + id: row.id, + scope: row.scope, + projectId: row.project_id, + ownerUserId, + createdByUserId, + updatedByUserId: metadataUserId(content.updatedByUserId) ?? createdByUserId, + summary: row.summary, + projectionClass: row.projection_class, + sourceEventCount: Array.isArray(row.source_event_ids_json) + ? row.source_event_ids_json.length + : JSON.parse(row.source_event_ids_json || '[]').length, + updatedAt: row.updated_at, + hitCount: row.hit_count ?? 0, + lastUsedAt: row.last_used_at ?? undefined, + status: row.status ?? 'active', + }; + }); +} + +function mapMemoryProjectRows(rows: MemoryProjectStatsRow[]): ContextMemoryProjectView[] { + return rows + .filter((row) => row.project_id) + .map((row) => ({ + projectId: row.project_id, + displayName: row.project_id, + totalRecords: row.total_records ?? 0, + recentSummaryCount: row.recent_summary_count ?? 0, + durableCandidateCount: row.durable_candidate_count ?? 0, + updatedAt: row.updated_at ?? undefined, + })); +} + +function buildMemoryProjectsFromRows(rows: Array & { + projection_class: 'recent_summary' | 'durable_memory_candidate'; + updated_at: number; +}>): ContextMemoryProjectView[] { + const projects = new Map(); + for (const row of rows) { + if (!row.project_id) continue; + const current = projects.get(row.project_id) ?? { + projectId: row.project_id, + displayName: row.project_id, + totalRecords: 0, + recentSummaryCount: 0, + durableCandidateCount: 0, + updatedAt: row.updated_at, + }; + current.totalRecords += 1; + if (row.projection_class === 'recent_summary') current.recentSummaryCount += 1; + if (row.projection_class === 'durable_memory_candidate') current.durableCandidateCount += 1; + current.updatedAt = Math.max(current.updatedAt ?? 0, row.updated_at ?? 0) || undefined; + projects.set(row.project_id, current); + } + return Array.from(projects.values()) + .sort((a, b) => (b.updatedAt ?? 0) - (a.updatedAt ?? 0) || b.totalRecords - a.totalRecords || a.projectId.localeCompare(b.projectId)) + .slice(0, 200); } function buildRemoteMemoryResponse( rows: Array<{ id: string; - scope: 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: SharedContextProjectionScope; project_id: string; projection_class: 'recent_summary' | 'durable_memory_candidate'; source_event_ids_json: string | string[]; @@ -181,7 +325,7 @@ function buildRemoteMemoryResponse( }>, query?: string, limit = 20, -): { stats: ContextMemoryStatsView; records: ContextMemoryRecordView[] } { +): ContextMemoryView { const normalizedQuery = query?.trim() ?? ''; const cleanRows = rows.filter((row) => !isMemoryNoiseSummary(row.summary)); const filtered = cleanRows.filter((row) => matchesMemoryQuery( @@ -198,6 +342,7 @@ function buildRemoteMemoryResponse( project_count: projectIds.size, }, filtered.length), records: mapMemoryRecordRows(filtered.slice(0, limit)), + projects: buildMemoryProjectsFromRows(cleanRows), }; } @@ -211,6 +356,7 @@ serverRoutes.get('/', requireAuth(), async (c) => { name: s.name, status: s.status, lastHeartbeatAt: s.last_heartbeat_at, + daemonVersion: s.daemon_version, createdAt: s.created_at, })); @@ -237,7 +383,7 @@ serverRoutes.delete('/:id', requireAuth(), async (c) => { // Notify daemon to self-destruct (best-effort — daemon may be offline) try { - WsBridge.get(serverId).sendToDaemon(JSON.stringify({ type: 'server.delete' })); + WsBridge.get(serverId).sendToDaemon(JSON.stringify({ type: DAEMON_COMMAND_TYPES.SERVER_DELETE })); } catch { /* daemon may be offline, continue with DB deletion */ } const deleted = await deleteServer(c.env.DB, serverId, userId); @@ -251,15 +397,21 @@ serverRoutes.post('/:id/upgrade', requireAuth(), async (c) => { const serverId = c.req.param('id') ?? ''; const dbServers = await getServersByUserId(c.env.DB, userId); if (!dbServers.find((s) => s.id === serverId)) return c.json({ error: 'not_found' }, 404); - try { - WsBridge.get(serverId).sendToDaemon(JSON.stringify({ - type: 'daemon.upgrade', - ...(process.env.APP_VERSION ? { targetVersion: process.env.APP_VERSION } : {}), - })); - return c.json({ ok: true }); - } catch { - return c.json({ error: 'daemon_offline' }, 503); + const result = WsBridge.get(serverId).requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'manual', + }); + if (!result.ok) { + return c.json({ error: result.reason ?? 'upgrade_request_failed', deliveryStatus: result.deliveryStatus }, 400); } + return c.json({ + ok: true, + upgradeId: result.upgradeId, + targetVersion: result.targetVersion, + deliveryStatus: result.deliveryStatus, + ...(result.nextAttemptAt ? { nextAttemptAt: result.nextAttemptAt } : {}), + ...(result.reason ? { reason: result.reason } : {}), + }); }); // POST /api/server/:id/heartbeat — authenticated via Bearer server token @@ -276,7 +428,9 @@ serverRoutes.post('/:id/heartbeat', async (c) => { ); if (!server) return c.json({ error: 'unauthorized' }, 401); - await updateServerHeartbeat(c.env.DB, serverId); + const body = await c.req.json().catch(() => null) as Record | null; + const daemonVersion = typeof body?.daemonVersion === 'string' ? body.daemonVersion : undefined; + await updateServerHeartbeat(c.env.DB, serverId, daemonVersion); return c.json({ ok: true }); }); @@ -494,12 +648,16 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { const safeEnterpriseId = isPersonal ? null : (serverRow.team_id ?? projection.namespace.enterpriseId ?? null); const safeWorkspaceId = isPersonal ? null : (projection.namespace.workspaceId ?? null); const safeUserId = isPersonal ? serverRow.user_id : (projection.namespace.userId ?? null); + const contentHash = computeProjectionContentHash({ + summary: projection.summary, + content: projection.content, + }); await c.env.DB.execute( `INSERT INTO shared_context_projections ( id, server_id, scope, enterprise_id, workspace_id, user_id, project_id, projection_class, source_event_ids_json, summary, content_json, - created_at, updated_at, replicated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11::jsonb, $12, $13, $14) + content_hash, origin, created_at, updated_at, replicated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11::jsonb, $12, $13, $14, $15, $16) ON CONFLICT (id) DO UPDATE SET scope = excluded.scope, enterprise_id = excluded.enterprise_id, @@ -510,6 +668,8 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { source_event_ids_json = excluded.source_event_ids_json, summary = excluded.summary, content_json = excluded.content_json, + content_hash = excluded.content_hash, + origin = excluded.origin, created_at = excluded.created_at, updated_at = excluded.updated_at, replicated_at = excluded.replicated_at`, @@ -525,6 +685,8 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { JSON.stringify(projection.sourceEventIds), projection.summary, JSON.stringify(projection.content), + contentHash, + projection.origin, projection.createdAt, projection.updatedAt, now, @@ -537,8 +699,8 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { await c.env.DB.execute( `INSERT INTO shared_context_records ( id, projection_id, server_id, scope, enterprise_id, workspace_id, user_id, project_id, - record_class, summary, content_json, status, created_at, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb, 'candidate', $12, $13) + record_class, summary, content_json, status, origin, created_at, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb, 'candidate', $12, $13, $14) ON CONFLICT (projection_id) DO UPDATE SET scope = excluded.scope, enterprise_id = excluded.enterprise_id, @@ -548,6 +710,7 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { record_class = excluded.record_class, summary = excluded.summary, content_json = excluded.content_json, + origin = excluded.origin, updated_at = excluded.updated_at`, [ `record:${projection.id}`, @@ -561,6 +724,7 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { projection.class, projection.summary, JSON.stringify(projection.content), + projection.origin, projection.createdAt, projection.updatedAt, ], @@ -584,6 +748,125 @@ serverRoutes.post('/:id/shared-context/processed', async (c) => { }); }); +serverRoutes.post('/:id/shared-context/owner-private', async (c) => { + const auth = c.req.header('Authorization'); + if (!auth?.startsWith('Bearer ')) return c.json({ error: 'unauthorized' }, 401); + const tokenHash = sha256Hex(auth.slice(7)); + + const serverRow = await c.env.DB.queryOne<{ id: string; user_id: string }>( + 'SELECT id, user_id FROM servers WHERE token_hash = $1 AND id = $2', + [tokenHash, c.req.param('id')], + ); + if (!serverRow) return c.json({ error: 'unauthorized' }, 401); + const featureFlags = parseMemoryFeatureFlagValuesJson( + await getUserPref(c.env.DB, serverRow.user_id, MEMORY_FEATURE_CONFIG_PREF_KEY), + ); + if (!isMemoryFeatureEnabled(c.env, MEMORY_FEATURES.userPrivateSync, featureFlags)) { + return c.json(sameShapeMemoryLookupEnvelope(), 404); + } + + const body = await c.req.json().catch(() => null); + const parsed = ownerPrivateReplicationSchema.safeParse(body); + if (!parsed.success) return c.json({ error: 'invalid_body' }, 400); + if (parsed.data.namespace.userId && parsed.data.namespace.userId !== serverRow.user_id) { + return c.json(sameShapeMemoryLookupEnvelope(), 404); + } + + const now = Date.now(); + let acceptedCount = 0; + for (const record of parsed.data.records) { + const idempotencyKey = record.idempotencyKey + ?? sha256Hex(`owner-private:v1:${serverRow.user_id}:${record.kind}:${record.fingerprint}:${record.text}`); + const recordId = record.id ?? sha256Hex(`owner-private-id:v1:${serverRow.user_id}:${idempotencyKey}`); + const createdAt = record.createdAt ?? now; + const updatedAt = record.updatedAt ?? createdAt; + await c.env.DB.execute( + `INSERT INTO owner_private_memories ( + id, owner_user_id, scope, kind, origin, fingerprint, text, content_json, + idempotency_key, source_server_id, created_at, updated_at, replicated_at + ) VALUES ($1, $2, 'user_private', $3, $4, $5, $6, $7::jsonb, $8, $9, $10, $11, $12) + ON CONFLICT (owner_user_id, idempotency_key) DO UPDATE SET + kind = excluded.kind, + origin = excluded.origin, + fingerprint = excluded.fingerprint, + text = excluded.text, + content_json = excluded.content_json, + source_server_id = excluded.source_server_id, + updated_at = excluded.updated_at, + replicated_at = excluded.replicated_at`, + [ + recordId, + serverRow.user_id, + record.kind, + record.origin, + record.fingerprint, + record.text, + JSON.stringify(record.content), + idempotencyKey, + serverRow.id, + createdAt, + updatedAt, + now, + ], + ); + acceptedCount += 1; + } + + return c.json({ ok: true, replicatedAt: now, memoryCount: acceptedCount }); +}); + +serverRoutes.post('/:id/shared-context/owner-private/search', async (c) => { + const auth = c.req.header('Authorization'); + if (!auth?.startsWith('Bearer ')) return c.json({ error: 'unauthorized' }, 401); + const tokenHash = sha256Hex(auth.slice(7)); + + const serverRow = await c.env.DB.queryOne<{ id: string; user_id: string }>( + 'SELECT id, user_id FROM servers WHERE token_hash = $1 AND id = $2', + [tokenHash, c.req.param('id')], + ); + if (!serverRow) return c.json({ error: 'unauthorized' }, 401); + const featureFlags = parseMemoryFeatureFlagValuesJson( + await getUserPref(c.env.DB, serverRow.user_id, MEMORY_FEATURE_CONFIG_PREF_KEY), + ); + if (!isMemoryFeatureEnabled(c.env, MEMORY_FEATURES.userPrivateSync, featureFlags)) { + return c.json(sameShapeSearchEnvelope()); + } + + const body = await c.req.json().catch(() => null); + const parsed = ownerPrivateSearchSchema.safeParse(body); + if (!parsed.success) return c.json({ error: 'invalid_body' }, 400); + const requestScope = isSearchRequestScope(parsed.data.scope) ? parsed.data.scope : 'owner_private'; + const scopes = expandSearchRequestScope(requestScope, { includeOwnerPrivate: true }); + if (!scopes.includes('user_private')) return c.json(sameShapeSearchEnvelope()); + const query = parsed.data.query.trim(); + const rows = await c.env.DB.query<{ + id: string; + kind: string; + origin: (typeof MEMORY_ORIGINS)[number]; + text: string; + updated_at: number; + }>( + `SELECT id, kind, origin, text, updated_at + FROM owner_private_memories + WHERE owner_user_id = $1 + ${query ? 'AND text ILIKE $2' : ''} + ORDER BY updated_at DESC + LIMIT $${query ? 3 : 2}`, + [serverRow.user_id, ...(query ? [`%${query}%`] : []), parsed.data.limit], + ); + return c.json({ + results: rows.map((row) => ({ + id: row.id, + scope: 'user_private' as const, + kind: row.kind, + origin: row.origin, + preview: row.text.slice(0, 240), + updatedAt: row.updated_at, + })), + nextCursor: null, + }); +}); + serverRoutes.delete('/:id/shared-context/personal-memory/:memoryId', requireAuth(), async (c) => { const userId = c.get('userId' as never) as string; const serverId = c.req.param('id') ?? ''; @@ -647,9 +930,26 @@ serverRoutes.get('/:id/shared-context/personal-memory', requireAuth(), async (c) LIMIT $${projectId ? (projectionClass ? 4 : 3) : (projectionClass ? 3 : 2)}`, [userId, ...(projectId ? [projectId] : []), ...(projectionClass ? [projectionClass] : []), limit], ); + const projectRows = await c.env.DB.query( + `SELECT project_id, + COUNT(*)::int AS total_records, + COUNT(*) FILTER (WHERE projection_class = 'recent_summary')::int AS recent_summary_count, + COUNT(*) FILTER (WHERE projection_class = 'durable_memory_candidate')::int AS durable_candidate_count, + MAX(updated_at) AS updated_at + FROM shared_context_projections + WHERE user_id = $1 + AND scope = 'personal' + ${projectId ? 'AND project_id = $2' : ''} + ${projectionClass ? `AND projection_class = $${projectId ? 3 : 2}` : ''} + GROUP BY project_id + ORDER BY MAX(updated_at) DESC + LIMIT 200`, + [userId, ...(projectId ? [projectId] : []), ...(projectionClass ? [projectionClass] : [])], + ); return c.json({ stats: buildMemoryStatsView(stats, stats?.total_records ?? 0), records: mapMemoryRecordRows(rows.filter((row) => !isMemoryNoiseSummary(row.summary))), + projects: mapMemoryProjectRows(projectRows), }); } @@ -684,8 +984,8 @@ serverRoutes.post('/:id/shared-context/authored-bindings', async (c) => { const token = auth.slice(7); const tokenHash = sha256Hex(token); - const serverRow = await c.env.DB.queryOne<{ id: string; team_id: string | null }>( - 'SELECT id, team_id FROM servers WHERE token_hash = $1 AND id = $2', + const serverRow = await c.env.DB.queryOne<{ id: string; team_id: string | null; user_id: string }>( + 'SELECT id, team_id, user_id FROM servers WHERE token_hash = $1 AND id = $2', [tokenHash, c.req.param('id')], ); if (!serverRow) return c.json({ error: 'unauthorized' }, 401); @@ -704,7 +1004,8 @@ serverRoutes.post('/:id/shared-context/authored-bindings', async (c) => { binding_id: string; version_id: string; binding_mode: RuntimeAuthoredContextBinding['mode']; - scope: RuntimeAuthoredContextBinding['scope']; + workspace_id: string | null; + enrollment_id: string | null; applicability_repo_id: string | null; applicability_language: string | null; applicability_path_pattern: string | null; @@ -716,11 +1017,8 @@ serverRoutes.post('/:id/shared-context/authored-bindings', async (c) => { b.id AS binding_id, v.id AS version_id, b.binding_mode, - CASE - WHEN b.enrollment_id IS NOT NULL THEN 'project_shared' - WHEN b.workspace_id IS NOT NULL THEN 'workspace_shared' - ELSE 'org_shared' - END AS scope, + b.workspace_id, + b.enrollment_id, b.applicability_repo_id, b.applicability_language, b.applicability_path_pattern, @@ -744,20 +1042,30 @@ serverRoutes.post('/:id/shared-context/authored-bindings', async (c) => { [enterpriseId, namespace.workspaceId ?? null, namespace.projectId], ); + const featureFlags = parseMemoryFeatureFlagValuesJson( + await getUserPref(c.env.DB, serverRow.user_id, MEMORY_FEATURE_CONFIG_PREF_KEY), + ); + const orgAuthoredEnabled = isMemoryFeatureEnabled(c.env, MEMORY_FEATURES.orgSharedAuthoredStandards, featureFlags); const bindings: RuntimeAuthoredContextBinding[] = rows .map((row) => ({ bindingId: row.binding_id, documentVersionId: row.version_id, mode: row.binding_mode, - scope: row.scope, + scope: authoredContextScopeForBinding({ + workspaceId: row.workspace_id, + enrollmentId: row.enrollment_id, + }), repository: row.applicability_repo_id ?? undefined, language: row.applicability_language ?? undefined, pathPattern: row.applicability_path_pattern ?? undefined, content: row.content_md, active: true, })) + .filter((binding) => binding.scope !== 'org_shared' || orgAuthoredEnabled) + .filter((binding) => !binding.repository || binding.repository === namespace.projectId) .filter((binding) => !binding.language || binding.language === language) - .filter((binding) => !binding.pathPattern || !!filePath); + .filter((binding) => !binding.pathPattern || (!!filePath && matchesAuthoredContextPathPattern(binding.pathPattern, filePath))) + .sort(compareRuntimeAuthoredContextBindings); return c.json({ bindings }); }); @@ -805,7 +1113,7 @@ serverRoutes.post('/:id/shared-context/resolve-namespace', async (c) => { id: string; enterprise_id: string; workspace_id: string | null; - scope: 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: AuthoredContextScope; status: 'active' | 'pending_removal' | 'removed'; }>( 'SELECT id, enterprise_id, workspace_id, scope, status FROM shared_project_enrollments WHERE enterprise_id = $1 AND canonical_repo_id = $2', diff --git a/server/src/routes/session-mgmt.ts b/server/src/routes/session-mgmt.ts index 65fbc293b..305b0be5a 100644 --- a/server/src/routes/session-mgmt.ts +++ b/server/src/routes/session-mgmt.ts @@ -17,6 +17,7 @@ export const sessionMgmtRoutes = new Hono<{ Bindings: Env; Variables: { userId: /** * POST /api/server/:id/session/start * POST /api/server/:id/session/stop + * POST /api/server/:id/session/cancel * POST /api/server/:id/session/send * * All commands are relayed to the daemon via WsBridge (JSON over WebSocket). @@ -24,7 +25,7 @@ export const sessionMgmtRoutes = new Hono<{ Bindings: Env; Variables: { userId: * * Permission model: * - start/stop: requires owner | admin - * - send: requires owner | admin | member + * - send/cancel: requires owner | admin | member */ // Apply auth middleware globally to all session routes @@ -311,6 +312,28 @@ sessionMgmtRoutes.post('/:id/session/stop', async (c) => { return relayToDaemon(c, 'session.stop'); }); +sessionMgmtRoutes.post('/:id/session/cancel', async (c) => { + const userId = c.get('userId' as never) as string; + const role = await resolveServerRole(c.env.DB, c.req.param('id')!, userId); + if (role === 'none') { + return c.json({ error: 'forbidden', reason: 'not_authorized_for_server' }, 403); + } + let body: Record = {}; + try { + const parsed = await c.req.json(); + body = parsed && typeof parsed === 'object' ? parsed as Record : {}; + } catch { + // body is optional; daemon will validate required fields + } + const sessionName = typeof body.sessionName === 'string' ? body.sessionName : undefined; + const session = typeof body.session === 'string' ? body.session : undefined; + const commandId = typeof body.commandId === 'string' ? body.commandId : undefined; + return relayToDaemon(c, DAEMON_COMMAND_TYPES.SESSION_CANCEL, { + ...(sessionName ? { sessionName } : session ? { session } : {}), + ...(commandId ? { commandId } : {}), + }); +}); + sessionMgmtRoutes.post('/:id/session/send', async (c) => { const userId = c.get('userId' as never) as string; const role = await resolveServerRole(c.env.DB, c.req.param('id')!, userId); diff --git a/server/src/routes/shared-context.ts b/server/src/routes/shared-context.ts index 53f29eadc..8e96a9729 100644 --- a/server/src/routes/shared-context.ts +++ b/server/src/routes/shared-context.ts @@ -6,14 +6,46 @@ import { logAudit } from '../security/audit.js'; import { parseRemoteUrl } from '../../../src/repo/detector.js'; import { parseCanonicalRepositoryKey } from '../../../src/agent/repository-identity-service.js'; import { classifyTimestampFreshness } from '../../../shared/context-freshness.js'; -import type { ContextMemoryRecordView, ContextMemoryStatsView } from '../../../shared/context-types.js'; +import type { ContextMemoryProjectView, ContextMemoryRecordView, ContextMemoryStatsView, ContextMemoryView } from '../../../shared/context-types.js'; import { computeRelevanceScore, applyRecallCapRule, type ProjectionClass } from '../../../shared/memory-scoring.js'; import { normalizeSharedContextRuntimeConfig } from '../../../shared/shared-context-runtime-config.js'; import { isTemplatePrompt, isTemplateOriginSummary, isImperativeCommand } from '../../../shared/template-prompt-patterns.js'; import { isMemoryNoiseSummary } from '../../../shared/memory-noise-patterns.js'; import { normalizeSummaryForFingerprint } from '../../../shared/memory-fingerprint.js'; +import { isMemoryOrigin, type MemoryOrigin } from '../../../shared/memory-origin.js'; +import { REPLICABLE_SHARED_PROJECTION_SCOPES } from '../../../shared/memory-scope.js'; +import { + MEMORY_FEATURE_CONFIG_PREF_KEY, + parseMemoryFeatureFlagValuesJson, + type MemoryFeatureFlag, + type MemoryFeatureFlagValues, +} from '../../../shared/feature-flags.js'; import { searchSemanticMemoryView } from '../util/semantic-memory-view.js'; +import { applyRuntimeAuthoredContextBudget } from '../memory/authored-context-runtime.js'; import { deleteEnterpriseMemoryProjection, deletePersonalMemoryProjection } from '../util/memory-delete.js'; +import { + authoredContextScopeForBinding, + compareRuntimeAuthoredContextBindings, + expandSearchRequestScope, + isMemoryFeatureEnabled, + isSearchRequestScope, + isAuthoredContextScope, + isSharedProjectionScope, + matchesAuthoredContextPathPattern, + MEMORY_FEATURES, + sameShapeMemoryLookupEnvelope, + sameShapeSearchEnvelope, + type AuthoredContextScope, + type MemoryScope, + type SearchRequestScope, + type SharedProjectionScope, +} from '../memory/scope-policy.js'; +import { + computeProjectionContentHash, + consumeCitationCountRateLimit, + deriveCitationIdempotencyKey, +} from '../memory/citation.js'; +import { getUserPref } from '../db/queries.js'; type EnterpriseRole = 'owner' | 'admin' | 'member'; type BindingMode = 'required' | 'advisory'; @@ -24,6 +56,19 @@ export const sharedContextRoutes = new Hono<{ Bindings: Env; Variables: { userId sharedContextRoutes.use('*', requireAuth()); type SharedContextRouteContext = Context<{ Bindings: Env; Variables: { userId: string; role: string } }>; +async function getCurrentUserMemoryFeatureFlags(c: SharedContextRouteContext): Promise { + const userId = c.get('userId' as never) as string; + return parseMemoryFeatureFlagValuesJson(await getUserPref(c.env.DB, userId, MEMORY_FEATURE_CONFIG_PREF_KEY)); +} + +function isUserMemoryFeatureEnabled( + c: SharedContextRouteContext, + feature: MemoryFeatureFlag, + flags: MemoryFeatureFlagValues, +): boolean { + return isMemoryFeatureEnabled(c.env, feature, flags); +} + async function getEnterpriseRole(db: Env['DB'], enterpriseId: string, userId: string): Promise { const row = await db.queryOne<{ role: EnterpriseRole }>( 'SELECT role FROM team_members WHERE team_id = $1 AND user_id = $2', @@ -39,10 +84,10 @@ async function requireEnterpriseRole( ): Promise<{ userId: string; role: EnterpriseRole } | Response> { const userId = c.get('userId' as never) as string; const role = await getEnterpriseRole(c.env.DB, enterpriseId, userId); - if (!role) return c.json({ error: 'forbidden', reason: 'not_a_team_member' }, 403); + if (!role) return sameShapeNotFound(c); const rank: Record = { owner: 3, admin: 2, member: 1 }; if (rank[role] < rank[minRole]) { - return c.json({ error: 'forbidden', required: minRole, actual: role }, 403); + return c.json({ error: 'forbidden' }, 403); } return { userId, role }; } @@ -89,6 +134,10 @@ async function readJsonBody(c: SharedContextRouteContext): Promise return await c.req.json().catch(() => null) as T | null; } +function sameShapeNotFound(c: SharedContextRouteContext): Response { + return c.json(sameShapeMemoryLookupEnvelope(), 404); +} + type EnrollmentVisibilityState = | 'unenrolled' | 'active' @@ -162,19 +211,44 @@ type MemoryStatsRow = { project_count?: number | null; }; +type MemoryProjectStatsRow = { + project_id: string; + total_records?: number | null; + recent_summary_count?: number | null; + durable_candidate_count?: number | null; + updated_at?: number | null; +}; + type MemoryRecordRow = { id: string; - scope: 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: SharedProjectionScope; project_id: string; projection_class: 'recent_summary' | 'durable_memory_candidate'; source_event_ids_json: string | string[]; summary: string; + content_json?: string | Record | null; updated_at: number; hit_count?: number | null; last_used_at?: number | null; status?: 'active' | 'archived' | null; }; +function parseRecordContent(raw: string | Record | null | undefined): Record { + if (!raw) return {}; + if (typeof raw === 'object' && !Array.isArray(raw)) return raw; + if (typeof raw !== 'string') return {}; + try { + const parsed = JSON.parse(raw) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as Record : {}; + } catch { + return {}; + } +} + +function metadataUserId(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + function buildMemoryStatsView( row: MemoryStatsRow | null | undefined, matchedRecords: number, @@ -192,26 +266,73 @@ function buildMemoryStatsView( } function mapMemoryRecordRows(rows: MemoryRecordRow[]): ContextMemoryRecordView[] { - return rows.map((row) => ({ - id: row.id, - scope: row.scope, - projectId: row.project_id, - summary: row.summary, - projectionClass: row.projection_class, - sourceEventCount: Array.isArray(row.source_event_ids_json) - ? row.source_event_ids_json.length - : JSON.parse(row.source_event_ids_json || '[]').length, - updatedAt: row.updated_at, - hitCount: row.hit_count ?? 0, - lastUsedAt: row.last_used_at ?? undefined, - status: row.status ?? 'active', - })); + return rows.map((row) => { + const content = parseRecordContent(row.content_json); + const ownerUserId = metadataUserId(content.ownerUserId) ?? metadataUserId(content.ownedByUserId) ?? metadataUserId(content.userId); + const createdByUserId = metadataUserId(content.createdByUserId) ?? metadataUserId(content.authorUserId) ?? ownerUserId; + return { + id: row.id, + scope: row.scope, + projectId: row.project_id, + ownerUserId, + createdByUserId, + updatedByUserId: metadataUserId(content.updatedByUserId) ?? createdByUserId, + summary: row.summary, + projectionClass: row.projection_class, + sourceEventCount: Array.isArray(row.source_event_ids_json) + ? row.source_event_ids_json.length + : JSON.parse(row.source_event_ids_json || '[]').length, + updatedAt: row.updated_at, + hitCount: row.hit_count ?? 0, + lastUsedAt: row.last_used_at ?? undefined, + status: row.status ?? 'active', + }; + }); +} + +function mapMemoryProjectRows(rows: MemoryProjectStatsRow[]): ContextMemoryProjectView[] { + return rows + .filter((row) => row.project_id) + .map((row) => ({ + projectId: row.project_id, + displayName: row.project_id, + totalRecords: row.total_records ?? 0, + recentSummaryCount: row.recent_summary_count ?? 0, + durableCandidateCount: row.durable_candidate_count ?? 0, + updatedAt: row.updated_at ?? undefined, + })); +} + +function buildMemoryProjectsFromRows(rows: Array & { + projection_class: 'recent_summary' | 'durable_memory_candidate'; + updated_at: number; +}>): ContextMemoryProjectView[] { + const projects = new Map(); + for (const row of rows) { + if (!row.project_id) continue; + const current = projects.get(row.project_id) ?? { + projectId: row.project_id, + displayName: row.project_id, + totalRecords: 0, + recentSummaryCount: 0, + durableCandidateCount: 0, + updatedAt: row.updated_at, + }; + current.totalRecords += 1; + if (row.projection_class === 'recent_summary') current.recentSummaryCount += 1; + if (row.projection_class === 'durable_memory_candidate') current.durableCandidateCount += 1; + current.updatedAt = Math.max(current.updatedAt ?? 0, row.updated_at ?? 0) || undefined; + projects.set(row.project_id, current); + } + return Array.from(projects.values()) + .sort((a, b) => (b.updatedAt ?? 0) - (a.updatedAt ?? 0) || b.totalRecords - a.totalRecords || a.projectId.localeCompare(b.projectId)) + .slice(0, 200); } function buildSharedMemoryResponse( rows: Array<{ id: string; - scope: 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: SharedProjectionScope; project_id: string; projection_class: 'recent_summary' | 'durable_memory_candidate'; source_event_ids_json: string | string[]; @@ -224,7 +345,7 @@ function buildSharedMemoryResponse( }>, query?: string, limit = 20, -): { stats: ContextMemoryStatsView; records: ContextMemoryRecordView[] } { +): ContextMemoryView { const normalizedQuery = query?.trim() ?? ''; const cleanRows = rows.filter((row) => !isMemoryNoiseSummary(row.summary)); const filtered = cleanRows.filter((row) => matchesMemoryQuery( @@ -241,6 +362,7 @@ function buildSharedMemoryResponse( project_count: projectIds.size, }, filtered.length), records: mapMemoryRecordRows(filtered.slice(0, limit)), + projects: buildMemoryProjectsFromRows(cleanRows), }; } @@ -300,9 +422,26 @@ sharedContextRoutes.get('/personal-memory', async (c) => { LIMIT $${projectId ? (projectionClass ? 4 : 3) : (projectionClass ? 3 : 2)}`, [userId, ...(projectId ? [projectId] : []), ...(projectionClass ? [projectionClass] : []), limit], ); + const projectRows = await c.env.DB.query( + `SELECT project_id, + COUNT(*)::int AS total_records, + COUNT(*) FILTER (WHERE projection_class = 'recent_summary')::int AS recent_summary_count, + COUNT(*) FILTER (WHERE projection_class = 'durable_memory_candidate')::int AS durable_candidate_count, + MAX(updated_at) AS updated_at + FROM shared_context_projections + WHERE user_id = $1 + AND scope = 'personal' + ${projectId ? 'AND project_id = $2' : ''} + ${projectionClass ? `AND projection_class = $${projectId ? 3 : 2}` : ''} + GROUP BY project_id + ORDER BY MAX(updated_at) DESC + LIMIT 200`, + [userId, ...(projectId ? [projectId] : []), ...(projectionClass ? [projectionClass] : [])], + ); return c.json({ stats: buildMemoryStatsView(stats, stats?.total_records ?? 0), records: mapMemoryRecordRows(rows.filter((row) => !isMemoryNoiseSummary(row.summary))), + projects: mapMemoryProjectRows(projectRows), }); } @@ -331,19 +470,294 @@ sharedContextRoutes.get('/personal-memory', async (c) => { return c.json(buildSharedMemoryResponse(rows, query, limit)); }); -function matchesPathPattern(pattern: string, filePath: string): boolean { - const normalizedPattern = pattern.replace(/\\/g, '/'); - const normalizedPath = filePath.replace(/\\/g, '/'); - if (normalizedPattern.endsWith('/**')) { - return normalizedPath.startsWith(normalizedPattern.slice(0, -3)); +sharedContextRoutes.post('/memory/search', async (c) => { + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + if (!isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.quickSearch, featureFlags)) { + return c.json(sameShapeSearchEnvelope()); + } + const userId = c.get('userId' as never) as string; + const body = await readJsonBody<{ + query?: string; + scope?: SearchRequestScope; + projectId?: string; + limit?: number; + }>(c); + const query = body?.query?.trim() ?? ''; + const requestedScope = isSearchRequestScope(body?.scope) ? body.scope : 'all_authorized'; + const limit = Math.max(1, Math.min(50, typeof body?.limit === 'number' ? body.limit : 20)); + const userPrivateSyncEnabled = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.userPrivateSync, featureFlags); + const scopes = expandSearchRequestScope(requestedScope, { includeOwnerPrivate: userPrivateSyncEnabled }); + if (scopes.length === 0) return c.json(sameShapeSearchEnvelope()); + + type SearchProjectionRow = { + id: string; + scope: Exclude; + project_id: string; + projection_class: ProjectionClass; + summary: string; + updated_at: number; + hit_count: number | null; + cite_count: number | null; + origin: MemoryOrigin | null; + }; + type OwnerPrivateRow = { + id: string; + kind: string; + origin: MemoryOrigin | null; + text: string; + updated_at: number; + }; + + const includeUserPrivate = userPrivateSyncEnabled && scopes.includes('user_private'); + const sharedScopes = scopes.filter((scope) => scope !== 'user_private' && isSharedProjectionScope(scope)); + const results: Array<{ + id: string; + scope: MemoryScope; + class: string; + preview: string; + origin?: MemoryOrigin; + projectId?: string; + updatedAt: number; + score: number; + }> = []; + + if (includeUserPrivate) { + const ownerRows = await c.env.DB.query( + `SELECT id, kind, origin, text, updated_at + FROM owner_private_memories + WHERE owner_user_id = $1 + ${query ? 'AND text ILIKE $2' : ''} + ORDER BY updated_at DESC + LIMIT $${query ? 3 : 2}`, + [userId, ...(query ? [`%${query}%`] : []), limit], + ); + for (const row of ownerRows) { + results.push({ + id: row.id, + scope: 'user_private', + class: row.kind, + preview: row.text.slice(0, 240), + origin: isMemoryOrigin(row.origin) ? row.origin : undefined, + updatedAt: row.updated_at, + score: row.updated_at, + }); + } } - if (normalizedPattern.includes('*')) { - const escaped = normalizedPattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*'); - return new RegExp(`^${escaped}$`).test(normalizedPath); + + if (sharedScopes.length > 0) { + const citeCountEnabled = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.citeCount, featureFlags); + const rows = await c.env.DB.query( + `SELECT p.id, p.scope, p.project_id, p.projection_class, p.summary, p.updated_at, p.origin, + p.hit_count, COALESCE(cc.cite_count, 0) AS cite_count + FROM shared_context_projections p + LEFT JOIN shared_context_projection_cite_counts cc ON cc.projection_id = p.id + WHERE COALESCE(p.status, 'active') = 'active' + AND ($1::text IS NULL OR p.project_id = $1) + AND ($2::text = '' OR p.summary ILIKE $3) + AND ( + (p.scope = 'personal' AND p.user_id = $4 AND p.scope = ANY($5::text[])) + OR ( + p.scope <> 'personal' + AND p.scope = ANY($5::text[]) + AND EXISTS ( + SELECT 1 FROM team_members tm + WHERE tm.team_id = p.enterprise_id AND tm.user_id = $4 + ) + ) + ) + ORDER BY (p.updated_at + CASE WHEN $7::boolean THEN LEAST(COALESCE(cc.cite_count, 0), 100) ELSE 0 END) DESC + LIMIT $6`, + [body?.projectId?.trim() || null, query, `%${query}%`, userId, sharedScopes, limit, citeCountEnabled], + ); + for (const row of rows.filter((entry) => !isMemoryNoiseSummary(entry.summary))) { + results.push({ + id: row.id, + scope: row.scope, + class: row.projection_class, + preview: row.summary.slice(0, 240), + origin: isMemoryOrigin(row.origin) ? row.origin : undefined, + projectId: row.project_id, + updatedAt: row.updated_at, + score: row.updated_at + (citeCountEnabled ? Math.min(row.cite_count ?? 0, 100) : 0), + }); + } + } + + results.sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return b.updatedAt - a.updatedAt; + }); + + return c.json({ + results: results.slice(0, limit).map((result) => ({ + id: result.id, + scope: result.scope, + class: result.class, + preview: result.preview, + origin: result.origin, + projectId: result.projectId, + updatedAt: result.updatedAt, + })), + nextCursor: null, + }); +}); + +type CitationProjectionRow = { + id: string; + scope: SharedProjectionScope; + enterprise_id: string | null; + user_id: string | null; + project_id: string; + summary: string; + content_json: string | Record | null; + content_hash: string | null; +}; + +async function getAuthorizedCitationProjection( + c: SharedContextRouteContext, + projectionId: string, + userId: string, +): Promise { + const row = await c.env.DB.queryOne( + `SELECT id, scope, enterprise_id, user_id, project_id, summary, content_json, content_hash + FROM shared_context_projections + WHERE id = $1 AND COALESCE(status, 'active') = 'active'`, + [projectionId], + ); + if (!row) return null; + if (row.scope === 'personal') return row.user_id === userId ? row : null; + if (!isSharedProjectionScope(row.scope)) return null; + if (!row.enterprise_id) return null; + const member = await c.env.DB.queryOne<{ role: EnterpriseRole }>( + 'SELECT role FROM team_members WHERE team_id = $1 AND user_id = $2', + [row.enterprise_id, userId], + ); + return member ? row : null; +} + +function parseProjectionContent(contentJson: CitationProjectionRow['content_json']): Record { + if (typeof contentJson !== 'string') return contentJson ?? {}; + try { + const parsed = JSON.parse(contentJson) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as Record : {}; + } catch { + return {}; } - return normalizedPattern === normalizedPath; } +async function getOrRepairProjectionContentHash( + c: SharedContextRouteContext, + projection: CitationProjectionRow, +): Promise { + const persisted = projection.content_hash?.trim(); + if (persisted) return persisted; + const computed = computeProjectionContentHash({ + summary: projection.summary, + content: parseProjectionContent(projection.content_json), + }); + await c.env.DB.execute( + `UPDATE shared_context_projections + SET content_hash = $1 + WHERE id = $2 AND (content_hash IS NULL OR content_hash = '')`, + [computed, projection.id], + ).catch(() => { /* best-effort repair; caller still uses the computed hash */ }); + return computed; +} + +sharedContextRoutes.post('/memory/citations', async (c) => { + const userId = c.get('userId' as never) as string; + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + if (!isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.citation, featureFlags)) return sameShapeNotFound(c); + const body = await readJsonBody<{ projectionId?: string; citingMessageId?: string }>(c); + const projectionId = body?.projectionId?.trim(); + const citingMessageId = body?.citingMessageId?.trim(); + if (!projectionId || !citingMessageId) return c.json({ error: 'invalid_body' }, 400); + + const projection = await getAuthorizedCitationProjection(c, projectionId, userId); + if (!projection) return sameShapeNotFound(c); + const contentHash = await getOrRepairProjectionContentHash(c, projection); + const scopeNamespace = `${projection.scope}:${projection.enterprise_id ?? projection.user_id ?? ''}:${projection.project_id}`; + const idempotencyKey = deriveCitationIdempotencyKey({ scopeNamespace, projectionId, citingMessageId }); + const citationId = randomHex(16); + const now = Date.now(); + const insert = await c.env.DB.execute( + `INSERT INTO shared_context_citations ( + id, projection_id, user_id, citing_message_id, idempotency_key, projection_content_hash, created_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (idempotency_key) DO NOTHING`, + [citationId, projectionId, userId, citingMessageId, idempotencyKey, contentHash, now], + ); + const inserted = insert.changes > 0; + const existingCitation = inserted + ? null + : await c.env.DB.queryOne<{ + id: string; + projection_id: string; + projection_content_hash: string; + created_at: number; + }>( + 'SELECT id, projection_id, projection_content_hash, created_at FROM shared_context_citations WHERE idempotency_key = $1 AND user_id = $2', + [idempotencyKey, userId], + ); + if (!inserted && !existingCitation) return sameShapeNotFound(c); + const countAllowed = inserted && isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.citeCount, featureFlags) + ? consumeCitationCountRateLimit({ env: c.env, userId, projectionId, now }).allowed + : false; + if (countAllowed) { + await c.env.DB.execute( + `INSERT INTO shared_context_projection_cite_counts (projection_id, cite_count, updated_at) + VALUES ($1, 1, $2) + ON CONFLICT (projection_id) DO UPDATE SET + cite_count = shared_context_projection_cite_counts.cite_count + 1, + updated_at = excluded.updated_at`, + [projectionId, now], + ); + } + const drift = existingCitation ? existingCitation.projection_content_hash !== contentHash : false; + const driftVisible = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.citeDriftBadge, featureFlags); + + return c.json({ + ok: true, + citation: { + id: existingCitation?.id ?? citationId, + projectionId, + createdAt: existingCitation?.created_at ?? now, + drift: driftVisible ? drift : false, + }, + deduped: !inserted, + }, inserted ? 201 : 200); +}); + +sharedContextRoutes.get('/memory/citations/:citationId', async (c) => { + const userId = c.get('userId' as never) as string; + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + if (!isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.citation, featureFlags)) return sameShapeNotFound(c); + const citationId = c.req.param('citationId'); + const row = await c.env.DB.queryOne<{ + id: string; + projection_id: string; + projection_content_hash: string; + created_at: number; + }>( + 'SELECT id, projection_id, projection_content_hash, created_at FROM shared_context_citations WHERE id = $1 AND user_id = $2', + [citationId, userId], + ); + if (!row) return sameShapeNotFound(c); + const projection = await getAuthorizedCitationProjection(c, row.projection_id, userId); + if (!projection) return sameShapeNotFound(c); + const currentHash = await getOrRepairProjectionContentHash(c, projection); + const driftVisible = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.citeDriftBadge, featureFlags); + return c.json({ + ok: true, + citation: { + id: row.id, + projectionId: row.projection_id, + createdAt: row.created_at, + drift: driftVisible ? currentHash !== row.projection_content_hash : false, + }, + }); +}); + function matchesRuntimeAuthoredContextRow( row: RuntimeAuthoredContextRow, filter: RuntimeAuthoredContextFilter, @@ -366,7 +780,7 @@ function matchesRuntimeAuthoredContextRow( } if (row.applicability_path_pattern) { if (!filter.filePath) return false; - if (!matchesPathPattern(row.applicability_path_pattern, filter.filePath)) return false; + if (!matchesAuthoredContextPathPattern(row.applicability_path_pattern, filter.filePath)) return false; } return true; } @@ -398,7 +812,7 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/projects', async (c) => { workspace_id: string | null; canonical_repo_id: string; display_name: string | null; - scope: 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: AuthoredContextScope; status: EnrollmentVisibilityState; }>( 'SELECT id, workspace_id, canonical_repo_id, display_name, scope, status FROM shared_project_enrollments WHERE enterprise_id = $1 ORDER BY id ASC', @@ -449,14 +863,14 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/documents', async (c) => { const enterpriseId = c.req.param('enterpriseId'); const auth = await requireEnterpriseRole(c, enterpriseId, 'member'); if (auth instanceof Response) return auth; - const docs = await c.env.DB.query<{ id: string; kind: DocumentKind; title: string }>( - 'SELECT id, kind, title FROM shared_context_documents WHERE enterprise_id = $1 ORDER BY title ASC', + const docs = await c.env.DB.query<{ id: string; kind: DocumentKind; title: string; created_by: string }>( + 'SELECT id, kind, title, created_by FROM shared_context_documents WHERE enterprise_id = $1 ORDER BY title ASC', [enterpriseId], ); const result = []; for (const doc of docs) { - const versions = await c.env.DB.query<{ id: string; version_number: number; status: string }>( - 'SELECT id, version_number, status FROM shared_context_document_versions WHERE document_id = $1 ORDER BY version_number DESC', + const versions = await c.env.DB.query<{ id: string; version_number: number; status: string; created_by: string }>( + 'SELECT id, version_number, status, created_by FROM shared_context_document_versions WHERE document_id = $1 ORDER BY version_number DESC', [doc.id], ); result.push({ @@ -464,10 +878,12 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/documents', async (c) => { enterpriseId, kind: doc.kind, title: doc.title, + createdByUserId: doc.created_by, versions: versions.map((version) => ({ id: version.id, versionNumber: version.version_number, status: version.status, + createdByUserId: version.created_by, })), }); } @@ -489,16 +905,30 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/document-bindings', async (c applicability_language: string | null; applicability_path_pattern: string | null; status: string; + created_by: string; }>( - 'SELECT id, workspace_id, enrollment_id, document_id, version_id, binding_mode, applicability_repo_id, applicability_language, applicability_path_pattern, status FROM shared_context_document_bindings WHERE enterprise_id = $1 ORDER BY id ASC', + 'SELECT id, workspace_id, enrollment_id, document_id, version_id, binding_mode, applicability_repo_id, applicability_language, applicability_path_pattern, status, created_by FROM shared_context_document_bindings WHERE enterprise_id = $1 ORDER BY id ASC', [enterpriseId], ); + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + const orgAuthoredEnabled = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.orgSharedAuthoredStandards, featureFlags); + const visibleRows = rows.filter((row) => { + const scope = authoredContextScopeForBinding({ + workspaceId: row.workspace_id, + enrollmentId: row.enrollment_id, + }); + return scope !== 'org_shared' || orgAuthoredEnabled; + }); return c.json({ enterpriseId, - bindings: rows.map((row) => ({ + bindings: visibleRows.map((row) => ({ id: row.id, workspaceId: row.workspace_id, enrollmentId: row.enrollment_id, + scope: authoredContextScopeForBinding({ + workspaceId: row.workspace_id, + enrollmentId: row.enrollment_id, + }), documentId: row.document_id, versionId: row.version_id, mode: row.binding_mode, @@ -506,14 +936,16 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/document-bindings', async (c applicabilityLanguage: row.applicability_language, applicabilityPathPattern: row.applicability_path_pattern, status: row.status, + createdByUserId: row.created_by, })), }); }); sharedContextRoutes.get('/enterprises/:enterpriseId/runtime-authored-context', async (c) => { const enterpriseId = c.req.param('enterpriseId'); - const auth = await requireEnterpriseRole(c, enterpriseId, 'member'); - if (auth instanceof Response) return auth; + const userId = c.get('userId' as never) as string; + const role = await getEnterpriseRole(c.env.DB, enterpriseId, userId); + if (!role) return sameShapeNotFound(c); const canonicalRepoId = c.req.query('canonicalRepoId')?.trim() ?? null; const workspaceId = c.req.query('workspaceId')?.trim() ?? null; const enrollmentId = c.req.query('enrollmentId')?.trim() ?? null; @@ -529,7 +961,7 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/runtime-authored-context', a b.applicability_language, b.applicability_path_pattern, v.id AS version_id, - v.content + v.content_md AS content FROM shared_context_document_bindings b JOIN shared_context_document_versions v ON v.id = b.version_id WHERE b.enterprise_id = $1 AND b.status = 'active' AND v.status = 'active' @@ -537,28 +969,49 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/runtime-authored-context', a [enterpriseId], ); - const bindings = rows.filter((row) => matchesRuntimeAuthoredContextRow(row, { - canonicalRepoId, - workspaceId, - enrollmentId, - language, - filePath, - })); - - return c.json({ - enterpriseId, - bindings: bindings.map((row) => ({ + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + const orgAuthoredEnabled = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.orgSharedAuthoredStandards, featureFlags); + const bindings = rows + .filter((row) => row.enrollment_id || row.workspace_id || orgAuthoredEnabled) + .filter((row) => matchesRuntimeAuthoredContextRow(row, { + canonicalRepoId, + workspaceId, + enrollmentId, + language, + filePath, + })) + .map((row) => ({ bindingId: row.binding_id, documentVersionId: row.version_id, mode: row.binding_mode, - scope: row.enrollment_id ? 'project_shared' : (row.workspace_id ? 'workspace_shared' : 'org_shared'), + scope: authoredContextScopeForBinding({ + workspaceId: row.workspace_id, + enrollmentId: row.enrollment_id, + }), repository: row.applicability_repo_id ?? undefined, language: row.applicability_language ?? undefined, pathPattern: row.applicability_path_pattern ?? undefined, content: row.content, active: true, superseded: false, - })), + })) + .sort(compareRuntimeAuthoredContextBindings); + const budgetBytesRaw = c.req.query('budgetBytes')?.trim(); + const budgetBytes = budgetBytesRaw ? Number(budgetBytesRaw) : undefined; + const budgeted = applyRuntimeAuthoredContextBudget(bindings, budgetBytes); + if (!budgeted.ok) { + return c.json({ + error: budgeted.error, + enterpriseId, + bindings: budgeted.bindings, + diagnostics: budgeted.diagnostics, + }, 409); + } + + return c.json({ + enterpriseId, + bindings: budgeted.bindings, + ...(budgeted.diagnostics.length > 0 ? { diagnostics: budgeted.diagnostics } : {}), }); }); @@ -598,7 +1051,7 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/diagnostics', async (c) => { b.applicability_language, b.applicability_path_pattern, v.id AS version_id, - v.content + v.content_md AS content FROM shared_context_document_bindings b JOIN shared_context_document_versions v ON v.id = b.version_id WHERE b.enterprise_id = $1 AND b.status = 'active' AND v.status = 'active' @@ -611,13 +1064,17 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/diagnostics', async (c) => { Date.now(), getRemoteProcessedFreshMs(), ); - const matchingBindings = bindings.filter((row) => matchesRuntimeAuthoredContextRow(row, { - canonicalRepoId, - workspaceId, - enrollmentId, - language, - filePath, - })); + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + const orgAuthoredEnabled = isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.orgSharedAuthoredStandards, featureFlags); + const matchingBindings = bindings + .filter((row) => row.enrollment_id || row.workspace_id || orgAuthoredEnabled) + .filter((row) => matchesRuntimeAuthoredContextRow(row, { + canonicalRepoId, + workspaceId, + enrollmentId, + language, + filePath, + })); return c.json({ enterpriseId, canonicalRepoId, @@ -690,12 +1147,12 @@ sharedContextRoutes.post('/enterprises/:enterpriseId/projects/enroll', async (c) canonicalRepoId?: string; displayName?: string; workspaceId?: string | null; - scope?: 'project_shared' | 'workspace_shared' | 'org_shared'; + scope?: AuthoredContextScope; }>(c); const canonicalRepoId = body?.canonicalRepoId?.trim(); if (!canonicalRepoId) return c.json({ error: 'canonical_repo_id_required' }, 400); const scope = body?.scope ?? 'project_shared'; - if (!['project_shared', 'workspace_shared', 'org_shared'].includes(scope)) return c.json({ error: 'invalid_scope' }, 400); + if (!isAuthoredContextScope(scope)) return c.json({ error: 'invalid_scope' }, 400); const enrollmentId = randomHex(16); const now = Date.now(); @@ -837,15 +1294,31 @@ sharedContextRoutes.get('/enterprises/:enterpriseId/memory', async (c) => { LIMIT $${canonicalRepoId ? (projectionClass ? 4 : 3) : (projectionClass ? 3 : 2)}`, [enterpriseId, ...(canonicalRepoId ? [canonicalRepoId] : []), ...(projectionClass ? [projectionClass] : []), limit], ); + const projectRows = await c.env.DB.query( + `SELECT project_id, + COUNT(*)::int AS total_records, + COUNT(*) FILTER (WHERE projection_class = 'recent_summary')::int AS recent_summary_count, + COUNT(*) FILTER (WHERE projection_class = 'durable_memory_candidate')::int AS durable_candidate_count, + MAX(updated_at) AS updated_at + FROM shared_context_projections + WHERE enterprise_id = $1 + ${canonicalRepoId ? 'AND project_id = $2' : ''} + ${projectionClass ? `AND projection_class = $${canonicalRepoId ? 3 : 2}` : ''} + GROUP BY project_id + ORDER BY MAX(updated_at) DESC + LIMIT 200`, + [enterpriseId, ...(canonicalRepoId ? [canonicalRepoId] : []), ...(projectionClass ? [projectionClass] : [])], + ); return c.json({ stats: buildMemoryStatsView(stats, stats?.total_records ?? 0), records: mapMemoryRecordRows(rows.filter((row) => !isMemoryNoiseSummary(row.summary))), + projects: mapMemoryProjectRows(projectRows), }); } const rows = await c.env.DB.query<{ id: string; - scope: 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: AuthoredContextScope; project_id: string; projection_class: 'recent_summary' | 'durable_memory_candidate'; source_event_ids_json: string | string[]; @@ -976,6 +1449,14 @@ sharedContextRoutes.post('/enterprises/:enterpriseId/document-bindings', async ( applicabilityPathPattern?: string | null; }>(c); if (!body?.documentId || !body?.versionId || !isBindingMode(body?.mode)) return c.json({ error: 'invalid_binding' }, 400); + const bindingScope = authoredContextScopeForBinding({ + workspaceId: body.workspaceId, + enrollmentId: body.enrollmentId, + }); + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + if (bindingScope === 'org_shared' && !isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.orgSharedAuthoredStandards, featureFlags)) { + return sameShapeNotFound(c); + } const bindingId = randomHex(16); const now = Date.now(); await c.env.DB.execute( @@ -988,6 +1469,7 @@ sharedContextRoutes.post('/enterprises/:enterpriseId/document-bindings', async ( enterpriseId, workspaceId: body.workspaceId ?? null, enrollmentId: body.enrollmentId ?? null, + scope: bindingScope, documentId: body.documentId, versionId: body.versionId, mode: body.mode, @@ -997,11 +1479,23 @@ sharedContextRoutes.post('/enterprises/:enterpriseId/document-bindings', async ( sharedContextRoutes.post('/document-bindings/:bindingId/deactivate', async (c) => { const bindingId = c.req.param('bindingId'); - const binding = await c.env.DB.queryOne<{ enterprise_id: string }>( - 'SELECT enterprise_id FROM shared_context_document_bindings WHERE id = $1', + const binding = await c.env.DB.queryOne<{ + enterprise_id: string; + workspace_id: string | null; + enrollment_id: string | null; + }>( + 'SELECT enterprise_id, workspace_id, enrollment_id FROM shared_context_document_bindings WHERE id = $1', [bindingId], ); - if (!binding) return c.json({ error: 'not_found' }, 404); + if (!binding) return sameShapeNotFound(c); + const bindingScope = authoredContextScopeForBinding({ + workspaceId: binding.workspace_id, + enrollmentId: binding.enrollment_id, + }); + const featureFlags = await getCurrentUserMemoryFeatureFlags(c); + if (bindingScope === 'org_shared' && !isUserMemoryFeatureEnabled(c, MEMORY_FEATURES.orgSharedAuthoredStandards, featureFlags)) { + return sameShapeNotFound(c); + } const auth = await requireEnterpriseRole(c, binding.enterprise_id, 'admin'); if (auth instanceof Response) return auth; const now = Date.now(); @@ -1119,12 +1613,12 @@ sharedContextRoutes.post('/:id/shared-context/memory/recall', async (c) => { FROM shared_context_projections p JOIN shared_context_embeddings e ON e.source_id = p.id AND e.source_kind = 'projection' JOIN team_members tm ON tm.team_id = p.enterprise_id AND tm.user_id = $2 - WHERE p.scope IN ('project_shared', 'workspace_shared', 'org_shared') - AND COALESCE(p.status, 'active') = 'active' - ${projectId ? 'AND p.project_id = $3' : ''} + JOIN unnest($3::text[]) AS allowed_scope(scope) ON allowed_scope.scope = p.scope + WHERE COALESCE(p.status, 'active') = 'active' + ${projectId ? 'AND p.project_id = $4' : ''} ORDER BY e.embedding <=> $1::vector - LIMIT $${projectId ? 4 : 3}`, - [vecSql, userId, ...(projectId ? [projectId] : []), candidateLimit], + LIMIT $${projectId ? 5 : 4}`, + [vecSql, userId, [...REPLICABLE_SHARED_PROJECTION_SCOPES], ...(projectId ? [projectId] : []), candidateLimit], ); } else { // Fallback: pg_trgm text similarity (for when embedding model is unavailable) @@ -1148,13 +1642,13 @@ sharedContextRoutes.post('/:id/shared-context/memory/recall', async (c) => { similarity(p.summary, $1) AS score, p.enterprise_id FROM shared_context_projections p JOIN team_members tm ON tm.team_id = p.enterprise_id AND tm.user_id = $2 - WHERE p.scope IN ('project_shared', 'workspace_shared', 'org_shared') - AND COALESCE(p.status, 'active') = 'active' - ${projectId ? 'AND p.project_id = $3' : ''} + JOIN unnest($3::text[]) AS allowed_scope(scope) ON allowed_scope.scope = p.scope + WHERE COALESCE(p.status, 'active') = 'active' + ${projectId ? 'AND p.project_id = $4' : ''} AND p.summary % $1 ORDER BY score DESC - LIMIT $${projectId ? 4 : 3}`, - [query, userId, ...(projectId ? [projectId] : []), candidateLimit], + LIMIT $${projectId ? 5 : 4}`, + [query, userId, [...REPLICABLE_SHARED_PROJECTION_SCOPES], ...(projectId ? [projectId] : []), candidateLimit], ); } diff --git a/server/src/util/metrics.ts b/server/src/util/metrics.ts new file mode 100644 index 000000000..548564706 --- /dev/null +++ b/server/src/util/metrics.ts @@ -0,0 +1,36 @@ +export type MetricLabels = Record; + +const counters = new Map(); +const MAX_COUNTERS = 1000; + +function labelsKey(labels?: MetricLabels): string { + if (!labels) return ''; + const entries = Object.entries(labels) + .filter(([, value]) => typeof value === 'string') + .sort(([a], [b]) => a.localeCompare(b)); + return entries.map(([key, value]) => `${key}=${value}`).join(','); +} + +function counterKey(name: string, labels?: MetricLabels): string { + const suffix = labelsKey(labels); + return suffix ? `${name}{${suffix}}` : name; +} + +export function incrementCounter(name: string, labels?: MetricLabels): void { + if (!name) return; + const key = counterKey(name, labels); + if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; + counters.set(key, (counters.get(key) ?? 0) + 1); +} + +export function getCounter(name: string, labels?: MetricLabels): number { + return counters.get(counterKey(name, labels)) ?? 0; +} + +export function snapshotCounters(): Record { + return Object.fromEntries(counters.entries()); +} + +export function resetMetricsForTests(): void { + counters.clear(); +} diff --git a/server/src/util/semantic-memory-view.ts b/server/src/util/semantic-memory-view.ts index c3ff4e3a8..5459eade1 100644 --- a/server/src/util/semantic-memory-view.ts +++ b/server/src/util/semantic-memory-view.ts @@ -1,12 +1,16 @@ -import type { ContextMemoryView } from '../../../shared/context-types.js'; +import type { ContextMemoryProjectView, ContextMemoryView } from '../../../shared/context-types.js'; import { computeRelevanceScore, type MemoryScoringWeights, type ProjectionClass } from '../../../shared/memory-scoring.js'; +import { + REPLICABLE_SHARED_PROJECTION_SCOPES, + type SharedContextProjectionScope, +} from '../../../shared/memory-scope.js'; import type { Database } from '../db/client.js'; import { embeddingToSql, generateEmbedding } from './embedding.js'; import { isMemoryNoiseSummary } from '../../../shared/memory-noise-patterns.js'; type MemoryScope = 'personal' | 'enterprise'; type ProjectionClassFilter = 'recent_summary' | 'durable_memory_candidate' | 'master_summary'; -type ProjectionScope = 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; +type ProjectionScope = SharedContextProjectionScope; type ProjectionStatus = 'active' | 'archived' | 'archived_dedup'; export interface SemanticMemoryViewInput { @@ -43,6 +47,14 @@ interface ScopedStatsRow { project_count: number; } +interface ScopedProjectStatsRow { + project_id: string; + total_records: number; + recent_summary_count: number; + durable_candidate_count: number; + updated_at: number; +} + function parseSourceEventCount(sourceEventIds: string | string[]): number { if (Array.isArray(sourceEventIds)) return sourceEventIds.length; try { @@ -70,7 +82,8 @@ export function buildScopedWhereClause(input: SemanticMemoryViewInput, includeAl conditions.push(`${alias}user_id = ${p(input.userId)}`); } else { if (!input.enterpriseId) throw new Error('enterpriseId is required for enterprise semantic memory search'); - conditions.push(`${alias}scope IN ('project_shared', 'workspace_shared', 'org_shared')`); + const sharedScopePlaceholders = REPLICABLE_SHARED_PROJECTION_SCOPES.map((scope) => p(scope)).join(', '); + conditions.push(`${alias}scope IN (${sharedScopePlaceholders})`); conditions.push(`${alias}enterprise_id = ${p(input.enterpriseId)}`); } @@ -107,6 +120,33 @@ async function loadScopedStats(db: Database, input: SemanticMemoryViewInput): Pr }; } +async function loadScopedProjectRows(db: Database, input: SemanticMemoryViewInput): Promise { + const { clause, params } = buildScopedWhereClause(input, false); + const rows = await db.query( + `SELECT project_id, + COUNT(*)::int AS total_records, + COUNT(*) FILTER (WHERE projection_class = 'recent_summary')::int AS recent_summary_count, + COUNT(*) FILTER (WHERE projection_class = 'durable_memory_candidate')::int AS durable_candidate_count, + MAX(updated_at) AS updated_at + FROM shared_context_projections + WHERE ${clause} + GROUP BY project_id + ORDER BY MAX(updated_at) DESC + LIMIT 200`, + params, + ); + return rows + .filter((row) => row.project_id) + .map((row) => ({ + projectId: row.project_id, + displayName: row.project_id, + totalRecords: row.total_records, + recentSummaryCount: row.recent_summary_count, + durableCandidateCount: row.durable_candidate_count, + updatedAt: row.updated_at, + })); +} + async function loadScopedVectorRows(db: Database, input: SemanticMemoryViewInput, queryEmbeddingSql: string, candidateLimit: number): Promise { const { clause, params } = buildScopedWhereClause(input, true); const vectorParam = `$${params.length + 1}`; @@ -136,7 +176,10 @@ export async function searchSemanticMemoryView(input: SemanticMemoryViewInput): const rows = await loadScopedVectorRows(input.db, input, embeddingToSql(embedding), candidateLimit); if (rows.length === 0) return null; - const stats = await loadScopedStats(input.db, input); + const [stats, projects] = await Promise.all([ + loadScopedStats(input.db, input), + loadScopedProjectRows(input.db, input), + ]); const currentProjectId = input.projectId ?? '__unknown_current_project__'; const ranked = rows .filter((row) => !isMemoryNoiseSummary(row.summary)) @@ -180,5 +223,6 @@ export async function searchSemanticMemoryView(input: SemanticMemoryViewInput): lastUsedAt: row.last_used_at ?? undefined, status: row.status ?? 'active', })), + projects, }; } diff --git a/server/src/ws/bridge.ts b/server/src/ws/bridge.ts index 736b91468..fae57e6b1 100644 --- a/server/src/ws/bridge.ts +++ b/server/src/ws/bridge.ts @@ -18,8 +18,40 @@ import type { Env } from '../env.js'; import { MemoryRateLimiter } from './rate-limiter.js'; import { sha256Hex } from '../security/crypto.js'; import { DAEMON_MSG } from '../../../shared/daemon-events.js'; +import { DAEMON_COMMAND_TYPES } from '../../../shared/daemon-command-types.js'; import { REPO_RELAY_TYPES } from '../../../shared/repo-types.js'; import { TRANSPORT_RELAY_TYPES, TRANSPORT_MSG } from '../../../shared/transport-events.js'; +import { + MEMORY_WS, + isMemoryManagementRequestType, + isMemoryManagementResponseType, +} from '../../../shared/memory-ws.js'; +import { + MEMORY_MANAGEMENT_CONTEXT_FIELD, + type AuthenticatedMemoryManagementContext, + type MemoryManagementBoundProject, + type MemoryManagementRole, +} from '../../../shared/memory-management-context.js'; +import { + MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES, + MEMORY_MANAGEMENT_ERROR_CODES, +} from '../../../shared/memory-management.js'; +import { + MEMORY_FEATURE_CONFIG_MSG, + MEMORY_FEATURE_CONFIG_PREF_KEY, + MEMORY_FEATURE_FLAGS, + encodeMemoryFeatureFlagValuesJson, + getMemoryFeatureFlagDefinition, + isMemoryFeatureFlag, + memoryFeatureFlagEnvKey, + parseMemoryFeatureFlagValuesJson, + computeEffectiveMemoryFeatureFlags, + resolveMemoryFeatureFlagValue, + type FeatureFlagValueSource, + type MemoryFeatureFlag, + type MemoryFeatureFlagResolutionLayers, + type MemoryFeatureFlagValues, +} from '../../../shared/feature-flags.js'; import { MSG_COMMAND_ACK, MSG_COMMAND_FAILED, @@ -52,15 +84,21 @@ import { type PreviewWsOpenedMessage, } from '../../../shared/preview-types.js'; import { LocalWebPreviewRegistry } from '../preview/registry.js'; -import { updateServerHeartbeat, updateServerStatus, upsertDiscussion, insertDiscussionRound, createSubSession, getSubSessionById, updateSubSession, upsertOrchestrationRun, updateProviderStatus, clearProviderStatus, updateProviderRemoteSessions, upsertSessionTextTailCacheEvent } from '../db/queries.js'; +import { updateServerHeartbeat, updateServerStatus, upsertDiscussion, insertDiscussionRound, createSubSession, getSubSessionById, updateSubSession, upsertOrchestrationRun, updateProviderStatus, clearProviderStatus, updateProviderRemoteSessions, upsertSessionTextTailCacheEvent, getUserPref, setUserPref } from '../db/queries.js'; import logger from '../util/logger.js'; +import { incrementCounter } from '../util/metrics.js'; import { pickReadableSessionDisplay } from '../../../shared/session-display.js'; import { isKnownTestSessionLike } from '../../../shared/test-session-guard.js'; import { PUSH_TIMELINE_EVENT_MAX_AGE_MS, TIMELINE_SUPPRESS_PUSH_FIELD } from '../../../shared/push-notifications.js'; +import { + DAEMON_UPGRADE_DELIVERY_STATUS, +} from '../../../shared/daemon-upgrade.js'; +import { DaemonUpgradeCoordinator, type DaemonUpgradeSource, type RequestDaemonUpgradeResult } from './daemon-upgrade-coordinator.js'; const AUTH_TIMEOUT_MS = 5000; const MAX_QUEUE_SIZE = 100; const MAX_BROWSER_PAYLOAD = 65536; // 64KB (subsession.rebuild_all can include many sessions) +const MAX_PENDING_MEMORY_MANAGEMENT_REQUESTS_PER_SOCKET = 32; // Desktop with pinned panels + many sessions can fire 60+ subscribe/repo/repo // detect / fs.git_status / chat.subscribe / ping messages on initial connect. // A reconnect within 10s doubles that. 120 was right at the cliff edge and @@ -290,7 +328,7 @@ export class WsBridge { private daemonWs: WebSocket | null = null; private authenticated = false; private daemonVersion: string | null = null; - private upgradeAttempts = 0; + private daemonUpgradeCoordinator = new DaemonUpgradeCoordinator(); private browserSockets = new Set(); private mobileSockets = new Set(); private queue: string[] = []; @@ -339,6 +377,9 @@ export class WsBridge { /** Per-request timeline.history / timeline.replay pending map — routes responses via requestId unicast. */ private pendingTimelineRequests = new Map }>(); + /** Per-request memory management pending map — routes sensitive admin responses via requestId unicast. */ + private pendingMemoryManagementRequests = new Map }>(); + /** Per-request HTTP timeline/history relay pending map. */ private pendingHttpTimelineRequests = new Map(); private pendingRecentTextBackfills = new Map>(); @@ -424,6 +465,360 @@ export class WsBridge { return WsBridge.instances; } + private registerMemoryManagementRequest(ws: WebSocket, msg: Record): string | null { + if (!isMemoryManagementRequestType(msg.type)) return null; + const userId = this.browserUserIds.get(ws)?.trim(); + if (!userId) { + safeSend(ws, JSON.stringify({ + type: 'error', + code: MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES.UNAUTHENTICATED, + message: 'memory management requests require an authenticated browser session', + originalType: msg.type, + })); + return null; + } + const pendingForSocket = [...this.pendingMemoryManagementRequests.values()].filter((pending) => pending.socket === ws).length; + if (pendingForSocket >= MAX_PENDING_MEMORY_MANAGEMENT_REQUESTS_PER_SOCKET) { + safeSend(ws, JSON.stringify({ + type: 'error', + code: MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES.TOO_MANY_PENDING_REQUESTS, + message: 'too many pending memory management requests', + originalType: msg.type, + })); + return null; + } + const requestId = typeof msg.requestId === 'string' && msg.requestId.trim() + ? msg.requestId.trim() + : null; + if (!requestId) { + safeSend(ws, JSON.stringify({ + type: 'error', + code: MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES.MISSING_REQUEST_ID, + message: 'memory management requests require requestId', + originalType: msg.type, + })); + return null; + } + const existing = this.pendingMemoryManagementRequests.get(requestId); + if (existing) { + safeSend(ws, JSON.stringify({ + type: 'error', + code: MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES.DUPLICATE_REQUEST_ID, + message: 'memory management requestId is already pending', + originalType: msg.type, + requestId, + })); + return null; + } + const timer = setTimeout(() => this.pendingMemoryManagementRequests.delete(requestId), 30_000); + this.pendingMemoryManagementRequests.set(requestId, { socket: ws, timer }); + return requestId; + } + + private clearPendingMemoryManagementRequest(requestId: string): WebSocket | undefined { + const pending = this.pendingMemoryManagementRequests.get(requestId); + if (!pending) return undefined; + clearTimeout(pending.timer); + this.pendingMemoryManagementRequests.delete(requestId); + return pending.socket; + } + + private failMemoryManagementForward(ws: WebSocket, msg: Record, requestId: string, error: unknown): void { + this.clearPendingMemoryManagementRequest(requestId); + logger.warn({ + serverId: this.serverId, + type: msg.type, + requestId, + error: error instanceof Error ? error.message : String(error), + }, 'memory management context injection failed'); + safeSend(ws, JSON.stringify({ + type: 'error', + code: MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES.CONTEXT_INJECTION_FAILED, + message: 'memory management request could not be authorized', + originalType: msg.type, + requestId, + })); + } + + private readMemoryFeatureEnvironmentDefaults(): MemoryFeatureFlagValues { + const environmentStartupDefault: MemoryFeatureFlagValues = {}; + for (const flag of MEMORY_FEATURE_FLAGS) { + const key = memoryFeatureFlagEnvKey(flag); + const raw = process.env[key]; + if (raw != null) environmentStartupDefault[flag] = raw === 'true' || raw === '1'; + } + return environmentStartupDefault; + } + + private memoryFeatureLayers(userConfig: MemoryFeatureFlagValues): MemoryFeatureFlagResolutionLayers { + return { + persistedConfig: userConfig, + environmentStartupDefault: this.readMemoryFeatureEnvironmentDefaults(), + }; + } + + private featureFlagValueSource(flag: MemoryFeatureFlag, layers: MemoryFeatureFlagResolutionLayers): FeatureFlagValueSource { + if (layers.runtimeConfigOverride?.[flag] !== undefined) return 'runtime_config_override'; + if (layers.persistedConfig?.[flag] !== undefined) return 'persisted_config'; + if (layers.environmentStartupDefault?.[flag] !== undefined) return 'environment_startup_default'; + return 'registry_default'; + } + + private requestedMemoryFeatureFlags(layers: MemoryFeatureFlagResolutionLayers): MemoryFeatureFlagValues { + const requested: MemoryFeatureFlagValues = {}; + for (const flag of MEMORY_FEATURE_FLAGS) { + requested[flag] = resolveMemoryFeatureFlagValue(flag, layers); + } + return requested; + } + + private buildMemoryFeatureAdminRecords(userConfig: MemoryFeatureFlagValues) { + const layers = this.memoryFeatureLayers(userConfig); + const requested = this.requestedMemoryFeatureFlags(layers); + const effective = computeEffectiveMemoryFeatureFlags(requested); + return MEMORY_FEATURE_FLAGS.map((flag) => { + const definition = getMemoryFeatureFlagDefinition(flag); + return { + flag, + requested: requested[flag] === true, + enabled: effective[flag], + source: this.featureFlagValueSource(flag, layers), + envKey: memoryFeatureFlagEnvKey(flag), + dependencies: definition.dependencies, + dependencyBlocked: requested[flag] === true && !effective[flag] + ? definition.dependencies.filter((dependency) => !effective[dependency]) + : [], + disabledBehavior: definition.disabledBehavior, + }; + }); + } + + private collectMemoryFeatureWithDependencies(flag: MemoryFeatureFlag, seen = new Set()): Set { + if (seen.has(flag)) return seen; + seen.add(flag); + for (const dependency of getMemoryFeatureFlagDefinition(flag).dependencies) { + this.collectMemoryFeatureWithDependencies(dependency, seen); + } + return seen; + } + + private async readUserMemoryFeatureFlags(userId: string): Promise { + if (!this.db) return {}; + return parseMemoryFeatureFlagValuesJson(await getUserPref(this.db, userId, MEMORY_FEATURE_CONFIG_PREF_KEY)); + } + + private async writeUserMemoryFeatureFlags(userId: string, flags: MemoryFeatureFlagValues): Promise { + if (!this.db) throw new Error('database_unavailable'); + const normalized = parseMemoryFeatureFlagValuesJson(encodeMemoryFeatureFlagValuesJson(flags)); + await setUserPref(this.db, userId, MEMORY_FEATURE_CONFIG_PREF_KEY, encodeMemoryFeatureFlagValuesJson(normalized)); + return normalized; + } + + private sendMemoryFeatureConfigApply(flags: MemoryFeatureFlagValues): void { + this.sendToDaemon(JSON.stringify({ + type: MEMORY_FEATURE_CONFIG_MSG.APPLY, + flags, + })); + } + + private async pushUserMemoryFeatureConfigToOnlineDaemons(userId: string, flags: MemoryFeatureFlagValues): Promise { + const entries = [...WsBridge.instances.values()]; + await Promise.all(entries.map(async (bridge) => { + if (!bridge.authenticated || !bridge.daemonWs || !bridge.db) return; + const row = await bridge.db.queryOne<{ user_id?: string }>( + 'SELECT user_id FROM servers WHERE id = $1', + [bridge.serverId], + ).catch(() => null); + if (row?.user_id !== userId) return; + try { + bridge.sendMemoryFeatureConfigApply(flags); + } catch (error) { + logger.warn({ err: error, serverId: bridge.serverId }, 'failed to apply global memory feature config to daemon'); + } + })); + } + + private async handleMemoryFeaturesQuery(ws: WebSocket, msg: Record): Promise { + if (msg.type !== MEMORY_WS.FEATURES_QUERY) return false; + const requestId = this.registerMemoryManagementRequest(ws, msg); + if (!requestId) return true; + const userId = this.browserUserIds.get(ws)?.trim(); + try { + const flags = userId ? await this.readUserMemoryFeatureFlags(userId) : {}; + this.clearPendingMemoryManagementRequest(requestId); + safeSend(ws, JSON.stringify({ + type: MEMORY_WS.FEATURES_RESPONSE, + requestId, + records: this.buildMemoryFeatureAdminRecords(flags), + })); + } catch (error) { + this.failMemoryManagementForward(ws, msg, requestId, error); + } + return true; + } + + private async handleMemoryFeaturesSet(ws: WebSocket, msg: Record): Promise { + if (msg.type !== MEMORY_WS.FEATURES_SET) return false; + const requestId = this.registerMemoryManagementRequest(ws, msg); + if (!requestId) return true; + const userId = this.browserUserIds.get(ws)?.trim(); + const flag = typeof msg.flag === 'string' ? msg.flag : undefined; + const enabled = msg.enabled; + const sendSetResponse = (payload: Record) => { + this.clearPendingMemoryManagementRequest(requestId); + safeSend(ws, JSON.stringify({ + type: MEMORY_WS.FEATURES_SET_RESPONSE, + requestId, + ...payload, + })); + }; + if (!userId) { + sendSetResponse({ success: false, error: MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES.UNAUTHENTICATED }); + return true; + } + if (!isMemoryFeatureFlag(flag) || typeof enabled !== 'boolean') { + sendSetResponse({ + success: false, + error: MEMORY_MANAGEMENT_ERROR_CODES.INVALID_FEATURE_FLAG, + errorCode: MEMORY_MANAGEMENT_ERROR_CODES.INVALID_FEATURE_FLAG, + }); + return true; + } + try { + const current = await this.readUserMemoryFeatureFlags(userId); + const updates: MemoryFeatureFlagValues = enabled + ? Object.fromEntries([...this.collectMemoryFeatureWithDependencies(flag)].map((dependency) => [dependency, true])) as MemoryFeatureFlagValues + : { [flag]: false }; + const next = await this.writeUserMemoryFeatureFlags(userId, { ...current, ...updates }); + await this.pushUserMemoryFeatureConfigToOnlineDaemons(userId, next); + const records = this.buildMemoryFeatureAdminRecords(next); + sendSetResponse({ + success: true, + flag, + requested: enabled, + enabled: records.find((record) => record.flag === flag)?.enabled ?? false, + records, + }); + } catch (error) { + logger.warn({ err: error, serverId: this.serverId, flag }, 'failed to persist global memory feature config'); + sendSetResponse({ + success: false, + flag, + requested: enabled, + error: MEMORY_MANAGEMENT_ERROR_CODES.FEATURE_CONFIG_WRITE_FAILED, + errorCode: MEMORY_MANAGEMENT_ERROR_CODES.FEATURE_CONFIG_WRITE_FAILED, + }); + } + return true; + } + + private roleFromMembership(role: unknown, elevatedRole: Exclude): MemoryManagementRole { + return role === 'owner' || role === 'admin' ? elevatedRole : 'user'; + } + + private async resolveMemoryManagementAuthorization(params: { + userId: string; + canonicalRepoId?: string; + projectDir?: string; + workspaceId?: string; + orgId?: string; + }): Promise<{ role: MemoryManagementRole; boundProjects: MemoryManagementBoundProject[] }> { + if (!this.db) return { role: 'user', boundProjects: [] }; + const { userId, canonicalRepoId, projectDir, workspaceId, orgId } = params; + try { + if (canonicalRepoId) { + const row = await this.db.queryOne<{ role?: string; workspace_id?: string | null; enterprise_id?: string | null }>( + `SELECT tm.role, e.workspace_id, e.enterprise_id + FROM shared_project_enrollments e + JOIN team_members tm ON tm.team_id = e.enterprise_id AND tm.user_id = $2 + WHERE e.canonical_repo_id = $1 + AND e.status = 'active' + ORDER BY CASE tm.role WHEN 'owner' THEN 0 WHEN 'admin' THEN 1 ELSE 2 END + LIMIT 1`, + [canonicalRepoId, userId], + ); + if (typeof row?.role === 'string') { + return { + role: this.roleFromMembership(row.role, 'workspace_admin'), + boundProjects: [{ + projectDir, + canonicalRepoId, + workspaceId: typeof row.workspace_id === 'string' ? row.workspace_id : undefined, + orgId: typeof row.enterprise_id === 'string' ? row.enterprise_id : undefined, + }], + }; + } + return { role: 'user', boundProjects: [] }; + } + + if (workspaceId) { + const row = await this.db.queryOne<{ role?: string; enterprise_id?: string | null }>( + `SELECT tm.role, w.enterprise_id + FROM shared_context_workspaces w + JOIN team_members tm ON tm.team_id = w.enterprise_id AND tm.user_id = $2 + WHERE w.id = $1`, + [workspaceId, userId], + ); + if (typeof row?.role === 'string') { + return { + role: this.roleFromMembership(row.role, 'workspace_admin'), + boundProjects: [{ + workspaceId, + orgId: typeof row.enterprise_id === 'string' ? row.enterprise_id : undefined, + }], + }; + } + return { role: 'user', boundProjects: [] }; + } + + if (orgId) { + const row = await this.db.queryOne<{ role?: string }>( + 'SELECT role FROM team_members WHERE team_id = $1 AND user_id = $2', + [orgId, userId], + ); + if (typeof row?.role === 'string') { + return { + role: this.roleFromMembership(row.role, 'org_admin'), + boundProjects: [{ orgId }], + }; + } + } + } catch (error) { + logger.warn({ err: error, serverId: this.serverId }, 'memory management authorization derivation failed'); + } + return { role: 'user', boundProjects: [] }; + } + + private async withMemoryManagementContext(ws: WebSocket, msg: Record, requestId: string): Promise> { + const userId = this.browserUserIds.get(ws)?.trim(); + if (!userId) return msg; + const canonicalRepoId = typeof msg.canonicalRepoId === 'string' && msg.canonicalRepoId.trim() + ? msg.canonicalRepoId.trim() + : undefined; + const projectDir = typeof msg.projectDir === 'string' && msg.projectDir.trim() ? msg.projectDir.trim() : undefined; + const workspaceId = typeof msg.workspaceId === 'string' && msg.workspaceId.trim() ? msg.workspaceId.trim() : undefined; + const orgId = typeof msg.orgId === 'string' && msg.orgId.trim() + ? msg.orgId.trim() + : (typeof msg.enterpriseId === 'string' && msg.enterpriseId.trim() ? msg.enterpriseId.trim() : undefined); + const authorization = await this.resolveMemoryManagementAuthorization({ userId, canonicalRepoId, projectDir, workspaceId, orgId }); + const context: AuthenticatedMemoryManagementContext = { + actorId: userId, + userId, + role: authorization.role, + serverId: this.serverId, + requestId, + source: 'server_bridge', + boundProjects: authorization.boundProjects, + }; + const { [MEMORY_MANAGEMENT_CONTEXT_FIELD]: _ignoredContext, managementContext: _ignoredLegacyContext, ...safeMsg } = msg; + void _ignoredContext; + void _ignoredLegacyContext; + return { + ...safeMsg, + [MEMORY_MANAGEMENT_CONTEXT_FIELD]: context, + }; + } + // ── Daemon connection ────────────────────────────────────────────────────── handleDaemonConnection(ws: WebSocket, db: Database, env: Env, onAuthenticated?: () => void): void { @@ -465,7 +860,10 @@ export class WsBridge { if (this.authTimer) clearTimeout(this.authTimer); const tokenHash = sha256Hex(msg.token); - const server = await db.queryOne<{ token_hash: string }>('SELECT token_hash FROM servers WHERE id = $1', [this.serverId]); + const server = await db.queryOne<{ token_hash: string; user_id?: string }>( + 'SELECT token_hash, user_id FROM servers WHERE id = $1', + [this.serverId], + ); if (!server || server.token_hash !== tokenHash) { logger.warn({ serverId: this.serverId }, 'Daemon auth failed'); @@ -484,8 +882,20 @@ export class WsBridge { updateServerHeartbeat(db, this.serverId, this.daemonVersion).catch((err) => logger.error({ err }, 'Failed to update heartbeat on auth'), ); + if (typeof server.user_id === 'string' && server.user_id.trim()) { + try { + this.sendMemoryFeatureConfigApply(await this.readUserMemoryFeatureFlags(server.user_id)); + } catch (err) { + logger.warn({ err, serverId: this.serverId }, 'failed to push global memory feature config on daemon auth'); + } + } + this.daemonUpgradeCoordinator.clearIfTargetVersionMatches(this.daemonVersion); + this.flushPendingDaemonUpgrade(ws); - // Auto-upgrade: on each reconnect, retry up to 3 times with 10-minute intervals. + // Auto-upgrade: on reconnect, retry up to 3 times, but never schedule + // more than one upgrade command per 15 minutes while the daemon remains + // on a mismatched version. This protects npm global install from + // reconnect storms and registry propagation windows. // Always target the server's exact version so dev↔stable mismatches converge to // the same channel in both directions. const serverVersion = process.env.APP_VERSION; @@ -496,20 +906,44 @@ export class WsBridge { && this.daemonVersion !== serverVersion, ); if (shouldUpgrade) { - this.upgradeAttempts = (this.upgradeAttempts ?? 0) + 1; - if (this.upgradeAttempts <= 3) { - logger.info({ serverId: this.serverId, daemonVersion: this.daemonVersion, serverVersion, attempt: this.upgradeAttempts }, 'Version mismatch — sending daemon.upgrade'); - setTimeout(() => { - try { ws.send(JSON.stringify({ type: 'daemon.upgrade', targetVersion: serverVersion })); } catch { /* ignore */ } - }, 5000); - // Schedule retry: if daemon reconnects with the same old version after 10 min, the counter is already incremented. - // If daemon doesn't reconnect (upgrade succeeded and restarted), the next auth will have matching version → no upgrade sent. - } else { - logger.warn({ serverId: this.serverId, daemonVersion: this.daemonVersion, serverVersion, attempts: this.upgradeAttempts }, 'Version mismatch — max upgrade attempts reached, giving up'); + const result = this.requestDaemonUpgrade({ + targetVersion: serverVersion, + source: 'auto', + isStillCurrent: () => this.daemonWs === ws && this.authenticated && this.daemonVersion !== serverVersion, + }); + if (result.deliveryStatus === DAEMON_UPGRADE_DELIVERY_STATUS.SENT) { + logger.info({ + serverId: this.serverId, + daemonVersion: this.daemonVersion, + serverVersion, + upgradeId: result.upgradeId, + }, 'Version mismatch — scheduling daemon.upgrade'); + } else if (result.deliveryStatus === DAEMON_UPGRADE_DELIVERY_STATUS.SUPPRESSED) { + logger.info({ + serverId: this.serverId, + daemonVersion: this.daemonVersion, + serverVersion, + nextAttemptAt: result.nextAttemptAt, + }, 'Version mismatch — auto daemon.upgrade suppressed by 15-minute interval'); + } else if (result.deliveryStatus === DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF) { + logger.warn({ + serverId: this.serverId, + daemonVersion: this.daemonVersion, + serverVersion, + reason: result.reason, + }, 'Version mismatch — auto daemon.upgrade in backoff'); + } else if (result.deliveryStatus === DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_PUBLICATION) { + logger.info({ + serverId: this.serverId, + daemonVersion: this.daemonVersion, + serverVersion, + nextAttemptAt: result.nextAttemptAt, + reason: result.reason, + }, 'Version mismatch — waiting for daemon upgrade target to appear on npm'); } } else { - // Version matches, daemon is newer, or auto-upgrade does not apply — reset retry counter - this.upgradeAttempts = 0; + // Version matches or auto-upgrade does not apply — reset retry state. + this.daemonUpgradeCoordinator.clearIfTargetVersionMatches(this.daemonVersion); } // Replay queued messages, skipping terminal.subscribe/unsubscribe — refs replay below is authoritative @@ -517,6 +951,14 @@ export class WsBridge { try { const parsed = JSON.parse(queued) as { type?: string }; if (parsed.type === 'terminal.subscribe' || parsed.type === 'terminal.unsubscribe') continue; + if (parsed.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE) { + this.requestDaemonUpgrade({ + targetVersion: (parsed as { targetVersion?: unknown }).targetVersion, + source: 'replay', + isStillCurrent: () => this.daemonWs === ws && this.authenticated, + }); + continue; + } ws.send(queued); } catch { /* ignore */ } } @@ -551,7 +993,11 @@ export class WsBridge { } if (msg.type === 'heartbeat') { - updateServerHeartbeat(db, this.serverId).catch((err) => + const heartbeatDaemonVersion = typeof msg.daemonVersion === 'string' + ? msg.daemonVersion + : this.daemonVersion; + if (typeof heartbeatDaemonVersion === 'string') this.daemonVersion = heartbeatDaemonVersion; + updateServerHeartbeat(db, this.serverId, heartbeatDaemonVersion).catch((err) => logger.error({ err }, 'Failed to update heartbeat'), ); // Ack heartbeat so daemon watchdog doesn't consider the connection dead @@ -657,7 +1103,7 @@ export class WsBridge { safeSend(ws, JSON.stringify({ type: TRANSPORT_MSG.SESSIONS_RESPONSE, providerId, sessions })); } - ws.on('message', (data) => { + ws.on('message', async (data) => { const raw = (data as Buffer).toString(); if (Buffer.byteLength(raw, 'utf8') > MAX_BROWSER_PAYLOAD) { logger.warn({ serverId: this.serverId }, 'Browser message too large — dropped'); @@ -714,6 +1160,37 @@ export class WsBridge { return; } + if (this.isBrowserForbiddenDaemonCommandType(msg.type)) { + logger.warn({ serverId: this.serverId, type: msg.type }, 'Browser attempted server-only daemon command — rejected'); + safeSend(ws, JSON.stringify({ + type: 'error', + code: 'server_only_command', + originalType: msg.type, + requestId: msg.requestId, + })); + return; + } + + if (msg.type === MEMORY_WS.FEATURES_QUERY) { + await this.handleMemoryFeaturesQuery(ws, msg); + return; + } + if (msg.type === MEMORY_WS.FEATURES_SET) { + await this.handleMemoryFeaturesSet(ws, msg); + return; + } + + if (isMemoryManagementRequestType(msg.type)) { + const requestId = this.registerMemoryManagementRequest(ws, msg); + if (!requestId) return; + try { + this.sendToDaemon(JSON.stringify(await this.withMemoryManagementContext(ws, msg, requestId))); + } catch (error) { + this.failMemoryManagementForward(ws, msg, requestId, error); + } + return; + } + // Track fs.ls requests for single-cast response routing if (msg.type === 'fs.ls' && typeof msg.requestId === 'string') { const reqId = msg.requestId; @@ -796,7 +1273,7 @@ export class WsBridge { return; } - // ── command.ack reliability: intercept session.send ──────────────── + // ── command.ack reliability: intercept user sends and cancels ─────── // // Three cases: // 1. daemon fully offline (past grace) → immediately command.failed @@ -805,7 +1282,7 @@ export class WsBridge { // // In all cases we record an inflight entry so that the later command.ack // (or timeout / disconnect) can correlate back to the right browser. - if (msg.type === 'session.send' && typeof msg.commandId === 'string') { + if ((msg.type === 'session.send' || msg.type === DAEMON_COMMAND_TYPES.SESSION_CANCEL) && typeof msg.commandId === 'string') { const sessionName = typeof msg.sessionName === 'string' ? msg.sessionName : (typeof msg.session === 'string' ? msg.session : ''); @@ -871,6 +1348,21 @@ export class WsBridge { return; } + if (isMemoryManagementResponseType(type)) { + const requestId = msg.requestId as string | undefined; + const pending = requestId ? this.pendingMemoryManagementRequests.get(requestId) : undefined; + if (!requestId || !pending) { + incrementCounter('mem.bridge.unrouted_response', { type: String(type) }); + logger.warn({ serverId: this.serverId, type, requestId }, 'memory management response missing pending request — dropped'); + return; + } + this.clearPendingMemoryManagementRequest(requestId); + if (pending.socket.readyState === WebSocket.OPEN) { + pending.socket.send(JSON.stringify(msg)); + } + return; + } + // ── fs.ls_response: single-cast back to requesting browser ──────────────── if (type === 'fs.ls_response') { const requestId = msg.requestId as string | undefined; @@ -1740,6 +2232,12 @@ export class WsBridge { this.pendingTimelineRequests.delete(reqId); } } + for (const [reqId, pending] of this.pendingMemoryManagementRequests) { + if (pending.socket === ws) { + clearTimeout(pending.timer); + this.pendingMemoryManagementRequests.delete(reqId); + } + } } /** @@ -2025,6 +2523,54 @@ export class WsBridge { return this.seenCommandAcks.has(commandId); } + requestDaemonUpgrade(input: { + targetVersion?: unknown; + source?: DaemonUpgradeSource; + isStillCurrent?: () => boolean; + } = {}): RequestDaemonUpgradeResult { + return this.daemonUpgradeCoordinator.request({ + targetVersion: input.targetVersion, + source: input.source ?? 'manual', + isDaemonReady: () => this.isDaemonReadyForUpgrade(), + isStillCurrent: input.isStillCurrent, + send: (message) => this.sendDirectToDaemon(message), + }); + } + + private flushPendingDaemonUpgrade(ws: WebSocket): void { + const result = this.daemonUpgradeCoordinator.flushPending({ + isDaemonReady: () => this.isDaemonReadyForUpgrade(), + isStillCurrent: () => this.daemonWs === ws && this.authenticated, + send: (message) => this.sendDirectToDaemon(message), + }); + if (result?.deliveryStatus === DAEMON_UPGRADE_DELIVERY_STATUS.SENT) { + logger.info({ + serverId: this.serverId, + targetVersion: result.targetVersion, + upgradeId: result.upgradeId, + }, 'Flushed pending daemon.upgrade after daemon auth'); + } else if (result?.deliveryStatus === DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_PUBLICATION) { + logger.info({ + serverId: this.serverId, + targetVersion: result.targetVersion, + nextAttemptAt: result.nextAttemptAt, + }, 'Pending daemon.upgrade is waiting for npm publication'); + } + } + + private isDaemonReadyForUpgrade(): boolean { + return Boolean(this.daemonWs && this.authenticated); + } + + private sendDirectToDaemon(message: Record): void { + if (!this.daemonWs || !this.authenticated) return; + try { + this.daemonWs.send(JSON.stringify(message)); + } catch (err) { + logger.error({ serverId: this.serverId, err }, 'Failed to send daemon upgrade command'); + } + } + /** Force-close the daemon WebSocket. Use after token rotation to evict the stale connection. */ kickDaemon(): void { if (this.daemonWs) { @@ -2035,6 +2581,14 @@ export class WsBridge { } sendToDaemon(message: string): void { + const parsed = this.parseJsonObject(message); + if (parsed?.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE) { + this.requestDaemonUpgrade({ + targetVersion: parsed.targetVersion, + source: 'manual', + }); + return; + } if (this.daemonWs && this.authenticated) { try { this.daemonWs.send(message); @@ -2048,6 +2602,21 @@ export class WsBridge { } } + private parseJsonObject(message: string): Record | null { + try { + const parsed = JSON.parse(message) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? parsed as Record + : null; + } catch { + return null; + } + } + + private isBrowserForbiddenDaemonCommandType(type: string): boolean { + return type === DAEMON_COMMAND_TYPES.SERVER_DELETE || type.startsWith('daemon.'); + } + requestTimelineHistory(params: { sessionName: string; limit?: number; diff --git a/server/src/ws/daemon-upgrade-coordinator.ts b/server/src/ws/daemon-upgrade-coordinator.ts new file mode 100644 index 000000000..590217f57 --- /dev/null +++ b/server/src/ws/daemon-upgrade-coordinator.ts @@ -0,0 +1,332 @@ +import { randomUUID } from 'node:crypto'; +import { DAEMON_COMMAND_TYPES } from '../../../shared/daemon-command-types.js'; +import { + DAEMON_UPGRADE_DELIVERY_STATUS, + normalizeDaemonUpgradeTargetVersion, + shouldSendDaemonUpgradeTargetVersion, + type DaemonUpgradeDeliveryStatus, +} from '../../../shared/daemon-upgrade.js'; +import { + daemonUpgradePublicationGate, + type DaemonUpgradePublicationGate, +} from './daemon-upgrade-publication-gate.js'; + +const AUTO_UPGRADE_SEND_DELAY_MS = 5_000; +const AUTO_UPGRADE_MIN_INTERVAL_MS = 15 * 60 * 1000; +const AUTO_UPGRADE_MAX_ATTEMPTS = 3; + +export type DaemonUpgradeSource = 'auto' | 'manual' | 'replay'; + +type UpgradeLifecycleState = 'pending_offline' | 'pending_publication' | 'scheduled' | 'sent' | 'superseded'; + +interface UpgradeState { + upgradeId: string; + targetVersion: string; + source: DaemonUpgradeSource; + status: UpgradeLifecycleState; + attempt: number; + createdAt: number; + updatedAt: number; + lastSentAt: number | null; + timer: ReturnType | null; + publicationResumeInput: RequestDaemonUpgradeInput | null; + publicationCallbackRegistered: boolean; +} + +export interface RequestDaemonUpgradeInput { + targetVersion?: unknown; + source: DaemonUpgradeSource; + isDaemonReady: () => boolean; + isStillCurrent?: () => boolean; + send: (message: Record) => void; + now?: number; +} + +export interface RequestDaemonUpgradeResult { + ok: boolean; + upgradeId?: string; + targetVersion?: string; + deliveryStatus: DaemonUpgradeDeliveryStatus; + nextAttemptAt?: string; + reason?: string; +} + +export class DaemonUpgradeCoordinator { + private current: UpgradeState | null = null; + private lastAutoSentAt: number | null = null; + + constructor(private readonly publicationGate: DaemonUpgradePublicationGate = daemonUpgradePublicationGate) {} + + request(input: RequestDaemonUpgradeInput): RequestDaemonUpgradeResult { + let targetVersion: string; + try { + targetVersion = normalizeDaemonUpgradeTargetVersion(input.targetVersion); + } catch { + return { + ok: false, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.INVALID_TARGET, + reason: 'invalid_target_version', + }; + } + + const now = input.now ?? Date.now(); + const current = this.current?.targetVersion === targetVersion ? this.current : null; + if (this.current && this.current.targetVersion !== targetVersion) { + this.supersedeCurrent(now); + } + + if (current?.status === 'pending_publication') { + current.source = input.source; + current.updatedAt = now; + if (!input.isDaemonReady()) { + current.status = 'pending_offline'; + return { + ok: true, + upgradeId: current.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE, + }; + } + const publication = this.ensureTargetPublished(current, input, now); + if (publication) return publication; + } else if (current && current.status !== 'superseded' && (current.status !== 'pending_offline' || !input.isDaemonReady())) { + if (input.source === 'auto') { + const nextAutoAt = this.nextAutoAttemptAt(now); + if (nextAutoAt > now) { + return { + ok: true, + upgradeId: current.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SUPPRESSED, + nextAttemptAt: new Date(nextAutoAt).toISOString(), + }; + } + if (current.attempt >= AUTO_UPGRADE_MAX_ATTEMPTS) { + return { + ok: true, + upgradeId: current.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'max_attempts_reached', + }; + } + } else { + return { + ok: true, + upgradeId: current.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.ALREADY_IN_PROGRESS, + }; + } + } + + const state = current ?? { + upgradeId: randomUUID(), + targetVersion, + source: input.source, + status: 'pending_offline' as UpgradeLifecycleState, + attempt: 0, + createdAt: now, + updatedAt: now, + lastSentAt: null, + timer: null, + publicationResumeInput: null, + publicationCallbackRegistered: false, + }; + state.source = input.source; + state.updatedAt = now; + this.current = state; + + if (!input.isDaemonReady()) { + state.status = 'pending_offline'; + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE, + }; + } + + if (input.source === 'auto') { + const publication = this.ensureTargetPublished(state, input, now); + if (publication) return publication; + return this.scheduleAutoSend(state, input, now); + } + + const publication = this.ensureTargetPublished(state, input, now); + if (publication) return publication; + this.sendNow(state, input, now); + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SENT, + }; + } + + flushPending(input: Omit): RequestDaemonUpgradeResult | null { + const state = this.current; + if (!state || (state.status !== 'pending_offline' && state.status !== 'pending_publication')) return null; + if (!input.isDaemonReady()) { + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion: state.targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE, + }; + } + const requestInput = { ...input, targetVersion: state.targetVersion, source: state.source }; + const now = Date.now(); + const publication = this.ensureTargetPublished(state, requestInput, now); + if (publication) return publication; + if (state.source === 'auto') { + return this.scheduleAutoSend(state, requestInput, now); + } + this.sendNow(state, requestInput, now); + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion: state.targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SENT, + }; + } + + clearIfTargetVersionMatches(targetVersion: string | null | undefined): void { + let normalized: string | null = null; + try { + normalized = targetVersion ? normalizeDaemonUpgradeTargetVersion(targetVersion) : null; + } catch { + return; + } + if (normalized && this.current?.targetVersion === normalized) { + this.clearCurrent(); + } + } + + parseQueuedUpgrade(raw: string): string | null { + try { + const parsed = JSON.parse(raw) as { type?: unknown; targetVersion?: unknown }; + if (parsed.type !== DAEMON_COMMAND_TYPES.DAEMON_UPGRADE) return null; + return normalizeDaemonUpgradeTargetVersion(parsed.targetVersion); + } catch { + return null; + } + } + + private scheduleAutoSend(state: UpgradeState, input: RequestDaemonUpgradeInput, now: number): RequestDaemonUpgradeResult { + const nextAutoAt = this.nextAutoAttemptAt(now); + if (nextAutoAt > now) { + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion: state.targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SUPPRESSED, + nextAttemptAt: new Date(nextAutoAt).toISOString(), + }; + } + if (state.attempt >= AUTO_UPGRADE_MAX_ATTEMPTS) { + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion: state.targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'max_attempts_reached', + }; + } + if (state.timer) clearTimeout(state.timer); + state.status = 'scheduled'; + state.attempt += 1; + state.updatedAt = now; + this.lastAutoSentAt = now; + state.timer = setTimeout(() => { + state.timer = null; + if (this.current !== state || !input.isDaemonReady() || input.isStillCurrent?.() === false) return; + this.sendNow(state, input, Date.now()); + }, AUTO_UPGRADE_SEND_DELAY_MS); + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion: state.targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SENT, + }; + } + + private ensureTargetPublished( + state: UpgradeState, + input: RequestDaemonUpgradeInput, + now: number, + ): RequestDaemonUpgradeResult | null { + state.publicationResumeInput = input; + const publication = this.publicationGate.ensurePublished( + state.targetVersion, + state.publicationCallbackRegistered + ? undefined + : () => { + state.publicationCallbackRegistered = false; + const resumeInput = state.publicationResumeInput; + state.publicationResumeInput = null; + if (resumeInput) this.resumeAfterPublication(state, resumeInput); + }, + ); + if (publication.status === 'available') { + state.publicationResumeInput = null; + state.publicationCallbackRegistered = false; + return null; + } + state.publicationCallbackRegistered = true; + state.status = 'pending_publication'; + state.updatedAt = now; + return { + ok: true, + upgradeId: state.upgradeId, + targetVersion: state.targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_PUBLICATION, + ...(publication.nextProbeAt ? { nextAttemptAt: publication.nextProbeAt } : {}), + ...(publication.reason ? { reason: publication.reason } : {}), + }; + } + + private resumeAfterPublication(state: UpgradeState, input: RequestDaemonUpgradeInput): void { + if (this.current !== state || state.status !== 'pending_publication') return; + if (!input.isDaemonReady() || input.isStillCurrent?.() === false) return; + const now = Date.now(); + if (state.source === 'auto') { + this.scheduleAutoSend(state, input, now); + return; + } + this.sendNow(state, input, now); + } + + private sendNow(state: UpgradeState, input: RequestDaemonUpgradeInput, now: number): void { + state.status = 'sent'; + state.lastSentAt = now; + state.updatedAt = now; + input.send(this.buildUpgradeMessage(state)); + } + + private buildUpgradeMessage(state: UpgradeState): Record { + return { + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: state.upgradeId, + ...(shouldSendDaemonUpgradeTargetVersion(state.targetVersion) ? { targetVersion: state.targetVersion } : {}), + }; + } + + private nextAutoAttemptAt(now: number): number { + return this.lastAutoSentAt == null ? now : this.lastAutoSentAt + AUTO_UPGRADE_MIN_INTERVAL_MS; + } + + private supersedeCurrent(now: number): void { + if (!this.current) return; + if (this.current.timer) clearTimeout(this.current.timer); + this.current.status = 'superseded'; + this.current.updatedAt = now; + this.current = null; + } + + private clearCurrent(): void { + if (this.current?.timer) clearTimeout(this.current.timer); + this.current = null; + this.lastAutoSentAt = null; + } +} diff --git a/server/src/ws/daemon-upgrade-publication-gate.ts b/server/src/ws/daemon-upgrade-publication-gate.ts new file mode 100644 index 000000000..4c76339d2 --- /dev/null +++ b/server/src/ws/daemon-upgrade-publication-gate.ts @@ -0,0 +1,162 @@ +import { DAEMON_UPGRADE_TARGET_LATEST } from '../../../shared/daemon-upgrade.js'; + +const NPM_TARBALL_BASE_URL = 'https://registry.npmjs.org/imcodes/-/'; +const DEFAULT_RETRY_DELAYS_MS = [15_000, 30_000, 60_000, 120_000, 300_000] as const; + +type ProbeStatus = 'available' | 'missing' | 'error'; + +export interface DaemonUpgradePublicationProbeResult { + status: ProbeStatus; + statusCode?: number; + error?: unknown; +} + +type DaemonUpgradePublicationProbe = (targetVersion: string, tarballUrl: string) => Promise; + +export interface DaemonUpgradePublicationGateResult { + status: 'available' | 'pending'; + nextProbeAt?: string; + reason?: string; +} + +interface PublicationRecord { + status: 'available' | 'pending'; + callbacks: Set<() => void>; + attempt: number; + inFlight: boolean; + timer: ReturnType | null; + nextProbeAt: number | null; + lastStatusCode?: number; +} + +export interface DaemonUpgradePublicationGateOptions { + probe?: DaemonUpgradePublicationProbe; + retryDelaysMs?: readonly number[]; +} + +async function defaultProbe(_targetVersion: string, tarballUrl: string): Promise { + try { + const res = await fetch(tarballUrl, { + method: 'HEAD', + redirect: 'follow', + }); + if (res.status >= 200 && res.status < 300) return { status: 'available', statusCode: res.status }; + if (res.status === 404) return { status: 'missing', statusCode: res.status }; + return { status: 'error', statusCode: res.status }; + } catch (error) { + return { status: 'error', error }; + } +} + +export function daemonUpgradeTarballUrl(targetVersion: string): string { + return `${NPM_TARBALL_BASE_URL}imcodes-${encodeURIComponent(targetVersion)}.tgz`; +} + +export class DaemonUpgradePublicationGate { + private readonly probe: DaemonUpgradePublicationProbe; + private readonly retryDelaysMs: readonly number[]; + private readonly records = new Map(); + + constructor(options: DaemonUpgradePublicationGateOptions = {}) { + this.probe = options.probe ?? defaultProbe; + this.retryDelaysMs = options.retryDelaysMs?.length ? options.retryDelaysMs : DEFAULT_RETRY_DELAYS_MS; + } + + ensurePublished(targetVersion: string, onPublished?: () => void): DaemonUpgradePublicationGateResult { + if (targetVersion === DAEMON_UPGRADE_TARGET_LATEST) return { status: 'available' }; + + const record = this.recordFor(targetVersion); + if (record.status === 'available') return { status: 'available' }; + + if (onPublished) record.callbacks.add(onPublished); + if (!record.inFlight && !record.timer) { + this.startProbe(targetVersion, record); + } + return { + status: 'pending', + ...(record.nextProbeAt ? { nextProbeAt: new Date(record.nextProbeAt).toISOString() } : {}), + reason: record.lastStatusCode === 404 ? 'target_version_not_published' : 'target_version_publication_pending', + }; + } + + markPublishedForTest(targetVersion: string): void { + const record = this.recordFor(targetVersion); + this.markPublished(targetVersion, record); + } + + clear(): void { + for (const record of this.records.values()) { + if (record.timer) clearTimeout(record.timer); + record.callbacks.clear(); + } + this.records.clear(); + } + + private recordFor(targetVersion: string): PublicationRecord { + const existing = this.records.get(targetVersion); + if (existing) return existing; + const next: PublicationRecord = { + status: 'pending', + callbacks: new Set(), + attempt: 0, + inFlight: false, + timer: null, + nextProbeAt: null, + }; + this.records.set(targetVersion, next); + return next; + } + + private startProbe(targetVersion: string, record: PublicationRecord): void { + record.inFlight = true; + record.nextProbeAt = null; + this.probe(targetVersion, daemonUpgradeTarballUrl(targetVersion)) + .then((result) => { + record.inFlight = false; + record.lastStatusCode = result.statusCode; + if (result.status === 'available') { + this.markPublished(targetVersion, record); + return; + } + this.scheduleRetry(targetVersion, record); + }) + .catch(() => { + record.inFlight = false; + record.lastStatusCode = undefined; + this.scheduleRetry(targetVersion, record); + }); + } + + private scheduleRetry(targetVersion: string, record: PublicationRecord): void { + if (record.status === 'available') return; + const delay = this.retryDelaysMs[Math.min(record.attempt, this.retryDelaysMs.length - 1)] ?? DEFAULT_RETRY_DELAYS_MS.at(-1)!; + record.attempt += 1; + record.nextProbeAt = Date.now() + delay; + record.timer = setTimeout(() => { + record.timer = null; + this.startProbe(targetVersion, record); + }, delay); + } + + private markPublished(targetVersion: string, record: PublicationRecord): void { + if (record.timer) clearTimeout(record.timer); + record.status = 'available'; + record.timer = null; + record.inFlight = false; + record.nextProbeAt = null; + const callbacks = [...record.callbacks]; + record.callbacks.clear(); + this.records.set(targetVersion, record); + for (const callback of callbacks) callback(); + } +} + +export const daemonUpgradePublicationGate = new DaemonUpgradePublicationGate(); + +export function resetDaemonUpgradePublicationGateForTest(): void { + daemonUpgradePublicationGate.clear(); +} + +export function markDaemonUpgradeTargetVersionPublishedForTest(targetVersion: string): void { + daemonUpgradePublicationGate.markPublishedForTest(targetVersion); +} diff --git a/server/test/ack-reliability.test.ts b/server/test/ack-reliability.test.ts index deb9f210c..83301c58d 100644 --- a/server/test/ack-reliability.test.ts +++ b/server/test/ack-reliability.test.ts @@ -10,6 +10,7 @@ import { ACK_TIMEOUT_MS, ACK_TIMEOUT_RETRY_LIMIT, } from '../../shared/ack-protocol.js'; +import { DAEMON_COMMAND_TYPES } from '../../shared/daemon-command-types.js'; class MockWs extends EventEmitter { sent: Array = []; @@ -120,6 +121,28 @@ describe('WsBridge — command ack reliability', () => { expect(bridge._getInflightCountForTest()).toBe(1); }); + it('dispatches session.cancel through the reliable priority command path', async () => { + const bridge = WsBridge.get(serverId); + const daemonWs = await connectAndAuthenticateDaemon(bridge, serverId); + const browser = addBrowserSubscriber(bridge, 'deck_test_brain'); + + browser.emit('message', Buffer.from(JSON.stringify({ + type: DAEMON_COMMAND_TYPES.SESSION_CANCEL, + sessionName: 'deck_test_brain', + commandId: 'C-CANCEL-1', + }))); + await flushAsync(); + + const forwarded = daemonWs.sentByType(DAEMON_COMMAND_TYPES.SESSION_CANCEL); + expect(forwarded).toHaveLength(1); + expect(forwarded[0]).toEqual({ + type: DAEMON_COMMAND_TYPES.SESSION_CANCEL, + sessionName: 'deck_test_brain', + commandId: 'C-CANCEL-1', + }); + expect(bridge._getInflightCountForTest()).toBe(1); + }); + it('does not forward an in-flight duplicate commandId to the daemon', async () => { const bridge = WsBridge.get(serverId); const daemonWs = await connectAndAuthenticateDaemon(bridge, serverId); diff --git a/server/test/bridge-memory-management.test.ts b/server/test/bridge-memory-management.test.ts new file mode 100644 index 000000000..a75885dd7 --- /dev/null +++ b/server/test/bridge-memory-management.test.ts @@ -0,0 +1,375 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; +import { WsBridge } from '../src/ws/bridge.js'; +import { MEMORY_WS } from '../../shared/memory-ws.js'; +import { MEMORY_MANAGEMENT_CONTEXT_FIELD } from '../../shared/memory-management-context.js'; +import { + MEMORY_FEATURE_CONFIG_MSG, + MEMORY_FEATURE_CONFIG_PREF_KEY, + MEMORY_FEATURE_FLAGS_BY_NAME, +} from '../../shared/feature-flags.js'; + +class MockWs extends EventEmitter { + sent: Array = []; + closed = false; + readyState = 1; + send(data: string | Buffer, _opts?: unknown, callback?: (err?: Error) => void) { + if (this.closed) { + const err = new Error('closed'); + if (callback) { callback(err); return; } + throw err; + } + this.sent.push(data); + callback?.(); + } + close() { this.closed = true; this.readyState = 3; this.emit('close'); } + sentJson(): Array> { + return this.sent.filter((entry): entry is string => typeof entry === 'string') + .map((entry) => JSON.parse(entry) as Record); + } +} + +function makeDb(queryOne?: (sql: string, params?: unknown[]) => Promise) { + return { + queryOne: queryOne ?? (async () => ({ token_hash: 'valid-hash' })), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +vi.mock('../src/security/crypto.js', () => ({ sha256Hex: () => 'valid-hash' })); +vi.mock('../src/routes/push.js', () => ({ dispatchPush: vi.fn() })); + +async function flush() { + for (let i = 0; i < 5; i++) await new Promise((resolve) => process.nextTick(resolve)); +} + +async function setup(db = makeDb()) { + const serverId = `memory-management-${Math.random().toString(36).slice(2)}`; + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + bridge.handleDaemonConnection(daemon as never, db, {} as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'token' })); + await flush(); + const browserA = new MockWs(); + const browserB = new MockWs(); + bridge.handleBrowserConnection(browserA as never, 'user-a', db); + bridge.handleBrowserConnection(browserB as never, 'user-b', db); + return { bridge, daemon, browserA, browserB }; +} + +describe('WsBridge memory management routing', () => { + beforeEach(() => { WsBridge.getAll().clear(); }); + afterEach(() => { WsBridge.getAll().clear(); vi.clearAllMocks(); }); + + it('single-casts memory management responses to the requesting browser only', async () => { + const { daemon, browserA, browserB } = await setup(); + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.SKILL_READ, requestId: 'req-skill', key: 'k', layer: 'user_default' })); + await flush(); + + daemon.emit('message', JSON.stringify({ type: MEMORY_WS.SKILL_READ_RESPONSE, requestId: 'req-skill', success: true, key: 'k', layer: 'user_default', content: 'secret skill' })); + await flush(); + + expect(browserA.sentJson().some((msg) => msg.type === MEMORY_WS.SKILL_READ_RESPONSE && msg.content === 'secret skill')).toBe(true); + expect(browserB.sentJson().some((msg) => msg.type === MEMORY_WS.SKILL_READ_RESPONSE)).toBe(false); + }); + + it('injects server-derived memory management context and does not trust browser actorId', async () => { + const { daemon, browserA } = await setup(); + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.OBSERVATION_PROMOTE, requestId: 'req-promote', id: 'obs-1', actorId: 'attacker', toScope: 'project_shared' })); + await flush(); + + const forwarded = daemon.sentJson().find((msg) => msg.type === MEMORY_WS.OBSERVATION_PROMOTE) as Record | undefined; + expect(forwarded).toBeTruthy(); + const ctx = forwarded?.[MEMORY_MANAGEMENT_CONTEXT_FIELD] as Record | undefined; + expect(ctx?.actorId).toBe('user-a'); + expect(ctx?.userId).toBe('user-a'); + expect(ctx?.role).toBe('user'); + expect(forwarded?.actorId).toBe('attacker'); + }); + + it('derives elevated memory management role from server membership instead of browser input', async () => { + const db = makeDb(async (sql: string, params?: unknown[]) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash' }; + if (sql.includes('FROM team_members') && params?.[0] === 'team-1' && params?.[1] === 'user-a') { + return { role: 'admin' }; + } + return null; + }); + const { daemon, browserA } = await setup(db); + browserA.emit('message', JSON.stringify({ + type: MEMORY_WS.OBSERVATION_PROMOTE, + requestId: 'req-promote-admin', + id: 'obs-1', + role: 'user', + enterpriseId: 'team-1', + toScope: 'org_shared', + })); + await flush(); + + const forwarded = daemon.sentJson().find((msg) => msg.type === MEMORY_WS.OBSERVATION_PROMOTE) as Record | undefined; + const ctx = forwarded?.[MEMORY_MANAGEMENT_CONTEXT_FIELD] as Record | undefined; + expect(ctx?.actorId).toBe('user-a'); + expect(ctx?.role).toBe('org_admin'); + expect(forwarded?.role).toBe('user'); + }); + + + it('rejects unauthenticated memory management requests before forwarding to daemon', async () => { + const serverId = `memory-management-${Math.random().toString(36).slice(2)}`; + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + const db = makeDb(); + bridge.handleDaemonConnection(daemon as never, db, {} as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'token' })); + await flush(); + const browser = new MockWs(); + bridge.handleBrowserConnection(browser as never, '', db); + + browser.emit('message', JSON.stringify({ type: MEMORY_WS.SKILL_QUERY, requestId: 'unauth-1' })); + await flush(); + + expect(daemon.sentJson().some((msg) => msg.type === MEMORY_WS.SKILL_QUERY)).toBe(false); + expect(browser.sentJson().some((msg) => msg.code === 'memory_management_unauthenticated')).toBe(true); + }); + + it('rejects duplicate memory management request ids without forwarding the duplicate', async () => { + const { daemon, browserA } = await setup(); + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.SKILL_QUERY, requestId: 'dup-1' })); + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.PREF_QUERY, requestId: 'dup-1' })); + await flush(); + + expect(daemon.sentJson().filter((msg) => msg.requestId === 'dup-1')).toHaveLength(1); + expect(browserA.sentJson().some((msg) => msg.code === 'duplicate_request_id')).toBe(true); + }); + + it('serves memory feature state from the user-global online preference store instead of daemon-local state', async () => { + const db = makeDb(async (sql: string, params?: unknown[]) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash', user_id: 'user-a' }; + if (sql.includes('FROM user_preferences')) { + expect(params).toEqual(['user-a', MEMORY_FEATURE_CONFIG_PREF_KEY]); + return { + value: JSON.stringify({ + [MEMORY_FEATURE_FLAGS_BY_NAME.namespaceRegistry]: true, + [MEMORY_FEATURE_FLAGS_BY_NAME.observationStore]: true, + [MEMORY_FEATURE_FLAGS_BY_NAME.preferences]: true, + }), + }; + } + return null; + }); + const { daemon, browserA } = await setup(db); + daemon.sent = []; + + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.FEATURES_QUERY, requestId: 'features-global' })); + await flush(); + + expect(daemon.sentJson().some((msg) => msg.type === MEMORY_WS.FEATURES_QUERY)).toBe(false); + const response = browserA.sentJson().find((msg) => msg.type === MEMORY_WS.FEATURES_RESPONSE && msg.requestId === 'features-global'); + expect(response).toBeTruthy(); + const records = response?.records as Array>; + expect(records.find((record) => record.flag === MEMORY_FEATURE_FLAGS_BY_NAME.preferences)).toMatchObject({ + requested: true, + enabled: true, + source: 'persisted_config', + }); + }); + + it('persists memory feature toggles globally and applies them to every online daemon owned by the user', async () => { + const writes: Array<{ userId: string; key: string; value: string }> = []; + const db = makeDb(async (sql: string) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash', user_id: 'user-a' }; + if (sql.includes('SELECT user_id FROM servers WHERE id = $1')) return { user_id: 'user-a' }; + if (sql.includes('FROM user_preferences')) return { value: '{}' }; + return null; + }); + const executeDb = db as unknown as { execute: (sql: string, params?: unknown[]) => Promise<{ changes: number }> }; + executeDb.execute = async (_sql: string, params?: unknown[]) => { + writes.push({ userId: String(params?.[0]), key: String(params?.[1]), value: String(params?.[2]) }); + return { changes: 1 }; + }; + const first = await setup(db); + const second = await setup(db); + first.daemon.sent = []; + second.daemon.sent = []; + + first.browserA.emit('message', JSON.stringify({ + type: MEMORY_WS.FEATURES_SET, + requestId: 'features-set-global', + flag: MEMORY_FEATURE_FLAGS_BY_NAME.preferences, + enabled: true, + })); + await flush(); + + const featureWrites = writes.filter((write) => write.key === MEMORY_FEATURE_CONFIG_PREF_KEY); + expect(featureWrites).toHaveLength(1); + const persisted = JSON.parse(featureWrites[0]?.value ?? '{}') as Record; + expect(persisted[MEMORY_FEATURE_FLAGS_BY_NAME.preferences]).toBe(true); + expect(persisted[MEMORY_FEATURE_FLAGS_BY_NAME.namespaceRegistry]).toBe(true); + expect(persisted[MEMORY_FEATURE_FLAGS_BY_NAME.observationStore]).toBe(true); + expect(first.daemon.sentJson().some((msg) => msg.type === MEMORY_WS.FEATURES_SET)).toBe(false); + const firstApply = first.daemon.sentJson().find((msg) => msg.type === MEMORY_FEATURE_CONFIG_MSG.APPLY); + const secondApply = second.daemon.sentJson().find((msg) => msg.type === MEMORY_FEATURE_CONFIG_MSG.APPLY); + expect(firstApply?.flags).toMatchObject({ [MEMORY_FEATURE_FLAGS_BY_NAME.preferences]: true }); + expect(secondApply?.flags).toMatchObject({ [MEMORY_FEATURE_FLAGS_BY_NAME.preferences]: true }); + expect(first.browserA.sentJson().find((msg) => msg.type === MEMORY_WS.FEATURES_SET_RESPONSE)).toMatchObject({ + requestId: 'features-set-global', + success: true, + flag: MEMORY_FEATURE_FLAGS_BY_NAME.preferences, + enabled: true, + }); + }); + + it('does not apply a user-global memory feature toggle to daemons owned by other users', async () => { + const makeOwnerDb = (ownerUserId: string) => makeDb(async (sql: string) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash', user_id: ownerUserId }; + if (sql.includes('SELECT user_id FROM servers WHERE id = $1')) return { user_id: ownerUserId }; + if (sql.includes('FROM user_preferences')) return { value: '{}' }; + return null; + }); + const owner = await setup(makeOwnerDb('user-a')); + const other = await setup(makeOwnerDb('user-b')); + owner.daemon.sent = []; + other.daemon.sent = []; + + owner.browserA.emit('message', JSON.stringify({ + type: MEMORY_WS.FEATURES_SET, + requestId: 'features-set-owner-only', + flag: MEMORY_FEATURE_FLAGS_BY_NAME.preferences, + enabled: true, + })); + await flush(); + + expect(owner.daemon.sentJson().some((msg) => msg.type === MEMORY_FEATURE_CONFIG_MSG.APPLY)).toBe(true); + expect(other.daemon.sentJson().some((msg) => msg.type === MEMORY_FEATURE_CONFIG_MSG.APPLY)).toBe(false); + }); + + it('enforces the per-socket pending memory management request limit', async () => { + const { daemon, browserA } = await setup(); + for (let i = 0; i < 33; i += 1) { + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.PREF_QUERY, requestId: `pending-${i}` })); + } + await flush(); + + expect(daemon.sentJson().filter((msg) => msg.type === MEMORY_WS.PREF_QUERY)).toHaveLength(32); + expect(browserA.sentJson().some((msg) => msg.code === 'too_many_memory_management_requests')).toBe(true); + }); + + it('strips browser-supplied management context fields before forwarding', async () => { + const { daemon, browserA } = await setup(); + browserA.emit('message', JSON.stringify({ + type: MEMORY_WS.SKILL_QUERY, + requestId: 'strip-1', + [MEMORY_MANAGEMENT_CONTEXT_FIELD]: { actorId: 'evil', userId: 'evil', role: 'org_admin', source: 'server_bridge' }, + managementContext: { actorId: 'evil', userId: 'evil', role: 'org_admin', source: 'server_bridge' }, + })); + await flush(); + + const forwarded = daemon.sentJson().find((msg) => msg.type === MEMORY_WS.SKILL_QUERY) as Record | undefined; + expect(forwarded).toBeTruthy(); + expect(forwarded?.managementContext).toBeUndefined(); + const ctx = forwarded?.[MEMORY_MANAGEMENT_CONTEXT_FIELD] as Record | undefined; + expect(ctx?.actorId).toBe('user-a'); + expect(ctx?.role).toBe('user'); + }); + + it('does not treat generic projectId as canonicalRepoId for role derivation', async () => { + const db = makeDb(async (sql: string, params?: unknown[]) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash' }; + if (sql.includes('shared_project_enrollments') && params?.[0] === 'repo-x') return { role: 'admin' }; + return null; + }); + const { daemon, browserA } = await setup(db); + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.SKILL_QUERY, requestId: 'alias-1', projectId: 'repo-x' })); + await flush(); + + const forwarded = daemon.sentJson().find((msg) => msg.type === MEMORY_WS.SKILL_QUERY) as Record | undefined; + const ctx = forwarded?.[MEMORY_MANAGEMENT_CONTEXT_FIELD] as Record | undefined; + expect(ctx?.role).toBe('user'); + expect((ctx?.boundProjects as Array> | undefined)?.[0]?.canonicalRepoId).toBeUndefined(); + }); + + it('does not forward unverified canonical project hints as authorized bindings', async () => { + const db = makeDb(async (sql: string) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash' }; + return null; + }); + const { daemon, browserA } = await setup(db); + browserA.emit('message', JSON.stringify({ + type: MEMORY_WS.SEARCH, + requestId: 'unauthorized-project', + canonicalRepoId: 'github.com/acme/private', + projectDir: '/tmp/acme-private', + repo: 'github.com/acme/private', + })); + await flush(); + + const forwarded = daemon.sentJson().find((msg) => msg.type === MEMORY_WS.SEARCH) as Record | undefined; + const ctx = forwarded?.[MEMORY_MANAGEMENT_CONTEXT_FIELD] as Record | undefined; + expect(ctx?.role).toBe('user'); + expect(ctx?.boundProjects).toEqual([]); + }); + + it('forwards active enrolled canonical projects with server-derived workspace/org bindings', async () => { + const db = makeDb(async (sql: string, params?: unknown[]) => { + if (sql.includes('token_hash')) return { token_hash: 'valid-hash' }; + if (sql.includes('shared_project_enrollments') && params?.[0] === 'github.com/acme/repo' && params?.[1] === 'user-a') { + return { role: 'member', workspace_id: 'workspace-1', enterprise_id: 'team-1' }; + } + return null; + }); + const { daemon, browserA } = await setup(db); + browserA.emit('message', JSON.stringify({ + type: MEMORY_WS.SEARCH, + requestId: 'authorized-project', + canonicalRepoId: 'github.com/acme/repo', + projectDir: '/work/repo', + repo: 'github.com/acme/repo', + })); + await flush(); + + const forwarded = daemon.sentJson().find((msg) => msg.type === MEMORY_WS.SEARCH) as Record | undefined; + const ctx = forwarded?.[MEMORY_MANAGEMENT_CONTEXT_FIELD] as Record | undefined; + expect(ctx?.role).toBe('user'); + expect(ctx?.boundProjects).toEqual([{ + canonicalRepoId: 'github.com/acme/repo', + projectDir: '/work/repo', + workspaceId: 'workspace-1', + orgId: 'team-1', + }]); + }); + + it('cleans up and single-casts an error if management context construction fails', async () => { + const { bridge, daemon, browserA, browserB } = await setup(); + vi.spyOn(bridge as unknown as { withMemoryManagementContext: (...args: unknown[]) => Promise> }, 'withMemoryManagementContext') + .mockRejectedValueOnce(new Error('context unavailable')); + + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.SKILL_QUERY, requestId: 'ctx-fail-1', canonicalRepoId: 'github.com/acme/repo' })); + await flush(); + + expect(daemon.sentJson().some((msg) => msg.requestId === 'ctx-fail-1')).toBe(false); + expect(browserA.sentJson().some((msg) => ( + msg.type === 'error' + && msg.code === 'context_injection_failed' + && msg.requestId === 'ctx-fail-1' + && msg.originalType === MEMORY_WS.SKILL_QUERY + ))).toBe(true); + expect(browserB.sentJson().some((msg) => msg.requestId === 'ctx-fail-1')).toBe(false); + + browserA.emit('message', JSON.stringify({ type: MEMORY_WS.PREF_QUERY, requestId: 'ctx-fail-1' })); + await flush(); + expect(daemon.sentJson().some((msg) => msg.type === MEMORY_WS.PREF_QUERY && msg.requestId === 'ctx-fail-1')).toBe(true); + }); + + it('drops unrouted memory management responses instead of broadcasting them', async () => { + const { daemon, browserA, browserB } = await setup(); + daemon.emit('message', JSON.stringify({ type: MEMORY_WS.PREF_RESPONSE, requestId: 'missing', records: [{ text: 'secret' }] })); + await flush(); + + expect(browserA.sentJson().some((msg) => msg.type === MEMORY_WS.PREF_RESPONSE)).toBe(false); + expect(browserB.sentJson().some((msg) => msg.type === MEMORY_WS.PREF_RESPONSE)).toBe(false); + }); +}); diff --git a/server/test/bridge.test.ts b/server/test/bridge.test.ts index 9ade4f216..cc7c4d459 100644 --- a/server/test/bridge.test.ts +++ b/server/test/bridge.test.ts @@ -1,6 +1,10 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { EventEmitter } from 'node:events'; import { WsBridge } from '../src/ws/bridge.js'; +import { + markDaemonUpgradeTargetVersionPublishedForTest, + resetDaemonUpgradePublicationGateForTest, +} from '../src/ws/daemon-upgrade-publication-gate.js'; import * as dbQueries from '../src/db/queries.js'; import { PUSH_TIMELINE_EVENT_MAX_AGE_MS, TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; @@ -85,10 +89,14 @@ describe('WsBridge', () => { beforeEach(() => { serverId = `test-${Math.random().toString(36).slice(2)}`; + resetDaemonUpgradePublicationGateForTest(); + markDaemonUpgradeTargetVersionPublishedForTest('2026.4.905-dev.877'); + markDaemonUpgradeTargetVersionPublishedForTest('2026.4.905'); }); afterEach(() => { WsBridge.getAll().clear(); + resetDaemonUpgradePublicationGateForTest(); vi.clearAllMocks(); }); @@ -197,6 +205,63 @@ describe('WsBridge', () => { expect(ws.sentStrings.some((msg) => msg.includes('"type":"daemon.upgrade"') && msg.includes('2026.4.905'))).toBe(true); }); + + it('rate-limits auto daemon.upgrade to at most once every 15 minutes', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.4.905-dev.877'; + + const bridge = WsBridge.get(serverId); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, makeDb('valid-hash'), {} as never); + + firstWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token', daemonVersion: '2026.4.904-dev.100' })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5000); + await flushAsync(); + + expect(firstWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + + const secondWs = new MockWs(); + bridge.handleDaemonConnection(secondWs as never, makeDb('valid-hash'), {} as never); + secondWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token', daemonVersion: '2026.4.904-dev.100' })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5000); + await flushAsync(); + + expect(secondWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + await vi.advanceTimersByTimeAsync(15 * 60 * 1000); + + const thirdWs = new MockWs(); + bridge.handleDaemonConnection(thirdWs as never, makeDb('valid-hash'), {} as never); + thirdWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token', daemonVersion: '2026.4.904-dev.100' })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5000); + await flushAsync(); + + expect(thirdWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + }); + + it('does not send a stale scheduled daemon.upgrade after the daemon socket is replaced', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.4.905-dev.877'; + + const bridge = WsBridge.get(serverId); + const staleWs = new MockWs(); + bridge.handleDaemonConnection(staleWs as never, makeDb('valid-hash'), {} as never); + staleWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token', daemonVersion: '2026.4.904-dev.100' })); + await flushAsync(); + + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, makeDb('valid-hash'), {} as never); + replacementWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token', daemonVersion: '2026.4.904-dev.100' })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5000); + await flushAsync(); + + expect(staleWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + }); }); describe('message relay daemon→browser', () => { @@ -315,6 +380,24 @@ describe('WsBridge', () => { expect(daemonWs.sentStrings.some((s) => s.includes('admin.shutdown'))).toBe(true); }); + it('rejects browser raw daemon.upgrade commands', async () => { + const { daemonWs, browserWs } = await setupBridge(); + browserWs.emit('message', JSON.stringify({ type: 'daemon.upgrade', targetVersion: '2026.4.905-dev.877', requestId: 'r1' })); + await flushAsync(); + + expect(daemonWs.sentStrings.some((s) => s.includes('daemon.upgrade'))).toBe(false); + expect(browserWs.sentStrings.some((s) => s.includes('server_only_command') && s.includes('r1'))).toBe(true); + }); + + it('rejects browser raw server.delete commands', async () => { + const { daemonWs, browserWs } = await setupBridge(); + browserWs.emit('message', JSON.stringify({ type: 'server.delete', requestId: 'r2' })); + await flushAsync(); + + expect(daemonWs.sentStrings.some((s) => s.includes('server.delete'))).toBe(false); + expect(browserWs.sentStrings.some((s) => s.includes('server_only_command') && s.includes('r2'))).toBe(true); + }); + it('drops oversized payload', async () => { const { daemonWs, browserWs } = await setupBridge(); browserWs.emit('message', JSON.stringify({ type: 'session.send', text: 'x'.repeat(70000) })); @@ -368,6 +451,35 @@ describe('WsBridge', () => { expect(daemonWs.sentStrings.some((s) => s.includes('get_sessions'))).toBe(true); }); + + it('keeps offline daemon.upgrade out of the ordinary queue and flushes it once on auth', async () => { + const bridge = WsBridge.get(serverId); + + bridge.sendToDaemon(JSON.stringify({ type: 'daemon.upgrade', targetVersion: '2026.4.905-dev.877' })); + bridge.sendToDaemon(JSON.stringify({ type: 'daemon.upgrade', targetVersion: '2026.4.905-dev.877' })); + + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + daemonWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 't' })); + await flushAsync(); + + const upgradeMessages = daemonWs.sentStrings.filter((s) => s.includes('"type":"daemon.upgrade"')); + expect(upgradeMessages).toHaveLength(1); + expect(upgradeMessages[0]).toContain('2026.4.905-dev.877'); + }); + + it('drops daemon.upgrade with an invalid targetVersion before it reaches the daemon', async () => { + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + daemonWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 't' })); + await flushAsync(); + + bridge.sendToDaemon(JSON.stringify({ type: 'daemon.upgrade', targetVersion: '2026.4.905-dev.877;touch /tmp/pwn' })); + await flushAsync(); + + expect(daemonWs.sentStrings.some((s) => s.includes('daemon.upgrade'))).toBe(false); + }); }); // ── Helpers shared by subscription / binary tests ───────────────────────── diff --git a/server/test/daemon-upgrade-coordinator.test.ts b/server/test/daemon-upgrade-coordinator.test.ts new file mode 100644 index 000000000..d6a2a73c4 --- /dev/null +++ b/server/test/daemon-upgrade-coordinator.test.ts @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { DAEMON_COMMAND_TYPES } from '../../shared/daemon-command-types.js'; +import { DAEMON_UPGRADE_DELIVERY_STATUS } from '../../shared/daemon-upgrade.js'; +import { DaemonUpgradeCoordinator } from '../src/ws/daemon-upgrade-coordinator.js'; +import { + DaemonUpgradePublicationGate, + daemonUpgradeTarballUrl, + type DaemonUpgradePublicationProbeResult, +} from '../src/ws/daemon-upgrade-publication-gate.js'; + +async function flushPromises(): Promise { + for (let i = 0; i < 5; i++) await Promise.resolve(); +} + +describe('DaemonUpgradeCoordinator npm publication gate', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('builds the exact npm tarball URL used for target-version publication probes', () => { + expect(daemonUpgradeTarballUrl('2026.4.905-dev.877')).toBe( + 'https://registry.npmjs.org/imcodes/-/imcodes-2026.4.905-dev.877.tgz', + ); + }); + + it('does not send daemon.upgrade until the target tarball is published, then caches the success', async () => { + vi.useFakeTimers(); + const targetVersion = '2026.4.905-dev.877'; + const probe = vi.fn<[], Promise>() + .mockResolvedValueOnce({ status: 'missing', statusCode: 404 }) + .mockResolvedValueOnce({ status: 'available', statusCode: 200 }); + const gate = new DaemonUpgradePublicationGate({ + probe: async () => probe(), + retryDelaysMs: [100], + }); + const coordinator = new DaemonUpgradeCoordinator(gate); + const sent: Record[] = []; + + const result = coordinator.request({ + targetVersion, + source: 'auto', + isDaemonReady: () => true, + isStillCurrent: () => true, + send: (message) => sent.push(message), + now: 0, + }); + + expect(result).toMatchObject({ + ok: true, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_PUBLICATION, + }); + expect(sent).toEqual([]); + expect(probe).toHaveBeenCalledTimes(1); + + await flushPromises(); + expect(sent).toEqual([]); + + await vi.advanceTimersByTimeAsync(100); + await flushPromises(); + expect(probe).toHaveBeenCalledTimes(2); + expect(sent).toEqual([]); + + await vi.advanceTimersByTimeAsync(5_000); + await flushPromises(); + expect(sent).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: expect.any(String), + targetVersion, + }]); + + const nextCoordinator = new DaemonUpgradeCoordinator(gate); + const nextSent: Record[] = []; + const nextResult = nextCoordinator.request({ + targetVersion, + source: 'manual', + isDaemonReady: () => true, + send: (message) => nextSent.push(message), + }); + + expect(nextResult.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + expect(nextSent).toHaveLength(1); + expect(probe).toHaveBeenCalledTimes(2); + }); + + it('coalesces repeated requests for an unpublished target into one in-flight HEAD probe', async () => { + let resolveProbe: ((result: DaemonUpgradePublicationProbeResult) => void) | null = null; + const probe = vi.fn(() => new Promise((resolve) => { + resolveProbe = resolve; + })); + const gate = new DaemonUpgradePublicationGate({ + probe: async () => probe(), + retryDelaysMs: [100], + }); + const coordinator = new DaemonUpgradeCoordinator(gate); + const targetVersion = '2026.4.906-dev.1'; + const sent: Record[] = []; + const request = { + targetVersion, + source: 'manual' as const, + isDaemonReady: () => true, + send: (message: Record) => sent.push(message), + }; + + coordinator.request(request); + coordinator.request(request); + + expect(probe).toHaveBeenCalledTimes(1); + expect(sent).toEqual([]); + + resolveProbe?.({ status: 'available', statusCode: 200 }); + await flushPromises(); + + expect(sent).toHaveLength(1); + }); +}); + diff --git a/server/test/memory-post11-migration.test.ts b/server/test/memory-post11-migration.test.ts new file mode 100644 index 000000000..264ec54e7 --- /dev/null +++ b/server/test/memory-post11-migration.test.ts @@ -0,0 +1,34 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; + +describe('post-1.1 memory migration coverage', () => { + const migration = readFileSync(new URL('../src/db/migrations/044_memory_scope_search_citations_org.sql', import.meta.url), 'utf8').toLowerCase(); + const hardeningMigration = readFileSync(new URL('../src/db/migrations/045_memory_post11_hardening.sql', import.meta.url), 'utf8').toLowerCase(); + + it('adds nullable fingerprint/origin parity columns for backfillable shared storage', () => { + expect(migration).toContain('add column if not exists summary_fingerprint text'); + expect(migration).toContain('add column if not exists origin text'); + expect(migration).toContain('idx_shared_context_projections_fingerprint'); + expect(migration).toContain('shared_context_projections_origin_check'); + }); + + it('creates server namespace/observation/audit tables matching daemon post-foundations schema', () => { + expect(migration).toContain('create table if not exists memory_context_namespaces'); + expect(migration).toContain('create table if not exists memory_context_observations'); + expect(migration).toContain('create table if not exists memory_observation_promotion_audit'); + expect(migration).toContain('uq_memory_context_observations_idempotency'); + expect(migration).toContain("action in ('web_ui_promote', 'cli_mem_promote', 'admin_api_promote')"); + }); + + it('hardens post-1.1 owner-private contracts and persistent citation drift markers', () => { + expect(hardeningMigration).toContain('delete from shared_context_records where scope ='); + expect(hardeningMigration).toContain('delete from shared_context_projections where scope ='); + expect(hardeningMigration).toContain('update shared_context_projections'); + expect(hardeningMigration).toContain('content_hash'); + expect(hardeningMigration).toContain('owner_private_memories_kind_check'); + expect(hardeningMigration).toContain('owner_private_memories_origin_check'); + expect(hardeningMigration).toContain('owner_private_memories_size_check'); + expect(hardeningMigration).toContain('shared_context_records_scope_no_user_private'); + expect(hardeningMigration).toContain('shared_context_projections_personal_identity_check'); + }); +}); diff --git a/server/test/memory-scope-authorization.test.ts b/server/test/memory-scope-authorization.test.ts new file mode 100644 index 000000000..17e42694d --- /dev/null +++ b/server/test/memory-scope-authorization.test.ts @@ -0,0 +1,426 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { Hono } from 'hono'; +import type { Env } from '../src/env.js'; +import type { Database } from '../src/db/client.js'; +import { sameShapeMemoryLookupEnvelope } from '../src/memory/scope-policy.js'; +import { computeProjectionContentHash, resetCitationCountRateLimiterForTests } from '../src/memory/citation.js'; + +vi.mock('../src/security/authorization.js', () => ({ + requireAuth: () => async (c: { req: { header: (name: string) => string | undefined }; set: (key: string, value: string) => void }, next: () => Promise) => { + c.set('userId', c.req.header('x-test-user') ?? 'user-member'); + c.set('role', 'member'); + await next(); + }, + resolveServerRole: vi.fn().mockResolvedValue('owner'), +})); + +vi.mock('../src/security/audit.js', () => ({ + logAudit: vi.fn().mockResolvedValue(undefined), +})); + +const randomHexMock = vi.hoisted(() => vi.fn()); + +vi.mock('../src/security/crypto.js', async (importOriginal) => { + const real = await importOriginal(); + return { ...real, randomHex: randomHexMock }; +}); + +function makeEnv(db: Database): Env { + return { + DB: db, + JWT_SIGNING_KEY: 'test-signing-key-32chars-padding!!', + BOT_ENCRYPTION_KEY: 'abcdef0123456789'.repeat(2), + SERVER_URL: 'https://app.im.codes', + ALLOWED_ORIGINS: '', + TRUSTED_PROXIES: '', + BIND_HOST: '127.0.0.1', + PORT: '3000', + NODE_ENV: 'test', + GITHUB_CLIENT_ID: '', + GITHUB_CLIENT_SECRET: '', + DATABASE_URL: '', + } as Env; +} + +function normalize(sql: string): string { + return sql.toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function makeMockDb() { + const executeLog: Array<{ sql: string; params: unknown[] }> = []; + const queryLog: Array<{ sql: string; params: unknown[] }> = []; + const projections = new Map; + content_hash?: string | null; + }>([ + ['shared-1', { + id: 'shared-1', + scope: 'org_shared', + enterprise_id: 'ent-1', + user_id: null, + project_id: 'github.com/acme/repo', + summary: 'Authorized summary', + origin: 'chat_compacted', + content_json: { note: 'raw source must not be returned' }, + content_hash: null, + }], + ]); + const citations = new Map(); + const citeCounts = new Map(); + const db: Database = { + queryOne: async (sql: string, params: unknown[] = []) => { + const s = normalize(sql); + if (s.includes('from shared_context_projections') && s.includes('content_hash')) { + if (params[0] === 'missing') return null; + return (projections.get(params[0] as string) ?? null) as T | null; + } + if (s.includes('select role from team_members where team_id = $1 and user_id = $2')) { + return params[0] === 'ent-1' && params[1] === 'user-member' ? ({ role: 'member' } as T) : null; + } + if (s.includes('select id, projection_id, projection_content_hash, created_at from shared_context_citations where idempotency_key = $1 and user_id = $2')) { + const citation = [...citations.values()].find((entry) => entry.idempotency_key === params[0] && entry.user_id === params[1]); + return citation ? ({ + id: citation.id, + projection_id: citation.projection_id, + projection_content_hash: citation.projection_content_hash, + created_at: citation.created_at, + } as T) : null; + } + if (s.includes('select id, projection_id, projection_content_hash, created_at from shared_context_citations where id = $1 and user_id = $2')) { + const citation = citations.get(params[0] as string); + if (!citation || citation.user_id !== params[1]) return null; + return { + id: citation.id, + projection_id: citation.projection_id, + projection_content_hash: citation.projection_content_hash, + created_at: citation.created_at, + } as T; + } + return null; + }, + query: async (sql: string, params: unknown[] = []) => { + queryLog.push({ sql, params }); + const s = normalize(sql); + if (s.includes('from shared_context_projections p') && s.includes('shared_context_projection_cite_counts')) { + const userId = params[3]; + if (userId !== 'user-member') return [] as T[]; + return [...projections.values()] + .filter((projection) => projection.scope !== 'personal' || projection.user_id === userId) + .filter((projection) => projection.scope === 'personal' || projection.enterprise_id === 'ent-1') + .map((projection) => ({ + id: projection.id, + scope: projection.scope, + project_id: projection.project_id, + projection_class: 'durable_memory_candidate', + summary: projection.summary, + origin: projection.origin, + updated_at: projection.id === 'shared-1' ? 10 : 1, + hit_count: 0, + cite_count: citeCounts.get(projection.id) ?? 0, + })) as T[]; + } + if (s.includes('from owner_private_memories')) return [] as T[]; + return [] as T[]; + }, + execute: async (sql: string, params: unknown[] = []) => { + executeLog.push({ sql, params }); + const s = normalize(sql); + if (s.includes('insert into shared_context_citations')) { + const idempotencyKey = params[4] as string; + if ([...citations.values()].some((entry) => entry.idempotency_key === idempotencyKey)) { + return { changes: 0 }; + } + citations.set(params[0] as string, { + id: params[0] as string, + projection_id: params[1] as string, + user_id: params[2] as string, + citing_message_id: params[3] as string, + idempotency_key: idempotencyKey, + projection_content_hash: params[5] as string, + created_at: params[6] as number, + }); + return { changes: 1 }; + } + if (s.includes('insert into shared_context_projection_cite_counts')) { + const projectionId = params[0] as string; + citeCounts.set(projectionId, (citeCounts.get(projectionId) ?? 0) + 1); + return { changes: 1 }; + } + return { changes: 1 }; + }, + exec: async () => {}, + close: async () => {}, + } as Database; + return { db, executeLog, queryLog, projections, citations, citeCounts }; +} + +async function buildApp(db: Database) { + const { sharedContextRoutes } = await import('../src/routes/shared-context.js'); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/shared-context', sharedContextRoutes); + return { app, env: makeEnv(db) }; +} + +describe('memory scope authorization and same-shape citation lookup', () => { + beforeEach(() => { + let randomCounter = 0; + randomHexMock.mockImplementation(() => `citation-id-${++randomCounter}`); + process.env.IMCODES_MEM_FEATURE_QUICK_SEARCH = 'true'; + process.env.IMCODES_MEM_FEATURE_CITATION = 'true'; + process.env.IMCODES_MEM_FEATURE_CITE_COUNT = 'true'; + }); + + afterEach(() => { + delete process.env.IMCODES_MEM_FEATURE_QUICK_SEARCH; + delete process.env.IMCODES_MEM_FEATURE_CITATION; + delete process.env.IMCODES_MEM_FEATURE_CITE_COUNT; + delete process.env.IMCODES_MEM_FEATURE_CITE_DRIFT_BADGE; + delete process.env.IMCODES_MEM_CITATION_COUNT_RATE_LIMIT; + delete process.env.IMCODES_MEM_CITATION_COUNT_RATE_LIMIT_WINDOW_MS; + resetCitationCountRateLimiterForTests(); + randomHexMock.mockReset(); + }); + + it('expands quick search through authorized scopes without raw source leakage', async () => { + const { db, queryLog } = makeMockDb(); + const { app, env } = await buildApp(db); + const res = await app.request('/api/shared-context/memory/search', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ query: 'summary', scope: 'all_authorized', limit: 5 }), + }, env); + + expect(res.status).toBe(200); + const json = await res.json() as { results: Array> }; + expect(json.results).toEqual([ + expect.objectContaining({ id: 'shared-1', scope: 'org_shared', preview: 'Authorized summary', origin: 'chat_compacted' }), + ]); + expect(JSON.stringify(json)).not.toContain('raw source'); + expect(JSON.stringify(json)).not.toContain('ent-1'); + const searchSql = queryLog.map((entry) => normalize(entry.sql)).find((entry) => entry.includes('from shared_context_projections p')); + expect(searchSql).toContain('exists ( select 1 from team_members'); + expect(searchSql).toContain("p.scope <> 'personal'"); + expect(searchSql).not.toContain("p.scope in ('project_shared', 'workspace_shared', 'org_shared')"); + expect(searchSql).toContain('order by (p.updated_at + case when $7::boolean then least(coalesce(cc.cite_count, 0), 100) else 0 end) desc'); + }); + + it('does not query owner-private memories from generic search when user-private sync is disabled', async () => { + process.env.IMCODES_MEM_FEATURE_USER_PRIVATE_SYNC = 'false'; + const { db, queryLog } = makeMockDb(); + const { app, env } = await buildApp(db); + + const res = await app.request('/api/shared-context/memory/search', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ query: 'summary', scope: 'all_authorized', limit: 5 }), + }, env); + + expect(res.status).toBe(200); + expect(queryLog.some((entry) => normalize(entry.sql).includes('from owner_private_memories'))).toBe(false); + expect(queryLog.some((entry) => normalize(entry.sql).includes('from shared_context_projections p'))).toBe(true); + }); + + it('returns identical envelopes for missing, unauthorized, and disabled citation attempts', async () => { + const { db, executeLog } = makeMockDb(); + const { app, env } = await buildApp(db); + + const missing = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'missing', citingMessageId: 'msg-1' }), + }, env); + const unauthorized = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-other' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-1' }), + }, env); + process.env.IMCODES_MEM_FEATURE_CITATION = 'false'; + const disabled = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-1' }), + }, env); + + expect(missing.status).toBe(404); + expect(unauthorized.status).toBe(404); + expect(disabled.status).toBe(404); + expect(await missing.json()).toEqual(sameShapeMemoryLookupEnvelope()); + expect(await unauthorized.json()).toEqual(sameShapeMemoryLookupEnvelope()); + expect(await disabled.json()).toEqual(sameShapeMemoryLookupEnvelope()); + expect(executeLog).toEqual([]); + }); + + it('increments cite count once per authoritative idempotency key', async () => { + const { db, executeLog, citeCounts } = makeMockDb(); + const { app, env } = await buildApp(db); + const first = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-2' }), + }, env); + const replay = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-2' }), + }, env); + const differentMessage = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-3' }), + }, env); + + expect(first.status).toBe(201); + expect(await first.json()).toMatchObject({ + ok: true, + deduped: false, + citation: { id: 'citation-id-1', projectionId: 'shared-1', drift: false }, + }); + expect(replay.status).toBe(200); + expect(await replay.json()).toMatchObject({ + ok: true, + deduped: true, + citation: { id: 'citation-id-1', projectionId: 'shared-1', drift: false }, + }); + expect(differentMessage.status).toBe(201); + expect(executeLog.some((entry) => normalize(entry.sql).includes('insert into shared_context_citations'))).toBe(true); + expect(executeLog.some((entry) => normalize(entry.sql).includes('insert into shared_context_projection_cite_counts'))).toBe(true); + expect(citeCounts.get('shared-1')).toBe(2); + }); + + it('rate-limits cite-count pumping while still accepting authorized citations', async () => { + process.env.IMCODES_MEM_CITATION_COUNT_RATE_LIMIT = '1'; + const { db, citeCounts } = makeMockDb(); + const { app, env } = await buildApp(db); + + const first = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-rate-1' }), + }, env); + const second = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-rate-2' }), + }, env); + + expect(first.status).toBe(201); + expect(second.status).toBe(201); + expect(citeCounts.get('shared-1')).toBe(1); + }); + + it('dedupes concurrent citation replays before the hot-row count increment', async () => { + const { db, citeCounts } = makeMockDb(); + const { app, env } = await buildApp(db); + + const responses = await Promise.all(Array.from({ length: 8 }, () => app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-concurrent' }), + }, env))); + + expect(responses.map((response) => response.status).sort()).toEqual([200, 200, 200, 200, 200, 200, 200, 201]); + expect(citeCounts.get('shared-1')).toBe(1); + }); + + it('does not increment cite count when cite-count is disabled', async () => { + process.env.IMCODES_MEM_FEATURE_CITE_COUNT = 'false'; + const { db, citeCounts } = makeMockDb(); + const { app, env } = await buildApp(db); + + const res = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-no-count' }), + }, env); + + expect(res.status).toBe(201); + expect(citeCounts.get('shared-1')).toBeUndefined(); + }); + + it('reports drift only for authorized citation lookup when drift badge is enabled', async () => { + process.env.IMCODES_MEM_FEATURE_CITE_DRIFT_BADGE = 'true'; + const { db, projections } = makeMockDb(); + const { app, env } = await buildApp(db); + + const created = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-drift' }), + }, env); + expect(created.status).toBe(201); + expect(await created.json()).toMatchObject({ citation: { drift: false } }); + + projections.get('shared-1')!.summary = 'Changed authorized summary'; + projections.get('shared-1')!.content_hash = computeProjectionContentHash({ + summary: 'Changed authorized summary', + content: projections.get('shared-1')!.content_json, + }); + const lookup = await app.request('/api/shared-context/memory/citations/citation-id-1', { + method: 'GET', + headers: { 'x-test-user': 'user-member' }, + }, env); + const unauthorized = await app.request('/api/shared-context/memory/citations/citation-id-1', { + method: 'GET', + headers: { 'x-test-user': 'user-other' }, + }, env); + + expect(lookup.status).toBe(200); + expect(await lookup.json()).toMatchObject({ citation: { id: 'citation-id-1', projectionId: 'shared-1', drift: true } }); + expect(unauthorized.status).toBe(404); + expect(await unauthorized.json()).toEqual(sameShapeMemoryLookupEnvelope()); + }); + + it('keeps citation lookup envelopes identical for missing, unauthorized, and disabled states', async () => { + const { db } = makeMockDb(); + const { app, env } = await buildApp(db); + + const created = await app.request('/api/shared-context/memory/citations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-test-user': 'user-member' }, + body: JSON.stringify({ projectionId: 'shared-1', citingMessageId: 'msg-envelope' }), + }, env); + expect(created.status).toBe(201); + + const missing = await app.request('/api/shared-context/memory/citations/missing-citation', { + method: 'GET', + headers: { 'x-test-user': 'user-member' }, + }, env); + const unauthorized = await app.request('/api/shared-context/memory/citations/citation-id-1', { + method: 'GET', + headers: { 'x-test-user': 'user-other' }, + }, env); + process.env.IMCODES_MEM_FEATURE_CITATION = 'false'; + const disabled = await app.request('/api/shared-context/memory/citations/citation-id-1', { + method: 'GET', + headers: { 'x-test-user': 'user-member' }, + }, env); + + const envelopes = [await missing.json(), await unauthorized.json(), await disabled.json()]; + expect(missing.status).toBe(404); + expect(unauthorized.status).toBe(404); + expect(disabled.status).toBe(404); + expect(envelopes).toEqual([ + sameShapeMemoryLookupEnvelope(), + sameShapeMemoryLookupEnvelope(), + sameShapeMemoryLookupEnvelope(), + ]); + for (const envelope of envelopes) { + expect(JSON.stringify(envelope)).not.toMatch(/drift|source|count|projectionId|enterprise|role/i); + } + }); +}); diff --git a/server/test/memory-scope-replication-check.test.ts b/server/test/memory-scope-replication-check.test.ts new file mode 100644 index 000000000..c511f5e43 --- /dev/null +++ b/server/test/memory-scope-replication-check.test.ts @@ -0,0 +1,248 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { Hono } from 'hono'; +import { sha256Hex } from '../src/security/crypto.js'; +import type { Env } from '../src/env.js'; +import type { Database } from '../src/db/client.js'; +import { serverRoutes } from '../src/routes/server.js'; +import { computeProjectionContentHash } from '../src/memory/citation.js'; +import { + MEMORY_FEATURE_CONFIG_PREF_KEY, + MEMORY_FEATURE_FLAGS_BY_NAME, +} from '../../shared/feature-flags.js'; + +function makeEnv(db: Database): Env { + return { + DB: db, + JWT_SIGNING_KEY: 'test-signing-key-32chars-padding!!', + BOT_ENCRYPTION_KEY: 'abcdef0123456789'.repeat(2), + SERVER_URL: 'https://app.im.codes', + ALLOWED_ORIGINS: '', + TRUSTED_PROXIES: '', + BIND_HOST: '127.0.0.1', + PORT: '3000', + NODE_ENV: 'test', + GITHUB_CLIENT_ID: '', + GITHUB_CLIENT_SECRET: '', + DATABASE_URL: '', + } as Env; +} + +function normalize(sql: string): string { + return sql.toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function makeMockDb(options: { userPrefs?: Record } = {}) { + const executeLog: Array<{ sql: string; params: unknown[] }> = []; + const tokenHash = sha256Hex('daemon-token'); + const db: Database = { + queryOne: async (sql: string, params: unknown[] = []) => { + const s = normalize(sql); + if (s.includes('from user_preferences')) { + const value = options.userPrefs?.[`${String(params[0])}:${String(params[1])}`]; + return value == null ? null : ({ value } as T); + } + if (s.includes('select id, user_id from servers where token_hash = $1 and id = $2')) { + return params[0] === tokenHash && params[1] === 'srv-1' + ? ({ id: 'srv-1', user_id: 'owner-1' } as T) + : null; + } + if (s.includes('select id, team_id, user_id from servers where token_hash = $1 and id = $2')) { + return params[0] === tokenHash && params[1] === 'srv-1' + ? ({ id: 'srv-1', team_id: 'ent-1', user_id: 'owner-1' } as T) + : null; + } + return null; + }, + query: async (sql: string, params: unknown[] = []) => { + const s = normalize(sql); + if (s.includes('from owner_private_memories')) { + return params[0] === 'owner-1' + ? ([{ id: 'mem-1', kind: 'preference', origin: 'user_note', text: 'Use pnpm', updated_at: 123 }] as T[]) + : [] as T[]; + } + return [] as T[]; + }, + execute: async (sql: string, params: unknown[] = []) => { + executeLog.push({ sql, params }); + return { changes: 1 }; + }, + exec: async () => {}, + close: async () => {}, + } as Database; + return { db, executeLog }; +} + +describe('user_private owner-only server replication', () => { + beforeEach(() => { + process.env.IMCODES_MEM_FEATURE_USER_PRIVATE_SYNC = 'true'; + }); + + afterEach(() => { + delete process.env.IMCODES_MEM_FEATURE_USER_PRIVATE_SYNC; + }); + + it('lets the user-global online feature config override the daemon env fallback', async () => { + const { db, executeLog } = makeMockDb({ + userPrefs: { + [`owner-1:${MEMORY_FEATURE_CONFIG_PREF_KEY}`]: JSON.stringify({ + [MEMORY_FEATURE_FLAGS_BY_NAME.userPrivateSync]: false, + }), + }, + }); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + const res = await app.request('/api/server/srv-1/shared-context/owner-private', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + namespace: { scope: 'user_private', userId: 'owner-1' }, + records: [{ kind: 'note', origin: 'user_note', fingerprint: 'fp-disabled', text: 'private' }], + }), + }, makeEnv(db)); + + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ ok: false, result: null, citation: null, error: 'not_found' }); + expect(executeLog).toEqual([]); + }); + + it('stores user_private records in the dedicated owner table, not shared projections', async () => { + const { db, executeLog } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + const res = await app.request('/api/server/srv-1/shared-context/owner-private', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + namespace: { scope: 'user_private', userId: 'owner-1' }, + records: [{ kind: 'preference', origin: 'user_note', fingerprint: 'fp-1', text: 'Use pnpm', content: { source: 'test' } }], + }), + }, makeEnv(db)); + + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ ok: true, memoryCount: 1 }); + expect(executeLog.some((entry) => normalize(entry.sql).includes('insert into owner_private_memories'))).toBe(true); + expect(executeLog.some((entry) => normalize(entry.sql).includes('shared_context_projections'))).toBe(false); + }); + + it('rejects missing or reserved origins on owner-private writes', async () => { + const { db, executeLog } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + for (const record of [ + { kind: 'note', fingerprint: 'fp-missing', text: 'missing origin' }, + { kind: 'note', origin: 'quick_search_cache', fingerprint: 'fp-reserved', text: 'reserved origin' }, + ]) { + const res = await app.request('/api/server/srv-1/shared-context/owner-private', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + namespace: { scope: 'user_private', userId: 'owner-1' }, + records: [record], + }), + }, makeEnv(db)); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'invalid_body' }); + } + expect(executeLog).toEqual([]); + }); + + it('bounds owner-private kind, text, content, and batch inputs before DB writes', async () => { + const { db, executeLog } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + for (const records of [ + [{ kind: 'unknown', origin: 'user_note', fingerprint: 'fp-kind', text: 'bad kind' }], + [{ kind: 'note', origin: 'user_note', fingerprint: 'fp-text', text: 'x'.repeat(32 * 1024 + 1) }], + [{ kind: 'note', origin: 'user_note', fingerprint: 'fp-content', text: 'content', content: { blob: 'x'.repeat(128 * 1024 + 1) } }], + Array.from({ length: 101 }, (_, index) => ({ kind: 'note', origin: 'user_note', fingerprint: `fp-${index}`, text: `note ${index}` })), + ]) { + const res = await app.request('/api/server/srv-1/shared-context/owner-private', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + namespace: { scope: 'user_private', userId: 'owner-1' }, + records, + }), + }, makeEnv(db)); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'invalid_body' }); + } + expect(executeLog).toEqual([]); + }); + + it('rejects namespace user mismatch with the same not-found lookup envelope', async () => { + const { db, executeLog } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + const res = await app.request('/api/server/srv-1/shared-context/owner-private', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + namespace: { scope: 'user_private', userId: 'other-user' }, + records: [{ kind: 'note', origin: 'user_note', fingerprint: 'fp-2', text: 'private' }], + }), + }, makeEnv(db)); + + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ ok: false, result: null, citation: null, error: 'not_found' }); + expect(executeLog).toEqual([]); + }); + + it('searches owner-private memory only for the daemon-authenticated owner', async () => { + const { db } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + const res = await app.request('/api/server/srv-1/shared-context/owner-private/search', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ query: 'pnpm', scope: 'owner_private' }), + }, makeEnv(db)); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + results: [{ id: 'mem-1', scope: 'user_private', kind: 'preference', origin: 'user_note', preview: 'Use pnpm', updatedAt: 123 }], + nextCursor: null, + }); + }); + + it('persists canonical content_hash on processed projection replication', async () => { + const { db, executeLog } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + const projection = { + id: 'projection-1', + namespace: { scope: 'personal', projectId: 'github.com/acme/repo', userId: 'owner-1' }, + class: 'recent_summary', + origin: 'chat_compacted', + sourceEventIds: ['evt-1'], + summary: 'Stable summary', + content: { b: 2, a: 1 }, + createdAt: 100, + updatedAt: 200, + }; + + const res = await app.request('/api/server/srv-1/shared-context/processed', { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + namespace: projection.namespace, + projections: [projection], + }), + }, makeEnv(db)); + + expect(res.status).toBe(200); + const insert = executeLog.find((entry) => normalize(entry.sql).includes('insert into shared_context_projections')); + expect(normalize(insert?.sql ?? '')).toContain('content_hash'); + expect(insert?.params[11]).toBe(computeProjectionContentHash({ + summary: projection.summary, + content: projection.content, + })); + }); +}); diff --git a/server/test/memory-search-auth.test.ts b/server/test/memory-search-auth.test.ts new file mode 100644 index 000000000..9651184bb --- /dev/null +++ b/server/test/memory-search-auth.test.ts @@ -0,0 +1,20 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; + +describe('memory search authorization source guard', () => { + it('keeps generic memory search gated from owner-private reads unless user-private sync is enabled', () => { + const source = readFileSync(new URL('../src/routes/shared-context.ts', import.meta.url), 'utf8'); + const routeStart = source.indexOf("sharedContextRoutes.post('/memory/search'"); + const routeEnd = source.indexOf('type CitationProjectionRow', routeStart); + expect(routeStart).toBeGreaterThanOrEqual(0); + expect(routeEnd).toBeGreaterThan(routeStart); + const route = source.slice(routeStart, routeEnd); + + expect(route).toContain('MEMORY_FEATURES.userPrivateSync'); + expect(route).toContain('includeOwnerPrivate: userPrivateSyncEnabled'); + expect(route).toContain('userPrivateSyncEnabled && scopes.includes'); + expect(route).toContain("p.scope <> 'personal'"); + expect(route).toContain('FROM owner_private_memories'); + expect(route.indexOf('userPrivateSyncEnabled && scopes.includes')).toBeLessThan(route.indexOf('FROM owner_private_memories')); + }); +}); diff --git a/server/test/p2p-config.integration.test.ts b/server/test/p2p-config.integration.test.ts index caa0c6186..ef63c9513 100644 --- a/server/test/p2p-config.integration.test.ts +++ b/server/test/p2p-config.integration.test.ts @@ -2,9 +2,9 @@ * E2E integration tests for P2P config per-session storage + legacy migration. * * Uses real PostgreSQL (testcontainers) + Hono app to verify: - * - Per-session config stored via /api/preferences/:key + * - Per-server/session config stored via /api/preferences/:key * - Legacy global key migrated to per-session key - * - Config isolated between sessions + * - Config isolated between sessions and servers * - Config shared across devices (same user, same API) */ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; @@ -13,6 +13,7 @@ import { runMigrations } from '../src/db/migrate.js'; import { buildApp } from '../src/index.js'; import { hashPassword, signJwt, randomHex } from '../src/security/crypto.js'; import type { Env } from '../src/env.js'; +import { P2P_SESSION_CONFIG_PREF_KEY, p2pSessionConfigPrefKey } from '../../shared/p2p-config-scope.js'; let db: Database; const JWT_KEY = 'test-jwt-key-p2p-config-integration-000'; @@ -79,7 +80,7 @@ describe('P2P config per-session storage', () => { }); it('saves and retrieves per-session P2P config', async () => { - const key = 'p2p_session_config:deck_proj_brain'; + const key = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-main'); const config = { sessions: { 'deck_sub_abc': { enabled: true, mode: 'audit' } }, rounds: 3 }; // Save @@ -102,8 +103,8 @@ describe('P2P config per-session storage', () => { }); it('isolates config between different sessions', async () => { - const key1 = 'p2p_session_config:deck_proj_brain'; - const key2 = 'p2p_session_config:deck_other_brain'; + const key1 = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-main'); + const key2 = p2pSessionConfigPrefKey('deck_other_brain', 'srv-main'); const config1 = { sessions: {}, rounds: 2 }; const config2 = { sessions: {}, rounds: 5 }; @@ -128,9 +129,38 @@ describe('P2P config per-session storage', () => { expect(d2.rounds).toBe(5); }); + it('isolates config between different servers with the same session name', async () => { + const key1 = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-one'); + const key2 = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-two'); + const config1 = { sessions: { deck_sub_one: { enabled: true, mode: 'audit' } }, rounds: 2 }; + const config2 = { sessions: { deck_sub_two: { enabled: true, mode: 'review' } }, rounds: 5 }; + + await app.request(`/api/preferences/${key1}`, { + method: 'PUT', + headers: csrfHeaders(token), + body: JSON.stringify({ value: JSON.stringify(config1) }), + }); + await app.request(`/api/preferences/${key2}`, { + method: 'PUT', + headers: csrfHeaders(token), + body: JSON.stringify({ value: JSON.stringify(config2) }), + }); + + const res1 = await app.request(`/api/preferences/${key1}`, { headers: csrfHeaders(token) }); + const res2 = await app.request(`/api/preferences/${key2}`, { headers: csrfHeaders(token) }); + + const d1 = JSON.parse((await res1.json() as { value: string }).value); + const d2 = JSON.parse((await res2.json() as { value: string }).value); + + expect(d1.sessions.deck_sub_one.mode).toBe('audit'); + expect(d1.rounds).toBe(2); + expect(d2.sessions.deck_sub_two.mode).toBe('review'); + expect(d2.rounds).toBe(5); + }); + it('legacy global key readable alongside per-session key', async () => { - const legacyKey = 'p2p_session_config'; - const sessionKey = 'p2p_session_config:deck_proj_brain'; + const legacyKey = P2P_SESSION_CONFIG_PREF_KEY; + const sessionKey = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-main'); const legacyConfig = { sessions: { 'deck_sub_old': { enabled: true, mode: 'review' } }, rounds: 1 }; // Save under legacy key (simulates old client) @@ -154,8 +184,8 @@ describe('P2P config per-session storage', () => { }); it('migration: writing per-session key after reading legacy key', async () => { - const legacyKey = 'p2p_session_config'; - const sessionKey = 'p2p_session_config:deck_proj_brain'; + const legacyKey = P2P_SESSION_CONFIG_PREF_KEY; + const sessionKey = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-main'); const config = { sessions: {}, rounds: 3, extraPrompt: 'be thorough' }; // Legacy save @@ -184,7 +214,7 @@ describe('P2P config per-session storage', () => { }); it('config accessible from different "devices" (same user token)', async () => { - const key = 'p2p_session_config:deck_proj_brain'; + const key = p2pSessionConfigPrefKey('deck_proj_brain', 'srv-main'); const config = { sessions: { 's1': { enabled: true, mode: 'brainstorm' } }, rounds: 2 }; // "Device 1" saves diff --git a/server/test/personal-cloud-memory.integration.test.ts b/server/test/personal-cloud-memory.integration.test.ts index 5b737eca8..f340497ed 100644 --- a/server/test/personal-cloud-memory.integration.test.ts +++ b/server/test/personal-cloud-memory.integration.test.ts @@ -265,6 +265,7 @@ describe('personal cloud memory — auth and data isolation', () => { id: randomHex(16), namespace: { scope: 'personal', projectId: 'my-repo' }, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds: ['e1', 'e2'], summary: 'Replicated from daemon', content: { trigger: 'idle' }, @@ -305,6 +306,7 @@ describe('personal cloud memory — auth and data isolation', () => { id: randomHex(16), namespace: { scope: 'personal', projectId: 'my-repo' }, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds: ['e1'], summary: 'Alice secret memory', content: {}, diff --git a/server/test/server-upgrade-route.test.ts b/server/test/server-upgrade-route.test.ts index 69fba7faf..8f8a7b56f 100644 --- a/server/test/server-upgrade-route.test.ts +++ b/server/test/server-upgrade-route.test.ts @@ -4,6 +4,7 @@ import type { Env } from '../src/env.js'; const mockGetServersByUserId = vi.fn(); const mockSendToDaemon = vi.fn(); +const mockRequestDaemonUpgrade = vi.fn(); vi.mock('../src/security/authorization.js', () => ({ requireAuth: () => async (c: { set: (key: string, value: string) => void }, next: () => Promise) => { @@ -25,6 +26,7 @@ vi.mock('../src/ws/bridge.js', () => ({ WsBridge: { get: () => ({ sendToDaemon: (...args: unknown[]) => mockSendToDaemon(...args), + requestDaemonUpgrade: (...args: unknown[]) => mockRequestDaemonUpgrade(...args), }), }, })); @@ -58,32 +60,123 @@ async function buildTestApp() { return app; } -describe('POST /api/server/:id/upgrade', () => { +describe('server routes', () => { beforeEach(() => { vi.clearAllMocks(); mockGetServersByUserId.mockResolvedValue([{ id: 'srv-1', name: 'Alpha' }]); + mockRequestDaemonUpgrade.mockReturnValue({ + ok: true, + upgradeId: 'upgrade-1', + targetVersion: 'latest', + deliveryStatus: 'sent', + }); delete process.env.APP_VERSION; }); - it('sends daemon.upgrade with the server app version as targetVersion', async () => { + it('GET /api/server returns persisted daemonVersion', async () => { + mockGetServersByUserId.mockResolvedValue([{ + id: 'srv-1', + name: 'Alpha', + status: 'online', + last_heartbeat_at: 123, + daemon_version: '2026.5.2047-dev.2025', + created_at: 99, + }]); + const app = await buildTestApp(); + + const res = await app.request('/api/server'); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + servers: [{ + id: 'srv-1', + name: 'Alpha', + status: 'online', + lastHeartbeatAt: 123, + daemonVersion: '2026.5.2047-dev.2025', + createdAt: 99, + }], + }); + }); + + it('requests daemon.upgrade with the server app version as targetVersion', async () => { process.env.APP_VERSION = '2026.4.905-dev.877'; + mockRequestDaemonUpgrade.mockReturnValue({ + ok: true, + upgradeId: 'upgrade-1', + targetVersion: '2026.4.905-dev.877', + deliveryStatus: 'sent', + }); const app = await buildTestApp(); const res = await app.request('/api/server/srv-1/upgrade', { method: 'POST' }); expect(res.status).toBe(200); - expect(mockSendToDaemon).toHaveBeenCalledWith(JSON.stringify({ - type: 'daemon.upgrade', + expect(await res.json()).toEqual({ + ok: true, + upgradeId: 'upgrade-1', + targetVersion: '2026.4.905-dev.877', + deliveryStatus: 'sent', + }); + expect(mockRequestDaemonUpgrade).toHaveBeenCalledWith({ targetVersion: '2026.4.905-dev.877', - })); + source: 'manual', + }); + expect(mockSendToDaemon).not.toHaveBeenCalled(); + }); + + it('requests latest only when APP_VERSION is unavailable', async () => { + const app = await buildTestApp(); + + const res = await app.request('/api/server/srv-1/upgrade', { method: 'POST' }); + + expect(res.status).toBe(200); + expect(mockRequestDaemonUpgrade).toHaveBeenCalledWith({ + targetVersion: undefined, + source: 'manual', + }); + }); + + it('returns 400 when the upgrade target is invalid', async () => { + process.env.APP_VERSION = '2026.4.905-dev.877;touch /tmp/pwn'; + mockRequestDaemonUpgrade.mockReturnValue({ + ok: false, + deliveryStatus: 'invalid_target', + reason: 'invalid_target_version', + }); + const app = await buildTestApp(); + + const res = await app.request('/api/server/srv-1/upgrade', { method: 'POST' }); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: 'invalid_target_version', + deliveryStatus: 'invalid_target', + }); }); - it('omits targetVersion only when APP_VERSION is unavailable', async () => { + it('surfaces npm publication gate state without pretending the upgrade was sent', async () => { + process.env.APP_VERSION = '2026.4.905-dev.877'; + mockRequestDaemonUpgrade.mockReturnValue({ + ok: true, + upgradeId: 'upgrade-1', + targetVersion: '2026.4.905-dev.877', + deliveryStatus: 'pending_publication', + nextAttemptAt: '2026-05-06T12:00:15.000Z', + reason: 'target_version_not_published', + }); const app = await buildTestApp(); const res = await app.request('/api/server/srv-1/upgrade', { method: 'POST' }); expect(res.status).toBe(200); - expect(mockSendToDaemon).toHaveBeenCalledWith(JSON.stringify({ type: 'daemon.upgrade' })); + expect(await res.json()).toEqual({ + ok: true, + upgradeId: 'upgrade-1', + targetVersion: '2026.4.905-dev.877', + deliveryStatus: 'pending_publication', + nextAttemptAt: '2026-05-06T12:00:15.000Z', + reason: 'target_version_not_published', + }); }); }); diff --git a/server/test/session-mgmt-routes.test.ts b/server/test/session-mgmt-routes.test.ts index ce8b5cd07..398caa871 100644 --- a/server/test/session-mgmt-routes.test.ts +++ b/server/test/session-mgmt-routes.test.ts @@ -227,6 +227,22 @@ describe('session-mgmt persistence routes', () => { }); }); + it('POST /session/cancel relays direct SDK cancel without /stop text', async () => { + const app = await buildApp(); + const res = await app.request('/api/server/srv-1/session/cancel', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sessionName: 'deck_proj_brain', commandId: 'cancel-1', text: '/stop' }), + }); + + expect(res.status).toBe(200); + expect(JSON.parse(String(sendToDaemonMock.mock.calls[0]?.[0]))).toEqual({ + type: DAEMON_COMMAND_TYPES.SESSION_CANCEL, + sessionName: 'deck_proj_brain', + commandId: 'cancel-1', + }); + }); + it('PATCH /sessions/:name/rename updates the project name and relays session.rename', async () => { const { updateProjectName } = await import('../src/db/queries.js'); const app = await buildApp(); diff --git a/server/test/shared-context-control-plane.test.ts b/server/test/shared-context-control-plane.test.ts index 5bfeeac9b..8f7876d4e 100644 --- a/server/test/shared-context-control-plane.test.ts +++ b/server/test/shared-context-control-plane.test.ts @@ -161,9 +161,16 @@ function makeMockDb() { const document = documents.get(version.document_id); return document ? ({ document_id: version.document_id, enterprise_id: document.enterprise_id } as T) : null; } - if (s.includes('select enterprise_id from shared_context_document_bindings where id = $1')) { + if ( + s.includes('select enterprise_id from shared_context_document_bindings where id = $1') + || s.includes('select enterprise_id, workspace_id, enrollment_id from shared_context_document_bindings where id = $1') + ) { const binding = bindings.get(params[0] as string); - return binding ? ({ enterprise_id: binding.enterprise_id } as T) : null; + return binding ? ({ + enterprise_id: binding.enterprise_id, + workspace_id: binding.workspace_id, + enrollment_id: binding.enrollment_id, + } as T) : null; } if (s.includes('select role from team_members where team_id = $1 and user_id = $2 and role in')) { const member = teamMembers.get(params[0] as string)?.get(params[1] as string); @@ -207,11 +214,12 @@ function makeMockDb() { status: entry.status, })) as T[]; } - if (s.includes('select id, kind, title from shared_context_documents where enterprise_id = $1')) { + if (s.includes('from shared_context_documents where enterprise_id = $1') && s.includes('select id, kind, title')) { return [...documents.values()].filter((entry) => entry.enterprise_id === params[0]).map((entry) => ({ id: entry.id, kind: entry.kind, title: entry.title, + created_by: entry.created_by, })) as T[]; } if (s.includes('from shared_context_projections where enterprise_id = $1') && s.includes('select id, scope, project_id, projection_class, source_event_ids_json, summary, content_json, updated_at')) { @@ -238,12 +246,12 @@ function makeMockDb() { status: 'active', })) as T[]; } - if (s.includes('select id, version_number, status from shared_context_document_versions where document_id = $1')) { + if (s.includes('from shared_context_document_versions where document_id = $1') && s.includes('select id, version_number, status')) { return [...versions.values()] .filter((entry) => entry.document_id === params[0]) - .map((entry) => ({ id: entry.id, version_number: entry.version_number, status: entry.status })) as T[]; + .map((entry) => ({ id: entry.id, version_number: entry.version_number, status: entry.status, created_by: entry.created_by })) as T[]; } - if (s.includes('select id, workspace_id, enrollment_id, document_id, version_id, binding_mode, applicability_repo_id, applicability_language, applicability_path_pattern, status from shared_context_document_bindings where enterprise_id = $1')) { + if (s.includes('from shared_context_document_bindings where enterprise_id = $1') && s.includes('select id, workspace_id, enrollment_id, document_id, version_id, binding_mode')) { return [...bindings.values()].filter((entry) => entry.enterprise_id === params[0]).map((entry) => ({ id: entry.id, workspace_id: entry.workspace_id, @@ -255,6 +263,7 @@ function makeMockDb() { applicability_language: entry.applicability_language, applicability_path_pattern: entry.applicability_path_pattern, status: entry.status, + created_by: entry.created_by, })) as T[]; } if (s.includes('from shared_context_document_bindings b join shared_context_document_versions v on v.id = b.version_id where b.enterprise_id = $1 and b.status = \'active\' and v.status = \'active\'')) { @@ -464,6 +473,7 @@ describe('shared-agent-context server control plane', () => { beforeEach(async () => { vi.clearAllMocks(); + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'true'; mockDb = makeMockDb(); app = await buildTestApp(makeEnv(mockDb.db)); }); @@ -744,6 +754,59 @@ describe('shared-agent-context server control plane', () => { expect(await res.json()).toEqual({ enterpriseId: 'team-1', bindings: [] }); }); + it('gates org-wide authored standards behind the explicit feature flag without affecting workspace bindings', async () => { + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'false'; + + const docRes = await app.request(...req('/api/shared-context/enterprises/team-1/documents', 'POST', { + kind: 'coding_standard', + title: 'Enterprise rules', + }, 'user-owner')); + const document = await docRes.json() as { id: string }; + const versionRes = await app.request(...req(`/api/shared-context/documents/${document.id}/versions`, 'POST', { + contentMd: 'Org-wide rule', + }, 'user-owner')); + const version = await versionRes.json() as { id: string }; + await app.request(...req(`/api/shared-context/document-versions/${version.id}/activate`, 'POST', {}, 'user-owner')); + + let res = await app.request(...req('/api/shared-context/enterprises/team-1/document-bindings', 'POST', { + documentId: document.id, + versionId: version.id, + mode: 'required', + }, 'user-owner')); + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ ok: false, result: null, citation: null, error: 'not_found' }); + + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'true'; + res = await app.request(...req('/api/shared-context/enterprises/team-1/document-bindings', 'POST', { + documentId: document.id, + versionId: version.id, + mode: 'required', + applicabilityRepoId: 'github.com/acme/repo', + }, 'user-owner')); + expect(res.status).toBe(201); + const binding = await res.json() as { id: string; workspaceId: string | null; enrollmentId: string | null }; + expect(binding.workspaceId).toBeNull(); + expect(binding.enrollmentId).toBeNull(); + + res = await app.request(...req('/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo', 'GET', undefined, 'user-member')); + expect(await res.json()).toEqual({ + enterpriseId: 'team-1', + bindings: [ + expect.objectContaining({ + bindingId: binding.id, + documentVersionId: version.id, + scope: 'org_shared', + mode: 'required', + content: 'Org-wide rule', + }), + ], + }); + + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'false'; + res = await app.request(...req('/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo', 'GET', undefined, 'user-member')); + expect(await res.json()).toEqual({ enterpriseId: 'team-1', bindings: [] }); + }); + it('requires explicit migration reason for host-change aliases and rejects unrelated repo aliases', async () => { let res = await app.request(...req('/api/shared-context/enterprises/team-1/repository-aliases', 'POST', { canonicalRepoId: 'github.com/acme/repo', @@ -869,6 +932,24 @@ describe('shared-agent-context server control plane', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo-2', + displayName: 'github.com/acme/repo-2', + totalRecords: 1, + recentSummaryCount: 1, + durableCandidateCount: 0, + updatedAt: 1700000000500, + }, + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 1, + recentSummaryCount: 1, + durableCandidateCount: 0, + updatedAt: 1700000000000, + }, + ], }); }); @@ -981,6 +1062,16 @@ describe('shared-agent-context server control plane', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 2, + recentSummaryCount: 1, + durableCandidateCount: 1, + updatedAt: 1700000000100, + }, + ], }); }); }); diff --git a/server/test/shared-context-org-authored-context.test.ts b/server/test/shared-context-org-authored-context.test.ts new file mode 100644 index 000000000..6f40fc3da --- /dev/null +++ b/server/test/shared-context-org-authored-context.test.ts @@ -0,0 +1,368 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { Hono } from 'hono'; +import type { Env } from '../src/env.js'; +import type { Database } from '../src/db/client.js'; +import { sameShapeMemoryLookupEnvelope } from '../src/memory/scope-policy.js'; + +vi.mock('../src/security/authorization.js', () => ({ + requireAuth: () => async (c: { req: { header: (name: string) => string | undefined }; set: (key: string, value: string) => void }, next: () => Promise) => { + c.set('userId', c.req.header('x-test-user') ?? 'user-member'); + c.set('role', 'member'); + await next(); + }, + resolveServerRole: vi.fn().mockResolvedValue('owner'), +})); + +vi.mock('../src/security/audit.js', () => ({ + logAudit: vi.fn().mockResolvedValue(undefined), +})); + +const randomHexMock = vi.hoisted(() => vi.fn()); + +vi.mock('../src/security/crypto.js', async (importOriginal) => { + const real = await importOriginal(); + return { ...real, randomHex: randomHexMock }; +}); + +type TeamRole = 'owner' | 'admin' | 'member'; +type BindingMode = 'required' | 'advisory'; + +type VersionRow = { + id: string; + document_id: string; + status: 'active' | 'draft' | 'superseded'; + content: string; +}; + +type BindingRow = { + id: string; + enterprise_id: string; + workspace_id: string | null; + enrollment_id: string | null; + document_id: string; + version_id: string; + binding_mode: BindingMode; + applicability_repo_id: string | null; + applicability_language: string | null; + applicability_path_pattern: string | null; + status: 'active' | 'inactive'; +}; + +function makeEnv(db: Database): Env { + return { + DB: db, + JWT_SIGNING_KEY: 'test-signing-key-32chars-padding!!', + BOT_ENCRYPTION_KEY: 'abcdef0123456789'.repeat(2), + SERVER_URL: 'https://app.im.codes', + ALLOWED_ORIGINS: '', + TRUSTED_PROXIES: '', + BIND_HOST: '127.0.0.1', + PORT: '3000', + NODE_ENV: 'test', + GITHUB_CLIENT_ID: '', + GITHUB_CLIENT_SECRET: '', + DATABASE_URL: '', + } as Env; +} + +function normalize(sql: string): string { + return sql.toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function makeMockDb() { + const teamMembers = new Map>([ + ['team-1', new Map([ + ['user-owner', { role: 'owner' }], + ['user-admin', { role: 'admin' }], + ['user-member', { role: 'member' }], + ])], + ['team-2', new Map([ + ['user-other-admin', { role: 'admin' }], + ])], + ]); + const versions = new Map([ + ['ver-project', { id: 'ver-project', document_id: 'doc-project', status: 'active', content: 'Project required rules' }], + ['ver-workspace', { id: 'ver-workspace', document_id: 'doc-workspace', status: 'active', content: 'Workspace advisory rules' }], + ['ver-org', { id: 'ver-org', document_id: 'doc-org', status: 'active', content: 'Org required rules' }], + ['ver-org-other', { id: 'ver-org-other', document_id: 'doc-org-other', status: 'active', content: 'Other repo org rules' }], + ]); + const bindings = new Map([ + ['bind-project', { + id: 'bind-project', + enterprise_id: 'team-1', + workspace_id: null, + enrollment_id: 'enr-1', + document_id: 'doc-project', + version_id: 'ver-project', + binding_mode: 'required', + applicability_repo_id: 'github.com/acme/repo', + applicability_language: 'typescript', + applicability_path_pattern: 'src/**', + status: 'active', + }], + ['bind-workspace', { + id: 'bind-workspace', + enterprise_id: 'team-1', + workspace_id: 'ws-1', + enrollment_id: null, + document_id: 'doc-workspace', + version_id: 'ver-workspace', + binding_mode: 'advisory', + applicability_repo_id: null, + applicability_language: 'typescript', + applicability_path_pattern: null, + status: 'active', + }], + ['bind-org', { + id: 'bind-org', + enterprise_id: 'team-1', + workspace_id: null, + enrollment_id: null, + document_id: 'doc-org', + version_id: 'ver-org', + binding_mode: 'required', + applicability_repo_id: null, + applicability_language: 'typescript', + applicability_path_pattern: 'src/**', + status: 'active', + }], + ['bind-org-other-repo', { + id: 'bind-org-other-repo', + enterprise_id: 'team-1', + workspace_id: null, + enrollment_id: null, + document_id: 'doc-org-other', + version_id: 'ver-org-other', + binding_mode: 'advisory', + applicability_repo_id: 'github.com/acme/other', + applicability_language: 'typescript', + applicability_path_pattern: null, + status: 'active', + }], + ]); + const executeLog: Array<{ sql: string; params: unknown[] }> = []; + + const db: Database = { + queryOne: async (sql: string, params: unknown[] = []) => { + const s = normalize(sql); + if (s.includes('select role from team_members where team_id = $1 and user_id = $2')) { + const member = teamMembers.get(params[0] as string)?.get(params[1] as string); + return member ? ({ role: member.role } as T) : null; + } + if (s.includes('select enterprise_id, workspace_id, enrollment_id from shared_context_document_bindings where id = $1')) { + const binding = bindings.get(params[0] as string); + return binding ? ({ + enterprise_id: binding.enterprise_id, + workspace_id: binding.workspace_id, + enrollment_id: binding.enrollment_id, + } as T) : null; + } + return null; + }, + query: async (sql: string, params: unknown[] = []) => { + const s = normalize(sql); + if (s.includes('from shared_context_document_bindings b join shared_context_document_versions v on v.id = b.version_id')) { + return [...bindings.values()] + .filter((binding) => binding.enterprise_id === params[0]) + .filter((binding) => binding.status === 'active') + .map((binding) => { + const version = versions.get(binding.version_id); + if (!version || version.status !== 'active') return null; + return { + binding_id: binding.id, + binding_mode: binding.binding_mode, + workspace_id: binding.workspace_id, + enrollment_id: binding.enrollment_id, + applicability_repo_id: binding.applicability_repo_id, + applicability_language: binding.applicability_language, + applicability_path_pattern: binding.applicability_path_pattern, + version_id: version.id, + content: version.content, + }; + }) + .filter(Boolean) as T[]; + } + return [] as T[]; + }, + execute: async (sql: string, params: unknown[] = []) => { + executeLog.push({ sql, params }); + const s = normalize(sql); + if (s.includes('insert into shared_context_document_bindings')) { + bindings.set(params[0] as string, { + id: params[0] as string, + enterprise_id: params[1] as string, + workspace_id: params[2] as string | null, + enrollment_id: params[3] as string | null, + document_id: params[4] as string, + version_id: params[5] as string, + binding_mode: params[6] as BindingMode, + applicability_repo_id: params[7] as string | null, + applicability_language: params[8] as string | null, + applicability_path_pattern: params[9] as string | null, + status: 'active', + }); + return { changes: 1 }; + } + if (s.includes("update shared_context_document_bindings set status = 'inactive'")) { + const binding = bindings.get(params[1] as string); + if (binding) binding.status = 'inactive'; + return { changes: binding ? 1 : 0 }; + } + return { changes: 1 }; + }, + exec: async () => {}, + close: async () => {}, + } as Database; + return { db, bindings, executeLog }; +} + +async function buildApp(db: Database) { + const { sharedContextRoutes } = await import('../src/routes/shared-context.js'); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/shared-context', sharedContextRoutes); + return { app, env: makeEnv(db) }; +} + +function req(path: string, method: string, body: unknown | undefined, userId: string) { + return [path, { + method, + headers: { 'content-type': 'application/json', 'x-test-user': userId }, + body: body === undefined ? undefined : JSON.stringify(body), + }] as const; +} + +describe('org_shared authored context standards', () => { + beforeEach(() => { + let counter = 0; + randomHexMock.mockImplementation(() => `generated-${++counter}`); + }); + + afterEach(() => { + delete process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS; + randomHexMock.mockReset(); + }); + + it('blocks org-wide mutation when disabled while leaving workspace binding mutation available', async () => { + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'false'; + const { db, bindings, executeLog } = makeMockDb(); + const { app, env } = await buildApp(db); + + const disabledOrg = await app.request(...req('/api/shared-context/enterprises/team-1/document-bindings', 'POST', { + documentId: 'doc-org', + versionId: 'ver-org', + mode: 'required', + }, 'user-admin'), env); + const workspace = await app.request(...req('/api/shared-context/enterprises/team-1/document-bindings', 'POST', { + documentId: 'doc-workspace', + versionId: 'ver-workspace', + workspaceId: 'ws-1', + mode: 'advisory', + }, 'user-admin'), env); + + expect(disabledOrg.status).toBe(404); + expect(await disabledOrg.json()).toEqual(sameShapeMemoryLookupEnvelope()); + expect(workspace.status).toBe(201); + expect(await workspace.json()).toMatchObject({ scope: 'workspace_shared', workspaceId: 'ws-1' }); + expect(bindings.get('generated-1')?.workspace_id).toBe('ws-1'); + expect(executeLog).toHaveLength(1); + }); + + it('enforces admin-only org mutation without role diagnostics and gates deactivation when disabled', async () => { + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'true'; + const { db, bindings } = makeMockDb(); + const { app, env } = await buildApp(db); + + const memberMutation = await app.request(...req('/api/shared-context/enterprises/team-1/document-bindings', 'POST', { + documentId: 'doc-org', + versionId: 'ver-org', + mode: 'required', + }, 'user-member'), env); + const adminMutation = await app.request(...req('/api/shared-context/enterprises/team-1/document-bindings', 'POST', { + documentId: 'doc-org', + versionId: 'ver-org', + mode: 'required', + }, 'user-admin'), env); + + expect(memberMutation.status).toBe(403); + expect(await memberMutation.json()).toEqual({ error: 'forbidden' }); + expect(adminMutation.status).toBe(201); + expect(await adminMutation.json()).toMatchObject({ id: 'generated-1', scope: 'org_shared' }); + expect(bindings.get('generated-1')?.workspace_id).toBeNull(); + expect(bindings.get('generated-1')?.enrollment_id).toBeNull(); + + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'false'; + const disabledDeactivate = await app.request(...req('/api/shared-context/document-bindings/generated-1/deactivate', 'POST', {}, 'user-admin'), env); + expect(disabledDeactivate.status).toBe(404); + expect(await disabledDeactivate.json()).toEqual(sameShapeMemoryLookupEnvelope()); + expect(bindings.get('generated-1')?.status).toBe('active'); + }); + + it('selects member-visible project, workspace, then org bindings with filter narrowing and no cross-enterprise leakage', async () => { + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'true'; + const { db } = makeMockDb(); + const { app, env } = await buildApp(db); + + const runtime = await app.request( + '/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo&workspaceId=ws-1&enrollmentId=enr-1&language=typescript&filePath=src/index.ts', + { method: 'GET', headers: { 'x-test-user': 'user-member' } }, + env, + ); + expect(runtime.status).toBe(200); + const json = await runtime.json() as { bindings: Array<{ bindingId: string; scope: string; mode: string; content: string }> }; + expect(json.bindings.map((binding) => binding.bindingId)).toEqual(['bind-project', 'bind-workspace', 'bind-org']); + expect(json.bindings.map((binding) => binding.scope)).toEqual(['project_shared', 'workspace_shared', 'org_shared']); + expect(json.bindings.find((binding) => binding.bindingId === 'bind-org')?.mode).toBe('required'); + expect(json.bindings.map((binding) => binding.bindingId)).not.toContain('bind-org-other-repo'); + + const narrowedByPath = await app.request( + '/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo&workspaceId=ws-1&enrollmentId=enr-1&language=typescript&filePath=docs/readme.md', + { method: 'GET', headers: { 'x-test-user': 'user-member' } }, + env, + ); + expect((await narrowedByPath.json() as { bindings: Array<{ bindingId: string }> }).bindings.map((binding) => binding.bindingId)).toEqual(['bind-workspace']); + + const advisoryTrimmed = await app.request( + '/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo&workspaceId=ws-1&enrollmentId=enr-1&language=typescript&filePath=src/index.ts&budgetBytes=45', + { method: 'GET', headers: { 'x-test-user': 'user-member' } }, + env, + ); + expect(advisoryTrimmed.status).toBe(200); + const advisoryJson = await advisoryTrimmed.json() as { bindings: Array<{ bindingId: string }>; diagnostics: Array<{ bindingId: string; reason: string }> }; + expect(advisoryJson.bindings.map((binding) => binding.bindingId)).toEqual(['bind-project', 'bind-org']); + expect(advisoryJson.diagnostics).toEqual([{ bindingId: 'bind-workspace', mode: 'advisory', reason: 'advisory_trimmed', bytes: 24 }]); + + const requiredOverBudget = await app.request( + '/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo&workspaceId=ws-1&enrollmentId=enr-1&language=typescript&filePath=src/index.ts&budgetBytes=30', + { method: 'GET', headers: { 'x-test-user': 'user-member' } }, + env, + ); + expect(requiredOverBudget.status).toBe(409); + expect(await requiredOverBudget.json()).toMatchObject({ + error: 'required_context_over_budget', + diagnostics: [{ bindingId: 'bind-workspace', mode: 'advisory', reason: 'advisory_trimmed', bytes: 24 }, { bindingId: 'bind-org', mode: 'required', reason: 'required_over_budget', bytes: 18 }], + }); + + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'false'; + const disabled = await app.request( + '/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo&workspaceId=ws-1&enrollmentId=enr-1&language=typescript&filePath=src/index.ts', + { method: 'GET', headers: { 'x-test-user': 'user-member' } }, + env, + ); + expect((await disabled.json() as { bindings: Array<{ bindingId: string }> }).bindings.map((binding) => binding.bindingId)).toEqual(['bind-project', 'bind-workspace']); + + const nonMember = await app.request( + '/api/shared-context/enterprises/team-1/runtime-authored-context?canonicalRepoId=github.com/acme/repo', + { method: 'GET', headers: { 'x-test-user': 'user-outsider' } }, + env, + ); + const otherEnterprise = await app.request( + '/api/shared-context/enterprises/team-2/runtime-authored-context?canonicalRepoId=github.com/acme/repo', + { method: 'GET', headers: { 'x-test-user': 'user-member' } }, + env, + ); + expect(nonMember.status).toBe(404); + expect(otherEnterprise.status).toBe(404); + expect(await nonMember.json()).toEqual(sameShapeMemoryLookupEnvelope()); + expect(await otherEnterprise.json()).toEqual(sameShapeMemoryLookupEnvelope()); + }); +}); diff --git a/server/test/shared-context-processed-remote.test.ts b/server/test/shared-context-processed-remote.test.ts index b0ee7f35a..c6a1c8fee 100644 --- a/server/test/shared-context-processed-remote.test.ts +++ b/server/test/shared-context-processed-remote.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { Hono } from 'hono'; import { sha256Hex } from '../src/security/crypto.js'; import { serverRoutes } from '../src/routes/server.js'; @@ -24,6 +24,11 @@ vi.mock('../src/security/authorization.js', () => ({ beforeEach(() => { generateEmbeddingMock.mockReset(); generateEmbeddingMock.mockResolvedValue(null); + process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS = 'true'; +}); + +afterEach(() => { + delete process.env.IMCODES_MEM_FEATURE_ORG_SHARED_AUTHORED_STANDARDS; }); function makeMockDb() { @@ -37,6 +42,8 @@ function makeMockDb() { version_id: 'doc-v2', binding_mode: 'required', scope: 'project_shared', + workspace_id: null, + enrollment_id: 'enr-1', applicability_repo_id: 'github.com/acme/repo', applicability_language: 'typescript', applicability_path_pattern: 'src/**', @@ -47,6 +54,8 @@ function makeMockDb() { version_id: 'doc-v1', binding_mode: 'advisory', scope: 'org_shared', + workspace_id: null, + enrollment_id: null, applicability_repo_id: null, applicability_language: null, applicability_path_pattern: null, @@ -143,7 +152,10 @@ function makeMockDb() { project_count: 1, } as T; } - if (normalized.includes("scope in ('project_shared', 'workspace_shared', 'org_shared')")) { + if ( + normalized.includes("scope in ('project_shared', 'workspace_shared', 'org_shared')") + || normalized.includes('scope in ($') + ) { return { total_records: 1, recent_summary_count: 0, @@ -180,7 +192,11 @@ function makeMockDb() { }, ] as T[]; } - if (normalized.includes("p.scope in ('project_shared', 'workspace_shared', 'org_shared')")) { + if ( + normalized.includes("p.scope in ('project_shared', 'workspace_shared', 'org_shared')") + || normalized.includes('p.scope in ($') + || normalized.includes('p.scope = any(') + ) { return [ { id: 'shared-projection-1', @@ -199,6 +215,28 @@ function makeMockDb() { ] as T[]; } } + if (normalized.includes('group by project_id') && normalized.includes("scope = 'personal'")) { + return [ + { + project_id: 'github.com/acme/repo', + total_records: 1, + recent_summary_count: 1, + durable_candidate_count: 0, + updated_at: 1700000000000, + }, + ] as T[]; + } + if (normalized.includes('group by project_id') && normalized.includes('enterprise_id =')) { + return [ + { + project_id: 'github.com/acme/repo', + total_records: 1, + recent_summary_count: 0, + durable_candidate_count: 1, + updated_at: 1700000002000, + }, + ] as T[]; + } if (normalized.includes("from shared_context_projections where user_id = $1 and scope = 'personal'")) { return [ { @@ -242,6 +280,7 @@ function makeMockDb() { user_id: params[5], project_id: params[6], projection_class: params[7], + origin: params[12], }); return { changes: 1 }; } @@ -256,6 +295,7 @@ function makeMockDb() { user_id: params[6], project_id: params[7], record_class: params[8], + origin: params[11], }); return { changes: 1 }; } @@ -317,6 +357,7 @@ describe('shared-context processed remote route', () => { enterpriseId: 'ent-1', }, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds: ['evt-1'], summary: 'summary', content: { foo: 'bar' }, @@ -331,6 +372,7 @@ describe('shared-context processed remote route', () => { enterpriseId: 'ent-1', }, class: 'durable_memory_candidate', + origin: 'chat_compacted', sourceEventIds: ['evt-2'], summary: 'decision', content: { kind: 'decision' }, @@ -351,6 +393,7 @@ describe('shared-context processed remote route', () => { expect.objectContaining({ projection_id: 'proj-2', record_class: 'durable_memory_candidate', + origin: 'chat_compacted', }), ]); expect(aliasRows).toHaveLength(0); @@ -383,6 +426,7 @@ describe('shared-context processed remote route', () => { enterpriseId: 'ent-1', }, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds: ['evt-bad'], summary: '**Assistant:** [API Error: Connection error. (cause: fetch failed)]', content: {}, @@ -397,6 +441,7 @@ describe('shared-context processed remote route', () => { enterpriseId: 'ent-1', }, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds: ['evt-good'], summary: 'useful summary', content: {}, @@ -415,6 +460,57 @@ describe('shared-context processed remote route', () => { expect(recordRows).toEqual([]); }); + it('rejects processed projection writes without an emit-safe explicit origin', async () => { + const { db, projectionRows, recordRows } = makeMockDb(); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/server', serverRoutes); + + for (const projection of [ + { + id: 'missing-origin', + namespace: { scope: 'project_shared', projectId: 'github.com/acme/repo', enterpriseId: 'ent-1' }, + class: 'recent_summary', + sourceEventIds: ['evt-missing'], + summary: 'missing origin', + content: {}, + createdAt: 100, + updatedAt: 101, + }, + { + id: 'reserved-origin', + namespace: { scope: 'project_shared', projectId: 'github.com/acme/repo', enterpriseId: 'ent-1' }, + class: 'recent_summary', + origin: 'quick_search_cache', + sourceEventIds: ['evt-reserved'], + summary: 'reserved origin', + content: {}, + createdAt: 100, + updatedAt: 101, + }, + ]) { + const response = await app.request('/api/server/srv-1/shared-context/processed', { + method: 'POST', + headers: { + authorization: 'Bearer daemon-token', + 'content-type': 'application/json', + }, + body: JSON.stringify({ + namespace: { + scope: 'project_shared', + projectId: 'github.com/acme/repo', + enterpriseId: 'ent-1', + }, + projections: [projection], + }), + }, makeEnv(db)); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ error: 'invalid_body' }); + } + expect(projectionRows).toEqual([]); + expect(recordRows).toEqual([]); + }); + it('sanitizes personal projections to the daemon owner and rejects mismatched namespace users', async () => { const { db, projectionRows, recordRows } = makeMockDb(); const app = new Hono<{ Bindings: Env }>(); @@ -443,6 +539,7 @@ describe('shared-context processed remote route', () => { workspaceId: 'wrong-ws', }, class: 'durable_memory_candidate', + origin: 'chat_compacted', sourceEventIds: ['evt-1'], summary: 'personal summary', content: { foo: 'bar' }, @@ -461,6 +558,7 @@ describe('shared-context processed remote route', () => { workspace_id: null, user_id: 'user-1', project_id: 'github.com/acme/repo', + origin: 'chat_compacted', })); expect(recordRows).toContainEqual(expect.objectContaining({ projection_id: 'personal-proj-1', @@ -468,6 +566,7 @@ describe('shared-context processed remote route', () => { enterprise_id: null, workspace_id: null, user_id: 'user-1', + origin: 'chat_compacted', })); const forbidden = await app.request('/api/server/srv-1/shared-context/processed', { @@ -491,6 +590,7 @@ describe('shared-context processed remote route', () => { userId: 'user-other', }, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds: ['evt-2'], summary: 'mismatch', content: { foo: 'bar' }, @@ -689,6 +789,16 @@ describe('shared-context processed remote route', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 1, + recentSummaryCount: 1, + durableCandidateCount: 0, + updatedAt: 1700000000000, + }, + ], }); }); @@ -726,6 +836,16 @@ describe('shared-context processed remote route', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 1, + recentSummaryCount: 1, + durableCandidateCount: 0, + updatedAt: 1700000000000, + }, + ], }); }); @@ -763,6 +883,16 @@ describe('shared-context processed remote route', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 1, + recentSummaryCount: 1, + durableCandidateCount: 0, + updatedAt: 1700000000000, + }, + ], }); }); @@ -826,6 +956,16 @@ describe('shared-context processed remote route', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 1, + recentSummaryCount: 1, + durableCandidateCount: 0, + updatedAt: 1700000000000, + }, + ], }); }); @@ -866,6 +1006,16 @@ describe('shared-context processed remote route', () => { status: 'active', }, ], + projects: [ + { + projectId: 'github.com/acme/repo', + displayName: 'github.com/acme/repo', + totalRecords: 1, + recentSummaryCount: 0, + durableCandidateCount: 1, + updatedAt: 1700000002000, + }, + ], }); }); diff --git a/shared/builtin-skill-manifest.ts b/shared/builtin-skill-manifest.ts new file mode 100644 index 000000000..7e4ca1ca6 --- /dev/null +++ b/shared/builtin-skill-manifest.ts @@ -0,0 +1,48 @@ +export const BUILTIN_SKILL_MANIFEST_VERSION = 1 as const; + +export interface BuiltinSkillManifestEntry { + name: string; + category: string; + path: string; + description?: string; + version?: string; +} + +export interface BuiltinSkillManifest { + version: typeof BUILTIN_SKILL_MANIFEST_VERSION; + skills: readonly BuiltinSkillManifestEntry[]; +} + +export const EMPTY_BUILTIN_SKILL_MANIFEST: BuiltinSkillManifest = { + version: BUILTIN_SKILL_MANIFEST_VERSION, + skills: [], +}; + +export function validateBuiltinSkillManifest(value: unknown): BuiltinSkillManifest { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Invalid built-in skill manifest: expected object'); + } + const record = value as Record; + if (record.version !== BUILTIN_SKILL_MANIFEST_VERSION) { + throw new Error('Invalid built-in skill manifest: unsupported version'); + } + if (!Array.isArray(record.skills)) { + throw new Error('Invalid built-in skill manifest: skills must be an array'); + } + for (const skill of record.skills) { + if (!skill || typeof skill !== 'object' || Array.isArray(skill)) { + throw new Error('Invalid built-in skill manifest: skill entry must be an object'); + } + const entry = skill as Record; + if (typeof entry.name !== 'string' || entry.name.trim().length === 0) { + throw new Error('Invalid built-in skill manifest: skill name is required'); + } + if (typeof entry.category !== 'string' || entry.category.trim().length === 0) { + throw new Error('Invalid built-in skill manifest: skill category is required'); + } + if (typeof entry.path !== 'string' || entry.path.trim().length === 0) { + throw new Error('Invalid built-in skill manifest: skill path is required'); + } + } + return record as unknown as BuiltinSkillManifest; +} diff --git a/shared/context-types.ts b/shared/context-types.ts index 08937f4b7..d0e295a4c 100644 --- a/shared/context-types.ts +++ b/shared/context-types.ts @@ -1,4 +1,9 @@ import type { MemoryScoringWeights } from './memory-scoring.js'; +import type { + AuthoredContextScope, + MemoryScope, +} from './memory-scope.js'; +import type { MemoryOrigin } from './memory-origin.js'; export type CanonicalRepositoryIdentityKind = 'git-origin' | 'local-fallback'; @@ -20,14 +25,16 @@ export interface RepositoryAlias { createdAt?: number; } -export type ContextScope = 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; +export type ContextScope = MemoryScope; export interface ContextNamespace { scope: ContextScope; - projectId: string; + projectId?: string; userId?: string; workspaceId?: string; enterpriseId?: string; + localTenant?: string; + canonicalRepoId?: string; } export interface SharedScopePolicyOverride { @@ -56,7 +63,7 @@ export interface RuntimeAuthoredContextBinding { bindingId: string; documentVersionId: string; mode: AuthoredContextBindingMode; - scope: Exclude; + scope: AuthoredContextScope; repository?: string; language?: string; pathPattern?: string; @@ -219,6 +226,8 @@ export interface ProcessedContextProjection { sourceEventIds: string[]; summary: string; content: Record; + contentHash?: string; + origin?: MemoryOrigin; createdAt: number; updatedAt: number; hitCount?: number; @@ -264,10 +273,23 @@ export interface ContextMemoryStatsView { pendingJobCount: number; } +export interface ContextMemoryProjectView { + projectId: string; + displayName?: string; + totalRecords: number; + recentSummaryCount: number; + durableCandidateCount: number; + pendingEventCount?: number; + updatedAt?: number; +} + export interface ContextMemoryRecordView { id: string; - scope: 'personal' | 'project_shared' | 'workspace_shared' | 'org_shared'; + scope: MemoryScope; projectId: string; + ownerUserId?: string; + createdByUserId?: string; + updatedByUserId?: string; summary: string; projectionClass: ProcessedContextClass; sourceEventCount: number; @@ -290,6 +312,7 @@ export interface ContextMemoryView { stats: ContextMemoryStatsView; records: ContextMemoryRecordView[]; pendingRecords?: ContextPendingEventView[]; + projects?: ContextMemoryProjectView[]; } export interface ProcessedContextReplicationBody { diff --git a/shared/daemon-command-types.ts b/shared/daemon-command-types.ts index 769465a0e..82577519c 100644 --- a/shared/daemon-command-types.ts +++ b/shared/daemon-command-types.ts @@ -1,4 +1,7 @@ export const DAEMON_COMMAND_TYPES = { + DAEMON_UPGRADE: 'daemon.upgrade', + SERVER_DELETE: 'server.delete', + SESSION_CANCEL: 'session.cancel', SESSION_UPDATE_TRANSPORT_CONFIG: 'session.update_transport_config', SUBSESSION_UPDATE_TRANSPORT_CONFIG: 'subsession.update_transport_config', } as const; diff --git a/shared/daemon-upgrade.ts b/shared/daemon-upgrade.ts new file mode 100644 index 000000000..a56ab8214 --- /dev/null +++ b/shared/daemon-upgrade.ts @@ -0,0 +1,31 @@ +export const DAEMON_UPGRADE_TARGET_LATEST = 'latest'; + +export const DAEMON_UPGRADE_DELIVERY_STATUS = { + SENT: 'sent', + PENDING_OFFLINE: 'pending_offline', + ALREADY_IN_PROGRESS: 'already_in_progress', + BACKOFF: 'backoff', + SUPPRESSED: 'suppressed', + PENDING_PUBLICATION: 'pending_publication', + INVALID_TARGET: 'invalid_target', +} as const; + +export type DaemonUpgradeDeliveryStatus = + (typeof DAEMON_UPGRADE_DELIVERY_STATUS)[keyof typeof DAEMON_UPGRADE_DELIVERY_STATUS]; + +const DAEMON_UPGRADE_TARGET_VERSION_RE = /^[0-9]+(?:\.[0-9]+){1,3}(?:-[0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*)?$/; + +export function normalizeDaemonUpgradeTargetVersion(value: unknown): string { + if (value == null || value === '') return DAEMON_UPGRADE_TARGET_LATEST; + if (typeof value !== 'string') throw new Error('invalid_target_version'); + const targetVersion = value.trim(); + if (targetVersion === DAEMON_UPGRADE_TARGET_LATEST) return targetVersion; + if (!DAEMON_UPGRADE_TARGET_VERSION_RE.test(targetVersion)) { + throw new Error('invalid_target_version'); + } + return targetVersion; +} + +export function shouldSendDaemonUpgradeTargetVersion(targetVersion: string): boolean { + return targetVersion !== DAEMON_UPGRADE_TARGET_LATEST; +} diff --git a/shared/feature-flags.ts b/shared/feature-flags.ts new file mode 100644 index 000000000..e85252461 --- /dev/null +++ b/shared/feature-flags.ts @@ -0,0 +1,300 @@ +export const MEMORY_FEATURE_FLAGS = [ + 'mem.feature.scope_registry_extensions', + 'mem.feature.user_private_sync', + 'mem.feature.self_learning', + 'mem.feature.namespace_registry', + 'mem.feature.observation_store', + 'mem.feature.quick_search', + 'mem.feature.citation', + 'mem.feature.cite_count', + 'mem.feature.cite_drift_badge', + 'mem.feature.md_ingest', + 'mem.feature.preferences', + 'mem.feature.skills', + 'mem.feature.skill_auto_creation', + 'mem.feature.org_shared_authored_standards', +] as const; + +export type MemoryFeatureFlag = (typeof MEMORY_FEATURE_FLAGS)[number]; + +export const FEATURE_FLAG_VALUE_PRECEDENCE = [ + 'runtime_config_override', + 'persisted_config', + 'environment_startup_default', + 'registry_default', +] as const; +export type FeatureFlagValueSource = (typeof FEATURE_FLAG_VALUE_PRECEDENCE)[number]; + +export type MemoryFeatureRuntimeSource = + | 'user_global_config' + | 'local_daemon_config' + | 'server_config' + | 'local_or_server_config'; + +export interface MemoryFeatureFlagDefinition { + flag: MemoryFeatureFlag; + defaultValue: boolean; + runtimeSource: MemoryFeatureRuntimeSource; + dependencies: readonly MemoryFeatureFlag[]; + requiredPrerequisites: readonly string[]; + observedBy: readonly string[]; + disabledBehavior: string; +} + +export type MemoryFeatureFlagValues = Partial>; +export type MemoryFeaturePrerequisites = Partial>; +export interface MemoryFeatureFlagResolutionLayers { + runtimeConfigOverride?: MemoryFeatureFlagValues; + persistedConfig?: MemoryFeatureFlagValues; + environmentStartupDefault?: MemoryFeatureFlagValues; + readFailed?: boolean; +} + +export const MEMORY_FEATURE_CONFIG_PREF_KEY = 'memory.feature_flags.global.v1' as const; + +export const MEMORY_FEATURE_CONFIG_MSG = { + APPLY: 'memory.features.apply', +} as const; + +const FLAG = { + scopeRegistryExtensions: 'mem.feature.scope_registry_extensions', + userPrivateSync: 'mem.feature.user_private_sync', + selfLearning: 'mem.feature.self_learning', + namespaceRegistry: 'mem.feature.namespace_registry', + observationStore: 'mem.feature.observation_store', + quickSearch: 'mem.feature.quick_search', + citation: 'mem.feature.citation', + citeCount: 'mem.feature.cite_count', + citeDriftBadge: 'mem.feature.cite_drift_badge', + mdIngest: 'mem.feature.md_ingest', + preferences: 'mem.feature.preferences', + skills: 'mem.feature.skills', + skillAutoCreation: 'mem.feature.skill_auto_creation', + orgSharedAuthoredStandards: 'mem.feature.org_shared_authored_standards', +} as const satisfies Record; + +export const MEMORY_FEATURE_FLAGS_BY_NAME = FLAG; + +export const MEMORY_FEATURE_FLAG_REGISTRY = { + [FLAG.scopeRegistryExtensions]: { + flag: FLAG.scopeRegistryExtensions, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [], + requiredPrerequisites: [], + observedBy: ['daemon', 'server', 'web', 'namespace_registry'], + disabledBehavior: 'Legacy scopes remain accepted; new user_private writes fail closed except migration/backfill reads.', + }, + [FLAG.userPrivateSync]: { + flag: FLAG.userPrivateSync, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.scopeRegistryExtensions, FLAG.namespaceRegistry, FLAG.observationStore], + requiredPrerequisites: [], + observedBy: ['daemon_replication_runner', 'server_owner_private_sync', 'startup_selection', 'memory_search'], + disabledBehavior: 'user_private remains daemon-local owner-only; no owner-private server reads or writes are attempted.', + }, + [FLAG.selfLearning]: { + flag: FLAG.selfLearning, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.namespaceRegistry, FLAG.observationStore], + requiredPrerequisites: [], + observedBy: ['materialization_pipeline', 'compression_pipeline'], + disabledBehavior: 'Classification, dedup, and durable extraction are skipped; projection commits remain readable.', + }, + [FLAG.namespaceRegistry]: { + flag: FLAG.namespaceRegistry, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [], + requiredPrerequisites: [], + observedBy: ['daemon_storage', 'server_storage'], + disabledBehavior: 'No new namespace records outside migration/backfill; legacy projection reads remain available.', + }, + [FLAG.observationStore]: { + flag: FLAG.observationStore, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.namespaceRegistry], + requiredPrerequisites: [], + observedBy: ['daemon_storage', 'server_storage', 'materialization', 'preferences', 'skills'], + disabledBehavior: 'No new observation rows; projections remain readable.', + }, + [FLAG.quickSearch]: { + flag: FLAG.quickSearch, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.namespaceRegistry], + requiredPrerequisites: [], + observedBy: ['web_search_ui', 'server_search_rpc', 'daemon_search_rpc'], + disabledBehavior: 'Search UI is hidden; endpoint returns the same disabled envelope without search jobs.', + }, + [FLAG.citation]: { + flag: FLAG.citation, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.quickSearch], + requiredPrerequisites: [], + observedBy: ['web_composer', 'citation_rpc'], + disabledBehavior: 'Citation UI is hidden and RPC rejects with the same disabled envelope; no citation rows.', + }, + [FLAG.citeCount]: { + flag: FLAG.citeCount, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.citation], + requiredPrerequisites: [], + observedBy: ['citation_store', 'search_ranking'], + disabledBehavior: 'No new count increments; existing counts are ignored in ranking without deleting data.', + }, + [FLAG.citeDriftBadge]: { + flag: FLAG.citeDriftBadge, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.citation], + requiredPrerequisites: [], + observedBy: ['web_citation_renderer'], + disabledBehavior: 'Drift badge is hidden; citation identity is preserved when citations are enabled.', + }, + [FLAG.mdIngest]: { + flag: FLAG.mdIngest, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.namespaceRegistry, FLAG.observationStore], + requiredPrerequisites: [], + observedBy: ['session_bootstrap', 'md_ingest_worker'], + disabledBehavior: 'No markdown reads, parses, or ingest jobs.', + }, + [FLAG.preferences]: { + flag: FLAG.preferences, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.namespaceRegistry, FLAG.observationStore], + requiredPrerequisites: [], + observedBy: ['daemon_send_handler', 'preference_store'], + disabledBehavior: '@pref: lines pass through as text and are not persisted or stripped.', + }, + [FLAG.skills]: { + flag: FLAG.skills, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.namespaceRegistry, FLAG.observationStore], + requiredPrerequisites: [], + observedBy: ['skill_loader', 'render_policy', 'admin_api'], + disabledBehavior: 'Loader returns an empty set; render policy skips skills; admin writes are rejected or disabled.', + }, + [FLAG.skillAutoCreation]: { + flag: FLAG.skillAutoCreation, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.skills, FLAG.selfLearning], + requiredPrerequisites: [], + observedBy: ['background_skill_review_worker'], + disabledBehavior: 'No skill-review jobs are claimed or created; existing skills still load when skills are enabled.', + }, + [FLAG.orgSharedAuthoredStandards]: { + flag: FLAG.orgSharedAuthoredStandards, + defaultValue: false, + runtimeSource: 'user_global_config', + dependencies: [FLAG.scopeRegistryExtensions], + requiredPrerequisites: ['shared_context_document_migrations', 'shared_context_version_migrations', 'shared_context_binding_migrations'], + observedBy: ['server_shared_context_routes', 'authored_context_resolver', 'web_diagnostics'], + disabledBehavior: 'Org-wide authored standard mutation/selection is rejected or skipped without leaking inventory.', + }, +} as const satisfies Record; + +const MEMORY_FEATURE_FLAG_SET: ReadonlySet = new Set(MEMORY_FEATURE_FLAGS); + +export function isMemoryFeatureFlag(value: unknown): value is MemoryFeatureFlag { + return typeof value === 'string' && MEMORY_FEATURE_FLAG_SET.has(value); +} + +export function sanitizeMemoryFeatureFlagValues(raw: unknown): MemoryFeatureFlagValues { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {}; + const flags: MemoryFeatureFlagValues = {}; + for (const [key, value] of Object.entries(raw)) { + if (!isMemoryFeatureFlag(key)) continue; + if (value === true || value === false) flags[key] = value; + } + return flags; +} + +export function parseMemoryFeatureFlagValuesJson(raw: string | null | undefined): MemoryFeatureFlagValues { + if (!raw?.trim()) return {}; + try { + return sanitizeMemoryFeatureFlagValues(JSON.parse(raw)); + } catch { + return {}; + } +} + +export function encodeMemoryFeatureFlagValuesJson(flags: MemoryFeatureFlagValues): string { + return JSON.stringify(sanitizeMemoryFeatureFlagValues(flags)); +} + +export function getMemoryFeatureFlagDefinition(flag: MemoryFeatureFlag): MemoryFeatureFlagDefinition { + return MEMORY_FEATURE_FLAG_REGISTRY[flag]; +} + +export function memoryFeatureFlagEnvKey(flag: MemoryFeatureFlag): string { + return `IMCODES_${flag.toUpperCase().replace(/[^A-Z0-9]+/g, '_')}`; +} + +export function resolveMemoryFeatureFlagValue( + flag: MemoryFeatureFlag, + layers: MemoryFeatureFlagResolutionLayers, +): boolean { + if (layers.readFailed) return false; + const runtime = layers.runtimeConfigOverride?.[flag]; + if (runtime !== undefined) return runtime; + const persisted = layers.persistedConfig?.[flag]; + if (persisted !== undefined) return persisted; + const environmentDefault = layers.environmentStartupDefault?.[flag]; + if (environmentDefault !== undefined) return environmentDefault; + return MEMORY_FEATURE_FLAG_REGISTRY[flag].defaultValue; +} + +export function resolveEffectiveMemoryFeatureFlags( + layers: MemoryFeatureFlagResolutionLayers, + prerequisites: MemoryFeaturePrerequisites = {}, +): Record { + if (layers.readFailed) { + return Object.fromEntries(MEMORY_FEATURE_FLAGS.map((flag) => [flag, false])) as Record; + } + const requested = Object.fromEntries( + MEMORY_FEATURE_FLAGS.map((flag) => [flag, resolveMemoryFeatureFlagValue(flag, layers)]), + ) as Record; + return computeEffectiveMemoryFeatureFlags(requested, prerequisites); +} + +export function resolveEffectiveMemoryFeatureFlagValue( + flag: MemoryFeatureFlag, + layers: MemoryFeatureFlagResolutionLayers, + prerequisites: MemoryFeaturePrerequisites = {}, +): boolean { + return resolveEffectiveMemoryFeatureFlags(layers, prerequisites)[flag]; +} + +export function computeEffectiveMemoryFeatureFlags( + requested: MemoryFeatureFlagValues, + prerequisites: MemoryFeaturePrerequisites = {}, +): Record { + const effective = Object.fromEntries(MEMORY_FEATURE_FLAGS.map((flag) => [flag, false])) as Record; + + const visit = (flag: MemoryFeatureFlag, stack: readonly MemoryFeatureFlag[]): boolean => { + if (effective[flag]) return true; + if (requested[flag] !== true) return false; + if (stack.includes(flag)) return false; + const definition = MEMORY_FEATURE_FLAG_REGISTRY[flag]; + const dependenciesEnabled = definition.dependencies.every((dependency) => visit(dependency, [...stack, flag])); + const prerequisitesAvailable = definition.requiredPrerequisites.every((name) => prerequisites[name] === true); + effective[flag] = dependenciesEnabled && prerequisitesAvailable; + return effective[flag]; + }; + + for (const flag of MEMORY_FEATURE_FLAGS) { + visit(flag, []); + } + return effective; +} diff --git a/shared/fs-error-codes.ts b/shared/fs-error-codes.ts new file mode 100644 index 000000000..86e96cca0 --- /dev/null +++ b/shared/fs-error-codes.ts @@ -0,0 +1,21 @@ +export const FS_GENERIC_ERROR_CODES = { + FORBIDDEN_PATH: 'forbidden_path', + FILE_TOO_LARGE: 'file_too_large', + INVALID_REQUEST: 'invalid_request', + INTERNAL_ERROR: 'internal_error', +} as const; + +export type FsGenericErrorCode = (typeof FS_GENERIC_ERROR_CODES)[keyof typeof FS_GENERIC_ERROR_CODES]; + +export const FS_GENERIC_ERROR_CODE_VALUES = [ + FS_GENERIC_ERROR_CODES.FORBIDDEN_PATH, + FS_GENERIC_ERROR_CODES.FILE_TOO_LARGE, + FS_GENERIC_ERROR_CODES.INVALID_REQUEST, + FS_GENERIC_ERROR_CODES.INTERNAL_ERROR, +] as const satisfies readonly FsGenericErrorCode[]; + +const FS_GENERIC_ERROR_CODE_SET: ReadonlySet = new Set(FS_GENERIC_ERROR_CODE_VALUES); + +export function isFsGenericErrorCode(value: unknown): value is FsGenericErrorCode { + return typeof value === 'string' && FS_GENERIC_ERROR_CODE_SET.has(value); +} diff --git a/shared/fs-read-error-codes.ts b/shared/fs-read-error-codes.ts new file mode 100644 index 000000000..4c28708f4 --- /dev/null +++ b/shared/fs-read-error-codes.ts @@ -0,0 +1,58 @@ +import { FS_GENERIC_ERROR_CODES } from './fs-error-codes.js'; +export { + FS_GENERIC_ERROR_CODES, + FS_GENERIC_ERROR_CODE_VALUES, + isFsGenericErrorCode, + type FsGenericErrorCode, +} from './fs-error-codes.js'; + +export const FS_READ_ERROR_CODES = { + ...FS_GENERIC_ERROR_CODES, + BINARY_FILE: 'binary_file', + PREVIEW_WORKER_QUEUE_FULL: 'preview_worker_queue_full', + PREVIEW_WORKER_TIMEOUT: 'preview_worker_timeout', + PREVIEW_WORKER_UNAVAILABLE: 'preview_worker_unavailable', + PREVIEW_WORKER_CRASHED: 'preview_worker_crashed', + STALE_READ: 'stale_read', +} as const; + +export type FsReadErrorCode = (typeof FS_READ_ERROR_CODES)[keyof typeof FS_READ_ERROR_CODES]; + +export const FS_READ_ERROR_CODE_VALUES = [ + FS_READ_ERROR_CODES.BINARY_FILE, + FS_READ_ERROR_CODES.FORBIDDEN_PATH, + FS_READ_ERROR_CODES.FILE_TOO_LARGE, + FS_READ_ERROR_CODES.PREVIEW_WORKER_QUEUE_FULL, + FS_READ_ERROR_CODES.PREVIEW_WORKER_TIMEOUT, + FS_READ_ERROR_CODES.PREVIEW_WORKER_UNAVAILABLE, + FS_READ_ERROR_CODES.PREVIEW_WORKER_CRASHED, + FS_READ_ERROR_CODES.STALE_READ, + FS_READ_ERROR_CODES.INVALID_REQUEST, + FS_READ_ERROR_CODES.INTERNAL_ERROR, +] as const satisfies readonly FsReadErrorCode[]; + +const FS_READ_ERROR_CODE_SET: ReadonlySet = new Set(FS_READ_ERROR_CODE_VALUES); + +export function isFsReadErrorCode(value: unknown): value is FsReadErrorCode { + return typeof value === 'string' && FS_READ_ERROR_CODE_SET.has(value); +} + +export const FS_READ_PREVIEW_REASONS = { + TOO_LARGE: 'too_large', + BINARY: 'binary', + UNKNOWN_TYPE: 'unknown_type', +} as const; + +export type FsReadPreviewReason = (typeof FS_READ_PREVIEW_REASONS)[keyof typeof FS_READ_PREVIEW_REASONS]; + +export const FS_READ_PREVIEW_REASON_VALUES = [ + FS_READ_PREVIEW_REASONS.TOO_LARGE, + FS_READ_PREVIEW_REASONS.BINARY, + FS_READ_PREVIEW_REASONS.UNKNOWN_TYPE, +] as const satisfies readonly FsReadPreviewReason[]; + +const FS_READ_PREVIEW_REASON_SET: ReadonlySet = new Set(FS_READ_PREVIEW_REASON_VALUES); + +export function isFsReadPreviewReason(value: unknown): value is FsReadPreviewReason { + return typeof value === 'string' && FS_READ_PREVIEW_REASON_SET.has(value); +} diff --git a/shared/imcodes-send.ts b/shared/imcodes-send.ts new file mode 100644 index 000000000..ba9e53056 --- /dev/null +++ b/shared/imcodes-send.ts @@ -0,0 +1,3 @@ +export const IMCODES_SESSION_ENV = 'IMCODES_SESSION'; +export const IMCODES_SESSION_LABEL_ENV = 'IMCODES_SESSION_LABEL'; +export const IMCODES_EXTERNAL_CLI_SENDER = '__imcodes_external_cli__'; diff --git a/shared/md-ingest.ts b/shared/md-ingest.ts new file mode 100644 index 000000000..6e6fe3084 --- /dev/null +++ b/shared/md-ingest.ts @@ -0,0 +1,183 @@ +import { computeMemoryFingerprint } from './memory-fingerprint.js'; +import { MEMORY_FEATURE_FLAGS_BY_NAME } from './feature-flags.js'; +import { MEMORY_DEFAULTS } from './memory-defaults.js'; +import type { MemoryOrigin } from './memory-origin.js'; +import { recordMemorySoftFailure, type MemoryTelemetryBuffer } from './memory-telemetry.js'; + +export const MD_INGEST_FEATURE_FLAG = MEMORY_FEATURE_FLAGS_BY_NAME.mdIngest; +export const MD_INGEST_ORIGIN = 'md_ingest' as const satisfies MemoryOrigin; +export const MD_INGEST_SUPPORTED_PATHS = [ + 'CLAUDE.md', + 'AGENTS.md', + '.imc/memory.md', + '.imcodes/memory.md', +] as const; +export type MdIngestSupportedPath = (typeof MD_INGEST_SUPPORTED_PATHS)[number]; + +export const MD_INGEST_SECTION_CLASSES = ['preference', 'workflow', 'code_pattern', 'note'] as const; +export type MdIngestSectionClass = (typeof MD_INGEST_SECTION_CLASSES)[number]; + +export interface MdIngestCaps { + maxBytes: number; + maxSections: number; + maxSectionBytes: number; + parserBudgetMs: number; + allowSymlinks: boolean; +} + +export const DEFAULT_MD_INGEST_CAPS: MdIngestCaps = { + maxBytes: MEMORY_DEFAULTS.markdownMaxBytes, + maxSections: MEMORY_DEFAULTS.markdownMaxSections, + maxSectionBytes: MEMORY_DEFAULTS.markdownMaxSectionBytes, + parserBudgetMs: MEMORY_DEFAULTS.markdownParserBudgetMs, + allowSymlinks: false, +}; + +export type MdIngestSkipReason = + | 'feature_disabled' + | 'unsupported_path' + | 'symlink_disallowed' + | 'size_capped' + | 'invalid_encoding' + | 'unsafe_prompt_instruction' + | 'section_count_capped' + | 'section_size_capped' + | 'parser_budget_exceeded'; + +export interface MdIngestSection { + class: MdIngestSectionClass; + heading: string; + text: string; + fingerprint: string; + origin: typeof MD_INGEST_ORIGIN; +} + +export interface MdIngestResult { + sections: MdIngestSection[]; + skipped: Array<{ reason: MdIngestSkipReason; heading?: string }>; + partial: boolean; +} + +export interface ParseMdIngestOptions { + path: string; + content: string | Uint8Array; + scopeKey: string; + featureEnabled: boolean; + isSymlink?: boolean; + caps?: Partial; + telemetry?: Pick; +} + +const SECTION_CLASS_BY_HEADING: Array<[RegExp, MdIngestSectionClass]> = [ + [/preferences?|prefs?/i, 'preference'], + [/workflow|process|playbook/i, 'workflow'], + [/code\s*patterns?|patterns?/i, 'code_pattern'], + [/notes?|memory/i, 'note'], +]; + +function capsWithDefaults(caps?: Partial): MdIngestCaps { + return { ...DEFAULT_MD_INGEST_CAPS, ...caps }; +} + +function utf8Bytes(text: string): number { + return new TextEncoder().encode(text).byteLength; +} + +export function isSupportedMdIngestPath(path: string): path is MdIngestSupportedPath { + const normalized = path.replace(/\\/g, '/').replace(/^\.\//, ''); + return MD_INGEST_SUPPORTED_PATHS.includes(normalized as MdIngestSupportedPath); +} + +function decodeUtf8(input: string | Uint8Array): string | null { + if (typeof input === 'string') return input; + try { + return new TextDecoder('utf-8', { fatal: true }).decode(input); + } catch { + return null; + } +} + +function classifyHeading(heading: string): MdIngestSectionClass { + return SECTION_CLASS_BY_HEADING.find(([pattern]) => pattern.test(heading))?.[1] ?? 'note'; +} + +function containsUnsafePromptInstruction(text: string): boolean { + return /ignore\s+(all\s+)?(previous|prior)\s+(system|developer)?\s*instructions|developer\s+message|system\s+prompt/i.test(text); +} + +export function parseMdIngestDocument(options: ParseMdIngestOptions): MdIngestResult { + const caps = capsWithDefaults(options.caps); + const startedAt = Date.now(); + const skipped: MdIngestResult['skipped'] = []; + const budgetExceeded = (): boolean => caps.parserBudgetMs < 0 || Date.now() - startedAt > caps.parserBudgetMs; + const skipAll = (reason: MdIngestSkipReason): MdIngestResult => { + recordMemorySoftFailure(options.telemetry, 'md_ingest', reason, { outcome: reason === 'feature_disabled' ? 'disabled' : 'rejected' }); + return { sections: [], skipped: [{ reason }], partial: false }; + }; + if (!options.featureEnabled) return skipAll('feature_disabled'); + if (!isSupportedMdIngestPath(options.path)) return skipAll('unsupported_path'); + if (options.isSymlink && !caps.allowSymlinks) return skipAll('symlink_disallowed'); + + const content = decodeUtf8(options.content); + if (content === null) return skipAll('invalid_encoding'); + if (utf8Bytes(content) > caps.maxBytes) return skipAll('size_capped'); + + const lines = content.replace(/\r\n?/g, '\n').split('\n'); + const rawSections: Array<{ heading: string; text: string }> = []; + let current: { heading: string; lines: string[] } | null = null; + for (const line of lines) { + if (budgetExceeded()) { + skipped.push({ reason: 'parser_budget_exceeded', heading: current?.heading }); + recordMemorySoftFailure(options.telemetry, 'md_ingest', 'parser_budget_exceeded', { outcome: 'dropped' }); + break; + } + const headingMatch = /^(#{1,3})\s+(.+?)\s*$/.exec(line); + if (headingMatch) { + if (current) rawSections.push({ heading: current.heading, text: current.lines.join('\n').trim() }); + current = { heading: headingMatch[2] ?? 'Notes', lines: [] }; + continue; + } + if (!current) current = { heading: 'Notes', lines: [] }; + current.lines.push(line); + } + if (current) rawSections.push({ heading: current.heading, text: current.lines.join('\n').trim() }); + + const sections: MdIngestSection[] = []; + for (const section of rawSections) { + if (budgetExceeded()) { + skipped.push({ reason: 'parser_budget_exceeded', heading: section.heading }); + recordMemorySoftFailure(options.telemetry, 'md_ingest', 'parser_budget_exceeded', { outcome: 'dropped' }); + break; + } + if (!section.text) continue; + if (sections.length >= caps.maxSections) { + skipped.push({ reason: 'section_count_capped', heading: section.heading }); + recordMemorySoftFailure(options.telemetry, 'md_ingest', 'section_count_capped', { outcome: 'dropped' }); + break; + } + if (utf8Bytes(section.text) > caps.maxSectionBytes) { + skipped.push({ reason: 'section_size_capped', heading: section.heading }); + recordMemorySoftFailure(options.telemetry, 'md_ingest', 'section_size_capped', { outcome: 'dropped' }); + continue; + } + if (containsUnsafePromptInstruction(section.text)) { + skipped.push({ reason: 'unsafe_prompt_instruction', heading: section.heading }); + recordMemorySoftFailure(options.telemetry, 'md_ingest', 'unsafe_prompt_instruction', { outcome: 'rejected' }); + continue; + } + const klass = classifyHeading(section.heading); + sections.push({ + class: klass, + heading: section.heading, + text: section.text, + fingerprint: computeMemoryFingerprint({ + kind: klass === 'preference' ? 'preference' : 'note', + content: section.text, + scopeKey: options.scopeKey, + }), + origin: MD_INGEST_ORIGIN, + }); + } + + return { sections, skipped, partial: skipped.length > 0 && sections.length > 0 }; +} diff --git a/shared/memory-content-hash.ts b/shared/memory-content-hash.ts new file mode 100644 index 000000000..313cd2b67 --- /dev/null +++ b/shared/memory-content-hash.ts @@ -0,0 +1,48 @@ +import { createHash } from 'node:crypto'; + +const PROJECTION_SYSTEM_METADATA_KEYS = new Set([ + 'ownerUserId', + 'ownedByUserId', + 'createdByUserId', + 'updatedByUserId', + 'authorUserId', +]); + +export function stableJson(value: unknown): string { + if (value == null) return 'null'; + if (typeof value !== 'object') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map((entry) => stableJson(entry)).join(',')}]`; + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`).join(',')}}`; +} + +/** + * Projection content may carry trusted management metadata used for owner / + * creator authorization. That metadata is not semantic memory content: it must + * not affect citation drift hashes, embedding sources, recall matching, or + * other model-visible payloads. + * + * Only top-level reserved keys are stripped. Nested fields remain caller data. + */ +export function projectionSemanticContent(content: unknown): unknown { + if (!content || typeof content !== 'object' || Array.isArray(content)) return content; + const record = content as Record; + let stripped = false; + const semantic: Record = {}; + for (const [key, value] of Object.entries(record)) { + if (PROJECTION_SYSTEM_METADATA_KEYS.has(key)) { + stripped = true; + continue; + } + semantic[key] = value; + } + return stripped ? semantic : content; +} + +export function sha256Text(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +export function computeProjectionContentHash(input: { summary: string; content: unknown }): string { + return sha256Text(`projection-content:v1:${input.summary.trim()}\n${stableJson(projectionSemanticContent(input.content))}`); +} diff --git a/shared/memory-counters.ts b/shared/memory-counters.ts new file mode 100644 index 000000000..7b792825b --- /dev/null +++ b/shared/memory-counters.ts @@ -0,0 +1,100 @@ +export const MEMORY_COUNTERS = [ + 'mem.startup.silent_failure', + 'mem.startup.budget_exceeded', + 'mem.startup.stage_dropped', + 'mem.master_compaction.skipped', + 'mem.shutdown.master_drain.contract_violation', + 'mem.shutdown.master_drain.timed_out', + 'mem.archive_fts.unavailable', + 'mem.archive_fts.match_failure', + 'mem.config.invalid_value', + 'mem.config.invalid_redact_pattern', + 'mem.write.retry_exhausted', + 'mem.search.empty_results', + 'mem.search.scope_filter_hit', + 'mem.search.unauthorized_lookup', + 'mem.search.disabled', + 'mem.citation.created', + 'mem.citation.drift_observed', + 'mem.citation.count_incremented', + 'mem.citation.count_deduped', + 'mem.citation.count_rejected', + 'mem.citation.count_rate_limited', + 'mem.ingest.skipped_unsafe', + 'mem.ingest.scope_clamped', + 'mem.ingest.scope_dropped', + 'mem.ingest.size_capped', + 'mem.ingest.section_count_capped', + 'mem.skill.sanitize_rejected', + 'mem.skill.resolver_miss', + 'mem.skill.registry_oversize', + 'mem.skill.evidence_filtered', + 'mem.skill.evidence_evicted', + 'mem.skill.evidence_reset_on_restart', + 'mem.skill.collision_escaped', + 'mem.skill.layer_conflict_resolved', + 'mem.skill.review_throttled', + 'mem.skill.review_not_eligible', + 'mem.skill.review_deduped', + 'mem.skill.review_failed', + 'mem.classify.failed', + 'mem.classify.dedup_merge', + 'mem.preferences.untrusted_origin', + 'mem.preferences.persisted', + 'mem.preferences.persistence_failed', + 'mem.preferences.duplicate_ignored', + 'mem.preferences.rejected_untrusted', + 'mem.preferences.unauthorized_delete', + 'mem.observation.duplicate_ignored', + 'mem.observation.unauthorized_promotion_attempt', + 'mem.observation.unauthorized_query', + 'mem.observation.cross_scope_promotion_blocked', + 'mem.observation.backfill_repaired', + 'mem.bridge.unrouted_response', + 'mem.management.unauthorized', + 'mem.cache.invalidate_published', + 'mem.materialization.repair_triggered', + 'mem.materialization.compression_admission_closed', + 'mem.materialization.retry_exhausted_archived', + 'mem.materialization.archive_failed', + 'mem.materialization.durable_projection_failed', + 'mem.compression.queue_prior_failure', + 'mem.compression.admission_closed', + 'mem.pinned_notes_overflow', + 'mem.telemetry.buffer_overflow', +] as const; + +export type MemoryCounter = (typeof MEMORY_COUNTERS)[number]; + +export const MEMORY_SOFT_FAIL_PATH_COUNTERS = { + startup_memory: 'mem.startup.silent_failure', + search: 'mem.search.empty_results', + citation: 'mem.citation.count_rejected', + cite_count: 'mem.citation.count_rejected', + md_ingest: 'mem.ingest.skipped_unsafe', + skills: 'mem.skill.sanitize_rejected', + skill_review: 'mem.skill.review_failed', + preferences: 'mem.preferences.rejected_untrusted', + materialization: 'mem.materialization.repair_triggered', + observations: 'mem.observation.backfill_repaired', + classification: 'mem.classify.failed', +} as const satisfies Record; + +export type MemorySoftFailPath = keyof typeof MEMORY_SOFT_FAIL_PATH_COUNTERS; + +export const MEMORY_COUNTER_LABEL_ENUMS = [ + 'MemoryOrigin', + 'SendOrigin', + 'MemoryFeatureFlag', + 'FingerprintKind', + 'ObservationClass', + 'SkillReviewTrigger', +] as const; + +export type MemoryCounterLabelEnum = (typeof MEMORY_COUNTER_LABEL_ENUMS)[number]; + +const MEMORY_COUNTER_SET: ReadonlySet = new Set(MEMORY_COUNTERS); + +export function isMemoryCounter(value: unknown): value is MemoryCounter { + return typeof value === 'string' && MEMORY_COUNTER_SET.has(value); +} diff --git a/shared/memory-defaults.ts b/shared/memory-defaults.ts new file mode 100644 index 000000000..bba22dfc4 --- /dev/null +++ b/shared/memory-defaults.ts @@ -0,0 +1,25 @@ +export const MEMORY_DEFAULTS = { + startupTotalTokens: 8000, + pinnedTokens: 1600, + durableTokens: 4000, + recentTokens: 2400, + skillTokens: 1000, + projectDocsTokens: 2000, + markdownMaxBytes: 51200, + markdownMaxSections: 30, + markdownMaxSectionBytes: 16 * 1024, + markdownParserBudgetMs: 5000, + skillMaxBytes: 4096, + skillRegistryMaxBytes: 1024 * 1024, + skillRegistryMaxEntries: 1024, + featureFlagPropagationP99Ms: 60000, + skillReviewToolIterationThreshold: 10, + skillReviewMinIntervalMs: 600000, + skillReviewDailyLimit: 6, + skillReviewManualMinIntervalMs: 60000, + skillReviewManualDailyLimit: 50, + citationIdempotencyRetentionDays: 180, + preferenceIdempotencyRetentionDays: 180, +} as const; + +export type MemoryDefaults = typeof MEMORY_DEFAULTS; diff --git a/shared/memory-fingerprint.ts b/shared/memory-fingerprint.ts index b86cd1f5d..8948535bc 100644 --- a/shared/memory-fingerprint.ts +++ b/shared/memory-fingerprint.ts @@ -23,6 +23,58 @@ import { createHash } from 'node:crypto'; * durable_memory_candidate) are never cross-matched. */ +export const FINGERPRINT_KINDS = ['summary', 'preference', 'skill', 'decision', 'note'] as const; +export type FingerprintKind = (typeof FINGERPRINT_KINDS)[number]; + +export const MEMORY_FINGERPRINT_VERSIONS = ['v1'] as const; +export type MemoryFingerprintVersion = (typeof MEMORY_FINGERPRINT_VERSIONS)[number]; + +export interface ComputeMemoryFingerprintArgs { + kind: FingerprintKind; + content: string; + scopeKey?: string; + version?: MemoryFingerprintVersion; +} + +const MEMORY_FINGERPRINT_DOMAIN = 'imcodes:memory-fingerprint'; +const FRONT_MATTER_PATTERN = /^\uFEFF?---[ \t]*\n[\s\S]*?\n---[ \t]*(?:\n|$)/; + +function normalizeUnicodeAndLineEndings(content: string): string { + return content.normalize('NFC').replace(/\r\n?/g, '\n').replace(/\u0000/g, '\uFFFD'); +} + +function collapseWhitespace(content: string): string { + return content.replace(/\s+/gu, ' ').trim(); +} + +function normalizeCaseFoldedText(content: string): string { + return collapseWhitespace(normalizeUnicodeAndLineEndings(content)).toLocaleLowerCase('en-US'); +} + +function stripPreferencePrefixes(content: string): string { + return normalizeUnicodeAndLineEndings(content) + .split('\n') + .map((line) => line.replace(/^\s*@pref:\s*/iu, '')) + .join('\n'); +} + +function stripSkillFrontMatter(content: string): string { + return normalizeUnicodeAndLineEndings(content).replace(FRONT_MATTER_PATTERN, ''); +} + +function normalizeSkillContent(content: string): string { + return stripSkillFrontMatter(content) + .split('\n') + .map((line) => line.trimEnd()) + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +function normalizeNoteContent(content: string): string { + return collapseWhitespace(normalizeUnicodeAndLineEndings(content)); +} + /** Normalize a summary for equality-based dedup. * - lowercase (case-insensitive) * - collapse all whitespace runs to a single space @@ -32,13 +84,46 @@ import { createHash } from 'node:crypto'; * collapse by accident. */ export function normalizeSummaryForFingerprint(summary: string): string { - return summary.toLowerCase().replace(/\s+/g, ' ').trim(); + return normalizeCaseFoldedText(summary); +} + +export function normalizeContentForFingerprint(kind: FingerprintKind, content: string): string { + switch (kind) { + case 'summary': + return normalizeSummaryForFingerprint(content); + case 'preference': + return normalizeCaseFoldedText(stripPreferencePrefixes(content)); + case 'skill': + return normalizeSkillContent(content); + case 'decision': + return normalizeCaseFoldedText(content); + case 'note': + return normalizeNoteContent(content); + } +} + +/** + * Canonical post-1.1 memory fingerprint API. + * + * The hash preimage includes version, kind, scope/namespace key, and normalized + * content. Including `scopeKey` prevents otherwise-identical memories from + * being deduplicated across authorization or namespace boundaries. + */ +export function computeMemoryFingerprint(args: ComputeMemoryFingerprintArgs): string { + const version = args.version ?? 'v1'; + const normalized = normalizeContentForFingerprint(args.kind, args.content); + const normalizedScope = normalizeUnicodeAndLineEndings(args.scopeKey ?? '').trim(); + const preimage = [MEMORY_FINGERPRINT_DOMAIN, version, args.kind, normalizedScope, normalized].join('\u0000'); + return createHash('sha256').update(preimage, 'utf8').digest('hex'); } /** Deterministic content key for a processed projection. * Same (namespaceKey, class, normalized summary) always produces the same * string. Opaque by design — callers should treat it as a fingerprint, not * a parsable structure. + * + * @deprecated Internal legacy projection helper. New memory call sites should + * use `computeMemoryFingerprint({ kind, content, scopeKey, version: 'v1' })`. */ export function fingerprintProjection(args: { namespaceKey: string; @@ -46,15 +131,17 @@ export function fingerprintProjection(args: { summary: string; }): string { const normalized = normalizeSummaryForFingerprint(args.summary); - // Use a simple null-separated join. The individual components never contain - // U+0000 by contract (namespaceKey is a slash-separated path, class is a - // fixed enum, summary is user-facing text), so this is unambiguous without - // needing a real hash function that would pull in crypto on hot paths. + // Keep the historical un-hashed key shape for existing local callers. return `${args.namespaceKey}\u0000${args.projectionClass}\u0000${normalized}`; } - -/** Return a stable SHA-256 hex fingerprint for already-normalized memory text. */ +/** + * Return a stable SHA-256 hex fingerprint for already-normalized memory text. + * + * @deprecated Internal summary-only helper. New memory call sites should use + * `computeMemoryFingerprint()` so the kind, version, and scope are in the + * fingerprint preimage. + */ export function computeFingerprint(normalizedSummary: string): string { return createHash('sha256').update(normalizedSummary, 'utf8').digest('hex'); } diff --git a/shared/memory-management-context.ts b/shared/memory-management-context.ts new file mode 100644 index 000000000..e06d6c8b6 --- /dev/null +++ b/shared/memory-management-context.ts @@ -0,0 +1,36 @@ +export const MEMORY_MANAGEMENT_CONTEXT_FIELD = '_memoryManagementContext' as const; + +export const MEMORY_MANAGEMENT_ROLES = ['user', 'workspace_admin', 'org_admin'] as const; +export type MemoryManagementRole = (typeof MEMORY_MANAGEMENT_ROLES)[number]; + +export interface MemoryManagementBoundProject { + projectDir?: string; + canonicalRepoId?: string; + workspaceId?: string; + orgId?: string; +} + +export interface AuthenticatedMemoryManagementContext { + actorId: string; + userId: string; + role: MemoryManagementRole; + serverId?: string; + requestId?: string; + boundProjects?: readonly MemoryManagementBoundProject[]; + source: 'server_bridge' | 'local_daemon'; +} + +export function isMemoryManagementRole(value: unknown): value is MemoryManagementRole { + return typeof value === 'string' && (MEMORY_MANAGEMENT_ROLES as readonly string[]).includes(value); +} + +export function isAuthenticatedMemoryManagementContext(value: unknown): value is AuthenticatedMemoryManagementContext { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const record = value as Record; + return typeof record.actorId === 'string' + && record.actorId.trim().length > 0 + && typeof record.userId === 'string' + && record.userId.trim().length > 0 + && isMemoryManagementRole(record.role) + && (record.source === 'server_bridge' || record.source === 'local_daemon'); +} diff --git a/shared/memory-management.ts b/shared/memory-management.ts new file mode 100644 index 000000000..271b5d773 --- /dev/null +++ b/shared/memory-management.ts @@ -0,0 +1,141 @@ +import type { FeatureFlagValueSource, MemoryFeatureFlag } from './feature-flags.js'; +import type { MemoryScope } from './memory-scope.js'; +import type { ObservationClass, ObservationState } from './memory-observation.js'; +import type { MemoryOrigin } from './memory-origin.js'; +import type { SkillRegistryEntry } from './skill-registry-types.js'; + +export const MEMORY_MANAGEMENT_ERROR_CODES = { + ACTION_FAILED: 'action_failed', + FEATURE_DISABLED: 'feature_disabled', + MISSING_PREFERENCE_TEXT: 'missing_preference_text', + MISSING_MEMORY_TEXT: 'missing_memory_text', + MISSING_ID: 'missing_id', + MEMORY_NOT_FOUND: 'memory_not_found', + PREFERENCE_NOT_FOUND: 'preference_not_found', + PREFERENCE_FORBIDDEN_OWNER: 'preference_forbidden_owner', + MISSING_PROJECT_DIR: 'missing_project_dir', + MISSING_PROJECT_IDENTITY: 'missing_project_identity', + INVALID_PROJECT_DIR: 'invalid_project_dir', + PROJECT_IDENTITY_MISMATCH: 'project_identity_mismatch', + INVALID_TARGET_SCOPE: 'invalid_target_scope', + PROMOTION_REQUIRES_AUTHORIZATION: 'promotion_requires_authorization', + MISSING_EXPECTED_FROM_SCOPE: 'missing_expected_from_scope', + OBSERVATION_FROM_SCOPE_MISMATCH: 'observation_from_scope_mismatch', + OBSERVATION_QUERY_FORBIDDEN: 'observation_query_forbidden', + UNSUPPORTED_MD_INGEST_SCOPE: 'unsupported_md_ingest_scope', + MANAGEMENT_REQUEST_UNROUTED: 'management_request_unrouted', + INVALID_FEATURE_FLAG: 'invalid_feature_flag', + FEATURE_CONFIG_WRITE_FAILED: 'feature_config_write_failed', + MISSING_OBSERVATION_TEXT: 'missing_observation_text', + OBSERVATION_NOT_FOUND: 'observation_not_found', + OBSERVATION_MUTATION_FORBIDDEN: 'observation_mutation_forbidden', + SKILL_PATH_NOT_READABLE: 'skill_path_not_readable', + SKILL_FILE_TOO_LARGE: 'skill_file_too_large', + SKILL_NOT_FOUND: 'skill_not_found', + SKILL_OUTSIDE_MANAGED_ROOTS: 'skill_outside_managed_roots', + REGISTRY_FILE_TOO_LARGE: 'registry_file_too_large', + REGISTRY_ENTRY_LIMIT_EXCEEDED: 'registry_entry_limit_exceeded', +} as const; + +export type MemoryManagementErrorCode = (typeof MEMORY_MANAGEMENT_ERROR_CODES)[keyof typeof MEMORY_MANAGEMENT_ERROR_CODES]; + +export const MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES = { + UNAUTHENTICATED: 'memory_management_unauthenticated', + TOO_MANY_PENDING_REQUESTS: 'too_many_memory_management_requests', + MISSING_REQUEST_ID: 'missing_request_id', + DUPLICATE_REQUEST_ID: 'duplicate_request_id', + CONTEXT_INJECTION_FAILED: 'context_injection_failed', +} as const; + +export type MemoryManagementBridgeErrorCode = (typeof MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES)[keyof typeof MEMORY_MANAGEMENT_BRIDGE_ERROR_CODES]; + +export interface MemoryFeatureAdminRecord { + flag: MemoryFeatureFlag; + enabled: boolean; + requested: boolean; + source: FeatureFlagValueSource; + envKey: string; + dependencies: readonly MemoryFeatureFlag[]; + dependencyBlocked: readonly MemoryFeatureFlag[]; + disabledBehavior: string; +} + +export interface MemoryFeatureAdminResponse { + requestId?: string; + records: MemoryFeatureAdminRecord[]; +} + +export interface MemoryFeatureSetResponse { + requestId?: string; + success: boolean; + flag?: MemoryFeatureFlag; + requested?: boolean; + enabled?: boolean; + records?: MemoryFeatureAdminRecord[]; + error?: string; + errorCode?: MemoryManagementErrorCode; +} + +export interface MemoryPreferenceAdminRecord { + id: string; + userId: string; + ownerUserId?: string; + createdByUserId?: string; + updatedByUserId?: string; + text: string; + fingerprint: string; + origin: MemoryOrigin; + state: ObservationState; + updatedAt: number; + createdAt: number; +} + +export interface MemoryPreferenceAdminResponse { + requestId?: string; + records: MemoryPreferenceAdminRecord[]; + featureEnabled?: boolean; +} + +export interface MemorySkillAdminRecord { + key: string; + layer: string; + name: string; + category: string; + description?: string; + displayPath: string; + uri: string; + fingerprint: string; + updatedAt: number; + enforcement?: string; + project?: SkillRegistryEntry['project']; +} + +export interface MemorySkillAdminResponse { + requestId?: string; + entries: MemorySkillAdminRecord[]; + sourceCounts?: Record; + featureEnabled?: boolean; +} + +export interface MemoryObservationAdminRecord { + id: string; + scope: MemoryScope; + class: ObservationClass; + origin: MemoryOrigin; + state: ObservationState; + ownerUserId?: string; + createdByUserId?: string; + updatedByUserId?: string; + text: string; + fingerprint: string; + namespaceId: string; + projectionId?: string; + updatedAt: number; + createdAt: number; +} + +export interface MemoryObservationAdminResponse { + requestId?: string; + records: MemoryObservationAdminRecord[]; + featureEnabled?: boolean; +} diff --git a/shared/memory-namespace.ts b/shared/memory-namespace.ts new file mode 100644 index 000000000..21db22c86 --- /dev/null +++ b/shared/memory-namespace.ts @@ -0,0 +1,334 @@ +import { + assertMemoryScopeIdentity, + getMemoryScopePolicy, + type MemoryScope, + type MemoryScopeIdentity, +} from './memory-scope.js'; +import type { ContextNamespace as LegacyContextNamespace } from './context-types.js'; + +export type MemoryNamespaceVisibility = 'owner_private' | 'shared_authorized'; + +export interface MemoryNamespaceInput { + scope: MemoryScope; + tenantId?: string; + userId?: string; + canonicalRepoId?: string; + projectId?: string; + workspaceId?: string; + orgId?: string; + rootSessionId?: string; + sessionTreeId?: string; + sessionId?: string; + name?: string; +} + +export interface ContextNamespace { + scope: MemoryScope; + key: string; + visibility: MemoryNamespaceVisibility; + tenantId?: string; + userId?: string; + projectId?: string; + canonicalRepoId?: string; + workspaceId?: string; + orgId?: string; + rootSessionId?: string; + sessionTreeId?: string; + sessionId?: string; + name?: string; +} + +export interface CanonicalNamespaceInput { + scope: MemoryScope; + localTenant?: string; + tenantId?: string; + userId?: string; + canonicalRepoId?: string; + projectId?: string; + workspaceId?: string; + orgId?: string; + enterpriseId?: string; + rootSessionId?: string; + sessionTreeId?: string; + sessionId?: string; + key?: string; + visibility?: 'private' | 'shared' | MemoryNamespaceVisibility; + name?: string; +} + +export interface ContextNamespaceBinding { + localTenant: string; + scope: MemoryScope; + userId?: string; + rootSessionId?: string; + sessionTreeId?: string; + sessionId?: string; + workspaceId?: string; + projectId?: string; + orgId?: string; + key: string; + visibility: 'private' | 'shared'; +} + +function encodeNamespaceSegment(value: string): string { + return encodeURIComponent(value.normalize('NFC').trim()).replace(/%2F/gi, '%252F'); +} + +function pushPart(parts: string[], label: string, value: string | undefined): void { + if (typeof value === 'string' && value.trim().length > 0) { + parts.push(`${label}:${encodeNamespaceSegment(value)}`); + } +} + +function scopeIdentityFor(input: MemoryNamespaceInput, projectId: string | undefined): MemoryScopeIdentity { + return { + tenant_id: input.tenantId, + user_id: input.userId, + project_id: projectId, + workspace_id: input.workspaceId, + org_id: input.orgId, + root_session_id: input.rootSessionId, + session_tree_id: input.sessionTreeId, + session_id: input.sessionId, + }; +} + +export function canonicalProjectIdForNamespace(input: Pick): string | undefined { + return input.canonicalRepoId ?? input.projectId; +} + +export function createMemoryNamespace(input: MemoryNamespaceInput): ContextNamespace { + const projectId = canonicalProjectIdForNamespace(input); + assertMemoryScopeIdentity(input.scope, scopeIdentityFor(input, projectId)); + + const parts = [`scope:${input.scope}`]; + pushPart(parts, 'tenant', input.tenantId); + pushPart(parts, 'user', input.userId); + pushPart(parts, 'project', projectId); + pushPart(parts, 'workspace', input.workspaceId); + pushPart(parts, 'org', input.orgId); + pushPart(parts, 'root_session', input.rootSessionId); + pushPart(parts, 'session_tree', input.sessionTreeId); + pushPart(parts, 'session', input.sessionId); + pushPart(parts, 'name', input.name ?? 'default'); + + const policy = getMemoryScopePolicy(input.scope); + return { + scope: input.scope, + key: parts.join('/'), + visibility: policy.ownerPrivate ? 'owner_private' : 'shared_authorized', + tenantId: input.tenantId, + userId: input.userId, + projectId, + canonicalRepoId: input.canonicalRepoId, + workspaceId: input.workspaceId, + orgId: input.orgId, + rootSessionId: input.rootSessionId, + sessionTreeId: input.sessionTreeId, + sessionId: input.sessionId, + name: input.name, + }; +} + +export function createUserPrivateNamespace(input: Omit & { projectId?: string }): ContextNamespace { + return createMemoryNamespace({ ...input, scope: 'user_private' }); +} + +export function createPersonalNamespace(input: Omit): ContextNamespace { + return createMemoryNamespace({ ...input, scope: 'personal' }); +} + +export function createProjectSharedNamespace(input: Omit): ContextNamespace { + return createMemoryNamespace({ ...input, scope: 'project_shared' }); +} + +export function createWorkspaceSharedNamespace(input: Omit): ContextNamespace { + return createMemoryNamespace({ ...input, scope: 'workspace_shared' }); +} + +export function createOrgSharedNamespace(input: Omit): ContextNamespace { + return createMemoryNamespace({ ...input, scope: 'org_shared' }); +} + +function clean(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function stripGitSuffix(value: string): string { + return value.endsWith('.git') ? value.slice(0, -4) : value; +} + +export function normalizeCanonicalRepoId(raw: string | undefined): string | undefined { + const value = clean(raw); + if (!value) return undefined; + const lower = value.toLowerCase(); + const sshMatch = lower.match(/^git@([^:]+):(.+)$/); + if (sshMatch) return stripGitSuffix(`${sshMatch[1]}/${sshMatch[2]}`).replace(/\/+/g, '/'); + try { + const url = new URL(lower); + if (url.hostname && url.pathname) { + const host = url.protocol === 'ssh:' ? url.hostname : url.host; + return stripGitSuffix(`${host}/${url.pathname.replace(/^\/+|\/+$/g, '')}`).replace(/\/+/g, '/'); + } + } catch { + // Plain canonical keys such as github.com/owner/repo are accepted below. + } + return stripGitSuffix(lower).replace(/^\/+|\/+$/g, '').replace(/\/+/g, '/'); +} + +function bindingVisibility(input: CanonicalNamespaceInput, ownerPrivate: boolean): 'private' | 'shared' { + if (input.visibility === 'private' || input.visibility === 'shared') return input.visibility; + return ownerPrivate ? 'private' : 'shared'; +} + +export function buildNamespaceKey(input: CanonicalNamespaceInput): string { + const projectId = normalizeCanonicalRepoId(input.canonicalRepoId ?? input.projectId); + const parts = [ + 'ctxns:v1', + input.scope, + clean(input.userId) ?? '', + clean(input.orgId ?? input.enterpriseId) ?? '', + clean(input.workspaceId) ?? '', + projectId ?? '', + clean(input.rootSessionId ?? input.sessionTreeId) ?? '', + clean(input.sessionId) ?? '', + clean(input.name ?? 'default') ?? 'default', + ]; + return parts.map((part) => encodeURIComponent(part)).join(':'); +} + +export function createContextNamespaceBinding(input: CanonicalNamespaceInput): ContextNamespaceBinding { + const projectId = normalizeCanonicalRepoId(input.canonicalRepoId ?? input.projectId); + const orgId = clean(input.orgId ?? input.enterpriseId); + assertMemoryScopeIdentity(input.scope, { + tenant_id: input.tenantId ?? input.localTenant, + user_id: input.userId, + project_id: projectId, + workspace_id: input.workspaceId, + org_id: orgId, + root_session_id: input.rootSessionId, + session_tree_id: input.sessionTreeId, + session_id: input.sessionId, + }); + const policy = getMemoryScopePolicy(input.scope); + return { + localTenant: clean(input.localTenant ?? input.tenantId) ?? 'daemon-local', + scope: input.scope, + userId: clean(input.userId), + rootSessionId: clean(input.rootSessionId), + sessionTreeId: clean(input.sessionTreeId ?? input.rootSessionId), + sessionId: clean(input.sessionId), + workspaceId: clean(input.workspaceId), + projectId, + orgId, + key: input.key?.trim() || buildNamespaceKey({ ...input, projectId }), + visibility: bindingVisibility(input, policy.ownerPrivate), + }; +} + +export function contextNamespaceToBinding(namespace: LegacyContextNamespace, options: { + localTenant?: string; + rootSessionId?: string; + sessionTreeId?: string; + sessionId?: string; + key?: string; +} = {}): ContextNamespaceBinding { + return createContextNamespaceBinding({ + localTenant: options.localTenant, + scope: namespace.scope as MemoryScope, + userId: namespace.userId, + workspaceId: namespace.workspaceId, + projectId: namespace.projectId, + orgId: namespace.enterpriseId, + enterpriseId: namespace.enterpriseId, + rootSessionId: options.rootSessionId, + sessionTreeId: options.sessionTreeId, + sessionId: options.sessionId, + key: options.key, + }); +} + +export function bindingToContextNamespace(binding: ContextNamespaceBinding): LegacyContextNamespace { + return { + scope: binding.scope as LegacyContextNamespace['scope'], + projectId: binding.projectId ?? '', + userId: binding.userId, + workspaceId: binding.workspaceId, + enterpriseId: binding.orgId, + }; +} + +export function bindSessionTreeContext(input: T, rootSessionId: string, sessionId?: string): ContextNamespaceBinding { + return createContextNamespaceBinding({ + ...input, + rootSessionId, + sessionTreeId: rootSessionId, + sessionId, + }); +} + +export function sameRootSessionTree(a: Pick, b: Pick): boolean { + const aRoot = a.rootSessionId ?? a.sessionTreeId; + const bRoot = b.rootSessionId ?? b.sessionTreeId; + return Boolean(aRoot && bRoot && aRoot === bRoot); +} + +export function sameCanonicalProject(a: Pick, b: Pick): boolean { + return Boolean(a.projectId && b.projectId && normalizeCanonicalRepoId(a.projectId) === normalizeCanonicalRepoId(b.projectId)); +} + +export interface RuntimeContextBinding { + userId?: string; + projectId?: string; + canonicalRepoId?: string; + rootSessionId?: string; + sessionTreeId?: string; + sessionId?: string; +} + +export function isSessionTreeBoundContext(binding: Pick): boolean { + return Boolean(binding.rootSessionId || binding.sessionTreeId || binding.sessionId); +} + +/** + * Decide whether a namespace binding is visible to a runtime session without + * introducing a new session-tree authorization scope. + * + * Session tree ids only bind context inside one tree. Cross-device project + * visibility comes from canonical project identity (`canonicalRepoId` / + * `projectId`), not from local paths, machine ids, or session ids. + */ +export function contextBindingVisibleToRuntime( + binding: ContextNamespaceBinding, + runtime: RuntimeContextBinding, +): boolean { + const runtimeProjectId = normalizeCanonicalRepoId(runtime.canonicalRepoId ?? runtime.projectId); + const runtimeBinding: Pick = { + projectId: runtimeProjectId, + rootSessionId: clean(runtime.rootSessionId), + sessionTreeId: clean(runtime.sessionTreeId ?? runtime.rootSessionId), + }; + + if (isSessionTreeBoundContext(binding)) { + return sameRootSessionTree(binding, runtimeBinding); + } + + if (binding.scope === 'user_private') { + return Boolean(binding.userId && runtime.userId && binding.userId === runtime.userId); + } + if (binding.scope === 'personal') { + return Boolean( + binding.userId + && runtime.userId + && binding.userId === runtime.userId + && sameCanonicalProject(binding, runtimeBinding), + ); + } + if (binding.scope === 'project_shared') { + return sameCanonicalProject(binding, runtimeBinding); + } + // Workspace/org membership authorization is enforced by the caller/server + // layer; this helper only prevents project/session identity drift. + return true; +} diff --git a/shared/memory-observation.ts b/shared/memory-observation.ts new file mode 100644 index 000000000..1064922ab --- /dev/null +++ b/shared/memory-observation.ts @@ -0,0 +1,146 @@ +import type { MemoryOrigin } from './memory-origin.js'; +import type { MemoryScope } from './memory-scope.js'; + +export const OBSERVATION_CLASSES = [ + 'fact', + 'decision', + 'bugfix', + 'feature', + 'refactor', + 'discovery', + 'preference', + 'skill_candidate', + 'workflow', + 'code_pattern', + 'note', +] as const; + +export type ObservationClass = (typeof OBSERVATION_CLASSES)[number]; + +export const OBSERVATION_STATES = ['candidate', 'active', 'superseded', 'rejected', 'promoted'] as const; +export type ObservationState = (typeof OBSERVATION_STATES)[number]; + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; +export interface JsonObject { + readonly [key: string]: JsonValue; +} + +export interface ObservationContent { + readonly [key: string]: JsonValue | undefined; + readonly text: string; + readonly title?: string; + readonly tags?: readonly string[]; +} + +export interface ContextObservationDraft { + namespaceId: string; + scope: MemoryScope; + class: ObservationClass; + origin: MemoryOrigin; + fingerprint: string; + content: ObservationContent; + sourceEventIds?: readonly string[]; + projectionId?: string; + state?: ObservationState; + confidence?: number; +} + +export interface ContextObservationInput { + namespaceId: string; + scope: MemoryScope; + class: ObservationClass; + origin: MemoryOrigin; + fingerprint: string; + content: Record; + text?: string; + textHash?: string; + sourceEventIds?: readonly string[]; + projectionId?: string; + state?: ObservationState; + confidence?: number; + id?: string; + now?: number; +} + +const OBSERVATION_CLASS_SET: ReadonlySet = new Set(OBSERVATION_CLASSES); +const OBSERVATION_STATE_SET: ReadonlySet = new Set(OBSERVATION_STATES); + +export function isObservationClass(value: unknown): value is ObservationClass { + return typeof value === 'string' && OBSERVATION_CLASS_SET.has(value); +} + +export function isObservationState(value: unknown): value is ObservationState { + return typeof value === 'string' && OBSERVATION_STATE_SET.has(value); +} + +function isJsonValue(value: unknown): value is JsonValue { + if (value === null) return true; + const valueType = typeof value; + if (valueType === 'string' || valueType === 'number' || valueType === 'boolean') { + return valueType !== 'number' || Number.isFinite(value); + } + if (Array.isArray(value)) return value.every(isJsonValue); + if (valueType === 'object') { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return false; + return Object.values(value as Record).every(isJsonValue); + } + return false; +} + +export function validateObservationContent( + observationClass: ObservationClass, + content: unknown, +): { ok: true; value: ObservationContent } | { ok: false; reason: string } { + if (!isObservationClass(observationClass)) { + return { ok: false, reason: `Unknown observation class: ${String(observationClass)}` }; + } + if (!isJsonValue(content) || content === null || Array.isArray(content) || typeof content !== 'object') { + return { ok: false, reason: 'Observation content must be a JSON object' }; + } + const record = content as Record; + if (record.class === 'memory_note') { + return { ok: false, reason: 'Use canonical observation class "note" instead of "memory_note"' }; + } + if (typeof record.text !== 'string' || record.text.trim().length === 0) { + return { ok: false, reason: 'Observation content requires non-empty text' }; + } + if (record.tags !== undefined && (!Array.isArray(record.tags) || !record.tags.every((tag) => typeof tag === 'string'))) { + return { ok: false, reason: 'Observation content tags must be strings' }; + } + return { ok: true, value: record as unknown as ObservationContent }; +} + +export function assertObservationContent(observationClass: ObservationClass, content: unknown): ObservationContent { + const result = validateObservationContent(observationClass, content); + if (result.ok) return result.value; + throw new Error(result.reason); +} + +export function normalizeObservationText(text: string): string { + return text.trim().replace(/\s+/g, ' ').toLowerCase(); +} + +export function normalizeObservationSourceIds(sourceEventIds: readonly string[] | undefined): string[] { + const out: string[] = []; + const seen = new Set(); + for (const raw of sourceEventIds ?? []) { + const value = raw.trim(); + if (!value || seen.has(value)) continue; + seen.add(value); + out.push(value); + } + return out; +} + +export function assertValidObservationInput(input: ContextObservationInput): void { + if (!input.namespaceId.trim()) throw new Error('namespaceId is required'); + if (!input.fingerprint.trim()) throw new Error('fingerprint is required'); + if (!isObservationClass(input.class)) throw new Error(`invalid observation class: ${String(input.class)}`); + if (!isObservationState(input.state ?? 'active')) throw new Error(`invalid observation state: ${String(input.state)}`); + if (input.confidence !== undefined && (!Number.isFinite(input.confidence) || input.confidence < 0 || input.confidence > 1)) { + throw new Error('confidence must be between 0 and 1'); + } + assertObservationContent(input.class, input.content); +} diff --git a/shared/memory-origin.ts b/shared/memory-origin.ts new file mode 100644 index 000000000..a906f1504 --- /dev/null +++ b/shared/memory-origin.ts @@ -0,0 +1,39 @@ +export const MEMORY_ORIGINS = [ + 'chat_compacted', + 'user_note', + 'skill_import', + 'manual_pin', + 'agent_learned', + 'md_ingest', +] as const; + +export type MemoryOrigin = (typeof MEMORY_ORIGINS)[number]; + +export const RESERVED_MEMORY_ORIGINS = ['quick_search_cache'] as const; +export type ReservedMemoryOrigin = (typeof RESERVED_MEMORY_ORIGINS)[number]; + +const MEMORY_ORIGIN_SET: ReadonlySet = new Set(MEMORY_ORIGINS); +const RESERVED_MEMORY_ORIGIN_SET: ReadonlySet = new Set(RESERVED_MEMORY_ORIGINS); + +export function isMemoryOrigin(value: unknown): value is MemoryOrigin { + return typeof value === 'string' && MEMORY_ORIGIN_SET.has(value); +} + +export function isReservedMemoryOrigin(value: unknown): value is ReservedMemoryOrigin { + return typeof value === 'string' && RESERVED_MEMORY_ORIGIN_SET.has(value); +} + +export function assertMemoryOrigin(value: unknown): MemoryOrigin { + if (isMemoryOrigin(value)) return value; + if (isReservedMemoryOrigin(value)) { + throw new Error(`Reserved memory origin is not emit-safe in this milestone: ${value}`); + } + throw new Error(`Unknown memory origin: ${String(value)}`); +} + +export function requireExplicitMemoryOrigin(value: unknown, context = 'memory write'): MemoryOrigin { + if (value == null || value === '') { + throw new Error(`Missing explicit memory origin for ${context}`); + } + return assertMemoryOrigin(value); +} diff --git a/shared/memory-project-options.ts b/shared/memory-project-options.ts new file mode 100644 index 000000000..05a3988e4 --- /dev/null +++ b/shared/memory-project-options.ts @@ -0,0 +1,61 @@ +export const MEMORY_PROJECT_RESOLUTION_STATUSES = [ + 'resolved', + 'needs_resolution', + 'canonical_only', + 'directory_only', + 'no_repo', + 'multiple_remotes', + 'unauthorized', + 'invalid_dir', + 'mismatch', + 'error', +] as const; + +export type MemoryProjectResolutionStatus = (typeof MEMORY_PROJECT_RESOLUTION_STATUSES)[number]; + +export const MEMORY_PROJECT_OPTION_SOURCES = [ + 'active_session', + 'recent_session', + 'enterprise_enrollment', + 'memory_index', + 'resolved_directory', + 'manual_resolved', +] as const; + +export type MemoryProjectOptionSource = (typeof MEMORY_PROJECT_OPTION_SOURCES)[number]; + +export interface MemoryProjectOption { + id: string; + displayName: string; + canonicalRepoId?: string; + projectDir?: string; + source: MemoryProjectOptionSource; + status: MemoryProjectResolutionStatus; + lastSeenAt?: number; +} + +export interface MemoryProjectCapabilities { + canFilterMemory: boolean; + canRunLocalTools: boolean; +} + +export function deriveMemoryProjectCapabilities(option: MemoryProjectOption | null | undefined): MemoryProjectCapabilities { + const hasCanonicalRepoId = Boolean(option?.canonicalRepoId?.trim()); + const hasProjectDir = Boolean(option?.projectDir?.trim()); + const resolved = option?.status === 'resolved'; + return { + canFilterMemory: hasCanonicalRepoId, + canRunLocalTools: hasCanonicalRepoId && hasProjectDir && resolved, + }; +} + +export interface MemoryProjectResolveResponsePayload { + requestId?: string; + success: boolean; + projectDir?: string; + canonicalRepoId?: string; + displayName?: string; + status: MemoryProjectResolutionStatus; + error?: string; + errorCode?: string; +} diff --git a/shared/memory-recall-format.ts b/shared/memory-recall-format.ts index 9cd5fd919..f8df802a3 100644 --- a/shared/memory-recall-format.ts +++ b/shared/memory-recall-format.ts @@ -12,6 +12,7 @@ export interface RelatedPastWorkRenderableItem { export const RELATED_PAST_WORK_HEADER = '[Related past work]'; export const STARTUP_PROJECT_MEMORY_HEADER = '# Recent project memory (reference only)'; +export const STARTUP_SKILL_INDEX_HEADER = '# Available skills (read on demand)'; export function formatRelatedPastWorkSummary(summary: string, maxLength = 200): string { return summary.split('\n')[0]?.slice(0, maxLength) ?? ''; diff --git a/shared/memory-render-kind.ts b/shared/memory-render-kind.ts new file mode 100644 index 000000000..0377d7bd5 --- /dev/null +++ b/shared/memory-render-kind.ts @@ -0,0 +1,8 @@ +export const MEMORY_RENDER_KINDS = ['summary', 'preference', 'note', 'skill', 'pinned', 'citation_preview'] as const; +export type MemoryRenderKind = (typeof MEMORY_RENDER_KINDS)[number]; + +const MEMORY_RENDER_KIND_SET: ReadonlySet = new Set(MEMORY_RENDER_KINDS); + +export function isMemoryRenderKind(value: unknown): value is MemoryRenderKind { + return typeof value === 'string' && MEMORY_RENDER_KIND_SET.has(value); +} diff --git a/shared/memory-render-policy.ts b/shared/memory-render-policy.ts new file mode 100644 index 000000000..bcdade635 --- /dev/null +++ b/shared/memory-render-policy.ts @@ -0,0 +1,97 @@ +import { MEMORY_DEFAULTS } from './memory-defaults.js'; +import { isMemoryRenderKind, type MemoryRenderKind } from './memory-render-kind.js'; +import type { MemoryTelemetryBuffer } from './memory-telemetry.js'; +import { renderSkillEnvelope } from './skill-envelope.js'; + +export interface MemoryRenderInput { + kind: MemoryRenderKind; + content: string; + authorizedRawSource?: boolean; + maxBytes?: number; +} + +export type MemoryRenderResult = { + ok: true; + text: string; + kind: MemoryRenderKind; +} | { + ok: false; + text: ''; + kind: MemoryRenderKind; + reason: string; +}; + +function utf8ByteLength(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function truncateUtf8(value: string, maxBytes: number): string { + let output = ''; + let used = 0; + const encoder = new TextEncoder(); + for (const char of value) { + const bytes = encoder.encode(char).byteLength; + if (used + bytes > maxBytes) break; + output += char; + used += bytes; + } + return output; +} + +function cap(value: string, maxBytes: number): string { + return utf8ByteLength(value) > maxBytes ? truncateUtf8(value, maxBytes) : value; +} + +export function renderMemoryContextItem(input: MemoryRenderInput): MemoryRenderResult { + if (!isMemoryRenderKind(input.kind)) { + return { ok: false, text: '', kind: input.kind, reason: 'unsupported_render_kind' }; + } + const maxBytes = Math.max(1, input.maxBytes ?? MEMORY_DEFAULTS.startupTotalTokens); + try { + switch (input.kind) { + case 'pinned': + return { ok: true, kind: input.kind, text: input.content }; + case 'skill': + return { ok: true, kind: input.kind, text: renderSkillEnvelope(input.content) }; + case 'citation_preview': + if (!input.authorizedRawSource) { + return { ok: false, text: '', kind: input.kind, reason: 'unauthorized_citation_preview' }; + } + return { ok: true, kind: input.kind, text: cap(input.content, maxBytes) }; + case 'summary': + case 'preference': + case 'note': + return { ok: true, kind: input.kind, text: cap(input.content.trim(), maxBytes) }; + } + } catch (error) { + return { + ok: false, + text: '', + kind: input.kind, + reason: error instanceof Error ? error.message : 'render_failed', + }; + } +} + +export interface RenderMemoryContextItemsOptions { + telemetry?: Pick; +} + +export function renderMemoryContextItems( + inputs: readonly MemoryRenderInput[], + options: RenderMemoryContextItemsOptions = {}, +): string[] { + const rendered: string[] = []; + for (const input of inputs) { + const result = renderMemoryContextItem(input); + if (result.ok) { + rendered.push(result.text); + continue; + } + options.telemetry?.enqueue('mem.startup.stage_dropped', { + outcome: 'dropped', + reason: 'render_failed', + }); + } + return rendered; +} diff --git a/shared/memory-retention.ts b/shared/memory-retention.ts new file mode 100644 index 000000000..cca05da0a --- /dev/null +++ b/shared/memory-retention.ts @@ -0,0 +1,77 @@ +export const MEMORY_RETENTION_TABLES = [ + 'shared_context_citations', + 'shared_context_projection_cite_counts', + 'observation_promotion_audit', + 'skill_review_jobs', + 'memory_telemetry_events', +] as const; + +export type MemoryRetentionTable = (typeof MEMORY_RETENTION_TABLES)[number]; + +export interface MemoryRetentionPolicy { + table: MemoryRetentionTable; + ttlMs: number; + timestampColumn: string; + batchSize: number; +} + +export const DEFAULT_MEMORY_RETENTION_POLICIES: readonly MemoryRetentionPolicy[] = [ + { table: 'shared_context_citations', ttlMs: 180 * 24 * 60 * 60 * 1000, timestampColumn: 'created_at', batchSize: 500 }, + { table: 'shared_context_projection_cite_counts', ttlMs: 365 * 24 * 60 * 60 * 1000, timestampColumn: 'updated_at', batchSize: 500 }, + { table: 'observation_promotion_audit', ttlMs: 365 * 24 * 60 * 60 * 1000, timestampColumn: 'created_at', batchSize: 500 }, + { table: 'skill_review_jobs', ttlMs: 30 * 24 * 60 * 60 * 1000, timestampColumn: 'updated_at', batchSize: 500 }, + { table: 'memory_telemetry_events', ttlMs: 14 * 24 * 60 * 60 * 1000, timestampColumn: 'created_at', batchSize: 1000 }, +]; + +export interface RetentionSweepPlanItem { + table: MemoryRetentionTable; + cutoff: number; + timestampColumn: string; + batchSize: number; +} + +export function buildMemoryRetentionSweepPlan(now: number, policies: readonly MemoryRetentionPolicy[] = DEFAULT_MEMORY_RETENTION_POLICIES): RetentionSweepPlanItem[] { + return policies.map((policy) => ({ + table: policy.table, + cutoff: now - policy.ttlMs, + timestampColumn: policy.timestampColumn, + batchSize: policy.batchSize, + })); +} + + +export interface MemoryRetentionSweepExecutor { + deleteBefore(item: RetentionSweepPlanItem): Promise | number; +} + +export interface MemoryRetentionSweepResult { + table: MemoryRetentionTable; + cutoff: number; + deleted: number; + ok: boolean; + error?: string; +} + +/** Best-effort, bounded retention sweep. Individual table failures are reported + * but do not abort the rest of the memory pipeline or shutdown path. */ +export async function runMemoryRetentionSweep( + executor: MemoryRetentionSweepExecutor, + plan: readonly RetentionSweepPlanItem[], +): Promise { + const results: MemoryRetentionSweepResult[] = []; + for (const item of plan) { + try { + const deleted = await executor.deleteBefore(item); + results.push({ table: item.table, cutoff: item.cutoff, deleted, ok: true }); + } catch (error) { + results.push({ + table: item.table, + cutoff: item.cutoff, + deleted: 0, + ok: false, + error: error instanceof Error ? error.message : String(error), + }); + } + } + return results; +} diff --git a/shared/memory-scope.ts b/shared/memory-scope.ts new file mode 100644 index 000000000..bb4605ad5 --- /dev/null +++ b/shared/memory-scope.ts @@ -0,0 +1,211 @@ +export const MEMORY_SCOPES = ['user_private', 'personal', 'project_shared', 'workspace_shared', 'org_shared'] as const; +export type MemoryScope = (typeof MEMORY_SCOPES)[number]; + +export type OwnerPrivateMemoryScope = 'user_private' | 'personal'; +export type ReplicableSharedProjectionScope = 'project_shared' | 'workspace_shared' | 'org_shared'; +export type AuthoredContextScope = 'project_shared' | 'workspace_shared' | 'org_shared'; + +export const OWNER_PRIVATE_MEMORY_SCOPES = ['user_private', 'personal'] as const satisfies readonly OwnerPrivateMemoryScope[]; +export const REPLICABLE_SHARED_PROJECTION_SCOPES = [ + 'project_shared', + 'workspace_shared', + 'org_shared', +] as const satisfies readonly ReplicableSharedProjectionScope[]; +export const AUTHORED_CONTEXT_SCOPES = [ + 'project_shared', + 'workspace_shared', + 'org_shared', +] as const satisfies readonly AuthoredContextScope[]; +export const SYNCED_PROJECTION_MEMORY_SCOPES = [ + 'personal', + ...REPLICABLE_SHARED_PROJECTION_SCOPES, +] as const satisfies readonly (OwnerPrivateMemoryScope | ReplicableSharedProjectionScope)[]; +export type SharedContextProjectionScope = (typeof SYNCED_PROJECTION_MEMORY_SCOPES)[number]; + +export const SEARCH_REQUEST_SCOPE_ALIASES = ['owner_private', 'shared', 'all_authorized'] as const; +export type SearchRequestScopeAlias = (typeof SEARCH_REQUEST_SCOPE_ALIASES)[number]; +export type SearchRequestScope = SearchRequestScopeAlias | MemoryScope; + +export const MEMORY_SCOPE_IDENTITY_FIELDS = [ + 'tenant_id', + 'user_id', + 'project_id', + 'workspace_id', + 'org_id', + 'root_session_id', + 'session_tree_id', + 'session_id', +] as const; +export type MemoryScopeIdentityField = (typeof MEMORY_SCOPE_IDENTITY_FIELDS)[number]; + +export type MemoryReplicationBehavior = + | 'daemon_local' + | 'owner_private_sync' + | 'shared_projection' + | 'authored_context'; + +export type RawSourceAccessPolicy = 'owner_only' | 'authorized_members' | 'admin_only' | 'none'; + +export interface MemoryScopePolicy { + scope: MemoryScope; + ownerPrivate: boolean; + requiredIdentityFields: readonly MemoryScopeIdentityField[]; + optionalIdentityFields: readonly MemoryScopeIdentityField[]; + forbiddenIdentityFields: readonly MemoryScopeIdentityField[]; + replication: MemoryReplicationBehavior; + requestExpansions: readonly SearchRequestScopeAlias[]; + rawSourceAccess: RawSourceAccessPolicy; + promotionTargets: readonly MemoryScope[]; + defaultSearchIncluded: boolean; + projectBound: boolean; +} + +export type MemoryScopeIdentity = Partial>; + +const PRIVATE_PROMOTION_TARGETS = ['project_shared', 'workspace_shared', 'org_shared'] as const satisfies readonly MemoryScope[]; + +export const MEMORY_SCOPE_POLICIES = { + user_private: { + scope: 'user_private', + ownerPrivate: true, + requiredIdentityFields: ['user_id'], + optionalIdentityFields: ['tenant_id', 'project_id', 'root_session_id', 'session_tree_id', 'session_id'], + forbiddenIdentityFields: ['workspace_id', 'org_id'], + replication: 'owner_private_sync', + requestExpansions: ['owner_private', 'all_authorized'], + rawSourceAccess: 'owner_only', + promotionTargets: ['personal', ...PRIVATE_PROMOTION_TARGETS], + defaultSearchIncluded: true, + projectBound: false, + }, + personal: { + scope: 'personal', + ownerPrivate: true, + requiredIdentityFields: ['user_id', 'project_id'], + optionalIdentityFields: ['tenant_id', 'root_session_id', 'session_tree_id', 'session_id'], + forbiddenIdentityFields: ['workspace_id', 'org_id'], + replication: 'daemon_local', + requestExpansions: ['owner_private', 'all_authorized'], + rawSourceAccess: 'owner_only', + promotionTargets: PRIVATE_PROMOTION_TARGETS, + defaultSearchIncluded: true, + projectBound: true, + }, + project_shared: { + scope: 'project_shared', + ownerPrivate: false, + requiredIdentityFields: ['project_id'], + optionalIdentityFields: ['tenant_id', 'workspace_id', 'org_id', 'root_session_id', 'session_tree_id', 'session_id'], + forbiddenIdentityFields: [], + replication: 'shared_projection', + requestExpansions: ['shared', 'all_authorized'], + rawSourceAccess: 'authorized_members', + promotionTargets: ['workspace_shared', 'org_shared'], + defaultSearchIncluded: true, + projectBound: true, + }, + workspace_shared: { + scope: 'workspace_shared', + ownerPrivate: false, + requiredIdentityFields: ['workspace_id'], + optionalIdentityFields: ['tenant_id', 'project_id', 'org_id', 'root_session_id', 'session_tree_id', 'session_id'], + forbiddenIdentityFields: [], + replication: 'shared_projection', + requestExpansions: ['shared', 'all_authorized'], + rawSourceAccess: 'authorized_members', + promotionTargets: ['org_shared'], + defaultSearchIncluded: true, + projectBound: false, + }, + org_shared: { + scope: 'org_shared', + ownerPrivate: false, + requiredIdentityFields: ['org_id'], + optionalIdentityFields: ['tenant_id', 'project_id', 'workspace_id', 'root_session_id', 'session_tree_id', 'session_id'], + forbiddenIdentityFields: [], + replication: 'authored_context', + requestExpansions: ['shared', 'all_authorized'], + rawSourceAccess: 'authorized_members', + promotionTargets: [], + defaultSearchIncluded: true, + projectBound: false, + }, +} as const satisfies Record; + +const MEMORY_SCOPE_SET: ReadonlySet = new Set(MEMORY_SCOPES); +const OWNER_PRIVATE_MEMORY_SCOPE_SET: ReadonlySet = new Set(OWNER_PRIVATE_MEMORY_SCOPES); +const REPLICABLE_SHARED_PROJECTION_SCOPE_SET: ReadonlySet = new Set(REPLICABLE_SHARED_PROJECTION_SCOPES); +const SHARED_CONTEXT_PROJECTION_SCOPE_SET: ReadonlySet = new Set(SYNCED_PROJECTION_MEMORY_SCOPES); +const AUTHORED_CONTEXT_SCOPE_SET: ReadonlySet = new Set(AUTHORED_CONTEXT_SCOPES); +const SEARCH_REQUEST_SCOPE_SET: ReadonlySet = new Set([...MEMORY_SCOPES, ...SEARCH_REQUEST_SCOPE_ALIASES]); + +function hasIdentityField(identity: MemoryScopeIdentity, field: MemoryScopeIdentityField): boolean { + const value = identity[field]; + return typeof value === 'string' && value.trim().length > 0; +} + +export function isMemoryScope(value: unknown): value is MemoryScope { + return typeof value === 'string' && MEMORY_SCOPE_SET.has(value); +} + +export function isSearchRequestScope(value: unknown): value is SearchRequestScope { + return typeof value === 'string' && SEARCH_REQUEST_SCOPE_SET.has(value); +} + +export function getMemoryScopePolicy(scope: MemoryScope): MemoryScopePolicy { + return MEMORY_SCOPE_POLICIES[scope]; +} + +export function isOwnerPrivateMemoryScope(scope: MemoryScope): scope is OwnerPrivateMemoryScope { + return OWNER_PRIVATE_MEMORY_SCOPE_SET.has(scope); +} + +export function isSharedProjectionScope(scope: MemoryScope): scope is ReplicableSharedProjectionScope { + return REPLICABLE_SHARED_PROJECTION_SCOPE_SET.has(scope); +} + +export function isReplicableSharedProjectionScope(value: unknown): value is ReplicableSharedProjectionScope { + return typeof value === 'string' && REPLICABLE_SHARED_PROJECTION_SCOPE_SET.has(value); +} + +export function isSharedContextProjectionScope(value: unknown): value is SharedContextProjectionScope { + return typeof value === 'string' && SHARED_CONTEXT_PROJECTION_SCOPE_SET.has(value); +} + +export function isAuthoredContextScope(value: unknown): value is AuthoredContextScope { + return typeof value === 'string' && AUTHORED_CONTEXT_SCOPE_SET.has(value); +} + +export function expandSearchRequestScope(requestScope: SearchRequestScope): readonly MemoryScope[] { + if (isMemoryScope(requestScope)) return [requestScope]; + switch (requestScope) { + case 'owner_private': + return OWNER_PRIVATE_MEMORY_SCOPES; + case 'shared': + return REPLICABLE_SHARED_PROJECTION_SCOPES; + case 'all_authorized': + return MEMORY_SCOPES; + } +} + +export function validateMemoryScopeIdentity(scope: MemoryScope, identity: MemoryScopeIdentity): { ok: true } | { ok: false; reason: string } { + const policy = getMemoryScopePolicy(scope); + const missing = policy.requiredIdentityFields.filter((field) => !hasIdentityField(identity, field)); + if (missing.length > 0) { + return { ok: false, reason: `Missing required identity field(s) for ${scope}: ${missing.join(', ')}` }; + } + const forbidden = policy.forbiddenIdentityFields.filter((field) => hasIdentityField(identity, field)); + if (forbidden.length > 0) { + return { ok: false, reason: `Forbidden identity field(s) for ${scope}: ${forbidden.join(', ')}` }; + } + return { ok: true }; +} + +export function assertMemoryScopeIdentity(scope: MemoryScope, identity: MemoryScopeIdentity): void { + const result = validateMemoryScopeIdentity(scope, identity); + if (!result.ok) throw new Error(result.reason); +} + +export function canPromoteMemoryScope(fromScope: MemoryScope, toScope: MemoryScope): boolean { + return getMemoryScopePolicy(fromScope).promotionTargets.includes(toScope); +} diff --git a/shared/memory-telemetry.ts b/shared/memory-telemetry.ts new file mode 100644 index 000000000..ec607a7a9 --- /dev/null +++ b/shared/memory-telemetry.ts @@ -0,0 +1,195 @@ +import { + MEMORY_COUNTERS, + MEMORY_SOFT_FAIL_PATH_COUNTERS, + type MemoryCounter, + type MemorySoftFailPath, +} from './memory-counters.js'; +import { isMemoryFeatureFlag, type MemoryFeatureFlag } from './feature-flags.js'; +import { isMemoryOrigin, type MemoryOrigin } from './memory-origin.js'; +import { isSendOrigin, type SendOrigin } from './send-origin.js'; +import { FINGERPRINT_KINDS, type FingerprintKind } from './memory-fingerprint.js'; +import { isObservationClass, type ObservationClass } from './memory-observation.js'; +import { isSkillReviewTrigger, type SkillReviewTrigger } from './skill-review-triggers.js'; + +export const MEMORY_TELEMETRY_LABEL_KEYS = [ + 'feature', + 'origin', + 'send_origin', + 'fingerprint_kind', + 'observation_class', + 'skill_review_trigger', + 'outcome', + 'reason', +] as const; + +export type MemoryTelemetryLabelKey = (typeof MEMORY_TELEMETRY_LABEL_KEYS)[number]; +export type MemoryTelemetryLabels = Partial>; + +export const MEMORY_SOFT_FAIL_SURFACES = Object.keys(MEMORY_SOFT_FAIL_PATH_COUNTERS).sort() as MemorySoftFailPath[]; +export type MemorySoftFailSurface = MemorySoftFailPath; + +export interface MemoryTelemetryEvent { + counter: MemoryCounter; + labels: MemoryTelemetryLabels; + value: number; + createdAt: number; +} + +export interface MemoryTelemetrySink { + record(event: MemoryTelemetryEvent): Promise | void; +} + +export interface MemoryTelemetryBufferOptions { + maxSize?: number; + sinkTimeoutMs?: number; + now?: () => number; + sink?: MemoryTelemetrySink; + onDrop?: (event: MemoryTelemetryEvent) => void; +} + +const MEMORY_COUNTER_SET: ReadonlySet = new Set(MEMORY_COUNTERS); +const FINGERPRINT_KIND_SET: ReadonlySet = new Set(FINGERPRINT_KINDS); +const MEMORY_TELEMETRY_LABEL_KEY_SET: ReadonlySet = new Set(MEMORY_TELEMETRY_LABEL_KEYS); +const MEMORY_SOFT_FAIL_SURFACE_SET: ReadonlySet = new Set(MEMORY_SOFT_FAIL_SURFACES); +const OUTCOME_VALUES = new Set(['success', 'disabled', 'deduped', 'rejected', 'dropped', 'failed', 'timeout']); +const REASON_PATTERN = /^[a-z][a-z0-9_]{0,63}$/; + +function isFingerprintKind(value: unknown): value is FingerprintKind { + return typeof value === 'string' && FINGERPRINT_KIND_SET.has(value); +} + +export function sanitizeMemoryTelemetryLabels(labels: MemoryTelemetryLabels = {}): MemoryTelemetryLabels { + const sanitized: MemoryTelemetryLabels = {}; + for (const [rawKey, rawValue] of Object.entries(labels)) { + if (!MEMORY_TELEMETRY_LABEL_KEY_SET.has(rawKey)) { + throw new Error(`Unsupported memory telemetry label: ${rawKey}`); + } + if (typeof rawValue !== 'string' || rawValue.length === 0) continue; + const key = rawKey as MemoryTelemetryLabelKey; + switch (key) { + case 'feature': + if (!isMemoryFeatureFlag(rawValue)) throw new Error(`Invalid memory feature telemetry label: ${rawValue}`); + sanitized[key] = rawValue satisfies MemoryFeatureFlag; + break; + case 'origin': + if (!isMemoryOrigin(rawValue)) throw new Error(`Invalid memory origin telemetry label: ${rawValue}`); + sanitized[key] = rawValue satisfies MemoryOrigin; + break; + case 'send_origin': + if (!isSendOrigin(rawValue)) throw new Error(`Invalid send origin telemetry label: ${rawValue}`); + sanitized[key] = rawValue satisfies SendOrigin; + break; + case 'fingerprint_kind': + if (!isFingerprintKind(rawValue)) throw new Error(`Invalid fingerprint kind telemetry label: ${rawValue}`); + sanitized[key] = rawValue satisfies FingerprintKind; + break; + case 'observation_class': + if (!isObservationClass(rawValue)) throw new Error(`Invalid observation class telemetry label: ${rawValue}`); + sanitized[key] = rawValue satisfies ObservationClass; + break; + case 'skill_review_trigger': + if (!isSkillReviewTrigger(rawValue)) throw new Error(`Invalid skill review trigger telemetry label: ${rawValue}`); + sanitized[key] = rawValue satisfies SkillReviewTrigger; + break; + case 'outcome': + if (!OUTCOME_VALUES.has(rawValue)) throw new Error(`Invalid memory telemetry outcome: ${rawValue}`); + sanitized[key] = rawValue; + break; + case 'reason': + if (!REASON_PATTERN.test(rawValue)) throw new Error(`Invalid memory telemetry reason: ${rawValue}`); + sanitized[key] = rawValue; + break; + } + } + return sanitized; +} + +export function isMemorySoftFailSurface(value: unknown): value is MemorySoftFailSurface { + return typeof value === 'string' && MEMORY_SOFT_FAIL_SURFACE_SET.has(value); +} + +export function counterForMemorySoftFailSurface(surface: MemorySoftFailSurface): MemoryCounter { + return MEMORY_SOFT_FAIL_PATH_COUNTERS[surface]; +} + +export function recordMemorySoftFailure( + telemetry: Pick | undefined, + surface: MemorySoftFailSurface, + reason: string, + labels: MemoryTelemetryLabels = {}, +): boolean { + if (!telemetry) return false; + return telemetry.enqueue(counterForMemorySoftFailSurface(surface), { + ...labels, + outcome: labels.outcome ?? 'failed', + reason, + }); +} + +export class MemoryTelemetryBuffer { + private readonly maxSize: number; + private readonly sinkTimeoutMs: number; + private readonly now: () => number; + private readonly sink?: MemoryTelemetrySink; + private readonly onDrop?: (event: MemoryTelemetryEvent) => void; + private queue: MemoryTelemetryEvent[] = []; + private flushing = false; + + constructor(options: MemoryTelemetryBufferOptions = {}) { + this.maxSize = Math.max(1, options.maxSize ?? 256); + this.sinkTimeoutMs = Math.max(1, options.sinkTimeoutMs ?? 250); + this.now = options.now ?? Date.now; + this.sink = options.sink; + this.onDrop = options.onDrop; + } + + get size(): number { + return this.queue.length; + } + + enqueue(counter: MemoryCounter, labels: MemoryTelemetryLabels = {}, value = 1): boolean { + if (!MEMORY_COUNTER_SET.has(counter)) { + throw new Error(`Unsupported memory counter: ${counter}`); + } + const event: MemoryTelemetryEvent = { + counter, + labels: sanitizeMemoryTelemetryLabels(labels), + value, + createdAt: this.now(), + }; + if (this.queue.length >= this.maxSize) { + this.onDrop?.(event); + return false; + } + this.queue.push(event); + void this.flush(); + return true; + } + + drain(): MemoryTelemetryEvent[] { + const events = this.queue; + this.queue = []; + return events; + } + + async flush(): Promise { + if (this.flushing || !this.sink) return; + this.flushing = true; + try { + while (this.queue.length > 0) { + const event = this.queue.shift(); + if (!event) break; + try { + await Promise.race([ + Promise.resolve(this.sink.record(event)), + new Promise((resolve) => setTimeout(resolve, this.sinkTimeoutMs)), + ]); + } catch { + // Telemetry is explicitly best-effort; sink failure must not affect memory behavior. + } + } + } finally { + this.flushing = false; + } + } +} diff --git a/shared/memory-ws.ts b/shared/memory-ws.ts index d4627788d..0ea950f64 100644 --- a/shared/memory-ws.ts +++ b/shared/memory-ws.ts @@ -1,13 +1,117 @@ export const MEMORY_WS = { SEARCH: 'memory.search', + SEARCH_RESPONSE: 'memory.search_response', ARCHIVE: 'memory.archive', ARCHIVE_RESPONSE: 'memory.archive_response', RESTORE: 'memory.restore', RESTORE_RESPONSE: 'memory.restore_response', + CREATE: 'memory.create', + CREATE_RESPONSE: 'memory.create_response', + UPDATE: 'memory.update', + UPDATE_RESPONSE: 'memory.update_response', + PIN: 'memory.pin', + PIN_RESPONSE: 'memory.pin_response', DELETE: 'memory.delete', DELETE_RESPONSE: 'memory.delete_response', PERSONAL_QUERY: 'shared_context.personal_memory.query', PERSONAL_RESPONSE: 'shared_context.personal_memory.response', + PROJECT_RESOLVE: 'memory.project.resolve', + PROJECT_RESOLVE_RESPONSE: 'memory.project.resolve_response', + FEATURES_QUERY: 'memory.features.query', + FEATURES_RESPONSE: 'memory.features.response', + FEATURES_SET: 'memory.features.set', + FEATURES_SET_RESPONSE: 'memory.features.set_response', + PREF_QUERY: 'memory.preferences.query', + PREF_RESPONSE: 'memory.preferences.response', + PREF_CREATE: 'memory.preferences.create', + PREF_CREATE_RESPONSE: 'memory.preferences.create_response', + PREF_UPDATE: 'memory.preferences.update', + PREF_UPDATE_RESPONSE: 'memory.preferences.update_response', + PREF_DELETE: 'memory.preferences.delete', + PREF_DELETE_RESPONSE: 'memory.preferences.delete_response', + SKILL_QUERY: 'memory.skills.query', + SKILL_RESPONSE: 'memory.skills.response', + SKILL_REBUILD: 'memory.skills.rebuild', + SKILL_REBUILD_RESPONSE: 'memory.skills.rebuild_response', + SKILL_READ: 'memory.skills.read', + SKILL_READ_RESPONSE: 'memory.skills.read_response', + SKILL_DELETE: 'memory.skills.delete', + SKILL_DELETE_RESPONSE: 'memory.skills.delete_response', + MD_INGEST_RUN: 'memory.md_ingest.run', + MD_INGEST_RUN_RESPONSE: 'memory.md_ingest.run_response', + OBSERVATION_QUERY: 'memory.observations.query', + OBSERVATION_RESPONSE: 'memory.observations.response', + OBSERVATION_UPDATE: 'memory.observations.update', + OBSERVATION_UPDATE_RESPONSE: 'memory.observations.update_response', + OBSERVATION_DELETE: 'memory.observations.delete', + OBSERVATION_DELETE_RESPONSE: 'memory.observations.delete_response', + OBSERVATION_PROMOTE: 'memory.observations.promote', + OBSERVATION_PROMOTE_RESPONSE: 'memory.observations.promote_response', } as const; export type MemoryWsType = typeof MEMORY_WS[keyof typeof MEMORY_WS]; + +export const MEMORY_MANAGEMENT_REQUEST_TYPES = [ + MEMORY_WS.SEARCH, + MEMORY_WS.ARCHIVE, + MEMORY_WS.RESTORE, + MEMORY_WS.CREATE, + MEMORY_WS.UPDATE, + MEMORY_WS.PIN, + MEMORY_WS.DELETE, + MEMORY_WS.PERSONAL_QUERY, + MEMORY_WS.PROJECT_RESOLVE, + MEMORY_WS.FEATURES_QUERY, + MEMORY_WS.FEATURES_SET, + MEMORY_WS.PREF_QUERY, + MEMORY_WS.PREF_CREATE, + MEMORY_WS.PREF_UPDATE, + MEMORY_WS.PREF_DELETE, + MEMORY_WS.SKILL_QUERY, + MEMORY_WS.SKILL_REBUILD, + MEMORY_WS.SKILL_READ, + MEMORY_WS.SKILL_DELETE, + MEMORY_WS.MD_INGEST_RUN, + MEMORY_WS.OBSERVATION_QUERY, + MEMORY_WS.OBSERVATION_UPDATE, + MEMORY_WS.OBSERVATION_DELETE, + MEMORY_WS.OBSERVATION_PROMOTE, +] as const satisfies readonly MemoryWsType[]; + +export const MEMORY_MANAGEMENT_RESPONSE_TYPES = [ + MEMORY_WS.ARCHIVE_RESPONSE, + MEMORY_WS.RESTORE_RESPONSE, + MEMORY_WS.CREATE_RESPONSE, + MEMORY_WS.UPDATE_RESPONSE, + MEMORY_WS.PIN_RESPONSE, + MEMORY_WS.DELETE_RESPONSE, + MEMORY_WS.PERSONAL_RESPONSE, + MEMORY_WS.PROJECT_RESOLVE_RESPONSE, + MEMORY_WS.FEATURES_RESPONSE, + MEMORY_WS.FEATURES_SET_RESPONSE, + MEMORY_WS.PREF_RESPONSE, + MEMORY_WS.PREF_CREATE_RESPONSE, + MEMORY_WS.PREF_UPDATE_RESPONSE, + MEMORY_WS.PREF_DELETE_RESPONSE, + MEMORY_WS.SKILL_RESPONSE, + MEMORY_WS.SKILL_REBUILD_RESPONSE, + MEMORY_WS.SKILL_READ_RESPONSE, + MEMORY_WS.SKILL_DELETE_RESPONSE, + MEMORY_WS.MD_INGEST_RUN_RESPONSE, + MEMORY_WS.OBSERVATION_RESPONSE, + MEMORY_WS.OBSERVATION_UPDATE_RESPONSE, + MEMORY_WS.OBSERVATION_DELETE_RESPONSE, + MEMORY_WS.OBSERVATION_PROMOTE_RESPONSE, + MEMORY_WS.SEARCH_RESPONSE, +] as const; + +const MEMORY_MANAGEMENT_REQUEST_TYPE_SET: ReadonlySet = new Set(MEMORY_MANAGEMENT_REQUEST_TYPES); +const MEMORY_MANAGEMENT_RESPONSE_TYPE_SET: ReadonlySet = new Set(MEMORY_MANAGEMENT_RESPONSE_TYPES); + +export function isMemoryManagementRequestType(type: unknown): type is (typeof MEMORY_MANAGEMENT_REQUEST_TYPES)[number] { + return typeof type === 'string' && MEMORY_MANAGEMENT_REQUEST_TYPE_SET.has(type); +} + +export function isMemoryManagementResponseType(type: unknown): type is (typeof MEMORY_MANAGEMENT_RESPONSE_TYPES)[number] { + return typeof type === 'string' && MEMORY_MANAGEMENT_RESPONSE_TYPE_SET.has(type); +} diff --git a/shared/p2p-config-scope.ts b/shared/p2p-config-scope.ts new file mode 100644 index 000000000..5c16b0048 --- /dev/null +++ b/shared/p2p-config-scope.ts @@ -0,0 +1,17 @@ +export const P2P_SESSION_CONFIG_PREF_KEY = 'p2p_session_config' as const; + +export function p2pScopedSessionKey(rootSession: string, serverId?: string | null): string { + return serverId ? `${serverId}:${rootSession}` : rootSession; +} + +export function p2pLegacySessionConfigPrefKey(rootSession: string): string { + return `${P2P_SESSION_CONFIG_PREF_KEY}:${rootSession}`; +} + +export function p2pSessionConfigPrefKey(rootSession: string, serverId?: string | null): string { + return `${P2P_SESSION_CONFIG_PREF_KEY}:${p2pScopedSessionKey(rootSession, serverId)}`; +} + +export function p2pSessionConfigLegacyPrefKeys(rootSession: string): readonly string[] { + return [p2pLegacySessionConfigPrefKey(rootSession), P2P_SESSION_CONFIG_PREF_KEY]; +} diff --git a/shared/preference-ingest.ts b/shared/preference-ingest.ts new file mode 100644 index 000000000..0edd0e0e9 --- /dev/null +++ b/shared/preference-ingest.ts @@ -0,0 +1,197 @@ +import { computeMemoryFingerprint } from './memory-fingerprint.js'; +import { MEMORY_FEATURE_FLAGS_BY_NAME, memoryFeatureFlagEnvKey } from './feature-flags.js'; +import type { MemoryOrigin } from './memory-origin.js'; +import type { ObservationClass, ObservationState } from './memory-observation.js'; +import { renderMemoryContextItem } from './memory-render-policy.js'; +import type { MemoryScope } from './memory-scope.js'; +import { + DEFAULT_SEND_ORIGIN, + isTrustedPreferenceWriteOrigin, + normalizeSendOrigin, + type SendOrigin, +} from './send-origin.js'; + +export const PREFERENCE_COMMAND_PREFIX = '@pref:'; +export const PREFERENCE_MAX_BYTES = 8 * 1024; +export const PREFERENCE_INGEST_SCOPE = 'user_private' as const satisfies MemoryScope; +export const PREFERENCE_INGEST_ORIGIN = 'user_note' as const satisfies MemoryOrigin; +export const PREFERENCE_INGEST_OBSERVATION_CLASS = 'preference' as const satisfies ObservationClass; +export const PREFERENCE_INGEST_OBSERVATION_STATE = 'active' as const satisfies ObservationState; +export const PREFERENCE_CONTEXT_START = ''; +export const PREFERENCE_CONTEXT_END = ''; +export const PREFERENCE_CONTEXT_MAX_ITEMS = 8; +export const PREFERENCE_CONTEXT_ITEM_MAX_BYTES = 1024; +export const PREFERENCE_IDEMPOTENCY_PREFIX = 'pref:v1'; + +export type PreferenceIngestOutcome = + | 'disabled_pass_through' + | 'no_preference' + | 'persist' + | 'duplicate_ignored' + | 'rejected_untrusted' + | 'rejected_oversize'; + +export interface PreferenceIngestRecord { + text: string; + fingerprint: string; + idempotencyKey: string; +} + +export interface PreferenceProviderContextRecord { + text: string; + fingerprint?: string; + updatedAt?: number; +} + +export interface PreferenceIngestResult { + outcome: PreferenceIngestOutcome; + providerText: string; + records: PreferenceIngestRecord[]; + telemetry: Array<{ + counter: 'mem.preferences.duplicate_ignored' | 'mem.preferences.rejected_untrusted'; + sendOrigin: SendOrigin; + }>; +} + +export interface ProcessPreferenceLinesOptions { + text: string; + featureEnabled: boolean; + sendOrigin?: unknown; + userId: string; + scopeKey: string; + messageId?: string; + seenIdempotencyKeys?: ReadonlySet; +} + +function utf8Bytes(text: string): number { + return new TextEncoder().encode(text).byteLength; +} + +function splitLeadingPreferenceLines(text: string): { preferences: string[]; rest: string } { + const lines = text.replace(/\r\n?/g, '\n').split('\n'); + const preferences: string[] = []; + let index = 0; + for (; index < lines.length; index++) { + const line = lines[index]; + if (!line.trim()) continue; + if (!line.trimStart().toLowerCase().startsWith(PREFERENCE_COMMAND_PREFIX)) break; + preferences.push(line.trimStart().slice(PREFERENCE_COMMAND_PREFIX.length).trim()); + } + return { preferences: preferences.filter(Boolean), rest: lines.slice(index).join('\n') }; +} + +export function buildPreferenceIdempotencyKey(input: { + userId: string; + scopeKey: string; + messageId?: string; + fingerprint: string; +}): string { + return [ + PREFERENCE_IDEMPOTENCY_PREFIX, + input.userId.trim(), + input.scopeKey.trim(), + input.messageId?.trim() || 'message:unknown', + input.fingerprint, + ].join('\u0000'); +} + +function normalizePreferenceContextText(text: string): string { + return text.trim().replace(/\s+/g, ' ').toLowerCase(); +} + +export function renderPreferenceProviderContext( + records: readonly PreferenceProviderContextRecord[], +): string { + const rendered: string[] = []; + const seen = new Set(); + const ordered = [...records].sort((left, right) => { + const leftTime = left.updatedAt ?? Number.MAX_SAFE_INTEGER; + const rightTime = right.updatedAt ?? Number.MAX_SAFE_INTEGER; + return rightTime - leftTime; + }); + for (const record of ordered) { + if (rendered.length >= PREFERENCE_CONTEXT_MAX_ITEMS) break; + const key = record.fingerprint?.trim() || normalizePreferenceContextText(record.text); + if (!key || seen.has(key)) continue; + const item = renderMemoryContextItem({ + kind: 'preference', + content: record.text, + maxBytes: PREFERENCE_CONTEXT_ITEM_MAX_BYTES, + }); + if (!item.ok || !item.text.trim()) continue; + seen.add(key); + rendered.push(`- ${item.text}`); + } + if (rendered.length === 0) return ''; + return [ + PREFERENCE_CONTEXT_START, + 'User-authored preferences for this and future turns. Follow them unless they conflict with higher-priority instructions or this turn explicitly overrides them.', + ...rendered, + PREFERENCE_CONTEXT_END, + ].join('\n'); +} + +export function prependPreferenceProviderContext(providerText: string, preferenceContext: string): string { + const context = preferenceContext.trim(); + if (!context) return providerText; + const text = providerText.trim(); + return text ? `${context}\n\n${text}` : context; +} + +/** + * Parse trusted leading @pref lines without touching the daemon receipt ack path. + * The caller can persist returned records asynchronously; disabled or untrusted + * paths preserve provider-bound text exactly as required by the send contract. + */ +export function processPreferenceLines(options: ProcessPreferenceLinesOptions): PreferenceIngestResult { + const sendOrigin = normalizeSendOrigin(options.sendOrigin ?? DEFAULT_SEND_ORIGIN); + if (!options.featureEnabled) { + return { outcome: 'disabled_pass_through', providerText: options.text, records: [], telemetry: [] }; + } + + const parsed = splitLeadingPreferenceLines(options.text); + if (parsed.preferences.length === 0) { + return { outcome: 'no_preference', providerText: options.text, records: [], telemetry: [] }; + } + + if (!isTrustedPreferenceWriteOrigin(sendOrigin)) { + return { + outcome: 'rejected_untrusted', + providerText: options.text, + records: [], + telemetry: [{ counter: 'mem.preferences.rejected_untrusted', sendOrigin }], + }; + } + + const records: PreferenceIngestRecord[] = []; + const telemetry: PreferenceIngestResult['telemetry'] = []; + let duplicateSeen = false; + for (const preference of parsed.preferences) { + if (utf8Bytes(preference) > PREFERENCE_MAX_BYTES) { + return { outcome: 'rejected_oversize', providerText: options.text, records: [], telemetry }; + } + const fingerprint = computeMemoryFingerprint({ kind: 'preference', content: preference, scopeKey: options.scopeKey }); + const idempotencyKey = buildPreferenceIdempotencyKey({ + userId: options.userId, + scopeKey: options.scopeKey, + messageId: options.messageId, + fingerprint, + }); + if (options.seenIdempotencyKeys?.has(idempotencyKey)) { + duplicateSeen = true; + telemetry.push({ counter: 'mem.preferences.duplicate_ignored', sendOrigin }); + continue; + } + records.push({ text: preference, fingerprint, idempotencyKey }); + } + + return { + outcome: records.length > 0 ? 'persist' : duplicateSeen ? 'duplicate_ignored' : 'no_preference', + providerText: parsed.rest, + records, + telemetry, + }; +} + +export const PREFERENCE_FEATURE_FLAG = MEMORY_FEATURE_FLAGS_BY_NAME.preferences; +export const PREFERENCE_FEATURE_ENV_KEY = memoryFeatureFlagEnvKey(PREFERENCE_FEATURE_FLAG); diff --git a/shared/self-learning.ts b/shared/self-learning.ts new file mode 100644 index 000000000..de95b1401 --- /dev/null +++ b/shared/self-learning.ts @@ -0,0 +1,109 @@ +import { isOwnerPrivateMemoryScope, type MemoryScope } from './memory-scope.js'; +import { MEMORY_FEATURE_FLAGS_BY_NAME } from './feature-flags.js'; +import type { ObservationClass } from './memory-observation.js'; + +export const SELF_LEARNING_FEATURE_FLAG = MEMORY_FEATURE_FLAGS_BY_NAME.selfLearning; + +export const SELF_LEARNING_CLASSIFICATION_PHASES = [ + 'classify', + 'dedup', + 'durable_signal', +] as const; +export type SelfLearningClassificationPhase = (typeof SELF_LEARNING_CLASSIFICATION_PHASES)[number]; + +export const DEDUP_DECISIONS = [ + 'new_observation', + 'merge_same_scope', + 'reject_cross_scope_merge', + 'reject_low_confidence', +] as const; +export type DedupDecision = (typeof DEDUP_DECISIONS)[number]; + +export const STARTUP_MEMORY_STATES = ['cold', 'warm', 'resumed'] as const; +export type StartupMemoryState = (typeof STARTUP_MEMORY_STATES)[number]; + +export interface SelfLearningCandidate { + scope: MemoryScope; + observationClass: ObservationClass; + text: string; + confidence: number; + sourceEventIds: readonly string[]; +} + +export interface SelfLearningDedupInput { + candidate: SelfLearningCandidate; + existing?: { scope: MemoryScope; sourceEventIds: readonly string[]; fingerprint: string }; + candidateFingerprint: string; +} + +export interface SelfLearningDedupResult { + decision: DedupDecision; + fingerprint: string; + sourceEventIds: readonly string[]; +} + +export function classifyStartupMemoryState(input: { hasExistingDurableMemory: boolean; resumedSession: boolean }): StartupMemoryState { + if (input.resumedSession) return 'resumed'; + return input.hasExistingDurableMemory ? 'warm' : 'cold'; +} + +export function canAutoPromoteBetweenScopes(fromScope: MemoryScope, toScope: MemoryScope): boolean { + if (isOwnerPrivateMemoryScope(fromScope) && fromScope !== toScope) return false; + return fromScope === toScope; +} + +export function dedupeSelfLearningCandidate(input: SelfLearningDedupInput): SelfLearningDedupResult { + if (input.candidate.confidence < 0.2) { + return { decision: 'reject_low_confidence', fingerprint: input.candidateFingerprint, sourceEventIds: input.candidate.sourceEventIds }; + } + if (!input.existing) { + return { decision: 'new_observation', fingerprint: input.candidateFingerprint, sourceEventIds: input.candidate.sourceEventIds }; + } + if (input.existing.scope !== input.candidate.scope) { + return { decision: 'reject_cross_scope_merge', fingerprint: input.candidateFingerprint, sourceEventIds: input.candidate.sourceEventIds }; + } + return { + decision: 'merge_same_scope', + fingerprint: input.existing.fingerprint, + sourceEventIds: [...new Set([...input.existing.sourceEventIds, ...input.candidate.sourceEventIds])], + }; +} + +export function withSelfLearningFailureIsolation(fallback: T, fn: () => T): { value: T; failed: boolean } { + try { + return { value: fn(), failed: false }; + } catch { + return { value: fallback, failed: true }; + } +} + +export interface SelfLearningPipelinePlanInput { + featureEnabled: boolean; + responseDelivered: boolean; + scope: MemoryScope; + startupState: StartupMemoryState; +} + +export type SelfLearningPipelineSkipReason = 'disabled' | 'not_delivered'; + +export type SelfLearningPipelinePlan = + | { + enabled: true; + foreground: false; + phases: readonly SelfLearningClassificationPhase[]; + startupState: StartupMemoryState; + scope: MemoryScope; + } + | { enabled: false; foreground: false; phases: readonly []; skipReason: SelfLearningPipelineSkipReason }; + +export function buildSelfLearningPipelinePlan(input: SelfLearningPipelinePlanInput): SelfLearningPipelinePlan { + if (!input.featureEnabled) return { enabled: false, foreground: false, phases: [], skipReason: 'disabled' }; + if (!input.responseDelivered) return { enabled: false, foreground: false, phases: [], skipReason: 'not_delivered' }; + return { + enabled: true, + foreground: false, + phases: SELF_LEARNING_CLASSIFICATION_PHASES, + startupState: input.startupState, + scope: input.scope, + }; +} diff --git a/shared/send-origin.ts b/shared/send-origin.ts new file mode 100644 index 000000000..27382cee9 --- /dev/null +++ b/shared/send-origin.ts @@ -0,0 +1,33 @@ +export const SEND_ORIGINS = [ + 'user_keyboard', + 'user_voice', + 'user_resend', + 'agent_output', + 'tool_output', + 'system_inject', +] as const; + +export type SendOrigin = (typeof SEND_ORIGINS)[number]; + +export const DEFAULT_SEND_ORIGIN: SendOrigin = 'system_inject'; + +export const TRUSTED_PREF_WRITE_ORIGINS = [ + 'user_keyboard', + 'user_voice', + 'user_resend', +] as const satisfies readonly SendOrigin[]; + +const SEND_ORIGIN_SET: ReadonlySet = new Set(SEND_ORIGINS); +const TRUSTED_PREF_WRITE_ORIGIN_SET: ReadonlySet = new Set(TRUSTED_PREF_WRITE_ORIGINS); + +export function isSendOrigin(value: unknown): value is SendOrigin { + return typeof value === 'string' && SEND_ORIGIN_SET.has(value); +} + +export function normalizeSendOrigin(value: unknown): SendOrigin { + return isSendOrigin(value) ? value : DEFAULT_SEND_ORIGIN; +} + +export function isTrustedPreferenceWriteOrigin(value: unknown): value is (typeof TRUSTED_PREF_WRITE_ORIGINS)[number] { + return typeof value === 'string' && TRUSTED_PREF_WRITE_ORIGIN_SET.has(value); +} diff --git a/shared/session-control-commands.ts b/shared/session-control-commands.ts new file mode 100644 index 000000000..3d8466473 --- /dev/null +++ b/shared/session-control-commands.ts @@ -0,0 +1,116 @@ +export const SESSION_COMPACT_COMMAND = '/compact' as const; +export const SESSION_CLEAR_COMMAND = '/clear' as const; +export const SESSION_STOP_COMMAND = '/stop' as const; +export const SESSION_CONTROL_METADATA_COMMAND_FIELD = 'controlCommand' as const; +export const SESSION_CONTROL_TIMELINE_STATE_STOPPING = 'stopping' as const; +export const SESSION_CONTROL_TIMELINE_REASON_USER_CANCEL = 'user_cancel' as const; + +export type SessionControlCommandId = 'compact' | 'clear' | 'stop'; +export type SessionControlHandling = 'provider-dispatched' | 'daemon-managed'; +export type SessionControlVisibility = 'visible' | 'hidden'; +export type SessionControlTimelineFeedback = + | { + state: typeof SESSION_CONTROL_TIMELINE_STATE_STOPPING; + reason: typeof SESSION_CONTROL_TIMELINE_REASON_USER_CANCEL; + } + | null; + +export interface SessionControlCommandDefinition { + id: SessionControlCommandId; + command: `/${string}`; + handling: SessionControlHandling; + timelineUserMessage: SessionControlVisibility; + optimisticUserMessage: SessionControlVisibility; + timelineFeedback: SessionControlTimelineFeedback; + daemonHandledReceiptAck: boolean; + resetsProcessPreferenceContext: boolean; + resetsTransportPreferenceContextOnSend: boolean; +} + +export const SESSION_CONTROL_COMMANDS = [ + { + id: 'compact', + command: SESSION_COMPACT_COMMAND, + handling: 'provider-dispatched', + timelineUserMessage: 'hidden', + optimisticUserMessage: 'hidden', + timelineFeedback: null, + daemonHandledReceiptAck: false, + resetsProcessPreferenceContext: true, + resetsTransportPreferenceContextOnSend: true, + }, + { + id: 'clear', + command: SESSION_CLEAR_COMMAND, + handling: 'daemon-managed', + timelineUserMessage: 'visible', + optimisticUserMessage: 'visible', + timelineFeedback: null, + daemonHandledReceiptAck: true, + resetsProcessPreferenceContext: true, + resetsTransportPreferenceContextOnSend: false, + }, + { + id: 'stop', + command: SESSION_STOP_COMMAND, + handling: 'daemon-managed', + timelineUserMessage: 'hidden', + optimisticUserMessage: 'hidden', + timelineFeedback: { + state: SESSION_CONTROL_TIMELINE_STATE_STOPPING, + reason: SESSION_CONTROL_TIMELINE_REASON_USER_CANCEL, + }, + daemonHandledReceiptAck: true, + resetsProcessPreferenceContext: false, + resetsTransportPreferenceContextOnSend: false, + }, +] as const satisfies readonly SessionControlCommandDefinition[]; + +export type KnownSessionControlCommand = typeof SESSION_CONTROL_COMMANDS[number]; + +export function classifySessionControlCommand(text: string): KnownSessionControlCommand | null { + const normalized = text.trim(); + return SESSION_CONTROL_COMMANDS.find((command) => command.command === normalized) ?? null; +} + +export function isSessionControlCommandText(text: string, id: SessionControlCommandId): boolean { + return classifySessionControlCommand(text)?.id === id; +} + +export function isSessionCompactCommandText(text: string): boolean { + return isSessionControlCommandText(text, 'compact'); +} + +export function isSessionClearCommandText(text: string): boolean { + return isSessionControlCommandText(text, 'clear'); +} + +export function isDaemonHandledSessionControlSend(text: string): boolean { + return classifySessionControlCommand(text)?.daemonHandledReceiptAck === true; +} + +export function shouldHideTimelineUserMessageForSessionControl(text: string): boolean { + return classifySessionControlCommand(text)?.timelineUserMessage === 'hidden'; +} + +export function shouldHideOptimisticUserMessageForSessionControl(text: string): boolean { + return classifySessionControlCommand(text)?.optimisticUserMessage === 'hidden'; +} + +export function getSessionControlTimelineFeedbackById( + id: SessionControlCommandId, +): SessionControlTimelineFeedback { + return SESSION_CONTROL_COMMANDS.find((command) => command.id === id)?.timelineFeedback ?? null; +} + +export function getSessionControlTimelineFeedback(text: string): SessionControlTimelineFeedback { + return classifySessionControlCommand(text)?.timelineFeedback ?? null; +} + +export function shouldResetProcessPreferenceContextForSessionControl(text: string): boolean { + return classifySessionControlCommand(text)?.resetsProcessPreferenceContext === true; +} + +export function shouldResetTransportPreferenceContextForSessionControl(text: string): boolean { + return classifySessionControlCommand(text)?.resetsTransportPreferenceContextOnSend === true; +} diff --git a/shared/session-model.ts b/shared/session-model.ts new file mode 100644 index 000000000..6c4c8eddb --- /dev/null +++ b/shared/session-model.ts @@ -0,0 +1,28 @@ +export interface SessionModelMetadata { + activeModel?: string | null; + requestedModel?: string | null; + modelDisplay?: string | null; + qwenModel?: string | null; +} + +function nonEmpty(value: string | null | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +/** + * Resolve the effective model a session is running for context-window and UI + * display decisions. Provider usage events are not guaranteed to include a + * model on every update, so all daemon/web callers must use the same fallback + * order before resolving model-family limits. + */ +export function resolveEffectiveSessionModel( + session: SessionModelMetadata | null | undefined, + ...fallbacks: Array +): string | undefined { + return nonEmpty(session?.activeModel) + ?? nonEmpty(session?.requestedModel) + ?? nonEmpty(session?.modelDisplay) + ?? nonEmpty(session?.qwenModel) + ?? fallbacks.map(nonEmpty).find((value): value is string => value !== undefined); +} diff --git a/shared/skill-envelope.ts b/shared/skill-envelope.ts new file mode 100644 index 000000000..5569b3d09 --- /dev/null +++ b/shared/skill-envelope.ts @@ -0,0 +1,114 @@ +import { MEMORY_DEFAULTS } from './memory-defaults.js'; + +export const SKILL_ENVELOPE_OPEN = '<<>>'; +export const SKILL_ENVELOPE_CLOSE = '<<>>'; +export const SKILL_ENVELOPE_COLLISION_PATTERN = /<</i, +] as const; + +export type SkillEnvelopeCollisionPolicy = typeof SKILL_ENVELOPE_COLLISION_POLICY | 'reject'; + +export interface SkillEnvelopeSanitizeResult { + ok: boolean; + content: string; + collision: boolean; + systemInstructionGuard: boolean; + truncated: boolean; + reason?: string; +} + +export interface SkillEnvelopeSanitizeOptions { + collisionPolicy?: SkillEnvelopeCollisionPolicy; + guardSystemInstructions?: boolean; + maxBytes?: number; +} + +const SKILL_DELIMITER_ESCAPE = '<< maxBytes) break; + output += char; + used += bytes; + } + return output; +} + +export function containsSkillEnvelopeDelimiter(content: string): boolean { + SKILL_ENVELOPE_COLLISION_PATTERN.lastIndex = 0; + return SKILL_ENVELOPE_COLLISION_PATTERN.test(content); +} + +export function violatesSkillSystemInstructionGuard(content: string): boolean { + return SKILL_SYSTEM_INSTRUCTION_GUARD_PATTERNS.some((pattern) => pattern.test(content)); +} + +function normalizeSanitizeOptions( + options: SkillEnvelopeCollisionPolicy | SkillEnvelopeSanitizeOptions | undefined, +): Required { + if (typeof options === 'string') { + return { + collisionPolicy: options, + guardSystemInstructions: true, + maxBytes: SKILL_MAX_BYTES, + }; + } + return { + collisionPolicy: options?.collisionPolicy ?? SKILL_ENVELOPE_COLLISION_POLICY, + guardSystemInstructions: options?.guardSystemInstructions ?? true, + maxBytes: Math.max(1, options?.maxBytes ?? SKILL_MAX_BYTES), + }; +} + +export function sanitizeSkillEnvelopeContent( + content: string, + options?: SkillEnvelopeCollisionPolicy | SkillEnvelopeSanitizeOptions, +): SkillEnvelopeSanitizeResult { + const resolved = normalizeSanitizeOptions(options); + const systemInstructionGuard = resolved.guardSystemInstructions && violatesSkillSystemInstructionGuard(content); + if (systemInstructionGuard) { + return { + ok: false, + content: '', + collision: false, + systemInstructionGuard: true, + truncated: false, + reason: 'Skill content attempts to act as system/developer instructions', + }; + } + const collision = containsSkillEnvelopeDelimiter(content); + if (collision && resolved.collisionPolicy === 'reject') { + return { + ok: false, + content: '', + collision, + systemInstructionGuard: false, + truncated: false, + reason: 'Skill content contains an imcodes skill envelope delimiter', + }; + } + const escaped = collision ? content.replace(SKILL_ENVELOPE_COLLISION_PATTERN, SKILL_DELIMITER_ESCAPE) : content; + const truncated = utf8ByteLength(escaped) > resolved.maxBytes; + const capped = truncated ? truncateUtf8(escaped, resolved.maxBytes) : escaped; + return { ok: true, content: capped, collision, systemInstructionGuard: false, truncated }; +} + +export function renderSkillEnvelope(content: string, options?: SkillEnvelopeCollisionPolicy | SkillEnvelopeSanitizeOptions): string { + const sanitized = sanitizeSkillEnvelopeContent(content, options); + if (!sanitized.ok) throw new Error(sanitized.reason ?? 'Skill content rejected'); + return `${SKILL_ENVELOPE_OPEN}\n${sanitized.content}\n${SKILL_ENVELOPE_CLOSE}`; +} diff --git a/shared/skill-precedence.ts b/shared/skill-precedence.ts new file mode 100644 index 000000000..b8aeecdaf --- /dev/null +++ b/shared/skill-precedence.ts @@ -0,0 +1,258 @@ +import { + DEFAULT_SHARED_SKILL_ENFORCEMENT, + SHARED_SKILL_LAYERS, + classifyUserSkillLayer, + isSharedSkillLayer, + skillMatchesProject, + type SkillLayer, + type SkillProjectContext, + type SkillSource, +} from './skill-store.js'; +import { + renderSkillEnvelope, + type SkillEnvelopeSanitizeOptions, +} from './skill-envelope.js'; + +export const ORDINARY_SKILL_PRECEDENCE = [ + 'project_escape_hatch', + 'user_project', + 'user_default', + 'workspace_shared', + 'org_shared', + 'builtin_fallback', +] as const satisfies readonly SkillLayer[]; + +export const ENFORCED_SKILL_POLICY_PRECEDENCE = [ + 'workspace_shared', + 'org_shared', +] as const satisfies readonly SkillLayer[]; + +export type SkillSelectionKind = 'ordinary' | 'additive' | 'enforced'; + +export interface SkillSelectionCandidate { + source: SkillSource; + key: string; + effectiveLayer: SkillLayer; +} + +export interface SelectedSkill { + source: SkillSource; + key: string; + effectiveLayer: SkillLayer; + selectionKind: SkillSelectionKind; +} + +export interface SkillLayerDiagnostic { + key: string; + consideredLayers: readonly SkillLayer[]; + selectedLayers: readonly SkillLayer[]; + hiddenByEnforcedLayer?: SkillLayer; + conflictResolved: boolean; +} + +export interface SkillSelectionResult { + selected: readonly SelectedSkill[]; + ordinary: readonly SelectedSkill[]; + additive: readonly SelectedSkill[]; + enforced: readonly SelectedSkill[]; + diagnostics: readonly SkillLayerDiagnostic[]; +} + +export interface RenderedSelectedSkill extends SelectedSkill { + text: string; +} + +export interface DroppedSelectedSkill extends SelectedSkill { + reason: string; +} + +export interface RenderSelectedSkillsResult { + rendered: readonly RenderedSelectedSkill[]; + dropped: readonly DroppedSelectedSkill[]; + text: string; +} + +const ORDINARY_SKILL_LAYER_RANK: ReadonlyMap = new Map( + ORDINARY_SKILL_PRECEDENCE.map((layer, index) => [layer, index]), +); + +const ENFORCED_SKILL_LAYER_RANK: ReadonlyMap = new Map( + ENFORCED_SKILL_POLICY_PRECEDENCE.map((layer, index) => [layer, index]), +); + +function rankLayer(layer: SkillLayer, ranks: ReadonlyMap): number { + return ranks.get(layer) ?? Number.MAX_SAFE_INTEGER; +} + +function isHigherPriority(candidate: SkillSelectionCandidate, current: SkillSelectionCandidate | undefined): boolean { + if (!current) return true; + return rankLayer(candidate.effectiveLayer, ORDINARY_SKILL_LAYER_RANK) + < rankLayer(current.effectiveLayer, ORDINARY_SKILL_LAYER_RANK); +} + +function isHigherEnforcedPriority(candidate: SkillSelectionCandidate, current: SkillSelectionCandidate | undefined): boolean { + if (!current) return true; + return rankLayer(candidate.effectiveLayer, ENFORCED_SKILL_LAYER_RANK) + < rankLayer(current.effectiveLayer, ENFORCED_SKILL_LAYER_RANK); +} + +function getEffectiveLayer(source: SkillSource, projectContext?: SkillProjectContext): SkillLayer | null { + if (source.layer === 'user_default' || source.layer === 'user_project') { + return classifyUserSkillLayer(source.metadata, projectContext); + } + if (source.layer === 'project_escape_hatch') { + return source.metadata.project && !skillMatchesProject(source.metadata, projectContext) + ? null + : 'project_escape_hatch'; + } + if (source.metadata.project && !skillMatchesProject(source.metadata, projectContext)) { + return null; + } + return source.layer; +} + +export function toSkillSelectionCandidates( + sources: readonly SkillSource[], + projectContext?: SkillProjectContext, +): readonly SkillSelectionCandidate[] { + const candidates: SkillSelectionCandidate[] = []; + for (const source of sources) { + const effectiveLayer = getEffectiveLayer(source, projectContext); + if (!effectiveLayer) continue; + candidates.push({ + source, + key: source.key, + effectiveLayer, + }); + } + return candidates; +} + +export function selectOrdinarySkillByKey( + sources: readonly SkillSource[], + projectContext?: SkillProjectContext, +): ReadonlyMap { + const selected = new Map(); + for (const candidate of toSkillSelectionCandidates(sources, projectContext)) { + const current = selected.get(candidate.key); + if (isHigherPriority(candidate, current)) { + selected.set(candidate.key, candidate); + } + } + return selected; +} + +function selectedFromCandidate(candidate: SkillSelectionCandidate, selectionKind: SkillSelectionKind): SelectedSkill { + return { + source: candidate.source, + key: candidate.key, + effectiveLayer: candidate.effectiveLayer, + selectionKind, + }; +} + +function buildDiagnostics( + grouped: ReadonlyMap, + selected: readonly SelectedSkill[], + enforcedByKey: ReadonlyMap, +): readonly SkillLayerDiagnostic[] { + const selectedByKey = new Map(); + for (const entry of selected) { + const layers = selectedByKey.get(entry.key) ?? []; + layers.push(entry.effectiveLayer); + selectedByKey.set(entry.key, layers); + } + return [...grouped].map(([key, candidates]) => { + const selectedLayers = selectedByKey.get(key) ?? []; + return { + key, + consideredLayers: candidates.map((candidate) => candidate.effectiveLayer), + selectedLayers, + hiddenByEnforcedLayer: enforcedByKey.get(key)?.effectiveLayer, + conflictResolved: candidates.length > selectedLayers.length || enforcedByKey.has(key), + }; + }); +} + +export function resolveSkillSelection( + sources: readonly SkillSource[], + projectContext?: SkillProjectContext, +): SkillSelectionResult { + const candidates = toSkillSelectionCandidates(sources, projectContext); + const grouped = new Map(); + for (const candidate of candidates) { + const entries = grouped.get(candidate.key) ?? []; + entries.push(candidate); + grouped.set(candidate.key, entries); + } + + const enforcedByKey = new Map(); + for (const candidate of candidates) { + if (!isSharedSkillLayer(candidate.effectiveLayer) || candidate.source.enforcement !== 'enforced') continue; + const current = enforcedByKey.get(candidate.key); + if (isHigherEnforcedPriority(candidate, current)) { + enforcedByKey.set(candidate.key, candidate); + } + } + + const ordinaryByKey = new Map(); + for (const candidate of candidates) { + if (enforcedByKey.has(candidate.key)) continue; + if (candidate.source.enforcement === 'enforced') continue; + const current = ordinaryByKey.get(candidate.key); + if (isHigherPriority(candidate, current)) { + ordinaryByKey.set(candidate.key, candidate); + } + } + + const additive: SelectedSkill[] = []; + for (const candidate of candidates) { + if (enforcedByKey.has(candidate.key)) continue; + if (!SHARED_SKILL_LAYERS.includes(candidate.effectiveLayer as never)) continue; + if ((candidate.source.enforcement ?? DEFAULT_SHARED_SKILL_ENFORCEMENT) !== 'additive') continue; + const ordinary = ordinaryByKey.get(candidate.key); + if (!ordinary || ordinary.source === candidate.source) continue; + const ordinaryIsUserOrProject = ordinary.effectiveLayer === 'project_escape_hatch' + || ordinary.effectiveLayer === 'user_project' + || ordinary.effectiveLayer === 'user_default'; + if (!ordinaryIsUserOrProject) continue; + additive.push(selectedFromCandidate(candidate, 'additive')); + } + + const enforced = [...enforcedByKey.values()].map((candidate) => selectedFromCandidate(candidate, 'enforced')); + const ordinary = [...ordinaryByKey.values()].map((candidate) => selectedFromCandidate(candidate, 'ordinary')); + const selected = [...enforced, ...ordinary, ...additive]; + return { + selected, + ordinary, + additive, + enforced, + diagnostics: buildDiagnostics(grouped, selected, enforcedByKey), + }; +} + +export function renderSelectedSkills( + selected: readonly SelectedSkill[], + options?: SkillEnvelopeSanitizeOptions, +): RenderSelectedSkillsResult { + const rendered: RenderedSelectedSkill[] = []; + const dropped: DroppedSelectedSkill[] = []; + for (const skill of selected) { + try { + rendered.push({ + ...skill, + text: renderSkillEnvelope(skill.source.content, options), + }); + } catch (error) { + dropped.push({ + ...skill, + reason: error instanceof Error ? error.message : 'skill_render_failed', + }); + } + } + return { + rendered, + dropped, + text: rendered.map((entry) => entry.text).join('\n\n'), + }; +} diff --git a/shared/skill-registry-types.ts b/shared/skill-registry-types.ts new file mode 100644 index 000000000..57d172414 --- /dev/null +++ b/shared/skill-registry-types.ts @@ -0,0 +1,52 @@ +import type { + SkillEnforcementMode, + SkillLayer, + SkillMetadata, + SkillProjectContext, + SkillSource, +} from './skill-store.js'; +import { createSkillSource } from './skill-store.js'; + +export const SKILL_REGISTRY_SCHEMA_VERSION = 1 as const; +export const SKILL_REGISTRY_FILE_NAME = 'registry.json' as const; +export const SKILL_URI_SCHEME = 'skill' as const; + +export interface SkillRegistryEntry { + schemaVersion: typeof SKILL_REGISTRY_SCHEMA_VERSION; + key: string; + layer: SkillLayer; + metadata: SkillMetadata; + /** Absolute local path for daemon resolution. Never render directly to provider context. */ + path?: string; + /** Provider-safe redacted path or opaque skill:// URI. */ + displayPath: string; + uri: `${typeof SKILL_URI_SCHEME}://${string}`; + fingerprint: string; + contentHash?: string; + mtimeMs?: number; + enforcement?: SkillEnforcementMode; + triggerKeywords?: string[]; + project?: SkillProjectContext; + updatedAt: number; +} + +export interface SkillRegistrySnapshot { + schemaVersion: typeof SKILL_REGISTRY_SCHEMA_VERSION; + generatedAt: number; + entries: SkillRegistryEntry[]; + sourceCounts?: Record; +} + +export function makeSkillUri(layer: SkillLayer, key: string): SkillRegistryEntry['uri'] { + return `${SKILL_URI_SCHEME}://${encodeURIComponent(layer)}/${encodeURIComponent(key)}`; +} + +export function skillRegistryEntryToSource(entry: SkillRegistryEntry, options: { displayPath?: boolean } = {}): SkillSource { + return createSkillSource({ + layer: entry.layer, + metadata: entry.metadata, + content: '', + path: options.displayPath ? entry.displayPath : (entry.path ?? entry.uri), + enforcement: entry.enforcement, + }); +} diff --git a/shared/skill-review-scheduler.ts b/shared/skill-review-scheduler.ts new file mode 100644 index 000000000..3b2e5d6a6 --- /dev/null +++ b/shared/skill-review-scheduler.ts @@ -0,0 +1,214 @@ +import { MEMORY_FEATURE_FLAGS_BY_NAME } from './feature-flags.js'; +import { MEMORY_DEFAULTS } from './memory-defaults.js'; +import { isSkillReviewTrigger, type SkillReviewTrigger } from './skill-review-triggers.js'; + +export const SKILL_AUTO_CREATION_FEATURE_FLAG = MEMORY_FEATURE_FLAGS_BY_NAME.skillAutoCreation; + +export interface SkillReviewSchedulerPolicy { + minIntervalMs: number; + dailyCap: number; + maxRetries: number; + backoffBaseMs: number; + maxConcurrentPerScope: number; + staleRunningMs: number; + toolIterationThreshold: number; +} + +export const DEFAULT_SKILL_REVIEW_SCHEDULER_POLICY: SkillReviewSchedulerPolicy = { + minIntervalMs: MEMORY_DEFAULTS.skillReviewMinIntervalMs, + dailyCap: MEMORY_DEFAULTS.skillReviewDailyLimit, + maxRetries: 3, + backoffBaseMs: 60 * 1000, + maxConcurrentPerScope: 1, + staleRunningMs: 15 * 60 * 1000, + toolIterationThreshold: MEMORY_DEFAULTS.skillReviewToolIterationThreshold, +}; + +export const SKILL_REVIEW_SCHEDULE_PHASES = [ + 'send_ack', + 'provider_delivery', + 'post_response_background', + 'stop', + 'approval_feedback', + 'shutdown', +] as const; + +export type SkillReviewSchedulePhase = (typeof SKILL_REVIEW_SCHEDULE_PHASES)[number]; + +export const SKILL_REVIEW_JOB_STATES = [ + 'pending', + 'running', + 'succeeded', + 'retry_wait', + 'failed', +] as const; + +export type SkillReviewJobState = (typeof SKILL_REVIEW_JOB_STATES)[number]; + +export interface SkillReviewState { + pendingKeys: ReadonlySet; + lastRunByScope: ReadonlyMap; + dailyCountByScope: ReadonlyMap; + runningCountByScope?: ReadonlyMap; +} + +export interface SkillReviewScheduleInput { + featureEnabled: boolean; + delivered: boolean; + shuttingDown?: boolean; + trigger: SkillReviewTrigger | string; + scopeKey: string; + responseId: string; + now: number; + state: SkillReviewState; + policy?: Partial; + phase?: SkillReviewSchedulePhase; + triggerEvidence?: { + toolIterationCount?: number; + }; +} + +export type SkillReviewScheduleDecision = + | { action: 'skip'; reason: 'disabled' | 'not_delivered' | 'not_background' | 'shutdown' | 'invalid_trigger' | 'below_trigger_threshold' | 'coalesced' | 'min_interval' | 'daily_cap' | 'per_scope_concurrency' } + | { action: 'enqueue'; idempotencyKey: string; nextAttemptAt: number; maxAttempts: number }; + +export interface SkillReviewJobSnapshot { + idempotencyKey: string; + scopeKey: string; + state: SkillReviewJobState; + attempt: number; + updatedAt: number; + nextAttemptAt?: number; +} + +export type SkillReviewClaimDecision = + | { action: 'skip'; reason: 'disabled' | 'shutdown' | 'not_due' | 'not_pending' | 'per_scope_concurrency' | 'attempts_exhausted' } + | { action: 'claim'; state: 'running'; attempt: number; claimedAt: number }; + +export interface SkillReviewRepairDecision { + idempotencyKey: string; + action: 'keep' | 'retry' | 'fail'; + state: SkillReviewJobState; + nextAttemptAt?: number; +} + +function policyWithDefaults(policy?: Partial): SkillReviewSchedulerPolicy { + return { ...DEFAULT_SKILL_REVIEW_SCHEDULER_POLICY, ...policy }; +} + +export function makeSkillReviewIdempotencyKey(input: { scopeKey: string; responseId: string; trigger: SkillReviewTrigger }): string { + return ['skill-review:v1', input.scopeKey.trim(), input.responseId.trim(), input.trigger].join('\u0000'); +} + +export function makeSkillReviewDailyCountKey(input: { scopeKey: string; now: number }): string { + const day = new Date(input.now).toISOString().slice(0, 10); + return ['skill-review:daily:v1', input.scopeKey.trim(), day].join('\u0000'); +} + +/** Skill auto-creation is post-delivery background work only; it never runs in the foreground send path. */ +export function decideSkillReviewSchedule(input: SkillReviewScheduleInput): SkillReviewScheduleDecision { + const policy = policyWithDefaults(input.policy); + if (!input.featureEnabled) return { action: 'skip', reason: 'disabled' }; + if (input.phase === 'shutdown') return { action: 'skip', reason: 'shutdown' }; + if (input.phase && input.phase !== 'post_response_background') return { action: 'skip', reason: 'not_background' }; + if (!input.delivered) return { action: 'skip', reason: 'not_delivered' }; + if (input.shuttingDown) return { action: 'skip', reason: 'shutdown' }; + if (!isSkillReviewTrigger(input.trigger)) return { action: 'skip', reason: 'invalid_trigger' }; + if ( + input.trigger === 'tool_iteration_count' + && Math.max(0, Math.floor(input.triggerEvidence?.toolIterationCount ?? 0)) < policy.toolIterationThreshold + ) { + return { action: 'skip', reason: 'below_trigger_threshold' }; + } + const idempotencyKey = makeSkillReviewIdempotencyKey({ scopeKey: input.scopeKey, responseId: input.responseId, trigger: input.trigger }); + if (input.state.pendingKeys.has(idempotencyKey)) return { action: 'skip', reason: 'coalesced' }; + const runningCount = input.state.runningCountByScope?.get(input.scopeKey) ?? 0; + if (runningCount >= policy.maxConcurrentPerScope) return { action: 'skip', reason: 'per_scope_concurrency' }; + const lastRun = input.state.lastRunByScope.get(input.scopeKey) ?? 0; + if (lastRun > 0 && input.now - lastRun < policy.minIntervalMs) return { action: 'skip', reason: 'min_interval' }; + const dailyCount = input.state.dailyCountByScope.get(makeSkillReviewDailyCountKey({ scopeKey: input.scopeKey, now: input.now })) ?? 0; + if (dailyCount >= policy.dailyCap) return { action: 'skip', reason: 'daily_cap' }; + return { action: 'enqueue', idempotencyKey, nextAttemptAt: input.now, maxAttempts: policy.maxRetries + 1 }; +} + +export function nextSkillReviewRetryAt(now: number, attempt: number, policy: Partial = {}): number { + const resolved = policyWithDefaults(policy); + const boundedAttempt = Math.max(0, Math.min(attempt, resolved.maxRetries)); + return now + resolved.backoffBaseMs * 2 ** boundedAttempt; +} + +export function decideSkillReviewClaim(input: { + featureEnabled: boolean; + shuttingDown?: boolean; + job: SkillReviewJobSnapshot; + now: number; + runningCountByScope: ReadonlyMap; + policy?: Partial; +}): SkillReviewClaimDecision { + const policy = policyWithDefaults(input.policy); + if (!input.featureEnabled) return { action: 'skip', reason: 'disabled' }; + if (input.shuttingDown) return { action: 'skip', reason: 'shutdown' }; + if (input.job.state !== 'pending' && input.job.state !== 'retry_wait') { + return { action: 'skip', reason: 'not_pending' }; + } + if (input.job.attempt > policy.maxRetries) { + return { action: 'skip', reason: 'attempts_exhausted' }; + } + if ((input.job.nextAttemptAt ?? 0) > input.now) { + return { action: 'skip', reason: 'not_due' }; + } + const runningCount = input.runningCountByScope.get(input.job.scopeKey) ?? 0; + if (runningCount >= policy.maxConcurrentPerScope) { + return { action: 'skip', reason: 'per_scope_concurrency' }; + } + return { + action: 'claim', + state: 'running', + attempt: input.job.attempt, + claimedAt: input.now, + }; +} + +export function repairSkillReviewJob(input: { + job: SkillReviewJobSnapshot; + now: number; + policy?: Partial; +}): SkillReviewRepairDecision { + const policy = policyWithDefaults(input.policy); + if (input.job.state === 'succeeded' || input.job.state === 'failed' || input.job.state === 'pending') { + return { + idempotencyKey: input.job.idempotencyKey, + action: 'keep', + state: input.job.state, + nextAttemptAt: input.job.nextAttemptAt, + }; + } + if (input.job.state === 'retry_wait') { + return { + idempotencyKey: input.job.idempotencyKey, + action: 'keep', + state: input.job.state, + nextAttemptAt: input.job.nextAttemptAt, + }; + } + if (input.now - input.job.updatedAt < policy.staleRunningMs) { + return { + idempotencyKey: input.job.idempotencyKey, + action: 'keep', + state: 'running', + }; + } + if (input.job.attempt >= policy.maxRetries) { + return { + idempotencyKey: input.job.idempotencyKey, + action: 'fail', + state: 'failed', + }; + } + return { + idempotencyKey: input.job.idempotencyKey, + action: 'retry', + state: 'retry_wait', + nextAttemptAt: nextSkillReviewRetryAt(input.now, input.job.attempt + 1, policy), + }; +} diff --git a/shared/skill-review-triggers.ts b/shared/skill-review-triggers.ts new file mode 100644 index 000000000..e24895e4b --- /dev/null +++ b/shared/skill-review-triggers.ts @@ -0,0 +1,12 @@ +export const SKILL_REVIEW_TRIGGERS = [ + 'tool_iteration_count', + 'manual_review', +] as const; + +export type SkillReviewTrigger = (typeof SKILL_REVIEW_TRIGGERS)[number]; + +const SKILL_REVIEW_TRIGGER_SET: ReadonlySet = new Set(SKILL_REVIEW_TRIGGERS); + +export function isSkillReviewTrigger(value: unknown): value is SkillReviewTrigger { + return typeof value === 'string' && SKILL_REVIEW_TRIGGER_SET.has(value); +} diff --git a/shared/skill-store.ts b/shared/skill-store.ts new file mode 100644 index 000000000..40d0e13a8 --- /dev/null +++ b/shared/skill-store.ts @@ -0,0 +1,606 @@ +import { join } from 'node:path'; +import { + validateBuiltinSkillManifest, + type BuiltinSkillManifestEntry, +} from './builtin-skill-manifest.js'; +import type { MemoryOrigin } from './memory-origin.js'; + +export const SKILL_FRONT_MATTER_DELIMITER = '---'; +export const SKILL_FILE_EXTENSION = '.md'; +export const PROJECT_SKILL_ESCAPE_HATCH_DIR = '.imc/skills'; +export const USER_SKILL_ROOT_DIR = '.imcodes/skills'; +export const DEFAULT_SKILL_CATEGORY = 'general'; +export const SKILL_IMPORT_ORIGIN = 'skill_import' satisfies MemoryOrigin; + +export const SKILL_LAYERS = [ + 'project_escape_hatch', + 'user_project', + 'user_default', + 'workspace_shared', + 'org_shared', + 'builtin_fallback', +] as const; +export type SkillLayer = (typeof SKILL_LAYERS)[number]; + +export const SHARED_SKILL_LAYERS = ['workspace_shared', 'org_shared'] as const; +export type SharedSkillLayer = (typeof SHARED_SKILL_LAYERS)[number]; + +export const SKILL_ENFORCEMENT_MODES = ['additive', 'enforced'] as const; +export type SkillEnforcementMode = (typeof SKILL_ENFORCEMENT_MODES)[number]; +export const DEFAULT_SHARED_SKILL_ENFORCEMENT = 'additive' as const satisfies SkillEnforcementMode; + +export const SKILL_ADMIN_ROLES = ['owner', 'admin', 'member', 'viewer'] as const; +export type SkillAdminRole = (typeof SKILL_ADMIN_ROLES)[number]; +export const SKILL_PUSH_SAFE_REJECTION_CODE = 'not_found_or_unauthorized' as const; +export const SKILL_PUSH_ACCEPTED_CODE = 'accepted' as const; +export const SKILL_PUSH_INVALID_REQUEST_CODE = 'invalid_request' as const; +export const SKILL_PUSH_INVALID_SCOPE_REASON = 'invalid_scope' as const; +export const SKILL_PUSH_INVALID_SKILL_REASON = 'invalid_skill' as const; + +export interface SkillProjectAssociation { + canonicalRepoId?: string; + projectId?: string; + workspaceId?: string; + orgId?: string; + rootPath?: string; +} + +export interface SkillProjectContext extends SkillProjectAssociation {} + +export interface SkillMetadata { + schemaVersion: 1; + name: string; + category: string; + description?: string; + project?: SkillProjectAssociation; + enforcement?: SkillEnforcementMode; +} + +export interface ParsedSkillMarkdown { + metadata: SkillMetadata; + content: string; + frontMatter: Record; +} + +export interface SkillSource { + id: string; + key: string; + layer: SkillLayer; + metadata: SkillMetadata; + content: string; + origin: typeof SKILL_IMPORT_ORIGIN; + path?: string; + enforcement?: SkillEnforcementMode; +} + +export interface SkillSourceInput { + layer: SkillLayer; + metadata: SkillMetadata | Record; + content: string; + path?: string; + enforcement?: SkillEnforcementMode; + fallbackName?: string; + fallbackCategory?: string; +} + +export interface SkillMarkdownSourceInput { + layer: SkillLayer; + markdown: string; + path?: string; + fallbackName?: string; + fallbackCategory?: string; + enforcement?: SkillEnforcementMode; +} + +export interface SharedSkillMirrorRecord { + layer: SharedSkillLayer; + scopeId: string; + markdown: string; + path?: string; + enforcement?: SkillEnforcementMode; +} + +export type SharedSkillPushAuthorizationResult = + | { ok: true; enforcement: SkillEnforcementMode } + | { ok: false; code: typeof SKILL_PUSH_SAFE_REJECTION_CODE }; + +export interface SharedSkillPushAuthorizationInput { + targetLayer: SharedSkillLayer | string; + actorRole: SkillAdminRole | string; + enforcement?: SkillEnforcementMode; +} + +export interface SharedSkillPushInput extends SharedSkillPushAuthorizationInput { + scopeId: string; + markdown: string; + path?: string; +} + +export type SharedSkillPushResult = + | { + ok: true; + code: typeof SKILL_PUSH_ACCEPTED_CODE; + record: SharedSkillMirrorRecord; + source: SkillSource; + } + | { + ok: false; + code: typeof SKILL_PUSH_SAFE_REJECTION_CODE; + } + | { + ok: false; + code: typeof SKILL_PUSH_INVALID_REQUEST_CODE; + reason: typeof SKILL_PUSH_INVALID_SCOPE_REASON | typeof SKILL_PUSH_INVALID_SKILL_REASON; + }; + +export interface BuiltinSkillLoadOptions { + builtinRoot?: string; + readSkillContent?: (path: string, entry: BuiltinSkillManifestEntry) => string; +} + +export interface SkillSelectionResult { + ordinary: SkillSource[]; + enforced: SkillSource[]; + skipped: Array<{ id: string; reason: 'project_mismatch' | 'lower_precedence' }>; +} + +export type SkillReviewWriteTarget = + | { action: 'update_user_skill'; source: SkillSource } + | { action: 'create_user_skill'; key: string }; + +const SKILL_LAYER_SET: ReadonlySet = new Set(SKILL_LAYERS); +const SHARED_SKILL_LAYER_SET: ReadonlySet = new Set(SHARED_SKILL_LAYERS); +const SKILL_ENFORCEMENT_MODE_SET: ReadonlySet = new Set(SKILL_ENFORCEMENT_MODES); +const SKILL_ADMIN_ROLE_SET: ReadonlySet = new Set(SKILL_ADMIN_ROLES); + +function asRecord(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`Invalid skill ${label}: expected object`); + } + return value as Record; +} + +function optionalString(record: Record, ...keys: string[]): string | undefined { + for (const key of keys) { + const value = record[key]; + if (value === undefined || value === null) continue; + if (typeof value !== 'string') { + throw new Error(`Invalid skill metadata: ${key} must be a string`); + } + const trimmed = value.trim(); + if (trimmed.length > 0) return trimmed; + } + return undefined; +} + +function optionalVersion(record: Record): 1 { + const value = record.schemaVersion ?? record.schema_version ?? record.version; + if (value === undefined || value === null) return 1; + if (value !== 1) { + throw new Error('Invalid skill metadata: schemaVersion must be 1'); + } + return 1; +} + +function normalizeSkillProjectAssociation(value: unknown): SkillProjectAssociation | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value === 'string') { + const canonicalRepoId = value.trim(); + if (canonicalRepoId.length === 0) return undefined; + return { canonicalRepoId }; + } + const record = asRecord(value, 'project association'); + const project = { + canonicalRepoId: optionalString(record, 'canonicalRepoId', 'canonical_repo_id', 'repo', 'repoId', 'repo_id'), + projectId: optionalString(record, 'projectId', 'project_id'), + workspaceId: optionalString(record, 'workspaceId', 'workspace_id'), + orgId: optionalString(record, 'orgId', 'org_id', 'enterpriseId', 'enterprise_id'), + rootPath: optionalString(record, 'rootPath', 'root_path'), + } satisfies SkillProjectAssociation; + return Object.values(project).some((entry) => entry !== undefined) ? project : undefined; +} + +function normalizeSkillEnforcement(value: unknown): SkillEnforcementMode | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== 'string' || !SKILL_ENFORCEMENT_MODE_SET.has(value)) { + throw new Error('Invalid skill metadata: enforcement must be additive or enforced'); + } + return value as SkillEnforcementMode; +} + +export function isSkillLayer(value: unknown): value is SkillLayer { + return typeof value === 'string' && SKILL_LAYER_SET.has(value); +} + +export function isSharedSkillLayer(value: unknown): value is SharedSkillLayer { + return typeof value === 'string' && SHARED_SKILL_LAYER_SET.has(value); +} + +export function isSkillEnforcementMode(value: unknown): value is SkillEnforcementMode { + return typeof value === 'string' && SKILL_ENFORCEMENT_MODE_SET.has(value); +} + +export function isSkillAdminRole(value: unknown): value is SkillAdminRole { + return typeof value === 'string' && SKILL_ADMIN_ROLE_SET.has(value); +} + +export function normalizeSkillMetadata( + value: SkillMetadata | Record, + fallback?: { name?: string; category?: string }, +): SkillMetadata { + const record = asRecord(value, 'metadata'); + const name = optionalString(record, 'name') ?? fallback?.name?.trim(); + const category = optionalString(record, 'category') ?? fallback?.category?.trim() ?? DEFAULT_SKILL_CATEGORY; + if (!name || name.length === 0) { + throw new Error('Invalid skill metadata: name is required'); + } + if (!category || category.length === 0) { + throw new Error('Invalid skill metadata: category is required'); + } + return { + schemaVersion: optionalVersion(record), + name, + category, + description: optionalString(record, 'description'), + project: normalizeSkillProjectAssociation(record.project), + enforcement: normalizeSkillEnforcement(record.enforcement), + }; +} + +function parseSkillScalar(value: string): unknown { + const trimmed = value.trim(); + if (trimmed.length === 0) return ''; + if (trimmed === 'true') return true; + if (trimmed === 'false') return false; + if (trimmed === 'null') return null; + if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) return Number(trimmed); + const singleQuoted = trimmed.match(/^'(.*)'$/s); + if (singleQuoted) return singleQuoted[1]?.replace(/''/g, "'") ?? ''; + const doubleQuoted = trimmed.match(/^"(.*)"$/s); + if (doubleQuoted) { + try { + return JSON.parse(trimmed); + } catch { + return doubleQuoted[1] ?? ''; + } + } + return trimmed; +} + +function parseSkillFrontMatter(rawFrontMatter: string): Record { + const root: Record = {}; + let currentObject: Record | null = null; + + for (const rawLine of rawFrontMatter.replace(/\r\n?/g, '\n').split('\n')) { + const trimmed = rawLine.trim(); + if (trimmed.length === 0 || trimmed.startsWith('#')) continue; + const topLevel = !/^\s/.test(rawLine); + const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*(.*)$/); + if (!match) { + throw new Error(`Invalid skill front matter: unsupported YAML line "${trimmed}"`); + } + const key = match[1]!; + const value = match[2] ?? ''; + if (topLevel) { + if (value.trim().length === 0) { + const nested: Record = {}; + root[key] = nested; + currentObject = nested; + continue; + } + root[key] = parseSkillScalar(value); + currentObject = null; + continue; + } + if (!currentObject) { + throw new Error(`Invalid skill front matter: nested key "${key}" has no parent`); + } + currentObject[key] = parseSkillScalar(value); + } + + return root; +} + +export function extractSkillFrontMatter(markdown: string): { frontMatter: Record; content: string } { + if (!markdown.startsWith(`${SKILL_FRONT_MATTER_DELIMITER}\n`) && !markdown.startsWith(`${SKILL_FRONT_MATTER_DELIMITER}\r\n`)) { + return { frontMatter: {}, content: markdown }; + } + const lineEnding = markdown.startsWith(`${SKILL_FRONT_MATTER_DELIMITER}\r\n`) ? '\r\n' : '\n'; + const close = `${lineEnding}${SKILL_FRONT_MATTER_DELIMITER}`; + const closeIndex = markdown.indexOf(close, SKILL_FRONT_MATTER_DELIMITER.length + lineEnding.length); + if (closeIndex < 0) { + throw new Error('Invalid skill front matter: missing closing delimiter'); + } + const rawFrontMatter = markdown.slice(SKILL_FRONT_MATTER_DELIMITER.length + lineEnding.length, closeIndex); + const afterClose = closeIndex + close.length; + const contentStart = markdown.startsWith(lineEnding, afterClose) ? afterClose + lineEnding.length : afterClose; + const parsed = rawFrontMatter.trim().length === 0 ? {} : parseSkillFrontMatter(rawFrontMatter); + return { frontMatter: asRecord(parsed, 'front matter'), content: markdown.slice(contentStart) }; +} + +export function parseSkillMarkdown( + markdown: string, + fallback?: { name?: string; category?: string }, +): ParsedSkillMarkdown { + const extracted = extractSkillFrontMatter(markdown); + return { + frontMatter: extracted.frontMatter, + content: extracted.content, + metadata: normalizeSkillMetadata(extracted.frontMatter, fallback), + }; +} + +export function normalizeSkillKeyPart(value: string): string { + return value.trim().toLowerCase(); +} + +export function makeSkillKey(category: string, name: string): string { + return `${normalizeSkillKeyPart(category)}/${normalizeSkillKeyPart(name)}`; +} + +export function normalizeSkillPathSegment(value: string): string { + const normalized = value.trim().toLowerCase().replace(/\s+/g, '-'); + if ( + normalized.length === 0 + || normalized === '.' + || normalized === '..' + || normalized.includes('/') + || normalized.includes('\\') + || !/^[a-z0-9][a-z0-9._-]*$/.test(normalized) + ) { + throw new Error(`Invalid skill path segment: ${value}`); + } + return normalized; +} + +export function getProjectSkillEscapeHatchDir(projectRoot: string): string { + return join(projectRoot, '.imc', 'skills'); +} + +export function getProjectSkillEscapeHatchPath(input: { projectRoot: string; category: string; skillName: string }): string { + return join( + getProjectSkillEscapeHatchDir(input.projectRoot), + normalizeSkillPathSegment(input.category), + `${normalizeSkillPathSegment(input.skillName)}${SKILL_FILE_EXTENSION}`, + ); +} + +export function getUserSkillRoot(homeDir: string): string { + return join(homeDir, '.imcodes', 'skills'); +} + +export function getUserSkillPath(input: { homeDir: string; category: string; skillName: string }): string { + return join( + getUserSkillRoot(input.homeDir), + normalizeSkillPathSegment(input.category), + `${normalizeSkillPathSegment(input.skillName)}${SKILL_FILE_EXTENSION}`, + ); +} + +export function skillHasProjectAssociation(metadata: SkillMetadata): boolean { + return metadata.project !== undefined; +} + +export function skillMatchesProject(metadata: SkillMetadata, context: SkillProjectContext | undefined): boolean { + if (!metadata.project) return true; + if (!context) return false; + const project = metadata.project; + const comparisons: Array = [ + 'canonicalRepoId', + 'projectId', + 'workspaceId', + 'orgId', + 'rootPath', + ]; + return comparisons.every((key) => { + const expected = project[key]; + if (expected === undefined) return true; + const actual = context[key]; + return typeof actual === 'string' && actual.trim() === expected; + }); +} + +export function classifyUserSkillLayer( + metadata: SkillMetadata, + context: SkillProjectContext | undefined, +): 'user_project' | 'user_default' | null { + if (!skillHasProjectAssociation(metadata)) return 'user_default'; + return skillMatchesProject(metadata, context) ? 'user_project' : null; +} + +export function createSkillSource(input: SkillSourceInput): SkillSource { + if (!isSkillLayer(input.layer)) { + throw new Error(`Invalid skill layer: ${String(input.layer)}`); + } + const metadata = normalizeSkillMetadata(input.metadata, { + name: input.fallbackName, + category: input.fallbackCategory, + }); + const key = makeSkillKey(metadata.category, metadata.name); + const enforcement = isSharedSkillLayer(input.layer) + ? (input.enforcement ?? metadata.enforcement ?? DEFAULT_SHARED_SKILL_ENFORCEMENT) + : input.enforcement ?? metadata.enforcement; + if (enforcement !== undefined && !isSkillEnforcementMode(enforcement)) { + throw new Error('Invalid skill enforcement mode'); + } + return { + id: `${input.layer}:${key}${input.path ? `:${input.path}` : ''}`, + key, + layer: input.layer, + metadata, + content: input.content, + origin: SKILL_IMPORT_ORIGIN, + path: input.path, + enforcement, + }; +} + +export function skillSourceFromMarkdown(input: SkillMarkdownSourceInput): SkillSource { + const parsed = parseSkillMarkdown(input.markdown, { + name: input.fallbackName, + category: input.fallbackCategory, + }); + return createSkillSource({ + layer: input.layer, + metadata: parsed.metadata, + content: parsed.content, + path: input.path, + enforcement: input.enforcement, + }); +} + +export function sharedSkillMirrorRecordToSource(record: SharedSkillMirrorRecord): SkillSource { + if (!isSharedSkillLayer(record.layer)) { + throw new Error(`Invalid shared skill mirror layer: ${String(record.layer)}`); + } + if (record.scopeId.trim().length === 0) { + throw new Error('Invalid shared skill mirror: scopeId is required'); + } + return skillSourceFromMarkdown({ + layer: record.layer, + markdown: record.markdown, + path: record.path, + enforcement: record.enforcement, + }); +} + +export function authorizeSharedSkillPush(input: SharedSkillPushAuthorizationInput): SharedSkillPushAuthorizationResult { + if (!isSharedSkillLayer(input.targetLayer)) { + return { ok: false, code: SKILL_PUSH_SAFE_REJECTION_CODE }; + } + if (input.actorRole !== 'owner' && input.actorRole !== 'admin') { + return { ok: false, code: SKILL_PUSH_SAFE_REJECTION_CODE }; + } + return { + ok: true, + enforcement: input.enforcement ?? DEFAULT_SHARED_SKILL_ENFORCEMENT, + }; +} + +/** + * Shared server helper for admin-pushed workspace/org skills. + * + * Authorization intentionally runs before scope/content parsing so unauthorized + * callers receive the same rejection shape for invalid layer, missing scope, + * malformed markdown, and non-existent inventory. + */ +export function prepareSharedSkillPush(input: SharedSkillPushInput): SharedSkillPushResult { + const authorized = authorizeSharedSkillPush(input); + if (!authorized.ok) return authorized; + + const scopeId = input.scopeId.trim(); + if (scopeId.length === 0) { + return { + ok: false, + code: SKILL_PUSH_INVALID_REQUEST_CODE, + reason: SKILL_PUSH_INVALID_SCOPE_REASON, + }; + } + + const record: SharedSkillMirrorRecord = { + layer: input.targetLayer as SharedSkillLayer, + scopeId, + markdown: input.markdown, + path: input.path, + enforcement: authorized.enforcement, + }; + + try { + return { + ok: true, + code: SKILL_PUSH_ACCEPTED_CODE, + record, + source: sharedSkillMirrorRecordToSource(record), + }; + } catch { + return { + ok: false, + code: SKILL_PUSH_INVALID_REQUEST_CODE, + reason: SKILL_PUSH_INVALID_SKILL_REASON, + }; + } +} + +export function loadBuiltinSkillSources(manifestValue: unknown, options: BuiltinSkillLoadOptions = {}): SkillSource[] { + const manifest = validateBuiltinSkillManifest(manifestValue); + if (manifest.skills.length === 0) return []; + if (!options.readSkillContent) { + throw new Error('Built-in skill manifest contains skills but no readSkillContent adapter was provided'); + } + return manifest.skills.map((entry) => { + const skillPath = options.builtinRoot ? join(options.builtinRoot, entry.path) : entry.path; + const markdown = options.readSkillContent?.(skillPath, entry); + if (markdown === undefined) { + throw new Error(`Built-in skill content missing: ${entry.path}`); + } + return skillSourceFromMarkdown({ + layer: 'builtin_fallback', + markdown, + path: skillPath, + fallbackName: entry.name, + fallbackCategory: entry.category, + }); + }); +} + +const ORDINARY_LAYER_PRIORITY: Record = { + project_escape_hatch: 0, + user_project: 1, + user_default: 2, + workspace_shared: 3, + org_shared: 4, + builtin_fallback: 5, +}; + +export function selectSkillSourcesForContext( + sources: readonly SkillSource[], + context?: SkillProjectContext, +): SkillSelectionResult { + const skipped: SkillSelectionResult['skipped'] = []; + const ordinaryByKey = new Map(); + const enforced: SkillSource[] = []; + + const sorted = [...sources].sort((a, b) => { + const priorityDiff = ORDINARY_LAYER_PRIORITY[a.layer] - ORDINARY_LAYER_PRIORITY[b.layer]; + if (priorityDiff !== 0) return priorityDiff; + return a.id.localeCompare(b.id); + }); + + for (const source of sorted) { + if (!skillMatchesProject(source.metadata, context)) { + skipped.push({ id: source.id, reason: 'project_mismatch' }); + continue; + } + if (source.enforcement === 'enforced') { + enforced.push(source); + continue; + } + if (ordinaryByKey.has(source.key)) { + skipped.push({ id: source.id, reason: 'lower_precedence' }); + continue; + } + ordinaryByKey.set(source.key, source); + } + + return { + ordinary: [...ordinaryByKey.values()], + enforced, + skipped, + }; +} + +export function chooseSkillReviewWriteTarget(input: { + candidateKey: string; + userSkillSources: readonly SkillSource[]; + context?: SkillProjectContext; +}): SkillReviewWriteTarget { + const matchingUserSkill = input.userSkillSources.find((source) => ( + source.key === input.candidateKey + && (source.layer === 'user_project' || source.layer === 'user_default') + && skillMatchesProject(source.metadata, input.context) + )); + if (matchingUserSkill) { + return { action: 'update_user_skill', source: matchingUserSkill }; + } + return { action: 'create_user_skill', key: input.candidateKey }; +} diff --git a/shared/test-session-guard.ts b/shared/test-session-guard.ts index 3da025af3..b42728146 100644 --- a/shared/test-session-guard.ts +++ b/shared/test-session-guard.ts @@ -16,6 +16,7 @@ const SESSION_NAME_PATTERNS: RegExp[] = [ /^deck_restorecheck[a-z0-9-]+_(brain|w\d+)$/i, /^deck_storecheck[a-z0-9-]+_(brain|w\d+)$/i, /^deck_shutdown[a-z0-9-]+_(brain|w\d+|probe)$/i, + /^deck_test_preview_[a-z0-9-]+_(brain|w\d+|probe)$/i, /^deck_sub_(?:cxsdk_e2e|cxsdk_effort|ccsdk_minimax_sub)$/i, ]; @@ -27,6 +28,7 @@ const PROJECT_NAME_PATTERNS: RegExp[] = [ /^restorecheck[a-z0-9-]+$/i, /^storecheck[a-z0-9-]+$/i, /^shutdown[a-z0-9-]+$/i, + /^imcodes-test-preview[-_]/i, /^e2e[-_]/i, ]; @@ -34,6 +36,7 @@ const PROJECT_DIR_PATTERNS: RegExp[] = [ /[/\\]tmp[/\\].*e2e/i, /[/\\]tmp[/\\].*modeaware/i, /[/\\]tmp[/\\].*bootmain/i, + /[/\\]tmp[/\\].*imcodes-test-preview/i, ]; function normalize(value: string | null | undefined): string | undefined { diff --git a/shared/usage-context-window.ts b/shared/usage-context-window.ts new file mode 100644 index 000000000..d0940bfc4 --- /dev/null +++ b/shared/usage-context-window.ts @@ -0,0 +1,10 @@ +export const USAGE_CONTEXT_WINDOW_SOURCES = { + PROVIDER: 'provider', +} as const; + +export type UsageContextWindowSource = + (typeof USAGE_CONTEXT_WINDOW_SOURCES)[keyof typeof USAGE_CONTEXT_WINDOW_SOURCES]; + +export function isUsageContextWindowSource(value: unknown): value is UsageContextWindowSource { + return Object.values(USAGE_CONTEXT_WINDOW_SOURCES).includes(value as UsageContextWindowSource); +} diff --git a/src/agent/providers/claude-code-sdk.ts b/src/agent/providers/claude-code-sdk.ts index c1caa0957..4e212a586 100644 --- a/src/agent/providers/claude-code-sdk.ts +++ b/src/agent/providers/claude-code-sdk.ts @@ -103,6 +103,14 @@ export class ClaudeCodeSdkProvider implements TransportProvider { reasoningEffort: true, supportedEffortLevels: CLAUDE_SDK_EFFORT_LEVELS, contextSupport: 'full-normalized-context-injection', + compact: { + execution: 'slash-command', + providerCommand: '/compact', + verified: true, + completion: 'status-only', + cancellation: 'provider-cancel', + reason: 'Verified with Claude Agent SDK 0.2.119 supportedCommands(): compact is a provider slash command, not an active RPC.', + }, }; private config: ProviderConfig | null = null; diff --git a/src/agent/providers/codex-sdk.ts b/src/agent/providers/codex-sdk.ts index 835eaaabb..1eba71985 100644 --- a/src/agent/providers/codex-sdk.ts +++ b/src/agent/providers/codex-sdk.ts @@ -12,6 +12,7 @@ import type { SessionConfig, SessionInfoUpdate, ProviderStatusUpdate, + ProviderUsageUpdate, ToolCallEvent, } from '../transport-provider.js'; import { @@ -22,6 +23,10 @@ import { } from '../transport-provider.js'; import type { AgentMessage, MessageDelta } from '../../../shared/agent-message.js'; import type { ProviderContextPayload } from '../../../shared/context-types.js'; +import { + SESSION_CONTROL_METADATA_COMMAND_FIELD, + isSessionControlCommandText, +} from '../../../shared/session-control-commands.js'; import type { TransportAttachment } from '../../../shared/transport-attachments.js'; import logger from '../../util/logger.js'; import { CODEX_SDK_EFFORT_LEVELS, type TransportEffortLevel } from '../../../shared/effort-levels.js'; @@ -30,6 +35,43 @@ import { getCodexBaseInstructions } from '../codex-runtime-config.js'; const CODEX_BIN = 'codex'; const CANCEL_INTERRUPT_TIMEOUT_MS = 1_500; +const COMPACT_START_ACCEPT_TIMEOUT_MS = 15_000; +const COMPACT_NO_SIGNAL_SETTLE_MS = 5_000; +const COMPACT_HARD_TIMEOUT_MS = 120_000; +const DEFAULT_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS = 32_000; +const MIN_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS = 4_000; +const MAX_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS = 128_000; + +function getCodexSdkContextInjectionMaxChars(): number { + const raw = process.env.IMCODES_CODEX_SDK_CONTEXT_MAX_CHARS; + if (raw === undefined || raw.trim() === '') return DEFAULT_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) return DEFAULT_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS; + if (parsed < MIN_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS) return MIN_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS; + if (parsed > MAX_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS) return MAX_CODEX_SDK_CONTEXT_INJECTION_MAX_CHARS; + return parsed; +} + +function capCodexSdkContextInjection(text: string, maxChars = getCodexSdkContextInjectionMaxChars()): string { + if (text.length <= maxChars) return text; + const marker = `\n\n[IM.codes: injected context truncated from ${text.length} to ${maxChars} chars to prevent SDK auto-compaction.]`; + if (maxChars <= marker.length + 16) return text.slice(0, maxChars); + return `${text.slice(0, maxChars - marker.length).trimEnd()}${marker}`; +} + +function buildCodexTurnInput(payload: ProviderContextPayload): string { + const contextParts: string[] = []; + const systemText = payload.systemText?.trim(); + const messagePreamble = payload.messagePreamble?.trim(); + if (systemText) contextParts.push(`Context instructions:\n${systemText}`); + if (messagePreamble) contextParts.push(messagePreamble); + if (contextParts.length === 0) return payload.assembledMessage; + + const contextText = capCodexSdkContextInjection(contextParts.join('\n\n')); + const userMessage = messagePreamble ? payload.userMessage : payload.assembledMessage; + const trimmedUserMessage = userMessage.trim(); + return trimmedUserMessage ? `${contextText}\n\n${trimmedUserMessage}` : contextText; +} /** * Provider-neutral fallback `baseInstructions` used when codex's own @@ -102,24 +144,148 @@ export interface CodexDiscoveredModel { interface CodexSdkSessionState { routeId: string; cwd: string; + env?: Record; model?: string; effort?: TransportEffortLevel; threadId?: string; loaded: boolean; runningTurnId?: string; + runningCompact: boolean; currentMessageId: string | null; currentText: string; pendingComplete?: AgentMessage; cancelled: boolean; cancelTimer: ReturnType | null; + compactSettleTimer: ReturnType | null; + compactHardTimer: ReturnType | null; + compactObserved: boolean; lastUsage?: { + /** + * Context-bar usage must represent the current prompt/window occupancy, + * not cumulative billing/thread totals. Codex app-server emits both + * `last` and `total`; `total` grows across turns and can exceed the model + * context window, so provider-neutral fields normalize from `last` when + * available and keep cumulative `total` fields only for diagnostics. + */ input_tokens: number; + cache_read_input_tokens: number; cached_input_tokens: number; output_tokens: number; + total_tokens?: number; + reasoning_output_tokens?: number; + model_context_window?: number; + codex_total_input_tokens?: number; + codex_total_cached_input_tokens?: number; + codex_total_output_tokens?: number; + codex_last_input_tokens?: number; + codex_last_cached_input_tokens?: number; + codex_last_output_tokens?: number; }; lastStatusSignature: string | null; } +function finiteNumber(value: unknown): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value)) return undefined; + return value; +} + +function normalizeCodexTokenUsage(params: Record): CodexSdkSessionState['lastUsage'] | undefined { + const tokenUsage = params.tokenUsage; + if (!tokenUsage || typeof tokenUsage !== 'object') return undefined; + + const total = tokenUsage.total && typeof tokenUsage.total === 'object' + ? tokenUsage.total as Record + : undefined; + const last = tokenUsage.last && typeof tokenUsage.last === 'object' + ? tokenUsage.last as Record + : undefined; + if (!total && !last) return undefined; + + const totalInput = finiteNumber(total?.inputTokens); + const totalCached = finiteNumber(total?.cachedInputTokens); + const totalOutput = finiteNumber(total?.outputTokens); + const lastInput = finiteNumber(last?.inputTokens); + const lastCached = finiteNumber(last?.cachedInputTokens); + const lastOutput = finiteNumber(last?.outputTokens); + + const inputTokens = lastInput ?? totalInput; + const cachedTokens = lastCached ?? totalCached; + const outputTokens = lastOutput ?? totalOutput; + if (inputTokens === undefined && cachedTokens === undefined && outputTokens === undefined) return undefined; + const cachedForUi = cachedTokens ?? 0; + // Codex/OpenAI-style `inputTokens` includes cached input as a subset + // (`totalTokens === inputTokens + outputTokens` in Codex JSONL). The web ctx + // bar renders `inputTokens + cacheTokens`, matching Anthropic's split fields. + // Therefore expose the uncached remainder as provider-neutral `input_tokens` + // and carry the raw Codex total separately for diagnostics. + const inputForUi = Math.max(0, (inputTokens ?? 0) - cachedForUi); + + const modelContextWindow = finiteNumber(tokenUsage.modelContextWindow) + // Backward-compat with older tests / adapters that briefly placed this + // beside `tokenUsage`; generated app-server types now nest it inside. + ?? finiteNumber(params.modelContextWindow); + + return { + input_tokens: inputForUi, + cache_read_input_tokens: cachedForUi, + // Keep Codex's native name too for diagnostics and direct provider users. + cached_input_tokens: cachedForUi, + output_tokens: outputTokens ?? 0, + ...(finiteNumber(total?.totalTokens) !== undefined ? { total_tokens: finiteNumber(total?.totalTokens)! } : {}), + ...(finiteNumber(total?.reasoningOutputTokens) !== undefined ? { reasoning_output_tokens: finiteNumber(total?.reasoningOutputTokens)! } : {}), + ...(modelContextWindow !== undefined && modelContextWindow > 0 ? { model_context_window: modelContextWindow } : {}), + ...(totalInput !== undefined ? { codex_total_input_tokens: totalInput } : {}), + ...(totalCached !== undefined ? { codex_total_cached_input_tokens: totalCached } : {}), + ...(totalOutput !== undefined ? { codex_total_output_tokens: totalOutput } : {}), + ...(lastInput !== undefined ? { codex_last_input_tokens: lastInput } : {}), + ...(lastCached !== undefined ? { codex_last_cached_input_tokens: lastCached } : {}), + ...(lastOutput !== undefined ? { codex_last_output_tokens: lastOutput } : {}), + }; +} + +function readParamThreadId(params: Record): string | undefined { + const threadId = params.threadId ?? params.thread_id ?? params.thread?.id ?? params.thread?.threadId ?? params.thread?.thread_id; + return typeof threadId === 'string' && threadId.trim() ? threadId : undefined; +} + +function readParamTurnId(params: Record): string | undefined { + const turnId = params.turnId ?? params.turn_id ?? params.turn?.id ?? params.turn?.turnId ?? params.turn?.turn_id; + return typeof turnId === 'string' && turnId.trim() ? turnId : undefined; +} + +function readThreadStatus(params: Record): string | undefined { + const raw = params.status ?? params.threadStatus ?? params.thread_status ?? params.thread?.status; + if (typeof raw === 'string' && raw.trim()) return raw.trim(); + if (!raw || typeof raw !== 'object') return undefined; + const nested = (raw as Record).type + ?? (raw as Record).kind + ?? (raw as Record).state + ?? (raw as Record).status; + return typeof nested === 'string' && nested.trim() ? nested.trim() : undefined; +} + +function normalizeStatusName(status: string | undefined): string { + return (status ?? '').replace(/[_\s-]+/g, '').toLowerCase(); +} + +function isThreadActiveStatus(status: string | undefined): boolean { + const normalized = normalizeStatusName(status); + return normalized === 'active' + || normalized === 'running' + || normalized === 'busy' + || normalized === 'compacting' + || normalized === 'inprogress'; +} + +function isThreadIdleStatus(status: string | undefined): boolean { + const normalized = normalizeStatusName(status); + return normalized === 'idle' + || normalized === 'ready' + || normalized === 'complete' + || normalized === 'completed' + || normalized === 'notloaded'; +} + function toolFromItem(item: Record, lifecycle: 'started' | 'completed'): ToolCallEvent | null { const meaningfulString = (value: unknown): string | undefined => { if (typeof value !== 'string') return undefined; @@ -276,6 +442,12 @@ export class CodexSdkProvider implements TransportProvider { reasoningEffort: true, supportedEffortLevels: CODEX_SDK_EFFORT_LEVELS, contextSupport: 'degraded-message-side-context-mapping', + compact: { + execution: 'sdk-rpc', + verified: true, + completion: 'provider-event', + cancellation: 'local-cancel', + }, }; private config: ProviderConfig | null = null; @@ -287,6 +459,7 @@ export class CodexSdkProvider implements TransportProvider { private toolCallCallbacks: Array<(sessionId: string, tool: ToolCallEvent) => void> = []; private sessionInfoCallbacks: Array<(sessionId: string, info: SessionInfoUpdate) => void> = []; private statusCallbacks: Array<(sessionId: string, status: ProviderStatusUpdate) => void> = []; + private usageCallbacks: Array<(sessionId: string, update: ProviderUsageUpdate) => void> = []; private child: ChildProcessWithoutNullStreams | null = null; private rl: ReadlineInterface | null = null; private nextRequestId = 1; @@ -303,7 +476,7 @@ export class CodexSdkProvider implements TransportProvider { execFile(resolved.executable, [...resolved.prependArgs, '--version'], { windowsHide: true }, (err) => (err ? reject(err) : resolve())); }); }); - await this.startAppServer(binaryPath); + await this.startAppServer(binaryPath, config); this.config = config; logger.info({ provider: this.id, resolved: resolved.executable, prepend: resolved.prependArgs }, 'Codex SDK provider connected via app-server'); } @@ -312,6 +485,10 @@ export class CodexSdkProvider implements TransportProvider { this.rejectPending(new Error('Codex app-server disconnected')); this.rl?.close(); this.rl = null; + for (const state of this.sessions.values()) { + this.clearCancelTimer(state); + this.clearCompactTimers(state); + } // `child.kill('SIGTERM')` only terminates the node wrapper; the native // codex binary it spawned lives on and leaks ~60MB per abandoned pair. // Walk the descendant tree and tree-kill instead. Fire-and-forget is @@ -331,16 +508,21 @@ export class CodexSdkProvider implements TransportProvider { this.sessions.set(routeId, { routeId, cwd: normalizeTransportCwd(config.cwd) ?? existing?.cwd ?? normalizeTransportCwd(process.cwd())!, + env: { ...(existing?.env ?? {}), ...((config.env as Record | undefined) ?? {}) }, model: typeof config.agentId === 'string' ? config.agentId : existing?.model, effort: config.effort ?? existing?.effort, threadId: config.resumeId ?? existing?.threadId, loaded: false, runningTurnId: undefined, + runningCompact: false, currentMessageId: null, currentText: '', pendingComplete: undefined, cancelled: false, cancelTimer: null, + compactSettleTimer: null, + compactHardTimer: null, + compactObserved: false, lastUsage: undefined, lastStatusSignature: null, }); @@ -352,6 +534,7 @@ export class CodexSdkProvider implements TransportProvider { const state = this.sessions.get(sessionId); if (!state) return; this.clearCancelTimer(state); + this.clearCompactTimers(state); if (state.threadId && state.loaded) { await this.request('thread/unsubscribe', { threadId: state.threadId }).catch(() => {}); this.threadToSession.delete(state.threadId); @@ -403,6 +586,14 @@ export class CodexSdkProvider implements TransportProvider { }; } + onUsage(cb: (sessionId: string, update: ProviderUsageUpdate) => void): () => void { + this.usageCallbacks.push(cb); + return () => { + const idx = this.usageCallbacks.indexOf(cb); + if (idx >= 0) this.usageCallbacks.splice(idx, 1); + }; + } + setSessionAgentId(sessionId: string, agentId: string): void { const state = this.sessions.get(sessionId); if (!state) return; @@ -424,7 +615,7 @@ export class CodexSdkProvider implements TransportProvider { if (!state) { throw this.makeError(PROVIDER_ERROR_CODES.SESSION_NOT_FOUND, `Unknown Codex SDK session: ${sessionId}`, false); } - if (state.runningTurnId) { + if (state.runningTurnId || state.runningCompact) { throw this.makeError(PROVIDER_ERROR_CODES.PROVIDER_ERROR, 'Codex SDK session is already busy', true); } @@ -436,12 +627,29 @@ export class CodexSdkProvider implements TransportProvider { state.lastUsage = undefined; state.lastStatusSignature = null; const payload = normalizeProviderPayload(payloadOrMessage, attachments, extraSystemPrompt); + if (this.isCompactCommand(payload)) { + await this.startCompact(sessionId, state); + return; + } await this.startTurn(sessionId, state, payload); } async cancel(sessionId: string): Promise { const state = this.sessions.get(sessionId); - if (!state?.threadId || !state.runningTurnId) return; + if (!state?.threadId) return; + if (state.runningCompact) { + state.cancelled = true; + const turnId = state.runningTurnId; + if (turnId) { + void this.request('turn/interrupt', { + threadId: state.threadId, + turnId, + }).catch(() => {}); + } + this.cancelCompactLocally(sessionId, state); + return; + } + if (!state.runningTurnId) return; state.cancelled = true; const turnId = state.runningTurnId; await this.request('turn/interrupt', { @@ -460,7 +668,7 @@ export class CodexSdkProvider implements TransportProvider { state.cancelTimer.unref?.(); } - private async startAppServer(binaryPath: string): Promise { + private async startAppServer(binaryPath: string, config: ProviderConfig): Promise { await this.disconnect().catch(() => {}); // Resolve npm .cmd shims into (node.exe, [scriptPath]) so spawn works // without shell:true (which has its own quoting issues on Windows). @@ -468,7 +676,7 @@ export class CodexSdkProvider implements TransportProvider { const args = [...resolved.prependArgs, 'app-server']; const child = spawn(resolved.executable, args, { stdio: ['pipe', 'pipe', 'pipe'], - env: process.env, + env: { ...process.env, ...((config.env as Record | undefined) ?? {}) }, windowsHide: true, }); this.child = child; @@ -509,13 +717,12 @@ export class CodexSdkProvider implements TransportProvider { private async startTurn(sessionId: string, state: CodexSdkSessionState, payload: ProviderContextPayload): Promise { try { await this.ensureThreadLoaded(sessionId, state); - const inputText = payload.systemText - ? `Context instructions:\n${payload.systemText}\n\n${payload.assembledMessage}` - : payload.assembledMessage; + const inputText = buildCodexTurnInput(payload); const result = await this.request('turn/start', { threadId: state.threadId, input: [{ type: 'text', text: inputText }], cwd: state.cwd, + ...this.sessionEnvironmentParams(state), approvalPolicy: 'never', sandboxPolicy: { type: 'dangerFullAccess' }, ...(state.model ? { model: state.model } : {}), @@ -528,6 +735,53 @@ export class CodexSdkProvider implements TransportProvider { } } + private isCompactCommand(payload: ProviderContextPayload): boolean { + // Codex slash commands are app-client controls, not ordinary model text. + // The daemon still forwards `/compact` through the ordinary transport send + // path; this provider adapter is the SDK boundary that maps the raw command + // to Codex app-server's native compaction RPC. Using `assembledMessage` + // here would be wrong because shared-context/preference preambles may wrap + // the provider-visible text, while `userMessage` preserves the user's raw + // command token. + return isSessionControlCommandText(payload.userMessage, 'compact'); + } + + private async startCompact(sessionId: string, state: CodexSdkSessionState): Promise { + try { + await this.ensureThreadLoaded(sessionId, state); + state.runningCompact = true; + state.compactObserved = false; + state.currentText = ''; + state.currentMessageId = null; + this.emitStatus(sessionId, state, { + status: 'compacting', + label: 'Compacting context...', + }); + this.armCompactHardTimeout(sessionId, state); + const result = await this.request('thread/compact/start', { + threadId: state.threadId, + }, COMPACT_START_ACCEPT_TIMEOUT_MS); + // Some Codex app-server builds accept `thread/compact/start` as a + // synchronous/no-op request when there is no compactable turn and never + // emit `thread/compacted` or a `contextCompaction` item. Do not leave the + // IM.codes runtime permanently busy in that accepted-but-silent state; + // give the native event stream a short grace window, then settle the UI + // as a completed native compact request. If a real compaction item/status + // arrives, `compactObserved` cancels this fallback and we wait for the + // native completion signal instead. + if (state.runningCompact && !state.compactObserved) { + this.armCompactNoSignalSettle(sessionId, state, readParamTurnId(result ?? {})); + } + } catch (err) { + this.clearCompactTimers(state); + this.clearStatus(sessionId, state); + state.runningCompact = false; + state.runningTurnId = undefined; + state.compactObserved = false; + this.emitError(sessionId, this.normalizeError(err)); + } + } + private async ensureThreadLoaded(sessionId: string, state: CodexSdkSessionState): Promise { if (state.threadId && state.loaded) return; @@ -547,6 +801,7 @@ export class CodexSdkProvider implements TransportProvider { // mid-flight. const result = await this.request('thread/resume', { threadId: state.threadId, + ...this.sessionEnvironmentParams(state), ...(state.model ? { model: state.model } : {}), baseInstructions, }); @@ -560,6 +815,7 @@ export class CodexSdkProvider implements TransportProvider { const result = await this.request('thread/start', { cwd: state.cwd, + ...this.sessionEnvironmentParams(state), approvalPolicy: 'never', sandbox: 'danger-full-access', personality: 'none', @@ -576,6 +832,10 @@ export class CodexSdkProvider implements TransportProvider { this.emitSessionInfo(sessionId, { resumeId: threadId, ...(state.model ? { model: state.model } : {}) }); } + private sessionEnvironmentParams(state: CodexSdkSessionState): { env?: Record } { + return state.env && Object.keys(state.env).length > 0 ? { env: state.env } : {}; + } + private handleLine(line: string): void { const trimmed = line.trim(); if (!trimmed) return; @@ -618,20 +878,53 @@ export class CodexSdkProvider implements TransportProvider { } if (method === 'thread/tokenUsage/updated') { - const sessionId = this.threadToSession.get(params.threadId); + const threadId = readParamThreadId(params); + const sessionId = threadId ? this.threadToSession.get(threadId) : undefined; const state = sessionId ? this.sessions.get(sessionId) : null; - const last = params.tokenUsage?.last; - if (!state || !last) return; - state.lastUsage = { - input_tokens: Number(last.inputTokens ?? 0), - cached_input_tokens: Number(last.cachedInputTokens ?? 0), - output_tokens: Number(last.outputTokens ?? 0), - }; + if (!sessionId || !state) return; + const normalizedUsage = normalizeCodexTokenUsage(params); + if (!normalizedUsage) return; + state.lastUsage = normalizedUsage; + for (const cb of this.usageCallbacks) cb(sessionId, { + usage: normalizedUsage, + ...(state.model ? { model: state.model } : {}), + }); + return; + } + + if (method === 'thread/compacted') { + const threadId = readParamThreadId(params); + const sessionId = threadId ? this.threadToSession.get(threadId) : undefined; + const state = sessionId ? this.sessions.get(sessionId) : null; + if (!sessionId || !state || !state.runningCompact) return; + this.completeCompact(sessionId, state, readParamTurnId(params)); + return; + } + + if (method === 'thread/status/changed') { + const threadId = readParamThreadId(params); + const sessionId = threadId ? this.threadToSession.get(threadId) : undefined; + const state = sessionId ? this.sessions.get(sessionId) : null; + if (!sessionId || !state || !state.runningCompact) return; + const status = readThreadStatus(params); + if (isThreadActiveStatus(status)) { + state.compactObserved = true; + this.clearCompactSettleTimer(state); + this.emitStatus(sessionId, state, { + status: 'compacting', + label: 'Compacting context...', + }); + return; + } + if (isThreadIdleStatus(status)) { + this.completeCompact(sessionId, state, readParamTurnId(params)); + } return; } if (method === 'item/agentMessage/delta') { - const sessionId = this.threadToSession.get(params.threadId); + const threadId = readParamThreadId(params); + const sessionId = threadId ? this.threadToSession.get(threadId) : undefined; const state = sessionId ? this.sessions.get(sessionId) : null; if (!sessionId || !state) return; this.clearStatus(sessionId, state); @@ -648,13 +941,30 @@ export class CodexSdkProvider implements TransportProvider { } if (method === 'item/started' || method === 'item/completed') { - const sessionId = this.threadToSession.get(params.threadId); + const threadId = readParamThreadId(params); + const sessionId = threadId ? this.threadToSession.get(threadId) : undefined; const state = sessionId ? this.sessions.get(sessionId) : null; if (!sessionId || !state) return; const item = params.item as Record | undefined; if (!item) return; + if (item.type === 'contextCompaction') { + state.runningCompact = true; + state.compactObserved = true; + this.clearCompactSettleTimer(state); + state.runningTurnId = readParamTurnId(params) ?? state.runningTurnId; + if (method === 'item/completed') { + this.completeCompact(sessionId, state, readParamTurnId(params)); + return; + } + this.emitStatus(sessionId, state, { + status: 'compacting', + label: 'Compacting context...', + }); + return; + } + if (item.type === 'reasoning') { this.emitStatus(sessionId, state, { status: 'thinking', @@ -690,7 +1000,8 @@ export class CodexSdkProvider implements TransportProvider { } if (method === 'turn/completed') { - const sessionId = this.threadToSession.get(params.threadId); + const threadId = readParamThreadId(params); + const sessionId = threadId ? this.threadToSession.get(threadId) : undefined; const state = sessionId ? this.sessions.get(sessionId) : null; if (!sessionId || !state) return; const turn = params.turn ?? {}; @@ -698,13 +1009,19 @@ export class CodexSdkProvider implements TransportProvider { if (status === 'failed') { this.clearCancelTimer(state); + this.clearCompactTimers(state); this.clearStatus(sessionId, state); + state.runningCompact = false; + state.compactObserved = false; state.runningTurnId = undefined; this.emitError(sessionId, this.makeError(PROVIDER_ERROR_CODES.PROVIDER_ERROR, turn.error?.message ?? 'Codex turn failed', false, turn.error)); return; } if (status === 'interrupted') { this.clearCancelTimer(state); + this.clearCompactTimers(state); + state.runningCompact = false; + state.compactObserved = false; if (!state.runningTurnId && state.cancelled) { state.cancelled = false; return; @@ -715,6 +1032,11 @@ export class CodexSdkProvider implements TransportProvider { return; } + if (state.runningCompact) { + this.completeCompact(sessionId, state, typeof turn.id === 'string' ? turn.id : undefined); + return; + } + this.clearCancelTimer(state); this.clearStatus(sessionId, state); state.pendingComplete = { @@ -741,18 +1063,63 @@ export class CodexSdkProvider implements TransportProvider { } } - private request(method: string, params: Record): Promise { + private request(method: string, params: Record, timeoutMs?: number): Promise { if (!this.child?.stdin.writable) { return Promise.reject(new Error('Codex app-server stdin is not writable')); } const id = this.nextRequestId++; const payload = JSON.stringify({ id, method, params }); return new Promise((resolve, reject) => { - this.pendingRequests.set(id, { resolve, reject }); + let timer: ReturnType | undefined; + if (timeoutMs && timeoutMs > 0 && Number.isFinite(timeoutMs)) { + timer = setTimeout(() => { + if (!this.pendingRequests.delete(id)) return; + reject(new Error(`Codex app-server request ${method} did not settle within ${Math.round(timeoutMs)}ms`)); + }, timeoutMs); + timer.unref?.(); + } + this.pendingRequests.set(id, { + resolve: (value) => { + if (timer) clearTimeout(timer); + resolve(value); + }, + reject: (error) => { + if (timer) clearTimeout(timer); + reject(error); + }, + }); this.child!.stdin.write(`${payload}\n`); }); } + private completeCompact(sessionId: string, state: CodexSdkSessionState, turnId?: string): void { + this.clearCancelTimer(state); + this.clearCompactTimers(state); + this.clearStatus(sessionId, state); + state.runningCompact = false; + state.runningTurnId = undefined; + state.compactObserved = false; + state.currentMessageId = null; + state.currentText = ''; + const completed: AgentMessage = { + id: turnId ? `${turnId}:context-compaction` : `${sessionId}:context-compaction:${Date.now()}`, + sessionId, + kind: 'system', + role: 'system', + content: 'Codex context compacted.', + timestamp: Date.now(), + status: 'complete', + metadata: { + provider: this.id, + event: 'thread/compacted', + [SESSION_CONTROL_METADATA_COMMAND_FIELD]: 'compact', + ...(state.threadId ? { resumeId: state.threadId } : {}), + ...(turnId ? { turnId } : {}), + }, + }; + for (const cb of this.completeCallbacks) cb(sessionId, completed); + } + /** * Expose the `account/rateLimits/read` RPC over the already-connected * app-server so callers (e.g. the daemon's rate-limit probe) can reuse @@ -869,6 +1236,19 @@ export class CodexSdkProvider implements TransportProvider { this.emitStatus(sessionId, state, { status: null, label: null }); } + private cancelCompactLocally(sessionId: string, state: CodexSdkSessionState): void { + this.clearCancelTimer(state); + this.clearCompactTimers(state); + this.clearStatus(sessionId, state); + state.runningCompact = false; + state.runningTurnId = undefined; + state.compactObserved = false; + state.currentMessageId = null; + state.currentText = ''; + state.pendingComplete = undefined; + this.emitError(sessionId, this.makeError(PROVIDER_ERROR_CODES.CANCELLED, 'Codex compact cancelled', true)); + } + private emitError(sessionId: string, error: ProviderError): void { for (const cb of this.errorCallbacks) cb(sessionId, error); } @@ -897,4 +1277,52 @@ export class CodexSdkProvider implements TransportProvider { clearTimeout(state.cancelTimer); state.cancelTimer = null; } + + private armCompactNoSignalSettle(sessionId: string, state: CodexSdkSessionState, turnId?: string): void { + this.clearCompactSettleTimer(state); + state.compactSettleTimer = setTimeout(() => { + if (!this.sessions.has(sessionId)) return; + if (!state.runningCompact || state.compactObserved) return; + this.completeCompact(sessionId, state, turnId); + }, COMPACT_NO_SIGNAL_SETTLE_MS); + state.compactSettleTimer.unref?.(); + } + + private armCompactHardTimeout(sessionId: string, state: CodexSdkSessionState): void { + this.clearCompactHardTimer(state); + state.compactHardTimer = setTimeout(() => { + if (!this.sessions.has(sessionId)) return; + if (!state.runningCompact) return; + this.clearCompactTimers(state); + this.clearStatus(sessionId, state); + state.runningCompact = false; + state.runningTurnId = undefined; + state.compactObserved = false; + state.currentMessageId = null; + state.currentText = ''; + this.emitError(sessionId, this.makeError( + PROVIDER_ERROR_CODES.PROVIDER_ERROR, + `Codex SDK compact did not complete within ${Math.round(COMPACT_HARD_TIMEOUT_MS)}ms`, + true, + )); + }, COMPACT_HARD_TIMEOUT_MS); + state.compactHardTimer.unref?.(); + } + + private clearCompactSettleTimer(state: CodexSdkSessionState): void { + if (!state.compactSettleTimer) return; + clearTimeout(state.compactSettleTimer); + state.compactSettleTimer = null; + } + + private clearCompactHardTimer(state: CodexSdkSessionState): void { + if (!state.compactHardTimer) return; + clearTimeout(state.compactHardTimer); + state.compactHardTimer = null; + } + + private clearCompactTimers(state: CodexSdkSessionState): void { + this.clearCompactSettleTimer(state); + this.clearCompactHardTimer(state); + } } diff --git a/src/agent/providers/copilot-sdk.ts b/src/agent/providers/copilot-sdk.ts index ee23c97ce..e7ddd6a94 100644 --- a/src/agent/providers/copilot-sdk.ts +++ b/src/agent/providers/copilot-sdk.ts @@ -22,6 +22,10 @@ import { } from '../transport-provider.js'; import type { AgentMessage, MessageDelta } from '../../../shared/agent-message.js'; import type { ProviderContextPayload } from '../../../shared/context-types.js'; +import { + SESSION_CONTROL_METADATA_COMMAND_FIELD, + isSessionControlCommandText, +} from '../../../shared/session-control-commands.js'; import type { TransportAttachment } from '../../../shared/transport-attachments.js'; import logger from '../../util/logger.js'; import { resolveBinaryWithWindowsFallbacks } from '../transport-paths.js'; @@ -46,6 +50,11 @@ type CopilotSessionLike = { abort(): Promise; setModel(model: string, options?: Record): Promise; on(handler: (event: Record) => void): () => void; + rpc?: { + history?: { + compact?: () => Promise; + }; + }; disconnect?(): Promise; }; @@ -61,6 +70,19 @@ type CopilotClientLike = { listModels(): Promise>; }; +type CopilotOperation = 'idle' | 'turn' | 'compact' | 'cancelling'; + +type CopilotCompactResultLike = { + success?: boolean; + error?: string; + tokensRemoved?: number; + messagesRemoved?: number; + summaryContent?: string; + checkpointNumber?: number; + checkpointPath?: string; + requestId?: string; +}; + interface PendingApproval { routeId: string; requestId: string; @@ -79,12 +101,27 @@ interface CopilotSessionState { currentMessageId: string | null; currentText: string; completionEmittedForCurrentTurn: boolean; + /** Per-turn token usage from copilot's `assistant.usage` event. The + * upstream `@github/copilot` SDK ships ALL four fields (verified against + * copilot-sdk/generated/session-events.d.ts:1554-1580) but the previous + * implementation only read `outputTokens`, leaving every copilot turn at + * input_tokens=0/cache=0 in `context_turn_usage`. The chat-header context + * bar showed "0 / N" because transport-relay's normalize step couldn't + * find any input_tokens to display. */ currentOutputTokens?: number; + currentInputTokens?: number; + currentCacheReadTokens?: number; + currentCacheWriteTokens?: number; + /** USD cost from copilot's billing breakdown — surfaced as `costUsd` in + * `usage.update` payload + `context_turn_usage.cost_usd`. */ + currentCostUsd?: number; currentInteractionId?: string; busy: boolean; + operation: CopilotOperation; backgroundTainted: boolean; cancelRequested: boolean; cancelErrorEmitted: boolean; + compactCompletionEmitted: boolean; rotationInProgress: boolean; generation: number; lastStatusSignature: string | null; @@ -229,6 +266,12 @@ export class CopilotSdkProvider implements TransportProvider { reasoningEffort: true, supportedEffortLevels: ['low', 'medium', 'high', 'max'], contextSupport: 'degraded-message-side-context-mapping', + compact: { + execution: 'sdk-rpc', + verified: true, + completion: 'rpc-result-or-provider-event', + cancellation: 'provider-cancel', + }, }; private config: ProviderConfig | null = null; @@ -355,9 +398,11 @@ export class CopilotSdkProvider implements TransportProvider { currentOutputTokens: undefined, currentInteractionId: undefined, busy: false, + operation: 'idle', backgroundTainted: false, cancelRequested: false, cancelErrorEmitted: false, + compactCompletionEmitted: false, rotationInProgress: false, generation: 0, lastStatusSignature: null, @@ -487,17 +532,14 @@ export class CopilotSdkProvider implements TransportProvider { throw this.makeError(PROVIDER_ERROR_CODES.PROVIDER_ERROR, 'Copilot session is already busy', true); } const payload = normalizeProviderPayload(payloadOrMessage, attachments, extraSystemPrompt); + if (isSessionControlCommandText(payload.userMessage, 'compact')) { + await this.compactHistory(state); + return; + } const prompt = [payload.systemText?.trim(), payload.assembledMessage?.trim()].filter(Boolean).join('\n\n'); const sdkAttachments = toAttachmentPayload(payload.attachments); - state.currentMessageId = null; - state.currentText = ''; - state.completionEmittedForCurrentTurn = false; - state.currentOutputTokens = undefined; - state.currentInteractionId = undefined; - state.backgroundTainted = false; - state.cancelRequested = false; - state.cancelErrorEmitted = false; - state.rotationInProgress = false; + this.resetTurnState(state); + state.operation = 'turn'; state.busy = true; try { if (state.model) { @@ -512,21 +554,137 @@ export class CopilotSdkProvider implements TransportProvider { }); } catch (error) { state.busy = false; + state.operation = 'idle'; throw error; } } + private resetTurnState(state: CopilotSessionState): void { + state.currentMessageId = null; + state.currentText = ''; + state.completionEmittedForCurrentTurn = false; + state.currentOutputTokens = undefined; + state.currentInputTokens = undefined; + state.currentCacheReadTokens = undefined; + state.currentCacheWriteTokens = undefined; + state.currentCostUsd = undefined; + state.currentInteractionId = undefined; + state.backgroundTainted = false; + state.cancelRequested = false; + state.cancelErrorEmitted = false; + state.compactCompletionEmitted = false; + state.rotationInProgress = false; + } + + private async compactHistory(state: CopilotSessionState): Promise { + const history = state.session.rpc?.history; + const compact = history?.compact; + if (typeof compact !== 'function') { + const error = this.makeError( + PROVIDER_ERROR_CODES.PROVIDER_ERROR, + 'Copilot compact failed: SDK history.compact is unavailable', + false, + ); + this.emitError(state.routeId, error); + throw error; + } + this.resetTurnState(state); + state.busy = true; + state.operation = 'compact'; + this.emitStatus(state.routeId, { status: 'compacting', label: 'Compacting conversation...' }); + try { + const result = await compact.call(history); + if (state.operation === 'compact' && !state.cancelRequested) { + this.completeCompactOnce(state, 'session.history.compact', result ?? {}); + } + this.finishCompactOperation(state); + } catch (error) { + const providerError = this.toCompactError(error); + this.finishCompactOperation(state); + if (state.cancelRequested) return; + this.emitError(state.routeId, providerError); + throw providerError; + } + } + + private completeCompactOnce(state: CopilotSessionState, event: string, data: CopilotCompactResultLike): void { + if (state.operation !== 'compact' || state.compactCompletionEmitted || state.cancelRequested) return; + state.compactCompletionEmitted = true; + state.completionEmittedForCurrentTurn = true; + if (data?.success === false) { + this.emitError(state.routeId, this.makeError( + PROVIDER_ERROR_CODES.PROVIDER_ERROR, + `Copilot compact failed: ${data.error ?? 'SDK reported no compactable changes'}`, + true, + data, + )); + return; + } + const complete: AgentMessage = { + id: `${state.sessionId}:context-compaction:${Date.now()}`, + sessionId: state.routeId, + kind: 'system', + role: 'system', + content: 'Copilot context compacted.', + timestamp: Date.now(), + status: 'complete', + metadata: { + provider: this.id, + event, + [SESSION_CONTROL_METADATA_COMMAND_FIELD]: 'compact', + resumeId: state.sessionId, + ...(typeof data?.tokensRemoved === 'number' ? { tokensRemoved: data.tokensRemoved } : {}), + ...(typeof data?.messagesRemoved === 'number' ? { messagesRemoved: data.messagesRemoved } : {}), + ...(typeof data?.summaryContent === 'string' ? { summaryContent: data.summaryContent } : {}), + ...(typeof data?.checkpointNumber === 'number' ? { checkpointNumber: data.checkpointNumber } : {}), + ...(typeof data?.checkpointPath === 'string' ? { checkpointPath: data.checkpointPath } : {}), + ...(typeof data?.requestId === 'string' ? { requestId: data.requestId } : {}), + }, + }; + for (const cb of this.completeCallbacks) cb(state.routeId, complete); + } + + private finishCompactOperation(state: CopilotSessionState): void { + if (state.operation === 'compact' || state.operation === 'cancelling') { + state.operation = 'idle'; + } + state.busy = false; + this.emitStatus(state.routeId, { status: null, label: null }); + } + + private toCompactError(error: unknown): ProviderError { + if (this.isProviderError(error)) { + return { + ...error, + message: /^Copilot compact failed:/i.test(error.message) + ? error.message + : `Copilot compact failed: ${error.message}`, + }; + } + const message = error instanceof Error ? error.message : String(error); + return this.makeError(PROVIDER_ERROR_CODES.PROVIDER_ERROR, `Copilot compact failed: ${message}`, false, error); + } + async cancel(sessionId: string): Promise { const state = this.getSessionState(sessionId); if (!state) return; + const wasCompact = state.operation === 'compact'; state.cancelRequested = true; + state.operation = 'cancelling'; try { await state.session.abort(); } finally { state.busy = false; + state.operation = 'idle'; + this.emitStatus(state.routeId, { status: null, label: null }); + if (wasCompact) state.compactCompletionEmitted = true; if (!state.cancelErrorEmitted) { state.cancelErrorEmitted = true; - this.emitError(state.routeId, this.makeError(PROVIDER_ERROR_CODES.CANCELLED, 'Copilot turn cancelled', true)); + this.emitError(state.routeId, this.makeError( + PROVIDER_ERROR_CODES.CANCELLED, + wasCompact ? 'Copilot compact cancelled' : 'Copilot turn cancelled', + true, + )); } } if (!state.backgroundTainted) return; @@ -640,9 +798,27 @@ export class CopilotSdkProvider implements TransportProvider { return; } case 'assistant.usage': { + // Capture the full token + cost breakdown from copilot's per-API-call + // usage event (schema: copilot-sdk/generated/session-events.d.ts:1554). + // Multiple usage events can fire per turn (sub-agent calls); we + // overwrite rather than accumulate to match the previous behavior of + // currentOutputTokens — accumulation would be more accurate but is a + // separate change that needs UI/contract review. + if (typeof event.data?.inputTokens === 'number') { + state.currentInputTokens = event.data.inputTokens; + } if (typeof event.data?.outputTokens === 'number') { state.currentOutputTokens = event.data.outputTokens; } + if (typeof event.data?.cacheReadTokens === 'number') { + state.currentCacheReadTokens = event.data.cacheReadTokens; + } + if (typeof event.data?.cacheWriteTokens === 'number') { + state.currentCacheWriteTokens = event.data.cacheWriteTokens; + } + if (typeof event.data?.cost === 'number') { + state.currentCostUsd = event.data.cost; + } if (isNonEmptyString(event.data?.interactionId)) { state.currentInteractionId = event.data.interactionId; } @@ -678,8 +854,25 @@ export class CopilotSdkProvider implements TransportProvider { } return; } + case 'session.compaction_start': { + if (state.operation === 'compact') { + this.emitStatus(routeId, { status: 'compacting', label: 'Compacting conversation...' }); + } + return; + } + case 'session.compaction_complete': { + if (state.operation === 'compact' && !state.cancelRequested) { + this.completeCompactOnce(state, 'session.compaction_complete', event.data ?? {}); + this.finishCompactOperation(state); + } + return; + } case 'session.idle': { + if (state.operation === 'compact') { + return; + } state.busy = false; + state.operation = 'idle'; if (state.cancelRequested && !state.cancelErrorEmitted) { state.cancelErrorEmitted = true; this.emitError(routeId, this.makeError(PROVIDER_ERROR_CODES.CANCELLED, 'Copilot turn cancelled', true)); @@ -697,9 +890,34 @@ export class CopilotSdkProvider implements TransportProvider { status: 'complete', metadata: { ...(state.model ? { model: state.model } : {}), - ...(typeof state.currentOutputTokens === 'number' - ? { usage: { output_tokens: state.currentOutputTokens } } + // Build usage with whichever fields we captured. transport-relay's + // normalizeUsageUpdatePayload reads input_tokens / output_tokens / + // cache_read_input_tokens / cache_creation_input_tokens (snake_case); + // we already collected copilot's camelCase fields and translate + // here so the chat header context bar + context_turn_usage row + // pick them up like every other provider. + ...(typeof state.currentInputTokens === 'number' + || typeof state.currentOutputTokens === 'number' + || typeof state.currentCacheReadTokens === 'number' + || typeof state.currentCacheWriteTokens === 'number' + ? { + usage: { + ...(typeof state.currentInputTokens === 'number' + ? { input_tokens: state.currentInputTokens } + : {}), + ...(typeof state.currentOutputTokens === 'number' + ? { output_tokens: state.currentOutputTokens } + : {}), + ...(typeof state.currentCacheReadTokens === 'number' + ? { cache_read_input_tokens: state.currentCacheReadTokens } + : {}), + ...(typeof state.currentCacheWriteTokens === 'number' + ? { cache_creation_input_tokens: state.currentCacheWriteTokens } + : {}), + }, + } : {}), + ...(typeof state.currentCostUsd === 'number' ? { costUsd: state.currentCostUsd } : {}), ...(state.currentInteractionId ? { interactionId: state.currentInteractionId } : {}), resumeId: state.sessionId, }, @@ -710,6 +928,7 @@ export class CopilotSdkProvider implements TransportProvider { } case 'session.error': { state.busy = false; + state.operation = 'idle'; const error = this.makeError( PROVIDER_ERROR_CODES.PROVIDER_ERROR, String(event.data?.message ?? 'Copilot session error'), @@ -785,12 +1004,14 @@ export class CopilotSdkProvider implements TransportProvider { state.currentMessageId = null; state.currentText = ''; state.completionEmittedForCurrentTurn = false; - state.currentOutputTokens = undefined; + state.currentOutputTokens = undefined; state.currentInputTokens = undefined; state.currentCacheReadTokens = undefined; state.currentCacheWriteTokens = undefined; state.currentCostUsd = undefined; state.currentInteractionId = undefined; state.busy = false; + state.operation = 'idle'; state.backgroundTainted = false; state.cancelRequested = false; state.cancelErrorEmitted = false; + state.compactCompletionEmitted = false; this.attachSession(state); this.emitSessionInfo(state.routeId, { resumeId: state.sessionId, @@ -837,12 +1058,14 @@ export class CopilotSdkProvider implements TransportProvider { state.currentMessageId = null; state.currentText = ''; state.completionEmittedForCurrentTurn = false; - state.currentOutputTokens = undefined; + state.currentOutputTokens = undefined; state.currentInputTokens = undefined; state.currentCacheReadTokens = undefined; state.currentCacheWriteTokens = undefined; state.currentCostUsd = undefined; state.currentInteractionId = undefined; state.busy = false; + state.operation = 'idle'; state.backgroundTainted = false; state.cancelRequested = false; state.cancelErrorEmitted = false; + state.compactCompletionEmitted = false; state.rotationInProgress = false; this.attachSession(state); try { diff --git a/src/agent/providers/cursor-headless-stream.ts b/src/agent/providers/cursor-headless-stream.ts index 74c8e9a65..bf939c0e4 100644 --- a/src/agent/providers/cursor-headless-stream.ts +++ b/src/agent/providers/cursor-headless-stream.ts @@ -86,6 +86,56 @@ function pickString(record: CursorRecord, ...keys: string[]): string | undefined return undefined; } +/** + * cursor-agent CLI emits per-turn usage in **camelCase** (verified against + * 2026.05.04-08e5280 by piping `echo "what is 1+1" | cursor-agent --print + * --output-format stream-json --force`): + * {"usage":{"inputTokens":1227,"outputTokens":13,"cacheReadTokens":10624,"cacheWriteTokens":0}} + * + * Everything downstream — `transport-relay.normalizeUsageUpdatePayload`, + * `ProviderUsageUpdate.usage`, the SQLite write path — expects + * `ProviderUsageUpdate.usage` shape (snake_case `input_tokens` / + * `output_tokens`). Cursor's cache counters are not provider-window occupancy: + * long-running sessions have shown `cacheReadTokens` exceeding the model + * context window while the live prompt is small. Preserve those counters under + * cursor-specific diagnostic keys instead of mapping them to canonical cache + * fields; otherwise the UI context meter reads cumulative/billing cache as + * live context and shows impossible values such as 1.3M / 1M. + * + * Without translation every cursor turn produced `undefined` token fields: + * the chat header context bar showed "0 / 1M (0.0%)", and `context_turn_usage` + * had zero rows for `cursor-headless` sessions even after the May 5 telemetry + * commit. Translate cursor's camelCase into the canonical snake_case here so + * the rest of the pipeline can treat cursor like every other provider. + */ +function normalizeCursorUsage(raw: CursorRecord | undefined): CursorRecord | undefined { + if (!raw) return undefined; + const result: CursorRecord = {}; + // Pass through any already-snake_case fields (defensive — cursor-agent could + // change shape in a future release without warning). + for (const k of Object.keys(raw)) result[k] = raw[k]; + // camelCase → snake_case mapping. Each mapping only fires when the + // canonical field is absent so a future cursor-agent emitting native + // snake_case won't be double-overwritten. + const map: Array<[string, string]> = [ + ['inputTokens', 'input_tokens'], + ['outputTokens', 'output_tokens'], + ['contextWindow', 'model_context_window'], + ]; + for (const [from, to] of map) { + if (typeof raw[from] === 'number' && typeof result[to] !== 'number') { + result[to] = raw[from]; + } + } + if (typeof raw.cacheReadTokens === 'number' && typeof result.cursor_cache_read_tokens !== 'number') { + result.cursor_cache_read_tokens = raw.cacheReadTokens; + } + if (typeof raw.cacheWriteTokens === 'number' && typeof result.cursor_cache_write_tokens !== 'number') { + result.cursor_cache_write_tokens = raw.cacheWriteTokens; + } + return result; +} + function pickRecord(value: unknown): CursorRecord | undefined { return isRecord(value) ? value : undefined; } @@ -245,7 +295,8 @@ function parseCursorRecord(record: unknown, fallbackSessionId?: string): CursorP ?? extractTextFromContent(record.text) ?? extractTextFromContent(pickRecord(record.message)?.content) ?? (typeof record.result === 'string' ? record.result : undefined); - const usage = pickRecord(record.usage) ?? pickRecord(pickRecord(record.message)?.usage); + const rawUsage = pickRecord(record.usage) ?? pickRecord(pickRecord(record.message)?.usage); + const usage = normalizeCursorUsage(rawUsage); return { kind: 'result.success', raw: record, diff --git a/src/agent/providers/cursor-headless.ts b/src/agent/providers/cursor-headless.ts index e0490d59b..0fc35f21e 100644 --- a/src/agent/providers/cursor-headless.ts +++ b/src/agent/providers/cursor-headless.ts @@ -108,6 +108,13 @@ export class CursorHeadlessProvider implements TransportProvider { attachments: false, reasoningEffort: false, contextSupport: 'degraded-message-side-context-mapping', + compact: { + execution: 'unsupported', + verified: true, + completion: 'none', + cancellation: 'none', + reason: 'Verified with cursor-agent 2026.05.05-84a231c: CLI help exposes no compact/compress command or compact API for the headless adapter.', + }, }; private config: ProviderConfig | null = null; diff --git a/src/agent/providers/gemini-sdk.ts b/src/agent/providers/gemini-sdk.ts index 0c89d091f..40ae668a8 100644 --- a/src/agent/providers/gemini-sdk.ts +++ b/src/agent/providers/gemini-sdk.ts @@ -127,6 +127,14 @@ interface GeminiSdkSessionState { /** Track last emitted signature per tool to deduplicate identical updates. */ emittedToolSignatures: Map; lastStatusSignature: string | null; + /** Most recent ACP `usage_update.tokens` for this session — captured here so + * the onComplete `metadata.usage` can carry per-turn token counts to the + * daemon's transport-relay (where they end up as `usage.update` timeline + * events and rows in `context_turn_usage`). Without this, gemini-sdk + * `metadata.usage` is empty and EVERY gemini-sdk turn is invisible in + * cost analytics — confirmed by inspecting the production SQLite + * (`context_turn_usage` had 0 gemini-sdk rows out of 599 total). */ + lastTurnUsage?: Record; } interface MergedToolCall { @@ -152,6 +160,13 @@ export class GeminiSdkProvider implements TransportProvider { attachments: false, reasoningEffort: false, contextSupport: 'degraded-message-side-context-mapping', + compact: { + execution: 'unsupported', + verified: true, + completion: 'none', + cancellation: 'none', + reason: 'Verified with Gemini CLI 0.39.1: regular CLI registers /compress with /compact and /summarize aliases, but the --acp command registry used by this adapter does not register compress/compact.', + }, }; private config: ProviderConfig | null = null; @@ -683,6 +698,13 @@ export class GeminiSdkProvider implements TransportProvider { ); return; } + // Snapshot + clear the per-turn ACP usage so each onComplete carries its + // OWN token counts (not the previous turn's). Transport-relay's + // normalizeUsageUpdatePayload reads `metadata.usage` to emit usage.update + // → recorded in context_turn_usage with the turn's eventId. + const turnUsage = state.lastTurnUsage; + state.lastTurnUsage = undefined; + if (stopReason === 'max_tokens' || stopReason === 'max_turn_requests') { // Still emit whatever text we accumulated — it's a partial but useful // response — and mark metadata so the UI can show the truncation cause. @@ -698,6 +720,7 @@ export class GeminiSdkProvider implements TransportProvider { stopReason, ...(state.model ? { model: state.model } : {}), ...(state.acpSessionId ? { resumeId: state.acpSessionId } : {}), + ...(turnUsage ? { usage: turnUsage } : {}), }, }; for (const cb of this.completeCallbacks) cb(sessionId, msg); @@ -716,6 +739,7 @@ export class GeminiSdkProvider implements TransportProvider { metadata: { ...(state.model ? { model: state.model } : {}), ...(state.acpSessionId ? { resumeId: state.acpSessionId } : {}), + ...(turnUsage ? { usage: turnUsage } : {}), }, }; for (const cb of this.completeCallbacks) cb(sessionId, msg); @@ -765,10 +789,21 @@ export class GeminiSdkProvider implements TransportProvider { // Map ACP's usage update onto our generic quota/session-info signal. // ACP's `UsageUpdate` is experimental; guard every field. const u = update as Record; + const tokens = (typeof u.tokens === 'object' && u.tokens) + ? (u.tokens as Record) + : undefined; + // Cache for the next onComplete so transport-relay can normalize it + // into a `usage.update` timeline event with input/output token data. + // ACP token field names vary across Gemini ACP versions; pass the + // raw map through and let transport-relay's normalizeUsageUpdatePayload + // pick up `input_tokens`/`output_tokens`/`cache_*` whichever form + // Gemini emitted. + if (tokens) { + const sessionState = this.sessions.get(routeId); + if (sessionState) sessionState.lastTurnUsage = tokens; + } this.emitSessionInfo(routeId, { - ...(typeof u.tokens === 'object' && u.tokens - ? { quotaMeta: u.tokens as Record as never } - : {}), + ...(tokens ? { quotaMeta: tokens as Record as never } : {}), }); return; } diff --git a/src/agent/providers/openclaw.ts b/src/agent/providers/openclaw.ts index 9a39ab88c..6210e4634 100644 --- a/src/agent/providers/openclaw.ts +++ b/src/agent/providers/openclaw.ts @@ -101,6 +101,13 @@ export class OpenClawProvider implements TransportProvider { reasoningEffort: true, supportedEffortLevels: OPENCLAW_THINKING_LEVELS, contextSupport: 'full-normalized-context-injection', + compact: { + execution: 'unsupported', + verified: true, + completion: 'none', + cancellation: 'none', + reason: 'Verified in this adapter/environment: OpenClaw exposes no compact RPC/command path here, and no local openclaw CLI is installed to test a provider slash command.', + }, }; // ── Private state ────────────────────────────────────────────────────────── diff --git a/src/agent/providers/qwen.ts b/src/agent/providers/qwen.ts index 541a00222..053a30f0d 100644 --- a/src/agent/providers/qwen.ts +++ b/src/agent/providers/qwen.ts @@ -29,9 +29,14 @@ import { DEFAULT_TRANSPORT_EFFORT, QWEN_EFFORT_LEVELS, type TransportEffortLevel import logger from '../../util/logger.js'; import { inferContextWindow } from '../../util/model-context.js'; import { normalizeTransportCwd, resolveExecutableForSpawn } from '../transport-paths.js'; +import { + SESSION_CONTROL_METADATA_COMMAND_FIELD, + isSessionCompactCommandText, +} from '../../../shared/session-control-commands.js'; const execFileAsync = promisify(execFile); const QWEN_BIN = 'qwen'; +const QWEN_COMPACT_SLASH_COMMAND = '/compress' as const; const TRANSIENT_RETRY_DELAY_MS = 250; const TRANSIENT_RETRY_MAX_ATTEMPTS = 1; const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -77,6 +82,23 @@ function extractSyntheticApiError(text: string | undefined): string | undefined return match?.[1]?.trim() || undefined; } +function toQwenCompactPayload(payload: ProviderContextPayload): ProviderContextPayload { + const { systemText: _systemText, messagePreamble: _messagePreamble, startupMemory: _startupMemory, memoryRecall: _memoryRecall, ...rest } = payload; + const { systemText: _contextSystemText, messagePreamble: _contextMessagePreamble, ...contextRest } = payload.context; + return { + ...rest, + userMessage: QWEN_COMPACT_SLASH_COMMAND, + assembledMessage: QWEN_COMPACT_SLASH_COMMAND, + context: { + ...contextRest, + requiredAuthoredContext: [], + advisoryAuthoredContext: [], + diagnostics: [], + }, + diagnostics: [], + }; +} + interface QwenSessionState { cwd: string; started: boolean; @@ -231,6 +253,14 @@ export class QwenProvider implements TransportProvider { reasoningEffort: true, supportedEffortLevels: QWEN_EFFORT_LEVELS, contextSupport: 'degraded-message-side-context-mapping', + compact: { + execution: 'slash-command', + providerCommand: QWEN_COMPACT_SLASH_COMMAND, + verified: true, + completion: 'command-result', + cancellation: 'provider-cancel', + reason: 'Verified with Qwen Code 0.14.5: non-interactive CLI supports /compress, not /compact; adapter translates the IM.codes /compact control command.', + }, }; private config: ProviderConfig | null = null; @@ -413,14 +443,25 @@ export class QwenProvider implements TransportProvider { state.emittedToolSignatures.clear(); state.lastStatusSignature = null; const payload = normalizeProviderPayload(payloadOrMessage, _attachments, extraSystemPrompt); + const isCompactControl = isSessionCompactCommandText(payload.userMessage); + const providerPayload = isCompactControl ? toQwenCompactPayload(payload) : payload; + const compactCompletionMetadata = isCompactControl + ? { + [SESSION_CONTROL_METADATA_COMMAND_FIELD]: 'compact', + event: 'session.history.compress', + providerCommand: QWEN_COMPACT_SLASH_COMMAND, + } + : undefined; const args = [ - '-p', payload.assembledMessage, + '-p', providerPayload.assembledMessage, '--output-format', 'stream-json', '--include-partial-messages', '--approval-mode', 'yolo', ]; - const effectivePrompt = payload.systemText?.trim() || state.description?.trim(); + const effectivePrompt = isCompactControl + ? undefined + : (providerPayload.systemText?.trim() || state.description?.trim()); if (effectivePrompt) { args.push('--append-system-prompt', effectivePrompt); } @@ -459,6 +500,12 @@ export class QwenProvider implements TransportProvider { }); state.child = child; this.sessions.set(sessionId, state); + if (isCompactControl) { + this.emitStatus(sessionId, state, { + status: 'compacting', + label: 'Compacting conversation...', + }); + } let completed = false; let sawError = false; @@ -487,6 +534,7 @@ export class QwenProvider implements TransportProvider { const emitError = (messageText: string, details?: unknown): void => { if (sawError || completed) return; sawError = true; + this.clearStatus(sessionId, state); const errorCode = state.cancelled ? PROVIDER_ERROR_CODES.CANCELLED : (this.isAuthFailureMessage(messageText) ? PROVIDER_ERROR_CODES.AUTH_FAILED : PROVIDER_ERROR_CODES.PROVIDER_ERROR); @@ -503,11 +551,12 @@ export class QwenProvider implements TransportProvider { state.pendingFinalText = undefined; state.pendingFinalMetadata = undefined; const finalMessageId = messageId || randomUUID(); + const isCompactCompletion = metadata?.[SESSION_CONTROL_METADATA_COMMAND_FIELD] === 'compact'; const msg: AgentMessage = { id: finalMessageId, sessionId, - kind: 'text', - role: 'assistant', + kind: isCompactCompletion ? 'system' : 'text', + role: isCompactCompletion ? 'system' : 'assistant', content: text, timestamp: Date.now(), status: 'complete', @@ -685,6 +734,7 @@ export class QwenProvider implements TransportProvider { state.pendingFinalMetadata = { ...(state.model || payload.message?.model ? { model: state.model ?? payload.message?.model } : {}), ...(payload.message?.usage ? { usage: sanitizeUsageForDisplay(payload.message.usage, state.model ?? payload.message?.model) } : {}), + ...(compactCompletionMetadata ?? {}), }; } return; @@ -739,6 +789,7 @@ export class QwenProvider implements TransportProvider { ...(state.pendingFinalMetadata ?? {}), ...(state.model ? { model: state.model } : {}), ...(!assistantUsage && sanitizedResultUsage ? { usage: sanitizedResultUsage } : {}), + ...(compactCompletionMetadata ?? {}), }; } } diff --git a/src/agent/repository-identity-service.ts b/src/agent/repository-identity-service.ts index 71a14b658..458f26a4f 100644 --- a/src/agent/repository-identity-service.ts +++ b/src/agent/repository-identity-service.ts @@ -17,6 +17,10 @@ function sha(input: string): string { return createHash('sha1').update(input).digest('hex').slice(0, 12); } +function canonicalPart(value: string): string { + return value.trim().toLowerCase(); +} + type CanonicalParts = { host: string; owner: string; @@ -28,9 +32,9 @@ export function parseCanonicalRepositoryKey(key: string): CanonicalParts | null const match = /^([^/]+)\/([^/]+)\/([^/]+)$/.exec(trimmed); if (!match) return null; return { - host: match[1].toLowerCase(), - owner: match[2], - repo: match[3], + host: canonicalPart(match[1]), + owner: canonicalPart(match[2]), + repo: canonicalPart(match[3]), }; } @@ -40,12 +44,15 @@ export class GitOriginRepositoryIdentityService implements RepositoryIdentitySer if (originUrl) { const parsed = parseRemoteUrl(originUrl); if (parsed) { + const host = canonicalPart(parsed.host); + const owner = canonicalPart(parsed.owner); + const repo = canonicalPart(parsed.repo); return { kind: 'git-origin', - key: `${parsed.host.toLowerCase()}/${parsed.owner}/${parsed.repo}`, - host: parsed.host.toLowerCase(), - owner: parsed.owner, - repo: parsed.repo, + key: `${host}/${owner}/${repo}`, + host, + owner, + repo, originUrl, }; } @@ -73,7 +80,7 @@ export class GitOriginRepositoryIdentityService implements RepositoryIdentitySer const canonical = parseCanonicalRepositoryKey(canonicalKey); const alias = parseRemoteUrl(aliasOriginUrl.trim()); if (!canonical || !alias) return null; - if (canonical.owner !== alias.owner || canonical.repo !== alias.repo) return null; + if (canonical.owner !== canonicalPart(alias.owner) || canonical.repo !== canonicalPart(alias.repo)) return null; return { aliasKey: aliasOriginUrl.trim(), canonicalKey, diff --git a/src/agent/runtime-context-bootstrap.ts b/src/agent/runtime-context-bootstrap.ts index 1b1ea830a..ce2bcd975 100644 --- a/src/agent/runtime-context-bootstrap.ts +++ b/src/agent/runtime-context-bootstrap.ts @@ -10,9 +10,21 @@ import { detectRepo } from '../repo/detector.js'; import { fetchBackendSharedContextNamespace } from '../context/backend-context-namespace.js'; import { getSharedContextRuntimeCredentials } from '../context/shared-context-runtime.js'; import type { MemorySearchResultItem } from '../context/memory-search.js'; -import { STARTUP_MEMORY_TOTAL_LIMIT, selectStartupMemoryItems } from '../context/startup-memory.js'; +import { + STARTUP_MEMORY_TOTAL_LIMIT, + selectStartupMemoryByPolicy, + selectStartupMemoryItems, + type StartupMemoryCandidate, +} from '../context/startup-memory.js'; +import { collectSkillStartupCandidates } from '../context/skill-startup-context.js'; import { getLocalProcessedFreshness } from '../store/context-store.js'; -import { buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { + STARTUP_PROJECT_MEMORY_HEADER, + STARTUP_SKILL_INDEX_HEADER, + buildStartupProjectMemoryText, + formatRelatedPastWorkSummary, +} from '../../shared/memory-recall-format.js'; +import { isMemoryScope } from '../../shared/memory-scope.js'; export interface TransportContextBootstrapInput { projectDir?: string; @@ -36,14 +48,14 @@ const repositoryIdentityService = new GitOriginRepositoryIdentityService(); export async function resolveTransportContextBootstrap( input: TransportContextBootstrapInput, ): Promise { + const projectDir = input.projectDir?.trim(); const explicitNamespace = parseExplicitContextNamespace(input.transportConfig); if (explicitNamespace) { return buildBootstrapResult(explicitNamespace, { diagnostics: ['namespace:explicit'], - }, input.startupMemoryAlreadyInjected); + }, input.startupMemoryAlreadyInjected, projectDir); } - const projectDir = input.projectDir?.trim(); let originUrl: string | null | undefined; if (projectDir) { try { @@ -69,7 +81,7 @@ export async function resolveTransportContextBootstrap( remoteProcessedFreshness: resolved.remoteProcessedFreshness, retryExhausted: resolved.retryExhausted, sharedPolicyOverride: resolved.sharedPolicyOverride, - }, input.startupMemoryAlreadyInjected); + }, input.startupMemoryAlreadyInjected, projectDir); } const personalNamespace: ContextNamespace = { scope: 'personal', @@ -79,7 +91,7 @@ export async function resolveTransportContextBootstrap( diagnostics: ['namespace:server-personal-fallback', ...(resolved?.diagnostics ?? [])], remoteProcessedFreshness: resolved?.remoteProcessedFreshness, retryExhausted: resolved?.retryExhausted, - }, input.startupMemoryAlreadyInjected); + }, input.startupMemoryAlreadyInjected, projectDir); } catch { const personalNamespace: ContextNamespace = { scope: 'personal', @@ -87,7 +99,7 @@ export async function resolveTransportContextBootstrap( }; return buildBootstrapResult(personalNamespace, { diagnostics: ['namespace:server-resolution-failed', 'namespace:git-origin'], - }, input.startupMemoryAlreadyInjected); + }, input.startupMemoryAlreadyInjected, projectDir); } } } @@ -98,30 +110,53 @@ export async function resolveTransportContextBootstrap( }; return buildBootstrapResult(fallbackNamespace, { diagnostics: [`namespace:${canonical.kind}`], - }, input.startupMemoryAlreadyInjected); + }, input.startupMemoryAlreadyInjected, projectDir); } function buildBootstrapResult( namespace: ContextNamespace, extras: Omit, skipStartupMemory = false, + projectDir?: string, ): TransportContextBootstrap { return { namespace, ...extras, localProcessedFreshness: getLocalProcessedFreshness(namespace), - startupMemory: skipStartupMemory ? undefined : buildTransportStartupMemory(namespace), + startupMemory: skipStartupMemory ? undefined : buildTransportStartupMemory(namespace, { + projectDir, + }), }; } export function buildTransportStartupMemory( namespace: ContextNamespace, - limit = STARTUP_MEMORY_TOTAL_LIMIT, + limitOrOptions: number | { limit?: number; projectDir?: string; homeDir?: string; skillsFeatureEnabled?: boolean } = STARTUP_MEMORY_TOTAL_LIMIT, ): TransportMemoryRecallArtifact | undefined { try { - const items = selectStartupMemoryItems(namespace, { totalLimit: limit }) - .map(toTransportMemoryRecallItem); - if (items.length === 0) return undefined; + const options = typeof limitOrOptions === 'number' + ? { limit: limitOrOptions } + : limitOrOptions; + const limit = options.limit ?? STARTUP_MEMORY_TOTAL_LIMIT; + const processedItems = selectStartupMemoryItems(namespace, { totalLimit: limit }); + const processedById = new Map(processedItems.map((item) => [item.id, item])); + const skillCandidates = collectSkillStartupCandidates({ + namespace, + projectDir: options.projectDir, + homeDir: options.homeDir, + featureEnabled: options.skillsFeatureEnabled, + }); + const selected = selectStartupMemoryByPolicy([ + ...processedItems.map(memorySearchItemToStartupCandidate), + ...skillCandidates, + ]); + const items = selected.selected.map((candidate) => { + const processed = processedById.get(candidate.id); + return processed + ? toTransportMemoryRecallItem(processed) + : startupCandidateToTransportMemoryRecallItem(candidate, namespace); + }); + if (items.length === 0 || selected.selected.length === 0) return undefined; return { reason: 'startup', runtimeFamily: 'transport', @@ -129,13 +164,40 @@ export function buildTransportStartupMemory( sourceKind: 'local_processed', injectionSurface: 'system-text', items, - injectedText: renderStartupMemoryText(items), + injectedText: renderStartupMemoryText(selected.selected, processedById), }; } catch { return undefined; } } +function memorySearchItemToStartupCandidate(item: MemorySearchResultItem): StartupMemoryCandidate { + return { + id: item.id, + source: item.projectionClass === 'durable_memory_candidate' ? 'durable' : 'recent', + text: item.summary, + updatedAt: item.updatedAt ?? item.createdAt, + fingerprint: `${item.projectionClass ?? 'recent_summary'}\u0000${item.summary}`, + }; +} + +function startupCandidateToTransportMemoryRecallItem( + candidate: StartupMemoryCandidate, + namespace: ContextNamespace, +): TransportMemoryRecallItem { + return { + id: candidate.id, + type: 'processed', + projectId: namespace.projectId ?? namespace.userId ?? namespace.enterpriseId ?? 'memory', + scope: namespace.scope, + ...(namespace.enterpriseId ? { enterpriseId: namespace.enterpriseId } : {}), + ...(namespace.workspaceId ? { workspaceId: namespace.workspaceId } : {}), + ...(namespace.userId ? { userId: namespace.userId } : {}), + summary: candidate.text, + ...(typeof candidate.updatedAt === 'number' ? { updatedAt: candidate.updatedAt } : {}), + }; +} + function toTransportMemoryRecallItem(item: MemorySearchResultItem): TransportMemoryRecallItem { return { id: item.id, @@ -156,8 +218,32 @@ function toTransportMemoryRecallItem(item: MemorySearchResultItem): TransportMem }; } -function renderStartupMemoryText(items: TransportMemoryRecallItem[]): string { - return buildStartupProjectMemoryText(items); +function renderStartupMemoryText( + selected: readonly StartupMemoryCandidate[], + processedById: ReadonlyMap, +): string { + const memoryItems = selected + .map((candidate) => processedById.get(candidate.id)) + .filter((item): item is MemorySearchResultItem => !!item) + .map(toTransportMemoryRecallItem); + const sections: string[] = []; + if (memoryItems.length > 0) { + sections.push(buildStartupProjectMemoryText(memoryItems)); + } + const skillBlocks = selected.filter((candidate) => candidate.source === 'skill'); + if (skillBlocks.length > 0) { + sections.push([ + STARTUP_SKILL_INDEX_HEADER, + '', + 'Read a listed skill file only when it is relevant to the current task; do not treat this index as the skill body.', + ...skillBlocks.map((candidate) => [ + `- [skill] ${formatRelatedPastWorkSummary(candidate.id, 120)}`, + candidate.text, + ].join('\n')), + '', + ].join('\n')); + } + return sections.join('\n\n'); } function parseExplicitContextNamespace( @@ -192,8 +278,5 @@ function extractNamespaceCandidate( } function isContextScope(value: string | undefined): value is ContextNamespace['scope'] { - return value === 'personal' - || value === 'project_shared' - || value === 'workspace_shared' - || value === 'org_shared'; + return isMemoryScope(value); } diff --git a/src/agent/session-manager.ts b/src/agent/session-manager.ts index 37c61b020..8b4309881 100644 --- a/src/agent/session-manager.ts +++ b/src/agent/session-manager.ts @@ -42,6 +42,7 @@ import { providerQuotaMetaEquals } from '../../shared/provider-quota.js'; import { resolveTransportContextBootstrap } from './runtime-context-bootstrap.js'; import { QWEN_AUTH_TYPES } from '../../shared/qwen-auth.js'; import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; +import { IMCODES_SESSION_ENV, IMCODES_SESSION_LABEL_ENV } from '../../shared/imcodes-send.js'; import { getAgentVersion } from './agent-version.js'; import { repoCache } from '../repo/cache.js'; @@ -956,6 +957,41 @@ const transportRuntimes = new Map(); const transportErrorRecoveryInFlight = new Map>(); const transportErrorRecoveryTimestamps = new Map(); +function buildTransportSessionEnv( + sessionName: string, + label: string | null | undefined, + extraEnv?: Record, +): Record { + return { + ...(extraEnv ?? {}), + [IMCODES_SESSION_ENV]: sessionName, + [IMCODES_SESSION_LABEL_ENV]: label?.trim() || sessionName, + }; +} + +function buildTransportImcodesIdentityPrompt( + sessionName: string, + label: string | null | undefined, +): string { + const displayLabel = label?.trim() || sessionName; + return [ + 'IM.codes session identity:', + `- Exact session name: ${sessionName}`, + `- Display label: ${displayLabel}`, + `- When invoking \`imcodes send\`, prefer $${IMCODES_SESSION_ENV}. If a SDK/tool environment lacks it, prefix the command with ${IMCODES_SESSION_ENV}=${sessionName}. Do not use display labels as sender identity unless the exact session name is unavailable, because labels can be duplicated.`, + ].join('\n'); +} + +function mergeTransportSystemPromptWithIdentity( + systemPrompt: string | undefined, + sessionName: string, + label: string | null | undefined, +): string { + return [systemPrompt?.trim(), buildTransportImcodesIdentityPrompt(sessionName, label)] + .filter(Boolean) + .join('\n\n'); +} + function queueTransportErrorResendEntries(sessionName: string, entries: PendingTransportMessage[]): number { if (entries.length === 0) return getResendCount(sessionName); const existingCommandIds = new Set(getResendEntries(sessionName).map((entry) => entry.commandId)); @@ -963,6 +999,7 @@ function queueTransportErrorResendEntries(sessionName: string, entries: PendingT if (existingCommandIds.has(entry.clientMessageId)) continue; enqueueResend(sessionName, { text: entry.text, + ...(entry.messagePreamble ? { messagePreamble: entry.messagePreamble } : {}), commandId: entry.clientMessageId, ...(entry.attachments?.length ? { attachments: entry.attachments } : {}), queuedAt: Date.now(), @@ -1379,11 +1416,11 @@ export async function restoreTransportSessions(providerId: string): Promise { const attachments = entry.attachments ?? []; - const result = attachments.length > 0 - ? runtime.send(entry.text, entry.commandId, attachments) - : runtime.send(entry.text, entry.commandId); + const result = entry.messagePreamble + ? runtime.send( + entry.text, + entry.commandId, + attachments.length > 0 ? attachments : undefined, + entry.messagePreamble, + ) + : (attachments.length > 0 + ? runtime.send(entry.text, entry.commandId, attachments) + : runtime.send(entry.text, entry.commandId)); if (result === 'sent') { timelineEmitter.emit( s.name, @@ -1669,11 +1713,11 @@ export async function launchTransportSession(opts: LaunchOpts): Promise { await runtime.initialize({ sessionKey: effectiveSessionKey, fresh: !!opts.fresh, - ...(transportEnv ? { env: transportEnv } : {}), + env: buildTransportSessionEnv(name, label, transportEnv), cwd: projectDir, label: label || name, description, - ...(transportSystemPrompt ? { systemPrompt: transportSystemPrompt } : {}), + systemPrompt: mergeTransportSystemPromptWithIdentity(transportSystemPrompt, name, label), ...(transportSettings ? { settings: transportSettings } : {}), contextNamespace: contextBootstrap.namespace, contextNamespaceDiagnostics: contextBootstrap.diagnostics, diff --git a/src/agent/tmux.ts b/src/agent/tmux.ts index f92bb3593..fe7ee5b0f 100644 --- a/src/agent/tmux.ts +++ b/src/agent/tmux.ts @@ -640,7 +640,10 @@ export function restoreWeztermPane(name: string, paneId: string): void { const XTERM_KEY_MAP: Record = { '\x1b[A': 'Up', '\x1b[B': 'Down', '\x1b[C': 'Right','\x1b[D': 'Left', + '\x1bOA': 'Up', '\x1bOB': 'Down', + '\x1bOC': 'Right','\x1bOD': 'Left', '\x1b[F': 'End', '\x1b[H': 'Home', + '\x1bOF': 'End', '\x1bOH': 'Home', '\x1b[1~': 'Home','\x1b[3~': 'DC', '\x1b[4~': 'End', '\x1b[5~': 'PPage', '\x1b[6~': 'NPage','\x1b[2~': 'IC', diff --git a/src/agent/transport-provider.ts b/src/agent/transport-provider.ts index e6d2c2acc..09e1be471 100644 --- a/src/agent/transport-provider.ts +++ b/src/agent/transport-provider.ts @@ -66,6 +66,31 @@ export type ProviderErrorCode = typeof PROVIDER_ERROR_CODES[keyof typeof PROVIDE // ── Supporting types ──────────────────────────────────────────────────────── +export type ProviderCompactExecution = 'sdk-rpc' | 'slash-command' | 'unsupported'; +export type ProviderCompactCompletion = + | 'rpc-result' + | 'provider-event' + | 'rpc-result-or-provider-event' + | 'command-result' + | 'status-only' + | 'none'; +export type ProviderCompactCancellation = 'provider-cancel' | 'local-cancel' | 'timeout-only' | 'none'; + +export interface ProviderCompactCapability { + /** How this provider executes the `/compact` control command. */ + execution: ProviderCompactExecution; + /** Provider-native slash command used when execution is `slash-command`. */ + providerCommand?: `/${string}`; + /** Whether the execution strategy is backed by this locked SDK/protocol version. */ + verified: boolean; + /** Where completion signals come from, if any. */ + completion: ProviderCompactCompletion; + /** Whether an in-flight compact can be cancelled or only locally abandoned. */ + cancellation: ProviderCompactCancellation; + /** Human-readable reason for unsupported or unverified behavior. */ + reason?: string; +} + /** * Provider capability flags. * Consumers MUST check the relevant flag before calling optional interface methods. @@ -89,6 +114,8 @@ export interface ProviderCapabilities { supportedEffortLevels?: readonly TransportEffortLevel[]; /** How well this provider can honor normalized shared-context payloads. */ contextSupport?: ProviderSupportClass; + /** Provider-specific `/compact` execution support. */ + compact?: ProviderCompactCapability; } /** @@ -223,6 +250,22 @@ export interface ProviderStatusUpdate { label?: string | null; } +/** Provider-reported token/context usage update. */ +export interface ProviderUsageUpdate { + /** Provider-native usage fields normalized enough for the daemon relay. */ + usage?: { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; + cached_input_tokens?: number; + model_context_window?: number; + [key: string]: unknown; + }; + /** Active model for resolving display context-window limits. */ + model?: string; +} + // ── TransportProvider interface ───────────────────────────────────────────── /** @@ -335,6 +378,14 @@ export interface TransportProvider { */ onStatus?(cb: (sessionId: string, status: ProviderStatusUpdate) => void): () => void; + /** + * Register a callback for token/context usage updates that can arrive + * independently from final assistant messages. Used by transports such as + * Codex SDK where tokenUsage notifications may race before or after + * item/turn completion. + */ + onUsage?(cb: (sessionId: string, update: ProviderUsageUpdate) => void): () => void; + /** * Register a callback for approval requests from the agent. * Only call when capabilities.approval is true. diff --git a/src/agent/transport-session-runtime.ts b/src/agent/transport-session-runtime.ts index b1d680bbc..17791bf67 100644 --- a/src/agent/transport-session-runtime.ts +++ b/src/agent/transport-session-runtime.ts @@ -6,6 +6,11 @@ import type { AgentMessage, MessageDelta } from '../../shared/agent-message.js'; import type { TransportProvider, ProviderError, SessionConfig, SessionInfoUpdate } from './transport-provider.js'; import type { ApprovalRequest } from './transport-provider.js'; import type { TransportEffortLevel } from '../../shared/effort-levels.js'; +import { + SESSION_CONTROL_METADATA_COMMAND_FIELD, + isSessionCompactCommandText, + shouldResetTransportPreferenceContextForSessionControl, +} from '../../shared/session-control-commands.js'; import type { TransportAttachment } from '../../shared/transport-attachments.js'; import { SharedContextDispatchError, @@ -32,6 +37,7 @@ import { clearRecentInjectionHistory, } from '../context/recent-injection-history.js'; import { getContextModelConfig } from '../context/context-model-config.js'; +import { PREFERENCE_CONTEXT_END, PREFERENCE_CONTEXT_START } from '../../shared/preference-ingest.js'; import { resolveRuntimeAuthoredContext } from '../context/shared-context-runtime.js'; import { buildTransportStartupMemory, type TransportContextBootstrap } from './runtime-context-bootstrap.js'; import { recordMemoryHits } from '../store/context-store.js'; @@ -40,7 +46,10 @@ import { incrementCounter } from '../util/metrics.js'; export interface PendingTransportMessage { clientMessageId: string; + /** User-visible task text, without daemon-rendered memory/context preambles. */ text: string; + /** Provider-visible per-turn context rendered through the shared context preamble path. */ + messagePreamble?: string; attachments?: TransportAttachment[]; } @@ -116,6 +125,10 @@ function withTimeoutOutcome( }); } +function isTransportSlashControl(message: string | undefined): boolean { + return message?.trim().startsWith('/') === true; +} + /** * Transport session runtime — manages a single conversation with a remote provider. * @@ -152,9 +165,15 @@ export class TransportSessionRuntime implements SessionRuntime { private _contextSharedPolicyOverride: SharedScopePolicyOverride | undefined; private _contextAuthoredContextLanguage: string | undefined; private _contextAuthoredContextFilePath: string | undefined; + private _projectDir: string | undefined; private _startupMemory: TransportMemoryRecallArtifact | null = null; private _startupMemoryTimelineEmitted = false; private _startupMemoryInjected = false; + /** Last provider-visible preference context block injected into this provider conversation. + * Preferences are stable session context, not per-turn recall; repeat injection + * bloats SDK prompt windows and can trigger provider auto-compaction. */ + private _lastInjectedPreferenceContextSignature: string | null = null; + private _preferenceContextInjectionAttempt: { previous: string | null } | null = null; private _contextBootstrapResolver: (() => Promise) | undefined; private _unsubscribes: Array<() => void> = []; private _onStatusChange?: (status: AgentStatus) => void; @@ -191,6 +210,9 @@ export class TransportSessionRuntime implements SessionRuntime { }), this.provider.onComplete((sid: string, message: AgentMessage) => { if (sid !== this._providerSessionId) return; + if (isTransportCompactionCompletion(message)) { + this._lastInjectedPreferenceContextSignature = null; + } this._sending = false; this._history.push(message); this._activeTurn?.resolve(); @@ -292,6 +314,7 @@ export class TransportSessionRuntime implements SessionRuntime { this._providerSessionId = await this.provider.createSession(config); this._description = config.description; this._systemPrompt = config.systemPrompt; + this._projectDir = config.cwd; this._agentId = config.agentId; this._effort = config.effort; this.applyContextBootstrap({ @@ -329,14 +352,24 @@ export class TransportSessionRuntime implements SessionRuntime { * * Returns 'sent' if dispatched immediately, 'queued' if enqueued. */ - send(message: string, clientMessageId?: string, attachments?: TransportAttachment[]): 'sent' | 'queued' { + send( + message: string, + clientMessageId?: string, + attachments?: TransportAttachment[], + messagePreamble?: string, + ): 'sent' | 'queued' { if (!this._providerSessionId) { throw new Error('TransportSessionRuntime not initialized — call initialize() first'); } + if (isSessionCompactCommandText(message) && this.provider.capabilities.compact?.execution === 'unsupported') { + const reason = this.provider.capabilities.compact.reason?.trim(); + throw new Error(reason || `${this.provider.id} does not support /compact`); + } const entry: PendingTransportMessage = { clientMessageId: clientMessageId ?? randomUUID(), text: message, + ...(messagePreamble?.trim() ? { messagePreamble: messagePreamble.trim() } : {}), ...(attachments?.length ? { attachments } : {}), }; @@ -355,6 +388,7 @@ export class TransportSessionRuntime implements SessionRuntime { const entry = this._pendingMessages.find((item) => item.clientMessageId === clientMessageId); if (!entry) return false; entry.text = nextText; + entry.messagePreamble = undefined; return true; } @@ -441,8 +475,13 @@ export class TransportSessionRuntime implements SessionRuntime { void promise.catch(() => {}); // prevent unhandled rejection this._activeTurn = { promise, resolve, reject }; + if (shouldResetTransportPreferenceContextForSessionControl(message)) { + this._lastInjectedPreferenceContextSignature = null; + } + void (async () => { await this.refreshContextBootstrap({ phase: 'dispatch' }); + const isSlashControl = isTransportSlashControl(message); const authority = resolveTransportDispatchAuthority(this.provider, { namespace: this._contextNamespace, remoteProcessedFreshness: this._contextRemoteProcessedFreshness, @@ -450,17 +489,27 @@ export class TransportSessionRuntime implements SessionRuntime { retryExhausted: this._contextRetryExhausted, sharedPolicyOverride: this._contextSharedPolicyOverride, }).authority; - const startupMemory = this._startupMemory ?? ( + const startupMemory = isSlashControl ? null : (this._startupMemory ?? ( !this._startupMemoryInjected && authority.authoritySource === 'processed_local' && this._contextNamespace - ? buildTransportStartupMemory(this._contextNamespace) + ? buildTransportStartupMemory(this._contextNamespace, { projectDir: this._projectDir }) : null - ); - const memoryRecallResult = await this.buildTransportMessageRecallResultWithinBudget(message, authority.authoritySource); + )); + const memoryRecallResult = isSlashControl + ? { + artifact: null, + statusPayload: buildMemoryContextStatusPayload(message.trim().slice(0, 200), 'skipped_control_message', 'message', { + runtimeFamily: 'transport', + authoritySource: authority.authoritySource, + sourceKind: 'local_processed', + }), + } + : await this.buildTransportMessageRecallResultWithinBudget(message, authority.authoritySource); const memoryRecall = memoryRecallResult.artifact; const dispatchResult = await dispatchSharedContextSend(this.provider, this._providerSessionId!, { userMessage: message, - description: this._description, - systemPrompt: this._systemPrompt, + messagePreamble: isSlashControl ? undefined : this.mergeMessagePreambles(dispatchedEntries, message), + description: isSlashControl ? undefined : this._description, + systemPrompt: isSlashControl ? undefined : this._systemPrompt, attachments, namespace: this._contextNamespace, namespaceDiagnostics: this._contextNamespaceDiagnostics, @@ -468,13 +517,14 @@ export class TransportSessionRuntime implements SessionRuntime { localProcessedFreshness: this._contextLocalProcessedFreshness, retryExhausted: this._contextRetryExhausted, sharedPolicyOverride: this._contextSharedPolicyOverride, - authoredContextRepository: this.resolveAuthoredContextRepository(), - authoredContextLanguage: this._contextAuthoredContextLanguage, - authoredContextFilePath: this._contextAuthoredContextFilePath, + authoredContextRepository: isSlashControl ? undefined : this.resolveAuthoredContextRepository(), + authoredContextLanguage: isSlashControl ? undefined : this._contextAuthoredContextLanguage, + authoredContextFilePath: isSlashControl ? undefined : this._contextAuthoredContextFilePath, ...(startupMemory ? { startupMemory } : {}), ...(memoryRecall ? { memoryRecall } : {}), }, { resolveAuthoredContext: (input) => { + if (isSlashControl) return Promise.resolve([]); if (!input.namespace) return Promise.resolve([]); return resolveRuntimeAuthoredContext(input.namespace, { language: input.authoredContextLanguage, @@ -492,6 +542,7 @@ export class TransportSessionRuntime implements SessionRuntime { } else if (memoryRecallResult.statusPayload) { this.emitMemoryContextStatusEvent(memoryRecallResult.statusPayload, clientMessageId); } + this._preferenceContextInjectionAttempt = null; if (!this._startupMemoryInjected && dispatchResult.payload?.startupMemory) { this._startupMemoryInjected = true; // Emit the "Historical context · injected" timeline card at the @@ -513,6 +564,10 @@ export class TransportSessionRuntime implements SessionRuntime { // Only handle if the provider didn't already fire onError callback. // Shared-context dispatch denial is surfaced here as a send failure // because the outer runtime contract is still send-oriented. + if (this._preferenceContextInjectionAttempt) { + this._lastInjectedPreferenceContextSignature = this._preferenceContextInjectionAttempt.previous; + this._preferenceContextInjectionAttempt = null; + } if (!this._sending || !this._activeTurn) return; this.setStatus('error'); this._sending = false; @@ -556,6 +611,45 @@ export class TransportSessionRuntime implements SessionRuntime { return true; } + private mergeMessagePreambles(entries: PendingTransportMessage[] | undefined, userMessage?: string): string | undefined { + if (!entries || entries.length === 0) return undefined; + const seen = new Set(); + const parts: string[] = []; + const isControlMessage = userMessage?.trim().startsWith('/') === true; + if (userMessage && shouldResetTransportPreferenceContextForSessionControl(userMessage)) { + // The compact control command must stay raw, and the next real turn + // should re-seed stable preferences because the provider may have + // discarded prior context during compaction. + this._lastInjectedPreferenceContextSignature = null; + } + for (const entry of entries) { + const preamble = entry.messagePreamble?.trim(); + if (!preamble) continue; + const filtered = this.filterOneShotPreferenceContext(preamble, isControlMessage); + if (!filtered || seen.has(filtered)) continue; + seen.add(filtered); + parts.push(filtered); + } + return parts.join('\n\n') || undefined; + } + + private filterOneShotPreferenceContext(preamble: string, isControlMessage: boolean): string | undefined { + const extracted = extractPreferenceContextBlocks(preamble); + if (extracted.blocks.length === 0) return preamble; + const signature = normalizePreferenceContextSignature(extracted.blocks); + if (isControlMessage) return extracted.withoutBlocks || undefined; + if (signature && signature === this._lastInjectedPreferenceContextSignature) { + return extracted.withoutBlocks || undefined; + } + if (signature) { + this._preferenceContextInjectionAttempt ??= { + previous: this._lastInjectedPreferenceContextSignature, + }; + this._lastInjectedPreferenceContextSignature = signature; + } + return preamble; + } + private async refreshContextBootstrap(options?: { phase?: 'initialize' | 'dispatch'; timeoutMs?: number; @@ -902,3 +996,49 @@ function toTransportMemoryRecallItem(item: MemorySearchResultItem): TransportMem ...(typeof item.updatedAt === 'number' ? { updatedAt: item.updatedAt } : {}), }; } + +function extractPreferenceContextBlocks(text: string): { blocks: string[]; withoutBlocks: string } { + const blocks: string[] = []; + const retained: string[] = []; + let cursor = 0; + while (cursor < text.length) { + const start = text.indexOf(PREFERENCE_CONTEXT_START, cursor); + if (start < 0) { + retained.push(text.slice(cursor)); + break; + } + const end = text.indexOf(PREFERENCE_CONTEXT_END, start + PREFERENCE_CONTEXT_START.length); + if (end < 0) { + retained.push(text.slice(cursor)); + break; + } + retained.push(text.slice(cursor, start)); + const blockEnd = end + PREFERENCE_CONTEXT_END.length; + blocks.push(text.slice(start, blockEnd).trim()); + cursor = blockEnd; + } + return { + blocks, + withoutBlocks: retained.join('').replace(/\n{3,}/g, '\n\n').trim(), + }; +} + +function normalizePreferenceContextSignature(blocks: readonly string[]): string { + return blocks.map((block) => block.replace(/\s+/g, ' ').trim()).filter(Boolean).join('\n'); +} + +function isTransportCompactionCompletion(message: AgentMessage): boolean { + const metadata = message.metadata; + const event = typeof metadata === 'object' && metadata !== null + ? (metadata as Record).event + : undefined; + return message.kind === 'system' + && message.role === 'system' + && ( + (typeof metadata === 'object' + && metadata !== null + && (metadata as Record)[SESSION_CONTROL_METADATA_COMMAND_FIELD] === 'compact') + || event === 'thread/compacted' + || event === 'session.history.compact' + ); +} diff --git a/src/bind/bind-flow.ts b/src/bind/bind-flow.ts index 77734f565..721162351 100644 --- a/src/bind/bind-flow.ts +++ b/src/bind/bind-flow.ts @@ -6,6 +6,7 @@ import { execSync } from 'child_process'; import logger from '../util/logger.js'; import { BACKEND } from '../agent/tmux.js'; import { restartWindowsDaemon } from '../util/windows-daemon.js'; +import { resolveDaemonLaunchTarget, renderSystemdExecStart, renderPlistProgramArguments } from '../util/launch-target.js'; const CREDS_DIR = join(homedir(), '.imcodes'); const CREDS_PATH = join(CREDS_DIR, 'server.json'); @@ -263,11 +264,13 @@ async function ensureTmux(): Promise { } async function installLaunchAgent(): Promise { - const nodeExec = process.execPath; - const script = process.argv[1]; const logPath = join(CREDS_DIR, 'daemon.log'); const launchAgentsDir = join(homedir(), 'Library', 'LaunchAgents'); + // Prefer the self-healing launcher when this install ships it. See + // `src/util/launch-target.ts` for rationale. + const target = resolveDaemonLaunchTarget(); + const plist = ` @@ -276,10 +279,7 @@ async function installLaunchAgent(): Promise { ${PLIST_LABEL} ProgramArguments - ${nodeExec} - ${script} - start - --foreground +${renderPlistProgramArguments(target)} EnvironmentVariables @@ -317,18 +317,20 @@ async function installLaunchAgent(): Promise { } async function installSystemdService(): Promise { - const nodeExec = process.execPath; - const script = process.argv[1]; const logPath = join(CREDS_DIR, 'daemon.log'); const serviceDir = join(homedir(), '.config', 'systemd', 'user'); const servicePath = join(serviceDir, 'imcodes.service'); + // Prefer the self-healing launcher when this install ships it. See + // `src/util/launch-target.ts` for rationale. + const target = resolveDaemonLaunchTarget(); + const unit = `[Unit] Description=IM.codes Daemon After=network.target [Service] -ExecStart=${nodeExec} ${script} start --foreground +ExecStart=${renderSystemdExecStart(target)} Restart=always RestartSec=5 KillMode=process diff --git a/src/context/live-context-ingestion.ts b/src/context/live-context-ingestion.ts index a402921e5..cadc43f69 100644 --- a/src/context/live-context-ingestion.ts +++ b/src/context/live-context-ingestion.ts @@ -6,6 +6,8 @@ import type { TransportContextBootstrap } from '../agent/runtime-context-bootstr import { MaterializationCoordinator, type MaterializationCoordinatorOptions } from './materialization-coordinator.js'; import { isMemoryNoiseTurn } from '../../shared/memory-noise-patterns.js'; import { createMemoryConfigResolver, rememberMemoryConfigProjectDir } from './memory-config-resolver.js'; +import { scheduleMarkdownMemoryIngest } from './md-ingest-worker.js'; +import { subscribeRuntimeMemoryCacheInvalidation } from './runtime-memory-cache-bus.js'; const BOOTSTRAP_CACHE_MS = 30_000; @@ -45,6 +47,7 @@ export class LiveContextIngestion { private readonly onError?: LiveContextIngestionOptions['onError']; private readonly sessionWork = new Map>(); private readonly bootstrapCache = new Map(); + private readonly unsubscribeCacheInvalidation: () => void; constructor(options: LiveContextIngestionOptions) { const memoryConfigResolver = options.memoryConfigResolver ?? (options.memoryConfig ? undefined : createMemoryConfigResolver({ @@ -61,6 +64,9 @@ export class LiveContextIngestion { this.sessionLookup = options.sessionLookup; this.resolveBootstrap = options.resolveBootstrap; this.onError = options.onError; + this.unsubscribeCacheInvalidation = subscribeRuntimeMemoryCacheInvalidation(() => { + this.bootstrapCache.clear(); + }); } handleTimelineEvent(event: TimelineEvent): Promise { @@ -75,6 +81,12 @@ export class LiveContextIngestion { return next; } + dispose(): void { + this.unsubscribeCacheInvalidation(); + this.bootstrapCache.clear(); + this.sessionWork.clear(); + } + async flushDueTargets(now = Date.now()): Promise { if (!isLiveContextMaterializationAdmissionOpen()) return; for (const job of this.coordinator.scheduleDueTargets(now)) { @@ -129,6 +141,16 @@ export class LiveContextIngestion { return; } + if (event.type === 'tool.result') { + const filteredReason = toolResultEvidenceFilteredReason(event); + if (filteredReason) { + this.coordinator.recordFilteredSkillReviewToolIteration(filteredReason); + } else { + this.coordinator.recordSkillReviewToolIteration(target); + } + return; + } + const mapped = mapTimelineEvent(event); if (!mapped) return; const result = this.coordinator.ingestEvent({ @@ -152,6 +174,7 @@ export class LiveContextIngestion { } const value = await this.resolveBootstrap(session); rememberMemoryConfigProjectDir(value.namespace, session.projectDir); + scheduleMarkdownMemoryIngest({ projectDir: session.projectDir, namespace: value.namespace }); this.bootstrapCache.set(session.name, { recordUpdatedAt: session.updatedAt, expiresAt: Date.now() + BOOTSTRAP_CACHE_MS, @@ -171,6 +194,16 @@ export class LiveContextIngestion { } } + +function toolResultEvidenceFilteredReason(event: TimelineEvent): 'hidden' | 'error' | null { + if (event.hidden === true || event.payload.hidden === true) return 'hidden'; + if (event.payload.error !== undefined && event.payload.error !== null && event.payload.error !== false) return 'error'; + const exitCode = event.payload.exit_code ?? event.payload.exitCode ?? event.payload.code; + if (typeof exitCode === 'number' && exitCode !== 0) return 'error'; + if (event.payload.success === false) return 'error'; + return null; +} + function toSessionTarget(sessionName: string, bootstrap: TransportContextBootstrap): ContextTargetRef { return { namespace: bootstrap.namespace, diff --git a/src/context/managed-skill-path.ts b/src/context/managed-skill-path.ts new file mode 100644 index 000000000..6aae70117 --- /dev/null +++ b/src/context/managed-skill-path.ts @@ -0,0 +1,101 @@ +import { lstatSync, realpathSync, statSync } from 'node:fs'; +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; +import { homedir } from 'node:os'; +import { getProjectSkillEscapeHatchDir, getUserSkillRoot } from '../../shared/skill-store.js'; + +export type ManagedSkillRootKind = 'user' | 'project'; +export type ManagedSkillPathRejectReason = + | 'nul_byte' + | 'outside_managed_root' + | 'managed_root_missing' + | 'symlink_component' + | 'not_file' + | 'oversize'; + +export class ManagedSkillPathError extends Error { + constructor(readonly reason: ManagedSkillPathRejectReason, message: string = reason) { + super(message); + this.name = 'ManagedSkillPathError'; + } +} + +export interface ManagedSkillPathAssertion { + rootKind: ManagedSkillRootKind; + path: string; + realPath: string; + root: string; + realRoot: string; + size: number; +} + +function isUnderRoot(path: string, root: string): boolean { + const rel = relative(root, path); + return rel === '' || (!!rel && !rel.startsWith('..') && !isAbsolute(rel)); +} + +function assertNoSymlinkDirectoryComponent(root: string, target: string): void { + const rel = relative(root, dirname(target)); + if (!rel || rel.startsWith('..') || isAbsolute(rel)) return; + let current = root; + for (const part of rel.split(/[\\/]+/).filter(Boolean)) { + current = `${current}${sep}${part}`; + const stat = lstatSync(current); + if (stat.isSymbolicLink()) { + throw new ManagedSkillPathError('symlink_component', `skill path contains symlink directory: ${current}`); + } + } +} + +export function assertManagedSkillPathSync(input: { + path: string; + projectDir?: string; + homeDir?: string; + maxBytes?: number; +}): ManagedSkillPathAssertion { + if (input.path.includes('\0')) throw new ManagedSkillPathError('nul_byte'); + const absolute = resolve(input.path); + const candidates: Array<{ kind: ManagedSkillRootKind; root: string }> = [ + { kind: 'user', root: resolve(getUserSkillRoot(input.homeDir ?? homedir())) }, + ]; + if (input.projectDir) { + candidates.push({ kind: 'project', root: resolve(getProjectSkillEscapeHatchDir(input.projectDir)) }); + } + + for (const candidate of candidates) { + if (!isUnderRoot(absolute, candidate.root)) continue; + let realRoot: string; + try { + realRoot = realpathSync(candidate.root); + } catch { + throw new ManagedSkillPathError('managed_root_missing'); + } + assertNoSymlinkDirectoryComponent(candidate.root, absolute); + let lstat; + try { + lstat = lstatSync(absolute); + } catch { + throw new ManagedSkillPathError('not_file'); + } + if (!lstat.isFile() || lstat.isSymbolicLink()) throw new ManagedSkillPathError('not_file'); + if (input.maxBytes !== undefined && lstat.size > input.maxBytes) throw new ManagedSkillPathError('oversize'); + let realPath: string; + try { + realPath = realpathSync(absolute); + } catch { + throw new ManagedSkillPathError('not_file'); + } + if (!isUnderRoot(realPath, realRoot)) throw new ManagedSkillPathError('outside_managed_root'); + const stat = statSync(realPath); + if (!stat.isFile()) throw new ManagedSkillPathError('not_file'); + if (input.maxBytes !== undefined && stat.size > input.maxBytes) throw new ManagedSkillPathError('oversize'); + return { + rootKind: candidate.kind, + path: absolute, + realPath, + root: candidate.root, + realRoot, + size: stat.size, + }; + } + throw new ManagedSkillPathError('outside_managed_root'); +} diff --git a/src/context/materialization-coordinator.ts b/src/context/materialization-coordinator.ts index f2ba4d036..f11f1847e 100644 --- a/src/context/materialization-coordinator.ts +++ b/src/context/materialization-coordinator.ts @@ -27,13 +27,14 @@ import { listProcessedProjections, pruneArchiveIfDue, queryProcessedProjections, + recordCompressionRun, recordContextEvent, countStagedTokens, setReplicationState, updateContextJob, writeProcessedProjection, } from '../store/context-store.js'; -import { serializeContextNamespace } from './context-keys.js'; +import { serializeContextNamespace, serializeContextTarget } from './context-keys.js'; import { countTokens } from './tokenizer.js'; import { loadMemoryConfig, type MemoryConfig } from './memory-config.js'; import { createMemoryConfigResolver, resolveMemoryConfigForNamespace, type MemoryConfigResolver } from './memory-config-resolver.js'; @@ -41,6 +42,23 @@ import { computeFingerprint } from '../../shared/memory-fingerprint.js'; import { warnOncePerHour } from '../util/rate-limited-warn.js'; import { incrementCounter } from '../util/metrics.js'; import { redactSummaryPreservingPinned } from '../util/redact-with-pinned-region.js'; +import { timelineEmitter } from '../daemon/timeline-emitter.js'; +import { + decideSkillReviewSchedule, + type SkillReviewSchedulerPolicy, + type SkillReviewState, +} from '../../shared/skill-review-scheduler.js'; +import type { SkillReviewTrigger } from '../../shared/skill-review-triggers.js'; +import { + MEMORY_FEATURE_FLAGS_BY_NAME, + memoryFeatureFlagEnvKey, + resolveMemoryFeatureFlagValue, +} from '../../shared/feature-flags.js'; +import { + getMemoryFeatureConfigStoreDiagnostics, + getPersistedMemoryFeatureFlagValues, + getRuntimeMemoryFeatureFlagValues, +} from '../store/memory-feature-config-store.js'; export interface MaterializationThresholds { autoTriggerTokens: number; @@ -66,6 +84,15 @@ export interface MaterializationCoordinatorOptions { compressor?: (input: import('./summary-compressor.js').CompressionInput) => Promise; /** Override archive writes for failure-injection tests. */ archiveEventsForMaterialization?: typeof archiveEventsForMaterialization; + /** + * Optional post-response skill review scheduler. The coordinator invokes it + * only after SDK-backed materialization has completed, so auto-creation stays + * on the existing isolated background path and never enters the send ack or + * provider-delivery foreground paths. + */ + skillReviewScheduler?: MaterializationSkillReviewScheduler; + /** Gate self-learning durable extraction/classification; defaults to the shared feature flag (default off). */ + selfLearningEnabled?: boolean | (() => boolean); } export interface MaterializationResult { @@ -76,6 +103,31 @@ export interface MaterializationResult { filteredOut?: boolean; } +export interface MaterializationSkillReviewJob { + idempotencyKey: string; + scopeKey: string; + responseId: string; + trigger: SkillReviewTrigger; + target: ContextTargetRef; + projectionId: string; + sourceEventIds: readonly string[]; + nextAttemptAt: number; + maxAttempts: number; + createdAt: number; +} + +interface SkillReviewTriggerEvidence { + toolIterationCount: number; +} + +export interface MaterializationSkillReviewScheduler { + featureEnabled: boolean | (() => boolean); + getState: (scopeKey: string) => SkillReviewState; + enqueue: (job: MaterializationSkillReviewJob) => void | Promise; + policy?: Partial; + isShuttingDown?: () => boolean; +} + const DEFAULT_THRESHOLDS: MaterializationThresholds = { autoTriggerTokens: 3000, minEventCount: 5, @@ -95,6 +147,7 @@ const DEFAULT_THRESHOLDS: MaterializationThresholds = { * producing the nested "--- Updated ---" chains observed in the field). */ const MAX_SDK_RETRY_ATTEMPTS = 3; +const MAX_SKILL_REVIEW_EVIDENCE_TARGETS = 256; export class MaterializationCoordinator { readonly thresholds: MaterializationThresholds; @@ -104,10 +157,15 @@ export class MaterializationCoordinator { private readonly thresholdOverrides: Partial; private readonly _compressor: MaterializationCoordinatorOptions['compressor']; private readonly _archiveEventsForMaterialization: typeof archiveEventsForMaterialization; + private readonly _skillReviewScheduler?: MaterializationSkillReviewScheduler; + private readonly _selfLearningEnabled?: boolean | (() => boolean); + private readonly skillReviewEvidenceByTarget = new Map(); constructor(options?: MaterializationCoordinatorOptions) { this._compressor = options?.compressor; this._archiveEventsForMaterialization = options?.archiveEventsForMaterialization ?? archiveEventsForMaterialization; + this._skillReviewScheduler = options?.skillReviewScheduler; + this._selfLearningEnabled = options?.selfLearningEnabled; this.resolveMemoryConfig = options?.memoryConfigResolver ?? createMemoryConfigResolver({ fixedConfig: options?.memoryConfig, fallbackCwd: options?.memoryConfigCwd, @@ -151,6 +209,27 @@ export class MaterializationCoordinator { return listDirtyTargets(namespace); } + recordSkillReviewToolIteration(target: ContextTargetRef, count = 1): void { + const safeCount = Math.max(0, Math.floor(count)); + if (safeCount <= 0) return; + const targetKey = serializeContextTarget(target); + const current = this.skillReviewEvidenceByTarget.get(targetKey)?.toolIterationCount ?? 0; + if (!this.skillReviewEvidenceByTarget.has(targetKey) && this.skillReviewEvidenceByTarget.size >= MAX_SKILL_REVIEW_EVIDENCE_TARGETS) { + const oldestKey = this.skillReviewEvidenceByTarget.keys().next().value as string | undefined; + if (oldestKey) { + this.skillReviewEvidenceByTarget.delete(oldestKey); + incrementCounter('mem.skill.evidence_evicted', { reason: 'lru_limit' }); + } + } + this.skillReviewEvidenceByTarget.set(targetKey, { + toolIterationCount: Math.min(1_000_000, current + safeCount), + }); + } + + recordFilteredSkillReviewToolIteration(reason: 'hidden' | 'error'): void { + incrementCounter('mem.skill.evidence_filtered', { reason }); + } + async materializeTarget(target: ContextTargetRef, trigger: ContextJobTrigger, now = Date.now()): Promise { const memoryConfig = this.configForTarget(target); const jobType = target.kind === 'project' ? 'materialize_project' : 'materialize_session'; @@ -215,6 +294,35 @@ export class MaterializationCoordinator { error: `compression_admission_closed: ${error.reason} — kept raw events for retry`, }); incrementCounter('mem.materialization.compression_admission_closed', { reason: error.reason }); + // Round-2 audit (0699ea64-3e6 finding A2/Commit B): admission_closed + // path used to early-return WITHOUT recording in + // context_compression_runs, leaving the schema's CHECK enum value + // 'admission_closed' as dead schema. Operators querying "how often + // does the upgrade-pending freeze block compactions" couldn't + // answer it. Now we record an SQLite row so the metric is queryable + // — but we explicitly do NOT emit a timeline event (would be chat + // noise: "nothing happened, but here's an event saying so"). + try { + recordCompressionRun({ + backend: 'none', + model: '', + usedBackup: false, + fromSdk: false, + namespaceKey: serializeContextNamespace(target.namespace), + targetKind: target.kind, + sessionName: target.sessionName ?? null, + trigger, + mode: 'auto', + eventCount: events.length, + inputTokens: 0, + outputTokens: 0, + targetTokens: 0, + durationMs: 0, + outcome: 'admission_closed', + errorCode: error.reason, + errorMessage: null, + }); + } catch { /* never escape */ } return { replicationQueued: false, }; @@ -222,15 +330,95 @@ export class MaterializationCoordinator { // Compressor itself threw (not just a provider failure the compressor // swallowed into a local-fallback result). Treat as fromSdk: false and // let the abandonment/retry logic below decide what to do. + const errMsg = error instanceof Error ? error.message : String(error); compression = { summary: '', model: 'local-fallback', backend: 'none', usedBackup: false, fromSdk: false, + inputTokens: 0, + outputTokens: 0, + targetTokens: 0, + durationMs: 0, + errorMessage: errMsg.slice(0, 500), }; } + // Persist a row in context_compression_runs so operators can query + // model/token costs later. Best-effort — recording failures must never + // break compression; recordCompressionRun swallows internally. + // + // Round-2 audit (0699ea64-3e6 finding A2/Commit B): when events.length===0 + // the compressor early-returns a fromSdk:false CompressionResult that + // used to be recorded as outcome='fallback' (because errorCode is also + // unset). That polluted SUM(outcome='fallback') analytics — those rows + // weren't real fallbacks, just no-ops. Skip recording + emit entirely + // for the no-op case; the path produces no projection and no LLM call, + // so there is nothing useful to log for cost analysis. + const outcome: 'success' | 'fallback' | 'error' = compression.fromSdk + ? (compression.usedBackup ? 'fallback' : 'success') + : (compression.errorCode || compression.errorMessage ? 'error' : 'fallback'); + if (events.length > 0) { + try { + recordCompressionRun({ + backend: compression.backend, + model: compression.model, + usedBackup: compression.usedBackup, + fromSdk: compression.fromSdk, + namespaceKey: serializeContextNamespace(target.namespace), + targetKind: target.kind, + sessionName: target.sessionName ?? null, + trigger, + mode: 'auto', + eventCount: events.length, + inputTokens: compression.inputTokens ?? 0, + outputTokens: compression.outputTokens ?? 0, + targetTokens: compression.targetTokens ?? 0, + durationMs: compression.durationMs ?? 0, + outcome, + errorCode: compression.errorCode ?? null, + errorMessage: compression.errorMessage ?? null, + }); + } catch { + // Telemetry must never escape — recordCompressionRun already swallows + // its own errors, but defense-in-depth. + } + } + + // Emit a `memory.compression` timeline event so the user can see in chat + // history that a compression just ran (web UI renders it COLLAPSED by + // default, click to expand). Persisted to JSONL via the timeline store. + // Only emit when we have a session-bound target — namespace-only + // (project-level) compactions don't have a chat thread to attach to. + // Same events.length>0 guard as the SQLite recording above. + if (target.sessionName && events.length > 0) { + try { + timelineEmitter.emit( + target.sessionName, + 'memory.compression', + { + backend: compression.backend, + model: compression.model, + usedBackup: compression.usedBackup, + fromSdk: compression.fromSdk, + trigger, + mode: 'auto', + eventCount: events.length, + inputTokens: compression.inputTokens ?? 0, + outputTokens: compression.outputTokens ?? 0, + targetTokens: compression.targetTokens ?? 0, + durationMs: compression.durationMs ?? 0, + outcome, + ...(compression.errorCode ? { errorCode: compression.errorCode } : {}), + }, + { source: 'daemon', confidence: 'high' }, + ); + } catch { + // Timeline emit must never escape compression. + } + } + // Only SDK-produced summaries are ever filtered by `isMemoryNoiseSummary`. // A fromSdk: false result means "no real summary was produced" — the // fallback branch below owns that case; we don't treat it as noise. @@ -273,7 +461,15 @@ export class MaterializationCoordinator { const sdkFailed = !compression.fromSdk; if (sdkFailed) { - const retryBudgetExhausted = priorFailures >= MAX_SDK_RETRY_ATTEMPTS; + // Round-2 audit (0699ea64-3e6 finding android#1/Commit B): + // `priorFailures >= MAX_SDK_RETRY_ATTEMPTS` was off-by-one — the comparison + // didn't include the CURRENT failure, so the constant `3` actually meant + // "permit 3 prior failures + give up on the 4th". Operators reading + // `attempt 3/3` reasonably expected "next failure ends the batch", but + // the daemon would still try once more. Counting the current failure + // (priorFailures + 1) makes the constant match its name. + const totalFailuresIncludingCurrent = priorFailures + 1; + const retryBudgetExhausted = totalFailuresIncludingCurrent >= MAX_SDK_RETRY_ATTEMPTS; // Any legacy tentative rows from earlier versions of this code get // scrubbed here — the new design never writes tentatives, so the only // tentative rows that can exist are pre-migration leftovers. @@ -288,7 +484,7 @@ export class MaterializationCoordinator { incrementCounter('mem.materialization.retry_exhausted_archived', { source: 'materializeTarget' }); deleteStagedEventsByIds(sourceEventIds); updateContextJob(job.id, 'completed', { now, - error: `SDK compression abandoned after ${priorFailures} consecutive failures — events discarded, no summary written`, + error: `SDK compression abandoned after ${totalFailuresIncludingCurrent} consecutive failures — events discarded, no summary written`, }); clearDirtyTarget(target); return { @@ -301,7 +497,7 @@ export class MaterializationCoordinator { // Retry path: keep staged events, leave dirty target in place, mark // job materialization_failed so the next trigger retries SDK. updateContextJob(job.id, 'materialization_failed', { now, - error: `SDK compression unavailable (attempt ${priorFailures + 1}/${MAX_SDK_RETRY_ATTEMPTS}) — kept raw events for retry`, + error: `SDK compression unavailable (attempt ${totalFailuresIncludingCurrent}/${MAX_SDK_RETRY_ATTEMPTS}) — kept raw events for retry`, }); return { replicationQueued: false, @@ -332,6 +528,7 @@ export class MaterializationCoordinator { const summaryProjection = writeProcessedProjection({ namespace: target.namespace, class: 'recent_summary', + origin: 'chat_compacted', sourceEventIds, summary: compression.summary, content: { @@ -353,17 +550,19 @@ export class MaterializationCoordinator { updatedAt: now, }); let durableProjection: ProcessedContextProjection | undefined; - try { - durableProjection = buildDurableProjection( - target.namespace, - events, - compression.summary, - sourceEventIds, - now, - ); - } catch (error) { - incrementCounter('mem.materialization.durable_projection_failed', { source: 'materializeTarget' }); - warnOncePerHour('mem.materialization.durable_projection_failed', { error: error instanceof Error ? error.message : String(error) }); + if (this.isSelfLearningEnabled()) { + try { + durableProjection = buildDurableProjection( + target.namespace, + events, + compression.summary, + sourceEventIds, + now, + ); + } catch (error) { + incrementCounter('mem.materialization.durable_projection_failed', { source: 'materializeTarget' }); + warnOncePerHour('mem.materialization.durable_projection_failed', { error: error instanceof Error ? error.message : String(error) }); + } } const replicationState = getReplicationState(target.namespace); @@ -380,6 +579,12 @@ export class MaterializationCoordinator { deleteStagedEventsByIds(sourceEventIds); updateContextJob(job.id, 'completed', { now }); clearDirtyTarget(target); + this.schedulePostResponseSkillReview({ + target, + projectionId: summaryProjection.id, + sourceEventIds, + now, + }); return { summaryProjection, @@ -436,6 +641,76 @@ export class MaterializationCoordinator { ); } + private schedulePostResponseSkillReview(input: { + target: ContextTargetRef; + projectionId: string; + sourceEventIds: readonly string[]; + now: number; + }): void { + const scheduler = this._skillReviewScheduler; + if (!scheduler) return; + try { + const featureEnabled = typeof scheduler.featureEnabled === 'function' + ? scheduler.featureEnabled() + : scheduler.featureEnabled; + const scopeKey = serializeContextNamespace(input.target.namespace); + const targetKey = serializeContextTarget(input.target); + const triggerEvidence = this.skillReviewEvidenceByTarget.get(targetKey) ?? { toolIterationCount: 0 }; + const responseId = [...input.sourceEventIds].reverse().find((id) => id.trim().length > 0) + ?? input.projectionId; + const decision = decideSkillReviewSchedule({ + featureEnabled, + delivered: true, + phase: 'post_response_background', + trigger: 'tool_iteration_count', + scopeKey, + responseId, + now: input.now, + state: scheduler.getState(scopeKey), + policy: scheduler.policy, + shuttingDown: scheduler.isShuttingDown?.() ?? false, + triggerEvidence, + }); + this.skillReviewEvidenceByTarget.delete(targetKey); + if (decision.action === 'skip') { + if (decision.reason === 'coalesced') { + incrementCounter('mem.skill.review_deduped', { source: 'materialization' }); + } else if (decision.reason === 'below_trigger_threshold' + || decision.reason === 'invalid_trigger' + || decision.reason === 'not_delivered' + || decision.reason === 'not_background') { + incrementCounter('mem.skill.review_not_eligible', { reason: decision.reason }); + } else if (decision.reason !== 'disabled' && decision.reason !== 'shutdown') { + incrementCounter('mem.skill.review_throttled', { reason: decision.reason }); + } + return; + } + const job: MaterializationSkillReviewJob = { + idempotencyKey: decision.idempotencyKey, + scopeKey, + responseId, + trigger: 'tool_iteration_count', + target: input.target, + projectionId: input.projectionId, + sourceEventIds: [...input.sourceEventIds], + nextAttemptAt: decision.nextAttemptAt, + maxAttempts: decision.maxAttempts, + createdAt: input.now, + }; + void Promise.resolve(scheduler.enqueue(job)).catch((error) => { + incrementCounter('mem.skill.review_failed', { source: 'materialization_enqueue' }); + warnOncePerHour('mem.skill.review_failed', { + error: error instanceof Error ? error.message : String(error), + }); + }); + } catch (error) { + incrementCounter('mem.skill.review_failed', { source: 'materialization_schedule' }); + warnOncePerHour('mem.skill.review_failed', { + error: error instanceof Error ? error.message : String(error), + }); + } + } + private selectTrigger(dirtyTarget: ContextDirtyTarget, now: number): ContextJobTrigger | undefined { const thresholds = this.thresholdsForTarget(dirtyTarget.target); if (this.isRateLimited(dirtyTarget.target, now)) return undefined; @@ -492,6 +767,19 @@ export class MaterializationCoordinator { return this.buildThresholds(this.configForTarget(target)); } + private isSelfLearningEnabled(): boolean { + if (typeof this._selfLearningEnabled === 'function') return this._selfLearningEnabled(); + if (typeof this._selfLearningEnabled === 'boolean') return this._selfLearningEnabled; + const flag = MEMORY_FEATURE_FLAGS_BY_NAME.selfLearning; + const raw = process.env[memoryFeatureFlagEnvKey(flag)]; + return resolveMemoryFeatureFlagValue(flag, { + runtimeConfigOverride: getRuntimeMemoryFeatureFlagValues(), + persistedConfig: getPersistedMemoryFeatureFlagValues(), + environmentStartupDefault: raw == null ? undefined : { [flag]: raw === 'true' || raw === '1' }, + readFailed: !!getMemoryFeatureConfigStoreDiagnostics().lastLoadIssue, + }); + } + private buildThresholds(memoryConfig: MemoryConfig): MaterializationThresholds { const configMinInterval = this.modelConfig.materializationMinIntervalMs; const thresholdOverrides = this.thresholdOverrides; @@ -566,6 +854,7 @@ export async function materializeMasterSummary(sessionName: string, namespace?: id: `master:${computeFingerprint(`${namespaceKey}:${sessionName}`)}`, namespace: resolvedNamespace, class: 'master_summary', + origin: 'chat_compacted', sourceEventIds, summary, content: { @@ -638,6 +927,7 @@ function buildDurableProjection( return writeProcessedProjection({ namespace, class: 'durable_memory_candidate', + origin: 'agent_learned', sourceEventIds, summary: buildDurableSummary(signals), content: { diff --git a/src/context/md-ingest-worker.ts b/src/context/md-ingest-worker.ts new file mode 100644 index 000000000..84dec793e --- /dev/null +++ b/src/context/md-ingest-worker.ts @@ -0,0 +1,151 @@ +import { lstat, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import type { ContextNamespace } from '../../shared/context-types.js'; +import type { MemoryScope } from '../../shared/memory-scope.js'; +import { + MD_INGEST_FEATURE_FLAG, + MD_INGEST_ORIGIN, + MD_INGEST_SUPPORTED_PATHS, + parseMdIngestDocument, +} from '../../shared/md-ingest.js'; +import { + MEMORY_FEATURE_FLAGS, + memoryFeatureFlagEnvKey, + resolveEffectiveMemoryFeatureFlagValue, + type MemoryFeatureFlag, + type MemoryFeatureFlagValues, +} from '../../shared/feature-flags.js'; +import { + getMemoryFeatureConfigStoreDiagnostics, + getPersistedMemoryFeatureFlagValues, + getRuntimeMemoryFeatureFlagValues, +} from '../store/memory-feature-config-store.js'; +import { writeProcessedProjection } from '../store/context-store.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { incrementCounter } from '../util/metrics.js'; +import { serializeContextNamespace } from './context-keys.js'; + +const scheduledKeys = new Set(); +const MD_INGEST_ALLOWED_SCOPES: ReadonlySet = new Set(['personal', 'project_shared']); + +function isMdIngestEnabled(): boolean { + const environmentStartupDefault = Object.fromEntries( + MEMORY_FEATURE_FLAGS.flatMap((flag): Array<[MemoryFeatureFlag, boolean]> => { + const raw = process.env[memoryFeatureFlagEnvKey(flag)]; + return raw == null ? [] : [[flag, raw === 'true' || raw === '1']]; + }), + ) as MemoryFeatureFlagValues; + return resolveEffectiveMemoryFeatureFlagValue(MD_INGEST_FEATURE_FLAG, { + runtimeConfigOverride: getRuntimeMemoryFeatureFlagValues(), + persistedConfig: getPersistedMemoryFeatureFlagValues(), + environmentStartupDefault, + readFailed: !!getMemoryFeatureConfigStoreDiagnostics().lastLoadIssue, + }); +} + +function validateMarkdownIngestNamespace(namespace: ContextNamespace): ContextNamespace | null { + if (MD_INGEST_ALLOWED_SCOPES.has(namespace.scope)) return namespace; + incrementCounter('mem.ingest.scope_dropped', { from: namespace.scope, reason: 'unsupported_scope' }); + warnOncePerHour('md_ingest.scope_dropped', { + scope: namespace.scope, + reason: 'unsupported_scope', + projectId: namespace.projectId, + }); + return null; +} + +export async function runMarkdownMemoryIngest(input: { + projectDir: string | undefined; + namespace: ContextNamespace; + actorUserId?: string; + featureEnabled?: boolean; + now?: number; +}): Promise<{ filesChecked: number; observationsWritten: number; droppedReason?: 'unsupported_scope' }> { + const projectDir = input.projectDir?.trim(); + if (!projectDir) return { filesChecked: 0, observationsWritten: 0 }; + const featureEnabled = input.featureEnabled ?? isMdIngestEnabled(); + if (!featureEnabled) return { filesChecked: 0, observationsWritten: 0 }; + + const namespace = validateMarkdownIngestNamespace(input.namespace); + if (!namespace) return { filesChecked: 0, observationsWritten: 0, droppedReason: 'unsupported_scope' }; + const scopeKey = serializeContextNamespace(namespace); + let filesChecked = 0; + let observationsWritten = 0; + + for (const relativePath of MD_INGEST_SUPPORTED_PATHS) { + const fullPath = join(projectDir, relativePath); + try { + const stat = await lstat(fullPath); + filesChecked += 1; + const content = stat.isSymbolicLink() ? new Uint8Array() : await readFile(fullPath); + const result = parseMdIngestDocument({ + path: relativePath, + content, + scopeKey, + featureEnabled, + isSymlink: stat.isSymbolicLink(), + }); + for (const section of result.sections) { + writeProcessedProjection({ + id: `md-ingest:${scopeKey}:${relativePath}:${section.fingerprint}`, + namespace, + class: 'durable_memory_candidate', + sourceEventIds: [`md-ingest:${relativePath}:${section.fingerprint}`], + summary: section.text, + content: { + text: section.text, + title: section.heading, + path: relativePath, + observationClass: section.class, + origin: MD_INGEST_ORIGIN, + fingerprint: section.fingerprint, + provenanceFingerprint: `${relativePath}:${section.fingerprint}`, + ...(input.actorUserId?.trim() ? { createdByUserId: input.actorUserId.trim(), updatedByUserId: input.actorUserId.trim() } : {}), + }, + origin: MD_INGEST_ORIGIN, + createdAt: input.now, + updatedAt: input.now, + }); + observationsWritten += 1; + } + } catch (error) { + const code = typeof error === 'object' && error && 'code' in error ? String((error as { code?: unknown }).code) : ''; + if (code === 'ENOENT') continue; + incrementCounter('mem.ingest.skipped_unsafe', { reason: 'read_failed' }); + warnOncePerHour('md_ingest.read_failed', { + path: relativePath, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return { filesChecked, observationsWritten }; +} + +export function scheduleMarkdownMemoryIngest(input: { + projectDir: string | undefined; + namespace: ContextNamespace; +}): void { + const projectDir = input.projectDir?.trim(); + if (!projectDir || !isMdIngestEnabled()) return; + const key = `${projectDir}\u0000${serializeContextNamespace(input.namespace)}`; + if (scheduledKeys.has(key)) return; + scheduledKeys.add(key); + const timer = setTimeout(() => { + void runMarkdownMemoryIngest(input) + .catch((error) => { + incrementCounter('mem.ingest.skipped_unsafe', { reason: 'worker_failed' }); + warnOncePerHour('md_ingest.worker_failed', { + error: error instanceof Error ? error.message : String(error), + }); + }) + .finally(() => { + scheduledKeys.delete(key); + }); + }, 0); + timer.unref?.(); +} + +export function resetMarkdownMemoryIngestForTests(): void { + scheduledKeys.clear(); +} diff --git a/src/context/memory-search.ts b/src/context/memory-search.ts index 957c3dd3b..e30f1c7f5 100644 --- a/src/context/memory-search.ts +++ b/src/context/memory-search.ts @@ -3,6 +3,7 @@ * Used by CLI (`imcodes memory`), WS command (`memory.search`), and web UI. */ import type { + ContextScope, ContextNamespace, LocalContextEvent, ProcessedContextClass, @@ -10,6 +11,7 @@ import type { ProcessedContextProjectionStatus, ContextMemoryStatsView, } from '../../shared/context-types.js'; +import { projectionSemanticContent } from '../../shared/memory-content-hash.js'; import { computeRelevanceScore, type ProjectionClass } from '../../shared/memory-scoring.js'; import { normalizeSummaryForFingerprint } from '../../shared/memory-fingerprint.js'; import { getContextModelConfig } from './context-model-config.js'; @@ -19,6 +21,7 @@ import { listContextEvents, listDirtyTargets, queryProcessedProjections, + LEGACY_DAEMON_LOCAL_USER_ID, getProjectionEmbeddings, saveProjectionEmbedding, } from '../store/context-store.js'; @@ -30,10 +33,16 @@ export interface MemorySearchQuery { query?: string; /** Filter by effective namespace. When provided, namespace fields are matched exactly. */ namespace?: ContextNamespace; + /** Filter by scope without requiring an exact namespace match. */ + scope?: ContextScope; /** Optional enterprise context used for ranking when search scope is broader than one namespace. */ currentEnterpriseId?: string; /** Filter by canonical repository ID (matches namespace.projectId). */ repo?: string; + /** Optional owner/user filter used by authenticated management reads. */ + userId?: string; + /** Include legacy local personal rows that have no durable owner id. */ + includeLegacyPersonalOwner?: boolean; /** Filter by projection class. */ projectionClass?: ProcessedContextClass; /** Include raw unprocessed staged events. */ @@ -87,6 +96,11 @@ export interface MemorySearchResult { stats: ContextMemoryStatsView; } +export interface AuthorizedMemorySearchQuery extends Omit { + /** Exact namespaces the management caller is authorized to search. */ + authorizedNamespaces: readonly ContextNamespace[]; +} + // ── Search implementation ──────────────────────────────────────────────────── /** @@ -336,6 +350,52 @@ export function searchLocalMemory(query: MemorySearchQuery): MemorySearchResult }; } +export function searchLocalMemoryAuthorized(query: AuthorizedMemorySearchQuery): MemorySearchResult { + const allItems: MemorySearchResultItem[] = []; + const seenProjectionIds = new Set(); + const requestedWindow = Math.max((query.limit ?? 50) + (query.offset ?? 0), query.limit ?? 50, 50); + + for (const namespace of query.authorizedNamespaces) { + const projections = queryProcessedProjections({ + scope: namespace.scope, + enterpriseId: namespace.enterpriseId, + workspaceId: namespace.workspaceId, + userId: namespace.userId, + projectId: namespace.projectId, + includeLegacyPersonalOwner: query.includeLegacyPersonalOwner, + projectionClass: query.projectionClass, + query: query.query, + includeArchived: query.includeArchived, + limit: requestedWindow, + }); + for (const projection of projections) { + if (seenProjectionIds.has(projection.id)) continue; + seenProjectionIds.add(projection.id); + const item = projectionToItem(projection); + if (matchesQuery(item, query)) { + allItems.push(item); + } + } + } + + allItems.sort((a, b) => (b.updatedAt ?? b.createdAt) - (a.updatedAt ?? a.createdAt)); + const stats = computeStats(allItems); + const offset = query.offset ?? 0; + const limit = query.limit ?? 50; + const paginated = allItems.slice(offset, offset + limit); + + return { + items: paginated, + stats: { + ...stats, + matchedRecords: allItems.length, + stagedEventCount: 0, + dirtyTargetCount: listDirtyTargets().length, + pendingJobCount: 0, + }, + }; +} + // ── Output formatting ──────────────────────────────────────────────────────── export function formatSearchResults(result: MemorySearchResult, format: MemorySearchFormat): string { @@ -395,11 +455,12 @@ function formatAge(timestamp: number): string { function collectProcessedProjections(query: MemorySearchQuery): ProcessedContextProjection[] { return queryProcessedProjections({ - scope: query.namespace?.scope, + scope: query.namespace?.scope ?? query.scope, enterpriseId: query.namespace?.enterpriseId, workspaceId: query.namespace?.workspaceId, - userId: query.namespace?.userId, + userId: query.namespace?.userId ?? query.userId, projectId: query.namespace?.projectId ?? query.repo, + includeLegacyPersonalOwner: query.includeLegacyPersonalOwner, projectionClass: query.projectionClass, query: query.query, includeArchived: query.includeArchived, @@ -421,26 +482,29 @@ function collectRawEvents(query: MemorySearchQuery): MemorySearchResultItem[] { } function projectionToItem(projection: ProcessedContextProjection): MemorySearchResultItem { - const content = projection.content; + const content = projectionSemanticContent(projection.content); + const contentRecord = content && typeof content === 'object' && !Array.isArray(content) + ? content as Record + : undefined; return { type: 'processed', id: projection.id, - projectId: projection.namespace.projectId, + projectId: projection.namespace.projectId ?? '', scope: projection.namespace.scope, enterpriseId: projection.namespace.enterpriseId, workspaceId: projection.namespace.workspaceId, userId: projection.namespace.userId, projectionClass: projection.class, summary: projection.summary, - content: typeof content === 'object' ? JSON.stringify(content) : undefined, + content: contentRecord ? JSON.stringify(contentRecord) : undefined, createdAt: projection.createdAt, updatedAt: projection.updatedAt, hitCount: projection.hitCount, lastUsedAt: projection.lastUsedAt, status: projection.status, - sourceEventCount: typeof content?.eventCount === 'number' ? content.eventCount : undefined, + sourceEventCount: typeof contentRecord?.eventCount === 'number' ? contentRecord.eventCount : undefined, sourceEventIds: projection.sourceEventIds, - processingModel: typeof content?.primaryContextModel === 'string' ? content.primaryContextModel : undefined, + processingModel: typeof contentRecord?.primaryContextModel === 'string' ? contentRecord.primaryContextModel : undefined, }; } @@ -448,7 +512,7 @@ function eventToItem(event: LocalContextEvent): MemorySearchResultItem { return { type: 'raw', id: event.id, - projectId: event.target.namespace.projectId, + projectId: event.target.namespace.projectId ?? '', scope: event.target.namespace.scope, enterpriseId: event.target.namespace.enterpriseId, workspaceId: event.target.namespace.workspaceId, @@ -494,10 +558,16 @@ function matchesNamespace( if (item.scope !== namespace.scope) return false; if ((item.enterpriseId ?? undefined) !== (namespace.enterpriseId ?? undefined)) return false; if ((item.workspaceId ?? undefined) !== (namespace.workspaceId ?? undefined)) return false; - if ((item.userId ?? undefined) !== (namespace.userId ?? undefined)) return false; + if ((item.userId ?? undefined) !== (namespace.userId ?? undefined)) { + if (!(query.includeLegacyPersonalOwner && namespace.scope === 'personal' && (!item.userId || item.userId === LEGACY_DAEMON_LOCAL_USER_ID))) return false; + } return true; } + if (query.scope && item.scope !== query.scope) return false; if (query.repo && item.projectId !== query.repo) return false; + if (query.userId && item.userId !== query.userId) { + if (!(query.includeLegacyPersonalOwner && item.scope === 'personal' && (!item.userId || item.userId === LEGACY_DAEMON_LOCAL_USER_ID))) return false; + } return true; } diff --git a/src/context/processed-context-replication.ts b/src/context/processed-context-replication.ts index 103f9cf40..b1279a8d4 100644 --- a/src/context/processed-context-replication.ts +++ b/src/context/processed-context-replication.ts @@ -136,7 +136,14 @@ function resolveStates(namespaces?: ContextNamespace[]): ContextReplicationState function selectPendingProjections(namespace: ContextNamespace, pendingIds: string[]): ProcessedContextProjection[] { const wanted = new Set(pendingIds); - return listProcessedProjections(namespace).filter((projection) => wanted.has(projection.id)); + return listProcessedProjections(namespace) + .filter((projection) => wanted.has(projection.id)) + .map((projection) => ({ + ...projection, + // Legacy rows created before post-1.1 origin metadata are backfilled at + // the replication boundary; new materialization/write paths set origin explicitly. + origin: projection.origin ?? 'chat_compacted', + })); } function isReplicableProjection(projection: ProcessedContextProjection): projection is ReplicableProcessedProjection { diff --git a/src/context/runtime-memory-cache-bus.ts b/src/context/runtime-memory-cache-bus.ts new file mode 100644 index 000000000..e0eec567e --- /dev/null +++ b/src/context/runtime-memory-cache-bus.ts @@ -0,0 +1,29 @@ +import type { ContextNamespace } from '../../shared/context-types.js'; +import { incrementCounter } from '../util/metrics.js'; + +export type RuntimeMemoryCacheInvalidationEvent = + | { kind: 'preference'; userId: string } + | { kind: 'observation'; observationId: string; namespace?: ContextNamespace } + | { kind: 'projection'; projectionId: string; namespace?: ContextNamespace } + | { kind: 'md_ingest'; projectDir: string; namespace: ContextNamespace } + | { kind: 'skill_registry' }; + +type Listener = (event: RuntimeMemoryCacheInvalidationEvent) => void; + +const listeners = new Set(); + +export function subscribeRuntimeMemoryCacheInvalidation(listener: Listener): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function publishRuntimeMemoryCacheInvalidation(event: RuntimeMemoryCacheInvalidationEvent): void { + incrementCounter('mem.cache.invalidate_published', { kind: event.kind }); + for (const listener of [...listeners]) { + try { + listener(event); + } catch { + // Cache invalidation is best-effort and must never block management mutation responses. + } + } +} diff --git a/src/context/skill-registry-builder.ts b/src/context/skill-registry-builder.ts new file mode 100644 index 000000000..c1654d233 --- /dev/null +++ b/src/context/skill-registry-builder.ts @@ -0,0 +1,242 @@ +import { existsSync, lstatSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'; +import { mkdirSync } from 'node:fs'; +import { dirname, join, relative } from 'node:path'; +import { homedir } from 'node:os'; +import { createHash } from 'node:crypto'; +import { + SKILL_REGISTRY_FILE_NAME, + SKILL_REGISTRY_SCHEMA_VERSION, + makeSkillUri, + type SkillRegistryEntry, + type SkillRegistrySnapshot, +} from '../../shared/skill-registry-types.js'; +import { + PROJECT_SKILL_ESCAPE_HATCH_DIR, + SKILL_FILE_EXTENSION, + classifyUserSkillLayer, + createSkillSource, + getProjectSkillEscapeHatchDir, + getUserSkillRoot, + parseSkillMarkdown, + type SkillLayer, + type SkillProjectContext, + type SkillSource, +} from '../../shared/skill-store.js'; +import { SKILL_MAX_BYTES } from '../../shared/skill-envelope.js'; +import { computeMemoryFingerprint } from '../../shared/memory-fingerprint.js'; +import { invalidateSkillRegistryCache } from './skill-registry.js'; +import { incrementCounter } from '../util/metrics.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { assertManagedSkillPathSync } from './managed-skill-path.js'; + +const MAX_SKILL_FILES = 64; +const MAX_SCAN_DEPTH = 4; +const SKILL_REGISTRY_BUILDER_SOURCE = 'skill-registry-builder'; + +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} + +function listMarkdownFiles(root: string, options: { maxFiles?: number; maxDepth?: number } = {}): string[] { + const maxFiles = Math.max(1, options.maxFiles ?? MAX_SKILL_FILES); + const maxDepth = Math.max(0, options.maxDepth ?? MAX_SCAN_DEPTH); + const files: string[] = []; + const visit = (dir: string, depth: number): void => { + if (files.length >= maxFiles || depth > maxDepth) return; + let entries: string[]; + try { + entries = readdirSync(dir).sort((a, b) => a.localeCompare(b)); + } catch { + return; + } + for (const entry of entries) { + if (files.length >= maxFiles) break; + const fullPath = join(dir, entry); + let stat; + try { + stat = lstatSync(fullPath); + } catch { + continue; + } + if (stat.isSymbolicLink()) continue; + if (stat.isDirectory()) { + visit(fullPath, depth + 1); + continue; + } + if (stat.isFile() && fullPath.endsWith(SKILL_FILE_EXTENSION) && stat.size <= SKILL_MAX_BYTES * 2) files.push(fullPath); + } + }; + if (existsSync(root)) visit(root, 0); + return files; +} + +function displayPathFor(path: string, input: { homeDir?: string; projectDir?: string }): string { + const home = input.homeDir?.replace(/[\\/]+$/, ''); + if (home && (path === home || path.startsWith(`${home}/`) || path.startsWith(`${home}\\`))) return `~${path.slice(home.length)}`; + const project = input.projectDir?.replace(/[\\/]+$/, ''); + if (project && (path === project || path.startsWith(`${project}/`) || path.startsWith(`${project}\\`))) { + const rel = relative(project, path); + if (rel && !rel.startsWith('..')) return rel; + } + return path; +} + +function fallbackNameFromPath(path: string): string | undefined { + const file = path.split(/[\\/]/).pop(); + return file?.endsWith(SKILL_FILE_EXTENSION) ? file.slice(0, -SKILL_FILE_EXTENSION.length) : file; +} + +export function skillRegistryEntryFromSource(source: SkillSource, input: { + path: string; + homeDir?: string; + projectDir?: string; + contentHash?: string; + mtimeMs?: number; + updatedAt?: number; +}): SkillRegistryEntry { + const fingerprint = computeMemoryFingerprint({ + kind: 'skill', + content: `${source.layer}\n${source.key}\n${source.metadata.description ?? ''}\n${input.contentHash ?? ''}`, + }); + return { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + key: source.key, + layer: source.layer, + metadata: source.metadata, + path: input.path, + displayPath: displayPathFor(input.path, input), + uri: makeSkillUri(source.layer, source.key), + fingerprint, + contentHash: input.contentHash, + mtimeMs: input.mtimeMs, + enforcement: source.enforcement, + project: source.metadata.project, + updatedAt: input.updatedAt ?? Date.now(), + }; +} + +function readSkillSource(path: string, layer: SkillLayer, fallback: { name?: string; category?: string }): SkillSource | null { + try { + const markdown = readFileSync(path, 'utf8'); + const parsed = parseSkillMarkdown(markdown, fallback); + return createSkillSource({ + layer, + metadata: parsed.metadata, + content: parsed.content, + path, + }); + } catch (error) { + incrementCounter('mem.skill.sanitize_rejected', { source: SKILL_REGISTRY_BUILDER_SOURCE }); + warnOncePerHour('skill_registry_builder.parse_failed', { path, error: error instanceof Error ? error.message : String(error) }); + return null; + } +} + +function writeRegistry(path: string, entries: SkillRegistryEntry[]): SkillRegistrySnapshot { + const snapshot: SkillRegistrySnapshot = { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: Date.now(), + entries: entries.sort((a, b) => `${a.layer}:${a.key}`.localeCompare(`${b.layer}:${b.key}`)), + }; + mkdirSync(dirname(path), { recursive: true }); + const tmpPath = `${path}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(tmpPath, `${JSON.stringify(snapshot, null, 2)}\n`, 'utf8'); + renameSync(tmpPath, path); + invalidateSkillRegistryCache(); + return snapshot; +} + +export function buildUserSkillRegistry(input: { homeDir?: string; context?: SkillProjectContext } = {}): SkillRegistrySnapshot { + const homeDir = input.homeDir ?? homedir(); + const root = getUserSkillRoot(homeDir); + const entries: SkillRegistryEntry[] = []; + for (const path of listMarkdownFiles(root)) { + try { + assertManagedSkillPathSync({ path, homeDir, maxBytes: SKILL_MAX_BYTES }); + } catch { + incrementCounter('mem.skill.sanitize_rejected', { source: SKILL_REGISTRY_BUILDER_SOURCE }); + continue; + } + const source = readSkillSource(path, 'user_default', { + name: fallbackNameFromPath(path), + category: relative(root, path).split(/[\\/]/)[0] || 'general', + }); + const layer = source ? classifyUserSkillLayer(source.metadata, input.context) : null; + if (!source || !layer) continue; + const normalized = { ...source, layer, id: `${layer}:${source.key}:${path}` } satisfies SkillSource; + const stat = statSync(path); + entries.push(skillRegistryEntryFromSource(normalized, { + path, + homeDir, + contentHash: sha256(readFileSync(path, 'utf8')), + mtimeMs: stat.mtimeMs, + updatedAt: stat.mtimeMs, + })); + } + return writeRegistry(join(root, SKILL_REGISTRY_FILE_NAME), entries); +} + +export function buildProjectSkillRegistry(input: { projectDir: string }): SkillRegistrySnapshot { + const root = getProjectSkillEscapeHatchDir(input.projectDir); + const entries: SkillRegistryEntry[] = []; + for (const path of listMarkdownFiles(root)) { + try { + assertManagedSkillPathSync({ path, projectDir: input.projectDir, maxBytes: SKILL_MAX_BYTES }); + } catch { + incrementCounter('mem.skill.sanitize_rejected', { source: SKILL_REGISTRY_BUILDER_SOURCE }); + continue; + } + const source = readSkillSource(path, 'project_escape_hatch', { + name: fallbackNameFromPath(path), + category: relative(root, path).split(/[\\/]/)[0] || 'project', + }); + if (!source) continue; + const stat = statSync(path); + entries.push(skillRegistryEntryFromSource(source, { + path, + projectDir: input.projectDir, + contentHash: sha256(readFileSync(path, 'utf8')), + mtimeMs: stat.mtimeMs, + updatedAt: stat.mtimeMs, + })); + } + return writeRegistry(join(root, SKILL_REGISTRY_FILE_NAME), entries); +} + +export function buildSkillRegistryEntryForWrittenUserSkill(input: { + homeDir: string; + path: string; + skillName: string; + category: string; + description?: string; + project?: SkillProjectContext; + now?: number; +}): SkillRegistryEntry { + const metadata = { + schemaVersion: 1 as const, + name: input.skillName, + category: input.category, + description: input.description, + project: input.project, + }; + const source = createSkillSource({ + layer: input.project ? 'user_project' : 'user_default', + metadata, + content: '', + path: input.path, + }); + return skillRegistryEntryFromSource(source, { + path: input.path, + homeDir: input.homeDir, + updatedAt: input.now ?? Date.now(), + }); +} + +export const SKILL_REGISTRY_BUILDER_TESTING = { + listMarkdownFiles, + displayPathFor, + fallbackNameFromPath, + constants: { + projectSkillEscapeHatchDir: PROJECT_SKILL_ESCAPE_HATCH_DIR, + }, +}; diff --git a/src/context/skill-registry.ts b/src/context/skill-registry.ts new file mode 100644 index 000000000..fdd978c2b --- /dev/null +++ b/src/context/skill-registry.ts @@ -0,0 +1,276 @@ +import { existsSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'; +import { mkdirSync } from 'node:fs'; +import { dirname, isAbsolute, join } from 'node:path'; +import { homedir } from 'node:os'; +import type { ContextNamespace } from '../../shared/context-types.js'; +import { + SKILL_REGISTRY_FILE_NAME, + SKILL_REGISTRY_SCHEMA_VERSION, + makeSkillUri, + type SkillRegistryEntry, + type SkillRegistrySnapshot, +} from '../../shared/skill-registry-types.js'; +import { + getProjectSkillEscapeHatchDir, + getUserSkillRoot, + isSkillLayer, + normalizeSkillMetadata, + type SkillProjectContext, +} from '../../shared/skill-store.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { incrementCounter } from '../util/metrics.js'; +import { MEMORY_DEFAULTS } from '../../shared/memory-defaults.js'; + +const EMPTY_SNAPSHOT: SkillRegistrySnapshot = { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: 0, + entries: [], + sourceCounts: {}, +}; + +type CacheEntry = { key: string; snapshot: SkillRegistrySnapshot }; +let cache: CacheEntry | null = null; + +const ALLOWED_REGISTRY_ENTRY_KEYS: ReadonlySet = new Set([ + 'schemaVersion', + 'key', + 'layer', + 'metadata', + 'path', + 'displayPath', + 'uri', + 'fingerprint', + 'contentHash', + 'mtimeMs', + 'enforcement', + 'triggerKeywords', + 'project', + 'updatedAt', +]); + +export interface SkillRegistryOptions { + namespace: ContextNamespace; + projectDir?: string; + homeDir?: string; +} + +function userRegistryPath(homeDir = homedir()): string { + return join(getUserSkillRoot(homeDir), SKILL_REGISTRY_FILE_NAME); +} + +function projectRegistryPath(projectDir: string | undefined): string | undefined { + const root = projectDir?.trim(); + return root ? join(getProjectSkillEscapeHatchDir(root), SKILL_REGISTRY_FILE_NAME) : undefined; +} + +function cacheKey(options: SkillRegistryOptions): string { + return [ + options.homeDir ?? homedir(), + options.projectDir ?? '', + options.namespace.scope, + options.namespace.projectId ?? '', + options.namespace.workspaceId ?? '', + options.namespace.enterpriseId ?? '', + options.namespace.userId ?? '', + ].join('\u0000'); +} + +function parseEntry(value: unknown): SkillRegistryEntry | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const record = value as Record; + const unknownKey = Object.keys(record).find((key) => !ALLOWED_REGISTRY_ENTRY_KEYS.has(key)); + if (unknownKey) throw new Error(`Unknown skill registry entry field: ${unknownKey}`); + if (record.schemaVersion !== SKILL_REGISTRY_SCHEMA_VERSION) return null; + if (typeof record.key !== 'string' || !record.key.trim()) return null; + if (!isSkillLayer(record.layer)) return null; + if (typeof record.displayPath !== 'string' || !record.displayPath.trim()) return null; + if (typeof record.uri !== 'string' || !record.uri.startsWith('skill://')) return null; + if (typeof record.fingerprint !== 'string' || !record.fingerprint.trim()) return null; + const metadata = normalizeSkillMetadata(record.metadata as Record); + return { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + key: record.key.trim(), + layer: record.layer, + metadata, + path: typeof record.path === 'string' && record.path.trim() ? record.path : undefined, + displayPath: sanitizeRegistryDisplayPath(record.displayPath, makeSkillUri(record.layer, record.key.trim())), + uri: record.uri as SkillRegistryEntry['uri'], + fingerprint: record.fingerprint.trim(), + contentHash: typeof record.contentHash === 'string' && record.contentHash.trim() ? record.contentHash : undefined, + mtimeMs: typeof record.mtimeMs === 'number' && Number.isFinite(record.mtimeMs) ? record.mtimeMs : undefined, + enforcement: record.enforcement === 'additive' || record.enforcement === 'enforced' ? record.enforcement : undefined, + triggerKeywords: Array.isArray(record.triggerKeywords) + ? record.triggerKeywords.filter((entry): entry is string => typeof entry === 'string' && entry.trim().length > 0).map((entry) => entry.trim()) + : undefined, + project: record.project && typeof record.project === 'object' && !Array.isArray(record.project) + ? record.project as SkillProjectContext + : undefined, + updatedAt: typeof record.updatedAt === 'number' && Number.isFinite(record.updatedAt) ? record.updatedAt : 0, + }; +} + +function sanitizeRegistryDisplayPath(displayPath: string, fallbackUri: SkillRegistryEntry['uri']): string { + const trimmed = displayPath.trim(); + if (!trimmed || trimmed.includes('\u0000')) return fallbackUri; + if (trimmed.startsWith('skill://')) return trimmed; + if (trimmed.startsWith('~/') || trimmed.startsWith('~\\')) return trimmed; + if (/^[a-zA-Z]:[\\/]/.test(trimmed) || trimmed.startsWith('\\\\') || isAbsolute(trimmed)) return fallbackUri; + const normalized = trimmed.replace(/\\/g, '/'); + if (normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) return fallbackUri; + return trimmed; +} + +function readRegistryFile(path: string | undefined): SkillRegistryEntry[] { + if (!path || !existsSync(path)) return []; + try { + const stat = statSync(path); + if (stat.size > MEMORY_DEFAULTS.skillRegistryMaxBytes) { + incrementCounter('mem.skill.registry_oversize', { source: 'skill_registry_read' }); + warnOncePerHour('skill_registry.oversize', { path, size: stat.size, maxBytes: MEMORY_DEFAULTS.skillRegistryMaxBytes }); + return []; + } + const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown; + const entries = typeof parsed === 'object' && parsed && Array.isArray((parsed as { entries?: unknown }).entries) + ? (parsed as { entries: unknown[] }).entries + : []; + if (entries.length > MEMORY_DEFAULTS.skillRegistryMaxEntries) { + incrementCounter('mem.skill.registry_oversize', { source: 'skill_registry_entries' }); + warnOncePerHour('skill_registry.too_many_entries', { path, entries: entries.length, maxEntries: MEMORY_DEFAULTS.skillRegistryMaxEntries }); + return []; + } + return entries.flatMap((entry) => { + try { + const parsed = parseEntry(entry); + return parsed ? [parsed] : []; + } catch (error) { + incrementCounter('mem.skill.sanitize_rejected', { source: 'skill_registry_entry' }); + warnOncePerHour('skill_registry.entry_rejected', { path, error: error instanceof Error ? error.message : String(error) }); + return []; + } + }); + } catch (error) { + incrementCounter('mem.skill.sanitize_rejected', { source: 'skill_registry_read' }); + warnOncePerHour('skill_registry.read_failed', { path, error: error instanceof Error ? error.message : String(error) }); + return []; + } +} + +function namespaceMatches(entry: SkillRegistryEntry, namespace: ContextNamespace): boolean { + const project = entry.metadata.project ?? entry.project; + if (!project) return true; + if (project.canonicalRepoId && project.canonicalRepoId !== namespace.projectId) return false; + if (project.projectId && project.projectId !== namespace.projectId) return false; + if (project.workspaceId && project.workspaceId !== namespace.workspaceId) return false; + if (project.orgId && project.orgId !== namespace.enterpriseId) return false; + return true; +} + +function mergeEntries(entries: SkillRegistryEntry[]): SkillRegistryEntry[] { + const byIdentity = new Map(); + for (const entry of entries) { + const id = `${entry.layer}\u0000${entry.key}\u0000${entry.path ?? entry.uri}`; + const prior = byIdentity.get(id); + if (!prior || entry.updatedAt >= prior.updatedAt) byIdentity.set(id, entry); + } + return [...byIdentity.values()].sort((a, b) => `${a.layer}:${a.key}`.localeCompare(`${b.layer}:${b.key}`)); +} + +export function getSkillRegistrySnapshot(options: SkillRegistryOptions): SkillRegistrySnapshot { + const key = cacheKey(options); + if (cache?.key === key) return cache.snapshot; + const entries = mergeEntries([ + ...readRegistryFile(projectRegistryPath(options.projectDir)), + ...readRegistryFile(userRegistryPath(options.homeDir)), + ]).filter((entry) => namespaceMatches(entry, options.namespace)); + const snapshot: SkillRegistrySnapshot = { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: Date.now(), + entries, + sourceCounts: entries.reduce>((acc, entry) => { + acc[entry.layer] = (acc[entry.layer] ?? 0) + 1; + return acc; + }, {}), + }; + cache = { key, snapshot }; + return snapshot; +} + +export function getSkillRegistryManagementSnapshot(options: { projectDir?: string; homeDir?: string } = {}): SkillRegistrySnapshot { + const entries = mergeEntries([ + ...readRegistryFile(projectRegistryPath(options.projectDir)), + ...readRegistryFile(userRegistryPath(options.homeDir)), + ]); + return { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: Date.now(), + entries, + sourceCounts: entries.reduce>((acc, entry) => { + acc[entry.layer] = (acc[entry.layer] ?? 0) + 1; + return acc; + }, {}), + }; +} + +export function writeSkillRegistryManagementSnapshot(path: string, entries: SkillRegistryEntry[]): SkillRegistrySnapshot { + const snapshot = { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: Date.now(), + entries: mergeEntries(entries), + } satisfies SkillRegistrySnapshot; + writeSnapshot(path, snapshot); + invalidateSkillRegistryCache(); + return snapshot; +} + +export function getSkillRegistryPathsForManagement(options: { projectDir?: string; homeDir?: string } = {}): { + user: string; + project?: string; +} { + return { + user: userRegistryPath(options.homeDir), + project: projectRegistryPath(options.projectDir), + }; +} + +export function invalidateSkillRegistryCache(): void { + cache = null; +} + +function readSnapshotForWrite(path: string): SkillRegistrySnapshot { + const entries = readRegistryFile(path); + return { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: Date.now(), + entries, + }; +} + +function writeSnapshot(path: string, snapshot: SkillRegistrySnapshot): void { + mkdirSync(dirname(path), { recursive: true }); + const tmpPath = `${path}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(tmpPath, `${JSON.stringify(snapshot, null, 2)}\n`, 'utf8'); + renameSync(tmpPath, path); +} + +export function upsertUserSkillRegistryEntry(entry: SkillRegistryEntry, options: { homeDir?: string } = {}): void { + const path = userRegistryPath(options.homeDir); + const snapshot = readSnapshotForWrite(path); + const nextEntries = mergeEntries([ + ...snapshot.entries.filter((existing) => !(existing.layer === entry.layer && existing.key === entry.key && existing.path === entry.path)), + entry, + ]); + writeSnapshot(path, { + schemaVersion: SKILL_REGISTRY_SCHEMA_VERSION, + generatedAt: Date.now(), + entries: nextEntries, + }); + invalidateSkillRegistryCache(); +} + +export const SKILL_REGISTRY_TESTING = { + userRegistryPath, + projectRegistryPath, + parseEntry, + readRegistryFile, + reset: invalidateSkillRegistryCache, +}; diff --git a/src/context/skill-resolver.ts b/src/context/skill-resolver.ts new file mode 100644 index 000000000..7c750d1cd --- /dev/null +++ b/src/context/skill-resolver.ts @@ -0,0 +1,105 @@ +import { readFileSync } from 'node:fs'; +import type { ContextNamespace } from '../../shared/context-types.js'; +import { renderSkillEnvelope } from '../../shared/skill-envelope.js'; +import { skillRegistryEntryToSource, type SkillRegistryEntry } from '../../shared/skill-registry-types.js'; +import { parseSkillMarkdown, type SkillProjectContext } from '../../shared/skill-store.js'; +import { resolveSkillSelection } from '../../shared/skill-precedence.js'; +import { getSkillRegistrySnapshot } from './skill-registry.js'; +import { incrementCounter } from '../util/metrics.js'; +import { assertManagedSkillPathSync, ManagedSkillPathError } from './managed-skill-path.js'; + +export type SkillResolveFailureReason = 'unknown_key' | 'stale_registry' | 'unauthorized' | 'oversize' | 'read_failed' | 'sanitize_rejected'; + +export type SkillResolveResult = + | { ok: true; key: string; layer: string; path: string; text: string; entry: SkillRegistryEntry } + | { ok: false; key: string; reason: SkillResolveFailureReason }; + +export interface SkillResolveOptions { + namespace: ContextNamespace; + key: string; + projectDir?: string; + homeDir?: string; + maxBytes?: number; +} + +function projectContext(namespace: ContextNamespace, projectDir?: string): SkillProjectContext { + return { + canonicalRepoId: namespace.projectId, + projectId: namespace.projectId, + workspaceId: namespace.workspaceId, + orgId: namespace.enterpriseId, + rootPath: projectDir, + }; +} + +function chooseEntry(options: SkillResolveOptions): SkillRegistryEntry | undefined { + const snapshot = getSkillRegistrySnapshot({ + namespace: options.namespace, + projectDir: options.projectDir, + homeDir: options.homeDir, + }); + const sources = snapshot.entries.map((entry) => skillRegistryEntryToSource(entry)); + const selected = resolveSkillSelection(sources, projectContext(options.namespace, options.projectDir)).selected; + const selectedSource = selected.find((entry) => entry.key === options.key.trim()); + if (!selectedSource) return undefined; + return snapshot.entries.find((entry) => entry.key === selectedSource.key && entry.layer === selectedSource.effectiveLayer); +} + +export function resolveSkillByKey(options: SkillResolveOptions): SkillResolveResult { + const key = options.key.trim(); + const entry = chooseEntry({ ...options, key }); + if (!entry?.path) { + incrementCounter('mem.skill.resolver_miss', { reason: 'unknown_key' }); + return { ok: false, key, reason: 'unknown_key' }; + } + let managedPath; + try { + managedPath = assertManagedSkillPathSync({ + path: entry.path, + projectDir: options.projectDir, + homeDir: options.homeDir, + maxBytes: options.maxBytes, + }); + } catch (error) { + const reason = error instanceof ManagedSkillPathError && error.reason === 'oversize' + ? 'oversize' + : (error instanceof ManagedSkillPathError && error.reason === 'not_file' ? 'stale_registry' : 'unauthorized'); + incrementCounter('mem.skill.resolver_miss', { reason }); + return { ok: false, key, reason }; + } + try { + const markdown = readFileSync(managedPath.realPath, 'utf8'); + const parsed = parseSkillMarkdown(markdown, { name: entry.metadata.name, category: entry.metadata.category }); + try { + return { + ok: true, + key, + layer: entry.layer, + path: entry.displayPath, + text: renderSkillEnvelope(parsed.content, { maxBytes: options.maxBytes }), + entry, + }; + } catch { + incrementCounter('mem.skill.sanitize_rejected', { source: 'skill_resolver' }); + return { ok: false, key, reason: 'sanitize_rejected' }; + } + } catch { + incrementCounter('mem.skill.resolver_miss', { reason: 'read_failed' }); + return { ok: false, key, reason: 'read_failed' }; + } +} + +export function resolveSkillsForTurn(input: Omit & { prompt: string; maxSkills?: number }): SkillResolveResult[] { + const prompt = input.prompt.toLowerCase(); + const snapshot = getSkillRegistrySnapshot({ namespace: input.namespace, projectDir: input.projectDir, homeDir: input.homeDir }); + const keys = snapshot.entries + .filter((entry) => { + const haystack = [entry.key, entry.metadata.name, entry.metadata.category, entry.metadata.description, ...(entry.triggerKeywords ?? [])] + .filter((value): value is string => typeof value === 'string') + .join(' ') + .toLowerCase(); + return haystack.split(/\s+/).some((token) => token.length >= 3 && prompt.includes(token)); + }) + .map((entry) => entry.key); + return [...new Set(keys)].slice(0, Math.max(1, input.maxSkills ?? 3)).map((key) => resolveSkillByKey({ ...input, key })); +} diff --git a/src/context/skill-review-worker.ts b/src/context/skill-review-worker.ts new file mode 100644 index 000000000..394a146bf --- /dev/null +++ b/src/context/skill-review-worker.ts @@ -0,0 +1,268 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { homedir } from 'node:os'; +import type { + MaterializationSkillReviewJob, + MaterializationSkillReviewScheduler, +} from './materialization-coordinator.js'; +import { + MEMORY_FEATURE_FLAGS, + MEMORY_FEATURE_FLAGS_BY_NAME, + memoryFeatureFlagEnvKey, + type MemoryFeatureFlag, + resolveEffectiveMemoryFeatureFlagValue, +} from '../../shared/feature-flags.js'; +import { + decideSkillReviewClaim, + makeSkillReviewDailyCountKey, + nextSkillReviewRetryAt, + type SkillReviewJobState, + type SkillReviewSchedulerPolicy, + type SkillReviewState, +} from '../../shared/skill-review-scheduler.js'; +import { computeMemoryFingerprint } from '../../shared/memory-fingerprint.js'; +import { + chooseSkillReviewWriteTarget, + getUserSkillPath, + makeSkillKey, +} from '../../shared/skill-store.js'; +import { skillRegistryEntryToSource } from '../../shared/skill-registry-types.js'; +import { sanitizeSkillEnvelopeContent } from '../../shared/skill-envelope.js'; +import { getProcessedProjectionById } from '../store/context-store.js'; +import { incrementCounter } from '../util/metrics.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { getSkillRegistrySnapshot, upsertUserSkillRegistryEntry } from './skill-registry.js'; +import { buildSkillRegistryEntryForWrittenUserSkill } from './skill-registry-builder.js'; +import { + getMemoryFeatureConfigStoreDiagnostics, + getPersistedMemoryFeatureFlagValues, + getRuntimeMemoryFeatureFlagValues, +} from '../store/memory-feature-config-store.js'; + +type StoredSkillReviewJob = MaterializationSkillReviewJob & { + state: SkillReviewJobState; + attempt: number; + updatedAt: number; +}; + +function readEnvFlag(flag: MemoryFeatureFlag): boolean | undefined { + const raw = process.env[memoryFeatureFlagEnvKey(flag)]; + if (raw == null) return undefined; + return raw === 'true' || raw === '1'; +} + +function effectiveSkillAutoCreationEnabled(): boolean { + const environmentStartupDefault = Object.fromEntries( + MEMORY_FEATURE_FLAGS.flatMap((flag): Array<[MemoryFeatureFlag, boolean]> => { + const value = readEnvFlag(flag); + return value === undefined ? [] : [[flag, value]]; + }), + ) as Partial>; + return resolveEffectiveMemoryFeatureFlagValue(MEMORY_FEATURE_FLAGS_BY_NAME.skillAutoCreation, { + runtimeConfigOverride: getRuntimeMemoryFeatureFlagValues(), + persistedConfig: getPersistedMemoryFeatureFlagValues(), + environmentStartupDefault, + readFailed: !!getMemoryFeatureConfigStoreDiagnostics().lastLoadIssue, + }); +} + +export class LocalSkillReviewWorker implements MaterializationSkillReviewScheduler { + readonly policy?: Partial; + private readonly jobs = new Map(); + private readonly lastRunByScope = new Map(); + private readonly dailyCountByScope = new Map(); + private readonly runningCountByScope = new Map(); + private timer: ReturnType | null = null; + private shuttingDown = false; + + constructor(private readonly options: { + homeDir?: string; + featureEnabled?: boolean | (() => boolean); + policy?: Partial; + } = {}) { + this.policy = options.policy; + } + + get featureEnabled(): boolean | (() => boolean) { + return this.options.featureEnabled ?? effectiveSkillAutoCreationEnabled; + } + + getState(scopeKey: string): SkillReviewState { + const pendingKeys = new Set(); + for (const job of this.jobs.values()) { + if (job.scopeKey !== scopeKey) continue; + if (job.state === 'pending' || job.state === 'retry_wait' || job.state === 'running') { + pendingKeys.add(job.idempotencyKey); + } + } + return { + pendingKeys, + lastRunByScope: this.lastRunByScope, + dailyCountByScope: this.dailyCountByScope, + runningCountByScope: this.runningCountByScope, + }; + } + + isShuttingDown(): boolean { + return this.shuttingDown; + } + + stop(): void { + this.shuttingDown = true; + if (this.timer) clearTimeout(this.timer); + this.timer = null; + } + + enqueue(job: MaterializationSkillReviewJob): void { + if (!this.jobs.has(job.idempotencyKey)) { + this.jobs.set(job.idempotencyKey, { + ...job, + state: 'pending', + attempt: 0, + updatedAt: job.createdAt, + }); + } + this.schedulePump(0); + } + + async drainDueJobsForTests(now = Date.now()): Promise { + await this.pump(now); + } + + private isEnabled(): boolean { + const enabled = this.featureEnabled; + return typeof enabled === 'function' ? enabled() : enabled; + } + + private schedulePump(delayMs: number): void { + if (this.timer || this.shuttingDown) return; + this.timer = setTimeout(() => { + this.timer = null; + void this.pump(Date.now()).catch((error) => { + incrementCounter('mem.skill.review_failed', { source: 'worker_pump' }); + warnOncePerHour('skill_review.worker_pump', { error: error instanceof Error ? error.message : String(error) }); + }); + }, delayMs); + this.timer.unref?.(); + } + + private async pump(now: number): Promise { + const enabled = this.isEnabled(); + let nextDelay: number | undefined; + for (const job of this.jobs.values()) { + const claim = decideSkillReviewClaim({ + featureEnabled: enabled, + shuttingDown: this.shuttingDown, + job, + now, + runningCountByScope: this.runningCountByScope, + policy: this.policy, + }); + if (claim.action === 'skip') { + if (job.state === 'retry_wait' && job.nextAttemptAt !== undefined) { + const delay = Math.max(0, job.nextAttemptAt - now); + nextDelay = nextDelay === undefined ? delay : Math.min(nextDelay, delay); + } + continue; + } + await this.runClaimedJob(job, now); + } + if (nextDelay !== undefined) this.schedulePump(nextDelay); + } + + private async runClaimedJob(job: StoredSkillReviewJob, now: number): Promise { + job.state = 'running'; + job.attempt += 1; + job.updatedAt = now; + this.runningCountByScope.set(job.scopeKey, (this.runningCountByScope.get(job.scopeKey) ?? 0) + 1); + try { + await this.writeSkill(job); + job.state = 'succeeded'; + job.updatedAt = Date.now(); + this.lastRunByScope.set(job.scopeKey, job.updatedAt); + const dailyCountKey = makeSkillReviewDailyCountKey({ scopeKey: job.scopeKey, now: job.updatedAt }); + this.dailyCountByScope.set(dailyCountKey, (this.dailyCountByScope.get(dailyCountKey) ?? 0) + 1); + } catch (error) { + incrementCounter('mem.skill.review_failed', { source: 'worker_write' }); + warnOncePerHour('skill_review.worker_write', { error: error instanceof Error ? error.message : String(error) }); + if (job.attempt >= job.maxAttempts) { + job.state = 'failed'; + } else { + job.state = 'retry_wait'; + job.nextAttemptAt = nextSkillReviewRetryAt(Date.now(), job.attempt, this.policy); + this.schedulePump(Math.max(0, job.nextAttemptAt - Date.now())); + } + job.updatedAt = Date.now(); + } finally { + const running = Math.max(0, (this.runningCountByScope.get(job.scopeKey) ?? 1) - 1); + if (running === 0) this.runningCountByScope.delete(job.scopeKey); + else this.runningCountByScope.set(job.scopeKey, running); + } + } + + private async writeSkill(job: MaterializationSkillReviewJob): Promise { + const projection = getProcessedProjectionById(job.projectionId); + if (!projection) throw new Error(`skill review projection not found: ${job.projectionId}`); + const candidateText = [ + '# Learned workflow', + '', + projection.summary, + '', + `Source projection: ${job.projectionId}`, + ].join('\n'); + const sanitized = sanitizeSkillEnvelopeContent(candidateText); + if (!sanitized.ok) { + incrementCounter('mem.skill.sanitize_rejected', { source: 'skill_review_worker' }); + throw new Error(sanitized.reason ?? 'skill review content rejected'); + } + const skillHash = computeMemoryFingerprint({ kind: 'skill', content: projection.summary }); + const skillName = `imcodes-learned-${skillHash.slice(0, 12)}`; + const homeDir = this.options.homeDir ?? homedir(); + const context = { + canonicalRepoId: job.target.namespace.projectId, + projectId: job.target.namespace.projectId, + workspaceId: job.target.namespace.workspaceId, + orgId: job.target.namespace.enterpriseId, + }; + const target = chooseSkillReviewWriteTarget({ + candidateKey: makeSkillKey('learned', skillName), + userSkillSources: getSkillRegistrySnapshot({ namespace: job.target.namespace, homeDir }).entries.map((entry) => skillRegistryEntryToSource(entry)), + context, + }); + const path = target.action === 'update_user_skill' && target.source.path + ? target.source.path + : getUserSkillPath({ + homeDir, + category: 'learned', + skillName, + }); + const projectFrontMatter = [ + 'project:', + ...(context.canonicalRepoId ? [` canonicalRepoId: ${JSON.stringify(context.canonicalRepoId)}`] : []), + ...(context.projectId ? [` projectId: ${JSON.stringify(context.projectId)}`] : []), + ...(context.workspaceId ? [` workspaceId: ${JSON.stringify(context.workspaceId)}`] : []), + ...(context.orgId ? [` orgId: ${JSON.stringify(context.orgId)}`] : []), + ]; + const markdown = [ + '---', + 'schemaVersion: 1', + `name: ${JSON.stringify(skillName)}`, + 'category: learned', + 'description: "Auto-created from post-response memory review."', + ...projectFrontMatter, + '---', + sanitized.content, + '', + ].join('\n'); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, markdown, 'utf8'); + upsertUserSkillRegistryEntry(buildSkillRegistryEntryForWrittenUserSkill({ + homeDir, + path, + skillName, + category: 'learned', + description: 'Auto-created from post-response memory review.', + project: context, + }), { homeDir }); + } +} diff --git a/src/context/skill-startup-context.ts b/src/context/skill-startup-context.ts new file mode 100644 index 000000000..cad79db7a --- /dev/null +++ b/src/context/skill-startup-context.ts @@ -0,0 +1,118 @@ +import { homedir } from 'node:os'; +import type { ContextNamespace } from '../../shared/context-types.js'; +import { + MEMORY_FEATURE_FLAGS, + MEMORY_FEATURE_FLAGS_BY_NAME, + memoryFeatureFlagEnvKey, + resolveEffectiveMemoryFeatureFlagValue, + type MemoryFeatureFlag, + type MemoryFeatureFlagValues, +} from '../../shared/feature-flags.js'; +import { computeMemoryFingerprint } from '../../shared/memory-fingerprint.js'; +import { violatesSkillSystemInstructionGuard } from '../../shared/skill-envelope.js'; +import { skillRegistryEntryToSource } from '../../shared/skill-registry-types.js'; +import type { SkillProjectContext } from '../../shared/skill-store.js'; +import { + resolveSkillSelection, + type SelectedSkill, +} from '../../shared/skill-precedence.js'; +import type { StartupMemoryCandidate } from './startup-memory.js'; +import { getSkillRegistrySnapshot } from './skill-registry.js'; +import { incrementCounter } from '../util/metrics.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { + getMemoryFeatureConfigStoreDiagnostics, + getPersistedMemoryFeatureFlagValues, + getRuntimeMemoryFeatureFlagValues, +} from '../store/memory-feature-config-store.js'; + +const SKILL_STARTUP_SOURCE = 'skill-startup-registry'; + +export interface SkillStartupContextOptions { + namespace: ContextNamespace; + projectDir?: string; + homeDir?: string; + featureEnabled?: boolean; +} + +function isSkillsFeatureEnabled(): boolean { + const flag = MEMORY_FEATURE_FLAGS_BY_NAME.skills; + const environmentStartupDefault = Object.fromEntries( + MEMORY_FEATURE_FLAGS.flatMap((candidate): Array<[MemoryFeatureFlag, boolean]> => { + const raw = process.env[memoryFeatureFlagEnvKey(candidate)]; + return raw == null ? [] : [[candidate, raw === 'true' || raw === '1']]; + }), + ) as MemoryFeatureFlagValues; + return resolveEffectiveMemoryFeatureFlagValue(flag, { + runtimeConfigOverride: getRuntimeMemoryFeatureFlagValues(), + persistedConfig: getPersistedMemoryFeatureFlagValues(), + environmentStartupDefault, + readFailed: !!getMemoryFeatureConfigStoreDiagnostics().lastLoadIssue, + }); +} + +function skillProjectContext(namespace: ContextNamespace, projectDir?: string): SkillProjectContext { + return { + canonicalRepoId: namespace.projectId, + projectId: namespace.projectId, + workspaceId: namespace.workspaceId, + orgId: namespace.enterpriseId, + rootPath: projectDir, + }; +} + +function sanitizeSkillDescriptor(value: string | undefined): string | undefined { + const oneLine = value?.replace(/\s+/g, ' ').trim(); + if (!oneLine) return undefined; + if (violatesSkillSystemInstructionGuard(oneLine)) return undefined; + return oneLine.length > 180 ? `${oneLine.slice(0, 177)}...` : oneLine; +} + +function renderSkillReference(entry: SelectedSkill): string { + const metadata = entry.source.metadata; + const description = sanitizeSkillDescriptor(metadata.description); + const path = entry.source.path ?? '(unavailable)'; + return [ + `skill: ${entry.key}`, + `layer: ${entry.effectiveLayer}`, + `selection: ${entry.selectionKind}`, + `path: ${path}`, + ...(description ? [`description: ${description}`] : []), + 'instruction: This is a registry hint only. Read this skill only when the current task is relevant; do not assume or execute its body until explicitly read.', + ].join('\n'); +} + +export function collectSkillStartupCandidates(options: SkillStartupContextOptions): StartupMemoryCandidate[] { + const featureEnabled = options.featureEnabled ?? isSkillsFeatureEnabled(); + if (!featureEnabled) return []; + try { + const context = skillProjectContext(options.namespace, options.projectDir); + const snapshot = getSkillRegistrySnapshot({ + namespace: options.namespace, + projectDir: options.projectDir, + homeDir: options.homeDir ?? homedir(), + }); + if (snapshot.entries.length === 0) return []; + const sources = snapshot.entries.map((entry) => skillRegistryEntryToSource(entry, { displayPath: true })); + const selection = resolveSkillSelection(sources, context); + return selection.selected.map((entry): StartupMemoryCandidate => ({ + id: `skill:${entry.effectiveLayer}:${entry.key}`, + source: 'skill', + text: renderSkillReference(entry), + fingerprint: computeMemoryFingerprint({ + kind: 'skill', + content: `${entry.selectionKind}\n${entry.effectiveLayer}\n${entry.key}\n${entry.source.path ?? ''}`, + }), + })); + } catch (error) { + incrementCounter('mem.startup.silent_failure', { source: SKILL_STARTUP_SOURCE }); + warnOncePerHour('skill_startup.registry_failed', { + error: error instanceof Error ? error.message : String(error), + }); + return []; + } +} + +export const SKILL_STARTUP_CONTEXT_TESTING = { + skillProjectContext, +}; diff --git a/src/context/startup-memory.ts b/src/context/startup-memory.ts index 59824aed9..f1c7cdd89 100644 --- a/src/context/startup-memory.ts +++ b/src/context/startup-memory.ts @@ -2,10 +2,48 @@ import type { ContextNamespace } from '../../shared/context-types.js'; import type { MemorySearchResultItem } from './memory-search.js'; import { searchLocalMemory } from './memory-search.js'; import { normalizeSummaryForFingerprint } from '../../shared/memory-fingerprint.js'; +import { MEMORY_DEFAULTS } from '../../shared/memory-defaults.js'; export const STARTUP_MEMORY_DURABLE_LIMIT = 7; export const STARTUP_MEMORY_RECENT_LIMIT = 8; export const STARTUP_MEMORY_TOTAL_LIMIT = 15; +export const STARTUP_MEMORY_STAGES = ['collect', 'prioritize', 'apply_quotas', 'trim', 'dedup', 'render'] as const; +export const STARTUP_BOOTSTRAP_SOURCES = [ + 'startup_memory', + 'preferences', + 'project_context', + 'user_context', + 'skills', +] as const; +export type StartupMemoryStage = (typeof STARTUP_MEMORY_STAGES)[number]; +export type StartupBootstrapSource = (typeof STARTUP_BOOTSTRAP_SOURCES)[number]; +export type StartupMemorySource = 'pinned' | 'durable' | 'recent' | 'project_docs' | 'preference' | 'user_context' | 'skill'; + +export interface StartupMemoryCandidate { + id: string; + source: StartupMemorySource; + text: string; + updatedAt?: number; + estimatedTokens?: number; + fingerprint?: string; +} + +export interface StartupMemoryPolicy { + totalTokens?: number; + pinnedTokens?: number; + durableTokens?: number; + recentTokens?: number; + projectDocsTokens?: number; + skillTokens?: number; +} + +export interface StartupMemorySelectionReport { + stages: readonly StartupMemoryStage[]; + bootstrapSources: readonly StartupBootstrapSource[]; + selected: StartupMemoryCandidate[]; + dropped: Array<{ id: string; source: StartupMemorySource; reason: 'duplicate' | 'source_quota' | 'total_budget' }>; + usedTokens: number; +} export interface StartupMemorySelectionOptions { durableLimit?: number; @@ -13,6 +51,133 @@ export interface StartupMemorySelectionOptions { totalLimit?: number; } +function tokenEstimate(candidate: StartupMemoryCandidate): number { + return Math.max(0, Math.ceil(candidate.estimatedTokens ?? Math.max(1, candidate.text.length / 4))); +} + +function candidateFingerprint(candidate: StartupMemoryCandidate): string { + return candidate.fingerprint ?? `${candidate.source}\u0000${normalizeSummaryForFingerprint(candidate.text)}`; +} + +function quotaForSource(policy: Required, source: StartupMemorySource): number { + switch (source) { + case 'pinned': return policy.pinnedTokens; + case 'durable': return policy.durableTokens; + case 'recent': return policy.recentTokens; + case 'project_docs': return policy.projectDocsTokens; + case 'preference': return policy.skillTokens; + case 'user_context': return policy.durableTokens; + case 'skill': return policy.skillTokens; + } +} + +const SOURCE_PRIORITY: Record = { + pinned: 0, + skill: 1, + preference: 2, + user_context: 3, + durable: 4, + project_docs: 5, + recent: 6, +}; + +function normalizeStartupPolicy(policy: StartupMemoryPolicy = {}): Required { + return { + totalTokens: policy.totalTokens ?? MEMORY_DEFAULTS.startupTotalTokens, + pinnedTokens: policy.pinnedTokens ?? MEMORY_DEFAULTS.pinnedTokens, + durableTokens: policy.durableTokens ?? MEMORY_DEFAULTS.durableTokens, + recentTokens: policy.recentTokens ?? MEMORY_DEFAULTS.recentTokens, + projectDocsTokens: policy.projectDocsTokens ?? MEMORY_DEFAULTS.projectDocsTokens, + skillTokens: policy.skillTokens ?? MEMORY_DEFAULTS.skillTokens, + }; +} + +export function selectStartupMemoryByPolicy( + candidates: readonly StartupMemoryCandidate[], + policyInput: StartupMemoryPolicy = {}, +): StartupMemorySelectionReport { + const policy = normalizeStartupPolicy(policyInput); + const dropped: StartupMemorySelectionReport['dropped'] = []; + const seen = new Set(); + const usedBySource = new Map(); + const selected: StartupMemoryCandidate[] = []; + let usedTokens = 0; + + const prioritized = [...candidates].sort((a, b) => { + const priorityDiff = SOURCE_PRIORITY[a.source] - SOURCE_PRIORITY[b.source]; + if (priorityDiff !== 0) return priorityDiff; + return (b.updatedAt ?? 0) - (a.updatedAt ?? 0); + }); + + for (const candidate of prioritized) { + const fingerprint = candidateFingerprint(candidate); + if (seen.has(fingerprint)) { + dropped.push({ id: candidate.id, source: candidate.source, reason: 'duplicate' }); + continue; + } + const tokens = tokenEstimate(candidate); + const sourceUsed = usedBySource.get(candidate.source) ?? 0; + if (sourceUsed + tokens > quotaForSource(policy, candidate.source)) { + dropped.push({ id: candidate.id, source: candidate.source, reason: 'source_quota' }); + continue; + } + if (usedTokens + tokens > policy.totalTokens) { + dropped.push({ id: candidate.id, source: candidate.source, reason: 'total_budget' }); + continue; + } + seen.add(fingerprint); + usedBySource.set(candidate.source, sourceUsed + tokens); + usedTokens += tokens; + selected.push(candidate); + } + + return { + stages: STARTUP_MEMORY_STAGES, + bootstrapSources: STARTUP_BOOTSTRAP_SOURCES, + selected, + dropped, + usedTokens, + }; +} + +export interface StartupBootstrapInput { + pinned?: readonly Omit[]; + durable?: readonly Omit[]; + recent?: readonly Omit[]; + projectContext?: readonly Omit[]; + userContext?: readonly Omit[]; + preferences?: readonly Omit[]; + skills?: readonly Omit[]; +} + +function tagStartupCandidates( + source: StartupMemorySource, + candidates: readonly Omit[] | undefined, +): StartupMemoryCandidate[] { + return (candidates ?? []).map((candidate) => ({ ...candidate, source })); +} + +/** + * Unified Wave 4/5 bootstrap entry point. It keeps preferences, user context, + * project docs, current startup memory, and future skills on the same named + * collect→prioritize→quota→trim→dedup→render path, so adding a source cannot + * bypass budget or duplicate handling. + */ +export function buildStartupBootstrapSelection( + input: StartupBootstrapInput, + policyInput: StartupMemoryPolicy = {}, +): StartupMemorySelectionReport { + return selectStartupMemoryByPolicy([ + ...tagStartupCandidates('pinned', input.pinned), + ...tagStartupCandidates('durable', input.durable), + ...tagStartupCandidates('recent', input.recent), + ...tagStartupCandidates('project_docs', input.projectContext), + ...tagStartupCandidates('user_context', input.userContext), + ...tagStartupCandidates('preference', input.preferences), + ...tagStartupCandidates('skill', input.skills), + ], policyInput); +} + export function selectStartupMemoryItems( namespace: ContextNamespace, options: StartupMemorySelectionOptions = {}, diff --git a/src/context/summary-compressor.ts b/src/context/summary-compressor.ts index 2a00ac955..8817af11e 100644 --- a/src/context/summary-compressor.ts +++ b/src/context/summary-compressor.ts @@ -51,6 +51,22 @@ export interface CompressionResult { backend: string; usedBackup: boolean; fromSdk: boolean; + /** Token telemetry — populated for every call so callers can persist a + * run row for later cost / efficiency analysis. `inputTokens` is the + * prompt size (including previous summary + serialized events). + * `outputTokens` is the produced summary size. `targetTokens` is the + * budget that was passed to the LLM. All counted via `countTokens`. */ + inputTokens: number; + outputTokens: number; + targetTokens: number; + /** Wall-clock duration of the SDK call (or local-fallback path). */ + durationMs: number; + /** Classified error code when `fromSdk: false` and the path was an + * exception (not a "no events" early return). One of the + * `CompressionErrorClassification.code` values. Undefined on success. */ + errorCode?: 'auth' | 'model' | 'session' | 'quota' | 'timeout' | 'empty_response' | 'agent_missing' | 'transient'; + /** Truncated last error message when the SDK path failed. */ + errorMessage?: string; } export type CompressionAdmissionReason = 'shutdown' | 'upgrade-pending' | 'test-reset'; @@ -157,7 +173,7 @@ const RETRY_MAX_DELAY_MS = 8000; /** Classify error as retryable (transient) or permanent. */ interface CompressionErrorClassification { retryable: boolean; - code: 'auth' | 'model' | 'session' | 'quota' | 'timeout' | 'empty_response' | 'transient'; + code: 'auth' | 'model' | 'session' | 'quota' | 'timeout' | 'empty_response' | 'agent_missing' | 'transient'; } function classifyCompressionError(err: unknown): CompressionErrorClassification { @@ -179,6 +195,41 @@ function classifyCompressionError(err: unknown): CompressionErrorClassification ) { return { retryable: false, code: 'quota' }; } + // Permanent: agent CLI binary missing on the host. + // + // Real-world hit: 213 (big@172.16.253.213) ran imcodes daemon configured with + // `claude-code-sdk` as the primary compression backend, but had no `claude` + // CLI installed. The Anthropic Claude Agent SDK threw + // "Claude Code native binary not found at claude. Please ensure + // Claude Code is installed via native installer..." + // Without classification, this fell through to "transient" and the retry + // loop kept the compression lane busy. Each retry was 1-8s × 3 = up to 27s + // of wasted active time per call, with the next materialization tick (10s) + // queuing another. Net effect: `getCompressionQueueState().idle` was never + // true, so the daemon-upgrade gate `if (!compressionState.idle) block` kept + // the daemon stuck on an old version that ALSO had a phantom-SIGTERM bug, + // producing a 13-second restart loop and a 4-hour outage. + // + // Patterns covered: + // - `Claude Code native binary not found at claude` (Anthropic SDK raw) + // - `Codex binary not found` / `Cursor binary not found` (imcodes provider + // wrappers — `src/agent/providers/{codex-sdk,cursor-headless}.ts`) + // - `spawn ENOENT` (Node.js child_process default) + // - `command not found` (shell-level) + // - PROVIDER_NOT_FOUND error code (imcodes shared error taxonomy) + // Permanent means: no retry, immediate circuit-breaker tick, fall through to + // backup backend → local fallback. Compression lane releases in <50 ms instead + // of holding for tens of seconds. + if ( + msg.includes('native binary not found') + || msg.includes('binary not found at') + || /\bbinary not found\b/.test(msg) + || /\bspawn\s+\S+\s+enoent\b/.test(msg) + || msg.includes('command not found') + || msg.includes('provider_not_found') + ) { + return { retryable: false, code: 'agent_missing' }; + } if (msg.includes('timed out') || msg.includes('timeout')) return { retryable: true, code: 'timeout' }; if (msg.includes('empty response')) return { retryable: true, code: 'empty_response' }; // Everything else (network, 5xx, transient provider failure) is retryable. @@ -206,7 +257,20 @@ async function sendWithRetry(prompt: string, selection: CompressionBackendSelect return await sendToProvider(selection, prompt); } catch (err) { lastErr = err; - if (!isRetryableError(err) || attempt === MAX_RETRIES_PER_BACKEND) { + const classification = classifyCompressionError(err); + if (!classification.retryable || attempt === MAX_RETRIES_PER_BACKEND) { + // Surface "agent CLI missing" loudly + actionably the FIRST time it + // happens — operators usually don't notice the per-call warn until + // hours later when something else (daemon-upgrade stall, drained + // memory recall) makes the missing CLI a visible problem. + if (classification.code === 'agent_missing') { + warnOncePerHour('mem.compression.agent_missing', { + backend: selection.backend, + error: err instanceof Error ? err.message : String(err), + hint: agentInstallHint(selection.backend), + }); + incrementCounter('mem.compression.agent_missing', { backend: selection.backend }); + } throw err; } // Tear down and retry with fresh provider @@ -220,6 +284,22 @@ async function sendWithRetry(prompt: string, selection: CompressionBackendSelect throw lastErr; } +/** Best-effort install hint per backend so operators have an actionable next step. */ +function agentInstallHint(backend: string): string { + switch (backend) { + case 'claude-code-sdk': + return 'install Claude Code CLI: npm install -g @anthropic-ai/claude-code'; + case 'codex-sdk': + return 'install Codex CLI (see codex.openai.com docs)'; + case 'qwen': + return 'install qwen CLI on PATH'; + case 'gemini-sdk': + return 'install Gemini CLI on PATH'; + default: + return `install the ${backend} CLI on PATH or configure a different primaryContextBackend`; + } +} + export function resetFailureTracking(): void { breakers.clear(); } @@ -340,16 +420,21 @@ const COMPRESSOR_SYSTEM_PROMPT = `You are a memory compression engine. Your outp export async function localOnlyCompressor(input: CompressionInput): Promise { // Tests expect local-only compression to behave as if SDK succeeded (fromSdk=true) // so the coordinator commits the result instead of entering retry mode. + const summary = ensurePinnedNotesSection( + buildLocalFallbackSummary(input.events, input.previousSummary), + input.pinnedNotes ?? [], + input.extraRedactPatterns ?? [], + ); return { - summary: ensurePinnedNotesSection( - buildLocalFallbackSummary(input.events, input.previousSummary), - input.pinnedNotes ?? [], - input.extraRedactPatterns ?? [], - ), + summary, model: 'local-only-test', backend: 'local', usedBackup: false, fromSdk: true, + inputTokens: 0, + outputTokens: countTokens(summary), + targetTokens: input.targetTokens ?? 0, + durationMs: 0, }; } @@ -488,6 +573,7 @@ function trimPreviousSummary(previousSummary: string | undefined, maxTokens = DE async function compressWithSdkInner(input: CompressionInput): Promise { const { events, modelConfig } = input; + const startedAt = Date.now(); const extraRedactPatterns = input.extraRedactPatterns ?? []; const previousSummary = trimPreviousSummary( input.previousSummary ? redactSummaryPreservingPinned(input.previousSummary, extraRedactPatterns) : undefined, @@ -495,9 +581,14 @@ async function compressWithSdkInner(input: CompressionInput): Promise 0 + ? pl.info.model_context_window + : undefined; + const contextWindow = resolveContextWindow( + modelContextWindow, + model, + 1_000_000, + { preferExplicit: modelContextWindow !== undefined }, + ); + const contextWindowSource = modelContextWindow !== undefined && contextWindow === modelContextWindow + ? USAGE_CONTEXT_WINDOW_SOURCES.PROVIDER + : undefined; timelineEmitter.emit(sessionName, 'usage.update', { - inputTokens: last.input_tokens, - cacheTokens: last.cached_input_tokens ?? 0, - outputTokens: last.output_tokens ?? 0, - contextWindow: resolveContextWindow(pl.info.model_context_window, model), + inputTokens: Math.max(0, usage.input_tokens - cachedInput), + cacheTokens: cachedInput, + outputTokens: usage.output_tokens ?? 0, + contextWindow, + ...(contextWindowSource ? { contextWindowSource } : {}), ...(model ? { model } : {}), }, { source: 'daemon', confidence: 'high', ...(ts ? { ts } : {}) }); } diff --git a/src/daemon/command-handler.ts b/src/daemon/command-handler.ts index f0541e36f..fef4bbaa4 100644 --- a/src/daemon/command-handler.ts +++ b/src/daemon/command-handler.ts @@ -27,7 +27,7 @@ import logger from '../util/logger.js'; import { getDefaultAckOutbox } from './ack-outbox.js'; import { COMMAND_ACK_ERROR_DUPLICATE_COMMAND_ID, MSG_COMMAND_ACK } from '../../shared/ack-protocol.js'; import { homedir } from 'os'; -import { readdir as fsReaddir, realpath as fsRealpath, readFile as fsReadFileRaw, stat as fsStat, writeFile as fsWriteFile } from 'node:fs/promises'; +import { lstat as fsLstat, readdir as fsReaddir, realpath as fsRealpath, readFile as fsReadFileRaw, stat as fsStat, unlink as fsUnlink, writeFile as fsWriteFile } from 'node:fs/promises'; import * as nodePath from 'node:path'; import { exec as execCb, execFile as execFileCb } from 'node:child_process'; import { promisify } from 'node:util'; @@ -44,9 +44,14 @@ import { TRANSPORT_MSG } from '../../shared/transport-events.js'; import { copyFile } from 'node:fs/promises'; import { randomUUID } from 'node:crypto'; import { ensureImcDir, imcSubDir } from '../util/imc-dir.js'; -import { buildWindowsCleanupScript, buildWindowsCleanupVbs, buildWindowsUpgradeBatch, buildWindowsUpgradeVbs } from '../util/windows-upgrade-script.js'; +import { + buildWindowsCleanupScript, + buildWindowsCleanupVbs, + buildWindowsUpgradeRunnerVbs, + resolveWindowsUpgradeRunnerPath, +} from '../util/windows-upgrade-script.js'; import { buildBashSharpRepair } from '../util/sharp-repair-script.js'; -import { UPGRADE_LOCK_FILE, encodeVbsAsUtf16, encodeCmdAsUtf8Bom } from '../util/windows-launch-artifacts.js'; +import { encodeVbsAsUtf16, encodeCmdAsUtf8Bom } from '../util/windows-launch-artifacts.js'; import { registerTempFile, removeTrackedTempFile } from '../store/temp-file-store.js'; import { sanitizeProjectName } from '../../shared/sanitize-project-name.js'; import { isTemplatePrompt, isTemplateOriginSummary, isImperativeCommand } from '../../shared/template-prompt-patterns.js'; @@ -62,9 +67,12 @@ import { getCodexRuntimeConfig } from '../agent/codex-runtime-config.js'; import { mergeCodexDisplayMetadata } from '../agent/codex-display.js'; import { P2P_TERMINAL_RUN_STATUSES } from '../../shared/p2p-status.js'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { DAEMON_UPGRADE_TARGET_LATEST, normalizeDaemonUpgradeTargetVersion } from '../../shared/daemon-upgrade.js'; import { CC_PRESET_MSG, type CcPreset } from '../../shared/cc-presets.js'; import { MEMORY_WS } from '../../shared/memory-ws.js'; +import { FS_WRITE_ERROR } from '../shared/transport/fs.js'; import { P2P_CONFIG_ERROR, P2P_CONFIG_MSG, MAX_P2P_PARTICIPANTS } from '../../shared/p2p-config-events.js'; +import { p2pScopedSessionKey } from '../../shared/p2p-config-scope.js'; import { DAEMON_COMMAND_TYPES } from '../../shared/daemon-command-types.js'; import { CLAUDE_SDK_EFFORT_LEVELS, @@ -77,7 +85,25 @@ import { type TransportEffortLevel, } from '../../shared/effort-levels.js'; import { getSavedP2pConfig, upsertSavedP2pConfig } from '../store/p2p-config-store.js'; -import { getProcessedProjectionStats, queryPendingContextEvents, queryProcessedProjections, recordMemoryHits } from '../store/context-store.js'; +import { + deleteContextObservation, + ensureContextNamespace, + getProcessedProjectionStats, + getProcessedProjectionById, + listMemoryProjectSummaries, + listContextNamespaces, + listContextObservations, + queryPendingContextEvents, + promoteContextObservation, + queryProcessedProjections, + recordMemoryHits, + updateProcessedProjectionSummary, + upsertPinnedNote, + updateContextObservationText, + writeContextObservation, + writeProcessedProjection, +} from '../store/context-store.js'; +import { serializeContextNamespace } from '../context/context-keys.js'; import { isKnownTestProjectName, isKnownTestSessionName, @@ -91,22 +117,263 @@ import { getContextModelConfig } from '../context/context-model-config.js'; import { getCompressionQueueState, resumeAcceptingCompression, stopAcceptingCompression } from '../context/summary-compressor.js'; import { closeLiveContextMaterializationAdmission, reopenLiveContextMaterializationAdmission } from '../context/live-context-ingestion.js'; import { getInflightMasterCompactionCount, resumeAcceptingMasterCompactions, stopAcceptingMasterCompactions } from './master-compaction-registry.js'; -import { detectRepo } from '../repo/detector.js'; +import { detectRepo, parseRemotes } from '../repo/detector.js'; import { GitOriginRepositoryIdentityService } from '../agent/repository-identity-service.js'; import { SUPERVISION_MODE, extractSessionSupervisionSnapshot, isSupportedSupervisionTargetSessionType, } from '../../shared/supervision-config.js'; +import { + PREFERENCE_FEATURE_FLAG, + PREFERENCE_INGEST_OBSERVATION_CLASS, + PREFERENCE_INGEST_OBSERVATION_STATE, + PREFERENCE_INGEST_ORIGIN, + PREFERENCE_INGEST_SCOPE, + PREFERENCE_IDEMPOTENCY_PREFIX, + prependPreferenceProviderContext, + processPreferenceLines, + renderPreferenceProviderContext, + type PreferenceIngestRecord, + type PreferenceProviderContextRecord, +} from '../../shared/preference-ingest.js'; +import { normalizeSendOrigin, type SendOrigin } from '../../shared/send-origin.js'; +import { + getMemoryFeatureFlagDefinition, + computeEffectiveMemoryFeatureFlags, + isMemoryFeatureFlag, + MEMORY_FEATURE_CONFIG_MSG, + MEMORY_FEATURE_FLAGS, + MEMORY_FEATURE_FLAGS_BY_NAME, + memoryFeatureFlagEnvKey, + resolveMemoryFeatureFlagValue, + sanitizeMemoryFeatureFlagValues, + type FeatureFlagValueSource, + type MemoryFeatureFlagValues, + type MemoryFeatureFlag, + type MemoryFeatureFlagResolutionLayers, +} from '../../shared/feature-flags.js'; +import { incrementCounter } from '../util/metrics.js'; +import { computeMemoryFingerprint } from '../../shared/memory-fingerprint.js'; +import { isMemoryScope, isOwnerPrivateMemoryScope, isSharedProjectionScope, type MemoryScope } from '../../shared/memory-scope.js'; +import { isObservationClass } from '../../shared/memory-observation.js'; +import { SKILL_MAX_BYTES } from '../../shared/skill-envelope.js'; +import { MD_INGEST_FEATURE_FLAG } from '../../shared/md-ingest.js'; +import { MEMORY_MANAGEMENT_ERROR_CODES, type MemoryManagementErrorCode } from '../../shared/memory-management.js'; +import type { MemoryProjectResolutionStatus } from '../../shared/memory-project-options.js'; +import { + MEMORY_MANAGEMENT_CONTEXT_FIELD, + isAuthenticatedMemoryManagementContext, + type AuthenticatedMemoryManagementContext, + type MemoryManagementBoundProject, +} from '../../shared/memory-management-context.js'; +import { + getSessionControlTimelineFeedbackById, + isDaemonHandledSessionControlSend, + isSessionControlCommandText, + shouldHideTimelineUserMessageForSessionControl, + shouldResetProcessPreferenceContextForSessionControl, +} from '../../shared/session-control-commands.js'; +import type { ContextMemoryStatsView, ContextNamespace } from '../../shared/context-types.js'; +import { publishRuntimeMemoryCacheInvalidation } from '../context/runtime-memory-cache-bus.js'; +import { assertManagedSkillPathSync, ManagedSkillPathError } from '../context/managed-skill-path.js'; +import { + getMemoryFeatureConfigStoreDiagnostics, + getPersistedMemoryFeatureFlagValues, + getRuntimeMemoryFeatureFlagValues, + setPersistedMemoryFeatureFlagValues, + setRuntimeMemoryFeatureFlagValues, +} from '../store/memory-feature-config-store.js'; const MAX_P2P_FILE_PULL_COUNT = 20; const processRecallRepositoryIdentityService = new GitOriginRepositoryIdentityService(); +const DAEMON_LOCAL_PREFERENCE_USER_ID = 'daemon-local'; function isEligibleSupervisionTaskText(text: string): boolean { const trimmed = text.trim(); return trimmed.length > 0 && !trimmed.startsWith('/'); } +function readBooleanEnv(value: string | undefined): boolean | undefined { + if (value == null) return undefined; + return value === 'true' || value === '1'; +} + +function isMemoryFeatureEnabled(flag: MemoryFeatureFlag): boolean { + return getEffectiveMemoryFeatureFlags()[flag]; +} + +function readMemoryFeatureEnvironmentDefaults(): MemoryFeatureFlagValues { + const environmentStartupDefault: MemoryFeatureFlagValues = {}; + for (const flag of MEMORY_FEATURE_FLAGS) { + const envValue = readBooleanEnv(process.env[memoryFeatureFlagEnvKey(flag)]); + if (envValue !== undefined) environmentStartupDefault[flag] = envValue; + } + return environmentStartupDefault; +} + +function readMemoryFeatureResolutionLayers(): MemoryFeatureFlagResolutionLayers { + const persistedConfig = getPersistedMemoryFeatureFlagValues(); + return { + runtimeConfigOverride: getRuntimeMemoryFeatureFlagValues(), + persistedConfig, + environmentStartupDefault: readMemoryFeatureEnvironmentDefaults(), + readFailed: !!getMemoryFeatureConfigStoreDiagnostics().lastLoadIssue, + }; +} + +function readRequestedMemoryFeatureFlags(layers: MemoryFeatureFlagResolutionLayers = readMemoryFeatureResolutionLayers()): MemoryFeatureFlagValues { + const requested: MemoryFeatureFlagValues = {}; + for (const flag of MEMORY_FEATURE_FLAGS) { + requested[flag] = resolveMemoryFeatureFlagValue(flag, layers); + } + return requested; +} + +function getEffectiveMemoryFeatureFlags(): Record { + return computeEffectiveMemoryFeatureFlags(readRequestedMemoryFeatureFlags()); +} + +function featureFlagValueSource(flag: MemoryFeatureFlag, layers: MemoryFeatureFlagResolutionLayers): FeatureFlagValueSource { + if (layers.runtimeConfigOverride?.[flag] !== undefined) return 'runtime_config_override'; + if (layers.persistedConfig?.[flag] !== undefined) return 'persisted_config'; + if (layers.environmentStartupDefault?.[flag] !== undefined) return 'environment_startup_default'; + return 'registry_default'; +} + +function isPreferenceFeatureEnabled(): boolean { + return isMemoryFeatureEnabled(PREFERENCE_FEATURE_FLAG); +} + +function preferenceUserIdForSend(cmd: Record, record: SessionRecord | null | undefined): string { + const fromCommand = typeof cmd.userId === 'string' ? cmd.userId.trim() : ''; + if (fromCommand) return fromCommand; + const fromNamespace = record?.contextNamespace?.userId?.trim(); + return fromNamespace || DAEMON_LOCAL_PREFERENCE_USER_ID; +} + +const processPreferenceContextSignatures = new Map(); + +function normalizePreferenceProviderContextSignature(context: string): string { + return context.replace(/\s+/g, ' ').trim(); +} + +function prepareProcessPreferenceProviderText(input: { + sessionName: string; + providerText: string; + preferenceContext: string; +}): string { + const context = input.preferenceContext.trim(); + if (!context) return input.providerText; + const trimmedText = input.providerText.trim(); + if (trimmedText.startsWith('/')) { + if (shouldResetProcessPreferenceContextForSessionControl(trimmedText)) { + processPreferenceContextSignatures.delete(input.sessionName); + } + return input.providerText; + } + const signature = normalizePreferenceProviderContextSignature(context); + if (!signature) return input.providerText; + if (processPreferenceContextSignatures.get(input.sessionName) === signature) { + return input.providerText; + } + processPreferenceContextSignatures.set(input.sessionName, signature); + return prependPreferenceProviderContext(input.providerText, context); +} + +function loadPreferenceProviderContext(input: { + enabled: boolean; + userId: string; + currentRecords: readonly PreferenceIngestRecord[]; +}): string { + if (!input.enabled) return ''; + const records: PreferenceProviderContextRecord[] = input.currentRecords.map((record) => ({ + text: record.text, + fingerprint: record.fingerprint, + })); + const scopeKey = `${PREFERENCE_INGEST_SCOPE}:${input.userId}`; + const idempotencyPrefix = [ + PREFERENCE_IDEMPOTENCY_PREFIX, + input.userId, + scopeKey, + '', + ].join('\u0000'); + try { + for (const observation of listContextObservations({ + scope: PREFERENCE_INGEST_SCOPE, + class: PREFERENCE_INGEST_OBSERVATION_CLASS, + })) { + if (observation.state !== PREFERENCE_INGEST_OBSERVATION_STATE) continue; + const preferenceText = typeof observation.content.text === 'string' + ? observation.content.text + : ''; + if (!preferenceText.trim()) continue; + const idempotencyKey = typeof observation.content.idempotencyKey === 'string' + ? observation.content.idempotencyKey + : ''; + if (!idempotencyKey.startsWith(idempotencyPrefix)) continue; + records.push({ + text: preferenceText, + fingerprint: observation.fingerprint, + updatedAt: observation.updatedAt, + }); + } + } catch (err) { + logger.warn({ err, userId: input.userId }, 'failed to load preference context for provider dispatch'); + } + return renderPreferenceProviderContext(records); +} + +function schedulePreferencePersistence(input: { + userId: string; + commandId: string; + records: readonly PreferenceIngestRecord[]; + sendOrigin: SendOrigin; +}): void { + if (input.records.length === 0) return; + setTimeout(() => { + try { + const namespace = ensureContextNamespace({ + scope: PREFERENCE_INGEST_SCOPE, + userId: input.userId, + name: 'preferences', + }); + for (const record of input.records) { + const alreadyPersisted = listContextObservations({ + namespaceId: namespace.id, + class: PREFERENCE_INGEST_OBSERVATION_CLASS, + }).some((observation) => ( + observation.fingerprint === record.fingerprint + && observation.content.idempotencyKey === record.idempotencyKey + )); + writeContextObservation({ + namespaceId: namespace.id, + scope: PREFERENCE_INGEST_SCOPE, + class: PREFERENCE_INGEST_OBSERVATION_CLASS, + origin: PREFERENCE_INGEST_ORIGIN, + fingerprint: record.fingerprint, + content: { + text: record.text, + ownerUserId: input.userId, + createdByUserId: input.userId, + updatedByUserId: input.userId, + idempotencyKey: record.idempotencyKey, + }, + text: record.text, + sourceEventIds: [input.commandId], + state: PREFERENCE_INGEST_OBSERVATION_STATE, + }); + incrementCounter(alreadyPersisted ? 'mem.preferences.duplicate_ignored' : 'mem.preferences.persisted', { + sendOrigin: input.sendOrigin, + }); + } + } catch (err) { + incrementCounter('mem.preferences.persistence_failed', { source: 'schedulePreferencePersistence' }); + logger.warn({ err }, 'preference ingest persistence failed after send receipt'); + } + }, 0); +} + /** * Reliable `command.ack` emission — enqueue into the on-disk outbox BEFORE the * network send so that a transient serverLink outage doesn't silently drop the @@ -343,15 +610,12 @@ function supportsTransportClear(agentType: string | undefined): agentType is 'cl || agentType === 'qwen'; } -// Note: an earlier `supportsTransportCompact` helper synthesized `/compact` -// behaviour daemon-side by replaying transport history, calling the memory -// compressor, and relaunching a fresh conversation. That was rolled back — -// transport SDKs (claude-code-sdk and friends) accept the literal `/compact` -// text and run their own native compaction, which is materially better than -// a daemon-side fresh relaunch (preserves SDK tool config, system prompts, -// and resume identity). The daemon's automatic materialization pipeline -// continues to record raw events into `context_event_archive` regardless, -// so no provenance is lost when the SDK compacts. +// `/compact` is provider-dispatched, not daemon-synthesized. Provider adapters +// that expose a compact RPC translate the raw command at the SDK boundary; +// verified slash-command providers receive the literal command; unsupported +// providers fail visibly. The daemon's automatic materialization pipeline still +// records raw events into `context_event_archive`, so provenance is preserved +// independently of provider-side compaction. function supportsProcessClear(agentType: string | undefined): agentType is 'claude-code' | 'codex' | 'opencode' { return agentType === 'claude-code' || agentType === 'codex' || agentType === 'opencode'; @@ -492,7 +756,16 @@ async function rewritePathsForSandbox(sessionName: string, text: string): Promis return result; } import { handleRepoCommand } from './repo-handler.js'; -import { handleFileUpload, handleFileDownload, createProjectFileHandle, lookupAttachment } from './file-transfer-handler.js'; +import { + handleFileUpload, + handleFileDownload, + tryCreateProjectFileHandle, + lookupAttachment, +} from './file-transfer-handler.js'; +import { getDefaultPreviewReadCoordinator, __resetPreviewReadCoordinatorForTests } from './file-preview-read-coordinator.js'; +import { isFilePreviewPathAllowed, resolveCanonical } from './file-preview-path-policy.js'; +import { FS_GENERIC_ERROR_CODES } from '../../shared/fs-error-codes.js'; +import { FS_READ_ERROR_CODES } from '../../shared/fs-read-error-codes.js'; import { REPO_MSG } from '../shared/repo-types.js'; import { handlePreviewCommand } from './preview-relay.js'; import { PREVIEW_MSG } from '../../shared/preview-types.js'; @@ -670,10 +943,26 @@ function resolveP2pConfigScopeSession(sessionName: string): string { return record?.parentSession ?? sessionName; } -async function resolveStructuredP2pSessionConfig(sessionName: string, clientConfig?: P2pSessionConfig): Promise { +function getP2pConfigStoreScope(serverLink: ServerLink, scopeSession: string): string { + const serverId = typeof (serverLink as unknown as { getServerId?: () => string }).getServerId === 'function' + ? (serverLink as unknown as { getServerId: () => string }).getServerId() + : undefined; + return p2pScopedSessionKey(scopeSession, serverId); +} + +async function resolveStructuredP2pSessionConfig( + sessionName: string, + serverLink: ServerLink, + clientConfig?: P2pSessionConfig, +): Promise { const scopeSession = resolveP2pConfigScopeSession(sessionName); - const saved = await getSavedP2pConfig(scopeSession); + const storeScope = getP2pConfigStoreScope(serverLink, scopeSession); + const saved = await getSavedP2pConfig(storeScope); if (saved?.sessions && typeof saved.sessions === 'object') return saved.sessions; + if (storeScope !== scopeSession) { + const legacySaved = await getSavedP2pConfig(scopeSession); + if (legacySaved?.sessions && typeof legacySaved.sessions === 'object') return legacySaved.sessions; + } return clientConfig; } @@ -882,9 +1171,45 @@ export function setRouterContext(ctx: RouterContext): void { } export function handleWebCommand(msg: unknown, serverLink: ServerLink): void { - if (!msg || typeof msg !== 'object') return; + // Input validation: anything that isn't a non-null object goes + // straight to the floor. We log a debug ping for arrays / primitives + // so a confused client gets diagnostic feedback without flooding. + if (!msg || typeof msg !== 'object' || Array.isArray(msg)) { + if (msg !== null && msg !== undefined) { + logger.debug({ kind: typeof msg, isArray: Array.isArray(msg) }, 'Ignoring non-object web command'); + } + return; + } const cmd = msg as Record; + // Top-level isolation: any synchronous throw inside a handler — e.g. + // a TypeError from `cmd.foo.bar` when `foo` is undefined, or a + // validation throw before the first await of an async function — + // would otherwise propagate out of the WebSocket onMessage callback + // and trip the global uncaughtException handler. That handler keeps + // the daemon alive but emits a noisy "UNCAUGHT EXCEPTION" line and + // broadcasts a daemon.error event to every connected browser, so to + // operators the daemon LOOKED crashed. Wrap the dispatch so a bad + // single command can't destabilize the whole connection. + // + // Note on async rejections: handlers in the switch use the + // `void handleX(...)` pattern so promise rejections propagate to + // process.on('unhandledRejection') in src/index.ts, which logs and + // forwards a daemon.error event but keeps the process alive. The + // rare-but-real "throw before first await" case STILL surfaces to + // browsers, but the daemon does not crash. Individual handlers + // already do their own try/catch where input validation matters. + try { + dispatchWebCommand(cmd, serverLink); + } catch (err) { + logger.warn( + { err, type: typeof cmd.type === 'string' ? cmd.type : '' }, + 'Web command handler threw synchronously — daemon stays alive', + ); + } +} + +function dispatchWebCommand(cmd: Record, serverLink: ServerLink): void { switch (cmd.type) { case 'inbound': void handleInbound(cmd); @@ -898,6 +1223,9 @@ export function handleWebCommand(msg: unknown, serverLink: ServerLink): void { case 'session.restart': void handleRestart(cmd, serverLink); break; + case DAEMON_COMMAND_TYPES.SESSION_CANCEL: + void handleSessionCancel(cmd, serverLink); + break; case DAEMON_COMMAND_TYPES.SESSION_UPDATE_TRANSPORT_CONFIG: void handleSessionTransportConfigUpdate(cmd, serverLink); break; @@ -1047,11 +1375,19 @@ export function handleWebCommand(msg: unknown, serverLink: ServerLink): void { case 'discussion.list': handleDiscussionList(serverLink); break; - case 'server.delete': + case DAEMON_COMMAND_TYPES.SERVER_DELETE: void handleServerDelete(); break; - case 'daemon.upgrade': - void handleDaemonUpgrade(cmd.targetVersion as string | undefined, serverLink); + case DAEMON_COMMAND_TYPES.DAEMON_UPGRADE: + try { + const normalizedTarget = normalizeDaemonUpgradeTargetVersion(cmd.targetVersion); + void handleDaemonUpgrade( + normalizedTarget === DAEMON_UPGRADE_TARGET_LATEST ? undefined : normalizedTarget, + serverLink, + ); + } catch { + logger.warn({ targetVersion: cmd.targetVersion }, 'daemon.upgrade rejected invalid targetVersion'); + } break; case 'file.search': void handleFileSearch(cmd, serverLink); @@ -1065,6 +1401,15 @@ export function handleWebCommand(msg: unknown, serverLink: ServerLink): void { case MEMORY_WS.RESTORE: void handleMemoryRestore(cmd, serverLink); break; + case MEMORY_WS.CREATE: + void handleMemoryCreate(cmd, serverLink); + break; + case MEMORY_WS.UPDATE: + void handleMemoryUpdate(cmd, serverLink); + break; + case MEMORY_WS.PIN: + void handleMemoryPin(cmd, serverLink); + break; case MEMORY_WS.DELETE: void handleMemoryDelete(cmd, serverLink); break; @@ -1104,9 +1449,60 @@ export function handleWebCommand(msg: unknown, serverLink: ServerLink): void { case SHARED_CONTEXT_RUNTIME_CONFIG_MSG.APPLY: void handleSharedContextRuntimeConfigApply(cmd); break; + case MEMORY_FEATURE_CONFIG_MSG.APPLY: + handleMemoryFeatureConfigApply(cmd); + break; case MEMORY_WS.PERSONAL_QUERY: void handlePersonalMemoryQuery(cmd, serverLink); break; + case MEMORY_WS.PROJECT_RESOLVE: + void handleMemoryProjectResolve(cmd, serverLink); + break; + case MEMORY_WS.FEATURES_QUERY: + handleMemoryFeaturesQuery(cmd, serverLink); + break; + case MEMORY_WS.FEATURES_SET: + handleMemoryFeaturesSet(cmd, serverLink); + break; + case MEMORY_WS.PREF_QUERY: + void handleMemoryPreferencesQuery(cmd, serverLink); + break; + case MEMORY_WS.PREF_CREATE: + void handleMemoryPreferenceCreate(cmd, serverLink); + break; + case MEMORY_WS.PREF_UPDATE: + void handleMemoryPreferenceUpdate(cmd, serverLink); + break; + case MEMORY_WS.PREF_DELETE: + void handleMemoryPreferenceDelete(cmd, serverLink); + break; + case MEMORY_WS.SKILL_QUERY: + void handleMemorySkillsQuery(cmd, serverLink); + break; + case MEMORY_WS.SKILL_REBUILD: + void handleMemorySkillsRebuild(cmd, serverLink); + break; + case MEMORY_WS.SKILL_READ: + void handleMemorySkillRead(cmd, serverLink); + break; + case MEMORY_WS.SKILL_DELETE: + void handleMemorySkillDelete(cmd, serverLink); + break; + case MEMORY_WS.MD_INGEST_RUN: + void handleMemoryMarkdownIngestRun(cmd, serverLink); + break; + case MEMORY_WS.OBSERVATION_QUERY: + void handleMemoryObservationsQuery(cmd, serverLink); + break; + case MEMORY_WS.OBSERVATION_UPDATE: + void handleMemoryObservationUpdate(cmd, serverLink); + break; + case MEMORY_WS.OBSERVATION_DELETE: + void handleMemoryObservationDelete(cmd, serverLink); + break; + case MEMORY_WS.OBSERVATION_PROMOTE: + void handleMemoryObservationPromote(cmd, serverLink); + break; case 'file.upload': void handleFileUpload(cmd, serverLink); break; @@ -1184,7 +1580,7 @@ async function handleP2pConfigSave(cmd: Record, serverLink: Ser return; } try { - await upsertSavedP2pConfig(scopeSession, config); + await upsertSavedP2pConfig(getP2pConfigStoreScope(serverLink, scopeSession), config); if (requestId) { serverLink?.send({ type: P2P_CONFIG_MSG.SAVE_RESPONSE, @@ -1492,6 +1888,87 @@ async function handleStop(cmd: Record, serverLink: ServerLink): try { serverLink.send({ type: 'session.error', project, message: `Shutdown failed: ${message}` }); } catch { /* ignore */ } } +function resolveSessionCommandName(cmd: Record): string | undefined { + return (typeof cmd.sessionName === 'string' && cmd.sessionName) + ? cmd.sessionName + : (typeof cmd.session === 'string' && cmd.session ? cmd.session : undefined); +} + +function markTransportCancelIdle(sessionName: string, error?: string): void { + timelineEmitter.emit(sessionName, 'session.state', { + state: 'idle', + pendingCount: 0, + pendingMessages: [], + pendingMessageEntries: [], + ...(error ? { error } : {}), + }, { source: 'daemon', confidence: 'high' }); +} + +function emitSessionControlTimelineFeedback(sessionName: string, controlId: 'stop'): void { + const feedback = getSessionControlTimelineFeedbackById(controlId); + if (!feedback) return; + timelineEmitter.emit(sessionName, 'session.state', { + state: feedback.state, + reason: feedback.reason, + }, { source: 'daemon', confidence: 'high' }); +} + +function cancelTransportTurnNow( + sessionName: string, + commandId: string | undefined, + serverLink: Pick | undefined, +): boolean { + const stopRuntime = getTransportRuntime(sessionName); + const stopRecord = getSession(sessionName); + const isTransportStop = !!stopRuntime + || stopRecord?.runtimeType === 'transport' + || (typeof stopRecord?.agentType === 'string' && isTransportAgent(stopRecord.agentType)); + if (!isTransportStop) return false; + + clearResend(sessionName); + if (commandId) emitCommandAck(sessionName, commandId, 'accepted', undefined, serverLink); + emitSessionControlTimelineFeedback(sessionName, 'stop'); + markTransportCancelIdle(sessionName); + + if (!stopRuntime) return true; + + void (async () => { + try { + supervisionAutomation.cancelSession(sessionName); + await stopRuntime.cancel(); + // Mark session for fresh start so daemon restart doesn't resume the + // stuck conversation. + if (stopRecord?.agentType === 'qwen') { + upsertSession({ ...stopRecord, qwenFreshOnResume: true, updatedAt: Date.now() }); + } + } catch (err) { + const errMsg = describeTransportSendError(err); + logger.error({ sessionName, err }, 'session.cancel (transport) failed'); + timelineEmitter.emit(sessionName, 'assistant.text', { text: `⚠️ Stop failed: ${errMsg}`, streaming: false, memoryExcluded: true }, { source: 'daemon', confidence: 'high' }); + markTransportCancelIdle(sessionName, errMsg); + } + })(); + + return true; +} + +async function handleSessionCancel(cmd: Record, serverLink: ServerLink): Promise { + const sessionName = resolveSessionCommandName(cmd); + const commandId = typeof cmd.commandId === 'string' && cmd.commandId.trim() + ? cmd.commandId.trim() + : undefined; + if (!sessionName) { + logger.warn('session.cancel: missing sessionName'); + return; + } + + if (cancelTransportTurnNow(sessionName, commandId, serverLink)) return; + + const errMsg = 'Transport session unavailable'; + logger.warn({ sessionName }, 'session.cancel: session is not a transport session'); + if (commandId) emitCommandAck(sessionName, commandId, 'error', errMsg, serverLink); +} + /** * Send a command to a session, handling `!`-prefixed shell commands: * - claude-code: send `!` first (with delayed-Enter), then send the rest of the command @@ -1610,7 +2087,7 @@ async function handleSend(cmd: Record, serverLink: ServerLink): || text.includes('@@all(') || text.includes('@@p2p-config('); const isDaemonHandledControlSend = trimmedText === '/stop' - || trimmedText === '/clear' + || isDaemonHandledSessionControlSend(trimmedText) || /^\/model\s+\S+/.test(trimmedText) || /^\/(?:thinking|effort)\s+\S+/.test(trimmedText); // For ordinary user turns, command.ack is a daemon-receipt acknowledgement: @@ -1627,51 +2104,14 @@ async function handleSend(cmd: Record, serverLink: ServerLink): } if (trimmedText === '/stop') { - const stopRuntime = getTransportRuntime(sessionName); - const stopRecord = getSession(sessionName); - const isTransportStop = !!stopRuntime - || stopRecord?.runtimeType === 'transport' - || (typeof stopRecord?.agentType === 'string' && isTransportAgent(stopRecord.agentType)); - if (isTransportStop) { - emitTransportUserMessage(text); - // `/stop` is a priority control lane: receipt ack and queue clearing - // happen before any P2P config read, pending relaunch wait, transport - // mutex, context bootstrap, recall, or provider cancel await. The - // cancel itself runs in the background; failures surface as timeline - // state, not as a delayed command ack. - clearResend(sessionName); - emitAcceptedReceiptAck(); - if (!stopRuntime) { - timelineEmitter.emit(sessionName, 'session.state', { - state: 'idle', - pendingCount: 0, - pendingMessages: [], - pendingMessageEntries: [], - }, { source: 'daemon', confidence: 'high' }); - return; - } - void (async () => { - try { - supervisionAutomation.cancelSession(sessionName); - await stopRuntime.cancel(); - // Mark session for fresh start so daemon restart doesn't resume the - // stuck conversation. - if (stopRecord?.agentType === 'qwen') { - upsertSession({ ...stopRecord, qwenFreshOnResume: true, updatedAt: Date.now() }); - } - } catch (err) { - const errMsg = describeTransportSendError(err); - logger.error({ sessionName, err }, 'session.stop (transport) failed'); - timelineEmitter.emit(sessionName, 'assistant.text', { text: `⚠️ Stop failed: ${errMsg}`, streaming: false, memoryExcluded: true }, { source: 'daemon', confidence: 'high' }); - timelineEmitter.emit(sessionName, 'session.state', { state: 'idle', error: errMsg }, { source: 'daemon', confidence: 'high' }); - } - })(); + if (cancelTransportTurnNow(sessionName, effectiveId, serverLink)) { + receiptAcked = true; return; } } const p2pSessionConfig = wantsStructuredP2pRouting - ? await resolveStructuredP2pSessionConfig(sessionName, clientP2pSessionConfig) + ? await resolveStructuredP2pSessionConfig(sessionName, serverLink, clientP2pSessionConfig) : undefined; // ── P2P start gates (mandatory) ── @@ -1963,12 +2403,37 @@ async function handleSend(cmd: Record, serverLink: ServerLink): // Transport sessions — route directly to the provider runtime, bypassing tmux. const transportRuntime = getTransportRuntime(sessionName); const record = (await import('../store/session-store.js')).getSession(sessionName); + const preferenceUserId = preferenceUserIdForSend(cmd, record); + const preferenceFeatureEnabled = isPreferenceFeatureEnabled(); + const preferenceIngest = processPreferenceLines({ + text, + featureEnabled: preferenceFeatureEnabled, + sendOrigin: cmd.origin, + userId: preferenceUserId, + scopeKey: `${PREFERENCE_INGEST_SCOPE}:${preferenceUserId}`, + messageId: effectiveId, + }); + for (const event of preferenceIngest.telemetry) { + incrementCounter(event.counter, { sendOrigin: event.sendOrigin }); + } + const displayText = preferenceIngest.providerText; + const preferenceMessagePreamble = loadPreferenceProviderContext({ + enabled: preferenceFeatureEnabled, + userId: preferenceUserId, + currentRecords: preferenceIngest.records, + }); + schedulePreferencePersistence({ + userId: preferenceUserId, + commandId: effectiveId, + records: preferenceIngest.records, + sendOrigin: normalizeSendOrigin(cmd.origin), + }); const supervisionSnapshot = isSupportedSupervisionTargetSessionType(record?.agentType) ? extractSessionSupervisionSnapshot(record?.transportConfig ?? null) : null; const shouldTrackSupervisionTaskRun = supervisionSnapshot != null && supervisionSnapshot.mode !== SUPERVISION_MODE.OFF - && isEligibleSupervisionTaskText(text); + && isEligibleSupervisionTaskText(displayText); const attachments: TransportAttachment[] = []; const transportUserEventId = (clientMessageId: string) => `transport-user:${clientMessageId}`; const isTransportSession = record?.runtimeType === 'transport' @@ -1991,9 +2456,14 @@ async function handleSend(cmd: Record, serverLink: ServerLink): { sessionName, providerId: record.providerId, commandId: effectiveId }, 'session.send: transport session has no runtime — queuing for resend after reconnect', ); - enqueueResend(sessionName, { text, commandId: effectiveId, queuedAt: Date.now() }); + enqueueResend(sessionName, { + text: displayText, + ...(preferenceMessagePreamble ? { messagePreamble: preferenceMessagePreamble } : {}), + commandId: effectiveId, + queuedAt: Date.now(), + }); if (shouldTrackSupervisionTaskRun) { - supervisionAutomation.queueTaskIntent(sessionName, effectiveId, text, supervisionSnapshot); + supervisionAutomation.queueTaskIntent(sessionName, effectiveId, displayText, supervisionSnapshot); } const queued = getResendEntries(sessionName); const infoMsg = `⏳ Provider ${providerLabel} not connected yet — will resend ${queued.length} queued message${queued.length === 1 ? '' : 's'} once reconnected.`; @@ -2050,9 +2520,14 @@ async function handleSend(cmd: Record, serverLink: ServerLink): { sessionName, providerId: record?.providerId, commandId: effectiveId }, 'session.send: transport runtime missing provider session id — queuing and auto-resuming', ); - enqueueResend(sessionName, { text, commandId: effectiveId, queuedAt: Date.now() }); + enqueueResend(sessionName, { + text: displayText, + ...(preferenceMessagePreamble ? { messagePreamble: preferenceMessagePreamble } : {}), + commandId: effectiveId, + queuedAt: Date.now(), + }); if (shouldTrackSupervisionTaskRun) { - supervisionAutomation.queueTaskIntent(sessionName, effectiveId, text, supervisionSnapshot); + supervisionAutomation.queueTaskIntent(sessionName, effectiveId, displayText, supervisionSnapshot); } const queued = getResendEntries(sessionName); const infoMsg = `⏳ Provider ${providerLabel} is restarting — will auto-resend ${queued.length} queued message${queued.length === 1 ? '' : 's'} once the runtime is back.`; @@ -2096,7 +2571,7 @@ async function handleSend(cmd: Record, serverLink: ServerLink): return; } if (transportRuntime) { - if (trimmedText === '/clear' && supportsTransportClear(record?.agentType)) { + if (isSessionControlCommandText(trimmedText, 'clear') && supportsTransportClear(record?.agentType)) { emitTransportUserMessage(text); // Fresh conversation must not replay stale queued messages from the prior // offline window — drop anything we had buffered for resend. @@ -2129,15 +2604,11 @@ async function handleSend(cmd: Record, serverLink: ServerLink): } return; } - // `/compact` is intentionally NOT intercepted here. Transport SDKs - // (claude-code-sdk, codex-sdk, copilot-sdk, cursor-headless, openclaw, - // qwen) accept the literal `/compact` text and run their own native - // compaction — preserving SDK tool config, system prompts, and resume - // identity. The daemon's automatic materialization pipeline already - // archives every memory-eligible event into `context_event_archive` - // independently, so daemon-side memory provenance is preserved whether - // or not the SDK compacts. Falling through to the default send path - // forwards `/compact` to the transport untouched. + // `/compact` is intentionally NOT handled as daemon-side compaction here. + // The transport runtime/provider capability decides whether it is translated + // to an SDK RPC, forwarded as a verified slash command, or rejected visibly. + // Falling through preserves the ordinary receipt-ack contract while keeping + // provider-specific compact semantics at the SDK boundary. const release = await getMutex(sessionName).acquire(); try { const modelMatch = trimmedText.match(/^\/model\s+(\S+)(?:\s+.*)?$/); @@ -2357,25 +2828,34 @@ async function handleSend(cmd: Record, serverLink: ServerLink): // send() is synchronous: dispatches immediately if idle, queues if busy. // Status changes come from transport runtime's onStatusChange callback. - const result = attachments.length > 0 - ? transportRuntime.send(text, effectiveId, attachments) - : transportRuntime.send(text, effectiveId); + const result = preferenceMessagePreamble + ? transportRuntime.send( + displayText, + effectiveId, + attachments.length > 0 ? attachments : undefined, + preferenceMessagePreamble, + ) + : (attachments.length > 0 + ? transportRuntime.send(displayText, effectiveId, attachments) + : transportRuntime.send(displayText, effectiveId)); if (shouldTrackSupervisionTaskRun) { if (result === 'queued') { - supervisionAutomation.queueTaskIntent(sessionName, effectiveId, text, supervisionSnapshot); + supervisionAutomation.queueTaskIntent(sessionName, effectiveId, displayText, supervisionSnapshot); } else if (result === 'sent') { - supervisionAutomation.registerTaskIntent(sessionName, effectiveId, text, supervisionSnapshot); + supervisionAutomation.registerTaskIntent(sessionName, effectiveId, displayText, supervisionSnapshot); } } if (result === 'sent') { - emitTransportUserMessage( - text, - { - clientMessageId: effectiveId, - ...(attachments.length > 0 ? { attachments } : {}), - }, - transportUserEventId(effectiveId), - ); + if (!shouldHideTimelineUserMessageForSessionControl(displayText)) { + emitTransportUserMessage( + displayText, + { + clientMessageId: effectiveId, + ...(attachments.length > 0 ? { attachments } : {}), + }, + transportUserEventId(effectiveId), + ); + } } if (result === 'queued') { timelineEmitter.emit(sessionName, 'session.state', { @@ -2393,7 +2873,8 @@ async function handleSend(cmd: Record, serverLink: ServerLink): } catch (err) { const errMsg = describeTransportSendError(err); logger.error({ sessionName, err }, 'session.send (transport) failed'); - timelineEmitter.emit(sessionName, 'assistant.text', { text: `⚠️ Send failed: ${errMsg}`, streaming: false, memoryExcluded: true }, { source: 'daemon', confidence: 'high' }); + const failureLabel = isSessionControlCommandText(displayText, 'compact') ? 'Compact failed' : 'Send failed'; + timelineEmitter.emit(sessionName, 'assistant.text', { text: `⚠️ ${failureLabel}: ${errMsg}`, streaming: false, memoryExcluded: true }, { source: 'daemon', confidence: 'high' }); timelineEmitter.emit(sessionName, 'session.state', { state: 'idle', error: errMsg }, { source: 'daemon', confidence: 'high' }); if (!receiptAcked) { emitCommandAckReliable(serverLink, { commandId: effectiveId, sessionName, status: 'error', error: errMsg }); @@ -2404,10 +2885,16 @@ async function handleSend(cmd: Record, serverLink: ServerLink): return; } - // Preserve raw @file references for normal sends. - const finalText = text; + // Preserve raw @file references for normal sends. Stable preferences are + // session context, not per-turn recall: for tmux/process agents inject them + // once per provider conversation, and reset the gate on clear/compact. + const finalText = prepareProcessPreferenceProviderText({ + sessionName, + providerText: displayText, + preferenceContext: preferenceMessagePreamble, + }); - if (text.trim() === '/clear' && record?.runtimeType !== 'transport' && supportsProcessClear(record?.agentType)) { + if (isSessionControlCommandText(text, 'clear') && record?.runtimeType !== 'transport' && supportsProcessClear(record?.agentType)) { emitTransportUserMessage(text); try { await runExclusiveSessionRelaunch(sessionName, async () => { @@ -2457,7 +2944,7 @@ async function handleSend(cmd: Record, serverLink: ServerLink): try { await sendProcessSessionMessage(sessionName, finalText, attachments, { - originalText: text, + originalText: displayText, commandId: effectiveId, isLegacy, ackAlreadySent: receiptAcked, @@ -2721,16 +3208,7 @@ async function handleInput(cmd: Record): Promise { const transportRuntime = getTransportRuntime(sessionName); if (transportRuntime) { if (data === '\x1b') { - try { - await transportRuntime.cancel(); - // Mark Qwen sessions for fresh start so restart doesn't resume stuck conversation - const rec = getSession(sessionName); - if (rec?.agentType === 'qwen') { - upsertSession({ ...rec, qwenFreshOnResume: true, updatedAt: Date.now() }); - } - } catch (err) { - logger.error({ sessionName, err }, 'session.input transport cancel failed'); - } + cancelTransportTurnNow(sessionName, undefined, undefined); } return; } @@ -3613,6 +4091,7 @@ function compareDaemonVersions(a: string, b: string): -1 | 0 | 1 { * may resolve to an older release than what's currently installed. */ async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLink): Promise { + const UPGRADE_MEMORY_FREEZE_TTL_MS = 15 * 60 * 1000; const activeRuns = getActiveP2pRunsBlockingDaemonUpgrade(); if (activeRuns.length > 0) { logger.warn({ @@ -3680,7 +4159,7 @@ async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLi } const { spawn } = await import('child_process'); - const { writeFileSync, mkdtempSync, existsSync } = await import('fs'); + const { writeFileSync, readFileSync, mkdtempSync, existsSync } = await import('fs'); const { join, dirname } = await import('path'); const { tmpdir, homedir } = await import('os'); @@ -3743,6 +4222,13 @@ async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLi reopenLiveContextMaterializationAdmission(); }; })(); + const scheduleUpgradeMemoryFreezeRelease = () => { + const timer = setTimeout(() => { + logger.warn({ targetVersion }, 'daemon.upgrade: releasing memory freeze after watchdog timeout'); + releaseUpgradeMemoryFreeze(); + }, UPGRADE_MEMORY_FREEZE_TTL_MS); + timer.unref?.(); + }; try { const postFreezeMasterCompactions = getInflightMasterCompactionCount(); @@ -3798,39 +4284,55 @@ if [ -n "$STALE_PID" ] && kill -0 "$STALE_PID" 2>/dev/null; then fi launchctl load -w "${plist}"`; } else if (process.platform === 'win32') { - // Windows: generate a CMD batch script + // Windows: drive the upgrade with a Node.js runner instead of a + // cmd.exe batch. The batch was the source of every Windows + // auto-upgrade outage we shipped (paren-counting in if-blocks, + // timeout-needs-stdin, del silent failures, codepage issues with + // non-ASCII %TEMP% / %USERPROFILE% paths). Node fs APIs use the + // Windows wide-char API natively, so Chinese / Cyrillic / etc. + // paths round-trip transparently. + // + // Layout: copy the bundled runner to %TEMP%/imcodes-upgrade-X/upgrade.mjs + // BEFORE spawning, so the in-flight `npm install -g` doesn't + // overwrite the runner's source under itself when the new + // package's files land at the same global path. const npmBin = join(dirname(process.execPath), 'npm.cmd'); const npmCmd = existsSync(npmBin) ? npmBin : 'npm'; const pkgSpec = targetVersion ? `imcodes@${targetVersion}` : 'imcodes@latest'; - const batchPath = join(scriptDir, 'upgrade.cmd'); - const upgradeVbsPath = join(scriptDir, 'upgrade.vbs'); + const targetVer = targetVersion ?? 'latest'; + + const runnerSrc = resolveWindowsUpgradeRunnerPath(); + const runnerCopy = join(scriptDir, 'upgrade.mjs'); + try { + // Read+write rather than cpSync so a broken runnerSrc fails loud. + writeFileSync(runnerCopy, readFileSync(runnerSrc)); + } catch (err) { + logger.error({ err, runnerSrc }, 'daemon.upgrade: failed to stage upgrade runner — cannot proceed'); + return; + } + + // Cleanup .cmd is still cmd.exe — but it's a 4-line idempotent rmdir + // with NO control flow. No parens, no timeout, no del — just one + // ping sleep and one rmdir. Kept because the runner self-cleans via + // its own deferred rmSync, but this is a belt-and-suspenders for + // the case where the runner crashes before reaching the finally block. const cleanupPath = join(scriptDir, 'cleanup.cmd'); const cleanupVbsPath = join(scriptDir, 'cleanup.vbs'); - const targetVer = targetVersion ?? 'latest'; - // .cmd files: UTF-8 + BOM, and the script itself switches to UTF-8 with - // `chcp 65001` before touching any non-ASCII paths. - // .vbs files: UTF-16 LE + BOM so wscript handles non-ASCII paths. writeFileSync(cleanupPath, encodeCmdAsUtf8Bom(buildWindowsCleanupScript(scriptDir))); writeFileSync(cleanupVbsPath, encodeVbsAsUtf16(buildWindowsCleanupVbs(cleanupPath))); - const vbsLauncherPath = join(homedir(), '.imcodes', 'daemon-launcher.vbs'); - const batch = buildWindowsUpgradeBatch({ - logFile, - scriptDir, - cleanupPath, - cleanupVbsPath, - npmCmd, - pkgSpec, - targetVer, - vbsLauncherPath, - upgradeLockFile: UPGRADE_LOCK_FILE, - }); - writeFileSync(batchPath, encodeCmdAsUtf8Bom(batch)); - writeFileSync(upgradeVbsPath, encodeVbsAsUtf16(buildWindowsUpgradeVbs(batchPath))); + // VBS launcher — runs the JS runner via `node upgrade.mjs ` + // hidden + detached. Bake all paths as args so the runner doesn't + // depend on env-var expansion or working directory. + const upgradeVbsPath = join(scriptDir, 'upgrade.vbs'); + const upgradeVbs = buildWindowsUpgradeRunnerVbs({ + nodeExe: process.execPath, + runnerPath: runnerCopy, + args: [logFile, npmCmd, pkgSpec, targetVer, scriptDir], + }); + writeFileSync(upgradeVbsPath, encodeVbsAsUtf16(upgradeVbs)); - // Launch via wscript on the wrapper VBS — this guarantees that ALL child - // processes spawned by the batch (wmic, find, tasklist, etc.) inherit a - // fully hidden parent and never flash console windows. + // Launch via wscript: hidden + fully detached, survives our exit. const child = spawn('wscript', [upgradeVbsPath], { detached: true, stdio: 'ignore', @@ -3838,8 +4340,17 @@ launchctl load -w "${plist}"`; }); child.unref(); - logger.info({ log: logFile }, 'daemon.upgrade: Windows upgrade script spawned'); + // Also kick off cleanup deferred 120 s — the runner cleans up too, + // but if it crashes before its finally block we still want %TEMP% tidy. + spawn('wscript', [cleanupVbsPath], { + detached: true, + stdio: 'ignore', + windowsHide: true, + }).unref(); + + logger.info({ log: logFile, runnerCopy }, 'daemon.upgrade: Windows JS upgrade runner spawned'); upgradeScriptSpawned = true; + scheduleUpgradeMemoryFreezeRelease(); return; } else { logger.warn('daemon.upgrade: unsupported platform, cannot restart service'); @@ -3894,7 +4405,10 @@ launchctl load -w "${plist}"`; // instead of evaporating in 60 s. Operators running into a stuck daemon // can grep `find /tmp -name 'imcodes-upgrade-*' -mmin -1440` after the // fact. Successful upgrades still clean up via the same timer; the only - // observable change is debugability. + // observable change is debugability. On Linux the delayed cleanup must run + // in its own transient user unit; a background `sleep 86400` spawned from + // imcodes.service stays in the daemon's cgroup and pollutes systemctl status + // until it exits. const CLEANUP_AFTER_SEC = 24 * 60 * 60; const script = `#!/bin/bash # imcodes daemon-upgrade script. Generated by daemon.upgrade. @@ -3903,8 +4417,35 @@ launchctl load -w "${plist}"`; # stuck or failed restart can be diagnosed post-hoc. LOG="${logFile}" +SCRIPT_DIR="${scriptDir}" +CLEANUP_AFTER_SEC=${CLEANUP_AFTER_SEC} log() { echo "[$(date '+%Y-%m-%dT%H:%M:%S%z')] $*" >> "$LOG"; } +schedule_self_cleanup() { + if [ -z "$SCRIPT_DIR" ] || [ ! -d "$SCRIPT_DIR" ]; then + return 0 + fi + + if [ "$(uname)" = "Linux" ]; then + if command -v systemd-run >/dev/null 2>&1; then + CLEANUP_LABEL=$(printf '%s' "$(basename "$SCRIPT_DIR")" | tr -c 'A-Za-z0-9_.-' '-') + CLEANUP_UNIT="imcodes-upgrade-cleanup-$CLEANUP_LABEL" + if systemd-run --user --unit="$CLEANUP_UNIT" --collect --quiet /bin/sh -c 'sleep "$1"; rm -rf "$2"' imcodes-upgrade-cleanup "$CLEANUP_AFTER_SEC" "$SCRIPT_DIR" >> "$LOG" 2>&1; then + log "[cleanup] scheduled via systemd-run user unit: $CLEANUP_UNIT" + return 0 + fi + log "[cleanup] systemd-run scheduling failed (non-fatal); leaving $SCRIPT_DIR for manual cleanup" + else + log "[cleanup] systemd-run unavailable; leaving $SCRIPT_DIR for manual cleanup" + fi + log "[cleanup] skipped background sleeper on Linux to avoid leaking into imcodes.service cgroup" + return 0 + fi + + (sleep "$CLEANUP_AFTER_SEC" && rm -rf "$SCRIPT_DIR") >/dev/null 2>&1 & + log "[cleanup] scheduled via background sleeper" +} + log "=== imcodes upgrade started ===" log "[step 0] daemon PID at gen time: ${oldDaemonPid}" log "[step 0] node bin: ${nodeBin}" @@ -3991,7 +4532,7 @@ release_upgrade_lock() { if ! acquire_upgrade_lock; then log "=== upgrade skipped: another upgrade is in progress ===" - sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & + schedule_self_cleanup exit 0 fi trap release_upgrade_lock EXIT @@ -4101,38 +4642,67 @@ log "[step 2] installing ${pkgSpec}" # \`npm install\` from inside the global package to repopulate it. Run with # --ignore-scripts again for the same reason. # -# ── Retry on ETARGET (npm CDN replication race) ──────────────────────── +# ── Retry on publish propagation / transient network failures ────────── # Real-world failure mode caught on 116.62.239.78: server publishes a # new dev release to npm and broadcasts \`daemon.upgrade { targetVersion }\` # almost immediately. npm origin has the version but the regional CDN # edge serving this daemon hasn't replicated yet — so the packument # response is a 200 missing the new version → npm exits with ETARGET. -# Pre-fix this killed the upgrade for that release entirely (no retry, -# next try only when the server broadcasts again). We now retry up to -# 4 times with 60/180/300s back-off (1m / 3m / 5m — the last gap is wide -# enough to outlast a slow regional CDN: 2m wasn't enough on a real -# replication-lagged edge node) and bust the packument cache between -# attempts so npm refetches origin instead of serving the stale 200. -# Total time-to-give-up: ~9 minutes from the first attempt. -# -# Non-ETARGET failures are NOT retried — they're typically deterministic -# (network down, ENOSPC, registry auth issue). Logging the per-attempt -# tail makes those diagnosable post-hoc without re-reading the giant -# main log. +# Pre-fix this either killed the upgrade for that release or, worse, ran +# \`npm cache clean --force\`, deleting every cached dependency on the box. +# The eventual successful install then had to redownload 200+ packages and +# took minutes. We now use a cheap \`npm view\` precheck for pinned versions, +# avoid full-cache wipes, and retry transient network failures like +# ECONNRESET/ETIMEDOUT/EAI_AGAIN. INSTALL_OUT="${scriptDir}/install-attempt.log" INSTALL_RC=1 ATTEMPT=0 -MAX_ATTEMPTS=4 +MAX_ATTEMPTS=5 # Indexed sequentially with $ATTEMPT (1-based), so element 0 is unused. -# 60s / 180s / 300s — see the comment block above for sizing rationale. -RETRY_DELAYS=(0 60 180 300) +# 15s / 30s / 60s / 120s keeps the common npm publish-CDN window quick +# without stretching a bad target into a 10-minute local stall. +RETRY_DELAYS=(0 15 30 60 120) + +is_etarget_output() { + grep -qiE 'code ETARGET|No matching version found' "$1" 2>/dev/null +} + +is_transient_npm_output() { + grep -qiE 'code (ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH|ENETUNREACH)|network aborted|socket timeout|fetch failed|network socket disconnected|5[0-9][0-9]' "$1" 2>/dev/null +} + while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do ATTEMPT=$((ATTEMPT + 1)) log "[step 2] install attempt $ATTEMPT/$MAX_ATTEMPTS" : > "$INSTALL_OUT" + + if [ "${targetVer}" != "latest" ]; then + log "[step 2] registry visibility precheck for ${pkgSpec}" + eval "$NPM_RUN view --prefer-online ${pkgSpec} version" >> "$INSTALL_OUT" 2>&1 + VIEW_RC=$? + cat "$INSTALL_OUT" >> "$LOG" + if [ "$VIEW_RC" -ne 0 ] && is_etarget_output "$INSTALL_OUT"; then + log "[step 2] ${pkgSpec} not visible in registry yet" + if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then + log "[step 2] target never became visible across $MAX_ATTEMPTS attempts — giving up before heavyweight install" + INSTALL_RC=$VIEW_RC + break + fi + DELAY=\${RETRY_DELAYS[$ATTEMPT]} + log "[step 2] waiting \${DELAY}s for npm publish propagation" + sleep "$DELAY" + continue + fi + if [ "$VIEW_RC" -ne 0 ]; then + log "[step 2] registry precheck failed (exit $VIEW_RC); trying install anyway" + fi + : > "$INSTALL_OUT" + fi + # --prefer-online: tell npm to revalidate cached packument metadata - # rather than serve potentially-stale entries. Belt & braces with the - # cache-clean we do between attempts. + # rather than serve potentially-stale entries. Do NOT use \`npm cache + # clean --force\` here: it wipes cached dependency tarballs too, which is + # exactly what made upgrades on large SDK dependency sets feel glacial. eval "$NPM_RUN install -g --ignore-scripts --prefer-online ${pkgSpec}" >> "$INSTALL_OUT" 2>&1 INSTALL_RC=$? # Always tee the attempt's output into the main log for forensics. @@ -4142,31 +4712,32 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do break fi log "[step 2] install attempt $ATTEMPT failed (exit $INSTALL_RC)" - # Detect ETARGET (case-insensitive — npm versions vary slightly). - IS_ETARGET=0 - if grep -qiE 'code ETARGET|No matching version found' "$INSTALL_OUT" 2>/dev/null; then - IS_ETARGET=1 + IS_RETRYABLE=0 + RETRY_REASON="non-retryable" + if is_etarget_output "$INSTALL_OUT"; then + IS_RETRYABLE=1 + RETRY_REASON="target-not-visible" + elif is_transient_npm_output "$INSTALL_OUT"; then + IS_RETRYABLE=1 + RETRY_REASON="transient-network" fi - if [ "$IS_ETARGET" -ne 1 ]; then - log "[step 2] non-ETARGET failure — not retrying. Tail of npm output:" + if [ "$IS_RETRYABLE" -ne 1 ]; then + log "[step 2] non-retryable npm failure — not retrying. Tail of npm output:" tail -20 "$INSTALL_OUT" | while IFS= read -r line; do log "[step 2] $line"; done break fi if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then - log "[step 2] ETARGET persisted across $MAX_ATTEMPTS attempts — registry never replicated ${pkgSpec}" + log "[step 2] retryable npm failure ($RETRY_REASON) persisted across $MAX_ATTEMPTS attempts" break fi DELAY=\${RETRY_DELAYS[$ATTEMPT]} - log "[step 2] ETARGET — npm CDN likely hasn't replicated ${pkgSpec} yet; retrying in \${DELAY}s after busting packument cache" - # Bust the cached packument so the next attempt forces an origin - # round-trip instead of revalidating into the stale cached 200. - eval "$NPM_RUN cache clean --force" >> "$LOG" 2>&1 || log "[step 2] cache clean returned non-zero (ignored)" + log "[step 2] retryable npm failure ($RETRY_REASON) — retrying in \${DELAY}s" sleep "$DELAY" done if [ "$INSTALL_RC" -ne 0 ]; then log "[step 2] install FAILED after $ATTEMPT attempts (final exit $INSTALL_RC) — keeping current daemon running" log "=== upgrade aborted ===" - sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & + schedule_self_cleanup exit 0 fi log "[step 2] install succeeded after $ATTEMPT attempt(s)" @@ -4182,7 +4753,7 @@ log "[step 3] installed version: $INSTALLED_VER, target: ${targetVer}" if [ "${targetVer}" != "latest" ] && [ "$INSTALLED_VER" != "${targetVer}" ]; then log "[step 3] version mismatch — keeping current daemon running" log "=== upgrade aborted ===" - sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & + schedule_self_cleanup exit 0 fi @@ -4214,13 +4785,13 @@ CMP=$? if [ "$CMP" = "1" ]; then log "[step 3] installed $INSTALLED_VER is OLDER than current $CURRENT_VER — refusing to downgrade" log "=== upgrade aborted ===" - sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & + schedule_self_cleanup exit 0 fi if [ "$CMP" = "0" ]; then log "[step 3] installed $INSTALLED_VER matches current — no restart needed" log "=== upgrade complete (no-op) ===" - sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & + schedule_self_cleanup exit 0 fi log "[step 3] version comparator: installed > current → restart" @@ -4260,12 +4831,30 @@ log "[step 3] version comparator: installed > current → restart" # clobber. log "[step 3.5] regenerating launch chain" NEW_IMCODES_SCRIPT="$GLOBAL_ROOT/imcodes/dist/src/index.js" +NEW_LAUNCHER="$GLOBAL_ROOT/imcodes/bin/imcodes-launch.sh" + +# Prefer the self-healing launcher (bin/imcodes-launch.sh) when the +# freshly-installed package ships it. Older installs (pre-launcher) fall +# back to the direct node ExecStart so we never break versions that +# don't ship the file. Either way the resulting unit/plist points at +# absolute paths from THIS install — consistent with the rest of step +# 3.5's contract. +if [ -f "$NEW_LAUNCHER" ]; then + LINUX_EXEC="ExecStart=$NEW_LAUNCHER start --foreground" + DARWIN_PROGRAM_ARGS="[\\"$NEW_LAUNCHER\\",\\"start\\",\\"--foreground\\"]" + log "[step 3.5] using self-healing launcher: $NEW_LAUNCHER" +else + LINUX_EXEC="ExecStart=$NODE $NEW_IMCODES_SCRIPT start --foreground" + DARWIN_PROGRAM_ARGS="[\\"$NODE\\",\\"$NEW_IMCODES_SCRIPT\\",\\"start\\",\\"--foreground\\"]" + log "[step 3.5] $NEW_LAUNCHER not present in this version — using direct node ExecStart" +fi + if [ ! -f "$NEW_IMCODES_SCRIPT" ]; then log "[step 3.5] $NEW_IMCODES_SCRIPT not found — skipping (will rely on existing launch chain)" elif [ "$(uname)" = "Linux" ]; then SVC="$HOME/.config/systemd/user/imcodes.service" if [ -f "$SVC" ]; then - NEW_EXEC="ExecStart=$NODE $NEW_IMCODES_SCRIPT start --foreground" + NEW_EXEC="$LINUX_EXEC" OLD_EXEC=$(grep -m1 '^ExecStart=' "$SVC" || echo '(none)') if [ "$OLD_EXEC" = "$NEW_EXEC" ]; then log "[step 3.5] systemd ExecStart already current" @@ -4296,7 +4885,7 @@ elif [ "$(uname)" = "Darwin" ]; then if [ -f "$PLIST" ]; then if command -v plutil >/dev/null 2>&1; then log "[step 3.5] rewriting plist ProgramArguments" - if plutil -replace ProgramArguments -json "[\\"$NODE\\",\\"$NEW_IMCODES_SCRIPT\\",\\"start\\",\\"--foreground\\"]" "$PLIST" >> "$LOG" 2>&1; then + if plutil -replace ProgramArguments -json "$DARWIN_PROGRAM_ARGS" "$PLIST" >> "$LOG" 2>&1; then log "[step 3.5] plutil rewrite OK" else log "[step 3.5] plutil rewrite FAILED (non-fatal)" @@ -4369,7 +4958,7 @@ fi log "=== upgrade script done ===" # Self-cleanup after 24 h so failures stay debuggable. -sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & +schedule_self_cleanup `; writeFileSync(scriptPath, script, { mode: 0o755 }); @@ -4383,6 +4972,7 @@ sleep ${CLEANUP_AFTER_SEC} && rm -rf "${scriptDir}" & logger.info({ log: logFile }, 'daemon.upgrade: upgrade script spawned, will restart in ~3 s'); upgradeScriptSpawned = true; + scheduleUpgradeMemoryFreezeRelease(); } finally { if (!upgradeScriptSpawned) { releaseUpgradeMemoryFreeze(); @@ -4404,13 +4994,7 @@ const WINDOWS_DRIVES_PATH = ':drives:'; const WINDOWS_DRIVES_ROOT = '__imcodes_windows_drives__'; function isPathAllowed(realPath: string): boolean { - // Block sensitive directories (e.g. ~/.ssh, ~/.gnupg) - const home = homedir(); - for (const dir of FS_DENIED_DIRS) { - const denied = nodePath.join(home, dir); - if (realPath === denied || realPath.startsWith(denied + nodePath.sep)) return false; - } - return true; + return isFilePreviewPathAllowed(realPath); } // ── P2P cancel/status handlers ──────────────────────────────────────────── @@ -4612,11 +5196,14 @@ const fsListCache = new Map( const fsListInflight = new Map>(); const fsListGenerations = new Map(); -function getFsListCacheKey(realPath: string, includeFiles: boolean, includeMetadata: boolean): string { - return `${realPath}::${includeFiles ? 'files' : 'dirs'}::${includeMetadata ? 'meta' : 'plain'}`; +function getFsListCacheKey(realPath: string, includeFiles: boolean, includeMetadata: boolean, allowDownloadHandles: boolean): string { + const metadataMode = includeMetadata + ? (allowDownloadHandles ? 'meta' : 'meta-no-downloads') + : 'plain'; + return `${realPath}::${includeFiles ? 'files' : 'dirs'}::${metadataMode}`; } -async function loadFsListSnapshot(real: string, includeFiles: boolean, includeMetadata: boolean): Promise { +async function loadFsListSnapshot(real: string, includeFiles: boolean, includeMetadata: boolean, allowDownloadHandles: boolean): Promise { const dirents = await fsReaddir(real, { withFileTypes: true }); const filtered = dirents.filter((d) => d.isDirectory() || (includeFiles && d.isFile())); @@ -4629,8 +5216,10 @@ async function loadFsListSnapshot(real: string, includeFiles: boolean, includeMe entry.size = fileStat.size; const ext = nodePath.extname(d.name).toLowerCase().slice(1); entry.mime = MIME_MAP[ext] || undefined; - const handle = createProjectFileHandle(filePath, d.name, entry.mime as string | undefined, fileStat.size); - entry.downloadId = handle.id; + if (allowDownloadHandles) { + const handle = await tryCreateProjectFileHandle(filePath, d.name, entry.mime as string | undefined, fileStat.size); + if (handle) entry.downloadId = handle.id; + } } catch { /* stat failed, skip metadata */ } } return entry; @@ -4649,9 +5238,9 @@ async function loadFsListSnapshot(real: string, includeFiles: boolean, includeMe }; } -async function getFsListSnapshot(real: string, includeFiles: boolean, includeMetadata: boolean): Promise { +async function getFsListSnapshot(real: string, includeFiles: boolean, includeMetadata: boolean, allowDownloadHandles: boolean): Promise { const dirSignature = await safeStatSignature(real); - const cacheKey = getFsListCacheKey(real, includeFiles, includeMetadata); + const cacheKey = getFsListCacheKey(real, includeFiles, includeMetadata, allowDownloadHandles); const cached = fsListCache.get(cacheKey); if (cached && cached.expiresAt > Date.now() && cached.value.dirSignature === dirSignature) { return cached.value; @@ -4662,7 +5251,7 @@ async function getFsListSnapshot(real: string, includeFiles: boolean, includeMet const inflight = fsListInflight.get(inflightKey); if (inflight) return await inflight; - const promise = loadFsListSnapshot(real, includeFiles, includeMetadata) + const promise = loadFsListSnapshot(real, includeFiles, includeMetadata, allowDownloadHandles) .then(async (value) => { const currentSignature = await safeStatSignature(real); if (getResourceGeneration(fsListGenerations, real) === generation && currentSignature === value.dirSignature) { @@ -4680,18 +5269,20 @@ async function getFsListSnapshot(real: string, includeFiles: boolean, includeMet function invalidateFsListCachesForPath(targetPath: string): void { const realTarget = normalizeFsPath(targetPath); bumpResourceGeneration(fsListGenerations, realTarget); - fsListCache.delete(getFsListCacheKey(realTarget, false, false)); - fsListCache.delete(getFsListCacheKey(realTarget, true, false)); - fsListCache.delete(getFsListCacheKey(realTarget, false, true)); - fsListCache.delete(getFsListCacheKey(realTarget, true, true)); + for (const includeFiles of [false, true]) { + fsListCache.delete(getFsListCacheKey(realTarget, includeFiles, false, true)); + fsListCache.delete(getFsListCacheKey(realTarget, includeFiles, true, true)); + fsListCache.delete(getFsListCacheKey(realTarget, includeFiles, true, false)); + } const parent = nodePath.dirname(realTarget); if (parent !== realTarget) { bumpResourceGeneration(fsListGenerations, parent); - fsListCache.delete(getFsListCacheKey(parent, false, false)); - fsListCache.delete(getFsListCacheKey(parent, true, false)); - fsListCache.delete(getFsListCacheKey(parent, false, true)); - fsListCache.delete(getFsListCacheKey(parent, true, true)); + for (const includeFiles of [false, true]) { + fsListCache.delete(getFsListCacheKey(parent, includeFiles, false, true)); + fsListCache.delete(getFsListCacheKey(parent, includeFiles, true, true)); + fsListCache.delete(getFsListCacheKey(parent, includeFiles, true, false)); + } } } @@ -4709,7 +5300,11 @@ async function handleFsList(cmd: Record, serverLink: ServerLink : (rawPath.startsWith('~') ? rawPath.replace(/^~/, homedir()) : rawPath); const resolved = isDrivesSentinel ? rawPath : nodePath.resolve(expanded); - const deadline = new Promise((_, reject) => setTimeout(() => reject(new Error('fs_list_timeout')), FS_LIST_DEADLINE_MS)); + let deadlineTimer: ReturnType | null = null; + const deadline = new Promise((_, reject) => { + deadlineTimer = setTimeout(() => reject(new Error('fs_list_timeout')), FS_LIST_DEADLINE_MS); + deadlineTimer.unref?.(); + }); try { await Promise.race([handleFsListInner(resolved, rawPath, requestId, includeFiles, includeMetadata, serverLink), deadline]); @@ -4720,6 +5315,8 @@ async function handleFsList(cmd: Record, serverLink: ServerLink } else { try { serverLink.send({ type: 'fs.ls_response', requestId, path: rawPath, status: 'error', error: msg }); } catch { /* ignore */ } } + } finally { + if (deadlineTimer) clearTimeout(deadlineTimer); } } @@ -4752,232 +5349,37 @@ async function handleFsListInner(resolved: string, rawPath: string, requestId: s return; } - let real: string; - try { - real = await fsRealpath(resolved); - } catch (err) { - if (process.platform === 'win32') { - logger.debug({ resolved, err }, 'fsRealpath failed on Windows, falling back to resolved path'); - real = resolved; - } else { - throw err; - } - } - - const allowed = isPathAllowed(real); - if (!allowed) { - try { serverLink.send({ type: 'fs.ls_response', requestId, path: rawPath, resolvedPath: real, status: 'error', error: 'forbidden_path' }); } catch { /* ignore */ } + const canonical = await resolveCanonical(resolved, includeMetadata ? 'lenient' : 'strict'); + if (!canonical) { + try { serverLink.send({ type: 'fs.ls_response', requestId, path: rawPath, status: 'error', error: FS_GENERIC_ERROR_CODES.FORBIDDEN_PATH }); } catch { /* ignore */ } return; } - const snapshot = await getFsListSnapshot(real, includeFiles, includeMetadata); + const snapshot = await getFsListSnapshot(canonical.realPath, includeFiles, includeMetadata, !canonical.usedFallback); try { serverLink.send({ type: 'fs.ls_response', requestId, path: rawPath, resolvedPath: snapshot.resolvedPath, status: 'ok', entries: snapshot.entries }); } catch { /* ignore */ } } -const FS_READ_SIZE_LIMIT = 100 * 1024 * 1024; // 100 MB - -// Video files are not transferred over WS — the browser fetches them via the -// HTTP download endpoint and plays them with