fix(context): refresh get_docs when packages change on disk

.changeset/olive-otters-watch.md

@@ -0,0 +1,5 @@
+---
+"@neuledge/context": patch
+---
+
+`serve` now picks up packages installed while it is running. `context add` writes to the data directory from a separate process, so a long-lived stdio server used to serve the package list it read at startup until the client reconnected, and `get_docs` could not see a package that was already installed.

packages/context/src/cli.ts

@@ -65,6 +65,7 @@ import {
   packageKey,
   readPackageInfo,
 } from "./store.js";
+import { watchDirectory } from "./watch.js";
 
 type SourceType = "file" | "url" | "git" | "local-dir" | "website";
 
@@ -506,19 +507,33 @@ function ensureDataDir(): void {
   mkdirSync(DATA_DIR, { recursive: true });
 }
 
-/** Load all packages from the data directory into the store. */
+/**
+ * Sync the store with the data directory: add or update what is on disk, drop
+ * what is not.
+ *
+ * Called once at startup and again whenever the directory changes, so removals
+ * have to be handled too, not only additions.
+ */
 function loadPackages(store: PackageStore): void {
-  if (!existsSync(DATA_DIR)) return;
+  const onDisk = new Set<string>();
 
-  for (const file of readdirSync(DATA_DIR)) {
-    if (!file.endsWith(".db")) continue;
-    try {
-      const info = readPackageInfo(join(DATA_DIR, file));
-      store.add(info);
-    } catch {
-      // Skip invalid packages
+  if (existsSync(DATA_DIR)) {
+    for (const file of readdirSync(DATA_DIR)) {
+      if (!file.endsWith(".db")) continue;
+      try {
+        const info = readPackageInfo(join(DATA_DIR, file));
+        store.add(info);
+        onDisk.add(packageKey(info));
+      } catch {
+        // Skip invalid packages
+      }
     }
   }
+
+  for (const pkg of store.list()) {
+    const key = packageKey(pkg);
+    if (!onDisk.has(key)) store.remove(key);
+  }
 }
 
 /**
@@ -1218,6 +1233,17 @@ program
 
       const server = new ContextServer(store, { allowedLibraries });
 
+      // `context add` runs in a separate process, so a long-lived stdio server
+      // would otherwise serve the package list it read at startup until the
+      // client reconnected. Skipped under --libs, which pins the session to a
+      // fixed library set on purpose.
+      if (!allowedLibraries) {
+        watchDirectory(DATA_DIR, () => {
+          loadPackages(store);
+          server.refreshGetDocsTool();
+        });
+      }
+
       if (options.http !== undefined) {
         const port =
           typeof options.http === "string"

packages/context/src/server.ts

@@ -223,8 +223,13 @@ export class ContextServer {
   /**
    * Update the get_docs tool to include newly installed packages.
    * If get_docs doesn't exist yet, register it for the first time.
+   *
+   * Public because packages can also arrive from outside this process: `context
+   * add` writes to the data directory while a stdio server is already running.
+   * `serve` watches that directory and calls this, so the tool's `library` enum
+   * stays current without the client reconnecting.
    */
-  private refreshGetDocsTool(): void {
+  refreshGetDocsTool(): void {
     const packages = this.visiblePackages();
 
     if (this.getDocsRegistration) {

packages/context/src/watch.test.ts

@@ -0,0 +1,94 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { watchDirectory } from "./watch.js";
+
+/** Give the watcher's debounce window time to close before asserting. */
+const settle = () => new Promise((r) => setTimeout(r, 400));
+
+describe("watchDirectory", () => {
+  let dir: string;
+  let stop: (() => void) | undefined;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), "context-watch-"));
+  });
+
+  afterEach(() => {
+    stop?.();
+    stop = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("fires when a file appears", async () => {
+    let calls = 0;
+    stop = watchDirectory(dir, () => {
+      calls++;
+    });
+
+    writeFileSync(join(dir, "demo.db"), "x");
+    await settle();
+
+    expect(calls).toBe(1);
+  });
+
+  it("collapses a burst into one call", async () => {
+    let calls = 0;
+    stop = watchDirectory(dir, () => {
+      calls++;
+    });
+
+    // What one install looks like: a temp file, then a rename into place.
+    writeFileSync(join(dir, ".downloading-demo.db"), "x");
+    writeFileSync(join(dir, "demo.db"), "x");
+    writeFileSync(join(dir, "other.db"), "x");
+    await settle();
+
+    expect(calls).toBe(1);
+  });
+
+  it("stops firing once stopped", async () => {
+    let calls = 0;
+    const cancel = watchDirectory(dir, () => {
+      calls++;
+    });
+    cancel();
+
+    writeFileSync(join(dir, "demo.db"), "x");
+    await settle();
+
+    expect(calls).toBe(0);
+  });
+
+  it("survives a callback that throws", async () => {
+    let calls = 0;
+    stop = watchDirectory(dir, () => {
+      calls++;
+      throw new Error("refresh failed");
+    });
+
+    writeFileSync(join(dir, "one.db"), "x");
+    await settle();
+    writeFileSync(join(dir, "two.db"), "x");
+    await settle();
+
+    // A throwing refresh must not tear the watcher down: the second write is
+    // still observed.
+    expect(calls).toBe(2);
+  });
+
+  it("is a no-op for a directory that does not exist", async () => {
+    let calls = 0;
+    const missing = join(dir, "absent");
+    stop = watchDirectory(missing, () => {
+      calls++;
+    });
+
+    mkdirSync(missing);
+    writeFileSync(join(missing, "demo.db"), "x");
+    await settle();
+
+    expect(calls).toBe(0);
+  });
+});

packages/context/src/watch.ts

@@ -0,0 +1,40 @@
+import { existsSync, watch } from "node:fs";
+
+/** Wait this long after the last event before treating a burst as one change. */
+const DEBOUNCE_MS = 200;
+
+/**
+ * Call `onChange` when the contents of `dir` change.
+ *
+ * Debounced, because a single install is several filesystem events: `context
+ * add` writes a temporary file and renames it into place. Without this, one
+ * install would rebuild the tool schema three or four times.
+ *
+ * The watcher is unref'd, so watching never keeps the process alive by itself.
+ * Returns a stop function; a missing directory is a no-op rather than an error,
+ * since nothing has been installed yet in that case.
+ */
+export function watchDirectory(dir: string, onChange: () => void): () => void {
+  if (!existsSync(dir)) return () => {};
+
+  let pending: NodeJS.Timeout | undefined;
+
+  const watcher = watch(dir, () => {
+    if (pending) clearTimeout(pending);
+    pending = setTimeout(() => {
+      pending = undefined;
+      try {
+        onChange();
+      } catch {
+        // A refresh that throws must not take the server down with it.
+      }
+    }, DEBOUNCE_MS);
+  });
+
+  watcher.unref();
+
+  return () => {
+    if (pending) clearTimeout(pending);
+    watcher.close();
+  };
+}