#123: Add hippocampus Chunker + minimal ingest pipeline (PageBuilder, Ingest, tests)

Author: devlux76

hippocampus/Ingest.ts

@@ -0,0 +1,100 @@
+import type { Book, ModelProfile, MetadataStore, VectorStore } from "../core/types";
+import { hashText } from "../core/crypto/hash";
+import type { KeyPair } from "../core/crypto/sign";
+import { EmbeddingRunner } from "../embeddings/EmbeddingRunner";
+import { chunkText } from "./Chunker";
+import { buildPage } from "./PageBuilder";
+import { runPromotionSweep } from "../core/SalienceEngine";
+
+export interface IngestOptions {
+  modelProfile: ModelProfile;
+  embeddingRunner: EmbeddingRunner;
+  vectorStore: VectorStore;
+  metadataStore: MetadataStore;
+  keyPair: KeyPair;
+  now?: number;
+}
+
+export interface IngestResult {
+  pages: Array<Awaited<ReturnType<typeof buildPage>>>;
+  book?: Book;
+}
+
+export async function ingestText(
+  text: string,
+  options: IngestOptions,
+): Promise<IngestResult> {
+  const {
+    modelProfile,
+    embeddingRunner,
+    vectorStore,
+    metadataStore,
+    keyPair,
+    now = Date.now(),
+  } = options;
+
+  const chunks = chunkText(text, modelProfile);
+  if (chunks.length === 0) {
+    return { pages: [], book: undefined };
+  }
+
+  const createdAt = new Date(now).toISOString();
+
+  // Precompute page IDs (content hashes) so we can link prev/next before signing.
+  const pageIds = await Promise.all(chunks.map((c) => hashText(c)));
+
+  const embeddings = await embeddingRunner.embed(chunks);
+  if (embeddings.length !== chunks.length) {
+    throw new Error("Embedding provider returned unexpected number of embeddings");
+  }
+
+  const offsets: number[] = [];
+  for (const embedding of embeddings) {
+    const offset = await vectorStore.appendVector(embedding);
+    offsets.push(offset);
+  }
+
+  const pages = await Promise.all(
+    chunks.map(async (content, idx) => {
+      const prevPageId = idx > 0 ? pageIds[idx - 1] : null;
+      const nextPageId = idx < pageIds.length - 1 ? pageIds[idx + 1] : null;
+
+      return buildPage({
+        content,
+        embedding: embeddings[idx],
+        embeddingOffset: offsets[idx],
+        embeddingDim: modelProfile.embeddingDimension,
+        creatorPubKey: keyPair.publicKey,
+        signingKey: keyPair.signingKey,
+        prevPageId,
+        nextPageId,
+        createdAt,
+      });
+    }),
+  );
+
+  // Persist pages and activity records.
+  for (const page of pages) {
+    await metadataStore.putPage(page);
+    await metadataStore.putPageActivity({
+      pageId: page.pageId,
+      queryHitCount: 0,
+      lastQueryAt: createdAt,
+    });
+  }
+
+  // Build a simple book containing all pages.
+  const bookId = await hashText(pageIds.join("|"));
+  const book: Book = {
+    bookId,
+    pageIds,
+    medoidPageId: pageIds[0],
+    meta: {},
+  };
+  await metadataStore.putBook(book);
+
+  // Run hotpath promotion for the newly ingested pages.
+  await runPromotionSweep(pageIds, metadataStore);
+
+  return { pages, book };
+}

hippocampus/PageBuilder.ts

@@ -0,0 +1,89 @@
+import type { Hash, Page } from "../core/types";
+import type { KeyPair } from "../core/crypto/sign";
+import { hashBinary, hashText } from "../core/crypto/hash";
+import { signData } from "../core/crypto/sign";
+
+export interface BuildPageOptions {
+  content: string;
+  embedding: Float32Array;
+  embeddingOffset: number;
+  embeddingDim: number;
+  creatorPubKey: string;
+  signingKey: CryptoKey;
+  prevPageId?: Hash | null;
+  nextPageId?: Hash | null;
+  createdAt?: string;
+}
+
+/**
+ * Build a Page entity from content + embedding.
+ *
+ * Creates deterministic `pageId`/`contentHash` from content, a `vectorHash` from
+ * the raw embedding bytes, and signs the page using the provided key.
+ */
+export async function buildPage(options: BuildPageOptions): Promise<Page> {
+  const {
+    content,
+    embedding,
+    embeddingOffset,
+    embeddingDim,
+    creatorPubKey,
+    signingKey,
+    prevPageId = null,
+    nextPageId = null,
+    createdAt = new Date().toISOString(),
+  } = options;
+
+  if (embedding.length !== embeddingDim) {
+    throw new Error(
+      `Embedding dimension mismatch: expected ${embeddingDim}, got ${embedding.length}`,
+    );
+  }
+
+  const contentHash = await hashText(content);
+  const pageId = contentHash;
+
+  const rawVector = embedding.buffer.slice(
+    embedding.byteOffset,
+    embedding.byteOffset + embedding.byteLength,
+  );
+  const vectorHash = await hashBinary(rawVector);
+
+  const unsignedPage = {
+    pageId,
+    content,
+    embeddingOffset,
+    embeddingDim,
+    contentHash,
+    vectorHash,
+    prevPageId: prevPageId ?? null,
+    nextPageId: nextPageId ?? null,
+    creatorPubKey,
+    createdAt,
+  } as const;
+
+  // Deterministic canonical representation used for signing.
+  const canonical = canonicalizePageForSigning(unsignedPage);
+  const signature = await signData(canonical, signingKey);
+
+  return {
+    ...unsignedPage,
+    signature,
+  };
+}
+
+function canonicalizePageForSigning(page: Omit<Page, "signature">): string {
+  // Keep key order stable for deterministic signing.
+  return JSON.stringify({
+    pageId: page.pageId,
+    content: page.content,
+    embeddingOffset: page.embeddingOffset,
+    embeddingDim: page.embeddingDim,
+    contentHash: page.contentHash,
+    vectorHash: page.vectorHash,
+    prevPageId: page.prevPageId ?? null,
+    nextPageId: page.nextPageId ?? null,
+    creatorPubKey: page.creatorPubKey,
+    createdAt: page.createdAt,
+  });
+}

tests/hippocampus/Ingest.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, it, beforeEach } from "vitest";
+import { IDBFactory, IDBKeyRange as FakeIDBKeyRange } from "fake-indexeddb";
+
+import { IndexedDbMetadataStore } from "../../storage/IndexedDbMetadataStore";
+import { MemoryVectorStore } from "../../storage/MemoryVectorStore";
+import { DeterministicDummyEmbeddingBackend } from "../../embeddings/DeterministicDummyEmbeddingBackend";
+import { EmbeddingRunner } from "../../embeddings/EmbeddingRunner";
+import { generateKeyPair } from "../../core/crypto/sign";
+import { ingestText } from "../../hippocampus/Ingest";
+import type { ModelProfile } from "../../core/ModelProfile";
+
+let dbCounter = 0;
+function freshDbName(): string {
+  return `cortex-ingest-test-${Date.now()}-${++dbCounter}`;
+}
+
+describe("hippocampus ingest", () => {
+  beforeEach(() => {
+    (globalThis as Record<string, unknown>)["indexedDB"] = new IDBFactory();
+    (globalThis as Record<string, unknown>)["IDBKeyRange"] = FakeIDBKeyRange;
+  });
+
+  it("persists pages, metadata, and book records", async () => {
+    const metadataStore = await IndexedDbMetadataStore.open(freshDbName());
+    const vectorStore = new MemoryVectorStore();
+    const keyPair = await generateKeyPair();
+
+    const backend = new DeterministicDummyEmbeddingBackend({ dimension: 4 });
+    const runner = new EmbeddingRunner(async () => ({
+      backend,
+      selectedKind: "dummy" as const,
+      reason: "forced" as const,
+      supportedKinds: ["dummy" as const],
+      measurements: [],
+    }));
+
+    const profile: ModelProfile = {
+      modelId: "test-model",
+      embeddingDimension: 4,
+      contextWindowTokens: 64,
+      truncationTokens: 48,
+      maxChunkTokens: 5,
+      source: "metadata",
+    };
+
+    const text = "One two three four five six seven eight nine ten.";
+
+    const result = await ingestText(text, {
+      modelProfile: profile,
+      embeddingRunner: runner,
+      vectorStore,
+      metadataStore,
+      keyPair,
+    });
+
+    expect(result.pages.length).toBeGreaterThanOrEqual(2);
+
+    // Stored page should match returned page
+    const stored = await metadataStore.getPage(result.pages[0].pageId);
+    expect(stored).toEqual(result.pages[0]);
+
+    // Activity record should be initialized
+    const activity = await metadataStore.getPageActivity(result.pages[0].pageId);
+    expect(activity).toEqual({
+      pageId: result.pages[0].pageId,
+      queryHitCount: 0,
+      lastQueryAt: result.pages[0].createdAt,
+    });
+
+    // Book should contain the pages
+    const storedBook = await metadataStore.getBook(result.book!.bookId);
+    expect(storedBook).toEqual(result.book);
+
+    // Vector store should have data stored for each page
+    expect(vectorStore.byteLength).toBeGreaterThan(0);
+  });
+});