Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@ These items **must** be completed to have a usable system. Without them, users c
- Link pages via `prevPageId`/`nextPageId`
- Initialise `PageActivity` records with zero counts

- [ ] **P0-C2:** Implement `hippocampus/Ingest.ts` (minimal version)
- Entry point: `ingestText(text, modelProfile, vectorStore, metadataStore, keyPair)`
- [x] **P0-C2:** Implement `hippocampus/Ingest.ts` (minimal version)
- Entry point: `ingestText(text, { modelProfile, embeddingRunner, vectorStore, metadataStore, keyPair, ... })`
- Chunk text via `Chunker`
- Batch embed chunks via `EmbeddingRunner`
- Persist vectors to `VectorStore`
Expand Down
3 changes: 2 additions & 1 deletion hippocampus/Ingest.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { Book, ModelProfile, MetadataStore, VectorStore } from "../core/types";
import type { Book, MetadataStore, VectorStore } from "../core/types";
import type { ModelProfile } from "../core/ModelProfile";
import { hashText } from "../core/crypto/hash";
import type { KeyPair } from "../core/crypto/sign";
import { EmbeddingRunner } from "../embeddings/EmbeddingRunner";
Expand Down
9 changes: 4 additions & 5 deletions hippocampus/PageBuilder.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import type { Hash, Page } from "../core/types";
import type { KeyPair } from "../core/crypto/sign";
import { hashBinary, hashText } from "../core/crypto/hash";
import { signData } from "../core/crypto/sign";

Expand Down Expand Up @@ -43,10 +42,10 @@ export async function buildPage(options: BuildPageOptions): Promise<Page> {
const contentHash = await hashText(content);
const pageId = contentHash;

const rawVector = embedding.buffer.slice(
embedding.byteOffset,
embedding.byteOffset + embedding.byteLength,
);
// Copy into a new ArrayBuffer-backed view so we never pass a SharedArrayBuffer
// into WebCrypto (and keep TypeScript happy).
const rawVector = new Uint8Array(embedding.byteLength);
rawVector.set(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength));
const vectorHash = await hashBinary(rawVector);

const unsignedPage = {
Expand Down
7 changes: 4 additions & 3 deletions tests/hippocampus/PageBuilder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { generateKeyPair } from "../../core/crypto/sign";
import { verifySignature } from "../../core/crypto/verify";
import { hashBinary, hashText } from "../../core/crypto/hash";

function canonicalizePageForSigning(page: Omit<Page, "signature">) {
function canonicalizePageForSigning(page: Page) {
return JSON.stringify({
pageId: page.pageId,
content: page.content,
Expand Down Expand Up @@ -44,9 +44,10 @@ describe("buildPage", () => {
expect(page.pageId).toBe(expectedContentHash);
expect(page.contentHash).toBe(expectedContentHash);

const rawVector = embedding.buffer.slice(
const rawVector = new Uint8Array(
embedding.buffer,
embedding.byteOffset,
embedding.byteOffset + embedding.byteLength,
embedding.byteLength,
);
const expectedVectorHash = await hashBinary(rawVector);
expect(page.vectorHash).toBe(expectedVectorHash);
Expand Down
Loading