feat(P2): Daydreamer background consolidation, community detection, and P2P sharing

PLAN.md

@@ -113,14 +113,18 @@ This document tracks the implementation status of each major module in CORTEX. I
 
 | Module | Status | Files | Notes |
 |--------|--------|-------|-------|
+| Idle Scheduler | ✅ Complete | `daydreamer/IdleScheduler.ts` | Cooperative background loop; interruptible; respects CPU budget |
+| Hebbian Updates | ✅ Complete | `daydreamer/HebbianUpdater.ts` | LTP (strengthen), LTD (decay), prune below threshold; recompute σ(v) for changed nodes; run promotion/eviction sweep |
+| Prototype Recomputation | ✅ Complete | `daydreamer/PrototypeRecomputer.ts` | Recalculate volume/shelf medoids and centroids; recompute salience for affected entries; run tier-quota promotion/eviction |
+| Full Neighbor Graph Recalc | ✅ Complete | `daydreamer/FullNeighborRecalc.ts` | Rebuild bounded neighbor lists for dirty volumes; batch size bounded by O(√(t log t)) per idle cycle; recompute salience after recalc. |
 | Idle Scheduler | ❌ Missing | `daydreamer/IdleScheduler.ts` (planned) | Cooperative background loop; interruptible; respects CPU budget |
 | Hebbian Updates | ❌ Missing | `daydreamer/HebbianUpdater.ts` (planned) | LTP (strengthen), LTD (decay), prune below threshold; recompute σ(v) for changed nodes; run promotion/eviction sweep |
 | Prototype Recomputation | ❌ Missing | `daydreamer/PrototypeRecomputer.ts` (planned) | Recalculate volume/shelf medoids and centroids; recompute salience for affected entries; run tier-quota promotion/eviction |
 | Full Neighbor Graph Recalc | ❌ Missing | `daydreamer/FullNeighborRecalc.ts` (planned) | Rebuild bounded neighbor lists for dirty volumes; batch size bounded by O(√(t log t)) per idle cycle; recompute salience after recalc. |
 | Experience Replay | ❌ Missing | `daydreamer/ExperienceReplay.ts` (planned) | Simulate queries to reinforce connections |
-| Cluster Stability | ❌ Missing | `daydreamer/ClusterStability.ts` (planned) | Detect/trigger split/merge for unstable clusters; run lightweight label propagation for community detection; store community labels in PageActivity |
+| Cluster Stability | ✅ Complete | `daydreamer/ClusterStability.ts` | Lightweight label propagation for community detection; stores community labels in PageActivity; detects oversized and empty communities |
 
-**Daydreamer Status:** 0/6 complete (0%)
+**Daydreamer Status:** 4/6 complete (66%)
 
 **Note:** Not a v1 blocker — system can ship without background consolidation (manual recalc only). Community detection is required before graph-community quota enforcement is active.
 
@@ -311,34 +315,35 @@ This document tracks the implementation status of each major module in CORTEX. I
 
 **Goal:** Idle maintenance keeps memory healthy, community-aware hotpath coverage stays diverse, and privacy-safe interest sharing is available.
 
-1. **Idle Scheduler** (`daydreamer/IdleScheduler.ts`)
+1. **Idle Scheduler** (`daydreamer/IdleScheduler.ts`) ✅ Complete
    - Cooperative, interruptible loop
    - CPU budget awareness
 
-2. **Hebbian Updater** (`daydreamer/HebbianUpdater.ts`)
+2. **Hebbian Updater** (`daydreamer/HebbianUpdater.ts`) ✅ Complete
    - LTP/LTD rules; edge pruning
    - Recompute σ(v) for changed nodes; run promotion/eviction sweep
 
-3. **Full Neighbor Graph Recalc** (`daydreamer/FullNeighborRecalc.ts`)
+3. **Full Neighbor Graph Recalc** (`daydreamer/FullNeighborRecalc.ts`) ✅ Complete
    - Rebuild neighbor lists for dirty volumes
    - O(√(t log t)) batch size per idle cycle
 
-4. **Prototype Recomputer** (`daydreamer/PrototypeRecomputer.ts`)
+4. **Prototype Recomputer** (`daydreamer/PrototypeRecomputer.ts`) ✅ Complete
    - Update volume/shelf prototypes
    - Tier-quota promotion/eviction after recomputation
 
-5. **Community Detection** (`daydreamer/ClusterStability.ts` — extend)
+5. **Community Detection** (`daydreamer/ClusterStability.ts`) ✅ Complete
    - Label propagation on semantic neighbor graph
    - Store community labels in `PageActivity.communityId`
-   - Wire community IDs into `SalienceEngine` promotion/eviction
+   - Community IDs wired into `SalienceEngine` promotion/eviction (already implemented in P0)
 
-6. **Smart Interest Sharing** (`sharing/*` planned)
+6. **Smart Interest Sharing** (`sharing/*`) ✅ Complete
    - `sharing/EligibilityClassifier.ts` — classify candidate nodes for share eligibility; block identity/PII-bearing nodes
    - `sharing/SubgraphExporter.ts` — export signed, topic-scoped graph slices from eligible nodes only
    - `sharing/SubgraphImporter.ts` — verify signatures/provenance and merge imported slices into local discovery index
    - `sharing/PeerExchange.ts` — opt-in peer transport for exchanging eligible graph slices
+   - `sharing/CuriosityBroadcaster.ts` — rate-limited broadcast of curiosity probes with fragment response handling
 
-**Exit Criteria:** System self-maintains over extended use; community-aware hotpath quotas enforced; privacy-safe smart sharing works end-to-end.
+**Exit Criteria:** System self-maintains over extended use; community-aware hotpath quotas enforced; privacy-safe smart sharing works end-to-end. ✅ **ACHIEVED**
 
 ---
 

TODO.md

@@ -503,13 +503,13 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** Need cooperative background loop that doesn't block foreground.
 
-- [ ] **P2-A1:** Implement `daydreamer/IdleScheduler.ts`
+- [x] **P2-A1:** Implement `daydreamer/IdleScheduler.ts`
   - Loop via `requestIdleCallback` (browser) or `setImmediate` (Node)
   - Interruptible (yield after N ms of work)
   - CPU budget awareness (pause if main thread busy)
   - Task queue (prioritize high-value work)
 
-- [ ] **P2-A2:** Add scheduler test coverage
+- [x] **P2-A2:** Add scheduler test coverage
   - `tests/daydreamer/IdleScheduler.test.ts`
   - Test cooperative yielding
   - Test interruption doesn't corrupt state
@@ -522,15 +522,15 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** Strengthen useful connections, decay unused ones. Edge changes alter σ(v) values and can trigger hotpath promotions or evictions.
 
-- [ ] **P2-B1:** Implement `daydreamer/HebbianUpdater.ts`
+- [x] **P2-B1:** Implement `daydreamer/HebbianUpdater.ts`
   - LTP: strengthen edges traversed during successful queries
   - LTD: decay all edges by small factor each pass
   - Prune: remove edges below threshold; keep Metroid degree within `HotpathPolicy`-derived bounds
   - After LTP/LTD: recompute σ(v) for all nodes whose incident edges changed (via `SalienceEngine.batchComputeSalience`)
   - Run promotion/eviction sweep for changed nodes via `SalienceEngine.runPromotionSweep`
   - Update `MetadataStore.putEdges`
 
-- [ ] **P2-B2:** Add Hebbian test coverage
+- [x] **P2-B2:** Add Hebbian test coverage
   - `tests/daydreamer/HebbianUpdater.test.ts`
   - Test strengthen increases weight
   - Test decay decreases weight
@@ -546,15 +546,15 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** Incremental fast semantic neighbor insert is approximate; need periodic full recalc. Recalc batch size must be bounded by H(t)-derived maintenance budget to avoid blocking the idle loop.
 
-- [ ] **P2-C1:** Implement `daydreamer/FullNeighborRecalc.ts`
+- [x] **P2-C1:** Implement `daydreamer/FullNeighborRecalc.ts`
   - Query `MetadataStore.needsNeighborRecalc(volumeId)` for dirty volumes; prioritise dirtiest first
   - Load all pages in volume; compute pairwise similarities
   - Bound batch: process at most `HotpathPolicy.computeCapacity(graphMass)` pairwise comparisons per idle cycle (O(√(t log t)))
   - Select policy-derived max neighbors for each page; update `MetadataStore.putSemanticNeighbors`
   - Clear dirty flag via `MetadataStore.clearNeighborRecalcFlag`
   - Recompute σ(v) for affected nodes via `SalienceEngine.batchComputeSalience`; run promotion sweep
 
-- [ ] **P2-C2:** Add neighbor graph recalc test coverage
+- [x] **P2-C2:** Add neighbor graph recalc test coverage
   - `tests/daydreamer/FullNeighborRecalc.test.ts`
   - Test dirty flag cleared after recalc
   - Test neighbor quality improved vs fast insert
@@ -569,14 +569,14 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** Keep volume/shelf prototypes accurate as pages/books change. Prototype updates change which entries should occupy the volume and shelf tier quotas.
 
-- [ ] **P2-D1:** Implement `daydreamer/PrototypeRecomputer.ts`
+- [x] **P2-D1:** Implement `daydreamer/PrototypeRecomputer.ts`
   - Recompute volume medoids (select medoid page per volume)
   - Recompute volume centroids (average of book embeddings)
   - Recompute shelf routing prototypes
   - Update vectors in `VectorStore` (append new, update offsets)
   - After recomputing each level: recompute salience for affected representative entries via `SalienceEngine`; run tier-quota promotion/eviction for that tier
 
-- [ ] **P2-D2:** Add prototype recomputer test coverage
+- [x] **P2-D2:** Add prototype recomputer test coverage
   - `tests/daydreamer/PrototypeRecomputer.test.ts`
   - Test medoid selection algorithm
   - Test centroid computation
@@ -590,7 +590,7 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** Validate Daydreamer improves system health and hotpath stays consistent.
 
-- [ ] **P2-E1:** Implement `tests/integration/Daydreamer.test.ts`
+- [x] **P2-E1:** Implement `tests/integration/Daydreamer.test.ts`
   - Ingest corpus
   - Run queries (generate edge traversals and PageActivity updates)
   - Run Daydreamer for N passes
@@ -607,20 +607,20 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** Without community detection, a single dense topic can fill the entire page-tier quota, crowding out unrelated memories. Community quotas ensure the hotpath is both hot (high salience) and diverse (topic-representative).
 
-- [ ] **P2-F1:** Add community detection to `daydreamer/ClusterStability.ts`
+- [x] **P2-F1:** Add community detection to `daydreamer/ClusterStability.ts`
   - Implement lightweight label propagation on the semantic neighbor graph
   - Run during idle passes when dirty-volume flags indicate meaningful structural change
   - Store community labels in `PageActivity.communityId` via `MetadataStore.putPageActivity`
   - Rerun when graph topology changes significantly (post-split, post-merge, post-full-recalc)
 
-- [ ] **P2-F2:** Wire community labels into `SalienceEngine` promotion/eviction
+- [x] **P2-F2:** Wire community labels into `SalienceEngine` promotion/eviction
   - `selectEvictionTarget` uses `communityId` to find weakest resident in the community bucket
   - Promotion checks community quota remaining before admitting
   - If community quota is full: candidate must beat weakest resident in that community
   - If community is unknown (`communityId` not yet set): place node in temporary pending pool borrowing from page-tier budget
   - Empty communities release their slots back to the page-tier budget
 
-- [ ] **P2-F3:** Add community-aware eviction tests
+- [x] **P2-F3:** Add community-aware eviction tests
   - `tests/daydreamer/ClusterStability.test.ts`
   - Test that a single dense community cannot consume all page-tier hotpath slots
   - Test that a new community (previously unknown) receives at least one slot
@@ -635,31 +635,31 @@ These items add idle background maintenance and privacy-safe interest sharing. T
 
 **Why:** When knowledge gaps are detected, CORTEX must be able to broadcast the incomplete Metroid as a curiosity probe to connected peers. Peers respond with relevant fragments, enabling collaborative learning. Additionally, interest sharing is a core product value for both app and library surfaces. v1 must share public-interest graph sections while preventing personal data leakage.
 
-- [ ] **P2-G0:** Implement `sharing/CuriosityBroadcaster.ts`
+- [x] **P2-G0:** Implement `sharing/CuriosityBroadcaster.ts`
   - Consume pending `CuriosityProbe` objects queued by `KnowledgeGapDetector`
   - Serialize and broadcast to connected peers via P2P transport
   - Handle responses: deserialize incoming graph fragments; pass to `SubgraphImporter` for integration
   - Rate-limit broadcasts to prevent spam
   - Include `knowledgeBoundary` field in probe so peers can target search precisely
 
-- [ ] **P2-G1:** Implement `sharing/EligibilityClassifier.ts`
+- [x] **P2-G1:** Implement `sharing/EligibilityClassifier.ts`
   - Classify candidate nodes as share-eligible vs blocked before export
   - Detect identity/PII-bearing content (person-specific identifiers, credentials, financial/health traces)
   - Emit deterministic eligibility decisions with reason codes for auditability
 
-- [ ] **P2-G2:** Implement `sharing/SubgraphExporter.ts`
+- [x] **P2-G2:** Implement `sharing/SubgraphExporter.ts`
   - Build topic-scoped graph slices from eligible nodes only
   - For curiosity responses: select graph fragment relevant to the received probe's `knowledgeBoundary`
   - Preserve node/edge signatures and provenance
   - Strip or coarsen personal metadata fields that are not needed for discovery
 
-- [ ] **P2-G3:** Implement `sharing/PeerExchange.ts` and `sharing/SubgraphImporter.ts`
+- [x] **P2-G3:** Implement `sharing/PeerExchange.ts` and `sharing/SubgraphImporter.ts`
   - Opt-in peer exchange over P2P transport
   - Verify signatures and schema on import; reject invalid or tampered payloads
   - Merge imported slices into discovery pathways without exposing sender identity metadata
   - After import, retry MetroidBuilder for any pending knowledge gaps that may be resolved by new data
 
-- [ ] **P2-G4:** Add sharing safety and discovery tests
+- [x] **P2-G4:** Add sharing safety and discovery tests
   - `tests/sharing/EligibilityClassifier.test.ts`
   - `tests/sharing/CuriosityBroadcaster.test.ts`
   - `tests/sharing/SubgraphExchange.test.ts`

core/crypto/uuid.ts

@@ -0,0 +1,48 @@
+// ---------------------------------------------------------------------------
+// uuid.ts — Minimal UUID v4 generation utility
+// ---------------------------------------------------------------------------
+//
+// Generates a RFC 4122 version 4 (random) UUID.
+// Uses crypto.randomUUID() when available (browsers and modern Node/Bun),
+// with a pure-JS fallback for environments that lack it.
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate a RFC 4122 version 4 UUID string.
+ *
+ * Prefers the platform's built-in crypto.randomUUID() when available,
+ * falling back to a pure-JS crypto.getRandomValues() implementation.
+ */
+export function randomUUID(): string {
+  if (
+    typeof crypto !== "undefined" &&
+    typeof (crypto as { randomUUID?: () => string }).randomUUID === "function"
+  ) {
+    return (crypto as { randomUUID: () => string }).randomUUID();
+  }
+
+  // Fallback: manually construct UUID v4 from random bytes
+  const bytes = new Uint8Array(16);
+  if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") {
+    crypto.getRandomValues(bytes);
+  } else {
+    // No secure RNG available: refuse to generate a UUID with weak randomness
+    throw new Error(
+      "randomUUID() requires a secure crypto.getRandomValues implementation; " +
+        "no suitable crypto API was found in this environment."
+    );
+  }
+
+  // Set version bits (v4) and variant bits (RFC 4122)
+  bytes[6] = (bytes[6] & 0x0f) | 0x40;
+  bytes[8] = (bytes[8] & 0x3f) | 0x80;
+
+  const hex = [...bytes].map((b) => b.toString(16).padStart(2, "0"));
+  return (
+    hex.slice(0, 4).join("") +
+    "-" + hex.slice(4, 6).join("") +
+    "-" + hex.slice(6, 8).join("") +
+    "-" + hex.slice(8, 10).join("") +
+    "-" + hex.slice(10).join("")
+  );
+}

core/types.ts

@@ -164,6 +164,8 @@ export interface MetadataStore {
 
   putVolume(volume: Volume): Promise<void>;
   getVolume(volumeId: Hash): Promise<Volume | undefined>;
+  /** Returns all volumes in the store. */
+  getAllVolumes(): Promise<Volume[]>;
   /**
    * Delete a volume record and clean up all reverse-index entries
    * (`bookToVolume` for each book in the volume, and the `volumeToShelf` entry).
@@ -174,9 +176,13 @@ export interface MetadataStore {
 
   putShelf(shelf: Shelf): Promise<void>;
   getShelf(shelfId: Hash): Promise<Shelf | undefined>;
+  /** Returns all shelves in the store. */
+  getAllShelves(): Promise<Shelf[]>;
 
   // --- Hebbian edges ---
   putEdges(edges: Edge[]): Promise<void>;
+  /** Remove a single directed edge. */
+  deleteEdge(fromPageId: Hash, toPageId: Hash): Promise<void>;
   getNeighbors(pageId: Hash, limit?: number): Promise<Edge[]>;
 
   // --- Reverse-index helpers ---