Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions src/dashboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,34 @@ export interface DashboardDeps {
agents?: () => AgentRollup[];
/** Optional: the compounding wiki-brain (leg 5). */
wiki?: WikiStore;
/** Optional: G1 activity-ledger X-ray — SAME math as the stop-hook receipt. */
xray?: () => XrayState | null;
}

/** Per-source rollup of the current session's activity events, plus the exact
* receipt text (buildReceipt) — the dashboard shows the SAME honest numbers
* the stop-hook prints, never a parallel aggregation. */
export interface XrayState {
bySource: Record<string, { events: number; raw: number; stored: number; saved: number }>;
receipt: string;
sessionStart: string | null;
trimmed: boolean;
}

/** Pure reduce over session events — one bucket per source; absent `source`
* is a legacy (pre-G1) event and counts as "mcp". */
export function xrayState(events: ActivityEvent[], receipt: string, sessionStart: string | null, trimmed: boolean): XrayState {
const bySource: XrayState["bySource"] = {};
for (const e of events) {
const key = e.source ?? "mcp";
const b = bySource[key] ?? { events: 0, raw: 0, stored: 0, saved: 0 };
b.events += 1;
b.raw += e.rawTokens ?? 0;
b.stored += e.storedTokens ?? 0;
b.saved += e.saved ?? 0;
bySource[key] = b;
}
return { bySource, receipt, sessionStart, trimmed };
}

/** Knowledge-graph summary: file count + the highest-fanout files (blast radius). */
Expand Down Expand Up @@ -150,6 +178,7 @@ export function dashboardState(deps: DashboardDeps): Record<string, unknown> {
recentLearnings: learnings.slice(-5).reverse().map((l) => ({ date: l.date, summary: l.summary.slice(0, 160) })),
knowledge: deps.knowledge ? knowledgeSummary(deps.knowledge) : null,
wiki: deps.wiki ? wikiState(deps.wiki) : null,
xray: deps.xray?.() ?? null,
skills: deps.skills
? deps.skills.list().map((s) => ({ name: s.name, uses: s.uses, triggers: s.triggers.slice(0, 6), updatedAt: s.updatedAt }))
: null,
Expand Down Expand Up @@ -200,6 +229,13 @@ const PAGE = `<!doctype html>
<div class="card" style="margin-top:.8rem"><div class="label">Knowledge graph (top blast radius)</div><div class="advice" id="kfiles"></div><table id="kg"></table></div>
<div class="card" style="margin-top:.8rem"><div class="label">Skills</div><table id="skills"></table></div>
<div class="card" style="margin-top:.8rem"><div class="label">Recent learnings</div><table id="recent"></table></div>
<div class="card" style="margin-top:.8rem"><div class="label">X-ray — G1 activity ledger (same math as the stop receipt)</div>
<div class="advice" id="xray-empty">no session data</div>
<div id="xray-wrap" style="display:none">
<table id="xray"></table>
<pre id="xray-receipt" style="white-space:pre-wrap; margin-top:.5rem; background:#161b22; padding:.5rem; border-radius:6px"></pre>
</div>
</div>
<div class="card" style="margin-top:.8rem"><div class="label">Team board</div><table id="board"></table></div>
<div class="card" style="margin-top:.8rem"><div class="label">Wiki-brain (browsable · click a page or a [[link]])</div>
<div id="wiki-empty" class="advice">no wiki yet — ingest with knitbrain_wiki_ingest</div>
Expand Down Expand Up @@ -300,6 +336,17 @@ async function tick() {
(s.recentLearnings.length ? s.recentLearnings.map(l => \`<tr><td>\${esc(l.date)}</td><td>\${esc(l.summary)}</td></tr>\`).join("") : "<tr><td colspan=2>—</td></tr>");
document.getElementById("board").innerHTML = "<tr><th>who</th><th>when</th><th>finding</th></tr>" +
(s.board.length ? s.board.map(b => \`<tr><td>\${esc(b.author)}</td><td>\${esc(b.ts.slice(11,19))}</td><td>\${esc(b.summary)}</td></tr>\`).join("") : "<tr><td colspan=3>—</td></tr>");
if (s.xray) {
document.getElementById("xray-empty").style.display = "none";
document.getElementById("xray-wrap").style.display = "";
const rows = Object.entries(s.xray.bySource);
document.getElementById("xray").innerHTML = "<tr><th>source</th><th>events</th><th>raw→stored</th><th>saved</th></tr>" +
(rows.length ? rows.map(([src, b]) => \`<tr><td>\${esc(src)}</td><td>\${b.events}</td><td>\${b.raw.toLocaleString()} → \${b.stored.toLocaleString()}</td><td>\${b.saved.toLocaleString()}</td></tr>\`).join("") : "<tr><td colspan=4>—</td></tr>");
document.getElementById("xray-receipt").textContent = s.xray.receipt;
} else {
document.getElementById("xray-empty").style.display = "";
document.getElementById("xray-wrap").style.display = "none";
}
if (s.wiki) {
wikiPages = s.wiki.pages || [];
wikiEdges = s.wiki.edges || [];
Expand Down
21 changes: 20 additions & 1 deletion src/engine/receipt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ export interface SessionMark {
savedAtStart: number;
usedAtStart: number;
retrievalsAtStart: number;
/** Output-side: cumulative model output tokens at session start (transcript
* probe) — lets the receipt PROVE actual output written this session. */
outputAtStart?: number;
reads: Record<string, { count: number; mtimeMs: number }>;
redirects: Record<string, number>;
}
Expand All @@ -41,14 +44,15 @@ function loadMark(root: string): SessionMark | null {
* zero-point so the receipt can report SESSION deltas, not lifetime totals. */
export function markSessionStart(
root: string,
snap: { savedTokens: number; usedTokens: number; retrievals: number },
snap: { savedTokens: number; usedTokens: number; retrievals: number; outputTokens?: number },
): void {
mkdirSync(root, { recursive: true });
const mark: SessionMark = {
startTs: new Date().toISOString(),
savedAtStart: snap.savedTokens,
usedAtStart: snap.usedTokens,
retrievalsAtStart: snap.retrievals,
...(snap.outputTokens !== undefined ? { outputAtStart: snap.outputTokens } : {}),
reads: {},
redirects: {},
};
Expand Down Expand Up @@ -182,6 +186,10 @@ export interface ReceiptInput {
eventsTrimmed: boolean;
/** Lifetime TOIN retrieval count (caller sums feedback stats). */
retrievalsTotal: number;
/** Output-side: cumulative model output tokens NOW (transcript probe). */
outputTokensNow?: number;
/** Whether a terse/caveman output mode is detectably active. */
terseActive?: boolean;
/** Injected clock for tests. */
now?: () => number;
}
Expand Down Expand Up @@ -251,6 +259,17 @@ export function buildReceipt(i: ReceiptInput): string {
// G6 forecast: only when it's a plan-billed session, we have a marker, the
// session has run long enough to trust a burn rate, and there's something
// to project (avoided > 0) — otherwise it's noise or a divide-by-garbage.
// Output side: we can PROVE what was written and that terse was on — but a
// verbose counterfactual is unknowable, so nothing here enters 'avoided'.
if (mark?.outputAtStart !== undefined && i.outputTokensNow !== undefined) {
const written = Math.max(0, i.outputTokensNow - mark.outputAtStart);
if (written > 0) {
lines.push(
`output written: ~${fmtTokens(written)} tok${i.terseActive ? " with terse mode ON (real output-side savings; no provable counterfactual — not counted in avoided)" : ""}`,
);
}
}

// G5 dollars: api-billing only — plan users' cost is quota, never shown $.
if (meter.billingMode === "api" && avoided > 0 && meter.model) {
const rate = RATE_PER_MTOK.find((r) => r.match.test(meter.model!));
Expand Down
26 changes: 23 additions & 3 deletions src/hooks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { createWikiStore } from "../engine/wiki.js";
import { createActivityLog } from "../engine/activity.js";
import { createFeedback } from "../engine/feedback.js";
import { markSessionStart, readSessionMark, recordRead, recordRedirect, buildReceipt } from "../engine/receipt.js";
import { currentContextTokens, currentContextModel } from "../engine/usage.js";
import { currentContextTokens, currentContextModel, readProjectUsage } from "../engine/usage.js";
import { ccrRoot, memoryRoot, meterRoot, wikiRoot, loopStatePath, activityRoot, feedbackRoot } from "../paths.js";
import { decideLoopStop } from "./stop.js";
import { decidePostToolUse, type PostToolUseInput } from "./posttooluse.js";
Expand Down Expand Up @@ -232,7 +232,13 @@ async function main(): Promise<void> {
const retrievals = createFeedback(feedbackRoot())
.stats()
.reduce((sum, s) => sum + s.retrievals, 0);
markSessionStart(meterRoot(), { savedTokens: r.savedTokens, usedTokens: r.usedTokens, retrievals });
const outputTokens = readProjectUsage(process.cwd())?.outputTokens;
markSessionStart(meterRoot(), {
savedTokens: r.savedTokens,
usedTokens: r.usedTokens,
retrievals,
...(outputTokens !== undefined ? { outputTokens } : {}),
});
} catch {
/* never break session start */
}
Expand Down Expand Up @@ -278,7 +284,21 @@ async function main(): Promise<void> {
const retrievalsTotal = createFeedback(feedbackRoot())
.stats()
.reduce((sum, s) => sum + s.retrievals, 0);
const receipt = buildReceipt({ meter: meterReading, mark, events, eventsTrimmed: trimmed, retrievalsTotal });
// Output-side proof: actual tokens written + terse-active detection
// (env or the project's terse rules file) — shown, never counted.
const outputTokensNow = readProjectUsage(process.cwd())?.outputTokens;
const terseActive =
(process.env["KNITBRAIN_TERSE"] ?? "") !== "" ||
existsSync(".claude/rules/knitbrain.md");
const receipt = buildReceipt({
meter: meterReading,
mark,
events,
eventsTrimmed: trimmed,
retrievalsTotal,
...(outputTokensNow !== undefined ? { outputTokensNow } : {}),
terseActive,
});
const out = adaptOutput(platform, "stop", { systemMessage: receipt });
if (out) process.stdout.write(JSON.stringify(out));
} catch {
Expand Down
78 changes: 55 additions & 23 deletions src/hub/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,39 +38,71 @@ export function loadHubConfig(): HubConfig | null {
* Mirror a board posting to the hub — FIRE-AND-FORGET. Never throws, never
* blocks: a dead/unreachable hub must never slow local work.
*/
/** Backoff schedule between attempts (ms) — index i is the wait before attempt i+2. */
const MIRROR_BACKOFF_MS = [250, 1000];

export function mirrorToHub(
cfg: HubConfig,
entry: { author: string; summary: string; original: string },
): void {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 2000);
void fetch(`${cfg.url.replace(/\/+$/, "")}/board`, {
method: "POST",
headers: { "content-type": "application/json", authorization: `Bearer ${cfg.token}` },
body: JSON.stringify({ ...entry, author: cfg.member || entry.author }),
signal: controller.signal,
})
.catch(() => {})
.finally(() => clearTimeout(timer));
void (async () => {
const url = `${cfg.url.replace(/\/+$/, "")}/board`;
const body = JSON.stringify({ ...entry, author: cfg.member || entry.author });
for (let attempt = 0; attempt < 3; attempt++) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 2000);
try {
const res = await fetch(url, {
method: "POST",
headers: { "content-type": "application/json", authorization: `Bearer ${cfg.token}` },
body,
signal: controller.signal,
});
clearTimeout(timer);
if (res.ok) return;
// 4xx (bad token, body too large, etc.) will never succeed on retry.
if (res.status >= 400 && res.status < 500) return;
// else: 5xx — fall through to retry.
} catch {
clearTimeout(timer);
// network error / abort (timeout) — fall through to retry.
}
const wait = MIRROR_BACKOFF_MS[attempt];
if (wait !== undefined) await new Promise((r) => setTimeout(r, wait));
}
// fire-and-forget: all errors swallowed after the final attempt.
})();
}

/** Backoff between the two fetchHubBoard attempts (ms). */
const BOARD_FETCH_BACKOFF_MS = 300;

/** Best-effort hub board fetch (for the dashboard's merged view). */
export async function fetchHubBoard(
cfg: HubConfig,
timeoutMs = 1500,
): Promise<Array<{ id: string; author: string; summary: string; ts: string }>> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetch(`${cfg.url.replace(/\/+$/, "")}/board`, {
headers: { authorization: `Bearer ${cfg.token}` },
signal: controller.signal,
});
if (!res.ok) return [];
return (await res.json()) as Array<{ id: string; author: string; summary: string; ts: string }>;
} catch {
return [];
} finally {
clearTimeout(timer);
const url = `${cfg.url.replace(/\/+$/, "")}/board`;
for (let attempt = 0; attempt < 2; attempt++) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetch(url, {
headers: { authorization: `Bearer ${cfg.token}` },
signal: controller.signal,
});
if (!res.ok) {
// 4xx will never succeed on retry; 5xx falls through to retry.
if (res.status >= 400 && res.status < 500) return [];
} else {
return (await res.json()) as Array<{ id: string; author: string; summary: string; ts: string }>;
}
} catch {
// network error / abort (timeout) — fall through to retry.
} finally {
clearTimeout(timer);
}
if (attempt === 0) await new Promise((r) => setTimeout(r, BOARD_FETCH_BACKOFF_MS));
}
return [];
}
50 changes: 49 additions & 1 deletion src/hub/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
import { join } from "node:path";
import { containsSecret } from "../engine/cleanse.js";
import { writeAtomic } from "../atomic.js";

/**
* Knit Brain HUB — the shared-sessions server a team points at.
Expand Down Expand Up @@ -30,6 +31,15 @@ export interface Hub {
/** Cap request bodies so one authenticated client can't exhaust hub memory. */
const MAX_BODY_BYTES = 1_000_000;

/** Default fixed-window rate limit: 60 req / 10s per remote address. */
const DEFAULT_RATE_LIMIT = { max: 60, windowMs: 10_000 };

/** Default board size cap (entries) before the rare rewrite-to-newest-N kicks in. */
const MAX_BOARD_ENTRIES = 500;

/** Bound the rate-limit address map itself — a botnet of distinct IPs must not grow this unbounded. */
const MAX_RATE_LIMIT_ENTRIES = 1000;

// Credential detection shared with the brain-write cleanse (single source in
// engine/cleanse.ts). A board original is stored byte-exact for the team, so we
// REJECT (never silently scrub — that would corrupt the original) a posting that
Expand Down Expand Up @@ -57,7 +67,13 @@ function readBody(req: IncomingMessage): Promise<string | typeof TOO_LARGE> {
});
}

export function createHub(root: string): Hub {
export function createHub(
root: string,
opts?: { now?: () => number; rateLimit?: { max: number; windowMs: number }; maxEntries?: number },
): Hub {
const now = opts?.now ?? Date.now;
const rateLimit = opts?.rateLimit ?? DEFAULT_RATE_LIMIT;
const maxEntries = opts?.maxEntries ?? MAX_BOARD_ENTRIES;
mkdirSync(root, { recursive: true });
const tokenPath = join(root, "token.txt");
// Append-only log: one JSON entry per line. Posting is O(1) (append a line)
Expand Down Expand Up @@ -100,6 +116,34 @@ export function createHub(root: string): Hub {
const append = (entry: HubEntry): void => {
writeFileSync(boardPath, JSON.stringify(entry) + "\n", { encoding: "utf8", flag: "a" });
};
// Sole write-amplification point: appends are O(1), but once the log passes
// maxEntries we rewrite it down to the newest maxEntries (rare — only at cap).
const trimBoard = (): void => {
const all = load();
if (all.length <= maxEntries) return;
const kept = all.slice(all.length - maxEntries);
writeAtomic(boardPath, kept.map((e) => JSON.stringify(e)).join("\n") + "\n");
};

// Fixed-window rate limiter, per remote address. Map insertion order gives
// us oldest-inserted-first eviction for free (bounded regardless of window
// resets) — a simple Map, not an LRU, since we only ever need "oldest key".
const rateBuckets = new Map<string, { windowStart: number; count: number }>();
const rateLimited = (addr: string): boolean => {
const t = now();
let bucket = rateBuckets.get(addr);
if (!bucket || t - bucket.windowStart >= rateLimit.windowMs) {
bucket = { windowStart: t, count: 0 };
rateBuckets.delete(addr); // re-insert at the end (fresh window = fresh recency)
rateBuckets.set(addr, bucket);
}
bucket.count++;
if (rateBuckets.size > MAX_RATE_LIMIT_ENTRIES) {
const oldestKey = rateBuckets.keys().next().value;
if (oldestKey !== undefined) rateBuckets.delete(oldestKey);
}
return bucket.count > rateLimit.max;
};

// SECURITY: constant-time comparison — no timing oracle on the team token.
const expected = createHash("sha256").update(`Bearer ${token}`).digest();
Expand All @@ -118,6 +162,9 @@ export function createHub(root: string): Hub {
if (req.method === "GET" && req.url === "/health") {
return json(res, 200, { status: "healthy", server: "knitbrain-hub" });
}
if (rateLimited(req.socket.remoteAddress ?? "?")) {
return json(res, 429, { error: "rate-limited" });
}
if (!authed(req)) return json(res, 401, { error: "missing or invalid token" });

if (req.method === "POST" && req.url === "/board") {
Expand All @@ -143,6 +190,7 @@ export function createHub(root: string): Hub {
ts: new Date().toISOString(),
};
append(entry);
trimBoard();
return json(res, 200, { id: entry.id });
}
if (req.method === "GET" && req.url === "/board") {
Expand Down
Loading
Loading