Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 81 additions & 8 deletions src/engine/activity.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
import { writeAtomic } from "../atomic.js";

Expand All @@ -17,6 +17,18 @@ export interface ActivityEvent {
/** Tokens knitbrain saved on this call (0 if not compressed). Universal —
* measured by knitbrain itself, so it works for every platform/plan. */
saved: number;
/** Where the event came from. Absent = legacy = "mcp" (pre-G1 JSONL lines
* still parse — every new field here is optional). */
source?: "mcp" | "hook" | "proxy";
/** File path the event concerns (read/redirect/optimize target). */
file?: string;
/** Original payload size before compression, for sink reporting. */
rawTokens?: number;
/** Size after compression. */
storedTokens?: number;
/** G1 receipt classification: an optimize (compress) or a redirect
* (oversized raw read steered to knitbrain_read). */
kind?: "optimize" | "redirect";
}

/** Per-agent optimization rollup — works for ANY MCP agent (Cursor, VS Code,
Expand All @@ -30,17 +42,43 @@ export interface AgentRollup {

export interface ActivityLog {
/** Record one tool call. Best-effort, never throws (must not break a tool). */
record(e: { agent: string; tool: string; summary: string; saved?: number }): void;
record(
e: {
agent: string;
tool: string;
summary: string;
saved?: number;
} & Partial<Pick<ActivityEvent, "source" | "file" | "rawTokens" | "storedTokens" | "kind">>,
): void;
/** Most recent events, newest first. */
recent(n?: number): ActivityEvent[];
/** Per-agent rollup across the log (universal optimization meter). */
rollup(): AgentRollup[];
/** Events at/after `ts` (ISO, string-compare) — for a receipt scoped to a
* session marker. `trimmed` warns the caller the log rotated past the
* session start (totals should fall back to exact meter numbers). */
since(ts: string): { events: ActivityEvent[]; trimmed: boolean };
}

/** Keep at most this many events on disk. */
const CAP = 200;
const CAP = 1000;
/** Cheap guard so appends stay O(1): only pay for the full readAll+trim
* pass once the file is actually large enough to matter. */
const TRIM_CHECK_BYTES = 300_000;
/** Protection ceiling: even a session-protected trim can't grow the file past
* this many lines — a never-ending session can't defeat bounding entirely. */
const PROTECTION_CEILING = 5000;

export function createActivityLog(root: string): ActivityLog {
export interface ActivityLogOptions {
/** Returns the current session's start ts (ISO), or null if none/unknown.
* When set, a due trim keeps the UNION of (last CAP lines) and (all lines
* with ts >= protectSince()) instead of a flat last-CAP cut — so a long
* session's early events survive until its receipt has read them. Errors
* are treated as null (fail-open to the plain CAP trim). */
protectSince?: () => string | null;
}

export function createActivityLog(root: string, opts: ActivityLogOptions = {}): ActivityLog {
mkdirSync(root, { recursive: true });
const path = join(root, "activity.jsonl");

Expand All @@ -58,15 +96,41 @@ export function createActivityLog(root: string): ActivityLog {
return out;
};

// Events are append-only so `all` is ts-ascending — both "last CAP" and
// "ts >= protectSince()" are therefore contiguous SUFFIXES of the array, and
// their union is just the suffix starting at the smaller of the two indices.
const trimKeep = (all: ActivityEvent[]): ActivityEvent[] => {
const tailStart = Math.max(0, all.length - CAP);
let protectTs: string | null = null;
try {
protectTs = opts.protectSince?.() ?? null;
} catch {
protectTs = null; // fail-open to the plain CAP trim
}
if (!protectTs) return all.slice(tailStart);
const protectStart = all.findIndex((x) => x.ts >= protectTs!);
const start = protectStart === -1 ? tailStart : Math.min(tailStart, protectStart);
const kept = all.slice(start);
// Ceiling: a never-ending session can't grow the file unbounded — past
// it, drop the oldest PROTECTED events first (keep the tail, since that's
// what recent()/rollup() serve).
return kept.length > PROTECTION_CEILING ? kept.slice(-PROTECTION_CEILING) : kept;
};

return {
record(e) {
try {
const ev: ActivityEvent = { ts: new Date().toISOString(), saved: 0, ...e };
appendFileSync(path, JSON.stringify(ev) + "\n", { encoding: "utf8", flag: "a" });
// Bounded: when the log grows past 2×CAP, rewrite the last CAP.
const all = readAll();
if (all.length > CAP * 2) {
writeAtomic(path, all.slice(-CAP).map((x) => JSON.stringify(x)).join("\n") + "\n");
// Bounded: when the log grows past 2×CAP, rewrite the kept set. The
// statSync guard keeps appends O(1) until the file is actually big
// enough that a trim could be due — avoids a readAll() every call.
const size = existsSync(path) ? statSync(path).size : 0;
if (size > TRIM_CHECK_BYTES) {
const all = readAll();
if (all.length > CAP * 2) {
writeAtomic(path, trimKeep(all).map((x) => JSON.stringify(x)).join("\n") + "\n");
}
}
} catch {
/* activity is observability — never break a tool call over it */
Expand All @@ -86,5 +150,14 @@ export function createActivityLog(root: string): ActivityLog {
}
return [...by.values()].sort((a, b) => b.saved - a.saved);
},
since(ts) {
const all = readAll();
if (all.length === 0) return { events: [], trimmed: false };
// Only claim rotation when a trim could actually have happened (log at
// CAP scale) — the oldest event being newer than the session start is
// otherwise the NORMAL case (session mark precedes its first event).
const trimmed = all.length >= CAP && all[0]!.ts > ts;
return { events: all.filter((e) => e.ts >= ts), trimmed };
},
};
}
48 changes: 39 additions & 9 deletions src/engine/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,18 +256,48 @@ export function detectDomains(files: string[]): string[] {

/**
* Loop-ready goal checkboxes derived from the charter goal — actionable, never
* the vague "design + implement + verify" boilerplate. When the project has
* real parts (detected domains or declared greenfield modules) each part is its
* own checkbox carrying the goal; otherwise the goal itself is ONE checkbox.
*
* Deliberately NOT split into sub-clauses: the loop runs ONE holistic verify
* gate after each box, so a box must independently pass the gate. Splitting a
* single goal into "A"/"B" clauses that only pass together would stall the loop
* on the first box — real modules are independent units; sentence clauses are not.
* the vague "design + implement + verify" boilerplate, and never N identical
* copies of the same sentence. When the project has real parts (detected
* domains or declared greenfield modules) AND the goal's own clauses cleanly
* name ≥2 of them ("build the API and the dashboard"), each part gets its own
* DISTINCT clause as its checkbox. Otherwise the goal stays ONE checkbox (a
* single goal can't be safely split into per-domain sub-clauses that only pass
* together would stall the loop's holistic verify gate on the first box — real
* modules are independent units; sentence clauses are not) with a note listing
* the domains it covers, for manual decomposition as work reveals structure.
*/
export function goalCheckboxes(goal: string, parts: string[]): string[] {
const g = goal.replace(/\s+/g, " ").trim() || "the current goal";
return parts.length > 0 ? parts.map((d) => `- [ ] ${d}: ${g}`) : [`- [ ] ${g}`];
if (parts.length === 0) return [`- [ ] ${g}`];

// Try to split the goal into per-domain clauses instead of repeating the
// whole goal on every box — "build the API and the dashboard" should become
// two distinct checkboxes, not the same sentence twice. Only commit to this
// if at least 2 parts land on genuinely DISTINCT clauses; a coincidental
// single match (or all parts matching one clause) isn't a real split.
const clauses = g
.split(/[,;]| and | with /i)
.map((c) => c.trim())
.filter((c) => c.length > 0);
const matched = new Map<string, string>(); // part -> clause
const usedClauses = new Set<string>();
for (const part of parts) {
const needle = part.toLowerCase();
const clause = clauses.find((c) => c.toLowerCase().includes(needle) && !usedClauses.has(c));
if (clause) {
matched.set(part, clause);
usedClauses.add(clause);
}
}
if (matched.size >= 2) {
return [...matched.entries()].map(([d, clause]) => `- [ ] ${d}: ${clause}`);
}

// No clean split — keep the goal as ONE checkbox, but note the domains it
// covers so the loop can decompose it further as work reveals structure.
// The note line does NOT start with "- [ ]" so parseGoalProgress ignores it
// (it only matches lines starting with a checkbox marker).
return [`- [ ] ${g}`, `(covers domains: ${parts.join(", ")} — decompose into per-domain boxes as you go)`];
}

/**
Expand Down
Loading
Loading