Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions src/commands/mine-local.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import {
import { extractPairs, type Pair } from "../skillify/extractors/index.js";
import { findAgentBin, type Agent } from "../skillify/gate-runner.js";
import { extractJsonBlock } from "../skillify/gate-parser.js";
import { permissionFlags, type ClaudeGrants } from "../hooks/wiki-worker-spawn.js";
import { resolveSkillsRoot, writeNewSkill, listSkills, parseFrontmatter } from "../skillify/skill-writer.js";
import { detectAgentSkillsRoots } from "../skillify/agent-roots.js";
import { fanOutSymlinks } from "../skillify/pull.js";
Expand Down Expand Up @@ -91,6 +92,12 @@ function runGateViaStdin(opts: {
bin: string;
prompt: string;
timeoutMs: number;
/**
* Dirs/tools to grant explicitly instead of the blanket `bypassPermissions`,
* which an enterprise policy can disable. See ClaudeGrants in
* ../hooks/wiki-worker-spawn.ts.
*/
grants?: ClaudeGrants;
}): Promise<{ stdout: string; stderr: string; errored: boolean; errorMessage?: string }> {
return new Promise((resolve) => {
if (opts.agent !== "claude_code") {
Expand All @@ -116,7 +123,7 @@ function runGateViaStdin(opts: {
"-p",
"--no-session-persistence",
"--model", "haiku",
"--permission-mode", "bypassPermissions",
...permissionFlags(opts.grants),
];
const child = spawn(opts.bin, args, {
stdio: ["pipe", "pipe", "pipe"],
Expand Down Expand Up @@ -586,7 +593,16 @@ async function runMineLocalImpl(args: string[]): Promise<void> {
const prompt = buildSessionPrompt(tail, s, verdictPath);
writeFileSync(join(sessionTmp, "prompt.txt"), prompt);

const gate = await runGateViaStdin({ agent: gateAgent, bin: gateBin, prompt, timeoutMs: GATE_TIMEOUT_MS });
// sessionTmp holds the verdict path named in the prompt and lives outside
// the session cwd, so grant it explicitly rather than relying on the
// bypass an enterprise policy can disable.
const gate = await runGateViaStdin({
agent: gateAgent,
bin: gateBin,
prompt,
timeoutMs: GATE_TIMEOUT_MS,
grants: { addDirs: [sessionTmp], allowedTools: ["Read", "Write"] },
});
try {
writeFileSync(join(sessionTmp, "gate-stdout.txt"), gate.stdout);
if (gate.stderr) writeFileSync(join(sessionTmp, "gate-stderr.txt"), gate.stderr);
Expand Down
14 changes: 12 additions & 2 deletions src/hooks/codex/wiki-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
* Invoked by stop.ts as: node wiki-worker.js <config.json>
*/

import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { readFileSync, writeFileSync, existsSync, statSync, utimesSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { buildTrailingPromptInvocation } from "../wiki-worker-spawn.js";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { finalizeSummary, releaseLock, readState } from "../summary-state.js";
import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { capLinesByBytes, markSummaryUnwritten, newRowsFromWindow, stampOffset, summaryWasWritten, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { redactSecrets } from "../shared/redact.js";
import { uploadSummary } from "../upload-summary.js";
import { log as _log } from "../../utils/debug.js";
Expand Down Expand Up @@ -239,6 +239,7 @@ async function main(): Promise<void> {
wlog("running codex exec");
let execSucceeded = false;
const summaryBeforeExec = existsSync(tmpSummary) ? readFileSync(tmpSummary, "utf-8") : null;
const summaryBaseline = markSummaryUnwritten(tmpSummary, { existsSync, utimesSync, statSync });
try {
const inv = buildTrailingPromptInvocation(cfg.codexBin, [
"exec",
Expand Down Expand Up @@ -271,6 +272,15 @@ async function main(): Promise<void> {
: "codex exec failed without producing a new summary; skipping upload");
return;
}
// Exit 0 is not proof of work: a child that cannot reach tmpDir (or simply
// declines) exits 0 having written nothing, leaving the pre-seeded base
// summary in place. Uploading it unchanged would still advance the offset
// and slice those events away forever, which is how a session gets stuck
// as a header-only placeholder run after run.
if (!summaryWasWritten(tmpSummary, summaryBaseline, summaryChanged, { statSync })) {
wlog("codex exec exited 0 but never wrote the summary; skipping upload to avoid advancing the offset");
return;
}
if (raw.trim()) {
// Stamp the offset ourselves so the persisted summary is authoritative
// and never depends on the LLM echoing the bookkeeping line.
Expand Down
14 changes: 12 additions & 2 deletions src/hooks/cursor/wiki-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@
* differs: codex shells `codex exec`, we shell `cursor-agent --print --model X`.
*/

import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { readFileSync, writeFileSync, existsSync, statSync, utimesSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { buildTrailingPromptInvocation } from "../wiki-worker-spawn.js";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { finalizeSummary, releaseLock, readState } from "../summary-state.js";
import { readSessionEventCache } from "../session-event-cache.js";
import { buildSessionPath } from "../../utils/session-path.js";
import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { capLinesByBytes, markSummaryUnwritten, newRowsFromWindow, stampOffset, summaryWasWritten, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { redactSecrets } from "../shared/redact.js";
import { uploadSummary } from "../upload-summary.js";
import { log as _log } from "../../utils/debug.js";
Expand Down Expand Up @@ -274,6 +274,7 @@ async function main(): Promise<void> {
wlog(`running cursor-agent --print (model=${cfg.cursorModel})`);
let execSucceeded = false;
const summaryBeforeExec = existsSync(tmpSummary) ? readFileSync(tmpSummary, "utf-8") : null;
const summaryBaseline = markSummaryUnwritten(tmpSummary, { existsSync, utimesSync, statSync });
try {
// cursor-agent --print is the non-interactive headless mode. --force
// auto-allows tools (matches the bypass-approvals semantic codex used).
Expand Down Expand Up @@ -312,6 +313,15 @@ async function main(): Promise<void> {
: "cursor-agent --print failed without producing a new summary; skipping upload");
return;
}
// Exit 0 is not proof of work: a child that cannot reach tmpDir (or simply
// declines) exits 0 having written nothing, leaving the pre-seeded base
// summary in place. Uploading it unchanged would still advance the offset
// and slice those events away forever, which is how a session gets stuck
// as a header-only placeholder run after run.
if (!summaryWasWritten(tmpSummary, summaryBaseline, summaryChanged, { statSync })) {
wlog("cursor-agent --print exited 0 but never wrote the summary; skipping upload to avoid advancing the offset");
return;
}
if (raw.trim()) {
// Stamp the offset ourselves so the persisted summary is authoritative
// and never depends on the LLM echoing the bookkeeping line.
Expand Down
14 changes: 12 additions & 2 deletions src/hooks/hermes/wiki-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
* differs: codex shells `codex exec`, we shell `hermes -z --provider X -m Y`.
*/

import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { readFileSync, writeFileSync, existsSync, statSync, utimesSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { finalizeSummary, releaseLock, readState } from "../summary-state.js";
import { readSessionEventCache } from "../session-event-cache.js";
import { buildSessionPath } from "../../utils/session-path.js";
import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { capLinesByBytes, markSummaryUnwritten, newRowsFromWindow, stampOffset, summaryWasWritten, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { redactSecrets } from "../shared/redact.js";
import { uploadSummary } from "../upload-summary.js";
import { log as _log } from "../../utils/debug.js";
Expand Down Expand Up @@ -276,6 +276,7 @@ async function main(): Promise<void> {
wlog(`running hermes -z (provider=${cfg.hermesProvider}, model=${cfg.hermesModel})`);
let execSucceeded = false;
const summaryBeforeExec = existsSync(tmpSummary) ? readFileSync(tmpSummary, "utf-8") : null;
const summaryBaseline = markSummaryUnwritten(tmpSummary, { existsSync, utimesSync, statSync });
try {
// hermes -z (--oneshot) is the non-interactive mode. --yolo
// auto-approves tool use within the spawned hermes process.
Expand Down Expand Up @@ -324,6 +325,15 @@ async function main(): Promise<void> {
: "hermes -z failed without producing a new summary; skipping upload");
return;
}
// Exit 0 is not proof of work: a child that cannot reach tmpDir (or simply
// declines) exits 0 having written nothing, leaving the pre-seeded base
// summary in place. Uploading it unchanged would still advance the offset
// and slice those events away forever, which is how a session gets stuck
// as a header-only placeholder run after run.
if (!summaryWasWritten(tmpSummary, summaryBaseline, summaryChanged, { statSync })) {
wlog("hermes -z exited 0 but never wrote the summary; skipping upload to avoid advancing the offset");
return;
}
if (raw.trim()) {
// Stamp the offset ourselves so the persisted summary is authoritative
// and never depends on the LLM echoing the bookkeeping line.
Expand Down
14 changes: 12 additions & 2 deletions src/hooks/pi/wiki-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
* we shell `pi --print --provider <p> --model <m>`. Same query/upload paths.
*/

import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { readFileSync, writeFileSync, existsSync, statSync, utimesSync, appendFileSync, mkdirSync, rmSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { buildTrailingPromptInvocation } from "../wiki-worker-spawn.js";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { finalizeSummary, releaseLock, readState } from "../summary-state.js";
import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { capLinesByBytes, markSummaryUnwritten, newRowsFromWindow, stampOffset, summaryWasWritten, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js";
import { redactSecrets } from "../shared/redact.js";
import { uploadSummary } from "../upload-summary.js";
import { log as _log } from "../../utils/debug.js";
Expand Down Expand Up @@ -224,6 +224,7 @@ async function main(): Promise<void> {
wlog(`running pi --print (provider=${cfg.piProvider}, model=${cfg.piModel})`);
let execSucceeded = false;
const summaryBeforeExec = existsSync(tmpSummary) ? readFileSync(tmpSummary, "utf-8") : null;
const summaryBaseline = markSummaryUnwritten(tmpSummary, { existsSync, utimesSync, statSync });
try {
// pi --print is the non-interactive mode; it bypasses extension
// discovery (modes/print-mode.js doesn't import ExtensionRunner),
Expand Down Expand Up @@ -263,6 +264,15 @@ async function main(): Promise<void> {
: "pi --print failed without producing a new summary; skipping upload");
return;
}
// Exit 0 is not proof of work: a child that cannot reach tmpDir (or simply
// declines) exits 0 having written nothing, leaving the pre-seeded base
// summary in place. Uploading it unchanged would still advance the offset
// and slice those events away forever, which is how a session gets stuck
// as a header-only placeholder run after run.
if (!summaryWasWritten(tmpSummary, summaryBaseline, summaryChanged, { statSync })) {
wlog("pi --print exited 0 but never wrote the summary; skipping upload to avoid advancing the offset");
return;
}
if (raw.trim()) {
// Stamp the offset ourselves so the persisted summary is authoritative
// and never depends on the LLM echoing the bookkeeping line.
Expand Down
57 changes: 57 additions & 0 deletions src/hooks/wiki-offset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,60 @@ function truncateUtf8(s: string, maxBytes: number): string {
const decoder = new TextDecoder("utf-8", { fatal: false });
return decoder.decode(buf.subarray(0, maxBytes)).replace(/�+$/, "");
}

/**
* Did THIS run's agent actually write the summary?
*
* Exit 0 is not proof of work: a child that cannot reach the scratch dir (an
* enterprise policy disabling bypassPermissions, say) exits 0 having written
* nothing, leaving the pre-seeded summary in place. Uploading that unchanged
* placeholder still advances the offset and slices the unread events away
* forever, so the worker must be able to tell "wrote nothing" from "wrote".
*
* Content equality alone cannot: an agent that legitimately regenerates the
* same text would read as a no-op and freeze the offset. So the baseline
* stamps the file a minute into the past and the check compares timestamps —
* any real write lands far outside any plausible filesystem granularity.
*
* When the filesystem will not cooperate (utimes throwing or silently doing
* nothing) the timestamp proves nothing, and the check falls back to content.
* That errs toward skipping: re-summarizing the same rows next run wastes work,
* whereas a wrong upload destroys events. Never the other way round.
*/
export interface SummaryBaseline {
mtimeMs: number;
/** False when utimes threw or left the timestamp unchanged. */
trusted: boolean;
}

const SUMMARY_BACKDATE_MS = 60_000;

export function markSummaryUnwritten(
path: string,
fs: Pick<typeof import("node:fs"), "existsSync" | "utimesSync" | "statSync">,
): SummaryBaseline {
if (!fs.existsSync(path)) return { mtimeMs: 0, trusted: true };
const sentinel = new Date(Date.now() - SUMMARY_BACKDATE_MS);
try {
fs.utimesSync(path, sentinel, sentinel);
const mtimeMs = fs.statSync(path).mtimeMs;
// A filesystem that ignores utimes reports something far from the sentinel.
return { mtimeMs, trusted: Math.abs(mtimeMs - sentinel.getTime()) < 2_000 };
} catch {
return { mtimeMs: 0, trusted: false };
}
}

export function summaryWasWritten(
path: string,
baseline: SummaryBaseline,
contentChanged: boolean,
fs: Pick<typeof import("node:fs"), "statSync">,
): boolean {
if (!baseline.trusted) return contentChanged;
try {
return fs.statSync(path).mtimeMs !== baseline.mtimeMs;
} catch {
return contentChanged;
}
}
70 changes: 58 additions & 12 deletions src/hooks/wiki-worker-spawn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,54 @@ import type { ExecFileSyncOptions } from "node:child_process";
import { binNeedsShell, shellFile } from "../utils/resolve-cli-bin.js";

/** Fixed flags for the summary-generation `claude -p` call (no user input). */
const CLAUDE_FLAGS = [
"--no-session-persistence",
"--model",
"haiku",
"--permission-mode",
"bypassPermissions",
] as const;
const CLAUDE_BASE_FLAGS = ["--no-session-persistence", "--model", "haiku"] as const;

/** Blanket grant, used only when the caller names no explicit grants. */
const CLAUDE_BYPASS_FLAGS = ["--permission-mode", "bypassPermissions"] as const;

export interface ClaudeInvocation {
file: string;
args: string[];
options: ExecFileSyncOptions;
}

/**
* Explicit grants for call sites that read/write files OUTSIDE the session cwd.
*
* `bypassPermissions` is NOT honored under an enterprise policy that sets
* `"disableBypassPermissionsMode": "disable"` (macOS:
* /Library/Application Support/ClaudeCode/managed-settings.json). The child then
* falls back to normal permissioning, refuses every path outside the working
* directory — the wiki worker's `$TMPDIR/deeplake-wiki-*` scratch dir, or a
* backfill's transcript + staging dir — and exits 0 having written nothing. On
* such a machine every summary stays a header-only stub and every backfill
* reports `no-summary`.
*
* Naming the dirs and tools instead is both policy-proof and least-privilege
* (the summarizer only ever needs Read + Write), so a caller that supplies
* grants gets them INSTEAD of the bypass, not in addition to it.
*/
export interface ClaudeGrants {
/** Directories to expose to the child (each becomes `--add-dir <dir>`). */
addDirs?: string[];
/** Tools to pre-approve, e.g. `["Read", "Write"]`. */
allowedTools?: string[];
}

/**
* `quotePaths` is for the Windows `.cmd` branch, where args are re-joined into a
* shell command line: a temp dir there routinely contains spaces
* (`C:\Users\First Last\AppData\Local\Temp`) and would otherwise split.
*/
export function permissionFlags(grants: ClaudeGrants | undefined, quotePaths = false): string[] {
if (!grants) return [...CLAUDE_BYPASS_FLAGS];
const flags: string[] = [];
for (const dir of grants.addDirs ?? []) flags.push("--add-dir", quotePaths ? `"${dir}"` : dir);
if (grants.allowedTools?.length) flags.push("--allowedTools", ...grants.allowedTools);
// An empty grants object would otherwise leave the child with no grant at all.
return flags.length > 0 ? flags : [...CLAUDE_BYPASS_FLAGS];
}

/**
* Build the `execFileSync` descriptor for the summary-generation claude call.
*
Expand All @@ -30,11 +64,15 @@ export interface ClaudeInvocation {
* prompt as a positional arg, no shell — so the already-working path stays
* byte-identical.
*/
export function buildClaudeInvocation(claudeBin: string, prompt: string): ClaudeInvocation {
export function buildClaudeInvocation(
claudeBin: string,
prompt: string,
grants?: ClaudeGrants,
): ClaudeInvocation {
if (binNeedsShell(claudeBin)) {
return {
file: shellFile(claudeBin),
args: ["-p", ...CLAUDE_FLAGS],
args: ["-p", ...CLAUDE_BASE_FLAGS, ...permissionFlags(grants, true)],
// windowsHide: the wiki worker is a detached, console-less process, so
// without CREATE_NO_WINDOW Windows allocates a visible console window
// (titled after the CLI exe) for the child. No-op on POSIX.
Expand All @@ -43,7 +81,7 @@ export function buildClaudeInvocation(claudeBin: string, prompt: string): Claude
}
return {
file: claudeBin,
args: ["-p", prompt, ...CLAUDE_FLAGS],
args: ["-p", prompt, ...CLAUDE_BASE_FLAGS, ...permissionFlags(grants)],
options: { stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
};
}
Expand Down Expand Up @@ -102,6 +140,14 @@ export function buildStdinPromptInvocation(bin: string, flags: string[], prompt:
}

/** Claude variant of {@link buildStdinPromptInvocation} (same fixed flags as the argv path). */
export function buildClaudeStdinInvocation(claudeBin: string, prompt: string): ClaudeInvocation {
return buildStdinPromptInvocation(claudeBin, ["-p", ...CLAUDE_FLAGS], prompt);
export function buildClaudeStdinInvocation(
claudeBin: string,
prompt: string,
grants?: ClaudeGrants,
): ClaudeInvocation {
return buildStdinPromptInvocation(
claudeBin,
["-p", ...CLAUDE_BASE_FLAGS, ...permissionFlags(grants)],
prompt,
);
}
Loading