diff --git a/db.js b/db.js index 751b08a0..c8923682 100644 --- a/db.js +++ b/db.js @@ -169,6 +169,17 @@ if (migrations.length > currentDbVersion) { // getHarness(), which treats null as Claude. if (!cols.has('runtime')) db.exec("ALTER TABLE session_cache ADD COLUMN runtime TEXT DEFAULT 'claude'"); if (!cols.has('sessionFile')) db.exec('ALTER TABLE session_cache ADD COLUMN sessionFile TEXT'); + // Resume state for Claude's incremental parser. Add by column presence even + // when a parallel branch already advanced db_version. Existing rows keep + // their cache/search data and get a full read on their next modification. + // Raw timestamp bounds are separate from created/modified, whose fallback + // to file times must not become an accumulator value on a later append. + for (const col of [ + 'customTitle TEXT', 'textContent TEXT', 'headHash TEXT', + 'indexedBytes INTEGER DEFAULT 0', 'firstTimestamp TEXT', 'lastTimestamp TEXT', + ]) { + if (!cols.has(col.split(' ')[0])) db.exec(`ALTER TABLE session_cache ADD COLUMN ${col}`); + } } // --- FTS5 full-text search --- @@ -206,17 +217,26 @@ const stmts = { `), // Session cache statements cacheCount: db.prepare('SELECT COUNT(*) as cnt FROM session_cache'), - cacheGetAll: db.prepare('SELECT * FROM session_cache'), + // Frequent sidebar/title refreshes do not need the potentially large search + // text or parser state. Keep the harness and transcript-location fields. + cacheGetAll: db.prepare(` + SELECT sessionId, folder, projectPath, summary, firstPrompt, created, modified, + messageCount, slug, aiTitle, fileMtime, runtime, sessionFile + FROM session_cache + `), cacheUpsert: db.prepare(` - INSERT INTO session_cache (sessionId, folder, projectPath, summary, firstPrompt, created, modified, messageCount, slug, aiTitle, fileMtime, runtime, sessionFile) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO session_cache (sessionId, folder, projectPath, summary, firstPrompt, created, modified, messageCount, slug, aiTitle, fileMtime, runtime, sessionFile, customTitle, textContent, headHash, indexedBytes, firstTimestamp, lastTimestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(sessionId) DO UPDATE SET folder = excluded.folder, projectPath = excluded.projectPath, summary = excluded.summary, firstPrompt = excluded.firstPrompt, created = excluded.created, modified = excluded.modified, messageCount = excluded.messageCount, slug = excluded.slug, aiTitle = excluded.aiTitle, fileMtime = excluded.fileMtime, - runtime = excluded.runtime, sessionFile = excluded.sessionFile + runtime = excluded.runtime, sessionFile = excluded.sessionFile, + customTitle = excluded.customTitle, textContent = excluded.textContent, + headHash = excluded.headHash, indexedBytes = excluded.indexedBytes, + firstTimestamp = excluded.firstTimestamp, lastTimestamp = excluded.lastTimestamp `), cacheGetByFolder: db.prepare('SELECT sessionId, fileMtime FROM session_cache WHERE folder = ?'), cacheGetSession: db.prepare('SELECT * FROM session_cache WHERE sessionId = ?'), @@ -304,7 +324,9 @@ const upsertCachedSessionsBatch = db.transaction((sessions) => { s.sessionId, s.folder, s.projectPath, s.summary, s.firstPrompt, s.created, s.modified, s.messageCount || 0, s.slug || null, s.aiTitle || null, s.fileMtime || null, - s.runtime || 'claude', s.sessionFile || null + s.runtime || 'claude', s.sessionFile || null, + s.customTitle || null, s.textContent || null, s.headHash || null, + s.indexedBytes || 0, s.firstTimestamp || null, s.lastTimestamp || null ); } }); diff --git a/harnesses/claude.js b/harnesses/claude.js index 1c8c1928..6f27e44a 100644 --- a/harnesses/claude.js +++ b/harnesses/claude.js @@ -13,6 +13,8 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); +const crypto = require('crypto'); +const { scanLines } = require('../jsonl-scan'); const { encodeProjectPath } = require('../encode-project-path'); const id = 'claude'; @@ -98,17 +100,18 @@ function transcriptPath({ sessionId, folder, sessionFile }) { // --- Project path derivation --- function extractCwdFromJsonl(filePath) { + let cwd = null; + const readCwd = (line) => { + try { + const entry = JSON.parse(line); + if (entry.cwd) { cwd = entry.cwd; return false; } + } catch {} + }; try { - const lines = fs.readFileSync(filePath, 'utf8').split('\n'); - for (const line of lines) { - if (!line) continue; - try { - const parsed = JSON.parse(line); - if (parsed.cwd) return parsed.cwd; - } catch {} - } - } catch {} - return null; + const { tail } = scanLines(filePath, 0, readCwd); + if (!cwd && tail) readCwd(tail); + } catch { return null; } + return cwd; } /** The project a folder belongs to, read out of any transcript it contains. */ @@ -224,74 +227,127 @@ function matchesLaunch(signals, { forkFrom, spawnedAt }) { // --- Transcript parsing --- -/** Parse a single .jsonl file into a session object (or null if invalid) */ -function readSessionFile(filePath, folder, projectPath) { +const HEAD_BYTES = 4096; // guard window for append-only transcripts +const TEXT_CONTENT_CAP = 8000; // how much body text the search index keeps +const TEXT_LINE_CAP = 500; + +function hashHead(fd, stat) { + const n = Math.min(HEAD_BYTES, stat.size); + if (n === 0) return ''; + const buf = Buffer.allocUnsafe(n); + if (fs.readSync(fd, buf, 0, n, 0) !== n) throw new Error('JSONL head changed during read'); + // Version the state: pre-harness rows lack raw timestamp accumulators. + // Include file identity so an atomic replacement with the same prefix resets. + return 'v2:' + crypto.createHash('sha1') + .update(`${stat.dev}:${stat.ino}:`).update(buf).digest('hex'); +} + +/** Accumulator. Every field is either a first-occurrence or a running total, + * which is what makes resuming mid-file valid. */ +function emptyState() { + return { summary: '', messageCount: 0, textContent: '', slug: null, customTitle: null, aiTitle: null, firstTimestamp: null, lastTimestamp: null }; +} + +function stateFrom(prev) { + return { + summary: prev.summary || '', + messageCount: prev.messageCount || 0, + textContent: prev.textContent || '', + slug: prev.slug || null, + customTitle: prev.customTitle || null, + aiTitle: prev.aiTitle || null, + firstTimestamp: prev.firstTimestamp || null, + lastTimestamp: prev.lastTimestamp || null, + }; +} + +function applyLine(line, st) { + let entry; + try { entry = JSON.parse(line); } catch { return; } + + if (entry.timestamp) { + if (!st.firstTimestamp || entry.timestamp < st.firstTimestamp) st.firstTimestamp = entry.timestamp; + if (!st.lastTimestamp || entry.timestamp > st.lastTimestamp) st.lastTimestamp = entry.timestamp; + } + + if (entry.slug && !st.slug) st.slug = entry.slug; + if (entry.type === 'custom-title' && entry.customTitle) st.customTitle = entry.customTitle; + if (entry.type === 'ai-title' && entry.aiTitle) st.aiTitle = entry.aiTitle; + + if (entry.type === 'user' || entry.type === 'assistant' || + (entry.type === 'message' && (entry.role === 'user' || entry.role === 'assistant'))) { + st.messageCount++; + } + + const msg = entry.message; + const text = typeof msg === 'string' ? msg : + (typeof msg?.content === 'string' ? msg.content : + (msg?.content?.[0]?.text || '')); + + if (!st.summary && (entry.type === 'user' || (entry.type === 'message' && entry.role === 'user'))) { + // Skip local command messages (! prefix) — use the next real user message + if (text && !/||/.test(text)) { + // Use scheduled task name if present + const taskMatch = text.match(/ lastTimestamp) lastTimestamp = entry.timestamp; - } - if (entry.slug && !slug) slug = entry.slug; - if (entry.type === 'custom-title' && entry.customTitle) { - customTitle = entry.customTitle; - } - if (entry.type === 'ai-title' && entry.aiTitle) { - aiTitle = entry.aiTitle; - } - if (entry.type === 'user' || entry.type === 'assistant' || - (entry.type === 'message' && (entry.role === 'user' || entry.role === 'assistant'))) { - messageCount++; - } - const msg = entry.message; - const text = typeof msg === 'string' ? msg : - (typeof msg?.content === 'string' ? msg.content : - (msg?.content?.[0]?.text || '')); - if (!summary && (entry.type === 'user' || (entry.type === 'message' && entry.role === 'user'))) { - // Skip local command messages (! prefix) — use the next real user message - if (text && !/||/.test(text)) { - // Use scheduled task name if present - const taskMatch = text.match(/ 0 + && prev.indexedBytes <= stat.size + && (prev.indexedBytes < stat.size || prev.fileMtime === fileMtime); + const st = canResume ? stateFrom(prev) : emptyState(); + const start = canResume ? prev.indexedBytes : 0; + // Hash and scan the same descriptor and only the size captured above. + // Concurrent appends are left for the next pass; read errors return null + // rather than saving partial metadata under a supposedly up-to-date mtime. + const { consumed, read, tail } = scanLines(fd, start, line => applyLine(line, st), stat.size); + if (tail) applyLine(tail, st); + const after = fs.fstatSync(fd); + if (after.size < stat.size || (after.size === stat.size && after.mtimeMs !== stat.mtimeMs)) return null; + if (!st.summary || st.messageCount < 1) return null; return { - sessionId, folder, projectPath, - runtime: id, - sessionFile: filePath, - summary, firstPrompt: summary, - // created/modified are display+sort values from message timestamps; - // fileMtime is the cache-invalidation key (compared against stat.mtime - // in refreshFolder). Old transcripts without timestamps fall back to stat. - created: firstTimestamp || stat.birthtime.toISOString(), - modified: lastTimestamp || stat.mtime.toISOString(), - fileMtime: stat.mtime.toISOString(), - messageCount, textContent, slug, customTitle, aiTitle, + sessionId, folder, projectPath, runtime: id, sessionFile: filePath, + summary: st.summary, firstPrompt: st.summary, + created: st.firstTimestamp || stat.birthtime.toISOString(), + modified: st.lastTimestamp || fileMtime, + fileMtime, + messageCount: st.messageCount, textContent: st.textContent, + slug: st.slug, customTitle: st.customTitle, aiTitle: st.aiTitle, + firstTimestamp: st.firstTimestamp, lastTimestamp: st.lastTimestamp, + headHash, + // A complete JSON value without a newline is displayed but cannot be + // accumulated safely: re-read it next time instead of double-counting it. + indexedBytes: tail ? 0 : consumed, + bytesRead: read + Math.min(HEAD_BYTES, stat.size), }; } catch { return null; + } finally { + if (fd !== null) { try { fs.closeSync(fd); } catch {} } } } diff --git a/harnesses/index.js b/harnesses/index.js index 40bb4b64..9ec5c383 100644 --- a/harnesses/index.js +++ b/harnesses/index.js @@ -23,7 +23,8 @@ // listTranscripts(dir) absolute transcript paths in a folder directory // sessionIdFromPath(file) transcript path → session id, without reading it // transcriptPath(row) cached row → absolute transcript path -// readSessionFile(file, folder, projectPath) → session row, or null +// readSessionFile(file, folder, projectPath, prev?) → session row, or null +// prev is optional cached parser state; harnesses may ignore it. // titleIndexPath() optional external title-index path // readSessionTitles() optional Map from that index // buildLaunchArgs({ sessionId, isNew, options }) → argv after the binary diff --git a/jsonl-scan.js b/jsonl-scan.js new file mode 100644 index 00000000..ecfd41a2 --- /dev/null +++ b/jsonl-scan.js @@ -0,0 +1,91 @@ +const fs = require('fs'); + +// Session .jsonl files routinely reach tens or hundreds of MB. Reading one into +// a JS string costs ~2x its size in RAM (V8 stores non-latin1 text as UTF-16), +// so any code that wants the first line — or one field — must not use +// readFileSync. These two helpers are the supported way to walk such a file. + +const CHUNK_BYTES = 256 * 1024; + +/** + * Walk the complete lines of a file from `startByte`, holding at most one chunk + * (plus the current line) in memory. + * + * `onLine(line)` may return false to stop early — useful when the caller only + * needs the first line carrying some field. + * + * Returns: + * consumed — offset just past the last complete line (a resume point) + * read — bytes actually pulled off disk + * tail — trailing bytes with no newline; NOT counted in `consumed`, + * because a partial line will be re-read on the next pass + * stopped — whether onLine asked to stop + * + * Accepts a path or an already-open descriptor. Reads only through endByte + * (default: the size at entry), so appends cannot make a scan run forever. + * I/O errors propagate: callers must not persist a partial scan as complete. + */ +function scanLines(filePath, startByte, onLine, endByte) { + const ownsFd = typeof filePath !== 'number'; + const fd = ownsFd ? fs.openSync(filePath, 'r') : filePath; + let consumed = startByte; + let read = 0; + try { + if (endByte === undefined) endByte = fs.fstatSync(fd).size; + let pending = []; + let pendingBytes = 0; + let pos = startByte; + while (pos < endByte) { + // Each chunk owns its bytes. Pending slices never alias a reused buffer. + const buf = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, endByte - pos)); + const n = fs.readSync(fd, buf, 0, buf.length, pos); + if (n === 0) throw new Error('JSONL file truncated during scan'); + read += n; + let from = 0; + let nl; + const data = buf.subarray(0, n); + while ((nl = data.indexOf(0x0A, from)) !== -1) { + const piece = data.subarray(from, nl); + // Concatenate once per line, avoiding quadratic copies of long lines. + const line = pendingBytes + ? Buffer.concat([...pending, piece], pendingBytes + piece.length).toString('utf8') + : piece.toString('utf8'); + pending = []; + pendingBytes = 0; + consumed = pos + nl + 1; + if (line && onLine(line) === false) { + return { consumed, read, tail: '', stopped: true }; + } + from = nl + 1; + } + if (from < n) { + pending.push(data.subarray(from)); + pendingBytes += n - from; + } + pos += n; + } + const tail = pendingBytes ? Buffer.concat(pending, pendingBytes).toString('utf8') : ''; + return { consumed, read, tail, stopped: false }; + } finally { + if (ownsFd) fs.closeSync(fd); + } +} + +/** Read at most `maxBytes` from the start of a file. Returns '' on failure. */ +function readHead(filePath, maxBytes) { + let fd = null; + try { + fd = fs.openSync(filePath, 'r'); + const buf = Buffer.allocUnsafe(maxBytes); + const n = fs.readSync(fd, buf, 0, maxBytes, 0); + return buf.toString('utf8', 0, n); + } catch { + return ''; + } finally { + if (fd !== null) { + try { fs.closeSync(fd); } catch {} + } + } +} + +module.exports = { scanLines, readHead }; diff --git a/main.js b/main.js index f19ff9f7..0a0fa002 100644 --- a/main.js +++ b/main.js @@ -8,6 +8,7 @@ const log = require('electron-log'); // getFolderIndexMtimeMs moved to session-cache.js const { startMcpServer, shutdownMcpServer, shutdownAll: shutdownAllMcp, resolvePendingDiff, rekeyMcpServer, cleanStaleLockFiles } = require('./mcp-bridge'); const { fetchAndTransformUsage } = require('./claude-auth'); +const { readHead } = require('./jsonl-scan'); const codexAuth = require('./codex-auth'); // SWITCHBOARD_DATA_DIR isolates a dev/test instance from the installed app: @@ -272,7 +273,7 @@ sessionCache.init({ getMainWindow: () => mainWindow, log, db: { - deleteCachedFolder, getCachedByFolder, upsertCachedSessions, deleteCachedSession, + deleteCachedFolder, getCachedByFolder, getCachedSession, upsertCachedSessions, deleteCachedSession, deleteSearchFolder, deleteSearchSession, upsertSearchEntries, setFolderMeta, getAllFolderMeta, getAllMeta, getAllCached, getSetting, setSetting, getMeta, setName, updateCachedAiTitle, updateSearchTitle, @@ -1187,7 +1188,7 @@ ipcMain.handle('open-terminal', async (_event, sessionId, projectPath, isNew, se if (!startFresh) { try { const jsonlPath = path.join(claudeProjectDir, sessionId + '.jsonl'); - const head = fs.readFileSync(jsonlPath, 'utf8').slice(0, 8000); + const head = readHead(jsonlPath, 8192); const firstLines = head.split('\n').filter(Boolean); for (const line of firstLines) { const entry = JSON.parse(line); diff --git a/schedule-runner.js b/schedule-runner.js index eb6adfc5..730de25a 100644 --- a/schedule-runner.js +++ b/schedule-runner.js @@ -3,6 +3,7 @@ const fs = require('fs'); const path = require('path'); const os = require('os'); const crypto = require('crypto'); +const { readHead } = require('./jsonl-scan'); const CLAUDE_DIR = path.join(os.homedir(), '.claude'); const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects'); @@ -100,7 +101,9 @@ function readProjectPathFromJsonl(folderPath) { try { const jsonlFiles = fs.readdirSync(folderPath).filter(f => f.endsWith('.jsonl')); for (const jf of jsonlFiles) { - const head = fs.readFileSync(path.join(folderPath, jf), 'utf8').slice(0, 4000); + // Head only: this runs every minute over every project folder, and a + // session .jsonl can be hundreds of MB. + const head = readHead(path.join(folderPath, jf), 4096); for (const line of head.split('\n').filter(Boolean)) { try { const entry = JSON.parse(line); diff --git a/session-cache.js b/session-cache.js index f203b257..c62b6fee 100644 --- a/session-cache.js +++ b/session-cache.js @@ -29,7 +29,7 @@ function resolveFolderPath(folder) { * Call init(ctx) once with the shared context object. */ let PROJECTS_DIR, activeSessions, getMainWindow, log; -let deleteCachedFolder, getCachedByFolder, upsertCachedSessions, deleteCachedSession; +let deleteCachedFolder, getCachedByFolder, getCachedSession, upsertCachedSessions, deleteCachedSession; let deleteSearchFolder, deleteSearchSession, upsertSearchEntries; let setFolderMeta, getAllFolderMeta, getAllMeta, getAllCached, getSetting, setSetting, getMeta, setName; let updateCachedAiTitle, updateSearchTitle; @@ -42,6 +42,7 @@ function init(ctx) { // DB functions deleteCachedFolder = ctx.db.deleteCachedFolder; getCachedByFolder = ctx.db.getCachedByFolder; + getCachedSession = ctx.db.getCachedSession; upsertCachedSessions = ctx.db.upsertCachedSessions; deleteCachedSession = ctx.db.deleteCachedSession; deleteSearchFolder = ctx.db.deleteSearchFolder; @@ -213,6 +214,10 @@ function refreshFolder(folder) { deleteCachedFolder(folder); return; } + // Never mark writes that arrive during this pass as already indexed. The + // Claude parser deliberately stops at its initial file size; a later append + // must remain visible to reconciliation even if its watcher event is missed. + const indexMtimeMs = getFolderIndexMtimeMs(folderPath); // For Claude a folder IS a project, and one with no readable cwd is unusable. // A codex folder is a date spanning many projects, so there is no folder-level @@ -220,7 +225,7 @@ function refreshFolder(folder) { // null (which is also what cache_meta records for it). const folderProject = h.deriveProjectPath(folderPath, folder); if (h.groupsByProject && !folderProject) { - setFolderMeta(folder, null, getFolderIndexMtimeMs(folderPath)); + setFolderMeta(folder, null, indexMtimeMs); return; } @@ -256,8 +261,12 @@ function refreshFolder(folder) { continue; // unchanged, skip } - // File is new or modified — re-read it - const sess = h.readSessionFile(filePath, folder, folderProject); + // File is new or modified — re-read it. The cached row carries the resume + // state, so an append costs the size of the append rather than the size of + // the file. Fetched per changed session on purpose: a folder holds + // thousands of sessions and only a couple change per flush. + const cachedRow = getCachedSession(sessionId); + const sess = h.readSessionFile(filePath, folder, folderProject, cachedRow); if (sess) { sessionsToUpsert.push(sess); // Title precedence: user rename (session_meta.name) > JSONL custom-title > JSONL ai-title. @@ -300,7 +309,7 @@ function refreshFolder(folder) { restoreProjectsWithNewSessions(sessionsToUpsert); // Update folder mtime - setFolderMeta(folder, folderProject, getFolderIndexMtimeMs(folderPath)); + setFolderMeta(folder, folderProject, indexMtimeMs); } /** diff --git a/test/db-schema-reconcile.test.js b/test/db-schema-reconcile.test.js index 2b06cc45..0b3a281d 100644 --- a/test/db-schema-reconcile.test.js +++ b/test/db-schema-reconcile.test.js @@ -175,3 +175,136 @@ test('a pre-existing runtime column is adopted, not duplicated', () => { fs.rmSync(dir, { recursive: true, force: true }); } }); + +for (const version of [4, 12]) { + test(`incremental columns upgrade a version-${version} database without losing metadata or search`, () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-db-incremental-')); + try { + const seed = runInElectronNode(` + const Database = require('better-sqlite3'); + const db = new Database(require('path').join(process.env.SWITCHBOARD_DATA_DIR, 'switchboard.db')); + db.exec(\`CREATE TABLE session_cache ( + sessionId TEXT PRIMARY KEY, folder TEXT NOT NULL, projectPath TEXT, + summary TEXT, firstPrompt TEXT, created TEXT, modified TEXT, + messageCount INTEGER DEFAULT 0, slug TEXT, aiTitle TEXT, fileMtime TEXT, + runtime TEXT DEFAULT 'claude', sessionFile TEXT + )\`); + db.exec('CREATE TABLE cache_meta (folder TEXT PRIMARY KEY, projectPath TEXT, indexMtimeMs REAL)'); + db.exec('CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)'); + db.prepare("INSERT INTO settings (key, value) VALUES ('db_version', ?)").run('${version}'); + db.prepare(\`INSERT INTO session_cache (sessionId, folder, summary, runtime, sessionFile) + VALUES ('kept', 'codex/day', 'Existing session', 'codex', '/original/rollout.jsonl')\`).run(); + db.prepare("INSERT INTO cache_meta VALUES ('codex/day', NULL, 123)").run(); + // A partial migration from another branch must be adopted too. + if (${version} > 4) { + db.exec('ALTER TABLE session_cache ADD COLUMN customTitle TEXT'); + db.prepare("UPDATE session_cache SET customTitle = 'Existing custom title'").run(); + } + db.exec('CREATE TABLE session_meta (sessionId TEXT PRIMARY KEY, name TEXT, starred INTEGER DEFAULT 0, archived INTEGER DEFAULT 0)'); + db.prepare("INSERT INTO session_meta VALUES ('kept', 'My name', 1, 1)").run(); + db.exec(\`CREATE VIRTUAL TABLE search_fts USING fts5(title, body, tokenize='trigram case_sensitive 0')\`); + db.exec('CREATE TABLE search_map (rowid INTEGER PRIMARY KEY, id TEXT NOT NULL, type TEXT NOT NULL, folder TEXT)'); + db.prepare("INSERT INTO search_map VALUES (1, 'kept', 'session', 'codex/day')").run(); + db.prepare("INSERT INTO search_fts(rowid, title, body) VALUES (1, 'Existing session', 'searchable history')").run(); + db.close(); + `, dir); + assert.equal(seed.status, 0, seed.stderr); + // Upgrade, then reopen again to check reconciliation is idempotent. + assert.equal(loadDbModule(dir).status, 0); + const result = runInElectronNode(` + const api = require('./db'); + const row = api.getCachedSession('kept'); + console.log(JSON.stringify({row, meta:api.getMeta('kept'), + folder:api.getFolderMeta('codex/day'), hits:api.searchByType('session','searchable'), + version:api.getSetting('db_version')})); + `, dir); + assert.equal(result.status, 0, result.stderr); + const state = JSON.parse(result.stdout.trim().split('\n').pop()); + assert.equal(state.row.runtime, 'codex'); + assert.equal(state.row.sessionFile, '/original/rollout.jsonl'); + assert.equal(state.row.indexedBytes, 0); + for (const col of ['textContent', 'headHash', 'firstTimestamp', 'lastTimestamp']) assert.equal(state.row[col], null); + assert.equal(state.row.customTitle, version > 4 ? 'Existing custom title' : null); + assert.equal(state.meta.name, 'My name'); + assert.equal(state.meta.starred, 1); + assert.equal(state.meta.archived, 1); + assert.equal(state.folder.indexMtimeMs, 123); + assert.equal(state.hits[0].id, 'kept'); + assert.equal(state.version, version); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); +} + +test('incremental refresh persists resume state and keeps Codex, search and displayed times intact', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-db-refresh-')); + try { + const result = runInElectronNode(` + const assert = require('node:assert/strict'); + const fs = require('fs'); + const path = require('path'); + const db = require('./db'); + const cache = require('./session-cache'); + const projectsDir = path.join(process.env.SWITCHBOARD_DATA_DIR, 'projects'); + const folder = 'project'; + const folderPath = path.join(projectsDir, folder); + fs.mkdirSync(folderPath, {recursive:true}); + const file = path.join(folderPath, 'session.jsonl'); + const line = x => JSON.stringify(x) + '\\n'; + fs.writeFileSync(file, + line({type:'user', cwd:'/project', message:'searchable question', timestamp:'2025-01-01T00:00:00Z'}) + + line({type:'assistant', message:'x'.repeat(2 * 1024 * 1024), timestamp:'2025-01-02T00:00:00Z'})); + db.upsertCachedSessions([{sessionId:'codex-session', folder:'codex/day', projectPath:'/codex', + runtime:'codex', sessionFile:'/codex/rollout.jsonl', summary:'Codex question', firstPrompt:'Codex question', + created:'2025-01-01T00:00:00Z', modified:'2025-01-01T00:00:00Z', fileMtime:'2025-01-01T00:00:00Z', messageCount:1}]); + db.setName('session', 'My name'); + cache.init({PROJECTS_DIR:projectsDir, activeSessions:new Map(), getMainWindow:()=>null, log:console, db}); + cache.refreshFolder(folder); + const first = db.getCachedSession('session'); + assert.equal(first.messageCount, 2); + assert.equal(first.indexedBytes, fs.statSync(file).size); + assert.equal(first.firstTimestamp, '2025-01-01T00:00:00Z'); + assert.equal(first.lastTimestamp, '2025-01-02T00:00:00Z'); + db.closeDb(); + delete require.cache[require.resolve('./db')]; + const reopened = require('./db'); + cache.init({PROJECTS_DIR:projectsDir, activeSessions:new Map(), getMainWindow:()=>null, log:console, db:reopened}); + fs.appendFileSync(file, line({type:'assistant', message:'appended answer', timestamp:'2025-01-03T00:00:00Z'}) + + line({type:'ai-title', aiTitle:'Automatic title'})); + const changedTime = new Date(Date.parse(first.fileMtime) + 1000); + fs.utimesSync(file, changedTime, changedTime); + const originalRead = fs.readSync; + let bytes = 0; + fs.readSync = function(...args) { const n = originalRead(...args); bytes += n; return n; }; + try { cache.refreshFolder(folder); } finally { fs.readSync = originalRead; } + // Includes cwd derivation (one bounded chunk), hash validation and the appended records. + assert.ok(bytes < 512 * 1024, 'refresh re-read the full transcript: ' + bytes); + const row = reopened.getCachedSession('session'); + assert.equal(row.messageCount, 3); + assert.equal(row.created, first.created); + assert.equal(row.modified, '2025-01-03T00:00:00Z'); + assert.equal(row.fileMtime, changedTime.toISOString()); + assert.equal(row.sessionFile, file); + assert.equal(row.runtime, 'claude'); + assert.equal(row.aiTitle, 'Automatic title'); + assert.ok(row.textContent.includes('searchable question')); + assert.ok(row.textContent.includes('appended answer')); + assert.equal(reopened.getMeta('session').name, 'My name'); + assert.equal(reopened.searchByType('session','appended answer')[0].id, 'session'); + const all = reopened.getAllCached(); + assert.equal(all.length, 2); + for (const item of all) { + assert.ok(!Object.hasOwn(item, 'textContent'), 'bulk queries must not load parser text'); + assert.ok(!Object.hasOwn(item, 'headHash')); + } + assert.equal(all.find(r=>r.sessionId==='codex-session').sessionFile, '/codex/rollout.jsonl'); + const projects = cache.buildProjectsFromCache(false); + assert.equal(projects.find(p=>p.projectPath==='/codex').sessions[0].runtime, 'codex'); + const visible = projects.find(p=>p.projectPath==='/project').sessions[0]; + assert.equal(visible.modified, row.modified); + assert.equal(visible.name, 'My name'); + console.log(JSON.stringify({bytesRead:bytes, messageCount:row.messageCount})); + reopened.closeDb(); + `, dir); + assert.equal(result.status, 0, result.stderr || result.stdout); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/test/jsonl-scan.test.js b/test/jsonl-scan.test.js new file mode 100644 index 00000000..fe3abd11 --- /dev/null +++ b/test/jsonl-scan.test.js @@ -0,0 +1,94 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { scanLines, readHead } = require('../jsonl-scan'); + +function withFile(content, fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-jsonl-scan-')); + const file = path.join(dir, 'session.jsonl'); + try { fs.writeFileSync(file, content); fn(file); } + finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +test('scanLines preserves UTF-8 across a chunk boundary and exact resume offsets', () => { + const first = 'x'.repeat(256 * 1024 - 1) + '😀'; + withFile(first + '\n\nsecond\npartial', file => { + const lines = []; + const scan = scanLines(file, 0, line => { lines.push(line); }); + assert.deepEqual(lines, [first, 'second']); + assert.equal(scan.consumed, Buffer.byteLength(first + '\n\nsecond\n')); + assert.equal(scan.tail, 'partial'); + fs.appendFileSync(file, '-finished\n'); + const appended = []; + const next = scanLines(file, scan.consumed, line => { appended.push(line); }); + assert.deepEqual(appended, ['partial-finished']); + assert.equal(next.consumed, fs.statSync(file).size); + }); +}); + +test('multi-megabyte lines require linear rather than quadratic buffer copying', () => { + const longLine = 'x'.repeat(3 * 1024 * 1024); + withFile(longLine + '\n', file => { + const concat = Buffer.concat; + let copiedBytes = 0; + Buffer.concat = function(list, ...args) { + copiedBytes += list.reduce((n, b) => n + b.length, 0); + return concat.call(Buffer, list, ...args); + }; + try { + const lines = []; + scanLines(file, 0, line => { lines.push(line); }); + assert.deepEqual(lines, [longLine]); + assert.ok(copiedBytes <= 2 * longLine.length, `${copiedBytes} bytes copied for ${longLine.length} bytes`); + } finally { Buffer.concat = concat; } + }); +}); + +test('early exit and readHead do not read an entire large transcript', () => { + withFile('first\n' + 'x'.repeat(1024 * 1024), file => { + const scan = scanLines(file, 0, () => false); + assert.equal(scan.consumed, 6); + assert.equal(scan.stopped, true); + assert.ok(scan.read <= 256 * 1024); + assert.equal(readHead(file, 5), 'first'); + }); +}); + +test('a scan stops at its initial size even when the callback appends', () => { + withFile('first\n', file => { + const lines = []; + const scan = scanLines(file, 0, line => { + lines.push(line); + fs.appendFileSync(file, 'later\n'); + }); + assert.deepEqual(lines, ['first']); + assert.equal(scan.consumed, 6); + const later = []; + scanLines(file, scan.consumed, line => { later.push(line); }); + assert.deepEqual(later, ['later']); + }); +}); + +test('read errors propagate instead of returning a successful partial scan', () => { + withFile('first\n' + 'x'.repeat(512 * 1024), file => { + const read = fs.readSync; + fs.readSync = function(fd, buf, offset, length, position) { + if (position >= 256 * 1024) throw new Error('simulated read failure'); + return read(fd, buf, offset, length, position); + }; + try { assert.throws(() => scanLines(file, 0, () => {}), /simulated read failure/); } + finally { fs.readSync = read; } + }); +}); + +test('truncation during a bounded scan fails and leaves a borrowed descriptor open', () => { + withFile('first\n' + 'x'.repeat(512 * 1024), file => { + const fd = fs.openSync(file, 'r'); + try { + assert.throws(() => scanLines(fd, 0, () => { fs.truncateSync(file, 6); }), /truncated/); + assert.equal(fs.fstatSync(fd).size, 6); + } finally { fs.closeSync(fd); } + }); +}); diff --git a/test/read-session-file.test.js b/test/read-session-file.test.js new file mode 100644 index 00000000..f17fbc0f --- /dev/null +++ b/test/read-session-file.test.js @@ -0,0 +1,217 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { readSessionFile } = require('../harnesses/claude'); + +const FOLDER = 'test-folder'; +const PROJECT = 'C:/test/project'; + +function line(obj) { + return JSON.stringify(obj) + '\n'; +} + +/** A session file with `n` user/assistant pairs, padded so it is worth measuring. */ +function buildSession(n, pad = 2000) { + let out = line({ type: 'user', slug: 'my-slug', message: 'first question' }); + for (let i = 0; i < n; i++) { + out += line({ type: 'assistant', message: { content: 'answer ' + i + ' ' + 'x'.repeat(pad) } }); + out += line({ type: 'user', message: 'question ' + i + ' ' + 'y'.repeat(pad) }); + } + return out; +} + +function withTmp(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-read-session-')); + try { + return fn(dir); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test('an appended session is re-indexed by reading only the appended bytes', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(40), 'utf8'); + const sizeBefore = fs.statSync(file).size; + + const first = readSessionFile(file, FOLDER, PROJECT); + assert.ok(first, 'first pass should return a session'); + assert.ok(first.indexedBytes > 0, 'first pass must report how far it indexed'); + + const appended = line({ type: 'assistant', message: { content: 'a late answer' } }); + fs.appendFileSync(file, appended, 'utf8'); + + const incremental = readSessionFile(file, FOLDER, PROJECT, first); + + // The whole point: the second pass must not re-read the file from the start. + assert.ok( + incremental.bytesRead < sizeBefore / 10, + `incremental pass read ${incremental.bytesRead} bytes of a ${sizeBefore}-byte file — ` + + 'it re-read the whole file instead of only the tail' + ); + }); +}); + +test('incremental indexing yields the same session as a full re-read', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(12), 'utf8'); + + const first = readSessionFile(file, FOLDER, PROJECT); + + fs.appendFileSync(file, line({ type: 'assistant', message: { content: 'tail answer' } }), 'utf8'); + fs.appendFileSync(file, line({ type: 'ai-title', aiTitle: 'A Generated Title' }), 'utf8'); + fs.appendFileSync(file, line({ type: 'custom-title', customTitle: 'My Rename' }), 'utf8'); + + const incremental = readSessionFile(file, FOLDER, PROJECT, first); + const full = readSessionFile(file, FOLDER, PROJECT); + + for (const field of ['summary', 'messageCount', 'slug', 'customTitle', 'aiTitle', 'textContent']) { + assert.deepEqual(incremental[field], full[field], `field "${field}" diverged from a full re-read`); + } + }); +}); + +test('a rewritten file falls back to a full re-read instead of resuming', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(10), 'utf8'); + const first = readSessionFile(file, FOLDER, PROJECT); + + // Same path, different content (e.g. Claude Code compacted the session). + // Resuming from the old offset here would produce a corrupt message count. + fs.writeFileSync(file, buildSession(10).replace('first question', 'a different opening'), 'utf8'); + + const after = readSessionFile(file, FOLDER, PROJECT, first); + const full = readSessionFile(file, FOLDER, PROJECT); + + assert.equal(after.summary, full.summary, 'stale head was not detected — resumed on a rewritten file'); + assert.equal(after.messageCount, full.messageCount); + }); +}); + +test('a single-message file with no trailing newline is still indexed', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, JSON.stringify({ type: 'user', message: 'only message' }), 'utf8'); + + const s = readSessionFile(file, FOLDER, PROJECT); + assert.ok(s, 'a file whose last line lacks a newline must not vanish from the sidebar'); + assert.equal(s.messageCount, 1); + }); +}); + +test('truncation and atomic replacement with an unchanged head reset resume state', () => { + withTmp(dir => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(40)); + const first = readSessionFile(file, FOLDER, PROJECT); + fs.writeFileSync(file, buildSession(2)); + const truncated = readSessionFile(file, FOLDER, PROJECT, first); + assert.equal(truncated.messageCount, 5); + + const replacement = path.join(dir, 'replacement'); + // Keep more than 4 KiB identical but change message counts beyond it. + const content = buildSession(2) + line({ type: 'assistant', message: 'x'.repeat(200000) }); + fs.writeFileSync(replacement, content); + fs.renameSync(replacement, file); + const replaced = readSessionFile(file, FOLDER, PROJECT, first); + assert.equal(replaced.messageCount, 6); + assert.equal(replaced.messageCount, readSessionFile(file, FOLDER, PROJECT).messageCount); + }); +}); + +test('same-size rewrites beyond the head reset when the modification time changes', () => { + withTmp(dir => { + const file = path.join(dir, 'session.jsonl'); + const content = buildSession(4) + line({ type: 'ai-title', aiTitle: 'original' }); + fs.writeFileSync(file, content); + const first = readSessionFile(file, FOLDER, PROJECT); + fs.writeFileSync(file, content.replace('original', 'replaced')); + const future = new Date(Date.parse(first.fileMtime) + 1000); + fs.utimesSync(file, future, future); + assert.equal(readSessionFile(file, FOLDER, PROJECT, first).aiTitle, 'replaced'); + }); +}); + +test('a partial final message is not counted twice when it is completed', () => { + withTmp(dir => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(4)); + const first = readSessionFile(file, FOLDER, PROJECT); + const append = JSON.stringify({ type: 'assistant', message: 'the final answer' }); + fs.appendFileSync(file, append.slice(0, -3)); + const partial = readSessionFile(file, FOLDER, PROJECT, first); + assert.equal(partial.messageCount, first.messageCount); + assert.equal(partial.indexedBytes, 0); + fs.appendFileSync(file, append.slice(-3)); + const noNewline = readSessionFile(file, FOLDER, PROJECT, partial); + assert.equal(noNewline.messageCount, first.messageCount + 1); + assert.equal(noNewline.indexedBytes, 0); + fs.appendFileSync(file, '\n'); + const completed = readSessionFile(file, FOLDER, PROJECT, noNewline); + assert.equal(completed.messageCount, first.messageCount + 1); + assert.equal(completed.indexedBytes, fs.statSync(file).size); + }); +}); + +test('incremental timestamps use message bounds rather than file-time fallbacks', () => { + withTmp(dir => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(4)); + const first = readSessionFile(file, FOLDER, PROJECT); + assert.equal(first.firstTimestamp, null); + fs.appendFileSync(file, line({ type: 'assistant', message: 'dated answer', timestamp: '2025-01-02T00:00:00Z' })); + const dated = readSessionFile(file, FOLDER, PROJECT, first); + assert.equal(dated.created, '2025-01-02T00:00:00Z'); + assert.equal(dated.modified, '2025-01-02T00:00:00Z'); + fs.appendFileSync(file, line({ type: 'user', message: 'earlier date', timestamp: '2025-01-01T00:00:00Z' })); + fs.appendFileSync(file, line({ type: 'ai-title', aiTitle: 'new title without activity' })); + const after = readSessionFile(file, FOLDER, PROJECT, dated); + const full = readSessionFile(file, FOLDER, PROJECT); + for (const field of ['created', 'modified', 'firstTimestamp', 'lastTimestamp', 'fileMtime', 'runtime', 'sessionFile']) { + assert.equal(after[field], full[field], field); + } + assert.equal(after.created, '2025-01-01T00:00:00Z'); + assert.equal(after.modified, '2025-01-02T00:00:00Z'); + }); +}); + +test('old parser state and read errors cannot produce a seemingly complete incremental result', () => { + withTmp(dir => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(100)); + const first = readSessionFile(file, FOLDER, PROJECT); + const legacy = { ...first, headHash: first.headHash.slice(3), messageCount: 999 }; + assert.equal(readSessionFile(file, FOLDER, PROJECT, legacy).messageCount, first.messageCount); + const read = fs.readSync; + fs.readSync = function(fd, buf, offset, length, position) { + if (position >= 256 * 1024) throw new Error('simulated read failure'); + return read(fd, buf, offset, length, position); + }; + try { assert.equal(readSessionFile(file, FOLDER, PROJECT), null); } + finally { fs.readSync = read; } + }); +}); + +test('cwd derivation supports an unterminated first record without reading the rest of a large file', () => { + const { deriveProjectPath } = require('../harnesses/claude'); + withTmp(dir => { + const file = path.join(dir, 'session.jsonl'); + const record = JSON.stringify({ type: 'user', cwd: PROJECT, message: 'hello' }); + fs.writeFileSync(file, record); + assert.equal(deriveProjectPath(dir), PROJECT); + fs.appendFileSync(file, '\n' + 'x'.repeat(1024 * 1024)); + const read = fs.readSync; + let bytes = 0; + fs.readSync = function(...args) { const n = read(...args); bytes += n; return n; }; + try { + assert.equal(deriveProjectPath(dir), PROJECT); + assert.ok(bytes <= 256 * 1024); + } finally { fs.readSync = read; } + }); +}); diff --git a/test/reconcile-cache.test.js b/test/reconcile-cache.test.js index 53aa96d5..f96c52ab 100644 --- a/test/reconcile-cache.test.js +++ b/test/reconcile-cache.test.js @@ -35,6 +35,7 @@ function makeFakeDb(metaMap, globalSettings = {}) { db: { deleteCachedFolder() {}, getCachedByFolder() { return []; }, + getCachedSession() { return null; }, upsertCachedSessions(sessions) { for (const s of sessions) { indexedFolders.add(s.folder); cachedRows.push(s); } }, @@ -99,6 +100,43 @@ test('reconcileCacheFromFilesystem indexes new and stale folders but skips up-to } }); +test('an append arriving during refresh remains eligible for the next reconciliation', () => { + const claude = require('../harnesses/claude'); + const originalRead = claude.readSessionFile; + const projectsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-concurrent-append-')); + const folder = 'project'; + const folderPath = path.join(projectsDir, folder); + try { + writeSession(folderPath, '/tmp/project'); + const file = path.join(folderPath, 'session.jsonl'); + const before = getFolderIndexMtimeMs(folderPath); + const metaMap = new Map(); + const fake = makeFakeDb(metaMap, { disabledHarnesses: ['codex'] }); + fake.db.getCachedByFolder = () => fake.cachedRows; + fake.db.getCachedSession = id => fake.cachedRows.find(r => r.sessionId === id) || null; + sessionCache.init({ PROJECTS_DIR: projectsDir, activeSessions: new Map(), + getMainWindow: () => null, log: console, db: fake.db }); + claude.readSessionFile = (...args) => { + const row = originalRead(...args); + fs.appendFileSync(file, JSON.stringify({ type: 'assistant', message: 'arrived during refresh' }) + '\n'); + const later = new Date(before + 5000); + fs.utimesSync(file, later, later); + return row; + }; + sessionCache.refreshFolder(folder); + claude.readSessionFile = originalRead; + assert.equal(fake.cachedRows[0].messageCount, 1); + assert.ok(metaMap.get(folder).indexMtimeMs < getFolderIndexMtimeMs(folderPath)); + fake.indexedFolders.clear(); + sessionCache.reconcileCacheFromFilesystem(); + assert.ok(fake.indexedFolders.has(folder), 'the late append must not be acknowledged without parsing it'); + assert.equal(fake.cachedRows.at(-1).messageCount, 2); + } finally { + claude.readSessionFile = originalRead; + fs.rmSync(projectsDir, { recursive: true, force: true }); + } +}); + test('a new session restores a hidden project', () => { const projectsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-restore-hidden-')); const projectPath = '/tmp/hidden-worktree';