From 3e5be1900c29dd9cd5b43fb9b7d4e34c4af7edd2 Mon Sep 17 00:00:00 2001 From: Davidb-2107 <79403282+Davidb-2107@users.noreply.github.com> Date: Thu, 30 Jul 2026 08:53:25 +0200 Subject: [PATCH] perf: stop reading whole session files, and index them incrementally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a machine with ~2.3 GB under ~/.claude/projects, the main process sat at a 139 MB median but spiked past 250 MB in 24% of samples, peaking at 498 MB (921 MB across all processes) — with a single terminal open, so neither the terminals nor the grid view were involved. Two causes, both "read the whole file to use a little of it". A session .jsonl held as a JS string costs ~2x its size in RAM, since V8 stores non-latin1 text as UTF-16. 1. Three sites read an entire file just to get its head: schedule-runner.js kept 4000 chars — every 60s, for every project folder main.js kept 8000 chars derive-project-path kept the first line carrying `cwd` The scheduler one dominated: a 61 MB session file allocated ~122 MB once a minute, which is the sawtooth in the main process. 2. readSessionFile re-read the file in full on every append, to produce ~9 KB of metadata. The projects watcher fires that on each write, so an active session re-read its whole history every few seconds. Session files are append-only (folder-index-state.js already relies on it; measured here: 0 rewrites and 0 truncations across 1214 files), and every field readSessionFile extracts is either a first occurrence or a running total. So it now resumes from the byte offset the previous pass reached, persisted alongside a 4 KB head hash and a size check that fall back to a full read if the file was rewritten or truncated. Adds jsonl-scan.js with the two supported ways to walk these files — scanLines (chunked, resumable, early-exit) and readHead. Measured on the same workload, main process over 4 minutes: before rss 136 -> 520 MB, heap peak 360 MB, 3 jumps of +354 MB after rss 145 -> 170 MB, heap peak 12 MB, 0 jumps A/B against the released build over 11 minutes, one terminal open: peak across all processes 921 -> 526 MB main process peak 498 -> 158 MB samples above 250 MB 24% -> 0% Re-indexing after an append: ~0 MB and 22 ms, from 152 MB and 447 ms. Co-Authored-By: Claude Opus 5 (1M context) --- db.js | 39 ++++++-- derive-project-path.js | 26 +++--- jsonl-scan.js | 91 ++++++++++++++++++ main.js | 5 +- read-session-file.js | 165 +++++++++++++++++++++++++-------- schedule-runner.js | 5 +- session-cache.js | 11 ++- test/read-session-file.test.js | 106 +++++++++++++++++++++ 8 files changed, 384 insertions(+), 64 deletions(-) create mode 100644 jsonl-scan.js create mode 100644 test/read-session-file.test.js diff --git a/db.js b/db.js index 4fabae44..cb9b6803 100644 --- a/db.js +++ b/db.js @@ -49,7 +49,11 @@ db.exec(` modified TEXT, messageCount INTEGER DEFAULT 0, slug TEXT, - aiTitle TEXT + aiTitle TEXT, + customTitle TEXT, + textContent TEXT, + headHash TEXT, + indexedBytes INTEGER DEFAULT 0 ) `); @@ -99,6 +103,19 @@ const migrations = [ try { db.exec('DELETE FROM session_cache'); } catch {} try { db.exec('DELETE FROM cache_meta'); } catch {} }, + // v4: Columns backing incremental (append-only) re-indexing. Session .jsonl + // files only ever grow, so an index pass can resume from the byte offset the + // previous pass stopped at instead of re-reading the whole file — which cost + // ~2x the file size in RAM on every append. The resume state needs the + // accumulators to survive across passes, hence customTitle/textContent. + // No cache wipe: indexedBytes stays NULL on existing rows, which reads as + // "cannot resume", so each session is fully re-read once and incrementally + // after that. + (db) => { + for (const col of ['customTitle TEXT', 'textContent TEXT', 'headHash TEXT', 'indexedBytes INTEGER DEFAULT 0']) { + try { db.exec(`ALTER TABLE session_cache ADD COLUMN ${col}`); } catch {} + } + }, ]; const currentDbVersion = (() => { @@ -150,16 +167,25 @@ const stmts = { `), // Session cache statements cacheCount: db.prepare('SELECT COUNT(*) as cnt FROM session_cache'), - cacheGetAll: db.prepare('SELECT * FROM session_cache'), + // Only the columns the sidebar renders. Deliberately excludes textContent and + // the resume state (headHash/indexedBytes): this runs on every projects + // refresh, and SELECT * would drag several MB of search-index text with it. + cacheGetAll: db.prepare(` + SELECT sessionId, folder, projectPath, summary, firstPrompt, created, modified, + messageCount, slug, aiTitle + FROM session_cache + `), cacheUpsert: db.prepare(` - INSERT INTO session_cache (sessionId, folder, projectPath, summary, firstPrompt, created, modified, messageCount, slug, aiTitle) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO session_cache (sessionId, folder, projectPath, summary, firstPrompt, created, modified, messageCount, slug, aiTitle, customTitle, textContent, headHash, indexedBytes) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(sessionId) DO UPDATE SET folder = excluded.folder, projectPath = excluded.projectPath, summary = excluded.summary, firstPrompt = excluded.firstPrompt, created = excluded.created, modified = excluded.modified, messageCount = excluded.messageCount, slug = excluded.slug, - aiTitle = excluded.aiTitle + aiTitle = excluded.aiTitle, customTitle = excluded.customTitle, + textContent = excluded.textContent, headHash = excluded.headHash, + indexedBytes = excluded.indexedBytes `), cacheGetByFolder: db.prepare('SELECT sessionId, modified FROM session_cache WHERE folder = ?'), cacheGetFolder: db.prepare('SELECT folder FROM session_cache WHERE sessionId = ?'), @@ -246,7 +272,8 @@ const upsertCachedSessionsBatch = db.transaction((sessions) => { stmts.cacheUpsert.run( s.sessionId, s.folder, s.projectPath, s.summary, s.firstPrompt, s.created, s.modified, s.messageCount || 0, - s.slug || null, s.aiTitle || null + s.slug || null, s.aiTitle || null, s.customTitle || null, + s.textContent || null, s.headHash || null, s.indexedBytes || 0 ); } }); diff --git a/derive-project-path.js b/derive-project-path.js index f563e35b..3da95b9f 100644 --- a/derive-project-path.js +++ b/derive-project-path.js @@ -1,18 +1,22 @@ const fs = require('fs'); const path = require('path'); +const { scanLines } = require('./jsonl-scan'); function extractCwdFromJsonl(filePath) { - try { - const lines = fs.readFileSync(filePath, 'utf8').split('\n'); - for (const line of lines) { - if (!line) continue; - try { - const parsed = JSON.parse(line); - if (parsed.cwd) return parsed.cwd; - } catch {} - } - } catch {} - return null; + // `cwd` is on the first line in practice, so stop at the first hit rather than + // reading the file — this runs per folder on every index pass, and a session + // .jsonl can be hundreds of MB. + let cwd = null; + scanLines(filePath, 0, (line) => { + try { + const parsed = JSON.parse(line); + if (parsed.cwd) { + cwd = parsed.cwd; + return false; + } + } catch {} + }); + return cwd; } function resolveWorktreePath(cwd) { diff --git a/jsonl-scan.js b/jsonl-scan.js new file mode 100644 index 00000000..19d9ea35 --- /dev/null +++ b/jsonl-scan.js @@ -0,0 +1,91 @@ +const fs = require('fs'); + +// Session .jsonl files routinely reach tens or hundreds of MB. Reading one into +// a JS string costs ~2x its size in RAM (V8 stores non-latin1 text as UTF-16), +// so any code that wants the first line — or one field — must not use +// readFileSync. These two helpers are the supported way to walk such a file. + +const CHUNK_BYTES = 256 * 1024; + +/** + * Walk the complete lines of a file from `startByte`, holding at most one chunk + * (plus the current line) in memory. + * + * `onLine(line)` may return false to stop early — useful when the caller only + * needs the first line carrying some field. + * + * Returns: + * consumed — offset just past the last complete line (a resume point) + * read — bytes actually pulled off disk + * tail — trailing bytes with no newline; NOT counted in `consumed`, + * because a partial line will be re-read on the next pass + * stopped — whether onLine asked to stop + */ +function scanLines(filePath, startByte, onLine) { + let fd = null; + let consumed = startByte; + let read = 0; + let tail = ''; + let stopped = false; + + try { + fd = fs.openSync(filePath, 'r'); + const buf = Buffer.allocUnsafe(CHUNK_BYTES); + let pending = Buffer.alloc(0); + let pos = startByte; + let n; + + while ((n = fs.readSync(fd, buf, 0, CHUNK_BYTES, pos)) > 0) { + pos += n; + read += n; + // `pending` carries the partial line from the previous chunk, so `data` + // always starts at offset `consumed`. That is what keeps the byte + // accounting exact across chunk boundaries and multi-byte UTF-8. + const data = pending.length + ? Buffer.concat([pending, buf.subarray(0, n)]) + : Buffer.from(buf.subarray(0, n)); + + let from = 0; + let nl; + while ((nl = data.indexOf(0x0A, from)) !== -1) { + if (nl > from && onLine(data.toString('utf8', from, nl)) === false) { + consumed += nl - from + 1; + stopped = true; + return { consumed, read, tail: '', stopped }; + } + consumed += nl - from + 1; + from = nl + 1; + } + pending = data.subarray(from); + } + + if (pending.length) tail = pending.toString('utf8'); + } catch { + // Fall through with whatever was gathered — callers treat this as "no data". + } finally { + if (fd !== null) { + try { fs.closeSync(fd); } catch {} + } + } + + return { consumed, read, tail, stopped }; +} + +/** Read at most `maxBytes` from the start of a file. Returns '' on failure. */ +function readHead(filePath, maxBytes) { + let fd = null; + try { + fd = fs.openSync(filePath, 'r'); + const buf = Buffer.allocUnsafe(maxBytes); + const n = fs.readSync(fd, buf, 0, maxBytes, 0); + return buf.toString('utf8', 0, n); + } catch { + return ''; + } finally { + if (fd !== null) { + try { fs.closeSync(fd); } catch {} + } + } +} + +module.exports = { scanLines, readHead }; diff --git a/main.js b/main.js index 2c587b77..85725e51 100644 --- a/main.js +++ b/main.js @@ -8,6 +8,7 @@ const log = require('electron-log'); // getFolderIndexMtimeMs moved to session-cache.js const { startMcpServer, shutdownMcpServer, shutdownAll: shutdownAllMcp, resolvePendingDiff, rekeyMcpServer, cleanStaleLockFiles } = require('./mcp-bridge'); const { fetchAndTransformUsage } = require('./claude-auth'); +const { readHead } = require('./jsonl-scan'); log.transports.file.level = app.isPackaged ? 'info' : 'debug'; log.transports.console.level = app.isPackaged ? 'info' : 'debug'; @@ -260,7 +261,7 @@ sessionCache.init({ getMainWindow: () => mainWindow, log, db: { - deleteCachedFolder, getCachedByFolder, upsertCachedSessions, deleteCachedSession, + deleteCachedFolder, getCachedByFolder, getCachedSession, upsertCachedSessions, deleteCachedSession, deleteSearchFolder, deleteSearchSession, upsertSearchEntries, setFolderMeta, getAllFolderMeta, getAllMeta, getAllCached, getSetting, getMeta, setName, }, @@ -996,7 +997,7 @@ ipcMain.handle('open-terminal', async (_event, sessionId, projectPath, isNew, se if (!isNew) { try { const jsonlPath = path.join(claudeProjectDir, sessionId + '.jsonl'); - const head = fs.readFileSync(jsonlPath, 'utf8').slice(0, 8000); + const head = readHead(jsonlPath, 8192); const firstLines = head.split('\n').filter(Boolean); for (const line of firstLines) { const entry = JSON.parse(line); diff --git a/read-session-file.js b/read-session-file.js index bb42318e..256e8c19 100644 --- a/read-session-file.js +++ b/read-session-file.js @@ -1,58 +1,141 @@ const path = require('path'); const fs = require('fs'); +const crypto = require('crypto'); +const { scanLines } = require('./jsonl-scan'); -/** Parse a single .jsonl file into a session object (or null if invalid) */ -function readSessionFile(filePath, folder, projectPath) { +// Session .jsonl files are append-only and can reach hundreds of MB, while the +// metadata we extract from them is a few KB. Reading one whole file into a JS +// string costs ~2x its size in RAM (V8 stores non-latin1 text as UTF-16), and +// the projects watcher re-triggers that read on every append. So we read in +// fixed-size chunks and, when the caller hands us the previously indexed row, +// resume from where that pass stopped instead of starting over. + +const HEAD_BYTES = 4096; // guard window — detects a file rewritten in place +const TEXT_CONTENT_CAP = 8000; // how much body text the search index keeps +const TEXT_LINE_CAP = 500; + +function hashHead(fd, size) { + const n = Math.min(HEAD_BYTES, size); + if (n === 0) return ''; + const buf = Buffer.allocUnsafe(n); + fs.readSync(fd, buf, 0, n, 0); + return crypto.createHash('sha1').update(buf).digest('hex'); +} + +/** Accumulator. Every field is either a first-occurrence or a running total, + * which is what makes resuming mid-file valid. */ +function emptyState() { + return { summary: '', messageCount: 0, textContent: '', slug: null, customTitle: null, aiTitle: null }; +} + +function stateFrom(prev) { + return { + summary: prev.summary || '', + messageCount: prev.messageCount || 0, + textContent: prev.textContent || '', + slug: prev.slug || null, + customTitle: prev.customTitle || null, + aiTitle: prev.aiTitle || null, + }; +} + +function applyLine(line, st) { + let entry; + try { entry = JSON.parse(line); } catch { return; } + + if (entry.slug && !st.slug) st.slug = entry.slug; + if (entry.type === 'custom-title' && entry.customTitle) st.customTitle = entry.customTitle; + if (entry.type === 'ai-title' && entry.aiTitle) st.aiTitle = entry.aiTitle; + + if (entry.type === 'user' || entry.type === 'assistant' || + (entry.type === 'message' && (entry.role === 'user' || entry.role === 'assistant'))) { + st.messageCount++; + } + + const msg = entry.message; + const text = typeof msg === 'string' ? msg : + (typeof msg?.content === 'string' ? msg.content : + (msg?.content?.[0]?.text || '')); + + if (!st.summary && (entry.type === 'user' || (entry.type === 'message' && entry.role === 'user'))) { + // Skip local command messages (! prefix) — use the next real user message + if (text && !/||/.test(text)) { + // Use scheduled task name if present + const taskMatch = text.match(/||/.test(text)) { - // Use scheduled task name if present - const taskMatch = text.match(/ 0 + && prev.indexedBytes <= stat.size; + + const st = canResume ? stateFrom(prev) : emptyState(); + const start = canResume ? prev.indexedBytes : 0; + + const { consumed, read, tail } = scanLines(filePath, start, (line) => applyLine(line, st)); + + // A trailing line with no newline is either a write caught in flight or a + // file that simply ends without one. Parse it so a single-message session + // still appears, but refuse to resume past it — the next pass would + // otherwise count that line a second time. + let resumable = true; + if (tail) { + applyLine(tail, st); + resumable = false; } - if (!summary || messageCount < 1) return null; + + if (!st.summary || st.messageCount < 1) return null; + return { sessionId, folder, projectPath, - summary, firstPrompt: summary, + summary: st.summary, firstPrompt: st.summary, created: stat.birthtime.toISOString(), modified: stat.mtime.toISOString(), - messageCount, textContent, slug, customTitle, aiTitle, + messageCount: st.messageCount, + textContent: st.textContent, + slug: st.slug, + customTitle: st.customTitle, + aiTitle: st.aiTitle, + headHash, + indexedBytes: resumable ? consumed : 0, + bytesRead: read, }; } catch { return null; + } finally { + if (fd !== null) { + try { fs.closeSync(fd); } catch {} + } } } diff --git a/schedule-runner.js b/schedule-runner.js index b8a51efc..c8d9d350 100644 --- a/schedule-runner.js +++ b/schedule-runner.js @@ -3,6 +3,7 @@ const fs = require('fs'); const path = require('path'); const os = require('os'); const crypto = require('crypto'); +const { readHead } = require('./jsonl-scan'); const CLAUDE_DIR = path.join(os.homedir(), '.claude'); const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects'); @@ -88,7 +89,9 @@ function scanSchedules(log) { try { const jsonlFiles = fs.readdirSync(folderPath).filter(f => f.endsWith('.jsonl')); for (const jf of jsonlFiles) { - const head = fs.readFileSync(path.join(folderPath, jf), 'utf8').slice(0, 4000); + // Head only: this runs every minute over every project folder, and a + // session .jsonl can be hundreds of MB. + const head = readHead(path.join(folderPath, jf), 4096); for (const line of head.split('\n').filter(Boolean)) { try { const entry = JSON.parse(line); diff --git a/session-cache.js b/session-cache.js index f066004e..bb37ec35 100644 --- a/session-cache.js +++ b/session-cache.js @@ -11,7 +11,7 @@ const { encodeProjectPath } = require('./encode-project-path'); * Call init(ctx) once with the shared context object. */ let PROJECTS_DIR, activeSessions, getMainWindow, log; -let deleteCachedFolder, getCachedByFolder, upsertCachedSessions, deleteCachedSession; +let deleteCachedFolder, getCachedByFolder, getCachedSession, upsertCachedSessions, deleteCachedSession; let deleteSearchFolder, deleteSearchSession, upsertSearchEntries; let setFolderMeta, getAllFolderMeta, getAllMeta, getAllCached, getSetting, getMeta, setName; @@ -23,6 +23,7 @@ function init(ctx) { // DB functions deleteCachedFolder = ctx.db.deleteCachedFolder; getCachedByFolder = ctx.db.getCachedByFolder; + getCachedSession = ctx.db.getCachedSession; upsertCachedSessions = ctx.db.upsertCachedSessions; deleteCachedSession = ctx.db.deleteCachedSession; deleteSearchFolder = ctx.db.deleteSearchFolder; @@ -106,8 +107,12 @@ function refreshFolder(folder) { continue; // unchanged, skip } - // File is new or modified — re-read it - const s = readSessionFile(filePath, folder, projectPath); + // File is new or modified — re-read it. The cached row carries the resume + // state, so an append costs the size of the append rather than the size of + // the file. Fetched per changed session on purpose: a folder holds + // thousands of sessions and only a couple change per flush. + const cachedRow = getCachedSession(sessionId); + const s = readSessionFile(filePath, folder, projectPath, cachedRow); if (s) { sessionsToUpsert.push(s); // Title precedence: user rename (session_meta.name) > JSONL custom-title > JSONL ai-title. diff --git a/test/read-session-file.test.js b/test/read-session-file.test.js new file mode 100644 index 00000000..09d4e078 --- /dev/null +++ b/test/read-session-file.test.js @@ -0,0 +1,106 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { readSessionFile } = require('../read-session-file'); + +const FOLDER = 'test-folder'; +const PROJECT = 'C:/test/project'; + +function line(obj) { + return JSON.stringify(obj) + '\n'; +} + +/** A session file with `n` user/assistant pairs, padded so it is worth measuring. */ +function buildSession(n, pad = 2000) { + let out = line({ type: 'user', slug: 'my-slug', message: 'first question' }); + for (let i = 0; i < n; i++) { + out += line({ type: 'assistant', message: { content: 'answer ' + i + ' ' + 'x'.repeat(pad) } }); + out += line({ type: 'user', message: 'question ' + i + ' ' + 'y'.repeat(pad) }); + } + return out; +} + +function withTmp(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-read-session-')); + try { + return fn(dir); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test('an appended session is re-indexed by reading only the appended bytes', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(40), 'utf8'); + const sizeBefore = fs.statSync(file).size; + + const first = readSessionFile(file, FOLDER, PROJECT); + assert.ok(first, 'first pass should return a session'); + assert.ok(first.indexedBytes > 0, 'first pass must report how far it indexed'); + + const appended = line({ type: 'assistant', message: { content: 'a late answer' } }); + fs.appendFileSync(file, appended, 'utf8'); + + const incremental = readSessionFile(file, FOLDER, PROJECT, first); + + // The whole point: the second pass must not re-read the file from the start. + assert.ok( + incremental.bytesRead < sizeBefore / 10, + `incremental pass read ${incremental.bytesRead} bytes of a ${sizeBefore}-byte file — ` + + 'it re-read the whole file instead of only the tail' + ); + }); +}); + +test('incremental indexing yields the same session as a full re-read', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(12), 'utf8'); + + const first = readSessionFile(file, FOLDER, PROJECT); + + fs.appendFileSync(file, line({ type: 'assistant', message: { content: 'tail answer' } }), 'utf8'); + fs.appendFileSync(file, line({ type: 'ai-title', aiTitle: 'A Generated Title' }), 'utf8'); + fs.appendFileSync(file, line({ type: 'custom-title', customTitle: 'My Rename' }), 'utf8'); + + const incremental = readSessionFile(file, FOLDER, PROJECT, first); + const full = readSessionFile(file, FOLDER, PROJECT); + + for (const field of ['summary', 'messageCount', 'slug', 'customTitle', 'aiTitle', 'textContent']) { + assert.deepEqual(incremental[field], full[field], `field "${field}" diverged from a full re-read`); + } + }); +}); + +test('a rewritten file falls back to a full re-read instead of resuming', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, buildSession(10), 'utf8'); + const first = readSessionFile(file, FOLDER, PROJECT); + + // Same path, different content (e.g. Claude Code compacted the session). + // Resuming from the old offset here would produce a corrupt message count. + fs.writeFileSync(file, buildSession(10).replace('first question', 'a different opening'), 'utf8'); + + const after = readSessionFile(file, FOLDER, PROJECT, first); + const full = readSessionFile(file, FOLDER, PROJECT); + + assert.equal(after.summary, full.summary, 'stale head was not detected — resumed on a rewritten file'); + assert.equal(after.messageCount, full.messageCount); + }); +}); + +test('a single-message file with no trailing newline is still indexed', () => { + withTmp((dir) => { + const file = path.join(dir, 'session.jsonl'); + fs.writeFileSync(file, JSON.stringify({ type: 'user', message: 'only message' }), 'utf8'); + + const s = readSessionFile(file, FOLDER, PROJECT); + assert.ok(s, 'a file whose last line lacks a newline must not vanish from the sidebar'); + assert.equal(s.messageCount, 1); + }); +});