From 28ce637f0dc3184933adc419e29554a779005484 Mon Sep 17 00:00:00 2001 From: Florian Meyer Date: Sun, 9 Aug 2026 15:13:40 +0200 Subject: [PATCH] fix: skip reading binary files during scans --- CHANGELOG.md | 7 +++++ package.json | 2 +- src/classify.js | 18 +++++++++-- src/scan.js | 7 +++-- test/ctxtrim.test.js | 74 +++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4f54a..659ae1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to this project are documented here, following [Keep a Changelog](https://keepachangelog.com/) and semantic versioning. +## [0.1.2] - 2026-08-09 + +### Fixed + +- Skip reading binary file contents during repository scans while preserving + their existing classification and zero-token accounting. + ## [0.1.1] - 2026-08-06 ### Changed diff --git a/package.json b/package.json index b80912c..eba3b63 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ctxtrim", - "version": "0.1.1", + "version": "0.1.2", "description": "Trim what bloats your AI coding context. Scan a repo, find the high-cost/low-value files ballooning your Claude Code / Cursor / Codex context, and write ignore files to cut token cost. Zero dependencies.", "type": "module", "bin": { diff --git a/src/classify.js b/src/classify.js index a89f39a..e648cfd 100644 --- a/src/classify.js +++ b/src/classify.js @@ -38,10 +38,10 @@ const GENERATED_MARKERS = /(@generated\b|DO NOT EDIT|Code generated by|autogener const dirParts = (rel) => rel.split(/[\\/]/).slice(0, -1); /** - * @returns {{category:string, trim:boolean, binary:boolean, reason:string}} - * category ∈ vendored|build|lockfile|minified|data|generated|large|binary|source + * Classify categories that can be determined without reading file contents. + * @returns {{category:string, trim:boolean, binary:boolean, reason:string}|null} */ -export function classify(rel, { size = 0, tokens = 0, sample = "", maxTokens = 2000 }) { +export function classifyPath(rel) { const name = basename(rel).toLowerCase(); const ext = extname(name); const parts = dirParts(rel).map((p) => p.toLowerCase()); @@ -52,6 +52,18 @@ export function classify(rel, { size = 0, tokens = 0, sample = "", maxTokens = 2 if (LOCKFILES.has(name)) return { category: "lockfile", trim: true, binary: false, reason: "dependency lockfile" }; if (MINIFIED.test(name)) return { category: "minified", trim: true, binary: false, reason: "minified / bundled / sourcemap" }; if (DATA_EXT.has(ext)) return { category: "data", trim: true, binary: false, reason: "data file" }; + return null; +} + +/** + * @returns {{category:string, trim:boolean, binary:boolean, reason:string}} + * category ∈ vendored|build|lockfile|minified|data|generated|large|binary|source + */ +export function classify(rel, { size = 0, tokens = 0, sample = "", maxTokens = 2000 }) { + const pathClassification = classifyPath(rel); + if (pathClassification) return pathClassification; + + const ext = extname(basename(rel).toLowerCase()); if (sample && GENERATED_MARKERS.test(sample)) return { category: "generated", trim: true, binary: false, reason: 'marked "generated / do not edit"' }; // Large structured data masquerading as source (big JSON/YAML/XML/SVG). if ([".json", ".yaml", ".yml", ".xml", ".svg"].includes(ext) && tokens > maxTokens) diff --git a/src/scan.js b/src/scan.js index 3d77ec0..4f1c63c 100644 --- a/src/scan.js +++ b/src/scan.js @@ -1,7 +1,7 @@ // Walk a repo, estimate each file's token cost, classify it, and aggregate. import { readdirSync, readFileSync, statSync, openSync, readSync, closeSync, existsSync } from "node:fs"; import { join, relative, sep } from "node:path"; -import { classify, ignorePattern } from "./classify.js"; +import { classify, classifyPath, ignorePattern } from "./classify.js"; const ALWAYS_SKIP = new Set([".git"]); const MAX_READ = 5_000_000; // bytes fully read; larger files are estimated from size @@ -50,8 +50,9 @@ export function scanRepo(target, opts = {}) { let size = 0; try { size = statSync(abs).size; } catch { continue; } const rel = relative(root, abs).split(sep).join("/"); - const info = fileInfo(abs, size); - const c = classify(rel, { size, tokens: info.tokens, sample: info.sample, maxTokens }); + const pathClassification = classifyPath(rel); + const info = pathClassification?.binary ? { tokens: 0, sample: "" } : fileInfo(abs, size); + const c = pathClassification ?? classify(rel, { size, tokens: info.tokens, sample: info.sample, maxTokens }); const tokens = c.binary ? 0 : info.tokens; // binaries carry no text tokens files.push({ rel, size, tokens, category: c.category, trim: c.trim, binary: c.binary, reason: c.reason }); } diff --git a/test/ctxtrim.test.js b/test/ctxtrim.test.js index 586fbe2..72793df 100644 --- a/test/ctxtrim.test.js +++ b/test/ctxtrim.test.js @@ -1,9 +1,12 @@ +import fs, { mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; import { test } from "node:test"; import assert from "node:assert/strict"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { scanRepo, estimateTokens } from "../src/scan.js"; -import { classify } from "../src/classify.js"; +import { classify, classifyPath } from "../src/classify.js"; import { merge, block } from "../src/ignore.js"; const repo = join(dirname(fileURLToPath(import.meta.url)), "fixtures", "sample-repo"); @@ -24,6 +27,75 @@ test("classify buckets files correctly", () => { assert.equal(classify("src/index.js", { tokens: 50 }).trim, false); // big json flagged as data assert.equal(classify("big.json", { tokens: 9000, maxTokens: 2000 }).category, "data"); + assert.deepEqual(classifyPath("logo.png"), classify("logo.png", {})); + assert.equal(classifyPath("generated.js"), null); + assert.equal(classifyPath("big.json"), null); +}); + +test("scan skips binary reads and retains content-dependent classification", (t) => { + const root = mkdtempSync(join(tmpdir(), "ctxtrim-binary-read-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + + const image = join(root, "pixel.png"); + const document = join(root, "manual.pdf"); + const generated = join(root, "generated.js"); + const largeJson = join(root, "large.json"); + writeFileSync(image, "not really an image"); + writeFileSync(document, ""); + truncateSync(document, 5_000_001); + writeFileSync(generated, "// @generated\nexport const value = 1;\n"); + writeFileSync(largeJson, JSON.stringify({ payload: "x".repeat(5_100_000) })); + + const readFiles = []; + const openedFiles = new Map(); + const partialReadFiles = []; + const readFileSync = fs.readFileSync; + const openSync = fs.openSync; + const readSync = fs.readSync; + t.mock.method(fs, "readFileSync", (...args) => { + readFiles.push(String(args[0])); + return readFileSync(...args); + }); + t.mock.method(fs, "openSync", (...args) => { + const fd = openSync(...args); + openedFiles.set(fd, String(args[0])); + return fd; + }); + t.mock.method(fs, "readSync", (...args) => { + partialReadFiles.push(openedFiles.get(args[0])); + return readSync(...args); + }); + syncBuiltinESMExports(); + + let result; + try { + result = scanRepo(root); + } finally { + t.mock.restoreAll(); + syncBuiltinESMExports(); + } + + assert.ok(!readFiles.includes(image)); + assert.ok(!readFiles.includes(document)); + assert.ok(![...openedFiles.values()].includes(document)); + assert.ok(!partialReadFiles.includes(document)); + assert.ok(readFiles.includes(generated)); + assert.ok([...openedFiles.values()].includes(largeJson)); + assert.ok(partialReadFiles.includes(largeJson)); + + for (const rel of ["pixel.png", "manual.pdf"]) { + const actual = result.files.find((file) => file.rel === rel); + const expected = classify(rel, {}); + assert.deepEqual({ + category: actual.category, + trim: actual.trim, + binary: actual.binary, + reason: actual.reason, + }, expected); + assert.equal(actual.tokens, 0); + } + assert.equal(result.files.find((file) => file.rel === "generated.js").category, "generated"); + assert.equal(result.files.find((file) => file.rel === "large.json").category, "data"); }); test("scan finds trimmable bloat and keeps source", () => {