Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
All notable changes to this project are documented here, following
[Keep a Changelog](https://keepachangelog.com/) and semantic versioning.

## [0.1.2] - 2026-08-09

### Fixed

- Skip reading binary file contents during repository scans while preserving
their existing classification and zero-token accounting.

## [0.1.1] - 2026-08-06

### Changed
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "ctxtrim",
"version": "0.1.1",
"version": "0.1.2",
"description": "Trim what bloats your AI coding context. Scan a repo, find the high-cost/low-value files ballooning your Claude Code / Cursor / Codex context, and write ignore files to cut token cost. Zero dependencies.",
"type": "module",
"bin": {
Expand Down
18 changes: 15 additions & 3 deletions src/classify.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ const GENERATED_MARKERS = /(@generated\b|DO NOT EDIT|Code generated by|autogener
const dirParts = (rel) => rel.split(/[\\/]/).slice(0, -1);

/**
* @returns {{category:string, trim:boolean, binary:boolean, reason:string}}
* category ∈ vendored|build|lockfile|minified|data|generated|large|binary|source
* Classify categories that can be determined without reading file contents.
* @returns {{category:string, trim:boolean, binary:boolean, reason:string}|null}
*/
export function classify(rel, { size = 0, tokens = 0, sample = "", maxTokens = 2000 }) {
export function classifyPath(rel) {
const name = basename(rel).toLowerCase();
const ext = extname(name);
const parts = dirParts(rel).map((p) => p.toLowerCase());
Expand All @@ -52,6 +52,18 @@ export function classify(rel, { size = 0, tokens = 0, sample = "", maxTokens = 2
if (LOCKFILES.has(name)) return { category: "lockfile", trim: true, binary: false, reason: "dependency lockfile" };
if (MINIFIED.test(name)) return { category: "minified", trim: true, binary: false, reason: "minified / bundled / sourcemap" };
if (DATA_EXT.has(ext)) return { category: "data", trim: true, binary: false, reason: "data file" };
return null;
}

/**
* @returns {{category:string, trim:boolean, binary:boolean, reason:string}}
* category ∈ vendored|build|lockfile|minified|data|generated|large|binary|source
*/
export function classify(rel, { size = 0, tokens = 0, sample = "", maxTokens = 2000 }) {
const pathClassification = classifyPath(rel);
if (pathClassification) return pathClassification;

const ext = extname(basename(rel).toLowerCase());
if (sample && GENERATED_MARKERS.test(sample)) return { category: "generated", trim: true, binary: false, reason: 'marked "generated / do not edit"' };
// Large structured data masquerading as source (big JSON/YAML/XML/SVG).
if ([".json", ".yaml", ".yml", ".xml", ".svg"].includes(ext) && tokens > maxTokens)
Expand Down
7 changes: 4 additions & 3 deletions src/scan.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Walk a repo, estimate each file's token cost, classify it, and aggregate.
import { readdirSync, readFileSync, statSync, openSync, readSync, closeSync, existsSync } from "node:fs";
import { join, relative, sep } from "node:path";
import { classify, ignorePattern } from "./classify.js";
import { classify, classifyPath, ignorePattern } from "./classify.js";

const ALWAYS_SKIP = new Set([".git"]);
const MAX_READ = 5_000_000; // bytes fully read; larger files are estimated from size
Expand Down Expand Up @@ -50,8 +50,9 @@ export function scanRepo(target, opts = {}) {
let size = 0;
try { size = statSync(abs).size; } catch { continue; }
const rel = relative(root, abs).split(sep).join("/");
const info = fileInfo(abs, size);
const c = classify(rel, { size, tokens: info.tokens, sample: info.sample, maxTokens });
const pathClassification = classifyPath(rel);
const info = pathClassification?.binary ? { tokens: 0, sample: "" } : fileInfo(abs, size);
const c = pathClassification ?? classify(rel, { size, tokens: info.tokens, sample: info.sample, maxTokens });
const tokens = c.binary ? 0 : info.tokens; // binaries carry no text tokens
files.push({ rel, size, tokens, category: c.category, trim: c.trim, binary: c.binary, reason: c.reason });
}
Expand Down
74 changes: 73 additions & 1 deletion test/ctxtrim.test.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import fs, { mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs";
import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import { test } from "node:test";
import assert from "node:assert/strict";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import { scanRepo, estimateTokens } from "../src/scan.js";
import { classify } from "../src/classify.js";
import { classify, classifyPath } from "../src/classify.js";
import { merge, block } from "../src/ignore.js";

const repo = join(dirname(fileURLToPath(import.meta.url)), "fixtures", "sample-repo");
Expand All @@ -24,6 +27,75 @@ test("classify buckets files correctly", () => {
assert.equal(classify("src/index.js", { tokens: 50 }).trim, false);
// big json flagged as data
assert.equal(classify("big.json", { tokens: 9000, maxTokens: 2000 }).category, "data");
assert.deepEqual(classifyPath("logo.png"), classify("logo.png", {}));
assert.equal(classifyPath("generated.js"), null);
assert.equal(classifyPath("big.json"), null);
});

test("scan skips binary reads and retains content-dependent classification", (t) => {
const root = mkdtempSync(join(tmpdir(), "ctxtrim-binary-read-"));
t.after(() => rmSync(root, { recursive: true, force: true }));

const image = join(root, "pixel.png");
const document = join(root, "manual.pdf");
const generated = join(root, "generated.js");
const largeJson = join(root, "large.json");
writeFileSync(image, "not really an image");
writeFileSync(document, "");
truncateSync(document, 5_000_001);
writeFileSync(generated, "// @generated\nexport const value = 1;\n");
writeFileSync(largeJson, JSON.stringify({ payload: "x".repeat(5_100_000) }));

const readFiles = [];
const openedFiles = new Map();
const partialReadFiles = [];
const readFileSync = fs.readFileSync;
const openSync = fs.openSync;
const readSync = fs.readSync;
t.mock.method(fs, "readFileSync", (...args) => {
readFiles.push(String(args[0]));
return readFileSync(...args);
});
t.mock.method(fs, "openSync", (...args) => {
const fd = openSync(...args);
openedFiles.set(fd, String(args[0]));
return fd;
});
t.mock.method(fs, "readSync", (...args) => {
partialReadFiles.push(openedFiles.get(args[0]));
return readSync(...args);
});
syncBuiltinESMExports();

let result;
try {
result = scanRepo(root);
} finally {
t.mock.restoreAll();
syncBuiltinESMExports();
}

assert.ok(!readFiles.includes(image));
assert.ok(!readFiles.includes(document));
assert.ok(![...openedFiles.values()].includes(document));
assert.ok(!partialReadFiles.includes(document));
assert.ok(readFiles.includes(generated));
assert.ok([...openedFiles.values()].includes(largeJson));
assert.ok(partialReadFiles.includes(largeJson));

for (const rel of ["pixel.png", "manual.pdf"]) {
const actual = result.files.find((file) => file.rel === rel);
const expected = classify(rel, {});
assert.deepEqual({
category: actual.category,
trim: actual.trim,
binary: actual.binary,
reason: actual.reason,
}, expected);
assert.equal(actual.tokens, 0);
}
assert.equal(result.files.find((file) => file.rel === "generated.js").category, "generated");
assert.equal(result.files.find((file) => file.rel === "large.json").category, "data");
});

test("scan finds trimmable bloat and keeps source", () => {
Expand Down
Loading