Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/db/crud.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,55 @@ describe("results CRUD", () => {
expect(ftsResults.length).toBe(1);
expect(ftsResults[0]!.title).toBe("TypeScript Guide");
});

it("redacts persisted content-result output, including legacy rows, without changing safe results", () => {
const syntheticValue = "synthetic-only-not-live-7a31";
const sensitiveLine = `needleword service_password = ${syntheticValue}`;
const safeLine = "needleword documents password handling without an assigned value";
const s = createSearch({ query: "needleword", providers: ["content", "google"] }, db);
const content = createResult(
{
searchId: s.id,
title: "config.txt",
url: "file:///synthetic/config.txt",
snippet: sensitiveLine,
source: "content",
provider: "Local Content",
rank: 1,
metadata: { line: 2, matches: [{ line: 2, text: sensitiveLine }] },
},
db,
);
const safe = createResult(
{
searchId: s.id,
title: "Guide",
url: "https://example.com/guide",
snippet: safeLine,
source: "google",
provider: "Google",
rank: 2,
},
db,
);

expect(content.snippet.includes(syntheticValue)).toBe(false);
expect(content.snippet).toBe("needleword service_password = [REDACTED]");
expect(JSON.stringify(content.metadata).includes(syntheticValue)).toBe(false);
expect(safe.snippet).toBe(safeLine);

db.prepare("UPDATE search_results SET snippet = ?, metadata = ? WHERE id = ?").run(
sensitiveLine,
JSON.stringify({ line: 2, matches: [{ line: 2, text: sensitiveLine }] }),
content.id,
);

const legacy = getResult(content.id, db)!;
expect(legacy.snippet.includes(syntheticValue)).toBe(false);
expect(legacy.snippet).toBe("needleword service_password = [REDACTED]");
expect(JSON.stringify(legacy.metadata).includes(syntheticValue)).toBe(false);
expect(listResults(s.id, {}, db).find((result) => result.id === content.id)).toEqual(legacy);
});
});

describe("saved searches CRUD", () => {
Expand Down
83 changes: 43 additions & 40 deletions src/db/results.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import type { Database } from "bun:sqlite";
import { getDb } from "./database.js";
import { type SearchResult, type SearchProviderName, generateId } from "../types/index.js";
import { redactContentSearchResult } from "../lib/redaction.js";

interface ResultRow {
id: string;
Expand All @@ -19,7 +20,7 @@ interface ResultRow {
}

function rowToResult(row: ResultRow): SearchResult {
return {
return redactContentSearchResult({
id: row.id,
searchId: row.search_id,
title: row.title,
Expand All @@ -33,7 +34,7 @@ function rowToResult(row: ResultRow): SearchResult {
thumbnail: row.thumbnail,
metadata: JSON.parse(row.metadata) as Record<string, unknown>,
createdAt: row.created_at,
};
});
}

export function createResult(
Expand All @@ -56,27 +57,7 @@ export function createResult(
const d = db ?? getDb();
const id = data.id ?? generateId();
const now = new Date().toISOString();

d.prepare(
`INSERT INTO search_results (id, search_id, title, url, snippet, source, provider, rank, score, published_at, thumbnail, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
).run(
id,
data.searchId,
data.title,
data.url,
data.snippet,
data.source,
data.provider,
data.rank,
data.score ?? null,
data.publishedAt ?? null,
data.thumbnail ?? null,
JSON.stringify(data.metadata ?? {}),
now,
);

return {
const result = redactContentSearchResult({
id,
searchId: data.searchId,
title: data.title,
Expand All @@ -90,7 +71,28 @@ export function createResult(
thumbnail: data.thumbnail ?? null,
metadata: data.metadata ?? {},
createdAt: now,
};
});

d.prepare(
`INSERT INTO search_results (id, search_id, title, url, snippet, source, provider, rank, score, published_at, thumbnail, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
).run(
result.id,
result.searchId,
result.title,
result.url,
result.snippet,
result.source,
result.provider,
result.rank,
result.score,
result.publishedAt,
result.thumbnail,
JSON.stringify(result.metadata),
result.createdAt,
);

return result;
}

export function createResults(
Expand Down Expand Up @@ -124,22 +126,7 @@ export function createResults(
try {
for (const data of results) {
const id = data.id ?? generateId();
stmt.run(
id,
data.searchId,
data.title,
data.url,
data.snippet,
data.source,
data.provider,
data.rank,
data.score ?? null,
data.publishedAt ?? null,
data.thumbnail ?? null,
JSON.stringify(data.metadata ?? {}),
now,
);
created.push({
const result = redactContentSearchResult({
id,
searchId: data.searchId,
title: data.title,
Expand All @@ -154,6 +141,22 @@ export function createResults(
metadata: data.metadata ?? {},
createdAt: now,
});
stmt.run(
result.id,
result.searchId,
result.title,
result.url,
result.snippet,
result.source,
result.provider,
result.rank,
result.score,
result.publishedAt,
result.thumbnail,
JSON.stringify(result.metadata),
result.createdAt,
);
created.push(result);
}
d.exec("COMMIT");
} catch (err) {
Expand Down
24 changes: 24 additions & 0 deletions src/lib/local/find.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,30 @@ describe("findLocal", () => {
expect(res.results[0]!.snippet).toContain("needleword");
});

test("redacts credential assignments without changing safe content or location metadata", () => {
const syntheticValue = "synthetic-only-not-live-7a31";
const sensitiveLine = `needleword service_password = ${syntheticValue}`;
const safeLine = "needleword documents password handling without an assigned value";
setup({
"config.txt": `heading\n${sensitiveLine}\nfooter`,
"guide.txt": safeLine,
});

const res = findLocal("needleword", { kind: "content", refresh: false }, db);
const sensitive = res.results.find((result) => result.path.endsWith("config.txt"))!;
const safe = res.results.find((result) => result.path.endsWith("guide.txt"))!;

expect(sensitive.path).toBe(join(root, "config.txt"));
expect(sensitive.line).toBe(2);
expect(sensitive.snippet?.includes(syntheticValue)).toBe(false);
expect(sensitive.snippet).toBe("needleword service_password = [REDACTED]");
expect(sensitive.matches).toEqual([
{ line: 2, text: "needleword service_password = [REDACTED]" },
]);
expect(safe.snippet).toBe(safeLine);
expect(safe.matches).toEqual([{ line: 1, text: safeLine }]);
});

test("respects limit", () => {
const files: Record<string, string> = {};
for (let i = 0; i < 30; i++) files[`dir/widget-${i}.ts`] = `widget ${i}`;
Expand Down
63 changes: 62 additions & 1 deletion src/lib/local/query.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ import { tmpdir } from "node:os";
import type { Database } from "bun:sqlite";
import { getIndexDbForTesting } from "../../db/index-db.js";
import { addRoot, indexRoot } from "./indexer.js";
import { searchFilePaths, searchFileContent, buildFtsQuery, tokenize } from "./query.js";
import {
searchFilePaths,
searchFileContent,
searchFileContentRegex,
buildFtsQuery,
tokenize,
} from "./query.js";

let root: string;
let db: Database;
Expand Down Expand Up @@ -349,4 +355,59 @@ describe("searchFileContent", () => {
const hits = searchFileContent("repeatedsymbol", {}, db);
expect(hits[0]!.matches.length).toBeLessThanOrEqual(5);
});

test("redacts emitted credential assignments in plain and regex searches while preserving safe prose", () => {
const syntheticValue = "synthetic-only-not-live-7a31";
const sensitiveLine = `needleword service_password = ${syntheticValue}`;
const safeLine = "needleword documents password handling without an assigned value";
setup({
"config.txt": `heading\n${sensitiveLine}\nfooter`,
"guide.txt": safeLine,
});

const plainHits = searchFileContent("needleword", {}, db);
const plainSensitive = plainHits.find((hit) => hit.relPath === "config.txt")!;
const plainSafe = plainHits.find((hit) => hit.relPath === "guide.txt")!;
expect(plainSensitive.line).toBe(2);
expect(plainSensitive.lineText.includes(syntheticValue)).toBe(false);
expect(plainSensitive.lineText).toBe("needleword service_password = [REDACTED]");
expect(plainSensitive.matches).toEqual([
{ line: 2, text: "needleword service_password = [REDACTED]" },
]);
expect(plainSafe.lineText).toBe(safeLine);
expect(plainSafe.matches).toEqual([{ line: 1, text: safeLine }]);

const regexHits = searchFileContentRegex("needleword.*service_password", {}, db);
expect(regexHits[0]!.line).toBe(2);
expect(regexHits[0]!.lineText.includes(syntheticValue)).toBe(false);
expect(regexHits[0]!.lineText).toBe("needleword service_password = [REDACTED]");
expect(regexHits[0]!.matches).toEqual([
{ line: 2, text: "needleword service_password = [REDACTED]" },
]);
});

test("redacts camelCase assignments without changing comparisons or type annotations", () => {
const syntheticValue = "synthetic-only-not-live-review";
const sensitiveLine = `needlecamel dbPassword = "${syntheticValue}"`;
const comparisonLine = 'needlecomparison if (password === "") return;';
const typeLine = "needletype password: string";
setup({
"camel.ts": sensitiveLine,
"comparison.ts": comparisonLine,
"type.ts": typeLine,
});

const plainHit = searchFileContent("needlecamel", {}, db)[0]!;
expect(plainHit.line).toBe(1);
expect(plainHit.lineText.includes(syntheticValue)).toBe(false);
expect(plainHit.lineText).toBe('needlecamel dbPassword = "[REDACTED]"');

const regexHit = searchFileContentRegex("needlecamel.*dbPassword", {}, db)[0]!;
expect(regexHit.line).toBe(1);
expect(regexHit.lineText.includes(syntheticValue)).toBe(false);
expect(regexHit.lineText).toBe('needlecamel dbPassword = "[REDACTED]"');

expect(searchFileContent("needlecomparison", {}, db)[0]!.lineText).toBe(comparisonLine);
expect(searchFileContent("needletype", {}, db)[0]!.lineText).toBe(typeLine);
});
});
12 changes: 10 additions & 2 deletions src/lib/local/query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { existsSync, readFileSync } from "node:fs";
import { getIndexDb } from "../../db/index-db.js";
import { getRoot } from "./indexer.js";
import { buildFtsQueryFromRegex, compileSearchRegex } from "./regex.js";
import { redactCredentialBearingText } from "../redaction.js";

export interface FileHit {
rootId: string;
Expand Down Expand Up @@ -57,6 +58,13 @@ const MAX_PATH_CANDIDATES = 20_000;
const MAX_CONTENT_CANDIDATES = 50_000;
const MAX_REGEX_CANDIDATES = 50_000;

function emittedLineMatch(line: number, rawText: string): LineMatch {
return {
line,
text: redactCredentialBearingText(rawText.trim()).slice(0, MAX_LINE_LENGTH),
};
}

export function tokenize(query: string): string[] {
// Control chars (NUL especially) would terminate FTS5's string parsing.
return query
Expand Down Expand Up @@ -304,7 +312,7 @@ function findLineMatches(
for (let i = 0; i < lines.length; i++) {
const text = lines[i]!;
const lower = text.toLowerCase();
const match: LineMatch = { line: i + 1, text: text.trim().slice(0, MAX_LINE_LENGTH) };
const match = emittedLineMatch(i + 1, text);

if (phrase.length > 0 && lower.includes(phrase)) phraseHits.push(match);
else if (lowered.every((t) => lower.includes(t))) allTokenHits.push(match);
Expand Down Expand Up @@ -424,7 +432,7 @@ export function searchFileContentRegex(
const matches: LineMatch[] = [];
for (let n = 0; n < lines.length && matches.length < MAX_MATCHES_PER_FILE; n++) {
if (regex.test(lines[n]!)) {
matches.push({ line: n + 1, text: lines[n]!.trim().slice(0, MAX_LINE_LENGTH) });
matches.push(emittedLineMatch(n + 1, lines[n]!));
}
}
if (matches.length === 0) continue;
Expand Down
Loading
Loading