diff --git a/src/db/crud.test.ts b/src/db/crud.test.ts index 3fd16e2..111b0d5 100644 --- a/src/db/crud.test.ts +++ b/src/db/crud.test.ts @@ -113,6 +113,55 @@ describe("results CRUD", () => { expect(ftsResults.length).toBe(1); expect(ftsResults[0]!.title).toBe("TypeScript Guide"); }); + + it("redacts persisted content-result output, including legacy rows, without changing safe results", () => { + const syntheticValue = "synthetic-only-not-live-7a31"; + const sensitiveLine = `needleword service_password = ${syntheticValue}`; + const safeLine = "needleword documents password handling without an assigned value"; + const s = createSearch({ query: "needleword", providers: ["content", "google"] }, db); + const content = createResult( + { + searchId: s.id, + title: "config.txt", + url: "file:///synthetic/config.txt", + snippet: sensitiveLine, + source: "content", + provider: "Local Content", + rank: 1, + metadata: { line: 2, matches: [{ line: 2, text: sensitiveLine }] }, + }, + db, + ); + const safe = createResult( + { + searchId: s.id, + title: "Guide", + url: "https://example.com/guide", + snippet: safeLine, + source: "google", + provider: "Google", + rank: 2, + }, + db, + ); + + expect(content.snippet.includes(syntheticValue)).toBe(false); + expect(content.snippet).toBe("needleword service_password = [REDACTED]"); + expect(JSON.stringify(content.metadata).includes(syntheticValue)).toBe(false); + expect(safe.snippet).toBe(safeLine); + + db.prepare("UPDATE search_results SET snippet = ?, metadata = ? WHERE id = ?").run( + sensitiveLine, + JSON.stringify({ line: 2, matches: [{ line: 2, text: sensitiveLine }] }), + content.id, + ); + + const legacy = getResult(content.id, db)!; + expect(legacy.snippet.includes(syntheticValue)).toBe(false); + expect(legacy.snippet).toBe("needleword service_password = [REDACTED]"); + expect(JSON.stringify(legacy.metadata).includes(syntheticValue)).toBe(false); + expect(listResults(s.id, {}, db).find((result) => result.id === content.id)).toEqual(legacy); + }); }); describe("saved searches CRUD", () => { diff --git a/src/db/results.ts b/src/db/results.ts index bd4518f..184f20f 100644 --- a/src/db/results.ts +++ b/src/db/results.ts @@ -1,6 +1,7 @@ import type { Database } from "bun:sqlite"; import { getDb } from "./database.js"; import { type SearchResult, type SearchProviderName, generateId } from "../types/index.js"; +import { redactContentSearchResult } from "../lib/redaction.js"; interface ResultRow { id: string; @@ -19,7 +20,7 @@ interface ResultRow { } function rowToResult(row: ResultRow): SearchResult { - return { + return redactContentSearchResult({ id: row.id, searchId: row.search_id, title: row.title, @@ -33,7 +34,7 @@ function rowToResult(row: ResultRow): SearchResult { thumbnail: row.thumbnail, metadata: JSON.parse(row.metadata) as Record, createdAt: row.created_at, - }; + }); } export function createResult( @@ -56,27 +57,7 @@ export function createResult( const d = db ?? getDb(); const id = data.id ?? generateId(); const now = new Date().toISOString(); - - d.prepare( - `INSERT INTO search_results (id, search_id, title, url, snippet, source, provider, rank, score, published_at, thumbnail, metadata, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - ).run( - id, - data.searchId, - data.title, - data.url, - data.snippet, - data.source, - data.provider, - data.rank, - data.score ?? null, - data.publishedAt ?? null, - data.thumbnail ?? null, - JSON.stringify(data.metadata ?? {}), - now, - ); - - return { + const result = redactContentSearchResult({ id, searchId: data.searchId, title: data.title, @@ -90,7 +71,28 @@ export function createResult( thumbnail: data.thumbnail ?? null, metadata: data.metadata ?? {}, createdAt: now, - }; + }); + + d.prepare( + `INSERT INTO search_results (id, search_id, title, url, snippet, source, provider, rank, score, published_at, thumbnail, metadata, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + result.id, + result.searchId, + result.title, + result.url, + result.snippet, + result.source, + result.provider, + result.rank, + result.score, + result.publishedAt, + result.thumbnail, + JSON.stringify(result.metadata), + result.createdAt, + ); + + return result; } export function createResults( @@ -124,22 +126,7 @@ export function createResults( try { for (const data of results) { const id = data.id ?? generateId(); - stmt.run( - id, - data.searchId, - data.title, - data.url, - data.snippet, - data.source, - data.provider, - data.rank, - data.score ?? null, - data.publishedAt ?? null, - data.thumbnail ?? null, - JSON.stringify(data.metadata ?? {}), - now, - ); - created.push({ + const result = redactContentSearchResult({ id, searchId: data.searchId, title: data.title, @@ -154,6 +141,22 @@ export function createResults( metadata: data.metadata ?? {}, createdAt: now, }); + stmt.run( + result.id, + result.searchId, + result.title, + result.url, + result.snippet, + result.source, + result.provider, + result.rank, + result.score, + result.publishedAt, + result.thumbnail, + JSON.stringify(result.metadata), + result.createdAt, + ); + created.push(result); } d.exec("COMMIT"); } catch (err) { diff --git a/src/lib/local/find.test.ts b/src/lib/local/find.test.ts index 29b62c4..ec95bc7 100644 --- a/src/lib/local/find.test.ts +++ b/src/lib/local/find.test.ts @@ -81,6 +81,30 @@ describe("findLocal", () => { expect(res.results[0]!.snippet).toContain("needleword"); }); + test("redacts credential assignments without changing safe content or location metadata", () => { + const syntheticValue = "synthetic-only-not-live-7a31"; + const sensitiveLine = `needleword service_password = ${syntheticValue}`; + const safeLine = "needleword documents password handling without an assigned value"; + setup({ + "config.txt": `heading\n${sensitiveLine}\nfooter`, + "guide.txt": safeLine, + }); + + const res = findLocal("needleword", { kind: "content", refresh: false }, db); + const sensitive = res.results.find((result) => result.path.endsWith("config.txt"))!; + const safe = res.results.find((result) => result.path.endsWith("guide.txt"))!; + + expect(sensitive.path).toBe(join(root, "config.txt")); + expect(sensitive.line).toBe(2); + expect(sensitive.snippet?.includes(syntheticValue)).toBe(false); + expect(sensitive.snippet).toBe("needleword service_password = [REDACTED]"); + expect(sensitive.matches).toEqual([ + { line: 2, text: "needleword service_password = [REDACTED]" }, + ]); + expect(safe.snippet).toBe(safeLine); + expect(safe.matches).toEqual([{ line: 1, text: safeLine }]); + }); + test("respects limit", () => { const files: Record = {}; for (let i = 0; i < 30; i++) files[`dir/widget-${i}.ts`] = `widget ${i}`; diff --git a/src/lib/local/query.test.ts b/src/lib/local/query.test.ts index 23eb367..4eef0ba 100644 --- a/src/lib/local/query.test.ts +++ b/src/lib/local/query.test.ts @@ -5,7 +5,13 @@ import { tmpdir } from "node:os"; import type { Database } from "bun:sqlite"; import { getIndexDbForTesting } from "../../db/index-db.js"; import { addRoot, indexRoot } from "./indexer.js"; -import { searchFilePaths, searchFileContent, buildFtsQuery, tokenize } from "./query.js"; +import { + searchFilePaths, + searchFileContent, + searchFileContentRegex, + buildFtsQuery, + tokenize, +} from "./query.js"; let root: string; let db: Database; @@ -349,4 +355,59 @@ describe("searchFileContent", () => { const hits = searchFileContent("repeatedsymbol", {}, db); expect(hits[0]!.matches.length).toBeLessThanOrEqual(5); }); + + test("redacts emitted credential assignments in plain and regex searches while preserving safe prose", () => { + const syntheticValue = "synthetic-only-not-live-7a31"; + const sensitiveLine = `needleword service_password = ${syntheticValue}`; + const safeLine = "needleword documents password handling without an assigned value"; + setup({ + "config.txt": `heading\n${sensitiveLine}\nfooter`, + "guide.txt": safeLine, + }); + + const plainHits = searchFileContent("needleword", {}, db); + const plainSensitive = plainHits.find((hit) => hit.relPath === "config.txt")!; + const plainSafe = plainHits.find((hit) => hit.relPath === "guide.txt")!; + expect(plainSensitive.line).toBe(2); + expect(plainSensitive.lineText.includes(syntheticValue)).toBe(false); + expect(plainSensitive.lineText).toBe("needleword service_password = [REDACTED]"); + expect(plainSensitive.matches).toEqual([ + { line: 2, text: "needleword service_password = [REDACTED]" }, + ]); + expect(plainSafe.lineText).toBe(safeLine); + expect(plainSafe.matches).toEqual([{ line: 1, text: safeLine }]); + + const regexHits = searchFileContentRegex("needleword.*service_password", {}, db); + expect(regexHits[0]!.line).toBe(2); + expect(regexHits[0]!.lineText.includes(syntheticValue)).toBe(false); + expect(regexHits[0]!.lineText).toBe("needleword service_password = [REDACTED]"); + expect(regexHits[0]!.matches).toEqual([ + { line: 2, text: "needleword service_password = [REDACTED]" }, + ]); + }); + + test("redacts camelCase assignments without changing comparisons or type annotations", () => { + const syntheticValue = "synthetic-only-not-live-review"; + const sensitiveLine = `needlecamel dbPassword = "${syntheticValue}"`; + const comparisonLine = 'needlecomparison if (password === "") return;'; + const typeLine = "needletype password: string"; + setup({ + "camel.ts": sensitiveLine, + "comparison.ts": comparisonLine, + "type.ts": typeLine, + }); + + const plainHit = searchFileContent("needlecamel", {}, db)[0]!; + expect(plainHit.line).toBe(1); + expect(plainHit.lineText.includes(syntheticValue)).toBe(false); + expect(plainHit.lineText).toBe('needlecamel dbPassword = "[REDACTED]"'); + + const regexHit = searchFileContentRegex("needlecamel.*dbPassword", {}, db)[0]!; + expect(regexHit.line).toBe(1); + expect(regexHit.lineText.includes(syntheticValue)).toBe(false); + expect(regexHit.lineText).toBe('needlecamel dbPassword = "[REDACTED]"'); + + expect(searchFileContent("needlecomparison", {}, db)[0]!.lineText).toBe(comparisonLine); + expect(searchFileContent("needletype", {}, db)[0]!.lineText).toBe(typeLine); + }); }); diff --git a/src/lib/local/query.ts b/src/lib/local/query.ts index a40945c..9506586 100644 --- a/src/lib/local/query.ts +++ b/src/lib/local/query.ts @@ -3,6 +3,7 @@ import { existsSync, readFileSync } from "node:fs"; import { getIndexDb } from "../../db/index-db.js"; import { getRoot } from "./indexer.js"; import { buildFtsQueryFromRegex, compileSearchRegex } from "./regex.js"; +import { redactCredentialBearingText } from "../redaction.js"; export interface FileHit { rootId: string; @@ -57,6 +58,13 @@ const MAX_PATH_CANDIDATES = 20_000; const MAX_CONTENT_CANDIDATES = 50_000; const MAX_REGEX_CANDIDATES = 50_000; +function emittedLineMatch(line: number, rawText: string): LineMatch { + return { + line, + text: redactCredentialBearingText(rawText.trim()).slice(0, MAX_LINE_LENGTH), + }; +} + export function tokenize(query: string): string[] { // Control chars (NUL especially) would terminate FTS5's string parsing. return query @@ -304,7 +312,7 @@ function findLineMatches( for (let i = 0; i < lines.length; i++) { const text = lines[i]!; const lower = text.toLowerCase(); - const match: LineMatch = { line: i + 1, text: text.trim().slice(0, MAX_LINE_LENGTH) }; + const match = emittedLineMatch(i + 1, text); if (phrase.length > 0 && lower.includes(phrase)) phraseHits.push(match); else if (lowered.every((t) => lower.includes(t))) allTokenHits.push(match); @@ -424,7 +432,7 @@ export function searchFileContentRegex( const matches: LineMatch[] = []; for (let n = 0; n < lines.length && matches.length < MAX_MATCHES_PER_FILE; n++) { if (regex.test(lines[n]!)) { - matches.push({ line: n + 1, text: lines[n]!.trim().slice(0, MAX_LINE_LENGTH) }); + matches.push(emittedLineMatch(n + 1, lines[n]!)); } } if (matches.length === 0) continue; diff --git a/src/lib/redaction.test.ts b/src/lib/redaction.test.ts new file mode 100644 index 0000000..7e9f196 --- /dev/null +++ b/src/lib/redaction.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, test } from "bun:test"; +import { + REDACTION_PLACEHOLDER, + redactCredentialBearingText, +} from "./redaction.js"; + +describe("redactCredentialBearingText", () => { + test("redacts sensitive assignments while preserving the key and quote shape", () => { + const value = "synthetic-only-not-live-7a31"; + expect(redactCredentialBearingText(`const service_password = "${value}";`)).toBe( + `const service_password = "${REDACTION_PLACEHOLDER}"`, + ); + }); + + test("leaves ordinary password-related prose unchanged", () => { + const safe = "This guide documents password handling without assigning a value."; + expect(redactCredentialBearingText(safe)).toBe(safe); + }); + + test("redacts camelCase credential assignments", () => { + const value = "synthetic-only-not-live-review"; + expect(redactCredentialBearingText(`dbPassword = "${value}"`)).toBe( + `dbPassword = "${REDACTION_PLACEHOLDER}"`, + ); + expect(redactCredentialBearingText(`clientSecret: ${value}`)).toBe( + `clientSecret: ${REDACTION_PLACEHOLDER}`, + ); + }); + + test("leaves comparisons and type annotations unchanged", () => { + const safeLines = [ + 'if (password === "") return;', + 'if (password !== "") return;', + 'const matches = password == "synthetic";', + "password: string", + ]; + + for (const safe of safeLines) { + expect(redactCredentialBearingText(safe)).toBe(safe); + } + }); + + test("redacts bearer values and URL user-info passwords", () => { + const value = "synthetic-only-not-live-7a31"; + const bearer = ["Bearer", value].join(" "); + const url = ["postgres://unit:", value, "@example.invalid/db"].join(""); + expect(redactCredentialBearingText(bearer)).toBe(`Bearer ${REDACTION_PLACEHOLDER}`); + expect(redactCredentialBearingText(url)).toBe( + `postgres://unit:${REDACTION_PLACEHOLDER}@example.invalid/db`, + ); + }); + + test("redacts common standalone credential token shapes", () => { + const values = [ + ["sk-", "synthetic_token_123456789"].join(""), + ["ghp_", "abcdefghijklmnopqrstuvwxyz123456"].join(""), + ["github_pat_", "abcdefghijklmnopqrstuvwxyz_123456"].join(""), + ["AKIA", "SYNTHETIC0000000"].join(""), + ["eyJheader", "eyJpayload", "signature"].join("."), + ]; + + for (const value of values) { + expect(redactCredentialBearingText(value)).toBe(REDACTION_PLACEHOLDER); + } + }); +}); diff --git a/src/lib/redaction.ts b/src/lib/redaction.ts new file mode 100644 index 0000000..3897437 --- /dev/null +++ b/src/lib/redaction.ts @@ -0,0 +1,100 @@ +import type { SearchResult } from "../types/index.js"; + +export const REDACTION_PLACEHOLDER = "[REDACTED]"; + +const SENSITIVE_KEY_SOURCE = + String.raw`\b[a-z0-9_-]*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|client[_-]?secret|(?:auth|access|refresh)[_-]?token|token|password|passwd|pwd|passphrase|private[_-]?key)\b`; + +const SENSITIVE_EQUALS_ASSIGNMENT_PATTERN = new RegExp( + `(${SENSITIVE_KEY_SOURCE}["'\\x60]?[\\s]*=[\\s]*)(?![=>])(["'\\x60]?).*$`, + "i", +); + +// Preserve ordinary TypeScript annotations while still protecting quoted JSON +// and unquoted YAML-style credential values. +const TYPE_ANNOTATION_VALUE_SOURCE = + String.raw`(?:(?:string|number|boolean|unknown|any|never|object|symbol|bigint|undefined|null)\b[\s|&]*)+[;,\])}]?\s*$`; +const SENSITIVE_COLON_ASSIGNMENT_PATTERN = new RegExp( + `(${SENSITIVE_KEY_SOURCE}["'\\x60]?[\\s]*:(?![\\s]*${TYPE_ANNOTATION_VALUE_SOURCE})[\\s]*)(["'\\x60]?).*$`, + "i", +); + +const SENSITIVE_ASSIGNMENT_PATTERNS = [ + SENSITIVE_EQUALS_ASSIGNMENT_PATTERN, + SENSITIVE_COLON_ASSIGNMENT_PATTERN, +] as const; + +const CREDENTIAL_URL_PATTERN = /([a-z][a-z0-9+.-]*:\/\/[^/\s:@]+:)([^@\s/]+)(@)/gi; +const BEARER_TOKEN_PATTERN = /(\bBearer\s+)[a-z0-9._~+/-]{8,}=*/gi; + +const INLINE_CREDENTIAL_PATTERNS: readonly RegExp[] = [ + /\bsk-[a-z0-9_-]{10,}\b/gi, + /\bgh[pousr]_[a-z0-9]{20,}\b/gi, + /\bgithub_pat_[a-z0-9_]{20,}\b/gi, + /\bAKIA[0-9A-Z]{16}\b/g, + /\beyJ[a-z0-9_-]+\.eyJ[a-z0-9_-]+\.[a-z0-9_-]+\b/gi, +]; + +/** + * Redact credential-bearing values from one emitted text field. + * + * Search matching still runs against the original source line. This function + * belongs at the output boundary so ranking and line coordinates remain exact. + */ +export function redactCredentialBearingText(text: string): string { + let redacted = text; + for (const pattern of SENSITIVE_ASSIGNMENT_PATTERNS) { + redacted = redacted.replace( + pattern, + (_match: string, prefix: string, quote: string) => + `${prefix}${quote}${REDACTION_PLACEHOLDER}${quote}`, + ); + } + + redacted = redacted.replace( + CREDENTIAL_URL_PATTERN, + (_match: string, prefix: string, _credential: string, suffix: string) => + `${prefix}${REDACTION_PLACEHOLDER}${suffix}`, + ); + redacted = redacted.replace( + BEARER_TOKEN_PATTERN, + (_match: string, prefix: string) => `${prefix}${REDACTION_PLACEHOLDER}`, + ); + + for (const pattern of INLINE_CREDENTIAL_PATTERNS) { + redacted = redacted.replace(pattern, REDACTION_PLACEHOLDER); + } + + return redacted; +} + +function redactContentMetadata(metadata: Record): Record { + const matches = metadata["matches"]; + if (!Array.isArray(matches)) return metadata; + + let changed = false; + const redactedMatches = matches.map((match) => { + if (typeof match !== "object" || match === null || Array.isArray(match)) return match; + const record = match as Record; + const text = record["text"]; + if (typeof text !== "string") return match; + + const redactedText = redactCredentialBearingText(text); + if (redactedText === text) return match; + changed = true; + return { ...record, text: redactedText }; + }); + + return changed ? { ...metadata, matches: redactedMatches } : metadata; +} + +/** Protect live, newly persisted, and historical local-content result output. */ +export function redactContentSearchResult(result: SearchResult): SearchResult { + if (result.source !== "content") return result; + + const snippet = redactCredentialBearingText(result.snippet); + const metadata = redactContentMetadata(result.metadata); + if (snippet === result.snippet && metadata === result.metadata) return result; + + return { ...result, snippet, metadata }; +}