From 0e05e89fd3be34a01afcc084c230fb28a2a4119f Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Tue, 11 Aug 2026 08:36:52 +1000 Subject: [PATCH 1/3] test: cover arbitrary Markdown entry content --- test/clipcase.test.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/test/clipcase.test.ts b/test/clipcase.test.ts index 08b8afd..cb04013 100644 --- a/test/clipcase.test.ts +++ b/test/clipcase.test.ts @@ -3,7 +3,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { test } from 'node:test'; -import { addEntry, createCase, exportCase, listCases, loadCase, searchCases } from '../src/index.js'; +import { addEntry, createCase, exportCase, listCases, loadCase, readEntryText, searchCases } from '../src/index.js'; import { findSecrets } from '../src/secrets.js'; import { loadConfig, writeConfig } from '../src/config.js'; async function tmp(): Promise { return fs.mkdtemp(path.join(os.tmpdir(), 'clipcase-test-')); } @@ -11,4 +11,24 @@ test('creates cases and captures deterministic entry metadata', async () => { co test('keeps identical same-second captures as distinct, ordered entries', async () => { const dir = await tmp(); await createCase(dir, 'collision'); const now = new Date('2026-01-01T00:00:01.100Z'); const first = await addEntry(dir, { caseName: 'collision', text: 'same content', now }); const second = await addEntry(dir, { caseName: 'collision', text: 'same content', now: new Date('2026-01-01T00:00:01.900Z') }); assert.equal(first.id, '20260101T000001Z-a636bd7cd420'); assert.equal(second.id, `${first.id}-000001`); assert.notEqual(first.path, second.path); const meta = await loadCase(dir, 'collision'); assert.deepEqual(meta.entries.map((entry) => entry.id), [first.id, second.id]); assert.deepEqual(meta.entries.map((entry) => entry.hash), [first.hash, first.hash]); assert.deepEqual(meta.entries.map((entry) => entry.bytes), [12, 12]); assert.equal((await searchCases(dir, 'same content')).length, 2); const exported = await exportCase(dir, 'collision'); assert.ok(exported.indexOf(`## ${first.id}\n`) < exported.indexOf(`## ${second.id}\n`)); }); test('blocks likely secrets unless explicitly allowed', async () => { const dir = await tmp(); await createCase(dir, 'secret-case'); await assert.rejects(() => addEntry(dir, { caseName: 'secret-case', text: 'token=abcdefghijklmnopqrstuvwxyz123456' }), /Refusing to save/); const entry = await addEntry(dir, { caseName: 'secret-case', text: 'token=abcdefghijklmnopqrstuvwxyz123456', allowSecret: true }); assert.ok(entry.id); assert.equal(findSecrets('AKIAABCDEFGHIJKLMNOP').length, 1); assert.equal(findSecrets('npm_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKL').length, 1); }); test('lists, searches, and exports case content', async () => { const dir = await tmp(); await createCase(dir, 'bug-login', 'Login Bug'); await addEntry(dir, { caseName: 'bug-login', text: 'expired cookie causes failure', source: 'terminal', tags: ['auth'] }); assert.equal((await listCases(dir)).length, 1); const results = await searchCases(dir, 'cookie'); assert.equal(results.length, 1); const exported = await exportCase(dir, 'bug-login'); assert.match(exported, /# Login Bug/); assert.match(exported, /expired cookie/); }); +test('round-trips arbitrary entry text and safely serializes metadata', async () => { + const dir = await tmp(); + await createCase(dir, 'hostile-markdown'); + const text = 'before\n```\nmiddle\n`````text\nafter\n'; + const source = 'terminal\nforged: field [link](https://example.test), "quoted"'; + const tags = ['comma, tag', 'brackets [x]', 'quote " and ` tick', 'line\nbreak']; + const entry = await addEntry(dir, { caseName: 'hostile-markdown', text, source, tags }); + const meta = await loadCase(dir, 'hostile-markdown'); + const stored = await fs.readFile(path.join(dir, 'hostile-markdown', entry.path), 'utf8'); + + assert.equal(await readEntryText(dir, meta, entry), text); + assert.equal(JSON.parse(stored.match(/^source: (.*)$/m)?.[1] ?? ''), source); + assert.deepEqual(JSON.parse(stored.match(/^tags: (.*)$/m)?.[1] ?? ''), [...tags].sort()); + assert.match(stored, /\n``````text\n/); + + const exported = await exportCase(dir, 'hostile-markdown'); + assert.match(exported, /- Source: `terminal\\nforged: field \[link\]\(https:\/\/example\.test\), \\"quoted\\"`/); + assert.match(exported, /- Tags: `brackets \[x\]` `comma, tag` `line\\nbreak` ``quote \\" and ` tick``/); + assert.match(exported, /\n``````text\nbefore\n```\nmiddle\n`````text\nafter\n``````\n/); +}); test('writes and loads local config', async () => { const dir = await tmp(); await writeConfig('notes', dir); const config = await loadConfig(dir); assert.equal(config.storageDir, path.join(dir, 'notes')); }); From 94afe883fe2fb0ce09e1223ef074be178ed1338e Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Tue, 11 Aug 2026 08:37:39 +1000 Subject: [PATCH 2/3] fix: serialize Markdown entries safely --- src/storage.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/storage.ts b/src/storage.ts index 86ea750..ccef28f 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { ClipcaseError } from './errors.js'; import { findSecrets } from './secrets.js'; import type { AddEntryInput, CaseMetadata, EntryMetadata } from './types.js'; -import { escapeMarkdown, formatTags, sha256, shortHash, slugify, toPosix } from './util.js'; +import { sha256, shortHash, slugify, toPosix } from './util.js'; const INDEX_FILE = 'index.json'; const LOCK_DIR = '.index.lock'; const LOCK_RETRY_MS = 25; @@ -18,7 +18,12 @@ export async function createCase(storageDir: string, name: string, title?: strin export async function listCases(storageDir: string): Promise { await ensureStore(storageDir); const names = await fs.readdir(storageDir).catch(() => [] as string[]); const cases: CaseMetadata[] = []; for (const name of names.sort()) { try { cases.push(await loadCase(storageDir, name)); } catch {} } return cases.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || a.name.localeCompare(b.name)); } export async function addEntry(storageDir: string, input: AddEntryInput): Promise { const findings = findSecrets(input.text); if (findings.length && !input.allowSecret) throw new ClipcaseError(`Refusing to save likely secret(s): ${findings.map((f) => f.label).join(', ')}. Re-run with --allow-secret if this is intentional.`, 3); return withCaseLock(storageDir, input.caseName, async () => { const meta = await loadCase(storageDir, input.caseName); const now = input.now ?? new Date(); const createdAt = now.toISOString(); const hash = sha256(input.text); const stamp = createdAt.replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); const baseId = `${stamp}-${shortHash(input.text)}`; let entry: EntryMetadata | undefined; for (let collision = 0; !entry; collision += 1) { const id = collision === 0 ? baseId : `${baseId}-${String(collision).padStart(6, '0')}`; const relPath = toPosix(path.join('entries', `${id}.md`)); const candidate: EntryMetadata = { id, caseName: meta.name, createdAt, source: input.source ?? 'stdin', tags: [...new Set(input.tags ?? [])].sort(), hash, bytes: Buffer.byteLength(input.text), path: relPath }; try { await fs.writeFile(path.join(caseDir(storageDir, meta.name), relPath), renderEntry(candidate, input.text), { flag: 'wx' }); entry = candidate; } catch (error) { if (!isAlreadyExists(error)) throw error; } } try { meta.entries.push(entry); meta.updatedAt = createdAt; await saveCase(storageDir, meta); return entry; } catch (error) { await fs.rm(path.join(caseDir(storageDir, meta.name), entry.path), { force: true }); throw error; } }); } function isAlreadyExists(error: unknown): boolean { return error instanceof Error && 'code' in error && error.code === 'EEXIST'; } -export function renderEntry(entry: EntryMetadata, text: string): string { return `---\nid: ${entry.id}\ncreatedAt: ${entry.createdAt}\nsource: ${entry.source}\ntags: [${entry.tags.join(', ')}]\nhash: ${entry.hash}\nbytes: ${entry.bytes}\n---\n\n# Entry ${entry.id}\n\n- Source: ${escapeMarkdown(entry.source)}\n- Tags: ${formatTags(entry.tags)}\n- SHA-256: \`${entry.hash}\`\n\n\`\`\`text\n${text.replace(/\n?$/, '\n')}\`\`\`\n`; } -export async function readEntryText(storageDir: string, meta: CaseMetadata, entry: EntryMetadata): Promise { const md = await fs.readFile(path.join(caseDir(storageDir, meta.name), entry.path), 'utf8'); const match = md.match(/```text\n([\s\S]*?)```\n?$/); return match ? match[1].replace(/\n$/, '') : md; } -export async function exportCase(storageDir: string, caseName: string): Promise { const meta = await loadCase(storageDir, caseName); const chunks = [`# ${meta.title}\n`, `- Case: ${meta.name}`, `- Created: ${meta.createdAt}`, `- Updated: ${meta.updatedAt}`, `- Entries: ${meta.entries.length}`, '']; for (const entry of meta.entries) { const text = await readEntryText(storageDir, meta, entry); chunks.push(`## ${entry.id}`, '', `- Source: ${entry.source}`, `- Tags: ${formatTags(entry.tags)}`, `- SHA-256: \`${entry.hash}\``, '', '```text', text, '```', ''); } return chunks.join('\n'); } +function fenceFor(text: string): string { const longest = Math.max(0, ...Array.from(text.matchAll(/`+/g), (match) => match[0].length)); return '`'.repeat(Math.max(3, longest + 1)); } +function serializedLabel(value: string): string { return JSON.stringify(value).slice(1, -1); } +function codeSpan(value: string): string { const serialized = serializedLabel(value); const longest = Math.max(0, ...Array.from(serialized.matchAll(/`+/g), (match) => match[0].length)); const delimiter = '`'.repeat(longest + 1); return `${delimiter}${serialized}${delimiter}`; } +function tagLabels(tags: string[]): string { return tags.length ? tags.map(codeSpan).join(' ') : '_none_'; } +function fencedText(text: string): string { const fence = fenceFor(text); return `${fence}text\n${text}${text.endsWith('\n') ? '' : '\n'}${fence}`; } +export function renderEntry(entry: EntryMetadata, text: string): string { return `---\nid: ${JSON.stringify(entry.id)}\ncreatedAt: ${JSON.stringify(entry.createdAt)}\nsource: ${JSON.stringify(entry.source)}\ntags: ${JSON.stringify(entry.tags)}\nhash: ${JSON.stringify(entry.hash)}\nbytes: ${entry.bytes}\n---\n\n# Entry ${entry.id}\n\n- Source: ${codeSpan(entry.source)}\n- Tags: ${tagLabels(entry.tags)}\n- SHA-256: \`${entry.hash}\`\n\n${fencedText(text)}\n`; } +export async function readEntryText(storageDir: string, meta: CaseMetadata, entry: EntryMetadata): Promise { const md = await fs.readFile(path.join(caseDir(storageDir, meta.name), entry.path)); const header = md.toString('utf8').match(/^(`{3,})text$/m); if (header) { const marker = Buffer.from(`${header[0]}\n`); const markerAt = md.indexOf(marker); if (markerAt >= 0) return md.subarray(markerAt + marker.length, markerAt + marker.length + entry.bytes).toString('utf8'); } const legacy = md.toString('utf8').match(/```text\n([\s\S]*?)```\n?$/); return legacy ? legacy[1].replace(/\n$/, '') : md.toString('utf8'); } +export async function exportCase(storageDir: string, caseName: string): Promise { const meta = await loadCase(storageDir, caseName); const chunks = [`# ${meta.title}\n`, `- Case: ${meta.name}`, `- Created: ${meta.createdAt}`, `- Updated: ${meta.updatedAt}`, `- Entries: ${meta.entries.length}`, '']; for (const entry of meta.entries) { const text = await readEntryText(storageDir, meta, entry); chunks.push(`## ${entry.id}`, '', `- Source: ${codeSpan(entry.source)}`, `- Tags: ${tagLabels(entry.tags)}`, `- SHA-256: \`${entry.hash}\``, '', fencedText(text), ''); } return chunks.join('\n'); } export async function searchCases(storageDir: string, query: string): Promise> { const needle = query.toLowerCase(); const results: Array<{ caseName: string; entry: EntryMetadata; preview: string }> = []; for (const meta of await listCases(storageDir)) for (const entry of meta.entries) { const text = await readEntryText(storageDir, meta, entry); const hay = `${entry.source} ${entry.tags.join(' ')} ${text}`.toLowerCase(); if (hay.includes(needle)) results.push({ caseName: meta.name, entry, preview: text.replace(/\s+/g, ' ').slice(0, 120) }); } return results.sort((a, b) => a.entry.createdAt.localeCompare(b.entry.createdAt)); } From 96c23c072ce5701550ba2469bb87a50fce758572 Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Tue, 11 Aug 2026 08:37:54 +1000 Subject: [PATCH 3/3] docs: describe robust entry serialization --- docs/STORAGE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/STORAGE.md b/docs/STORAGE.md index 64908da..2807ca6 100644 --- a/docs/STORAGE.md +++ b/docs/STORAGE.md @@ -7,6 +7,16 @@ ClipCase stores plain files so users can inspect, diff, back up, or delete casef - `index.json` contains case metadata and entry metadata. - `entries/*.md` contains front matter, hash metadata, and fenced plaintext. +Entry front matter uses JSON-compatible YAML scalars for string values and the +tag array. This keeps multiline labels and punctuation such as commas, brackets, +and quotes inside their original values. Human-readable source and tag labels are +likewise rendered as escaped Markdown code spans. + +The plaintext fence is at least three backticks and is always one backtick longer +than the longest run found in the captured text. Readers use the indexed byte +length to recover the exact capture, including whether it ended with a newline. +Older entries that use the original fixed triple-backtick fence remain readable. + Entry IDs are timestamp plus content hash prefix: `YYYYMMDDTHHMMSSZ-<12 hex>`. If that ID already exists, ClipCase appends a zero-padded collision counter, starting at `-000001`. This preserves both identical captures made within the same second