From 702f3940cf06030719c410849ac76526b4fbd425 Mon Sep 17 00:00:00 2001 From: Harish Anantharaj Date: Tue, 1 Sep 2026 09:18:28 +0530 Subject: [PATCH] feat(cli): add an export command that dumps the kv layer as import-compatible JSON --- .changeset/cli-export-command.md | 9 +++ ARCHITECTURE.md | 2 +- README.md | 1 + docs/CLI.md | 68 ++++++++++++++++--- src/cli/readonly-fs.test.ts | 2 +- src/cli/run.test.ts | 111 ++++++++++++++++++++++++++++++- src/cli/run.ts | 87 ++++++++++++++++++++++-- 7 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 .changeset/cli-export-command.md diff --git a/.changeset/cli-export-command.md b/.changeset/cli-export-command.md new file mode 100644 index 0000000..461359a --- /dev/null +++ b/.changeset/cli-export-command.md @@ -0,0 +1,9 @@ +--- +"@libredb/libredb": minor +--- + +Add a CLI `export` command: `libredb export ` dumps a database's key-value layer as JSON in the exact shape `import` consumes, so `export` -> `import` round-trips. + +The dump is one flat JSON object of string values. It covers the key-value layer, which is the raw layer — `document` and `relational` data therefore appears as the internal prefixed entries those lenses store (a document `l1` in collection `logs` is the key `logs:l1` holding its JSON); there is no per-lens export. LibreDB's reserved `\x00`-prefixed catalog namespace is deliberately left out, because `import` refuses to write reserved keys: a restored file holds every row but no catalog entry, so a byte-exact copy is still a file copy, not a dump. A database holding raw non-UTF-8 bytes (only reachable by writing through the kernel API directly) is refused rather than dumped with replacement characters that would import back as different data. + +Like every other read command, `export` opens through the read-only filesystem adapter: it takes no lock, creates no `.lock`, and leaves the database byte-identical, so it can dump a file a live writer holds open. The only thing it writes is the destination JSON file, which is overwritten if it already exists. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c4f7d09..456254e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -797,7 +797,7 @@ by staying small and correct, not by absorbing every feature. | `lens/catalog.ts` | edge | reserved namespace, registry, validate-on-reopen | | `adapter/node-fs.ts` | edge | the real `node:fs` WAL adapter (fd reads, directory fsync, lock file) | | `adapter/opfs.ts` | edge | the browser OPFS WAL adapter | -| `cli/` | tooling | the libredb CLI (inspect, stats, get, scan, set, delete, import) and the read-only filesystem | +| `cli/` | tooling | the libredb CLI (inspect, stats, get, scan, export, set, delete, import) and the read-only filesystem | | `index.ts` | public | the Node npm export surface | | `browser.ts` | public | the browser export surface (no Node built-ins) | | `sim/` | test harness | simulated filesystem and crash-recovery oracle (DST) | diff --git a/README.md b/README.md index 5af7e29..ea7fb64 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ npx libredb inspect data.libredb # namespaces, kinds, and table schemas npx libredb stats data.libredb # file size and namespace counts npx libredb get data.libredb user:1 # print one value npx libredb scan data.libredb user: # print key=value under a prefix +npx libredb export data.libredb dump.json # dump every key as import-compatible JSON npx libredb set data.libredb user:1 Ada # set a key npx libredb delete data.libredb user:1 # remove a key npx libredb import data.libredb seed.json # bulk-set from a JSON object, atomically diff --git a/docs/CLI.md b/docs/CLI.md index 229f2bd..f29d573 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -36,6 +36,7 @@ Usage: libredb stats Summarize the file: size and namespace counts libredb get Print the value stored at a key libredb scan Print key=value for every key under a prefix + libredb export Dump every key to a JSON object (the shape import reads) libredb set Set a key to a value libredb delete Remove a key libredb import Bulk-set keys from a JSON object (one atomic commit) @@ -97,6 +98,50 @@ user:1=Ada user:2=Grace ``` +#### `export ` — JSON dump + +The counterpart of `import` below: it writes the **same JSON shape** `import` +reads — one object of string values — so a dump round-trips back into a database. + +```sh +$ libredb export app.libredb backup.json +export 4 keys + +$ cat backup.json +{ + "color": "teal", + "logs:l1": "{\"message\":\"hi\"}", + "user:1": "Ada", + "user:2": "Grace" +} + +$ libredb import restored.libredb backup.json # restore into a fresh file +import 4 keys +``` + +What a dump covers, exactly: + +- **The key-value layer** — the raw layer, so `document` and `relational` data + appears as the **internal prefixed entries** those lenses store (a document + `l1` in collection `logs` is the key `logs:l1` holding its JSON). There is no + per-lens export in v1: one dump, one flat object. +- **Not the reserved namespace.** LibreDB's `\x00`-prefixed catalog space is left + out, because `import` refuses to write reserved keys (see [Safety](#safety)). + A restored file therefore holds every row but no catalog entry, so `inspect` + lists nothing until a lens registers a namespace again. For a *byte-exact* + copy, copy the file — see [Backup and restore](#backup-and-restore). +- **UTF-8 text only.** Every lens and every CLI command writes well-formed + UTF-8. If a database holds raw non-UTF-8 bytes — only reachable by writing + through the kernel API directly — `export` refuses rather than emit + replacement characters that would import back as different data. + +Escaping is `JSON.stringify`'s (quotes, backslashes, control characters, +Unicode), so values are stored verbatim rather than terminal-escaped the way +`get`/`scan` print them; `--raw` does not apply. The output file is **overwritten** +if it exists, like a shell redirect, and its parent directory must already exist. +`import` **merges** into its target, so restore into a fresh path unless you mean +to overlay. + ### Write commands These take an advisory lock (see [Safety](#safety)) and commit through the WAL. @@ -139,9 +184,10 @@ The CLI touches real database files, so it is deliberately careful: - **Reads never mutate the file.** Opening a database runs crash recovery, which would normally truncate a torn tail — a write. Read commands - (`inspect`/`stats`/`get`/`scan`) open through a **read-only filesystem adapter**: - recovery drops a torn tail *in memory only*; the bytes on disk are left exactly - as found. + (`inspect`/`stats`/`get`/`scan`/`export`) open through a **read-only filesystem + adapter**: recovery drops a torn tail *in memory only*; the bytes on disk are + left exactly as found. That adapter also has no lock at all, so a read never + creates a `.lock` — `export` can dump a file a live writer holds open. - **A wrong path cannot destroy a file.** Opening a file that is not a LibreDB database (a typo, a text file) fails with a clear error and leaves the file byte-for-byte untouched — the on-disk `LRDB` header is checked before anything @@ -179,9 +225,13 @@ file copy — with one rule. - **Restore:** copy the file back and open it — recovery replays it like any reopen. Nothing else to do. -- **Export as text:** `libredb scan ""` is not supported (an empty prefix - is refused); scan per namespace prefix, or use the programmatic lenses for a - structured export. A first-class `export` command is on the roadmap. +- **Export as JSON:** `libredb export ` dumps the key-value + layer as an import-compatible object — see + [`export`](#export-path-filejson--json-dump) for exactly what it covers. + Restore it with `libredb import`. This is a *logical* dump and not a + replacement for the file copy above: the copy is byte-exact (it carries the + catalog and the log itself), while a dump carries only the keys `import` can + write back. --- @@ -203,9 +253,9 @@ libredb get app.libredb migration:done >/dev/null 2>&1 || libredb set app.libred ## Notes & limitations -- `get`/`scan`/`set`/`delete`/`import` operate on the **key-value layer** (UTF-8 - string keys and values). `inspect`/`stats` read the **catalog** for the richer - document/relational view. +- `get`/`scan`/`export`/`set`/`delete`/`import` operate on the **key-value layer** + (UTF-8 string keys and values). `inspect`/`stats` read the **catalog** for the + richer document/relational view. - There is no interactive `repl` (it was intentionally left out for now). - The CLI is one of three identical front-ends — see the [standalone binary](./BINARY.md) and [Docker image](./DOCKER.md) for the same diff --git a/src/cli/readonly-fs.test.ts b/src/cli/readonly-fs.test.ts index 3ce13f6..e3649fc 100644 --- a/src/cli/readonly-fs.test.ts +++ b/src/cli/readonly-fs.test.ts @@ -1,7 +1,7 @@ /** * readonly-fs.test.ts — the CLI's read-only filesystem adapter. * - * Inspection commands (inspect/get/scan/stats) must never mutate the file they + * Inspection commands (inspect/get/scan/stats/export) must never mutate the file they * read. That matters because open() runs recovery, which would truncate a torn * tail and so write to a "read-only" target. This adapter satisfies the kernel's * FileSystem seam for reads only: size and read work; append and fsync refuse; diff --git a/src/cli/run.test.ts b/src/cli/run.test.ts index b214613..498b4d2 100644 --- a/src/cli/run.test.ts +++ b/src/cli/run.test.ts @@ -4,9 +4,10 @@ * run(argv, io) is the whole CLI as a pure function: it takes an argument vector * and an IO sink and returns an exit code, so every command and error path is * testable without spawning a process. These cover the read commands (inspect, - * stats, get, scan) against real .libredb files, plus usage and error handling. + * stats, get, scan, export) against real .libredb files, plus usage and error + * handling. */ -import { appendFileSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; @@ -14,6 +15,7 @@ import { afterEach, expect, test } from "bun:test"; import { LOCK_SENTINEL } from "../adapter/node-fs.ts"; import { open } from "../index.ts"; +import { isReservedKey } from "../lens/catalog.ts"; import { doc } from "../lens/document.ts"; import { kv } from "../lens/kv.ts"; import { table } from "../lens/relational.ts"; @@ -160,6 +162,7 @@ test("reading never mutates the file (read-only open)", () => { cli("get", path, "user:1"); cli("scan", path, "user:"); cli("stats", path); + cli("export", path, `${path}.export.json`); expect(Bun.file(path).size).toBe(before); }); @@ -260,6 +263,110 @@ test("import rejects malformed JSON as a usage error (exit 2)", () => { expect(r.err.join("\n")).toMatch(/json/i); }); +/** Read a dump written by `export` back as the object `import` would consume. */ +const readDump = (file: string): Record => + JSON.parse(readFileSync(file, "utf8")) as Record; + +test("export dumps the kv layer in the shape import consumes, and the round trip restores it", () => { + const source = fixture(); + // Widen the shared fixture into a mixed-namespace, mixed-encoding one: raw kv + // pairs, a document row, a table row, and the serialization cases a dump has + // to survive. "__proto__" is here because `object[key] = value` on a plain + // object would silently drop it (the inherited setter defines no own property). + const db = open({ path: source }); + kv(db).set("quote", '"quoted" and a \\ backslash'); + kv(db).set("control", "line\nbreak\ttab\u001b[2J"); + kv(db).set("unicode", "üñíçödé \u{1f600} \u{10ffff}"); + kv(db).set("empty-value", ""); + kv(db).set("", "the empty key is a legal key"); + kv(db).set("__proto__", "not a prototype"); + table(db, "people", { primaryKey: "id", columns: { id: "string", name: "string" } }).insert({ + id: "p1", + name: "Ada", + }); + db.close(); + + const dump = `${source}.export.json`; + const exported = cli("export", source, dump); + expect(exported.code).toBe(0); + expect(exported.out).toEqual(["export 10 keys"]); + + const dumped = readDump(dump); + // Raw kv pairs, plus document and table rows as their internal prefixed + // entries (v1 exports the kv layer; there is no per-lens serializer). + expect(dumped["user:1"]).toBe("Ada"); + expect(dumped["logs:l1"]).toBe('{"message":"hi"}'); + expect(dumped["people:p1"]).toBe('{"id":"p1","name":"Ada"}'); + expect(dumped["quote"]).toBe('"quoted" and a \\ backslash'); + // Verbatim, NOT sanitized: get/scan would print that ESC as \x1b so an + // untrusted value cannot drive a terminal, but a dump is data that has to + // import back unchanged; JSON.stringify escapes it as \u001b in the file. + expect(dumped["control"]).toBe("line\nbreak\ttab\u001b[2J"); + expect(dumped["unicode"]).toBe("üñíçödé \u{1f600} \u{10ffff}"); + expect(dumped["empty-value"]).toBe(""); + expect(dumped[""]).toBe("the empty key is a legal key"); + expect(dumped["__proto__"]).toBe("not a prototype"); + expect(Object.keys(dumped)).toHaveLength(10); + expect(Object.values(dumped).every((value) => typeof value === "string")).toBe(true); + // The reserved catalog namespace is NOT dumped: import refuses those keys, so + // emitting them would produce a file import cannot read. + expect(Object.keys(dumped).filter(isReservedKey)).toEqual([]); + + // The round trip: dump -> a brand-new database -> dump again. Comparing the two + // dumps proves every exported key AND value survived, not just the exit codes. + const restored = `${source}.restored.libredb`; + const imported = cli("import", restored, dump); + expect(imported.code).toBe(0); + expect(imported.out).toEqual(["import 10 keys"]); + const roundTripped = `${source}.round-trip.json`; + expect(cli("export", restored, roundTripped).code).toBe(0); + expect(readDump(roundTripped)).toEqual(dumped); + // And the restored database really answers reads with the same values. + expect(cli("get", restored, "user:1").out).toEqual(["Ada"]); + expect(cli("get", restored, "people:p1").out).toEqual(['{"id":"p1","name":"Ada"}']); +}); + +test("export leaves the database byte-identical and creates no lock file", () => { + const path = fixture(); + const before = new Uint8Array(readFileSync(path)); + const r = cli("export", path, `${path}.export.json`); + expect(r.code).toBe(0); + // Byte-for-byte, not merely the same size: export opens through the read-only + // filesystem adapter, which has no lock() at all, so a read can neither write + // nor announce itself. + expect(new Uint8Array(readFileSync(path))).toEqual(before); + expect(existsSync(`${path}.lock`)).toBe(false); +}); + +test("export refuses a database holding raw non-UTF-8 bytes instead of dumping replacement characters", () => { + const dir = mkdtempSync(join(tmpdir(), "libredb-cli-")); + dirs.push(dir); + const path = join(dir, "raw.libredb"); + const db = open({ path }); + // Only reachable by writing through the kernel directly: 0x80 is a bare UTF-8 + // continuation byte. Decoded loosely it becomes U+FFFD, which would import + // back as a different key — so export refuses the whole dump instead. + db.transact((tx) => tx.set(new Uint8Array([0x80]), new TextEncoder().encode("v"))); + db.close(); + const r = cli("export", path, `${path}.export.json`); + expect(r.code).toBe(1); + expect(r.err.join("\n")).toMatch(/not valid UTF-8/i); +}); + +test("export overwrites an existing output file rather than appending to it", () => { + const path = fixture(); + const dump = `${path}.export.json`; + writeFileSync(dump, "stale bytes from an earlier dump"); + expect(cli("export", path, dump).code).toBe(0); + expect(readDump(dump)["user:1"]).toBe("Ada"); // it parses at all: the old bytes are gone +}); + +test("export with no file is a usage error", () => { + const r = cli("export", fixture()); + expect(r.code).toBe(2); + expect(r.err.join("\n")).toMatch(/file/i); +}); + /** A lock file naming a live holder: this very test process. */ const liveLock = (): string => `${LOCK_SENTINEL}\n${process.pid}\n${hostname()}\nnonce\n`; diff --git a/src/cli/run.ts b/src/cli/run.ts index 2f9ec2a..8020501 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -7,13 +7,14 @@ * bin shim (main.ts) is the only place that touches the real process. * * This is open-edge tooling over the public API, not kernel code: it adds no - * durability logic. Read commands (inspect/stats/get/scan) open through the - * read-only filesystem adapter so inspecting a file never mutates it. Write + * durability logic. Read commands (inspect/stats/get/scan/export) open through + * the read-only filesystem adapter so inspecting a file never mutates it. Write * commands (set/delete/import) rely on the kernel's exclusive open lock (a * second writer fails loudly; --force clears a lock whose holder is gone), and - * import commits all keys in one transaction so a bulk load is atomic. + * import commits all keys in one transaction so a bulk load is atomic — export + * is its inverse, reading the whole dump back out in one transaction. */ -import { readFileSync, statSync } from "node:fs"; +import { readFileSync, statSync, writeFileSync } from "node:fs"; import { parseArgs } from "node:util"; import { forceUnlock } from "../adapter/node-fs.ts"; @@ -67,6 +68,7 @@ const USAGE = [ " libredb stats Summarize the file: size and namespace counts", " libredb get Print the value stored at a key", " libredb scan Print key=value for every key under a prefix", + " libredb export Dump every key to a JSON object (the shape import reads)", " libredb set Set a key to a value", " libredb delete Remove a key", " libredb import Bulk-set keys from a JSON object (one atomic commit)", @@ -169,6 +171,82 @@ function scan({ path, args, io, raw }: Ctx): number { }); } +/** + * The byte range `export` scans: the whole keyspace a UTF-8 string can occupy. + * + * The kernel orders arbitrary byte keys with no maximum, so a half-open + * `[start, end)` cannot literally say "everything" — and it does not need to. A + * JSON dump can only carry keys that are UTF-8 TEXT, and no valid UTF-8 encoding + * begins with a byte above 0xF4 (the lead byte of U+10FFFF), so 0xF5 is above + * every key a lens or a CLI command can write. The start is the EMPTY key: it + * sorts before everything (including the reserved namespace, which is why + * reserved keys are filtered by predicate below rather than excluded by bound) + * and is itself a legal key. + */ +const EXPORT_START = new Uint8Array(); +const EXPORT_END = new Uint8Array([0xf5]); + +const strictUtf8 = new TextDecoder("utf-8", { fatal: true }); + +/** + * Decode one stored byte string for the dump, refusing bytes that are not valid + * UTF-8. Every lens and every CLI command writes well-formed UTF-8, so this can + * only fire for a key or value written as raw bytes straight through the kernel + * — and there a lossy decode would put U+FFFD in the dump, which imports back as + * DIFFERENT data (two distinct raw keys would collapse onto one JSON key). + * Export reads through the kernel directly, exactly as import writes through it, + * so it holds the same line import does with `assertWellFormedText`. + */ +const decodeText = (bytes: Uint8Array, what: string): string => { + try { + return strictUtf8.decode(bytes); + } catch { + throw new Error( + `libredb: a stored ${what} is not valid UTF-8 text (only reachable by writing raw bytes through the kernel ` + + `API); export refuses rather than emit replacement characters that would import back as different data`, + ); + } +}; + +function exportKeys({ path, args, io }: Ctx): number { + const [file] = args; + if (file === undefined) { + io.err("missing "); + return 2; + } + const pairs = withReadDb(path, (db) => + // ONE transaction for the whole dump, so the file is a single consistent + // snapshot — the read counterpart of import's one-transaction write. It + // reads the kernel range directly because the kv lens cannot express this + // scan: its range() takes STRING bounds, and no string encodes EXPORT_END. + db.transact((tx) => { + const rows: [string, string][] = []; + for (const entry of tx.getRange(EXPORT_START, EXPORT_END)) { + const key = decodeText(entry.key, "key"); + // Skip LibreDB's reserved namespace (the catalog): import refuses those + // keys, so dumping them would produce a file import cannot read. Testing + // the published isReservedKey predicate rather than a hardcoded prefix is + // what keeps export correct if the reserved namespace ever grows. + if (isReservedKey(key)) continue; + rows.push([key, decodeText(entry.value, "value")]); + } + return rows; + }), + ); + // Object.fromEntries, never `object[key] = value`: assigning "__proto__" on a + // plain object hits the inherited setter and defines NO own property, so that + // one key would silently vanish from the dump. fromEntries defines own + // properties, and JSON.parse does too — so the key survives the round trip. + // JSON.stringify does every bit of the escaping (quotes, backslashes, control + // characters, and it can never emit a lone surrogate); nothing here builds + // JSON text by hand. Indented with a trailing newline because a dump is a file + // humans read and diff. The write TRUNCATES an existing file, like a shell + // redirect; this is the only thing export writes. + writeFileSync(file, `${JSON.stringify(Object.fromEntries(pairs), null, 2)}\n`); + io.out(`export ${pairs.length} keys`); + return 0; +} + function set({ path, args, io, force }: Ctx): number { const [key, value] = args; if (key === undefined || value === undefined) { @@ -265,6 +343,7 @@ const commands = new Map number>([ ["stats", stats], ["get", get], ["scan", scan], + ["export", exportKeys], ["set", set], ["delete", remove], ["import", importKeys],