From 0fefad471cf1a4355d31ec326c147c047c983049 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 22 Feb 2026 20:46:08 +0000 Subject: [PATCH] feat(atxp): compress backup files into zip archive before upload The backup tool now uses jszip to create DEFLATE-compressed zip archives instead of sending raw JSON payloads. This reduces transfer size and bandwidth usage. Includes server migration instructions for the corresponding endpoint changes. https://claude.ai/code/session_01HZSR7WQPTnnuo8Pmi8UwAw --- package-lock.json | 119 +++++++++++++++--- packages/atxp/SERVER_MIGRATION_BACKUP_ZIP.md | 123 +++++++++++++++++++ packages/atxp/package.json | 1 + packages/atxp/src/commands/backup.ts | 43 +++++-- packages/atxp/src/commands/commands.test.ts | 65 ++++++++++ 5 files changed, 321 insertions(+), 30 deletions(-) create mode 100644 packages/atxp/SERVER_MIGRATION_BACKUP_ZIP.md diff --git a/package-lock.json b/package-lock.json index 3c80191..fb6aa0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4825,6 +4825,16 @@ "@types/node": "*" } }, + "node_modules/@types/jszip": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@types/jszip/-/jszip-3.4.0.tgz", + "integrity": "sha512-GFHqtQQP3R4NNuvZH3hNCYD0NbyBZ42bkN7kO3NDrU/SnvIZWMS8Bp38XCsRKBT5BXvgm0y1zqpZWp/ZkRzBzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "jszip": "*" + } + }, "node_modules/@types/node": { "version": "22.18.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.1.tgz", @@ -6506,6 +6516,12 @@ "url": "https://opencollective.com/core-js" } }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cors": { "version": "2.8.6", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", @@ -8293,6 +8309,12 @@ "node": ">=16.x" } }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -8689,6 +8711,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -9029,6 +9057,48 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -9083,6 +9153,15 @@ "node": ">= 0.8.0" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lighthouse-logger": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-1.4.2.tgz", @@ -10511,6 +10590,12 @@ "node": ">=6" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -10804,6 +10889,12 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -11889,6 +11980,12 @@ "node": ">= 0.8.0" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -13981,6 +14078,7 @@ "chalk": "^5.3.0", "fs-extra": "^11.2.0", "inquirer": "^9.2.12", + "jszip": "^3.10.1", "open": "^9.1.0", "ora": "^7.0.1", "qrcode-terminal": "^0.12.0" @@ -13991,6 +14089,7 @@ "devDependencies": { "@types/fs-extra": "^11.0.4", "@types/inquirer": "^9.0.7", + "@types/jszip": "^3.4.0", "@types/node": "^22.13.0", "@types/qrcode-terminal": "^0.12.2", "tsx": "^4.19.2", @@ -14035,26 +14134,6 @@ "globals": "^16.3.0", "vitest": "^1.0.0" } - }, - "packages/create-atxp/node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "packages/create-atxp/node_modules/qrcode-terminal": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/qrcode-terminal/-/qrcode-terminal-0.12.0.tgz", - "integrity": "sha512-EXtzRZmC+YGmGlDFbXKxQiMZNwCLEO6BANKXG4iCtSIM0yqc/pappSx3RIKr4r0uh5JsBckOXeKrB3Iz7mdQpQ==", - "bin": { - "qrcode-terminal": "bin/qrcode-terminal.js" - } } } } diff --git a/packages/atxp/SERVER_MIGRATION_BACKUP_ZIP.md b/packages/atxp/SERVER_MIGRATION_BACKUP_ZIP.md new file mode 100644 index 0000000..0c6c935 --- /dev/null +++ b/packages/atxp/SERVER_MIGRATION_BACKUP_ZIP.md @@ -0,0 +1,123 @@ +# Server Migration: Backup Endpoint Zip Compression + +The CLI backup tool now sends and expects zip archives instead of JSON payloads. The server endpoints need to be updated accordingly. + +## Summary of Changes + +The CLI `backup push` command now: +- Collects `.md` files and creates a **zip archive** (DEFLATE, level 9) using `jszip` +- Sends the zip as a binary body with `Content-Type: application/zip` + +The CLI `backup pull` command now: +- Sends `Accept: application/zip` header +- Expects a binary zip archive response (not JSON) +- Extracts files from the zip on the client side + +The `backup status` endpoint is **unchanged**. + +--- + +## Endpoint Changes Required + +### `PUT /backup/files` (push) + +**Before:** +- `Content-Type: application/json` +- Body: `{ "files": [{ "path": "SOUL.md", "content": "# Soul" }, ...] }` + +**After:** +- `Content-Type: application/zip` +- Body: raw zip archive binary + +**Server handling:** + +1. Read the request body as a binary buffer (not JSON) +2. Extract the zip archive (e.g., using `jszip`, `yauzl`, or `adm-zip` in Node; `zipfile` in Python) +3. Each entry in the zip is a `.md` file with its relative path preserved as the zip entry name +4. Store the extracted files the same way the old JSON files were stored +5. The response format is **unchanged**: `{ "fileCount": , "syncedAt": "" }` + +**Example server pseudocode (Node.js):** +```js +// Before: +// const { files } = JSON.parse(body); + +// After: +const JSZip = require('jszip'); +const zip = await JSZip.loadAsync(requestBodyBuffer); +const files = []; +for (const [name, entry] of Object.entries(zip.files)) { + if (entry.dir) continue; + const content = await entry.async('string'); + files.push({ path: name, content }); +} +// Then store `files` exactly as before +``` + +### `GET /backup/files` (pull) + +**Before:** +- Response `Content-Type: application/json` +- Body: `{ "files": [{ "path": "SOUL.md", "content": "# Soul" }, ...] }` + +**After:** +- Client sends `Accept: application/zip` header +- Response `Content-Type: application/zip` +- Body: raw zip archive binary containing all backed-up files + +**If no backup exists**, return an empty body (0 bytes) with a 200 status. The client handles this as "no backup found." + +**Example server pseudocode (Node.js):** +```js +// Before: +// res.json({ files }); + +// After: +const JSZip = require('jszip'); +const zip = new JSZip(); +for (const file of storedFiles) { + zip.file(file.path, file.content); +} +const zipBuffer = await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + compressionOptions: { level: 9 }, +}); +res.setHeader('Content-Type', 'application/zip'); +res.send(zipBuffer); +``` + +### `GET /backup/status` (status) + +**No changes required.** This endpoint continues to return JSON: +```json +{ "fileCount": 5, "syncedAt": "2026-02-22T...", "totalBytes": 12345 } +``` + +Note: `totalBytes` should reflect the **uncompressed** size of the stored files (sum of all file contents in bytes), not the zip archive size. This is what users see in `backup status` output. + +--- + +## Migration Strategy + +Since `push` replaces the entire server snapshot and there is no versioning, the migration is straightforward: + +1. **Deploy the server update first** with support for `application/zip` on both endpoints +2. Optionally support both `application/json` (legacy) and `application/zip` during a transition period by checking the `Content-Type` header on `PUT /backup/files`: + - If `application/zip` -> extract from zip + - If `application/json` -> parse JSON (legacy fallback) +3. For `GET /backup/files`, check the `Accept` header: + - If `Accept: application/zip` -> respond with zip + - Otherwise -> respond with JSON (legacy fallback) +4. Once CLI version with zip support is widely deployed, remove JSON fallback + +--- + +## Storage Consideration + +Server-side, you can choose to either: + +- **Store files individually** (extract on receive, re-zip on serve) -- same as current behavior but with zip/unzip at the boundary +- **Store the zip blob directly** -- simpler, saves CPU on push, just store and serve the raw zip. On status requests, you'd need to either cache metadata or extract to count files/compute sizes. + +The second option is recommended for simplicity since backups are small (markdown files only). diff --git a/packages/atxp/package.json b/packages/atxp/package.json index 6d1d2f8..9cb9659 100644 --- a/packages/atxp/package.json +++ b/packages/atxp/package.json @@ -38,6 +38,7 @@ "chalk": "^5.3.0", "fs-extra": "^11.2.0", "inquirer": "^9.2.12", + "jszip": "^3.10.1", "open": "^9.1.0", "ora": "^7.0.1", "qrcode-terminal": "^0.12.0" diff --git a/packages/atxp/src/commands/backup.ts b/packages/atxp/src/commands/backup.ts index 134a12d..b4e78e2 100644 --- a/packages/atxp/src/commands/backup.ts +++ b/packages/atxp/src/commands/backup.ts @@ -1,5 +1,6 @@ import chalk from 'chalk'; import fs from 'fs'; +import JSZip from 'jszip'; import path from 'path'; import { getConnection } from '../config.js'; @@ -37,6 +38,7 @@ function showBackupHelp(): void { console.log(); console.log(chalk.bold('Details:')); console.log(' Backs up all .md files (recursively) from the given directory.'); + console.log(' Files are compressed into a zip archive before upload.'); console.log(' Each push replaces the previous server snapshot entirely.'); console.log(' Pull writes server files to the local directory (non-destructive).'); console.log(); @@ -104,15 +106,23 @@ async function pushBackup(pathArg: string): Promise { } const totalBytes = files.reduce((sum, f) => sum + Buffer.byteLength(f.content, 'utf-8'), 0); - console.log(chalk.gray(`\nPushing ${files.length} file(s) (${formatBytes(totalBytes)})...`)); + console.log(chalk.gray(`\nCompressing ${files.length} file(s) (${formatBytes(totalBytes)})...`)); + + const zip = new JSZip(); + for (const file of files) { + zip.file(file.path, file.content); + } + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 9 } }); + + console.log(chalk.gray(`Pushing zip archive (${formatBytes(zipBuffer.length)})...`)); const res = await fetch(`${baseUrl}/backup/files`, { method: 'PUT', headers: { 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json', + 'Content-Type': 'application/zip', }, - body: JSON.stringify({ files: files.map(f => ({ path: f.path, content: f.content })) }), + body: new Uint8Array(zipBuffer), }); if (!res.ok) { @@ -144,6 +154,7 @@ async function pullBackup(pathArg: string): Promise { const res = await fetch(`${baseUrl}/backup/files`, { headers: { 'Authorization': `Bearer ${token}`, + 'Accept': 'application/zip', }, }); @@ -153,9 +164,20 @@ async function pullBackup(pathArg: string): Promise { process.exit(1); } - const data = await res.json() as { files: BackupFile[] }; + const zipBuffer = Buffer.from(await res.arrayBuffer()); + + if (zipBuffer.length === 0) { + console.log(chalk.yellow('No backup found on server. Push one first with:')); + console.log(chalk.cyan(' npx atxp backup push --path ')); + return; + } + + console.log(chalk.gray(`Extracting zip archive (${formatBytes(zipBuffer.length)})...`)); + + const zip = await JSZip.loadAsync(zipBuffer); + const fileNames = Object.keys(zip.files).filter(name => !zip.files[name].dir); - if (!data.files || data.files.length === 0) { + if (fileNames.length === 0) { console.log(chalk.yellow('No backup found on server. Push one first with:')); console.log(chalk.cyan(' npx atxp backup push --path ')); return; @@ -164,19 +186,20 @@ async function pullBackup(pathArg: string): Promise { // Create target directory if needed fs.mkdirSync(resolvedPath, { recursive: true }); - for (const file of data.files) { - const filePath = path.join(resolvedPath, file.path); + for (const name of fileNames) { + const content = await zip.files[name].async('string'); + const filePath = path.join(resolvedPath, name); const fileDir = path.dirname(filePath); fs.mkdirSync(fileDir, { recursive: true }); - fs.writeFileSync(filePath, file.content, 'utf-8'); + fs.writeFileSync(filePath, content, 'utf-8'); - console.log(chalk.gray(` ${file.path}`)); + console.log(chalk.gray(` ${name}`)); } console.log(); console.log(chalk.green.bold('Backup pulled successfully!')); - console.log(' ' + chalk.bold('Files written:') + ' ' + data.files.length); + console.log(' ' + chalk.bold('Files written:') + ' ' + fileNames.length); console.log(' ' + chalk.bold('Directory:') + ' ' + resolvedPath); } diff --git a/packages/atxp/src/commands/commands.test.ts b/packages/atxp/src/commands/commands.test.ts index e9d82ea..a742d5f 100644 --- a/packages/atxp/src/commands/commands.test.ts +++ b/packages/atxp/src/commands/commands.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import fs from 'fs'; +import JSZip from 'jszip'; import os from 'os'; import path from 'path'; import { collectMdFiles } from './backup.js'; @@ -246,6 +247,70 @@ describe('Tool Commands', () => { expect(`${baseUrl}/backup/files`).toBe('https://accounts.atxp.ai/backup/files'); expect(`${baseUrl}/backup/status`).toBe('https://accounts.atxp.ai/backup/status'); }); + + it('should create a zip archive from collected files', async () => { + fs.writeFileSync(path.join(tmpDir, 'SOUL.md'), '# Soul'); + const subDir = path.join(tmpDir, 'memory'); + fs.mkdirSync(subDir); + fs.writeFileSync(path.join(subDir, 'session.md'), '# Session notes'); + + const files = collectMdFiles(tmpDir); + + const zip = new JSZip(); + for (const file of files) { + zip.file(file.path, file.content); + } + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 9 } }); + + expect(zipBuffer.length).toBeGreaterThan(0); + expect(zipBuffer.length).toBeLessThan( + files.reduce((sum, f) => sum + Buffer.byteLength(f.content, 'utf-8'), 0) + 1024 + ); + }); + + it('should round-trip files through zip compression', async () => { + fs.writeFileSync(path.join(tmpDir, 'README.md'), '# README\nSome content here.'); + const subDir = path.join(tmpDir, 'memory'); + fs.mkdirSync(subDir); + fs.writeFileSync(path.join(subDir, 'log.md'), '## Log\n- Entry 1\n- Entry 2'); + + const files = collectMdFiles(tmpDir); + + // Create zip + const zip = new JSZip(); + for (const file of files) { + zip.file(file.path, file.content); + } + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 9 } }); + + // Extract zip + const extracted = await JSZip.loadAsync(zipBuffer); + const extractedNames = Object.keys(extracted.files).filter(n => !extracted.files[n].dir); + + expect(extractedNames.sort()).toEqual(files.map(f => f.path).sort()); + + for (const file of files) { + const content = await extracted.files[file.path].async('string'); + expect(content).toBe(file.content); + } + }); + + it('should produce a smaller zip than raw JSON payload', async () => { + // Create a file with repetitive content that compresses well + const repeatedContent = '# Memory\n\n' + 'This is a repeated line of memory content.\n'.repeat(100); + fs.writeFileSync(path.join(tmpDir, 'MEMORY.md'), repeatedContent); + + const files = collectMdFiles(tmpDir); + const jsonSize = Buffer.byteLength(JSON.stringify({ files }), 'utf-8'); + + const zip = new JSZip(); + for (const file of files) { + zip.file(file.path, file.content); + } + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 9 } }); + + expect(zipBuffer.length).toBeLessThan(jsonSize); + }); }); describe('common command behavior', () => {