diff --git a/package.json b/package.json index e4b925d..97b36f2 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "smoke:codex-parser": "node scripts/codex-parser-smoke.js", "smoke:parser-reparse": "node scripts/parser-reparse-smoke.js", "smoke:chatgpt-import": "node scripts/chatgpt-import-smoke.js", + "smoke:claude-import": "node scripts/claude-import-smoke.js", "smoke:skills-permissions": "node scripts/skills-backup-permission-smoke.js", "smoke:referenced-attachments": "node scripts/referenced-attachments-smoke.js", "github:traffic": "node scripts/github-traffic-report.js", diff --git a/scripts/claude-import-smoke.js b/scripts/claude-import-smoke.js new file mode 100644 index 0000000..a9186d2 --- /dev/null +++ b/scripts/claude-import-smoke.js @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// Smoke test for the Claude export importer (src/claude-export.ts). +// +// Builds a small synthetic Claude export (an extracted folder — openExportReader +// accepts folders as well as zips), then exercises preflight and the full import +// pipeline with the store's persistence layer stubbed in-memory, and asserts that +// a second run deduplicates everything. +// +// Run: node scripts/claude-import-smoke.js (or: npm run smoke:claude-import) + +const fs = require('fs') +const os = require('os') +const path = require('path') + +require('ts-node').register({ + transpileOnly: true, + compilerOptions: { module: 'commonjs', moduleResolution: 'node' }, +}) + +function assert(cond, message) { + if (!cond) { + console.error(`FAIL: ${message}`) + process.exit(1) + } + console.log(`ok: ${message}`) +} + +function buildFixture(root) { + fs.mkdirSync(root, { recursive: true }) + fs.mkdirSync(path.join(root, 'design_chats'), { recursive: true }) + fs.mkdirSync(path.join(root, 'projects'), { recursive: true }) + + fs.writeFileSync(path.join(root, 'users.json'), JSON.stringify([ + { uuid: 'user-1', full_name: 'Tester', email_address: 'tester@example.com' }, + ])) + + const conversations = [ + { + uuid: 'conv-1', + name: 'First conversation', + summary: '', + created_at: '2025-01-01T10:00:00.000000Z', + updated_at: '2025-01-01T10:05:00.000000Z', + account: { uuid: 'user-1' }, + chat_messages: [ + { + uuid: 'm1', sender: 'human', created_at: '2025-01-01T10:00:00.000000Z', + text: 'Please analyze the attached file.', + content: [{ type: 'text', text: 'Please analyze the attached file.' }], + attachments: [{ file_name: 'notes.txt', file_size: 12, file_type: 'txt', extracted_content: 'hello world!' }], + files: [{ file_name: 'logo.png', file_uuid: 'f-1' }], + parent_message_uuid: '00000000-0000-4000-8000-000000000000', + }, + { + uuid: 'm2', sender: 'assistant', created_at: '2025-01-01T10:01:00.000000Z', + text: 'Here is my analysis.', + content: [ + { type: 'thinking', thinking: 'Let me reason about this carefully.' }, + { type: 'tool_use', name: 'web_search', input: { query: 'analysis' }, message: 'Searching' }, + { type: 'tool_result', name: 'web_search', content: [{ type: 'knowledge', title: 'Result', text: 'a fact', url: 'https://x' }] }, + { type: 'token_budget', remaining: null }, + { type: 'text', text: 'Here is my analysis.' }, + ], + attachments: [], files: [], parent_message_uuid: 'm1', + }, + ], + }, + { + // Empty conversation — should be skipped (no messages). + uuid: 'conv-empty', name: 'Empty', created_at: '2025-01-02T00:00:00Z', updated_at: '2025-01-02T00:00:00Z', + account: { uuid: 'user-1' }, chat_messages: [], + }, + ] + fs.writeFileSync(path.join(root, 'conversations.json'), JSON.stringify(conversations)) + + const designChat = { + uuid: 'design-1', + title: 'Design chat', + project: { uuid: 'p-1', name: 'Project' }, + created_at: '2025-03-01T09:00:00.000Z', + updated_at: '2025-03-01T09:10:00.000Z', + messages: [ + { + uuid: 'dm1', role: 'user', created_at: '2025-03-01T09:00:00.000Z', + content: { + role: 'user', content: 'What do you think of this mockup?', + attachments: [{ id: 'a1', name: 'mock.png', path: 'uploads/mock.png', type: 'image' }], + timestamp: '2025-03-01T09:00:00.000Z', + }, + }, + { + uuid: 'dm2', role: 'assistant', created_at: '2025-03-01T09:01:00.000Z', + content: { + role: 'assistant', content: 'It looks great.', + contentBlocks: [ + { type: 'thinking', text: 'Consider the layout.' }, + { type: 'tool_call', toolCall: { name: 'view_image', type: 'edit', input: { path: 'uploads/mock.png' } } }, + { type: 'text', text: 'It looks great.' }, + ], + timestamp: '2025-03-01T09:01:00.000Z', + }, + }, + ], + } + fs.writeFileSync(path.join(root, 'design_chats', 'design-1.json'), JSON.stringify(designChat)) + + // A project file (metadata only — not imported as a chat). + fs.writeFileSync(path.join(root, 'projects', 'p-1.json'), JSON.stringify({ uuid: 'p-1', name: 'Project', docs: [] })) +} + +async function main() { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'claude-import-smoke-')) + const fixture = path.join(tmp, 'export') + process.env.DATAMOAT_HOME = path.join(tmp, 'home') + buildFixture(fixture) + + // Stub the store's persistence so no encrypted vault is required. + const store = require('../src/store') + let sessionsStore = [] + const msgStore = new Map() + store.hasVaultSession = () => true + store.ensureDirs = () => {} + store.loadSessions = async () => sessionsStore + store.saveSessions = async s => { sessionsStore = s.slice() } + store.appendMessages = async (session, msgs) => { msgStore.set(session.uid, msgs) } + store.replaceSessionMessages = async (session, msgs) => { msgStore.set(session.uid, msgs) } + store.appendRawRecords = async () => {} + + const { preflightClaudeExport, runClaudeExportImport } = require('../src/claude-export') + + const pre = await preflightClaudeExport(fixture) + assert(pre.ok, 'preflight succeeds') + assert(pre.counts.conversations === 2, `preflight counts 2 conversations (got ${pre.counts.conversations})`) + assert(pre.counts.designChats === 1, `preflight counts 1 design chat (got ${pre.counts.designChats})`) + assert(pre.counts.threads === 2, `preflight counts 2 non-empty threads (got ${pre.counts.threads})`) + assert(pre.counts.thoughts === 2, `preflight counts 2 thinking blocks (got ${pre.counts.thoughts})`) + assert(pre.counts.toolUses === 2, `preflight counts 2 tool uses (got ${pre.counts.toolUses})`) + assert(Object.keys(pre.counts.unknownBlockTypes).length === 0, 'no unknown block types') + + const job1 = await runClaudeExportImport(fixture) + assert(job1.phase === 'completed', `import completes (phase=${job1.phase})`) + assert(job1.imported.sessions === 2, `imports 2 sessions (got ${job1.imported.sessions})`) + assert(sessionsStore.length === 2, `2 sessions in store (got ${sessionsStore.length})`) + + const conv = sessionsStore.find(s => s.id === 'conv-1') + assert(conv && conv.modelProvider === 'anthropic', 'conversation session has anthropic provider') + assert(conv && conv.source === 'claude-export', 'session source is claude-export') + const convMsgs = msgStore.get(conv.uid) + assert(convMsgs[0].role === 'user', 'human maps to user role') + assert(convMsgs[1].role === 'assistant', 'assistant role preserved') + assert(convMsgs[1].hasThinking === true, 'thinking detected on assistant message') + const types = convMsgs[1].content.map(b => b.type) + assert(types.includes('thinking') && types.includes('tool_use') && types.includes('tool_result') && types.includes('text'), + `assistant blocks mapped (${types.join(',')})`) + assert(!types.includes('token_budget'), 'token_budget dropped') + assert(convMsgs[0].content.some(b => b.type === 'file'), 'attachment mapped to file block') + + const design = sessionsStore.find(s => s.sourceClient === 'Claude design chat') + assert(design, 'design chat imported as a session') + const designMsgs = msgStore.get(design.uid) + assert(designMsgs.length === 2 && designMsgs[1].hasThinking, 'design chat assistant thinking mapped') + + // Second run — everything deduplicates. + const job2 = await runClaudeExportImport(fixture) + assert(job2.imported.sessions === 0, `re-run imports nothing (got ${job2.imported.sessions})`) + assert(job2.skipped.duplicates === 2, `re-run skips 2 duplicates (got ${job2.skipped.duplicates})`) + assert(sessionsStore.length === 2, 'session count stable after re-run') + + fs.rmSync(tmp, { recursive: true, force: true }) + console.log('\nAll Claude import smoke checks passed.') +} + +main().catch(err => { console.error(err); process.exit(1) }) diff --git a/src/chatgpt-export.ts b/src/chatgpt-export.ts index b51873f..228f138 100644 --- a/src/chatgpt-export.ts +++ b/src/chatgpt-export.ts @@ -1,9 +1,22 @@ import * as crypto from 'crypto' import * as fs from 'fs' import * as path from 'path' -import * as zlib from 'zlib' -import { Readable } from 'stream' import { STATE_DIR } from './config' +import { + ExportArchiveReader as ChatGptExportReader, + ExportEntry, + nowIso, + openExportReader, + positiveByteLimit, + readJsonFile, + readJsonFromExport, + readStreamSample, + safeError, + sha256Hex, + sniffMediaType, + toPosixPath, + writePrivateJson, +} from './export-archive' import { appendMessages, appendRawRecords, @@ -32,9 +45,6 @@ const CHATGPT_IMPORT_JOB_VERSION = 1 const CHATGPT_PARSER_VERSION = 2 const CHATGPT_IMPORTS_FILE = path.join(STATE_DIR, 'chatgpt-export-imports.json') const CHATGPT_IMPORT_JOB_FILE = path.join(STATE_DIR, 'chatgpt-export-import-job.json') -const MAX_ZIP_ENTRY_COUNT = 100_000 -const MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES = 12 * 1024 * 1024 * 1024 -const MAX_ZIP_SINGLE_ENTRY_BYTES = MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES const LARGE_ASSET_STREAM_THRESHOLD_BYTES = positiveByteLimit( process.env.DATAMOAT_CHATGPT_LARGE_ASSET_STREAM_THRESHOLD_BYTES, 16 * 1024 * 1024, @@ -42,35 +52,6 @@ const LARGE_ASSET_STREAM_THRESHOLD_BYTES = positiveByteLimit( const STRONG_DUPLICATE_MIN_TEXT_CHARS = 200 const STRONG_DUPLICATE_MIN_MESSAGES = 3 -function positiveByteLimit(value: string | undefined, fallback: number): number { - const parsed = Number(value) - return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback -} - -type ZipEntry = { - path: string - compressedSize: number - uncompressedSize: number - method: number - flags: number - localHeaderOffset: number -} - -type ExportEntry = { - path: string - size: number -} - -type ChatGptExportReader = { - kind: 'folder' | 'zip' - rootPath: string - listEntries(): ExportEntry[] - has(relativePath: string): boolean - readBuffer(relativePath: string): Buffer - createReadStream(relativePath: string): NodeJS.ReadableStream - close?(): void -} - type ChatGptManifest = { version?: number manifest_file?: string @@ -241,42 +222,6 @@ type ChatGptImportsState = { attachmentIds: Record } -function nowIso(): string { - return new Date().toISOString() -} - -function safeError(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} - -function sha256Hex(value: string | Buffer): string { - return crypto.createHash('sha256').update(value).digest('hex') -} - -function writePrivateJson(filePath: string, value: unknown): void { - fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 }) - const tmpPath = `${filePath}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp` - fs.writeFileSync(tmpPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 }) - const fd = fs.openSync(tmpPath, 'r') - try { - fs.fsyncSync(fd) - } catch { - /* non-fatal */ - } finally { - fs.closeSync(fd) - } - fs.renameSync(tmpPath, filePath) - try { fs.chmodSync(filePath, 0o600) } catch { /* non-fatal */ } -} - -function readJsonFile(filePath: string): T | null { - try { - return JSON.parse(fs.readFileSync(filePath, 'utf8')) as T - } catch { - return null - } -} - function readJob(): ChatGptImportJob | null { return readJsonFile(CHATGPT_IMPORT_JOB_FILE) } @@ -323,291 +268,6 @@ function writeImportsState(state: ChatGptImportsState): void { }) } -function toPosixPath(value: string): string { - return value.replace(/\\/g, '/') -} - -function safeZipPath(name: string): string { - if (name.includes('\0')) throw new Error('zip entry path contains NUL byte') - const normalized = toPosixPath(name) - if (!normalized || normalized.startsWith('/') || normalized.startsWith('~')) { - throw new Error(`unsafe zip entry path: ${name}`) - } - if (/^[A-Za-z]:\//.test(normalized)) throw new Error(`unsafe zip entry path: ${name}`) - const parts = normalized.split('/').filter(Boolean) - if (parts.some(part => part === '..')) throw new Error(`unsafe zip entry path: ${name}`) - return parts.join('/') -} - -function listFolderEntries(root: string): ExportEntry[] { - const entries: ExportEntry[] = [] - const walk = (dirPath: string): void => { - for (const entry of fs.readdirSync(dirPath, { withFileTypes: true })) { - if (entry.name === '.DS_Store' || entry.name.startsWith('._')) continue - const absolute = path.join(dirPath, entry.name) - if (entry.isDirectory()) { - walk(absolute) - continue - } - if (!entry.isFile()) continue - const relative = toPosixPath(path.relative(root, absolute)) - entries.push({ path: relative, size: fs.statSync(absolute).size }) - } - } - walk(root) - return entries.sort((a, b) => a.path.localeCompare(b.path)) -} - -function folderReader(rootPath: string): ChatGptExportReader { - const resolved = path.resolve(rootPath) - const entries = listFolderEntries(resolved) - const entrySet = new Set(entries.map(entry => entry.path)) - const resolveEntryPath = (relativePath: string): string => { - const safe = safeZipPath(relativePath) - if (!entrySet.has(safe)) throw new Error(`export file not found: ${relativePath}`) - const absolute = path.join(resolved, ...safe.split('/')) - const normalized = path.resolve(absolute) - if (!normalized.startsWith(`${resolved}${path.sep}`) && normalized !== resolved) { - throw new Error(`unsafe export file path: ${relativePath}`) - } - return normalized - } - return { - kind: 'folder', - rootPath: resolved, - listEntries: () => entries, - has: relativePath => entrySet.has(toPosixPath(relativePath)), - readBuffer(relativePath: string): Buffer { - return fs.readFileSync(resolveEntryPath(relativePath)) - }, - createReadStream(relativePath: string): NodeJS.ReadableStream { - return fs.createReadStream(resolveEntryPath(relativePath)) - }, - } -} - -function readAt(fd: number, offset: number, length: number): Buffer { - const buffer = Buffer.alloc(length) - const bytesRead = fs.readSync(fd, buffer, 0, length, offset) - if (bytesRead !== length) throw new Error('unexpected end of zip file') - return buffer -} - -function uint64ToSafeNumber(value: bigint, label: string): number { - if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`zip64 ${label} is too large`) - return Number(value) -} - -function zip64ExtraValues( - extra: Buffer, - fields: { uncompressed: number; compressed: number; localHeaderOffset: number }, -): { uncompressedSize: number; compressedSize: number; localHeaderOffset: number } { - let uncompressedSize = fields.uncompressed - let compressedSize = fields.compressed - let localHeaderOffset = fields.localHeaderOffset - let cursor = 0 - while (cursor + 4 <= extra.length) { - const headerId = extra.readUInt16LE(cursor) - const dataSize = extra.readUInt16LE(cursor + 2) - const dataStart = cursor + 4 - const dataEnd = dataStart + dataSize - if (dataEnd > extra.length) break - if (headerId === 0x0001) { - let valueCursor = dataStart - const readZip64Value = (label: string): number => { - if (valueCursor + 8 > dataEnd) throw new Error(`zip64 extra field is truncated: ${label}`) - const value = uint64ToSafeNumber(extra.readBigUInt64LE(valueCursor), label) - valueCursor += 8 - return value - } - if (fields.uncompressed === 0xffffffff) uncompressedSize = readZip64Value('uncompressed size') - if (fields.compressed === 0xffffffff) compressedSize = readZip64Value('compressed size') - if (fields.localHeaderOffset === 0xffffffff) localHeaderOffset = readZip64Value('local header offset') - return { uncompressedSize, compressedSize, localHeaderOffset } - } - cursor = dataEnd - } - if ( - fields.uncompressed === 0xffffffff - || fields.compressed === 0xffffffff - || fields.localHeaderOffset === 0xffffffff - ) { - throw new Error('zip64 extra field is missing') - } - return { uncompressedSize, compressedSize, localHeaderOffset } -} - -function zipCentralDirectoryLocation( - fd: number, - stat: fs.Stats, - eocdOffset: number, - eocd: Buffer, -): { entriesTotal: number; centralSize: number; centralOffset: number } { - const diskNumber = eocd.readUInt16LE(4) - const centralDisk = eocd.readUInt16LE(6) - const entriesThisDisk = eocd.readUInt16LE(8) - const entriesTotal32 = eocd.readUInt16LE(10) - const centralSize32 = eocd.readUInt32LE(12) - const centralOffset32 = eocd.readUInt32LE(16) - if (diskNumber !== 0 || centralDisk !== 0 || entriesThisDisk !== entriesTotal32) { - throw new Error('multi-disk zip files are not supported') - } - - const needsZip64 = entriesTotal32 === 0xffff || centralSize32 === 0xffffffff || centralOffset32 === 0xffffffff - if (!needsZip64) { - return { - entriesTotal: entriesTotal32, - centralSize: centralSize32, - centralOffset: centralOffset32, - } - } - - if (eocdOffset < 20) throw new Error('zip64 locator is missing') - const locator = readAt(fd, eocdOffset - 20, 20) - if (locator.readUInt32LE(0) !== 0x07064b50) throw new Error('zip64 locator is missing') - const locatorDisk = locator.readUInt32LE(4) - const zip64EocdOffset = uint64ToSafeNumber(locator.readBigUInt64LE(8), 'end-of-central-directory offset') - const totalDisks = locator.readUInt32LE(16) - if (locatorDisk !== 0 || totalDisks !== 1) throw new Error('multi-disk zip files are not supported') - if (zip64EocdOffset < 0 || zip64EocdOffset + 56 > stat.size) throw new Error('invalid zip64 end-of-central-directory offset') - - const zip64Header = readAt(fd, zip64EocdOffset, 56) - if (zip64Header.readUInt32LE(0) !== 0x06064b50) throw new Error('zip64 end-of-central-directory record not found') - const zip64Disk = zip64Header.readUInt32LE(16) - const zip64CentralDisk = zip64Header.readUInt32LE(20) - const zip64EntriesThisDisk = uint64ToSafeNumber(zip64Header.readBigUInt64LE(24), 'entry count') - const zip64EntriesTotal = uint64ToSafeNumber(zip64Header.readBigUInt64LE(32), 'entry count') - if (zip64Disk !== 0 || zip64CentralDisk !== 0 || zip64EntriesThisDisk !== zip64EntriesTotal) { - throw new Error('multi-disk zip files are not supported') - } - return { - entriesTotal: zip64EntriesTotal, - centralSize: uint64ToSafeNumber(zip64Header.readBigUInt64LE(40), 'central directory size'), - centralOffset: uint64ToSafeNumber(zip64Header.readBigUInt64LE(48), 'central directory offset'), - } -} - -function parseZipEntries(zipPath: string, fd: number): Map { - const stat = fs.fstatSync(fd) - const tailLength = Math.min(stat.size, 66_000) - const tail = readAt(fd, stat.size - tailLength, tailLength) - let eocdOffset = -1 - for (let i = tail.length - 22; i >= 0; i -= 1) { - if (tail.readUInt32LE(i) === 0x06054b50) { - eocdOffset = stat.size - tailLength + i - break - } - } - if (eocdOffset < 0) throw new Error('zip end-of-central-directory record not found') - const eocd = readAt(fd, eocdOffset, 22) - const { entriesTotal, centralSize, centralOffset } = zipCentralDirectoryLocation(fd, stat, eocdOffset, eocd) - if (entriesTotal > MAX_ZIP_ENTRY_COUNT) throw new Error('zip has too many files') - const central = readAt(fd, centralOffset, centralSize) - const entries = new Map() - let cursor = 0 - let totalUncompressed = 0 - for (let index = 0; index < entriesTotal; index += 1) { - if (central.readUInt32LE(cursor) !== 0x02014b50) throw new Error(`invalid zip central directory in ${path.basename(zipPath)}`) - const flags = central.readUInt16LE(cursor + 8) - const method = central.readUInt16LE(cursor + 10) - const compressedSize32 = central.readUInt32LE(cursor + 20) - const uncompressedSize32 = central.readUInt32LE(cursor + 24) - const nameLength = central.readUInt16LE(cursor + 28) - const extraLength = central.readUInt16LE(cursor + 30) - const commentLength = central.readUInt16LE(cursor + 32) - const localHeaderOffset32 = central.readUInt32LE(cursor + 42) - const name = central.subarray(cursor + 46, cursor + 46 + nameLength).toString('utf8') - const extra = central.subarray(cursor + 46 + nameLength, cursor + 46 + nameLength + extraLength) - cursor += 46 + nameLength + extraLength + commentLength - if (name.endsWith('/')) continue - const { compressedSize, uncompressedSize, localHeaderOffset } = zip64ExtraValues(extra, { - compressed: compressedSize32, - uncompressed: uncompressedSize32, - localHeaderOffset: localHeaderOffset32, - }) - if (flags & 0x1) throw new Error(`encrypted zip entry is not supported: ${name}`) - if (method !== 0 && method !== 8) throw new Error(`unsupported zip compression method ${method}: ${name}`) - if (uncompressedSize > MAX_ZIP_SINGLE_ENTRY_BYTES) throw new Error(`zip entry is too large: ${name}`) - totalUncompressed += uncompressedSize - if (totalUncompressed > MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES) throw new Error('zip uncompressed size is too large') - const safe = safeZipPath(name) - entries.set(safe, { path: safe, compressedSize, uncompressedSize, method, flags, localHeaderOffset }) - } - return entries -} - -function zipEntryDataOffset(fd: number, entry: ZipEntry, relativePath: string): number { - const local = readAt(fd, entry.localHeaderOffset, 30) - if (local.readUInt32LE(0) !== 0x04034b50) throw new Error(`invalid zip local header: ${relativePath}`) - const nameLength = local.readUInt16LE(26) - const extraLength = local.readUInt16LE(28) - return entry.localHeaderOffset + 30 + nameLength + extraLength -} - -function zipReader(zipPath: string): ChatGptExportReader { - const resolved = path.resolve(zipPath) - const fd = fs.openSync(resolved, 'r') - let closed = false - const entries = parseZipEntries(resolved, fd) - const list = Array.from(entries.values()).map(entry => ({ path: entry.path, size: entry.uncompressedSize })) - .sort((a, b) => a.path.localeCompare(b.path)) - return { - kind: 'zip', - rootPath: resolved, - listEntries: () => list, - has: relativePath => entries.has(toPosixPath(relativePath)), - readBuffer(relativePath: string): Buffer { - const safe = safeZipPath(relativePath) - const entry = entries.get(safe) - if (!entry) throw new Error(`export file not found: ${relativePath}`) - const dataOffset = zipEntryDataOffset(fd, entry, relativePath) - const compressed = readAt(fd, dataOffset, entry.compressedSize) - const data = entry.method === 0 ? compressed : zlib.inflateRawSync(compressed) - if (data.length !== entry.uncompressedSize) throw new Error(`zip entry size mismatch: ${relativePath}`) - return data - }, - createReadStream(relativePath: string): NodeJS.ReadableStream { - const safe = safeZipPath(relativePath) - const entry = entries.get(safe) - if (!entry) throw new Error(`export file not found: ${relativePath}`) - if (entry.compressedSize === 0) return Readable.from([]) - const dataOffset = zipEntryDataOffset(fd, entry, relativePath) - const compressed = fs.createReadStream(resolved, { - start: dataOffset, - end: dataOffset + entry.compressedSize - 1, - }) - return entry.method === 0 ? compressed : compressed.pipe(zlib.createInflateRaw()) - }, - close() { - if (closed) return - closed = true - fs.closeSync(fd) - }, - } -} - -function openExportReader(sourcePath: string): ChatGptExportReader { - const resolved = path.resolve(sourcePath) - const stat = fs.statSync(resolved) - if (stat.isDirectory()) return folderReader(resolved) - if (!stat.isFile()) throw new Error('ChatGPT export path must be a zip file or extracted folder') - const fd = fs.openSync(resolved, 'r') - try { - const signature = readAt(fd, 0, Math.min(4, stat.size)) - if (signature.length < 4 || signature.readUInt32LE(0) !== 0x04034b50) { - throw new Error('ChatGPT export file must be a .zip file') - } - } finally { - fs.closeSync(fd) - } - return zipReader(resolved) -} - -function readJsonFromExport(reader: ChatGptExportReader, relativePath: string, fallback: T): T { - if (!reader.has(relativePath)) return fallback - return JSON.parse(reader.readBuffer(relativePath).toString('utf8')) as T -} - function emptyCounts(entries: ExportEntry[]): ChatGptExportCounts { return { files: entries.length, @@ -1069,44 +729,6 @@ function buildGraphSummary(conversation: ChatGptConversation, assetsJson: Assets } } -function mediaTypeFromName(name: string): string | null { - const lower = String(name || '').toLowerCase() - if (/\.(mp4|m4v)$/i.test(lower)) return 'video/mp4' - if (/\.(mov|qt)$/i.test(lower)) return 'video/quicktime' - if (/\.webm$/i.test(lower)) return 'video/webm' - if (/\.mp3$/i.test(lower)) return 'audio/mpeg' - if (/\.wav$/i.test(lower)) return 'audio/wav' - if (/\.zip$/i.test(lower)) return 'application/zip' - if (/\.md$/i.test(lower)) return 'text/markdown' - if (/\.csv$/i.test(lower)) return 'text/csv' - if (/\.json$/i.test(lower)) return 'application/json' - if (/\.tex$/i.test(lower)) return 'text/x-tex' - return null -} - -function sniffMediaType(buffer: Buffer, fallbackName = ''): string { - if (buffer.length >= 8 && buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png' - if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) return 'image/jpeg' - if (buffer.length >= 6 && (buffer.subarray(0, 6).toString('ascii') === 'GIF87a' || buffer.subarray(0, 6).toString('ascii') === 'GIF89a')) return 'image/gif' - if (buffer.length >= 12 && buffer.subarray(0, 4).toString('ascii') === 'RIFF' && buffer.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp' - if (buffer.length >= 5 && buffer.subarray(0, 5).toString('ascii') === '%PDF-') return 'application/pdf' - if (buffer.length >= 12 && buffer.subarray(4, 8).toString('ascii') === 'ftyp') return /\.mov$/i.test(fallbackName) ? 'video/quicktime' : 'video/mp4' - if (buffer.length >= 4 && buffer[0] === 0x1a && buffer[1] === 0x45 && buffer[2] === 0xdf && buffer[3] === 0xa3) return 'video/webm' - const named = mediaTypeFromName(fallbackName) - if (named) return named - const prefix = buffer.subarray(0, Math.min(buffer.length, 512)).toString('utf8').trimStart().toLowerCase() - if (prefix.startsWith(' 0) { - let printable = 0 - for (const byte of sample) { - if (byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte < 127) || byte >= 0xc2) printable += 1 - } - if (printable / sample.length > 0.85) return 'text/plain' - } - return 'application/octet-stream' -} - function loadAssetNames(reader: ChatGptExportReader): Record { return readJsonFromExport>(reader, 'conversation_asset_file_names.json', {}) } @@ -1115,25 +737,6 @@ function assetMessageReferences(assetsJson: AssetsJson): number { return Object.values(assetsJson).reduce((sum, files) => sum + files.length, 0) } -async function readStreamSample(stream: NodeJS.ReadableStream, maxBytes: number): Promise { - const chunks: Buffer[] = [] - let bytes = 0 - for await (const rawChunk of stream as AsyncIterable) { - const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) - if (bytes < maxBytes) { - const needed = Math.min(maxBytes - bytes, chunk.length) - chunks.push(chunk.subarray(0, needed)) - bytes += needed - } - if (bytes >= maxBytes) break - } - const destroyable = stream as unknown as { destroy?: () => void } - if (typeof destroyable.destroy === 'function') { - destroyable.destroy() - } - return Buffer.concat(chunks) -} - async function analyzeAssetEntry( reader: ChatGptExportReader, entry: ExportEntry, diff --git a/src/claude-export.ts b/src/claude-export.ts new file mode 100644 index 0000000..d4b631f --- /dev/null +++ b/src/claude-export.ts @@ -0,0 +1,1036 @@ +// Importer for Claude data exports (the ZIP you download from claude.ai → +// Settings → Privacy → "Export data"). It mirrors the ChatGPT export importer +// (`chatgpt-export.ts`) — preflight, resumable-ish progress job, dedup ledger — +// but adapts to Claude's format: +// * a single `conversations.json` array (not sharded) +// * flat, linear `chat_messages` (no branch tree) +// * ISO-8601 timestamps, `sender: human|assistant` +// * text-only attachments (`extracted_content`); no binary asset files +// * an additional `design_chats/*.json` set that uses a different, richer +// message shape and is imported through the same pipeline. +// +// The generic zip/folder reader and helpers live in `export-archive.ts`. + +import * as crypto from 'crypto' +import * as fs from 'fs' +import * as path from 'path' +import { STATE_DIR } from './config' +import { + ExportArchiveReader, + ExportEntry, + nowIso, + openExportReader, + readJsonFile, + safeError, + sha256Hex, + writePrivateJson, +} from './export-archive' +import { + appendMessages, + appendRawRecords, + ensureDirs, + hasVaultSession, + loadSessions, + makeVaultPath, + replaceSessionMessages, + saveSessions, +} from './store' +import type { ContentBlock, Message, RawRecord, Session } from './types' + +const CLAUDE_EXPORT_SOURCE = 'claude-export' as const +const CLAUDE_IMPORTS_VERSION = 1 +const CLAUDE_IMPORT_JOB_VERSION = 1 +const CLAUDE_PARSER_VERSION = 1 +const CLAUDE_IMPORTS_FILE = path.join(STATE_DIR, 'claude-export-imports.json') +const CLAUDE_IMPORT_JOB_FILE = path.join(STATE_DIR, 'claude-export-import-job.json') +const STRONG_DUPLICATE_MIN_TEXT_CHARS = 200 +const STRONG_DUPLICATE_MIN_MESSAGES = 3 + +// --------------------------------------------------------------------------- +// Raw export shapes (conversations.json) +// --------------------------------------------------------------------------- + +type ClaudeContentBlock = { + type?: string + text?: string + thinking?: string + name?: string + input?: unknown + content?: unknown + title?: string + [key: string]: unknown +} + +type ClaudeChatMessage = { + uuid?: string + text?: string + content?: ClaudeContentBlock[] + sender?: string + created_at?: string + updated_at?: string + attachments?: Array> + files?: Array> + parent_message_uuid?: string + [key: string]: unknown +} + +type ClaudeConversation = { + uuid?: string + name?: string + summary?: string + created_at?: string + updated_at?: string + model?: string | null + account?: { uuid?: string } + chat_messages?: ClaudeChatMessage[] +} + +// --------------------------------------------------------------------------- +// Raw export shapes (design_chats/*.json) +// --------------------------------------------------------------------------- + +type ClaudeDesignInnerBlock = { + type?: string + text?: string + message?: unknown + toolCall?: { name?: string; type?: string; input?: unknown; output?: unknown } + [key: string]: unknown +} + +type ClaudeDesignInner = { + role?: string + content?: string + attachments?: Array> + contentBlocks?: ClaudeDesignInnerBlock[] + timestamp?: string + [key: string]: unknown +} + +type ClaudeDesignMessage = { + uuid?: string + role?: string + content?: ClaudeDesignInner + created_at?: string +} + +type ClaudeDesignChat = { + uuid?: string + title?: string + project?: { uuid?: string; name?: string } + created_at?: string + updated_at?: string + messages?: ClaudeDesignMessage[] +} + +// A normalized thread — the common shape both parsers reduce to before the +// shared prepare/dedup/import pipeline runs. +type ClaudeThread = { + kind: 'conversation' | 'design' + file: string + id: string + title: string | undefined + createdAt: string | undefined + updatedAt: string | undefined + messages: Message[] + raw: unknown +} + +// --------------------------------------------------------------------------- +// Public result / job types +// --------------------------------------------------------------------------- + +export type ClaudeExportCounts = { + files: number + totalBytes: number + conversations: number + designChats: number + projects: number + threads: number + messages: number + attachments: number + thoughts: number + toolUses: number + unknownBlockTypes: Record +} + +export type ClaudeExportPreflightResult = { + ok: boolean + sourcePath: string + sourceKind: 'folder' | 'zip' + status: 'ready' | 'failed' + format: 'claude-export' + counts: ClaudeExportCounts + warnings: string[] + errors: string[] + files: { + conversations: boolean + users: boolean + projects: number + designChats: number + } +} + +export type ClaudeImportPhase = + | 'idle' + | 'preflight' + | 'reading-export' + | 'importing-conversations' + | 'completed' + | 'failed' + +export type ClaudeImportJob = { + version: typeof CLAUDE_IMPORT_JOB_VERSION + id: string + sourcePath: string + sourceKind: 'folder' | 'zip' + phase: ClaudeImportPhase + startedAt: string + updatedAt: string + completedAt?: string + currentConversation?: string + lastError?: string + counts: ClaudeExportCounts + imported: { sessions: number; messages: number; rawRecords: number; attachments: number } + updated: { sessions: number; messages: number } + skipped: { sessions: number; messages: number; duplicates: number } + failed: { sessions: number; attachments: number } + cursor: { conversationIndex: number; attachmentIndex: number } + warnings: string[] + done: boolean +} + +type ClaudeImportedConversationRecord = { + parserVersion?: number + sourceAccount?: string + conversationId: string + destinationUid: string + currentPathFingerprint: string + rawConversationHash: string + strongFingerprint: boolean + firstImportedAt: string + lastImportedAt: string + lastAction: 'imported' | 'updated' | 'skipped' +} + +type ClaudeImportsState = { + version: typeof CLAUDE_IMPORTS_VERSION + updatedAt: string + conversations: Record + currentPathFingerprints: Record + rawConversationHashes: Record +} + +type PreparedThread = { + session: Session + messages: Message[] + rawRecord: RawRecord + currentPathFingerprint: string + rawConversationHash: string + strongFingerprint: boolean +} + +// --------------------------------------------------------------------------- +// State files (dedup ledger + progress job) +// --------------------------------------------------------------------------- + +function defaultImportsState(): ClaudeImportsState { + return { + version: CLAUDE_IMPORTS_VERSION, + updatedAt: nowIso(), + conversations: {}, + currentPathFingerprints: {}, + rawConversationHashes: {}, + } +} + +function readImportsState(): ClaudeImportsState { + const raw = readJsonFile>(CLAUDE_IMPORTS_FILE) + if (!raw || typeof raw !== 'object') return defaultImportsState() + return { + version: CLAUDE_IMPORTS_VERSION, + updatedAt: typeof raw.updatedAt === 'string' ? raw.updatedAt : nowIso(), + conversations: raw.conversations && typeof raw.conversations === 'object' ? raw.conversations : {}, + currentPathFingerprints: raw.currentPathFingerprints && typeof raw.currentPathFingerprints === 'object' ? raw.currentPathFingerprints : {}, + rawConversationHashes: raw.rawConversationHashes && typeof raw.rawConversationHashes === 'object' ? raw.rawConversationHashes : {}, + } +} + +function writeImportsState(state: ClaudeImportsState): void { + writePrivateJson(CLAUDE_IMPORTS_FILE, { ...state, version: CLAUDE_IMPORTS_VERSION, updatedAt: nowIso() }) +} + +function readJob(): ClaudeImportJob | null { + return readJsonFile(CLAUDE_IMPORT_JOB_FILE) +} + +function writeJob(job: ClaudeImportJob): void { + writePrivateJson(CLAUDE_IMPORT_JOB_FILE, job) +} + +function updateJob(job: ClaudeImportJob, patch: Partial): ClaudeImportJob { + const next = { ...job, ...patch, updatedAt: nowIso() } + writeJob(next) + return next +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +function isoFromClaudeTime(value: unknown, fallback?: string): string { + if (typeof value === 'string' && value.trim()) { + const parsed = Date.parse(value) + if (!Number.isNaN(parsed)) return new Date(parsed).toISOString() + } + return fallback ?? new Date(0).toISOString() +} + +function stringifyCompact(value: unknown): string { + if (value === undefined || value === null) return '' + if (typeof value === 'string') return value + try { + return JSON.stringify(value) + } catch { + return String(value) + } +} + +function pushText(blocks: ContentBlock[], text: unknown): void { + if (typeof text !== 'string') return + const trimmed = text.trim() + if (!trimmed) return + const previous = blocks[blocks.length - 1] + if (previous?.type === 'text' && typeof previous.text === 'string') { + previous.text = `${previous.text}\n${trimmed}` + } else { + blocks.push({ type: 'text', text: trimmed }) + } +} + +function claudeRole(sender: string | undefined): Message['role'] { + if (sender === 'human' || sender === 'user') return 'user' + if (sender === 'assistant' || sender === 'system' || sender === 'tool') return sender === 'system' || sender === 'tool' ? sender : 'assistant' + return 'assistant' +} + +function mediaTypeFromClaudeFileType(fileType: unknown, name = ''): string | undefined { + const raw = typeof fileType === 'string' ? fileType.trim().toLowerCase() : '' + const map: Record = { + txt: 'text/plain', + md: 'text/markdown', + csv: 'text/csv', + json: 'application/json', + pdf: 'application/pdf', + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + image: 'image/*', + } + if (raw && map[raw]) return map[raw] + if (raw.includes('/')) return raw + const lower = name.toLowerCase() + if (/\.pdf$/.test(lower)) return 'application/pdf' + if (/\.(png|jpe?g|gif|webp)$/.test(lower)) return 'image/*' + if (/\.(md|markdown)$/.test(lower)) return 'text/markdown' + if (/\.csv$/.test(lower)) return 'text/csv' + if (/\.json$/.test(lower)) return 'application/json' + if (/\.txt$/.test(lower)) return 'text/plain' + return undefined +} + +function toolResultText(content: unknown): string { + if (typeof content === 'string') return content + if (Array.isArray(content)) { + return content + .map(item => { + if (typeof item === 'string') return item + if (item && typeof item === 'object') { + const record = item as Record + const parts = [record.title, record.text, record.url].filter((value): value is string => typeof value === 'string' && value.trim() !== '') + return parts.join(' — ') + } + return '' + }) + .filter(Boolean) + .join('\n') + } + return stringifyCompact(content) +} + +// --------------------------------------------------------------------------- +// Content-block translation +// --------------------------------------------------------------------------- + +// conversations.json message content[] → ContentBlock[] +function collectConversationBlocks(content: ClaudeContentBlock[] | undefined, blocks: ContentBlock[]): void { + if (!Array.isArray(content)) return + for (const block of content) { + if (!block || typeof block !== 'object') continue + const type = typeof block.type === 'string' ? block.type : '' + switch (type) { + case 'text': + pushText(blocks, block.text) + break + case 'thinking': + if (typeof block.thinking === 'string' && block.thinking.trim()) { + blocks.push({ type: 'thinking', thinking: block.thinking.trim() }) + } + break + case 'tool_use': + blocks.push({ + type: 'tool_use', + name: typeof block.name === 'string' ? block.name : 'claude_tool', + input: block.input, + text: typeof block.message === 'string' ? block.message : undefined, + content: block, + }) + break + case 'tool_result': + blocks.push({ + type: 'tool_result', + name: typeof block.name === 'string' ? block.name : undefined, + content: block.content, + text: toolResultText(block.content), + }) + break + case 'voice_note': + pushText(blocks, [block.title, block.text].filter(v => typeof v === 'string').join('\n')) + break + case 'token_budget': + // Pure metadata (remaining token budget) — nothing to render. + break + default: { + const fallback = typeof block.text === 'string' ? block.text : stringifyCompact(block) + if (fallback && fallback !== '{}') blocks.push({ type: 'other', content: block, text: fallback }) + } + } + } +} + +// design_chats inner.contentBlocks[] → ContentBlock[] +function collectDesignBlocks(inner: ClaudeDesignInner, blocks: ContentBlock[]): void { + const contentBlocks = Array.isArray(inner.contentBlocks) ? inner.contentBlocks : [] + if (contentBlocks.length === 0) { + pushText(blocks, inner.content) + return + } + for (const block of contentBlocks) { + if (!block || typeof block !== 'object') continue + const type = typeof block.type === 'string' ? block.type : '' + switch (type) { + case 'text': + pushText(blocks, block.text) + break + case 'thinking': + if (typeof block.text === 'string' && block.text.trim()) { + blocks.push({ type: 'thinking', thinking: block.text.trim() }) + } + break + case 'tool_call': { + const call = block.toolCall || {} + blocks.push({ + type: 'tool_use', + name: typeof call.name === 'string' ? call.name : 'design_tool', + input: call.input, + text: typeof call.type === 'string' ? call.type : undefined, + content: call, + }) + break + } + case 'error': + if (typeof block.message === 'string') pushText(blocks, block.message) + break + case 'user_interjection': { + const msg = block.message as Record | undefined + const text = msg && typeof msg.content === 'string' ? msg.content : stringifyCompact(block.message) + pushText(blocks, text) + break + } + default: { + const fallback = typeof block.text === 'string' ? block.text : stringifyCompact(block) + if (fallback && fallback !== '{}') blocks.push({ type: 'other', content: block, text: fallback }) + } + } + } +} + +// Text-only attachments and file references (both export shapes) → ContentBlock[] +function appendAttachmentBlocks( + blocks: ContentBlock[], + attachments: Array> | undefined, + files: Array> | undefined, +): number { + let count = 0 + for (const attachment of attachments ?? []) { + if (!attachment || typeof attachment !== 'object') continue + const fileName = typeof attachment.file_name === 'string' ? attachment.file_name + : typeof attachment.name === 'string' ? attachment.name : 'attachment' + const fileType = attachment.file_type ?? attachment.type + const extracted = typeof attachment.extracted_content === 'string' ? attachment.extracted_content + : typeof attachment.content === 'string' ? attachment.content : '' + blocks.push({ + type: 'file', + attachmentName: fileName, + mediaType: mediaTypeFromClaudeFileType(fileType, fileName), + text: extracted ? `${fileName}\n${extracted}` : fileName, + content: attachment, + }) + count += 1 + } + for (const file of files ?? []) { + if (!file || typeof file !== 'object') continue + const fileName = typeof file.file_name === 'string' ? file.file_name + : typeof file.name === 'string' ? file.name : 'file' + blocks.push({ + type: 'file', + attachmentName: fileName, + mediaType: mediaTypeFromClaudeFileType(file.type, fileName), + text: fileName, + content: file, + }) + count += 1 + } + return count +} + +// --------------------------------------------------------------------------- +// Message construction +// --------------------------------------------------------------------------- + +function messageFromChatMessage(message: ClaudeChatMessage, conversation: ClaudeConversation): Message | null { + const role = claudeRole(message.sender) + const blocks: ContentBlock[] = [] + collectConversationBlocks(message.content, blocks) + if (blocks.length === 0) pushText(blocks, message.text) + appendAttachmentBlocks(blocks, message.attachments, message.files) + if (blocks.length === 0) return null + const timestamp = isoFromClaudeTime(message.created_at, isoFromClaudeTime(conversation.created_at)) + return { + id: message.uuid || sha256Hex(stringifyCompact(message)).slice(0, 24), + role, + timestamp, + content: blocks, + hasThinking: blocks.some(block => block.type === 'thinking'), + sourceEventType: 'claude.message', + unknownAttrs: { + claudeMessageUuid: message.uuid ?? null, + claudeParentMessageUuid: message.parent_message_uuid ?? null, + claudeSender: message.sender ?? null, + }, + } +} + +function messageFromDesignMessage(message: ClaudeDesignMessage, chat: ClaudeDesignChat): Message | null { + const inner: ClaudeDesignInner = message.content && typeof message.content === 'object' ? message.content : {} + const role = claudeRole(message.role || inner.role) + const blocks: ContentBlock[] = [] + collectDesignBlocks(inner, blocks) + appendAttachmentBlocks(blocks, inner.attachments, undefined) + if (blocks.length === 0) return null + const timestamp = isoFromClaudeTime(inner.timestamp || message.created_at, isoFromClaudeTime(chat.created_at)) + return { + id: message.uuid || sha256Hex(stringifyCompact(message)).slice(0, 24), + role, + timestamp, + content: blocks, + hasThinking: blocks.some(block => block.type === 'thinking'), + sourceEventType: 'claude.design-message', + unknownAttrs: { + claudeDesignMessageUuid: message.uuid ?? null, + claudeDesignAuthorName: inner.authorName ?? null, + }, + } +} + +// --------------------------------------------------------------------------- +// Loading / normalizing threads +// --------------------------------------------------------------------------- + +function loadConversationThreads(reader: ExportArchiveReader): ClaudeThread[] { + const files = reader.listEntries() + .map(entry => entry.path) + .filter(file => file === 'conversations.json' || /^conversations-\d+\.json$/.test(file)) + .sort() + const threads: ClaudeThread[] = [] + for (const file of files) { + const parsed = JSON.parse(reader.readBuffer(file).toString('utf8')) as unknown + if (!Array.isArray(parsed)) throw new Error(`${file} is not a Claude conversations array`) + for (const raw of parsed) { + if (!raw || typeof raw !== 'object') continue + const conversation = raw as ClaudeConversation + const messages = (conversation.chat_messages ?? []) + .map(message => messageFromChatMessage(message, conversation)) + .filter((message): message is Message => message !== null) + threads.push({ + kind: 'conversation', + file, + id: String(conversation.uuid || '').trim(), + title: conversation.name || undefined, + createdAt: conversation.created_at, + updatedAt: conversation.updated_at, + messages, + raw: conversation, + }) + } + } + return threads +} + +function loadDesignThreads(reader: ExportArchiveReader): ClaudeThread[] { + const files = reader.listEntries() + .map(entry => entry.path) + .filter(file => /^design_chats\/.+\.json$/.test(file)) + .sort() + const threads: ClaudeThread[] = [] + for (const file of files) { + const parsed = JSON.parse(reader.readBuffer(file).toString('utf8')) as unknown + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue + const chat = parsed as ClaudeDesignChat + const messages = (chat.messages ?? []) + .map(message => messageFromDesignMessage(message, chat)) + .filter((message): message is Message => message !== null) + threads.push({ + kind: 'design', + file, + id: String(chat.uuid || '').trim(), + title: chat.title || chat.project?.name || undefined, + createdAt: chat.created_at, + updatedAt: chat.updated_at, + messages, + raw: chat, + }) + } + return threads +} + +function loadThreads(reader: ExportArchiveReader): ClaudeThread[] { + return [...loadConversationThreads(reader), ...loadDesignThreads(reader)] +} + +// --------------------------------------------------------------------------- +// Account / identity / dedup +// --------------------------------------------------------------------------- + +function userHash(reader: ExportArchiveReader): string | undefined { + if (!reader.has('users.json')) return undefined + try { + const raw = reader.readBuffer('users.json').toString('utf8') + const parsed = JSON.parse(raw) as unknown + const first = Array.isArray(parsed) ? parsed[0] : parsed + const record = (first && typeof first === 'object' ? first : {}) as Record + const candidate = String(record.uuid || record.email_address || record.email || raw).trim() + return `claude:${sha256Hex(candidate).slice(0, 12)}` + } catch { + return undefined + } +} + +// A thread's stable identity namespace — includes kind so a conversation and a +// design chat can never collide even if uuids were ever shared. +function threadIdentity(thread: ClaudeThread, rawHash: string): string { + const explicit = thread.id + if (explicit) return `${thread.kind}:${explicit}` + const title = String(thread.title || '').trim() + const created = String(thread.createdAt || '').trim() + return `${thread.kind}:missing-${sha256Hex(`${title}\0${created}\0${rawHash}`).slice(0, 16)}` +} + +function stableSessionUid(sourceAccount: string | undefined, identity: string): string { + return sha256Hex(`${CLAUDE_EXPORT_SOURCE}\0${sourceAccount ?? ''}\0${identity}`).slice(0, 24) +} + +function conversationRecordKey(sourceAccount: string | undefined, identity: string): string { + return `${sourceAccount ?? ''}\0${identity}` +} + +function scopedImportIndexKey(sourceAccount: string | undefined, value: string): string { + return `${sourceAccount ?? ''}\0${value}` +} + +function sameAccountRecord( + record: ClaudeImportedConversationRecord | undefined, + sourceAccount: string | undefined, +): ClaudeImportedConversationRecord | undefined { + if (!record) return undefined + return (record.sourceAccount ?? '') === (sourceAccount ?? '') ? record : undefined +} + +function currentPathFingerprint(messages: Message[]): { hash: string; strong: boolean; textChars: number } { + let textChars = 0 + const normalized = messages.map(message => { + const content = message.content.map(block => { + if (block.type === 'text') { + const text = String(block.text || '').normalize('NFKC').replace(/\s+/g, ' ').trim() + textChars += text.length + return { type: block.type, text } + } + if (block.type === 'thinking') { + const text = String(block.thinking || '').normalize('NFKC').replace(/\s+/g, ' ').trim() + textChars += text.length + return { type: block.type, thinking: text } + } + return { + type: block.type, + name: block.name, + attachmentName: block.attachmentName, + text: String(block.text || '').normalize('NFKC').replace(/\s+/g, ' ').trim(), + } + }) + return { role: message.role, content } + }) + return { + hash: sha256Hex(JSON.stringify(normalized)), + strong: messages.length >= STRONG_DUPLICATE_MIN_MESSAGES || textChars >= STRONG_DUPLICATE_MIN_TEXT_CHARS, + textChars, + } +} + +// --------------------------------------------------------------------------- +// Thread → Session +// --------------------------------------------------------------------------- + +function prepareThread(thread: ClaudeThread, sourceAccount: string | undefined): PreparedThread { + const rawConversationHash = sha256Hex(JSON.stringify(thread.raw)) + const identity = threadIdentity(thread, rawConversationHash) + const messages = thread.messages + const fallbackTime = isoFromClaudeTime(thread.createdAt) + const firstTimestamp = messages[0]?.timestamp || fallbackTime + const lastTimestamp = messages[messages.length - 1]?.timestamp || isoFromClaudeTime(thread.updatedAt, firstTimestamp) + const uid = stableSessionUid(sourceAccount, identity) + const originalPath = `claude-export://${thread.kind}/${sourceAccount ?? 'unknown'}/${thread.id || identity}` + const fingerprint = currentPathFingerprint(messages) + const defaultTitle = thread.kind === 'design' ? 'Untitled Claude design chat' : 'Untitled Claude conversation' + const session: Session = { + uid, + id: thread.id || identity, + source: CLAUDE_EXPORT_SOURCE, + sourceClient: thread.kind === 'design' ? 'Claude design chat' : 'Claude export', + sourceAccount, + appVersion: `export parser-v${CLAUDE_PARSER_VERSION}`, + model: 'Claude', + modelProvider: 'anthropic', + title: thread.title || undefined, + firstTimestamp, + lastTimestamp, + cwd: thread.title || defaultTitle, + messageCount: messages.length, + hasThinking: messages.some(message => message.hasThinking), + vaultPath: makeVaultPath(CLAUDE_EXPORT_SOURCE, uid), + originalPath, + } + const rawRecord: RawRecord = { + v: 1, + source: CLAUDE_EXPORT_SOURCE, + sourcePath: originalPath, + sourceByteOffset: 0, + capturedAt: nowIso(), + rawHash: rawConversationHash, + raw: { + type: thread.kind === 'design' ? 'claude-design-snapshot' : 'claude-conversation-snapshot', + thread: thread.raw, + }, + } + return { + session, + messages, + rawRecord, + currentPathFingerprint: fingerprint.hash, + rawConversationHash, + strongFingerprint: fingerprint.strong, + } +} + +// --------------------------------------------------------------------------- +// Counting / preflight +// --------------------------------------------------------------------------- + +function emptyCounts(entries: ExportEntry[]): ClaudeExportCounts { + return { + files: entries.length, + totalBytes: entries.reduce((sum, entry) => sum + entry.size, 0), + conversations: 0, + designChats: 0, + projects: 0, + threads: 0, + messages: 0, + attachments: 0, + thoughts: 0, + toolUses: 0, + unknownBlockTypes: {}, + } +} + +const KNOWN_CONVERSATION_BLOCK_TYPES = new Set(['text', 'thinking', 'tool_use', 'tool_result', 'voice_note', 'token_budget']) + +function summarizeExport(reader: ExportArchiveReader, threads: ClaudeThread[]): ClaudeExportCounts { + const entries = reader.listEntries() + const counts = emptyCounts(entries) + counts.projects = entries.filter(entry => /^projects\/.+\.json$/.test(entry.path)).length + for (const thread of threads) { + if (thread.kind === 'design') counts.designChats += 1 + else counts.conversations += 1 + if (thread.messages.length === 0) continue + counts.threads += 1 + counts.messages += thread.messages.length + for (const message of thread.messages) { + for (const block of message.content) { + if (block.type === 'thinking') counts.thoughts += 1 + if (block.type === 'tool_use') counts.toolUses += 1 + if (block.type === 'file') counts.attachments += 1 + } + } + // Track unknown raw block types for visibility. + if (thread.kind === 'conversation') { + const conversation = thread.raw as ClaudeConversation + for (const message of conversation.chat_messages ?? []) { + for (const block of message.content ?? []) { + const type = typeof block?.type === 'string' ? block.type : '' + if (type && !KNOWN_CONVERSATION_BLOCK_TYPES.has(type)) { + counts.unknownBlockTypes[type] = (counts.unknownBlockTypes[type] || 0) + 1 + } + } + } + } + } + return counts +} + +export async function preflightClaudeExport(sourcePath: string): Promise { + const warnings: string[] = [] + const errors: string[] = [] + let reader: ExportArchiveReader | null = null + try { + reader = openExportReader(sourcePath) + const entries = reader.listEntries() + const hasConversations = reader.has('conversations.json') || entries.some(entry => /^conversations-\d+\.json$/.test(entry.path)) + const designChatCount = entries.filter(entry => /^design_chats\/.+\.json$/.test(entry.path)).length + if (!hasConversations && designChatCount === 0) { + errors.push('No conversations.json or design_chats were found in this export') + } + const threads = loadThreads(reader) + const counts = summarizeExport(reader, threads) + if (!reader.has('users.json')) warnings.push('users.json is missing; imported sessions will not be grouped by account') + const emptyThreads = threads.filter(thread => thread.messages.length === 0).length + if (emptyThreads > 0) warnings.push(`${emptyThreads} conversations/design chats have no messages and will be skipped`) + if (counts.projects > 0) warnings.push(`${counts.projects} project files were found; project metadata is preserved but not imported as chat sessions`) + if (Object.keys(counts.unknownBlockTypes).length > 0) { + warnings.push(`Unrecognized content block types will be preserved as raw: ${Object.keys(counts.unknownBlockTypes).join(', ')}`) + } + return { + ok: errors.length === 0, + sourcePath: path.resolve(sourcePath), + sourceKind: reader.kind, + status: errors.length === 0 ? 'ready' : 'failed', + format: 'claude-export', + counts, + warnings, + errors, + files: { + conversations: hasConversations, + users: reader.has('users.json'), + projects: counts.projects, + designChats: designChatCount, + }, + } + } catch (error) { + errors.push(safeError(error)) + const entries = reader?.listEntries() ?? [] + return { + ok: false, + sourcePath: path.resolve(sourcePath), + sourceKind: reader?.kind ?? (fs.existsSync(sourcePath) && fs.statSync(sourcePath).isDirectory() ? 'folder' : 'zip'), + status: 'failed', + format: 'claude-export', + counts: emptyCounts(entries), + warnings, + errors, + files: { + conversations: !!reader?.has('conversations.json'), + users: !!reader?.has('users.json'), + projects: entries.filter(entry => /^projects\/.+\.json$/.test(entry.path)).length, + designChats: entries.filter(entry => /^design_chats\/.+\.json$/.test(entry.path)).length, + }, + } + } finally { + reader?.close?.() + } +} + +// --------------------------------------------------------------------------- +// Import job +// --------------------------------------------------------------------------- + +function newJob(sourcePath: string, sourceKind: 'folder' | 'zip', counts: ClaudeExportCounts, warnings: string[]): ClaudeImportJob { + const startedAt = nowIso() + return { + version: CLAUDE_IMPORT_JOB_VERSION, + id: crypto.randomUUID(), + sourcePath: path.resolve(sourcePath), + sourceKind, + phase: 'preflight', + startedAt, + updatedAt: startedAt, + counts, + imported: { sessions: 0, messages: 0, rawRecords: 0, attachments: 0 }, + updated: { sessions: 0, messages: 0 }, + skipped: { sessions: 0, messages: 0, duplicates: 0 }, + failed: { sessions: 0, attachments: 0 }, + cursor: { conversationIndex: 0, attachmentIndex: 0 }, + warnings, + done: false, + } +} + +export function currentClaudeImportJob(): ClaudeImportJob | null { + return readJob() +} + +export async function runClaudeExportImport(sourcePath: string): Promise { + if (!hasVaultSession()) throw new Error('current vault must be unlocked before importing Claude exports') + ensureDirs() + const preflight = await preflightClaudeExport(sourcePath) + if (!preflight.ok) throw new Error(preflight.errors.join(' · ') || 'Claude export preflight failed') + let job = newJob(sourcePath, preflight.sourceKind, preflight.counts, preflight.warnings) + writeJob(job) + let reader: ExportArchiveReader | null = null + try { + reader = openExportReader(sourcePath) + job = updateJob(job, { phase: 'reading-export' }) + const sourceAccount = userHash(reader) + const allThreads = loadThreads(reader) + const threads = allThreads.filter(thread => thread.messages.length > 0) + const importsState = readImportsState() + + job = updateJob(job, { phase: 'importing-conversations' }) + const sessions = await loadSessions() + const sessionByUid = new Map(sessions.map(session => [session.uid, session])) + const merged = [...sessions] + const indexByUid = new Map(merged.map((session, index) => [session.uid, index])) + let sessionsDirty = false + const flushSessions = async (): Promise => { + if (!sessionsDirty) return + await saveSessions(merged) + sessionsDirty = false + } + + for (let index = 0; index < threads.length; index += 1) { + const thread = threads[index] + job.cursor.conversationIndex = index + job.currentConversation = thread.title || thread.id || thread.file + writeJob(job) + try { + const prepared = prepareThread(thread, sourceAccount) + const identity = threadIdentity(thread, prepared.rawConversationHash) + const key = conversationRecordKey(sourceAccount, identity) + const existingRecord = importsState.conversations[key] + const sameRaw = importsState.rawConversationHashes[scopedImportIndexKey(sourceAccount, prepared.rawConversationHash)] + || sameAccountRecord(importsState.rawConversationHashes[prepared.rawConversationHash], sourceAccount) + const sameStrongFingerprint = prepared.strongFingerprint + ? importsState.currentPathFingerprints[scopedImportIndexKey(sourceAccount, prepared.currentPathFingerprint)] + || sameAccountRecord(importsState.currentPathFingerprints[prepared.currentPathFingerprint], sourceAccount) + : undefined + const destinationUid = existingRecord?.destinationUid || sameRaw?.destinationUid || sameStrongFingerprint?.destinationUid || prepared.session.uid + const existingSession = sessionByUid.get(destinationUid) || sessionByUid.get(prepared.session.uid) + const rawAlreadyStored = !!sameRaw + + const existingCurrentParser = existingRecord?.parserVersion === CLAUDE_PARSER_VERSION + const sameRawCurrentParser = sameRaw?.parserVersion === CLAUDE_PARSER_VERSION + const sameStrongCurrentParser = sameStrongFingerprint?.parserVersion === CLAUDE_PARSER_VERSION + if ( + (existingRecord && existingCurrentParser && existingRecord.currentPathFingerprint === prepared.currentPathFingerprint) + || (sameRaw && sameRawCurrentParser) + || (sameStrongFingerprint && sameStrongCurrentParser && sameStrongFingerprint.conversationId !== identity) + ) { + job.skipped.sessions += 1 + job.skipped.duplicates += 1 + job.skipped.messages += prepared.messages.length + const record = existingRecord || sameRaw || sameStrongFingerprint! + record.lastImportedAt = nowIso() + record.lastAction = 'skipped' + importsState.conversations[key] = record + writeImportsState(importsState) + writeJob(job) + continue + } + + const destinationSession: Session = { + ...prepared.session, + uid: destinationUid, + vaultPath: makeVaultPath(CLAUDE_EXPORT_SOURCE, destinationUid), + } + + if (existingSession || existingRecord) { + await replaceSessionMessages(destinationSession, prepared.messages) + const existingIndex = indexByUid.get(destinationUid) + if (existingIndex !== undefined) merged[existingIndex] = destinationSession + else { + merged.push(destinationSession) + indexByUid.set(destinationUid, merged.length - 1) + } + job.updated.sessions += 1 + job.updated.messages += prepared.messages.length + } else { + await appendMessages(destinationSession, prepared.messages) + merged.push(destinationSession) + indexByUid.set(destinationUid, merged.length - 1) + sessionByUid.set(destinationUid, destinationSession) + job.imported.sessions += 1 + job.imported.messages += prepared.messages.length + } + sessionsDirty = true + + if (!rawAlreadyStored) { + await appendRawRecords(CLAUDE_EXPORT_SOURCE, destinationUid, [{ + ...prepared.rawRecord, + sourcePath: destinationSession.originalPath, + }]) + job.imported.rawRecords += 1 + } + await saveSessions(merged) + sessionsDirty = false + const importedAt = nowIso() + const record: ClaudeImportedConversationRecord = { + parserVersion: CLAUDE_PARSER_VERSION, + sourceAccount, + conversationId: identity, + destinationUid, + currentPathFingerprint: prepared.currentPathFingerprint, + rawConversationHash: prepared.rawConversationHash, + strongFingerprint: prepared.strongFingerprint, + firstImportedAt: existingRecord?.firstImportedAt ?? importedAt, + lastImportedAt: importedAt, + lastAction: existingSession || existingRecord ? 'updated' : 'imported', + } + importsState.conversations[key] = record + importsState.rawConversationHashes[scopedImportIndexKey(sourceAccount, prepared.rawConversationHash)] = record + if (prepared.strongFingerprint) importsState.currentPathFingerprints[scopedImportIndexKey(sourceAccount, prepared.currentPathFingerprint)] = record + writeImportsState(importsState) + writeJob(job) + } catch (error) { + job.failed.sessions += 1 + job.lastError = safeError(error) + writeJob(job) + } + } + await flushSessions() + job.currentConversation = undefined + job = updateJob(job, { + phase: 'completed', + completedAt: nowIso(), + lastError: undefined, + done: true, + }) + return job + } catch (error) { + job = updateJob(job, { + phase: 'failed', + lastError: safeError(error), + done: true, + }) + return job + } finally { + reader?.close?.() + } +} diff --git a/src/config.ts b/src/config.ts index eb59619..5444994 100644 --- a/src/config.ts +++ b/src/config.ts @@ -51,7 +51,7 @@ export const AUTH_FILE = path.join(DATAMOAT_ROOT, 'auth.json') export const BOOTSTRAP_CAPTURE_DIR = path.join(DATAMOAT_ROOT, 'bootstrap-capture') export const WATCHED_SOURCES: readonly WatchedSource[] = ['claude-cli', 'codex-cli', 'claude-app', 'openclaw', 'cursor'] -export const ALL_SOURCES: readonly Source[] = [...WATCHED_SOURCES, 'chatgpt-export'] +export const ALL_SOURCES: readonly Source[] = [...WATCHED_SOURCES, 'chatgpt-export', 'claude-export'] function discoverClaudeAppRoots(): string[] { if (process.platform !== 'win32') { diff --git a/src/electron/main.ts b/src/electron/main.ts index 66fd20c..5336af1 100644 --- a/src/electron/main.ts +++ b/src/electron/main.ts @@ -2003,6 +2003,23 @@ function installDesktopIpc(): void { if (result.canceled || !result.filePaths[0]) return { canceled: true } return { canceled: false, path: result.filePaths[0] } }) + ipcMain.handle('datamoat:claudeExport:selectSource', async () => { + const owner = usableWindow() + const options: OpenDialogOptions = { + title: 'Choose Claude export zip or folder', + buttonLabel: 'Choose Export', + properties: ['openFile', 'openDirectory'], + filters: [ + { name: 'Claude export ZIP', extensions: ['zip'] }, + { name: 'All Files', extensions: ['*'] }, + ], + } + const result = owner + ? await dialog.showOpenDialog(owner, options) + : await dialog.showOpenDialog(options) + if (result.canceled || !result.filePaths[0]) return { canceled: true } + return { canceled: false, path: result.filePaths[0] } + }) } type ExportPdfWorkerArgs = { diff --git a/src/electron/preload.ts b/src/electron/preload.ts index 69675fd..59be099 100644 --- a/src/electron/preload.ts +++ b/src/electron/preload.ts @@ -14,6 +14,9 @@ contextBridge.exposeInMainWorld('datamoatDesktop', { chatgptExport: { selectSource: () => ipcRenderer.invoke('datamoat:chatgptExport:selectSource'), }, + claudeExport: { + selectSource: () => ipcRenderer.invoke('datamoat:claudeExport:selectSource'), + }, clipboard: { write: (text: string) => ipcRenderer.invoke('datamoat:clipboard:write', text), }, diff --git a/src/export-archive.ts b/src/export-archive.ts new file mode 100644 index 0000000..f90876f --- /dev/null +++ b/src/export-archive.ts @@ -0,0 +1,426 @@ +// Shared archive-reading utilities for provider export importers (ChatGPT, Claude, …). +// +// This module owns the provider-agnostic pieces: reading a ZIP file or an +// extracted folder as a uniform entry list, magic-byte media-type sniffing, +// atomic private JSON writes, and small hashing/time helpers. It was factored +// out of `chatgpt-export.ts` so a second importer can reuse the exact same, +// battle-tested zip/zip64 reader instead of duplicating it. + +import * as crypto from 'crypto' +import * as fs from 'fs' +import * as path from 'path' +import * as zlib from 'zlib' +import { Readable } from 'stream' + +export const MAX_ZIP_ENTRY_COUNT = 100_000 +export const MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES = 12 * 1024 * 1024 * 1024 +export const MAX_ZIP_SINGLE_ENTRY_BYTES = MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES + +export function positiveByteLimit(value: string | undefined, fallback: number): number { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback +} + +type ZipEntry = { + path: string + compressedSize: number + uncompressedSize: number + method: number + flags: number + localHeaderOffset: number +} + +export type ExportEntry = { + path: string + size: number +} + +export type ExportArchiveReader = { + kind: 'folder' | 'zip' + rootPath: string + listEntries(): ExportEntry[] + has(relativePath: string): boolean + readBuffer(relativePath: string): Buffer + createReadStream(relativePath: string): NodeJS.ReadableStream + close?(): void +} + +export function nowIso(): string { + return new Date().toISOString() +} + +export function safeError(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +export function sha256Hex(value: string | Buffer): string { + return crypto.createHash('sha256').update(value).digest('hex') +} + +export function writePrivateJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 }) + const tmpPath = `${filePath}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp` + fs.writeFileSync(tmpPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 }) + const fd = fs.openSync(tmpPath, 'r') + try { + fs.fsyncSync(fd) + } catch { + /* non-fatal */ + } finally { + fs.closeSync(fd) + } + fs.renameSync(tmpPath, filePath) + try { fs.chmodSync(filePath, 0o600) } catch { /* non-fatal */ } +} + +export function readJsonFile(filePath: string): T | null { + try { + return JSON.parse(fs.readFileSync(filePath, 'utf8')) as T + } catch { + return null + } +} + +export function toPosixPath(value: string): string { + return value.replace(/\\/g, '/') +} + +function safeZipPath(name: string): string { + if (name.includes('\0')) throw new Error('zip entry path contains NUL byte') + const normalized = toPosixPath(name) + if (!normalized || normalized.startsWith('/') || normalized.startsWith('~')) { + throw new Error(`unsafe zip entry path: ${name}`) + } + if (/^[A-Za-z]:\//.test(normalized)) throw new Error(`unsafe zip entry path: ${name}`) + const parts = normalized.split('/').filter(Boolean) + if (parts.some(part => part === '..')) throw new Error(`unsafe zip entry path: ${name}`) + return parts.join('/') +} + +function listFolderEntries(root: string): ExportEntry[] { + const entries: ExportEntry[] = [] + const walk = (dirPath: string): void => { + for (const entry of fs.readdirSync(dirPath, { withFileTypes: true })) { + if (entry.name === '.DS_Store' || entry.name.startsWith('._')) continue + const absolute = path.join(dirPath, entry.name) + if (entry.isDirectory()) { + walk(absolute) + continue + } + if (!entry.isFile()) continue + const relative = toPosixPath(path.relative(root, absolute)) + entries.push({ path: relative, size: fs.statSync(absolute).size }) + } + } + walk(root) + return entries.sort((a, b) => a.path.localeCompare(b.path)) +} + +function folderReader(rootPath: string): ExportArchiveReader { + const resolved = path.resolve(rootPath) + const entries = listFolderEntries(resolved) + const entrySet = new Set(entries.map(entry => entry.path)) + const resolveEntryPath = (relativePath: string): string => { + const safe = safeZipPath(relativePath) + if (!entrySet.has(safe)) throw new Error(`export file not found: ${relativePath}`) + const absolute = path.join(resolved, ...safe.split('/')) + const normalized = path.resolve(absolute) + if (!normalized.startsWith(`${resolved}${path.sep}`) && normalized !== resolved) { + throw new Error(`unsafe export file path: ${relativePath}`) + } + return normalized + } + return { + kind: 'folder', + rootPath: resolved, + listEntries: () => entries, + has: relativePath => entrySet.has(toPosixPath(relativePath)), + readBuffer(relativePath: string): Buffer { + return fs.readFileSync(resolveEntryPath(relativePath)) + }, + createReadStream(relativePath: string): NodeJS.ReadableStream { + return fs.createReadStream(resolveEntryPath(relativePath)) + }, + } +} + +function readAt(fd: number, offset: number, length: number): Buffer { + const buffer = Buffer.alloc(length) + const bytesRead = fs.readSync(fd, buffer, 0, length, offset) + if (bytesRead !== length) throw new Error('unexpected end of zip file') + return buffer +} + +function uint64ToSafeNumber(value: bigint, label: string): number { + if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`zip64 ${label} is too large`) + return Number(value) +} + +function zip64ExtraValues( + extra: Buffer, + fields: { uncompressed: number; compressed: number; localHeaderOffset: number }, +): { uncompressedSize: number; compressedSize: number; localHeaderOffset: number } { + let uncompressedSize = fields.uncompressed + let compressedSize = fields.compressed + let localHeaderOffset = fields.localHeaderOffset + let cursor = 0 + while (cursor + 4 <= extra.length) { + const headerId = extra.readUInt16LE(cursor) + const dataSize = extra.readUInt16LE(cursor + 2) + const dataStart = cursor + 4 + const dataEnd = dataStart + dataSize + if (dataEnd > extra.length) break + if (headerId === 0x0001) { + let valueCursor = dataStart + const readZip64Value = (label: string): number => { + if (valueCursor + 8 > dataEnd) throw new Error(`zip64 extra field is truncated: ${label}`) + const value = uint64ToSafeNumber(extra.readBigUInt64LE(valueCursor), label) + valueCursor += 8 + return value + } + if (fields.uncompressed === 0xffffffff) uncompressedSize = readZip64Value('uncompressed size') + if (fields.compressed === 0xffffffff) compressedSize = readZip64Value('compressed size') + if (fields.localHeaderOffset === 0xffffffff) localHeaderOffset = readZip64Value('local header offset') + return { uncompressedSize, compressedSize, localHeaderOffset } + } + cursor = dataEnd + } + if ( + fields.uncompressed === 0xffffffff + || fields.compressed === 0xffffffff + || fields.localHeaderOffset === 0xffffffff + ) { + throw new Error('zip64 extra field is missing') + } + return { uncompressedSize, compressedSize, localHeaderOffset } +} + +function zipCentralDirectoryLocation( + fd: number, + stat: fs.Stats, + eocdOffset: number, + eocd: Buffer, +): { entriesTotal: number; centralSize: number; centralOffset: number } { + const diskNumber = eocd.readUInt16LE(4) + const centralDisk = eocd.readUInt16LE(6) + const entriesThisDisk = eocd.readUInt16LE(8) + const entriesTotal32 = eocd.readUInt16LE(10) + const centralSize32 = eocd.readUInt32LE(12) + const centralOffset32 = eocd.readUInt32LE(16) + if (diskNumber !== 0 || centralDisk !== 0 || entriesThisDisk !== entriesTotal32) { + throw new Error('multi-disk zip files are not supported') + } + + const needsZip64 = entriesTotal32 === 0xffff || centralSize32 === 0xffffffff || centralOffset32 === 0xffffffff + if (!needsZip64) { + return { + entriesTotal: entriesTotal32, + centralSize: centralSize32, + centralOffset: centralOffset32, + } + } + + if (eocdOffset < 20) throw new Error('zip64 locator is missing') + const locator = readAt(fd, eocdOffset - 20, 20) + if (locator.readUInt32LE(0) !== 0x07064b50) throw new Error('zip64 locator is missing') + const locatorDisk = locator.readUInt32LE(4) + const zip64EocdOffset = uint64ToSafeNumber(locator.readBigUInt64LE(8), 'end-of-central-directory offset') + const totalDisks = locator.readUInt32LE(16) + if (locatorDisk !== 0 || totalDisks !== 1) throw new Error('multi-disk zip files are not supported') + if (zip64EocdOffset < 0 || zip64EocdOffset + 56 > stat.size) throw new Error('invalid zip64 end-of-central-directory offset') + + const zip64Header = readAt(fd, zip64EocdOffset, 56) + if (zip64Header.readUInt32LE(0) !== 0x06064b50) throw new Error('zip64 end-of-central-directory record not found') + const zip64Disk = zip64Header.readUInt32LE(16) + const zip64CentralDisk = zip64Header.readUInt32LE(20) + const zip64EntriesThisDisk = uint64ToSafeNumber(zip64Header.readBigUInt64LE(24), 'entry count') + const zip64EntriesTotal = uint64ToSafeNumber(zip64Header.readBigUInt64LE(32), 'entry count') + if (zip64Disk !== 0 || zip64CentralDisk !== 0 || zip64EntriesThisDisk !== zip64EntriesTotal) { + throw new Error('multi-disk zip files are not supported') + } + return { + entriesTotal: zip64EntriesTotal, + centralSize: uint64ToSafeNumber(zip64Header.readBigUInt64LE(40), 'central directory size'), + centralOffset: uint64ToSafeNumber(zip64Header.readBigUInt64LE(48), 'central directory offset'), + } +} + +function parseZipEntries(zipPath: string, fd: number): Map { + const stat = fs.fstatSync(fd) + const tailLength = Math.min(stat.size, 66_000) + const tail = readAt(fd, stat.size - tailLength, tailLength) + let eocdOffset = -1 + for (let i = tail.length - 22; i >= 0; i -= 1) { + if (tail.readUInt32LE(i) === 0x06054b50) { + eocdOffset = stat.size - tailLength + i + break + } + } + if (eocdOffset < 0) throw new Error('zip end-of-central-directory record not found') + const eocd = readAt(fd, eocdOffset, 22) + const { entriesTotal, centralSize, centralOffset } = zipCentralDirectoryLocation(fd, stat, eocdOffset, eocd) + if (entriesTotal > MAX_ZIP_ENTRY_COUNT) throw new Error('zip has too many files') + const central = readAt(fd, centralOffset, centralSize) + const entries = new Map() + let cursor = 0 + let totalUncompressed = 0 + for (let index = 0; index < entriesTotal; index += 1) { + if (central.readUInt32LE(cursor) !== 0x02014b50) throw new Error(`invalid zip central directory in ${path.basename(zipPath)}`) + const flags = central.readUInt16LE(cursor + 8) + const method = central.readUInt16LE(cursor + 10) + const compressedSize32 = central.readUInt32LE(cursor + 20) + const uncompressedSize32 = central.readUInt32LE(cursor + 24) + const nameLength = central.readUInt16LE(cursor + 28) + const extraLength = central.readUInt16LE(cursor + 30) + const commentLength = central.readUInt16LE(cursor + 32) + const localHeaderOffset32 = central.readUInt32LE(cursor + 42) + const name = central.subarray(cursor + 46, cursor + 46 + nameLength).toString('utf8') + const extra = central.subarray(cursor + 46 + nameLength, cursor + 46 + nameLength + extraLength) + cursor += 46 + nameLength + extraLength + commentLength + if (name.endsWith('/')) continue + const { compressedSize, uncompressedSize, localHeaderOffset } = zip64ExtraValues(extra, { + compressed: compressedSize32, + uncompressed: uncompressedSize32, + localHeaderOffset: localHeaderOffset32, + }) + if (flags & 0x1) throw new Error(`encrypted zip entry is not supported: ${name}`) + if (method !== 0 && method !== 8) throw new Error(`unsupported zip compression method ${method}: ${name}`) + if (uncompressedSize > MAX_ZIP_SINGLE_ENTRY_BYTES) throw new Error(`zip entry is too large: ${name}`) + totalUncompressed += uncompressedSize + if (totalUncompressed > MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES) throw new Error('zip uncompressed size is too large') + const safe = safeZipPath(name) + entries.set(safe, { path: safe, compressedSize, uncompressedSize, method, flags, localHeaderOffset }) + } + return entries +} + +function zipEntryDataOffset(fd: number, entry: ZipEntry, relativePath: string): number { + const local = readAt(fd, entry.localHeaderOffset, 30) + if (local.readUInt32LE(0) !== 0x04034b50) throw new Error(`invalid zip local header: ${relativePath}`) + const nameLength = local.readUInt16LE(26) + const extraLength = local.readUInt16LE(28) + return entry.localHeaderOffset + 30 + nameLength + extraLength +} + +function zipReader(zipPath: string): ExportArchiveReader { + const resolved = path.resolve(zipPath) + const fd = fs.openSync(resolved, 'r') + let closed = false + const entries = parseZipEntries(resolved, fd) + const list = Array.from(entries.values()).map(entry => ({ path: entry.path, size: entry.uncompressedSize })) + .sort((a, b) => a.path.localeCompare(b.path)) + return { + kind: 'zip', + rootPath: resolved, + listEntries: () => list, + has: relativePath => entries.has(toPosixPath(relativePath)), + readBuffer(relativePath: string): Buffer { + const safe = safeZipPath(relativePath) + const entry = entries.get(safe) + if (!entry) throw new Error(`export file not found: ${relativePath}`) + const dataOffset = zipEntryDataOffset(fd, entry, relativePath) + const compressed = readAt(fd, dataOffset, entry.compressedSize) + const data = entry.method === 0 ? compressed : zlib.inflateRawSync(compressed) + if (data.length !== entry.uncompressedSize) throw new Error(`zip entry size mismatch: ${relativePath}`) + return data + }, + createReadStream(relativePath: string): NodeJS.ReadableStream { + const safe = safeZipPath(relativePath) + const entry = entries.get(safe) + if (!entry) throw new Error(`export file not found: ${relativePath}`) + if (entry.compressedSize === 0) return Readable.from([]) + const dataOffset = zipEntryDataOffset(fd, entry, relativePath) + const compressed = fs.createReadStream(resolved, { + start: dataOffset, + end: dataOffset + entry.compressedSize - 1, + }) + return entry.method === 0 ? compressed : compressed.pipe(zlib.createInflateRaw()) + }, + close() { + if (closed) return + closed = true + fs.closeSync(fd) + }, + } +} + +// Opens a provider export located at `sourcePath`, which may be either a `.zip` +// archive or an already-extracted folder. Throws for anything else. +export function openExportReader(sourcePath: string): ExportArchiveReader { + const resolved = path.resolve(sourcePath) + const stat = fs.statSync(resolved) + if (stat.isDirectory()) return folderReader(resolved) + if (!stat.isFile()) throw new Error('export path must be a zip file or extracted folder') + const fd = fs.openSync(resolved, 'r') + try { + const signature = readAt(fd, 0, Math.min(4, stat.size)) + if (signature.length < 4 || signature.readUInt32LE(0) !== 0x04034b50) { + throw new Error('export file must be a .zip file') + } + } finally { + fs.closeSync(fd) + } + return zipReader(resolved) +} + +export function readJsonFromExport(reader: ExportArchiveReader, relativePath: string, fallback: T): T { + if (!reader.has(relativePath)) return fallback + return JSON.parse(reader.readBuffer(relativePath).toString('utf8')) as T +} + +function mediaTypeFromName(name: string): string | null { + const lower = String(name || '').toLowerCase() + if (/\.(mp4|m4v)$/i.test(lower)) return 'video/mp4' + if (/\.(mov|qt)$/i.test(lower)) return 'video/quicktime' + if (/\.webm$/i.test(lower)) return 'video/webm' + if (/\.mp3$/i.test(lower)) return 'audio/mpeg' + if (/\.wav$/i.test(lower)) return 'audio/wav' + if (/\.zip$/i.test(lower)) return 'application/zip' + if (/\.md$/i.test(lower)) return 'text/markdown' + if (/\.csv$/i.test(lower)) return 'text/csv' + if (/\.json$/i.test(lower)) return 'application/json' + if (/\.tex$/i.test(lower)) return 'text/x-tex' + return null +} + +export function sniffMediaType(buffer: Buffer, fallbackName = ''): string { + if (buffer.length >= 8 && buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png' + if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) return 'image/jpeg' + if (buffer.length >= 6 && (buffer.subarray(0, 6).toString('ascii') === 'GIF87a' || buffer.subarray(0, 6).toString('ascii') === 'GIF89a')) return 'image/gif' + if (buffer.length >= 12 && buffer.subarray(0, 4).toString('ascii') === 'RIFF' && buffer.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp' + if (buffer.length >= 5 && buffer.subarray(0, 5).toString('ascii') === '%PDF-') return 'application/pdf' + if (buffer.length >= 12 && buffer.subarray(4, 8).toString('ascii') === 'ftyp') return /\.mov$/i.test(fallbackName) ? 'video/quicktime' : 'video/mp4' + if (buffer.length >= 4 && buffer[0] === 0x1a && buffer[1] === 0x45 && buffer[2] === 0xdf && buffer[3] === 0xa3) return 'video/webm' + const named = mediaTypeFromName(fallbackName) + if (named) return named + const prefix = buffer.subarray(0, Math.min(buffer.length, 512)).toString('utf8').trimStart().toLowerCase() + if (prefix.startsWith(' 0) { + let printable = 0 + for (const byte of sample) { + if (byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte < 127) || byte >= 0xc2) printable += 1 + } + if (printable / sample.length > 0.85) return 'text/plain' + } + return 'application/octet-stream' +} + +export async function readStreamSample(stream: NodeJS.ReadableStream, maxBytes: number): Promise { + const chunks: Buffer[] = [] + let bytes = 0 + for await (const rawChunk of stream as AsyncIterable) { + const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) + if (bytes < maxBytes) { + const needed = Math.min(maxBytes - bytes, chunk.length) + chunks.push(chunk.subarray(0, needed)) + bytes += needed + } + if (bytes >= maxBytes) break + } + const destroyable = stream as unknown as { destroy?: () => void } + if (typeof destroyable.destroy === 'function') { + destroyable.destroy() + } + return Buffer.concat(chunks) +} diff --git a/src/session-titles.ts b/src/session-titles.ts index 62824cd..1317b12 100644 --- a/src/session-titles.ts +++ b/src/session-titles.ts @@ -176,7 +176,7 @@ export function titleForSession(session: Session): string | undefined { if (session.source === 'claude-app') { return claudeAppSessionTitle(session.originalPath) || cleanTitle(session.title) } - if (session.source === 'chatgpt-export') { + if (session.source === 'chatgpt-export' || session.source === 'claude-export') { return cleanTitle(session.title) || cleanTitle(session.cwd) } return cleanTitle(session.title) diff --git a/src/transfer-import.ts b/src/transfer-import.ts index dccdace..9e383d1 100644 --- a/src/transfer-import.ts +++ b/src/transfer-import.ts @@ -1085,6 +1085,8 @@ function isTransferTransientPath(relativePath: string): boolean { // restored backup must not inherit the source machine's "imported" banner. // (The dedup ledger chatgpt-export-imports.json is intentionally left alone.) || normalized === 'state/chatgpt-export-import-job.json' + // Same rationale for the Claude export import-job display state. + || normalized === 'state/claude-export-import-job.json' // UI preferences (language/theme/export format) follow the machine, not the // backup. A vault restored onto an English machine must re-detect the OS // language instead of inheriting the source machine's language choice. @@ -1232,6 +1234,8 @@ function cleanMachineBoundTransferredState(root: string): void { 'state/transfer-replace-journal.json', // Stale chatgpt-export import-progress display from a previous machine/vault. 'state/chatgpt-export-import-job.json', + // Stale claude-export import-progress display from a previous machine/vault. + 'state/claude-export-import-job.json', // Machine-local UI prefs (language/theme) — re-detect OS language on restore. 'state/ui-preferences.json', 'state/bootstrap-capture.json', diff --git a/src/types.ts b/src/types.ts index 42cff8e..ebb7b94 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,5 @@ -export type Source = 'claude-cli' | 'codex-cli' | 'claude-app' | 'openclaw' | 'cursor' | 'chatgpt-export' -export type WatchedSource = Exclude +export type Source = 'claude-cli' | 'codex-cli' | 'claude-app' | 'openclaw' | 'cursor' | 'chatgpt-export' | 'claude-export' +export type WatchedSource = Exclude export interface ConversationBranchSummary { id: string diff --git a/src/ui/index.html b/src/ui/index.html index b4996b9..dfcfd38 100644 --- a/src/ui/index.html +++ b/src/ui/index.html @@ -275,6 +275,9 @@ .filter-btn[data-src="chatgpt-export"] { color: #19c6a7; } + .filter-btn[data-src="claude-export"] { + color: #d88965; + } .filter-btn:hover { background: var(--surface2); transform: none; @@ -1060,6 +1063,7 @@ .source-dot.openclaw { background: var(--openclaw); } .source-dot.cursor { background: var(--cursor); } .source-dot.chatgpt-export { background: var(--chatgpt); } + .source-dot.claude-export { background: var(--claude); } .session-model { font-family: var(--font-mono); @@ -1524,6 +1528,7 @@ .badge.openclaw { color: var(--openclaw); border-color: rgba(162,138,187,0.24); background: rgba(162,138,187,0.09); } .badge.cursor { color: var(--cursor); border-color: rgba(183,154,106,0.24); background: rgba(183,154,106,0.09); } .badge.chatgpt-export { color: var(--chatgpt); border-color: rgba(116,182,165,0.24); background: rgba(116,182,165,0.09); } + .badge.claude-export { color: var(--claude); border-color: rgba(192,140,102,0.24); background: rgba(192,140,102,0.09); } .conversation-shell.split-active .conv-header { position: relative; top: auto; @@ -3456,6 +3461,7 @@ .backup-path-dot.openclaw { background: var(--openclaw); } .backup-path-dot.cursor { background: var(--cursor); } .backup-path-dot.chatgpt-export { background: var(--chatgpt); } + .backup-path-dot.claude-export { background: var(--claude); } .backup-path-items { display: grid; gap: 8px; @@ -3975,6 +3981,7 @@ +
loading…
@@ -4361,6 +4368,40 @@
+
+
+
+
Import claude-export
+
waiting for export
+
+
waiting
+
+
Step 1 checks the Claude export without changing your vault. Step 2 imports conversations, design chats, and raw snapshots.
+
+ +
+ + +
+
No export selected yet. The picker accepts Claude export zip files and extracted export folders.
+ + +
+ + +
+
+ +
+
@@ -4439,6 +4480,10 @@ let chatGptImportBusy = false let chatGptImportPollTimer = null let chatGptImportPreflightTimer = null +let claudeImportData = null +let claudeImportBusy = false +let claudeImportPollTimer = null +let claudeImportPreflightTimer = null let releasesUrl = '' let settingsAppVersion = '' let noticeActionHandler = null @@ -4500,6 +4545,7 @@ 'openclaw': 'OpenClaw', 'cursor': 'Cursor', 'chatgpt-export': 'chatgpt export', + 'claude-export': 'claude export', } const CURRENT_RELEASE_NOTES_VERSION = '2.0.14' @@ -4695,6 +4741,15 @@ chatgptNoExportPaste: 'No export selected yet. Paste the chatgpt-export zip or extracted folder path below.', chatgptManualHelp: 'File picker is unavailable in this browser view, so paste the export path here.', chatgptSelected: 'Selected', + claudeDesignChats: 'design chats', + thoughts: 'thoughts', + toolUses: 'tool uses', + claudeChooseExport: 'Choose a Claude export zip or extracted folder.', + claudeImported: 'The Claude export has been imported into this vault. Conversations, design chats, and raw snapshots are now available.', + claudeChecked: 'Check complete. Nothing has been imported yet. Click Import into Vault to write this Claude export into DataMoat.', + claudeNoExportPicker: 'No export selected yet. Step 1 opens a picker and checks the Claude export before anything is imported.', + claudeNoExportPaste: 'No export selected yet. Paste the Claude export zip or extracted folder path below.', + claudeSelected: 'Selected', chooseDifferentExport: '1. Choose Different Export', checkSelectedExport: 'Check Selected Export', importComplete: 'Imported', @@ -4830,6 +4885,15 @@ chatgptNoExportPaste: '尚未选择 export。请在下面粘贴 chatgpt-export zip 或解压文件夹路径。', chatgptManualHelp: '这个浏览器视图无法使用文件选择器,请在这里粘贴 export 路径。', chatgptSelected: '已选择', + claudeDesignChats: '设计对话', + thoughts: '思考', + toolUses: '工具调用', + claudeChooseExport: '选择 Claude export zip 或解压后的文件夹。', + claudeImported: 'Claude export 已导入这个 vault。对话、设计对话和原始快照现在都可用。', + claudeChecked: '检查完成。现在还没有导入。点击导入到 Vault 才会写入 DataMoat。', + claudeNoExportPicker: '尚未选择 export。第 1 步会打开选择器,并在导入前检查 Claude export。', + claudeNoExportPaste: '尚未选择 export。请在下面粘贴 Claude export zip 或解压文件夹路径。', + claudeSelected: '已选择', chooseDifferentExport: '1. 选择其他 Export', checkSelectedExport: '检查已选 Export', importComplete: '已导入', @@ -4965,6 +5029,15 @@ chatgptNoExportPaste: 'まだ export が選択されていません。chatgpt-export zip または展開済みフォルダのパスを下に貼り付けてください。', chatgptManualHelp: 'このブラウザ表示ではファイル選択が使えないため、export パスをここに貼り付けてください。', chatgptSelected: '選択済み', + claudeDesignChats: 'デザインチャット', + thoughts: '思考', + toolUses: 'ツール使用', + claudeChooseExport: 'Claude export の zip または展開済みフォルダを選択してください。', + claudeImported: 'Claude export はこの vault にインポート済みです。会話、デザインチャット、raw スナップショットが利用できます。', + claudeChecked: '確認完了。まだインポートしていません。Vault にインポートを押すと DataMoat に書き込みます。', + claudeNoExportPicker: 'まだ export が選択されていません。手順 1 で選択画面を開き、インポート前に Claude export を確認します。', + claudeNoExportPaste: 'まだ export が選択されていません。Claude export zip または展開済みフォルダのパスを下に貼り付けてください。', + claudeSelected: '選択済み', chooseDifferentExport: '1. 別の Export を選択', checkSelectedExport: '選択した Export を確認', importComplete: 'インポート済み', @@ -5135,6 +5208,9 @@ chatgptPickerUnavailable: 'File picker is unavailable here. Paste the chatgpt-export zip or folder path.', chatgptPathRequired: 'chatgpt-export zip or folder path required', chatgptImportReview: 'chatgpt-export import finished with items that need review.', + claudePickerUnavailable: 'File picker is unavailable here. Paste the Claude export zip or folder path.', + claudePathRequired: 'Claude export zip or folder path required', + claudeImportReview: 'Claude export import finished with items that need review.', }, 'zh-CN': { manualUpdatePackagedReplace: '下载最新版本、替换这个 app,然后自动重新打开 DataMoat。之后可以加入签名来减少 Windows 警告。', @@ -5263,6 +5339,9 @@ chatgptPickerUnavailable: '这里无法使用文件选择器。请粘贴 chatgpt-export zip 或文件夹路径。', chatgptPathRequired: '需要 chatgpt-export zip 或文件夹路径', chatgptImportReview: 'chatgpt-export 导入完成,但有项目需要检查。', + claudePickerUnavailable: '这里无法使用文件选择器。请粘贴 Claude export zip 或文件夹路径。', + claudePathRequired: '需要 Claude export zip 或文件夹路径', + claudeImportReview: 'Claude export 导入完成,但有项目需要检查。', }, ja: { manualUpdatePackagedReplace: '最新版をダウンロードし、この app を置き換えてから DataMoat を自動で再起動します。署名は後で追加して Windows 警告を減らせます。', @@ -5391,6 +5470,9 @@ chatgptPickerUnavailable: 'ここではファイル選択が使えません。chatgpt-export zip またはフォルダのパスを貼り付けてください。', chatgptPathRequired: 'chatgpt-export zip またはフォルダのパスが必要です', chatgptImportReview: 'chatgpt-export のインポートは完了しましたが、確認が必要な項目があります。', + claudePickerUnavailable: 'ここではファイル選択が使えません。Claude export zip またはフォルダのパスを貼り付けてください。', + claudePathRequired: 'Claude export zip またはフォルダのパスが必要です', + claudeImportReview: 'Claude export のインポートは完了しましたが、確認が必要な項目があります。', }, } for (const [language, values] of Object.entries(appTextSettingsExtra)) { @@ -11197,6 +11279,268 @@ } } +function currentClaudeImportJob() { + const data = claudeImportData + return data?.job || (data?.version ? data : null) +} + +function claudeImportJobIsRunning(job = currentClaudeImportJob()) { + return !!job && !job.done && job.phase !== 'failed' && job.phase !== 'completed' +} + +function claudeImportStatsHtml(counts = {}) { + if (!counts) return '' + const items = [ + [appTr('conversations'), counts.conversations], + [appTr('claudeDesignChats'), counts.designChats], + [appTr('messages'), counts.messages], + [appTr('thoughts'), counts.thoughts], + [appTr('toolUses'), counts.toolUses], + [appTr('assets'), counts.attachments], + ] + return items.map(([label, value]) => ` +
${fmtNumber(value || 0)}${esc(label)}
+ `).join('') +} + +function renderClaudeImportPanel() { + const stateEl = document.getElementById('claude-import-state') + const badgeEl = document.getElementById('claude-import-badge') + const messageEl = document.getElementById('claude-import-message') + const warningEl = document.getElementById('claude-import-warning') + const statsEl = document.getElementById('claude-import-stats') + const progressEl = document.getElementById('claude-import-progress') + const progressLabelEl = document.getElementById('claude-import-progress-label') + const progressCountEl = document.getElementById('claude-import-progress-count') + const progressFillEl = document.getElementById('claude-import-progress-fill') + const selectBtn = document.getElementById('claude-import-select-btn') + const checkBtn = document.getElementById('claude-import-check-btn') + const startBtn = document.getElementById('claude-import-start-btn') + const pathSummaryEl = document.getElementById('claude-import-path-summary') + const manualRowEl = document.getElementById('claude-import-manual-row') + const manualHelpEl = document.getElementById('claude-import-manual-help') + if (!stateEl || !badgeEl || !messageEl || !warningEl || !statsEl || !progressEl || !progressLabelEl || !progressCountEl || !progressFillEl || !selectBtn || !checkBtn || !startBtn || !pathSummaryEl || !manualRowEl || !manualHelpEl) return + + const data = claudeImportData + const job = currentClaudeImportJob() + const status = job?.phase || data?.status || 'waiting for export' + const running = claudeImportBusy || claudeImportJobIsRunning(job) + const cardEl = document.getElementById('claude-import-card') + if (cardEl && job?.phase === 'completed') feelPulseElement(cardEl) + const selectedPath = claudeImportInputPath() + const hasPicker = !!window.datamoatDesktop?.claudeExport?.selectSource + const displayStatus = job?.phase === 'completed' + ? 'imported' + : data?.ok && !job + ? 'checked only' + : status + stateEl.textContent = appStatusText(displayStatus) + badgeEl.textContent = running ? appTr('working') : job?.phase === 'completed' ? appTr('imported') : data?.ok ? appTr('checked') : status === 'failed' ? appTr('failed') : appTr('waiting') + badgeEl.classList.toggle('off', !(data?.ok || job?.phase === 'completed') || status === 'failed') + messageEl.textContent = job?.phase === 'completed' + ? appTr('claudeImported') + : data?.ok + ? appTr('claudeChecked') + : data?.error || data?.errors?.join(' · ') || appTr('claudeChooseExport') + pathSummaryEl.textContent = selectedPath + ? `${appTr('claudeSelected')}: ${selectedPath}` + : hasPicker + ? appTr('claudeNoExportPicker') + : appTr('claudeNoExportPaste') + manualRowEl.hidden = hasPicker + manualHelpEl.hidden = hasPicker + selectBtn.hidden = !hasPicker + + const warnings = job?.phase === 'completed' + ? [] + : Array.isArray(job?.warnings) ? job.warnings : Array.isArray(data?.warnings) ? data.warnings : [] + const jobError = job?.phase === 'failed' ? job?.lastError : '' + if (warnings.length > 0 || data?.error || jobError) { + warningEl.hidden = false + warningEl.textContent = data?.error || jobError || warnings.join(' · ') + } else { + warningEl.hidden = true + warningEl.textContent = '' + } + + const counts = job?.counts || data?.counts + const imported = job?.imported + const updated = job?.updated + const skipped = job?.skipped + statsEl.innerHTML = job + ? [ + `
${fmtNumber(imported?.sessions || 0)}${esc(appTr('newSessions'))}
`, + `
${fmtNumber(updated?.sessions || 0)}${esc(appTr('updated'))}
`, + `
${fmtNumber(skipped?.duplicates || 0)}${esc(appTr('duplicates'))}
`, + `
${fmtNumber(imported?.messages || 0)}${esc(appTr('messages'))}
`, + ].join('') + : claudeImportStatsHtml(counts) + + if (running || job) { + const total = Number(job?.counts?.threads || 0) + const done = Number(job?.imported?.sessions || 0) + Number(job?.updated?.sessions || 0) + Number(job?.skipped?.sessions || 0) + Number(job?.failed?.sessions || 0) + const pct = total > 0 + ? Math.max(12, Math.min(100, Math.round((done / total) * 100))) + : job?.phase === 'completed' ? 100 : 18 + progressEl.hidden = false + progressLabelEl.textContent = appStatusText(job?.phase || 'checking export') + progressCountEl.textContent = total > 0 ? `${fmtNumber(done)} / ${fmtNumber(total)}` : (running ? appTr('working') : '') + progressFillEl.style.width = `${pct}%` + } else { + progressEl.hidden = true + progressFillEl.style.width = '0' + progressCountEl.textContent = '' + } + + const ready = data?.ok || job?.phase === 'completed' + selectBtn.textContent = selectedPath ? appTr('chooseDifferentExport') : appTr('chooseCheckExport') + selectBtn.disabled = claudeImportBusy + checkBtn.hidden = !selectedPath && hasPicker + checkBtn.textContent = data?.ok ? appTr('checkAgain') : appTr('checkSelectedExport') + checkBtn.disabled = claudeImportBusy || !selectedPath + startBtn.textContent = job?.phase === 'completed' ? appTr('importComplete') : appTr('importVault') + startBtn.disabled = claudeImportBusy || !ready || job?.phase === 'completed' +} + +async function loadClaudeImportPanel() { + claudeImportBusy = true + renderClaudeImportPanel() + try { + claudeImportData = await jsonOrThrow(await apiFetch('/api/claude-export/import/status')) + scheduleClaudeImportPollIfRunning() + } catch { + claudeImportData = null + } finally { + claudeImportBusy = false + renderClaudeImportPanel() + } +} + +function scheduleClaudeImportPollIfRunning(delay = 900) { + if (claudeImportPollTimer) { + window.clearTimeout(claudeImportPollTimer) + claudeImportPollTimer = null + } + if (!settingsOpen) return + if (!claudeImportJobIsRunning()) return + claudeImportPollTimer = window.setTimeout(async () => { + claudeImportPollTimer = null + try { + claudeImportData = await jsonOrThrow(await apiFetch('/api/claude-export/import/status')) + renderClaudeImportPanel() + } catch { + // Keep last visible progress. + } + scheduleClaudeImportPollIfRunning() + }, delay) +} + +function claudeImportInputPath() { + return document.getElementById('claude-import-path')?.value.trim() || '' +} + +async function selectClaudeImportSource() { + const selector = window.datamoatDesktop?.claudeExport?.selectSource + if (!selector) { + claudeImportData = { error: appTr('claudePickerUnavailable') } + renderClaudeImportPanel() + return + } + claudeImportBusy = true + renderClaudeImportPanel() + try { + const selected = await selector() + if (!selected || selected.canceled || !selected.path) return + const input = document.getElementById('claude-import-path') + if (input) input.value = selected.path + await preflightClaudeImport() + } catch (error) { + claudeImportData = { error: error instanceof Error ? error.message : String(error) } + } finally { + claudeImportBusy = false + renderClaudeImportPanel() + } +} + +async function preflightClaudeImport() { + const sourcePath = claudeImportInputPath() + if (!sourcePath) { + claudeImportData = { error: appTr('claudePathRequired') } + renderClaudeImportPanel() + return + } + claudeImportBusy = true + renderClaudeImportPanel() + try { + claudeImportData = await jsonOrThrow(await apiFetch('/api/claude-export/import/preflight', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sourcePath }), + })) + } catch (error) { + claudeImportData = { error: error instanceof Error ? error.message : String(error) } + } finally { + claudeImportBusy = false + renderClaudeImportPanel() + } +} + +function scheduleClaudeImportPreflight() { + if (claudeImportPreflightTimer) { + window.clearTimeout(claudeImportPreflightTimer) + claudeImportPreflightTimer = null + } + const sourcePath = claudeImportInputPath() + if (!sourcePath) { + claudeImportData = null + renderClaudeImportPanel() + return + } + claudeImportData = { status: 'checking', warnings: [], counts: null } + renderClaudeImportPanel() + claudeImportPreflightTimer = window.setTimeout(() => { + claudeImportPreflightTimer = null + void preflightClaudeImport() + }, 450) +} + +async function startClaudeImport() { + const sourcePath = claudeImportInputPath() + if (!sourcePath) { + claudeImportData = { error: appTr('claudePathRequired') } + renderClaudeImportPanel() + return + } + claudeImportBusy = true + claudeImportData = { job: { phase: 'reading-export', done: false, counts: {}, imported: {}, updated: {}, skipped: {}, failed: {}, warnings: [] } } + renderClaudeImportPanel() + scheduleClaudeImportPollIfRunning(500) + try { + claudeImportData = await jsonOrThrow(await apiFetch('/api/claude-export/import/start', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sourcePath }), + })) + scheduleClaudeImportPollIfRunning() + await loadSessions() + const completed = claudeImportData?.phase === 'completed' || claudeImportData?.job?.phase === 'completed' + const failedSessions = Number(claudeImportData?.failed?.sessions || claudeImportData?.job?.failed?.sessions || 0) + if (completed && failedSessions === 0) { + setSourceFilter('claude-export') + closeSettings() + markSourceReaction('claude-export') + feelPulseSelector('.session-item.search-result, .session-item[data-session-id]') + } else { + setUpdatePanelMessage(appTr('claudeImportReview'), true) + } + } catch (error) { + claudeImportData = { error: error instanceof Error ? error.message : String(error) } + } finally { + claudeImportBusy = false + renderClaudeImportPanel() + } +} + function sourceRootFromOriginalPath(source, originalPath) { if (!source || !originalPath) return null const normalized = String(originalPath).replace(/\\/g, '/') @@ -11750,6 +12094,7 @@ void loadReferencedAttachmentPanel() void loadTransferPanels() void loadChatGptImportPanel() + void loadClaudeImportPanel() } function openSettingsToChatGptExport() { @@ -12135,6 +12480,17 @@ void preflightChatGptImport() } }) +document.getElementById('claude-import-select-btn').addEventListener('click', () => { void selectClaudeImportSource() }) +document.getElementById('claude-import-check-btn').addEventListener('click', () => { void preflightClaudeImport() }) +document.getElementById('claude-import-start-btn').addEventListener('click', () => { void startClaudeImport() }) +document.getElementById('claude-import-path').addEventListener('input', () => scheduleClaudeImportPreflight()) +document.getElementById('claude-import-path').addEventListener('change', () => { void preflightClaudeImport() }) +document.getElementById('claude-import-path').addEventListener('keydown', event => { + if (event.key === 'Enter') { + event.preventDefault() + void preflightClaudeImport() + } +}) document.getElementById('reinstall-source-path').addEventListener('input', e => { const input = e.target if (!(input instanceof HTMLInputElement)) return diff --git a/src/ui/server.ts b/src/ui/server.ts index 664a32d..d2e43d7 100644 --- a/src/ui/server.ts +++ b/src/ui/server.ts @@ -132,6 +132,11 @@ import { readChatGptBranchMessages, runChatGptExportImport, } from '../chatgpt-export' +import { + currentClaudeImportJob, + preflightClaudeExport, + runClaudeExportImport, +} from '../claude-export' import { cleanupAllSourceArchivePending } from '../source-archive' import { readUiPreferences, @@ -329,7 +334,7 @@ const SEARCH_MAX_RESULTS = 50 const SEARCH_MAX_CONCURRENCY = 14 const SEARCH_MEMORY_CACHE_LIMIT = 16 const SEARCH_MEMORY_CACHE_TTL_MS = 10 * 60 * 1000 -const SEARCH_SOURCE_FILTERS: Source[] = ['claude-cli', 'codex-cli', 'claude-app', 'openclaw', 'cursor', 'chatgpt-export'] +const SEARCH_SOURCE_FILTERS: Source[] = ['claude-cli', 'codex-cli', 'claude-app', 'openclaw', 'cursor', 'chatgpt-export', 'claude-export'] const BACKGROUND_CAPTURE_RETRY_THROTTLE_MS = 30000 const SOURCE_ARCHIVE_PENDING_CLEANUP_DELAY_MS = 15000 const CSRF_COOKIE = 'dm_csrf' @@ -3120,6 +3125,32 @@ export async function startUIServer(): Promise<{ port: number; url: string }> { } }) + app.post('/api/claude-export/import/preflight', requireAuth, async (req, res) => { + const sourcePath = String((req.body?.sourcePath || req.body?.path || '')).trim() + if (!sourcePath) return res.status(400).json({ error: 'Claude export zip or folder path required' }) + try { + res.json(await preflightClaudeExport(sourcePath)) + } catch (error) { + res.status(400).json({ error: error instanceof Error ? error.message : String(error) }) + } + }) + + app.get('/api/claude-export/import/status', requireAuth, (_req, res) => { + res.json(currentClaudeImportJob() ?? { phase: 'idle', done: false }) + }) + + app.post('/api/claude-export/import/start', requireAuth, async (req, res) => { + const sourcePath = String((req.body?.sourcePath || req.body?.path || '')).trim() + if (!sourcePath) return res.status(400).json({ error: 'Claude export zip or folder path required' }) + try { + const job = await runClaudeExportImport(sourcePath) + res.json({ ok: job.phase === 'completed', job }) + } catch (error) { + writeLog('error', 'claude-export-import', 'start_failed', { error }) + res.status(400).json({ error: error instanceof Error ? error.message : String(error), job: currentClaudeImportJob() }) + } + }) + app.get('/api/update/status', requireAuth, (_req, res) => { res.json(loadUpdateState()) }) @@ -3370,6 +3401,7 @@ function fallbackDisplayModel(session: Session): string { if (session.source === 'codex-cli') return 'Codex' if (session.source === 'cursor') return 'Cursor' if (session.source === 'chatgpt-export') return 'ChatGPT' + if (session.source === 'claude-export') return 'Claude' if (session.source === 'openclaw') return 'OpenClaw' return session.source }