|
| 1 | +#!/usr/bin/env npx tsx |
| 2 | +/** |
| 3 | + * Migrate Photos to Content-Addressed Blob Storage |
| 4 | + * |
| 5 | + * Moves photos from data/photos/ to data/blobs/ using SHA-256 hashing |
| 6 | + * for deduplication, and creates blob + media records in SQLite. |
| 7 | + * |
| 8 | + * Usage: |
| 9 | + * npx tsx scripts/migrate-photos-to-blobs.ts [--dry-run] [--keep-originals] |
| 10 | + */ |
| 11 | + |
| 12 | +import * as fs from 'fs'; |
| 13 | +import * as path from 'path'; |
| 14 | +import { fileURLToPath } from 'url'; |
| 15 | +import { blobService } from '../server/src/services/blob.service.js'; |
| 16 | +import { idMappingService } from '../server/src/services/id-mapping.service.js'; |
| 17 | +import { sqliteService } from '../server/src/db/sqlite.service.js'; |
| 18 | + |
| 19 | +const __dirname = path.dirname(fileURLToPath(import.meta.url)); |
| 20 | +const ROOT_DIR = path.resolve(__dirname, '..'); |
| 21 | +const PHOTOS_DIR = path.join(ROOT_DIR, 'data/photos'); |
| 22 | + |
| 23 | +// Parse command line arguments |
| 24 | +const args = process.argv.slice(2); |
| 25 | +const dryRun = args.includes('--dry-run'); |
| 26 | +const keepOriginals = args.includes('--keep-originals'); |
| 27 | + |
| 28 | +console.log('Photo Migration to Blob Storage'); |
| 29 | +console.log('================================'); |
| 30 | +console.log(`Mode: ${dryRun ? 'DRY RUN' : 'LIVE'}`); |
| 31 | +console.log(`Keep originals: ${keepOriginals ? 'YES' : 'NO'}`); |
| 32 | +console.log(); |
| 33 | + |
| 34 | +// Initialize SQLite |
| 35 | +sqliteService.initDb(); |
| 36 | + |
| 37 | +// Track statistics |
| 38 | +const stats = { |
| 39 | + found: 0, |
| 40 | + migrated: 0, |
| 41 | + skipped: 0, |
| 42 | + errors: 0, |
| 43 | + duplicates: 0, |
| 44 | + totalBytes: 0, |
| 45 | +}; |
| 46 | + |
| 47 | +/** |
| 48 | + * Parse photo filename to extract person ID and source |
| 49 | + * |
| 50 | + * Filename patterns: |
| 51 | + * - {fsId}.jpg -> FamilySearch primary photo |
| 52 | + * - {fsId}-wiki.jpg -> Wikipedia photo |
| 53 | + * - {fsId}-ancestry.jpg -> Ancestry photo |
| 54 | + * - {fsId}-{source}.jpg -> Other source |
| 55 | + * |
| 56 | + * FamilySearch IDs are typically: XXXX-XXX (4 alphanum, dash, 3 alphanum) |
| 57 | + */ |
| 58 | +function parseFilename(filename: string): { fsId: string; source: string } | null { |
| 59 | + // Match FamilySearch ID pattern (e.g., 9S8X-B4M) optionally followed by -source |
| 60 | + const match = filename.match(/^([A-Z0-9]{4}-[A-Z0-9]{3})(?:-([a-z]+))?\.(?:jpg|jpeg|png|gif|webp)$/i); |
| 61 | + if (!match) return null; |
| 62 | + |
| 63 | + const fsId = match[1]; |
| 64 | + const sourceSuffix = match[2]; |
| 65 | + |
| 66 | + let source = 'familysearch'; |
| 67 | + if (sourceSuffix === 'wiki') source = 'wikipedia'; |
| 68 | + else if (sourceSuffix === 'ancestry') source = 'ancestry'; |
| 69 | + else if (sourceSuffix === 'wikitree') source = 'wikitree'; |
| 70 | + else if (sourceSuffix === 'findagrave') source = 'findagrave'; |
| 71 | + else if (sourceSuffix) source = sourceSuffix; |
| 72 | + |
| 73 | + return { fsId, source }; |
| 74 | +} |
| 75 | + |
| 76 | +/** |
| 77 | + * Get source URL for a photo |
| 78 | + */ |
| 79 | +function getSourceUrl(fsId: string, source: string): string | undefined { |
| 80 | + const urls: Record<string, (id: string) => string> = { |
| 81 | + familysearch: (id) => `https://www.familysearch.org/tree/person/details/${id}`, |
| 82 | + ancestry: (id) => `https://www.ancestry.com/family-tree/person/${id}`, |
| 83 | + wikipedia: () => '', // Would need Wikipedia page URL |
| 84 | + wikitree: (id) => `https://www.wikitree.com/wiki/${id}`, |
| 85 | + findagrave: (id) => `https://www.findagrave.com/memorial/${id}`, |
| 86 | + }; |
| 87 | + return urls[source]?.(fsId) || undefined; |
| 88 | +} |
| 89 | + |
| 90 | +// Main migration |
| 91 | +async function migrate() { |
| 92 | + // Check if photos directory exists |
| 93 | + if (!fs.existsSync(PHOTOS_DIR)) { |
| 94 | + console.log('No photos directory found. Nothing to migrate.'); |
| 95 | + return; |
| 96 | + } |
| 97 | + |
| 98 | + // Get all photo files |
| 99 | + const files = fs.readdirSync(PHOTOS_DIR).filter((f) => { |
| 100 | + const ext = path.extname(f).toLowerCase(); |
| 101 | + return ['.jpg', '.jpeg', '.png', '.gif', '.webp'].includes(ext); |
| 102 | + }); |
| 103 | + |
| 104 | + stats.found = files.length; |
| 105 | + console.log(`Found ${stats.found} photos to migrate\n`); |
| 106 | + |
| 107 | + if (stats.found === 0) { |
| 108 | + console.log('No photos to migrate.'); |
| 109 | + return; |
| 110 | + } |
| 111 | + |
| 112 | + // Process each photo |
| 113 | + for (const filename of files) { |
| 114 | + const parsed = parseFilename(filename); |
| 115 | + if (!parsed) { |
| 116 | + console.log(` SKIP: ${filename} (unrecognized filename pattern)`); |
| 117 | + stats.skipped++; |
| 118 | + continue; |
| 119 | + } |
| 120 | + |
| 121 | + const { fsId, source } = parsed; |
| 122 | + const filePath = path.join(PHOTOS_DIR, filename); |
| 123 | + const fileStats = fs.statSync(filePath); |
| 124 | + |
| 125 | + // Resolve FamilySearch ID to canonical ULID |
| 126 | + const canonicalId = idMappingService.getCanonicalId('familysearch', fsId); |
| 127 | + if (!canonicalId) { |
| 128 | + console.log(` SKIP: ${filename} (no canonical ID for ${fsId})`); |
| 129 | + stats.skipped++; |
| 130 | + continue; |
| 131 | + } |
| 132 | + |
| 133 | + // Check if media already exists for this person+source |
| 134 | + const existingMedia = sqliteService.queryOne<{ media_id: string }>( |
| 135 | + `SELECT media_id FROM media WHERE person_id = @personId AND source = @source`, |
| 136 | + { personId: canonicalId, source } |
| 137 | + ); |
| 138 | + |
| 139 | + if (existingMedia) { |
| 140 | + console.log(` DUP: ${filename} (already has ${source} photo)`); |
| 141 | + stats.duplicates++; |
| 142 | + continue; |
| 143 | + } |
| 144 | + |
| 145 | + stats.totalBytes += fileStats.size; |
| 146 | + |
| 147 | + if (dryRun) { |
| 148 | + console.log( |
| 149 | + ` WOULD: ${filename} -> ${canonicalId} (${source}, ${(fileStats.size / 1024).toFixed(1)}KB)` |
| 150 | + ); |
| 151 | + stats.migrated++; |
| 152 | + continue; |
| 153 | + } |
| 154 | + |
| 155 | + // Store in blob storage |
| 156 | + const blob = blobService.storeBlobFromFile(filePath); |
| 157 | + |
| 158 | + // Determine if this should be primary (FamilySearch photos are primary by default) |
| 159 | + const isPrimary = source === 'familysearch'; |
| 160 | + |
| 161 | + // Create media record |
| 162 | + const mediaId = blobService.createMedia(canonicalId, blob.hash, source, { |
| 163 | + sourceUrl: getSourceUrl(fsId, source), |
| 164 | + isPrimary, |
| 165 | + }); |
| 166 | + |
| 167 | + console.log( |
| 168 | + ` OK: ${filename} -> ${blob.hash.substring(0, 12)}... (${source}, ${ |
| 169 | + blob.isNew ? 'new' : 'dup' |
| 170 | + })` |
| 171 | + ); |
| 172 | + stats.migrated++; |
| 173 | + |
| 174 | + // Delete original if not keeping |
| 175 | + if (!keepOriginals) { |
| 176 | + fs.unlinkSync(filePath); |
| 177 | + } |
| 178 | + } |
| 179 | + |
| 180 | + // Summary |
| 181 | + console.log('\n================================'); |
| 182 | + console.log('Migration Summary'); |
| 183 | + console.log('================================'); |
| 184 | + console.log(`Photos found: ${stats.found}`); |
| 185 | + console.log(`Photos migrated: ${stats.migrated}`); |
| 186 | + console.log(`Photos skipped: ${stats.skipped}`); |
| 187 | + console.log(`Duplicates: ${stats.duplicates}`); |
| 188 | + console.log(`Errors: ${stats.errors}`); |
| 189 | + console.log(`Total size: ${(stats.totalBytes / 1024 / 1024).toFixed(2)} MB`); |
| 190 | + |
| 191 | + if (dryRun) { |
| 192 | + console.log('\n(DRY RUN - no changes made)'); |
| 193 | + } else { |
| 194 | + // Show storage stats |
| 195 | + const storageStats = blobService.getStorageStats(); |
| 196 | + console.log('\nBlob Storage Stats:'); |
| 197 | + console.log(` Blobs: ${storageStats.blobCount}`); |
| 198 | + console.log(` Media: ${storageStats.mediaCount}`); |
| 199 | + console.log(` Total: ${(storageStats.totalSize / 1024 / 1024).toFixed(2)} MB`); |
| 200 | + } |
| 201 | +} |
| 202 | + |
| 203 | +migrate() |
| 204 | + .catch(console.error) |
| 205 | + .finally(() => { |
| 206 | + sqliteService.closeDb(); |
| 207 | + }); |
0 commit comments