Skip to content

Commit 331c01e

Browse files
committed
feat: add photo migration to content-addressed blob storage (Phase 15.4)
- Add scripts/migrate-photos-to-blobs.ts for migrating photos - Photos stored by SHA-256 hash for automatic deduplication - Creates blob records and media records linking to persons - Supports --dry-run and --keep-originals flags - Parses filename patterns to determine source (familysearch, wikipedia, ancestry) Storage structure: data/blobs/{hash[:2]}/{hash}.{ext} Migrated 9 existing photos to new structure.
1 parent 53fd4a9 commit 331c01e

1 file changed

Lines changed: 207 additions & 0 deletions

File tree

scripts/migrate-photos-to-blobs.ts

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
#!/usr/bin/env npx tsx
2+
/**
3+
* Migrate Photos to Content-Addressed Blob Storage
4+
*
5+
* Moves photos from data/photos/ to data/blobs/ using SHA-256 hashing
6+
* for deduplication, and creates blob + media records in SQLite.
7+
*
8+
* Usage:
9+
* npx tsx scripts/migrate-photos-to-blobs.ts [--dry-run] [--keep-originals]
10+
*/
11+
12+
import * as fs from 'fs';
13+
import * as path from 'path';
14+
import { fileURLToPath } from 'url';
15+
import { blobService } from '../server/src/services/blob.service.js';
16+
import { idMappingService } from '../server/src/services/id-mapping.service.js';
17+
import { sqliteService } from '../server/src/db/sqlite.service.js';
18+
19+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
20+
const ROOT_DIR = path.resolve(__dirname, '..');
21+
const PHOTOS_DIR = path.join(ROOT_DIR, 'data/photos');
22+
23+
// Parse command line arguments
24+
const args = process.argv.slice(2);
25+
const dryRun = args.includes('--dry-run');
26+
const keepOriginals = args.includes('--keep-originals');
27+
28+
console.log('Photo Migration to Blob Storage');
29+
console.log('================================');
30+
console.log(`Mode: ${dryRun ? 'DRY RUN' : 'LIVE'}`);
31+
console.log(`Keep originals: ${keepOriginals ? 'YES' : 'NO'}`);
32+
console.log();
33+
34+
// Initialize SQLite
35+
sqliteService.initDb();
36+
37+
// Track statistics
38+
const stats = {
39+
found: 0,
40+
migrated: 0,
41+
skipped: 0,
42+
errors: 0,
43+
duplicates: 0,
44+
totalBytes: 0,
45+
};
46+
47+
/**
48+
* Parse photo filename to extract person ID and source
49+
*
50+
* Filename patterns:
51+
* - {fsId}.jpg -> FamilySearch primary photo
52+
* - {fsId}-wiki.jpg -> Wikipedia photo
53+
* - {fsId}-ancestry.jpg -> Ancestry photo
54+
* - {fsId}-{source}.jpg -> Other source
55+
*
56+
* FamilySearch IDs are typically: XXXX-XXX (4 alphanum, dash, 3 alphanum)
57+
*/
58+
function parseFilename(filename: string): { fsId: string; source: string } | null {
59+
// Match FamilySearch ID pattern (e.g., 9S8X-B4M) optionally followed by -source
60+
const match = filename.match(/^([A-Z0-9]{4}-[A-Z0-9]{3})(?:-([a-z]+))?\.(?:jpg|jpeg|png|gif|webp)$/i);
61+
if (!match) return null;
62+
63+
const fsId = match[1];
64+
const sourceSuffix = match[2];
65+
66+
let source = 'familysearch';
67+
if (sourceSuffix === 'wiki') source = 'wikipedia';
68+
else if (sourceSuffix === 'ancestry') source = 'ancestry';
69+
else if (sourceSuffix === 'wikitree') source = 'wikitree';
70+
else if (sourceSuffix === 'findagrave') source = 'findagrave';
71+
else if (sourceSuffix) source = sourceSuffix;
72+
73+
return { fsId, source };
74+
}
75+
76+
/**
77+
* Get source URL for a photo
78+
*/
79+
function getSourceUrl(fsId: string, source: string): string | undefined {
80+
const urls: Record<string, (id: string) => string> = {
81+
familysearch: (id) => `https://www.familysearch.org/tree/person/details/${id}`,
82+
ancestry: (id) => `https://www.ancestry.com/family-tree/person/${id}`,
83+
wikipedia: () => '', // Would need Wikipedia page URL
84+
wikitree: (id) => `https://www.wikitree.com/wiki/${id}`,
85+
findagrave: (id) => `https://www.findagrave.com/memorial/${id}`,
86+
};
87+
return urls[source]?.(fsId) || undefined;
88+
}
89+
90+
// Main migration
91+
async function migrate() {
92+
// Check if photos directory exists
93+
if (!fs.existsSync(PHOTOS_DIR)) {
94+
console.log('No photos directory found. Nothing to migrate.');
95+
return;
96+
}
97+
98+
// Get all photo files
99+
const files = fs.readdirSync(PHOTOS_DIR).filter((f) => {
100+
const ext = path.extname(f).toLowerCase();
101+
return ['.jpg', '.jpeg', '.png', '.gif', '.webp'].includes(ext);
102+
});
103+
104+
stats.found = files.length;
105+
console.log(`Found ${stats.found} photos to migrate\n`);
106+
107+
if (stats.found === 0) {
108+
console.log('No photos to migrate.');
109+
return;
110+
}
111+
112+
// Process each photo
113+
for (const filename of files) {
114+
const parsed = parseFilename(filename);
115+
if (!parsed) {
116+
console.log(` SKIP: ${filename} (unrecognized filename pattern)`);
117+
stats.skipped++;
118+
continue;
119+
}
120+
121+
const { fsId, source } = parsed;
122+
const filePath = path.join(PHOTOS_DIR, filename);
123+
const fileStats = fs.statSync(filePath);
124+
125+
// Resolve FamilySearch ID to canonical ULID
126+
const canonicalId = idMappingService.getCanonicalId('familysearch', fsId);
127+
if (!canonicalId) {
128+
console.log(` SKIP: ${filename} (no canonical ID for ${fsId})`);
129+
stats.skipped++;
130+
continue;
131+
}
132+
133+
// Check if media already exists for this person+source
134+
const existingMedia = sqliteService.queryOne<{ media_id: string }>(
135+
`SELECT media_id FROM media WHERE person_id = @personId AND source = @source`,
136+
{ personId: canonicalId, source }
137+
);
138+
139+
if (existingMedia) {
140+
console.log(` DUP: ${filename} (already has ${source} photo)`);
141+
stats.duplicates++;
142+
continue;
143+
}
144+
145+
stats.totalBytes += fileStats.size;
146+
147+
if (dryRun) {
148+
console.log(
149+
` WOULD: ${filename} -> ${canonicalId} (${source}, ${(fileStats.size / 1024).toFixed(1)}KB)`
150+
);
151+
stats.migrated++;
152+
continue;
153+
}
154+
155+
// Store in blob storage
156+
const blob = blobService.storeBlobFromFile(filePath);
157+
158+
// Determine if this should be primary (FamilySearch photos are primary by default)
159+
const isPrimary = source === 'familysearch';
160+
161+
// Create media record
162+
const mediaId = blobService.createMedia(canonicalId, blob.hash, source, {
163+
sourceUrl: getSourceUrl(fsId, source),
164+
isPrimary,
165+
});
166+
167+
console.log(
168+
` OK: ${filename} -> ${blob.hash.substring(0, 12)}... (${source}, ${
169+
blob.isNew ? 'new' : 'dup'
170+
})`
171+
);
172+
stats.migrated++;
173+
174+
// Delete original if not keeping
175+
if (!keepOriginals) {
176+
fs.unlinkSync(filePath);
177+
}
178+
}
179+
180+
// Summary
181+
console.log('\n================================');
182+
console.log('Migration Summary');
183+
console.log('================================');
184+
console.log(`Photos found: ${stats.found}`);
185+
console.log(`Photos migrated: ${stats.migrated}`);
186+
console.log(`Photos skipped: ${stats.skipped}`);
187+
console.log(`Duplicates: ${stats.duplicates}`);
188+
console.log(`Errors: ${stats.errors}`);
189+
console.log(`Total size: ${(stats.totalBytes / 1024 / 1024).toFixed(2)} MB`);
190+
191+
if (dryRun) {
192+
console.log('\n(DRY RUN - no changes made)');
193+
} else {
194+
// Show storage stats
195+
const storageStats = blobService.getStorageStats();
196+
console.log('\nBlob Storage Stats:');
197+
console.log(` Blobs: ${storageStats.blobCount}`);
198+
console.log(` Media: ${storageStats.mediaCount}`);
199+
console.log(` Total: ${(storageStats.totalSize / 1024 / 1024).toFixed(2)} MB`);
200+
}
201+
}
202+
203+
migrate()
204+
.catch(console.error)
205+
.finally(() => {
206+
sqliteService.closeDb();
207+
});

0 commit comments

Comments
 (0)