Skip to content

Commit 1eb5168

Browse files
atomanticatomantic
committed
feat: normalize FamilySearch as downstream provider (Phase 15.19)
Treat FamilySearch like other providers (Ancestry, WikiTree) instead of as a special "native" source. SparseTree is now the canonical source with all providers as equal downstream data sources. Changes: - Photo storage: FamilySearch now uses -{provider} suffix like others - New: {personId}-familysearch.jpg (consistent with other providers) - Legacy fallback: still checks {personId}.jpg for backwards compat - ID resolution: removed FamilySearch priority in resolveId() - All providers now checked in alphabetical order - Augmentation: added linkFamilySearch(), parseFamilySearchUrl() methods - getFamilySearchPhotoPath(), hasFamilySearchPhoto() helpers - Migration script: scripts/migrate-fs-photos.ts - Renames {id}.jpg -> {id}-familysearch.jpg - Supports --dry-run for preview Files modified: - server/src/services/augmentation.service.ts - server/src/services/id-mapping.service.ts - server/src/services/multi-platform-comparison.service.ts - server/src/services/scraper.service.ts - server/src/routes/person.routes.ts - scripts/migrate-fs-photos.ts (new) - PLAN.md
1 parent 9fea217 commit 1eb5168

7 files changed

Lines changed: 266 additions & 20 deletions

File tree

‎PLAN.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ High-level project roadmap. For detailed phase documentation, see [docs/roadmap.
3434
| 15.14 | Code quality refactoring | 📋 |
3535
| 15.16 | Ancestry photo upload | ✅ |
3636
| 15.17 | Data integrity + bulk discovery | ✅ |
37+
| 15.18 | Separate provider download from auto-apply | ✅ |
38+
| 15.19 | Normalize FamilySearch as downstream provider | ✅ |
3739
| 16 | Multi-platform sync architecture | 📋 |
3840
| 17 | Real-time event system (Socket.IO) | 📋 |
3941

‎scripts/migrate-fs-photos.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/**
2+
* Migration Script: Rename FamilySearch photos to use -familysearch suffix
3+
*
4+
* This script migrates legacy FamilySearch photos from the old naming convention
5+
* ({personId}.jpg) to the new standardized convention ({personId}-familysearch.jpg).
6+
*
7+
* This makes FamilySearch photos consistent with other providers:
8+
* - {personId}-ancestry.jpg
9+
* - {personId}-wikitree.jpg
10+
* - {personId}-wiki.jpg
11+
* - {personId}-familysearch.jpg <- NEW
12+
*
13+
* Run with: npx tsx scripts/migrate-fs-photos.ts [--dry-run]
14+
*/
15+
16+
import fs from 'fs';
17+
import path from 'path';
18+
19+
const DATA_DIR = path.resolve(import.meta.dirname, '../data');
20+
const PHOTOS_DIR = path.join(DATA_DIR, 'photos');
21+
22+
// Suffixes that indicate a photo belongs to a specific provider (not FamilySearch)
23+
const PROVIDER_SUFFIXES = ['-wiki', '-ancestry', '-wikitree', '-linkedin', '-familysearch'];
24+
25+
interface MigrationResult {
26+
total: number;
27+
migrated: number;
28+
skipped: number;
29+
errors: string[];
30+
}
31+
32+
function isLegacyFsPhoto(filename: string): boolean {
33+
// Check if it's a jpg or png
34+
if (!filename.endsWith('.jpg') && !filename.endsWith('.png')) {
35+
return false;
36+
}
37+
38+
// Check if it already has a provider suffix
39+
const baseName = filename.replace(/\.(jpg|png)$/, '');
40+
for (const suffix of PROVIDER_SUFFIXES) {
41+
if (baseName.endsWith(suffix)) {
42+
return false; // Already has a provider suffix
43+
}
44+
}
45+
46+
return true; // No suffix = legacy FamilySearch photo
47+
}
48+
49+
function migratePhotos(dryRun: boolean): MigrationResult {
50+
const result: MigrationResult = {
51+
total: 0,
52+
migrated: 0,
53+
skipped: 0,
54+
errors: [],
55+
};
56+
57+
if (!fs.existsSync(PHOTOS_DIR)) {
58+
console.log('📁 Photos directory does not exist, nothing to migrate');
59+
return result;
60+
}
61+
62+
const files = fs.readdirSync(PHOTOS_DIR);
63+
64+
for (const file of files) {
65+
if (!isLegacyFsPhoto(file)) {
66+
continue;
67+
}
68+
69+
result.total++;
70+
71+
const ext = path.extname(file);
72+
const baseName = file.replace(ext, '');
73+
const newName = `${baseName}-familysearch${ext}`;
74+
75+
const oldPath = path.join(PHOTOS_DIR, file);
76+
const newPath = path.join(PHOTOS_DIR, newName);
77+
78+
// Check if new file already exists
79+
if (fs.existsSync(newPath)) {
80+
console.log(`⏭️ Skipping ${file} - ${newName} already exists`);
81+
result.skipped++;
82+
continue;
83+
}
84+
85+
if (dryRun) {
86+
console.log(`📸 Would rename: ${file} → ${newName}`);
87+
result.migrated++;
88+
} else {
89+
try {
90+
fs.renameSync(oldPath, newPath);
91+
console.log(`✅ Renamed: ${file} → ${newName}`);
92+
result.migrated++;
93+
} catch (err) {
94+
const message = `Failed to rename ${file}: ${(err as Error).message}`;
95+
console.error(`❌ ${message}`);
96+
result.errors.push(message);
97+
}
98+
}
99+
}
100+
101+
return result;
102+
}
103+
104+
function main(): void {
105+
const args = process.argv.slice(2);
106+
const dryRun = args.includes('--dry-run');
107+
108+
console.log('');
109+
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
110+
console.log('📸 FamilySearch Photo Migration');
111+
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
112+
console.log('');
113+
console.log(`Renaming legacy photos from {id}.jpg to {id}-familysearch.jpg`);
114+
console.log(`Photos directory: ${PHOTOS_DIR}`);
115+
console.log('');
116+
117+
if (dryRun) {
118+
console.log('🔍 DRY RUN MODE - no changes will be made');
119+
console.log('');
120+
}
121+
122+
const result = migratePhotos(dryRun);
123+
124+
console.log('');
125+
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
126+
console.log('Summary:');
127+
console.log(` Total legacy photos found: ${result.total}`);
128+
console.log(` ${dryRun ? 'Would migrate' : 'Migrated'}: ${result.migrated}`);
129+
console.log(` Skipped (already exists): ${result.skipped}`);
130+
console.log(` Errors: ${result.errors.length}`);
131+
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
132+
console.log('');
133+
134+
if (dryRun && result.migrated > 0) {
135+
console.log('ℹ️ Run without --dry-run to perform the migration');
136+
console.log('');
137+
}
138+
}
139+
140+
main();

‎server/src/routes/person.routes.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -470,13 +470,14 @@ personRoutes.get('/:dbId/:personId/claims', async (req, res, next) => {
470470
// =============================================================================
471471

472472
/**
473-
* Get the photo suffix for a provider (e.g., '-ancestry', '-wikitree')
473+
* Get the photo suffix for a provider (e.g., '-ancestry', '-wikitree', '-familysearch')
474+
* All providers now use consistent suffixed naming.
474475
*/
475476
function getPhotoSuffix(provider: BuiltInProvider): string {
476477
switch (provider) {
477478
case 'ancestry': return '-ancestry';
478479
case 'wikitree': return '-wikitree';
479-
case 'familysearch': return ''; // FamilySearch photos have no suffix
480+
case 'familysearch': return '-familysearch';
480481
default: return `-${provider}`;
481482
}
482483
}
@@ -524,7 +525,14 @@ personRoutes.post('/:dbId/:personId/use-photo/:provider', async (req, res, next)
524525
const suffix = getPhotoSuffix(provider as BuiltInProvider);
525526
const jpgPath = path.join(PHOTOS_DIR, `${canonical}${suffix}.jpg`);
526527
const pngPath = path.join(PHOTOS_DIR, `${canonical}${suffix}.png`);
527-
const sourcePath = fs.existsSync(jpgPath) ? jpgPath : fs.existsSync(pngPath) ? pngPath : null;
528+
let sourcePath = fs.existsSync(jpgPath) ? jpgPath : fs.existsSync(pngPath) ? pngPath : null;
529+
530+
// Legacy fallback for FamilySearch: check unsuffixed path
531+
if (!sourcePath && provider === 'familysearch') {
532+
const legacyJpgPath = path.join(PHOTOS_DIR, `${canonical}.jpg`);
533+
const legacyPngPath = path.join(PHOTOS_DIR, `${canonical}.png`);
534+
sourcePath = fs.existsSync(legacyJpgPath) ? legacyJpgPath : fs.existsSync(legacyPngPath) ? legacyPngPath : null;
535+
}
528536

529537
if (!sourcePath) {
530538
return res.status(404).json({

‎server/src/services/augmentation.service.ts‎

Lines changed: 95 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -920,11 +920,85 @@ export const augmentationService = {
920920
return this.getLinkedInPhotoPath(personId) !== null;
921921
},
922922

923+
getFamilySearchPhotoPath(personId: string): string | null {
924+
// New standardized path with -familysearch suffix
925+
const jpgPath = path.join(PHOTOS_DIR, `${personId}-familysearch.jpg`);
926+
const pngPath = path.join(PHOTOS_DIR, `${personId}-familysearch.png`);
927+
if (fs.existsSync(jpgPath)) return jpgPath;
928+
if (fs.existsSync(pngPath)) return pngPath;
929+
// Legacy fallback: check for photos without suffix
930+
const legacyJpgPath = path.join(PHOTOS_DIR, `${personId}.jpg`);
931+
const legacyPngPath = path.join(PHOTOS_DIR, `${personId}.png`);
932+
if (fs.existsSync(legacyJpgPath)) return legacyJpgPath;
933+
if (fs.existsSync(legacyPngPath)) return legacyPngPath;
934+
return null;
935+
},
936+
937+
hasFamilySearchPhoto(personId: string): boolean {
938+
return this.getFamilySearchPhotoPath(personId) !== null;
939+
},
940+
923941
parseLinkedInUrl(url: string): string | null {
924942
const match = url.match(/linkedin\.com\/in\/([A-Za-z0-9_-]+)/);
925943
return match ? match[1] : null;
926944
},
927945

946+
/**
947+
* Parse FamilySearch URL to extract the person ID
948+
* Format: https://www.familysearch.org/tree/person/details/XXXX-XXX
949+
*/
950+
parseFamilySearchUrl(url: string): string | null {
951+
const match = url.match(/familysearch\.org\/tree\/person\/(?:details|vitals|sources|memories)\/([A-Z0-9-]+)/i);
952+
return match ? match[1] : null;
953+
},
954+
955+
/**
956+
* Link a FamilySearch profile to a person
957+
* This registers FamilySearch as an augmentation platform like other providers
958+
*/
959+
async linkFamilySearch(personId: string, familySearchUrl: string): Promise<PersonAugmentation> {
960+
logger.start('augment', `Linking FamilySearch for ${personId}: ${familySearchUrl}`);
961+
962+
const familySearchId = this.parseFamilySearchUrl(familySearchUrl);
963+
if (!familySearchId) {
964+
throw new Error('Invalid FamilySearch URL format. Expected: https://www.familysearch.org/tree/person/details/XXXX-XXX');
965+
}
966+
967+
// Get existing augmentation or create new
968+
const existing = this.getAugmentation(personId) || {
969+
id: personId,
970+
platforms: [],
971+
photos: [],
972+
descriptions: [],
973+
updatedAt: new Date().toISOString(),
974+
};
975+
976+
// Add or update FamilySearch platform reference
977+
const existingPlatform = existing.platforms.find(p => p.platform === 'familysearch');
978+
if (existingPlatform) {
979+
existingPlatform.url = familySearchUrl;
980+
existingPlatform.externalId = familySearchId;
981+
existingPlatform.linkedAt = new Date().toISOString();
982+
} else {
983+
existing.platforms.push({
984+
platform: 'familysearch',
985+
url: familySearchUrl,
986+
externalId: familySearchId,
987+
linkedAt: new Date().toISOString(),
988+
});
989+
}
990+
991+
existing.updatedAt = new Date().toISOString();
992+
this.saveAugmentation(existing);
993+
994+
// Also register external identity in SQLite
995+
registerExternalIdentityIfEnabled(personId, 'familysearch', familySearchId, familySearchUrl);
996+
997+
logger.ok('augment', `Linked FamilySearch ${familySearchId} to ${personId}`);
998+
999+
return existing;
1000+
},
1001+
9281002
async scrapeLinkedIn(url: string): Promise<{ headline?: string; company?: string; photoUrl?: string; profileId: string }> {
9291003
const profileId = this.parseLinkedInUrl(url);
9301004
if (!profileId) {
@@ -1235,10 +1309,18 @@ export const augmentationService = {
12351309
}
12361310

12371311
// Check if we already have a photo from this platform locally - skip if we do
1238-
const photoSuffix = platform === 'wikipedia' ? 'wiki' : platform === 'familysearch' ? '' : platform;
1239-
const jpgPath = photoSuffix ? path.join(PHOTOS_DIR, `${personId}-${photoSuffix}.jpg`) : path.join(PHOTOS_DIR, `${personId}.jpg`);
1240-
const pngPath = photoSuffix ? path.join(PHOTOS_DIR, `${personId}-${photoSuffix}.png`) : path.join(PHOTOS_DIR, `${personId}.png`);
1241-
const existingLocalPath = fs.existsSync(jpgPath) ? jpgPath : fs.existsSync(pngPath) ? pngPath : null;
1312+
// All providers now use consistent suffixed naming: -{provider}
1313+
const photoSuffix = platform === 'wikipedia' ? 'wiki' : platform;
1314+
const jpgPath = path.join(PHOTOS_DIR, `${personId}-${photoSuffix}.jpg`);
1315+
const pngPath = path.join(PHOTOS_DIR, `${personId}-${photoSuffix}.png`);
1316+
let existingLocalPath = fs.existsSync(jpgPath) ? jpgPath : fs.existsSync(pngPath) ? pngPath : null;
1317+
1318+
// Legacy fallback for FamilySearch: check unsuffixed photos
1319+
if (!existingLocalPath && platform === 'familysearch') {
1320+
const legacyJpgPath = path.join(PHOTOS_DIR, `${personId}.jpg`);
1321+
const legacyPngPath = path.join(PHOTOS_DIR, `${personId}.png`);
1322+
existingLocalPath = fs.existsSync(legacyJpgPath) ? legacyJpgPath : fs.existsSync(legacyPngPath) ? legacyPngPath : null;
1323+
}
12421324

12431325
if (existingLocalPath) {
12441326
logger.photo('augment', `Photo from ${platform} already exists locally: ${existingLocalPath}`);
@@ -1265,9 +1347,15 @@ export const augmentationService = {
12651347

12661348
// Special case for FamilySearch - use the already-scraped photo
12671349
if (platform === 'familysearch') {
1268-
const fsJpgPath = path.join(PHOTOS_DIR, `${personId}.jpg`);
1269-
const fsPngPath = path.join(PHOTOS_DIR, `${personId}.png`);
1270-
const fsPhotoPath = fs.existsSync(fsJpgPath) ? fsJpgPath : fs.existsSync(fsPngPath) ? fsPngPath : null;
1350+
// Check new suffixed path first, then legacy unsuffixed path
1351+
const fsJpgPath = path.join(PHOTOS_DIR, `${personId}-familysearch.jpg`);
1352+
const fsPngPath = path.join(PHOTOS_DIR, `${personId}-familysearch.png`);
1353+
const legacyJpgPath = path.join(PHOTOS_DIR, `${personId}.jpg`);
1354+
const legacyPngPath = path.join(PHOTOS_DIR, `${personId}.png`);
1355+
const fsPhotoPath = fs.existsSync(fsJpgPath) ? fsJpgPath :
1356+
fs.existsSync(fsPngPath) ? fsPngPath :
1357+
fs.existsSync(legacyJpgPath) ? legacyJpgPath :
1358+
fs.existsSync(legacyPngPath) ? legacyPngPath : null;
12711359

12721360
if (!fsPhotoPath) {
12731361
throw new Error('No FamilySearch photo available for this person');

‎server/src/services/id-mapping.service.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,8 +245,9 @@ function resolveId(id: string, source?: string): string | undefined {
245245
return getCanonicalId(source, id);
246246
}
247247

248-
// Try common sources in order
249-
const sources = ['familysearch', 'ancestry', 'wikitree', 'geni', '23andme'];
248+
// Try common sources in alphabetical order (no provider priority)
249+
// All providers are treated equally - SparseTree is the canonical source
250+
const sources = ['23andme', 'ancestry', 'familysearch', 'geni', 'wikitree'];
250251
for (const s of sources) {
251252
const canonical = getCanonicalId(s, id);
252253
if (canonical) return canonical;

‎server/src/services/multi-platform-comparison.service.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,14 @@ function downloadImage(url: string, destPath: string): Promise<void> {
8181
}
8282

8383
/**
84-
* Get the photo suffix for a provider (e.g., '-ancestry', '-wikitree')
84+
* Get the photo suffix for a provider (e.g., '-ancestry', '-wikitree', '-familysearch')
85+
* All providers now use a consistent suffixed naming convention.
8586
*/
8687
function getPhotoSuffix(provider: BuiltInProvider): string {
8788
switch (provider) {
8889
case 'ancestry': return '-ancestry';
8990
case 'wikitree': return '-wikitree';
90-
case 'familysearch': return ''; // FamilySearch photos have no suffix
91+
case 'familysearch': return '-familysearch';
9192
default: return `-${provider}`;
9293
}
9394
}

‎server/src/services/scraper.service.ts‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,26 +80,32 @@ export const scraperService = {
8080
return this.getPhotoPath(personId) !== null;
8181
},
8282

83-
/** Check if a FamilySearch-specific photo exists (base photo without provider suffix) */
83+
/** Check if a FamilySearch-specific photo exists */
8484
hasFsPhoto(personId: string): boolean {
8585
const extensions = ['.jpg', '.png'];
86-
// Check by canonical ULID
86+
// Check new suffixed path first
87+
for (const ext of extensions) {
88+
if (fs.existsSync(path.join(PHOTOS_DIR, `${personId}-familysearch${ext}`))) return true;
89+
}
90+
// Legacy fallback: check unsuffixed path
8791
for (const ext of extensions) {
8892
if (fs.existsSync(path.join(PHOTOS_DIR, `${personId}${ext}`))) return true;
8993
}
9094
// Check by FamilySearch ID (legacy)
9195
const fsId = idMappingService.getExternalId(personId, 'familysearch');
9296
if (fsId) {
9397
for (const ext of extensions) {
98+
if (fs.existsSync(path.join(PHOTOS_DIR, `${fsId}-familysearch${ext}`))) return true;
9499
if (fs.existsSync(path.join(PHOTOS_DIR, `${fsId}${ext}`))) return true;
95100
}
96101
}
97102
return false;
98103
},
99104

100105
getPhotoPath(personId: string): string | null {
101-
// Photo patterns to check: base, -wiki, -ancestry, -wikitree suffixes
102-
const suffixes = ['', '-wiki', '-ancestry', '-wikitree'];
106+
// Photo patterns to check: all providers use suffixed naming
107+
// Include empty suffix for legacy FamilySearch photos
108+
const suffixes = ['-familysearch', '-wiki', '-ancestry', '-wikitree', ''];
103109
const extensions = ['.jpg', '.png'];
104110

105111
// Check by canonical ULID first
@@ -205,8 +211,8 @@ export const scraperService = {
205211
sendProgress({ phase: 'downloading', message: 'Downloading photo...', personId: canonicalId });
206212

207213
const ext = data.photoUrl.includes('.png') ? 'png' : 'jpg';
208-
// Store photo with canonical ID, not FamilySearch ID
209-
const photoPath = path.join(PHOTOS_DIR, `${canonicalId}.${ext}`);
214+
// Store photo with canonical ID and -familysearch suffix for consistency
215+
const photoPath = path.join(PHOTOS_DIR, `${canonicalId}-familysearch.${ext}`);
210216

211217
await downloadImage(data.photoUrl, photoPath).catch(err => {
212218
logger.error('scraper', `Failed to download photo for ${canonicalId}: ${err.message}`);

0 commit comments

Comments
 (0)