From 2d00ce56734c0450b0e7ed871ad829312e8c701a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 31 Jan 2026 21:03:47 +0000 Subject: [PATCH 01/27] docs: archive changelog for v0.6.33 [skip ci] --- .changelog/{v0.6.x.md => v0.6.33.md} | 54 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) rename .changelog/{v0.6.x.md => v0.6.33.md} (90%) diff --git a/.changelog/v0.6.x.md b/.changelog/v0.6.33.md similarity index 90% rename from .changelog/v0.6.x.md rename to .changelog/v0.6.33.md index 12076e3f..d6ea7b02 100644 --- a/.changelog/v0.6.x.md +++ b/.changelog/v0.6.33.md @@ -1,4 +1,4 @@ -# Release v0.6.x - FamilySearch Normalization & Architecture Refinement +# Release v0.6.33 - FamilySearch Normalization & Architecture Refinement Released: YYYY-MM-DD @@ -8,7 +8,7 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry ## New Features -### Ancestry Update Automation Page (v0.6.x) +### Ancestry Update Automation Page (v0.6.33) - New database-scoped page at `/db/:dbId/ancestry-update` for bulk Ancestry synchronization - BFS traversal of direct-ancestor sparse tree from selected root person - Configurable generation depth (1-4 or full tree) @@ -18,7 +18,7 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - Adds "Ancestry Update" link to database sidebar - API endpoints: GET `/api/ancestry-update/status`, GET `/:dbId/events` (SSE), POST `/:dbId/cancel`, GET `/:dbId/validate/:personId` -### Ancestry Free Hints Automation (v0.6.x) +### Ancestry Free Hints Automation (v0.6.33) - New "Hints" button on Ancestry row in Provider Data table to process free record hints - Playwright automation navigates to hints page, reviews each hint, accepts and saves to tree - Supports checkbox selection for adding related people from hints @@ -26,7 +26,7 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - SSE-based progress events for real-time UI updates - API endpoints: POST/GET `/api/ancestry-hints/:dbId/:personId`, cancel, status -### Ancestry Vital Info Upload (v0.6.x) +### Ancestry Vital Info Upload (v0.6.33) - Extended Ancestry upload to support birth date, birth place, death date, death place, and name (not just photos) - Upload dialog now shows field-by-field comparison between local SparseTree data and cached Ancestry data - Browser automation uses Ancestry's Quick Edit sidebar to update person facts @@ -39,69 +39,69 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - Added `linkFamilySearch()` and `getFamilySearchPhotoPath()` to augmentation service - ID resolution uses alphabetical order (no FamilySearch priority) -### Architecture: Source of Truth Principle (v0.6.x) +### Architecture: Source of Truth Principle (v0.6.33) - SparseTree SQLite is now the canonical source of truth - Provider cache (`data/provider-cache/`) is for comparison only, not a fallback data source - All data reads come from SQLite; provider cache shows what providers have vs what SparseTree has ## Bug Fixes -### SQLite Vital Event Row ID Preservation (v0.6.x) +### SQLite Vital Event Row ID Preservation (v0.6.33) - Changed vital_event INSERT from `INSERT OR REPLACE` to `ON CONFLICT DO UPDATE` - Preserves row IDs which are critical for local_override references - INSERT OR REPLACE was deleting and re-inserting rows, orphaning override references -### Improved Vital Event Date Parsing (v0.6.x) +### Improved Vital Event Date Parsing (v0.6.33) - Now uses actual `birth.date` and `death.date` objects instead of parsing lifespan string - Captures full dates (e.g., "1979-07-31") rather than just years from lifespan ("1979-2020") - Added `parseYearFromDate()` function to handle various date formats including BC notation -### Ancestry Update Log Memory Fix (v0.6.x) +### Ancestry Update Log Memory Fix (v0.6.33) - Limited execution log entries to 500 to prevent browser memory issues on large trees - Shows notice when older entries are truncated -### FamilySearch Upload Improvements (v0.6.x) +### FamilySearch Upload Improvements (v0.6.33) - Auto-refresh FamilySearch cache after successful upload so UI shows updated data - Improved photo comparison to detect already-uploaded photos via symlink matching - Fixed Google login button selector for updated FamilySearch login page - Improved vitals page editing: handles conclusion dialogs and edit buttons - Added autocomplete handling for date/place fields (clicks first suggestion) -### Scraper Auto-Login (v0.6.x) +### Scraper Auto-Login (v0.6.33) - Scraper now attempts auto-login via Google OAuth when landing on FamilySearch login page - Added `handleLoginIfNeeded()` function to detect login pages and complete OAuth flow - Navigates directly to target URL after successful login -### Provider Comparison Not Using Local Overrides (v0.6.x) +### Provider Comparison Not Using Local Overrides (v0.6.33) - Fixed comparison service using original database values instead of user-applied overrides - Added `applyLocalOverrides()` to fetch and apply vital_event overrides before comparison - FamilySearch/Ancestry now correctly show "match" when user has applied their values -### Date Format Differences Showing as Different (v0.6.x) +### Date Format Differences Showing as Different (v0.6.33) - Added date normalization to expand month abbreviations (e.g., "AUG" → "august") - "29 AUG 1933" now correctly matches "29 August 1933" - Reduces false positives in provider comparison diff counts -### Unicode Escape in View Mode Selector (v0.6.x) +### Unicode Escape in View Mode Selector (v0.6.33) - Fixed checkmark showing as literal `\u2713` text instead of ✓ in tree view mode dropdown - JSX text content requires actual Unicode characters, not escape sequences -### Fan Chart Ancestor Count (v0.6.x) +### Fan Chart Ancestor Count (v0.6.33) - Fixed footer showing incorrect count like "109 of 62 ancestors shown" - Now correctly counts only ancestors within the visible generation range -### Stale Browser Connection Detection (v0.6.x) +### Stale Browser Connection Detection (v0.6.33) - Added `verifyAndReconnect()` to browser service that checks both Playwright state and actual browser process - Augmentation service now detects and recovers from stale connections before scraping - Fixes silent failures when browser was closed but connection state was still "connected" -### Auto-Login on Sync (v0.6.x) +### Auto-Login on Sync (v0.6.33) - FamilySearch sync now attempts automatic login when auth token is missing - Uses `providerService.ensureAuthenticated()` to try Google SSO or stored credentials - Eliminates need for manual browser login before sync operations - Applies to both `refreshPerson()` and `fetchPersonDisplayName()` methods -### FamilySearch Scraper Not Extracting Data (v0.6.x) +### FamilySearch Scraper Not Extracting Data (v0.6.33) - Updated scraper selectors to match current FamilySearch DOM structure - Name now extracted using `[data-testid="fullName"]` - Birth/death dates now extracted from `[data-testid="conclusionDisplay:BIRTH"]` and `[data-testid="conclusionDisplay:DEATH"]` containers @@ -111,7 +111,7 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - Added detailed debug logging for photo extraction troubleshooting - Improved UI feedback: loading toast during scrape, success toast shows extracted fields -### "Use" Button Not Updating SparseTree Row (v0.6.x) +### "Use" Button Not Updating SparseTree Row (v0.6.33) - Fixed clicking "Use" on provider field values not updating the SparseTree row display - Root causes fixed: - Client: `onFieldChanged` callback now refetches person data after applying override @@ -119,7 +119,7 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - Added `applyLocalOverridesToPerson()` to database service for consistent override application - Birth/death date/place values from providers now properly display after clicking "Use" -### "Set as Primary" Photo Not Updating (v0.6.x) +### "Set as Primary" Photo Not Updating (v0.6.33) - Fixed photo not updating after clicking "Set as Primary" on provider photos - Root causes fixed: - `getPhotoPath()` now checks for primary photo (`{personId}.jpg`) before provider-suffixed photos @@ -129,27 +129,27 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - Added client API methods: `getFsPhotoUrl()`, `hasFsPhotoProvider()` - Added `photoVersion` state for cache-busting photo URLs after changes -### HMR Context Resilience (v0.6.x) +### HMR Context Resilience (v0.6.33) - Fixed "useSidebar must be used within a SidebarProvider" error during HMR transitions - `useSidebar()` and `useTheme()` hooks now return safe defaults during dev hot reloads - Prevents crashes when React context identity changes during development ## Improvements -### FamilySearch Upload Performance (v0.6.x) +### FamilySearch Upload Performance (v0.6.33) - Eliminated redundant post-login delays in FamilySearch upload automation - After Google OAuth redirect, now checks if already on target page before re-navigating - Replaced fixed `waitForTimeout` delays with element-based `waitForSelector` for responsiveness - Added PM2 timestamps to server logs for better debugging (`time: true` in ecosystem config) -### Page-Level Padding Control (v0.6.x) +### Page-Level Padding Control (v0.6.33) - Moved `p-6` padding from Layout to individual pages for flexibility - Full-height pages (tree views, PersonDetail, SparseTreePage) control their own layout - Standard content pages receive consistent padding via their root element - Fixed missing padding on SparseTreePage and PersonDetail - Removed double padding from AIProviders page (external component handles its own padding) -### Tree Visualization Parity with Ancestry.com (v0.6.x) +### Tree Visualization Parity with Ancestry.com (v0.6.33) - Added 5 tree view modes matching Ancestry.com functionality - **Fan Chart** (default): Radial pedigree with lineage-colored wedges (paternal=blue/teal, maternal=red/coral) - **Horizontal Pedigree**: Root left, ancestors right with expandable nodes @@ -160,7 +160,7 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - New utilities: lineageColors, treeLayout, arcGenerator - Fixed classic view bugs: dynamic centering, ResizeObserver instead of setTimeout -### Vertical Family View Improvements (v0.6.x) +### Vertical Family View Improvements (v0.6.33) - Refactored layout to use multi-pass collision resolution algorithm - Parents properly avoid overlapping when multiple branches are expanded - Added horizontal coupling bars between parent pairs (matching Ancestry.com style) @@ -170,13 +170,13 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry - Fixed expand/collapse button positioning to overlap card tops consistently - Moved spouse connection line from middle of card to name/date area -### Avatar Placeholders (v0.6.x) +### Avatar Placeholders (v0.6.33) - Replaced emoji-based avatar placeholders with proper SVG components throughout tree views - New `AvatarPlaceholder` component renders gender-aware silhouettes (male/female/unknown profiles) - Updated PersonCard, FocusNavigatorView, GenerationalColumnsView, PedigreeChartView, and SparseTreePage - Softened gender color palette for better visual harmony (less saturated blue/pink) -### Legacy Code Removal (v0.6.x) +### Legacy Code Removal (v0.6.33) - Removed all legacy fallback code from services (no more dual-path lookups) - Services updated: `familysearch-refresh`, `multi-platform-comparison`, `augmentation`, `scraper`, `person.routes` - Architecture policy: use migration scripts instead of maintaining legacy paths @@ -213,4 +213,4 @@ This release normalizes FamilySearch as a downstream provider (equal to Ancestry ## Full Changelog -**Full Diff**: https://github.com/atomantic/SparseTree/compare/v0.5.4...v0.6.x +**Full Diff**: https://github.com/atomantic/SparseTree/compare/v0.5.4...v0.6.33 From 828203ae3dd59ac8129449918008d66fa9125e07 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 31 Jan 2026 21:04:01 +0000 Subject: [PATCH 02/27] build: prep v0.7.0 for next release [skip ci] --- client/package.json | 2 +- package-lock.json | 10 +++++----- package.json | 2 +- server/package.json | 2 +- shared/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/package.json b/client/package.json index ec49f197..56822cb0 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "@fsf/client", - "version": "0.6.33", + "version": "0.7.0", "type": "module", "scripts": { "dev": "vite --port 6373", diff --git a/package-lock.json b/package-lock.json index 69379f8a..3707f2f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "sparsetree", - "version": "0.6.33", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sparsetree", - "version": "0.6.33", + "version": "0.7.0", "license": "ISC", "workspaces": [ "shared", @@ -59,7 +59,7 @@ }, "client": { "name": "@fsf/client", - "version": "0.6.33", + "version": "0.7.0", "dependencies": { "@fsf/shared": "*", "d3": "^7.9.0", @@ -8360,7 +8360,7 @@ }, "server": { "name": "@fsf/server", - "version": "0.6.33", + "version": "0.7.0", "dependencies": { "@fsf/shared": "*", "better-sqlite3": "^12.6.2", @@ -8383,7 +8383,7 @@ }, "shared": { "name": "@fsf/shared", - "version": "0.6.33", + "version": "0.7.0", "devDependencies": { "typescript": "^5.7.2" } diff --git a/package.json b/package.json index 75978404..9433c4b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sparsetree", - "version": "0.6.33", + "version": "0.7.0", "private": true, "description": "", "main": "index.js", diff --git a/server/package.json b/server/package.json index 586b8d75..7f28272a 100644 --- a/server/package.json +++ b/server/package.json @@ -1,6 +1,6 @@ { "name": "@fsf/server", - "version": "0.6.33", + "version": "0.7.0", "type": "module", "main": "dist/index.js", "scripts": { diff --git a/shared/package.json b/shared/package.json index 550d6bc5..3974367f 100644 --- a/shared/package.json +++ b/shared/package.json @@ -1,6 +1,6 @@ { "name": "@fsf/shared", - "version": "0.6.33", + "version": "0.7.0", "type": "module", "main": "types/index.js", "types": "types/index.d.ts", From e6926ce4ce9dc95be6af55d49bc5720b31809498 Mon Sep 17 00:00:00 2001 From: Adam Eivy Date: Wed, 4 Feb 2026 14:21:54 -0800 Subject: [PATCH 03/27] feat(ai-discovery): add dismiss functionality and debug endpoints - Add ability to dismiss AI candidates to exclude from future discoveries - Store dismissed candidates in SQLite with reasoning and tags - Batch dismiss, undo dismiss, and clear all dismissed endpoints - Debug endpoints for troubleshooting AI run failures - Fix stdin piping for large prompts to CLI providers - Fix JSON parsing to handle noisy CLI output with banners - Capture error details on failed runs for debugging --- .changelog/v0.7.x.md | 33 +++ client/src/components/ai/AiDiscoveryModal.tsx | 175 +++++++++--- client/src/services/api.ts | 34 ++- .../db/migrations/005_discovery_dismissed.ts | 40 +++ server/src/db/migrations/index.ts | 2 + server/src/db/schema.sql | 14 + server/src/routes/ai-discovery.routes.ts | 119 ++++++++- server/src/services/ai-discovery.service.ts | 248 ++++++++++++++---- 8 files changed, 581 insertions(+), 84 deletions(-) create mode 100644 .changelog/v0.7.x.md create mode 100644 server/src/db/migrations/005_discovery_dismissed.ts diff --git a/.changelog/v0.7.x.md b/.changelog/v0.7.x.md new file mode 100644 index 00000000..9d6c01da --- /dev/null +++ b/.changelog/v0.7.x.md @@ -0,0 +1,33 @@ +# Release v0.7.x - AI Discovery Improvements + +Released: YYYY-MM-DD + +## Overview + +This release improves the AI Discovery feature with dismiss functionality, debugging capabilities, and better CLI integration. + +## New Features + +### AI Discovery Dismiss Functionality (v0.7.0) +- Dismiss AI-discovered candidates to exclude them from future discoveries +- Batch dismiss all unselected candidates at once +- View dismissed candidates with original AI reasoning and tags +- Undo dismiss to restore candidates +- Clear all dismissed candidates for a database +- Dismissed candidates stored in SQLite with `discovery_dismissed` table +- API endpoints: POST `/dismiss`, POST `/dismiss-batch`, GET `/dismissed`, DELETE `/dismissed/:personId`, DELETE `/dismissed` + +### AI Discovery Debug Endpoints (v0.7.0) +- New debug endpoints for troubleshooting AI run failures +- GET `/api/ai-discovery/debug/runs` - list recent AI runs with metadata +- GET `/api/ai-discovery/debug/runs/:runId` - detailed run info including prompt preview and output +- Error details capture (last 10 lines of stderr/stdout) stored on failed runs + +## Bug Fixes + +### CLI Provider Integration (v0.7.0) +- Fixed large prompt handling via stdin piping for CLI providers (codex, claude-code) +- Added `-` flag detection in provider args to enable stdin mode +- Fixed JSON parsing to find LAST valid JSON array in noisy CLI output +- Handles CLI banners and headers that could interfere with response parsing +- Improved error capture with `errorDetails` field for debugging failures diff --git a/client/src/components/ai/AiDiscoveryModal.tsx b/client/src/components/ai/AiDiscoveryModal.tsx index 9e9191c1..dce5ce83 100644 --- a/client/src/components/ai/AiDiscoveryModal.tsx +++ b/client/src/components/ai/AiDiscoveryModal.tsx @@ -1,8 +1,8 @@ -import { useState } from 'react'; -import { X, Sparkles, Check, Loader2, ChevronDown, ChevronUp } from 'lucide-react'; +import { useState, useEffect } from 'react'; +import { X, Sparkles, Check, Loader2, ChevronDown, ChevronUp, ThumbsDown } from 'lucide-react'; import { Link } from 'react-router-dom'; import { api } from '../../services/api'; -import type { DiscoveryResult } from '../../services/api'; +import type { DiscoveryResult, DiscoveryCandidate } from '../../services/api'; import toast from 'react-hot-toast'; interface AiDiscoveryModalProps { @@ -16,17 +16,27 @@ export function AiDiscoveryModal({ dbId, onClose, onComplete }: AiDiscoveryModal const [result, setResult] = useState(null); const [selectedCandidates, setSelectedCandidates] = useState>(new Set()); const [expandedCandidates, setExpandedCandidates] = useState>(new Set()); - const [sampleSize, setSampleSize] = useState(100); + const [dismissedThisSession, setDismissedThisSession] = useState>(new Set()); + const [dismissedCount, setDismissedCount] = useState(0); + const [sampleSize, setSampleSize] = useState(200); const [excludeBiblical, setExcludeBiblical] = useState(true); + const [maxGenerations, setMaxGenerations] = useState(undefined); const [customPrompt, setCustomPrompt] = useState(''); const [applying, setApplying] = useState(false); const [error, setError] = useState(null); + // Load dismissed count on mount + useEffect(() => { + api.getDismissedCandidates(dbId).then(data => { + setDismissedCount(data.count); + }).catch(() => {}); + }, [dbId]); + const startDiscovery = async () => { setStatus('running'); setError(null); - api.quickDiscovery(dbId, sampleSize, { excludeBiblical, customPrompt: customPrompt || undefined }) + api.quickDiscovery(dbId, sampleSize, { excludeBiblical, maxGenerations, customPrompt: customPrompt || undefined }) .then(data => { setResult(data); setStatus('completed'); @@ -92,6 +102,55 @@ export function AiDiscoveryModal({ dbId, onClose, onComplete }: AiDiscoveryModal }); }; + const dismissCandidate = async (candidate: DiscoveryCandidate) => { + api.dismissDiscoveryCandidate(dbId, candidate.personId, candidate.whyInteresting, candidate.suggestedTags) + .then(() => { + setDismissedThisSession(prev => new Set([...prev, candidate.personId])); + setDismissedCount(prev => prev + 1); + // Remove from selected if it was selected + setSelectedCandidates(prev => { + const next = new Set(prev); + next.delete(candidate.personId); + return next; + }); + toast.success(`Dismissed "${candidate.name}"`); + }) + .catch(err => { + toast.error(`Failed to dismiss: ${err.message}`); + }); + }; + + const dismissUnselected = async () => { + if (!result) return; + + const candidatesToDismiss = result.candidates.filter( + c => !selectedCandidates.has(c.personId) && !dismissedThisSession.has(c.personId) + ); + + if (candidatesToDismiss.length === 0) { + toast.error('No candidates to dismiss'); + return; + } + + api.dismissDiscoveryBatch(dbId, candidatesToDismiss.map(c => ({ + personId: c.personId, + whyInteresting: c.whyInteresting, + suggestedTags: c.suggestedTags, + }))) + .then(data => { + const dismissedIds = candidatesToDismiss.map(c => c.personId); + setDismissedThisSession(prev => new Set([...prev, ...dismissedIds])); + setDismissedCount(prev => prev + data.dismissed); + toast.success(`Dismissed ${data.dismissed} candidates`); + }) + .catch(err => { + toast.error(`Failed to dismiss: ${err.message}`); + }); + }; + + // Filter out dismissed candidates from display + const visibleCandidates = result?.candidates.filter(c => !dismissedThisSession.has(c.personId)) || []; + const getConfidenceColor = (confidence: 'high' | 'medium' | 'low') => { switch (confidence) { case 'high': return 'bg-green-500/20 text-green-400 border-green-500/30'; @@ -135,14 +194,14 @@ export function AiDiscoveryModal({ dbId, onClose, onComplete }: AiDiscoveryModal
setSampleSize(parseInt(e.target.value))} className="flex-1" /> - {sampleSize} people + {sampleSize} people

Prioritizes ancestors with biographical information and listed occupations. @@ -160,6 +219,29 @@ export function AiDiscoveryModal({ dbId, onClose, onComplete }: AiDiscoveryModal (born before 500 AD) +

+ +
+ setMaxGenerations(e.target.value ? parseInt(e.target.value) : undefined)} + placeholder="All" + className="w-24 px-3 py-1.5 text-sm border border-app-border rounded-md bg-app-bg text-app-text placeholder-app-text-subtle" + /> + + {maxGenerations ? `Up to ${maxGenerations} generations from root` : 'Search all generations'} + +
+

+ Limit discovery to ancestors within a specific number of generations. +

+
+