From fe351fbc626b301ab8db3371e6ebffa0abbf2799 Mon Sep 17 00:00:00 2001 From: Bryandero98 Date: Thu, 3 Sep 2026 17:57:21 -0500 Subject: [PATCH] fix: count iMessage handle frequency in SQL instead of loading 2,000 full rows suggestTribeImports() pulled up to 2,000 full human_activity_events rows (participants + metadata JSONB, title, summary) across the wire just to count occurrences of metadata.handle in a Node Map. Adds countEventsByHandle(), a GROUP BY aggregate that returns only handle + count, and switches suggestTribeImports() to it. Refs #6026 --- server/services/humanActivity.db.test.js | 28 +++++++++++++++++++++++- server/services/humanActivity.js | 28 ++++++++++++++++++++++++ server/services/tribeContacts.js | 13 ++++++----- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/server/services/humanActivity.db.test.js b/server/services/humanActivity.db.test.js index 9073cb1f11..f6ca6de7f1 100644 --- a/server/services/humanActivity.db.test.js +++ b/server/services/humanActivity.db.test.js @@ -14,7 +14,7 @@ import { describe, it, expect, afterAll } from 'vitest'; import { checkHealth, ensureSchema, close, query, withTransaction } from '../lib/db.js'; import { requireDbOrSkip } from '../lib/dbTestGate.js'; -import { recordEvents, listEvents, getDaySummary, stripParticipantsForAccount } from './humanActivity.js'; +import { recordEvents, listEvents, getDaySummary, stripParticipantsForAccount, countEventsByHandle } from './humanActivity.js'; let dbReady = false; let skipReason = ''; @@ -292,3 +292,29 @@ describe.skipIf(!runDb)('source-scoped read plan (#5715)', () => { expectOrderedIndexScan(plan, 'idx_human_activity_source_kind_happened'); }); }); + +describe.skipIf(!runDb)('countEventsByHandle — SQL-aggregated handle frequency (#6026)', () => { + it('groups and counts events per metadata.handle without loading full rows', async () => { + await recordEvents([ + mk({ dedupeKey: 'handle-a-1', metadata: { handle: '+15551112222' } }), + mk({ dedupeKey: 'handle-a-2', metadata: { handle: '+15551112222' } }), + mk({ dedupeKey: 'handle-a-3', metadata: { handle: '+15551112222' } }), + mk({ dedupeKey: 'handle-b-1', metadata: { handle: '+15553334444' } }), + // No handle in metadata — must not surface as a spurious group. + mk({ dedupeKey: 'handle-none' }), + ]); + const rows = await countEventsByHandle({ source: SOURCE, eventLimit: 2000 }); + const byHandle = new Map(rows.map((r) => [r.handle, r.eventCount])); + expect(byHandle.get('+15551112222')).toBe(3); + expect(byHandle.get('+15553334444')).toBe(1); + expect(byHandle.has('')).toBe(false); + expect(byHandle.has(undefined)).toBe(false); + // Ranked most-frequent first. + expect(rows[0].handle).toBe('+15551112222'); + }); + + it('returns nothing for a source with no matching events', async () => { + const rows = await countEventsByHandle({ source: `${SOURCE}-empty` }); + expect(rows).toEqual([]); + }); +}); diff --git a/server/services/humanActivity.js b/server/services/humanActivity.js index 8b4c966dfd..b1ece68a6f 100644 --- a/server/services/humanActivity.js +++ b/server/services/humanActivity.js @@ -574,6 +574,34 @@ export async function listConversations({ source, q, limit } = {}) { })); } +/** + * Rank `metadata.handle` values by recent event frequency for one source, + * aggregated in SQL rather than loaded into Node. `eventLimit` bounds the + * same recency window a caller would otherwise pass to `listEvents` before + * counting client-side — only the handle column crosses the wire, and + * Postgres does the GROUP BY instead of a per-row Map in Node. + */ +export async function countEventsByHandle({ source, eventLimit } = {}) { + if (!source) return []; + await ensureReady(); + const cap = Math.min(Math.max(Number(eventLimit) || 2000, 1), 5000); + const result = await query( + `SELECT handle, COUNT(*)::int AS event_count + FROM ( + SELECT metadata->>'handle' AS handle + FROM human_activity_events + WHERE source = $1 + ORDER BY happened_at DESC + LIMIT $2 + ) recent + WHERE handle IS NOT NULL AND handle <> '' + GROUP BY handle + ORDER BY event_count DESC`, + [String(source), cap], + ); + return result.rows.map((row) => ({ handle: row.handle, eventCount: Number(row.event_count) || 0 })); +} + /** * Source-level stats for a manager UI: total events, conversation count, date * range. Cheap aggregates over the machine-local activity store. diff --git a/server/services/tribeContacts.js b/server/services/tribeContacts.js index d134f649e6..699478d092 100644 --- a/server/services/tribeContacts.js +++ b/server/services/tribeContacts.js @@ -118,18 +118,19 @@ export async function suggestTribeImports({ limit = 50 } = {}) { const tribeIndex = buildPersonMatchIndex(people); const { contacts, index: contactIndex } = await loadContactIndex(); - // Pull recent-ish imessage events for handle frequency (cap to keep it cheap). - const events = await humanActivity.listEvents({ source: 'imessage', limit: 2000 }); + // Recent-ish imessage handle frequency, counted in SQL (#6026) rather than + // pulling up to 2,000 full event rows across the wire to count in Node. + const handleRows = await humanActivity.countEventsByHandle({ source: 'imessage', eventLimit: 2000 }); const handleCounts = new Map(); - for (const ev of events) { - const h = ev.metadata?.handle; - if (!h) continue; + for (const { handle: h, eventCount } of handleRows) { const key = normalizePhone(h) || normalizeIdentifier(h); if (!key) continue; // Skip handles already in Tribe. const id = identityFromHandle(h); if (matchPerson(id, tribeIndex)) continue; - handleCounts.set(key, (handleCounts.get(key) || 0) + 1); + // Two raw handles (e.g. "+15551234567" vs "5551234567") can normalize to + // the same key — sum rather than let the later one clobber the count. + handleCounts.set(key, (handleCounts.get(key) || 0) + eventCount); } // Also surface contacts with phones/emails not in Tribe (even without iMessage).