diff --git a/src/extractors/cc_session.ts b/src/extractors/cc_session.ts index 8f7c4cb..d981bbb 100644 --- a/src/extractors/cc_session.ts +++ b/src/extractors/cc_session.ts @@ -20,6 +20,7 @@ import { appendEvents, readSyncState, writeSyncState } from '../lib/events'; import { resolveOrUnknown } from '../lib/identity'; +import { isGhostUserId, canonicalSessionId } from '../lib/ghost_user'; import type { NewEvent } from '../lib/events'; const COLLECTOR_BASE = @@ -414,6 +415,15 @@ export interface SyncSummary { newSessions: number; eventsEmitted: number; unresolvedUsers: string[]; + /** + * Sessions skipped because they were `cc-status`-only fragments under a + * hostname-fallback "ghost" user_id whose session_id is also present + * (with a real transcript) under a non-ghost user_id on the same Matrix-Riven + * machine. See `lib/ghost_user.ts` for the detection rules + rationale. + * Sole-identity ghosts (their session_id has no non-ghost peer) are NOT + * skipped — their data is real and we want it. + */ + redundantGhostsSkipped: number; errors: Array<{ user?: string; date?: string; sessionId?: string; error: string }>; } @@ -425,6 +435,7 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay newSessions: 0, eventsEmitted: 0, unresolvedUsers: [], + redundantGhostsSkipped: 0, errors: [] }; @@ -441,6 +452,40 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay summary.users = users.length; const lookback = opts?.lookbackDays ?? 14; const today = new Date(); + const cutoff = new Date(today.getTime() - lookback * 86_400_000); + const cutoffStr = cutoff.toISOString().slice(0, 10); + + // ---- Pre-pass: index non-ghost (real-email) users' transcript session_ids. + // Used in the main loop to skip "redundant ghost" sessions — cc-status + // fragments under hostname-fallback user_ids whose canonical session_id + // is already owned by a real-email user (which holds the actual transcript). + // Sole-identity ghosts (no non-ghost peer for any of their sessions) pass + // through untouched — see lib/ghost_user.ts. + const nonGhostTranscriptIds = new Set(); + for (const email of users) { + if (isGhostUserId(email)) continue; + let dates: string[]; + try { + dates = await listDates(email); + } catch { + continue; // missing dates listing on a real user is not fatal here + } + for (const d of dates.filter((x) => x >= cutoffStr)) { + let sess: SessionFileRef[]; + try { + sess = await listSessions(email, d); + } catch { + continue; + } + for (const s of sess) { + // Only record entries that are TRANSCRIPTS (not cc-status fragments). + // The collector represents each as basename-minus-extension; cc-status + // entries carry the `.cc-status` suffix on `id`. + if (s.id.endsWith('.cc-status')) continue; + nonGhostTranscriptIds.add(s.id); + } + } + } for (const email of users) { let resolved = await resolveOrUnknown('email', email); @@ -448,6 +493,7 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay const userState = state.users[email] ?? {}; const lastMtime = userState.lastSyncedMtime ?? ''; let highestMtime = lastMtime; + const isGhost = isGhostUserId(email); let dates: string[]; try { @@ -457,8 +503,6 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay continue; } // Filter to lookback window — collector may list very old dates. - const cutoff = new Date(today.getTime() - lookback * 86_400_000); - const cutoffStr = cutoff.toISOString().slice(0, 10); const datesInWindow = dates.filter((d) => d >= cutoffStr).sort(); for (const date of datesInWindow) { @@ -475,6 +519,18 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay } for (const sess of sessions) { if (lastMtime && sess.mtime <= lastMtime) continue; + + // Redundant-ghost filter: this session is under a hostname-fallback + // user_id AND its canonical session_id is already covered by a + // real-email user's transcript on the same Matrix-Riven instance + // (built in the pre-pass above). Skip ingestion to avoid emitting + // duplicate cc.* events under both identities. + if (isGhost && nonGhostTranscriptIds.has(canonicalSessionId(sess.id))) { + summary.redundantGhostsSkipped++; + if (sess.mtime > highestMtime) highestMtime = sess.mtime; + continue; + } + try { const raw = await fetchSessionRaw(email, date, sess.id, sess.ext); const parsed = parseSession(email, date, sess.id, resolved.name, raw); diff --git a/src/lib/ghost_user.ts b/src/lib/ghost_user.ts new file mode 100644 index 0000000..3a68294 --- /dev/null +++ b/src/lib/ghost_user.ts @@ -0,0 +1,106 @@ +/** + * Helpers for detecting / handling "ghost user_id" forms produced by the + * Matrix-Riven uploader when `git config user.email` is unset on a client + * machine. + * + * Matrix-Riven's `getUserId()` (packages/shared/src/identity.ts) tries + * `git config user.email` first and falls back to `${unix_user}@${hostname}` + * (with an optional `.local` mDNS suffix on macOS). Pre-PR #3 on the realtime + * path that fallback ALSO fired even when `~/.riven/digital-twin.json` + * carried a real `identity.user_id` — which produced same-machine pairs + * where the Stop hook tagged transcripts with the real email and the + * realtime hook tagged cc-status snapshots with the hostname form. + * + * Two categories of ghost we see on the collector: + * + * 1. **Redundant ghost** — a hostname-form user_id whose session_ids are + * ALSO present (with full transcripts) under a real-email user_id on + * the same machine. The ghost only carries cc-status fragments; the + * paired real-email user has the bytes. The team extractor should + * skip these to avoid double-counting on the workboard. + * + * 2. **Sole-identity ghost** — a hostname-form user_id that IS the + * person's only upload identity (their `git config user.email` is + * genuinely never set; they're not running `riven digital-twin login` + * either). Their transcripts only exist under this id. Must be + * preserved. + * + * The redundant-vs-sole distinction is made by the extractor at sync time + * by cross-referencing session_ids against the global non-ghost transcript + * index — see `syncCcSessions` in `cc_session.ts`. + */ + +/** + * Email TLDs we accept as "looks like a real email" — empirically observed + * across the team's user list. Add more as needed; rule of thumb is "domain + * has a top-level public suffix" — but a hand-maintained allowlist is more + * predictable than `psl`-style heuristics for this small team. + */ +const REAL_EMAIL_TLDS = [ + '.com', '.cn', '.io', '.org', '.net', '.dev', '.ai', '.co', + '.app', '.me', '.us', '.uk', '.eu', '.gov', '.edu', +]; + +/** + * Hostname suffixes that strongly imply this is NOT a real email — even + * if the suffix happens to contain a dot (Mac mDNS `.local` is the common + * case here). + */ +const HOSTNAME_SUFFIXES = ['.local']; + +/** + * Return true when `userId` matches the `${unix_user}@${hostname}` fallback + * shape from Matrix-Riven's `getUserId()` — i.e. it does NOT look like a + * real email. + * + * isGhostUserId('hrdai@qq.com') → false (real email) + * isGhostUserId('horton2048@users.noreply.github.com') → false + * isGhostUserId('19723@hut') → true (bare hostname) + * isGhostUserId('blink@BlinkdeMacBook-Air.local') → true (mDNS .local) + * isGhostUserId('lv@lvjiawendeMacBook-Air.local') → true + * + * Note: this is a SHAPE test, not a "this user is fake" judgment. A + * sole-identity ghost (someone whose only upload identity is the + * hostname form) STILL matches — the redundant-vs-sole judgment is made + * downstream by the session-id cross-reference. + */ +export function isGhostUserId(userId: string): boolean { + const at = userId.lastIndexOf('@'); + if (at < 0) return false; + const host = userId.slice(at + 1); + if (host.length === 0) return false; + // Explicit hostname suffix → ghost. + for (const suf of HOSTNAME_SUFFIXES) { + if (host.endsWith(suf)) return true; + } + // Real email TLD → real. + const hostLower = host.toLowerCase(); + for (const tld of REAL_EMAIL_TLDS) { + if (hostLower.endsWith(tld)) return false; + } + // No dot at all → bare hostname like `hut` → ghost. + if (!host.includes('.')) return true; + // Has a dot but no known TLD → unknown, lean conservative (treat as real + // so we don't accidentally drop a legitimate user with an exotic TLD). + return false; +} + +/** + * Strip the trailing `.cc-status` suffix that Matrix-Riven appends to + * cc-status-snapshot session entries. The canonical session id is the + * bare UUID/ULID; transcript and cc-status files for the same CC session + * share that id with different suffix: + * + * .jsonl → transcript + * .cc-status.jsonl → cc-status snapshot + * + * The `/api/sessions` listing exposes the basename-minus-extension as + * `id`, so we see two entries with different `id` values for the same + * underlying session. Use this helper to canonicalize. + */ +export function canonicalSessionId(rawId: string): string { + if (rawId.endsWith('.cc-status')) { + return rawId.slice(0, -'.cc-status'.length); + } + return rawId; +} diff --git a/tests/lib/ghost_user.test.ts b/tests/lib/ghost_user.test.ts new file mode 100644 index 0000000..b013c1a --- /dev/null +++ b/tests/lib/ghost_user.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect } from 'vitest'; +import { isGhostUserId, canonicalSessionId } from '../../src/lib/ghost_user'; + +describe('isGhostUserId', () => { + it('returns false for real email TLDs', () => { + expect(isGhostUserId('hrdai@qq.com')).toBe(false); + expect(isGhostUserId('charleyplztrybest@outlook.com')).toBe(false); + expect(isGhostUserId('liboze2026@163.com')).toBe(false); + expect(isGhostUserId('chenjr@nb-ai.com')).toBe(false); + expect(isGhostUserId('witkowskiloeser@gmail.com')).toBe(false); + expect(isGhostUserId('2383145798@qq.com')).toBe(false); + // `users.noreply.github.com` ends in `.com` → real email TLD → not ghost. + expect(isGhostUserId('horton2048@users.noreply.github.com')).toBe(false); + }); + + it('returns true for hostname-fallback shape (bare hostname, no TLD)', () => { + expect(isGhostUserId('19723@hut')).toBe(true); + expect(isGhostUserId('asus@yilinFormula1')).toBe(true); + }); + + it('returns true for .local mDNS hostname suffix', () => { + expect(isGhostUserId('blink@BlinkdeMacBook-Air.local')).toBe(true); + expect(isGhostUserId('lv@lvjiawendeMacBook-Air.local')).toBe(true); + expect(isGhostUserId('zhangziyi@zhangziyideMacBook-Air-2.local')).toBe(true); + expect(isGhostUserId('alexpeng@pengchengdeMacBook-Air.local')).toBe(true); + }); + + it('handles edge cases', () => { + expect(isGhostUserId('')).toBe(false); + expect(isGhostUserId('no-at-sign')).toBe(false); + expect(isGhostUserId('user@')).toBe(false); + }); + + it('leans conservative on unknown TLDs (dot present, not in allowlist → real)', () => { + // An exotic TLD we don't recognise — better to ingest than to drop. + expect(isGhostUserId('person@company.example')).toBe(false); + }); +}); + +describe('canonicalSessionId', () => { + it('strips trailing .cc-status', () => { + expect(canonicalSessionId('8c7b8e3b-ea8d-4482-9d52-c78cc302c35c.cc-status')).toBe( + '8c7b8e3b-ea8d-4482-9d52-c78cc302c35c', + ); + }); + + it('leaves transcript ids unchanged', () => { + expect(canonicalSessionId('8c7b8e3b-ea8d-4482-9d52-c78cc302c35c')).toBe( + '8c7b8e3b-ea8d-4482-9d52-c78cc302c35c', + ); + }); + + it('leaves unknown suffixes alone', () => { + expect(canonicalSessionId('foo.bar')).toBe('foo.bar'); + }); +});