feat(extractor): filter redundant ghost user_ids during cc_session sync

src/extractors/cc_session.ts

@@ -20,6 +20,7 @@
 
 import { appendEvents, readSyncState, writeSyncState } from '../lib/events';
 import { resolveOrUnknown } from '../lib/identity';
+import { isGhostUserId, canonicalSessionId } from '../lib/ghost_user';
 import type { NewEvent } from '../lib/events';
 
 const COLLECTOR_BASE =
@@ -414,6 +415,15 @@ export interface SyncSummary {
   newSessions: number;
   eventsEmitted: number;
   unresolvedUsers: string[];
+  /**
+   * Sessions skipped because they were `cc-status`-only fragments under a
+   * hostname-fallback "ghost" user_id whose session_id is also present
+   * (with a real transcript) under a non-ghost user_id on the same Matrix-Riven
+   * machine. See `lib/ghost_user.ts` for the detection rules + rationale.
+   * Sole-identity ghosts (their session_id has no non-ghost peer) are NOT
+   * skipped — their data is real and we want it.
+   */
+  redundantGhostsSkipped: number;
   errors: Array<{ user?: string; date?: string; sessionId?: string; error: string }>;
 }
 
@@ -425,6 +435,7 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay
     newSessions: 0,
     eventsEmitted: 0,
     unresolvedUsers: [],
+    redundantGhostsSkipped: 0,
     errors: []
   };
 
@@ -441,13 +452,48 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay
   summary.users = users.length;
   const lookback = opts?.lookbackDays ?? 14;
   const today = new Date();
+  const cutoff = new Date(today.getTime() - lookback * 86_400_000);
+  const cutoffStr = cutoff.toISOString().slice(0, 10);
+
+  // ---- Pre-pass: index non-ghost (real-email) users' transcript session_ids.
+  // Used in the main loop to skip "redundant ghost" sessions — cc-status
+  // fragments under hostname-fallback user_ids whose canonical session_id
+  // is already owned by a real-email user (which holds the actual transcript).
+  // Sole-identity ghosts (no non-ghost peer for any of their sessions) pass
+  // through untouched — see lib/ghost_user.ts.
+  const nonGhostTranscriptIds = new Set<string>();
+  for (const email of users) {
+    if (isGhostUserId(email)) continue;
+    let dates: string[];
+    try {
+      dates = await listDates(email);
+    } catch {
+      continue; // missing dates listing on a real user is not fatal here
+    }
+    for (const d of dates.filter((x) => x >= cutoffStr)) {
+      let sess: SessionFileRef[];
+      try {
+        sess = await listSessions(email, d);
+      } catch {
+        continue;
+      }
+      for (const s of sess) {
+        // Only record entries that are TRANSCRIPTS (not cc-status fragments).
+        // The collector represents each as basename-minus-extension; cc-status
+        // entries carry the `.cc-status` suffix on `id`.
+        if (s.id.endsWith('.cc-status')) continue;
+        nonGhostTranscriptIds.add(s.id);
+      }
+    }
+  }
 
   for (const email of users) {
     let resolved = await resolveOrUnknown('email', email);
     if (resolved.unresolved) summary.unresolvedUsers.push(email);
     const userState = state.users[email] ?? {};
     const lastMtime = userState.lastSyncedMtime ?? '';
     let highestMtime = lastMtime;
+    const isGhost = isGhostUserId(email);
 
     let dates: string[];
     try {
@@ -457,8 +503,6 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay
       continue;
     }
     // Filter to lookback window — collector may list very old dates.
-    const cutoff = new Date(today.getTime() - lookback * 86_400_000);
-    const cutoffStr = cutoff.toISOString().slice(0, 10);
     const datesInWindow = dates.filter((d) => d >= cutoffStr).sort();
 
     for (const date of datesInWindow) {
@@ -475,6 +519,18 @@ export async function syncCcSessions(opts?: { limitUsers?: string[]; lookbackDay
       }
       for (const sess of sessions) {
         if (lastMtime && sess.mtime <= lastMtime) continue;
+
+        // Redundant-ghost filter: this session is under a hostname-fallback
+        // user_id AND its canonical session_id is already covered by a
+        // real-email user's transcript on the same Matrix-Riven instance
+        // (built in the pre-pass above). Skip ingestion to avoid emitting
+        // duplicate cc.* events under both identities.
+        if (isGhost && nonGhostTranscriptIds.has(canonicalSessionId(sess.id))) {
+          summary.redundantGhostsSkipped++;
+          if (sess.mtime > highestMtime) highestMtime = sess.mtime;
+          continue;
+        }
+
         try {
           const raw = await fetchSessionRaw(email, date, sess.id, sess.ext);
           const parsed = parseSession(email, date, sess.id, resolved.name, raw);

src/lib/ghost_user.ts

@@ -0,0 +1,106 @@
+/**
+ * Helpers for detecting / handling "ghost user_id" forms produced by the
+ * Matrix-Riven uploader when `git config user.email` is unset on a client
+ * machine.
+ *
+ * Matrix-Riven's `getUserId()` (packages/shared/src/identity.ts) tries
+ * `git config user.email` first and falls back to `${unix_user}@${hostname}`
+ * (with an optional `.local` mDNS suffix on macOS). Pre-PR #3 on the realtime
+ * path that fallback ALSO fired even when `~/.riven/digital-twin.json`
+ * carried a real `identity.user_id` — which produced same-machine pairs
+ * where the Stop hook tagged transcripts with the real email and the
+ * realtime hook tagged cc-status snapshots with the hostname form.
+ *
+ * Two categories of ghost we see on the collector:
+ *
+ *   1. **Redundant ghost** — a hostname-form user_id whose session_ids are
+ *      ALSO present (with full transcripts) under a real-email user_id on
+ *      the same machine. The ghost only carries cc-status fragments; the
+ *      paired real-email user has the bytes. The team extractor should
+ *      skip these to avoid double-counting on the workboard.
+ *
+ *   2. **Sole-identity ghost** — a hostname-form user_id that IS the
+ *      person's only upload identity (their `git config user.email` is
+ *      genuinely never set; they're not running `riven digital-twin login`
+ *      either). Their transcripts only exist under this id. Must be
+ *      preserved.
+ *
+ * The redundant-vs-sole distinction is made by the extractor at sync time
+ * by cross-referencing session_ids against the global non-ghost transcript
+ * index — see `syncCcSessions` in `cc_session.ts`.
+ */
+
+/**
+ * Email TLDs we accept as "looks like a real email" — empirically observed
+ * across the team's user list. Add more as needed; rule of thumb is "domain
+ * has a top-level public suffix" — but a hand-maintained allowlist is more
+ * predictable than `psl`-style heuristics for this small team.
+ */
+const REAL_EMAIL_TLDS = [
+  '.com', '.cn', '.io', '.org', '.net', '.dev', '.ai', '.co',
+  '.app', '.me', '.us', '.uk', '.eu', '.gov', '.edu',
+];
+
+/**
+ * Hostname suffixes that strongly imply this is NOT a real email — even
+ * if the suffix happens to contain a dot (Mac mDNS `.local` is the common
+ * case here).
+ */
+const HOSTNAME_SUFFIXES = ['.local'];
+
+/**
+ * Return true when `userId` matches the `${unix_user}@${hostname}` fallback
+ * shape from Matrix-Riven's `getUserId()` — i.e. it does NOT look like a
+ * real email.
+ *
+ *   isGhostUserId('hrdai@qq.com')                    → false (real email)
+ *   isGhostUserId('horton2048@users.noreply.github.com') → false
+ *   isGhostUserId('19723@hut')                       → true  (bare hostname)
+ *   isGhostUserId('blink@BlinkdeMacBook-Air.local')  → true  (mDNS .local)
+ *   isGhostUserId('lv@lvjiawendeMacBook-Air.local')  → true
+ *
+ * Note: this is a SHAPE test, not a "this user is fake" judgment. A
+ * sole-identity ghost (someone whose only upload identity is the
+ * hostname form) STILL matches — the redundant-vs-sole judgment is made
+ * downstream by the session-id cross-reference.
+ */
+export function isGhostUserId(userId: string): boolean {
+  const at = userId.lastIndexOf('@');
+  if (at < 0) return false;
+  const host = userId.slice(at + 1);
+  if (host.length === 0) return false;
+  // Explicit hostname suffix → ghost.
+  for (const suf of HOSTNAME_SUFFIXES) {
+    if (host.endsWith(suf)) return true;
+  }
+  // Real email TLD → real.
+  const hostLower = host.toLowerCase();
+  for (const tld of REAL_EMAIL_TLDS) {
+    if (hostLower.endsWith(tld)) return false;
+  }
+  // No dot at all → bare hostname like `hut` → ghost.
+  if (!host.includes('.')) return true;
+  // Has a dot but no known TLD → unknown, lean conservative (treat as real
+  // so we don't accidentally drop a legitimate user with an exotic TLD).
+  return false;
+}
+
+/**
+ * Strip the trailing `.cc-status` suffix that Matrix-Riven appends to
+ * cc-status-snapshot session entries. The canonical session id is the
+ * bare UUID/ULID; transcript and cc-status files for the same CC session
+ * share that id with different suffix:
+ *
+ *   <uuid>.jsonl              → transcript
+ *   <uuid>.cc-status.jsonl    → cc-status snapshot
+ *
+ * The `/api/sessions` listing exposes the basename-minus-extension as
+ * `id`, so we see two entries with different `id` values for the same
+ * underlying session. Use this helper to canonicalize.
+ */
+export function canonicalSessionId(rawId: string): string {
+  if (rawId.endsWith('.cc-status')) {
+    return rawId.slice(0, -'.cc-status'.length);
+  }
+  return rawId;
+}

tests/lib/ghost_user.test.ts

@@ -0,0 +1,56 @@
+import { describe, it, expect } from 'vitest';
+import { isGhostUserId, canonicalSessionId } from '../../src/lib/ghost_user';
+
+describe('isGhostUserId', () => {
+  it('returns false for real email TLDs', () => {
+    expect(isGhostUserId('hrdai@qq.com')).toBe(false);
+    expect(isGhostUserId('charleyplztrybest@outlook.com')).toBe(false);
+    expect(isGhostUserId('liboze2026@163.com')).toBe(false);
+    expect(isGhostUserId('chenjr@nb-ai.com')).toBe(false);
+    expect(isGhostUserId('witkowskiloeser@gmail.com')).toBe(false);
+    expect(isGhostUserId('2383145798@qq.com')).toBe(false);
+    // `users.noreply.github.com` ends in `.com` → real email TLD → not ghost.
+    expect(isGhostUserId('horton2048@users.noreply.github.com')).toBe(false);
+  });
+
+  it('returns true for hostname-fallback shape (bare hostname, no TLD)', () => {
+    expect(isGhostUserId('19723@hut')).toBe(true);
+    expect(isGhostUserId('asus@yilinFormula1')).toBe(true);
+  });
+
+  it('returns true for .local mDNS hostname suffix', () => {
+    expect(isGhostUserId('blink@BlinkdeMacBook-Air.local')).toBe(true);
+    expect(isGhostUserId('lv@lvjiawendeMacBook-Air.local')).toBe(true);
+    expect(isGhostUserId('zhangziyi@zhangziyideMacBook-Air-2.local')).toBe(true);
+    expect(isGhostUserId('alexpeng@pengchengdeMacBook-Air.local')).toBe(true);
+  });
+
+  it('handles edge cases', () => {
+    expect(isGhostUserId('')).toBe(false);
+    expect(isGhostUserId('no-at-sign')).toBe(false);
+    expect(isGhostUserId('user@')).toBe(false);
+  });
+
+  it('leans conservative on unknown TLDs (dot present, not in allowlist → real)', () => {
+    // An exotic TLD we don't recognise — better to ingest than to drop.
+    expect(isGhostUserId('person@company.example')).toBe(false);
+  });
+});
+
+describe('canonicalSessionId', () => {
+  it('strips trailing .cc-status', () => {
+    expect(canonicalSessionId('8c7b8e3b-ea8d-4482-9d52-c78cc302c35c.cc-status')).toBe(
+      '8c7b8e3b-ea8d-4482-9d52-c78cc302c35c',
+    );
+  });
+
+  it('leaves transcript ids unchanged', () => {
+    expect(canonicalSessionId('8c7b8e3b-ea8d-4482-9d52-c78cc302c35c')).toBe(
+      '8c7b8e3b-ea8d-4482-9d52-c78cc302c35c',
+    );
+  });
+
+  it('leaves unknown suffixes alone', () => {
+    expect(canonicalSessionId('foo.bar')).toBe('foo.bar');
+  });
+});