diff --git a/apps/chronicle/server/src/chronicle_server/app.py b/apps/chronicle/server/src/chronicle_server/app.py index a468c7a..69208f8 100644 --- a/apps/chronicle/server/src/chronicle_server/app.py +++ b/apps/chronicle/server/src/chronicle_server/app.py @@ -19,8 +19,10 @@ from chronicle_server.generate import router as generate_router from chronicle_server.health import router as health_router from chronicle_server.interpret import router as interpret_router +from chronicle_server.people import router as people_router from chronicle_server.search import router as search_router from chronicle_server.sources import router as sources_router +from chronicle_server.topics import router as topics_router from chronicle_server.workspaces import router as workspaces_router if TYPE_CHECKING: @@ -79,6 +81,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # generate before events so /events/generate is not captured by /events/{id} app.include_router(generate_router, prefix="/api") app.include_router(events_router, prefix="/api") + # topics before any catch-all; generate path is under the same router + app.include_router(topics_router, prefix="/api") + # people: list/merge-candidates before {id} + app.include_router(people_router, prefix="/api") # Stash settings early so tests can inspect before lifespan if needed. app.state.settings = resolved return app diff --git a/apps/chronicle/server/src/chronicle_server/chronicle.py b/apps/chronicle/server/src/chronicle_server/chronicle.py index 95ee4be..df8bd91 100644 --- a/apps/chronicle/server/src/chronicle_server/chronicle.py +++ b/apps/chronicle/server/src/chronicle_server/chronicle.py @@ -25,12 +25,13 @@ router = APIRouter(tags=["chronicle"]) VALID_UNITS: tuple[str, ...] = ("hour", "day", "week", "month", "quarter", "year") -VALID_LANES = frozenset({"messages", "attachments", "people", "top_people", "events"}) +VALID_LANES = frozenset({"messages", "attachments", "people", "top_people", "events", "topics"}) MAX_BUCKETS = 2000 MIN_PIXEL_WIDTH = 320 MAX_PIXEL_WIDTH = 8192 DENSITY_PIXEL_WIDTH = 1600 TOP_PEOPLE_LIMIT = 8 +TOP_TOPICS_LIMIT = 6 EVENTS_LANE_CAP = 500 # Approximate unit widths for bucket-count estimation (pixel-width rule). @@ -92,6 +93,21 @@ class TopPeopleLaneData(BaseModel): contacts: list[TopPeopleContactSeries] +class TopicSeries(BaseModel): + """Activity bucket series for one topic (member emails per bucket).""" + + topic_id: str + label: str + origin: str + buckets: list[BucketPoint] + + +class TopicsLaneData(BaseModel): + """topics lane: top visible topics by member volume within viewport+scope.""" + + topics: list[TopicSeries] + + class EventLaneMark(BaseModel): """Sparse event diamond mark for the events lane (not a bucket count).""" @@ -142,7 +158,7 @@ def _check_lanes(cls, value: list[str]) -> list[str]: return value -LanePayload = list[BucketPoint] | TopPeopleLaneData | EventsLaneData +LanePayload = list[BucketPoint] | TopPeopleLaneData | EventsLaneData | TopicsLaneData class BucketsResponse(BaseModel): @@ -526,6 +542,97 @@ def _top_people_buckets( return TopPeopleLaneData(contacts=[by_contact[cid] for cid in order]) +def _topics_buckets( + pool: ConnectionPool, + *, + unit: str, + viewport_from: str, + viewport_to: str, + scope_conds: list[str], + scope_params: dict[str, Any], +) -> TopicsLaneData: + """Top 6 visible topics by member volume within viewport+scope. + + One set-based statement: members ⨝ emails ⨝ topics, ranked CTE like + top_people. Hidden topics excluded. Bare emails columns are unambiguous + (app_topic_members / app_topics do not share those names). + """ + email_where = [ + "date IS NOT NULL", + "date >= %(viewport_from)s", + "date < %(viewport_to)s", + *scope_conds, + ] + sql = f""" + WITH filtered AS ( + SELECT m.topic_id, e.date + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE t.hidden = FALSE + AND {" AND ".join(email_where)} + ), + ranked AS ( + SELECT topic_id, count(*)::int AS total + FROM filtered + GROUP BY topic_id + ORDER BY total DESC, topic_id + LIMIT %(top_limit)s + ), + bucketed AS ( + SELECT + f.topic_id, + date_trunc(%(unit)s, f.date) AS bucket, + count(*)::int AS count + FROM filtered f + JOIN ranked r ON r.topic_id = f.topic_id + GROUP BY f.topic_id, 2 + ) + SELECT + r.topic_id::text, + t.label, + t.origin, + b.bucket, + coalesce(b.count, 0)::int AS count, + r.total + FROM ranked r + JOIN app_topics t ON t.id = r.topic_id + LEFT JOIN bucketed b ON b.topic_id = r.topic_id + ORDER BY r.total DESC, r.topic_id, b.bucket + """ + params = { + **scope_params, + "unit": unit, + "viewport_from": viewport_from, + "viewport_to": viewport_to, + "top_limit": TOP_TOPICS_LIMIT, + } + with pool.connection() as conn: + rows = conn.execute(sql, params).fetchall() + + by_topic: dict[str, TopicSeries] = {} + order: list[str] = [] + for row in rows: + topic_id = str(row[0]) + label = str(row[1]) if row[1] is not None else topic_id + origin = str(row[2]) + if topic_id not in by_topic: + by_topic[topic_id] = TopicSeries( + topic_id=topic_id, + label=label, + origin=origin, + buckets=[], + ) + order.append(topic_id) + bucket = _iso_utc(row[3]) + if bucket is None: + continue + by_topic[topic_id].buckets.append( + BucketPoint(bucket=bucket, count=int(row[4])), + ) + return TopicsLaneData(topics=[by_topic[tid] for tid in order]) + + def _extent( pool: ConnectionPool, scope_conds: list[str], @@ -673,6 +780,15 @@ def get_buckets(pool: ConnectionPool, body: BucketsRequest) -> BucketsResponse: scope_conds=scope_conds, scope_params=scope_params, ) + elif lane == "topics": + lane_data[lane] = _topics_buckets( + pool, + unit=unit, + viewport_from=viewport_from_s, + viewport_to=viewport_to_s, + scope_conds=scope_conds, + scope_params=scope_params, + ) elif lane == "events": lane_data[lane] = _events_lane( pool, @@ -783,6 +899,15 @@ def _fetch_lanes_for_viewport( scope_conds=scope_conds, scope_params=scope_params, ) + elif lane == "topics": + lane_data[lane] = _topics_buckets( + pool, + unit=unit, + viewport_from=viewport_from_s, + viewport_to=viewport_to_s, + scope_conds=scope_conds, + scope_params=scope_params, + ) elif lane == "events": lane_data[lane] = _events_lane( pool, diff --git a/apps/chronicle/server/src/chronicle_server/db.py b/apps/chronicle/server/src/chronicle_server/db.py index 1e7c847..b33ea8b 100644 --- a/apps/chronicle/server/src/chronicle_server/db.py +++ b/apps/chronicle/server/src/chronicle_server/db.py @@ -113,6 +113,29 @@ 'direct','supported','conflicting','unresolved')), citations JSONB NOT NULL DEFAULT '[]' ); +CREATE TABLE IF NOT EXISTS app_topics ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + label TEXT NOT NULL, + description TEXT, + origin TEXT NOT NULL DEFAULT 'automatic' + CHECK (origin IN ('automatic','curated','manual')), + parent_id UUID REFERENCES app_topics(id), + hidden BOOLEAN NOT NULL DEFAULT FALSE, + centroid vector(768), + top_terms TEXT[] NOT NULL DEFAULT '{}', + generation INT NOT NULL DEFAULT 1, + member_count INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS app_topic_members ( + topic_id UUID NOT NULL REFERENCES app_topics(id) ON DELETE CASCADE, + email_id UUID NOT NULL, + distance REAL, + origin TEXT NOT NULL DEFAULT 'automatic' CHECK (origin IN ('automatic','manual')), + PRIMARY KEY (topic_id, email_id) +); +CREATE INDEX IF NOT EXISTS app_topic_members_email_idx ON app_topic_members (email_id); """ diff --git a/apps/chronicle/server/src/chronicle_server/files.py b/apps/chronicle/server/src/chronicle_server/files.py index bac1934..460be58 100644 --- a/apps/chronicle/server/src/chronicle_server/files.py +++ b/apps/chronicle/server/src/chronicle_server/files.py @@ -2,6 +2,7 @@ from __future__ import annotations +import difflib import re from datetime import datetime from pathlib import Path @@ -10,7 +11,7 @@ from uuid import UUID import structlog -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import FileResponse, JSONResponse, Response from pydantic import BaseModel, Field, field_validator @@ -30,6 +31,15 @@ _LIST_DEFAULT_LIMIT = 50 _LIST_MAX_LIMIT = 200 _OCCURRENCE_BOUND = 20 +_DIFF_HUNK_CAP = 200 + +# Trailing version/copy tokens stripped from filename stems (Table 25 / FI-004). +_STEM_VERSION_SUFFIX = re.compile( + r"(?:[-_ ]?(?:v?\d+|final|draft|copy|\(\d+\)))+$", + re.IGNORECASE, +) +# Currency / number patterns on changed lines (spec §A.2 amount changes). +_AMOUNT_RE = re.compile(r"[$€£]?\s?\d[\d,.]*") # Content-type family → ILIKE patterns (module constant per task). # "other" is handled as NOT matching any of the named families. @@ -123,6 +133,8 @@ class AttachmentListItem(BaseModel): sha256: str duplicate_count: int occurrences: list[AttachmentOccurrence] | None = None + # Probable version-family size (self included). Cheap flag for UI "N versions". + family_count: int = 1 class AttachmentListResponse(BaseModel): @@ -136,6 +148,137 @@ class PreviewDenied(BaseModel): reason: str +class FamilyCandidate(BaseModel): + id: str + filename: str + date: str | None = None + sender: str | None = None + size: int | None = None + sha256: str + confidence: Literal["exact-duplicate", "probable-version"] + signals: list[str] + + +class AttachmentFamilyResponse(BaseModel): + id: str + stem: str + candidates: list[FamilyCandidate] + + +class DiffLine(BaseModel): + kind: Literal["same", "add", "del"] + text: str + + +class DiffHunk(BaseModel): + a_start: int + b_start: int + lines: list[DiffLine] + + +class AmountChange(BaseModel): + kind: Literal["add", "del"] + text: str + amounts: list[str] + + +class AttachmentMeta(BaseModel): + id: str + filename: str + content_type: str | None = None + size: int | None = None + date: str | None = None + sender: str | None = None + sha256: str + source_message_id: str | None = None + + +class AttachmentCompareResponse(BaseModel): + a: AttachmentMeta + b: AttachmentMeta + hunks: list[DiffHunk] + truncated: bool + amount_changes: list[AmountChange] + + +# --- pure helpers (version families + compare) --- + + +def filename_stem(filename: str) -> str: + """Normalize filename to a version-family stem (Table 25 / FI-004). + + Lowercase, strip extension, strip trailing version/copy tokens: + ``[-_ ]?(v?\\d+|final|draft|copy|(\\d+))+``. + """ + name = (filename or "").lower().strip() + if not name: + return "" + # Strip last extension only when it looks like one (short alphanumeric). + if "." in name: + base, ext = name.rsplit(".", 1) + if base and ext and all(c.isalnum() or c in "+-" for c in ext): + name = base + # Repeatedly strip trailing version tokens (handles stacked suffixes). + while True: + stripped = _STEM_VERSION_SUFFIX.sub("", name) + if stripped == name: + break + name = stripped + return name + + +def diff_lines(a: str, b: str, context: int = 2) -> dict[str, Any]: + """Line-level LCS diff via difflib; return hunks + truncated flag. + + Each hunk: ``{a_start, b_start, lines: [{kind: same|add|del, text}]}``. + Cap at 200 hunks. + """ + a_lines = (a or "").splitlines() + b_lines = (b or "").splitlines() + matcher = difflib.SequenceMatcher(None, a_lines, b_lines, autojunk=False) + hunks: list[dict[str, Any]] = [] + truncated = False + for group in matcher.get_grouped_opcodes(n=max(0, context)): + if len(hunks) >= _DIFF_HUNK_CAP: + truncated = True + break + a_start = group[0][1] + b_start = group[0][3] + lines: list[dict[str, str]] = [] + for tag, i1, i2, j1, j2 in group: + if tag == "equal": + for line in a_lines[i1:i2]: + lines.append({"kind": "same", "text": line}) + elif tag == "delete": + for line in a_lines[i1:i2]: + lines.append({"kind": "del", "text": line}) + elif tag == "insert": + for line in b_lines[j1:j2]: + lines.append({"kind": "add", "text": line}) + elif tag == "replace": + for line in a_lines[i1:i2]: + lines.append({"kind": "del", "text": line}) + for line in b_lines[j1:j2]: + lines.append({"kind": "add", "text": line}) + hunks.append({"a_start": a_start, "b_start": b_start, "lines": lines}) + return {"hunks": hunks, "truncated": truncated} + + +def amount_changes_from_hunks(hunks: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Collect add/del lines that contain currency/number patterns.""" + out: list[dict[str, Any]] = [] + for hunk in hunks: + for line in hunk.get("lines") or []: + kind = line.get("kind") + if kind not in ("add", "del"): + continue + text = line.get("text") or "" + amounts = _AMOUNT_RE.findall(text) + if amounts: + out.append({"kind": kind, "text": text, "amounts": amounts}) + return out + + # --- helpers --- @@ -404,6 +547,8 @@ def list_attachments( shas = list({r[4] for r in page}) occurrences_map = _fetch_occurrences(conn, shas) + family_counts = _family_counts_for_page(conn, page) + items: list[AttachmentListItem] = [] for r in page: ( @@ -440,6 +585,7 @@ def list_attachments( sha256=sha256, duplicate_count=int(duplicate_count), occurrences=(occurrences_map.get(sha256) if body.group_duplicates else None), + family_count=family_counts.get(int(att_id), 1), ) items.append(item) @@ -456,6 +602,277 @@ def list_attachments( ) +def _family_counts_for_page( + conn: Any, + page: list[Any], +) -> dict[int, int]: + """Cheap per-page family_count (self included) via one candidate query.""" + if not page: + return {} + att_ids = [int(r[0]) for r in page] + # Seed contexts: one row per attachment id (prefer newest email). + seeds = conn.execute( + """ + SELECT DISTINCT ON (a.id) + a.id, a.filename, e.sender_address, e.thread_id + FROM attachments a + JOIN email_attachments ea ON ea.attachment_id = a.id + JOIN emails e ON e.id = ea.email_id + WHERE a.id = ANY(%(ids)s) + ORDER BY a.id, e.date DESC NULLS LAST, e.id DESC + """, + {"ids": att_ids}, + ).fetchall() + if not seeds: + return {aid: 1 for aid in att_ids} + + senders = list({r[2] for r in seeds if r[2]}) + threads = list({r[3] for r in seeds if r[3]}) + if not senders and not threads: + return {aid: 1 for aid in att_ids} + + conds: list[str] = [] + params: dict[str, Any] = {} + if senders: + conds.append("e.sender_address = ANY(%(senders)s)") + params["senders"] = senders + if threads: + conds.append("e.thread_id = ANY(%(threads)s)") + params["threads"] = threads + where = " OR ".join(conds) + + cand_rows = conn.execute( + f""" + SELECT DISTINCT ON (a.id) + a.id, a.filename, e.sender_address, e.thread_id + FROM attachments a + JOIN email_attachments ea ON ea.attachment_id = a.id + JOIN emails e ON e.id = ea.email_id + WHERE {where} + ORDER BY a.id, e.date DESC NULLS LAST, e.id DESC + """, + params, + ).fetchall() + + # Precompute stem + (sender, thread) for candidates. + cand_info: list[tuple[int, str, str | None, str | None]] = [ + (int(r[0]), filename_stem(str(r[1] or "")), r[2], r[3]) for r in cand_rows + ] + + counts: dict[int, int] = {} + for seed in seeds: + sid = int(seed[0]) + stem = filename_stem(str(seed[1] or "")) + s_sender = seed[2] + s_thread = seed[3] + n = 0 + for cid, c_stem, c_sender, c_thread in cand_info: + if c_stem != stem or not stem: + continue + same_sender = bool(s_sender and c_sender and s_sender == c_sender) + same_thread = bool(s_thread and c_thread and s_thread == c_thread) + if same_sender or same_thread or cid == sid: + n += 1 + counts[sid] = max(n, 1) + return counts + + +def _decode_att_key(att_sid: str) -> int: + try: + kind, key = decode_source_id(att_sid) + except ValueError as exc: + raise _404() from exc + if kind != "att" or not isinstance(key, int): + raise _404() + return key + + +def get_attachment_family(pool: ConnectionPool, att_sid: str) -> AttachmentFamilyResponse: + """Probable version family for an attachment (stem + same sender/thread).""" + att_key = _decode_att_key(att_sid) + + with pool.connection() as conn: + seed = conn.execute( + """ + SELECT DISTINCT ON (a.id) + a.id, a.filename, a.size, a.sha256, + e.date, e.sender_name, e.sender_address, e.thread_id + FROM attachments a + JOIN email_attachments ea ON ea.attachment_id = a.id + JOIN emails e ON e.id = ea.email_id + WHERE a.id = %(id)s + ORDER BY a.id, e.date DESC NULLS LAST, e.id DESC + """, + {"id": att_key}, + ).fetchone() + if seed is None: + raise _404("attachment not found") + + stem = filename_stem(str(seed[1] or "")) + seed_sender = seed[6] + seed_thread = seed[7] + seed_sha = str(seed[3]) + + conds: list[str] = [] + params: dict[str, Any] = {} + if seed_sender: + conds.append("e.sender_address = %(sender)s") + params["sender"] = seed_sender + if seed_thread: + conds.append("e.thread_id = %(thread)s") + params["thread"] = seed_thread + if not conds: + # No sender/thread anchor — family is just the seed itself. + cand_rows = [seed] + else: + # One statement: candidates sharing sender OR thread (stem filtered in Python). + where = " OR ".join(conds) + cand_rows = conn.execute( + f""" + SELECT DISTINCT ON (a.id) + a.id, a.filename, a.size, a.sha256, + e.date, e.sender_name, e.sender_address, e.thread_id + FROM attachments a + JOIN email_attachments ea ON ea.attachment_id = a.id + JOIN emails e ON e.id = ea.email_id + WHERE {where} + ORDER BY a.id, e.date DESC NULLS LAST, e.id DESC + """, + params, + ).fetchall() + + candidates: list[FamilyCandidate] = [] + for r in cand_rows: + rid = int(r[0]) + c_stem = filename_stem(str(r[1] or "")) + # Empty stem: only the seed itself is a family member. + if not stem: + if rid != att_key: + continue + elif c_stem != stem: + continue + c_sender = r[6] + c_thread = r[7] + c_sha = str(r[3]) + signals: list[str] = ["stem"] if stem else [] + same_sender = bool(seed_sender and c_sender and seed_sender == c_sender) + same_thread = bool(seed_thread and c_thread and seed_thread == c_thread) + if same_sender: + signals.append("sender") + if same_thread: + signals.append("thread") + # Self always included; others need sender or thread signal. + if rid != att_key and not (same_sender or same_thread): + continue + if c_sha == seed_sha: + confidence: Literal["exact-duplicate", "probable-version"] = "exact-duplicate" + if "sha256" not in signals: + signals.append("sha256") + else: + confidence = "probable-version" + sender_label = r[5] or r[6] + candidates.append( + FamilyCandidate( + id=encode_source_id("att", rid), + filename=str(r[1] or ""), + date=_iso(r[4]), + sender=sender_label, + size=r[2], + sha256=c_sha, + confidence=confidence, + signals=signals, + ) + ) + + # Stable order: date asc, then id. + candidates.sort(key=lambda c: (c.date or "", c.id)) + + return AttachmentFamilyResponse( + id=att_sid, + stem=stem, + candidates=candidates, + ) + + +def get_attachment_compare( + pool: ConnectionPool, + a_sid: str, + b_sid: str, +) -> AttachmentCompareResponse: + """Metadata side-by-side + extracted-text line diff + amount changes.""" + a_key = _decode_att_key(a_sid) + b_key = _decode_att_key(b_sid) + + with pool.connection() as conn: + + def load_one(key: int) -> tuple[Any, ...] | None: + return conn.execute( + """ + SELECT DISTINCT ON (a.id) + a.id, a.filename, a.content_type, a.size, a.sha256, + e.date, e.sender_name, e.sender_address, e.id AS email_id, + ac.status AS ext_status, ac.markdown, ac.reason AS ext_reason + FROM attachments a + JOIN email_attachments ea ON ea.attachment_id = a.id + JOIN emails e ON e.id = ea.email_id + LEFT JOIN attachment_contents ac ON ac.attachment_id = a.id + WHERE a.id = %(id)s + ORDER BY a.id, e.date DESC NULLS LAST, e.id DESC + """, + {"id": key}, + ).fetchone() + + row_a = load_one(a_key) + row_b = load_one(b_key) + + if row_a is None: + raise _404(f"attachment not found: {a_sid}") + if row_b is None: + raise _404(f"attachment not found: {b_sid}") + + def meta(row: tuple[Any, ...], sid: str) -> AttachmentMeta: + sender = row[6] or row[7] + email_id = row[8] + return AttachmentMeta( + id=sid, + filename=str(row[1] or ""), + content_type=row[2], + size=row[3], + date=_iso(row[5]), + sender=sender, + sha256=str(row[4]), + source_message_id=encode_source_id("msg", email_id) if email_id else None, + ) + + md_a = row_a[10] + md_b = row_b[10] + status_a = row_a[9] + status_b = row_b[9] + if md_a is None or status_a != "extracted": + reason = row_a[11] or status_a or "no extraction" + raise HTTPException( + status_code=404, + detail=f"missing extraction for {a_sid}: {reason}", + ) + if md_b is None or status_b != "extracted": + reason = row_b[11] or status_b or "no extraction" + raise HTTPException( + status_code=404, + detail=f"missing extraction for {b_sid}: {reason}", + ) + + diff = diff_lines(str(md_a), str(md_b), context=2) + amounts = amount_changes_from_hunks(diff["hunks"]) + + return AttachmentCompareResponse( + a=meta(row_a, a_sid), + b=meta(row_b, b_sid), + hunks=[DiffHunk.model_validate(h) for h in diff["hunks"]], + truncated=bool(diff["truncated"]), + amount_changes=[AmountChange.model_validate(x) for x in amounts], + ) + + def _load_attachment_row(pool: ConnectionPool, att_key: int) -> tuple[str, str | None, str] | None: """Return (storage_path, content_type, filename) or None.""" with pool.connection() as conn: @@ -507,6 +924,29 @@ def post_attachments_list( return list_attachments(pool, body, secret_key) +@router.get("/attachments/compare", response_model=AttachmentCompareResponse) +def get_attachments_compare( + request: Request, + a: str = Query(..., description="Attachment source id (att_…)"), + b: str = Query(..., description="Attachment source id (att_…)"), + _user: str = Depends(require_user), +) -> AttachmentCompareResponse: + """Side-by-side metadata + extracted-text diff (version-family compare).""" + pool: ConnectionPool = request.app.state.pool + return get_attachment_compare(pool, a, b) + + +@router.get("/attachments/{att_sid}/family", response_model=AttachmentFamilyResponse) +def get_attachments_family( + att_sid: str, + request: Request, + _user: str = Depends(require_user), +) -> AttachmentFamilyResponse: + """Probable version family for an attachment (Table 25 / FI-004).""" + pool: ConnectionPool = request.app.state.pool + return get_attachment_family(pool, att_sid) + + @router.get("/attachments/{att_sid}/preview", response_model=None) def get_attachment_preview( att_sid: str, diff --git a/apps/chronicle/server/src/chronicle_server/health.py b/apps/chronicle/server/src/chronicle_server/health.py index 834bd69..044dd27 100644 --- a/apps/chronicle/server/src/chronicle_server/health.py +++ b/apps/chronicle/server/src/chronicle_server/health.py @@ -112,9 +112,35 @@ def get_archive_health(pool: ConnectionPool) -> dict[str, Any]: }, ).fetchall() + # Topics coverage (tables may be empty pre-generation). + try: + topics_count_row = conn.execute("SELECT count(*)::int FROM app_topics").fetchone() + assigned_row = conn.execute( + "SELECT count(DISTINCT email_id)::int FROM app_topic_members" + ).fetchone() + last_gen_row = conn.execute( + """ + SELECT at FROM app_audit + WHERE action = 'topics_generate' + ORDER BY at DESC + LIMIT 1 + """ + ).fetchone() + topics_n = int(topics_count_row[0]) if topics_count_row else 0 + assigned_n = int(assigned_row[0]) if assigned_row else 0 + last_generated = _iso(last_gen_row[0]) if last_gen_row else None + except Exception: + # Tables not yet present (pre-migration) — report zeros. + topics_n = 0 + assigned_n = 0 + last_generated = None + assert threading_row is not None assert chunk_emb_row is not None + embedded_n = int(summary["embedding"]["embedded"]) + coverage_ratio = (assigned_n / embedded_n) if embedded_n > 0 else 0.0 + db = MailDB._from_pool(pool) import_records = db.import_history(limit=20) imports = [ @@ -173,6 +199,11 @@ def get_archive_health(pool: ConnectionPool) -> dict[str, Any]: "missing": chunk_emb_row[1], }, }, + "topics": { + "topics": topics_n, + "coverage": coverage_ratio, + "last_generated": last_generated, + }, "imports": imports, "audit_tail": audit_tail, "generated_at": datetime.now(UTC).isoformat(), diff --git a/apps/chronicle/server/src/chronicle_server/people.py b/apps/chronicle/server/src/chronicle_server/people.py new file mode 100644 index 0000000..192328d --- /dev/null +++ b/apps/chronicle/server/src/chronicle_server/people.py @@ -0,0 +1,690 @@ +"""People & Organizations: thin authenticated proxies over MailDB contacts. + +Identity resolution, archive span, activity, topics, and merge/unmerge curation +all flow through the existing contacts subsystem — no duplicated resolution logic. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, Literal +from uuid import UUID + +import structlog +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from pydantic import BaseModel, Field + +from chronicle_server.auth import require_user +from chronicle_server.cursor import decode_cursor, encode_cursor +from chronicle_server.db import audit +from chronicle_server.ids import encode_source_id + +if TYPE_CHECKING: + from psycopg_pool import ConnectionPool + +logger = structlog.get_logger() + +router = APIRouter(tags=["people"]) + +_LIST_DEFAULT_LIMIT = 50 +_LIST_MAX_LIMIT = 500 +_CANDIDATES_DEFAULT_LIMIT = 20 +_CANDIDATES_MAX_LIMIT = 100 +_TOP_TOPICS = 5 +_GRAPH_DEFAULT_MAX_NODES = 25 +_GRAPH_HARD_MAX_NODES = 50 +_GRAPH_EVIDENCE_THREAD_CAP = 20 + +ContactKind = Literal["human", "organization", "automated", "mailing_list", "unknown"] +VALID_KINDS: frozenset[str] = frozenset( + {"human", "organization", "automated", "mailing_list", "unknown"} +) + + +# --- request models --- + + +class ContactPatch(BaseModel): + kind: ContactKind | None = None + tags: list[str] | None = None + notes: str | None = None + display_name: str | None = None + + +class MergeRequest(BaseModel): + source_id: str = Field(min_length=1) + target_id: str = Field(min_length=1) + + +class UnmergeRequest(BaseModel): + merge_id: str = Field(min_length=1) + + +# --- helpers --- + + +def _iso(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + if hasattr(value, "isoformat"): + return str(value.isoformat()) + return str(value) + + +def _serialize_contact(card: dict[str, Any]) -> dict[str, Any]: + """Normalize a MailDB contact card for JSON (ISO datetimes, stable ids).""" + out = dict(card) + for key in ("first_seen", "last_seen", "classified_at"): + if key in out: + out[key] = _iso(out[key]) + if "id" in out and out["id"] is not None: + out["id"] = str(out["id"]) + if "merge_id" in out and out["merge_id"] is not None: + out["merge_id"] = str(out["merge_id"]) + return out + + +def _maildb(pool: ConnectionPool) -> Any: + from maildb import MailDB + + return MailDB._from_pool(pool) + + +def _clamp_limit(limit: int, *, default: int, maximum: int) -> int: + if limit < 1: + return default + return min(limit, maximum) + + +def _parse_offset_cursor(cursor: str | None, secret_key: str) -> int: + if not cursor: + return 0 + try: + payload = decode_cursor(cursor, secret_key) + except ValueError as exc: + raise HTTPException(status_code=422, detail="invalid cursor") from exc + offset = payload.get("offset", 0) + if not isinstance(offset, int) or offset < 0: + raise HTTPException(status_code=422, detail="invalid cursor offset") + return offset + + +def _address_rows(pool: ConnectionPool, contact_id: UUID) -> list[dict[str, Any]]: + """Per-address volumes and owner flag (PE-002).""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT address, is_user, messages_from, messages_to, first_seen, last_seen + FROM contact_addresses + WHERE contact_id = %(id)s + ORDER BY (messages_from + messages_to) DESC, address + """, + {"id": contact_id}, + ).fetchall() + return [ + { + "address": str(r[0]), + "is_user": bool(r[1]), + "messages_from": int(r[2] or 0), + "messages_to": int(r[3] or 0), + "first_seen": _iso(r[4]), + "last_seen": _iso(r[5]), + } + for r in rows + ] + + +def _activity_buckets(pool: ConnectionPool, addresses: list[str]) -> list[dict[str, Any]]: + """Monthly sent-message buckets for contact addresses (one statement).""" + if not addresses: + return [] + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT date_trunc('month', e.date) AS bucket, count(*)::int AS count + FROM emails e + WHERE lower(e.sender_address) = ANY(%(addrs)s) + AND e.date IS NOT NULL + GROUP BY 1 + ORDER BY 1 + """, + {"addrs": [a.lower() for a in addresses]}, + ).fetchall() + return [{"bucket": _iso(r[0]), "count": int(r[1])} for r in rows if r[0] is not None] + + +def _top_topics(pool: ConnectionPool, addresses: list[str]) -> list[dict[str, Any]]: + """Top topics by authored membership; empty when topics not generated / table missing.""" + if not addresses: + return [] + try: + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT t.id, t.label, count(*)::int AS cnt + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE lower(e.sender_address) = ANY(%(addrs)s) + AND t.hidden = FALSE + GROUP BY t.id, t.label + ORDER BY cnt DESC, t.label ASC + LIMIT %(lim)s + """, + {"addrs": [a.lower() for a in addresses], "lim": _TOP_TOPICS}, + ).fetchall() + except Exception as exc: + logger.debug("people_topics_empty", error=str(exc)) + return [] + return [{"id": str(r[0]), "label": str(r[1]), "count": int(r[2])} for r in rows] + + +def _thread_count(pool: ConnectionPool, addresses: list[str]) -> int: + if not addresses: + return 0 + with pool.connection() as conn: + row = conn.execute( + """ + SELECT count(DISTINCT e.thread_id)::int + FROM emails e + WHERE lower(e.sender_address) = ANY(%(addrs)s) + AND e.thread_id IS NOT NULL + """, + {"addrs": [a.lower() for a in addresses]}, + ).fetchone() + return int(row[0]) if row else 0 + + +def _merge_history(pool: ConnectionPool, contact_id: UUID) -> list[dict[str, Any]]: + """Merges where this contact is the surviving target (unmerge-able).""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, source_id, target_id, merged_at + FROM contact_merges + WHERE target_id = %(id)s + ORDER BY merged_at DESC + """, + {"id": contact_id}, + ).fetchall() + return [ + { + "id": str(r[0]), + "source_id": str(r[1]), + "target_id": str(r[2]), + "merged_at": _iso(r[3]), + } + for r in rows + ] + + +def _parse_optional_date(raw: str | None, *, field: str) -> datetime | None: + """Parse optional ISO date or datetime query param to timezone-aware UTC.""" + if raw is None or raw == "": + return None + text = raw.strip() + if not text: + return None + try: + if len(text) == 10 and text[4] == "-" and text[7] == "-": + dt = datetime(int(text[0:4]), int(text[5:7]), int(text[8:10]), tzinfo=UTC) + else: + dt = datetime.fromisoformat(text.replace("Z", "+00:00")) + dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC) + except (ValueError, TypeError) as exc: + raise HTTPException(status_code=422, detail=f"invalid {field}") from exc + return dt + + +def _date_only(value: Any) -> str | None: + """ISO calendar date (YYYY-MM-DD) for edge first/last fields.""" + if value is None: + return None + if isinstance(value, datetime): + dt = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return dt.astimezone(UTC).date().isoformat() + if hasattr(value, "isoformat"): + s = str(value.isoformat()) + return s[:10] if len(s) >= 10 else s + s = str(value) + return s[:10] if len(s) >= 10 else s + + +def _ego_graph( + pool: ConnectionPool, + *, + contact_id: UUID, + ego_label: str, + ego_kind: str, + max_nodes: int, + date_from: datetime | None, + date_to: datetime | None, +) -> dict[str, Any]: + """Bounded depth-1 co-participation graph around one contact (PE-003). + + One set-based pipeline: ego threads (sender or recipient) → co-senders → + identity join → rank/cap. Deeper neighborhoods arrive on demand later. + """ + ego_id = str(contact_id) + with pool.connection() as conn: + rows = conn.execute( + """ + WITH ego_addrs AS ( + SELECT lower(btrim(address)) AS address + FROM contact_addresses + WHERE contact_id = %(cid)s + AND address IS NOT NULL + AND btrim(address) <> '' + ), + ego_threads AS ( + SELECT DISTINCT e.thread_id + FROM emails e + WHERE e.thread_id IS NOT NULL + AND (%(date_from)s::timestamptz IS NULL + OR e.date >= %(date_from)s::timestamptz) + AND (%(date_to)s::timestamptz IS NULL + OR e.date <= %(date_to)s::timestamptz) + AND ( + lower(btrim(e.sender_address)) IN (SELECT address FROM ego_addrs) + OR EXISTS ( + SELECT 1 + FROM ego_addrs a + WHERE e.recipients @> jsonb_build_object( + 'to', to_jsonb(ARRAY[a.address]) + ) + OR e.recipients @> jsonb_build_object( + 'cc', to_jsonb(ARRAY[a.address]) + ) + OR e.recipients @> jsonb_build_object( + 'bcc', to_jsonb(ARRAY[a.address]) + ) + ) + ) + ), + co_raw AS ( + SELECT + CASE + WHEN ca.contact_id IS NOT NULL THEN ca.contact_id::text + ELSE 'addr:' || lower(btrim(e.sender_address)) + END AS node_id, + CASE + WHEN ca.contact_id IS NOT NULL THEN + coalesce( + nullif(btrim(c.display_name), ''), + lower(btrim(e.sender_address)) + ) + ELSE lower(btrim(e.sender_address)) + END AS label, + CASE + WHEN ca.contact_id IS NOT NULL THEN coalesce(c.kind, 'unknown') + ELSE 'address' + END AS kind, + e.thread_id, + e.date + FROM emails e + JOIN ego_threads t ON t.thread_id = e.thread_id + LEFT JOIN contact_addresses ca + ON lower(btrim(ca.address)) = lower(btrim(e.sender_address)) + LEFT JOIN contacts c ON c.id = ca.contact_id + WHERE e.sender_address IS NOT NULL + AND btrim(e.sender_address) <> '' + AND lower(btrim(e.sender_address)) NOT IN (SELECT address FROM ego_addrs) + AND (%(date_from)s::timestamptz IS NULL + OR e.date >= %(date_from)s::timestamptz) + AND (%(date_to)s::timestamptz IS NULL + OR e.date <= %(date_to)s::timestamptz) + ), + co_threads AS ( + SELECT node_id, + thread_id, + min(date) AS thread_first, + max(date) AS thread_last, + (array_agg(label ORDER BY label))[1] AS label, + (array_agg(kind ORDER BY kind))[1] AS kind + FROM co_raw + GROUP BY node_id, thread_id + ), + agg AS ( + SELECT + node_id, + (array_agg(label ORDER BY label))[1] AS label, + (array_agg(kind ORDER BY kind))[1] AS kind, + count(*)::int AS shared_thread_count, + min(thread_first) AS first_shared, + max(thread_last) AS last_shared, + (array_agg(thread_id ORDER BY thread_id))[1:%(ev_cap)s] AS sample_threads + FROM co_threads + GROUP BY node_id + ), + ranked AS ( + SELECT + a.*, + count(*) OVER ()::int AS total_coparticipants, + row_number() OVER ( + ORDER BY a.shared_thread_count DESC, a.node_id ASC + ) AS rn + FROM agg a + ) + SELECT node_id, label, kind, shared_thread_count, + first_shared, last_shared, sample_threads, + total_coparticipants + FROM ranked + WHERE rn <= %(max_nodes)s + ORDER BY shared_thread_count DESC, node_id ASC + """, + { + "cid": contact_id, + "date_from": date_from, + "date_to": date_to, + "max_nodes": max_nodes, + "ev_cap": _GRAPH_EVIDENCE_THREAD_CAP, + }, + ).fetchall() + + total = int(rows[0][7]) if rows else 0 + truncated = total > max_nodes + + nodes: list[dict[str, Any]] = [ + { + "id": ego_id, + "label": ego_label, + "kind": ego_kind, + "is_ego": True, + } + ] + edges: list[dict[str, Any]] = [] + for r in rows: + node_id = str(r[0]) + label = str(r[1] or node_id) + kind = str(r[2] or "unknown") + shared = int(r[3] or 0) + first = _date_only(r[4]) + last = _date_only(r[5]) + sample = r[6] or [] + if not isinstance(sample, (list, tuple)): + sample = list(sample) if sample else [] + thr_ids = [encode_source_id("thr", str(tid)) for tid in sample if tid is not None][ + :_GRAPH_EVIDENCE_THREAD_CAP + ] + + nodes.append( + { + "id": node_id, + "label": label, + "kind": kind, + "is_ego": False, + } + ) + edges.append( + { + "source": ego_id, + "target": node_id, + "kind": "thread_co_participation", + "shared_threads": shared, + "first": first, + "last": last, + "evidence": {"thread_ids": thr_ids}, + } + ) + + return { + "nodes": nodes, + "edges": edges, + "truncated": truncated, + "total_coparticipants": total, + } + + +def _enrich_card(pool: ConnectionPool, card: dict[str, Any]) -> dict[str, Any]: + """Full profile: card + address classes, activity, topics, merges.""" + contact_id = UUID(str(card["id"])) + addresses = list(card.get("addresses") or []) + addr_rows = _address_rows(pool, contact_id) + address_classes: dict[str, str] = { + r["address"]: ("owner" if r["is_user"] else "external") for r in addr_rows + } + # Fallback for addresses present on the card but missing detail rows. + for addr in addresses: + if addr not in address_classes: + address_classes[addr] = "external" + + out = _serialize_contact(card) + out["address_classes"] = address_classes + out["address_details"] = addr_rows + out["activity"] = _activity_buckets(pool, addresses) + out["topics"] = _top_topics(pool, addresses) + out["thread_count"] = _thread_count(pool, addresses) + merges = _merge_history(pool, contact_id) + if merges: + out["merges"] = merges + return out + + +# --- endpoints --- + + +@router.get("/people") +def people_list( + request: Request, + q: str | None = Query(default=None), + kind: str | None = Query(default=None), + needs_review: bool = Query(default=False), + limit: int = Query(default=_LIST_DEFAULT_LIMIT, ge=1, le=_LIST_MAX_LIMIT), + cursor: str | None = Query(default=None), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Search contacts (include_total on first page). Offset cursor, limit ≤ 500.""" + if kind is not None and kind not in VALID_KINDS: + raise HTTPException( + status_code=422, + detail=f"kind must be one of: {', '.join(sorted(VALID_KINDS))}", + ) + pool: ConnectionPool = request.app.state.pool + secret_key: str = request.app.state.settings.secret_key + offset = _parse_offset_cursor(cursor, secret_key) + lim = _clamp_limit(limit, default=_LIST_DEFAULT_LIMIT, maximum=_LIST_MAX_LIMIT) + + db = _maildb(pool) + include_total = offset == 0 + results, total = db.contacts_search( + query=q if q else None, + kind=kind, + needs_review=needs_review, + limit=lim, + offset=offset, + include_total=include_total, + ) + items = [_serialize_contact(r) for r in results] + next_cursor: str | None = None + if len(items) >= lim: + next_cursor = encode_cursor({"offset": offset + lim}, secret_key) + + return { + "items": items, + "total": total, + "next_cursor": next_cursor, + "limit": lim, + "offset": offset, + } + + +@router.get("/people/merge-candidates") +def people_merge_candidates( + request: Request, + limit: int = Query(default=_CANDIDATES_DEFAULT_LIMIT, ge=1, le=_CANDIDATES_MAX_LIMIT), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Contact pairs that share a normalized name variant.""" + pool: ConnectionPool = request.app.state.pool + lim = _clamp_limit(limit, default=_CANDIDATES_DEFAULT_LIMIT, maximum=_CANDIDATES_MAX_LIMIT) + db = _maildb(pool) + pairs = db.merge_candidates(limit=lim) + return {"items": pairs} + + +@router.get("/people/{contact_id}") +def people_get( + contact_id: UUID, + request: Request, + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Full contact card + activity, topics, address classes (PE-002).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + card = db.get_contact(contact_id=contact_id) + if card is None: + raise HTTPException(status_code=404, detail="Contact not found") + return _enrich_card(pool, card) + + +@router.get("/people/{contact_id}/graph") +def people_graph( + contact_id: UUID, + request: Request, + depth: int = Query(default=1, ge=1), + max_nodes: int = Query(default=_GRAPH_DEFAULT_MAX_NODES, ge=1, le=_GRAPH_HARD_MAX_NODES), + date_from: str | None = Query(default=None), + date_to: str | None = Query(default=None), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Bounded depth-1 ego co-participation graph (spec §7.3, PE-003, PERF-002). + + depth=1 only — deeper neighborhoods arrive on demand later. Never a global graph. + """ + if depth != 1: + raise HTTPException( + status_code=422, + detail="depth=1 only; deeper neighborhoods arrive on demand later", + ) + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + card = db.get_contact(contact_id=contact_id) + if card is None: + raise HTTPException(status_code=404, detail="Contact not found") + + df = _parse_optional_date(date_from, field="date_from") + dt = _parse_optional_date(date_to, field="date_to") + # Inclusive end-of-day when only a calendar date was supplied. + if dt is not None and date_to is not None and len(date_to.strip()) == 10: + dt = dt.replace(hour=23, minute=59, second=59, microsecond=999999) + + ego_label = str(card.get("display_name") or "") + if not ego_label: + addrs = card.get("addresses") or [] + ego_label = str(addrs[0]) if addrs else str(contact_id) + ego_kind = str(card.get("kind") or "unknown") + lim = min(max(1, max_nodes), _GRAPH_HARD_MAX_NODES) + + return _ego_graph( + pool, + contact_id=contact_id, + ego_label=ego_label, + ego_kind=ego_kind, + max_nodes=lim, + date_from=df, + date_to=dt, + ) + + +@router.patch("/people/{contact_id}") +def people_patch( + contact_id: UUID, + body: ContactPatch, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Curation write: kind/tags/notes/display_name (audited).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + try: + card = db.update_contact( + contact_id=contact_id, + kind=body.kind, + tags=body.tags, + notes=body.notes, + display_name=body.display_name, + ) + except ValueError as exc: + msg = str(exc) + if "does not exist" in msg: + raise HTTPException(status_code=404, detail=msg) from exc + raise HTTPException(status_code=422, detail=msg) from exc + + audit( + pool, + username=user, + action="people_update", + detail={ + "contact_id": str(contact_id), + "kind": body.kind, + "tags": body.tags, + "notes": body.notes is not None, + "display_name": body.display_name is not None, + }, + ) + return _enrich_card(pool, card) + + +@router.post("/people/merge") +def people_merge( + body: MergeRequest, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Merge source into target; audited (PE-001).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + try: + card = db.merge_contacts(source_id=body.source_id, target_id=body.target_id) + except ValueError as exc: + msg = str(exc) + if "does not exist" in msg: + raise HTTPException(status_code=404, detail=msg) from exc + raise HTTPException(status_code=422, detail=msg) from exc + + merge_id = card.get("merge_id") + audit( + pool, + username=user, + action="people_merge", + detail={ + "source_id": body.source_id, + "target_id": body.target_id, + "merge_id": str(merge_id) if merge_id is not None else None, + }, + ) + return _enrich_card(pool, card) + + +@router.post("/people/unmerge") +def people_unmerge( + body: UnmergeRequest, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Reverse a prior merge by merge_id; audited (PE-001).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + try: + result = db.unmerge_contacts(merge_id=body.merge_id) + except ValueError as exc: + msg = str(exc) + if "does not exist" in msg: + raise HTTPException(status_code=404, detail=msg) from exc + raise HTTPException(status_code=422, detail=msg) from exc + + audit( + pool, + username=user, + action="people_unmerge", + detail={"merge_id": body.merge_id}, + ) + source = result.get("source") or {} + target = result.get("target") or {} + return { + "source": _enrich_card(pool, source) if source.get("id") else source, + "target": _enrich_card(pool, target) if target.get("id") else target, + } diff --git a/apps/chronicle/server/src/chronicle_server/topics.py b/apps/chronicle/server/src/chronicle_server/topics.py new file mode 100644 index 0000000..f7df21e --- /dev/null +++ b/apps/chronicle/server/src/chronicle_server/topics.py @@ -0,0 +1,1770 @@ +"""Topics subsystem: clustering, assignment, curation, and generation. + +Deterministic k-means over an embedding sample produces centroid-backed +``app_topics``; one set-based full-corpus membership assignment; term-derived +labels with optional gateway polish; curation CRUD with manual precedence +(TA-005). + +``POST /api/topics/generate`` is synchronous and may take minutes on a full +corpus (the set-based assignment scans every embedded email). Data Health +records the run via the ``topics_generate`` audit action. +""" + +from __future__ import annotations + +import json +import re +import time +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any, Literal +from uuid import UUID + +import numpy as np +import structlog +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from pydantic import BaseModel, Field, field_validator + +from chronicle_server.auth import require_user +from chronicle_server.chronicle import VALID_UNITS, choose_aggregation +from chronicle_server.cursor import decode_cursor, encode_cursor +from chronicle_server.db import audit +from chronicle_server.gateway import ModelGateway +from chronicle_server.ids import decode_source_id, encode_source_id, msg_key_to_uuid +from chronicle_server.scope import QueryScope, scope_filters + +if TYPE_CHECKING: + from psycopg_pool import ConnectionPool + + from chronicle_server.config import ChronicleSettings + +logger = structlog.get_logger() + +router = APIRouter(tags=["topics"]) + +TopicOrigin = Literal["automatic", "curated", "manual"] +MemberOrigin = Literal["automatic", "manual"] + +_MAX_KMEANS_ITERS = 25 +_CENTROID_SHIFT_EPS = 1e-4 +_EMBED_DIM = 768 +_TOP_TERMS = 5 +_LABEL_TERMS = 3 +_MAX_LABEL_WORDS = 4 +_MIN_TERM_LEN = 3 +_MEMBERS_DEFAULT_LIMIT = 50 +_MEMBERS_MAX_LIMIT = 200 +_RIVER_DEFAULT_TOP = 8 +_RIVER_MAX_TOP = 48 +_RIVER_DEFAULT_PIXEL_WIDTH = 920 + +# Common English stopwords for subject TF labeling (lowercase). +STOPWORDS: frozenset[str] = frozenset( + { + "a", + "about", + "above", + "after", + "again", + "against", + "all", + "am", + "an", + "and", + "any", + "are", + "as", + "at", + "be", + "because", + "been", + "before", + "being", + "below", + "between", + "both", + "but", + "by", + "can", + "could", + "did", + "do", + "does", + "doing", + "down", + "during", + "each", + "few", + "for", + "from", + "further", + "had", + "has", + "have", + "having", + "he", + "her", + "here", + "hers", + "herself", + "him", + "himself", + "his", + "how", + "i", + "if", + "in", + "into", + "is", + "it", + "its", + "itself", + "just", + "me", + "more", + "most", + "my", + "myself", + "no", + "nor", + "not", + "now", + "of", + "off", + "on", + "once", + "only", + "or", + "other", + "our", + "ours", + "ourselves", + "out", + "over", + "own", + "re", + "same", + "she", + "should", + "so", + "some", + "such", + "than", + "that", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "there", + "these", + "they", + "this", + "those", + "through", + "to", + "too", + "under", + "until", + "up", + "very", + "was", + "we", + "were", + "what", + "when", + "where", + "which", + "while", + "who", + "whom", + "why", + "will", + "with", + "would", + "you", + "your", + "yours", + "yourself", + "yourselves", + "fw", + "fwd", + "subject", + } +) + +_TERM_RE = re.compile(r"[a-z0-9]+") + +LABEL_POLISH_POLICY = ( + "Given topic term lists, output ONLY a STRICT JSON array of short labels " + "(one per topic, same order). Each label is at most 4 words. " + "No other keys, no prose." +) + + +# --- request / response models --- + + +class GenerateRequest(BaseModel): + k: int = Field(default=24, ge=4, le=48) + sample: int = Field(default=20000, ge=1) + seed: int = 13 + + +class GenerateResponse(BaseModel): + topics: int + assigned: int + skipped_curated: int + took_ms: int + + +class TopicCreate(BaseModel): + label: str = Field(min_length=1) + description: str | None = None + + +class TopicPatch(BaseModel): + label: str | None = Field(default=None, min_length=1) + description: str | None = None + hidden: bool | None = None + parent_id: str | None = None + + @field_validator("parent_id") + @classmethod + def _empty_parent_ok(cls, value: str | None) -> str | None: + if value is not None and value.strip() == "": + return None + return value + + +class MemberAddRequest(BaseModel): + email_sid: str = Field(min_length=1) + + +# --- pure helpers (testable without DB) --- + + +def pca_project_2d(centroids: np.ndarray) -> np.ndarray: + """Deterministic PCA of centroid matrix to 2D via SVD. + + Returns an ``(n, 2)`` array with coordinates normalized into ``[-1, 1]``. + Topics without centroids must be filtered out before calling. + """ + arr = np.asarray(centroids, dtype=np.float64) + if arr.ndim != 2 or arr.shape[0] == 0: + return np.zeros((0, 2), dtype=np.float64) + n, d = arr.shape + if n == 1: + return np.zeros((1, 2), dtype=np.float64) + + centered = arr - arr.mean(axis=0, keepdims=True) + # Economy SVD is deterministic for a fixed matrix (numpy). + _u, s, vt = np.linalg.svd(centered, full_matrices=False) + k = min(2, s.shape[0], d, n) + # Project: X_c @ V_k (equivalent to U[:, :k] * S[:k]) + coords = centered @ vt[:k].T + if k < 2: + coords = np.pad(coords, ((0, 0), (0, 2 - k))) + max_abs = float(np.max(np.abs(coords))) if coords.size else 0.0 + if max_abs > 0.0 and np.isfinite(max_abs): + coords = coords / max_abs + return np.asarray(coords, dtype=np.float64) + + +def _parse_viewport_dt(value: str) -> datetime: + """Parse ISO datetime/date for viewport bounds; raise 422 on failure.""" + raw = value.strip() + if not raw: + raise HTTPException(status_code=422, detail="empty datetime") + # Accept trailing Z. + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(raw) + except ValueError as exc: + raise HTTPException(status_code=422, detail=f"invalid datetime: {value}") from exc + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +def _scope_from_query( + *, + mailboxes: list[str] | None, + senders: list[str] | None, + scope_from: str | None, + scope_to: str | None, +) -> QueryScope: + payload: dict[str, Any] = { + "mailboxes": list(mailboxes or []), + "senders": list(senders or []), + } + if scope_from or scope_to: + date: dict[str, str] = {} + if scope_from: + date["from"] = scope_from + if scope_to: + date["to"] = scope_to + payload["date"] = date + return QueryScope.model_validate(payload) + + +def _resolve_river_unit(unit: str, viewport_from: datetime, viewport_to: datetime) -> str: + if unit == "auto": + duration = viewport_to - viewport_from + if duration.total_seconds() <= 0: + duration = timedelta(days=1) + return choose_aggregation(duration, _RIVER_DEFAULT_PIXEL_WIDTH) + if unit not in VALID_UNITS: + raise HTTPException( + status_code=422, + detail=f"unit must be auto or one of {', '.join(VALID_UNITS)}", + ) + return unit + + +def parse_embedding(raw: Any) -> list[float] | None: + """Parse pgvector embedding (string or list) into floats.""" + if raw is None: + return None + if isinstance(raw, list | tuple | np.ndarray): + return [float(x) for x in raw] + if isinstance(raw, str): + return [float(x) for x in raw.strip("[]").split(",") if x.strip()] + try: + return [float(x) for x in raw] + except (TypeError, ValueError): + return None + + +def l2_normalize(vectors: np.ndarray) -> np.ndarray: + """Row-wise L2 normalize; zero rows stay zero.""" + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + norms = np.maximum(norms, 1e-12) + return np.asarray(vectors / norms, dtype=np.float64) + + +def kmeans_plus_plus_init( + vectors: np.ndarray, + k: int, + rng: np.random.Generator, +) -> np.ndarray: + """k-means++ centroid initialization (seeded).""" + n = vectors.shape[0] + centroids = np.empty((k, vectors.shape[1]), dtype=np.float64) + first = int(rng.integers(0, n)) + centroids[0] = vectors[first] + # Squared Euclidean distances to nearest chosen centroid. + closest = np.full(n, np.inf, dtype=np.float64) + for i in range(1, k): + diff = vectors - centroids[i - 1] + dist_sq = np.sum(diff * diff, axis=1) + closest = np.minimum(closest, dist_sq) + total = float(closest.sum()) + if total <= 0.0 or not np.isfinite(total): + centroids[i] = vectors[int(rng.integers(0, n))] + continue + probs = closest / total + idx = int(rng.choice(n, p=probs)) + centroids[i] = vectors[idx] + return centroids + + +def run_kmeans( + vectors: np.ndarray, + k: int, + *, + seed: int = 13, + max_iters: int = _MAX_KMEANS_ITERS, + eps: float = _CENTROID_SHIFT_EPS, +) -> tuple[np.ndarray, np.ndarray]: + """Deterministic L2-normalized k-means (cosine-equivalent). + + Returns ``(centroids, labels)`` where centroids are L2-normalized and + labels is an int array of length n (cluster index per sample). + + Raises ValueError when ``k > n`` or ``n == 0`` or ``k < 1``. + """ + if k < 1: + raise ValueError("k must be >= 1") + n = vectors.shape[0] + if n == 0: + raise ValueError("empty sample") + if k > n: + raise ValueError(f"k ({k}) exceeds sample size ({n})") + + data = l2_normalize(np.asarray(vectors, dtype=np.float64)) + rng = np.random.default_rng(seed) + centroids = l2_normalize(kmeans_plus_plus_init(data, k, rng)) + + labels = np.zeros(n, dtype=np.int32) + for _ in range(max_iters): + # Assign: nearest centroid by squared Euclidean (≡ cosine on unit sphere). + # (n, k) distances via ||x||^2 + ||c||^2 - 2 x·c; norms ≈ 1. + dots = data @ centroids.T # (n, k) + dist_sq = 2.0 - 2.0 * dots + labels = np.argmin(dist_sq, axis=1).astype(np.int32) + + new_centroids = np.zeros_like(centroids) + for j in range(k): + mask = labels == j + if not np.any(mask): + # Re-seed empty cluster from a random point. + new_centroids[j] = data[int(rng.integers(0, n))] + else: + new_centroids[j] = data[mask].mean(axis=0) + new_centroids = l2_normalize(new_centroids) + + shift = float(np.linalg.norm(new_centroids - centroids)) + centroids = new_centroids + if shift < eps: + break + + return centroids, labels + + +def extract_top_terms( + subjects: list[str | None], + *, + top_n: int = _TOP_TERMS, + stopwords: frozenset[str] = STOPWORDS, +) -> list[str]: + """Top TF terms from subjects: lowercase, alnum ≥ 3, stopword-filtered.""" + counts: dict[str, int] = {} + for subject in subjects: + if not subject: + continue + for token in _TERM_RE.findall(subject.lower()): + if len(token) < _MIN_TERM_LEN or token in stopwords: + continue + counts[token] = counts.get(token, 0) + 1 + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + return [term for term, _ in ranked[:top_n]] + + +def label_from_terms(terms: list[str], n: int = _LABEL_TERMS) -> str: + """Comma-join of the first *n* terms, or a placeholder when empty.""" + chosen = terms[:n] + if not chosen: + return "untitled topic" + return ", ".join(chosen) + + +def _largest_json_array(text: str) -> str | None: + """Return the largest balanced ``[...]`` substring, or None.""" + best: str | None = None + depth = 0 + start: int | None = None + for i, ch in enumerate(text): + if ch == "[": + if depth == 0: + start = i + depth += 1 + elif ch == "]": + if depth > 0: + depth -= 1 + if depth == 0 and start is not None: + candidate = text[start : i + 1] + if best is None or len(candidate) > len(best): + best = candidate + return best + + +def validate_label_array(raw: Any, expected: int) -> list[str] | None: + """Whitelist-validate a JSON array of ≤4-word label strings. + + Returns None on any structural failure (wrong type, wrong length, empty). + """ + if not isinstance(raw, list) or len(raw) != expected: + return None + out: list[str] = [] + for item in raw: + if not isinstance(item, str): + return None + label = " ".join(item.split()) + if not label: + return None + words = label.split() + if len(words) > _MAX_LABEL_WORDS: + return None + out.append(label) + return out + + +def parse_label_polish_response(content: str, expected: int) -> list[str] | None: + """Extract and validate gateway label array. None on failure.""" + if not content or not content.strip(): + return None + block = _largest_json_array(content) + if block is None: + return None + try: + raw = json.loads(block) + except (json.JSONDecodeError, TypeError, ValueError): + return None + return validate_label_array(raw, expected) + + +def vector_literal(vec: np.ndarray | list[float]) -> str: + """Format a float vector as a pgvector text literal ``[a,b,...]``.""" + arr = np.asarray(vec, dtype=np.float64).ravel() + return "[" + ",".join(f"{float(x):.8g}" for x in arr) + "]" + + +def _iso(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + if hasattr(value, "isoformat"): + return str(value.isoformat()) + return str(value) + + +def _gateway_from_request(request: Request, settings: ChronicleSettings) -> ModelGateway: + transport = getattr(request.app.state, "chat_transport", None) + return ModelGateway(settings, transport) + + +def _model_available(request: Request, gateway: ModelGateway) -> bool: + forced = getattr(request.app.state, "model_available", None) + if forced is not None: + return bool(forced) + return gateway.availability() + + +def _complete_chat(gateway: ModelGateway, messages: list[dict[str, str]]) -> str: + """One non-streaming completion: collect all transport deltas.""" + settings = gateway._settings # noqa: SLF001 + transport = gateway._transport # noqa: SLF001 + parts: list[str] = [] + for delta in transport(settings.answer_model, messages, False): + if delta: + parts.append(str(delta)) + return "".join(parts) + + +def _topic_row_dict(row: Any) -> dict[str, Any]: + return { + "id": str(row[0]), + "label": str(row[1]), + "description": row[2], + "origin": str(row[3]), + "parent_id": str(row[4]) if row[4] is not None else None, + "hidden": bool(row[5]), + "top_terms": list(row[6]) if row[6] is not None else [], + "generation": int(row[7]), + "member_count": int(row[8]), + "created_at": _iso(row[9]), + "updated_at": _iso(row[10]), + } + + +_TOPIC_SELECT = """ + SELECT id, label, description, origin, parent_id, hidden, + top_terms, generation, member_count, created_at, updated_at + FROM app_topics +""" + + +# --- generation --- + + +def generate_topics( + pool: ConnectionPool, + body: GenerateRequest, + *, + username: str, + gateway: ModelGateway | None = None, + model_available: bool = False, +) -> GenerateResponse: + """Sample embeddings, cluster, label, replace-regenerate, full-corpus assign. + + Runtime note: the full-corpus assignment is a single set-based scan of every + email with an embedding and may take minutes on large archives. The endpoint + is synchronous; Data Health surfaces the run via audit ``topics_generate``. + """ + started = time.perf_counter() + k = body.k + sample_n = body.sample + seed = body.seed + + with pool.connection() as conn: + # Stable pseudo-random sample via md5(id::text) order (deterministic). + sample_rows = conn.execute( + """ + SELECT id, embedding, subject + FROM emails + WHERE embedding IS NOT NULL + ORDER BY md5(id::text) + LIMIT %(sample)s + """, + {"sample": sample_n}, + ).fetchall() + + if not sample_rows: + audit( + pool, + username=username, + action="topics_generate", + detail={ + "topics": 0, + "assigned": 0, + "skipped_curated": 0, + "seed": seed, + "k": k, + "sample": sample_n, + "took_ms": int((time.perf_counter() - started) * 1000), + }, + ) + return GenerateResponse(topics=0, assigned=0, skipped_curated=0, took_ms=0) + + ids: list[UUID] = [] + subjects: list[str | None] = [] + vectors_list: list[list[float]] = [] + for row in sample_rows: + emb = parse_embedding(row[1]) + if emb is None or len(emb) != _EMBED_DIM: + continue + ids.append(row[0]) + subjects.append(row[2] if row[2] is not None else None) + vectors_list.append(emb) + + if not vectors_list: + took_ms = int((time.perf_counter() - started) * 1000) + audit( + pool, + username=username, + action="topics_generate", + detail={ + "topics": 0, + "assigned": 0, + "skipped_curated": 0, + "seed": seed, + "k": k, + "sample": sample_n, + "took_ms": took_ms, + }, + ) + return GenerateResponse(topics=0, assigned=0, skipped_curated=0, took_ms=took_ms) + + n = len(vectors_list) + effective_k = min(k, n) + if k > n: + # Guard: cannot form more clusters than samples; use n. + logger.info("topics_k_clamped", requested_k=k, sample_n=n) + effective_k = n + + vectors = np.asarray(vectors_list, dtype=np.float64) + centroids, labels = run_kmeans(vectors, effective_k, seed=seed) + + # Per-cluster subjects from the sample → top terms + default labels. + cluster_subjects: list[list[str | None]] = [[] for _ in range(effective_k)] + for i, lab in enumerate(labels): + cluster_subjects[int(lab)].append(subjects[i]) + + top_terms_list: list[list[str]] = [] + term_labels: list[str] = [] + for j in range(effective_k): + terms = extract_top_terms(cluster_subjects[j]) + top_terms_list.append(terms) + term_labels.append(label_from_terms(terms)) + + labels_final = list(term_labels) + if model_available and gateway is not None and effective_k > 0: + term_payload = [ + {"index": j, "terms": top_terms_list[j], "fallback": term_labels[j]} + for j in range(effective_k) + ] + messages = [ + {"role": "system", "content": LABEL_POLISH_POLICY}, + { + "role": "user", + "content": json.dumps(term_payload, ensure_ascii=False), + }, + ] + try: + content = _complete_chat(gateway, messages) + polished = parse_label_polish_response(content, effective_k) + if polished is not None: + labels_final = polished + except Exception as exc: + logger.info("topics_label_polish_failed", error=str(exc)) + + # Surviving topics: curated/manual origin, hidden automatic, or any + # with manual members — never deleted or relabeled (TA-005). + surviving = conn.execute( + """ + SELECT t.id + FROM app_topics t + WHERE t.origin IN ('curated', 'manual') + OR t.hidden = TRUE + OR EXISTS ( + SELECT 1 FROM app_topic_members m + WHERE m.topic_id = t.id AND m.origin = 'manual' + ) + """ + ).fetchall() + surviving_ids = {row[0] for row in surviving} + skipped_curated = len(surviving_ids) + + # Delete replaceable automatic topics (cascade memberships). + conn.execute( + """ + DELETE FROM app_topics t + WHERE t.origin = 'automatic' + AND t.hidden = FALSE + AND NOT EXISTS ( + SELECT 1 FROM app_topic_members m + WHERE m.topic_id = t.id AND m.origin = 'manual' + ) + """ + ) + + gen_row = conn.execute("SELECT COALESCE(MAX(generation), 0) FROM app_topics").fetchone() + next_gen = int(gen_row[0]) + 1 if gen_row else 1 + + new_topic_ids: list[UUID] = [] + for j in range(effective_k): + centroid_lit = vector_literal(centroids[j]) + inserted = conn.execute( + """ + INSERT INTO app_topics ( + label, origin, centroid, top_terms, generation, member_count + ) VALUES ( + %(label)s, 'automatic', %(centroid)s::vector, + %(top_terms)s, %(generation)s, 0 + ) + RETURNING id + """, + { + "label": labels_final[j], + "centroid": centroid_lit, + "top_terms": top_terms_list[j], + "generation": next_gen, + }, + ).fetchone() + if inserted is None: + raise RuntimeError("topic insert returned no id") + new_topic_ids.append(inserted[0]) + + # Drop previous automatic memberships on automatic topics before reassign. + conn.execute( + """ + DELETE FROM app_topic_members m + USING app_topics t + WHERE m.topic_id = t.id + AND t.origin = 'automatic' + AND m.origin = 'automatic' + """ + ) + + # Full-corpus nearest-centroid assignment (one set-based statement). + assign_result = conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + SELECT t.id, e.id, t.centroid <=> e.embedding, 'automatic' + FROM emails e + CROSS JOIN LATERAL ( + SELECT id, centroid FROM app_topics + WHERE origin = 'automatic' AND centroid IS NOT NULL + ORDER BY centroid <=> e.embedding + LIMIT 1 + ) t + WHERE e.embedding IS NOT NULL + ON CONFLICT (topic_id, email_id) DO NOTHING + """ + ) + assigned = assign_result.rowcount if assign_result.rowcount is not None else 0 + + # One grouped UPDATE for all member_counts. + conn.execute( + """ + UPDATE app_topics t + SET member_count = c.cnt + FROM ( + SELECT t2.id AS id, count(m.email_id)::int AS cnt + FROM app_topics t2 + LEFT JOIN app_topic_members m ON m.topic_id = t2.id + GROUP BY t2.id + ) c + WHERE t.id = c.id + """ + ) + + topics_created = len(new_topic_ids) + conn.commit() + + took_ms = int((time.perf_counter() - started) * 1000) + audit( + pool, + username=username, + action="topics_generate", + detail={ + "topics": topics_created, + "assigned": assigned, + "skipped_curated": skipped_curated, + "seed": seed, + "k": k, + "sample": sample_n, + "took_ms": took_ms, + }, + ) + logger.info( + "topics_generate", + topics=topics_created, + assigned=assigned, + skipped_curated=skipped_curated, + took_ms=took_ms, + seed=seed, + k=k, + ) + return GenerateResponse( + topics=topics_created, + assigned=assigned, + skipped_curated=skipped_curated, + took_ms=took_ms, + ) + + +# --- CRUD helpers --- + + +def list_topics( + pool: ConnectionPool, + *, + include_hidden: bool = False, +) -> list[dict[str, Any]]: + """Return topics as a forest: parents with nested children. + + Visible by default (``hidden=false``); ``include_hidden`` shows all. + """ + with pool.connection() as conn: + if include_hidden: + rows = conn.execute( + _TOPIC_SELECT + " ORDER BY member_count DESC, label ASC, id ASC" + ).fetchall() + else: + rows = conn.execute( + _TOPIC_SELECT + + " WHERE hidden = FALSE ORDER BY member_count DESC, label ASC, id ASC" + ).fetchall() + + topics = [_topic_row_dict(r) for r in rows] + by_id: dict[str, dict[str, Any]] = { + t["id"]: { + "id": t["id"], + "label": t["label"], + "origin": t["origin"], + "member_count": t["member_count"], + "hidden": t["hidden"], + "top_terms": t["top_terms"], + "parent_id": t["parent_id"], + "children": [], + } + for t in topics + } + roots: list[dict[str, Any]] = [] + for t in topics: + node = by_id[t["id"]] + parent_id = t["parent_id"] + if parent_id and parent_id in by_id: + by_id[parent_id]["children"].append(node) + else: + roots.append(node) + + # Drop internal parent_id from API payload (tree structure carries it). + def _strip(node: dict[str, Any]) -> dict[str, Any]: + children = [_strip(c) for c in node["children"]] + return { + "id": node["id"], + "label": node["label"], + "origin": node["origin"], + "member_count": node["member_count"], + "hidden": node["hidden"], + "top_terms": node["top_terms"], + "children": children, + } + + return [_strip(r) for r in roots] + + +def get_topic_detail(pool: ConnectionPool, topic_id: UUID) -> dict[str, Any]: + """Topic detail + monthly activity series + 5 representative members.""" + with pool.connection() as conn: + row = conn.execute( + _TOPIC_SELECT + " WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if row is None: + raise HTTPException(status_code=404, detail="Topic not found") + + topic = _topic_row_dict(row) + + activity_rows = conn.execute( + """ + SELECT date_trunc('month', e.date) AS bucket, count(*)::int AS count + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + WHERE m.topic_id = %(id)s + AND e.date IS NOT NULL + GROUP BY 1 + ORDER BY 1 + """, + {"id": topic_id}, + ).fetchall() + + # Closest members when centroid exists; otherwise newest. + if topic["origin"] != "manual": + member_rows = conn.execute( + """ + SELECT e.id, e.subject, e.sender_name, e.sender_address, + e.date, e.source_account, e.thread_id, m.distance + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE m.topic_id = %(id)s + ORDER BY m.distance ASC NULLS LAST, e.date DESC NULLS LAST + LIMIT 5 + """, + {"id": topic_id}, + ).fetchall() + else: + member_rows = conn.execute( + """ + SELECT e.id, e.subject, e.sender_name, e.sender_address, + e.date, e.source_account, e.thread_id, m.distance + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + WHERE m.topic_id = %(id)s + ORDER BY e.date DESC NULLS LAST + LIMIT 5 + """, + {"id": topic_id}, + ).fetchall() + + activity = [ + {"bucket": _iso(r[0]), "count": int(r[1])} for r in activity_rows if r[0] is not None + ] + members = [ + { + "id": encode_source_id("msg", r[0]), + "subject": r[1], + "sender_name": r[2], + "sender_address": r[3], + "date": _iso(r[4]), + "mailbox": r[5], + "thread_id": r[6], + "distance": float(r[7]) if r[7] is not None else None, + } + for r in member_rows + ] + return { + **{ + "id": topic["id"], + "label": topic["label"], + "description": topic["description"], + "origin": topic["origin"], + "parent_id": topic["parent_id"], + "hidden": topic["hidden"], + "top_terms": topic["top_terms"], + "generation": topic["generation"], + "member_count": topic["member_count"], + "created_at": topic["created_at"], + "updated_at": topic["updated_at"], + }, + "activity": activity, + "members": members, + } + + +def patch_topic( + pool: ConnectionPool, + topic_id: UUID, + body: TopicPatch, + *, + username: str, +) -> dict[str, Any]: + """Rename/describe/hide/parent; automatic → curated on any change (Table 21).""" + fields: list[str] = [] + params: dict[str, Any] = {"id": topic_id} + + if body.label is not None: + fields.append("label = %(label)s") + params["label"] = body.label + if body.description is not None: + fields.append("description = %(description)s") + params["description"] = body.description + if body.hidden is not None: + fields.append("hidden = %(hidden)s") + params["hidden"] = body.hidden + if "parent_id" in body.model_fields_set: + if body.parent_id is None: + fields.append("parent_id = NULL") + else: + try: + parent_uuid = UUID(body.parent_id) + except ValueError as exc: + raise HTTPException(status_code=422, detail="invalid parent_id") from exc + if parent_uuid == topic_id: + raise HTTPException(status_code=422, detail="parent_id cannot be self") + fields.append("parent_id = %(parent_id)s") + params["parent_id"] = parent_uuid + + if not fields: + raise HTTPException(status_code=422, detail="no fields to update") + + with pool.connection() as conn: + existing = conn.execute( + "SELECT origin FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if existing is None: + raise HTTPException(status_code=404, detail="Topic not found") + + if "parent_id" in params: + parent_row = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(parent_id)s", + {"parent_id": params["parent_id"]}, + ).fetchone() + if parent_row is None: + raise HTTPException(status_code=404, detail="Parent topic not found") + + origin = str(existing[0]) + if origin == "automatic": + fields.append("origin = 'curated'") + + fields.append("updated_at = now()") + sql = f"UPDATE app_topics SET {', '.join(fields)} WHERE id = %(id)s RETURNING id" + conn.execute(sql, params) + conn.commit() + + audit( + pool, + username=username, + action="topics_patch", + detail={"topic_id": str(topic_id), "fields": list(body.model_fields_set)}, + ) + return get_topic_detail(pool, topic_id) + + +def create_manual_topic( + pool: ConnectionPool, + body: TopicCreate, + *, + username: str, +) -> dict[str, Any]: + """Create a manual topic (no centroid; members added manually).""" + with pool.connection() as conn: + row = conn.execute( + """ + INSERT INTO app_topics (label, description, origin) + VALUES (%(label)s, %(description)s, 'manual') + RETURNING id + """, + {"label": body.label, "description": body.description}, + ).fetchone() + assert row is not None + topic_id = row[0] + conn.commit() + + audit( + pool, + username=username, + action="topics_create", + detail={"topic_id": str(topic_id), "label": body.label}, + ) + return get_topic_detail(pool, topic_id) + + +def delete_topic( + pool: ConnectionPool, + topic_id: UUID, + *, + username: str, +) -> dict[str, str]: + """Hard-delete only origin='manual'; others → 403 (hide instead).""" + with pool.connection() as conn: + row = conn.execute( + "SELECT origin FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if row is None: + raise HTTPException(status_code=404, detail="Topic not found") + if str(row[0]) != "manual": + raise HTTPException( + status_code=403, + detail="Only manual topics can be deleted; hide instead", + ) + conn.execute("DELETE FROM app_topics WHERE id = %(id)s", {"id": topic_id}) + conn.commit() + + audit( + pool, + username=username, + action="topics_delete", + detail={"topic_id": str(topic_id)}, + ) + return {"status": "deleted"} + + +def add_member( + pool: ConnectionPool, + topic_id: UUID, + email_sid: str, + *, + username: str, +) -> dict[str, Any]: + """Manual include: insert membership with origin='manual'.""" + try: + kind, key = decode_source_id(email_sid) + except ValueError as exc: + raise HTTPException(status_code=404, detail=f"Unknown source_id: {email_sid}") from exc + if kind != "msg" or not isinstance(key, int): + raise HTTPException(status_code=422, detail="email_sid must be a msg_ source id") + email_id = msg_key_to_uuid(key) + + with pool.connection() as conn: + trow = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if trow is None: + raise HTTPException(status_code=404, detail="Topic not found") + erow = conn.execute( + "SELECT 1 FROM emails WHERE id = %(id)s", + {"id": email_id}, + ).fetchone() + if erow is None: + raise HTTPException(status_code=404, detail=f"Unknown source_id: {email_sid}") + + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(topic_id)s, %(email_id)s, NULL, 'manual') + ON CONFLICT (topic_id, email_id) DO UPDATE + SET origin = 'manual', + distance = COALESCE(app_topic_members.distance, EXCLUDED.distance) + """, + {"topic_id": topic_id, "email_id": email_id}, + ) + conn.execute( + """ + UPDATE app_topics + SET member_count = ( + SELECT count(*)::int FROM app_topic_members WHERE topic_id = %(id)s + ), + updated_at = now() + WHERE id = %(id)s + """, + {"id": topic_id}, + ) + conn.commit() + + audit( + pool, + username=username, + action="topics_member_add", + detail={"topic_id": str(topic_id), "email_sid": email_sid}, + ) + return {"status": "ok", "topic_id": str(topic_id), "email_sid": email_sid} + + +def remove_member( + pool: ConnectionPool, + topic_id: UUID, + email_sid: str, + *, + username: str, +) -> dict[str, Any]: + """Exclude: DELETE membership row (manual or automatic).""" + try: + kind, key = decode_source_id(email_sid) + except ValueError as exc: + raise HTTPException(status_code=404, detail=f"Unknown source_id: {email_sid}") from exc + if kind != "msg" or not isinstance(key, int): + raise HTTPException(status_code=422, detail="email_sid must be a msg_ source id") + email_id = msg_key_to_uuid(key) + + with pool.connection() as conn: + trow = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if trow is None: + raise HTTPException(status_code=404, detail="Topic not found") + deleted = conn.execute( + """ + DELETE FROM app_topic_members + WHERE topic_id = %(topic_id)s AND email_id = %(email_id)s + """, + {"topic_id": topic_id, "email_id": email_id}, + ) + if deleted.rowcount == 0: + raise HTTPException(status_code=404, detail="Membership not found") + conn.execute( + """ + UPDATE app_topics + SET member_count = ( + SELECT count(*)::int FROM app_topic_members WHERE topic_id = %(id)s + ), + updated_at = now() + WHERE id = %(id)s + """, + {"id": topic_id}, + ) + conn.commit() + + audit( + pool, + username=username, + action="topics_member_remove", + detail={"topic_id": str(topic_id), "email_sid": email_sid}, + ) + return {"status": "ok", "topic_id": str(topic_id), "email_sid": email_sid} + + +# --- Atlas analytics (river / matrix / projection / members) --- + + +def topics_river( + pool: ConnectionPool, + *, + viewport_from: str, + viewport_to: str, + unit: str = "auto", + top: int = _RIVER_DEFAULT_TOP, + scope: QueryScope | None = None, +) -> dict[str, Any]: + """Per-topic bucket series for the top-N visible topics in range. + + Reuses the Chronicle topics-lane ranked CTE shape with parameterized ``top``. + ``mode_hint`` is always ``absolute`` — normalization is client-side. + """ + top_n = max(1, min(int(top), _RIVER_MAX_TOP)) + vf = _parse_viewport_dt(viewport_from) + vt = _parse_viewport_dt(viewport_to) + if not (vt > vf): + raise HTTPException(status_code=422, detail="to must be after from") + resolved_unit = _resolve_river_unit(unit, vf, vt) + scope = scope or QueryScope() + scope_conds, scope_params = scope_filters(scope) + + # Bare emails columns are unambiguous (members/topics lack those names). + email_where = [ + "date IS NOT NULL", + "date >= %(viewport_from)s", + "date < %(viewport_to)s", + *scope_conds, + ] + + sql = f""" + WITH filtered AS ( + SELECT m.topic_id, e.date + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE t.hidden = FALSE + AND {" AND ".join(email_where)} + ), + ranked AS ( + SELECT topic_id, count(*)::int AS total + FROM filtered + GROUP BY topic_id + ORDER BY total DESC, topic_id + LIMIT %(top_limit)s + ), + bucketed AS ( + SELECT + f.topic_id, + date_trunc(%(unit)s, f.date) AS bucket, + count(*)::int AS count + FROM filtered f + JOIN ranked r ON r.topic_id = f.topic_id + GROUP BY f.topic_id, 2 + ) + SELECT + r.topic_id::text, + t.label, + t.origin, + b.bucket, + coalesce(b.count, 0)::int AS count, + r.total + FROM ranked r + JOIN app_topics t ON t.id = r.topic_id + LEFT JOIN bucketed b ON b.topic_id = r.topic_id + ORDER BY r.total DESC, r.topic_id, b.bucket + """ + params = { + **scope_params, + "unit": resolved_unit, + "viewport_from": vf.isoformat(), + "viewport_to": vt.isoformat(), + "top_limit": top_n, + } + with pool.connection() as conn: + rows = conn.execute(sql, params).fetchall() + + by_topic: dict[str, dict[str, Any]] = {} + order: list[str] = [] + for row in rows: + topic_id = str(row[0]) + label = str(row[1]) if row[1] is not None else topic_id + origin = str(row[2]) + if topic_id not in by_topic: + by_topic[topic_id] = { + "topic_id": topic_id, + "label": label, + "origin": origin, + "buckets": [], + } + order.append(topic_id) + bucket = _iso(row[3]) + if bucket is None: + continue + by_topic[topic_id]["buckets"].append({"bucket": bucket, "count": int(row[4])}) + + return { + "unit": resolved_unit, + "mode_hint": "absolute", + "from": _iso(vf), + "to": _iso(vt), + "topics": [by_topic[tid] for tid in order], + } + + +def topics_matrix( + pool: ConnectionPool, + *, + by: str = "year", + scope: QueryScope | None = None, +) -> dict[str, Any]: + """Visible topics × year member counts (one grouped statement) + totals.""" + if by != "year": + raise HTTPException(status_code=422, detail="only by=year is supported") + scope = scope or QueryScope() + scope_conds, scope_params = scope_filters(scope) + extra_sql = (" AND " + " AND ".join(scope_conds)) if scope_conds else "" + + sql = f""" + SELECT + t.id::text AS topic_id, + t.label, + t.origin, + extract(year FROM e.date)::int AS year, + count(*)::int AS count + FROM app_topics t + JOIN app_topic_members m ON m.topic_id = t.id + JOIN emails e ON e.id = m.email_id + WHERE t.hidden = FALSE + AND e.date IS NOT NULL + {extra_sql} + GROUP BY t.id, t.label, t.origin, extract(year FROM e.date) + ORDER BY t.member_count DESC, t.label ASC, year ASC + """ + with pool.connection() as conn: + rows = conn.execute(sql, scope_params).fetchall() + + # topic_id → {label, origin, cells: {year: count}} + topics_map: dict[str, dict[str, Any]] = {} + years_set: set[int] = set() + for row in rows: + tid = str(row[0]) + year = int(row[3]) + count = int(row[4]) + years_set.add(year) + if tid not in topics_map: + topics_map[tid] = { + "topic_id": tid, + "label": str(row[1]), + "origin": str(row[2]), + "cells": {}, + } + topics_map[tid]["cells"][str(year)] = count + + years = sorted(years_set) + rows_out: list[dict[str, Any]] = [] + col_totals: dict[str, int] = {str(y): 0 for y in years} + grand_total = 0 + for tid, info in topics_map.items(): + row_total = 0 + cells: dict[str, int] = {} + for y in years: + c = int(info["cells"].get(str(y), 0)) + cells[str(y)] = c + row_total += c + col_totals[str(y)] = col_totals[str(y)] + c + grand_total += row_total + rows_out.append( + { + "topic_id": tid, + "label": info["label"], + "origin": info["origin"], + "cells": cells, + "row_total": row_total, + } + ) + + return { + "by": "year", + "columns": [str(y) for y in years], + "rows": rows_out, + "column_totals": col_totals, + "grand_total": grand_total, + } + + +def topics_projection(pool: ConnectionPool) -> dict[str, Any]: + """Centroid PCA (numpy SVD) of automatic+curated topics to 2D. + + Manual topics (no centroid) are excluded; response notes the count. + NEVER returns per-source points (TA-003 — LOD stops at topics). + """ + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, label, origin, member_count, centroid + FROM app_topics + WHERE hidden = FALSE + AND centroid IS NOT NULL + AND origin IN ('automatic', 'curated') + ORDER BY member_count DESC, label ASC, id ASC + """ + ).fetchall() + excluded_row = conn.execute( + """ + SELECT count(*)::int + FROM app_topics + WHERE hidden = FALSE + AND (centroid IS NULL OR origin = 'manual') + """ + ).fetchone() + excluded_manual = int(excluded_row[0]) if excluded_row else 0 + + topic_meta: list[tuple[str, str, str, int]] = [] + vectors: list[list[float]] = [] + for row in rows: + emb = parse_embedding(row[4]) + if emb is None or len(emb) != _EMBED_DIM: + excluded_manual += 1 + continue + topic_meta.append((str(row[0]), str(row[1]), str(row[2]), int(row[3]))) + vectors.append(emb) + + points: list[dict[str, Any]] = [] + if vectors: + coords = pca_project_2d(np.asarray(vectors, dtype=np.float64)) + for i, (tid, label, origin, mcount) in enumerate(topic_meta): + points.append( + { + "topic_id": tid, + "label": label, + "origin": origin, + "member_count": mcount, + "x": float(coords[i, 0]), + "y": float(coords[i, 1]), + } + ) + + note = None + if excluded_manual > 0: + note = ( + f"{excluded_manual} topic(s) excluded (manual or missing centroid); " + "projection is topic-level only — no per-source points (TA-003)." + ) + else: + note = "Projection is topic-level only — no per-source points (TA-003)." + + return { + "points": points, + "excluded_without_centroid": excluded_manual, + "note": note, + } + + +def _parse_members_cursor(token: str, secret_key: str) -> tuple[str | None, UUID]: + try: + payload = decode_cursor(token, secret_key) + except ValueError as exc: + raise HTTPException(status_code=400, detail="invalid cursor") from exc + if "id" not in payload: + raise HTTPException(status_code=400, detail="invalid cursor") + try: + last_id = UUID(str(payload["id"])) + except (ValueError, TypeError, AttributeError) as exc: + raise HTTPException(status_code=400, detail="invalid cursor") from exc + d = payload.get("d") + if d is not None and not isinstance(d, str): + raise HTTPException(status_code=400, detail="invalid cursor") + return d if isinstance(d, str) else None, last_id + + +def list_topic_members( + pool: ConnectionPool, + topic_id: UUID, + *, + secret_key: str, + cursor: str | None = None, + limit: int = _MEMBERS_DEFAULT_LIMIT, +) -> dict[str, Any]: + """Paginated definitive member list for a topic (keyset on date DESC, id DESC). + + Atlas "Open sources" uses this instead of Research Desk topic scoping + (Phase 5 polish). TA-004 — every aggregated view can open this list. + """ + lim = max(1, min(int(limit), _MEMBERS_MAX_LIMIT)) + with pool.connection() as conn: + trow = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if trow is None: + raise HTTPException(status_code=404, detail="Topic not found") + + conditions = ["m.topic_id = %(topic_id)s"] + params: dict[str, Any] = {"topic_id": topic_id, "lim": lim + 1} + + if cursor: + cursor_d, cursor_id = _parse_members_cursor(cursor, secret_key) + params["cursor_id"] = cursor_id + if cursor_d is not None: + params["cursor_d"] = cursor_d + # DESC NULLS LAST: after (d, id) come earlier dates, same-date lower ids, + # then all NULL dates. + conditions.append( + "(" + " (e.date IS NOT NULL AND (e.date < %(cursor_d)s" + " OR (e.date = %(cursor_d)s AND e.id < %(cursor_id)s)))" + " OR e.date IS NULL" + ")" + ) + else: + conditions.append("e.date IS NULL AND e.id < %(cursor_id)s") + + where_sql = " AND ".join(conditions) + rows = conn.execute( + f""" + SELECT e.id, e.subject, e.sender_name, e.sender_address, + e.date, e.source_account, e.thread_id, m.distance + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + WHERE {where_sql} + ORDER BY e.date DESC NULLS LAST, e.id DESC + LIMIT %(lim)s + """, + params, + ).fetchall() + + page = rows[:lim] + has_more = len(rows) > lim + items = [ + { + "id": encode_source_id("msg", r[0]), + "subject": r[1], + "sender_name": r[2], + "sender_address": r[3], + "date": _iso(r[4]), + "mailbox": r[5], + "thread_id": r[6], + "distance": float(r[7]) if r[7] is not None else None, + } + for r in page + ] + next_cursor: str | None = None + if has_more and page: + last = page[-1] + next_cursor = encode_cursor( + {"d": _iso(last[4]), "id": str(last[0])}, + secret_key, + ) + return {"items": items, "next_cursor": next_cursor} + + +# --- routes --- + + +@router.post("/topics/generate", response_model=GenerateResponse) +def topics_generate( + body: GenerateRequest, + request: Request, + user: str = Depends(require_user), +) -> GenerateResponse: + """Cluster sample + full-corpus assign. Synchronous; may take minutes.""" + pool: ConnectionPool = request.app.state.pool + settings: ChronicleSettings = request.app.state.settings + gateway = _gateway_from_request(request, settings) + available = _model_available(request, gateway) + return generate_topics( + pool, + body, + username=user, + gateway=gateway if available else None, + model_available=available, + ) + + +@router.get("/topics") +def topics_list( + request: Request, + include_hidden: bool = Query(default=False), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic tree (parents + children); visible by default.""" + pool: ConnectionPool = request.app.state.pool + return {"topics": list_topics(pool, include_hidden=include_hidden)} + + +@router.get("/topics/river") +def topics_river_endpoint( + request: Request, + from_: str = Query(..., alias="from"), + to: str = Query(...), + unit: str = Query(default="auto"), + top: int = Query(default=_RIVER_DEFAULT_TOP, ge=1, le=_RIVER_MAX_TOP), + mb: str | None = Query(default=None, description="CSV mailboxes"), + sd: str | None = Query(default=None, description="CSV senders"), + df: str | None = Query(default=None, description="Scope date from"), + dt: str | None = Query(default=None, description="Scope date to"), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic river: top-N visible topics × time buckets (mode_hint=absolute).""" + pool: ConnectionPool = request.app.state.pool + mailboxes = [s.strip() for s in (mb or "").split(",") if s.strip()] + senders = [s.strip() for s in (sd or "").split(",") if s.strip()] + scope = _scope_from_query( + mailboxes=mailboxes, + senders=senders, + scope_from=df, + scope_to=dt, + ) + return topics_river( + pool, + viewport_from=from_, + viewport_to=to, + unit=unit, + top=top, + scope=scope, + ) + + +@router.get("/topics/matrix") +def topics_matrix_endpoint( + request: Request, + by: str = Query(default="year"), + mb: str | None = Query(default=None, description="CSV mailboxes"), + sd: str | None = Query(default=None, description="CSV senders"), + df: str | None = Query(default=None, description="Scope date from"), + dt: str | None = Query(default=None, description="Scope date to"), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic × year matrix with row/column totals.""" + pool: ConnectionPool = request.app.state.pool + mailboxes = [s.strip() for s in (mb or "").split(",") if s.strip()] + senders = [s.strip() for s in (sd or "").split(",") if s.strip()] + scope = _scope_from_query( + mailboxes=mailboxes, + senders=senders, + scope_from=df, + scope_to=dt, + ) + return topics_matrix(pool, by=by, scope=scope) + + +@router.get("/topics/projection") +def topics_projection_endpoint( + request: Request, + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Centroid PCA to 2D; topics only — never per-source points (TA-003).""" + pool: ConnectionPool = request.app.state.pool + return topics_projection(pool) + + +@router.post("/topics") +def topics_create( + body: TopicCreate, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Create a manual topic.""" + pool: ConnectionPool = request.app.state.pool + return create_manual_topic(pool, body, username=user) + + +@router.get("/topics/{topic_id}/members") +def topics_members_list( + topic_id: UUID, + request: Request, + cursor: str | None = Query(default=None), + limit: int = Query(default=_MEMBERS_DEFAULT_LIMIT, ge=1, le=_MEMBERS_MAX_LIMIT), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Paginated definitive member list (keyset). Atlas TA-004 source list.""" + pool: ConnectionPool = request.app.state.pool + secret_key: str = request.app.state.settings.secret_key + return list_topic_members( + pool, + topic_id, + secret_key=secret_key, + cursor=cursor, + limit=limit, + ) + + +@router.get("/topics/{topic_id}") +def topics_get( + topic_id: UUID, + request: Request, + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic detail, monthly activity, representative members.""" + pool: ConnectionPool = request.app.state.pool + return get_topic_detail(pool, topic_id) + + +@router.patch("/topics/{topic_id}") +def topics_patch( + topic_id: UUID, + body: TopicPatch, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Rename/describe/hide/parent; flips automatic → curated.""" + pool: ConnectionPool = request.app.state.pool + return patch_topic(pool, topic_id, body, username=user) + + +@router.delete("/topics/{topic_id}") +def topics_delete( + topic_id: UUID, + request: Request, + user: str = Depends(require_user), +) -> dict[str, str]: + """Hard-delete manual topics only.""" + pool: ConnectionPool = request.app.state.pool + return delete_topic(pool, topic_id, username=user) + + +@router.post("/topics/{topic_id}/members") +def topics_member_add( + topic_id: UUID, + body: MemberAddRequest, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Manual membership include.""" + pool: ConnectionPool = request.app.state.pool + return add_member(pool, topic_id, body.email_sid, username=user) + + +@router.delete("/topics/{topic_id}/members/{email_sid}") +def topics_member_remove( + topic_id: UUID, + email_sid: str, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Remove membership (manual exclude).""" + pool: ConnectionPool = request.app.state.pool + return remove_member(pool, topic_id, email_sid, username=user) diff --git a/apps/chronicle/server/tests/test_chronicle.py b/apps/chronicle/server/tests/test_chronicle.py index 863ef67..013e8d3 100644 --- a/apps/chronicle/server/tests/test_chronicle.py +++ b/apps/chronicle/server/tests/test_chronicle.py @@ -510,6 +510,16 @@ def test_buckets_request_accepts_events_lane() -> None: assert "events" in req.lanes +def test_buckets_request_accepts_topics_lane() -> None: + req = BucketsRequest.model_validate( + { + "viewport": {"from": "2020-01-01", "to": "2021-01-01"}, + "lanes": ["messages", "topics"], + } + ) + assert "topics" in req.lanes + + def _cleanup_events_for_lane(pool: ConnectionPool) -> None: with pool.connection() as conn: conn.execute("DELETE FROM app_events") @@ -645,6 +655,198 @@ def test_events_lane_cap_and_dismissed_exclusion( _cleanup_events_for_lane(db_pool) +def _cleanup_topics_lane(pool: ConnectionPool, email_ids: list[Any]) -> None: + with pool.connection() as conn: + conn.execute("DELETE FROM app_topic_members") + conn.execute("DELETE FROM app_topics") + for eid in email_ids: + conn.execute("DELETE FROM emails WHERE id = %(id)s", {"id": eid}) + conn.commit() + + +def test_topics_lane_top6_and_viewport(db_client: TestClient, db_pool: ConnectionPool) -> None: + """Topics lane: top-6 by volume, viewport restriction, origin on each series.""" + email_ids: list[Any] = [] + try: + with db_pool.connection() as conn: + # 8 topics; volumes 1..8 so top-6 drops the two lowest. + topic_ids = [] + for i in range(8): + row = conn.execute( + """ + INSERT INTO app_topics (label, origin, hidden, member_count) + VALUES (%(label)s, 'automatic', FALSE, 0) + RETURNING id + """, + {"label": f"TopicLane {i}"}, + ).fetchone() + assert row is not None + topic_ids.append(row[0]) + + # Hidden topic with high volume must not appear. + hidden_row = conn.execute( + """ + INSERT INTO app_topics (label, origin, hidden, member_count) + VALUES ('Hidden High', 'automatic', TRUE, 0) + RETURNING id + """ + ).fetchone() + assert hidden_row is not None + hidden_id = hidden_row[0] + + for i, tid in enumerate(topic_ids): + vol = i + 1 # topic 7 has most + for n in range(vol): + eid = uuid4() + email_ids.append(eid) + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, labels, source_account, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, 'topics lane', + 'A', 'a@example.com', 'example.com', + '{}'::jsonb, %(date)s, 'b', NULL, + false, %(labels)s, 'topics-lane@example.com', now() + ) + """, + { + "id": eid, + "mid": f"", + "tid": f"thr-{eid}", + "date": datetime(2020, 3, 1 + (n % 20), 12, 0, tzinfo=UTC), + "labels": ["INBOX"], + }, + ) + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(tid)s, %(eid)s, 0.1, 'automatic') + """, + {"tid": tid, "eid": eid}, + ) + + # Hidden: 50 members inside viewport + for _n in range(50): + eid = uuid4() + email_ids.append(eid) + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, labels, source_account, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, 'hidden', + 'A', 'a@example.com', 'example.com', + '{}'::jsonb, %(date)s, 'b', NULL, + false, %(labels)s, 'topics-lane@example.com', now() + ) + """, + { + "id": eid, + "mid": f"", + "tid": f"thr-{eid}", + "date": datetime(2020, 5, 1, 12, 0, tzinfo=UTC), + "labels": ["INBOX"], + }, + ) + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(tid)s, %(eid)s, 0.1, 'automatic') + """, + {"tid": hidden_id, "eid": eid}, + ) + + # Outside-viewport member for topic 7 only — should not count in ranking. + eid_out = uuid4() + email_ids.append(eid_out) + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, labels, source_account, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, 'outside', + 'A', 'a@example.com', 'example.com', + '{}'::jsonb, %(date)s, 'b', NULL, + false, %(labels)s, 'topics-lane@example.com', now() + ) + """, + { + "id": eid_out, + "mid": f"", + "tid": f"thr-{eid_out}", + "date": datetime(2018, 1, 1, 12, 0, tzinfo=UTC), + "labels": ["INBOX"], + }, + ) + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(tid)s, %(eid)s, 0.1, 'automatic') + """, + {"tid": topic_ids[7], "eid": eid_out}, + ) + conn.commit() + + _login(db_client) + payload = { + "scope": {"mailboxes": ["topics-lane@example.com"]}, + "viewport": {"from": "2020-01-01T00:00:00Z", "to": "2021-01-01T00:00:00Z"}, + "pixel_width": 920, + "aggregation": "month", + "lanes": ["topics"], + } + r = db_client.post("/api/chronicle/buckets", json=payload) + assert r.status_code == 200, r.text + lane = r.json()["lanes"]["topics"] + assert isinstance(lane, dict) + topics = lane["topics"] + assert isinstance(topics, list) + assert len(topics) <= 6 + assert len(topics) == 6 + # Hidden excluded + labels = {t["label"] for t in topics} + assert "Hidden High" not in labels + # Lowest volumes dropped (TopicLane 0 and 1) + assert "TopicLane 0" not in labels + assert "TopicLane 1" not in labels + assert "TopicLane 7" in labels + # Ranked by volume desc; origin present + totals = [] + for t in topics: + assert "topic_id" in t + assert "label" in t + assert "origin" in t + assert isinstance(t["buckets"], list) + totals.append(sum(pt["count"] for pt in t["buckets"])) + assert totals == sorted(totals, reverse=True) + + # Outside viewport → empty + r_empty = db_client.post( + "/api/chronicle/buckets", + json={ + **payload, + "viewport": { + "from": "2010-01-01T00:00:00Z", + "to": "2011-01-01T00:00:00Z", + }, + }, + ) + assert r_empty.status_code == 200 + assert r_empty.json()["lanes"]["topics"]["topics"] == [] + finally: + _cleanup_topics_lane(db_pool, email_ids) + + # --- compare mode --- diff --git a/apps/chronicle/server/tests/test_files.py b/apps/chronicle/server/tests/test_files.py index 4973459..a36b20d 100644 --- a/apps/chronicle/server/tests/test_files.py +++ b/apps/chronicle/server/tests/test_files.py @@ -6,7 +6,15 @@ from typing import TYPE_CHECKING, Any from uuid import uuid4 -from chronicle_server.files import CONTENT_TYPE_FAMILY_PATTERNS, _match_magic +import pytest + +from chronicle_server.files import ( + CONTENT_TYPE_FAMILY_PATTERNS, + _match_magic, + amount_changes_from_hunks, + diff_lines, + filename_stem, +) from chronicle_server.ids import encode_source_id from tests.conftest import PASSWORD, USERNAME @@ -27,6 +35,8 @@ def test_attachments_require_auth(client: TestClient) -> None: assert client.post("/api/attachments/list", json={}).status_code == 401 assert client.get("/api/attachments/att_1/preview").status_code == 401 assert client.get("/api/attachments/att_1/download").status_code == 401 + assert client.get("/api/attachments/att_1/family").status_code == 401 + assert client.get("/api/attachments/compare?a=att_1&b=att_2").status_code == 401 # --- unit: family mapping + magic --- @@ -71,10 +81,12 @@ def _seed_attachment( reason: str | None = None, markdown: str | None = "extracted text", email_id: Any | None = None, + thread_id: str | None = None, ) -> dict[str, Any]: eid = email_id or uuid4() message_id = f"" sha = sha256 or hashlib.sha256(f"{eid}:{filename}:{storage_path}".encode()).hexdigest() + tid = thread_id if thread_id is not None else f"thread-{eid}" with pool.connection() as conn: # Email may already exist when linking another attachment. @@ -97,7 +109,7 @@ def _seed_attachment( { "id": eid, "mid": message_id, - "tid": f"thread-{eid}", + "tid": tid, "subject": subject, "sname": sender_name, "saddr": sender_address, @@ -610,3 +622,309 @@ def test_download_disposition_and_audit( assert detail.get("attachment_id") == seed["att_sid"] finally: _cleanup_seeds(db_pool, [seed]) + + +# --- filename_stem (Table 25 / FI-004) --- + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("Final_Roof_Estimate_v3.pdf", "final_roof_estimate"), + ("final_roof_estimate.pdf", "final_roof_estimate"), + ("roof_photos.zip", "roof_photos"), + ("invoice_final.docx", "invoice"), + ("invoice-draft.pdf", "invoice"), + ("report copy.pdf", "report"), + ("report (2).pdf", "report"), + ("report(3).xlsx", "report"), + ("Budget_v12_final.pdf", "budget"), + ("notes.txt", "notes"), + ("UPPER_CASE_V2.PDF", "upper_case"), + ("file-v1-copy-final.pdf", "file"), + ("noext", "noext"), + ("", ""), + ("estimate_copy_2.pdf", "estimate"), + ], +) +def test_filename_stem_table(filename: str, expected: str) -> None: + assert filename_stem(filename) == expected + + +def test_filename_stem_equivalence_examples() -> None: + assert filename_stem("Final_Roof_Estimate_v3.pdf") == filename_stem("final_roof_estimate.pdf") + assert filename_stem("Final_Roof_Estimate_v3.pdf") != filename_stem("roof_photos.zip") + + +# --- diff_lines + amount_changes (pure) --- + + +def test_diff_lines_basic_hunks() -> None: + a = "line1\nline2\nline3\n" + b = "line1\nline2 changed\nline3\nline4\n" + result = diff_lines(a, b, context=2) + assert "hunks" in result + assert result["truncated"] is False + assert len(result["hunks"]) >= 1 + kinds = {ln["kind"] for h in result["hunks"] for ln in h["lines"]} + assert "del" in kinds + assert "add" in kinds + # same context lines present + assert any( + ln["kind"] == "same" and ln["text"] == "line1" for h in result["hunks"] for ln in h["lines"] + ) + + +def test_diff_lines_identical() -> None: + result = diff_lines("a\nb\n", "a\nb\n", context=2) + # SequenceMatcher get_grouped_opcodes yields no groups when fully equal + assert result["hunks"] == [] + assert result["truncated"] is False + + +def test_diff_lines_hunk_cap() -> None: + # Many independent one-line changes → many hunks when context=0 + a_lines = [f"keep-{i}" if i % 2 == 0 else f"old-{i}" for i in range(500)] + b_lines = [f"keep-{i}" if i % 2 == 0 else f"new-{i}" for i in range(500)] + result = diff_lines("\n".join(a_lines), "\n".join(b_lines), context=0) + assert len(result["hunks"]) <= 200 + if len(result["hunks"]) == 200: + assert result["truncated"] is True + + +@pytest.mark.parametrize( + ("text", "expect_match"), + [ + ("Total: $1,234.56", True), + ("€ 99", True), + ("£5000", True), + ("price 42", True), + ("no money here", False), + ("$ 12.00 due", True), + # Pattern matches the digit "3" inside "v3" + ("version v3 only", True), + ("just words", False), + ], +) +def test_amount_regex_on_changed_lines(text: str, expect_match: bool) -> None: + hunks = [{"a_start": 0, "b_start": 0, "lines": [{"kind": "add", "text": text}]}] + changes = amount_changes_from_hunks(hunks) + if expect_match: + assert len(changes) == 1 + assert changes[0]["amounts"] + else: + assert changes == [] + + +def test_amount_changes_from_diff() -> None: + a = "Quote total: $1,000\nNotes\n" + b = "Quote total: $1,250.00\nNotes\nExtra €40 fee\n" + diff = diff_lines(a, b, context=1) + amounts = amount_changes_from_hunks(diff["hunks"]) + joined = " ".join(c["text"] for c in amounts) + found = " ".join(x for c in amounts for x in c["amounts"]) + assert "$1,000" in joined or "$1,000" in found + assert "1,250" in joined or "1,250" in found + assert "€40" in joined or "40" in found + assert all(c["kind"] in ("add", "del") for c in amounts) + + +# --- version family endpoint --- + + +def test_family_statement_shape_and_confidence( + db_pool: ConnectionPool, db_client: TestClient +) -> None: + shared_thread = "thread-roof-family" + # v3 estimate (seed) — self is exact-duplicate (same sha256 as seed) + s1 = _seed_attachment( + db_pool, + filename="Final_Roof_Estimate_v3.pdf", + content_type="application/pdf", + storage_path="fam/v3.pdf", + sha256=hashlib.sha256(b"estimate-v3-content").hexdigest(), + sender_address="contractor@example.com", + sender_name="Contractor", + date="2015-06-01T12:00:00+00:00", + thread_id=shared_thread, + markdown="Estimate total: $10,000", + ) + # earlier version same sender, different content (probable-version; never merged) + s2 = _seed_attachment( + db_pool, + filename="final_roof_estimate.pdf", + content_type="application/pdf", + storage_path="fam/v1.pdf", + sha256=hashlib.sha256(b"estimate-v1-content").hexdigest(), + sender_address="contractor@example.com", + sender_name="Contractor", + date="2015-05-01T12:00:00+00:00", + thread_id="thread-other-1", + markdown="Estimate total: $9,500", + ) + # same thread, version via thread signal + s3 = _seed_attachment( + db_pool, + filename="Final_Roof_Estimate_v2.pdf", + content_type="application/pdf", + storage_path="fam/v2.pdf", + sha256=hashlib.sha256(b"estimate-v2-content").hexdigest(), + sender_address="other-on-thread@example.com", + sender_name="Other", + date="2015-05-15T12:00:00+00:00", + thread_id=shared_thread, + markdown="Estimate total: $9,800", + ) + # same stem shape but unrelated sender/thread — must NOT join family + s4 = _seed_attachment( + db_pool, + filename="final_roof_estimate_draft.pdf", + content_type="application/pdf", + storage_path="fam/unrelated.pdf", + sha256=hashlib.sha256(b"unrelated").hexdigest(), + sender_address="stranger@example.com", + sender_name="Stranger", + date="2015-07-01T12:00:00+00:00", + thread_id="thread-stranger", + markdown="nope", + ) + # different stem entirely + s5 = _seed_attachment( + db_pool, + filename="roof_photos.zip", + content_type="application/zip", + storage_path="fam/photos.zip", + sha256=hashlib.sha256(b"photos").hexdigest(), + sender_address="contractor@example.com", + sender_name="Contractor", + date="2015-06-03T12:00:00+00:00", + thread_id=shared_thread, + markdown=None, + status="failed", + reason="unsupported", + ) + seeds = [s1, s2, s3, s4, s5] + try: + _login(db_client) + r = db_client.get(f"/api/attachments/{s1['att_sid']}/family") + assert r.status_code == 200 + body = r.json() + assert body["id"] == s1["att_sid"] + assert body["stem"] == "final_roof_estimate" + ids = {c["id"] for c in body["candidates"]} + assert s1["att_sid"] in ids + assert s2["att_sid"] in ids + assert s3["att_sid"] in ids + assert s4["att_sid"] not in ids + assert s5["att_sid"] not in ids + # Distinct content → separate candidates (never auto-collapsed) + assert len(ids) == 3 + + by_id = {c["id"]: c for c in body["candidates"]} + # seed self: exact-duplicate + assert by_id[s1["att_sid"]]["confidence"] == "exact-duplicate" + assert "sha256" in by_id[s1["att_sid"]]["signals"] + + # s2: different content → probable-version via sender + assert by_id[s2["att_sid"]]["confidence"] == "probable-version" + assert "stem" in by_id[s2["att_sid"]]["signals"] + assert "sender" in by_id[s2["att_sid"]]["signals"] + assert by_id[s2["att_sid"]]["sha256"] != by_id[s1["att_sid"]]["sha256"] + + # s3: probable-version via thread + assert by_id[s3["att_sid"]]["confidence"] == "probable-version" + assert "thread" in by_id[s3["att_sid"]]["signals"] + + for c in body["candidates"]: + assert "filename" in c + assert "date" in c + assert "sender" in c + assert "size" in c + assert "sha256" in c + assert c["confidence"] in ("exact-duplicate", "probable-version") + assert isinstance(c["signals"], list) + + # list surfaces family_count > 1 cheaply + r_list = db_client.post( + "/api/attachments/list", + json={"filters": {"filename": "Roof_Estimate"}}, + ) + assert r_list.status_code == 200 + items = r_list.json()["items"] + target_ids = {s1["att_sid"], s2["att_sid"], s3["att_sid"]} + fam_items = [it for it in items if it["id"] in target_ids] + assert fam_items + assert all(it.get("family_count", 1) > 1 for it in fam_items) + finally: + _cleanup_seeds(db_pool, seeds) + + +def test_compare_diff_and_amounts(db_pool: ConnectionPool, db_client: TestClient) -> None: + s_a = _seed_attachment( + db_pool, + filename="quote_v1.pdf", + content_type="application/pdf", + storage_path="cmp/a.pdf", + sha256=hashlib.sha256(b"cmp-a").hexdigest(), + date="2015-01-01T12:00:00+00:00", + markdown="Roof quote\nTotal: $8,000\nNotes: draft\n", + ) + s_b = _seed_attachment( + db_pool, + filename="quote_v2.pdf", + content_type="application/pdf", + storage_path="cmp/b.pdf", + sha256=hashlib.sha256(b"cmp-b").hexdigest(), + date="2015-02-01T12:00:00+00:00", + markdown="Roof quote\nTotal: $9,500.00\nNotes: final\nExtra fee: €120\n", + ) + try: + _login(db_client) + r = db_client.get(f"/api/attachments/compare?a={s_a['att_sid']}&b={s_b['att_sid']}") + assert r.status_code == 200 + body = r.json() + assert body["a"]["id"] == s_a["att_sid"] + assert body["b"]["id"] == s_b["att_sid"] + assert body["a"]["filename"] == "quote_v1.pdf" + assert body["b"]["filename"] == "quote_v2.pdf" + assert "sha256" in body["a"] + assert isinstance(body["hunks"], list) + assert body["truncated"] is False + # At least one del and one add across hunks + kinds = {ln["kind"] for h in body["hunks"] for ln in h["lines"]} + assert "del" in kinds + assert "add" in kinds + assert body["amount_changes"] + amount_texts = " ".join(c["text"] for c in body["amount_changes"]) + assert "$8,000" in amount_texts or "8,000" in amount_texts + assert "$9,500" in amount_texts or "9,500" in amount_texts + finally: + _cleanup_seeds(db_pool, [s_a, s_b]) + + +def test_compare_404_missing_extraction(db_pool: ConnectionPool, db_client: TestClient) -> None: + s_ok = _seed_attachment( + db_pool, + filename="ok.pdf", + content_type="application/pdf", + storage_path="cmp/ok.pdf", + markdown="hello", + status="extracted", + ) + s_fail = _seed_attachment( + db_pool, + filename="fail.pdf", + content_type="application/pdf", + storage_path="cmp/fail.pdf", + markdown=None, + status="failed", + reason="timeout", + ) + try: + _login(db_client) + r = db_client.get(f"/api/attachments/compare?a={s_ok['att_sid']}&b={s_fail['att_sid']}") + assert r.status_code == 404 + detail = r.json().get("detail", "") + assert "missing extraction" in detail.lower() or "timeout" in detail.lower() + finally: + _cleanup_seeds(db_pool, [s_ok, s_fail]) diff --git a/apps/chronicle/server/tests/test_health.py b/apps/chronicle/server/tests/test_health.py index 7afdf76..e561fc1 100644 --- a/apps/chronicle/server/tests/test_health.py +++ b/apps/chronicle/server/tests/test_health.py @@ -56,6 +56,13 @@ def _assert_health_shape(body: dict[str, Any]) -> None: _assert_nonneg_int(emb[bucket]["embedded"]) _assert_nonneg_int(emb[bucket]["missing"]) + assert "topics" in body + topics = body["topics"] + _assert_nonneg_int(topics["topics"]) + assert isinstance(topics["coverage"], (int, float)) + assert topics["coverage"] >= 0 + assert "last_generated" in topics # may be null + assert "imports" in body assert isinstance(body["imports"], list) assert len(body["imports"]) <= 20 diff --git a/apps/chronicle/server/tests/test_people.py b/apps/chronicle/server/tests/test_people.py new file mode 100644 index 0000000..e243745 --- /dev/null +++ b/apps/chronicle/server/tests/test_people.py @@ -0,0 +1,836 @@ +# tests/test_people.py +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +from chronicle_server.cursor import encode_cursor +from chronicle_server.ids import encode_source_id +from chronicle_server.people import ( + _activity_buckets, + _ego_graph, + _enrich_card, + _top_topics, +) +from tests.conftest import PASSWORD, USERNAME + +if TYPE_CHECKING: + from fastapi.testclient import TestClient + from psycopg_pool import ConnectionPool + + +def _login(client: TestClient) -> None: + r = client.post("/api/auth/login", json={"username": USERNAME, "password": PASSWORD}) + assert r.status_code == 200 + + +def _sample_card(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "id": str(uuid4()), + "display_name": "Alice Example", + "kind": "human", + "kind_source": "manual", + "tags": ["vip"], + "human_probability": 0.85, + "addresses": ["alice@example.com"], + "name_variants": ["Alice"], + "messages_from": 10, + "messages_to": 3, + "first_seen": datetime(2015, 1, 1, tzinfo=UTC), + "last_seen": datetime(2018, 6, 1, tzinfo=UTC), + "notes": "notes", + "metadata": {}, + "classification_signals": {"bidirectional": 0.15, "personal_name": 0.1}, + "classified_at": datetime(2020, 1, 1, tzinfo=UTC), + } + base.update(overrides) + return base + + +# --- auth --- + + +def test_people_list_requires_auth(client: TestClient) -> None: + r = client.get("/api/people") + assert r.status_code == 401 + + +def test_people_get_requires_auth(client: TestClient) -> None: + r = client.get(f"/api/people/{uuid4()}") + assert r.status_code == 401 + + +def test_people_merge_requires_auth(client: TestClient) -> None: + r = client.post( + "/api/people/merge", + json={"source_id": str(uuid4()), "target_id": str(uuid4())}, + ) + assert r.status_code == 401 + + +# --- proxy shapes (stub pool + mocked MailDB) --- + + +def test_people_list_proxy_shape(client: TestClient, stub_pool: MagicMock) -> None: + card = _sample_card() + mock_db = MagicMock() + mock_db.contacts_search.return_value = ([card], 1) + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get("/api/people?q=Alice&limit=10") + assert r.status_code == 200, r.text + body = r.json() + assert "items" in body + assert body["total"] == 1 + assert body["offset"] == 0 + assert body["limit"] == 10 + assert body["items"][0]["display_name"] == "Alice Example" + assert body["items"][0]["first_seen"].endswith("Z") or "2015" in body["items"][0]["first_seen"] + kwargs = mock_db.contacts_search.call_args.kwargs + assert kwargs["query"] == "Alice" + assert kwargs["include_total"] is True + assert kwargs["limit"] == 10 + assert kwargs["offset"] == 0 + + +def test_people_list_cursor_offset_and_no_total_on_later_page( + client: TestClient, +) -> None: + card = _sample_card() + mock_db = MagicMock() + mock_db.contacts_search.return_value = ([card], None) + secret = "test-secret-key-not-for-production" + cursor = encode_cursor({"offset": 50}, secret) + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people?cursor={cursor}&limit=50") + assert r.status_code == 200 + body = r.json() + assert body["total"] is None + assert body["offset"] == 50 + kwargs = mock_db.contacts_search.call_args.kwargs + assert kwargs["offset"] == 50 + assert kwargs["include_total"] is False + + +def test_people_list_invalid_kind_422(client: TestClient) -> None: + with patch("chronicle_server.people._maildb", return_value=MagicMock()): + _login(client) + r = client.get("/api/people?kind=robot") + assert r.status_code == 422 + + +def test_people_list_needs_review(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.contacts_search.return_value = ([], 0) + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get("/api/people?needs_review=true") + assert r.status_code == 200 + assert mock_db.contacts_search.call_args.kwargs["needs_review"] is True + + +def test_people_get_enriches_address_classes_and_activity(client: TestClient) -> None: + cid = uuid4() + card = _sample_card(id=str(cid), addresses=["alice@example.com", "me@owner.com"]) + mock_db = MagicMock() + mock_db.get_contact.return_value = card + + addr_rows = [ + ("alice@example.com", False, 10, 2, None, None), + ("me@owner.com", True, 0, 5, None, None), + ] + activity_rows = [ + (datetime(2016, 3, 1, tzinfo=UTC), 4), + (datetime(2016, 4, 1, tzinfo=UTC), 2), + ] + + def fake_execute(sql: str, params: dict[str, Any] | None = None) -> MagicMock: + result = MagicMock() + sql_l = sql.lower() + if "contact_addresses" in sql_l: + result.fetchall.return_value = addr_rows + result.fetchone.return_value = None + elif "date_trunc" in sql_l: + result.fetchall.return_value = activity_rows + result.fetchone.return_value = None + elif "app_topic_members" in sql_l: + result.fetchall.return_value = [] + result.fetchone.return_value = None + elif "count(distinct" in sql_l or "thread_id" in sql_l: + result.fetchone.return_value = (7,) + result.fetchall.return_value = [] + elif "contact_merges" in sql_l: + result.fetchall.return_value = [] + result.fetchone.return_value = None + else: + result.fetchall.return_value = [] + result.fetchone.return_value = None + return result + + conn = MagicMock() + conn.execute = fake_execute + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + + with ( + patch("chronicle_server.people._maildb", return_value=mock_db), + patch.object(client.app.state, "pool", pool, create=True), + ): + # Ensure pool is used from request.app.state + client.app.state.pool = pool + _login(client) + r = client.get(f"/api/people/{cid}") + + assert r.status_code == 200, r.text + body = r.json() + assert body["address_classes"]["alice@example.com"] == "external" + assert body["address_classes"]["me@owner.com"] == "owner" + assert len(body["activity"]) == 2 + assert body["activity"][0]["count"] == 4 + assert body["topics"] == [] + assert body["thread_count"] == 7 + assert "address_details" in body + assert any(d["is_user"] for d in body["address_details"]) + + +def test_people_get_404(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.get_contact.return_value = None + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people/{uuid4()}") + assert r.status_code == 404 + + +def test_people_patch_calls_update_and_audits(client: TestClient, stub_pool: MagicMock) -> None: + cid = uuid4() + card = _sample_card(id=str(cid), kind="human", kind_source="manual") + mock_db = MagicMock() + mock_db.update_contact.return_value = card + + # Enrichment queries return empty. + conn = MagicMock() + empty = MagicMock(fetchall=MagicMock(return_value=[]), fetchone=MagicMock(return_value=(0,))) + conn.execute = MagicMock(return_value=empty) + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + # audit() opens its own connection on the real pool path — use stub + track INSERT. + stub_pool.connection = connection + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.patch( + f"/api/people/{cid}", + json={"kind": "human", "display_name": "Alice Renamed", "tags": ["a"]}, + ) + assert r.status_code == 200, r.text + kwargs = mock_db.update_contact.call_args.kwargs + assert kwargs["kind"] == "human" + assert kwargs["display_name"] == "Alice Renamed" + assert kwargs["tags"] == ["a"] + # audit INSERT should have been attempted + assert conn.execute.called + + +def test_people_merge_unmerge_proxy(client: TestClient) -> None: + src = str(uuid4()) + tgt = str(uuid4()) + mid = str(uuid4()) + card = _sample_card(id=tgt, merge_id=mid) + mock_db = MagicMock() + mock_db.merge_contacts.return_value = card + mock_db.unmerge_contacts.return_value = { + "source": _sample_card(id=src), + "target": _sample_card(id=tgt), + } + + conn = MagicMock() + empty = MagicMock(fetchall=MagicMock(return_value=[]), fetchone=MagicMock(return_value=(0,))) + conn.execute = MagicMock(return_value=empty) + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + client.app.state.pool.connection = connection + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.post("/api/people/merge", json={"source_id": src, "target_id": tgt}) + assert r.status_code == 200, r.text + assert mock_db.merge_contacts.called + + r2 = client.post("/api/people/unmerge", json={"merge_id": mid}) + assert r2.status_code == 200, r2.text + assert mock_db.unmerge_contacts.called + body = r2.json() + assert "source" in body and "target" in body + + +def test_people_merge_candidates(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.merge_candidates.return_value = [ + { + "norm_name": "alice", + "a": { + "display_name": "Alice", + "primary_address": "a@x.com", + "msg_count": 5, + "contact_id": str(uuid4()), + }, + "b": { + "display_name": "A. Example", + "primary_address": "b@x.com", + "msg_count": 3, + "contact_id": str(uuid4()), + }, + } + ] + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get("/api/people/merge-candidates?limit=5") + assert r.status_code == 200 + body = r.json() + assert len(body["items"]) == 1 + mock_db.merge_candidates.assert_called_once_with(limit=5) + + +def test_activity_buckets_one_statement() -> None: + """_activity_buckets issues a single date_trunc statement.""" + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = [ + (datetime(2016, 1, 1, tzinfo=UTC), 3), + ] + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + buckets = _activity_buckets(pool, ["alice@example.com"]) + assert len(buckets) == 1 + assert buckets[0]["count"] == 3 + assert conn.execute.call_count == 1 + sql = conn.execute.call_args[0][0] + assert "date_trunc" in sql.lower() + + +def test_topics_empty_tolerance() -> None: + conn = MagicMock() + conn.execute.side_effect = RuntimeError("relation app_topics does not exist") + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + assert _top_topics(pool, ["alice@example.com"]) == [] + + +def test_owner_class_derivation() -> None: + cid = uuid4() + card = _sample_card(id=str(cid), addresses=["a@x.com", "me@x.com"]) + addr_rows = [ + ("a@x.com", False, 1, 0, None, None), + ("me@x.com", True, 0, 1, None, None), + ] + + def fake_execute(sql: str, params: dict[str, Any] | None = None) -> MagicMock: + result = MagicMock() + sql_l = sql.lower() + if "contact_addresses" in sql_l: + result.fetchall.return_value = addr_rows + elif "date_trunc" in sql_l or "app_topic" in sql_l or "contact_merges" in sql_l: + result.fetchall.return_value = [] + elif "thread" in sql_l: + result.fetchone.return_value = (0,) + else: + result.fetchall.return_value = [] + result.fetchone.return_value = (0,) + return result + + conn = MagicMock() + conn.execute = fake_execute + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + out = _enrich_card(pool, card) + assert out["address_classes"]["a@x.com"] == "external" + assert out["address_classes"]["me@x.com"] == "owner" + + +# --- DB-backed merge round-trip + audit --- + + +def _insert_throwaway_contact( + pool: ConnectionPool, + *, + display_name: str, + address: str, + is_user: bool = False, +) -> str: + cid = uuid4() + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO contacts (id, display_name, kind, kind_source, tags, notes) + VALUES (%(id)s, %(name)s, 'unknown', 'heuristic', '{}', NULL) + """, + {"id": cid, "name": display_name}, + ) + conn.execute( + """ + INSERT INTO contact_addresses ( + address, contact_id, name_variants, is_user, + first_seen, last_seen, messages_from, messages_to + ) VALUES ( + %(addr)s, %(cid)s, %(variants)s, %(is_user)s, + now(), now(), 5, 1 + ) + """, + { + "addr": address, + "cid": cid, + "variants": [display_name], + "is_user": is_user, + }, + ) + conn.commit() + return str(cid) + + +def _cleanup_contacts(pool: ConnectionPool, contact_ids: list[str], addresses: list[str]) -> None: + with pool.connection() as conn: + for cid in contact_ids: + conn.execute( + """ + DELETE FROM contact_merges + WHERE source_id = %(id)s OR target_id = %(id)s + """, + {"id": cid}, + ) + conn.execute( + "DELETE FROM contact_addresses WHERE contact_id = %(id)s", + {"id": cid}, + ) + conn.execute("DELETE FROM contacts WHERE id = %(id)s", {"id": cid}) + for addr in addresses: + conn.execute( + "DELETE FROM contact_addresses WHERE address = %(a)s", + {"a": addr}, + ) + conn.execute("DELETE FROM app_audit WHERE action LIKE 'people%%'") + conn.commit() + + +def test_merge_unmerge_roundtrip_and_audit(db_client: TestClient, db_pool: ConnectionPool) -> None: + suffix = uuid4().hex[:8] + addr_a = f"throwaway-a-{suffix}@people-test.example" + addr_b = f"throwaway-b-{suffix}@people-test.example" + name_a = f"ThrowA {suffix}" + name_b = f"ThrowB {suffix}" + ids: list[str] = [] + try: + id_a = _insert_throwaway_contact(db_pool, display_name=name_a, address=addr_a) + id_b = _insert_throwaway_contact(db_pool, display_name=name_b, address=addr_b) + ids = [id_a, id_b] + + _login(db_client) + r = db_client.post( + "/api/people/merge", + json={"source_id": id_a, "target_id": id_b}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["id"] == id_b + merge_id = body.get("merge_id") + assert merge_id + # Address classes present after enrich + assert addr_a in body["address_classes"] or any( + d["address"] == addr_a for d in body.get("address_details", []) + ) + assert "merges" in body or merge_id + + with db_pool.connection() as conn: + audit_rows = conn.execute( + """ + SELECT action, username, detail FROM app_audit + WHERE action = 'people_merge' + ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + assert audit_rows is not None + assert audit_rows[0] == "people_merge" + assert audit_rows[1] == USERNAME + + r2 = db_client.post("/api/people/unmerge", json={"merge_id": merge_id}) + assert r2.status_code == 200, r2.text + restored = r2.json() + assert restored["source"]["id"] == id_a + assert restored["target"]["id"] == id_b + + with db_pool.connection() as conn: + unmerge_rows = conn.execute( + """ + SELECT action FROM app_audit + WHERE action = 'people_unmerge' + ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + assert unmerge_rows is not None + assert unmerge_rows[0] == "people_unmerge" + + # Owner class derivation on DB card with is_user address + owner_addr = f"throwaway-owner-{suffix}@people-test.example" + id_owner = _insert_throwaway_contact( + db_pool, + display_name=f"Owner {suffix}", + address=owner_addr, + is_user=True, + ) + ids.append(id_owner) + r3 = db_client.get(f"/api/people/{id_owner}") + assert r3.status_code == 200 + card = r3.json() + assert card["address_classes"].get(owner_addr) == "owner" + finally: + _cleanup_contacts( + db_pool, + ids, + [addr_a, addr_b, f"throwaway-owner-{suffix}@people-test.example"], + ) + + +def test_people_list_db_search(db_client: TestClient, db_pool: ConnectionPool) -> None: + suffix = uuid4().hex[:8] + addr = f"list-search-{suffix}@people-test.example" + name = f"ListSearch {suffix}" + cid = _insert_throwaway_contact(db_pool, display_name=name, address=addr) + try: + _login(db_client) + r = db_client.get(f"/api/people?q={suffix}") + assert r.status_code == 200, r.text + body = r.json() + assert any(item["id"] == cid for item in body["items"]) + assert body["total"] is not None + finally: + _cleanup_contacts(db_pool, [cid], [addr]) + + +# --- ego graph --- + + +def test_people_graph_requires_auth(client: TestClient) -> None: + r = client.get(f"/api/people/{uuid4()}/graph") + assert r.status_code == 401 + + +def test_people_graph_depth_not_one_422(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.get_contact.return_value = _sample_card() + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people/{uuid4()}/graph?depth=2") + assert r.status_code == 422 + assert "depth=1" in r.json()["detail"] + + +def test_people_graph_404(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.get_contact.return_value = None + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people/{uuid4()}/graph") + assert r.status_code == 404 + + +def _graph_row( + node_id: str, + label: str, + kind: str, + shared: int, + first: datetime, + last: datetime, + threads: list[str], + total: int, +) -> tuple[Any, ...]: + return (node_id, label, kind, shared, first, last, threads, total) + + +def test_people_graph_shape_two_coparticipants(client: TestClient) -> None: + """Graph shape: ego + 2 co-participants, edges with evidence thr_* ids.""" + ego_id = uuid4() + bob_id = str(uuid4()) + card = _sample_card(id=str(ego_id), display_name="Ego Person", kind="human") + mock_db = MagicMock() + mock_db.get_contact.return_value = card + + thr_a, thr_b, thr_c = "thread-a", "thread-b", "thread-c" + rows = [ + _graph_row( + bob_id, + "Bob Co", + "human", + 3, + datetime(2014, 1, 1, tzinfo=UTC), + datetime(2016, 6, 1, tzinfo=UTC), + [thr_a, thr_b, thr_c], + 2, + ), + _graph_row( + "addr:stranger@example.com", + "stranger@example.com", + "address", + 1, + datetime(2015, 3, 1, tzinfo=UTC), + datetime(2015, 3, 2, tzinfo=UTC), + ["thread-x"], + 2, + ), + ] + + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = rows + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + + with ( + patch("chronicle_server.people._maildb", return_value=mock_db), + patch.object(client.app.state, "pool", pool, create=True), + ): + client.app.state.pool = pool + _login(client) + r = client.get(f"/api/people/{ego_id}/graph?max_nodes=25") + + assert r.status_code == 200, r.text + body = r.json() + assert body["truncated"] is False + assert body["total_coparticipants"] == 2 + assert len(body["nodes"]) == 3 # ego + 2 + ego_nodes = [n for n in body["nodes"] if n["is_ego"]] + assert len(ego_nodes) == 1 + assert ego_nodes[0]["id"] == str(ego_id) + assert ego_nodes[0]["label"] == "Ego Person" + assert any(n["id"] == bob_id and n["kind"] == "human" for n in body["nodes"]) + assert any( + n["id"] == "addr:stranger@example.com" and n["kind"] == "address" for n in body["nodes"] + ) + assert len(body["edges"]) == 2 + for edge in body["edges"]: + assert edge["source"] == str(ego_id) + assert edge["kind"] == "thread_co_participation" + assert "shared_threads" in edge + assert "first" in edge and "last" in edge + thr_ids = edge["evidence"]["thread_ids"] + assert all(tid.startswith("thr_") for tid in thr_ids) + # Encoded form matches encode_source_id + if edge["target"] == bob_id: + assert edge["shared_threads"] == 3 + assert thr_ids == [ + encode_source_id("thr", thr_a), + encode_source_id("thr", thr_b), + encode_source_id("thr", thr_c), + ] + assert edge["first"] == "2014-01-01" + assert edge["last"] == "2016-06-01" + + +def test_people_graph_ranking_cap_truncated() -> None: + """Ranking by shared_threads desc + cap sets truncated and total honesty.""" + ego_id = uuid4() + # Simulate SQL already capped to max_nodes=2 but total=5 + rows = [ + _graph_row( + str(uuid4()), + "Top", + "human", + 10, + datetime(2010, 1, 1, tzinfo=UTC), + datetime(2011, 1, 1, tzinfo=UTC), + ["t1"], + 5, + ), + _graph_row( + str(uuid4()), + "Second", + "human", + 5, + datetime(2010, 1, 1, tzinfo=UTC), + datetime(2011, 1, 1, tzinfo=UTC), + ["t2"], + 5, + ), + ] + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = rows + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + + out = _ego_graph( + pool, + contact_id=ego_id, + ego_label="Ego", + ego_kind="human", + max_nodes=2, + date_from=None, + date_to=None, + ) + assert out["truncated"] is True + assert out["total_coparticipants"] == 5 + assert len(out["edges"]) == 2 + assert out["edges"][0]["shared_threads"] >= out["edges"][1]["shared_threads"] + # SQL receives max_nodes + params = conn.execute.call_args[0][1] + assert params["max_nodes"] == 2 + + +def test_people_graph_evidence_thread_ids_bounded() -> None: + """Evidence thread ids are thr_* and capped at 20.""" + ego_id = uuid4() + raw_threads = [f"thread-{i}" for i in range(25)] + rows = [ + _graph_row( + "addr:x@y.com", + "x@y.com", + "address", + 25, + datetime(2012, 1, 1, tzinfo=UTC), + datetime(2013, 1, 1, tzinfo=UTC), + raw_threads, # server still slices to 20 + 1, + ) + ] + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = rows + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + + out = _ego_graph( + pool, + contact_id=ego_id, + ego_label="Ego", + ego_kind="human", + max_nodes=25, + date_from=None, + date_to=None, + ) + thr_ids = out["edges"][0]["evidence"]["thread_ids"] + assert len(thr_ids) == 20 + assert all(t.startswith("thr_") for t in thr_ids) + params = conn.execute.call_args[0][1] + assert params["ev_cap"] == 20 + + +def test_people_graph_date_filtering_passed_to_sql(client: TestClient) -> None: + ego_id = uuid4() + card = _sample_card(id=str(ego_id)) + mock_db = MagicMock() + mock_db.get_contact.return_value = card + + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = [] + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + + with ( + patch("chronicle_server.people._maildb", return_value=mock_db), + patch.object(client.app.state, "pool", pool, create=True), + ): + client.app.state.pool = pool + _login(client) + r = client.get( + f"/api/people/{ego_id}/graph?date_from=2014-01-01&date_to=2018-12-31&max_nodes=10" + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["nodes"][0]["is_ego"] is True + assert body["edges"] == [] + assert body["total_coparticipants"] == 0 + assert body["truncated"] is False + params = conn.execute.call_args[0][1] + assert params["date_from"] is not None + assert params["date_to"] is not None + assert params["date_from"].year == 2014 + assert params["date_to"].year == 2018 + assert params["max_nodes"] == 10 + sql = conn.execute.call_args[0][0].lower() + assert "ego_threads" in sql or "ego_addrs" in sql + assert "thread_co_participation" not in sql # edge kind is response-only + + +def test_people_graph_invalid_date_422(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.get_contact.return_value = _sample_card() + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people/{uuid4()}/graph?date_from=not-a-date") + assert r.status_code == 422 + + +def test_people_graph_max_nodes_hard_cap(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.get_contact.return_value = _sample_card() + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people/{uuid4()}/graph?max_nodes=100") + # FastAPI Query le=50 → 422 + assert r.status_code == 422 diff --git a/apps/chronicle/server/tests/test_topics.py b/apps/chronicle/server/tests/test_topics.py new file mode 100644 index 0000000..2084081 --- /dev/null +++ b/apps/chronicle/server/tests/test_topics.py @@ -0,0 +1,745 @@ +# tests/test_topics.py +from __future__ import annotations + +from collections.abc import Iterator +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import numpy as np +import pytest + +from chronicle_server.ids import encode_source_id +from chronicle_server.topics import ( + extract_top_terms, + label_from_terms, + parse_label_polish_response, + pca_project_2d, + run_kmeans, + validate_label_array, + vector_literal, +) +from tests.conftest import PASSWORD, USERNAME + +if TYPE_CHECKING: + from fastapi.testclient import TestClient + from psycopg_pool import ConnectionPool + + +def _login(client: TestClient) -> None: + r = client.post("/api/auth/login", json={"username": USERNAME, "password": PASSWORD}) + assert r.status_code == 200 + + +# --- pure k-means / labels --- + + +def test_kmeans_deterministic_same_seed() -> None: + rng = np.random.default_rng(0) + data = rng.normal(size=(40, 8)).astype(np.float64) + # Two well-separated blobs for stability. + data[20:] += 5.0 + c1, l1 = run_kmeans(data, k=2, seed=13) + c2, l2 = run_kmeans(data, k=2, seed=13) + np.testing.assert_allclose(c1, c2) + np.testing.assert_array_equal(l1, l2) + + +def test_kmeans_k_greater_than_sample_raises() -> None: + data = np.eye(3, dtype=np.float64) + with pytest.raises(ValueError, match="exceeds sample size"): + run_kmeans(data, k=5, seed=1) + + +def test_extract_top_terms_stopwords_and_min_len() -> None: + subjects = [ + "The Meeting about Renovation and Plumbing", + "RE: Renovation quote for plumbing", + "fw: meeting notes", + "a to of", + ] + terms = extract_top_terms(subjects, top_n=5) + assert "the" not in terms + assert "about" not in terms + assert "and" not in terms + assert "for" not in terms + # short tokens dropped + assert all(len(t) >= 3 for t in terms) + assert "renovation" in terms + assert "plumbing" in terms + assert label_from_terms(terms).count(",") <= 2 + + +def test_label_polish_whitelist_and_failure_fallback() -> None: + ok = validate_label_array(["House renovation", "Travel plans"], 2) + assert ok == ["House renovation", "Travel plans"] + # too many words + assert validate_label_array(["one two three four five"], 1) is None + # wrong length + assert validate_label_array(["only one"], 2) is None + # non-string + assert validate_label_array([1, 2], 2) is None + + content = 'Here you go: ["Kitchen remodel", "Family travel"]' + parsed = parse_label_polish_response(content, 2) + assert parsed == ["Kitchen remodel", "Family travel"] + assert parse_label_polish_response("not json", 2) is None + assert parse_label_polish_response('["too many words in this label"]', 1) is None + + +# --- DB helpers --- + + +def _unit_vec(i: int, dim: int = 768) -> list[float]: + v = [0.0] * dim + v[i % dim] = 1.0 + return v + + +def _insert_email_with_embedding( + pool: ConnectionPool, + *, + subject: str, + embedding: list[float], + date: datetime | None = None, + source_account: str = "topics-test@example.com", + sender_address: str = "alice@topics-test.example", +) -> dict[str, Any]: + email_id = uuid4() + message_id = f"" + tid = f"thread-topics-{email_id}" + when = date or datetime(2020, 6, 15, 12, 0, tzinfo=UTC) + emb_lit = vector_literal(embedding) + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, attachments, labels, source_account, + embedding, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, %(subject)s, + 'Alice', %(saddr)s, 'topics-test.example', + '{"to": [], "cc": [], "bcc": []}'::jsonb, %(date)s, 'body', NULL, + false, NULL, ARRAY['INBOX'], %(acct)s, + %(emb)s::vector, now() + ) + """, + { + "id": email_id, + "mid": message_id, + "tid": tid, + "subject": subject, + "saddr": sender_address, + "date": when, + "acct": source_account, + "emb": emb_lit, + }, + ) + conn.commit() + return { + "email_id": email_id, + "msg_sid": encode_source_id("msg", email_id), + "subject": subject, + "source_account": source_account, + } + + +def _cleanup_topics_fixtures(pool: ConnectionPool, seeds: list[dict[str, Any]]) -> None: + with pool.connection() as conn: + conn.execute("DELETE FROM app_topic_members") + conn.execute("DELETE FROM app_topics") + for seed in seeds: + conn.execute("DELETE FROM emails WHERE id = %(id)s", {"id": seed["email_id"]}) + conn.execute("DELETE FROM app_audit WHERE action LIKE 'topics%%'") + conn.commit() + + +# --- generation / precedence / assignment --- + + +def test_generate_and_assignment_seeded_corpus( + db_client: TestClient, db_pool: ConnectionPool +) -> None: + seeds: list[dict[str, Any]] = [] + try: + # Two clusters along orthogonal axes. + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Kitchen renovation plumbing quote {i}", + embedding=_unit_vec(0), + date=datetime(2020, 3, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Family travel itinerary flights {i}", + embedding=_unit_vec(10), + date=datetime(2020, 4, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + + _login(db_client) + r = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 100, "seed": 13}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["topics"] >= 1 + assert body["assigned"] >= 12 + assert "took_ms" in body + + # Distances populated; member_count correct. + with db_pool.connection() as conn: + rows = conn.execute( + """ + SELECT t.id, t.label, t.origin, t.member_count, t.centroid IS NOT NULL + FROM app_topics t + WHERE t.origin = 'automatic' + """ + ).fetchall() + assert len(rows) >= 1 + total_members = sum(int(r[3]) for r in rows) + assert total_members >= 12 + for r_ in rows: + assert r_[2] == "automatic" + assert r_[4] is True + + dist_rows = conn.execute( + """ + SELECT count(*) FILTER (WHERE distance IS NOT NULL)::int, + count(*)::int + FROM app_topic_members + WHERE origin = 'automatic' + """ + ).fetchone() + assert dist_rows is not None + assert dist_rows[0] == dist_rows[1] + assert dist_rows[1] >= 12 + + # Origin visible on list payload. + listed = db_client.get("/api/topics") + assert listed.status_code == 200 + topics = listed.json()["topics"] + assert isinstance(topics, list) + assert all("origin" in t for t in topics) + + # Deterministic: second generate with same seed yields same centroid count path. + r2 = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 100, "seed": 13}, + ) + assert r2.status_code == 200, r2.text + assert r2.json()["topics"] == body["topics"] + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_regeneration_precedence_matrix(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(8): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Alpha beta gamma document {i}", + embedding=_unit_vec(i % 3), + date=datetime(2021, 1, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + r = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 7}, + ) + assert r.status_code == 200, r.text + + listed = db_client.get("/api/topics").json()["topics"] + assert listed + auto_id = listed[0]["id"] + auto_label = listed[0]["label"] + + # Curate by rename → origin curated. + patch = db_client.patch( + f"/api/topics/{auto_id}", + json={"label": "Curated Keep Me"}, + ) + assert patch.status_code == 200, patch.text + assert patch.json()["origin"] == "curated" + assert patch.json()["label"] == "Curated Keep Me" + + # Manual topic. + created = db_client.post( + "/api/topics", + json={"label": "Manual Archive Topic", "description": "hand built"}, + ) + assert created.status_code == 200, created.text + manual_id = created.json()["id"] + assert created.json()["origin"] == "manual" + + # Manual membership on a surviving automatic topic (create fresh auto via re-gen + # after we add manual member to a new auto). First add manual member to curated. + email_sid = seeds[0]["msg_sid"] + add = db_client.post( + f"/api/topics/{auto_id}/members", + json={"email_sid": email_sid}, + ) + assert add.status_code == 200, add.text + + # Also create a pure-automatic topic path: generate again after insert more? Use + # a second automatic by regenerating and picking an automatic one, then add + # manual member to it. + r_mid = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 7}, + ) + assert r_mid.status_code == 200 + + # After regen: curated and manual survive with labels intact. + with db_pool.connection() as conn: + curated = conn.execute( + "SELECT label, origin FROM app_topics WHERE id = %(id)s", + {"id": auto_id}, + ).fetchone() + assert curated is not None + assert curated[0] == "Curated Keep Me" + assert curated[1] == "curated" + + manual = conn.execute( + "SELECT label, origin FROM app_topics WHERE id = %(id)s", + {"id": manual_id}, + ).fetchone() + assert manual is not None + assert manual[0] == "Manual Archive Topic" + assert manual[1] == "manual" + + # Manual membership row survives reassignment. + mem = conn.execute( + """ + SELECT origin FROM app_topic_members + WHERE topic_id = %(tid)s AND email_id = %(eid)s + """, + {"tid": auto_id, "eid": seeds[0]["email_id"]}, + ).fetchone() + assert mem is not None + assert mem[0] == "manual" + + # Plain automatic topics were replaced (old label gone unless recreated). + # Curated id still present; generation increased for new automatic topics. + with db_pool.connection() as conn: + autos = conn.execute( + "SELECT id, label FROM app_topics WHERE origin = 'automatic'" + ).fetchall() + # New automatic topics exist after regen. + assert len(autos) >= 1 + # Original automatic label may differ after recreation — curated kept. + assert auto_label # just ensure we captured it + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_gateway_polish_fake_transport(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Contract negotiation deadline {i}", + embedding=_unit_vec(2), + ) + ) + _login(db_client) + + def fake_transport( + model: str, messages: list[dict[str, str]], stream: bool + ) -> Iterator[str]: + # Emit one polished label per cluster — over-provision; parse keeps length match. + yield '["Contract talks"]' + + db_client.app.state.chat_transport = fake_transport # type: ignore[attr-defined] + db_client.app.state.model_available = True # type: ignore[attr-defined] + + # k=4 but sample may produce fewer effective clusters; force k=4 with enough + # diversity if possible. With identical embeddings, kmeans still creates k + # centroids (empty clusters reseeded) — labels_final length = effective_k. + # For polish to apply, response length must match effective_k. + # Use k=4 and a response with 4 labels. + def fake_transport4( + model: str, messages: list[dict[str, str]], stream: bool + ) -> Iterator[str]: + yield '["Alpha one", "Beta two", "Gamma three", "Delta four"]' + + db_client.app.state.chat_transport = fake_transport4 # type: ignore[attr-defined] + + r = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 3}, + ) + assert r.status_code == 200, r.text + # At least one topic created; polish may or may not stick depending on k vs n. + assert r.json()["topics"] >= 1 + + # Failure fallback: bad transport keeps term labels (no crash). + def bad_transport( + model: str, messages: list[dict[str, str]], stream: bool + ) -> Iterator[str]: + yield "NOT VALID JSON AT ALL" + + db_client.app.state.chat_transport = bad_transport # type: ignore[attr-defined] + r2 = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 3}, + ) + assert r2.status_code == 200, r2.text + assert r2.json()["topics"] >= 1 + finally: + db_client.app.state.model_available = False # type: ignore[attr-defined] + _cleanup_topics_fixtures(db_pool, seeds) + + +# --- CRUD --- + + +def test_crud_rename_delete_members_auth( + db_client: TestClient, db_pool: ConnectionPool, client: TestClient +) -> None: + seeds: list[dict[str, Any]] = [] + try: + seeds.append( + _insert_email_with_embedding( + db_pool, + subject="Solo message for membership", + embedding=_unit_vec(5), + ) + ) + + # Auth required. + assert client.get("/api/topics").status_code == 401 + assert client.post("/api/topics/generate", json={}).status_code == 401 + + _login(db_client) + + created = db_client.post( + "/api/topics", + json={"label": "Hand Topic", "description": "d"}, + ) + assert created.status_code == 200 + tid = created.json()["id"] + assert created.json()["origin"] == "manual" + + # Manual delete ok. + deleted = db_client.delete(f"/api/topics/{tid}") + assert deleted.status_code == 200 + + # Recreate + auto via generate for delete guard. + for i in range(4): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Auto cluster subject word{i}", + embedding=_unit_vec(1), + ) + ) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 20, "seed": 1}, + ) + assert gen.status_code == 200 + topics = db_client.get("/api/topics").json()["topics"] + auto = next(t for t in topics if t["origin"] == "automatic") + forbidden = db_client.delete(f"/api/topics/{auto['id']}") + assert forbidden.status_code == 403 + + # Rename flips to curated. + patched = db_client.patch( + f"/api/topics/{auto['id']}", + json={"label": "Renamed Auto"}, + ) + assert patched.status_code == 200 + assert patched.json()["origin"] == "curated" + assert patched.json()["label"] == "Renamed Auto" + + # Member add/remove + audit. + sid = seeds[0]["msg_sid"] + add = db_client.post( + f"/api/topics/{auto['id']}/members", + json={"email_sid": sid}, + ) + assert add.status_code == 200 + rem = db_client.delete(f"/api/topics/{auto['id']}/members/{sid}") + assert rem.status_code == 200 + + with db_pool.connection() as conn: + actions = { + r[0] + for r in conn.execute( + """ + SELECT action FROM app_audit + WHERE action LIKE 'topics%%' + """ + ).fetchall() + } + assert "topics_member_add" in actions + assert "topics_member_remove" in actions + assert "topics_patch" in actions + assert "topics_generate" in actions + + # Detail endpoint shape. + detail = db_client.get(f"/api/topics/{auto['id']}") + assert detail.status_code == 200 + body = detail.json() + assert "activity" in body + assert "members" in body + assert body["origin"] in ("curated", "automatic", "manual") + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +# --- Atlas analytics (4.2) --- + + +def test_pca_project_2d_deterministic() -> None: + rng = np.random.default_rng(42) + data = rng.normal(size=(12, 16)).astype(np.float64) + a = pca_project_2d(data) + b = pca_project_2d(data) + np.testing.assert_allclose(a, b) + assert a.shape == (12, 2) + assert float(np.max(np.abs(a))) <= 1.0 + 1e-9 + + +def test_river_top_n_and_unit(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(5): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"River alpha topic word {i}", + embedding=_unit_vec(0), + date=datetime(2020, 3, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + for i in range(3): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"River beta topic word {i}", + embedding=_unit_vec(20), + date=datetime(2020, 6, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 11}, + ) + assert gen.status_code == 200, gen.text + + r = db_client.get( + "/api/topics/river", + params={ + "from": "2020-01-01T00:00:00Z", + "to": "2021-01-01T00:00:00Z", + "unit": "month", + "top": 2, + }, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["mode_hint"] == "absolute" + assert body["unit"] == "month" + assert isinstance(body["topics"], list) + assert len(body["topics"]) <= 2 + for series in body["topics"]: + assert "topic_id" in series + assert "label" in series + assert "origin" in series + assert isinstance(series["buckets"], list) + for b in series["buckets"]: + assert "bucket" in b and "count" in b + + # unit=auto resolves to a valid unit. + r_auto = db_client.get( + "/api/topics/river", + params={ + "from": "2020-01-01T00:00:00Z", + "to": "2021-01-01T00:00:00Z", + "unit": "auto", + "top": 8, + }, + ) + assert r_auto.status_code == 200, r_auto.text + assert r_auto.json()["unit"] in ( + "hour", + "day", + "week", + "month", + "quarter", + "year", + ) + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_matrix_totals(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(4): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Matrix year twenty {i}", + embedding=_unit_vec(1), + date=datetime(2020, 5, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + for i in range(2): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Matrix year twentyone {i}", + embedding=_unit_vec(1), + date=datetime(2021, 5, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 5}, + ) + assert gen.status_code == 200, gen.text + + r = db_client.get("/api/topics/matrix", params={"by": "year"}) + assert r.status_code == 200, r.text + body = r.json() + assert body["by"] == "year" + assert "2020" in body["columns"] or "2021" in body["columns"] + assert isinstance(body["rows"], list) + assert "column_totals" in body + assert "grand_total" in body + # Totals reconcile: sum of row_totals == grand_total == sum of col totals. + row_sum = sum(int(row["row_total"]) for row in body["rows"]) + col_sum = sum(int(v) for v in body["column_totals"].values()) + assert row_sum == body["grand_total"] + assert col_sum == body["grand_total"] + assert body["grand_total"] >= 6 + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_projection_determinism_shape_and_manual_exclusion( + db_client: TestClient, db_pool: ConnectionPool +) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Projection cluster subject {i}", + embedding=_unit_vec(i % 2), + date=datetime(2019, 1, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 9}, + ) + assert gen.status_code == 200, gen.text + + # Manual topic without centroid must be excluded. + created = db_client.post( + "/api/topics", + json={"label": "Manual No Centroid"}, + ) + assert created.status_code == 200 + manual_id = created.json()["id"] + + p1 = db_client.get("/api/topics/projection") + assert p1.status_code == 200, p1.text + body1 = p1.json() + p2 = db_client.get("/api/topics/projection") + assert p2.status_code == 200 + body2 = p2.json() + # Determinism: same centroids → same coords. + assert body1["points"] == body2["points"] + + # Shape: topic-level only (no source_id / email_id fields). + assert "points" in body1 + for pt in body1["points"]: + assert set(pt.keys()) >= { + "topic_id", + "label", + "origin", + "member_count", + "x", + "y", + } + assert "source_id" not in pt + assert "email_id" not in pt + assert "sources" not in pt + assert -1.0 - 1e-9 <= float(pt["x"]) <= 1.0 + 1e-9 + assert -1.0 - 1e-9 <= float(pt["y"]) <= 1.0 + 1e-9 + assert pt["topic_id"] != manual_id + + assert body1["excluded_without_centroid"] >= 1 + assert "note" in body1 + assert "TA-003" in body1["note"] or "topic-level" in body1["note"].lower() + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_members_keyset_endpoint(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Member list item {i}", + embedding=_unit_vec(3), + date=datetime(2022, 1, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 2}, + ) + assert gen.status_code == 200, gen.text + topics = db_client.get("/api/topics").json()["topics"] + assert topics + tid = topics[0]["id"] + + page1 = db_client.get( + f"/api/topics/{tid}/members", + params={"limit": 2}, + ) + assert page1.status_code == 200, page1.text + b1 = page1.json() + assert len(b1["items"]) <= 2 + assert "next_cursor" in b1 + ids1 = [it["id"] for it in b1["items"]] + assert all(sid.startswith("msg_") for sid in ids1) + + if b1["next_cursor"]: + page2 = db_client.get( + f"/api/topics/{tid}/members", + params={"limit": 2, "cursor": b1["next_cursor"]}, + ) + assert page2.status_code == 200, page2.text + ids2 = [it["id"] for it in page2.json()["items"]] + # Keyset pages must not overlap. + assert set(ids1).isdisjoint(set(ids2)) + finally: + _cleanup_topics_fixtures(db_pool, seeds) diff --git a/apps/chronicle/web/src/App.tsx b/apps/chronicle/web/src/App.tsx index 936ddd8..795dc48 100644 --- a/apps/chronicle/web/src/App.tsx +++ b/apps/chronicle/web/src/App.tsx @@ -7,10 +7,13 @@ import { FilesPage } from './files/FilesPage' import { SourcePage } from './reader/SourcePage' import { ResearchDeskPage } from './research/ResearchDeskPage' import { ResearchNavShortcut } from './research/ResearchNavShortcut' +import { PeoplePage } from './people/PeoplePage' +import { PersonProfilePage } from './people/PersonProfilePage' import { ChroniclePage } from './routes/ChroniclePage' import { DataHealthPage } from './routes/DataHealthPage' import { StubPage } from './routes/StubPage' import { Workstation } from './shell/Workstation' +import { TopicsPage } from './topics/TopicsPage' import { WorkspacePage } from './workspaces/WorkspacePage' import { WorkspacesListPage } from './workspaces/WorkspacesListPage' @@ -35,8 +38,9 @@ export function App() { /> } /> } /> - } /> - } /> + } /> + } /> + } /> } /> } /> } /> diff --git a/apps/chronicle/web/src/acceptance/workflowB.acceptance.test.tsx b/apps/chronicle/web/src/acceptance/workflowB.acceptance.test.tsx new file mode 100644 index 0000000..e587180 --- /dev/null +++ b/apps/chronicle/web/src/acceptance/workflowB.acceptance.test.tsx @@ -0,0 +1,408 @@ +/** + * Workflow B — explore an unknown period (spec §19.2). + * Mocked-API end-to-end: 2007 burst → Focus → Topic Atlas (same scope) → + * topic select → Chronicle → save workspace with scope. + */ +import { QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { BrowserRouter } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ChronicleBuckets, QueryScope } from '../api/types' +import { App } from '../App' +import { + createTestQueryClient, + mockArchiveSummary, + mockSessionOk, +} from '../test/test-utils' +import { resetUrlSyncForTests, writeStoreToUrlNow } from '../workingset/useUrlSync' +import { resetWorkingSetStore, useWorkingSetStore } from '../workingset/store' + +const EXTENT_FROM = '2005-01-01T00:00:00.000Z' +const EXTENT_TO = '2010-01-01T00:00:00.000Z' +const TOPIC_ID = 'topic-2007-burst' +const WS_ID = 'ws-workflow-b' + +const burst2007 = { + fromMs: Date.UTC(2007, 0, 1), + toMs: Date.UTC(2008, 0, 1), +} + +function mockBuckets(overrides: Partial = {}): Response { + const body: ChronicleBuckets = { + scope_fingerprint: 'qs_workflow_b', + aggregation: 'year', + unit: 'year', + viewport: { from: EXTENT_FROM, to: EXTENT_TO }, + lanes: { + messages: [ + { bucket: '2005-01-01T00:00:00.000Z', count: 40 }, + { bucket: '2006-01-01T00:00:00.000Z', count: 55 }, + // Activity burst in 2007 + { bucket: '2007-01-01T00:00:00.000Z', count: 480 }, + { bucket: '2008-01-01T00:00:00.000Z', count: 70 }, + { bucket: '2009-01-01T00:00:00.000Z', count: 60 }, + ], + attachments: [{ bucket: '2007-01-01T00:00:00.000Z', count: 42 }], + top_people: { + contacts: [ + { + contact_id: 'contact-alice', + display_name: 'Alice', + buckets: [{ bucket: '2007-01-01T00:00:00.000Z', count: 30 }], + }, + ], + }, + topics: { + topics: [ + { + topic_id: TOPIC_ID, + label: 'Kitchen remodel', + origin: 'automatic', + buckets: [{ bucket: '2007-01-01T00:00:00.000Z', count: 80 }], + }, + ], + }, + events: { events: [], truncated: false }, + }, + density: { + unit: 'year', + buckets: [ + { bucket: '2005-01-01T00:00:00.000Z', count: 40 }, + { bucket: '2006-01-01T00:00:00.000Z', count: 55 }, + { bucket: '2007-01-01T00:00:00.000Z', count: 480 }, + { bucket: '2008-01-01T00:00:00.000Z', count: 70 }, + { bucket: '2009-01-01T00:00:00.000Z', count: 60 }, + ], + }, + extent: { from: EXTENT_FROM, to: EXTENT_TO }, + generated_at: '2026-01-01T00:00:00.000Z', + ...overrides, + } + return { + ok: true, + status: 200, + json: async () => body, + } as Response +} + +function mockSourceList() { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: 'msg_2007_1', + subject: 'Kitchen plans', + sender_name: 'Alice', + sender_address: 'alice@example.com', + date: '2007-03-15T12:00:00Z', + mailbox: 'me@example.com', + has_attachment: true, + attachment_count: 1, + thread_id: 'thr_kitchen', + }, + { + id: 'msg_2007_2', + subject: 'Tile samples', + sender_name: 'Bob', + sender_address: 'bob@example.com', + date: '2007-04-01T12:00:00Z', + mailbox: 'me@example.com', + has_attachment: false, + attachment_count: 0, + thread_id: 'thr_kitchen', + }, + ], + next_cursor: null, + scope_fingerprint: 'qs_workflow_b', + }), + } as Response +} + +function installWorkflowBFetch() { + const createdWorkspaces: { name: string; scope: QueryScope }[] = [] + + const fetchMock = vi.fn().mockImplementation(async (url: string, init?: RequestInit) => { + const u = String(url) + const method = (init?.method || 'GET').toUpperCase() + + if (u.includes('/api/auth/session')) return mockSessionOk() + if (u.includes('/api/archive/summary')) return mockArchiveSummary({ + date_range: { from: '2005-01-01T00:00:00', to: '2009-12-31T00:00:00' }, + }) + if (u.includes('/api/chronicle/buckets')) return mockBuckets() + if (u.includes('/api/sources/list')) return mockSourceList() + + if (u.includes('/api/topics/') && u.includes('/members')) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: 'msg_2007_1', + subject: 'Kitchen plans', + sender_name: 'Alice', + sender_address: 'alice@example.com', + date: '2007-03-15T12:00:00Z', + mailbox: 'me@example.com', + has_attachment: true, + attachment_count: 1, + thread_id: 'thr_kitchen', + }, + ], + next_cursor: null, + }), + } as Response + } + if (u.includes(`/api/topics/${TOPIC_ID}`) && !u.includes('/members')) { + return { + ok: true, + status: 200, + json: async () => ({ + id: TOPIC_ID, + label: 'Kitchen remodel', + origin: 'automatic', + description: null, + parent_id: null, + hidden: false, + member_count: 12, + generation: 1, + top_terms: ['kitchen', 'tile'], + created_at: null, + updated_at: null, + activity: [{ bucket: '2007-01-01T00:00:00Z', count: 80 }], + members: [ + { + id: 'msg_2007_1', + subject: 'Kitchen plans', + sender_name: 'Alice', + sender_address: 'alice@example.com', + date: '2007-03-15T12:00:00Z', + mailbox: 'me@example.com', + thread_id: 'thr_kitchen', + distance: 0.1, + }, + ], + }), + } as Response + } + if (u.includes('/api/topics/river')) { + return { + ok: true, + status: 200, + json: async () => ({ + unit: 'month', + mode_hint: 'absolute', + from: '2007-01-01T00:00:00Z', + to: '2008-01-01T00:00:00Z', + topics: [ + { + topic_id: TOPIC_ID, + label: 'Kitchen remodel', + origin: 'automatic', + buckets: [ + { bucket: '2007-03-01T00:00:00Z', count: 20 }, + { bucket: '2007-04-01T00:00:00Z', count: 30 }, + ], + }, + ], + }), + } as Response + } + if (u.includes('/api/topics') && method === 'GET' && !u.includes('river') && !u.includes('matrix') && !u.includes('projection')) { + return { + ok: true, + status: 200, + json: async () => ({ + topics: [ + { + id: TOPIC_ID, + label: 'Kitchen remodel', + origin: 'automatic', + member_count: 12, + hidden: false, + children: [], + top_terms: ['kitchen', 'tile'], + }, + ], + }), + } as Response + } + + if (u.includes('/api/workspaces') && method === 'GET' && !u.includes('/blocks')) { + return { + ok: true, + status: 200, + json: async () => ({ items: [] }), + } as Response + } + if (u.includes('/api/workspaces') && method === 'POST' && !u.includes('/blocks')) { + const body = JSON.parse(String(init?.body || '{}')) as { + name: string + scope: QueryScope + } + createdWorkspaces.push(body) + return { + ok: true, + status: 201, + json: async () => ({ + id: WS_ID, + name: body.name, + scope: body.scope, + version: 1, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + counts: { blocks: 0, pins: 0, notes: 0, answers: 0, headings: 0 }, + }), + } as Response + } + if (u.includes(`/api/workspaces/${WS_ID}`)) { + const last = createdWorkspaces[createdWorkspaces.length - 1] + return { + ok: true, + status: 200, + json: async () => ({ + id: WS_ID, + name: last?.name ?? '2007 period', + scope: last?.scope ?? {}, + version: 1, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + counts: { blocks: 0, pins: 0, notes: 0, answers: 0, headings: 0 }, + blocks: [], + }), + } as Response + } + + throw new Error(`unexpected fetch: ${method} ${u}`) + }) + + vi.stubGlobal('fetch', fetchMock) + return { fetchMock, createdWorkspaces } +} + +function renderFullApp() { + const client = createTestQueryClient() + return { + client, + ...render( + + + + + , + ), + } +} + +function scopeSnapshot() { + const s = useWorkingSetStore.getState() + return { + scope: structuredClone(s.scope), + focus: s.focus ? { ...s.focus } : null, + viewport: s.viewport ? { ...s.viewport } : null, + } +} + +describe('Workflow B — explore an unknown period (§19.2)', () => { + beforeEach(() => { + window.history.replaceState(null, '', '/') + resetWorkingSetStore() + resetUrlSyncForTests() + }) + + afterEach(() => { + vi.unstubAllGlobals() + window.history.replaceState(null, '', '/') + resetWorkingSetStore() + resetUrlSyncForTests() + }) + + it('runs end-to-end: 2007 burst → Focus → Topics river → Chronicle → workspace', async () => { + const { createdWorkspaces } = installWorkflowBFetch() + renderFullApp() + + // 1. Full-archive Chronicle; activity burst in 2007 (fixture) + expect(await screen.findByTestId('workstation-shell')).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Chronicle' })).toBeInTheDocument() + expect(await screen.findByTestId('timeline-toolbar')).toBeInTheDocument() + + // Double-click burst → Focus mode (store path mirrors canvas double-click) + useWorkingSetStore.getState().setFocus(burst2007) + writeStoreToUrlNow() + + expect(await screen.findByTestId('focus-mode')).toBeInTheDocument() + expect(screen.getByTestId('focus-period-label')).toHaveTextContent(/2007/) + // Authoritative source list in focus + expect(await screen.findByTestId('source-list')).toBeInTheDocument() + expect(screen.getByTestId('focus-source-sequence')).toBeInTheDocument() + + // Apply focus as scope date so chips reflect the period across lens hops + fireEvent.click(screen.getByTestId('focus-set-scope-date')) + await waitFor(() => { + expect(screen.queryByTestId('focus-mode')).not.toBeInTheDocument() + }) + expect(await screen.findByTestId('scope-chip-date')).toBeInTheDocument() + const snapAfterFocus = scopeSnapshot() + expect(snapAfterFocus.scope.date).toBeTruthy() + + // 2. Open Topic Atlas via nav — same scope chips; river for the range + fireEvent.click(screen.getByRole('link', { name: 'Topics' })) + expect(await screen.findByTestId('topics-page')).toBeInTheDocument() + // Scope bar chips unchanged + expect(screen.getByTestId('scope-chip-date')).toBeInTheDocument() + const snapOnTopics = scopeSnapshot() + expect(snapOnTopics.scope).toEqual(snapAfterFocus.scope) + + fireEvent.click(screen.getByTestId('topic-view-river')) + expect(await screen.findByTestId('river-view')).toBeInTheDocument() + // River renders for the range without re-entering it + await waitFor(() => { + expect(screen.queryByTestId('river-empty')).not.toBeInTheDocument() + }) + + // 3. Select a topic cluster → back to Chronicle — scope intact + // Prefer hierarchy select for reliable selection + member list + fireEvent.click(screen.getByTestId('topic-view-hierarchy')) + expect(await screen.findByTestId('hierarchy-view')).toBeInTheDocument() + fireEvent.click(await screen.findByTestId(`topic-treeitem-${TOPIC_ID}`)) + + // Topic card exposes authoritative member list + expect(await screen.findByTestId('topic-card')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('topic-open-sources')) + expect(await screen.findByTestId('topic-member-list')).toBeInTheDocument() + // Focus source list was reachable earlier; topic member list is the + // authoritative aggregated-visual list for the Atlas hop. + expect(screen.getByTestId('topic-list-member-msg_2007_1')).toBeInTheDocument() + + fireEvent.click(screen.getByRole('link', { name: 'Chronicle' })) + expect(await screen.findByRole('heading', { name: 'Chronicle' })).toBeInTheDocument() + expect(screen.getByTestId('scope-chip-date')).toBeInTheDocument() + const snapBack = scopeSnapshot() + expect(snapBack.scope).toEqual(snapAfterFocus.scope) + + // 4. Save the period as a workspace (create with current scope) + fireEvent.click(screen.getByRole('link', { name: 'Workspaces' })) + expect(await screen.findByTestId('workspaces-list-page')).toBeInTheDocument() + fireEvent.change(screen.getByTestId('new-workspace-name'), { + target: { value: '2007 exploration' }, + }) + // use current scope checked by default + expect(screen.getByTestId('use-current-scope')).toBeChecked() + fireEvent.click(screen.getByTestId('create-workspace')) + + await waitFor(() => { + expect(createdWorkspaces.length).toBe(1) + }) + const ws = createdWorkspaces[0]! + expect(ws.name).toBe('2007 exploration') + expect(ws.scope).toEqual(snapAfterFocus.scope) + + // Pass condition: working set identical across lens changes; authoritative lists reachable + expect(snapOnTopics.scope).toEqual(snapAfterFocus.scope) + expect(snapBack.scope).toEqual(snapAfterFocus.scope) + }) +}) diff --git a/apps/chronicle/web/src/acceptance/workflowD.acceptance.test.tsx b/apps/chronicle/web/src/acceptance/workflowD.acceptance.test.tsx new file mode 100644 index 0000000..a0faeba --- /dev/null +++ b/apps/chronicle/web/src/acceptance/workflowD.acceptance.test.tsx @@ -0,0 +1,396 @@ +/** + * Workflow D — investigate a person (spec §19.4). + * Mocked-API: Chronicle people-lane → profile → unmerge → ego graph evidence. + */ +import { QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { BrowserRouter } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ChronicleBuckets, ContactCard } from '../api/types' +import { App } from '../App' +import { + createTestQueryClient, + mockArchiveSummary, + mockSessionOk, +} from '../test/test-utils' +import { resetUrlSyncForTests } from '../workingset/useUrlSync' +import { resetWorkingSetStore, useWorkingSetStore } from '../workingset/store' + +const EXTENT_FROM = '2010-01-01T00:00:00.000Z' +const EXTENT_TO = '2020-01-01T00:00:00.000Z' +const CONTACT_ID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' +const MERGE_ID = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' +const SOURCE_ID = 'cccccccc-cccc-cccc-cccc-cccccccccccc' +const NEIGHBOR_ID = 'dddddddd-dddd-dddd-dddd-dddddddddddd' +const THR_A = 'thr_evidence_a' +const THR_B = 'thr_evidence_b' + +function mockCard(overrides: Partial = {}): ContactCard { + return { + id: CONTACT_ID, + display_name: 'Alice Merged', + kind: 'human', + kind_source: 'heuristic', + tags: [], + human_probability: 0.9, + addresses: ['alice@example.com', 'wrongly-merged@example.com'], + name_variants: ['Alice'], + messages_from: 40, + messages_to: 10, + first_seen: '2012-01-01T00:00:00Z', + last_seen: '2018-12-31T00:00:00Z', + notes: null, + metadata: {}, + classification_signals: { bidirectional: 0.2 }, + classified_at: '2020-01-01T00:00:00Z', + address_classes: { + 'alice@example.com': 'external', + 'wrongly-merged@example.com': 'external', + }, + address_details: [ + { + address: 'alice@example.com', + is_user: false, + messages_from: 30, + messages_to: 5, + first_seen: '2012-01-01T00:00:00Z', + last_seen: '2018-12-31T00:00:00Z', + }, + { + address: 'wrongly-merged@example.com', + is_user: false, + messages_from: 10, + messages_to: 5, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2016-01-01T00:00:00Z', + }, + ], + activity: [ + { bucket: '2015-01-01T00:00:00Z', count: 8 }, + { bucket: '2016-01-01T00:00:00Z', count: 12 }, + ], + topics: [{ id: 'topic-alice', label: 'Renovation', count: 5 }], + thread_count: 18, + merges: [ + { + id: MERGE_ID, + source_id: SOURCE_ID, + target_id: CONTACT_ID, + merged_at: '2021-06-01T00:00:00Z', + }, + ], + ...overrides, + } +} + +function mockBuckets(): Response { + const body: ChronicleBuckets = { + scope_fingerprint: 'qs_workflow_d', + aggregation: 'year', + unit: 'year', + viewport: { from: EXTENT_FROM, to: EXTENT_TO }, + lanes: { + messages: [{ bucket: '2015-01-01T00:00:00.000Z', count: 100 }], + attachments: [], + top_people: { + contacts: [ + { + contact_id: CONTACT_ID, + display_name: 'Alice Merged', + buckets: [{ bucket: '2015-01-01T00:00:00.000Z', count: 40 }], + }, + ], + }, + events: { events: [], truncated: false }, + }, + density: { + unit: 'year', + buckets: [{ bucket: '2015-01-01T00:00:00.000Z', count: 100 }], + }, + extent: { from: EXTENT_FROM, to: EXTENT_TO }, + generated_at: '2026-01-01T00:00:00.000Z', + } + return { + ok: true, + status: 200, + json: async () => body, + } as Response +} + +function installWorkflowDFetch() { + let card = mockCard() + const unmergePosts: unknown[] = [] + + const graphWithEvidence = { + nodes: [ + { id: CONTACT_ID, label: 'Alice Merged', kind: 'human', is_ego: true }, + { id: NEIGHBOR_ID, label: 'Bob Builder', kind: 'human', is_ego: false }, + ], + edges: [ + { + source: CONTACT_ID, + target: NEIGHBOR_ID, + kind: 'thread_co_participation', + shared_threads: 2, + first: '2015-01-01T00:00:00Z', + last: '2016-06-01T00:00:00Z', + evidence: { thread_ids: [THR_A, THR_B] }, + }, + ], + truncated: false, + total_coparticipants: 1, + } + + const fetchMock = vi.fn().mockImplementation(async (url: string, init?: RequestInit) => { + const u = String(url) + const method = (init?.method || 'GET').toUpperCase() + + if (u.includes('/api/auth/session')) return mockSessionOk() + if (u.includes('/api/archive/summary')) return mockArchiveSummary() + if (u.includes('/api/chronicle/buckets')) return mockBuckets() + + if (u.includes('/api/people/unmerge') && method === 'POST') { + const body = JSON.parse(String(init?.body || '{}')) as { merge_id?: string } + unmergePosts.push(body) + card = mockCard({ + addresses: ['alice@example.com'], + address_classes: { 'alice@example.com': 'external' }, + address_details: [ + { + address: 'alice@example.com', + is_user: false, + messages_from: 30, + messages_to: 5, + first_seen: '2012-01-01T00:00:00Z', + last_seen: '2018-12-31T00:00:00Z', + }, + ], + merges: [], + display_name: 'Alice Example', + }) + return { + ok: true, + status: 200, + json: async () => ({ + source: mockCard({ + id: SOURCE_ID, + display_name: 'Wrong Identity', + addresses: ['wrongly-merged@example.com'], + merges: [], + }), + target: card, + }), + } as Response + } + + if (u.includes(`/api/people/${CONTACT_ID}/graph`)) { + return { + ok: true, + status: 200, + json: async () => graphWithEvidence, + } as Response + } + + if (u.includes(`/api/people/${CONTACT_ID}`) && method === 'GET') { + return { + ok: true, + status: 200, + json: async () => card, + } as Response + } + + if ( + u.includes('/api/people') && + method === 'GET' && + !u.includes('/graph') && + !u.includes('/merge') + ) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: CONTACT_ID, + display_name: card.display_name, + kind: 'human', + kind_source: 'heuristic', + tags: [], + human_probability: 0.9, + addresses: card.addresses, + name_variants: ['Alice'], + messages_from: 40, + messages_to: 10, + first_seen: card.first_seen, + last_seen: card.last_seen, + }, + ], + next_cursor: null, + }), + } as Response + } + + if (u.includes(`/api/threads/${THR_A}`) || u.includes(`/api/threads/${THR_B}`)) { + const tid = u.includes(THR_A) ? THR_A : THR_B + return { + ok: true, + status: 200, + json: async () => ({ + id: tid, + subject: tid === THR_A ? 'Quote discussion' : 'Site visit', + date_range: { + from: tid === THR_A ? '2015-02-01T00:00:00Z' : '2016-03-01T00:00:00Z', + to: tid === THR_A ? '2015-03-01T00:00:00Z' : '2016-04-01T00:00:00Z', + }, + participants: [ + { name: 'Alice', address: 'alice@example.com' }, + { name: 'Bob', address: 'bob@example.com' }, + ], + messages: [ + { + id: tid === THR_A ? 'msg_thr_a' : 'msg_thr_b', + subject: tid === THR_A ? 'Quote discussion' : 'Site visit', + sender_name: 'Bob', + sender_address: 'bob@example.com', + date: tid === THR_A ? '2015-02-01T12:00:00Z' : '2016-03-01T12:00:00Z', + }, + ], + }), + } as Response + } + + if (u.includes('/api/sources/list')) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [], + next_cursor: null, + scope_fingerprint: 'qs_workflow_d', + }), + } as Response + } + + // merge-candidates rail on people list + if (u.includes('/api/people/merge-candidates')) { + return { + ok: true, + status: 200, + json: async () => ({ items: [] }), + } as Response + } + + throw new Error(`unexpected fetch: ${method} ${u}`) + }) + + vi.stubGlobal('fetch', fetchMock) + return { fetchMock, unmergePosts, getCard: () => card } +} + +function renderFullApp() { + const client = createTestQueryClient() + return { + client, + ...render( + + + + + , + ), + } +} + +describe('Workflow D — investigate a person (§19.4)', () => { + beforeEach(() => { + window.history.replaceState(null, '', '/') + resetWorkingSetStore() + resetUrlSyncForTests() + }) + + afterEach(() => { + vi.unstubAllGlobals() + window.history.replaceState(null, '', '/') + resetWorkingSetStore() + resetUrlSyncForTests() + }) + + it('runs end-to-end: people-lane → profile → unmerge → ego evidence', async () => { + const { fetchMock, unmergePosts } = installWorkflowDFetch() + renderFullApp() + + // 1. Chronicle with top_people contact in fixture (people-lane row) + expect(await screen.findByTestId('workstation-shell')).toBeInTheDocument() + expect(await screen.findByTestId('timeline-toolbar')).toBeInTheDocument() + + // Switch to table alt so people-lane contact column is reachable + fireEvent.click(screen.getByRole('button', { name: /View as table/i })) + const table = await screen.findByTestId('timeline-table') + expect(table.querySelector(`[data-contact-col="${CONTACT_ID}"]`)).toBeTruthy() + expect(table.querySelector(`[data-contact-col="${CONTACT_ID}"]`)).toHaveTextContent( + /Alice/, + ) + + // Open person from people-lane context → profile + // (canvas row → people list with same contact → profile) + fireEvent.click(screen.getByRole('link', { name: 'People' })) + expect(await screen.findByTestId('people-list')).toBeInTheDocument() + fireEvent.click(await screen.findByTestId(`people-row-${CONTACT_ID}`)) + + expect(await screen.findByTestId('person-profile')).toBeInTheDocument() + expect(window.location.pathname).toBe(`/people/${CONTACT_ID}`) + + // Profile renders aliases/span/activity/topics + ego co-participants + expect(screen.getByTestId('person-identity')).toBeInTheDocument() + expect(screen.getByTestId('person-span')).toBeInTheDocument() + expect(screen.getByTestId('person-activity')).toBeInTheDocument() + expect(screen.getByTestId('person-topics')).toBeInTheDocument() + expect(await screen.findByTestId('ego-graph')).toBeInTheDocument() + expect(screen.getByText('wrongly-merged@example.com')).toBeInTheDocument() + + // 2. Correct identity: unmerge wrongly-merged address + expect(screen.getByTestId('person-merge-history')).toBeInTheDocument() + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(true) + fireEvent.click(screen.getByTestId(`unmerge-${MERGE_ID}`)) + + await waitFor(() => { + expect(unmergePosts.length).toBe(1) + }) + const unmergeBody = unmergePosts[0] as { merge_id: string } + // Pass condition: unmerge carries audit-relevant merge_id + expect(unmergeBody.merge_id).toBe(MERGE_ID) + + // Profile refetch reflects the split (mock swap) + await waitFor(() => { + expect(screen.queryByText('wrongly-merged@example.com')).not.toBeInTheDocument() + }) + expect(screen.getByText('alice@example.com')).toBeInTheDocument() + confirmSpy.mockRestore() + + // 3. Ego graph: only evidence-backed edges; select edge → thread list + date range + expect(await screen.findByTestId('ego-svg')).toBeInTheDocument() + // Render guard: edge exists only with evidence in fixture + const edge = screen.getByTestId(`ego-edge-${NEIGHBOR_ID}`) + expect(edge).toBeInTheDocument() + fireEvent.click(edge) + + expect(await screen.findByTestId('ego-evidence')).toBeInTheDocument() + // Date range visible on evidence panel + expect(screen.getByTestId('ego-evidence')).toHaveTextContent(/2015/) + expect(screen.getByTestId('ego-evidence')).toHaveTextContent(/2016/) + expect(await screen.findByTestId(`ego-evidence-row-${THR_A}`)).toBeInTheDocument() + expect(screen.getByTestId(`ego-evidence-row-${THR_B}`)).toBeInTheDocument() + + // Fixture invariant: graph response edges all have thread evidence + const graphOk = fetchMock.mock.calls.some((c) => String(c[0]).includes('/graph')) + expect(graphOk).toBe(true) + // No edge without evidence in fixtures — assert rows present for every thread_id + for (const tid of [THR_A, THR_B]) { + expect(screen.getByTestId(`ego-evidence-row-${tid}`)).toBeInTheDocument() + } + + // Store still healthy after profile hops + expect(useWorkingSetStore.getState()).toBeTruthy() + }) +}) diff --git a/apps/chronicle/web/src/api/types.ts b/apps/chronicle/web/src/api/types.ts index 7285a14..d968431 100644 --- a/apps/chronicle/web/src/api/types.ts +++ b/apps/chronicle/web/src/api/types.ts @@ -191,6 +191,132 @@ export interface TopPeopleLane { contacts: TopPeopleContact[] } +/** topics lane: activity bucket series for one topic. */ +export interface TopicSeries { + topic_id: string + label: string + origin: string + buckets: BucketPoint[] +} + +/** topics lane payload (top topics by volume; multirow). */ +export interface TopicsLane { + topics: TopicSeries[] +} + +/** Topic origin (Table 21). */ +export type TopicOrigin = 'automatic' | 'curated' | 'manual' + +/** GET /api/topics tree node. */ +export interface TopicTreeNode { + id: string + label: string + origin: TopicOrigin | string + member_count: number + hidden: boolean + top_terms: string[] + children: TopicTreeNode[] +} + +export interface TopicListResponse { + topics: TopicTreeNode[] +} + +export interface TopicActivityBucket { + bucket: string + count: number +} + +export interface TopicMemberEnvelope { + id: string + subject: string | null + sender_name: string | null + sender_address: string | null + date: string | null + mailbox: string | null + thread_id: string | null + distance: number | null +} + +/** GET /api/topics/{id} */ +export interface TopicDetail { + id: string + label: string + description: string | null + origin: TopicOrigin | string + parent_id: string | null + hidden: boolean + top_terms: string[] + generation: number + member_count: number + created_at: string | null + updated_at: string | null + activity: TopicActivityBucket[] + members: TopicMemberEnvelope[] +} + +export interface TopicPatchRequest { + label?: string + description?: string | null + hidden?: boolean + parent_id?: string | null +} + +/** GET /api/topics/river */ +export interface TopicRiverSeries { + topic_id: string + label: string + origin: TopicOrigin | string + buckets: BucketPoint[] +} + +export interface TopicRiverResponse { + unit: string + mode_hint: 'absolute' | string + from: string | null + to: string | null + topics: TopicRiverSeries[] +} + +/** GET /api/topics/matrix */ +export interface TopicMatrixRow { + topic_id: string + label: string + origin: TopicOrigin | string + cells: Record + row_total: number +} + +export interface TopicMatrixResponse { + by: string + columns: string[] + rows: TopicMatrixRow[] + column_totals: Record + grand_total: number +} + +/** GET /api/topics/projection — topic-level only (TA-003). */ +export interface TopicProjectionPoint { + topic_id: string + label: string + origin: TopicOrigin | string + member_count: number + x: number + y: number +} + +export interface TopicProjectionResponse { + points: TopicProjectionPoint[] + excluded_without_centroid: number + note: string +} + +/** GET /api/topics/{id}/members */ +export interface TopicMembersResponse { + items: TopicMemberEnvelope[] + next_cursor: string | null +} + /** Sparse event diamond mark on the events lane. */ export interface EventLaneMark { event_id: string @@ -210,8 +336,8 @@ export interface EventsLane { truncated: boolean } -/** A bars-style lane is BucketPoint[]; top_people / events are nested objects. */ -export type LaneData = BucketPoint[] | TopPeopleLane | EventsLane +/** A bars-style lane is BucketPoint[]; multirow / events are nested objects. */ +export type LaneData = BucketPoint[] | TopPeopleLane | TopicsLane | EventsLane export function isTopPeopleLane(data: LaneData | undefined): data is TopPeopleLane { return ( @@ -222,6 +348,15 @@ export function isTopPeopleLane(data: LaneData | undefined): data is TopPeopleLa ) } +export function isTopicsLane(data: LaneData | undefined): data is TopicsLane { + return ( + data != null && + typeof data === 'object' && + !Array.isArray(data) && + Array.isArray((data as TopicsLane).topics) + ) +} + export function isEventsLane(data: LaneData | undefined): data is EventsLane { return ( data != null && @@ -922,3 +1057,149 @@ export interface WorkspaceManifestRow { subject_or_filename?: string | null excerpt_hash?: string | null } + +/** People & Organizations (GET /api/people) */ + +export type ContactKind = + | 'human' + | 'organization' + | 'automated' + | 'mailing_list' + | 'unknown' + +export type AddressClass = 'owner' | 'external' + +export interface ContactSummary { + id: string + display_name: string | null + kind: ContactKind | string + kind_source: string | null + tags: string[] + human_probability: number | null + addresses: string[] + name_variants: string[] + messages_from: number + messages_to: number + first_seen: string | null + last_seen: string | null +} + +export interface PeopleListResponse { + items: ContactSummary[] + total: number | null + next_cursor: string | null + limit: number + offset: number +} + +export interface ContactAddressDetail { + address: string + is_user: boolean + messages_from: number + messages_to: number + first_seen: string | null + last_seen: string | null +} + +export interface ContactActivityBucket { + bucket: string + count: number +} + +export interface ContactTopic { + id: string + label: string + count: number +} + +export interface ContactMergeRecord { + id: string + source_id: string + target_id: string + merged_at: string | null +} + +/** GET /api/people/:id */ +export interface ContactCard extends ContactSummary { + notes: string | null + metadata: Record + classification_signals: Record | null + classified_at: string | null + address_classes: Record + address_details: ContactAddressDetail[] + activity: ContactActivityBucket[] + topics: ContactTopic[] + thread_count: number + merges?: ContactMergeRecord[] + merge_id?: string | null +} + +export interface ContactPatchRequest { + kind?: ContactKind + tags?: string[] + notes?: string | null + display_name?: string | null +} + +export interface MergeCandidateSide { + display_name: string | null + primary_address: string | null + msg_count: number + contact_id: string +} + +export interface MergeCandidatePair { + norm_name: string + a: MergeCandidateSide + b: MergeCandidateSide +} + +export interface MergeCandidatesResponse { + items: MergeCandidatePair[] +} + +export interface MergeRequest { + source_id: string + target_id: string +} + +export interface UnmergeRequest { + merge_id: string +} + +export interface UnmergeResponse { + source: ContactCard + target: ContactCard +} + +/** GET /api/people/:id/graph — depth-1 co-participation ego network (PE-003). */ + +export type EgoGraphNodeKind = ContactKind | 'address' | string + +export interface EgoGraphNode { + id: string + label: string + kind: EgoGraphNodeKind + is_ego: boolean +} + +export interface EgoGraphEdgeEvidence { + thread_ids: string[] +} + +export interface EgoGraphEdge { + source: string + target: string + kind: 'thread_co_participation' | string + shared_threads: number + first: string | null + last: string | null + evidence: EgoGraphEdgeEvidence +} + +export interface EgoGraphResponse { + nodes: EgoGraphNode[] + edges: EgoGraphEdge[] + truncated: boolean + total_coparticipants: number +} diff --git a/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx b/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx index 7ffef0c..13c63fa 100644 --- a/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx +++ b/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx @@ -10,7 +10,6 @@ import { } from 'react' import type { BucketPoint, EventLaneMark, LaneData } from '../api/types' -import { isTopPeopleLane } from '../api/types' import { CreateEventFromBrush } from '../events/CreateEventFromBrush' import { useWorkingSetStore } from '../workingset/store' import { @@ -23,13 +22,16 @@ import { MARKS_LANE_H, MULTIROW_HEADER_H, MULTIROW_ROW_H, - PEOPLE_CYAN, barsPoints, canvasHeightForLanes, eventPositionMs, eventsMarks, laneAtY as laneAtYFromLayout, layoutLanes, + multirowBucketsForHit, + multirowHitPrefix, + multirowMarkColor, + multirowSeriesForLane, originGlyph, type LaneSpec, } from './laneModel' @@ -234,9 +236,8 @@ function buildAriaLabel( const n = eventsMarks(laneData).length return `${n} events` } - const tp = laneData[spec.key] - const n = isTopPeopleLane(tp) ? tp.contacts.length : 0 - return `${n} contacts` + const n = multirowSeriesForLane(spec, laneData).length + return `${n} ${spec.key === 'topics' ? 'topics' : 'contacts'}` }) return `Timeline, ${range}, ${parts.join(', ')}` } @@ -456,38 +457,39 @@ export function TimelineCanvas({ ctx.textAlign = 'start' } } else { - // multirow: top_people activity spans - const data = laneData[spec.key] - const contacts = isTopPeopleLane(data) ? data.contacts : [] + // multirow: top_people / topics activity spans + const series = multirowSeriesForLane(spec, laneData) + const markColor = multirowMarkColor(spec.key) + const hitPrefix = multirowHitPrefix(spec.key) ctx.fillStyle = COLORS.muted ctx.font = '10px Inter, system-ui, sans-serif' ctx.textBaseline = 'middle' ctx.fillText(spec.label, 4, top + MULTIROW_HEADER_H / 2) - contacts.forEach((contact, i) => { + series.forEach((row, i) => { const rowTop = top + MULTIROW_HEADER_H + i * MULTIROW_ROW_H - const rowMax = maxCount(contact.buckets) || 1 + const rowMax = maxCount(row.buckets) || 1 ctx.fillStyle = COLORS.muted ctx.font = '10px Inter, system-ui, sans-serif' ctx.textBaseline = 'middle' ctx.fillText( - truncateLabel(contact.display_name, 16), + truncateLabel(row.label, 16), 4, rowTop + MULTIROW_ROW_H / 2, ) - for (const pt of contact.buckets) { + for (const pt of row.buckets) { if (pt.count <= 0) continue const t = Date.parse(pt.bucket) if (!Number.isFinite(t)) continue const x = LANE_LABEL_W + xForTime(t, viewport, plotW) const opacity = Math.min(1, Math.max(0.15, pt.count / rowMax)) - ctx.fillStyle = PEOPLE_CYAN + ctx.fillStyle = markColor ctx.globalAlpha = opacity const markH = MULTIROW_ROW_H - 4 ctx.fillRect(x, rowTop + 2, bw, markH) ctx.globalAlpha = 1 - const hitLane = `top_people:${contact.contact_id}` + const hitLane = `${hitPrefix}:${row.id}` if ( selectedBucket && selectedBucket.lane === hitLane && @@ -536,9 +538,8 @@ export function TimelineCanvas({ if (spec.kind === 'bars') total += sumCounts(barsPoints(laneData, spec.key)) else if (spec.kind === 'marks') total += eventsMarks(laneData).length else { - const tp = laneData[spec.key] - if (isTopPeopleLane(tp)) { - for (const c of tp.contacts) total += sumCounts(c.buckets) + for (const row of multirowSeriesForLane(spec, laneData)) { + total += sumCounts(row.buckets) } } } @@ -730,13 +731,8 @@ export function TimelineCanvas({ if (!onSelectBucket) return let points: BucketPoint[] = [] - if (hitKey.startsWith('top_people:')) { - const contactId = hitKey.slice('top_people:'.length) - const tp = laneData.top_people - if (isTopPeopleLane(tp)) { - const contact = tp.contacts.find((c) => c.contact_id === contactId) - points = contact?.buckets ?? [] - } + if (hitKey.startsWith('top_people:') || hitKey.startsWith('topics:')) { + points = multirowBucketsForHit(hitKey, laneData) } else { points = barsPoints(laneData, hitKey) } @@ -759,13 +755,8 @@ export function TimelineCanvas({ if (hitKey) { const unitTyped = parseUnit(unit) let points: BucketPoint[] = [] - if (hitKey.startsWith('top_people:')) { - const contactId = hitKey.slice('top_people:'.length) - const tp = laneData.top_people - if (isTopPeopleLane(tp)) { - const contact = tp.contacts.find((c) => c.contact_id === contactId) - points = contact?.buckets ?? [] - } + if (hitKey.startsWith('top_people:') || hitKey.startsWith('topics:')) { + points = multirowBucketsForHit(hitKey, laneData) } else { points = barsPoints(laneData, hitKey) } diff --git a/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx b/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx index 0f162d5..b38af3d 100644 --- a/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx +++ b/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx @@ -82,4 +82,43 @@ describe('TimelineTable', () => { expect(screen.getByText('Alice')).toBeInTheDocument() expect(screen.getByText('Bob')).toBeInTheDocument() }) + + it('includes per-topic columns for topics lane', () => { + const viewport = { + fromMs: Date.UTC(2014, 0, 1), + toMs: Date.UTC(2016, 0, 1), + } + render( + , + ) + + const table = screen.getByTestId('timeline-table') + expect(table.querySelector('[data-topic-col="t1"]')?.textContent).toBe('House') + expect(table.querySelector('[data-topic-col="t2"]')?.textContent).toBe('Travel') + expect(table.querySelector('[data-topic-cell="t1"]')?.textContent).toBe('3') + expect(table.querySelector('[data-topic-cell="t2"]')?.textContent).toBe('7') + }) }) diff --git a/apps/chronicle/web/src/chronicle/TimelineTable.tsx b/apps/chronicle/web/src/chronicle/TimelineTable.tsx index cf391eb..61de79d 100644 --- a/apps/chronicle/web/src/chronicle/TimelineTable.tsx +++ b/apps/chronicle/web/src/chronicle/TimelineTable.tsx @@ -1,6 +1,7 @@ -import type { BucketPoint, LaneData, TopPeopleContact } from '../api/types' -import { isBucketSeries, isTopPeopleLane } from '../api/types' -import type { LaneSpec } from './laneModel' +import type { BucketPoint, LaneData } from '../api/types' +import { isBucketSeries } from '../api/types' +import type { LaneSpec, MultirowSeries } from './laneModel' +import { multirowSeriesForLane } from './laneModel' import { formatPeriodLabel, formatPeriodRange, type Viewport } from './timeScale' export interface TimelineTableProps { @@ -15,19 +16,36 @@ type MergedRow = { bucket: string /** bars lane key → count */ bars: Record - /** contact_id → count for top_people */ - contacts: Record + /** multirow series id → count (top_people contacts / topics) */ + multi: Record +} + +/** Collect multirow columns from all multirow lanes in catalog order. */ +export function multirowColumns( + lanes: LaneSpec[], + laneData: Record, +): { laneKey: string; series: MultirowSeries }[] { + const out: { laneKey: string; series: MultirowSeries }[] = [] + for (const spec of lanes) { + if (spec.kind !== 'multirow') continue + for (const s of multirowSeriesForLane(spec, laneData)) { + out.push({ laneKey: spec.key, series: s }) + } + } + return out } /** Collect contact columns from top_people (stable order from server). */ export function topPeopleContacts( laneData: Record, -): TopPeopleContact[] { - const tp = laneData.top_people - return isTopPeopleLane(tp) ? tp.contacts : [] +): MultirowSeries[] { + return multirowSeriesForLane( + { key: 'top_people', label: 'Top people', kind: 'multirow' }, + laneData, + ) } -/** Merge all bar lanes + top_people per-contact counts by bucket timestamp. */ +/** Merge all bar lanes + multirow per-series counts by bucket timestamp. */ export function mergeBucketRows( lanes: LaneSpec[], laneData: Record, @@ -37,7 +55,7 @@ export function mergeBucketRows( const ensure = (bucket: string): MergedRow => { let row = map.get(bucket) if (!row) { - row = { bucket, bars: {}, contacts: {} } + row = { bucket, bars: {}, multi: {} } map.set(bucket, row) } return row @@ -50,11 +68,12 @@ export function mergeBucketRows( for (const p of points) { ensure(p.bucket).bars[spec.key] = p.count } - } else if (spec.key === 'top_people') { - const contacts = topPeopleContacts(laneData) - for (const c of contacts) { - for (const p of c.buckets) { - ensure(p.bucket).contacts[c.contact_id] = p.count + } else if (spec.kind === 'multirow') { + const series = multirowSeriesForLane(spec, laneData) + for (const s of series) { + for (const p of s.buckets) { + // Prefix id with lane key to avoid contact/topic id collisions. + ensure(p.bucket).multi[`${spec.key}:${s.id}`] = p.count } } } @@ -73,8 +92,8 @@ function bucketLabel(iso: string): string { /** * Accessible table alternative for the timeline canvas (A11Y-002). - * One row per bucket; columns follow configured lanes. For top_people, - * one column per contact (contact × bucket counts). + * One row per bucket; columns follow configured lanes. For multirow lanes + * (top_people / topics), one column per series (series × bucket counts). */ export function TimelineTable({ viewport, @@ -83,10 +102,9 @@ export function TimelineTable({ laneData, }: TimelineTableProps) { const rows = mergeBucketRows(lanes, laneData) - const contacts = - lanes.some((s) => s.key === 'top_people') ? topPeopleContacts(laneData) : [] + const multiCols = multirowColumns(lanes, laneData) const barLanes = lanes.filter((s) => s.kind === 'bars') - const colCount = 1 + barLanes.length + contacts.length + const colCount = 1 + barLanes.length + multiCols.length const caption = `Timeline buckets (${unit}), ${formatPeriodRange(viewport)}` return ( @@ -108,15 +126,16 @@ export function TimelineTable({ {spec.label} ))} - {contacts.map((c) => ( + {multiCols.map(({ laneKey, series: s }) => ( - {c.display_name} + {s.label} ))} @@ -145,13 +164,14 @@ export function TimelineTable({ {(row.bars[spec.key] ?? 0).toLocaleString()} ))} - {contacts.map((c) => ( + {multiCols.map(({ laneKey, series: s }) => ( - {(row.contacts[c.contact_id] ?? 0).toLocaleString()} + {(row.multi[`${laneKey}:${s.id}`] ?? 0).toLocaleString()} ))} diff --git a/apps/chronicle/web/src/chronicle/laneModel.test.ts b/apps/chronicle/web/src/chronicle/laneModel.test.ts index dcaf17c..78ff314 100644 --- a/apps/chronicle/web/src/chronicle/laneModel.test.ts +++ b/apps/chronicle/web/src/chronicle/laneModel.test.ts @@ -83,6 +83,54 @@ describe('laneModel', () => { }) expect(laneAtY(AXIS_H + LANE_GAP + 10, layout)).toBe('events') }) + + it('topics lane is multirow with topic-purple hit keys', () => { + const multi = specsForKeys(['topics']) + expect(multi[0]?.kind).toBe('multirow') + expect(multi[0]?.key).toBe('topics') + const hMulti = canvasHeightForLanes(multi, { + topics: { + topics: [ + { + topic_id: 't1', + label: 'House', + origin: 'automatic', + buckets: [], + }, + { + topic_id: 't2', + label: 'Travel', + origin: 'curated', + buckets: [], + }, + ], + }, + }) + expect(hMulti).toBe( + AXIS_H + LANE_GAP + MULTIROW_HEADER_H + 2 * MULTIROW_ROW_H, + ) + const layout = layoutLanes(multi, { + topics: { + topics: [ + { + topic_id: 't1', + label: 'House', + origin: 'automatic', + buckets: [], + }, + { + topic_id: 't2', + label: 'Travel', + origin: 'curated', + buckets: [], + }, + ], + }, + }) + const multiTop = layout[0]!.top + MULTIROW_HEADER_H + 1 + expect(laneAtY(multiTop, layout)).toBe('topics:t1') + expect(laneAtY(multiTop + MULTIROW_ROW_H, layout)).toBe('topics:t2') + }) }) describe('eventPositionMs day-floor', () => { diff --git a/apps/chronicle/web/src/chronicle/laneModel.ts b/apps/chronicle/web/src/chronicle/laneModel.ts index 26a799b..1514869 100644 --- a/apps/chronicle/web/src/chronicle/laneModel.ts +++ b/apps/chronicle/web/src/chronicle/laneModel.ts @@ -1,11 +1,27 @@ -import type { BucketPoint, EventLaneMark, LaneData, TopPeopleLane } from '../api/types' -import { isBucketSeries, isEventsLane, isTopPeopleLane } from '../api/types' +import type { + BucketPoint, + EventLaneMark, + LaneData, + TopicSeries, + TopPeopleContact, + TopPeopleLane, + TopicsLane, +} from '../api/types' +import { + isBucketSeries, + isEventsLane, + isTopPeopleLane, + isTopicsLane, +} from '../api/types' import type { LaneKey } from '../workingset/urlState' export type LaneKind = 'bars' | 'multirow' | 'marks' +/** Catalog keys include `topics` even when URL codec lags (server-first lane). */ +export type ChronicleLaneKey = LaneKey | 'topics' + export interface LaneSpec { - key: LaneKey + key: ChronicleLaneKey label: string kind: LaneKind } @@ -16,6 +32,7 @@ export const LANE_CATALOG: readonly LaneSpec[] = [ { key: 'attachments', label: 'Attachments', kind: 'bars' }, { key: 'people', label: 'People (distinct)', kind: 'bars' }, { key: 'top_people', label: 'Top people', kind: 'multirow' }, + { key: 'topics', label: 'Topics', kind: 'multirow' }, { key: 'events', label: 'Events', kind: 'marks' }, ] as const @@ -32,6 +49,9 @@ export const EVENT_AMBER = '#E0A84A' export const PEOPLE_CYAN = '#56d4dd' +/** Spec topic purple (#A78BFA) for topics multirow marks. */ +export const TOPIC_PURPLE = '#A78BFA' + export const AXIS_H = 28 export const LANE_H = 72 export const LANE_LABEL_W = 72 @@ -45,14 +65,51 @@ export const MARKS_LANE_H = 56 export function specsForKeys(keys: readonly string[]): LaneSpec[] { const out: LaneSpec[] = [] for (const key of keys) { - const spec = CATALOG_BY_KEY.get(key as LaneKey) + const spec = CATALOG_BY_KEY.get(key as ChronicleLaneKey) if (spec) out.push(spec) } return out } -export function multirowHeight(contactCount: number): number { - return MULTIROW_HEADER_H + MULTIROW_ROW_H * Math.max(0, contactCount) +export function multirowHeight(rowCount: number): number { + return MULTIROW_HEADER_H + MULTIROW_ROW_H * Math.max(0, rowCount) +} + +/** Multirow series rows for top_people or topics (shared canvas/table shape). */ +export interface MultirowSeries { + id: string + label: string + buckets: BucketPoint[] +} + +export function multirowSeriesForLane( + spec: LaneSpec, + laneData: Record | undefined, +): MultirowSeries[] { + const data = laneData?.[spec.key] + if (spec.key === 'topics' && isTopicsLane(data)) { + return data.topics.map((t) => ({ + id: t.topic_id, + label: t.label, + buckets: t.buckets, + })) + } + if (isTopPeopleLane(data)) { + return data.contacts.map((c) => ({ + id: c.contact_id, + label: c.display_name, + buckets: c.buckets, + })) + } + return [] +} + +export function multirowHitPrefix(laneKey: string): string { + return laneKey === 'topics' ? 'topics' : 'top_people' +} + +export function multirowMarkColor(laneKey: string): string { + return laneKey === 'topics' ? TOPIC_PURPLE : PEOPLE_CYAN } export function laneContentHeight( @@ -61,9 +118,7 @@ export function laneContentHeight( ): number { if (spec.kind === 'bars') return LANE_H if (spec.kind === 'marks') return MARKS_LANE_H - const data = laneData?.[spec.key] - const n = isTopPeopleLane(data) ? data.contacts.length : 0 - return multirowHeight(n) + return multirowHeight(multirowSeriesForLane(spec, laneData).length) } /** Total canvas CSS height for the given lane set and data. */ @@ -79,7 +134,7 @@ export function canvasHeightForLanes( } export interface LaneLayoutRow { - /** Hit-test id: lane key, or `top_people:` for multirow rows. */ + /** Hit-test id: lane key, or `top_people:` / `topics:` for multirow. */ hitKey: string /** Y of the top of this hit region (local canvas coords). */ top: number @@ -88,8 +143,10 @@ export interface LaneLayoutRow { /** Lane key for data lookup. */ laneKey: string kind: LaneKind - /** Contact id when multirow row. */ + /** Contact id when top_people multirow row. */ contactId?: string + /** Topic id when topics multirow row. */ + topicId?: string } export interface LaneLayoutBlock { @@ -119,16 +176,17 @@ export function layoutLanes( kind: spec.kind, }) } else { - const data = laneData?.[spec.key] - const contacts = isTopPeopleLane(data) ? data.contacts : [] - contacts.forEach((c, i) => { + const series = multirowSeriesForLane(spec, laneData) + const prefix = multirowHitPrefix(spec.key) + series.forEach((row, i) => { rows.push({ - hitKey: `top_people:${c.contact_id}`, + hitKey: `${prefix}:${row.id}`, top: y + MULTIROW_HEADER_H + i * MULTIROW_ROW_H, height: MULTIROW_ROW_H, laneKey: spec.key, kind: 'multirow', - contactId: c.contact_id, + contactId: prefix === 'top_people' ? row.id : undefined, + topicId: prefix === 'topics' ? row.id : undefined, }) }) } @@ -169,6 +227,13 @@ export function topPeopleData( return isTopPeopleLane(data) ? data : undefined } +export function topicsData( + laneData: Record | undefined, +): TopicsLane | undefined { + const data = laneData?.topics + return isTopicsLane(data) ? data : undefined +} + export function eventsMarks( laneData: Record | undefined, ): EventLaneMark[] { @@ -176,6 +241,26 @@ export function eventsMarks( return isEventsLane(data) ? data.events : [] } +/** Resolve multirow series by hit key (`top_people:…` or `topics:…`). */ +export function multirowBucketsForHit( + hitKey: string, + laneData: Record | undefined, +): BucketPoint[] { + if (hitKey.startsWith('topics:')) { + const topicId = hitKey.slice('topics:'.length) + const topics = topicsData(laneData) + const row = topics?.topics.find((t: TopicSeries) => t.topic_id === topicId) + return row?.buckets ?? [] + } + if (hitKey.startsWith('top_people:')) { + const contactId = hitKey.slice('top_people:'.length) + const tp = topPeopleData(laneData) + const row = tp?.contacts.find((c: TopPeopleContact) => c.contact_id === contactId) + return row?.buckets ?? [] + } + return [] +} + /** * Origin glyph (text, not color-only) for event marks. * A=analyst, ⚙=automatic, S=source, I=imported. diff --git a/apps/chronicle/web/src/files/FamilyPanel.tsx b/apps/chronicle/web/src/files/FamilyPanel.tsx new file mode 100644 index 0000000..2017221 --- /dev/null +++ b/apps/chronicle/web/src/files/FamilyPanel.tsx @@ -0,0 +1,126 @@ +import { useQuery } from '@tanstack/react-query' + +import { getAttachmentFamily } from './api' +import { formatBytes } from './format' +import type { FamilyCandidate } from './types' + +export interface FamilyPanelProps { + attSid: string + onClose: () => void + onCompare: (a: string, b: string) => void + onSelect: (attSid: string) => void +} + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[11px] text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +function confidenceLabel(c: FamilyCandidate): string { + if (c.confidence === 'exact-duplicate') return 'exact duplicate' + return 'probable version' +} + +export function FamilyPanel({ + attSid, + onClose, + onCompare, + onSelect, +}: FamilyPanelProps) { + const query = useQuery({ + queryKey: ['attachments', 'family', attSid], + queryFn: ({ signal }) => getAttachmentFamily(attSid, signal), + retry: false, + }) + + const candidates = query.data?.candidates ?? [] + + return ( +
+
+
+

+ Version family +

+ {query.data?.stem ? ( +

+ stem: {query.data.stem} +

+ ) : null} +
+ +
+ + {query.isLoading ? ( +

+ Loading family… +

+ ) : null} + {query.isError ? ( +

+ Failed to load version family +

+ ) : null} + + {candidates.length > 0 ? ( +
    + {candidates.map((c) => ( +
  • +
    + + + {confidenceLabel(c)} + {c.signals.length > 0 + ? ` · signals: ${c.signals.join(', ')}` + : ''} + +
    +

    + {c.date ? c.date.slice(0, 10) : '—'} · {c.sender || '—'} ·{' '} + {formatBytes(c.size)} +

    +
    + {candidates + .filter((other) => other.id !== c.id) + .map((other) => ( + + ))} +
    +
    • + ))} +
+ ) : null} + + {!query.isLoading && !query.isError && candidates.length === 0 ? ( +

No family candidates

+ ) : null} +
+ ) +} diff --git a/apps/chronicle/web/src/files/FilesPage.test.tsx b/apps/chronicle/web/src/files/FilesPage.test.tsx index 6ebe96a..f6f8447 100644 --- a/apps/chronicle/web/src/files/FilesPage.test.tsx +++ b/apps/chronicle/web/src/files/FilesPage.test.tsx @@ -259,6 +259,152 @@ describe('FilesPage', () => { expect(screen.getByTestId('files-table')).toBeInTheDocument() }) }) + + it('family badge opens panel; compare shows text-prefixed diff kinds', async () => { + const v1 = item({ + id: 'att_v1', + filename: 'final_roof_estimate.pdf', + family_count: 2, + sha256: 'sha1', + } as AttachmentListItem & { family_count: number }) + const v2 = item({ + id: 'att_v2', + filename: 'Final_Roof_Estimate_v3.pdf', + family_count: 2, + sha256: 'sha2', + } as AttachmentListItem & { family_count: number }) + + const familyBody = { + id: 'att_v1', + stem: 'final_roof_estimate', + candidates: [ + { + id: 'att_v1', + filename: 'final_roof_estimate.pdf', + date: '2015-05-01T00:00:00Z', + sender: 'Alice', + size: 100, + sha256: 'sha1', + confidence: 'exact-duplicate', + signals: ['stem', 'sha256'], + }, + { + id: 'att_v2', + filename: 'Final_Roof_Estimate_v3.pdf', + date: '2015-06-01T00:00:00Z', + sender: 'Alice', + size: 120, + sha256: 'sha2', + confidence: 'probable-version', + signals: ['stem', 'sender'], + }, + ], + } + + const compareBody = { + a: { + id: 'att_v1', + filename: 'final_roof_estimate.pdf', + content_type: 'application/pdf', + size: 100, + date: '2015-05-01T00:00:00Z', + sender: 'Alice', + sha256: 'sha1', + source_message_id: 'msg_1', + }, + b: { + id: 'att_v2', + filename: 'Final_Roof_Estimate_v3.pdf', + content_type: 'application/pdf', + size: 120, + date: '2015-06-01T00:00:00Z', + sender: 'Alice', + sha256: 'sha2', + source_message_id: 'msg_2', + }, + hunks: [ + { + a_start: 0, + b_start: 0, + lines: [ + { kind: 'same', text: 'Roof quote' }, + { kind: 'del', text: 'Total: $8,000' }, + { kind: 'add', text: 'Total: $9,500.00' }, + ], + }, + ], + truncated: false, + amount_changes: [ + { kind: 'del', text: 'Total: $8,000', amounts: ['$8,000'] }, + { kind: 'add', text: 'Total: $9,500.00', amounts: ['$9,500.00'] }, + ], + } + + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + const u = String(url) + if (u.includes('/api/attachments/list')) { + return { + ok: true, + status: 200, + json: async () => listResponse([v1, v2]), + } as Response + } + if (u.includes('/family')) { + return { + ok: true, + status: 200, + json: async () => familyBody, + } as Response + } + if (u.includes('/compare')) { + return { + ok: true, + status: 200, + json: async () => compareBody, + } as Response + } + throw new Error(`unexpected ${u}`) + }), + ) + + renderFiles() + await screen.findByTestId('file-row-att_v1') + expect(screen.getByTestId('family-badge-att_v1')).toHaveTextContent('2 versions') + fireEvent.click(screen.getByTestId('family-badge-att_v1')) + + expect(await screen.findByTestId('family-panel')).toBeInTheDocument() + expect(await screen.findByTestId('family-stem')).toHaveTextContent( + 'final_roof_estimate', + ) + expect(await screen.findByTestId('family-confidence-att_v2')).toHaveTextContent( + /probable version/i, + ) + expect(screen.getByTestId('family-confidence-att_v2')).toHaveTextContent( + /signals:.*stem/i, + ) + + fireEvent.click(screen.getByTestId('family-compare-att_v1-att_v2')) + expect(await screen.findByTestId('version-compare-view')).toBeInTheDocument() + expect(await screen.findByTestId('compare-metadata')).toBeInTheDocument() + expect(screen.getByTestId('compare-amount-changes')).toHaveTextContent('$8,000') + expect(screen.getByTestId('compare-amount-changes')).toHaveTextContent('$9,500') + + // Text prefixes on diff kinds (not color-only) + const delLines = screen.getAllByTestId('diff-line-del') + expect(delLines[0]?.textContent?.startsWith('−')).toBe(true) + const addLines = screen.getAllByTestId('diff-line-add') + expect(addLines[0]?.textContent?.startsWith('+')).toBe(true) + const sameLines = screen.getAllByTestId('diff-line-same') + expect(sameLines[0]?.textContent).toMatch(/Roof quote/) + + // Preview links intact + expect(screen.getByTestId('compare-preview-a')).toHaveAttribute( + 'href', + expect.stringContaining('/api/attachments/att_v1/preview'), + ) + }) }) describe('PreviewPanel', () => { diff --git a/apps/chronicle/web/src/files/FilesPage.tsx b/apps/chronicle/web/src/files/FilesPage.tsx index 9e68355..96220a4 100644 --- a/apps/chronicle/web/src/files/FilesPage.tsx +++ b/apps/chronicle/web/src/files/FilesPage.tsx @@ -11,6 +11,7 @@ import type { } from '../api/types' import { useWorkingSetStore } from '../workingset/store' import type { FilesViewMode } from '../workingset/urlState' +import { FamilyPanel } from './FamilyPanel' import { contentTypeFamily, formatBytes, @@ -18,6 +19,10 @@ import { previewUrl, truncateFilename, } from './format' +import { VersionCompareView } from './VersionCompareView' + +/** List items may include server family_count (task 4.5). */ +type FileRowItem = AttachmentListItem & { family_count?: number } const btnClass = 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' @@ -69,12 +74,14 @@ function FilesTable({ expanded, onToggleExpand, onSelect, + onOpenFamily, }: { - items: AttachmentListItem[] + items: FileRowItem[] groupDuplicates: boolean expanded: Set onToggleExpand: (id: string) => void - onSelect: (item: AttachmentListItem) => void + onSelect: (item: FileRowItem) => void + onOpenFamily: (id: string) => void }) { return (
@@ -89,6 +96,7 @@ function FilesTable({ Source Extraction Dup + Versions @@ -104,6 +112,7 @@ function FilesTable({ failed={failed} onToggleExpand={onToggleExpand} onSelect={onSelect} + onOpenFamily={onOpenFamily} /> ) })} @@ -120,15 +129,18 @@ function FragmentRow({ failed, onToggleExpand, onSelect, + onOpenFamily, }: { - item: AttachmentListItem + item: FileRowItem groupDuplicates: boolean isExpanded: boolean failed: boolean onToggleExpand: (id: string) => void - onSelect: (item: AttachmentListItem) => void + onSelect: (item: FileRowItem) => void + onOpenFamily: (id: string) => void }) { const sender = item.sender_name || item.sender_address || '—' + const familyCount = item.family_count ?? 1 return ( <> — )} + + {familyCount > 1 ? ( + + ) : ( + — + )} + {groupDuplicates && isExpanded && item.occurrences ? ( - +

Occurrences (exact duplicates)

@@ -322,6 +351,10 @@ export function FilesPage() { const [status, setStatus] = useState('') const [groupDuplicates, setGroupDuplicates] = useState(false) const [expanded, setExpanded] = useState>(() => new Set()) + const [familyAttSid, setFamilyAttSid] = useState(null) + const [comparePair, setComparePair] = useState<{ a: string; b: string } | null>( + null, + ) // Keep draft in sync when URL fq changes (back/forward). useEffect(() => { setFilenameDraft(filesQuery) @@ -383,12 +416,12 @@ export function FilesPage() { }) const items = useMemo( - () => query.data?.pages.flatMap((p) => p.items) ?? [], + () => (query.data?.pages.flatMap((p) => p.items) ?? []) as FileRowItem[], [query.data], ) const onSelect = useCallback( - (item: AttachmentListItem) => { + (item: FileRowItem) => { setSelection({ kind: 'attachment', sid: item.id }) }, [setSelection], @@ -403,6 +436,27 @@ export function FilesPage() { }) }, []) + const onOpenFamily = useCallback((id: string) => { + setComparePair(null) + setFamilyAttSid(id) + }, []) + + const onCompare = useCallback((a: string, b: string) => { + setComparePair({ a, b }) + }, []) + + if (comparePair) { + return ( +
+ setComparePair(null)} + /> +
+ ) + } + return (
@@ -510,9 +564,21 @@ export function FilesPage() { expanded={expanded} onToggleExpand={onToggleExpand} onSelect={onSelect} + onOpenFamily={onOpenFamily} /> )} + {familyAttSid ? ( + setFamilyAttSid(null)} + onCompare={onCompare} + onSelect={(sid) => { + setSelection({ kind: 'attachment', sid }) + }} + /> + ) : null} + {query.hasNextPage ? (
+
+ + {query.isLoading ? ( +

+ Loading comparison… +

+ ) : null} + + {query.isError ? ( +
+ Failed to compare (extraction may be missing) + +
+ ) : null} + + {query.data ? ( + <> +
+ + +
+ + {query.data.amount_changes.length > 0 ? ( +
+

+ Changed amounts +

+
    + {query.data.amount_changes.map((ch, i) => ( +
  • + + {ch.kind === 'add' ? '+' : '−'} + + {ch.text} + + ({ch.amounts.join(', ')}) + +
    • + ))} +
+
+ ) : ( +

+ No amount changes detected +

+ )} + +
+

+ Text diff + {query.data.truncated ? ' (truncated)' : ''} +

+ {query.data.hunks.length === 0 ? ( +

No differences

+ ) : ( +
+ {query.data.hunks.map((hunk, hi) => ( +
+

+ @@ a:{hunk.a_start} b:{hunk.b_start} @@ +

+ {hunk.lines.map((line, li) => ( + + ))} +
+ ))} +
+ )} +
+ + ) : null} +
+ ) +} diff --git a/apps/chronicle/web/src/files/api.ts b/apps/chronicle/web/src/files/api.ts new file mode 100644 index 0000000..dc59581 --- /dev/null +++ b/apps/chronicle/web/src/files/api.ts @@ -0,0 +1,27 @@ +import { apiGet } from '../api/client' +import type { + AttachmentCompareResponse, + AttachmentFamilyResponse, +} from './types' + +export function getAttachmentFamily( + attSid: string, + signal?: AbortSignal, +): Promise { + return apiGet( + `/api/attachments/${encodeURIComponent(attSid)}/family`, + signal, + ) +} + +export function getAttachmentCompare( + a: string, + b: string, + signal?: AbortSignal, +): Promise { + const sp = new URLSearchParams({ a, b }) + return apiGet( + `/api/attachments/compare?${sp}`, + signal, + ) +} diff --git a/apps/chronicle/web/src/files/types.ts b/apps/chronicle/web/src/files/types.ts new file mode 100644 index 0000000..f798d6a --- /dev/null +++ b/apps/chronicle/web/src/files/types.ts @@ -0,0 +1,64 @@ +/** Local types for version families + attachment compare (task 4.5). */ + +export type FamilyConfidence = 'exact-duplicate' | 'probable-version' + +export interface FamilyCandidate { + id: string + filename: string + date: string | null + sender: string | null + size: number | null + sha256: string + confidence: FamilyConfidence + signals: string[] +} + +export interface AttachmentFamilyResponse { + id: string + stem: string + candidates: FamilyCandidate[] +} + +export type DiffLineKind = 'same' | 'add' | 'del' + +export interface DiffLine { + kind: DiffLineKind + text: string +} + +export interface DiffHunk { + a_start: number + b_start: number + lines: DiffLine[] +} + +export interface AmountChange { + kind: 'add' | 'del' + text: string + amounts: string[] +} + +export interface AttachmentMeta { + id: string + filename: string + content_type: string | null + size: number | null + date: string | null + sender: string | null + sha256: string + source_message_id: string | null +} + +export interface AttachmentCompareResponse { + a: AttachmentMeta + b: AttachmentMeta + hunks: DiffHunk[] + truncated: boolean + amount_changes: AmountChange[] +} + +/** List item may include family_count from the server. */ +export interface AttachmentListItemWithFamily { + id: string + family_count?: number +} diff --git a/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx b/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx index 3296879..a6e5691 100644 --- a/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx +++ b/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx @@ -426,4 +426,222 @@ describe('InspectorPanel flow', () => { expect(body.content.title).toMatch(/Hello world/) expect(await screen.findByTestId('pin-status')).toHaveTextContent('Pinned') }) + + const TOPIC_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' + + function topicDetail(overrides: Record = {}) { + return { + id: TOPIC_ID, + label: 'Kitchen remodel', + description: 'home projects', + origin: 'automatic', + parent_id: null, + hidden: false, + top_terms: ['kitchen', 'remodel'], + generation: 1, + member_count: 12, + created_at: null, + updated_at: null, + activity: [ + { bucket: '2020-01-01T00:00:00Z', count: 4 }, + { bucket: '2020-02-01T00:00:00Z', count: 8 }, + ], + members: [ + { + id: 'msg_topic_rep', + subject: 'Quote for cabinets', + sender_name: 'Bob', + sender_address: 'bob@example.com', + date: '2020-01-15T00:00:00Z', + mailbox: 'me@example.com', + thread_id: null, + distance: 0.1, + }, + ], + ...overrides, + } + } + + it('topic selection kind t: shows topic card with origin and rename tooltip', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + if (String(url).includes(`/api/topics/${TOPIC_ID}`)) { + return { + ok: true, + status: 200, + json: async () => topicDetail(), + } as Response + } + throw new Error(`unexpected: ${url}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + expect(await screen.findByTestId('topic-card')).toBeInTheDocument() + expect(screen.getByTestId('topic-origin-badge')).toHaveTextContent(/Automatic/i) + expect(screen.getByTestId('topic-rename-tooltip')).toHaveTextContent( + /flips origin to curated/i, + ) + expect(screen.getByTestId('topic-activity-sparkline')).toBeInTheDocument() + }) + + it('rename → curated flow with tooltip', async () => { + let origin = 'automatic' + let label = 'Kitchen remodel' + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string, init?: RequestInit) => { + const u = String(url) + const method = (init?.method || 'GET').toUpperCase() + if (u.includes(`/api/topics/${TOPIC_ID}`) && method === 'PATCH') { + const body = JSON.parse(String(init?.body || '{}')) as { label?: string } + label = body.label ?? label + origin = 'curated' + return { + ok: true, + status: 200, + json: async () => topicDetail({ label, origin }), + } as Response + } + if (u.includes(`/api/topics/${TOPIC_ID}`)) { + return { + ok: true, + status: 200, + json: async () => topicDetail({ label, origin }), + } as Response + } + throw new Error(`unexpected: ${method} ${u}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + await screen.findByTestId('topic-card') + fireEvent.click(screen.getByTestId('topic-label')) + fireEvent.change(screen.getByTestId('topic-rename-input'), { + target: { value: 'Home kitchen' }, + }) + fireEvent.submit(screen.getByTestId('topic-rename-form')) + await waitFor(() => { + expect(screen.getByTestId('topic-origin-badge')).toHaveTextContent(/Curated/i) + }) + expect(screen.getByTestId('topic-rename-tooltip')).toHaveTextContent(/Curated/i) + }) + + it('representative member click → source (message) selection', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + const u = String(url) + if (u.includes(`/api/topics/${TOPIC_ID}`) && !u.includes('/members')) { + return { + ok: true, + status: 200, + json: async () => topicDetail(), + } as Response + } + if (u.includes('/api/sources/msg_topic_rep')) { + return { + ok: true, + status: 200, + json: async () => ({ + ...msgSource, + envelope: { ...msgSource.envelope, id: 'msg_topic_rep', subject: 'Quote for cabinets' }, + }), + } as Response + } + throw new Error(`unexpected: ${u}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + await screen.findByTestId('topic-card') + fireEvent.click(screen.getByTestId('topic-member-msg_topic_rep')) + await waitFor(() => { + expect(useWorkingSetStore.getState().selection).toEqual({ + kind: 'message', + sid: 'msg_topic_rep', + }) + }) + expect(await screen.findByTestId('message-card')).toBeInTheDocument() + }) + + it('member list pagination (TA-004 definitive list)', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + const u = String(url) + if (u.includes(`/api/topics/${TOPIC_ID}/members`)) { + if (u.includes('cursor=')) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: 'msg_page2', + subject: 'Second page', + sender_name: null, + sender_address: null, + date: null, + mailbox: null, + thread_id: null, + distance: null, + }, + ], + next_cursor: null, + }), + } as Response + } + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: 'msg_page1', + subject: 'First page', + sender_name: null, + sender_address: null, + date: null, + mailbox: null, + thread_id: null, + distance: null, + }, + ], + next_cursor: 'cursor_abc', + }), + } as Response + } + if (u.includes(`/api/topics/${TOPIC_ID}`)) { + return { + ok: true, + status: 200, + json: async () => topicDetail(), + } as Response + } + throw new Error(`unexpected: ${u}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + await screen.findByTestId('topic-card') + fireEvent.click(screen.getByTestId('topic-open-sources')) + expect(await screen.findByTestId('topic-member-list')).toBeInTheDocument() + expect(screen.getByText('First page')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('topic-members-more')) + expect(await screen.findByText('Second page')).toBeInTheDocument() + }) }) diff --git a/apps/chronicle/web/src/inspector/InspectorPanel.tsx b/apps/chronicle/web/src/inspector/InspectorPanel.tsx index acbba59..2fbb098 100644 --- a/apps/chronicle/web/src/inspector/InspectorPanel.tsx +++ b/apps/chronicle/web/src/inspector/InspectorPanel.tsx @@ -9,6 +9,7 @@ import { downloadUrl } from '../files/format' import { PreviewPanel } from '../files/PreviewPanel' import { useWorkingSetStore } from '../workingset/store' import { formatPeriodLabel, UNIT_MS, type Unit } from '../chronicle/timeScale' +import { TopicCard } from '../topics/TopicCard' import { PinToWorkspace } from '../workspaces/PinToWorkspace' import { MessageCard } from './MessageCard' import { SourceList } from './SourceList' @@ -205,6 +206,15 @@ export function InspectorPanel({ bucketCount }: InspectorPanelProps) { ) } + if (selection.kind === 'topic') { + return ( + setSelection(null)} + /> + ) + } + // Bucket selection const { bucketIso, lane } = selection const dateFrom = bucketIso diff --git a/apps/chronicle/web/src/people/ActivityBars.tsx b/apps/chronicle/web/src/people/ActivityBars.tsx new file mode 100644 index 0000000..941c200 --- /dev/null +++ b/apps/chronicle/web/src/people/ActivityBars.tsx @@ -0,0 +1,54 @@ +/** Compact monthly activity bar row (simple SVG). */ + +export interface ActivityBarsProps { + buckets: { bucket: string; count: number }[] + height?: number + className?: string +} + +export function ActivityBars({ + buckets, + height = 32, + className = '', +}: ActivityBarsProps) { + if (!buckets.length) { + return ( +

+ No sent-message activity +

+ ) + } + const max = Math.max(...buckets.map((b) => b.count), 1) + const barW = Math.max(2, Math.min(8, Math.floor(280 / buckets.length))) + const gap = 1 + const width = buckets.length * (barW + gap) + + return ( + + {buckets.map((b, i) => { + const h = Math.max(1, (b.count / max) * (height - 2)) + return ( + + + {b.bucket.slice(0, 7)}: {b.count} + + + ) + })} + + ) +} diff --git a/apps/chronicle/web/src/people/EgoGraph.test.tsx b/apps/chronicle/web/src/people/EgoGraph.test.tsx new file mode 100644 index 0000000..9899009 --- /dev/null +++ b/apps/chronicle/web/src/people/EgoGraph.test.tsx @@ -0,0 +1,276 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import type { + ContactCard, + EgoGraphResponse, + ThreadResponse, +} from '../api/types' +import { EgoGraph } from './EgoGraph' + +const EGO_ID = '11111111-1111-1111-1111-111111111111' +const BOB_ID = '22222222-2222-2222-2222-222222222222' +const THR_A = 'thr_YQ' +const THR_B = 'thr_Yg' + +function mockCard(overrides: Partial = {}): ContactCard { + return { + id: EGO_ID, + display_name: 'Ego Person', + kind: 'human', + kind_source: 'manual', + tags: [], + human_probability: 0.9, + addresses: ['ego@example.com'], + name_variants: [], + messages_from: 10, + messages_to: 2, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2018-12-31T00:00:00Z', + notes: null, + metadata: {}, + classification_signals: null, + classified_at: null, + address_classes: { 'ego@example.com': 'external' }, + address_details: [], + activity: [], + topics: [], + thread_count: 5, + ...overrides, + } +} + +function mockGraph(overrides: Partial = {}): EgoGraphResponse { + return { + nodes: [ + { id: EGO_ID, label: 'Ego Person', kind: 'human', is_ego: true }, + { id: BOB_ID, label: 'Alice Chen', kind: 'human', is_ego: false }, + { + id: 'addr:stranger@example.com', + label: 'stranger@example.com', + kind: 'address', + is_ego: false, + }, + ], + edges: [ + { + source: EGO_ID, + target: BOB_ID, + kind: 'thread_co_participation', + shared_threads: 12, + first: '2014-01-01', + last: '2018-06-01', + evidence: { thread_ids: [THR_A, THR_B] }, + }, + { + source: EGO_ID, + target: 'addr:stranger@example.com', + kind: 'thread_co_participation', + shared_threads: 3, + first: '2015-01-01', + last: '2015-06-01', + evidence: { thread_ids: [THR_A] }, + }, + ], + truncated: true, + total_coparticipants: 40, + ...overrides, + } +} + +function mockThread(tid: string, subject: string): ThreadResponse { + return { + thread_id: tid, + subject, + date_range: { from: '2015-01-01T00:00:00Z', to: '2015-02-01T00:00:00Z' }, + participants: [{ name: 'Alice Chen', address: 'alice@example.com' }], + message_count: 2, + messages: [ + { + id: `msg_${tid}`, + subject, + sender_name: 'Alice Chen', + sender_address: 'alice@example.com', + recipients: {}, + date: '2015-01-01T00:00:00Z', + mailbox: 'inbox', + labels: [], + has_attachment: false, + }, + ], + truncated: false, + } +} + +function renderEgo( + graph: EgoGraphResponse = mockGraph(), + card: ContactCard = mockCard(), +) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.includes('/graph')) { + return { + ok: true, + status: 200, + json: async () => graph, + } as Response + } + if (url.includes(`/api/threads/${THR_A}`)) { + return { + ok: true, + status: 200, + json: async () => mockThread(THR_A, 'Roof repair plan'), + } as Response + } + if (url.includes(`/api/threads/${THR_B}`)) { + return { + ok: true, + status: 200, + json: async () => mockThread(THR_B, 'Follow-up quotes'), + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }) + vi.stubGlobal('fetch', fetchMock) + + const view = render( + + + + } + /> + Source
} + /> + + + , + ) + return { ...view, fetchMock } +} + +describe('EgoGraph', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('renders from mock with sizes/widths monotonic in shared_threads', async () => { + renderEgo() + await screen.findByTestId('ego-svg') + expect(screen.getByTestId('ego-truncated')).toHaveTextContent( + 'Showing 2 of 40 co-participants', + ) + + const edgeBob = screen.getByTestId(`ego-edge-${BOB_ID}`) + const edgeAddr = screen.getByTestId( + 'ego-edge-addr:stranger@example.com', + ) + const wBob = Number(edgeBob.getAttribute('data-width')) + const wAddr = Number(edgeAddr.getAttribute('data-width')) + expect(wBob).toBeGreaterThan(wAddr) + expect(wBob).toBeLessThanOrEqual(4) + expect(wAddr).toBeGreaterThanOrEqual(1) + + const nodeBob = screen.getByTestId(`ego-node-${BOB_ID}`) + const nodeAddr = screen.getByTestId( + 'ego-node-addr:stranger@example.com', + ) + const rBob = Number(nodeBob.getAttribute('data-radius')) + const rAddr = Number(nodeAddr.getAttribute('data-radius')) + expect(rBob).toBeGreaterThan(rAddr) + }) + + it('edge click loads evidence rows via thread fetch', async () => { + const { fetchMock } = renderEgo() + await screen.findByTestId('ego-svg') + fireEvent.click(screen.getByTestId(`ego-edge-${BOB_ID}`)) + + expect(await screen.findByTestId('ego-evidence')).toHaveTextContent( + '12 shared threads with Alice Chen (2014–2018)', + ) + + await waitFor(() => { + const thrCalls = fetchMock.mock.calls.filter((c) => + String(c[0]).includes('/api/threads/'), + ) + expect(thrCalls.length).toBeGreaterThanOrEqual(2) + }) + + expect( + await screen.findByTestId(`ego-evidence-row-${THR_A}`), + ).toHaveTextContent('Roof repair plan') + expect(screen.getByTestId(`ego-evidence-row-${THR_B}`)).toHaveTextContent( + 'Follow-up quotes', + ) + }) + + it('table alternative has name / shared threads / first / last parity', async () => { + renderEgo() + await screen.findByTestId('ego-view-toggle') + fireEvent.click(screen.getByTestId('ego-view-toggle')) + const table = await screen.findByTestId('ego-table') + expect(table).toHaveTextContent('Name') + expect(table).toHaveTextContent('Shared threads') + expect(table).toHaveTextContent('First') + expect(table).toHaveTextContent('Last') + expect(table).toHaveTextContent('Alice Chen') + expect(table).toHaveTextContent('12') + expect(table).toHaveTextContent('2014-01-01') + expect(table).toHaveTextContent('2018-06-01') + expect(table).toHaveTextContent('stranger@example.com') + }) + + it('nodes and edges are focusable with shared-thread labels', async () => { + renderEgo() + await screen.findByTestId('ego-svg') + const node = screen.getByTestId(`ego-node-${BOB_ID}`) + expect(node).toHaveAttribute('tabindex', '0') + expect(node).toHaveAttribute( + 'aria-label', + 'Alice Chen — 12 shared threads', + ) + const edge = screen.getByTestId(`ego-edge-${BOB_ID}`) + expect(edge).toHaveAttribute('tabindex', '0') + expect(edge).toHaveAttribute( + 'aria-label', + 'Alice Chen — 12 shared threads', + ) + }) + + it('address node shows not yet a contact text', async () => { + renderEgo() + await screen.findByTestId('ego-svg') + fireEvent.click( + screen.getByTestId('ego-node-addr:stranger@example.com'), + ) + expect(await screen.findByTestId('ego-address-note')).toHaveTextContent( + 'not yet a contact', + ) + }) + + it('evidence row links to thread in the reader', async () => { + renderEgo() + await screen.findByTestId('ego-svg') + fireEvent.click(screen.getByTestId(`ego-edge-${BOB_ID}`)) + await waitFor(() => { + const row = screen.getByTestId(`ego-evidence-row-${THR_A}`) + expect(row.tagName.toLowerCase()).toBe('a') + expect(row).toHaveAttribute( + 'href', + `/source/${encodeURIComponent(`msg_${THR_A}`)}?thread=1`, + ) + }) + }) +}) diff --git a/apps/chronicle/web/src/people/EgoGraph.tsx b/apps/chronicle/web/src/people/EgoGraph.tsx new file mode 100644 index 0000000..12a3c46 --- /dev/null +++ b/apps/chronicle/web/src/people/EgoGraph.tsx @@ -0,0 +1,550 @@ +/** + * Depth-1 ego co-participation graph (spec §7.3). + * Edge selection drills to exact shared threads (PE-003). + * No relationship-quality labels (PE-004). + */ + +import { useQueries, useQuery } from '@tanstack/react-query' +import { useCallback, useMemo, useState } from 'react' +import { Link, useNavigate } from 'react-router' + +import { apiGet } from '../api/client' +import type { + ContactCard, + EgoGraphEdge, + EgoGraphNode, + EgoGraphResponse, + ThreadResponse, +} from '../api/types' +import { PEOPLE_CYAN } from '../chronicle/laneModel' +import { layoutEgoGraph } from './egoLayout' + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[12px] text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +const MAX_NODE_OPTIONS = [10, 25, 50] as const +const LABEL_ALWAYS_THRESHOLD = 12 +const ACCENT = '#5aa7ff' +const EGO_RING = '#e8ecf4' + +export interface EgoGraphProps { + contactId: string + card: ContactCard +} + +function yearOf(iso: string | null | undefined): string { + if (!iso) return '?' + return iso.slice(0, 4) +} + +function dateInputValue(iso: string | null | undefined): string { + if (!iso) return '' + return iso.slice(0, 10) +} + +function sharedForNode( + nodeId: string, + edges: EgoGraphEdge[], + egoId: string, +): number { + if (nodeId === egoId) return 0 + const e = edges.find((x) => x.target === nodeId || x.source === nodeId) + return e?.shared_threads ?? 0 +} + +function edgeWidth(shared: number, maxShared: number): number { + if (maxShared <= 0) return 1 + const t = Math.sqrt(shared / maxShared) + return 1 + t * 3 // 1–4px +} + +function nodeRadius(shared: number, maxShared: number, isEgo: boolean): number { + if (isEgo) return 14 + if (maxShared <= 0) return 6 + return 5 + 10 * Math.sqrt(shared / maxShared) +} + +export function EgoGraph({ contactId, card }: EgoGraphProps) { + const navigate = useNavigate() + const [maxNodes, setMaxNodes] = useState(25) + const [dateFrom, setDateFrom] = useState(() => dateInputValue(card.first_seen)) + const [dateTo, setDateTo] = useState(() => dateInputValue(card.last_seen)) + const [asTable, setAsTable] = useState(false) + const [selectedEdge, setSelectedEdge] = useState(null) + const [selectedNodeId, setSelectedNodeId] = useState(null) + const [hoverId, setHoverId] = useState(null) + + const graphQuery = useQuery({ + queryKey: ['people', contactId, 'graph', maxNodes, dateFrom, dateTo], + queryFn: ({ signal }) => { + const sp = new URLSearchParams() + sp.set('depth', '1') + sp.set('max_nodes', String(maxNodes)) + if (dateFrom) sp.set('date_from', dateFrom) + if (dateTo) sp.set('date_to', dateTo) + return apiGet( + `/api/people/${encodeURIComponent(contactId)}/graph?${sp}`, + signal, + ) + }, + enabled: !!contactId, + retry: false, + }) + + const data = graphQuery.data + const edges = data?.edges ?? [] + const nodes = data?.nodes ?? [] + const egoNode = nodes.find((n) => n.is_ego) + const egoId = egoNode?.id ?? contactId + const neighbors = nodes.filter((n) => !n.is_ego) + + const maxShared = useMemo( + () => edges.reduce((m, e) => Math.max(m, e.shared_threads), 0) || 1, + [edges], + ) + + const layout = useMemo(() => { + const neighborInputs = neighbors.map((n) => ({ + id: n.id, + shared_threads: sharedForNode(n.id, edges, egoId), + })) + return layoutEgoGraph(neighborInputs) + }, [neighbors, edges, egoId]) + + const posById = useMemo(() => { + const m = new Map() + for (const p of layout) { + if (p.is_ego) m.set(egoId, { x: p.x, y: p.y }) + else m.set(p.id, { x: p.x, y: p.y }) + } + return m + }, [layout, egoId]) + + const evidenceIds = selectedEdge?.evidence?.thread_ids?.slice(0, 20) ?? [] + + const threadQueries = useQueries({ + queries: evidenceIds.map((tid) => ({ + queryKey: ['threads', tid], + queryFn: ({ signal }: { signal?: AbortSignal }) => + apiGet( + `/api/threads/${encodeURIComponent(tid)}`, + signal, + ), + enabled: !!selectedEdge && evidenceIds.length > 0, + retry: false as const, + })), + }) + + const nodeById = useMemo(() => { + const m = new Map() + for (const n of nodes) m.set(n.id, n) + return m + }, [nodes]) + + const onEdgeSelect = useCallback( + (edge: EgoGraphEdge) => { + setSelectedEdge(edge) + setSelectedNodeId(edge.target === egoId ? edge.source : edge.target) + }, + [egoId], + ) + + /** Neighbor click: evidence panel; contact nodes also go to profile. */ + const onNeighborActivate = useCallback( + (node: EgoGraphNode) => { + if (node.is_ego) return + setSelectedNodeId(node.id) + const edge = edges.find( + (e) => e.target === node.id || e.source === node.id, + ) + if (edge) setSelectedEdge(edge) + if (!node.id.startsWith('addr:')) { + void navigate(`/people/${encodeURIComponent(node.id)}`) + } + }, + [edges, navigate], + ) + + const selectedNeighbor = selectedNodeId + ? nodeById.get(selectedNodeId) + : null + + const showAlwaysLabels = neighbors.length <= LABEL_ALWAYS_THRESHOLD + + const toSvg = (x: number, y: number, size = 360) => ({ + cx: size / 2 + x * (size / 2 - 28), + cy: size / 2 + y * (size / 2 - 28), + }) + + const size = 360 + + return ( +
+
+

+ Co-participants +

+ + Thread co-participation · depth 1 + +
+ +
+ + + + +
+ + {graphQuery.isLoading ? ( +

+ Loading co-participants… +

+ ) : null} + + {graphQuery.isError ? ( +
+ Failed to load graph + +
+ ) : null} + + {data?.truncated ? ( +

+ Showing {neighbors.length} of {data.total_coparticipants}{' '} + co-participants (capped at {maxNodes}) +

+ ) : data ? ( +

+ {data.total_coparticipants} co-participant + {data.total_coparticipants === 1 ? '' : 's'} +

+ ) : null} + + {data && neighbors.length === 0 && !graphQuery.isLoading ? ( +

+ No co-participants in this span +

+ ) : null} + + {data && asTable ? ( +
+ + + + + + + + + + + + {edges.map((edge) => { + const nid = edge.target === egoId ? edge.source : edge.target + const node = nodeById.get(nid) + const label = node?.label ?? nid + return ( + { + setSelectedEdge(edge) + setSelectedNodeId(nid) + }} + data-testid={`ego-table-row-${nid}`} + > + + + + + + ) + })} + +
+ Co-participants by shared thread count +
+ Name + + Shared threads + + First + + Last +
+ {label} + + {edge.shared_threads} + + {edge.first ?? '—'} + + {edge.last ?? '—'} +
+
+ ) : null} + + {data && !asTable && neighbors.length > 0 ? ( + + {/* edges under nodes */} + {edges.map((edge) => { + const a = posById.get(edge.source) + const b = posById.get(edge.target) + if (!a || !b) return null + const pa = toSvg(a.x, a.y, size) + const pb = toSvg(b.x, b.y, size) + const w = edgeWidth(edge.shared_threads, maxShared) + const targetId = + edge.target === egoId ? edge.source : edge.target + const targetNode = nodeById.get(targetId) + const label = targetNode?.label ?? targetId + const selected = + selectedEdge?.target === edge.target && + selectedEdge?.source === edge.source + return ( + onEdgeSelect(edge)} + onKeyDown={(ev) => { + if (ev.key === 'Enter' || ev.key === ' ') { + ev.preventDefault() + onEdgeSelect(edge) + } + }} + /> + ) + })} + + {nodes.map((node) => { + const pos = posById.get(node.id) + if (!pos) return null + const { cx, cy } = toSvg(pos.x, pos.y, size) + const shared = sharedForNode(node.id, edges, egoId) + const r = nodeRadius(shared, maxShared, node.is_ego) + const focused = + hoverId === node.id || selectedNodeId === node.id + const showLabel = + showAlwaysLabels || focused || node.is_ego + const ariaShared = node.is_ego + ? `${node.label} — ego` + : `${node.label} — ${shared} shared threads` + return ( + + {node.is_ego ? ( + + ) : null} + { + if (node.is_ego) return + onNeighborActivate(node) + }} + onKeyDown={(ev) => { + if (ev.key === 'Enter' || ev.key === ' ') { + ev.preventDefault() + if (node.is_ego) return + onNeighborActivate(node) + } + }} + onFocus={() => setHoverId(node.id)} + onBlur={() => + setHoverId((id) => (id === node.id ? null : id)) + } + onMouseEnter={() => setHoverId(node.id)} + onMouseLeave={() => + setHoverId((id) => (id === node.id ? null : id)) + } + > + {ariaShared} + + {showLabel ? ( + + {node.label.length > 22 + ? `${node.label.slice(0, 20)}…` + : node.label} + + ) : null} + + ) + })} + + ) : null} + + {/* Address-only node note when selected without profile */} + {selectedNeighbor?.id.startsWith('addr:') ? ( +

+ {selectedNeighbor.label} — not yet a contact +

+ ) : null} + + {selectedEdge ? ( +
+

+ {selectedEdge.shared_threads} shared thread + {selectedEdge.shared_threads === 1 ? '' : 's'} with{' '} + {selectedNeighbor?.label ?? + (selectedEdge.target === egoId + ? selectedEdge.source + : selectedEdge.target)}{' '} + ({yearOf(selectedEdge.first)}–{yearOf(selectedEdge.last)}) +

+
    + {evidenceIds.map((tid, i) => { + const q = threadQueries[i] + const t = q?.data + const subject = t?.subject ?? (q?.isLoading ? 'Loading…' : tid) + const date = t?.date_range?.from ?? t?.messages?.[0]?.date ?? null + const sender = + t?.participants?.[0]?.name || + t?.participants?.[0]?.address || + t?.messages?.[0]?.sender_name || + t?.messages?.[0]?.sender_address || + '' + const firstMsg = t?.messages?.[0]?.id + return ( +
  • + {firstMsg ? ( + + {subject || '(no subject)'} + + {date ? date.slice(0, 10) : ''} + {sender ? ` · ${sender}` : ''} + + + ) : ( +
    + {q?.isError ? `Failed to load ${tid}` : subject} +
    + )} +
    • + ) + })} + {evidenceIds.length === 0 ? ( +
  • No thread ids
    • + ) : null} +
+
+ ) : null} +
+ ) +} diff --git a/apps/chronicle/web/src/people/PeoplePage.test.tsx b/apps/chronicle/web/src/people/PeoplePage.test.tsx new file mode 100644 index 0000000..c9c4745 --- /dev/null +++ b/apps/chronicle/web/src/people/PeoplePage.test.tsx @@ -0,0 +1,193 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { + ContactSummary, + MergeCandidatesResponse, + PeopleListResponse, +} from '../api/types' +import { PeoplePage } from './PeoplePage' + +function contact(overrides: Partial = {}): ContactSummary { + return { + id: 'c1', + display_name: 'Alice Example', + kind: 'human', + kind_source: 'manual', + tags: [], + human_probability: 0.87, + addresses: ['alice@example.com'], + name_variants: ['Alice'], + messages_from: 10, + messages_to: 2, + first_seen: '2015-01-01T00:00:00Z', + last_seen: '2018-06-01T00:00:00Z', + ...overrides, + } +} + +function listResponse( + items: ContactSummary[], + total: number | null = items.length, +): PeopleListResponse { + return { + items, + total, + next_cursor: null, + limit: 50, + offset: 0, + } +} + +function renderPeople(initial = '/people') { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + return render( + + + + } /> + Profile} /> + + + , + ) +} + +describe('PeoplePage', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + beforeEach(() => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.includes('/api/people/merge-candidates')) { + const body: MergeCandidatesResponse = { + items: [ + { + norm_name: 'alice', + a: { + display_name: 'Alice A', + primary_address: 'a@x.com', + msg_count: 5, + contact_id: 'ca', + }, + b: { + display_name: 'Alice B', + primary_address: 'b@x.com', + msg_count: 3, + contact_id: 'cb', + }, + }, + ], + } + return { + ok: true, + status: 200, + json: async () => body, + } as Response + } + if (url.includes('/api/people')) { + const u = new URL(url, 'http://local') + const q = u.searchParams.get('q') + const kind = u.searchParams.get('kind') + const review = u.searchParams.get('needs_review') + let items = [ + contact({ id: 'c1', kind: 'human', display_name: 'Alice' }), + contact({ + id: 'c2', + kind: 'organization', + display_name: 'Acme Corp', + human_probability: 0.2, + }), + contact({ + id: 'c3', + kind: 'unknown', + display_name: 'Bob Unknown', + human_probability: 0.5, + }), + ] + if (q) items = items.filter((i) => i.display_name?.includes(q)) + if (kind) items = items.filter((i) => i.kind === kind) + if (review === 'true') items = items.filter((i) => i.kind === 'unknown') + return { + ok: true, + status: 200, + json: async () => listResponse(items), + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }), + ) + }) + + it('renders index rows with kind badge, probability, volume, span', async () => { + renderPeople() + expect(await screen.findByTestId('people-page')).toBeInTheDocument() + expect(await screen.findByTestId('people-row-c1')).toBeInTheDocument() + expect(screen.getByTestId('kind-badge-c1')).toHaveTextContent('human') + expect(screen.getByTestId('people-row-c1')).toHaveTextContent('0.87') + expect(screen.getByTestId('people-row-c1')).toHaveTextContent('12') + expect(screen.getByTestId('people-row-c1')).toHaveTextContent('2015-01-01') + }) + + it('search updates pq and refilters', async () => { + renderPeople() + await screen.findByTestId('people-row-c1') + fireEvent.change(screen.getByTestId('people-search'), { + target: { value: 'Acme' }, + }) + fireEvent.click(screen.getByTestId('people-search-submit')) + await waitFor(() => { + expect(screen.getByTestId('people-row-c2')).toBeInTheDocument() + }) + expect(screen.queryByTestId('people-row-c1')).not.toBeInTheDocument() + }) + + it('kind filter chips filter list; organizations tab preset', async () => { + renderPeople('/people?kind=organization') + expect(await screen.findByTestId('people-row-c2')).toBeInTheDocument() + expect(screen.queryByTestId('people-row-c1')).not.toBeInTheDocument() + expect(screen.getByTestId('people-kind-organization')).toHaveAttribute( + 'aria-selected', + 'true', + ) + }) + + it('needs-review toggle filters to unknown queue', async () => { + renderPeople() + await screen.findByTestId('people-row-c1') + fireEvent.click(screen.getByTestId('people-needs-review')) + await waitFor(() => { + expect(screen.getByTestId('people-row-c3')).toBeInTheDocument() + }) + expect(screen.queryByTestId('people-row-c1')).not.toBeInTheDocument() + }) + + it('merge candidates rail with confirm dialog naming direction', async () => { + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(false) + renderPeople() + expect(await screen.findByTestId('merge-pair-ca')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('merge-into-b-ca')) + expect(confirmSpy).toHaveBeenCalled() + const msg = String(confirmSpy.mock.calls[0]?.[0] ?? '') + expect(msg).toMatch(/Merge Alice A into Alice B/) + confirmSpy.mockRestore() + }) + + it('row click navigates to profile', async () => { + renderPeople() + fireEvent.click(await screen.findByTestId('people-row-c1')) + expect(await screen.findByText('Profile')).toBeInTheDocument() + }) +}) diff --git a/apps/chronicle/web/src/people/PeoplePage.tsx b/apps/chronicle/web/src/people/PeoplePage.tsx new file mode 100644 index 0000000..ae267fe --- /dev/null +++ b/apps/chronicle/web/src/people/PeoplePage.tsx @@ -0,0 +1,348 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useCallback, useMemo, useState } from 'react' +import { Link, useNavigate, useSearchParams } from 'react-router' + +import type { ContactKind, ContactSummary, MergeCandidatePair } from '../api/types' +import { listMergeCandidates, listPeople, mergePeople } from './api' + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[12px] text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +const KINDS: { id: '' | ContactKind; label: string }[] = [ + { id: '', label: 'All' }, + { id: 'human', label: 'Human' }, + { id: 'organization', label: 'Organizations' }, + { id: 'automated', label: 'Automated' }, + { id: 'mailing_list', label: 'Mailing list' }, + { id: 'unknown', label: 'Unknown' }, +] + +function volume(c: ContactSummary): number { + return (c.messages_from ?? 0) + (c.messages_to ?? 0) +} + +function formatProb(p: number | null | undefined): string { + if (p == null || !Number.isFinite(p)) return '—' + return p.toFixed(2) +} + +function formatSpan(first: string | null, last: string | null): string { + const f = first ? first.slice(0, 10) : '—' + const l = last ? last.slice(0, 10) : '—' + return `${f} → ${l}` +} + +/** + * People & Organizations index: search, kind chips, needs-review queue, + * merge-candidates rail. Organizations tab = kind=organization preset. + */ +export function PeoplePage() { + const [searchParams, setSearchParams] = useSearchParams() + const navigate = useNavigate() + const qc = useQueryClient() + + const pq = searchParams.get('pq') ?? '' + const kind = searchParams.get('kind') ?? '' + const needsReview = searchParams.get('review') === '1' + const [draft, setDraft] = useState(pq) + + const setParam = useCallback( + (key: string, value: string | null) => { + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev) + if (value == null || value === '') next.delete(key) + else next.set(key, value) + return next + }, + { replace: true }, + ) + }, + [setSearchParams], + ) + + const commitSearch = useCallback(() => { + setParam('pq', draft.trim() || null) + }, [draft, setParam]) + + // Kind counts from a first unfiltered page (same q / needs_review, no kind). + const countsQuery = useQuery({ + queryKey: ['people', 'counts', pq, needsReview], + queryFn: ({ signal }) => + listPeople( + { + q: pq || undefined, + needs_review: needsReview || undefined, + limit: 200, + }, + signal, + ), + retry: false, + }) + + const listQuery = useQuery({ + queryKey: ['people', 'list', pq, kind, needsReview], + queryFn: ({ signal }) => + listPeople( + { + q: pq || undefined, + kind: kind || undefined, + needs_review: needsReview || undefined, + limit: 50, + }, + signal, + ), + retry: false, + }) + + const candidatesQuery = useQuery({ + queryKey: ['people', 'merge-candidates'], + queryFn: ({ signal }) => listMergeCandidates(20, signal), + retry: false, + }) + + const kindCounts = useMemo(() => { + const counts: Record = {} + for (const item of countsQuery.data?.items ?? []) { + const k = item.kind || 'unknown' + counts[k] = (counts[k] ?? 0) + 1 + } + return counts + }, [countsQuery.data]) + + const mergeMut = useMutation({ + mutationFn: (body: { source_id: string; target_id: string }) => + mergePeople(body), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ['people'] }) + }, + }) + + const onMerge = useCallback( + (pair: MergeCandidatePair, direction: 'a_into_b' | 'b_into_a') => { + const source = direction === 'a_into_b' ? pair.a : pair.b + const target = direction === 'a_into_b' ? pair.b : pair.a + const sourceName = source.display_name || source.primary_address || source.contact_id + const targetName = target.display_name || target.primary_address || target.contact_id + const ok = window.confirm( + `Merge ${sourceName} into ${targetName}? This moves addresses to the target contact.`, + ) + if (!ok) return + mergeMut.mutate({ + source_id: source.contact_id, + target_id: target.contact_id, + }) + }, + [mergeMut], + ) + + const items = listQuery.data?.items ?? [] + + return ( +
+
+

+ People & Organizations +

+

+ Identity profiles over the contacts subsystem. Org domain controls + use tags/notes (MVP). +

+
+ +
+ setDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') commitSearch() + }} + placeholder="Search name or address…" + className="min-w-[12rem] flex-1 rounded-md border border-steel bg-graphite-900 px-2 py-1 text-[12px] text-text-primary" + data-testid="people-search" + aria-label="Search people" + /> + + +
+ +
+ {KINDS.map((k) => { + const selected = kind === k.id + const count = + k.id === '' + ? countsQuery.data?.items?.length + : kindCounts[k.id] + return ( + + ) + })} +
+ +
+
+ {listQuery.isLoading ? ( +

Loading…

+ ) : listQuery.isError ? ( +

+ Failed to load people +

+ ) : items.length === 0 ? ( +

No contacts match.

+ ) : ( + + + + + + + + + + + + {items.map((c) => ( + void navigate(`/people/${c.id}`)} + > + + + + + + + ))} + +
NameKindHuman pVolumeSpan
+ {c.display_name || c.addresses[0] || c.id} + + + {c.kind} + + + {formatProb(c.human_probability)} + {volume(c)} + {formatSpan(c.first_seen, c.last_seen)} +
+ )} + {listQuery.data?.total != null ? ( +

+ {listQuery.data.total} matching +

+ ) : null} +
+ + +
+
+ ) +} diff --git a/apps/chronicle/web/src/people/PersonProfilePage.test.tsx b/apps/chronicle/web/src/people/PersonProfilePage.test.tsx new file mode 100644 index 0000000..8cdb7bb --- /dev/null +++ b/apps/chronicle/web/src/people/PersonProfilePage.test.tsx @@ -0,0 +1,333 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ContactCard } from '../api/types' +import { resetWorkingSetStore, useWorkingSetStore } from '../workingset/store' +import { PersonProfilePage } from './PersonProfilePage' + +const CONTACT_ID = '11111111-1111-1111-1111-111111111111' +const TOPIC_ID = '22222222-2222-2222-2222-222222222222' +const MERGE_ID = '33333333-3333-3333-3333-333333333333' +const TARGET_ID = '44444444-4444-4444-4444-444444444444' + +function mockCard(overrides: Partial = {}): ContactCard { + return { + id: CONTACT_ID, + display_name: 'Alice Example', + kind: 'human', + kind_source: 'heuristic', + tags: ['vip'], + human_probability: 0.9, + addresses: ['alice@example.com', 'me@owner.com'], + name_variants: ['Alice'], + messages_from: 20, + messages_to: 5, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2019-12-31T00:00:00Z', + notes: 'Analyst note', + metadata: {}, + classification_signals: { + bidirectional: 0.15, + personal_name: 0.1, + }, + classified_at: '2020-01-01T00:00:00Z', + address_classes: { + 'alice@example.com': 'external', + 'me@owner.com': 'owner', + }, + address_details: [ + { + address: 'alice@example.com', + is_user: false, + messages_from: 20, + messages_to: 0, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2019-12-31T00:00:00Z', + }, + { + address: 'me@owner.com', + is_user: true, + messages_from: 0, + messages_to: 5, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2019-12-31T00:00:00Z', + }, + ], + activity: [ + { bucket: '2016-01-01T00:00:00Z', count: 4 }, + { bucket: '2016-02-01T00:00:00Z', count: 2 }, + ], + topics: [{ id: TOPIC_ID, label: 'Kitchen remodel', count: 7 }], + thread_count: 12, + merges: [ + { + id: MERGE_ID, + source_id: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + target_id: CONTACT_ID, + merged_at: '2021-06-01T00:00:00Z', + }, + ], + ...overrides, + } +} + +function emptyGraph(contactId = CONTACT_ID, label = 'Alice Example') { + return { + nodes: [{ id: contactId, label, kind: 'human', is_ego: true }], + edges: [] as [], + truncated: false, + total_coparticipants: 0, + } +} + +/** Fetch mock that serves the person card and an empty ego graph. */ +function personFetch(card: ContactCard, extra?: (url: string, init?: RequestInit) => Response | null) { + return vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (extra) { + const hit = extra(url, init) + if (hit) return hit + } + if (url.includes('/graph')) { + return { + ok: true, + status: 200, + json: async () => emptyGraph(card.id, card.display_name ?? 'Contact'), + } as Response + } + if (url.includes(`/api/people/${CONTACT_ID}`) && !url.includes('merge')) { + return { + ok: true, + status: 200, + json: async () => card, + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }) +} + +function renderProfile(initial = `/people/${CONTACT_ID}`) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + return render( + + + + } /> + Chronicle} /> + Research} /> + Topics} /> + + + , + ) +} + +describe('PersonProfilePage', () => { + beforeEach(() => { + resetWorkingSetStore() + }) + + afterEach(() => { + vi.unstubAllGlobals() + resetWorkingSetStore() + }) + + it('renders profile sections from mocked card incl. signals and owner badges', async () => { + const card = mockCard() + vi.stubGlobal('fetch', personFetch(card)) + + renderProfile() + expect(await screen.findByTestId('person-profile')).toBeInTheDocument() + expect(screen.getByTestId('person-identity')).toBeInTheDocument() + expect(screen.getByTestId('person-span')).toHaveTextContent('threads: 12') + expect(screen.getByTestId('person-activity')).toBeInTheDocument() + expect(screen.getByTestId('activity-bars')).toBeInTheDocument() + expect(screen.getByTestId('person-topics')).toHaveTextContent('Kitchen remodel') + expect(screen.getByTestId('address-class-me@owner.com')).toHaveTextContent( + 'owner', + ) + expect( + screen.getByTestId('address-class-alice@example.com'), + ).toHaveTextContent('external') + expect(await screen.findByTestId('ego-graph')).toBeInTheDocument() + + // Expand signals details + fireEvent.click(screen.getByTestId('person-signals').querySelector('summary')!) + expect(screen.getByTestId('person-signals')).toHaveTextContent('bidirectional') + expect(screen.getByTestId('person-signals')).toHaveTextContent('0.15') + }) + + it('kind PATCH with kind_source explanation', async () => { + const card = mockCard({ kind_source: 'heuristic', kind: 'unknown' }) + const fetchMock = personFetch(card, (_url, init) => { + if (init?.method === 'PATCH') { + const body = JSON.parse(String(init.body)) as { kind?: string } + return { + ok: true, + status: 200, + json: async () => + mockCard({ + kind: body.kind as ContactCard['kind'], + kind_source: 'manual', + }), + } as Response + } + return null + }) + vi.stubGlobal('fetch', fetchMock) + + renderProfile() + await screen.findByTestId('person-kind-source') + expect(screen.getByTestId('person-kind-source')).toHaveTextContent( + 'kind_source: heuristic', + ) + + fireEvent.change(screen.getByTestId('person-kind-select'), { + target: { value: 'human' }, + }) + await waitFor(() => { + const patches = fetchMock.mock.calls.filter( + (c) => (c[1] as RequestInit | undefined)?.method === 'PATCH', + ) + expect(patches.length).toBeGreaterThan(0) + }) + await waitFor(() => { + expect(screen.getByTestId('person-kind-source')).toHaveTextContent('manual') + expect(screen.getByTestId('person-kind-source')).toHaveTextContent( + 'locked against machinery', + ) + }) + }) + + it('merge flow confirm dialog naming direction', async () => { + const card = mockCard() + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(false) + vi.stubGlobal( + 'fetch', + personFetch(card, (url) => { + if (url.includes('/api/people?') && url.includes('q=')) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: TARGET_ID, + display_name: 'Target Person', + kind: 'human', + kind_source: 'manual', + tags: [], + human_probability: 0.5, + addresses: ['t@x.com'], + name_variants: [], + messages_from: 3, + messages_to: 1, + first_seen: null, + last_seen: null, + }, + ], + total: 1, + next_cursor: null, + limit: 8, + offset: 0, + }), + } as Response + } + return null + }), + ) + + renderProfile() + await screen.findByTestId('person-merge') + fireEvent.change(screen.getByTestId('merge-target-search'), { + target: { value: 'Target' }, + }) + await screen.findByTestId(`merge-target-${TARGET_ID}`) + fireEvent.click(screen.getByTestId(`merge-target-${TARGET_ID}`)) + fireEvent.click(screen.getByTestId('merge-confirm')) + expect(confirmSpy).toHaveBeenCalled() + const msg = String(confirmSpy.mock.calls[0]?.[0] ?? '') + expect(msg).toMatch(/Merge Alice Example into Target Person/) + confirmSpy.mockRestore() + }) + + it('unmerge from merge history', async () => { + const card = mockCard() + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(true) + const fetchMock = personFetch(card, (url, init) => { + if (init?.method === 'POST' && url.includes('/api/people/unmerge')) { + return { + ok: true, + status: 200, + json: async () => ({ + source: mockCard({ id: 'src', merges: undefined }), + target: mockCard({ merges: [] }), + }), + } as Response + } + return null + }) + vi.stubGlobal('fetch', fetchMock) + + renderProfile() + await screen.findByTestId('person-merge-history') + fireEvent.click(screen.getByTestId(`unmerge-${MERGE_ID}`)) + await waitFor(() => { + const posts = fetchMock.mock.calls.filter( + (c) => + String(c[0]).includes('/api/people/unmerge') && + (c[1] as RequestInit)?.method === 'POST', + ) + expect(posts.length).toBe(1) + const body = JSON.parse(String((posts[0]?.[1] as RequestInit).body)) + expect(body.merge_id).toBe(MERGE_ID) + }) + confirmSpy.mockRestore() + }) + + it('Open in Chronicle sets sender scope and viewport from span', async () => { + const card = mockCard() + vi.stubGlobal('fetch', personFetch(card)) + + renderProfile() + await screen.findByTestId('open-in-chronicle') + fireEvent.click(screen.getByTestId('open-in-chronicle')) + expect(await screen.findByTestId('chronicle')).toBeInTheDocument() + // Navigation encodes senders in URL; store may also be hydrated by useUrlSync + // on real app — here we only verify the button navigates to Chronicle. + }) + + it('View correspondence sets scope senders and navigates to research exact mode', async () => { + const card = mockCard() + vi.stubGlobal('fetch', personFetch(card)) + + renderProfile() + await screen.findByTestId('view-correspondence') + fireEvent.click(screen.getByTestId('view-correspondence')) + expect(await screen.findByTestId('research')).toBeInTheDocument() + const state = useWorkingSetStore.getState() + expect(state.scope.senders).toEqual( + expect.arrayContaining(['alice@example.com', 'me@owner.com']), + ) + expect(state.mode).toBe('exact') + expect(state.query).toBe('') + expect(state.grouping).toBe('thread') + }) + + it('topic link navigates with tsel', async () => { + const card = mockCard() + vi.stubGlobal('fetch', personFetch(card)) + + renderProfile() + fireEvent.click(await screen.findByTestId(`topic-link-${TOPIC_ID}`)) + expect(await screen.findByTestId('topics')).toBeInTheDocument() + }) +}) diff --git a/apps/chronicle/web/src/people/PersonProfilePage.tsx b/apps/chronicle/web/src/people/PersonProfilePage.tsx new file mode 100644 index 0000000..6c788ee --- /dev/null +++ b/apps/chronicle/web/src/people/PersonProfilePage.tsx @@ -0,0 +1,568 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { Link, useNavigate, useParams } from 'react-router' + +import type { ContactCard, ContactKind, ContactSummary } from '../api/types' +import { encodeState, toIsoSeconds } from '../workingset/urlState' +import { useWorkingSetStore } from '../workingset/store' +import { + getPerson, + listPeople, + mergePeople, + patchPerson, + unmergePeople, +} from './api' +import { ActivityBars } from './ActivityBars' +import { EgoGraph } from './EgoGraph' + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[12px] text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +const KINDS: ContactKind[] = [ + 'human', + 'organization', + 'automated', + 'mailing_list', + 'unknown', +] + +/** + * Person / organization profile (spec Table 23 sections). + * Org domain controls covered by tags/notes MVP. + */ +export function PersonProfilePage() { + const { id = '' } = useParams<{ id: string }>() + const navigate = useNavigate() + const qc = useQueryClient() + const setScope = useWorkingSetStore((s) => s.setScope) + const setMode = useWorkingSetStore((s) => s.setMode) + const setQuery = useWorkingSetStore((s) => s.setQuery) + const setGrouping = useWorkingSetStore((s) => s.setGrouping) + + const [nameDraft, setNameDraft] = useState('') + const [notesDraft, setNotesDraft] = useState('') + const [tagDraft, setTagDraft] = useState('') + const [mergeSearch, setMergeSearch] = useState('') + const [mergeTarget, setMergeTarget] = useState(null) + + const query = useQuery({ + queryKey: ['people', id], + queryFn: ({ signal }) => getPerson(id, signal), + enabled: !!id, + retry: false, + }) + + const card = query.data + + useEffect(() => { + if (card) { + setNameDraft(card.display_name ?? '') + setNotesDraft(card.notes ?? '') + } + }, [card]) + + const patchMut = useMutation({ + mutationFn: (body: { + kind?: ContactKind + tags?: string[] + notes?: string | null + display_name?: string | null + }) => patchPerson(id, body), + onSuccess: (data) => { + void qc.setQueryData(['people', id], data) + void qc.invalidateQueries({ queryKey: ['people', 'list'] }) + }, + }) + + const mergeMut = useMutation({ + mutationFn: (body: { source_id: string; target_id: string }) => + mergePeople(body), + onSuccess: (data) => { + void qc.invalidateQueries({ queryKey: ['people'] }) + void navigate(`/people/${data.id}`) + }, + }) + + const unmergeMut = useMutation({ + mutationFn: (mergeId: string) => unmergePeople({ merge_id: mergeId }), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ['people'] }) + void query.refetch() + }, + }) + + const mergeSearchQuery = useQuery({ + queryKey: ['people', 'merge-search', mergeSearch], + queryFn: ({ signal }) => + listPeople({ q: mergeSearch, limit: 8 }, signal), + enabled: mergeSearch.trim().length >= 2, + retry: false, + }) + + const signalRows = useMemo(() => { + const signals = card?.classification_signals + if (!signals || typeof signals !== 'object') return [] + return Object.entries(signals).map(([signal, weight]) => ({ + signal, + weight: typeof weight === 'number' ? weight : Number(weight), + })) + }, [card?.classification_signals]) + + const primaryAddress = useMemo(() => { + if (!card) return null + const details = card.address_details ?? [] + if (details.length > 0) { + const sorted = [...details].sort( + (a, b) => + b.messages_from + b.messages_to - (a.messages_from + a.messages_to), + ) + return sorted[0]?.address ?? card.addresses[0] ?? null + } + return card.addresses[0] ?? null + }, [card]) + + const openInChronicle = useCallback( + (c: ContactCard) => { + const addr = primaryAddress + if (!addr) return + let fromMs: number + let toMs: number + if (c.first_seen && c.last_seen) { + fromMs = Date.parse(c.first_seen) + toMs = Date.parse(c.last_seen) + if (!Number.isFinite(fromMs) || !Number.isFinite(toMs) || toMs <= fromMs) { + toMs = fromMs + 24 * 3600 * 1000 + } + } else if (c.activity?.length) { + const times = c.activity + .map((a) => Date.parse(a.bucket)) + .filter((t) => Number.isFinite(t)) + fromMs = Math.min(...times) + toMs = Math.max(...times) + 30 * 24 * 3600 * 1000 + } else { + const now = Date.now() + fromMs = now - 365 * 24 * 3600 * 1000 + toMs = now + } + const params = encodeState({ + scope: { senders: [addr] }, + viewport: { fromMs, toMs }, + aggregation: 'auto', + view: 'canvas', + selection: null, + lanes: ['messages', 'top_people'], + }) + if (!params.get('vf')) params.set('vf', toIsoSeconds(fromMs)) + if (!params.get('vt')) params.set('vt', toIsoSeconds(toMs)) + const qs = params.toString() + void navigate({ pathname: '/', search: qs ? `?${qs}` : '' }) + }, + [navigate, primaryAddress], + ) + + const openInResearch = useCallback( + (c: ContactCard) => { + const addrs = c.addresses ?? [] + if (!addrs.length) return + setScope({ senders: [...addrs] }) + setQuery('') + setMode('exact') + setGrouping('thread') + void navigate('/research') + }, + [navigate, setGrouping, setMode, setQuery, setScope], + ) + + const openTopic = useCallback( + (topicId: string) => { + void navigate(`/topics?tsel=${encodeURIComponent(topicId)}`) + }, + [navigate], + ) + + const saveName = useCallback(() => { + if (!card) return + const next = nameDraft.trim() + if (next === (card.display_name ?? '')) return + patchMut.mutate({ display_name: next || null }) + }, [card, nameDraft, patchMut]) + + const saveNotes = useCallback(() => { + if (!card) return + if (notesDraft === (card.notes ?? '')) return + patchMut.mutate({ notes: notesDraft }) + }, [card, notesDraft, patchMut]) + + const addTag = useCallback(() => { + if (!card) return + const t = tagDraft.trim() + if (!t) return + const tags = card.tags ?? [] + if (tags.includes(t)) { + setTagDraft('') + return + } + patchMut.mutate({ tags: [...tags, t] }) + setTagDraft('') + }, [card, tagDraft, patchMut]) + + const removeTag = useCallback( + (tag: string) => { + if (!card) return + patchMut.mutate({ tags: (card.tags ?? []).filter((t) => t !== tag) }) + }, + [card, patchMut], + ) + + const confirmMergeInto = useCallback(() => { + if (!mergeTarget || !id) return + const sourceName = + card?.display_name || card?.addresses[0] || id + const targetName = + mergeTarget.display_name || mergeTarget.addresses[0] || mergeTarget.id + const ok = window.confirm( + `Merge ${sourceName} into ${targetName}? Addresses move to the target.`, + ) + if (!ok) return + mergeMut.mutate({ source_id: id, target_id: mergeTarget.id }) + }, [mergeTarget, id, card, mergeMut]) + + if (query.isLoading) { + return ( +
+ Loading profile… +
+ ) + } + + if (query.isError || !card) { + return ( +
+ Failed to load contact + +
+ ) + } + + const kindLocked = card.kind_source === 'manual' + + return ( +
+
+ + ← People + + {card.kind === 'organization' ? ( + Organization profile + ) : null} +
+ + {/* Identity */} +
+

Identity

+
+ setNameDraft(e.target.value)} + onBlur={saveName} + onKeyDown={(e) => { + if (e.key === 'Enter') saveName() + }} + className="min-w-[12rem] rounded-md border border-steel bg-graphite-900 px-2 py-1 text-sm text-text-primary" + data-testid="person-display-name" + aria-label="Display name" + /> + + + kind_source: {card.kind_source ?? '—'} + {kindLocked ? ' (manual — locked against machinery)' : ''} + +
+ +
+ {(card.tags ?? []).map((tag) => ( + + {tag} + + + ))} + setTagDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') addTag() + }} + placeholder="Add tag" + className="w-24 rounded border border-steel bg-graphite-900 px-1 text-[11px]" + data-testid="person-tag-input" + /> + +
+ +