From 53901f41ebc639bcceae054fd2cee275170ce92b Mon Sep 17 00:00:00 2001 From: Claude Code Agent Date: Mon, 13 Jul 2026 21:52:58 -0700 Subject: [PATCH 1/5] =?UTF-8?q?feat(chronicle):=20topics=20subsystem=20?= =?UTF-8?q?=E2=80=94=20clustering,=20set-based=20assignment,=20curation=20?= =?UTF-8?q?precedence=20(Phase=204=20task=204.1,=20#105)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit app_topics (pgvector centroids) + app_topic_members; deterministic seeded k-means++ over an md5-stable embedding sample; TF-term labels with whitelisted gateway polish; full-corpus membership in one CROSS JOIN LATERAL statement; regeneration replaces only untouched automatic topics — curated/manual topics and manual memberships provably survive; CRUD flips automatic→curated on any edit; Chronicle topics lane (top-6 ranked CTE, topic purple). Implemented via cheap-coder (grok-4.5 @ max). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XP7M17NHxw6mjpTFtMqwZu --- .../server/src/chronicle_server/app.py | 3 + .../server/src/chronicle_server/chronicle.py | 129 +- .../server/src/chronicle_server/db.py | 23 + .../server/src/chronicle_server/health.py | 31 + .../server/src/chronicle_server/topics.py | 1241 +++++++++++++++++ apps/chronicle/server/tests/test_chronicle.py | 202 +++ apps/chronicle/server/tests/test_health.py | 7 + apps/chronicle/server/tests/test_topics.py | 502 +++++++ apps/chronicle/web/src/api/types.ts | 26 +- .../web/src/chronicle/TimelineCanvas.tsx | 53 +- .../web/src/chronicle/TimelineTable.test.tsx | 39 + .../web/src/chronicle/TimelineTable.tsx | 78 +- .../web/src/chronicle/laneModel.test.ts | 48 + apps/chronicle/web/src/chronicle/laneModel.ts | 117 +- .../plans/2026-07-13-life-chronicle-plan.md | 16 +- 15 files changed, 2432 insertions(+), 83 deletions(-) create mode 100644 apps/chronicle/server/src/chronicle_server/topics.py create mode 100644 apps/chronicle/server/tests/test_topics.py diff --git a/apps/chronicle/server/src/chronicle_server/app.py b/apps/chronicle/server/src/chronicle_server/app.py index a468c7a..d6788c9 100644 --- a/apps/chronicle/server/src/chronicle_server/app.py +++ b/apps/chronicle/server/src/chronicle_server/app.py @@ -21,6 +21,7 @@ from chronicle_server.interpret import router as interpret_router from chronicle_server.search import router as search_router from chronicle_server.sources import router as sources_router +from chronicle_server.topics import router as topics_router from chronicle_server.workspaces import router as workspaces_router if TYPE_CHECKING: @@ -79,6 +80,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # generate before events so /events/generate is not captured by /events/{id} app.include_router(generate_router, prefix="/api") app.include_router(events_router, prefix="/api") + # topics before any catch-all; generate path is under the same router + app.include_router(topics_router, prefix="/api") # Stash settings early so tests can inspect before lifespan if needed. app.state.settings = resolved return app diff --git a/apps/chronicle/server/src/chronicle_server/chronicle.py b/apps/chronicle/server/src/chronicle_server/chronicle.py index 95ee4be..df8bd91 100644 --- a/apps/chronicle/server/src/chronicle_server/chronicle.py +++ b/apps/chronicle/server/src/chronicle_server/chronicle.py @@ -25,12 +25,13 @@ router = APIRouter(tags=["chronicle"]) VALID_UNITS: tuple[str, ...] = ("hour", "day", "week", "month", "quarter", "year") -VALID_LANES = frozenset({"messages", "attachments", "people", "top_people", "events"}) +VALID_LANES = frozenset({"messages", "attachments", "people", "top_people", "events", "topics"}) MAX_BUCKETS = 2000 MIN_PIXEL_WIDTH = 320 MAX_PIXEL_WIDTH = 8192 DENSITY_PIXEL_WIDTH = 1600 TOP_PEOPLE_LIMIT = 8 +TOP_TOPICS_LIMIT = 6 EVENTS_LANE_CAP = 500 # Approximate unit widths for bucket-count estimation (pixel-width rule). @@ -92,6 +93,21 @@ class TopPeopleLaneData(BaseModel): contacts: list[TopPeopleContactSeries] +class TopicSeries(BaseModel): + """Activity bucket series for one topic (member emails per bucket).""" + + topic_id: str + label: str + origin: str + buckets: list[BucketPoint] + + +class TopicsLaneData(BaseModel): + """topics lane: top visible topics by member volume within viewport+scope.""" + + topics: list[TopicSeries] + + class EventLaneMark(BaseModel): """Sparse event diamond mark for the events lane (not a bucket count).""" @@ -142,7 +158,7 @@ def _check_lanes(cls, value: list[str]) -> list[str]: return value -LanePayload = list[BucketPoint] | TopPeopleLaneData | EventsLaneData +LanePayload = list[BucketPoint] | TopPeopleLaneData | EventsLaneData | TopicsLaneData class BucketsResponse(BaseModel): @@ -526,6 +542,97 @@ def _top_people_buckets( return TopPeopleLaneData(contacts=[by_contact[cid] for cid in order]) +def _topics_buckets( + pool: ConnectionPool, + *, + unit: str, + viewport_from: str, + viewport_to: str, + scope_conds: list[str], + scope_params: dict[str, Any], +) -> TopicsLaneData: + """Top 6 visible topics by member volume within viewport+scope. + + One set-based statement: members ⨝ emails ⨝ topics, ranked CTE like + top_people. Hidden topics excluded. Bare emails columns are unambiguous + (app_topic_members / app_topics do not share those names). + """ + email_where = [ + "date IS NOT NULL", + "date >= %(viewport_from)s", + "date < %(viewport_to)s", + *scope_conds, + ] + sql = f""" + WITH filtered AS ( + SELECT m.topic_id, e.date + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE t.hidden = FALSE + AND {" AND ".join(email_where)} + ), + ranked AS ( + SELECT topic_id, count(*)::int AS total + FROM filtered + GROUP BY topic_id + ORDER BY total DESC, topic_id + LIMIT %(top_limit)s + ), + bucketed AS ( + SELECT + f.topic_id, + date_trunc(%(unit)s, f.date) AS bucket, + count(*)::int AS count + FROM filtered f + JOIN ranked r ON r.topic_id = f.topic_id + GROUP BY f.topic_id, 2 + ) + SELECT + r.topic_id::text, + t.label, + t.origin, + b.bucket, + coalesce(b.count, 0)::int AS count, + r.total + FROM ranked r + JOIN app_topics t ON t.id = r.topic_id + LEFT JOIN bucketed b ON b.topic_id = r.topic_id + ORDER BY r.total DESC, r.topic_id, b.bucket + """ + params = { + **scope_params, + "unit": unit, + "viewport_from": viewport_from, + "viewport_to": viewport_to, + "top_limit": TOP_TOPICS_LIMIT, + } + with pool.connection() as conn: + rows = conn.execute(sql, params).fetchall() + + by_topic: dict[str, TopicSeries] = {} + order: list[str] = [] + for row in rows: + topic_id = str(row[0]) + label = str(row[1]) if row[1] is not None else topic_id + origin = str(row[2]) + if topic_id not in by_topic: + by_topic[topic_id] = TopicSeries( + topic_id=topic_id, + label=label, + origin=origin, + buckets=[], + ) + order.append(topic_id) + bucket = _iso_utc(row[3]) + if bucket is None: + continue + by_topic[topic_id].buckets.append( + BucketPoint(bucket=bucket, count=int(row[4])), + ) + return TopicsLaneData(topics=[by_topic[tid] for tid in order]) + + def _extent( pool: ConnectionPool, scope_conds: list[str], @@ -673,6 +780,15 @@ def get_buckets(pool: ConnectionPool, body: BucketsRequest) -> BucketsResponse: scope_conds=scope_conds, scope_params=scope_params, ) + elif lane == "topics": + lane_data[lane] = _topics_buckets( + pool, + unit=unit, + viewport_from=viewport_from_s, + viewport_to=viewport_to_s, + scope_conds=scope_conds, + scope_params=scope_params, + ) elif lane == "events": lane_data[lane] = _events_lane( pool, @@ -783,6 +899,15 @@ def _fetch_lanes_for_viewport( scope_conds=scope_conds, scope_params=scope_params, ) + elif lane == "topics": + lane_data[lane] = _topics_buckets( + pool, + unit=unit, + viewport_from=viewport_from_s, + viewport_to=viewport_to_s, + scope_conds=scope_conds, + scope_params=scope_params, + ) elif lane == "events": lane_data[lane] = _events_lane( pool, diff --git a/apps/chronicle/server/src/chronicle_server/db.py b/apps/chronicle/server/src/chronicle_server/db.py index 1e7c847..b33ea8b 100644 --- a/apps/chronicle/server/src/chronicle_server/db.py +++ b/apps/chronicle/server/src/chronicle_server/db.py @@ -113,6 +113,29 @@ 'direct','supported','conflicting','unresolved')), citations JSONB NOT NULL DEFAULT '[]' ); +CREATE TABLE IF NOT EXISTS app_topics ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + label TEXT NOT NULL, + description TEXT, + origin TEXT NOT NULL DEFAULT 'automatic' + CHECK (origin IN ('automatic','curated','manual')), + parent_id UUID REFERENCES app_topics(id), + hidden BOOLEAN NOT NULL DEFAULT FALSE, + centroid vector(768), + top_terms TEXT[] NOT NULL DEFAULT '{}', + generation INT NOT NULL DEFAULT 1, + member_count INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS app_topic_members ( + topic_id UUID NOT NULL REFERENCES app_topics(id) ON DELETE CASCADE, + email_id UUID NOT NULL, + distance REAL, + origin TEXT NOT NULL DEFAULT 'automatic' CHECK (origin IN ('automatic','manual')), + PRIMARY KEY (topic_id, email_id) +); +CREATE INDEX IF NOT EXISTS app_topic_members_email_idx ON app_topic_members (email_id); """ diff --git a/apps/chronicle/server/src/chronicle_server/health.py b/apps/chronicle/server/src/chronicle_server/health.py index 834bd69..044dd27 100644 --- a/apps/chronicle/server/src/chronicle_server/health.py +++ b/apps/chronicle/server/src/chronicle_server/health.py @@ -112,9 +112,35 @@ def get_archive_health(pool: ConnectionPool) -> dict[str, Any]: }, ).fetchall() + # Topics coverage (tables may be empty pre-generation). + try: + topics_count_row = conn.execute("SELECT count(*)::int FROM app_topics").fetchone() + assigned_row = conn.execute( + "SELECT count(DISTINCT email_id)::int FROM app_topic_members" + ).fetchone() + last_gen_row = conn.execute( + """ + SELECT at FROM app_audit + WHERE action = 'topics_generate' + ORDER BY at DESC + LIMIT 1 + """ + ).fetchone() + topics_n = int(topics_count_row[0]) if topics_count_row else 0 + assigned_n = int(assigned_row[0]) if assigned_row else 0 + last_generated = _iso(last_gen_row[0]) if last_gen_row else None + except Exception: + # Tables not yet present (pre-migration) — report zeros. + topics_n = 0 + assigned_n = 0 + last_generated = None + assert threading_row is not None assert chunk_emb_row is not None + embedded_n = int(summary["embedding"]["embedded"]) + coverage_ratio = (assigned_n / embedded_n) if embedded_n > 0 else 0.0 + db = MailDB._from_pool(pool) import_records = db.import_history(limit=20) imports = [ @@ -173,6 +199,11 @@ def get_archive_health(pool: ConnectionPool) -> dict[str, Any]: "missing": chunk_emb_row[1], }, }, + "topics": { + "topics": topics_n, + "coverage": coverage_ratio, + "last_generated": last_generated, + }, "imports": imports, "audit_tail": audit_tail, "generated_at": datetime.now(UTC).isoformat(), diff --git a/apps/chronicle/server/src/chronicle_server/topics.py b/apps/chronicle/server/src/chronicle_server/topics.py new file mode 100644 index 0000000..1072692 --- /dev/null +++ b/apps/chronicle/server/src/chronicle_server/topics.py @@ -0,0 +1,1241 @@ +"""Topics subsystem: clustering, assignment, curation, and generation. + +Deterministic k-means over an embedding sample produces centroid-backed +``app_topics``; one set-based full-corpus membership assignment; term-derived +labels with optional gateway polish; curation CRUD with manual precedence +(TA-005). + +``POST /api/topics/generate`` is synchronous and may take minutes on a full +corpus (the set-based assignment scans every embedded email). Data Health +records the run via the ``topics_generate`` audit action. +""" + +from __future__ import annotations + +import json +import re +import time +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, Literal +from uuid import UUID + +import numpy as np +import structlog +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from pydantic import BaseModel, Field, field_validator + +from chronicle_server.auth import require_user +from chronicle_server.db import audit +from chronicle_server.gateway import ModelGateway +from chronicle_server.ids import decode_source_id, encode_source_id, msg_key_to_uuid + +if TYPE_CHECKING: + from psycopg_pool import ConnectionPool + + from chronicle_server.config import ChronicleSettings + +logger = structlog.get_logger() + +router = APIRouter(tags=["topics"]) + +TopicOrigin = Literal["automatic", "curated", "manual"] +MemberOrigin = Literal["automatic", "manual"] + +_MAX_KMEANS_ITERS = 25 +_CENTROID_SHIFT_EPS = 1e-4 +_EMBED_DIM = 768 +_TOP_TERMS = 5 +_LABEL_TERMS = 3 +_MAX_LABEL_WORDS = 4 +_MIN_TERM_LEN = 3 + +# Common English stopwords for subject TF labeling (lowercase). +STOPWORDS: frozenset[str] = frozenset( + { + "a", + "about", + "above", + "after", + "again", + "against", + "all", + "am", + "an", + "and", + "any", + "are", + "as", + "at", + "be", + "because", + "been", + "before", + "being", + "below", + "between", + "both", + "but", + "by", + "can", + "could", + "did", + "do", + "does", + "doing", + "down", + "during", + "each", + "few", + "for", + "from", + "further", + "had", + "has", + "have", + "having", + "he", + "her", + "here", + "hers", + "herself", + "him", + "himself", + "his", + "how", + "i", + "if", + "in", + "into", + "is", + "it", + "its", + "itself", + "just", + "me", + "more", + "most", + "my", + "myself", + "no", + "nor", + "not", + "now", + "of", + "off", + "on", + "once", + "only", + "or", + "other", + "our", + "ours", + "ourselves", + "out", + "over", + "own", + "re", + "same", + "she", + "should", + "so", + "some", + "such", + "than", + "that", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "there", + "these", + "they", + "this", + "those", + "through", + "to", + "too", + "under", + "until", + "up", + "very", + "was", + "we", + "were", + "what", + "when", + "where", + "which", + "while", + "who", + "whom", + "why", + "will", + "with", + "would", + "you", + "your", + "yours", + "yourself", + "yourselves", + "fw", + "fwd", + "subject", + } +) + +_TERM_RE = re.compile(r"[a-z0-9]+") + +LABEL_POLISH_POLICY = ( + "Given topic term lists, output ONLY a STRICT JSON array of short labels " + "(one per topic, same order). Each label is at most 4 words. " + "No other keys, no prose." +) + + +# --- request / response models --- + + +class GenerateRequest(BaseModel): + k: int = Field(default=24, ge=4, le=48) + sample: int = Field(default=20000, ge=1) + seed: int = 13 + + +class GenerateResponse(BaseModel): + topics: int + assigned: int + skipped_curated: int + took_ms: int + + +class TopicCreate(BaseModel): + label: str = Field(min_length=1) + description: str | None = None + + +class TopicPatch(BaseModel): + label: str | None = Field(default=None, min_length=1) + description: str | None = None + hidden: bool | None = None + parent_id: str | None = None + + @field_validator("parent_id") + @classmethod + def _empty_parent_ok(cls, value: str | None) -> str | None: + if value is not None and value.strip() == "": + return None + return value + + +class MemberAddRequest(BaseModel): + email_sid: str = Field(min_length=1) + + +# --- pure helpers (testable without DB) --- + + +def parse_embedding(raw: Any) -> list[float] | None: + """Parse pgvector embedding (string or list) into floats.""" + if raw is None: + return None + if isinstance(raw, list | tuple | np.ndarray): + return [float(x) for x in raw] + if isinstance(raw, str): + return [float(x) for x in raw.strip("[]").split(",") if x.strip()] + try: + return [float(x) for x in raw] + except (TypeError, ValueError): + return None + + +def l2_normalize(vectors: np.ndarray) -> np.ndarray: + """Row-wise L2 normalize; zero rows stay zero.""" + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + norms = np.maximum(norms, 1e-12) + return np.asarray(vectors / norms, dtype=np.float64) + + +def kmeans_plus_plus_init( + vectors: np.ndarray, + k: int, + rng: np.random.Generator, +) -> np.ndarray: + """k-means++ centroid initialization (seeded).""" + n = vectors.shape[0] + centroids = np.empty((k, vectors.shape[1]), dtype=np.float64) + first = int(rng.integers(0, n)) + centroids[0] = vectors[first] + # Squared Euclidean distances to nearest chosen centroid. + closest = np.full(n, np.inf, dtype=np.float64) + for i in range(1, k): + diff = vectors - centroids[i - 1] + dist_sq = np.sum(diff * diff, axis=1) + closest = np.minimum(closest, dist_sq) + total = float(closest.sum()) + if total <= 0.0 or not np.isfinite(total): + centroids[i] = vectors[int(rng.integers(0, n))] + continue + probs = closest / total + idx = int(rng.choice(n, p=probs)) + centroids[i] = vectors[idx] + return centroids + + +def run_kmeans( + vectors: np.ndarray, + k: int, + *, + seed: int = 13, + max_iters: int = _MAX_KMEANS_ITERS, + eps: float = _CENTROID_SHIFT_EPS, +) -> tuple[np.ndarray, np.ndarray]: + """Deterministic L2-normalized k-means (cosine-equivalent). + + Returns ``(centroids, labels)`` where centroids are L2-normalized and + labels is an int array of length n (cluster index per sample). + + Raises ValueError when ``k > n`` or ``n == 0`` or ``k < 1``. + """ + if k < 1: + raise ValueError("k must be >= 1") + n = vectors.shape[0] + if n == 0: + raise ValueError("empty sample") + if k > n: + raise ValueError(f"k ({k}) exceeds sample size ({n})") + + data = l2_normalize(np.asarray(vectors, dtype=np.float64)) + rng = np.random.default_rng(seed) + centroids = l2_normalize(kmeans_plus_plus_init(data, k, rng)) + + labels = np.zeros(n, dtype=np.int32) + for _ in range(max_iters): + # Assign: nearest centroid by squared Euclidean (≡ cosine on unit sphere). + # (n, k) distances via ||x||^2 + ||c||^2 - 2 x·c; norms ≈ 1. + dots = data @ centroids.T # (n, k) + dist_sq = 2.0 - 2.0 * dots + labels = np.argmin(dist_sq, axis=1).astype(np.int32) + + new_centroids = np.zeros_like(centroids) + for j in range(k): + mask = labels == j + if not np.any(mask): + # Re-seed empty cluster from a random point. + new_centroids[j] = data[int(rng.integers(0, n))] + else: + new_centroids[j] = data[mask].mean(axis=0) + new_centroids = l2_normalize(new_centroids) + + shift = float(np.linalg.norm(new_centroids - centroids)) + centroids = new_centroids + if shift < eps: + break + + return centroids, labels + + +def extract_top_terms( + subjects: list[str | None], + *, + top_n: int = _TOP_TERMS, + stopwords: frozenset[str] = STOPWORDS, +) -> list[str]: + """Top TF terms from subjects: lowercase, alnum ≥ 3, stopword-filtered.""" + counts: dict[str, int] = {} + for subject in subjects: + if not subject: + continue + for token in _TERM_RE.findall(subject.lower()): + if len(token) < _MIN_TERM_LEN or token in stopwords: + continue + counts[token] = counts.get(token, 0) + 1 + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + return [term for term, _ in ranked[:top_n]] + + +def label_from_terms(terms: list[str], n: int = _LABEL_TERMS) -> str: + """Comma-join of the first *n* terms, or a placeholder when empty.""" + chosen = terms[:n] + if not chosen: + return "untitled topic" + return ", ".join(chosen) + + +def _largest_json_array(text: str) -> str | None: + """Return the largest balanced ``[...]`` substring, or None.""" + best: str | None = None + depth = 0 + start: int | None = None + for i, ch in enumerate(text): + if ch == "[": + if depth == 0: + start = i + depth += 1 + elif ch == "]": + if depth > 0: + depth -= 1 + if depth == 0 and start is not None: + candidate = text[start : i + 1] + if best is None or len(candidate) > len(best): + best = candidate + return best + + +def validate_label_array(raw: Any, expected: int) -> list[str] | None: + """Whitelist-validate a JSON array of ≤4-word label strings. + + Returns None on any structural failure (wrong type, wrong length, empty). + """ + if not isinstance(raw, list) or len(raw) != expected: + return None + out: list[str] = [] + for item in raw: + if not isinstance(item, str): + return None + label = " ".join(item.split()) + if not label: + return None + words = label.split() + if len(words) > _MAX_LABEL_WORDS: + return None + out.append(label) + return out + + +def parse_label_polish_response(content: str, expected: int) -> list[str] | None: + """Extract and validate gateway label array. None on failure.""" + if not content or not content.strip(): + return None + block = _largest_json_array(content) + if block is None: + return None + try: + raw = json.loads(block) + except (json.JSONDecodeError, TypeError, ValueError): + return None + return validate_label_array(raw, expected) + + +def vector_literal(vec: np.ndarray | list[float]) -> str: + """Format a float vector as a pgvector text literal ``[a,b,...]``.""" + arr = np.asarray(vec, dtype=np.float64).ravel() + return "[" + ",".join(f"{float(x):.8g}" for x in arr) + "]" + + +def _iso(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + if hasattr(value, "isoformat"): + return str(value.isoformat()) + return str(value) + + +def _gateway_from_request(request: Request, settings: ChronicleSettings) -> ModelGateway: + transport = getattr(request.app.state, "chat_transport", None) + return ModelGateway(settings, transport) + + +def _model_available(request: Request, gateway: ModelGateway) -> bool: + forced = getattr(request.app.state, "model_available", None) + if forced is not None: + return bool(forced) + return gateway.availability() + + +def _complete_chat(gateway: ModelGateway, messages: list[dict[str, str]]) -> str: + """One non-streaming completion: collect all transport deltas.""" + settings = gateway._settings # noqa: SLF001 + transport = gateway._transport # noqa: SLF001 + parts: list[str] = [] + for delta in transport(settings.answer_model, messages, False): + if delta: + parts.append(str(delta)) + return "".join(parts) + + +def _topic_row_dict(row: Any) -> dict[str, Any]: + return { + "id": str(row[0]), + "label": str(row[1]), + "description": row[2], + "origin": str(row[3]), + "parent_id": str(row[4]) if row[4] is not None else None, + "hidden": bool(row[5]), + "top_terms": list(row[6]) if row[6] is not None else [], + "generation": int(row[7]), + "member_count": int(row[8]), + "created_at": _iso(row[9]), + "updated_at": _iso(row[10]), + } + + +_TOPIC_SELECT = """ + SELECT id, label, description, origin, parent_id, hidden, + top_terms, generation, member_count, created_at, updated_at + FROM app_topics +""" + + +# --- generation --- + + +def generate_topics( + pool: ConnectionPool, + body: GenerateRequest, + *, + username: str, + gateway: ModelGateway | None = None, + model_available: bool = False, +) -> GenerateResponse: + """Sample embeddings, cluster, label, replace-regenerate, full-corpus assign. + + Runtime note: the full-corpus assignment is a single set-based scan of every + email with an embedding and may take minutes on large archives. The endpoint + is synchronous; Data Health surfaces the run via audit ``topics_generate``. + """ + started = time.perf_counter() + k = body.k + sample_n = body.sample + seed = body.seed + + with pool.connection() as conn: + # Stable pseudo-random sample via md5(id::text) order (deterministic). + sample_rows = conn.execute( + """ + SELECT id, embedding, subject + FROM emails + WHERE embedding IS NOT NULL + ORDER BY md5(id::text) + LIMIT %(sample)s + """, + {"sample": sample_n}, + ).fetchall() + + if not sample_rows: + audit( + pool, + username=username, + action="topics_generate", + detail={ + "topics": 0, + "assigned": 0, + "skipped_curated": 0, + "seed": seed, + "k": k, + "sample": sample_n, + "took_ms": int((time.perf_counter() - started) * 1000), + }, + ) + return GenerateResponse(topics=0, assigned=0, skipped_curated=0, took_ms=0) + + ids: list[UUID] = [] + subjects: list[str | None] = [] + vectors_list: list[list[float]] = [] + for row in sample_rows: + emb = parse_embedding(row[1]) + if emb is None or len(emb) != _EMBED_DIM: + continue + ids.append(row[0]) + subjects.append(row[2] if row[2] is not None else None) + vectors_list.append(emb) + + if not vectors_list: + took_ms = int((time.perf_counter() - started) * 1000) + audit( + pool, + username=username, + action="topics_generate", + detail={ + "topics": 0, + "assigned": 0, + "skipped_curated": 0, + "seed": seed, + "k": k, + "sample": sample_n, + "took_ms": took_ms, + }, + ) + return GenerateResponse(topics=0, assigned=0, skipped_curated=0, took_ms=took_ms) + + n = len(vectors_list) + effective_k = min(k, n) + if k > n: + # Guard: cannot form more clusters than samples; use n. + logger.info("topics_k_clamped", requested_k=k, sample_n=n) + effective_k = n + + vectors = np.asarray(vectors_list, dtype=np.float64) + centroids, labels = run_kmeans(vectors, effective_k, seed=seed) + + # Per-cluster subjects from the sample → top terms + default labels. + cluster_subjects: list[list[str | None]] = [[] for _ in range(effective_k)] + for i, lab in enumerate(labels): + cluster_subjects[int(lab)].append(subjects[i]) + + top_terms_list: list[list[str]] = [] + term_labels: list[str] = [] + for j in range(effective_k): + terms = extract_top_terms(cluster_subjects[j]) + top_terms_list.append(terms) + term_labels.append(label_from_terms(terms)) + + labels_final = list(term_labels) + if model_available and gateway is not None and effective_k > 0: + term_payload = [ + {"index": j, "terms": top_terms_list[j], "fallback": term_labels[j]} + for j in range(effective_k) + ] + messages = [ + {"role": "system", "content": LABEL_POLISH_POLICY}, + { + "role": "user", + "content": json.dumps(term_payload, ensure_ascii=False), + }, + ] + try: + content = _complete_chat(gateway, messages) + polished = parse_label_polish_response(content, effective_k) + if polished is not None: + labels_final = polished + except Exception as exc: + logger.info("topics_label_polish_failed", error=str(exc)) + + # Surviving topics: curated/manual origin, hidden automatic, or any + # with manual members — never deleted or relabeled (TA-005). + surviving = conn.execute( + """ + SELECT t.id + FROM app_topics t + WHERE t.origin IN ('curated', 'manual') + OR t.hidden = TRUE + OR EXISTS ( + SELECT 1 FROM app_topic_members m + WHERE m.topic_id = t.id AND m.origin = 'manual' + ) + """ + ).fetchall() + surviving_ids = {row[0] for row in surviving} + skipped_curated = len(surviving_ids) + + # Delete replaceable automatic topics (cascade memberships). + conn.execute( + """ + DELETE FROM app_topics t + WHERE t.origin = 'automatic' + AND t.hidden = FALSE + AND NOT EXISTS ( + SELECT 1 FROM app_topic_members m + WHERE m.topic_id = t.id AND m.origin = 'manual' + ) + """ + ) + + gen_row = conn.execute("SELECT COALESCE(MAX(generation), 0) FROM app_topics").fetchone() + next_gen = int(gen_row[0]) + 1 if gen_row else 1 + + new_topic_ids: list[UUID] = [] + for j in range(effective_k): + centroid_lit = vector_literal(centroids[j]) + inserted = conn.execute( + """ + INSERT INTO app_topics ( + label, origin, centroid, top_terms, generation, member_count + ) VALUES ( + %(label)s, 'automatic', %(centroid)s::vector, + %(top_terms)s, %(generation)s, 0 + ) + RETURNING id + """, + { + "label": labels_final[j], + "centroid": centroid_lit, + "top_terms": top_terms_list[j], + "generation": next_gen, + }, + ).fetchone() + if inserted is None: + raise RuntimeError("topic insert returned no id") + new_topic_ids.append(inserted[0]) + + # Drop previous automatic memberships on automatic topics before reassign. + conn.execute( + """ + DELETE FROM app_topic_members m + USING app_topics t + WHERE m.topic_id = t.id + AND t.origin = 'automatic' + AND m.origin = 'automatic' + """ + ) + + # Full-corpus nearest-centroid assignment (one set-based statement). + assign_result = conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + SELECT t.id, e.id, t.centroid <=> e.embedding, 'automatic' + FROM emails e + CROSS JOIN LATERAL ( + SELECT id, centroid FROM app_topics + WHERE origin = 'automatic' AND centroid IS NOT NULL + ORDER BY centroid <=> e.embedding + LIMIT 1 + ) t + WHERE e.embedding IS NOT NULL + ON CONFLICT (topic_id, email_id) DO NOTHING + """ + ) + assigned = assign_result.rowcount if assign_result.rowcount is not None else 0 + + # One grouped UPDATE for all member_counts. + conn.execute( + """ + UPDATE app_topics t + SET member_count = c.cnt + FROM ( + SELECT t2.id AS id, count(m.email_id)::int AS cnt + FROM app_topics t2 + LEFT JOIN app_topic_members m ON m.topic_id = t2.id + GROUP BY t2.id + ) c + WHERE t.id = c.id + """ + ) + + topics_created = len(new_topic_ids) + conn.commit() + + took_ms = int((time.perf_counter() - started) * 1000) + audit( + pool, + username=username, + action="topics_generate", + detail={ + "topics": topics_created, + "assigned": assigned, + "skipped_curated": skipped_curated, + "seed": seed, + "k": k, + "sample": sample_n, + "took_ms": took_ms, + }, + ) + logger.info( + "topics_generate", + topics=topics_created, + assigned=assigned, + skipped_curated=skipped_curated, + took_ms=took_ms, + seed=seed, + k=k, + ) + return GenerateResponse( + topics=topics_created, + assigned=assigned, + skipped_curated=skipped_curated, + took_ms=took_ms, + ) + + +# --- CRUD helpers --- + + +def list_topics( + pool: ConnectionPool, + *, + include_hidden: bool = False, +) -> list[dict[str, Any]]: + """Return topics as a forest: parents with nested children. + + Visible by default (``hidden=false``); ``include_hidden`` shows all. + """ + with pool.connection() as conn: + if include_hidden: + rows = conn.execute( + _TOPIC_SELECT + " ORDER BY member_count DESC, label ASC, id ASC" + ).fetchall() + else: + rows = conn.execute( + _TOPIC_SELECT + + " WHERE hidden = FALSE ORDER BY member_count DESC, label ASC, id ASC" + ).fetchall() + + topics = [_topic_row_dict(r) for r in rows] + by_id: dict[str, dict[str, Any]] = { + t["id"]: { + "id": t["id"], + "label": t["label"], + "origin": t["origin"], + "member_count": t["member_count"], + "hidden": t["hidden"], + "top_terms": t["top_terms"], + "parent_id": t["parent_id"], + "children": [], + } + for t in topics + } + roots: list[dict[str, Any]] = [] + for t in topics: + node = by_id[t["id"]] + parent_id = t["parent_id"] + if parent_id and parent_id in by_id: + by_id[parent_id]["children"].append(node) + else: + roots.append(node) + + # Drop internal parent_id from API payload (tree structure carries it). + def _strip(node: dict[str, Any]) -> dict[str, Any]: + children = [_strip(c) for c in node["children"]] + return { + "id": node["id"], + "label": node["label"], + "origin": node["origin"], + "member_count": node["member_count"], + "hidden": node["hidden"], + "top_terms": node["top_terms"], + "children": children, + } + + return [_strip(r) for r in roots] + + +def get_topic_detail(pool: ConnectionPool, topic_id: UUID) -> dict[str, Any]: + """Topic detail + monthly activity series + 5 representative members.""" + with pool.connection() as conn: + row = conn.execute( + _TOPIC_SELECT + " WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if row is None: + raise HTTPException(status_code=404, detail="Topic not found") + + topic = _topic_row_dict(row) + + activity_rows = conn.execute( + """ + SELECT date_trunc('month', e.date) AS bucket, count(*)::int AS count + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + WHERE m.topic_id = %(id)s + AND e.date IS NOT NULL + GROUP BY 1 + ORDER BY 1 + """, + {"id": topic_id}, + ).fetchall() + + # Closest members when centroid exists; otherwise newest. + if topic["origin"] != "manual": + member_rows = conn.execute( + """ + SELECT e.id, e.subject, e.sender_name, e.sender_address, + e.date, e.source_account, e.thread_id, m.distance + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE m.topic_id = %(id)s + ORDER BY m.distance ASC NULLS LAST, e.date DESC NULLS LAST + LIMIT 5 + """, + {"id": topic_id}, + ).fetchall() + else: + member_rows = conn.execute( + """ + SELECT e.id, e.subject, e.sender_name, e.sender_address, + e.date, e.source_account, e.thread_id, m.distance + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + WHERE m.topic_id = %(id)s + ORDER BY e.date DESC NULLS LAST + LIMIT 5 + """, + {"id": topic_id}, + ).fetchall() + + activity = [ + {"bucket": _iso(r[0]), "count": int(r[1])} for r in activity_rows if r[0] is not None + ] + members = [ + { + "id": encode_source_id("msg", r[0]), + "subject": r[1], + "sender_name": r[2], + "sender_address": r[3], + "date": _iso(r[4]), + "mailbox": r[5], + "thread_id": r[6], + "distance": float(r[7]) if r[7] is not None else None, + } + for r in member_rows + ] + return { + **{ + "id": topic["id"], + "label": topic["label"], + "description": topic["description"], + "origin": topic["origin"], + "parent_id": topic["parent_id"], + "hidden": topic["hidden"], + "top_terms": topic["top_terms"], + "generation": topic["generation"], + "member_count": topic["member_count"], + "created_at": topic["created_at"], + "updated_at": topic["updated_at"], + }, + "activity": activity, + "members": members, + } + + +def patch_topic( + pool: ConnectionPool, + topic_id: UUID, + body: TopicPatch, + *, + username: str, +) -> dict[str, Any]: + """Rename/describe/hide/parent; automatic → curated on any change (Table 21).""" + fields: list[str] = [] + params: dict[str, Any] = {"id": topic_id} + + if body.label is not None: + fields.append("label = %(label)s") + params["label"] = body.label + if body.description is not None: + fields.append("description = %(description)s") + params["description"] = body.description + if body.hidden is not None: + fields.append("hidden = %(hidden)s") + params["hidden"] = body.hidden + if "parent_id" in body.model_fields_set: + if body.parent_id is None: + fields.append("parent_id = NULL") + else: + try: + parent_uuid = UUID(body.parent_id) + except ValueError as exc: + raise HTTPException(status_code=422, detail="invalid parent_id") from exc + if parent_uuid == topic_id: + raise HTTPException(status_code=422, detail="parent_id cannot be self") + fields.append("parent_id = %(parent_id)s") + params["parent_id"] = parent_uuid + + if not fields: + raise HTTPException(status_code=422, detail="no fields to update") + + with pool.connection() as conn: + existing = conn.execute( + "SELECT origin FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if existing is None: + raise HTTPException(status_code=404, detail="Topic not found") + + if "parent_id" in params: + parent_row = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(parent_id)s", + {"parent_id": params["parent_id"]}, + ).fetchone() + if parent_row is None: + raise HTTPException(status_code=404, detail="Parent topic not found") + + origin = str(existing[0]) + if origin == "automatic": + fields.append("origin = 'curated'") + + fields.append("updated_at = now()") + sql = f"UPDATE app_topics SET {', '.join(fields)} WHERE id = %(id)s RETURNING id" + conn.execute(sql, params) + conn.commit() + + audit( + pool, + username=username, + action="topics_patch", + detail={"topic_id": str(topic_id), "fields": list(body.model_fields_set)}, + ) + return get_topic_detail(pool, topic_id) + + +def create_manual_topic( + pool: ConnectionPool, + body: TopicCreate, + *, + username: str, +) -> dict[str, Any]: + """Create a manual topic (no centroid; members added manually).""" + with pool.connection() as conn: + row = conn.execute( + """ + INSERT INTO app_topics (label, description, origin) + VALUES (%(label)s, %(description)s, 'manual') + RETURNING id + """, + {"label": body.label, "description": body.description}, + ).fetchone() + assert row is not None + topic_id = row[0] + conn.commit() + + audit( + pool, + username=username, + action="topics_create", + detail={"topic_id": str(topic_id), "label": body.label}, + ) + return get_topic_detail(pool, topic_id) + + +def delete_topic( + pool: ConnectionPool, + topic_id: UUID, + *, + username: str, +) -> dict[str, str]: + """Hard-delete only origin='manual'; others → 403 (hide instead).""" + with pool.connection() as conn: + row = conn.execute( + "SELECT origin FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if row is None: + raise HTTPException(status_code=404, detail="Topic not found") + if str(row[0]) != "manual": + raise HTTPException( + status_code=403, + detail="Only manual topics can be deleted; hide instead", + ) + conn.execute("DELETE FROM app_topics WHERE id = %(id)s", {"id": topic_id}) + conn.commit() + + audit( + pool, + username=username, + action="topics_delete", + detail={"topic_id": str(topic_id)}, + ) + return {"status": "deleted"} + + +def add_member( + pool: ConnectionPool, + topic_id: UUID, + email_sid: str, + *, + username: str, +) -> dict[str, Any]: + """Manual include: insert membership with origin='manual'.""" + try: + kind, key = decode_source_id(email_sid) + except ValueError as exc: + raise HTTPException(status_code=404, detail=f"Unknown source_id: {email_sid}") from exc + if kind != "msg" or not isinstance(key, int): + raise HTTPException(status_code=422, detail="email_sid must be a msg_ source id") + email_id = msg_key_to_uuid(key) + + with pool.connection() as conn: + trow = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if trow is None: + raise HTTPException(status_code=404, detail="Topic not found") + erow = conn.execute( + "SELECT 1 FROM emails WHERE id = %(id)s", + {"id": email_id}, + ).fetchone() + if erow is None: + raise HTTPException(status_code=404, detail=f"Unknown source_id: {email_sid}") + + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(topic_id)s, %(email_id)s, NULL, 'manual') + ON CONFLICT (topic_id, email_id) DO UPDATE + SET origin = 'manual', + distance = COALESCE(app_topic_members.distance, EXCLUDED.distance) + """, + {"topic_id": topic_id, "email_id": email_id}, + ) + conn.execute( + """ + UPDATE app_topics + SET member_count = ( + SELECT count(*)::int FROM app_topic_members WHERE topic_id = %(id)s + ), + updated_at = now() + WHERE id = %(id)s + """, + {"id": topic_id}, + ) + conn.commit() + + audit( + pool, + username=username, + action="topics_member_add", + detail={"topic_id": str(topic_id), "email_sid": email_sid}, + ) + return {"status": "ok", "topic_id": str(topic_id), "email_sid": email_sid} + + +def remove_member( + pool: ConnectionPool, + topic_id: UUID, + email_sid: str, + *, + username: str, +) -> dict[str, Any]: + """Exclude: DELETE membership row (manual or automatic).""" + try: + kind, key = decode_source_id(email_sid) + except ValueError as exc: + raise HTTPException(status_code=404, detail=f"Unknown source_id: {email_sid}") from exc + if kind != "msg" or not isinstance(key, int): + raise HTTPException(status_code=422, detail="email_sid must be a msg_ source id") + email_id = msg_key_to_uuid(key) + + with pool.connection() as conn: + trow = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if trow is None: + raise HTTPException(status_code=404, detail="Topic not found") + deleted = conn.execute( + """ + DELETE FROM app_topic_members + WHERE topic_id = %(topic_id)s AND email_id = %(email_id)s + """, + {"topic_id": topic_id, "email_id": email_id}, + ) + if deleted.rowcount == 0: + raise HTTPException(status_code=404, detail="Membership not found") + conn.execute( + """ + UPDATE app_topics + SET member_count = ( + SELECT count(*)::int FROM app_topic_members WHERE topic_id = %(id)s + ), + updated_at = now() + WHERE id = %(id)s + """, + {"id": topic_id}, + ) + conn.commit() + + audit( + pool, + username=username, + action="topics_member_remove", + detail={"topic_id": str(topic_id), "email_sid": email_sid}, + ) + return {"status": "ok", "topic_id": str(topic_id), "email_sid": email_sid} + + +# --- routes --- + + +@router.post("/topics/generate", response_model=GenerateResponse) +def topics_generate( + body: GenerateRequest, + request: Request, + user: str = Depends(require_user), +) -> GenerateResponse: + """Cluster sample + full-corpus assign. Synchronous; may take minutes.""" + pool: ConnectionPool = request.app.state.pool + settings: ChronicleSettings = request.app.state.settings + gateway = _gateway_from_request(request, settings) + available = _model_available(request, gateway) + return generate_topics( + pool, + body, + username=user, + gateway=gateway if available else None, + model_available=available, + ) + + +@router.get("/topics") +def topics_list( + request: Request, + include_hidden: bool = Query(default=False), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic tree (parents + children); visible by default.""" + pool: ConnectionPool = request.app.state.pool + return {"topics": list_topics(pool, include_hidden=include_hidden)} + + +@router.post("/topics") +def topics_create( + body: TopicCreate, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Create a manual topic.""" + pool: ConnectionPool = request.app.state.pool + return create_manual_topic(pool, body, username=user) + + +@router.get("/topics/{topic_id}") +def topics_get( + topic_id: UUID, + request: Request, + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic detail, monthly activity, representative members.""" + pool: ConnectionPool = request.app.state.pool + return get_topic_detail(pool, topic_id) + + +@router.patch("/topics/{topic_id}") +def topics_patch( + topic_id: UUID, + body: TopicPatch, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Rename/describe/hide/parent; flips automatic → curated.""" + pool: ConnectionPool = request.app.state.pool + return patch_topic(pool, topic_id, body, username=user) + + +@router.delete("/topics/{topic_id}") +def topics_delete( + topic_id: UUID, + request: Request, + user: str = Depends(require_user), +) -> dict[str, str]: + """Hard-delete manual topics only.""" + pool: ConnectionPool = request.app.state.pool + return delete_topic(pool, topic_id, username=user) + + +@router.post("/topics/{topic_id}/members") +def topics_member_add( + topic_id: UUID, + body: MemberAddRequest, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Manual membership include.""" + pool: ConnectionPool = request.app.state.pool + return add_member(pool, topic_id, body.email_sid, username=user) + + +@router.delete("/topics/{topic_id}/members/{email_sid}") +def topics_member_remove( + topic_id: UUID, + email_sid: str, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Remove membership (manual exclude).""" + pool: ConnectionPool = request.app.state.pool + return remove_member(pool, topic_id, email_sid, username=user) diff --git a/apps/chronicle/server/tests/test_chronicle.py b/apps/chronicle/server/tests/test_chronicle.py index 863ef67..013e8d3 100644 --- a/apps/chronicle/server/tests/test_chronicle.py +++ b/apps/chronicle/server/tests/test_chronicle.py @@ -510,6 +510,16 @@ def test_buckets_request_accepts_events_lane() -> None: assert "events" in req.lanes +def test_buckets_request_accepts_topics_lane() -> None: + req = BucketsRequest.model_validate( + { + "viewport": {"from": "2020-01-01", "to": "2021-01-01"}, + "lanes": ["messages", "topics"], + } + ) + assert "topics" in req.lanes + + def _cleanup_events_for_lane(pool: ConnectionPool) -> None: with pool.connection() as conn: conn.execute("DELETE FROM app_events") @@ -645,6 +655,198 @@ def test_events_lane_cap_and_dismissed_exclusion( _cleanup_events_for_lane(db_pool) +def _cleanup_topics_lane(pool: ConnectionPool, email_ids: list[Any]) -> None: + with pool.connection() as conn: + conn.execute("DELETE FROM app_topic_members") + conn.execute("DELETE FROM app_topics") + for eid in email_ids: + conn.execute("DELETE FROM emails WHERE id = %(id)s", {"id": eid}) + conn.commit() + + +def test_topics_lane_top6_and_viewport(db_client: TestClient, db_pool: ConnectionPool) -> None: + """Topics lane: top-6 by volume, viewport restriction, origin on each series.""" + email_ids: list[Any] = [] + try: + with db_pool.connection() as conn: + # 8 topics; volumes 1..8 so top-6 drops the two lowest. + topic_ids = [] + for i in range(8): + row = conn.execute( + """ + INSERT INTO app_topics (label, origin, hidden, member_count) + VALUES (%(label)s, 'automatic', FALSE, 0) + RETURNING id + """, + {"label": f"TopicLane {i}"}, + ).fetchone() + assert row is not None + topic_ids.append(row[0]) + + # Hidden topic with high volume must not appear. + hidden_row = conn.execute( + """ + INSERT INTO app_topics (label, origin, hidden, member_count) + VALUES ('Hidden High', 'automatic', TRUE, 0) + RETURNING id + """ + ).fetchone() + assert hidden_row is not None + hidden_id = hidden_row[0] + + for i, tid in enumerate(topic_ids): + vol = i + 1 # topic 7 has most + for n in range(vol): + eid = uuid4() + email_ids.append(eid) + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, labels, source_account, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, 'topics lane', + 'A', 'a@example.com', 'example.com', + '{}'::jsonb, %(date)s, 'b', NULL, + false, %(labels)s, 'topics-lane@example.com', now() + ) + """, + { + "id": eid, + "mid": f"", + "tid": f"thr-{eid}", + "date": datetime(2020, 3, 1 + (n % 20), 12, 0, tzinfo=UTC), + "labels": ["INBOX"], + }, + ) + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(tid)s, %(eid)s, 0.1, 'automatic') + """, + {"tid": tid, "eid": eid}, + ) + + # Hidden: 50 members inside viewport + for _n in range(50): + eid = uuid4() + email_ids.append(eid) + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, labels, source_account, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, 'hidden', + 'A', 'a@example.com', 'example.com', + '{}'::jsonb, %(date)s, 'b', NULL, + false, %(labels)s, 'topics-lane@example.com', now() + ) + """, + { + "id": eid, + "mid": f"", + "tid": f"thr-{eid}", + "date": datetime(2020, 5, 1, 12, 0, tzinfo=UTC), + "labels": ["INBOX"], + }, + ) + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(tid)s, %(eid)s, 0.1, 'automatic') + """, + {"tid": hidden_id, "eid": eid}, + ) + + # Outside-viewport member for topic 7 only — should not count in ranking. + eid_out = uuid4() + email_ids.append(eid_out) + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, labels, source_account, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, 'outside', + 'A', 'a@example.com', 'example.com', + '{}'::jsonb, %(date)s, 'b', NULL, + false, %(labels)s, 'topics-lane@example.com', now() + ) + """, + { + "id": eid_out, + "mid": f"", + "tid": f"thr-{eid_out}", + "date": datetime(2018, 1, 1, 12, 0, tzinfo=UTC), + "labels": ["INBOX"], + }, + ) + conn.execute( + """ + INSERT INTO app_topic_members (topic_id, email_id, distance, origin) + VALUES (%(tid)s, %(eid)s, 0.1, 'automatic') + """, + {"tid": topic_ids[7], "eid": eid_out}, + ) + conn.commit() + + _login(db_client) + payload = { + "scope": {"mailboxes": ["topics-lane@example.com"]}, + "viewport": {"from": "2020-01-01T00:00:00Z", "to": "2021-01-01T00:00:00Z"}, + "pixel_width": 920, + "aggregation": "month", + "lanes": ["topics"], + } + r = db_client.post("/api/chronicle/buckets", json=payload) + assert r.status_code == 200, r.text + lane = r.json()["lanes"]["topics"] + assert isinstance(lane, dict) + topics = lane["topics"] + assert isinstance(topics, list) + assert len(topics) <= 6 + assert len(topics) == 6 + # Hidden excluded + labels = {t["label"] for t in topics} + assert "Hidden High" not in labels + # Lowest volumes dropped (TopicLane 0 and 1) + assert "TopicLane 0" not in labels + assert "TopicLane 1" not in labels + assert "TopicLane 7" in labels + # Ranked by volume desc; origin present + totals = [] + for t in topics: + assert "topic_id" in t + assert "label" in t + assert "origin" in t + assert isinstance(t["buckets"], list) + totals.append(sum(pt["count"] for pt in t["buckets"])) + assert totals == sorted(totals, reverse=True) + + # Outside viewport → empty + r_empty = db_client.post( + "/api/chronicle/buckets", + json={ + **payload, + "viewport": { + "from": "2010-01-01T00:00:00Z", + "to": "2011-01-01T00:00:00Z", + }, + }, + ) + assert r_empty.status_code == 200 + assert r_empty.json()["lanes"]["topics"]["topics"] == [] + finally: + _cleanup_topics_lane(db_pool, email_ids) + + # --- compare mode --- diff --git a/apps/chronicle/server/tests/test_health.py b/apps/chronicle/server/tests/test_health.py index 7afdf76..e561fc1 100644 --- a/apps/chronicle/server/tests/test_health.py +++ b/apps/chronicle/server/tests/test_health.py @@ -56,6 +56,13 @@ def _assert_health_shape(body: dict[str, Any]) -> None: _assert_nonneg_int(emb[bucket]["embedded"]) _assert_nonneg_int(emb[bucket]["missing"]) + assert "topics" in body + topics = body["topics"] + _assert_nonneg_int(topics["topics"]) + assert isinstance(topics["coverage"], (int, float)) + assert topics["coverage"] >= 0 + assert "last_generated" in topics # may be null + assert "imports" in body assert isinstance(body["imports"], list) assert len(body["imports"]) <= 20 diff --git a/apps/chronicle/server/tests/test_topics.py b/apps/chronicle/server/tests/test_topics.py new file mode 100644 index 0000000..3d5940c --- /dev/null +++ b/apps/chronicle/server/tests/test_topics.py @@ -0,0 +1,502 @@ +# tests/test_topics.py +from __future__ import annotations + +from collections.abc import Iterator +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import numpy as np +import pytest + +from chronicle_server.ids import encode_source_id +from chronicle_server.topics import ( + extract_top_terms, + label_from_terms, + parse_label_polish_response, + run_kmeans, + validate_label_array, + vector_literal, +) +from tests.conftest import PASSWORD, USERNAME + +if TYPE_CHECKING: + from fastapi.testclient import TestClient + from psycopg_pool import ConnectionPool + + +def _login(client: TestClient) -> None: + r = client.post("/api/auth/login", json={"username": USERNAME, "password": PASSWORD}) + assert r.status_code == 200 + + +# --- pure k-means / labels --- + + +def test_kmeans_deterministic_same_seed() -> None: + rng = np.random.default_rng(0) + data = rng.normal(size=(40, 8)).astype(np.float64) + # Two well-separated blobs for stability. + data[20:] += 5.0 + c1, l1 = run_kmeans(data, k=2, seed=13) + c2, l2 = run_kmeans(data, k=2, seed=13) + np.testing.assert_allclose(c1, c2) + np.testing.assert_array_equal(l1, l2) + + +def test_kmeans_k_greater_than_sample_raises() -> None: + data = np.eye(3, dtype=np.float64) + with pytest.raises(ValueError, match="exceeds sample size"): + run_kmeans(data, k=5, seed=1) + + +def test_extract_top_terms_stopwords_and_min_len() -> None: + subjects = [ + "The Meeting about Renovation and Plumbing", + "RE: Renovation quote for plumbing", + "fw: meeting notes", + "a to of", + ] + terms = extract_top_terms(subjects, top_n=5) + assert "the" not in terms + assert "about" not in terms + assert "and" not in terms + assert "for" not in terms + # short tokens dropped + assert all(len(t) >= 3 for t in terms) + assert "renovation" in terms + assert "plumbing" in terms + assert label_from_terms(terms).count(",") <= 2 + + +def test_label_polish_whitelist_and_failure_fallback() -> None: + ok = validate_label_array(["House renovation", "Travel plans"], 2) + assert ok == ["House renovation", "Travel plans"] + # too many words + assert validate_label_array(["one two three four five"], 1) is None + # wrong length + assert validate_label_array(["only one"], 2) is None + # non-string + assert validate_label_array([1, 2], 2) is None + + content = 'Here you go: ["Kitchen remodel", "Family travel"]' + parsed = parse_label_polish_response(content, 2) + assert parsed == ["Kitchen remodel", "Family travel"] + assert parse_label_polish_response("not json", 2) is None + assert parse_label_polish_response('["too many words in this label"]', 1) is None + + +# --- DB helpers --- + + +def _unit_vec(i: int, dim: int = 768) -> list[float]: + v = [0.0] * dim + v[i % dim] = 1.0 + return v + + +def _insert_email_with_embedding( + pool: ConnectionPool, + *, + subject: str, + embedding: list[float], + date: datetime | None = None, + source_account: str = "topics-test@example.com", + sender_address: str = "alice@topics-test.example", +) -> dict[str, Any]: + email_id = uuid4() + message_id = f"" + tid = f"thread-topics-{email_id}" + when = date or datetime(2020, 6, 15, 12, 0, tzinfo=UTC) + emb_lit = vector_literal(embedding) + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO emails ( + id, message_id, thread_id, subject, + sender_name, sender_address, sender_domain, + recipients, date, body_text, body_html, + has_attachment, attachments, labels, source_account, + embedding, created_at + ) VALUES ( + %(id)s, %(mid)s, %(tid)s, %(subject)s, + 'Alice', %(saddr)s, 'topics-test.example', + '{"to": [], "cc": [], "bcc": []}'::jsonb, %(date)s, 'body', NULL, + false, NULL, ARRAY['INBOX'], %(acct)s, + %(emb)s::vector, now() + ) + """, + { + "id": email_id, + "mid": message_id, + "tid": tid, + "subject": subject, + "saddr": sender_address, + "date": when, + "acct": source_account, + "emb": emb_lit, + }, + ) + conn.commit() + return { + "email_id": email_id, + "msg_sid": encode_source_id("msg", email_id), + "subject": subject, + "source_account": source_account, + } + + +def _cleanup_topics_fixtures(pool: ConnectionPool, seeds: list[dict[str, Any]]) -> None: + with pool.connection() as conn: + conn.execute("DELETE FROM app_topic_members") + conn.execute("DELETE FROM app_topics") + for seed in seeds: + conn.execute("DELETE FROM emails WHERE id = %(id)s", {"id": seed["email_id"]}) + conn.execute("DELETE FROM app_audit WHERE action LIKE 'topics%%'") + conn.commit() + + +# --- generation / precedence / assignment --- + + +def test_generate_and_assignment_seeded_corpus( + db_client: TestClient, db_pool: ConnectionPool +) -> None: + seeds: list[dict[str, Any]] = [] + try: + # Two clusters along orthogonal axes. + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Kitchen renovation plumbing quote {i}", + embedding=_unit_vec(0), + date=datetime(2020, 3, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Family travel itinerary flights {i}", + embedding=_unit_vec(10), + date=datetime(2020, 4, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + + _login(db_client) + r = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 100, "seed": 13}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["topics"] >= 1 + assert body["assigned"] >= 12 + assert "took_ms" in body + + # Distances populated; member_count correct. + with db_pool.connection() as conn: + rows = conn.execute( + """ + SELECT t.id, t.label, t.origin, t.member_count, t.centroid IS NOT NULL + FROM app_topics t + WHERE t.origin = 'automatic' + """ + ).fetchall() + assert len(rows) >= 1 + total_members = sum(int(r[3]) for r in rows) + assert total_members >= 12 + for r_ in rows: + assert r_[2] == "automatic" + assert r_[4] is True + + dist_rows = conn.execute( + """ + SELECT count(*) FILTER (WHERE distance IS NOT NULL)::int, + count(*)::int + FROM app_topic_members + WHERE origin = 'automatic' + """ + ).fetchone() + assert dist_rows is not None + assert dist_rows[0] == dist_rows[1] + assert dist_rows[1] >= 12 + + # Origin visible on list payload. + listed = db_client.get("/api/topics") + assert listed.status_code == 200 + topics = listed.json()["topics"] + assert isinstance(topics, list) + assert all("origin" in t for t in topics) + + # Deterministic: second generate with same seed yields same centroid count path. + r2 = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 100, "seed": 13}, + ) + assert r2.status_code == 200, r2.text + assert r2.json()["topics"] == body["topics"] + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_regeneration_precedence_matrix(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(8): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Alpha beta gamma document {i}", + embedding=_unit_vec(i % 3), + date=datetime(2021, 1, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + r = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 7}, + ) + assert r.status_code == 200, r.text + + listed = db_client.get("/api/topics").json()["topics"] + assert listed + auto_id = listed[0]["id"] + auto_label = listed[0]["label"] + + # Curate by rename → origin curated. + patch = db_client.patch( + f"/api/topics/{auto_id}", + json={"label": "Curated Keep Me"}, + ) + assert patch.status_code == 200, patch.text + assert patch.json()["origin"] == "curated" + assert patch.json()["label"] == "Curated Keep Me" + + # Manual topic. + created = db_client.post( + "/api/topics", + json={"label": "Manual Archive Topic", "description": "hand built"}, + ) + assert created.status_code == 200, created.text + manual_id = created.json()["id"] + assert created.json()["origin"] == "manual" + + # Manual membership on a surviving automatic topic (create fresh auto via re-gen + # after we add manual member to a new auto). First add manual member to curated. + email_sid = seeds[0]["msg_sid"] + add = db_client.post( + f"/api/topics/{auto_id}/members", + json={"email_sid": email_sid}, + ) + assert add.status_code == 200, add.text + + # Also create a pure-automatic topic path: generate again after insert more? Use + # a second automatic by regenerating and picking an automatic one, then add + # manual member to it. + r_mid = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 7}, + ) + assert r_mid.status_code == 200 + + # After regen: curated and manual survive with labels intact. + with db_pool.connection() as conn: + curated = conn.execute( + "SELECT label, origin FROM app_topics WHERE id = %(id)s", + {"id": auto_id}, + ).fetchone() + assert curated is not None + assert curated[0] == "Curated Keep Me" + assert curated[1] == "curated" + + manual = conn.execute( + "SELECT label, origin FROM app_topics WHERE id = %(id)s", + {"id": manual_id}, + ).fetchone() + assert manual is not None + assert manual[0] == "Manual Archive Topic" + assert manual[1] == "manual" + + # Manual membership row survives reassignment. + mem = conn.execute( + """ + SELECT origin FROM app_topic_members + WHERE topic_id = %(tid)s AND email_id = %(eid)s + """, + {"tid": auto_id, "eid": seeds[0]["email_id"]}, + ).fetchone() + assert mem is not None + assert mem[0] == "manual" + + # Plain automatic topics were replaced (old label gone unless recreated). + # Curated id still present; generation increased for new automatic topics. + with db_pool.connection() as conn: + autos = conn.execute( + "SELECT id, label FROM app_topics WHERE origin = 'automatic'" + ).fetchall() + # New automatic topics exist after regen. + assert len(autos) >= 1 + # Original automatic label may differ after recreation — curated kept. + assert auto_label # just ensure we captured it + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_gateway_polish_fake_transport(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Contract negotiation deadline {i}", + embedding=_unit_vec(2), + ) + ) + _login(db_client) + + def fake_transport( + model: str, messages: list[dict[str, str]], stream: bool + ) -> Iterator[str]: + # Emit one polished label per cluster — over-provision; parse keeps length match. + yield '["Contract talks"]' + + db_client.app.state.chat_transport = fake_transport # type: ignore[attr-defined] + db_client.app.state.model_available = True # type: ignore[attr-defined] + + # k=4 but sample may produce fewer effective clusters; force k=4 with enough + # diversity if possible. With identical embeddings, kmeans still creates k + # centroids (empty clusters reseeded) — labels_final length = effective_k. + # For polish to apply, response length must match effective_k. + # Use k=4 and a response with 4 labels. + def fake_transport4( + model: str, messages: list[dict[str, str]], stream: bool + ) -> Iterator[str]: + yield '["Alpha one", "Beta two", "Gamma three", "Delta four"]' + + db_client.app.state.chat_transport = fake_transport4 # type: ignore[attr-defined] + + r = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 3}, + ) + assert r.status_code == 200, r.text + # At least one topic created; polish may or may not stick depending on k vs n. + assert r.json()["topics"] >= 1 + + # Failure fallback: bad transport keeps term labels (no crash). + def bad_transport( + model: str, messages: list[dict[str, str]], stream: bool + ) -> Iterator[str]: + yield "NOT VALID JSON AT ALL" + + db_client.app.state.chat_transport = bad_transport # type: ignore[attr-defined] + r2 = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 3}, + ) + assert r2.status_code == 200, r2.text + assert r2.json()["topics"] >= 1 + finally: + db_client.app.state.model_available = False # type: ignore[attr-defined] + _cleanup_topics_fixtures(db_pool, seeds) + + +# --- CRUD --- + + +def test_crud_rename_delete_members_auth( + db_client: TestClient, db_pool: ConnectionPool, client: TestClient +) -> None: + seeds: list[dict[str, Any]] = [] + try: + seeds.append( + _insert_email_with_embedding( + db_pool, + subject="Solo message for membership", + embedding=_unit_vec(5), + ) + ) + + # Auth required. + assert client.get("/api/topics").status_code == 401 + assert client.post("/api/topics/generate", json={}).status_code == 401 + + _login(db_client) + + created = db_client.post( + "/api/topics", + json={"label": "Hand Topic", "description": "d"}, + ) + assert created.status_code == 200 + tid = created.json()["id"] + assert created.json()["origin"] == "manual" + + # Manual delete ok. + deleted = db_client.delete(f"/api/topics/{tid}") + assert deleted.status_code == 200 + + # Recreate + auto via generate for delete guard. + for i in range(4): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Auto cluster subject word{i}", + embedding=_unit_vec(1), + ) + ) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 20, "seed": 1}, + ) + assert gen.status_code == 200 + topics = db_client.get("/api/topics").json()["topics"] + auto = next(t for t in topics if t["origin"] == "automatic") + forbidden = db_client.delete(f"/api/topics/{auto['id']}") + assert forbidden.status_code == 403 + + # Rename flips to curated. + patched = db_client.patch( + f"/api/topics/{auto['id']}", + json={"label": "Renamed Auto"}, + ) + assert patched.status_code == 200 + assert patched.json()["origin"] == "curated" + assert patched.json()["label"] == "Renamed Auto" + + # Member add/remove + audit. + sid = seeds[0]["msg_sid"] + add = db_client.post( + f"/api/topics/{auto['id']}/members", + json={"email_sid": sid}, + ) + assert add.status_code == 200 + rem = db_client.delete(f"/api/topics/{auto['id']}/members/{sid}") + assert rem.status_code == 200 + + with db_pool.connection() as conn: + actions = { + r[0] + for r in conn.execute( + """ + SELECT action FROM app_audit + WHERE action LIKE 'topics%%' + """ + ).fetchall() + } + assert "topics_member_add" in actions + assert "topics_member_remove" in actions + assert "topics_patch" in actions + assert "topics_generate" in actions + + # Detail endpoint shape. + detail = db_client.get(f"/api/topics/{auto['id']}") + assert detail.status_code == 200 + body = detail.json() + assert "activity" in body + assert "members" in body + assert body["origin"] in ("curated", "automatic", "manual") + finally: + _cleanup_topics_fixtures(db_pool, seeds) diff --git a/apps/chronicle/web/src/api/types.ts b/apps/chronicle/web/src/api/types.ts index 7285a14..7d245f5 100644 --- a/apps/chronicle/web/src/api/types.ts +++ b/apps/chronicle/web/src/api/types.ts @@ -191,6 +191,19 @@ export interface TopPeopleLane { contacts: TopPeopleContact[] } +/** topics lane: activity bucket series for one topic. */ +export interface TopicSeries { + topic_id: string + label: string + origin: string + buckets: BucketPoint[] +} + +/** topics lane payload (top topics by volume; multirow). */ +export interface TopicsLane { + topics: TopicSeries[] +} + /** Sparse event diamond mark on the events lane. */ export interface EventLaneMark { event_id: string @@ -210,8 +223,8 @@ export interface EventsLane { truncated: boolean } -/** A bars-style lane is BucketPoint[]; top_people / events are nested objects. */ -export type LaneData = BucketPoint[] | TopPeopleLane | EventsLane +/** A bars-style lane is BucketPoint[]; multirow / events are nested objects. */ +export type LaneData = BucketPoint[] | TopPeopleLane | TopicsLane | EventsLane export function isTopPeopleLane(data: LaneData | undefined): data is TopPeopleLane { return ( @@ -222,6 +235,15 @@ export function isTopPeopleLane(data: LaneData | undefined): data is TopPeopleLa ) } +export function isTopicsLane(data: LaneData | undefined): data is TopicsLane { + return ( + data != null && + typeof data === 'object' && + !Array.isArray(data) && + Array.isArray((data as TopicsLane).topics) + ) +} + export function isEventsLane(data: LaneData | undefined): data is EventsLane { return ( data != null && diff --git a/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx b/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx index 7ffef0c..13c63fa 100644 --- a/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx +++ b/apps/chronicle/web/src/chronicle/TimelineCanvas.tsx @@ -10,7 +10,6 @@ import { } from 'react' import type { BucketPoint, EventLaneMark, LaneData } from '../api/types' -import { isTopPeopleLane } from '../api/types' import { CreateEventFromBrush } from '../events/CreateEventFromBrush' import { useWorkingSetStore } from '../workingset/store' import { @@ -23,13 +22,16 @@ import { MARKS_LANE_H, MULTIROW_HEADER_H, MULTIROW_ROW_H, - PEOPLE_CYAN, barsPoints, canvasHeightForLanes, eventPositionMs, eventsMarks, laneAtY as laneAtYFromLayout, layoutLanes, + multirowBucketsForHit, + multirowHitPrefix, + multirowMarkColor, + multirowSeriesForLane, originGlyph, type LaneSpec, } from './laneModel' @@ -234,9 +236,8 @@ function buildAriaLabel( const n = eventsMarks(laneData).length return `${n} events` } - const tp = laneData[spec.key] - const n = isTopPeopleLane(tp) ? tp.contacts.length : 0 - return `${n} contacts` + const n = multirowSeriesForLane(spec, laneData).length + return `${n} ${spec.key === 'topics' ? 'topics' : 'contacts'}` }) return `Timeline, ${range}, ${parts.join(', ')}` } @@ -456,38 +457,39 @@ export function TimelineCanvas({ ctx.textAlign = 'start' } } else { - // multirow: top_people activity spans - const data = laneData[spec.key] - const contacts = isTopPeopleLane(data) ? data.contacts : [] + // multirow: top_people / topics activity spans + const series = multirowSeriesForLane(spec, laneData) + const markColor = multirowMarkColor(spec.key) + const hitPrefix = multirowHitPrefix(spec.key) ctx.fillStyle = COLORS.muted ctx.font = '10px Inter, system-ui, sans-serif' ctx.textBaseline = 'middle' ctx.fillText(spec.label, 4, top + MULTIROW_HEADER_H / 2) - contacts.forEach((contact, i) => { + series.forEach((row, i) => { const rowTop = top + MULTIROW_HEADER_H + i * MULTIROW_ROW_H - const rowMax = maxCount(contact.buckets) || 1 + const rowMax = maxCount(row.buckets) || 1 ctx.fillStyle = COLORS.muted ctx.font = '10px Inter, system-ui, sans-serif' ctx.textBaseline = 'middle' ctx.fillText( - truncateLabel(contact.display_name, 16), + truncateLabel(row.label, 16), 4, rowTop + MULTIROW_ROW_H / 2, ) - for (const pt of contact.buckets) { + for (const pt of row.buckets) { if (pt.count <= 0) continue const t = Date.parse(pt.bucket) if (!Number.isFinite(t)) continue const x = LANE_LABEL_W + xForTime(t, viewport, plotW) const opacity = Math.min(1, Math.max(0.15, pt.count / rowMax)) - ctx.fillStyle = PEOPLE_CYAN + ctx.fillStyle = markColor ctx.globalAlpha = opacity const markH = MULTIROW_ROW_H - 4 ctx.fillRect(x, rowTop + 2, bw, markH) ctx.globalAlpha = 1 - const hitLane = `top_people:${contact.contact_id}` + const hitLane = `${hitPrefix}:${row.id}` if ( selectedBucket && selectedBucket.lane === hitLane && @@ -536,9 +538,8 @@ export function TimelineCanvas({ if (spec.kind === 'bars') total += sumCounts(barsPoints(laneData, spec.key)) else if (spec.kind === 'marks') total += eventsMarks(laneData).length else { - const tp = laneData[spec.key] - if (isTopPeopleLane(tp)) { - for (const c of tp.contacts) total += sumCounts(c.buckets) + for (const row of multirowSeriesForLane(spec, laneData)) { + total += sumCounts(row.buckets) } } } @@ -730,13 +731,8 @@ export function TimelineCanvas({ if (!onSelectBucket) return let points: BucketPoint[] = [] - if (hitKey.startsWith('top_people:')) { - const contactId = hitKey.slice('top_people:'.length) - const tp = laneData.top_people - if (isTopPeopleLane(tp)) { - const contact = tp.contacts.find((c) => c.contact_id === contactId) - points = contact?.buckets ?? [] - } + if (hitKey.startsWith('top_people:') || hitKey.startsWith('topics:')) { + points = multirowBucketsForHit(hitKey, laneData) } else { points = barsPoints(laneData, hitKey) } @@ -759,13 +755,8 @@ export function TimelineCanvas({ if (hitKey) { const unitTyped = parseUnit(unit) let points: BucketPoint[] = [] - if (hitKey.startsWith('top_people:')) { - const contactId = hitKey.slice('top_people:'.length) - const tp = laneData.top_people - if (isTopPeopleLane(tp)) { - const contact = tp.contacts.find((c) => c.contact_id === contactId) - points = contact?.buckets ?? [] - } + if (hitKey.startsWith('top_people:') || hitKey.startsWith('topics:')) { + points = multirowBucketsForHit(hitKey, laneData) } else { points = barsPoints(laneData, hitKey) } diff --git a/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx b/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx index 0f162d5..b38af3d 100644 --- a/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx +++ b/apps/chronicle/web/src/chronicle/TimelineTable.test.tsx @@ -82,4 +82,43 @@ describe('TimelineTable', () => { expect(screen.getByText('Alice')).toBeInTheDocument() expect(screen.getByText('Bob')).toBeInTheDocument() }) + + it('includes per-topic columns for topics lane', () => { + const viewport = { + fromMs: Date.UTC(2014, 0, 1), + toMs: Date.UTC(2016, 0, 1), + } + render( + , + ) + + const table = screen.getByTestId('timeline-table') + expect(table.querySelector('[data-topic-col="t1"]')?.textContent).toBe('House') + expect(table.querySelector('[data-topic-col="t2"]')?.textContent).toBe('Travel') + expect(table.querySelector('[data-topic-cell="t1"]')?.textContent).toBe('3') + expect(table.querySelector('[data-topic-cell="t2"]')?.textContent).toBe('7') + }) }) diff --git a/apps/chronicle/web/src/chronicle/TimelineTable.tsx b/apps/chronicle/web/src/chronicle/TimelineTable.tsx index cf391eb..61de79d 100644 --- a/apps/chronicle/web/src/chronicle/TimelineTable.tsx +++ b/apps/chronicle/web/src/chronicle/TimelineTable.tsx @@ -1,6 +1,7 @@ -import type { BucketPoint, LaneData, TopPeopleContact } from '../api/types' -import { isBucketSeries, isTopPeopleLane } from '../api/types' -import type { LaneSpec } from './laneModel' +import type { BucketPoint, LaneData } from '../api/types' +import { isBucketSeries } from '../api/types' +import type { LaneSpec, MultirowSeries } from './laneModel' +import { multirowSeriesForLane } from './laneModel' import { formatPeriodLabel, formatPeriodRange, type Viewport } from './timeScale' export interface TimelineTableProps { @@ -15,19 +16,36 @@ type MergedRow = { bucket: string /** bars lane key → count */ bars: Record - /** contact_id → count for top_people */ - contacts: Record + /** multirow series id → count (top_people contacts / topics) */ + multi: Record +} + +/** Collect multirow columns from all multirow lanes in catalog order. */ +export function multirowColumns( + lanes: LaneSpec[], + laneData: Record, +): { laneKey: string; series: MultirowSeries }[] { + const out: { laneKey: string; series: MultirowSeries }[] = [] + for (const spec of lanes) { + if (spec.kind !== 'multirow') continue + for (const s of multirowSeriesForLane(spec, laneData)) { + out.push({ laneKey: spec.key, series: s }) + } + } + return out } /** Collect contact columns from top_people (stable order from server). */ export function topPeopleContacts( laneData: Record, -): TopPeopleContact[] { - const tp = laneData.top_people - return isTopPeopleLane(tp) ? tp.contacts : [] +): MultirowSeries[] { + return multirowSeriesForLane( + { key: 'top_people', label: 'Top people', kind: 'multirow' }, + laneData, + ) } -/** Merge all bar lanes + top_people per-contact counts by bucket timestamp. */ +/** Merge all bar lanes + multirow per-series counts by bucket timestamp. */ export function mergeBucketRows( lanes: LaneSpec[], laneData: Record, @@ -37,7 +55,7 @@ export function mergeBucketRows( const ensure = (bucket: string): MergedRow => { let row = map.get(bucket) if (!row) { - row = { bucket, bars: {}, contacts: {} } + row = { bucket, bars: {}, multi: {} } map.set(bucket, row) } return row @@ -50,11 +68,12 @@ export function mergeBucketRows( for (const p of points) { ensure(p.bucket).bars[spec.key] = p.count } - } else if (spec.key === 'top_people') { - const contacts = topPeopleContacts(laneData) - for (const c of contacts) { - for (const p of c.buckets) { - ensure(p.bucket).contacts[c.contact_id] = p.count + } else if (spec.kind === 'multirow') { + const series = multirowSeriesForLane(spec, laneData) + for (const s of series) { + for (const p of s.buckets) { + // Prefix id with lane key to avoid contact/topic id collisions. + ensure(p.bucket).multi[`${spec.key}:${s.id}`] = p.count } } } @@ -73,8 +92,8 @@ function bucketLabel(iso: string): string { /** * Accessible table alternative for the timeline canvas (A11Y-002). - * One row per bucket; columns follow configured lanes. For top_people, - * one column per contact (contact × bucket counts). + * One row per bucket; columns follow configured lanes. For multirow lanes + * (top_people / topics), one column per series (series × bucket counts). */ export function TimelineTable({ viewport, @@ -83,10 +102,9 @@ export function TimelineTable({ laneData, }: TimelineTableProps) { const rows = mergeBucketRows(lanes, laneData) - const contacts = - lanes.some((s) => s.key === 'top_people') ? topPeopleContacts(laneData) : [] + const multiCols = multirowColumns(lanes, laneData) const barLanes = lanes.filter((s) => s.kind === 'bars') - const colCount = 1 + barLanes.length + contacts.length + const colCount = 1 + barLanes.length + multiCols.length const caption = `Timeline buckets (${unit}), ${formatPeriodRange(viewport)}` return ( @@ -108,15 +126,16 @@ export function TimelineTable({ {spec.label} ))} - {contacts.map((c) => ( + {multiCols.map(({ laneKey, series: s }) => ( - {c.display_name} + {s.label} ))} @@ -145,13 +164,14 @@ export function TimelineTable({ {(row.bars[spec.key] ?? 0).toLocaleString()} ))} - {contacts.map((c) => ( + {multiCols.map(({ laneKey, series: s }) => ( - {(row.contacts[c.contact_id] ?? 0).toLocaleString()} + {(row.multi[`${laneKey}:${s.id}`] ?? 0).toLocaleString()} ))} diff --git a/apps/chronicle/web/src/chronicle/laneModel.test.ts b/apps/chronicle/web/src/chronicle/laneModel.test.ts index dcaf17c..78ff314 100644 --- a/apps/chronicle/web/src/chronicle/laneModel.test.ts +++ b/apps/chronicle/web/src/chronicle/laneModel.test.ts @@ -83,6 +83,54 @@ describe('laneModel', () => { }) expect(laneAtY(AXIS_H + LANE_GAP + 10, layout)).toBe('events') }) + + it('topics lane is multirow with topic-purple hit keys', () => { + const multi = specsForKeys(['topics']) + expect(multi[0]?.kind).toBe('multirow') + expect(multi[0]?.key).toBe('topics') + const hMulti = canvasHeightForLanes(multi, { + topics: { + topics: [ + { + topic_id: 't1', + label: 'House', + origin: 'automatic', + buckets: [], + }, + { + topic_id: 't2', + label: 'Travel', + origin: 'curated', + buckets: [], + }, + ], + }, + }) + expect(hMulti).toBe( + AXIS_H + LANE_GAP + MULTIROW_HEADER_H + 2 * MULTIROW_ROW_H, + ) + const layout = layoutLanes(multi, { + topics: { + topics: [ + { + topic_id: 't1', + label: 'House', + origin: 'automatic', + buckets: [], + }, + { + topic_id: 't2', + label: 'Travel', + origin: 'curated', + buckets: [], + }, + ], + }, + }) + const multiTop = layout[0]!.top + MULTIROW_HEADER_H + 1 + expect(laneAtY(multiTop, layout)).toBe('topics:t1') + expect(laneAtY(multiTop + MULTIROW_ROW_H, layout)).toBe('topics:t2') + }) }) describe('eventPositionMs day-floor', () => { diff --git a/apps/chronicle/web/src/chronicle/laneModel.ts b/apps/chronicle/web/src/chronicle/laneModel.ts index 26a799b..1514869 100644 --- a/apps/chronicle/web/src/chronicle/laneModel.ts +++ b/apps/chronicle/web/src/chronicle/laneModel.ts @@ -1,11 +1,27 @@ -import type { BucketPoint, EventLaneMark, LaneData, TopPeopleLane } from '../api/types' -import { isBucketSeries, isEventsLane, isTopPeopleLane } from '../api/types' +import type { + BucketPoint, + EventLaneMark, + LaneData, + TopicSeries, + TopPeopleContact, + TopPeopleLane, + TopicsLane, +} from '../api/types' +import { + isBucketSeries, + isEventsLane, + isTopPeopleLane, + isTopicsLane, +} from '../api/types' import type { LaneKey } from '../workingset/urlState' export type LaneKind = 'bars' | 'multirow' | 'marks' +/** Catalog keys include `topics` even when URL codec lags (server-first lane). */ +export type ChronicleLaneKey = LaneKey | 'topics' + export interface LaneSpec { - key: LaneKey + key: ChronicleLaneKey label: string kind: LaneKind } @@ -16,6 +32,7 @@ export const LANE_CATALOG: readonly LaneSpec[] = [ { key: 'attachments', label: 'Attachments', kind: 'bars' }, { key: 'people', label: 'People (distinct)', kind: 'bars' }, { key: 'top_people', label: 'Top people', kind: 'multirow' }, + { key: 'topics', label: 'Topics', kind: 'multirow' }, { key: 'events', label: 'Events', kind: 'marks' }, ] as const @@ -32,6 +49,9 @@ export const EVENT_AMBER = '#E0A84A' export const PEOPLE_CYAN = '#56d4dd' +/** Spec topic purple (#A78BFA) for topics multirow marks. */ +export const TOPIC_PURPLE = '#A78BFA' + export const AXIS_H = 28 export const LANE_H = 72 export const LANE_LABEL_W = 72 @@ -45,14 +65,51 @@ export const MARKS_LANE_H = 56 export function specsForKeys(keys: readonly string[]): LaneSpec[] { const out: LaneSpec[] = [] for (const key of keys) { - const spec = CATALOG_BY_KEY.get(key as LaneKey) + const spec = CATALOG_BY_KEY.get(key as ChronicleLaneKey) if (spec) out.push(spec) } return out } -export function multirowHeight(contactCount: number): number { - return MULTIROW_HEADER_H + MULTIROW_ROW_H * Math.max(0, contactCount) +export function multirowHeight(rowCount: number): number { + return MULTIROW_HEADER_H + MULTIROW_ROW_H * Math.max(0, rowCount) +} + +/** Multirow series rows for top_people or topics (shared canvas/table shape). */ +export interface MultirowSeries { + id: string + label: string + buckets: BucketPoint[] +} + +export function multirowSeriesForLane( + spec: LaneSpec, + laneData: Record | undefined, +): MultirowSeries[] { + const data = laneData?.[spec.key] + if (spec.key === 'topics' && isTopicsLane(data)) { + return data.topics.map((t) => ({ + id: t.topic_id, + label: t.label, + buckets: t.buckets, + })) + } + if (isTopPeopleLane(data)) { + return data.contacts.map((c) => ({ + id: c.contact_id, + label: c.display_name, + buckets: c.buckets, + })) + } + return [] +} + +export function multirowHitPrefix(laneKey: string): string { + return laneKey === 'topics' ? 'topics' : 'top_people' +} + +export function multirowMarkColor(laneKey: string): string { + return laneKey === 'topics' ? TOPIC_PURPLE : PEOPLE_CYAN } export function laneContentHeight( @@ -61,9 +118,7 @@ export function laneContentHeight( ): number { if (spec.kind === 'bars') return LANE_H if (spec.kind === 'marks') return MARKS_LANE_H - const data = laneData?.[spec.key] - const n = isTopPeopleLane(data) ? data.contacts.length : 0 - return multirowHeight(n) + return multirowHeight(multirowSeriesForLane(spec, laneData).length) } /** Total canvas CSS height for the given lane set and data. */ @@ -79,7 +134,7 @@ export function canvasHeightForLanes( } export interface LaneLayoutRow { - /** Hit-test id: lane key, or `top_people:` for multirow rows. */ + /** Hit-test id: lane key, or `top_people:` / `topics:` for multirow. */ hitKey: string /** Y of the top of this hit region (local canvas coords). */ top: number @@ -88,8 +143,10 @@ export interface LaneLayoutRow { /** Lane key for data lookup. */ laneKey: string kind: LaneKind - /** Contact id when multirow row. */ + /** Contact id when top_people multirow row. */ contactId?: string + /** Topic id when topics multirow row. */ + topicId?: string } export interface LaneLayoutBlock { @@ -119,16 +176,17 @@ export function layoutLanes( kind: spec.kind, }) } else { - const data = laneData?.[spec.key] - const contacts = isTopPeopleLane(data) ? data.contacts : [] - contacts.forEach((c, i) => { + const series = multirowSeriesForLane(spec, laneData) + const prefix = multirowHitPrefix(spec.key) + series.forEach((row, i) => { rows.push({ - hitKey: `top_people:${c.contact_id}`, + hitKey: `${prefix}:${row.id}`, top: y + MULTIROW_HEADER_H + i * MULTIROW_ROW_H, height: MULTIROW_ROW_H, laneKey: spec.key, kind: 'multirow', - contactId: c.contact_id, + contactId: prefix === 'top_people' ? row.id : undefined, + topicId: prefix === 'topics' ? row.id : undefined, }) }) } @@ -169,6 +227,13 @@ export function topPeopleData( return isTopPeopleLane(data) ? data : undefined } +export function topicsData( + laneData: Record | undefined, +): TopicsLane | undefined { + const data = laneData?.topics + return isTopicsLane(data) ? data : undefined +} + export function eventsMarks( laneData: Record | undefined, ): EventLaneMark[] { @@ -176,6 +241,26 @@ export function eventsMarks( return isEventsLane(data) ? data.events : [] } +/** Resolve multirow series by hit key (`top_people:…` or `topics:…`). */ +export function multirowBucketsForHit( + hitKey: string, + laneData: Record | undefined, +): BucketPoint[] { + if (hitKey.startsWith('topics:')) { + const topicId = hitKey.slice('topics:'.length) + const topics = topicsData(laneData) + const row = topics?.topics.find((t: TopicSeries) => t.topic_id === topicId) + return row?.buckets ?? [] + } + if (hitKey.startsWith('top_people:')) { + const contactId = hitKey.slice('top_people:'.length) + const tp = topPeopleData(laneData) + const row = tp?.contacts.find((c: TopPeopleContact) => c.contact_id === contactId) + return row?.buckets ?? [] + } + return [] +} + /** * Origin glyph (text, not color-only) for event marks. * A=analyst, ⚙=automatic, S=source, I=imported. diff --git a/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md b/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md index e0703ef..2bb8623 100644 --- a/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md +++ b/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md @@ -75,9 +75,17 @@ Tasks are cheap-coder-sized (≈ one focused PR or less). One PR per phase, one **Phase 3 exit criterion (spec §20.1):** generated events are reviewable, evidence-backed, and versioned. -### Phase 4 — Secondary exploration (elaborate at start) +### Phase 4 — Secondary exploration (PR: `cheap-coder/chronicle-phase-4`) -Topics subsystem (clustering job over embeddings, hierarchy, curation with manual precedence), Topic Atlas (hierarchy default, projection with LOD, river, matrix), person/organization profiles + ego graph (contacts synergy), version families. +| Task | Scope | +| --- | --- | +| 4.1 | Topics subsystem: `app_topics`/`app_topic_members` (pgvector centroids), deterministic k-means over an embedding sample + one set-based full-corpus assignment pass, term-based labels with optional gateway polish, curation CRUD with manual precedence (TA-005), topics lane | +| 4.2 | Topic Atlas UI: hierarchy default (TA-001), topic detail + river (Chronicle-linked), topic×year matrix with table alternative, centroid-PCA projection with exploratory disclaimer (TA-003 LOD: topics only, never all sources) | +| 4.3 | People & Organizations UI: person profile over the contacts subsystem (identity, span, activity → Chronicle, topics, sources), merge/split curation UI (PE-001), org view by domain/kind | +| 4.4 | Ego graph: bounded co-participation edges endpoint (set-based), SVG graph with edge→thread-evidence drill (PE-003), no relationship-quality labels (PE-004) | +| 4.5 | Version families (Table 25 probable-version heuristic + compare view) + workflows B and D acceptance tests | + +**Phase 4 exit criterion (spec §20.1):** every secondary lens preserves scope and exposes authoritative source lists. ### Phase 5 — Hardening (elaborate at start) @@ -94,7 +102,7 @@ WCAG 2.2 AA audit, security review (CSP, sanitizer corpus, IDOR/enumeration), pe ## 5. STATE — live progress (update after every task) -**Next up:** Phase 4 (secondary exploration) — elaborate task table, then 4.1. Goal mode active (2026-07-13): user delegated review+merge of all phases to Claude via /goal. +**Next up:** Phase 4, Task 4.2 (Topic Atlas UI). Goal mode active (2026-07-13): user delegated review+merge of all phases to Claude via /goal. | Date | Task | PR | Outcome | | --- | --- | --- | --- | @@ -123,3 +131,5 @@ WCAG 2.2 AA audit, security review (CSP, sanitizer corpus, IDOR/enumeration), pe | 2026-07-13 | 3.3 reconstruction: claim-evidence matrix, adopt/suggestions, stale checks, dismissed list | chronicle-phase-3 | Approved; 222 server / 203 web tests | | 2026-07-13 | 3.4 compare mode: aligned/small-multiples, normalize, event anchor | chronicle-phase-3 | Approved; 228 server / 238 web tests | | 2026-07-13 | 3.5 workflow A acceptance + criterion 4; keyboard glue | chronicle-phase-3 | Approved; Phase 3 complete (244 web tests, 0 todos) | +| 2026-07-13 | Phase 3 PR #109 merged (36b7e97) | #109 | CI green | +| 2026-07-13 | 4.1 topics subsystem: seeded k-means, one-statement assignment, manual precedence, topics lane | chronicle-phase-4 | Approved; 238 server / 246 web tests | From 98780a870754125627d4d5991add95b447da5cb2 Mon Sep 17 00:00:00 2001 From: Claude Code Agent Date: Mon, 13 Jul 2026 22:01:49 -0700 Subject: [PATCH 2/5] =?UTF-8?q?feat(chronicle):=20Topic=20Atlas=20?= =?UTF-8?q?=E2=80=94=20hierarchy,=20river,=20matrix,=20centroid=20projecti?= =?UTF-8?q?on=20(Phase=204=20task=204.2,=20#105)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /topics lens: keyboard-navigable hierarchy (default) with visible origins and hidden group; topic inspector with rename→curated, representative members, and a definitive keyset member list (TA-004); river with normalize legend; sortable topic×year matrix; deterministic topics-only PCA projection carrying the §6.3 exploratory disclaimer. Working set preserved across all views. Implemented via cheap-coder (grok-4.5 @ max). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XP7M17NHxw6mjpTFtMqwZu --- .../server/src/chronicle_server/topics.py | 531 +++++++++++++++++- apps/chronicle/server/tests/test_topics.py | 243 ++++++++ apps/chronicle/web/src/App.tsx | 3 +- apps/chronicle/web/src/api/types.ts | 113 ++++ .../web/src/inspector/InspectorPanel.test.tsx | 218 +++++++ .../web/src/inspector/InspectorPanel.tsx | 10 + .../web/src/topics/HierarchyView.tsx | 308 ++++++++++ apps/chronicle/web/src/topics/MatrixView.tsx | 202 +++++++ .../web/src/topics/ProjectionView.tsx | 197 +++++++ apps/chronicle/web/src/topics/RiverView.tsx | 229 ++++++++ apps/chronicle/web/src/topics/TopicCard.tsx | 354 ++++++++++++ .../web/src/topics/TopicsPage.test.tsx | 286 ++++++++++ apps/chronicle/web/src/topics/TopicsPage.tsx | 202 +++++++ apps/chronicle/web/src/topics/api.ts | 111 ++++ apps/chronicle/web/src/topics/originBadge.tsx | 23 + .../web/src/workingset/urlState.test.ts | 40 +- apps/chronicle/web/src/workingset/urlState.ts | 67 +++ .../plans/2026-07-13-life-chronicle-plan.md | 3 +- 18 files changed, 3136 insertions(+), 4 deletions(-) create mode 100644 apps/chronicle/web/src/topics/HierarchyView.tsx create mode 100644 apps/chronicle/web/src/topics/MatrixView.tsx create mode 100644 apps/chronicle/web/src/topics/ProjectionView.tsx create mode 100644 apps/chronicle/web/src/topics/RiverView.tsx create mode 100644 apps/chronicle/web/src/topics/TopicCard.tsx create mode 100644 apps/chronicle/web/src/topics/TopicsPage.test.tsx create mode 100644 apps/chronicle/web/src/topics/TopicsPage.tsx create mode 100644 apps/chronicle/web/src/topics/api.ts create mode 100644 apps/chronicle/web/src/topics/originBadge.tsx diff --git a/apps/chronicle/server/src/chronicle_server/topics.py b/apps/chronicle/server/src/chronicle_server/topics.py index 1072692..f7df21e 100644 --- a/apps/chronicle/server/src/chronicle_server/topics.py +++ b/apps/chronicle/server/src/chronicle_server/topics.py @@ -15,7 +15,7 @@ import json import re import time -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, Literal from uuid import UUID @@ -25,9 +25,12 @@ from pydantic import BaseModel, Field, field_validator from chronicle_server.auth import require_user +from chronicle_server.chronicle import VALID_UNITS, choose_aggregation +from chronicle_server.cursor import decode_cursor, encode_cursor from chronicle_server.db import audit from chronicle_server.gateway import ModelGateway from chronicle_server.ids import decode_source_id, encode_source_id, msg_key_to_uuid +from chronicle_server.scope import QueryScope, scope_filters if TYPE_CHECKING: from psycopg_pool import ConnectionPool @@ -48,6 +51,11 @@ _LABEL_TERMS = 3 _MAX_LABEL_WORDS = 4 _MIN_TERM_LEN = 3 +_MEMBERS_DEFAULT_LIMIT = 50 +_MEMBERS_MAX_LIMIT = 200 +_RIVER_DEFAULT_TOP = 8 +_RIVER_MAX_TOP = 48 +_RIVER_DEFAULT_PIXEL_WIDTH = 920 # Common English stopwords for subject TF labeling (lowercase). STOPWORDS: frozenset[str] = frozenset( @@ -236,6 +244,85 @@ class MemberAddRequest(BaseModel): # --- pure helpers (testable without DB) --- +def pca_project_2d(centroids: np.ndarray) -> np.ndarray: + """Deterministic PCA of centroid matrix to 2D via SVD. + + Returns an ``(n, 2)`` array with coordinates normalized into ``[-1, 1]``. + Topics without centroids must be filtered out before calling. + """ + arr = np.asarray(centroids, dtype=np.float64) + if arr.ndim != 2 or arr.shape[0] == 0: + return np.zeros((0, 2), dtype=np.float64) + n, d = arr.shape + if n == 1: + return np.zeros((1, 2), dtype=np.float64) + + centered = arr - arr.mean(axis=0, keepdims=True) + # Economy SVD is deterministic for a fixed matrix (numpy). + _u, s, vt = np.linalg.svd(centered, full_matrices=False) + k = min(2, s.shape[0], d, n) + # Project: X_c @ V_k (equivalent to U[:, :k] * S[:k]) + coords = centered @ vt[:k].T + if k < 2: + coords = np.pad(coords, ((0, 0), (0, 2 - k))) + max_abs = float(np.max(np.abs(coords))) if coords.size else 0.0 + if max_abs > 0.0 and np.isfinite(max_abs): + coords = coords / max_abs + return np.asarray(coords, dtype=np.float64) + + +def _parse_viewport_dt(value: str) -> datetime: + """Parse ISO datetime/date for viewport bounds; raise 422 on failure.""" + raw = value.strip() + if not raw: + raise HTTPException(status_code=422, detail="empty datetime") + # Accept trailing Z. + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(raw) + except ValueError as exc: + raise HTTPException(status_code=422, detail=f"invalid datetime: {value}") from exc + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +def _scope_from_query( + *, + mailboxes: list[str] | None, + senders: list[str] | None, + scope_from: str | None, + scope_to: str | None, +) -> QueryScope: + payload: dict[str, Any] = { + "mailboxes": list(mailboxes or []), + "senders": list(senders or []), + } + if scope_from or scope_to: + date: dict[str, str] = {} + if scope_from: + date["from"] = scope_from + if scope_to: + date["to"] = scope_to + payload["date"] = date + return QueryScope.model_validate(payload) + + +def _resolve_river_unit(unit: str, viewport_from: datetime, viewport_to: datetime) -> str: + if unit == "auto": + duration = viewport_to - viewport_from + if duration.total_seconds() <= 0: + duration = timedelta(days=1) + return choose_aggregation(duration, _RIVER_DEFAULT_PIXEL_WIDTH) + if unit not in VALID_UNITS: + raise HTTPException( + status_code=422, + detail=f"unit must be auto or one of {', '.join(VALID_UNITS)}", + ) + return unit + + def parse_embedding(raw: Any) -> list[float] | None: """Parse pgvector embedding (string or list) into floats.""" if raw is None: @@ -1138,6 +1225,362 @@ def remove_member( return {"status": "ok", "topic_id": str(topic_id), "email_sid": email_sid} +# --- Atlas analytics (river / matrix / projection / members) --- + + +def topics_river( + pool: ConnectionPool, + *, + viewport_from: str, + viewport_to: str, + unit: str = "auto", + top: int = _RIVER_DEFAULT_TOP, + scope: QueryScope | None = None, +) -> dict[str, Any]: + """Per-topic bucket series for the top-N visible topics in range. + + Reuses the Chronicle topics-lane ranked CTE shape with parameterized ``top``. + ``mode_hint`` is always ``absolute`` — normalization is client-side. + """ + top_n = max(1, min(int(top), _RIVER_MAX_TOP)) + vf = _parse_viewport_dt(viewport_from) + vt = _parse_viewport_dt(viewport_to) + if not (vt > vf): + raise HTTPException(status_code=422, detail="to must be after from") + resolved_unit = _resolve_river_unit(unit, vf, vt) + scope = scope or QueryScope() + scope_conds, scope_params = scope_filters(scope) + + # Bare emails columns are unambiguous (members/topics lack those names). + email_where = [ + "date IS NOT NULL", + "date >= %(viewport_from)s", + "date < %(viewport_to)s", + *scope_conds, + ] + + sql = f""" + WITH filtered AS ( + SELECT m.topic_id, e.date + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE t.hidden = FALSE + AND {" AND ".join(email_where)} + ), + ranked AS ( + SELECT topic_id, count(*)::int AS total + FROM filtered + GROUP BY topic_id + ORDER BY total DESC, topic_id + LIMIT %(top_limit)s + ), + bucketed AS ( + SELECT + f.topic_id, + date_trunc(%(unit)s, f.date) AS bucket, + count(*)::int AS count + FROM filtered f + JOIN ranked r ON r.topic_id = f.topic_id + GROUP BY f.topic_id, 2 + ) + SELECT + r.topic_id::text, + t.label, + t.origin, + b.bucket, + coalesce(b.count, 0)::int AS count, + r.total + FROM ranked r + JOIN app_topics t ON t.id = r.topic_id + LEFT JOIN bucketed b ON b.topic_id = r.topic_id + ORDER BY r.total DESC, r.topic_id, b.bucket + """ + params = { + **scope_params, + "unit": resolved_unit, + "viewport_from": vf.isoformat(), + "viewport_to": vt.isoformat(), + "top_limit": top_n, + } + with pool.connection() as conn: + rows = conn.execute(sql, params).fetchall() + + by_topic: dict[str, dict[str, Any]] = {} + order: list[str] = [] + for row in rows: + topic_id = str(row[0]) + label = str(row[1]) if row[1] is not None else topic_id + origin = str(row[2]) + if topic_id not in by_topic: + by_topic[topic_id] = { + "topic_id": topic_id, + "label": label, + "origin": origin, + "buckets": [], + } + order.append(topic_id) + bucket = _iso(row[3]) + if bucket is None: + continue + by_topic[topic_id]["buckets"].append({"bucket": bucket, "count": int(row[4])}) + + return { + "unit": resolved_unit, + "mode_hint": "absolute", + "from": _iso(vf), + "to": _iso(vt), + "topics": [by_topic[tid] for tid in order], + } + + +def topics_matrix( + pool: ConnectionPool, + *, + by: str = "year", + scope: QueryScope | None = None, +) -> dict[str, Any]: + """Visible topics × year member counts (one grouped statement) + totals.""" + if by != "year": + raise HTTPException(status_code=422, detail="only by=year is supported") + scope = scope or QueryScope() + scope_conds, scope_params = scope_filters(scope) + extra_sql = (" AND " + " AND ".join(scope_conds)) if scope_conds else "" + + sql = f""" + SELECT + t.id::text AS topic_id, + t.label, + t.origin, + extract(year FROM e.date)::int AS year, + count(*)::int AS count + FROM app_topics t + JOIN app_topic_members m ON m.topic_id = t.id + JOIN emails e ON e.id = m.email_id + WHERE t.hidden = FALSE + AND e.date IS NOT NULL + {extra_sql} + GROUP BY t.id, t.label, t.origin, extract(year FROM e.date) + ORDER BY t.member_count DESC, t.label ASC, year ASC + """ + with pool.connection() as conn: + rows = conn.execute(sql, scope_params).fetchall() + + # topic_id → {label, origin, cells: {year: count}} + topics_map: dict[str, dict[str, Any]] = {} + years_set: set[int] = set() + for row in rows: + tid = str(row[0]) + year = int(row[3]) + count = int(row[4]) + years_set.add(year) + if tid not in topics_map: + topics_map[tid] = { + "topic_id": tid, + "label": str(row[1]), + "origin": str(row[2]), + "cells": {}, + } + topics_map[tid]["cells"][str(year)] = count + + years = sorted(years_set) + rows_out: list[dict[str, Any]] = [] + col_totals: dict[str, int] = {str(y): 0 for y in years} + grand_total = 0 + for tid, info in topics_map.items(): + row_total = 0 + cells: dict[str, int] = {} + for y in years: + c = int(info["cells"].get(str(y), 0)) + cells[str(y)] = c + row_total += c + col_totals[str(y)] = col_totals[str(y)] + c + grand_total += row_total + rows_out.append( + { + "topic_id": tid, + "label": info["label"], + "origin": info["origin"], + "cells": cells, + "row_total": row_total, + } + ) + + return { + "by": "year", + "columns": [str(y) for y in years], + "rows": rows_out, + "column_totals": col_totals, + "grand_total": grand_total, + } + + +def topics_projection(pool: ConnectionPool) -> dict[str, Any]: + """Centroid PCA (numpy SVD) of automatic+curated topics to 2D. + + Manual topics (no centroid) are excluded; response notes the count. + NEVER returns per-source points (TA-003 — LOD stops at topics). + """ + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, label, origin, member_count, centroid + FROM app_topics + WHERE hidden = FALSE + AND centroid IS NOT NULL + AND origin IN ('automatic', 'curated') + ORDER BY member_count DESC, label ASC, id ASC + """ + ).fetchall() + excluded_row = conn.execute( + """ + SELECT count(*)::int + FROM app_topics + WHERE hidden = FALSE + AND (centroid IS NULL OR origin = 'manual') + """ + ).fetchone() + excluded_manual = int(excluded_row[0]) if excluded_row else 0 + + topic_meta: list[tuple[str, str, str, int]] = [] + vectors: list[list[float]] = [] + for row in rows: + emb = parse_embedding(row[4]) + if emb is None or len(emb) != _EMBED_DIM: + excluded_manual += 1 + continue + topic_meta.append((str(row[0]), str(row[1]), str(row[2]), int(row[3]))) + vectors.append(emb) + + points: list[dict[str, Any]] = [] + if vectors: + coords = pca_project_2d(np.asarray(vectors, dtype=np.float64)) + for i, (tid, label, origin, mcount) in enumerate(topic_meta): + points.append( + { + "topic_id": tid, + "label": label, + "origin": origin, + "member_count": mcount, + "x": float(coords[i, 0]), + "y": float(coords[i, 1]), + } + ) + + note = None + if excluded_manual > 0: + note = ( + f"{excluded_manual} topic(s) excluded (manual or missing centroid); " + "projection is topic-level only — no per-source points (TA-003)." + ) + else: + note = "Projection is topic-level only — no per-source points (TA-003)." + + return { + "points": points, + "excluded_without_centroid": excluded_manual, + "note": note, + } + + +def _parse_members_cursor(token: str, secret_key: str) -> tuple[str | None, UUID]: + try: + payload = decode_cursor(token, secret_key) + except ValueError as exc: + raise HTTPException(status_code=400, detail="invalid cursor") from exc + if "id" not in payload: + raise HTTPException(status_code=400, detail="invalid cursor") + try: + last_id = UUID(str(payload["id"])) + except (ValueError, TypeError, AttributeError) as exc: + raise HTTPException(status_code=400, detail="invalid cursor") from exc + d = payload.get("d") + if d is not None and not isinstance(d, str): + raise HTTPException(status_code=400, detail="invalid cursor") + return d if isinstance(d, str) else None, last_id + + +def list_topic_members( + pool: ConnectionPool, + topic_id: UUID, + *, + secret_key: str, + cursor: str | None = None, + limit: int = _MEMBERS_DEFAULT_LIMIT, +) -> dict[str, Any]: + """Paginated definitive member list for a topic (keyset on date DESC, id DESC). + + Atlas "Open sources" uses this instead of Research Desk topic scoping + (Phase 5 polish). TA-004 — every aggregated view can open this list. + """ + lim = max(1, min(int(limit), _MEMBERS_MAX_LIMIT)) + with pool.connection() as conn: + trow = conn.execute( + "SELECT 1 FROM app_topics WHERE id = %(id)s", + {"id": topic_id}, + ).fetchone() + if trow is None: + raise HTTPException(status_code=404, detail="Topic not found") + + conditions = ["m.topic_id = %(topic_id)s"] + params: dict[str, Any] = {"topic_id": topic_id, "lim": lim + 1} + + if cursor: + cursor_d, cursor_id = _parse_members_cursor(cursor, secret_key) + params["cursor_id"] = cursor_id + if cursor_d is not None: + params["cursor_d"] = cursor_d + # DESC NULLS LAST: after (d, id) come earlier dates, same-date lower ids, + # then all NULL dates. + conditions.append( + "(" + " (e.date IS NOT NULL AND (e.date < %(cursor_d)s" + " OR (e.date = %(cursor_d)s AND e.id < %(cursor_id)s)))" + " OR e.date IS NULL" + ")" + ) + else: + conditions.append("e.date IS NULL AND e.id < %(cursor_id)s") + + where_sql = " AND ".join(conditions) + rows = conn.execute( + f""" + SELECT e.id, e.subject, e.sender_name, e.sender_address, + e.date, e.source_account, e.thread_id, m.distance + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + WHERE {where_sql} + ORDER BY e.date DESC NULLS LAST, e.id DESC + LIMIT %(lim)s + """, + params, + ).fetchall() + + page = rows[:lim] + has_more = len(rows) > lim + items = [ + { + "id": encode_source_id("msg", r[0]), + "subject": r[1], + "sender_name": r[2], + "sender_address": r[3], + "date": _iso(r[4]), + "mailbox": r[5], + "thread_id": r[6], + "distance": float(r[7]) if r[7] is not None else None, + } + for r in page + ] + next_cursor: str | None = None + if has_more and page: + last = page[-1] + next_cursor = encode_cursor( + {"d": _iso(last[4]), "id": str(last[0])}, + secret_key, + ) + return {"items": items, "next_cursor": next_cursor} + + # --- routes --- @@ -1172,6 +1615,72 @@ def topics_list( return {"topics": list_topics(pool, include_hidden=include_hidden)} +@router.get("/topics/river") +def topics_river_endpoint( + request: Request, + from_: str = Query(..., alias="from"), + to: str = Query(...), + unit: str = Query(default="auto"), + top: int = Query(default=_RIVER_DEFAULT_TOP, ge=1, le=_RIVER_MAX_TOP), + mb: str | None = Query(default=None, description="CSV mailboxes"), + sd: str | None = Query(default=None, description="CSV senders"), + df: str | None = Query(default=None, description="Scope date from"), + dt: str | None = Query(default=None, description="Scope date to"), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic river: top-N visible topics × time buckets (mode_hint=absolute).""" + pool: ConnectionPool = request.app.state.pool + mailboxes = [s.strip() for s in (mb or "").split(",") if s.strip()] + senders = [s.strip() for s in (sd or "").split(",") if s.strip()] + scope = _scope_from_query( + mailboxes=mailboxes, + senders=senders, + scope_from=df, + scope_to=dt, + ) + return topics_river( + pool, + viewport_from=from_, + viewport_to=to, + unit=unit, + top=top, + scope=scope, + ) + + +@router.get("/topics/matrix") +def topics_matrix_endpoint( + request: Request, + by: str = Query(default="year"), + mb: str | None = Query(default=None, description="CSV mailboxes"), + sd: str | None = Query(default=None, description="CSV senders"), + df: str | None = Query(default=None, description="Scope date from"), + dt: str | None = Query(default=None, description="Scope date to"), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Topic × year matrix with row/column totals.""" + pool: ConnectionPool = request.app.state.pool + mailboxes = [s.strip() for s in (mb or "").split(",") if s.strip()] + senders = [s.strip() for s in (sd or "").split(",") if s.strip()] + scope = _scope_from_query( + mailboxes=mailboxes, + senders=senders, + scope_from=df, + scope_to=dt, + ) + return topics_matrix(pool, by=by, scope=scope) + + +@router.get("/topics/projection") +def topics_projection_endpoint( + request: Request, + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Centroid PCA to 2D; topics only — never per-source points (TA-003).""" + pool: ConnectionPool = request.app.state.pool + return topics_projection(pool) + + @router.post("/topics") def topics_create( body: TopicCreate, @@ -1183,6 +1692,26 @@ def topics_create( return create_manual_topic(pool, body, username=user) +@router.get("/topics/{topic_id}/members") +def topics_members_list( + topic_id: UUID, + request: Request, + cursor: str | None = Query(default=None), + limit: int = Query(default=_MEMBERS_DEFAULT_LIMIT, ge=1, le=_MEMBERS_MAX_LIMIT), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Paginated definitive member list (keyset). Atlas TA-004 source list.""" + pool: ConnectionPool = request.app.state.pool + secret_key: str = request.app.state.settings.secret_key + return list_topic_members( + pool, + topic_id, + secret_key=secret_key, + cursor=cursor, + limit=limit, + ) + + @router.get("/topics/{topic_id}") def topics_get( topic_id: UUID, diff --git a/apps/chronicle/server/tests/test_topics.py b/apps/chronicle/server/tests/test_topics.py index 3d5940c..2084081 100644 --- a/apps/chronicle/server/tests/test_topics.py +++ b/apps/chronicle/server/tests/test_topics.py @@ -14,6 +14,7 @@ extract_top_terms, label_from_terms, parse_label_polish_response, + pca_project_2d, run_kmeans, validate_label_array, vector_literal, @@ -500,3 +501,245 @@ def test_crud_rename_delete_members_auth( assert body["origin"] in ("curated", "automatic", "manual") finally: _cleanup_topics_fixtures(db_pool, seeds) + + +# --- Atlas analytics (4.2) --- + + +def test_pca_project_2d_deterministic() -> None: + rng = np.random.default_rng(42) + data = rng.normal(size=(12, 16)).astype(np.float64) + a = pca_project_2d(data) + b = pca_project_2d(data) + np.testing.assert_allclose(a, b) + assert a.shape == (12, 2) + assert float(np.max(np.abs(a))) <= 1.0 + 1e-9 + + +def test_river_top_n_and_unit(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(5): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"River alpha topic word {i}", + embedding=_unit_vec(0), + date=datetime(2020, 3, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + for i in range(3): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"River beta topic word {i}", + embedding=_unit_vec(20), + date=datetime(2020, 6, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 11}, + ) + assert gen.status_code == 200, gen.text + + r = db_client.get( + "/api/topics/river", + params={ + "from": "2020-01-01T00:00:00Z", + "to": "2021-01-01T00:00:00Z", + "unit": "month", + "top": 2, + }, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["mode_hint"] == "absolute" + assert body["unit"] == "month" + assert isinstance(body["topics"], list) + assert len(body["topics"]) <= 2 + for series in body["topics"]: + assert "topic_id" in series + assert "label" in series + assert "origin" in series + assert isinstance(series["buckets"], list) + for b in series["buckets"]: + assert "bucket" in b and "count" in b + + # unit=auto resolves to a valid unit. + r_auto = db_client.get( + "/api/topics/river", + params={ + "from": "2020-01-01T00:00:00Z", + "to": "2021-01-01T00:00:00Z", + "unit": "auto", + "top": 8, + }, + ) + assert r_auto.status_code == 200, r_auto.text + assert r_auto.json()["unit"] in ( + "hour", + "day", + "week", + "month", + "quarter", + "year", + ) + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_matrix_totals(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(4): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Matrix year twenty {i}", + embedding=_unit_vec(1), + date=datetime(2020, 5, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + for i in range(2): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Matrix year twentyone {i}", + embedding=_unit_vec(1), + date=datetime(2021, 5, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 5}, + ) + assert gen.status_code == 200, gen.text + + r = db_client.get("/api/topics/matrix", params={"by": "year"}) + assert r.status_code == 200, r.text + body = r.json() + assert body["by"] == "year" + assert "2020" in body["columns"] or "2021" in body["columns"] + assert isinstance(body["rows"], list) + assert "column_totals" in body + assert "grand_total" in body + # Totals reconcile: sum of row_totals == grand_total == sum of col totals. + row_sum = sum(int(row["row_total"]) for row in body["rows"]) + col_sum = sum(int(v) for v in body["column_totals"].values()) + assert row_sum == body["grand_total"] + assert col_sum == body["grand_total"] + assert body["grand_total"] >= 6 + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_projection_determinism_shape_and_manual_exclusion( + db_client: TestClient, db_pool: ConnectionPool +) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Projection cluster subject {i}", + embedding=_unit_vec(i % 2), + date=datetime(2019, 1, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 9}, + ) + assert gen.status_code == 200, gen.text + + # Manual topic without centroid must be excluded. + created = db_client.post( + "/api/topics", + json={"label": "Manual No Centroid"}, + ) + assert created.status_code == 200 + manual_id = created.json()["id"] + + p1 = db_client.get("/api/topics/projection") + assert p1.status_code == 200, p1.text + body1 = p1.json() + p2 = db_client.get("/api/topics/projection") + assert p2.status_code == 200 + body2 = p2.json() + # Determinism: same centroids → same coords. + assert body1["points"] == body2["points"] + + # Shape: topic-level only (no source_id / email_id fields). + assert "points" in body1 + for pt in body1["points"]: + assert set(pt.keys()) >= { + "topic_id", + "label", + "origin", + "member_count", + "x", + "y", + } + assert "source_id" not in pt + assert "email_id" not in pt + assert "sources" not in pt + assert -1.0 - 1e-9 <= float(pt["x"]) <= 1.0 + 1e-9 + assert -1.0 - 1e-9 <= float(pt["y"]) <= 1.0 + 1e-9 + assert pt["topic_id"] != manual_id + + assert body1["excluded_without_centroid"] >= 1 + assert "note" in body1 + assert "TA-003" in body1["note"] or "topic-level" in body1["note"].lower() + finally: + _cleanup_topics_fixtures(db_pool, seeds) + + +def test_members_keyset_endpoint(db_client: TestClient, db_pool: ConnectionPool) -> None: + seeds: list[dict[str, Any]] = [] + try: + for i in range(6): + seeds.append( + _insert_email_with_embedding( + db_pool, + subject=f"Member list item {i}", + embedding=_unit_vec(3), + date=datetime(2022, 1, 1 + i, 12, 0, tzinfo=UTC), + ) + ) + _login(db_client) + gen = db_client.post( + "/api/topics/generate", + json={"k": 4, "sample": 50, "seed": 2}, + ) + assert gen.status_code == 200, gen.text + topics = db_client.get("/api/topics").json()["topics"] + assert topics + tid = topics[0]["id"] + + page1 = db_client.get( + f"/api/topics/{tid}/members", + params={"limit": 2}, + ) + assert page1.status_code == 200, page1.text + b1 = page1.json() + assert len(b1["items"]) <= 2 + assert "next_cursor" in b1 + ids1 = [it["id"] for it in b1["items"]] + assert all(sid.startswith("msg_") for sid in ids1) + + if b1["next_cursor"]: + page2 = db_client.get( + f"/api/topics/{tid}/members", + params={"limit": 2, "cursor": b1["next_cursor"]}, + ) + assert page2.status_code == 200, page2.text + ids2 = [it["id"] for it in page2.json()["items"]] + # Keyset pages must not overlap. + assert set(ids1).isdisjoint(set(ids2)) + finally: + _cleanup_topics_fixtures(db_pool, seeds) diff --git a/apps/chronicle/web/src/App.tsx b/apps/chronicle/web/src/App.tsx index 936ddd8..5c53910 100644 --- a/apps/chronicle/web/src/App.tsx +++ b/apps/chronicle/web/src/App.tsx @@ -11,6 +11,7 @@ import { ChroniclePage } from './routes/ChroniclePage' import { DataHealthPage } from './routes/DataHealthPage' import { StubPage } from './routes/StubPage' import { Workstation } from './shell/Workstation' +import { TopicsPage } from './topics/TopicsPage' import { WorkspacePage } from './workspaces/WorkspacePage' import { WorkspacesListPage } from './workspaces/WorkspacesListPage' @@ -35,7 +36,7 @@ export function App() { /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/apps/chronicle/web/src/api/types.ts b/apps/chronicle/web/src/api/types.ts index 7d245f5..7f331ca 100644 --- a/apps/chronicle/web/src/api/types.ts +++ b/apps/chronicle/web/src/api/types.ts @@ -204,6 +204,119 @@ export interface TopicsLane { topics: TopicSeries[] } +/** Topic origin (Table 21). */ +export type TopicOrigin = 'automatic' | 'curated' | 'manual' + +/** GET /api/topics tree node. */ +export interface TopicTreeNode { + id: string + label: string + origin: TopicOrigin | string + member_count: number + hidden: boolean + top_terms: string[] + children: TopicTreeNode[] +} + +export interface TopicListResponse { + topics: TopicTreeNode[] +} + +export interface TopicActivityBucket { + bucket: string + count: number +} + +export interface TopicMemberEnvelope { + id: string + subject: string | null + sender_name: string | null + sender_address: string | null + date: string | null + mailbox: string | null + thread_id: string | null + distance: number | null +} + +/** GET /api/topics/{id} */ +export interface TopicDetail { + id: string + label: string + description: string | null + origin: TopicOrigin | string + parent_id: string | null + hidden: boolean + top_terms: string[] + generation: number + member_count: number + created_at: string | null + updated_at: string | null + activity: TopicActivityBucket[] + members: TopicMemberEnvelope[] +} + +export interface TopicPatchRequest { + label?: string + description?: string | null + hidden?: boolean + parent_id?: string | null +} + +/** GET /api/topics/river */ +export interface TopicRiverSeries { + topic_id: string + label: string + origin: TopicOrigin | string + buckets: BucketPoint[] +} + +export interface TopicRiverResponse { + unit: string + mode_hint: 'absolute' | string + from: string | null + to: string | null + topics: TopicRiverSeries[] +} + +/** GET /api/topics/matrix */ +export interface TopicMatrixRow { + topic_id: string + label: string + origin: TopicOrigin | string + cells: Record + row_total: number +} + +export interface TopicMatrixResponse { + by: string + columns: string[] + rows: TopicMatrixRow[] + column_totals: Record + grand_total: number +} + +/** GET /api/topics/projection — topic-level only (TA-003). */ +export interface TopicProjectionPoint { + topic_id: string + label: string + origin: TopicOrigin | string + member_count: number + x: number + y: number +} + +export interface TopicProjectionResponse { + points: TopicProjectionPoint[] + excluded_without_centroid: number + note: string +} + +/** GET /api/topics/{id}/members */ +export interface TopicMembersResponse { + items: TopicMemberEnvelope[] + next_cursor: string | null +} + /** Sparse event diamond mark on the events lane. */ export interface EventLaneMark { event_id: string diff --git a/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx b/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx index 3296879..a6e5691 100644 --- a/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx +++ b/apps/chronicle/web/src/inspector/InspectorPanel.test.tsx @@ -426,4 +426,222 @@ describe('InspectorPanel flow', () => { expect(body.content.title).toMatch(/Hello world/) expect(await screen.findByTestId('pin-status')).toHaveTextContent('Pinned') }) + + const TOPIC_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' + + function topicDetail(overrides: Record = {}) { + return { + id: TOPIC_ID, + label: 'Kitchen remodel', + description: 'home projects', + origin: 'automatic', + parent_id: null, + hidden: false, + top_terms: ['kitchen', 'remodel'], + generation: 1, + member_count: 12, + created_at: null, + updated_at: null, + activity: [ + { bucket: '2020-01-01T00:00:00Z', count: 4 }, + { bucket: '2020-02-01T00:00:00Z', count: 8 }, + ], + members: [ + { + id: 'msg_topic_rep', + subject: 'Quote for cabinets', + sender_name: 'Bob', + sender_address: 'bob@example.com', + date: '2020-01-15T00:00:00Z', + mailbox: 'me@example.com', + thread_id: null, + distance: 0.1, + }, + ], + ...overrides, + } + } + + it('topic selection kind t: shows topic card with origin and rename tooltip', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + if (String(url).includes(`/api/topics/${TOPIC_ID}`)) { + return { + ok: true, + status: 200, + json: async () => topicDetail(), + } as Response + } + throw new Error(`unexpected: ${url}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + expect(await screen.findByTestId('topic-card')).toBeInTheDocument() + expect(screen.getByTestId('topic-origin-badge')).toHaveTextContent(/Automatic/i) + expect(screen.getByTestId('topic-rename-tooltip')).toHaveTextContent( + /flips origin to curated/i, + ) + expect(screen.getByTestId('topic-activity-sparkline')).toBeInTheDocument() + }) + + it('rename → curated flow with tooltip', async () => { + let origin = 'automatic' + let label = 'Kitchen remodel' + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string, init?: RequestInit) => { + const u = String(url) + const method = (init?.method || 'GET').toUpperCase() + if (u.includes(`/api/topics/${TOPIC_ID}`) && method === 'PATCH') { + const body = JSON.parse(String(init?.body || '{}')) as { label?: string } + label = body.label ?? label + origin = 'curated' + return { + ok: true, + status: 200, + json: async () => topicDetail({ label, origin }), + } as Response + } + if (u.includes(`/api/topics/${TOPIC_ID}`)) { + return { + ok: true, + status: 200, + json: async () => topicDetail({ label, origin }), + } as Response + } + throw new Error(`unexpected: ${method} ${u}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + await screen.findByTestId('topic-card') + fireEvent.click(screen.getByTestId('topic-label')) + fireEvent.change(screen.getByTestId('topic-rename-input'), { + target: { value: 'Home kitchen' }, + }) + fireEvent.submit(screen.getByTestId('topic-rename-form')) + await waitFor(() => { + expect(screen.getByTestId('topic-origin-badge')).toHaveTextContent(/Curated/i) + }) + expect(screen.getByTestId('topic-rename-tooltip')).toHaveTextContent(/Curated/i) + }) + + it('representative member click → source (message) selection', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + const u = String(url) + if (u.includes(`/api/topics/${TOPIC_ID}`) && !u.includes('/members')) { + return { + ok: true, + status: 200, + json: async () => topicDetail(), + } as Response + } + if (u.includes('/api/sources/msg_topic_rep')) { + return { + ok: true, + status: 200, + json: async () => ({ + ...msgSource, + envelope: { ...msgSource.envelope, id: 'msg_topic_rep', subject: 'Quote for cabinets' }, + }), + } as Response + } + throw new Error(`unexpected: ${u}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + await screen.findByTestId('topic-card') + fireEvent.click(screen.getByTestId('topic-member-msg_topic_rep')) + await waitFor(() => { + expect(useWorkingSetStore.getState().selection).toEqual({ + kind: 'message', + sid: 'msg_topic_rep', + }) + }) + expect(await screen.findByTestId('message-card')).toBeInTheDocument() + }) + + it('member list pagination (TA-004 definitive list)', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (url: string) => { + const u = String(url) + if (u.includes(`/api/topics/${TOPIC_ID}/members`)) { + if (u.includes('cursor=')) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: 'msg_page2', + subject: 'Second page', + sender_name: null, + sender_address: null, + date: null, + mailbox: null, + thread_id: null, + distance: null, + }, + ], + next_cursor: null, + }), + } as Response + } + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: 'msg_page1', + subject: 'First page', + sender_name: null, + sender_address: null, + date: null, + mailbox: null, + thread_id: null, + distance: null, + }, + ], + next_cursor: 'cursor_abc', + }), + } as Response + } + if (u.includes(`/api/topics/${TOPIC_ID}`)) { + return { + ok: true, + status: 200, + json: async () => topicDetail(), + } as Response + } + throw new Error(`unexpected: ${u}`) + }), + ) + useWorkingSetStore.getState().setSelection({ + kind: 'topic', + topicId: TOPIC_ID, + }) + renderPanel() + await screen.findByTestId('topic-card') + fireEvent.click(screen.getByTestId('topic-open-sources')) + expect(await screen.findByTestId('topic-member-list')).toBeInTheDocument() + expect(screen.getByText('First page')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('topic-members-more')) + expect(await screen.findByText('Second page')).toBeInTheDocument() + }) }) diff --git a/apps/chronicle/web/src/inspector/InspectorPanel.tsx b/apps/chronicle/web/src/inspector/InspectorPanel.tsx index acbba59..2fbb098 100644 --- a/apps/chronicle/web/src/inspector/InspectorPanel.tsx +++ b/apps/chronicle/web/src/inspector/InspectorPanel.tsx @@ -9,6 +9,7 @@ import { downloadUrl } from '../files/format' import { PreviewPanel } from '../files/PreviewPanel' import { useWorkingSetStore } from '../workingset/store' import { formatPeriodLabel, UNIT_MS, type Unit } from '../chronicle/timeScale' +import { TopicCard } from '../topics/TopicCard' import { PinToWorkspace } from '../workspaces/PinToWorkspace' import { MessageCard } from './MessageCard' import { SourceList } from './SourceList' @@ -205,6 +206,15 @@ export function InspectorPanel({ bucketCount }: InspectorPanelProps) { ) } + if (selection.kind === 'topic') { + return ( + setSelection(null)} + /> + ) + } + // Bucket selection const { bucketIso, lane } = selection const dateFrom = bucketIso diff --git a/apps/chronicle/web/src/topics/HierarchyView.tsx b/apps/chronicle/web/src/topics/HierarchyView.tsx new file mode 100644 index 0000000..c90b2d8 --- /dev/null +++ b/apps/chronicle/web/src/topics/HierarchyView.tsx @@ -0,0 +1,308 @@ +import { useCallback, useMemo, useRef, useState, type KeyboardEvent } from 'react' + +import type { TopicTreeNode } from '../api/types' +import { OriginBadge } from './originBadge' + +export interface HierarchyViewProps { + topics: TopicTreeNode[] + selectedId: string | null + onSelect: (id: string) => void +} + +type FlatNode = { + id: string + label: string + origin: string + member_count: number + hidden: boolean + depth: number + hasChildren: boolean + /** Synthetic group for hidden topics. */ + isHiddenGroup?: boolean +} + +function flattenVisible( + nodes: TopicTreeNode[], + expanded: Set, + depth: number, +): FlatNode[] { + const out: FlatNode[] = [] + for (const n of nodes) { + const hasChildren = (n.children?.length ?? 0) > 0 + out.push({ + id: n.id, + label: n.label, + origin: n.origin, + member_count: n.member_count, + hidden: n.hidden, + depth, + hasChildren, + }) + if (hasChildren && expanded.has(n.id)) { + out.push(...flattenVisible(n.children, expanded, depth + 1)) + } + } + return out +} + +function filterTree(nodes: TopicTreeNode[], q: string): TopicTreeNode[] { + if (!q) return nodes + const lower = q.toLowerCase() + const walk = (list: TopicTreeNode[]): TopicTreeNode[] => { + const out: TopicTreeNode[] = [] + for (const n of list) { + const kids = walk(n.children ?? []) + const selfMatch = + n.label.toLowerCase().includes(lower) || + (n.top_terms ?? []).some((t) => t.toLowerCase().includes(lower)) + if (selfMatch || kids.length > 0) { + out.push({ ...n, children: kids }) + } + } + return out + } + return walk(nodes) +} + +/** + * Keyboard-navigable topic hierarchy (roles tree/treeitem; WAI-ARIA arrows). + * Hidden topics under a collapsed "Hidden (N)" group. TA-001 default view. + */ +export function HierarchyView({ topics, selectedId, onSelect }: HierarchyViewProps) { + const [filter, setFilter] = useState('') + const [expanded, setExpanded] = useState>(() => new Set()) + const [hiddenOpen, setHiddenOpen] = useState(false) + const [focusId, setFocusId] = useState(null) + const treeRef = useRef(null) + + const { visible, hidden } = useMemo(() => { + const filtered = filterTree(topics, filter.trim()) + const vis: TopicTreeNode[] = [] + const hid: TopicTreeNode[] = [] + const split = (list: TopicTreeNode[]) => { + for (const n of list) { + if (n.hidden) hid.push(n) + else vis.push({ ...n, children: (n.children ?? []).filter((c) => !c.hidden) }) + } + } + split(filtered) + // Also collect nested hidden from original filtered tree. + const collectHidden = (list: TopicTreeNode[]) => { + for (const n of list) { + if (n.hidden && !hid.some((h) => h.id === n.id)) hid.push(n) + collectHidden(n.children ?? []) + } + } + collectHidden(filtered) + return { visible: vis, hidden: hid } + }, [topics, filter]) + + const flat = useMemo(() => { + const rows = flattenVisible(visible, expanded, 0) + if (hidden.length > 0) { + const groupId = '__hidden__' + rows.push({ + id: groupId, + label: `Hidden (${hidden.length})`, + origin: '', + member_count: hidden.length, + hidden: true, + depth: 0, + hasChildren: true, + isHiddenGroup: true, + }) + if (hiddenOpen) { + for (const h of hidden) { + rows.push({ + id: h.id, + label: h.label, + origin: h.origin, + member_count: h.member_count, + hidden: true, + depth: 1, + hasChildren: false, + }) + } + } + } + return rows + }, [visible, hidden, expanded, hiddenOpen]) + + const toggleExpand = useCallback((id: string, isHiddenGroup?: boolean) => { + if (isHiddenGroup) { + setHiddenOpen((v) => !v) + return + } + setExpanded((prev) => { + const next = new Set(prev) + if (next.has(id)) next.delete(id) + else next.add(id) + return next + }) + }, []) + + const focusIndex = flat.findIndex((r) => r.id === (focusId ?? selectedId)) + const safeFocus = focusIndex >= 0 ? focusIndex : 0 + + const onKeyDown = (e: KeyboardEvent) => { + if (flat.length === 0) return + const current = flat[safeFocus] + if (!current) return + + if (e.key === 'ArrowDown') { + e.preventDefault() + const next = flat[Math.min(flat.length - 1, safeFocus + 1)] + if (next) setFocusId(next.id) + } else if (e.key === 'ArrowUp') { + e.preventDefault() + const prev = flat[Math.max(0, safeFocus - 1)] + if (prev) setFocusId(prev.id) + } else if (e.key === 'ArrowRight') { + e.preventDefault() + if (current.hasChildren) { + if (current.isHiddenGroup) { + if (!hiddenOpen) setHiddenOpen(true) + else { + const child = flat[safeFocus + 1] + if (child) setFocusId(child.id) + } + } else if (!expanded.has(current.id)) { + setExpanded((prev) => new Set(prev).add(current.id)) + } else { + const child = flat[safeFocus + 1] + if (child && child.depth > current.depth) setFocusId(child.id) + } + } + } else if (e.key === 'ArrowLeft') { + e.preventDefault() + if (current.isHiddenGroup && hiddenOpen) { + setHiddenOpen(false) + } else if (current.hasChildren && expanded.has(current.id)) { + setExpanded((prev) => { + const next = new Set(prev) + next.delete(current.id) + return next + }) + } else if (current.depth > 0) { + // Move to parent: previous row with smaller depth. + for (let i = safeFocus - 1; i >= 0; i--) { + if (flat[i]!.depth < current.depth) { + setFocusId(flat[i]!.id) + break + } + } + } + } else if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + if (current.isHiddenGroup) { + setHiddenOpen((v) => !v) + } else { + onSelect(current.id) + } + } else if (e.key === 'Home') { + e.preventDefault() + if (flat[0]) setFocusId(flat[0].id) + } else if (e.key === 'End') { + e.preventDefault() + const last = flat[flat.length - 1] + if (last) setFocusId(last.id) + } + } + + return ( +
+ +
+ {flat.length === 0 ? ( +

No topics

+ ) : ( +
    + {flat.map((row) => { + const isExpanded = row.isHiddenGroup + ? hiddenOpen + : expanded.has(row.id) + const isSelected = !row.isHiddenGroup && row.id === selectedId + const isFocused = row.id === (focusId ?? selectedId ?? flat[0]?.id) + return ( +
  • { + setFocusId(row.id) + if (row.isHiddenGroup) toggleExpand(row.id, true) + else onSelect(row.id) + }} + onDoubleClick={() => { + if (row.hasChildren) toggleExpand(row.id, row.isHiddenGroup) + }} + > + {row.hasChildren ? ( + + ) : ( + + )} + + {row.label} + + {!row.isHiddenGroup && row.origin ? ( + + ) : null} + + {row.member_count.toLocaleString()} + +
    • + ) + })} +
+ )} +
+
+ ) +} diff --git a/apps/chronicle/web/src/topics/MatrixView.tsx b/apps/chronicle/web/src/topics/MatrixView.tsx new file mode 100644 index 0000000..0bec964 --- /dev/null +++ b/apps/chronicle/web/src/topics/MatrixView.tsx @@ -0,0 +1,202 @@ +import { useMemo, useState } from 'react' + +import type { TopicMatrixResponse } from '../api/types' +import { OriginBadge } from './originBadge' + +export interface MatrixViewProps { + data: TopicMatrixResponse | undefined + loading?: boolean + selectedId: string | null + onSelect: (id: string) => void +} + +type SortKey = 'label' | 'row_total' | string + +/** + * Topic × year matrix — accessible table with sort + normalize-per-row. + * This IS the accessible alternative for itself (Table 22). + */ +export function MatrixView({ data, loading, selectedId, onSelect }: MatrixViewProps) { + const [normalize, setNormalize] = useState(false) + const [sortKey, setSortKey] = useState('row_total') + const [sortDir, setSortDir] = useState<'asc' | 'desc'>('desc') + + const columns = data?.columns ?? [] + const rows = useMemo(() => { + const list = [...(data?.rows ?? [])] + list.sort((a, b) => { + let av: string | number + let bv: string | number + if (sortKey === 'label') { + av = a.label + bv = b.label + } else if (sortKey === 'row_total') { + av = a.row_total + bv = b.row_total + } else { + av = a.cells[sortKey] ?? 0 + bv = b.cells[sortKey] ?? 0 + } + if (typeof av === 'string' && typeof bv === 'string') { + return sortDir === 'asc' ? av.localeCompare(bv) : bv.localeCompare(av) + } + const an = Number(av) + const bn = Number(bv) + return sortDir === 'asc' ? an - bn : bn - an + }) + return list + }, [data?.rows, sortKey, sortDir]) + + const toggleSort = (key: SortKey) => { + if (sortKey === key) setSortDir((d) => (d === 'asc' ? 'desc' : 'asc')) + else { + setSortKey(key) + setSortDir(key === 'label' ? 'asc' : 'desc') + } + } + + if (loading) { + return ( +

+ Loading matrix… +

+ ) + } + + if (!data || rows.length === 0) { + return ( +

+ No topic×year data +

+ ) + } + + return ( +
+
+ + + Total: {data.grand_total.toLocaleString()} + +
+
+ + + + + + + {columns.map((col) => ( + + ))} + + + + + {rows.map((row) => { + const denom = row.row_total > 0 ? row.row_total : 1 + return ( + onSelect(row.topic_id)} + data-testid={`matrix-row-${row.topic_id}`} + > + + + {columns.map((col) => { + const raw = row.cells[col] ?? 0 + const display = normalize ? raw / denom : raw + return ( + + ) + })} + + + ) + })} + + + + + + ))} + + + +
+ Topic by year member counts. Sortable columns. Values are + {normalize ? ' row-normalized shares' : ' absolute counts'}. +
+ + + Origin + + + + +
+ {row.label} + + + { + e.stopPropagation() + onSelect(row.topic_id) + }} + > + {normalize ? display.toFixed(2) : raw.toLocaleString()} + + {row.row_total.toLocaleString()} +
+ Column total + + {columns.map((col) => ( + + {(data.column_totals[col] ?? 0).toLocaleString()} + + {data.grand_total.toLocaleString()} +
+
+
+ ) +} diff --git a/apps/chronicle/web/src/topics/ProjectionView.tsx b/apps/chronicle/web/src/topics/ProjectionView.tsx new file mode 100644 index 0000000..6177945 --- /dev/null +++ b/apps/chronicle/web/src/topics/ProjectionView.tsx @@ -0,0 +1,197 @@ +import { useMemo, useState } from 'react' + +import type { TopicProjectionResponse } from '../api/types' +import { TOPIC_PURPLE } from '../chronicle/laneModel' +import { OriginBadge } from './originBadge' + +export interface ProjectionViewProps { + data: TopicProjectionResponse | undefined + loading?: boolean + selectedId: string | null + onSelect: (id: string) => void +} + +const DISCLAIMER = + 'Exploratory projection — positions change when embeddings or clustering change; not an objective map of meaning' + +/** + * Centroid-level PCA scatter (SVG, not canvas). Topic circles only — never + * individual sources (TA-003). Zoom not required. + */ +export function ProjectionView({ + data, + loading, + selectedId, + onSelect, +}: ProjectionViewProps) { + const [asList, setAsList] = useState(false) + const [hoverId, setHoverId] = useState(null) + + const points = data?.points ?? [] + + const sized = useMemo(() => { + const maxCount = points.reduce((m, p) => Math.max(m, p.member_count), 0) || 1 + return points.map((p) => ({ + ...p, + // sqrt scale for area-like sizing + r: 4 + 12 * Math.sqrt(p.member_count / maxCount), + })) + }, [points]) + + if (loading) { + return ( +

+ Loading projection… +

+ ) + } + + return ( +
+

+ {DISCLAIMER} +

+ {data?.note ? ( +

+ {data.note} +

+ ) : null} +
+ + + {points.length} topic{points.length === 1 ? '' : 's'} + +
+ + {points.length === 0 ? ( +

+ No centroid-backed topics to project +

+ ) : asList ? ( +
+ + + + + + + + + + + + + {points.map((p) => ( + onSelect(p.topic_id)} + data-testid={`projection-list-row-${p.topic_id}`} + > + + + + + + + ))} + +
Topic projection as list
+ Topic + + Origin + + Members + + x + + y +
+ {p.label} + + + {p.member_count}{p.x.toFixed(3)}{p.y.toFixed(3)}
+
+ ) : ( + + {/* axes crosshair */} + + + {sized.map((p) => { + const cx = 200 + p.x * 160 + const cy = 160 - p.y * 130 + const selected = p.topic_id === selectedId + const hovered = p.topic_id === hoverId + return ( + + onSelect(p.topic_id)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + onSelect(p.topic_id) + } + }} + onFocus={() => setHoverId(p.topic_id)} + onBlur={() => setHoverId((id) => (id === p.topic_id ? null : id))} + onMouseEnter={() => setHoverId(p.topic_id)} + onMouseLeave={() => + setHoverId((id) => (id === p.topic_id ? null : id)) + } + > + + {p.label} · {p.origin} · {p.member_count} + + + {(hovered || selected) && ( + + {p.label} + + )} + + ) + })} + + )} +
+ ) +} diff --git a/apps/chronicle/web/src/topics/RiverView.tsx b/apps/chronicle/web/src/topics/RiverView.tsx new file mode 100644 index 0000000..6c5c3f2 --- /dev/null +++ b/apps/chronicle/web/src/topics/RiverView.tsx @@ -0,0 +1,229 @@ +import { useMemo, useState } from 'react' + +import type { TopicRiverResponse, TopicRiverSeries } from '../api/types' +import { TOPIC_PURPLE } from '../chronicle/laneModel' + +export interface RiverViewProps { + data: TopicRiverResponse | undefined + loading?: boolean + selectedId: string | null + onSelect: (id: string) => void +} + +/** + * Topic river: area-less multirow bands (thin wrapper over multirow series shape). + * Absolute | normalized toggle with explicit legend (Table 22 river safeguards). + */ +export function RiverView({ data, loading, selectedId, onSelect }: RiverViewProps) { + const [normalized, setNormalized] = useState(false) + const [asTable, setAsTable] = useState(false) + + const topics = data?.topics ?? [] + + const buckets = useMemo(() => { + const set = new Set() + for (const t of topics) { + for (const b of t.buckets) set.add(b.bucket) + } + return Array.from(set).sort() + }, [topics]) + + const series = useMemo(() => { + return topics.map((t) => { + const byBucket = new Map(t.buckets.map((b) => [b.bucket, b.count])) + const counts = buckets.map((b) => byBucket.get(b) ?? 0) + const total = counts.reduce((a, c) => a + c, 0) + const values = normalized + ? counts.map((c) => (total > 0 ? c / total : 0)) + : counts + const peak = values.reduce((m, v) => Math.max(m, v), 0) + return { ...t, values, peak, total } + }) + }, [topics, buckets, normalized]) + + if (loading) { + return ( +
+

+ Loading river… +

+
+ ) + } + + if (topics.length === 0) { + return ( +
+

+ No topic activity in this range +

+
+ ) + } + + return ( +
+
+
+ + +
+

+ {normalized + ? 'Legend: each band is share of that topic’s activity (0–1 per topic)' + : 'Legend: absolute message counts per bucket (mode_hint=absolute)'} + {data?.unit ? ` · unit=${data.unit}` : ''} +

+ +
+ + {asTable ? ( + + ) : ( +
+ {series.map((t) => ( + onSelect(t.topic_id)} + /> + ))} +
+ )} +
+ ) +} + +function RiverBand({ + series, + values, + peak, + selected, + onSelect, +}: { + series: TopicRiverSeries & { values: number[]; peak: number } + values: number[] + peak: number + selected: boolean + onSelect: () => void +}) { + const max = peak > 0 ? peak : 1 + return ( + + ) +} + +function RiverTable({ + series, + buckets, + normalized, + onSelect, + selectedId, +}: { + series: Array + buckets: string[] + normalized: boolean + onSelect: (id: string) => void + selectedId: string | null +}) { + return ( +
+ + + + + + {buckets.map((b) => ( + + ))} + + + + {series.map((t) => ( + onSelect(t.topic_id)} + > + + {t.values.map((v, i) => ( + + ))} + + ))} + +
Topic river as table
+ Topic + + {b.slice(0, 10)} +
+ {t.label} + + {normalized ? v.toFixed(2) : v} +
+
+ ) +} diff --git a/apps/chronicle/web/src/topics/TopicCard.tsx b/apps/chronicle/web/src/topics/TopicCard.tsx new file mode 100644 index 0000000..f4b771a --- /dev/null +++ b/apps/chronicle/web/src/topics/TopicCard.tsx @@ -0,0 +1,354 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { useNavigate } from 'react-router' + +import type { TopicDetail, TopicMemberEnvelope } from '../api/types' +import { encodeState, toIsoSeconds } from '../workingset/urlState' +import { useWorkingSetStore } from '../workingset/store' +import { getTopic, listTopicMembers, patchTopic } from './api' +import { OriginBadge } from './originBadge' + +export interface TopicCardProps { + topicId: string + onClose?: () => void +} + +const RENAME_TOOLTIP = + 'Renaming an automatic topic flips its origin to curated. Manual changes take precedence over regeneration (Table 21 / TA-002).' + +/** + * Inspector topic card: rename (→ curated), hide/unhide, activity sparkline, + * representative members → source selection, definitive member list (TA-004), + * Open in Chronicle (viewport + topics lane). + * + * "Open sources in Research" is deferred: Research Desk topic scoping needs a + * QueryScope.topics field (forbidden in scope.py this task). Instead, Open + * sources opens the paginated member list inside the Atlas via + * GET /api/topics/{id}/members (TA-004 definitive list). Phase 5 polish adds + * Research-lens topic scoping. + */ +export function TopicCard({ topicId, onClose }: TopicCardProps) { + const setSelection = useWorkingSetStore((s) => s.setSelection) + const navigate = useNavigate() + const qc = useQueryClient() + const [editing, setEditing] = useState(false) + const [labelDraft, setLabelDraft] = useState('') + const [showMembers, setShowMembers] = useState(false) + const [memberItems, setMemberItems] = useState([]) + const [memberCursor, setMemberCursor] = useState(null) + const [membersLoading, setMembersLoading] = useState(false) + + const query = useQuery({ + queryKey: ['topics', topicId], + queryFn: ({ signal }) => getTopic(topicId, signal), + retry: false, + }) + + useEffect(() => { + if (query.data) setLabelDraft(query.data.label) + }, [query.data]) + + const patchMut = useMutation({ + mutationFn: (body: { label?: string; hidden?: boolean; description?: string | null }) => + patchTopic(topicId, body), + onSuccess: (data) => { + void qc.invalidateQueries({ queryKey: ['topics'] }) + void qc.setQueryData(['topics', topicId], data) + setEditing(false) + }, + }) + + const detail = query.data + + const loadMembers = useCallback( + async (cursor: string | null, append: boolean) => { + setMembersLoading(true) + try { + const res = await listTopicMembers(topicId, { cursor, limit: 25 }) + setMemberItems((prev) => (append ? [...prev, ...res.items] : res.items)) + setMemberCursor(res.next_cursor) + } finally { + setMembersLoading(false) + } + }, + [topicId], + ) + + const openMembers = useCallback(() => { + setShowMembers(true) + void loadMembers(null, false) + }, [loadMembers]) + + const openInChronicle = useCallback( + (d: TopicDetail) => { + // MVP: set viewport from activity date extent and enable topics lane. + // Topics are not a QueryScope field yet. + const activity = d.activity ?? [] + let fromMs: number + let toMs: number + if (activity.length > 0) { + const times = activity + .map((a) => Date.parse(a.bucket)) + .filter((t) => Number.isFinite(t)) + fromMs = Math.min(...times) + toMs = Math.max(...times) + 30 * 24 * 3600 * 1000 // pad one month + } else { + const now = Date.now() + fromMs = now - 365 * 24 * 3600 * 1000 + toMs = now + } + const scope = useWorkingSetStore.getState().scope + const params = encodeState({ + scope, + viewport: { fromMs, toMs }, + aggregation: 'auto', + view: 'canvas', + selection: null, + lanes: ['messages', 'topics'], + }) + // Ensure vf/vt present even if encode uses second precision. + if (!params.get('vf')) params.set('vf', toIsoSeconds(fromMs)) + if (!params.get('vt')) params.set('vt', toIsoSeconds(toMs)) + const qs = params.toString() + void navigate({ pathname: '/', search: qs ? `?${qs}` : '' }) + }, + [navigate], + ) + + const sparkline = useMemo(() => { + if (!detail?.activity?.length) return null + const counts = detail.activity.map((a) => a.count) + const max = Math.max(...counts, 1) + return counts.map((c, i) => ({ + i, + h: Math.max(2, (c / max) * 28), + })) + }, [detail?.activity]) + + if (query.isLoading) { + return ( +
+
+
+
+ ) + } + + if (query.isError || !detail) { + return ( +
+

Failed to load topic

+ +
+ ) + } + + return ( +
+
+
+ {editing ? ( +
{ + e.preventDefault() + const label = labelDraft.trim() + if (!label || label === detail.label) { + setEditing(false) + return + } + patchMut.mutate({ label }) + }} + data-testid="topic-rename-form" + > + setLabelDraft(e.target.value)} + className="min-w-0 flex-1 rounded border border-steel bg-graphite-900 px-1.5 py-0.5 text-sm text-text-primary" + data-testid="topic-rename-input" + autoFocus + aria-label="Topic label" + /> + + +
+ ) : ( + + )} + {detail.origin === 'automatic' || detail.origin === 'curated' ? ( +

+ {detail.origin === 'automatic' + ? 'Rename flips origin to curated' + : 'Curated — manual changes preserved on regenerate'} +

+ ) : null} +
+ +
+ + {detail.description ? ( +

+ {detail.description} +

+ ) : null} + +

+ {detail.member_count.toLocaleString()} members +

+ + {detail.top_terms?.length ? ( +

+ Terms: {detail.top_terms.join(', ')} +

+ ) : null} + + {sparkline ? ( +
+ {sparkline.map((s) => ( +
+ ))} +
+ ) : null} + +
+

Representative members

+
    + {(detail.members ?? []).map((m) => ( +
  • + +
    • + ))} + {(detail.members ?? []).length === 0 ? ( +
  • No members
    • + ) : null} +
+
+ +
+ + + + {onClose ? ( + + ) : null} +
+ + {showMembers ? ( +
+

+ All members (definitive list · TA-004) +

+
    + {memberItems.map((m) => ( +
  • + +
    • + ))} +
+ {memberCursor ? ( + + ) : null} +
+ ) : null} +
+ ) +} diff --git a/apps/chronicle/web/src/topics/TopicsPage.test.tsx b/apps/chronicle/web/src/topics/TopicsPage.test.tsx new file mode 100644 index 0000000..d229c68 --- /dev/null +++ b/apps/chronicle/web/src/topics/TopicsPage.test.tsx @@ -0,0 +1,286 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { resetWorkingSetStore, useWorkingSetStore } from '../workingset/store' +import { TopicsPage } from './TopicsPage' +import { HierarchyView } from './HierarchyView' +import { RiverView } from './RiverView' +import { MatrixView } from './MatrixView' +import { ProjectionView } from './ProjectionView' +import type { + TopicMatrixResponse, + TopicProjectionResponse, + TopicRiverResponse, + TopicTreeNode, +} from '../api/types' + +const TOPIC_A = '11111111-1111-1111-1111-111111111111' +const TOPIC_B = '22222222-2222-2222-2222-222222222222' +const TOPIC_HIDDEN = '33333333-3333-3333-3333-333333333333' +const TOPIC_CHILD = '44444444-4444-4444-4444-444444444444' + +const tree: TopicTreeNode[] = [ + { + id: TOPIC_A, + label: 'Kitchen remodel', + origin: 'automatic', + member_count: 42, + hidden: false, + top_terms: ['kitchen', 'remodel'], + children: [ + { + id: TOPIC_CHILD, + label: 'Cabinets', + origin: 'curated', + member_count: 8, + hidden: false, + top_terms: ['cabinet'], + children: [], + }, + ], + }, + { + id: TOPIC_B, + label: 'Travel plans', + origin: 'manual', + member_count: 12, + hidden: false, + top_terms: ['travel'], + children: [], + }, + { + id: TOPIC_HIDDEN, + label: 'Spam cluster', + origin: 'automatic', + member_count: 3, + hidden: true, + top_terms: ['spam'], + children: [], + }, +] + +function mockFetch(handlers: Record) { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + for (const [key, body] of Object.entries(handlers)) { + if (url.includes(key)) { + return { + ok: true, + status: 200, + json: async () => body, + } as Response + } + } + return { + ok: false, + status: 404, + json: async () => ({ detail: 'not found' }), + } as Response + }), + ) +} + +function renderPage(initial = '/topics') { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + return render( + + + + } /> + + + , + ) +} + +describe('TopicsPage / Topic Atlas views', () => { + beforeEach(() => { + resetWorkingSetStore() + }) + + afterEach(() => { + vi.unstubAllGlobals() + resetWorkingSetStore() + }) + + it('defaults to hierarchy view (TA-001)', async () => { + mockFetch({ '/api/topics': { topics: tree } }) + renderPage('/topics') + expect(await screen.findByTestId('hierarchy-view')).toBeInTheDocument() + expect(screen.getByTestId('topic-view-hierarchy')).toHaveAttribute( + 'aria-selected', + 'true', + ) + }) + + it('hierarchy shows origin badges and member counts', async () => { + mockFetch({ '/api/topics': { topics: tree } }) + renderPage('/topics') + await screen.findByTestId('hierarchy-view') + const badges = screen.getAllByTestId('topic-origin-badge') + expect(badges.some((b) => b.textContent?.match(/Automatic/i))).toBe(true) + expect(badges.some((b) => b.textContent?.match(/Manual/i))).toBe(true) + expect(screen.getAllByTestId('topic-member-count').length).toBeGreaterThan(0) + }) + + it('hidden topics under collapsed Hidden (N) group', async () => { + mockFetch({ '/api/topics': { topics: tree } }) + renderPage('/topics') + await screen.findByTestId('topic-hidden-group') + expect(screen.getByTestId('topic-hidden-group')).toHaveTextContent(/Hidden \(1\)/) + expect(screen.queryByTestId(`topic-treeitem-${TOPIC_HIDDEN}`)).not.toBeInTheDocument() + fireEvent.click(screen.getByTestId('topic-hidden-group')) + expect(screen.getByTestId(`topic-treeitem-${TOPIC_HIDDEN}`)).toBeInTheDocument() + }) + + it('keyboard arrow-right expands into children (WAI-ARIA tree)', async () => { + const onSelect = vi.fn() + render( + , + ) + const treeEl = screen.getByTestId('topic-tree') + treeEl.focus() + // Focus first node (Kitchen) which has children + fireEvent.keyDown(treeEl, { key: 'ArrowRight' }) + // Expanded — child should appear + expect(await screen.findByTestId(`topic-treeitem-${TOPIC_CHILD}`)).toBeInTheDocument() + fireEvent.keyDown(treeEl, { key: 'ArrowDown' }) + fireEvent.keyDown(treeEl, { key: 'Enter' }) + expect(onSelect).toHaveBeenCalled() + }) + + it('selecting a node sets selection kind topic', async () => { + mockFetch({ '/api/topics': { topics: tree } }) + renderPage('/topics') + await screen.findByTestId(`topic-treeitem-${TOPIC_A}`) + fireEvent.click(screen.getByTestId(`topic-treeitem-${TOPIC_A}`)) + await waitFor(() => { + const sel = useWorkingSetStore.getState().selection + expect(sel).toEqual({ kind: 'topic', topicId: TOPIC_A }) + }) + }) + + it('river normalize legend and absolute/normalized toggle', () => { + const data: TopicRiverResponse = { + unit: 'month', + mode_hint: 'absolute', + from: '2020-01-01T00:00:00Z', + to: '2021-01-01T00:00:00Z', + topics: [ + { + topic_id: TOPIC_A, + label: 'Kitchen', + origin: 'automatic', + buckets: [ + { bucket: '2020-01-01T00:00:00Z', count: 10 }, + { bucket: '2020-02-01T00:00:00Z', count: 5 }, + ], + }, + ], + } + render( + , + ) + expect(screen.getByTestId('river-legend')).toHaveTextContent(/absolute/i) + fireEvent.click(screen.getByRole('button', { name: /Normalized/i })) + expect(screen.getByTestId('river-legend')).toHaveTextContent(/share/i) + }) + + it('matrix sort and normalize-per-row', () => { + const data: TopicMatrixResponse = { + by: 'year', + columns: ['2020', '2021'], + rows: [ + { + topic_id: TOPIC_A, + label: 'Alpha', + origin: 'automatic', + cells: { '2020': 10, '2021': 0 }, + row_total: 10, + }, + { + topic_id: TOPIC_B, + label: 'Beta', + origin: 'curated', + cells: { '2020': 2, '2021': 8 }, + row_total: 10, + }, + ], + column_totals: { '2020': 12, '2021': 8 }, + grand_total: 20, + } + render( + , + ) + expect(screen.getByTestId('matrix-view')).toBeInTheDocument() + // Default sort row_total desc — both equal, labels stable enough + fireEvent.click(screen.getByTestId('matrix-sort-label')) + const rows = screen.getAllByTestId(/matrix-row-/) + expect(rows[0]).toHaveTextContent('Alpha') + fireEvent.click(screen.getByTestId('matrix-normalize-toggle')) + // 10/10 = 1.00 for Alpha 2020 + expect(screen.getByTestId(`matrix-cell-${TOPIC_A}-2020`)).toHaveTextContent('1.00') + }) + + it('projection disclaimer and focusable points (TA-003)', () => { + const data: TopicProjectionResponse = { + points: [ + { + topic_id: TOPIC_A, + label: 'Kitchen', + origin: 'automatic', + member_count: 40, + x: 0.5, + y: -0.2, + }, + ], + excluded_without_centroid: 1, + note: '1 topic(s) excluded; topic-level only (TA-003).', + } + render( + , + ) + expect(screen.getByTestId('projection-disclaimer')).toHaveTextContent( + /Exploratory projection/i, + ) + const point = screen.getByTestId(`projection-point-${TOPIC_A}`) + expect(point).toHaveAttribute('tabindex', '0') + expect(point).toHaveAttribute('aria-label') + // No source points — only topic circles. + expect(screen.queryByTestId(/projection-source/)).not.toBeInTheDocument() + }) + + it('tv param selects river view', async () => { + mockFetch({ + '/api/topics/river': { + unit: 'month', + mode_hint: 'absolute', + from: null, + to: null, + topics: [], + }, + '/api/topics': { topics: tree }, + }) + renderPage('/topics?tv=river') + expect(await screen.findByTestId('river-view')).toBeInTheDocument() + }) +}) + +describe('HierarchyView filter', () => { + it('filters tree client-side', () => { + render( + , + ) + fireEvent.change(screen.getByTestId('topic-tree-filter'), { + target: { value: 'Travel' }, + }) + expect(screen.getByTestId(`topic-treeitem-${TOPIC_B}`)).toBeInTheDocument() + expect(screen.queryByTestId(`topic-treeitem-${TOPIC_A}`)).not.toBeInTheDocument() + }) +}) diff --git a/apps/chronicle/web/src/topics/TopicsPage.tsx b/apps/chronicle/web/src/topics/TopicsPage.tsx new file mode 100644 index 0000000..baad563 --- /dev/null +++ b/apps/chronicle/web/src/topics/TopicsPage.tsx @@ -0,0 +1,202 @@ +import { useQuery } from '@tanstack/react-query' +import { useCallback, useMemo } from 'react' +import { useSearchParams } from 'react-router' + +import type { QueryScope } from '../api/types' +import { toIsoSeconds } from '../workingset/urlState' +import type { TopicViewMode } from '../workingset/urlState' +import { parseTopicView } from '../workingset/urlState' +import { useWorkingSetStore } from '../workingset/store' +import { + getTopicMatrix, + getTopicProjection, + getTopicRiver, + listTopics, +} from './api' +import { HierarchyView } from './HierarchyView' +import { MatrixView } from './MatrixView' +import { ProjectionView } from './ProjectionView' +import { RiverView } from './RiverView' + +const VIEWS: { id: TopicViewMode; label: string }[] = [ + { id: 'hierarchy', label: 'Hierarchy' }, + { id: 'river', label: 'River' }, + { id: 'matrix', label: 'Matrix' }, + { id: 'projection', label: 'Projection' }, +] + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[12px] text-text-primary enabled:hover:bg-graphite-900 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +/** + * Topic Atlas lens (config rail + canvas + inspector selection). + * Working set (scope bar) is untouched — G-002. + */ +export function TopicsPage() { + const scope = useWorkingSetStore((s) => s.scope) + const viewport = useWorkingSetStore((s) => s.viewport) + const setSelection = useWorkingSetStore((s) => s.setSelection) + const [searchParams, setSearchParams] = useSearchParams() + + const topicView = parseTopicView(searchParams.get('tv')) + const tsel = searchParams.get('tsel') + + const setTopicView = useCallback( + (view: TopicViewMode) => { + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev) + if (view === 'hierarchy') next.delete('tv') + else next.set('tv', view) + return next + }, + { replace: true }, + ) + }, + [setSearchParams], + ) + + const selectTopic = useCallback( + (id: string) => { + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev) + next.set('tsel', id) + return next + }, + { replace: true }, + ) + setSelection({ kind: 'topic', topicId: id }) + }, + [setSearchParams, setSelection], + ) + + const listQuery = useQuery({ + queryKey: ['topics', 'list', true], + queryFn: ({ signal }) => listTopics(true, signal), + retry: false, + }) + + const riverRange = useMemo(() => { + if (viewport && viewport.toMs > viewport.fromMs) { + return { + from: toIsoSeconds(viewport.fromMs), + to: toIsoSeconds(viewport.toMs), + } + } + // Fall back to scope date or last decade. + const from = scope.date?.from + ? `${scope.date.from}T00:00:00Z` + : toIsoSeconds(Date.UTC(2010, 0, 1)) + const to = scope.date?.to + ? `${scope.date.to}T00:00:00Z` + : toIsoSeconds(Date.now()) + return { from, to } + }, [viewport, scope.date?.from, scope.date?.to]) + + const riverQuery = useQuery({ + queryKey: ['topics', 'river', riverRange, scope], + queryFn: ({ signal }) => + getTopicRiver( + { + from: riverRange.from, + to: riverRange.to, + unit: 'auto', + top: 8, + scope: scope as QueryScope, + }, + signal, + ), + enabled: topicView === 'river', + retry: false, + }) + + const matrixQuery = useQuery({ + queryKey: ['topics', 'matrix', scope], + queryFn: ({ signal }) => getTopicMatrix({ by: 'year', scope }, signal), + enabled: topicView === 'matrix', + retry: false, + }) + + const projectionQuery = useQuery({ + queryKey: ['topics', 'projection'], + queryFn: ({ signal }) => getTopicProjection(signal), + enabled: topicView === 'projection', + retry: false, + }) + + return ( +
+
+

Topic Atlas

+
+ {VIEWS.map((v) => ( + + ))} +
+
+ +
+ {topicView === 'hierarchy' && ( + listQuery.isLoading ? ( +

Loading topics…

+ ) : listQuery.isError ? ( +

+ Failed to load topics +

+ ) : ( + + ) + )} + {topicView === 'river' && ( + + )} + {topicView === 'matrix' && ( + + )} + {topicView === 'projection' && ( + + )} +
+
+ ) +} diff --git a/apps/chronicle/web/src/topics/api.ts b/apps/chronicle/web/src/topics/api.ts new file mode 100644 index 0000000..e2a6ebb --- /dev/null +++ b/apps/chronicle/web/src/topics/api.ts @@ -0,0 +1,111 @@ +/** + * Topics Atlas API helpers (authenticated fetch wrappers). + */ + +import { apiFetch, apiGet } from '../api/client' +import type { + TopicDetail, + TopicListResponse, + TopicMatrixResponse, + TopicMembersResponse, + TopicPatchRequest, + TopicProjectionResponse, + TopicRiverResponse, +} from '../api/types' +import type { QueryScope } from '../api/types' + +function scopeQuery(scope: QueryScope | undefined): Record { + const q: Record = {} + if (!scope) return q + if (scope.mailboxes?.length) q.mb = scope.mailboxes.join(',') + if (scope.senders?.length) q.sd = scope.senders.join(',') + if (scope.date?.from) q.df = scope.date.from + if (scope.date?.to) q.dt = scope.date.to + return q +} + +function toSearch(params: Record): string { + const sp = new URLSearchParams() + for (const [k, v] of Object.entries(params)) { + if (v == null || v === '') continue + sp.set(k, String(v)) + } + const s = sp.toString() + return s ? `?${s}` : '' +} + +export function listTopics( + includeHidden = true, + signal?: AbortSignal, +): Promise { + const q = includeHidden ? '?include_hidden=true' : '' + return apiGet(`/api/topics${q}`, signal) +} + +export function getTopic(id: string, signal?: AbortSignal): Promise { + return apiGet(`/api/topics/${encodeURIComponent(id)}`, signal) +} + +export function patchTopic( + id: string, + body: TopicPatchRequest, + signal?: AbortSignal, +): Promise { + return apiFetch(`/api/topics/${encodeURIComponent(id)}`, { + method: 'PATCH', + body: JSON.stringify(body), + signal, + }) +} + +export function getTopicRiver( + opts: { + from: string + to: string + unit?: string + top?: number + scope?: QueryScope + }, + signal?: AbortSignal, +): Promise { + const q = toSearch({ + from: opts.from, + to: opts.to, + unit: opts.unit ?? 'auto', + top: opts.top ?? 8, + ...scopeQuery(opts.scope), + }) + return apiGet(`/api/topics/river${q}`, signal) +} + +export function getTopicMatrix( + opts: { by?: string; scope?: QueryScope } = {}, + signal?: AbortSignal, +): Promise { + const q = toSearch({ + by: opts.by ?? 'year', + ...scopeQuery(opts.scope), + }) + return apiGet(`/api/topics/matrix${q}`, signal) +} + +export function getTopicProjection( + signal?: AbortSignal, +): Promise { + return apiGet('/api/topics/projection', signal) +} + +export function listTopicMembers( + id: string, + opts: { cursor?: string | null; limit?: number } = {}, + signal?: AbortSignal, +): Promise { + const q = toSearch({ + cursor: opts.cursor ?? undefined, + limit: opts.limit ?? 50, + }) + return apiGet( + `/api/topics/${encodeURIComponent(id)}/members${q}`, + signal, + ) +} diff --git a/apps/chronicle/web/src/topics/originBadge.tsx b/apps/chronicle/web/src/topics/originBadge.tsx new file mode 100644 index 0000000..5df936a --- /dev/null +++ b/apps/chronicle/web/src/topics/originBadge.tsx @@ -0,0 +1,23 @@ +/** Origin badge text for topics (TA-002 — always visible). */ + +const ORIGIN_LABEL: Record = { + automatic: 'Automatic', + curated: 'Curated', + manual: 'Manual', +} + +export function originLabel(origin: string): string { + return ORIGIN_LABEL[origin] ?? origin +} + +export function OriginBadge({ origin }: { origin: string }) { + return ( + + {originLabel(origin)} + + ) +} diff --git a/apps/chronicle/web/src/workingset/urlState.test.ts b/apps/chronicle/web/src/workingset/urlState.test.ts index a4d69fd..9905ff8 100644 --- a/apps/chronicle/web/src/workingset/urlState.test.ts +++ b/apps/chronicle/web/src/workingset/urlState.test.ts @@ -162,6 +162,38 @@ describe('encodeState / decodeState', () => { expect(decoded.filesView).toBe('table') expect(decoded.filesQuery).toBe('x') }) + + it('roundtrips topic atlas tv and tsel params', () => { + const state: UrlWorkingState = { + ...DEFAULT_URL_STATE, + topicView: 'projection', + topicSelected: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + } + const encoded = encodeState(state) + expect(encoded.get('tv')).toBe('projection') + expect(encoded.get('tsel')).toBe('aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee') + const decoded = decodeState(encoded) + expect(decoded.topicView).toBe('projection') + expect(decoded.topicSelected).toBe('aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee') + }) + + it('omits default hierarchy tv and empty tsel', () => { + const params = encodeState({ + ...DEFAULT_URL_STATE, + topicView: 'hierarchy', + topicSelected: null, + }) + expect(params.has('tv')).toBe(false) + expect(params.has('tsel')).toBe(false) + }) + + it('decodes unknown tv as hierarchy', () => { + const decoded = decodeState( + new URLSearchParams({ tv: 'map', tsel: 'topic-1' }), + ) + expect(decoded.topicView).toBe('hierarchy') + expect(decoded.topicSelected).toBe('topic-1') + }) }) describe('isScopePristine', () => { @@ -177,7 +209,7 @@ describe('isScopePristine', () => { }) describe('selection codec (sel)', () => { - it('roundtrips bucket, message, attachment, and event selections', () => { + it('roundtrips bucket, message, attachment, event, and topic selections', () => { const bucket = { kind: 'bucket' as const, lane: 'messages', @@ -189,10 +221,16 @@ describe('selection codec (sel)', () => { kind: 'event' as const, eventId: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890', } + const topic = { + kind: 'topic' as const, + topicId: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + } expect(decodeSelection(encodeSelection(bucket))).toEqual(bucket) expect(decodeSelection(encodeSelection(msg))).toEqual(msg) expect(decodeSelection(encodeSelection(att))).toEqual(att) expect(decodeSelection(encodeSelection(evt))).toEqual(evt) + expect(decodeSelection(encodeSelection(topic))).toEqual(topic) + expect(encodeSelection(topic)).toBe('t:aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee') expect(encodeSelection(null)).toBeNull() expect(decodeSelection(null)).toBeNull() }) diff --git a/apps/chronicle/web/src/workingset/urlState.ts b/apps/chronicle/web/src/workingset/urlState.ts index 6f9be11..3373c87 100644 --- a/apps/chronicle/web/src/workingset/urlState.ts +++ b/apps/chronicle/web/src/workingset/urlState.ts @@ -38,6 +38,16 @@ export type ViewMode = 'canvas' | 'table' /** Files lens view mode (URL param `fv`). */ export type FilesViewMode = 'table' | 'gallery' +/** Topic Atlas view mode (URL param `tv`). Default hierarchy — TA-001. */ +export type TopicViewMode = 'hierarchy' | 'river' | 'matrix' | 'projection' + +const TOPIC_VIEWS: ReadonlySet = new Set([ + 'hierarchy', + 'river', + 'matrix', + 'projection', +]) + /** Research Desk grouping (URL param `grp`). */ export type ResearchGrouping = 'none' | 'thread' | 'year' | 'mailbox' @@ -56,6 +66,7 @@ export type Selection = | { kind: 'message'; sid: string } | { kind: 'attachment'; sid: string } | { kind: 'event'; eventId: string } + | { kind: 'topic'; topicId: string } | null /** Serializable working-set slice for the URL codec. */ @@ -91,6 +102,16 @@ export interface UrlWorkingState { filesView?: FilesViewMode /** Files lens filename query (URL param `fq`). */ filesQuery?: string + /** + * Topic Atlas view (URL param `tv`). Optional so store-driven encodes that + * omit it preserve the current location value. Default hierarchy (TA-001). + */ + topicView?: TopicViewMode + /** + * Topic Atlas selected topic id (URL param `tsel`). Optional preserve like + * filesView; keeps topic context when inspector selection becomes a message. + */ + topicSelected?: string | null } export const DEFAULT_URL_STATE: UrlWorkingState = { @@ -107,6 +128,8 @@ export const DEFAULT_URL_STATE: UrlWorkingState = { grouping: 'none', filesView: 'table', filesQuery: '', + topicView: 'hierarchy', + topicSelected: null, } /** @@ -115,6 +138,7 @@ export const DEFAULT_URL_STATE: UrlWorkingState = { * - message: `m:` * - attachment: `a:` * - event: `e:` + * - topic: `t:` */ export function encodeSelection(selection: Selection): string | null { if (!selection) return null @@ -134,6 +158,10 @@ export function encodeSelection(selection: Selection): string | null { if (!selection.eventId) return null return `e:${selection.eventId}` } + if (selection.kind === 'topic') { + if (!selection.topicId) return null + return `t:${selection.topicId}` + } return null } @@ -170,9 +198,20 @@ export function decodeSelection(raw: string | null): Selection { if (!eventId || !/^[A-Za-z0-9_-]+$/.test(eventId)) return null return { kind: 'event', eventId } } + if (raw.startsWith('t:')) { + const topicId = raw.slice(2) + // UUID (with hyphens) or opaque non-empty id. + if (!topicId || !/^[A-Za-z0-9_-]+$/.test(topicId)) return null + return { kind: 'topic', topicId } + } return null } +export function parseTopicView(value: string | null): TopicViewMode { + if (value && TOPIC_VIEWS.has(value)) return value as TopicViewMode + return 'hierarchy' +} + /** Format epoch ms as UTC ISO datetime with second precision (`…Z`, no ms). */ export function toIsoSeconds(ms: number): string { return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z') @@ -393,6 +432,30 @@ export function encodeState(state: UrlWorkingState): URLSearchParams { const fq = (filesQuery ?? '').trim() if (fq) params.set('fq', fq) + // Topic Atlas: tv / tsel. When omitted from state, preserve from location + // so store-driven rewrites (selection etc.) do not wipe atlas params. + let topicView = state.topicView + let topicSelected = state.topicSelected + if ( + typeof window !== 'undefined' && + (topicView === undefined || topicSelected === undefined) + ) { + try { + const current = new URLSearchParams(window.location.search) + if (topicView === undefined) { + topicView = parseTopicView(current.get('tv')) + } + if (topicSelected === undefined) { + topicSelected = current.get('tsel') + } + } catch { + /* ignore */ + } + } + if (topicView && topicView !== 'hierarchy') params.set('tv', topicView) + const tsel = (topicSelected ?? '').trim() + if (tsel) params.set('tsel', tsel) + return params } @@ -449,6 +512,8 @@ export function decodeState(params: URLSearchParams): UrlWorkingState { const filesView: FilesViewMode = params.get('fv') === 'gallery' ? 'gallery' : 'table' const filesQuery = params.get('fq') ?? '' + const topicView = parseTopicView(params.get('tv')) + const topicSelected = params.get('tsel') return { scope, @@ -464,6 +529,8 @@ export function decodeState(params: URLSearchParams): UrlWorkingState { grouping, filesView, filesQuery, + topicView, + topicSelected, } } diff --git a/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md b/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md index 2bb8623..c0059b5 100644 --- a/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md +++ b/docs/superpowers/plans/2026-07-13-life-chronicle-plan.md @@ -102,7 +102,7 @@ WCAG 2.2 AA audit, security review (CSP, sanitizer corpus, IDOR/enumeration), pe ## 5. STATE — live progress (update after every task) -**Next up:** Phase 4, Task 4.2 (Topic Atlas UI). Goal mode active (2026-07-13): user delegated review+merge of all phases to Claude via /goal. +**Next up:** Phase 4, Task 4.3 (People & Organizations UI). Goal mode active (2026-07-13): user delegated review+merge of all phases to Claude via /goal. | Date | Task | PR | Outcome | | --- | --- | --- | --- | @@ -133,3 +133,4 @@ WCAG 2.2 AA audit, security review (CSP, sanitizer corpus, IDOR/enumeration), pe | 2026-07-13 | 3.5 workflow A acceptance + criterion 4; keyboard glue | chronicle-phase-3 | Approved; Phase 3 complete (244 web tests, 0 todos) | | 2026-07-13 | Phase 3 PR #109 merged (36b7e97) | #109 | CI green | | 2026-07-13 | 4.1 topics subsystem: seeded k-means, one-statement assignment, manual precedence, topics lane | chronicle-phase-4 | Approved; 238 server / 246 web tests | +| 2026-07-13 | 4.2 Topic Atlas: hierarchy/river/matrix/projection + members list | chronicle-phase-4 | Approved; 243 server / 263 web tests | From d925a3c193b9b34840489280744345a4caeeb0fd Mon Sep 17 00:00:00 2001 From: Claude Code Agent Date: Mon, 13 Jul 2026 22:09:23 -0700 Subject: [PATCH 3/5] feat(chronicle): People & Organizations lens over the contacts subsystem (Phase 4 task 4.3, #105) /people index (search, kind filters, curation queue, merge candidates) and /people/:id profile (identity with owner/external address classes and classifier signal explainability, span, monthly activity, top topics, correspondence hand-off to Research, merge/unmerge with confirm+audit). All identity logic proxies the existing contacts subsystem. Copy audited against relationship-quality language. Implemented via cheap-coder (grok-4.5 @ max). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XP7M17NHxw6mjpTFtMqwZu --- .../server/src/chronicle_server/app.py | 3 + .../server/src/chronicle_server/people.py | 420 +++++++++++++ apps/chronicle/server/tests/test_people.py | 546 +++++++++++++++++ apps/chronicle/web/src/App.tsx | 5 +- apps/chronicle/web/src/api/types.ts | 114 ++++ .../chronicle/web/src/people/ActivityBars.tsx | 54 ++ .../web/src/people/PeoplePage.test.tsx | 193 ++++++ apps/chronicle/web/src/people/PeoplePage.tsx | 348 +++++++++++ .../web/src/people/PersonProfilePage.test.tsx | 367 ++++++++++++ .../web/src/people/PersonProfilePage.tsx | 564 ++++++++++++++++++ apps/chronicle/web/src/people/api.ts | 97 +++ .../web/src/people/copyAudit.test.ts | 37 ++ .../web/src/workingset/urlState.test.ts | 19 + apps/chronicle/web/src/workingset/urlState.ts | 22 + .../plans/2026-07-13-life-chronicle-plan.md | 3 +- 15 files changed, 2790 insertions(+), 2 deletions(-) create mode 100644 apps/chronicle/server/src/chronicle_server/people.py create mode 100644 apps/chronicle/server/tests/test_people.py create mode 100644 apps/chronicle/web/src/people/ActivityBars.tsx create mode 100644 apps/chronicle/web/src/people/PeoplePage.test.tsx create mode 100644 apps/chronicle/web/src/people/PeoplePage.tsx create mode 100644 apps/chronicle/web/src/people/PersonProfilePage.test.tsx create mode 100644 apps/chronicle/web/src/people/PersonProfilePage.tsx create mode 100644 apps/chronicle/web/src/people/api.ts create mode 100644 apps/chronicle/web/src/people/copyAudit.test.ts diff --git a/apps/chronicle/server/src/chronicle_server/app.py b/apps/chronicle/server/src/chronicle_server/app.py index d6788c9..69208f8 100644 --- a/apps/chronicle/server/src/chronicle_server/app.py +++ b/apps/chronicle/server/src/chronicle_server/app.py @@ -19,6 +19,7 @@ from chronicle_server.generate import router as generate_router from chronicle_server.health import router as health_router from chronicle_server.interpret import router as interpret_router +from chronicle_server.people import router as people_router from chronicle_server.search import router as search_router from chronicle_server.sources import router as sources_router from chronicle_server.topics import router as topics_router @@ -82,6 +83,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.include_router(events_router, prefix="/api") # topics before any catch-all; generate path is under the same router app.include_router(topics_router, prefix="/api") + # people: list/merge-candidates before {id} + app.include_router(people_router, prefix="/api") # Stash settings early so tests can inspect before lifespan if needed. app.state.settings = resolved return app diff --git a/apps/chronicle/server/src/chronicle_server/people.py b/apps/chronicle/server/src/chronicle_server/people.py new file mode 100644 index 0000000..68b112c --- /dev/null +++ b/apps/chronicle/server/src/chronicle_server/people.py @@ -0,0 +1,420 @@ +"""People & Organizations: thin authenticated proxies over MailDB contacts. + +Identity resolution, archive span, activity, topics, and merge/unmerge curation +all flow through the existing contacts subsystem — no duplicated resolution logic. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, Literal +from uuid import UUID + +import structlog +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from pydantic import BaseModel, Field + +from chronicle_server.auth import require_user +from chronicle_server.cursor import decode_cursor, encode_cursor +from chronicle_server.db import audit + +if TYPE_CHECKING: + from psycopg_pool import ConnectionPool + +logger = structlog.get_logger() + +router = APIRouter(tags=["people"]) + +_LIST_DEFAULT_LIMIT = 50 +_LIST_MAX_LIMIT = 500 +_CANDIDATES_DEFAULT_LIMIT = 20 +_CANDIDATES_MAX_LIMIT = 100 +_TOP_TOPICS = 5 + +ContactKind = Literal["human", "organization", "automated", "mailing_list", "unknown"] +VALID_KINDS: frozenset[str] = frozenset( + {"human", "organization", "automated", "mailing_list", "unknown"} +) + + +# --- request models --- + + +class ContactPatch(BaseModel): + kind: ContactKind | None = None + tags: list[str] | None = None + notes: str | None = None + display_name: str | None = None + + +class MergeRequest(BaseModel): + source_id: str = Field(min_length=1) + target_id: str = Field(min_length=1) + + +class UnmergeRequest(BaseModel): + merge_id: str = Field(min_length=1) + + +# --- helpers --- + + +def _iso(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + if hasattr(value, "isoformat"): + return str(value.isoformat()) + return str(value) + + +def _serialize_contact(card: dict[str, Any]) -> dict[str, Any]: + """Normalize a MailDB contact card for JSON (ISO datetimes, stable ids).""" + out = dict(card) + for key in ("first_seen", "last_seen", "classified_at"): + if key in out: + out[key] = _iso(out[key]) + if "id" in out and out["id"] is not None: + out["id"] = str(out["id"]) + if "merge_id" in out and out["merge_id"] is not None: + out["merge_id"] = str(out["merge_id"]) + return out + + +def _maildb(pool: ConnectionPool) -> Any: + from maildb import MailDB + + return MailDB._from_pool(pool) + + +def _clamp_limit(limit: int, *, default: int, maximum: int) -> int: + if limit < 1: + return default + return min(limit, maximum) + + +def _parse_offset_cursor(cursor: str | None, secret_key: str) -> int: + if not cursor: + return 0 + try: + payload = decode_cursor(cursor, secret_key) + except ValueError as exc: + raise HTTPException(status_code=422, detail="invalid cursor") from exc + offset = payload.get("offset", 0) + if not isinstance(offset, int) or offset < 0: + raise HTTPException(status_code=422, detail="invalid cursor offset") + return offset + + +def _address_rows(pool: ConnectionPool, contact_id: UUID) -> list[dict[str, Any]]: + """Per-address volumes and owner flag (PE-002).""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT address, is_user, messages_from, messages_to, first_seen, last_seen + FROM contact_addresses + WHERE contact_id = %(id)s + ORDER BY (messages_from + messages_to) DESC, address + """, + {"id": contact_id}, + ).fetchall() + return [ + { + "address": str(r[0]), + "is_user": bool(r[1]), + "messages_from": int(r[2] or 0), + "messages_to": int(r[3] or 0), + "first_seen": _iso(r[4]), + "last_seen": _iso(r[5]), + } + for r in rows + ] + + +def _activity_buckets(pool: ConnectionPool, addresses: list[str]) -> list[dict[str, Any]]: + """Monthly sent-message buckets for contact addresses (one statement).""" + if not addresses: + return [] + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT date_trunc('month', e.date) AS bucket, count(*)::int AS count + FROM emails e + WHERE lower(e.sender_address) = ANY(%(addrs)s) + AND e.date IS NOT NULL + GROUP BY 1 + ORDER BY 1 + """, + {"addrs": [a.lower() for a in addresses]}, + ).fetchall() + return [{"bucket": _iso(r[0]), "count": int(r[1])} for r in rows if r[0] is not None] + + +def _top_topics(pool: ConnectionPool, addresses: list[str]) -> list[dict[str, Any]]: + """Top topics by authored membership; empty when topics not generated / table missing.""" + if not addresses: + return [] + try: + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT t.id, t.label, count(*)::int AS cnt + FROM app_topic_members m + JOIN emails e ON e.id = m.email_id + JOIN app_topics t ON t.id = m.topic_id + WHERE lower(e.sender_address) = ANY(%(addrs)s) + AND t.hidden = FALSE + GROUP BY t.id, t.label + ORDER BY cnt DESC, t.label ASC + LIMIT %(lim)s + """, + {"addrs": [a.lower() for a in addresses], "lim": _TOP_TOPICS}, + ).fetchall() + except Exception as exc: + logger.debug("people_topics_empty", error=str(exc)) + return [] + return [{"id": str(r[0]), "label": str(r[1]), "count": int(r[2])} for r in rows] + + +def _thread_count(pool: ConnectionPool, addresses: list[str]) -> int: + if not addresses: + return 0 + with pool.connection() as conn: + row = conn.execute( + """ + SELECT count(DISTINCT e.thread_id)::int + FROM emails e + WHERE lower(e.sender_address) = ANY(%(addrs)s) + AND e.thread_id IS NOT NULL + """, + {"addrs": [a.lower() for a in addresses]}, + ).fetchone() + return int(row[0]) if row else 0 + + +def _merge_history(pool: ConnectionPool, contact_id: UUID) -> list[dict[str, Any]]: + """Merges where this contact is the surviving target (unmerge-able).""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, source_id, target_id, merged_at + FROM contact_merges + WHERE target_id = %(id)s + ORDER BY merged_at DESC + """, + {"id": contact_id}, + ).fetchall() + return [ + { + "id": str(r[0]), + "source_id": str(r[1]), + "target_id": str(r[2]), + "merged_at": _iso(r[3]), + } + for r in rows + ] + + +def _enrich_card(pool: ConnectionPool, card: dict[str, Any]) -> dict[str, Any]: + """Full profile: card + address classes, activity, topics, merges.""" + contact_id = UUID(str(card["id"])) + addresses = list(card.get("addresses") or []) + addr_rows = _address_rows(pool, contact_id) + address_classes: dict[str, str] = { + r["address"]: ("owner" if r["is_user"] else "external") for r in addr_rows + } + # Fallback for addresses present on the card but missing detail rows. + for addr in addresses: + if addr not in address_classes: + address_classes[addr] = "external" + + out = _serialize_contact(card) + out["address_classes"] = address_classes + out["address_details"] = addr_rows + out["activity"] = _activity_buckets(pool, addresses) + out["topics"] = _top_topics(pool, addresses) + out["thread_count"] = _thread_count(pool, addresses) + merges = _merge_history(pool, contact_id) + if merges: + out["merges"] = merges + return out + + +# --- endpoints --- + + +@router.get("/people") +def people_list( + request: Request, + q: str | None = Query(default=None), + kind: str | None = Query(default=None), + needs_review: bool = Query(default=False), + limit: int = Query(default=_LIST_DEFAULT_LIMIT, ge=1, le=_LIST_MAX_LIMIT), + cursor: str | None = Query(default=None), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Search contacts (include_total on first page). Offset cursor, limit ≤ 500.""" + if kind is not None and kind not in VALID_KINDS: + raise HTTPException( + status_code=422, + detail=f"kind must be one of: {', '.join(sorted(VALID_KINDS))}", + ) + pool: ConnectionPool = request.app.state.pool + secret_key: str = request.app.state.settings.secret_key + offset = _parse_offset_cursor(cursor, secret_key) + lim = _clamp_limit(limit, default=_LIST_DEFAULT_LIMIT, maximum=_LIST_MAX_LIMIT) + + db = _maildb(pool) + include_total = offset == 0 + results, total = db.contacts_search( + query=q if q else None, + kind=kind, + needs_review=needs_review, + limit=lim, + offset=offset, + include_total=include_total, + ) + items = [_serialize_contact(r) for r in results] + next_cursor: str | None = None + if len(items) >= lim: + next_cursor = encode_cursor({"offset": offset + lim}, secret_key) + + return { + "items": items, + "total": total, + "next_cursor": next_cursor, + "limit": lim, + "offset": offset, + } + + +@router.get("/people/merge-candidates") +def people_merge_candidates( + request: Request, + limit: int = Query(default=_CANDIDATES_DEFAULT_LIMIT, ge=1, le=_CANDIDATES_MAX_LIMIT), + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Contact pairs that share a normalized name variant.""" + pool: ConnectionPool = request.app.state.pool + lim = _clamp_limit(limit, default=_CANDIDATES_DEFAULT_LIMIT, maximum=_CANDIDATES_MAX_LIMIT) + db = _maildb(pool) + pairs = db.merge_candidates(limit=lim) + return {"items": pairs} + + +@router.get("/people/{contact_id}") +def people_get( + contact_id: UUID, + request: Request, + _user: str = Depends(require_user), +) -> dict[str, Any]: + """Full contact card + activity, topics, address classes (PE-002).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + card = db.get_contact(contact_id=contact_id) + if card is None: + raise HTTPException(status_code=404, detail="Contact not found") + return _enrich_card(pool, card) + + +@router.patch("/people/{contact_id}") +def people_patch( + contact_id: UUID, + body: ContactPatch, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Curation write: kind/tags/notes/display_name (audited).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + try: + card = db.update_contact( + contact_id=contact_id, + kind=body.kind, + tags=body.tags, + notes=body.notes, + display_name=body.display_name, + ) + except ValueError as exc: + msg = str(exc) + if "does not exist" in msg: + raise HTTPException(status_code=404, detail=msg) from exc + raise HTTPException(status_code=422, detail=msg) from exc + + audit( + pool, + username=user, + action="people_update", + detail={ + "contact_id": str(contact_id), + "kind": body.kind, + "tags": body.tags, + "notes": body.notes is not None, + "display_name": body.display_name is not None, + }, + ) + return _enrich_card(pool, card) + + +@router.post("/people/merge") +def people_merge( + body: MergeRequest, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Merge source into target; audited (PE-001).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + try: + card = db.merge_contacts(source_id=body.source_id, target_id=body.target_id) + except ValueError as exc: + msg = str(exc) + if "does not exist" in msg: + raise HTTPException(status_code=404, detail=msg) from exc + raise HTTPException(status_code=422, detail=msg) from exc + + merge_id = card.get("merge_id") + audit( + pool, + username=user, + action="people_merge", + detail={ + "source_id": body.source_id, + "target_id": body.target_id, + "merge_id": str(merge_id) if merge_id is not None else None, + }, + ) + return _enrich_card(pool, card) + + +@router.post("/people/unmerge") +def people_unmerge( + body: UnmergeRequest, + request: Request, + user: str = Depends(require_user), +) -> dict[str, Any]: + """Reverse a prior merge by merge_id; audited (PE-001).""" + pool: ConnectionPool = request.app.state.pool + db = _maildb(pool) + try: + result = db.unmerge_contacts(merge_id=body.merge_id) + except ValueError as exc: + msg = str(exc) + if "does not exist" in msg: + raise HTTPException(status_code=404, detail=msg) from exc + raise HTTPException(status_code=422, detail=msg) from exc + + audit( + pool, + username=user, + action="people_unmerge", + detail={"merge_id": body.merge_id}, + ) + source = result.get("source") or {} + target = result.get("target") or {} + return { + "source": _enrich_card(pool, source) if source.get("id") else source, + "target": _enrich_card(pool, target) if target.get("id") else target, + } diff --git a/apps/chronicle/server/tests/test_people.py b/apps/chronicle/server/tests/test_people.py new file mode 100644 index 0000000..30f2101 --- /dev/null +++ b/apps/chronicle/server/tests/test_people.py @@ -0,0 +1,546 @@ +# tests/test_people.py +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +from chronicle_server.cursor import encode_cursor +from chronicle_server.people import ( + _activity_buckets, + _enrich_card, + _top_topics, +) +from tests.conftest import PASSWORD, USERNAME + +if TYPE_CHECKING: + from fastapi.testclient import TestClient + from psycopg_pool import ConnectionPool + + +def _login(client: TestClient) -> None: + r = client.post("/api/auth/login", json={"username": USERNAME, "password": PASSWORD}) + assert r.status_code == 200 + + +def _sample_card(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "id": str(uuid4()), + "display_name": "Alice Example", + "kind": "human", + "kind_source": "manual", + "tags": ["vip"], + "human_probability": 0.85, + "addresses": ["alice@example.com"], + "name_variants": ["Alice"], + "messages_from": 10, + "messages_to": 3, + "first_seen": datetime(2015, 1, 1, tzinfo=UTC), + "last_seen": datetime(2018, 6, 1, tzinfo=UTC), + "notes": "notes", + "metadata": {}, + "classification_signals": {"bidirectional": 0.15, "personal_name": 0.1}, + "classified_at": datetime(2020, 1, 1, tzinfo=UTC), + } + base.update(overrides) + return base + + +# --- auth --- + + +def test_people_list_requires_auth(client: TestClient) -> None: + r = client.get("/api/people") + assert r.status_code == 401 + + +def test_people_get_requires_auth(client: TestClient) -> None: + r = client.get(f"/api/people/{uuid4()}") + assert r.status_code == 401 + + +def test_people_merge_requires_auth(client: TestClient) -> None: + r = client.post( + "/api/people/merge", + json={"source_id": str(uuid4()), "target_id": str(uuid4())}, + ) + assert r.status_code == 401 + + +# --- proxy shapes (stub pool + mocked MailDB) --- + + +def test_people_list_proxy_shape(client: TestClient, stub_pool: MagicMock) -> None: + card = _sample_card() + mock_db = MagicMock() + mock_db.contacts_search.return_value = ([card], 1) + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get("/api/people?q=Alice&limit=10") + assert r.status_code == 200, r.text + body = r.json() + assert "items" in body + assert body["total"] == 1 + assert body["offset"] == 0 + assert body["limit"] == 10 + assert body["items"][0]["display_name"] == "Alice Example" + assert body["items"][0]["first_seen"].endswith("Z") or "2015" in body["items"][0]["first_seen"] + kwargs = mock_db.contacts_search.call_args.kwargs + assert kwargs["query"] == "Alice" + assert kwargs["include_total"] is True + assert kwargs["limit"] == 10 + assert kwargs["offset"] == 0 + + +def test_people_list_cursor_offset_and_no_total_on_later_page( + client: TestClient, +) -> None: + card = _sample_card() + mock_db = MagicMock() + mock_db.contacts_search.return_value = ([card], None) + secret = "test-secret-key-not-for-production" + cursor = encode_cursor({"offset": 50}, secret) + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people?cursor={cursor}&limit=50") + assert r.status_code == 200 + body = r.json() + assert body["total"] is None + assert body["offset"] == 50 + kwargs = mock_db.contacts_search.call_args.kwargs + assert kwargs["offset"] == 50 + assert kwargs["include_total"] is False + + +def test_people_list_invalid_kind_422(client: TestClient) -> None: + with patch("chronicle_server.people._maildb", return_value=MagicMock()): + _login(client) + r = client.get("/api/people?kind=robot") + assert r.status_code == 422 + + +def test_people_list_needs_review(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.contacts_search.return_value = ([], 0) + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get("/api/people?needs_review=true") + assert r.status_code == 200 + assert mock_db.contacts_search.call_args.kwargs["needs_review"] is True + + +def test_people_get_enriches_address_classes_and_activity(client: TestClient) -> None: + cid = uuid4() + card = _sample_card(id=str(cid), addresses=["alice@example.com", "me@owner.com"]) + mock_db = MagicMock() + mock_db.get_contact.return_value = card + + addr_rows = [ + ("alice@example.com", False, 10, 2, None, None), + ("me@owner.com", True, 0, 5, None, None), + ] + activity_rows = [ + (datetime(2016, 3, 1, tzinfo=UTC), 4), + (datetime(2016, 4, 1, tzinfo=UTC), 2), + ] + + def fake_execute(sql: str, params: dict[str, Any] | None = None) -> MagicMock: + result = MagicMock() + sql_l = sql.lower() + if "contact_addresses" in sql_l: + result.fetchall.return_value = addr_rows + result.fetchone.return_value = None + elif "date_trunc" in sql_l: + result.fetchall.return_value = activity_rows + result.fetchone.return_value = None + elif "app_topic_members" in sql_l: + result.fetchall.return_value = [] + result.fetchone.return_value = None + elif "count(distinct" in sql_l or "thread_id" in sql_l: + result.fetchone.return_value = (7,) + result.fetchall.return_value = [] + elif "contact_merges" in sql_l: + result.fetchall.return_value = [] + result.fetchone.return_value = None + else: + result.fetchall.return_value = [] + result.fetchone.return_value = None + return result + + conn = MagicMock() + conn.execute = fake_execute + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + + with ( + patch("chronicle_server.people._maildb", return_value=mock_db), + patch.object(client.app.state, "pool", pool, create=True), + ): + # Ensure pool is used from request.app.state + client.app.state.pool = pool + _login(client) + r = client.get(f"/api/people/{cid}") + + assert r.status_code == 200, r.text + body = r.json() + assert body["address_classes"]["alice@example.com"] == "external" + assert body["address_classes"]["me@owner.com"] == "owner" + assert len(body["activity"]) == 2 + assert body["activity"][0]["count"] == 4 + assert body["topics"] == [] + assert body["thread_count"] == 7 + assert "address_details" in body + assert any(d["is_user"] for d in body["address_details"]) + + +def test_people_get_404(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.get_contact.return_value = None + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get(f"/api/people/{uuid4()}") + assert r.status_code == 404 + + +def test_people_patch_calls_update_and_audits(client: TestClient, stub_pool: MagicMock) -> None: + cid = uuid4() + card = _sample_card(id=str(cid), kind="human", kind_source="manual") + mock_db = MagicMock() + mock_db.update_contact.return_value = card + + # Enrichment queries return empty. + conn = MagicMock() + empty = MagicMock(fetchall=MagicMock(return_value=[]), fetchone=MagicMock(return_value=(0,))) + conn.execute = MagicMock(return_value=empty) + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + # audit() opens its own connection on the real pool path — use stub + track INSERT. + stub_pool.connection = connection + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.patch( + f"/api/people/{cid}", + json={"kind": "human", "display_name": "Alice Renamed", "tags": ["a"]}, + ) + assert r.status_code == 200, r.text + kwargs = mock_db.update_contact.call_args.kwargs + assert kwargs["kind"] == "human" + assert kwargs["display_name"] == "Alice Renamed" + assert kwargs["tags"] == ["a"] + # audit INSERT should have been attempted + assert conn.execute.called + + +def test_people_merge_unmerge_proxy(client: TestClient) -> None: + src = str(uuid4()) + tgt = str(uuid4()) + mid = str(uuid4()) + card = _sample_card(id=tgt, merge_id=mid) + mock_db = MagicMock() + mock_db.merge_contacts.return_value = card + mock_db.unmerge_contacts.return_value = { + "source": _sample_card(id=src), + "target": _sample_card(id=tgt), + } + + conn = MagicMock() + empty = MagicMock(fetchall=MagicMock(return_value=[]), fetchone=MagicMock(return_value=(0,))) + conn.execute = MagicMock(return_value=empty) + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + client.app.state.pool.connection = connection + + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.post("/api/people/merge", json={"source_id": src, "target_id": tgt}) + assert r.status_code == 200, r.text + assert mock_db.merge_contacts.called + + r2 = client.post("/api/people/unmerge", json={"merge_id": mid}) + assert r2.status_code == 200, r2.text + assert mock_db.unmerge_contacts.called + body = r2.json() + assert "source" in body and "target" in body + + +def test_people_merge_candidates(client: TestClient) -> None: + mock_db = MagicMock() + mock_db.merge_candidates.return_value = [ + { + "norm_name": "alice", + "a": { + "display_name": "Alice", + "primary_address": "a@x.com", + "msg_count": 5, + "contact_id": str(uuid4()), + }, + "b": { + "display_name": "A. Example", + "primary_address": "b@x.com", + "msg_count": 3, + "contact_id": str(uuid4()), + }, + } + ] + with patch("chronicle_server.people._maildb", return_value=mock_db): + _login(client) + r = client.get("/api/people/merge-candidates?limit=5") + assert r.status_code == 200 + body = r.json() + assert len(body["items"]) == 1 + mock_db.merge_candidates.assert_called_once_with(limit=5) + + +def test_activity_buckets_one_statement() -> None: + """_activity_buckets issues a single date_trunc statement.""" + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = [ + (datetime(2016, 1, 1, tzinfo=UTC), 3), + ] + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + buckets = _activity_buckets(pool, ["alice@example.com"]) + assert len(buckets) == 1 + assert buckets[0]["count"] == 3 + assert conn.execute.call_count == 1 + sql = conn.execute.call_args[0][0] + assert "date_trunc" in sql.lower() + + +def test_topics_empty_tolerance() -> None: + conn = MagicMock() + conn.execute.side_effect = RuntimeError("relation app_topics does not exist") + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + assert _top_topics(pool, ["alice@example.com"]) == [] + + +def test_owner_class_derivation() -> None: + cid = uuid4() + card = _sample_card(id=str(cid), addresses=["a@x.com", "me@x.com"]) + addr_rows = [ + ("a@x.com", False, 1, 0, None, None), + ("me@x.com", True, 0, 1, None, None), + ] + + def fake_execute(sql: str, params: dict[str, Any] | None = None) -> MagicMock: + result = MagicMock() + sql_l = sql.lower() + if "contact_addresses" in sql_l: + result.fetchall.return_value = addr_rows + elif "date_trunc" in sql_l or "app_topic" in sql_l or "contact_merges" in sql_l: + result.fetchall.return_value = [] + elif "thread" in sql_l: + result.fetchone.return_value = (0,) + else: + result.fetchall.return_value = [] + result.fetchone.return_value = (0,) + return result + + conn = MagicMock() + conn.execute = fake_execute + + from contextlib import contextmanager + + @contextmanager + def connection() -> Any: + yield conn + + pool = MagicMock() + pool.connection = connection + out = _enrich_card(pool, card) + assert out["address_classes"]["a@x.com"] == "external" + assert out["address_classes"]["me@x.com"] == "owner" + + +# --- DB-backed merge round-trip + audit --- + + +def _insert_throwaway_contact( + pool: ConnectionPool, + *, + display_name: str, + address: str, + is_user: bool = False, +) -> str: + cid = uuid4() + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO contacts (id, display_name, kind, kind_source, tags, notes) + VALUES (%(id)s, %(name)s, 'unknown', 'heuristic', '{}', NULL) + """, + {"id": cid, "name": display_name}, + ) + conn.execute( + """ + INSERT INTO contact_addresses ( + address, contact_id, name_variants, is_user, + first_seen, last_seen, messages_from, messages_to + ) VALUES ( + %(addr)s, %(cid)s, %(variants)s, %(is_user)s, + now(), now(), 5, 1 + ) + """, + { + "addr": address, + "cid": cid, + "variants": [display_name], + "is_user": is_user, + }, + ) + conn.commit() + return str(cid) + + +def _cleanup_contacts(pool: ConnectionPool, contact_ids: list[str], addresses: list[str]) -> None: + with pool.connection() as conn: + for cid in contact_ids: + conn.execute( + """ + DELETE FROM contact_merges + WHERE source_id = %(id)s OR target_id = %(id)s + """, + {"id": cid}, + ) + conn.execute( + "DELETE FROM contact_addresses WHERE contact_id = %(id)s", + {"id": cid}, + ) + conn.execute("DELETE FROM contacts WHERE id = %(id)s", {"id": cid}) + for addr in addresses: + conn.execute( + "DELETE FROM contact_addresses WHERE address = %(a)s", + {"a": addr}, + ) + conn.execute("DELETE FROM app_audit WHERE action LIKE 'people%%'") + conn.commit() + + +def test_merge_unmerge_roundtrip_and_audit(db_client: TestClient, db_pool: ConnectionPool) -> None: + suffix = uuid4().hex[:8] + addr_a = f"throwaway-a-{suffix}@people-test.example" + addr_b = f"throwaway-b-{suffix}@people-test.example" + name_a = f"ThrowA {suffix}" + name_b = f"ThrowB {suffix}" + ids: list[str] = [] + try: + id_a = _insert_throwaway_contact(db_pool, display_name=name_a, address=addr_a) + id_b = _insert_throwaway_contact(db_pool, display_name=name_b, address=addr_b) + ids = [id_a, id_b] + + _login(db_client) + r = db_client.post( + "/api/people/merge", + json={"source_id": id_a, "target_id": id_b}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["id"] == id_b + merge_id = body.get("merge_id") + assert merge_id + # Address classes present after enrich + assert addr_a in body["address_classes"] or any( + d["address"] == addr_a for d in body.get("address_details", []) + ) + assert "merges" in body or merge_id + + with db_pool.connection() as conn: + audit_rows = conn.execute( + """ + SELECT action, username, detail FROM app_audit + WHERE action = 'people_merge' + ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + assert audit_rows is not None + assert audit_rows[0] == "people_merge" + assert audit_rows[1] == USERNAME + + r2 = db_client.post("/api/people/unmerge", json={"merge_id": merge_id}) + assert r2.status_code == 200, r2.text + restored = r2.json() + assert restored["source"]["id"] == id_a + assert restored["target"]["id"] == id_b + + with db_pool.connection() as conn: + unmerge_rows = conn.execute( + """ + SELECT action FROM app_audit + WHERE action = 'people_unmerge' + ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + assert unmerge_rows is not None + assert unmerge_rows[0] == "people_unmerge" + + # Owner class derivation on DB card with is_user address + owner_addr = f"throwaway-owner-{suffix}@people-test.example" + id_owner = _insert_throwaway_contact( + db_pool, + display_name=f"Owner {suffix}", + address=owner_addr, + is_user=True, + ) + ids.append(id_owner) + r3 = db_client.get(f"/api/people/{id_owner}") + assert r3.status_code == 200 + card = r3.json() + assert card["address_classes"].get(owner_addr) == "owner" + finally: + _cleanup_contacts( + db_pool, + ids, + [addr_a, addr_b, f"throwaway-owner-{suffix}@people-test.example"], + ) + + +def test_people_list_db_search(db_client: TestClient, db_pool: ConnectionPool) -> None: + suffix = uuid4().hex[:8] + addr = f"list-search-{suffix}@people-test.example" + name = f"ListSearch {suffix}" + cid = _insert_throwaway_contact(db_pool, display_name=name, address=addr) + try: + _login(db_client) + r = db_client.get(f"/api/people?q={suffix}") + assert r.status_code == 200, r.text + body = r.json() + assert any(item["id"] == cid for item in body["items"]) + assert body["total"] is not None + finally: + _cleanup_contacts(db_pool, [cid], [addr]) diff --git a/apps/chronicle/web/src/App.tsx b/apps/chronicle/web/src/App.tsx index 5c53910..795dc48 100644 --- a/apps/chronicle/web/src/App.tsx +++ b/apps/chronicle/web/src/App.tsx @@ -7,6 +7,8 @@ import { FilesPage } from './files/FilesPage' import { SourcePage } from './reader/SourcePage' import { ResearchDeskPage } from './research/ResearchDeskPage' import { ResearchNavShortcut } from './research/ResearchNavShortcut' +import { PeoplePage } from './people/PeoplePage' +import { PersonProfilePage } from './people/PersonProfilePage' import { ChroniclePage } from './routes/ChroniclePage' import { DataHealthPage } from './routes/DataHealthPage' import { StubPage } from './routes/StubPage' @@ -37,7 +39,8 @@ export function App() { } /> } /> } /> - } /> + } /> + } /> } /> } /> } /> diff --git a/apps/chronicle/web/src/api/types.ts b/apps/chronicle/web/src/api/types.ts index 7f331ca..0315b2c 100644 --- a/apps/chronicle/web/src/api/types.ts +++ b/apps/chronicle/web/src/api/types.ts @@ -1057,3 +1057,117 @@ export interface WorkspaceManifestRow { subject_or_filename?: string | null excerpt_hash?: string | null } + +/** People & Organizations (GET /api/people) */ + +export type ContactKind = + | 'human' + | 'organization' + | 'automated' + | 'mailing_list' + | 'unknown' + +export type AddressClass = 'owner' | 'external' + +export interface ContactSummary { + id: string + display_name: string | null + kind: ContactKind | string + kind_source: string | null + tags: string[] + human_probability: number | null + addresses: string[] + name_variants: string[] + messages_from: number + messages_to: number + first_seen: string | null + last_seen: string | null +} + +export interface PeopleListResponse { + items: ContactSummary[] + total: number | null + next_cursor: string | null + limit: number + offset: number +} + +export interface ContactAddressDetail { + address: string + is_user: boolean + messages_from: number + messages_to: number + first_seen: string | null + last_seen: string | null +} + +export interface ContactActivityBucket { + bucket: string + count: number +} + +export interface ContactTopic { + id: string + label: string + count: number +} + +export interface ContactMergeRecord { + id: string + source_id: string + target_id: string + merged_at: string | null +} + +/** GET /api/people/:id */ +export interface ContactCard extends ContactSummary { + notes: string | null + metadata: Record + classification_signals: Record | null + classified_at: string | null + address_classes: Record + address_details: ContactAddressDetail[] + activity: ContactActivityBucket[] + topics: ContactTopic[] + thread_count: number + merges?: ContactMergeRecord[] + merge_id?: string | null +} + +export interface ContactPatchRequest { + kind?: ContactKind + tags?: string[] + notes?: string | null + display_name?: string | null +} + +export interface MergeCandidateSide { + display_name: string | null + primary_address: string | null + msg_count: number + contact_id: string +} + +export interface MergeCandidatePair { + norm_name: string + a: MergeCandidateSide + b: MergeCandidateSide +} + +export interface MergeCandidatesResponse { + items: MergeCandidatePair[] +} + +export interface MergeRequest { + source_id: string + target_id: string +} + +export interface UnmergeRequest { + merge_id: string +} + +export interface UnmergeResponse { + source: ContactCard + target: ContactCard +} diff --git a/apps/chronicle/web/src/people/ActivityBars.tsx b/apps/chronicle/web/src/people/ActivityBars.tsx new file mode 100644 index 0000000..941c200 --- /dev/null +++ b/apps/chronicle/web/src/people/ActivityBars.tsx @@ -0,0 +1,54 @@ +/** Compact monthly activity bar row (simple SVG). */ + +export interface ActivityBarsProps { + buckets: { bucket: string; count: number }[] + height?: number + className?: string +} + +export function ActivityBars({ + buckets, + height = 32, + className = '', +}: ActivityBarsProps) { + if (!buckets.length) { + return ( +

+ No sent-message activity +

+ ) + } + const max = Math.max(...buckets.map((b) => b.count), 1) + const barW = Math.max(2, Math.min(8, Math.floor(280 / buckets.length))) + const gap = 1 + const width = buckets.length * (barW + gap) + + return ( + + {buckets.map((b, i) => { + const h = Math.max(1, (b.count / max) * (height - 2)) + return ( + + + {b.bucket.slice(0, 7)}: {b.count} + + + ) + })} + + ) +} diff --git a/apps/chronicle/web/src/people/PeoplePage.test.tsx b/apps/chronicle/web/src/people/PeoplePage.test.tsx new file mode 100644 index 0000000..c9c4745 --- /dev/null +++ b/apps/chronicle/web/src/people/PeoplePage.test.tsx @@ -0,0 +1,193 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { + ContactSummary, + MergeCandidatesResponse, + PeopleListResponse, +} from '../api/types' +import { PeoplePage } from './PeoplePage' + +function contact(overrides: Partial = {}): ContactSummary { + return { + id: 'c1', + display_name: 'Alice Example', + kind: 'human', + kind_source: 'manual', + tags: [], + human_probability: 0.87, + addresses: ['alice@example.com'], + name_variants: ['Alice'], + messages_from: 10, + messages_to: 2, + first_seen: '2015-01-01T00:00:00Z', + last_seen: '2018-06-01T00:00:00Z', + ...overrides, + } +} + +function listResponse( + items: ContactSummary[], + total: number | null = items.length, +): PeopleListResponse { + return { + items, + total, + next_cursor: null, + limit: 50, + offset: 0, + } +} + +function renderPeople(initial = '/people') { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + return render( + + + + } /> + Profile
} /> + + + , + ) +} + +describe('PeoplePage', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + beforeEach(() => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.includes('/api/people/merge-candidates')) { + const body: MergeCandidatesResponse = { + items: [ + { + norm_name: 'alice', + a: { + display_name: 'Alice A', + primary_address: 'a@x.com', + msg_count: 5, + contact_id: 'ca', + }, + b: { + display_name: 'Alice B', + primary_address: 'b@x.com', + msg_count: 3, + contact_id: 'cb', + }, + }, + ], + } + return { + ok: true, + status: 200, + json: async () => body, + } as Response + } + if (url.includes('/api/people')) { + const u = new URL(url, 'http://local') + const q = u.searchParams.get('q') + const kind = u.searchParams.get('kind') + const review = u.searchParams.get('needs_review') + let items = [ + contact({ id: 'c1', kind: 'human', display_name: 'Alice' }), + contact({ + id: 'c2', + kind: 'organization', + display_name: 'Acme Corp', + human_probability: 0.2, + }), + contact({ + id: 'c3', + kind: 'unknown', + display_name: 'Bob Unknown', + human_probability: 0.5, + }), + ] + if (q) items = items.filter((i) => i.display_name?.includes(q)) + if (kind) items = items.filter((i) => i.kind === kind) + if (review === 'true') items = items.filter((i) => i.kind === 'unknown') + return { + ok: true, + status: 200, + json: async () => listResponse(items), + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }), + ) + }) + + it('renders index rows with kind badge, probability, volume, span', async () => { + renderPeople() + expect(await screen.findByTestId('people-page')).toBeInTheDocument() + expect(await screen.findByTestId('people-row-c1')).toBeInTheDocument() + expect(screen.getByTestId('kind-badge-c1')).toHaveTextContent('human') + expect(screen.getByTestId('people-row-c1')).toHaveTextContent('0.87') + expect(screen.getByTestId('people-row-c1')).toHaveTextContent('12') + expect(screen.getByTestId('people-row-c1')).toHaveTextContent('2015-01-01') + }) + + it('search updates pq and refilters', async () => { + renderPeople() + await screen.findByTestId('people-row-c1') + fireEvent.change(screen.getByTestId('people-search'), { + target: { value: 'Acme' }, + }) + fireEvent.click(screen.getByTestId('people-search-submit')) + await waitFor(() => { + expect(screen.getByTestId('people-row-c2')).toBeInTheDocument() + }) + expect(screen.queryByTestId('people-row-c1')).not.toBeInTheDocument() + }) + + it('kind filter chips filter list; organizations tab preset', async () => { + renderPeople('/people?kind=organization') + expect(await screen.findByTestId('people-row-c2')).toBeInTheDocument() + expect(screen.queryByTestId('people-row-c1')).not.toBeInTheDocument() + expect(screen.getByTestId('people-kind-organization')).toHaveAttribute( + 'aria-selected', + 'true', + ) + }) + + it('needs-review toggle filters to unknown queue', async () => { + renderPeople() + await screen.findByTestId('people-row-c1') + fireEvent.click(screen.getByTestId('people-needs-review')) + await waitFor(() => { + expect(screen.getByTestId('people-row-c3')).toBeInTheDocument() + }) + expect(screen.queryByTestId('people-row-c1')).not.toBeInTheDocument() + }) + + it('merge candidates rail with confirm dialog naming direction', async () => { + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(false) + renderPeople() + expect(await screen.findByTestId('merge-pair-ca')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('merge-into-b-ca')) + expect(confirmSpy).toHaveBeenCalled() + const msg = String(confirmSpy.mock.calls[0]?.[0] ?? '') + expect(msg).toMatch(/Merge Alice A into Alice B/) + confirmSpy.mockRestore() + }) + + it('row click navigates to profile', async () => { + renderPeople() + fireEvent.click(await screen.findByTestId('people-row-c1')) + expect(await screen.findByText('Profile')).toBeInTheDocument() + }) +}) diff --git a/apps/chronicle/web/src/people/PeoplePage.tsx b/apps/chronicle/web/src/people/PeoplePage.tsx new file mode 100644 index 0000000..ae267fe --- /dev/null +++ b/apps/chronicle/web/src/people/PeoplePage.tsx @@ -0,0 +1,348 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useCallback, useMemo, useState } from 'react' +import { Link, useNavigate, useSearchParams } from 'react-router' + +import type { ContactKind, ContactSummary, MergeCandidatePair } from '../api/types' +import { listMergeCandidates, listPeople, mergePeople } from './api' + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[12px] text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +const KINDS: { id: '' | ContactKind; label: string }[] = [ + { id: '', label: 'All' }, + { id: 'human', label: 'Human' }, + { id: 'organization', label: 'Organizations' }, + { id: 'automated', label: 'Automated' }, + { id: 'mailing_list', label: 'Mailing list' }, + { id: 'unknown', label: 'Unknown' }, +] + +function volume(c: ContactSummary): number { + return (c.messages_from ?? 0) + (c.messages_to ?? 0) +} + +function formatProb(p: number | null | undefined): string { + if (p == null || !Number.isFinite(p)) return '—' + return p.toFixed(2) +} + +function formatSpan(first: string | null, last: string | null): string { + const f = first ? first.slice(0, 10) : '—' + const l = last ? last.slice(0, 10) : '—' + return `${f} → ${l}` +} + +/** + * People & Organizations index: search, kind chips, needs-review queue, + * merge-candidates rail. Organizations tab = kind=organization preset. + */ +export function PeoplePage() { + const [searchParams, setSearchParams] = useSearchParams() + const navigate = useNavigate() + const qc = useQueryClient() + + const pq = searchParams.get('pq') ?? '' + const kind = searchParams.get('kind') ?? '' + const needsReview = searchParams.get('review') === '1' + const [draft, setDraft] = useState(pq) + + const setParam = useCallback( + (key: string, value: string | null) => { + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev) + if (value == null || value === '') next.delete(key) + else next.set(key, value) + return next + }, + { replace: true }, + ) + }, + [setSearchParams], + ) + + const commitSearch = useCallback(() => { + setParam('pq', draft.trim() || null) + }, [draft, setParam]) + + // Kind counts from a first unfiltered page (same q / needs_review, no kind). + const countsQuery = useQuery({ + queryKey: ['people', 'counts', pq, needsReview], + queryFn: ({ signal }) => + listPeople( + { + q: pq || undefined, + needs_review: needsReview || undefined, + limit: 200, + }, + signal, + ), + retry: false, + }) + + const listQuery = useQuery({ + queryKey: ['people', 'list', pq, kind, needsReview], + queryFn: ({ signal }) => + listPeople( + { + q: pq || undefined, + kind: kind || undefined, + needs_review: needsReview || undefined, + limit: 50, + }, + signal, + ), + retry: false, + }) + + const candidatesQuery = useQuery({ + queryKey: ['people', 'merge-candidates'], + queryFn: ({ signal }) => listMergeCandidates(20, signal), + retry: false, + }) + + const kindCounts = useMemo(() => { + const counts: Record = {} + for (const item of countsQuery.data?.items ?? []) { + const k = item.kind || 'unknown' + counts[k] = (counts[k] ?? 0) + 1 + } + return counts + }, [countsQuery.data]) + + const mergeMut = useMutation({ + mutationFn: (body: { source_id: string; target_id: string }) => + mergePeople(body), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ['people'] }) + }, + }) + + const onMerge = useCallback( + (pair: MergeCandidatePair, direction: 'a_into_b' | 'b_into_a') => { + const source = direction === 'a_into_b' ? pair.a : pair.b + const target = direction === 'a_into_b' ? pair.b : pair.a + const sourceName = source.display_name || source.primary_address || source.contact_id + const targetName = target.display_name || target.primary_address || target.contact_id + const ok = window.confirm( + `Merge ${sourceName} into ${targetName}? This moves addresses to the target contact.`, + ) + if (!ok) return + mergeMut.mutate({ + source_id: source.contact_id, + target_id: target.contact_id, + }) + }, + [mergeMut], + ) + + const items = listQuery.data?.items ?? [] + + return ( +
+
+

+ People & Organizations +

+

+ Identity profiles over the contacts subsystem. Org domain controls + use tags/notes (MVP). +

+
+ +
+ setDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') commitSearch() + }} + placeholder="Search name or address…" + className="min-w-[12rem] flex-1 rounded-md border border-steel bg-graphite-900 px-2 py-1 text-[12px] text-text-primary" + data-testid="people-search" + aria-label="Search people" + /> + + +
+ +
+ {KINDS.map((k) => { + const selected = kind === k.id + const count = + k.id === '' + ? countsQuery.data?.items?.length + : kindCounts[k.id] + return ( + + ) + })} +
+ +
+
+ {listQuery.isLoading ? ( +

Loading…

+ ) : listQuery.isError ? ( +

+ Failed to load people +

+ ) : items.length === 0 ? ( +

No contacts match.

+ ) : ( + + + + + + + + + + + + {items.map((c) => ( + void navigate(`/people/${c.id}`)} + > + + + + + + + ))} + +
NameKindHuman pVolumeSpan
+ {c.display_name || c.addresses[0] || c.id} + + + {c.kind} + + + {formatProb(c.human_probability)} + {volume(c)} + {formatSpan(c.first_seen, c.last_seen)} +
+ )} + {listQuery.data?.total != null ? ( +

+ {listQuery.data.total} matching +

+ ) : null} +
+ + +
+
+ ) +} diff --git a/apps/chronicle/web/src/people/PersonProfilePage.test.tsx b/apps/chronicle/web/src/people/PersonProfilePage.test.tsx new file mode 100644 index 0000000..a1e5466 --- /dev/null +++ b/apps/chronicle/web/src/people/PersonProfilePage.test.tsx @@ -0,0 +1,367 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ContactCard } from '../api/types' +import { resetWorkingSetStore, useWorkingSetStore } from '../workingset/store' +import { PersonProfilePage } from './PersonProfilePage' + +const CONTACT_ID = '11111111-1111-1111-1111-111111111111' +const TOPIC_ID = '22222222-2222-2222-2222-222222222222' +const MERGE_ID = '33333333-3333-3333-3333-333333333333' +const TARGET_ID = '44444444-4444-4444-4444-444444444444' + +function mockCard(overrides: Partial = {}): ContactCard { + return { + id: CONTACT_ID, + display_name: 'Alice Example', + kind: 'human', + kind_source: 'heuristic', + tags: ['vip'], + human_probability: 0.9, + addresses: ['alice@example.com', 'me@owner.com'], + name_variants: ['Alice'], + messages_from: 20, + messages_to: 5, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2019-12-31T00:00:00Z', + notes: 'Analyst note', + metadata: {}, + classification_signals: { + bidirectional: 0.15, + personal_name: 0.1, + }, + classified_at: '2020-01-01T00:00:00Z', + address_classes: { + 'alice@example.com': 'external', + 'me@owner.com': 'owner', + }, + address_details: [ + { + address: 'alice@example.com', + is_user: false, + messages_from: 20, + messages_to: 0, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2019-12-31T00:00:00Z', + }, + { + address: 'me@owner.com', + is_user: true, + messages_from: 0, + messages_to: 5, + first_seen: '2014-01-01T00:00:00Z', + last_seen: '2019-12-31T00:00:00Z', + }, + ], + activity: [ + { bucket: '2016-01-01T00:00:00Z', count: 4 }, + { bucket: '2016-02-01T00:00:00Z', count: 2 }, + ], + topics: [{ id: TOPIC_ID, label: 'Kitchen remodel', count: 7 }], + thread_count: 12, + merges: [ + { + id: MERGE_ID, + source_id: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + target_id: CONTACT_ID, + merged_at: '2021-06-01T00:00:00Z', + }, + ], + ...overrides, + } +} + +function renderProfile(initial = `/people/${CONTACT_ID}`) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + return render( + + + + } /> + Chronicle
} /> + Research
} /> + Topics} /> + + + , + ) +} + +describe('PersonProfilePage', () => { + beforeEach(() => { + resetWorkingSetStore() + }) + + afterEach(() => { + vi.unstubAllGlobals() + resetWorkingSetStore() + }) + + it('renders profile sections from mocked card incl. signals and owner badges', async () => { + const card = mockCard() + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.includes(`/api/people/${CONTACT_ID}`) && !url.includes('merge')) { + return { + ok: true, + status: 200, + json: async () => card, + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }), + ) + + renderProfile() + expect(await screen.findByTestId('person-profile')).toBeInTheDocument() + expect(screen.getByTestId('person-identity')).toBeInTheDocument() + expect(screen.getByTestId('person-span')).toHaveTextContent('threads: 12') + expect(screen.getByTestId('person-activity')).toBeInTheDocument() + expect(screen.getByTestId('activity-bars')).toBeInTheDocument() + expect(screen.getByTestId('person-topics')).toHaveTextContent('Kitchen remodel') + expect(screen.getByTestId('address-class-me@owner.com')).toHaveTextContent( + 'owner', + ) + expect( + screen.getByTestId('address-class-alice@example.com'), + ).toHaveTextContent('external') + + // Expand signals details + fireEvent.click(screen.getByTestId('person-signals').querySelector('summary')!) + expect(screen.getByTestId('person-signals')).toHaveTextContent('bidirectional') + expect(screen.getByTestId('person-signals')).toHaveTextContent('0.15') + }) + + it('kind PATCH with kind_source explanation', async () => { + const card = mockCard({ kind_source: 'heuristic', kind: 'unknown' }) + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (init?.method === 'PATCH') { + const body = JSON.parse(String(init.body)) as { kind?: string } + return { + ok: true, + status: 200, + json: async () => + mockCard({ + kind: body.kind as ContactCard['kind'], + kind_source: 'manual', + }), + } as Response + } + if (url.includes(`/api/people/${CONTACT_ID}`)) { + return { + ok: true, + status: 200, + json: async () => card, + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }) + vi.stubGlobal('fetch', fetchMock) + + renderProfile() + await screen.findByTestId('person-kind-source') + expect(screen.getByTestId('person-kind-source')).toHaveTextContent( + 'kind_source: heuristic', + ) + + fireEvent.change(screen.getByTestId('person-kind-select'), { + target: { value: 'human' }, + }) + await waitFor(() => { + const patches = fetchMock.mock.calls.filter( + (c) => (c[1] as RequestInit | undefined)?.method === 'PATCH', + ) + expect(patches.length).toBeGreaterThan(0) + }) + await waitFor(() => { + expect(screen.getByTestId('person-kind-source')).toHaveTextContent('manual') + expect(screen.getByTestId('person-kind-source')).toHaveTextContent( + 'locked against machinery', + ) + }) + }) + + it('merge flow confirm dialog naming direction', async () => { + const card = mockCard() + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(false) + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.includes('/api/people?') && url.includes('q=')) { + return { + ok: true, + status: 200, + json: async () => ({ + items: [ + { + id: TARGET_ID, + display_name: 'Target Person', + kind: 'human', + kind_source: 'manual', + tags: [], + human_probability: 0.5, + addresses: ['t@x.com'], + name_variants: [], + messages_from: 3, + messages_to: 1, + first_seen: null, + last_seen: null, + }, + ], + total: 1, + next_cursor: null, + limit: 8, + offset: 0, + }), + } as Response + } + if (url.includes(`/api/people/${CONTACT_ID}`)) { + return { + ok: true, + status: 200, + json: async () => card, + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }), + ) + + renderProfile() + await screen.findByTestId('person-merge') + fireEvent.change(screen.getByTestId('merge-target-search'), { + target: { value: 'Target' }, + }) + await screen.findByTestId(`merge-target-${TARGET_ID}`) + fireEvent.click(screen.getByTestId(`merge-target-${TARGET_ID}`)) + fireEvent.click(screen.getByTestId('merge-confirm')) + expect(confirmSpy).toHaveBeenCalled() + const msg = String(confirmSpy.mock.calls[0]?.[0] ?? '') + expect(msg).toMatch(/Merge Alice Example into Target Person/) + confirmSpy.mockRestore() + }) + + it('unmerge from merge history', async () => { + const card = mockCard() + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(true) + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (init?.method === 'POST' && url.includes('/api/people/unmerge')) { + return { + ok: true, + status: 200, + json: async () => ({ + source: mockCard({ id: 'src', merges: undefined }), + target: mockCard({ merges: [] }), + }), + } as Response + } + if (url.includes(`/api/people/${CONTACT_ID}`)) { + return { + ok: true, + status: 200, + json: async () => card, + } as Response + } + return { + ok: false, + status: 404, + json: async () => ({}), + } as Response + }) + vi.stubGlobal('fetch', fetchMock) + + renderProfile() + await screen.findByTestId('person-merge-history') + fireEvent.click(screen.getByTestId(`unmerge-${MERGE_ID}`)) + await waitFor(() => { + const posts = fetchMock.mock.calls.filter( + (c) => + String(c[0]).includes('/api/people/unmerge') && + (c[1] as RequestInit)?.method === 'POST', + ) + expect(posts.length).toBe(1) + const body = JSON.parse(String((posts[0]?.[1] as RequestInit).body)) + expect(body.merge_id).toBe(MERGE_ID) + }) + confirmSpy.mockRestore() + }) + + it('Open in Chronicle sets sender scope and viewport from span', async () => { + const card = mockCard() + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => card, + })), + ) + + renderProfile() + await screen.findByTestId('open-in-chronicle') + fireEvent.click(screen.getByTestId('open-in-chronicle')) + expect(await screen.findByTestId('chronicle')).toBeInTheDocument() + // Navigation encodes senders in URL; store may also be hydrated by useUrlSync + // on real app — here we only verify the button navigates to Chronicle. + }) + + it('View correspondence sets scope senders and navigates to research exact mode', async () => { + const card = mockCard() + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => card, + })), + ) + + renderProfile() + await screen.findByTestId('view-correspondence') + fireEvent.click(screen.getByTestId('view-correspondence')) + expect(await screen.findByTestId('research')).toBeInTheDocument() + const state = useWorkingSetStore.getState() + expect(state.scope.senders).toEqual( + expect.arrayContaining(['alice@example.com', 'me@owner.com']), + ) + expect(state.mode).toBe('exact') + expect(state.query).toBe('') + expect(state.grouping).toBe('thread') + }) + + it('topic link navigates with tsel', async () => { + const card = mockCard() + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => card, + })), + ) + + renderProfile() + fireEvent.click(await screen.findByTestId(`topic-link-${TOPIC_ID}`)) + expect(await screen.findByTestId('topics')).toBeInTheDocument() + }) +}) diff --git a/apps/chronicle/web/src/people/PersonProfilePage.tsx b/apps/chronicle/web/src/people/PersonProfilePage.tsx new file mode 100644 index 0000000..77f3af2 --- /dev/null +++ b/apps/chronicle/web/src/people/PersonProfilePage.tsx @@ -0,0 +1,564 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { Link, useNavigate, useParams } from 'react-router' + +import type { ContactCard, ContactKind, ContactSummary } from '../api/types' +import { encodeState, toIsoSeconds } from '../workingset/urlState' +import { useWorkingSetStore } from '../workingset/store' +import { + getPerson, + listPeople, + mergePeople, + patchPerson, + unmergePeople, +} from './api' +import { ActivityBars } from './ActivityBars' + +const btnClass = + 'rounded-md border border-steel bg-graphite-800 px-2 py-1 text-[12px] text-text-primary enabled:hover:bg-graphite-900 disabled:cursor-not-allowed disabled:opacity-40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-action' + +const KINDS: ContactKind[] = [ + 'human', + 'organization', + 'automated', + 'mailing_list', + 'unknown', +] + +/** + * Person / organization profile (spec Table 23 sections). + * Org domain controls covered by tags/notes MVP. + */ +export function PersonProfilePage() { + const { id = '' } = useParams<{ id: string }>() + const navigate = useNavigate() + const qc = useQueryClient() + const setScope = useWorkingSetStore((s) => s.setScope) + const setMode = useWorkingSetStore((s) => s.setMode) + const setQuery = useWorkingSetStore((s) => s.setQuery) + const setGrouping = useWorkingSetStore((s) => s.setGrouping) + + const [nameDraft, setNameDraft] = useState('') + const [notesDraft, setNotesDraft] = useState('') + const [tagDraft, setTagDraft] = useState('') + const [mergeSearch, setMergeSearch] = useState('') + const [mergeTarget, setMergeTarget] = useState(null) + + const query = useQuery({ + queryKey: ['people', id], + queryFn: ({ signal }) => getPerson(id, signal), + enabled: !!id, + retry: false, + }) + + const card = query.data + + useEffect(() => { + if (card) { + setNameDraft(card.display_name ?? '') + setNotesDraft(card.notes ?? '') + } + }, [card]) + + const patchMut = useMutation({ + mutationFn: (body: { + kind?: ContactKind + tags?: string[] + notes?: string | null + display_name?: string | null + }) => patchPerson(id, body), + onSuccess: (data) => { + void qc.setQueryData(['people', id], data) + void qc.invalidateQueries({ queryKey: ['people', 'list'] }) + }, + }) + + const mergeMut = useMutation({ + mutationFn: (body: { source_id: string; target_id: string }) => + mergePeople(body), + onSuccess: (data) => { + void qc.invalidateQueries({ queryKey: ['people'] }) + void navigate(`/people/${data.id}`) + }, + }) + + const unmergeMut = useMutation({ + mutationFn: (mergeId: string) => unmergePeople({ merge_id: mergeId }), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ['people'] }) + void query.refetch() + }, + }) + + const mergeSearchQuery = useQuery({ + queryKey: ['people', 'merge-search', mergeSearch], + queryFn: ({ signal }) => + listPeople({ q: mergeSearch, limit: 8 }, signal), + enabled: mergeSearch.trim().length >= 2, + retry: false, + }) + + const signalRows = useMemo(() => { + const signals = card?.classification_signals + if (!signals || typeof signals !== 'object') return [] + return Object.entries(signals).map(([signal, weight]) => ({ + signal, + weight: typeof weight === 'number' ? weight : Number(weight), + })) + }, [card?.classification_signals]) + + const primaryAddress = useMemo(() => { + if (!card) return null + const details = card.address_details ?? [] + if (details.length > 0) { + const sorted = [...details].sort( + (a, b) => + b.messages_from + b.messages_to - (a.messages_from + a.messages_to), + ) + return sorted[0]?.address ?? card.addresses[0] ?? null + } + return card.addresses[0] ?? null + }, [card]) + + const openInChronicle = useCallback( + (c: ContactCard) => { + const addr = primaryAddress + if (!addr) return + let fromMs: number + let toMs: number + if (c.first_seen && c.last_seen) { + fromMs = Date.parse(c.first_seen) + toMs = Date.parse(c.last_seen) + if (!Number.isFinite(fromMs) || !Number.isFinite(toMs) || toMs <= fromMs) { + toMs = fromMs + 24 * 3600 * 1000 + } + } else if (c.activity?.length) { + const times = c.activity + .map((a) => Date.parse(a.bucket)) + .filter((t) => Number.isFinite(t)) + fromMs = Math.min(...times) + toMs = Math.max(...times) + 30 * 24 * 3600 * 1000 + } else { + const now = Date.now() + fromMs = now - 365 * 24 * 3600 * 1000 + toMs = now + } + const params = encodeState({ + scope: { senders: [addr] }, + viewport: { fromMs, toMs }, + aggregation: 'auto', + view: 'canvas', + selection: null, + lanes: ['messages', 'top_people'], + }) + if (!params.get('vf')) params.set('vf', toIsoSeconds(fromMs)) + if (!params.get('vt')) params.set('vt', toIsoSeconds(toMs)) + const qs = params.toString() + void navigate({ pathname: '/', search: qs ? `?${qs}` : '' }) + }, + [navigate, primaryAddress], + ) + + const openInResearch = useCallback( + (c: ContactCard) => { + const addrs = c.addresses ?? [] + if (!addrs.length) return + setScope({ senders: [...addrs] }) + setQuery('') + setMode('exact') + setGrouping('thread') + void navigate('/research') + }, + [navigate, setGrouping, setMode, setQuery, setScope], + ) + + const openTopic = useCallback( + (topicId: string) => { + void navigate(`/topics?tsel=${encodeURIComponent(topicId)}`) + }, + [navigate], + ) + + const saveName = useCallback(() => { + if (!card) return + const next = nameDraft.trim() + if (next === (card.display_name ?? '')) return + patchMut.mutate({ display_name: next || null }) + }, [card, nameDraft, patchMut]) + + const saveNotes = useCallback(() => { + if (!card) return + if (notesDraft === (card.notes ?? '')) return + patchMut.mutate({ notes: notesDraft }) + }, [card, notesDraft, patchMut]) + + const addTag = useCallback(() => { + if (!card) return + const t = tagDraft.trim() + if (!t) return + const tags = card.tags ?? [] + if (tags.includes(t)) { + setTagDraft('') + return + } + patchMut.mutate({ tags: [...tags, t] }) + setTagDraft('') + }, [card, tagDraft, patchMut]) + + const removeTag = useCallback( + (tag: string) => { + if (!card) return + patchMut.mutate({ tags: (card.tags ?? []).filter((t) => t !== tag) }) + }, + [card, patchMut], + ) + + const confirmMergeInto = useCallback(() => { + if (!mergeTarget || !id) return + const sourceName = + card?.display_name || card?.addresses[0] || id + const targetName = + mergeTarget.display_name || mergeTarget.addresses[0] || mergeTarget.id + const ok = window.confirm( + `Merge ${sourceName} into ${targetName}? Addresses move to the target.`, + ) + if (!ok) return + mergeMut.mutate({ source_id: id, target_id: mergeTarget.id }) + }, [mergeTarget, id, card, mergeMut]) + + if (query.isLoading) { + return ( +
+ Loading profile… +
+ ) + } + + if (query.isError || !card) { + return ( +
+ Failed to load contact + +
+ ) + } + + const kindLocked = card.kind_source === 'manual' + + return ( +
+
+ + ← People + + {card.kind === 'organization' ? ( + Organization profile + ) : null} +
+ + {/* Identity */} +
+

Identity

+
+ setNameDraft(e.target.value)} + onBlur={saveName} + onKeyDown={(e) => { + if (e.key === 'Enter') saveName() + }} + className="min-w-[12rem] rounded-md border border-steel bg-graphite-900 px-2 py-1 text-sm text-text-primary" + data-testid="person-display-name" + aria-label="Display name" + /> + + + kind_source: {card.kind_source ?? '—'} + {kindLocked ? ' (manual — locked against machinery)' : ''} + +
+ +
+ {(card.tags ?? []).map((tag) => ( + + {tag} + + + ))} + setTagDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') addTag() + }} + placeholder="Add tag" + className="w-24 rounded border border-steel bg-graphite-900 px-1 text-[11px]" + data-testid="person-tag-input" + /> + +
+ +