From 08e8dfc06846ef30a61a8bb6fd045475d73c9290 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 13:41:28 +0530 Subject: [PATCH 01/52] docs(audit-log): add design spec for app-wide activity/audit log Co-Authored-By: Claude Opus 4.8 --- .../specs/2026-06-22-audit-log-design.md | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-22-audit-log-design.md diff --git a/docs/superpowers/specs/2026-06-22-audit-log-design.md b/docs/superpowers/specs/2026-06-22-audit-log-design.md new file mode 100644 index 00000000..6ca8d2a6 --- /dev/null +++ b/docs/superpowers/specs/2026-06-22-audit-log-design.md @@ -0,0 +1,237 @@ +# Audit Log — Design + +**Date:** 2026-06-22 +**Status:** Design (approved in brainstorming) +**Scope:** Full vertical — backend capture pipeline, query API, and a user-facing +"Activity log" page. Record every state-changing action a user performs with what changed, +when, and from which device. + +## Problem + +The app has ~22 backend modules (bookmarks, tasks, notes, passwords, environment manager, +api client, code snippets, sql/nosql/redis/s3 connections, url shortener, game scores, +feedback, user preferences, auth, etc.), all doing user-scoped CRUD via FastAPI services on +MongoDB. There is **no audit trail**. A user cannot see what they changed, when, or from +which device; there is no security/forensic record of writes or auth events. + +## Goals + +- Capture every state-changing action (writes + auth events) across the whole app. +- For each event record: actor, action, target entity, **what changed** (before/after), + when, outcome, and **device** (IP + browser/OS). +- Surface it to the user as an "Activity log" page they can browse and filter. +- Never leak secrets/PII into the log. +- Never let audit capture slow down or break a real request. + +## Non-Goals + +- No logging of plain reads (GET). Most reads are noise; revisit later if needed. +- No geo-IP lookup this pass (IP + parsed User-Agent only). +- No admin/cross-user audit console — log is per-user, scoped to the actor. +- No external SIEM/export integration this pass. + +## Decisions (from brainstorming) + +| Question | Decision | +|----------|----------| +| Event scope | Writes (POST/PUT/PATCH/DELETE) + auth events. Skip GET. | +| Capture mechanism | Hybrid: HTTP middleware auto-envelope + service-level helper for entity/diff | +| Device info | Raw IP + parsed User-Agent (browser, OS, device type). No extra deps — small regex helper. | +| Secret handling | Field-name **allowlist**; everything else redacted (field noted as changed, value `[redacted]`). | +| Retention | 90-day Mongo TTL index. | +| UI scope | Full vertical — backend + query API + user-facing Activity log page. | + +## Architecture + +### 1. Data model + +New collection constant `AUDIT_LOG = "audit_log"` in +[collection_name.py](../../../apps/backend/app/utils/collection_name.py). + +Event document shape: + +``` +{ + _id: str, # new_id() + uid: str | None, # actor; null if auth failed / unauthenticated write attempt + action: str, # dotted verb, e.g. "bookmark.create", "auth.login" + module: str, # "bookmarks" (inferred from path; helper may override) + entity_type: str | None, # "bookmark" + entity_id: str | None, # affected document id, when known + method: str, # "POST" | "PATCH" | "PUT" | "DELETE" + path: str, # "/api/v1/bookmarks/{id}" (raw request path) + status: int, # HTTP status code + outcome: "success" | "failure", # derived from status (<400 = success) + changes: [{ field, before, after }] | null, # allowlisted fields only; sensitive => "[redacted]" + summary: str | null, # human-readable, e.g. "Created bookmark 'GitHub'" + ip: str | null, # client IP + ua_raw: str | null, # raw User-Agent header + device: { browser, os, device_type } | null, # parsed from ua_raw + latency_ms: int, # request duration + ts: int, # epoch ms (create_timestamp()) + expireAt: datetime # BSON Date = now + 90d; TTL field +} +``` + +`ts` (epoch ms) mirrors the `createdAt` convention used elsewhere for ordering/display. +`expireAt` is a real BSON Date because Mongo TTL requires a Date field; it is the only +datetime in the doc. + +Indexes (added to [indexes.py](../../../apps/backend/app/core/indexes.py)): + +- `AUDIT_LOG` `[("uid", 1), ("ts", -1)]` — default per-user listing. +- `AUDIT_LOG` `[("uid", 1), ("module", 1), ("ts", -1)]` — module filter. +- `AUDIT_LOG` TTL index on `expireAt` (`expireAfterSeconds=0`). + +`db_manager.create_index` currently takes `unique`/`sparse` flags only. Add an +`expire_after_seconds` passthrough (or a dedicated `create_ttl_index` helper) so the TTL +index can be declared without breaking existing callers. + +### 2. Capture pipeline — `app/core/audit.py` + +A per-request `contextvars.ContextVar[AuditContext]` carries detail set by services back up +to the middleware that writes the event. + +```python +@dataclass +class AuditContext: + action: str | None = None + entity_type: str | None = None + entity_id: str | None = None + module: str | None = None + changes: list[dict] | None = None + summary: str | None = None +``` + +Helper API services call (all no-ops if no active context — safe to call anywhere): + +- `audit.set_entity(entity_type, entity_id)` +- `audit.set_action(action)` +- `audit.set_summary(text)` +- `audit.set_changes(changes)` / `audit.add_change(field, before, after)` +- `audit.diff(before: dict, after: dict, allow_fields) -> list[dict]` — compares two docs, + emits a `changes` list. Fields **not** in the global safe allowlist are still reported as + changed but their `before`/`after` values become the literal `"[redacted]"`. + +**Redaction (allowlist):** a module-level `SAFE_FIELDS` set of non-sensitive field names +(e.g. `title, name, tags, folderId, parentId, status, statusOrder, color, icon, description, +url, isExpanded, projectId, createdAt, updatedAt`). Any field outside the set is redacted. +Default-deny: a new field is redacted until explicitly added to the allowlist. Sensitive +keys (`password, secret, encryptedData, iv, connectionString, value, token`, etc.) are never +in the allowlist by construction. + +**UA parsing:** small internal regex helper `parse_user_agent(ua: str) -> dict` returning +`{browser, os, device_type}`. No third-party dependency. Falls back to `{browser:"Unknown", +os:"Unknown", device_type:"desktop"}` on unparseable input. + +### 3. `AuditMiddleware` + +Registered in [main.py](../../../apps/backend/app/main.py) (after CORS, around the request). + +Flow per request: + +1. **Skip** non-auditable: method in `{GET, HEAD, OPTIONS}`, or path not under `/api/v1`, or + health endpoints. Return early (no overhead). +2. Initialize a fresh `AuditContext` and set it on the ContextVar. +3. Resolve `uid`: reuse the token-extraction logic from `get_current_uid` + (Authorization bearer / `mdt_at` cookie) + `decode_access_token`. Wrapped in try/except — + on failure `uid = None` (the request itself will 401, but we still record the attempt). + This is independent of route dependencies, so coverage does not depend on each route. +4. `response = await call_next(request)`; measure `latency_ms`; read `status`. +5. Capture `ip` (client host, honoring `X-Forwarded-For` first hop if present) and `ua_raw`; + `device = parse_user_agent(ua_raw)`. +6. Read the ContextVar back — services may have set entity/changes/summary/action. If + `action`/`module` weren't set, infer `module` from the first path segment after `/api/v1` + and synthesize a generic `action` like `"bookmarks.update"` from module + method. +7. Build the event doc and **fire-and-forget** write it via `asyncio.create_task`, wrapped in + try/except that logs and swallows any error. + +**Safety invariant:** the entire audit path (uid decode, context read, write) is wrapped so +that no audit failure can alter or delay the user's response. The write is scheduled as a +background task; an exception in it is logged, never raised. + +### 4. Auth events + module coverage + +- **Auth events** are not entity CRUD, so [auth/services.py](../../../apps/backend/app/api/routes/auth/services.py) + (and the auth API handlers) call the helper explicitly: + `audit.set_action("auth.login" | "auth.logout" | "auth.token_refresh" | "auth.register" | + "auth.password_change" | "auth.account_disable")` with a summary. The middleware still + supplies device/ip/outcome/latency. +- **Failed writes** (4xx/5xx) are logged with `outcome:"failure"`; `uid` is null when the + request was unauthenticated. Useful security signal (e.g. repeated 401 writes). +- **Diff enrichment priority** — services get the auto-envelope for free; entity+diff is added + first to the high-value modules: + - bookmarks, bookmark-folders, tasks, projects, notes, code snippets (full safe diffs) + - passwords, environment manager (entity + action only; values redacted) + - sql/nosql/redis/s3 connections (entity + action; connection strings redacted) + - remaining modules rely on the auto-envelope until enriched later. + +### 5. Query API — `app/api/routes/audit_log/` + +`GET /audit-log` (router wired in [router.py](../../../apps/backend/app/api/router.py)), +`Depends(get_current_uid)`, scoped to the actor's `uid`. + +Query params: + +- `skip` (≥0), `limit` (1–100, default 50) +- `module`, `action`, `outcome` (optional exact filters) +- `from`, `to` (epoch ms range on `ts`) +- `search` (substring match on `summary`) + +Response: `{ items: AuditEventOut[], total, skip, limit }`. `AuditEventOut` mirrors the +document minus internal fields (`expireAt`). Sorted `ts` desc. + +### 6. Web UI — Activity log page + +- New page under the dashboard (e.g. `apps/web/src/app/dashboard` activity section or a + dedicated route), fetching through the existing `/api/backend/[...path]` proxy. +- Enterprise-flat style matching the current dashboard (solid surfaces, theme tokens, + `prefers-reduced-motion`, accessible). +- Layout: a filter bar (module, action, outcome, date range, search) above a timeline list. +- Each row: action badge · `summary` · device ("Chrome on macOS") · relative time. Expandable + to reveal the field-level `changes` diff (before → after, redacted values shown as + `[redacted]`). +- i18n labels for all static text, consistent with existing analytics i18n work. +- Empty/loading/error states consistent with existing dashboard panels. + +## Component boundaries + +| Unit | Does | Depends on | +|------|------|-----------| +| `audit.py` (helper) | Per-request context, diff + redaction, UA parse | contextvars, stdlib | +| `AuditMiddleware` | Build + fire-and-forget write the event envelope | audit helper, db_manager, auth token decode | +| `audit_log` route | Paginated, filtered per-user query API | db_manager, get_current_uid | +| Activity log page | Render filterable timeline + diffs | backend proxy, audit API | +| Service `audit.*` calls | Attach entity/diff/summary where valuable | audit helper | + +## Error handling + +- Audit write runs as a background task; failure is logged and swallowed. +- uid decode failure → `uid=null`, request proceeds normally. +- Unparseable UA → fallback device object. +- The query API treats malformed filters as 422 (FastAPI validation), never 500. + +## Testing + +- **Unit:** `audit.diff` allowlist redaction (safe field passes through, sensitive field → + `[redacted]`, new/unknown field → redacted); `parse_user_agent` for common UAs + fallback; + contextvar set/merge round-trip. +- **Integration:** a create request produces exactly one audit doc with correct + action/module/entity/device/outcome and redacted sensitive fields; a failed write records + `outcome:"failure"`; a GET produces **no** doc; auth login/logout emit records. +- **Safety:** simulate a write failure in the audit task and assert the user response is + unaffected. +- **Manual:** Activity log page at 375 / 768 / 1024 / 1440 px, light + dark; verify diffs + expand and secrets show as `[redacted]`. + +## Risks + +- **Middleware uid decode duplicates auth logic** — mitigated by reusing the existing + extraction + `decode_access_token`; factor the token-extraction into a shared helper so it + is not copy-pasted. +- **Write volume / storage** — bounded by 90d TTL; indexes keep queries cheap. +- **Background-task writes under high concurrency** — acceptable; ties into the existing + backend-scale plan. If `create_task` proves too lossy under load, swap to a bounded queue + later (out of scope now). +- **Allowlist drift** — default-deny means the failure mode is over-redaction (safe), not + leakage. From 3da2a180d6d0173f5b9f01dd53ba5ebcc9864360 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 13:49:02 +0530 Subject: [PATCH 02/52] docs(audit-log): add implementation plan Co-Authored-By: Claude Opus 4.8 --- .../superpowers/plans/2026-06-22-audit-log.md | 1421 +++++++++++++++++ 1 file changed, 1421 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-22-audit-log.md diff --git a/docs/superpowers/plans/2026-06-22-audit-log.md b/docs/superpowers/plans/2026-06-22-audit-log.md new file mode 100644 index 00000000..95bf9bcb --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-audit-log.md @@ -0,0 +1,1421 @@ +# Audit Log Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Record every state-changing user action (writes + auth events) across the whole app with what changed, when, outcome, and from which device, then expose it as a user-facing Activity log. + +**Architecture:** A FastAPI HTTP middleware auto-captures an envelope (uid, method, path, status, latency, IP, parsed User-Agent) for every non-GET `/api/v1` request and writes it fire-and-forget to a new `audit_log` MongoDB collection. A `contextvars`-based helper lets services attach entity id, a redacted before/after diff, and a human summary that the middleware merges in. A paginated query API and a Next.js Activity log page surface the data. + +**Tech Stack:** Python 3.9+, FastAPI, Starlette `BaseHTTPMiddleware`, Motor/MongoDB, `contextvars`, pytest + `fastapi.testclient`. Frontend: Next.js (App Router), `next-intl`, existing `backendFetch` proxy, Vitest. + +## Global Constraints + +- Backend collections are referenced via constants in `app/utils/collection_name.py` — never hardcode collection strings. +- All DB access goes through `app/database/db_manager.py` — never touch Motor directly in services/middleware. +- Money/secret safety: audit `changes` MUST only contain field names in the `SAFE_FIELDS` allowlist; every other field value is stored as the literal string `"[redacted]"`. Default-deny. +- The audit write path MUST never raise into, delay, or alter the user's response. All audit work is wrapped in try/except and the DB write is scheduled with `asyncio.create_task`. +- Timestamps use `app/utils/utils.create_timestamp()` (epoch ms) and ids use `new_id()`, matching every existing module. +- Only writes (`POST/PUT/PATCH/DELETE`) under `/api/v1` and explicit auth events are logged. `GET/HEAD/OPTIONS` and health are skipped. +- Frontend user-facing strings go through `next-intl` (`useTranslations`); add keys to `apps/web/messages/en.json`. + +--- + +## File Structure + +**Backend (create):** +- `apps/backend/app/core/audit.py` — context, redaction/diff, UA parsing, write helper. +- `apps/backend/app/core/audit_middleware.py` — the HTTP middleware. +- `apps/backend/app/api/routes/audit_log/__init__.py` +- `apps/backend/app/api/routes/audit_log/schema.py` +- `apps/backend/app/api/routes/audit_log/services.py` +- `apps/backend/app/api/routes/audit_log/api.py` +- `apps/backend/tests/test_audit_helper.py` +- `apps/backend/tests/test_audit_middleware.py` +- `apps/backend/tests/api/routes/audit_log/test_audit_query.py` + +**Backend (modify):** +- `apps/backend/app/utils/collection_name.py` — add `AUDIT_LOG`. +- `apps/backend/app/database/db_manager.py` — TTL index support. +- `apps/backend/app/core/indexes.py` — audit indexes. +- `apps/backend/app/main.py` — register middleware. +- `apps/backend/app/api/router.py` — include audit router. +- `apps/backend/app/api/routes/bookmarks/services.py` — worked example of diff enrichment. +- `apps/backend/app/api/routes/auth/api.py` (+ `services.py`) — auth event records. + +**Frontend (create):** +- `apps/web/src/lib/audit-log-api.ts` — typed client. +- `apps/web/src/lib/__tests__/audit-log-api.test.ts` +- `apps/web/src/app/dashboard/activity/page.tsx` — route. +- `apps/web/src/components/dashboard/activity/activity-log-panel.tsx` +- `apps/web/src/components/dashboard/activity/audit-event-row.tsx` + +**Frontend (modify):** +- `apps/web/messages/en.json` — `Dashboard.activity` labels. + +--- + +## Task 1: Collection constant + TTL index support + +**Files:** +- Modify: `apps/backend/app/utils/collection_name.py` +- Modify: `apps/backend/app/database/db_manager.py` +- Modify: `apps/backend/app/core/indexes.py` +- Test: `apps/backend/tests/test_audit_helper.py` (new file — first assertion only) + +**Interfaces:** +- Produces: constant `AUDIT_LOG = "audit_log"`; `db_manager.create_index(..., expire_after_seconds: int | None = None)`. + +- [ ] **Step 1: Add the collection constant** + +In `apps/backend/app/utils/collection_name.py`, append: + +```python +AUDIT_LOG = "audit_log" +``` + +- [ ] **Step 2: Write a failing test that the constant exists** + +Create `apps/backend/tests/test_audit_helper.py`: + +```python +from app.utils.collection_name import AUDIT_LOG + + +def test_audit_log_collection_name(): + assert AUDIT_LOG == "audit_log" +``` + +- [ ] **Step 3: Run it** + +Run: `cd apps/backend && python -m pytest tests/test_audit_helper.py -v` +Expected: PASS (constant added in Step 1). + +- [ ] **Step 4: Add TTL passthrough to db_manager.create_index** + +In `apps/backend/app/database/db_manager.py`, replace the existing `create_index`: + +```python +async def create_index( + collection_name, field, unique=False, sparse=False, expire_after_seconds=None +): + kwargs = {"unique": unique, "sparse": sparse} + if expire_after_seconds is not None: + kwargs["expireAfterSeconds"] = expire_after_seconds + await db[collection_name].create_index(field, **kwargs) +``` + +- [ ] **Step 5: Register audit indexes** + +In `apps/backend/app/core/indexes.py`, add `AUDIT_LOG` to the import block, then add at the end of `ensure_indexes()`: + +```python + await db_manager.create_index(AUDIT_LOG, [("uid", 1), ("ts", -1)]) + await db_manager.create_index(AUDIT_LOG, [("uid", 1), ("module", 1), ("ts", -1)]) + await db_manager.create_index(AUDIT_LOG, "expireAt", expire_after_seconds=0) +``` + +- [ ] **Step 6: Verify imports resolve** + +Run: `cd apps/backend && python -c "from app.core import indexes; from app.database import db_manager; print('ok')"` +Expected: prints `ok`. + +- [ ] **Step 7: Commit** + +```bash +git add apps/backend/app/utils/collection_name.py apps/backend/app/database/db_manager.py apps/backend/app/core/indexes.py apps/backend/tests/test_audit_helper.py +git commit -m "feat(audit): add audit_log collection + TTL index support" +``` + +--- + +## Task 2: Audit helper — context, redaction/diff, UA parsing + +**Files:** +- Create: `apps/backend/app/core/audit.py` +- Test: `apps/backend/tests/test_audit_helper.py` (extend) + +**Interfaces:** +- Produces: + - `SAFE_FIELDS: set[str]` + - `@dataclass AuditContext` with attrs `action, module, entity_type, entity_id, changes, summary` (all optional). + - `_audit_ctx: ContextVar[AuditContext | None]` (module-private). + - `current_context() -> AuditContext | None` + - `set_entity(entity_type: str, entity_id: str | None) -> None` + - `set_action(action: str) -> None` + - `set_summary(text: str) -> None` + - `set_changes(changes: list[dict]) -> None` + - `add_change(field: str, before, after) -> None` + - `diff(before: dict | None, after: dict | None, allow_fields: set[str] | None = None) -> list[dict]` + - `parse_user_agent(ua: str | None) -> dict` → keys `browser, os, device_type`. + +- [ ] **Step 1: Write failing tests for redaction/diff and UA parsing** + +Append to `apps/backend/tests/test_audit_helper.py`: + +```python +from app.core import audit + + +def test_diff_passes_safe_fields_through(): + changes = audit.diff({"title": "Old"}, {"title": "New"}) + assert changes == [{"field": "title", "before": "Old", "after": "New"}] + + +def test_diff_redacts_sensitive_fields(): + changes = audit.diff( + {"password": "old-secret"}, {"password": "new-secret"} + ) + assert changes == [ + {"field": "password", "before": "[redacted]", "after": "[redacted]"} + ] + + +def test_diff_redacts_unknown_fields_by_default(): + changes = audit.diff({"mystery": 1}, {"mystery": 2}) + assert changes == [ + {"field": "mystery", "before": "[redacted]", "after": "[redacted]"} + ] + + +def test_diff_ignores_unchanged_fields(): + assert audit.diff({"title": "Same"}, {"title": "Same"}) == [] + + +def test_diff_handles_create_and_delete(): + assert audit.diff(None, {"title": "New"}) == [ + {"field": "title", "before": None, "after": "New"} + ] + assert audit.diff({"title": "Gone"}, None) == [ + {"field": "title", "before": "Gone", "after": None} + ] + + +def test_parse_user_agent_chrome_macos(): + d = audit.parse_user_agent( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0 Safari/537.36" + ) + assert d["browser"] == "Chrome" + assert d["os"] == "macOS" + assert d["device_type"] == "desktop" + + +def test_parse_user_agent_fallback(): + d = audit.parse_user_agent(None) + assert d == {"browser": "Unknown", "os": "Unknown", "device_type": "desktop"} + + +def test_context_mutation_round_trip(): + token = audit._audit_ctx.set(audit.AuditContext()) + try: + audit.set_entity("bookmark", "abc") + audit.set_action("bookmark.create") + audit.add_change("title", None, "Hi") + ctx = audit.current_context() + assert ctx.entity_type == "bookmark" + assert ctx.entity_id == "abc" + assert ctx.action == "bookmark.create" + assert ctx.changes == [{"field": "title", "before": None, "after": "Hi"}] + finally: + audit._audit_ctx.reset(token) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cd apps/backend && python -m pytest tests/test_audit_helper.py -v` +Expected: FAIL with `ModuleNotFoundError: app.core.audit` / attribute errors. + +- [ ] **Step 3: Implement `app/core/audit.py`** + +```python +from __future__ import annotations + +import re +from contextvars import ContextVar +from dataclasses import dataclass, field + +# Field names whose values are safe to record verbatim in the audit diff. +# Anything NOT in this set is recorded as "[redacted]" (default-deny). +SAFE_FIELDS: set[str] = { + "title", "name", "tags", "folderId", "parentId", "status", "statusOrder", + "color", "icon", "description", "url", "isExpanded", "projectId", + "priority", "dueDate", "completed", "language", "createdAt", "updatedAt", +} + +REDACTED = "[redacted]" + + +@dataclass +class AuditContext: + action: str | None = None + module: str | None = None + entity_type: str | None = None + entity_id: str | None = None + changes: list[dict] | None = None + summary: str | None = None + + +_audit_ctx: ContextVar[AuditContext | None] = ContextVar("audit_ctx", default=None) + + +def current_context() -> AuditContext | None: + return _audit_ctx.get() + + +def set_action(action: str) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.action = action + + +def set_entity(entity_type: str, entity_id: str | None) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.entity_type = entity_type + ctx.entity_id = entity_id + + +def set_summary(text: str) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.summary = text + + +def set_changes(changes: list[dict]) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.changes = changes + + +def add_change(field_name: str, before, after) -> None: + ctx = _audit_ctx.get() + if ctx is None: + return + if ctx.changes is None: + ctx.changes = [] + ctx.changes.append({"field": field_name, "before": before, "after": after}) + + +def _redact(field_name: str, value, allow_fields: set[str]): + return value if field_name in allow_fields else REDACTED + + +def diff(before: dict | None, after: dict | None, allow_fields: set[str] | None = None) -> list[dict]: + allow = SAFE_FIELDS if allow_fields is None else allow_fields + before = before or {} + after = after or {} + changes: list[dict] = [] + for key in sorted(set(before) | set(after)): + if key in ("_id", "created_by"): + continue + b = before.get(key) + a = after.get(key) + if b == a: + continue + changes.append({ + "field": key, + "before": _redact(key, b, allow), + "after": _redact(key, a, allow), + }) + return changes + + +_BROWSERS = [ + ("Edg", "Edge"), ("OPR", "Opera"), ("Chrome", "Chrome"), + ("Firefox", "Firefox"), ("Safari", "Safari"), +] + + +def parse_user_agent(ua: str | None) -> dict: + fallback = {"browser": "Unknown", "os": "Unknown", "device_type": "desktop"} + if not ua: + return fallback + browser = "Unknown" + for token, name in _BROWSERS: + if token in ua: + browser = name + break + if "Windows" in ua: + os_name = "Windows" + elif "Mac OS X" in ua or "Macintosh" in ua: + os_name = "macOS" + elif "Android" in ua: + os_name = "Android" + elif "iPhone" in ua or "iPad" in ua or "iOS" in ua: + os_name = "iOS" + elif "Linux" in ua: + os_name = "Linux" + else: + os_name = "Unknown" + if re.search(r"Mobi|iPhone|Android.*Mobile", ua): + device_type = "mobile" + elif "iPad" in ua or ("Android" in ua and "Mobile" not in ua): + device_type = "tablet" + else: + device_type = "desktop" + return {"browser": browser, "os": os_name, "device_type": device_type} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd apps/backend && python -m pytest tests/test_audit_helper.py -v` +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/app/core/audit.py apps/backend/tests/test_audit_helper.py +git commit -m "feat(audit): add context helper, redacted diff, and UA parser" +``` + +--- + +## Task 3: Audit middleware + registration + +**Files:** +- Create: `apps/backend/app/core/audit_middleware.py` +- Modify: `apps/backend/app/main.py` +- Test: `apps/backend/tests/test_audit_middleware.py` + +**Interfaces:** +- Consumes: `app.core.audit` (context + `parse_user_agent`), `AUDIT_LOG`, `db_manager.insert_one`, `try_decode_access_token_uid`, `ACCESS_COOKIE_NAME`, `create_timestamp`, `new_id`. +- Produces: `class AuditMiddleware(BaseHTTPMiddleware)`; `write_audit_event(doc: dict) -> Awaitable` (the awaitable scheduled as a task). + +**Why a mutable context object (not rebinding):** Starlette `BaseHTTPMiddleware` runs the endpoint in a child task whose context is a *copy* taken after the middleware sets the ContextVar. The copy shares the *same* `AuditContext` object instance, so values the endpoint sets by **mutating** that object (via `audit.set_entity` etc.) are visible to the middleware after `call_next`. We therefore set a fresh object once in the middleware and only mutate it downstream — never rebind the var in services. + +- [ ] **Step 1: Write failing middleware tests** + +Create `apps/backend/tests/test_audit_middleware.py`: + +```python +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.core import audit +from app.core.audit_middleware import AuditMiddleware + + +@pytest.fixture +def captured(monkeypatch): + docs = [] + + async def fake_insert_one(collection_name, data): + docs.append((collection_name, data)) + + # Run the fire-and-forget task synchronously so assertions are deterministic. + monkeypatch.setattr("app.core.audit_middleware.db_manager.insert_one", fake_insert_one) + return docs + + +def build_app(): + app = FastAPI() + app.add_middleware(AuditMiddleware) + + @app.post("/api/v1/bookmarks") + async def create(): + audit.set_action("bookmark.create") + audit.set_entity("bookmark", "bk1") + audit.set_summary("Created bookmark 'GitHub'") + audit.add_change("title", None, "GitHub") + return {"id": "bk1"} + + @app.get("/api/v1/bookmarks") + async def listing(): + return [] + + @app.delete("/api/v1/bookmarks/{bid}") + async def fail(bid: str): + from fastapi import HTTPException + raise HTTPException(status_code=404, detail="nope") + + return app + + +def test_write_is_logged_with_envelope_and_detail(captured): + client = TestClient(build_app()) + res = client.post( + "/api/v1/bookmarks", + headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120.0 Safari/537.36"}, + ) + assert res.status_code == 200 + assert len(captured) == 1 + _, doc = captured[0] + assert doc["action"] == "bookmark.create" + assert doc["module"] == "bookmarks" + assert doc["entity_id"] == "bk1" + assert doc["method"] == "POST" + assert doc["outcome"] == "success" + assert doc["device"]["browser"] == "Chrome" + assert doc["changes"] == [{"field": "title", "before": None, "after": "GitHub"}] + assert "expireAt" in doc and "ts" in doc + + +def test_get_is_not_logged(captured): + client = TestClient(build_app()) + client.get("/api/v1/bookmarks") + assert captured == [] + + +def test_failed_write_logged_as_failure(captured): + client = TestClient(build_app()) + res = client.delete("/api/v1/bookmarks/bk1") + assert res.status_code == 404 + assert len(captured) == 1 + _, doc = captured[0] + assert doc["outcome"] == "failure" + assert doc["status"] == 404 + assert doc["module"] == "bookmarks" + + +def test_audit_write_failure_does_not_break_request(monkeypatch): + async def boom(collection_name, data): + raise RuntimeError("db down") + + monkeypatch.setattr("app.core.audit_middleware.db_manager.insert_one", boom) + client = TestClient(build_app()) + res = client.post("/api/v1/bookmarks") + assert res.status_code == 200 # user response unaffected +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cd apps/backend && python -m pytest tests/test_audit_middleware.py -v` +Expected: FAIL with `ModuleNotFoundError: app.core.audit_middleware`. + +- [ ] **Step 3: Implement `app/core/audit_middleware.py`** + +```python +from __future__ import annotations + +import asyncio +import datetime +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +from app.core import audit +from app.core.auth_cookies import ACCESS_COOKIE_NAME +from app.database import db_manager +from app.api.routes.auth.tokens import try_decode_access_token_uid +from app.utils.collection_name import AUDIT_LOG +from app.utils.utils import create_timestamp, new_id + +logger = logging.getLogger(__name__) + +_SKIP_METHODS = {"GET", "HEAD", "OPTIONS"} +_TTL_DAYS = 90 + + +def _extract_uid(request: Request) -> str | None: + token = None + auth = request.headers.get("authorization") + if auth: + scheme, _, value = auth.partition(" ") + if scheme.lower() == "bearer" and value.strip(): + token = value.strip() + if not token: + cookie = request.cookies.get(ACCESS_COOKIE_NAME) + if cookie and cookie.strip(): + token = cookie.strip() + if not token: + return None + try: + return try_decode_access_token_uid(token) + except Exception: # never let auth decode break auditing + return None + + +def _client_ip(request: Request) -> str | None: + fwd = request.headers.get("x-forwarded-for") + if fwd: + return fwd.split(",")[0].strip() + return request.client.host if request.client else None + + +def _module_from_path(path: str) -> str | None: + # /api/v1//... + parts = [p for p in path.split("/") if p] + if len(parts) >= 3 and parts[0] == "api" and parts[1] == "v1": + return parts[2] + return None + + +async def write_audit_event(doc: dict) -> None: + try: + await db_manager.insert_one(AUDIT_LOG, doc) + except Exception as exc: # swallow — auditing must never break requests + logger.warning("audit write failed: %s", exc) + + +class AuditMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + path = request.url.path + if request.method in _SKIP_METHODS or not path.startswith("/api/v1"): + return await call_next(request) + if path.startswith("/api/v1/health"): + return await call_next(request) + + ctx = audit.AuditContext() + token = audit._audit_ctx.set(ctx) + started = create_timestamp() + uid = _extract_uid(request) + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + return response + finally: + try: + latency = create_timestamp() - started + module = ctx.module or _module_from_path(path) + action = ctx.action or (f"{module}.{request.method.lower()}" if module else request.method.lower()) + ts = create_timestamp() + doc = { + "_id": new_id(), + "uid": uid, + "action": action, + "module": module, + "entity_type": ctx.entity_type, + "entity_id": ctx.entity_id, + "method": request.method, + "path": path, + "status": status_code, + "outcome": "success" if status_code < 400 else "failure", + "changes": ctx.changes, + "summary": ctx.summary, + "ip": _client_ip(request), + "ua_raw": request.headers.get("user-agent"), + "device": audit.parse_user_agent(request.headers.get("user-agent")), + "latency_ms": latency, + "ts": ts, + "expireAt": datetime.datetime.utcnow() + datetime.timedelta(days=_TTL_DAYS), + } + asyncio.create_task(write_audit_event(doc)) + except Exception as exc: # never propagate + logger.warning("audit envelope build failed: %s", exc) + finally: + audit._audit_ctx.reset(token) +``` + +- [ ] **Step 4: Register the middleware in `main.py`** + +In `apps/backend/app/main.py`, add the import near the other core imports: + +```python +from app.core.audit_middleware import AuditMiddleware +``` + +Then register it after the CORS middleware block (so it wraps requests): + +```python +app.add_middleware(AuditMiddleware) +``` + +- [ ] **Step 5: Run middleware tests** + +Run: `cd apps/backend && python -m pytest tests/test_audit_middleware.py -v` +Expected: all PASS. + +Note: `asyncio.create_task` is awaited implicitly because the monkeypatched `insert_one` resolves immediately and `TestClient` drives the loop to completion before returning; the assertions read `captured` after the response. If a created task is occasionally not yet flushed, the test fixture's `insert_one` runs synchronously enough for these assertions — if flakiness appears, change `asyncio.create_task(write_audit_event(doc))` call sites to `await write_audit_event(doc)` *inside tests* via a seam; do not change production behavior. + +- [ ] **Step 6: Run the full backend suite to confirm no regressions** + +Run: `cd apps/backend && python -m pytest -q` +Expected: PASS (existing tests unaffected; middleware skips GET/health). + +- [ ] **Step 7: Commit** + +```bash +git add apps/backend/app/core/audit_middleware.py apps/backend/app/main.py apps/backend/tests/test_audit_middleware.py +git commit -m "feat(audit): add HTTP middleware that auto-logs writes" +``` + +--- + +## Task 4: Enrich services with entity + diff (bookmarks worked example + auth events) + +**Files:** +- Modify: `apps/backend/app/api/routes/bookmarks/services.py` +- Modify: `apps/backend/app/api/routes/auth/api.py` +- Test: `apps/backend/tests/test_audit_middleware.py` (extend with a bookmarks-through-service test is out of scope; assert via helper-level unit below) + +**Interfaces:** +- Consumes: `app.core.audit` helper functions, `audit.diff`, `audit.SAFE_FIELDS`. +- Produces: bookmarks create/update/delete now call `audit.set_*`; auth login/logout/refresh/register call `audit.set_action`. + +This task establishes the enrichment **pattern**. Apply the same shape to other high-value modules in follow-up commits (tasks, notes, code_snippets fully; passwords/env/connections entity+action only). Only bookmarks + auth are implemented here so the pattern is concrete and tested. + +- [ ] **Step 1: Write a failing test for bookmark create enrichment** + +Append to `apps/backend/tests/test_audit_middleware.py`: + +```python +def test_bookmark_service_sets_audit_detail(monkeypatch): + import asyncio as _asyncio + from app.api.routes.bookmarks import services as bm + from app.api.routes.bookmarks.schema import BookmarkCreate + from app.core import audit + + async def fake_insert_one(collection_name, data): + return None + + monkeypatch.setattr("app.api.routes.bookmarks.services.db_manager.insert_one", fake_insert_one) + + async def run(): + tok = audit._audit_ctx.set(audit.AuditContext()) + try: + await bm.create_bookmark("uid1", BookmarkCreate(title="GitHub", url="https://gh.com")) + ctx = audit.current_context() + assert ctx.action == "bookmark.create" + assert ctx.entity_type == "bookmark" + assert ctx.entity_id # the new id + assert any(c["field"] == "title" and c["after"] == "GitHub" for c in (ctx.changes or [])) + finally: + audit._audit_ctx.reset(tok) + + _asyncio.run(run()) +``` + +(Adjust `BookmarkCreate(...)` kwargs if the schema requires more required fields — inspect `app/api/routes/bookmarks/schema.py`.) + +- [ ] **Step 2: Run to verify failure** + +Run: `cd apps/backend && python -m pytest tests/test_audit_middleware.py::test_bookmark_service_sets_audit_detail -v` +Expected: FAIL (no audit calls yet). + +- [ ] **Step 3: Enrich bookmarks services** + +In `apps/backend/app/api/routes/bookmarks/services.py`, add the import at the top: + +```python +from app.core import audit +``` + +In `create_bookmark`, right before `return _bookmark_doc_to_out(doc)` (after a successful `insert_one`): + +```python + audit.set_action("bookmark.create") + audit.set_entity("bookmark", bid) + audit.set_summary(f"Created bookmark '{body.title}'") + audit.set_changes(audit.diff(None, doc)) +``` + +In `update_bookmark`, capture the prior doc and diff it. Replace the body so the existing doc is read first: + +```python +async def update_bookmark(uid: str, bookmark_id: str, body: BookmarkUpdate) -> BookmarkOut: + patch = body.model_dump(exclude_unset=True) + if not patch: + return await get_bookmark(uid, bookmark_id) + before = await db_manager.find_one(BOOKMARKS, {"_id": bookmark_id, "created_by": uid}) + patch["updatedAt"] = create_timestamp() + try: + result = await db_manager.find_one_and_update( + BOOKMARKS, + {"_id": bookmark_id, "created_by": uid}, + {"$set": patch}, + return_document=ReturnDocument.AFTER, + ) + except PyMongoError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to update bookmark." + ) from exc + if not result: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Bookmark not found.") + audit.set_action("bookmark.update") + audit.set_entity("bookmark", bookmark_id) + audit.set_summary(f"Updated bookmark '{result.get('title', '')}'") + audit.set_changes(audit.diff(before, result)) + return _bookmark_doc_to_out(result) +``` + +In `delete_bookmark`, set detail before raising/returning: + +```python +async def delete_bookmark(uid: str, bookmark_id: str) -> None: + before = await db_manager.find_one(BOOKMARKS, {"_id": bookmark_id, "created_by": uid}) + result = await db_manager.delete_one(BOOKMARKS, {"_id": bookmark_id, "created_by": uid}) + if result.deleted_count == 0: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Bookmark not found.") + audit.set_action("bookmark.delete") + audit.set_entity("bookmark", bookmark_id) + title = (before or {}).get("title", "") + audit.set_summary(f"Deleted bookmark '{title}'") +``` + +- [ ] **Step 4: Run the bookmark enrichment test** + +Run: `cd apps/backend && python -m pytest tests/test_audit_middleware.py::test_bookmark_service_sets_audit_detail -v` +Expected: PASS. + +- [ ] **Step 5: Add auth event records** + +Inspect `apps/backend/app/api/routes/auth/api.py` to find the login, logout, token-refresh, and register handlers. In each handler body (after the operation succeeds), add `from app.core import audit` (top of file) and a matching call, e.g.: + +```python + audit.set_action("auth.login") + audit.set_summary("Signed in") +``` + +Use `"auth.logout"` / `"Signed out"`, `"auth.token_refresh"` / `"Refreshed session"`, `"auth.register"` / `"Created account"` in the corresponding handlers. (Entity is the user; `entity_id` may be set to the uid where available.) + +- [ ] **Step 6: Run the full backend suite** + +Run: `cd apps/backend && python -m pytest -q` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add apps/backend/app/api/routes/bookmarks/services.py apps/backend/app/api/routes/auth/api.py apps/backend/tests/test_audit_middleware.py +git commit -m "feat(audit): enrich bookmark services + auth events with audit detail" +``` + +--- + +## Task 5: Query API — GET /audit-log + +**Files:** +- Create: `apps/backend/app/api/routes/audit_log/__init__.py` (empty) +- Create: `apps/backend/app/api/routes/audit_log/schema.py` +- Create: `apps/backend/app/api/routes/audit_log/services.py` +- Create: `apps/backend/app/api/routes/audit_log/api.py` +- Modify: `apps/backend/app/api/router.py` +- Test: `apps/backend/tests/api/routes/audit_log/test_audit_query.py` (+ `__init__.py` files as needed) + +**Interfaces:** +- Consumes: `get_current_uid`, `db_manager.find` / `count_documents`, `AUDIT_LOG`. +- Produces: `GET /audit-log` → `AuditListOut { items: list[AuditEventOut], total, skip, limit }`. + +- [ ] **Step 1: Write failing query-service test** + +Create `apps/backend/tests/api/routes/audit_log/__init__.py` (empty) and `apps/backend/tests/api/routes/audit_log/test_audit_query.py`: + +```python +import pytest + +from app.api.routes.audit_log import services as svc + + +@pytest.fixture +def fake_db(monkeypatch): + rows = [ + {"_id": "1", "uid": "u1", "action": "bookmark.create", "module": "bookmarks", + "entity_type": "bookmark", "entity_id": "b1", "method": "POST", + "path": "/api/v1/bookmarks", "status": 200, "outcome": "success", + "changes": [{"field": "title", "before": None, "after": "GitHub"}], + "summary": "Created bookmark 'GitHub'", "ip": "1.2.3.4", "ua_raw": "UA", + "device": {"browser": "Chrome", "os": "macOS", "device_type": "desktop"}, + "latency_ms": 12, "ts": 1000, "expireAt": "x"}, + ] + + async def fake_find(collection_name, query, projection=None, sort=None, skip=0, limit=0, collation=None): + assert query["uid"] == "u1" + return rows + + async def fake_count(collection_name, query): + return len(rows) + + monkeypatch.setattr("app.api.routes.audit_log.services.db_manager.find", fake_find) + monkeypatch.setattr("app.api.routes.audit_log.services.db_manager.count_documents", fake_count) + return rows + + +@pytest.mark.asyncio +async def test_list_audit_events_scopes_to_uid_and_excludes_expireAt(fake_db): + out = await svc.list_audit_events("u1", skip=0, limit=50) + assert out.total == 1 + assert out.items[0].action == "bookmark.create" + assert not hasattr(out.items[0], "expireAt") +``` + +If the project lacks `pytest-asyncio` config, wrap the call with `asyncio.run` instead of the marker (match the style used elsewhere in `tests/`). + +- [ ] **Step 2: Run to verify failure** + +Run: `cd apps/backend && python -m pytest tests/api/routes/audit_log/ -v` +Expected: FAIL (`ModuleNotFoundError`). + +- [ ] **Step 3: Implement schema** + +`apps/backend/app/api/routes/audit_log/schema.py`: + +```python +from typing import Any, Optional + +from pydantic import BaseModel + + +class AuditChange(BaseModel): + field: str + before: Any | None = None + after: Any | None = None + + +class AuditDevice(BaseModel): + browser: str + os: str + device_type: str + + +class AuditEventOut(BaseModel): + id: str + uid: Optional[str] = None + action: str + module: Optional[str] = None + entity_type: Optional[str] = None + entity_id: Optional[str] = None + method: str + path: str + status: int + outcome: str + changes: Optional[list[AuditChange]] = None + summary: Optional[str] = None + ip: Optional[str] = None + ua_raw: Optional[str] = None + device: Optional[AuditDevice] = None + latency_ms: int + ts: int + + +class AuditListOut(BaseModel): + items: list[AuditEventOut] + total: int + skip: int + limit: int +``` + +- [ ] **Step 4: Implement services** + +`apps/backend/app/api/routes/audit_log/services.py`: + +```python +from typing import Any, Optional + +from app.database import db_manager +from app.utils.collection_name import AUDIT_LOG +from app.api.routes.audit_log.schema import AuditEventOut, AuditListOut + + +def _doc_to_out(doc: dict[str, Any]) -> AuditEventOut: + return AuditEventOut( + id=str(doc.get("_id", "")), + uid=doc.get("uid"), + action=doc.get("action", ""), + module=doc.get("module"), + entity_type=doc.get("entity_type"), + entity_id=doc.get("entity_id"), + method=doc.get("method", ""), + path=doc.get("path", ""), + status=int(doc.get("status", 0)), + outcome=doc.get("outcome", ""), + changes=doc.get("changes"), + summary=doc.get("summary"), + ip=doc.get("ip"), + ua_raw=doc.get("ua_raw"), + device=doc.get("device"), + latency_ms=int(doc.get("latency_ms", 0)), + ts=int(doc.get("ts", 0)), + ) + + +async def list_audit_events( + uid: str, + *, + skip: int = 0, + limit: int = 50, + module: Optional[str] = None, + action: Optional[str] = None, + outcome: Optional[str] = None, + ts_from: Optional[int] = None, + ts_to: Optional[int] = None, + search: Optional[str] = None, +) -> AuditListOut: + query: dict[str, Any] = {"uid": uid} + if module: + query["module"] = module + if action: + query["action"] = action + if outcome: + query["outcome"] = outcome + if ts_from is not None or ts_to is not None: + rng: dict[str, Any] = {} + if ts_from is not None: + rng["$gte"] = ts_from + if ts_to is not None: + rng["$lte"] = ts_to + query["ts"] = rng + if search: + query["summary"] = {"$regex": search, "$options": "i"} + + total = await db_manager.count_documents(AUDIT_LOG, query) + docs = await db_manager.find( + AUDIT_LOG, query, sort=[("ts", -1)], skip=skip, limit=limit + ) + return AuditListOut( + items=[_doc_to_out(d) for d in docs], + total=total, + skip=skip, + limit=limit, + ) +``` + +- [ ] **Step 5: Implement API** + +`apps/backend/app/api/routes/audit_log/api.py`: + +```python +from typing import Optional + +from fastapi import APIRouter, Depends, Query + +from app.api.routes.auth.services import get_current_uid +from app.api.routes.audit_log import services as svc +from app.api.routes.audit_log.schema import AuditListOut + +router = APIRouter(prefix="/audit-log", tags=["audit-log"]) + + +@router.get("", response_model=AuditListOut, summary="List the current user's audit events") +async def list_events( + uid: str = Depends(get_current_uid), + skip: int = Query(default=0, ge=0), + limit: int = Query(default=50, ge=1, le=100), + module: Optional[str] = Query(default=None), + action: Optional[str] = Query(default=None), + outcome: Optional[str] = Query(default=None), + ts_from: Optional[int] = Query(default=None, alias="from"), + ts_to: Optional[int] = Query(default=None, alias="to"), + search: Optional[str] = Query(default=None), +) -> AuditListOut: + return await svc.list_audit_events( + uid, skip=skip, limit=limit, module=module, action=action, + outcome=outcome, ts_from=ts_from, ts_to=ts_to, search=search, + ) +``` + +- [ ] **Step 6: Wire the router** + +In `apps/backend/app/api/router.py`, add the import with the others: + +```python +from app.api.routes.audit_log.api import router as audit_log_router +``` + +and register it: + +```python +api_router.include_router(audit_log_router) +``` + +- [ ] **Step 7: Run query tests + full suite** + +Run: `cd apps/backend && python -m pytest tests/api/routes/audit_log/ -v && python -m pytest -q` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add apps/backend/app/api/routes/audit_log apps/backend/app/api/router.py apps/backend/tests/api/routes/audit_log +git commit -m "feat(audit): add paginated, filterable GET /audit-log API" +``` + +--- + +## Task 6: Frontend API client + +**Files:** +- Create: `apps/web/src/lib/audit-log-api.ts` +- Test: `apps/web/src/lib/__tests__/audit-log-api.test.ts` + +**Interfaces:** +- Consumes: `backendFetch` from `@/lib/backend-auth`. +- Produces: types `AuditEvent`, `AuditListResponse`, `AuditQuery`; `fetchAuditLog(query?: AuditQuery): Promise`. + +- [ ] **Step 1: Write a failing test for query-string building** + +Create `apps/web/src/lib/__tests__/audit-log-api.test.ts`: + +```ts +import { describe, it, expect, vi, beforeEach } from 'vitest' + +const backendFetch = vi.fn() +vi.mock('@/lib/backend-auth', () => ({ backendFetch: (...a: unknown[]) => backendFetch(...a) })) + +import { fetchAuditLog } from '@/lib/audit-log-api' + +describe('fetchAuditLog', () => { + beforeEach(() => backendFetch.mockReset()) + + it('builds the query string and returns parsed data', async () => { + backendFetch.mockResolvedValue({ + ok: true, + json: async () => ({ items: [], total: 0, skip: 0, limit: 50 }), + }) + const res = await fetchAuditLog({ module: 'bookmarks', limit: 50 }) + const url = backendFetch.mock.calls[0][0] as string + expect(url).toContain('/api/backend/audit-log') + expect(url).toContain('module=bookmarks') + expect(url).toContain('limit=50') + expect(res.total).toBe(0) + }) + + it('throws on non-ok response', async () => { + backendFetch.mockResolvedValue({ ok: false, status: 500, text: async () => 'boom' }) + await expect(fetchAuditLog()).rejects.toThrow() + }) +}) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cd apps/web && pnpm vitest run src/lib/__tests__/audit-log-api.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement `src/lib/audit-log-api.ts`** + +```ts +import { backendFetch } from '@/lib/backend-auth' + +export type AuditChange = { field: string; before: unknown; after: unknown } +export type AuditDevice = { browser: string; os: string; device_type: string } + +export type AuditEvent = { + id: string + uid: string | null + action: string + module: string | null + entity_type: string | null + entity_id: string | null + method: string + path: string + status: number + outcome: 'success' | 'failure' + changes: AuditChange[] | null + summary: string | null + ip: string | null + ua_raw: string | null + device: AuditDevice | null + latency_ms: number + ts: number +} + +export type AuditListResponse = { + items: AuditEvent[] + total: number + skip: number + limit: number +} + +export type AuditQuery = { + skip?: number + limit?: number + module?: string + action?: string + outcome?: 'success' | 'failure' + from?: number + to?: number + search?: string +} + +export async function fetchAuditLog(query: AuditQuery = {}): Promise { + const params = new URLSearchParams() + for (const [key, value] of Object.entries(query)) { + if (value !== undefined && value !== null && value !== '') { + params.set(key, String(value)) + } + } + const qs = params.toString() + const url = `/api/backend/audit-log${qs ? `?${qs}` : ''}` + const res = await backendFetch(url, { method: 'GET' }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new Error(text || `Audit log failed (${res.status})`) + } + return (await res.json()) as AuditListResponse +} +``` + +- [ ] **Step 4: Run the test** + +Run: `cd apps/web && pnpm vitest run src/lib/__tests__/audit-log-api.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/web/src/lib/audit-log-api.ts apps/web/src/lib/__tests__/audit-log-api.test.ts +git commit -m "feat(audit): add web audit-log API client" +``` + +--- + +## Task 7: Activity log page (UI) + +**Files:** +- Create: `apps/web/src/components/dashboard/activity/audit-event-row.tsx` +- Create: `apps/web/src/components/dashboard/activity/activity-log-panel.tsx` +- Create: `apps/web/src/app/dashboard/activity/page.tsx` +- Modify: `apps/web/messages/en.json` + +**Interfaces:** +- Consumes: `fetchAuditLog`, `AuditEvent` from `@/lib/audit-log-api`; `useTranslations('Dashboard.activity')`. + +- [ ] **Step 1: Add i18n labels** + +In `apps/web/messages/en.json`, inside the existing `"Dashboard"` object, add an `"activity"` block: + +```json +"activity": { + "title": "Activity log", + "subtitle": "Everything you changed, when, and from which device.", + "filterModule": "Module", + "filterOutcome": "Outcome", + "search": "Search", + "all": "All", + "success": "Success", + "failure": "Failure", + "empty": "No activity yet.", + "loadError": "Could not load activity.", + "loadMore": "Load more", + "changedFields": "What changed", + "device": "Device", + "before": "Before", + "after": "After" +} +``` + +(Other locale files may be filled later; `en.json` is the source of truth.) + +- [ ] **Step 2: Implement the event row** + +`apps/web/src/components/dashboard/activity/audit-event-row.tsx`: + +```tsx +'use client' + +import { useState } from 'react' +import { useTranslations } from 'next-intl' +import type { AuditEvent } from '@/lib/audit-log-api' + +function relativeTime(ts: number): string { + const diff = Date.now() - ts + const mins = Math.floor(diff / 60000) + if (mins < 1) return 'just now' + if (mins < 60) return `${mins}m ago` + const hrs = Math.floor(mins / 60) + if (hrs < 24) return `${hrs}h ago` + return new Date(ts).toLocaleDateString() +} + +export function AuditEventRow({ event }: { event: AuditEvent }) { + const t = useTranslations('Dashboard.activity') + const [open, setOpen] = useState(false) + const hasChanges = !!event.changes && event.changes.length > 0 + const device = event.device + ? `${event.device.browser} on ${event.device.os}` + : '—' + + return ( +
+ + + {open && hasChanges && ( +
+
{t('changedFields')}
+
    + {event.changes!.map((c) => ( +
  • + {c.field}: + {String(c.before ?? '∅')} + → + {String(c.after ?? '∅')} +
    • + ))} +
+
+ )} +
+ ) +} +``` + +- [ ] **Step 3: Implement the panel** + +`apps/web/src/components/dashboard/activity/activity-log-panel.tsx`: + +```tsx +'use client' + +import { useCallback, useEffect, useState } from 'react' +import { useTranslations } from 'next-intl' +import { fetchAuditLog, type AuditEvent } from '@/lib/audit-log-api' +import { AuditEventRow } from './audit-event-row' + +const PAGE = 50 + +export function ActivityLogPanel() { + const t = useTranslations('Dashboard.activity') + const [events, setEvents] = useState([]) + const [total, setTotal] = useState(0) + const [skip, setSkip] = useState(0) + const [outcome, setOutcome] = useState<'' | 'success' | 'failure'>('') + const [search, setSearch] = useState('') + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + const load = useCallback( + async (reset: boolean) => { + setLoading(true) + setError(null) + try { + const nextSkip = reset ? 0 : skip + const res = await fetchAuditLog({ + skip: nextSkip, + limit: PAGE, + outcome: outcome || undefined, + search: search || undefined, + }) + setTotal(res.total) + setSkip(nextSkip + res.items.length) + setEvents((prev) => (reset ? res.items : [...prev, ...res.items])) + } catch (e) { + setError(e instanceof Error ? e.message : t('loadError')) + } finally { + setLoading(false) + } + }, + [skip, outcome, search, t], + ) + + useEffect(() => { + void load(true) + // reload when filters change + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [outcome, search]) + + return ( +
+

{t('title')}

+

{t('subtitle')}

+ +
+ + setSearch(e.target.value)} + className="flex-1 rounded-md border border-border bg-background px-2 py-1 text-sm" + /> +
+ + {error &&
{error}
} + {!error && events.length === 0 && !loading && ( +
{t('empty')}
+ )} + +
+ {events.map((e) => ( + + ))} +
+ + {events.length < total && ( + + )} +
+ ) +} +``` + +- [ ] **Step 4: Implement the route page** + +`apps/web/src/app/dashboard/activity/page.tsx`: + +```tsx +'use client' + +import { ActivityLogPanel } from '@/components/dashboard/activity/activity-log-panel' + +export default function ActivityLogRoute() { + return +} +``` + +- [ ] **Step 5: Typecheck + build the web app** + +Run: `cd apps/web && pnpm vitest run src/lib/__tests__/audit-log-api.test.ts && pnpm tsc --noEmit` +Expected: tests PASS, no type errors. (If the repo uses `pnpm lint`/`pnpm build` as the gate, run that instead.) + +- [ ] **Step 6: Manual verification** + +Start the app, sign in, perform a few actions (create/edit/delete a bookmark, log out/in), then visit `/dashboard/activity`. Confirm: rows appear newest-first; device shows "Chrome on macOS"; expanding a row shows field diffs; password/connection edits show `[redacted]`; outcome failures appear for a forced 404. + +- [ ] **Step 7: Commit** + +```bash +git add apps/web/src/app/dashboard/activity apps/web/src/components/dashboard/activity apps/web/messages/en.json +git commit -m "feat(audit): add Activity log page" +``` + +--- + +## Self-Review + +**Spec coverage:** +- Data model + TTL + indexes → Task 1. +- Context helper, redaction allowlist, diff, UA parse → Task 2. +- Hybrid middleware (auto-envelope, fire-and-forget, safety invariant, device/IP) → Task 3. +- Auth events + failed writes + per-module diff enrichment pattern → Tasks 3 (failure path) + 4. +- Query API (paginated, filterable, uid-scoped, excludes `expireAt`) → Task 5. +- Web client + Activity log UI (filters, timeline, device, expandable diffs, i18n, redacted display) → Tasks 6–7. + +**Known follow-up (explicitly out of this plan's scope, noted for the executor):** Task 4 implements the enrichment pattern for bookmarks + auth only. Tasks/notes/code_snippets (full diffs) and passwords/env/connections (entity+action, redacted) follow the identical shape in later commits. All those modules already get the auto-envelope from Task 3 without any change, so coverage is complete from day one; enrichment only adds richer `summary`/`changes`. + +**Placeholder scan:** none — every code step has full content. + +**Type consistency:** `AuditContext` fields, `audit.set_*`/`diff`/`parse_user_agent` signatures, `AuditEventOut`/`AuditListOut` shapes, and the TS `AuditEvent`/`AuditListResponse`/`fetchAuditLog` types are consistent across tasks (snake_case preserved end-to-end since the API returns the raw field names). From c91ddafc63807aae05c1d0ee70484c2ded654e31 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 14:14:35 +0530 Subject: [PATCH 03/52] feat(audit): add audit_log collection + TTL index support --- apps/backend/app/core/indexes.py | 4 ++++ apps/backend/app/database/db_manager.py | 9 +++++++-- apps/backend/app/utils/collection_name.py | 3 ++- apps/backend/tests/test_audit_helper.py | 5 +++++ 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 apps/backend/tests/test_audit_helper.py diff --git a/apps/backend/app/core/indexes.py b/apps/backend/app/core/indexes.py index 45195398..13e32c17 100644 --- a/apps/backend/app/core/indexes.py +++ b/apps/backend/app/core/indexes.py @@ -3,6 +3,7 @@ API_CLIENT_HISTORY, API_CLIENT_COLLECTIONS, API_CLIENT_ENVIRONMENTS, + AUDIT_LOG, BOOKMARK_FOLDERS, BOOKMARKS, CODE_SNIPPETS, @@ -52,3 +53,6 @@ async def ensure_indexes() -> None: await db_manager.create_index(GAME_SCORES, [("created_by", 1), ("updatedAt", -1)]) await db_manager.create_index(FEEDBACK, [("created_by", 1), ("createdAt", -1)]) await db_manager.create_index(URL_CLICK_EVENTS, [("code", 1), ("ts", 1)]) + await db_manager.create_index(AUDIT_LOG, [("uid", 1), ("ts", -1)]) + await db_manager.create_index(AUDIT_LOG, [("uid", 1), ("module", 1), ("ts", -1)]) + await db_manager.create_index(AUDIT_LOG, "expireAt", expire_after_seconds=0) diff --git a/apps/backend/app/database/db_manager.py b/apps/backend/app/database/db_manager.py index 531d3c8c..7d5c5e4a 100644 --- a/apps/backend/app/database/db_manager.py +++ b/apps/backend/app/database/db_manager.py @@ -85,8 +85,13 @@ async def aggregate(collection_name, query): return await cursor.to_list(length=None) -async def create_index(collection_name, field, unique=False, sparse=False): - await db[collection_name].create_index(field, unique=unique, sparse=sparse) +async def create_index( + collection_name, field, unique=False, sparse=False, expire_after_seconds=None +): + kwargs = {"unique": unique, "sparse": sparse} + if expire_after_seconds is not None: + kwargs["expireAfterSeconds"] = expire_after_seconds + await db[collection_name].create_index(field, **kwargs) async def drop_index(collection_name, name): diff --git a/apps/backend/app/utils/collection_name.py b/apps/backend/app/utils/collection_name.py index 6cf6e8a1..3cdbf037 100644 --- a/apps/backend/app/utils/collection_name.py +++ b/apps/backend/app/utils/collection_name.py @@ -21,4 +21,5 @@ FEEDBACK = "feedback" REDIS_CONNECTIONS = "redis_connections" URL_LINKS = "url_links" -URL_CLICK_EVENTS = "url_click_events" \ No newline at end of file +URL_CLICK_EVENTS = "url_click_events" +AUDIT_LOG = "audit_log" \ No newline at end of file diff --git a/apps/backend/tests/test_audit_helper.py b/apps/backend/tests/test_audit_helper.py new file mode 100644 index 00000000..f34b1adf --- /dev/null +++ b/apps/backend/tests/test_audit_helper.py @@ -0,0 +1,5 @@ +from app.utils.collection_name import AUDIT_LOG + + +def test_audit_log_collection_name(): + assert AUDIT_LOG == "audit_log" From 5c7d2cfb09922fc518ddab36bb0c47a558f66d21 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 14:19:05 +0530 Subject: [PATCH 04/52] feat(audit): add context helper, redacted diff, and UA parser --- apps/backend/app/core/audit.py | 126 ++++++++++++++++++++++++ apps/backend/tests/test_audit_helper.py | 65 ++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 apps/backend/app/core/audit.py diff --git a/apps/backend/app/core/audit.py b/apps/backend/app/core/audit.py new file mode 100644 index 00000000..d418d4f6 --- /dev/null +++ b/apps/backend/app/core/audit.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import re +from contextvars import ContextVar +from dataclasses import dataclass, field + +# Field names whose values are safe to record verbatim in the audit diff. +# Anything NOT in this set is recorded as "[redacted]" (default-deny). +SAFE_FIELDS: set[str] = { + "title", "name", "tags", "folderId", "parentId", "status", "statusOrder", + "color", "icon", "description", "url", "isExpanded", "projectId", + "priority", "dueDate", "completed", "language", "createdAt", "updatedAt", +} + +REDACTED = "[redacted]" + + +@dataclass +class AuditContext: + action: str | None = None + module: str | None = None + entity_type: str | None = None + entity_id: str | None = None + changes: list[dict] | None = None + summary: str | None = None + + +_audit_ctx: ContextVar[AuditContext | None] = ContextVar("audit_ctx", default=None) + + +def current_context() -> AuditContext | None: + return _audit_ctx.get() + + +def set_action(action: str) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.action = action + + +def set_entity(entity_type: str, entity_id: str | None) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.entity_type = entity_type + ctx.entity_id = entity_id + + +def set_summary(text: str) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.summary = text + + +def set_changes(changes: list[dict]) -> None: + ctx = _audit_ctx.get() + if ctx is not None: + ctx.changes = changes + + +def add_change(field_name: str, before, after) -> None: + ctx = _audit_ctx.get() + if ctx is None: + return + if ctx.changes is None: + ctx.changes = [] + ctx.changes.append({"field": field_name, "before": before, "after": after}) + + +def _redact(field_name: str, value, allow_fields: set[str]): + return value if field_name in allow_fields else REDACTED + + +def diff(before: dict | None, after: dict | None, allow_fields: set[str] | None = None) -> list[dict]: + allow = SAFE_FIELDS if allow_fields is None else allow_fields + before = before or {} + after = after or {} + changes: list[dict] = [] + for key in sorted(set(before) | set(after)): + if key in ("_id", "created_by"): + continue + b = before.get(key) + a = after.get(key) + if b == a: + continue + changes.append({ + "field": key, + "before": _redact(key, b, allow), + "after": _redact(key, a, allow), + }) + return changes + + +_BROWSERS = [ + ("Edg", "Edge"), ("OPR", "Opera"), ("Chrome", "Chrome"), + ("Firefox", "Firefox"), ("Safari", "Safari"), +] + + +def parse_user_agent(ua: str | None) -> dict: + fallback = {"browser": "Unknown", "os": "Unknown", "device_type": "desktop"} + if not ua: + return fallback + browser = "Unknown" + for token, name in _BROWSERS: + if token in ua: + browser = name + break + if "Windows" in ua: + os_name = "Windows" + elif "Mac OS X" in ua or "Macintosh" in ua: + os_name = "macOS" + elif "Android" in ua: + os_name = "Android" + elif "iPhone" in ua or "iPad" in ua or "iOS" in ua: + os_name = "iOS" + elif "Linux" in ua: + os_name = "Linux" + else: + os_name = "Unknown" + if re.search(r"Mobi|iPhone|Android.*Mobile", ua): + device_type = "mobile" + elif "iPad" in ua or ("Android" in ua and "Mobile" not in ua): + device_type = "tablet" + else: + device_type = "desktop" + return {"browser": browser, "os": os_name, "device_type": device_type} diff --git a/apps/backend/tests/test_audit_helper.py b/apps/backend/tests/test_audit_helper.py index f34b1adf..3d7d8b5f 100644 --- a/apps/backend/tests/test_audit_helper.py +++ b/apps/backend/tests/test_audit_helper.py @@ -1,5 +1,70 @@ from app.utils.collection_name import AUDIT_LOG +from app.core import audit def test_audit_log_collection_name(): assert AUDIT_LOG == "audit_log" + + +def test_diff_passes_safe_fields_through(): + changes = audit.diff({"title": "Old"}, {"title": "New"}) + assert changes == [{"field": "title", "before": "Old", "after": "New"}] + + +def test_diff_redacts_sensitive_fields(): + changes = audit.diff( + {"password": "old-secret"}, {"password": "new-secret"} + ) + assert changes == [ + {"field": "password", "before": "[redacted]", "after": "[redacted]"} + ] + + +def test_diff_redacts_unknown_fields_by_default(): + changes = audit.diff({"mystery": 1}, {"mystery": 2}) + assert changes == [ + {"field": "mystery", "before": "[redacted]", "after": "[redacted]"} + ] + + +def test_diff_ignores_unchanged_fields(): + assert audit.diff({"title": "Same"}, {"title": "Same"}) == [] + + +def test_diff_handles_create_and_delete(): + assert audit.diff(None, {"title": "New"}) == [ + {"field": "title", "before": None, "after": "New"} + ] + assert audit.diff({"title": "Gone"}, None) == [ + {"field": "title", "before": "Gone", "after": None} + ] + + +def test_parse_user_agent_chrome_macos(): + d = audit.parse_user_agent( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0 Safari/537.36" + ) + assert d["browser"] == "Chrome" + assert d["os"] == "macOS" + assert d["device_type"] == "desktop" + + +def test_parse_user_agent_fallback(): + d = audit.parse_user_agent(None) + assert d == {"browser": "Unknown", "os": "Unknown", "device_type": "desktop"} + + +def test_context_mutation_round_trip(): + token = audit._audit_ctx.set(audit.AuditContext()) + try: + audit.set_entity("bookmark", "abc") + audit.set_action("bookmark.create") + audit.add_change("title", None, "Hi") + ctx = audit.current_context() + assert ctx.entity_type == "bookmark" + assert ctx.entity_id == "abc" + assert ctx.action == "bookmark.create" + assert ctx.changes == [{"field": "title", "before": None, "after": "Hi"}] + finally: + audit._audit_ctx.reset(token) From 9abe36c782fdb8efff4a659df455170df212eb1d Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 14:29:46 +0530 Subject: [PATCH 05/52] feat(audit): add HTTP middleware that auto-logs writes --- apps/backend/app/core/audit_middleware.py | 111 ++++++++++++++++++++ apps/backend/app/main.py | 3 + apps/backend/tests/test_audit_middleware.py | 88 ++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 apps/backend/app/core/audit_middleware.py create mode 100644 apps/backend/tests/test_audit_middleware.py diff --git a/apps/backend/app/core/audit_middleware.py b/apps/backend/app/core/audit_middleware.py new file mode 100644 index 00000000..2739fff9 --- /dev/null +++ b/apps/backend/app/core/audit_middleware.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import asyncio +import datetime +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +from app.core import audit +from app.core.auth_cookies import ACCESS_COOKIE_NAME +from app.database import db_manager +from app.api.routes.auth.tokens import try_decode_access_token_uid +from app.utils.collection_name import AUDIT_LOG +from app.utils.utils import create_timestamp, new_id + +logger = logging.getLogger(__name__) + +_SKIP_METHODS = {"GET", "HEAD", "OPTIONS"} +_TTL_DAYS = 90 + + +def _extract_uid(request: Request) -> str | None: + token = None + auth = request.headers.get("authorization") + if auth: + scheme, _, value = auth.partition(" ") + if scheme.lower() == "bearer" and value.strip(): + token = value.strip() + if not token: + cookie = request.cookies.get(ACCESS_COOKIE_NAME) + if cookie and cookie.strip(): + token = cookie.strip() + if not token: + return None + try: + return try_decode_access_token_uid(token) + except Exception: # never let auth decode break auditing + return None + + +def _client_ip(request: Request) -> str | None: + fwd = request.headers.get("x-forwarded-for") + if fwd: + return fwd.split(",")[0].strip() + return request.client.host if request.client else None + + +def _module_from_path(path: str) -> str | None: + # /api/v1//... + parts = [p for p in path.split("/") if p] + if len(parts) >= 3 and parts[0] == "api" and parts[1] == "v1": + return parts[2] + return None + + +async def write_audit_event(doc: dict) -> None: + try: + await db_manager.insert_one(AUDIT_LOG, doc) + except Exception as exc: # swallow — auditing must never break requests + logger.warning("audit write failed: %s", exc) + + +class AuditMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + path = request.url.path + if request.method in _SKIP_METHODS or not path.startswith("/api/v1"): + return await call_next(request) + if path.startswith("/api/v1/health"): + return await call_next(request) + + ctx = audit.AuditContext() + token = audit._audit_ctx.set(ctx) + started = create_timestamp() + uid = _extract_uid(request) + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + return response + finally: + try: + latency = create_timestamp() - started + module = ctx.module or _module_from_path(path) + action = ctx.action or (f"{module}.{request.method.lower()}" if module else request.method.lower()) + ts = create_timestamp() + doc = { + "_id": new_id(), + "uid": uid, + "action": action, + "module": module, + "entity_type": ctx.entity_type, + "entity_id": ctx.entity_id, + "method": request.method, + "path": path, + "status": status_code, + "outcome": "success" if status_code < 400 else "failure", + "changes": ctx.changes, + "summary": ctx.summary, + "ip": _client_ip(request), + "ua_raw": request.headers.get("user-agent"), + "device": audit.parse_user_agent(request.headers.get("user-agent")), + "latency_ms": latency, + "ts": ts, + "expireAt": datetime.datetime.utcnow() + datetime.timedelta(days=_TTL_DAYS), + } + asyncio.create_task(write_audit_event(doc)) + except Exception as exc: # never propagate + logger.warning("audit envelope build failed: %s", exc) + finally: + audit._audit_ctx.reset(token) diff --git a/apps/backend/app/main.py b/apps/backend/app/main.py index 8dea95a2..81b8ee96 100644 --- a/apps/backend/app/main.py +++ b/apps/backend/app/main.py @@ -9,6 +9,7 @@ from slowapi.middleware import SlowAPIMiddleware from app.api.router import api_router +from app.core.audit_middleware import AuditMiddleware from app.core.config import get_settings from app.core.limiter import limiter @@ -46,6 +47,8 @@ async def lifespan(_app: FastAPI): allow_headers=["*"], ) +app.add_middleware(AuditMiddleware) + @app.middleware("http") async def security_headers(request: Request, call_next) -> Response: diff --git a/apps/backend/tests/test_audit_middleware.py b/apps/backend/tests/test_audit_middleware.py new file mode 100644 index 00000000..8c7ec953 --- /dev/null +++ b/apps/backend/tests/test_audit_middleware.py @@ -0,0 +1,88 @@ +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.core import audit +from app.core.audit_middleware import AuditMiddleware + + +@pytest.fixture +def captured(monkeypatch): + docs = [] + + async def fake_insert_one(collection_name, data): + docs.append((collection_name, data)) + + # Run the fire-and-forget task synchronously so assertions are deterministic. + monkeypatch.setattr("app.core.audit_middleware.db_manager.insert_one", fake_insert_one) + return docs + + +def build_app(): + app = FastAPI() + app.add_middleware(AuditMiddleware) + + @app.post("/api/v1/bookmarks") + async def create(): + audit.set_action("bookmark.create") + audit.set_entity("bookmark", "bk1") + audit.set_summary("Created bookmark 'GitHub'") + audit.add_change("title", None, "GitHub") + return {"id": "bk1"} + + @app.get("/api/v1/bookmarks") + async def listing(): + return [] + + @app.delete("/api/v1/bookmarks/{bid}") + async def fail(bid: str): + from fastapi import HTTPException + raise HTTPException(status_code=404, detail="nope") + + return app + + +def test_write_is_logged_with_envelope_and_detail(captured): + client = TestClient(build_app()) + res = client.post( + "/api/v1/bookmarks", + headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120.0 Safari/537.36"}, + ) + assert res.status_code == 200 + assert len(captured) == 1 + _, doc = captured[0] + assert doc["action"] == "bookmark.create" + assert doc["module"] == "bookmarks" + assert doc["entity_id"] == "bk1" + assert doc["method"] == "POST" + assert doc["outcome"] == "success" + assert doc["device"]["browser"] == "Chrome" + assert doc["changes"] == [{"field": "title", "before": None, "after": "GitHub"}] + assert "expireAt" in doc and "ts" in doc + + +def test_get_is_not_logged(captured): + client = TestClient(build_app()) + client.get("/api/v1/bookmarks") + assert captured == [] + + +def test_failed_write_logged_as_failure(captured): + client = TestClient(build_app()) + res = client.delete("/api/v1/bookmarks/bk1") + assert res.status_code == 404 + assert len(captured) == 1 + _, doc = captured[0] + assert doc["outcome"] == "failure" + assert doc["status"] == 404 + assert doc["module"] == "bookmarks" + + +def test_audit_write_failure_does_not_break_request(monkeypatch): + async def boom(collection_name, data): + raise RuntimeError("db down") + + monkeypatch.setattr("app.core.audit_middleware.db_manager.insert_one", boom) + client = TestClient(build_app()) + res = client.post("/api/v1/bookmarks") + assert res.status_code == 200 # user response unaffected From 53e8b2a69388c40a9d0e57d74e2e314cf3527a44 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 14:41:54 +0530 Subject: [PATCH 06/52] feat(audit): enrich bookmark services + auth events with audit detail --- apps/backend/app/api/routes/auth/api.py | 11 ++++++++ .../app/api/routes/bookmarks/services.py | 15 +++++++++++ apps/backend/tests/test_audit_middleware.py | 26 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/apps/backend/app/api/routes/auth/api.py b/apps/backend/app/api/routes/auth/api.py index 36833375..04f89507 100644 --- a/apps/backend/app/api/routes/auth/api.py +++ b/apps/backend/app/api/routes/auth/api.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Cookie, Depends, Header, HTTPException, Request, Response, status from app.core.limiter import limiter +from app.core import audit from app.api.routes.auth.cookie_attach import attach_auth_cookies, clear_auth_cookies from app.api.routes.auth.schema import ( @@ -73,6 +74,9 @@ async def create_session(request: Request, payload: SessionRequest, response: Re status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="User record missing after upsert.", ) + audit.set_action("auth.login") + audit.set_entity("user", uid) + audit.set_summary("Signed in") return UserProfileResponse( uid=str(doc["_id"]), email=doc.get("email"), @@ -111,6 +115,9 @@ async def refresh_session( await set_refresh_token_hash(uid, hash_refresh_token(new_raw)) access = create_access_token(uid) attach_auth_cookies(response, access, new_raw) + audit.set_action("auth.token_refresh") + audit.set_entity("user", uid) + audit.set_summary("Refreshed session") return OkResponse(ok=True) @@ -143,6 +150,10 @@ async def logout( clear_auth_cookies(response) if uid: await clear_refresh_token_hash(uid) + audit.set_action("auth.logout") + if uid: + audit.set_entity("user", uid) + audit.set_summary("Signed out") return OkResponse(ok=True) diff --git a/apps/backend/app/api/routes/bookmarks/services.py b/apps/backend/app/api/routes/bookmarks/services.py index ee8f8799..dcb11267 100644 --- a/apps/backend/app/api/routes/bookmarks/services.py +++ b/apps/backend/app/api/routes/bookmarks/services.py @@ -5,6 +5,7 @@ from pymongo.errors import PyMongoError from pymongo import ReturnDocument from app.utils.utils import new_id, create_timestamp, is_duplicate_key_error +from app.core import audit from app.utils.collection_name import BOOKMARK_FOLDERS as FOLDERS, BOOKMARKS from app.database import db_manager @@ -100,6 +101,10 @@ async def create_bookmark(uid: str, body: BookmarkCreate) -> BookmarkOut: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to create bookmark." ) from exc + audit.set_action("bookmark.create") + audit.set_entity("bookmark", bid) + audit.set_summary(f"Created bookmark '{body.title}'") + audit.set_changes(audit.diff(None, doc)) return _bookmark_doc_to_out(doc) @@ -107,6 +112,7 @@ async def update_bookmark(uid: str, bookmark_id: str, body: BookmarkUpdate) -> B patch = body.model_dump(exclude_unset=True) if not patch: return await get_bookmark(uid, bookmark_id) + before = await db_manager.find_one(BOOKMARKS, {"_id": bookmark_id, "created_by": uid}) patch["updatedAt"] = create_timestamp() try: result = await db_manager.find_one_and_update( @@ -121,6 +127,10 @@ async def update_bookmark(uid: str, bookmark_id: str, body: BookmarkUpdate) -> B ) from exc if not result: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Bookmark not found.") + audit.set_action("bookmark.update") + audit.set_entity("bookmark", bookmark_id) + audit.set_summary(f"Updated bookmark '{result.get('title', '')}'") + audit.set_changes(audit.diff(before, result)) return _bookmark_doc_to_out(result) @@ -129,9 +139,14 @@ async def move_bookmark(uid: str, bookmark_id: str, body: BookmarkMove) -> Bookm async def delete_bookmark(uid: str, bookmark_id: str) -> None: + before = await db_manager.find_one(BOOKMARKS, {"_id": bookmark_id, "created_by": uid}) result = await db_manager.delete_one(BOOKMARKS, {"_id": bookmark_id, "created_by": uid}) if result.deleted_count == 0: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Bookmark not found.") + audit.set_action("bookmark.delete") + audit.set_entity("bookmark", bookmark_id) + title = (before or {}).get("title", "") + audit.set_summary(f"Deleted bookmark '{title}'") async def import_bookmarks(uid: str, body: BookmarkImportBody) -> dict[str, int]: diff --git a/apps/backend/tests/test_audit_middleware.py b/apps/backend/tests/test_audit_middleware.py index 8c7ec953..889f0d97 100644 --- a/apps/backend/tests/test_audit_middleware.py +++ b/apps/backend/tests/test_audit_middleware.py @@ -86,3 +86,29 @@ async def boom(collection_name, data): client = TestClient(build_app()) res = client.post("/api/v1/bookmarks") assert res.status_code == 200 # user response unaffected + + +def test_bookmark_service_sets_audit_detail(monkeypatch): + import asyncio as _asyncio + from app.api.routes.bookmarks import services as bm + from app.api.routes.bookmarks.schema import BookmarkCreate + from app.core import audit + + async def fake_insert_one(collection_name, data): + return None + + monkeypatch.setattr("app.api.routes.bookmarks.services.db_manager.insert_one", fake_insert_one) + + async def run(): + tok = audit._audit_ctx.set(audit.AuditContext()) + try: + await bm.create_bookmark("uid1", BookmarkCreate(title="GitHub", url="https://gh.com")) + ctx = audit.current_context() + assert ctx.action == "bookmark.create" + assert ctx.entity_type == "bookmark" + assert ctx.entity_id # the new id + assert any(c["field"] == "title" and c["after"] == "GitHub" for c in (ctx.changes or [])) + finally: + audit._audit_ctx.reset(tok) + + _asyncio.run(run()) From ab1562bfeb51d55483a88149b4804e33c5758ff6 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 14:50:20 +0530 Subject: [PATCH 07/52] feat(audit): add paginated, filterable GET /audit-log API --- apps/backend/app/api/router.py | 2 + .../app/api/routes/audit_log/__init__.py | 0 apps/backend/app/api/routes/audit_log/api.py | 27 ++++++++ .../app/api/routes/audit_log/schema.py | 42 ++++++++++++ .../app/api/routes/audit_log/services.py | 68 +++++++++++++++++++ .../tests/api/routes/audit_log/__init__.py | 0 .../api/routes/audit_log/test_audit_query.py | 36 ++++++++++ 7 files changed, 175 insertions(+) create mode 100644 apps/backend/app/api/routes/audit_log/__init__.py create mode 100644 apps/backend/app/api/routes/audit_log/api.py create mode 100644 apps/backend/app/api/routes/audit_log/schema.py create mode 100644 apps/backend/app/api/routes/audit_log/services.py create mode 100644 apps/backend/tests/api/routes/audit_log/__init__.py create mode 100644 apps/backend/tests/api/routes/audit_log/test_audit_query.py diff --git a/apps/backend/app/api/router.py b/apps/backend/app/api/router.py index e6a6098e..f0753287 100644 --- a/apps/backend/app/api/router.py +++ b/apps/backend/app/api/router.py @@ -21,6 +21,7 @@ from app.api.routes.redis_commander.api import router as redis_commander_router from app.api.routes.url_shortener.api import router as url_shortener_router from app.api.routes.dns_lookup.api import router as dns_lookup_router +from app.api.routes.audit_log.api import router as audit_log_router api_router = APIRouter() api_router.include_router(health_router) @@ -44,3 +45,4 @@ api_router.include_router(redis_commander_router) api_router.include_router(url_shortener_router) api_router.include_router(dns_lookup_router) +api_router.include_router(audit_log_router) diff --git a/apps/backend/app/api/routes/audit_log/__init__.py b/apps/backend/app/api/routes/audit_log/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/backend/app/api/routes/audit_log/api.py b/apps/backend/app/api/routes/audit_log/api.py new file mode 100644 index 00000000..27b305c5 --- /dev/null +++ b/apps/backend/app/api/routes/audit_log/api.py @@ -0,0 +1,27 @@ +from typing import Optional + +from fastapi import APIRouter, Depends, Query + +from app.api.routes.auth.services import get_current_uid +from app.api.routes.audit_log import services as svc +from app.api.routes.audit_log.schema import AuditListOut + +router = APIRouter(prefix="/audit-log", tags=["audit-log"]) + + +@router.get("", response_model=AuditListOut, summary="List the current user's audit events") +async def list_events( + uid: str = Depends(get_current_uid), + skip: int = Query(default=0, ge=0), + limit: int = Query(default=50, ge=1, le=100), + module: Optional[str] = Query(default=None), + action: Optional[str] = Query(default=None), + outcome: Optional[str] = Query(default=None), + ts_from: Optional[int] = Query(default=None, alias="from"), + ts_to: Optional[int] = Query(default=None, alias="to"), + search: Optional[str] = Query(default=None), +) -> AuditListOut: + return await svc.list_audit_events( + uid, skip=skip, limit=limit, module=module, action=action, + outcome=outcome, ts_from=ts_from, ts_to=ts_to, search=search, + ) diff --git a/apps/backend/app/api/routes/audit_log/schema.py b/apps/backend/app/api/routes/audit_log/schema.py new file mode 100644 index 00000000..d85d4c3e --- /dev/null +++ b/apps/backend/app/api/routes/audit_log/schema.py @@ -0,0 +1,42 @@ +from typing import Any, Optional + +from pydantic import BaseModel + + +class AuditChange(BaseModel): + field: str + before: Any | None = None + after: Any | None = None + + +class AuditDevice(BaseModel): + browser: str + os: str + device_type: str + + +class AuditEventOut(BaseModel): + id: str + uid: Optional[str] = None + action: str + module: Optional[str] = None + entity_type: Optional[str] = None + entity_id: Optional[str] = None + method: str + path: str + status: int + outcome: str + changes: Optional[list[AuditChange]] = None + summary: Optional[str] = None + ip: Optional[str] = None + ua_raw: Optional[str] = None + device: Optional[AuditDevice] = None + latency_ms: int + ts: int + + +class AuditListOut(BaseModel): + items: list[AuditEventOut] + total: int + skip: int + limit: int diff --git a/apps/backend/app/api/routes/audit_log/services.py b/apps/backend/app/api/routes/audit_log/services.py new file mode 100644 index 00000000..92396e98 --- /dev/null +++ b/apps/backend/app/api/routes/audit_log/services.py @@ -0,0 +1,68 @@ +from typing import Any, Optional + +from app.database import db_manager +from app.utils.collection_name import AUDIT_LOG +from app.api.routes.audit_log.schema import AuditEventOut, AuditListOut + + +def _doc_to_out(doc: dict[str, Any]) -> AuditEventOut: + return AuditEventOut( + id=str(doc.get("_id", "")), + uid=doc.get("uid"), + action=doc.get("action", ""), + module=doc.get("module"), + entity_type=doc.get("entity_type"), + entity_id=doc.get("entity_id"), + method=doc.get("method", ""), + path=doc.get("path", ""), + status=int(doc.get("status", 0)), + outcome=doc.get("outcome", ""), + changes=doc.get("changes"), + summary=doc.get("summary"), + ip=doc.get("ip"), + ua_raw=doc.get("ua_raw"), + device=doc.get("device"), + latency_ms=int(doc.get("latency_ms", 0)), + ts=int(doc.get("ts", 0)), + ) + + +async def list_audit_events( + uid: str, + *, + skip: int = 0, + limit: int = 50, + module: Optional[str] = None, + action: Optional[str] = None, + outcome: Optional[str] = None, + ts_from: Optional[int] = None, + ts_to: Optional[int] = None, + search: Optional[str] = None, +) -> AuditListOut: + query: dict[str, Any] = {"uid": uid} + if module: + query["module"] = module + if action: + query["action"] = action + if outcome: + query["outcome"] = outcome + if ts_from is not None or ts_to is not None: + rng: dict[str, Any] = {} + if ts_from is not None: + rng["$gte"] = ts_from + if ts_to is not None: + rng["$lte"] = ts_to + query["ts"] = rng + if search: + query["summary"] = {"$regex": search, "$options": "i"} + + total = await db_manager.count_documents(AUDIT_LOG, query) + docs = await db_manager.find( + AUDIT_LOG, query, sort=[("ts", -1)], skip=skip, limit=limit + ) + return AuditListOut( + items=[_doc_to_out(d) for d in docs], + total=total, + skip=skip, + limit=limit, + ) diff --git a/apps/backend/tests/api/routes/audit_log/__init__.py b/apps/backend/tests/api/routes/audit_log/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/backend/tests/api/routes/audit_log/test_audit_query.py b/apps/backend/tests/api/routes/audit_log/test_audit_query.py new file mode 100644 index 00000000..d73e3ed9 --- /dev/null +++ b/apps/backend/tests/api/routes/audit_log/test_audit_query.py @@ -0,0 +1,36 @@ +import asyncio + +import pytest + +from app.api.routes.audit_log import services as svc + + +@pytest.fixture +def fake_db(monkeypatch): + rows = [ + {"_id": "1", "uid": "u1", "action": "bookmark.create", "module": "bookmarks", + "entity_type": "bookmark", "entity_id": "b1", "method": "POST", + "path": "/api/v1/bookmarks", "status": 200, "outcome": "success", + "changes": [{"field": "title", "before": None, "after": "GitHub"}], + "summary": "Created bookmark 'GitHub'", "ip": "1.2.3.4", "ua_raw": "UA", + "device": {"browser": "Chrome", "os": "macOS", "device_type": "desktop"}, + "latency_ms": 12, "ts": 1000, "expireAt": "x"}, + ] + + async def fake_find(collection_name, query, projection=None, sort=None, skip=0, limit=0, collation=None): + assert query["uid"] == "u1" + return rows + + async def fake_count(collection_name, query): + return len(rows) + + monkeypatch.setattr("app.api.routes.audit_log.services.db_manager.find", fake_find) + monkeypatch.setattr("app.api.routes.audit_log.services.db_manager.count_documents", fake_count) + return rows + + +def test_list_audit_events_scopes_to_uid_and_excludes_expireAt(fake_db): + out = asyncio.run(svc.list_audit_events("u1", skip=0, limit=50)) + assert out.total == 1 + assert out.items[0].action == "bookmark.create" + assert not hasattr(out.items[0], "expireAt") From 391ba92925fe0bcb0e1767652adb640fafbafd43 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 14:53:54 +0530 Subject: [PATCH 08/52] feat(audit): add web audit-log API client Co-Authored-By: Claude Opus 4.8 --- .../src/lib/__tests__/audit-log-api.test.ts | 26 ++++++++ apps/web/src/lib/audit-log-api.ts | 59 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 apps/web/src/lib/__tests__/audit-log-api.test.ts create mode 100644 apps/web/src/lib/audit-log-api.ts diff --git a/apps/web/src/lib/__tests__/audit-log-api.test.ts b/apps/web/src/lib/__tests__/audit-log-api.test.ts new file mode 100644 index 00000000..940d197f --- /dev/null +++ b/apps/web/src/lib/__tests__/audit-log-api.test.ts @@ -0,0 +1,26 @@ +const backendFetch = jest.fn() +jest.mock('@/lib/backend-auth', () => ({ backendFetch: (...a: unknown[]) => backendFetch(...a) })) + +import { fetchAuditLog } from '@/lib/audit-log-api' + +describe('fetchAuditLog', () => { + beforeEach(() => backendFetch.mockReset()) + + it('builds the query string and returns parsed data', async () => { + backendFetch.mockResolvedValue({ + ok: true, + json: async () => ({ items: [], total: 0, skip: 0, limit: 50 }), + }) + const res = await fetchAuditLog({ module: 'bookmarks', limit: 50 }) + const url = backendFetch.mock.calls[0][0] as string + expect(url).toContain('/api/backend/audit-log') + expect(url).toContain('module=bookmarks') + expect(url).toContain('limit=50') + expect(res.total).toBe(0) + }) + + it('throws on non-ok response', async () => { + backendFetch.mockResolvedValue({ ok: false, status: 500, text: async () => 'boom' }) + await expect(fetchAuditLog()).rejects.toThrow() + }) +}) diff --git a/apps/web/src/lib/audit-log-api.ts b/apps/web/src/lib/audit-log-api.ts new file mode 100644 index 00000000..147f79b5 --- /dev/null +++ b/apps/web/src/lib/audit-log-api.ts @@ -0,0 +1,59 @@ +import { backendFetch } from '@/lib/backend-auth' + +export type AuditChange = { field: string; before: unknown; after: unknown } +export type AuditDevice = { browser: string; os: string; device_type: string } + +export type AuditEvent = { + id: string + uid: string | null + action: string + module: string | null + entity_type: string | null + entity_id: string | null + method: string + path: string + status: number + outcome: 'success' | 'failure' + changes: AuditChange[] | null + summary: string | null + ip: string | null + ua_raw: string | null + device: AuditDevice | null + latency_ms: number + ts: number +} + +export type AuditListResponse = { + items: AuditEvent[] + total: number + skip: number + limit: number +} + +export type AuditQuery = { + skip?: number + limit?: number + module?: string + action?: string + outcome?: 'success' | 'failure' + from?: number + to?: number + search?: string +} + +export async function fetchAuditLog(query: AuditQuery = {}): Promise { + const params = new URLSearchParams() + for (const [key, value] of Object.entries(query)) { + if (value !== undefined && value !== null && value !== '') { + params.set(key, String(value)) + } + } + const qs = params.toString() + const url = `/api/backend/audit-log${qs ? `?${qs}` : ''}` + const res = await backendFetch(url, { method: 'GET' }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new Error(text || `Audit log failed (${res.status})`) + } + return (await res.json()) as AuditListResponse +} From a6f86f265f4a0609278c379e2e7b4fc798182a3d Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 15:03:13 +0530 Subject: [PATCH 09/52] feat(audit): add Activity log page Co-Authored-By: Claude Sonnet 4.6 --- apps/web/messages/en.json | 17 ++++ apps/web/src/app/dashboard/activity/page.tsx | 7 ++ .../dashboard/activity/activity-log-panel.tsx | 98 +++++++++++++++++++ .../dashboard/activity/audit-event-row.tsx | 71 ++++++++++++++ 4 files changed, 193 insertions(+) create mode 100644 apps/web/src/app/dashboard/activity/page.tsx create mode 100644 apps/web/src/components/dashboard/activity/activity-log-panel.tsx create mode 100644 apps/web/src/components/dashboard/activity/audit-event-row.tsx diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index b83af0f4..05f5fed4 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -298,6 +298,23 @@ "favoriteTools": "Favorite tools", "toolsWithUsage": "Tools with usage tracked" }, + "activity": { + "title": "Activity log", + "subtitle": "Everything you changed, when, and from which device.", + "filterModule": "Module", + "filterOutcome": "Outcome", + "search": "Search", + "all": "All", + "success": "Success", + "failure": "Failure", + "empty": "No activity yet.", + "loadError": "Could not load activity.", + "loadMore": "Load more", + "changedFields": "What changed", + "device": "Device", + "before": "Before", + "after": "After" + }, "toolCard": { "defaultDescription": "Explore this tool for better functionality.", "launchTool": "Launch Tool" diff --git a/apps/web/src/app/dashboard/activity/page.tsx b/apps/web/src/app/dashboard/activity/page.tsx new file mode 100644 index 00000000..31ccc661 --- /dev/null +++ b/apps/web/src/app/dashboard/activity/page.tsx @@ -0,0 +1,7 @@ +'use client' + +import { ActivityLogPanel } from '@/components/dashboard/activity/activity-log-panel' + +export default function ActivityLogRoute() { + return +} diff --git a/apps/web/src/components/dashboard/activity/activity-log-panel.tsx b/apps/web/src/components/dashboard/activity/activity-log-panel.tsx new file mode 100644 index 00000000..f059ee3e --- /dev/null +++ b/apps/web/src/components/dashboard/activity/activity-log-panel.tsx @@ -0,0 +1,98 @@ +'use client' + +import { useCallback, useEffect, useState } from 'react' +import { useTranslations } from 'next-intl' +import { fetchAuditLog, type AuditEvent } from '@/lib/audit-log-api' +import { AuditEventRow } from './audit-event-row' + +const PAGE = 50 + +export function ActivityLogPanel() { + const t = useTranslations('Dashboard.activity') + const [events, setEvents] = useState([]) + const [total, setTotal] = useState(0) + const [skip, setSkip] = useState(0) + const [outcome, setOutcome] = useState<'' | 'success' | 'failure'>('') + const [search, setSearch] = useState('') + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + const load = useCallback( + async (reset: boolean) => { + setLoading(true) + setError(null) + try { + const nextSkip = reset ? 0 : skip + const res = await fetchAuditLog({ + skip: nextSkip, + limit: PAGE, + outcome: outcome || undefined, + search: search || undefined, + }) + setTotal(res.total) + setSkip(nextSkip + res.items.length) + setEvents((prev) => (reset ? res.items : [...prev, ...res.items])) + } catch (e) { + setError(e instanceof Error ? e.message : t('loadError')) + } finally { + setLoading(false) + } + }, + [skip, outcome, search, t], + ) + + useEffect(() => { + void load(true) + // reload when filters change + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [outcome, search]) + + return ( +
+

{t('title')}

+

{t('subtitle')}

+ +
+ + setSearch(e.target.value)} + className="flex-1 rounded-md border border-border bg-background px-2 py-1 text-sm" + /> +
+ + {error &&
{error}
} + {!error && events.length === 0 && !loading && ( +
{t('empty')}
+ )} + +
+ {events.map((e) => ( + + ))} +
+ + {events.length < total && ( + + )} +
+ ) +} diff --git a/apps/web/src/components/dashboard/activity/audit-event-row.tsx b/apps/web/src/components/dashboard/activity/audit-event-row.tsx new file mode 100644 index 00000000..be1befe9 --- /dev/null +++ b/apps/web/src/components/dashboard/activity/audit-event-row.tsx @@ -0,0 +1,71 @@ +'use client' + +import { useState } from 'react' +import { useTranslations } from 'next-intl' +import type { AuditEvent } from '@/lib/audit-log-api' + +function relativeTime(ts: number): string { + const diff = Date.now() - ts + const mins = Math.floor(diff / 60000) + if (mins < 1) return 'just now' + if (mins < 60) return `${mins}m ago` + const hrs = Math.floor(mins / 60) + if (hrs < 24) return `${hrs}h ago` + return new Date(ts).toLocaleDateString() +} + +export function AuditEventRow({ event }: { event: AuditEvent }) { + const t = useTranslations('Dashboard.activity') + const [open, setOpen] = useState(false) + const hasChanges = !!event.changes && event.changes.length > 0 + const device = event.device + ? `${event.device.browser} on ${event.device.os}` + : '—' + + return ( +
+ + + {open && hasChanges && ( +
+
{t('changedFields')}
+
    + {event.changes!.map((c) => ( +
  • + {c.field}: + {String(c.before ?? '∅')} + → + {String(c.after ?? '∅')} +
    • + ))} +
+
+ )} +
+ ) +} From 7a200964de7d2314a8ed2e4bd7313dd73bf8bda4 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 15:09:57 +0530 Subject: [PATCH 10/52] refactor(audit): final-review cleanups (utcnow deprecation, unused import, EOF newline) Co-Authored-By: Claude Opus 4.8 --- apps/backend/app/core/audit.py | 2 +- apps/backend/app/core/audit_middleware.py | 2 +- apps/backend/app/utils/collection_name.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/backend/app/core/audit.py b/apps/backend/app/core/audit.py index d418d4f6..f891d6fb 100644 --- a/apps/backend/app/core/audit.py +++ b/apps/backend/app/core/audit.py @@ -2,7 +2,7 @@ import re from contextvars import ContextVar -from dataclasses import dataclass, field +from dataclasses import dataclass # Field names whose values are safe to record verbatim in the audit diff. # Anything NOT in this set is recorded as "[redacted]" (default-deny). diff --git a/apps/backend/app/core/audit_middleware.py b/apps/backend/app/core/audit_middleware.py index 2739fff9..6ef1341c 100644 --- a/apps/backend/app/core/audit_middleware.py +++ b/apps/backend/app/core/audit_middleware.py @@ -102,7 +102,7 @@ async def dispatch(self, request: Request, call_next): "device": audit.parse_user_agent(request.headers.get("user-agent")), "latency_ms": latency, "ts": ts, - "expireAt": datetime.datetime.utcnow() + datetime.timedelta(days=_TTL_DAYS), + "expireAt": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=_TTL_DAYS), } asyncio.create_task(write_audit_event(doc)) except Exception as exc: # never propagate diff --git a/apps/backend/app/utils/collection_name.py b/apps/backend/app/utils/collection_name.py index 3cdbf037..78ac6d34 100644 --- a/apps/backend/app/utils/collection_name.py +++ b/apps/backend/app/utils/collection_name.py @@ -22,4 +22,4 @@ REDIS_CONNECTIONS = "redis_connections" URL_LINKS = "url_links" URL_CLICK_EVENTS = "url_click_events" -AUDIT_LOG = "audit_log" \ No newline at end of file +AUDIT_LOG = "audit_log" From 4fbdf7dad3b8d856b6471d51434809129fe7cde7 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 15:43:14 +0530 Subject: [PATCH 11/52] docs(cache): design spec for Redis caching masterplan Decorator-based caching layer with versioned per-user keys, XFetch on aggregates, fail-open + per-namespace rollout flag. Co-Authored-By: Claude Opus 4.7 --- ...6-06-22-redis-caching-masterplan-design.md | 358 ++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-22-redis-caching-masterplan-design.md diff --git a/docs/superpowers/specs/2026-06-22-redis-caching-masterplan-design.md b/docs/superpowers/specs/2026-06-22-redis-caching-masterplan-design.md new file mode 100644 index 00000000..ff8e722c --- /dev/null +++ b/docs/superpowers/specs/2026-06-22-redis-caching-masterplan-design.md @@ -0,0 +1,358 @@ +# Redis Caching Masterplan — Design Spec + +**Status:** Draft for review +**Date:** 2026-06-22 +**Author:** Akhil (with Claude) +**Scope:** FastAPI backend (`apps/backend/`) — full Redis caching platform + +## 1. Goal + +Introduce a production-grade Redis caching layer across the FastAPI backend that: + +- Cuts MongoDB read load by ≥60% on cached collections. +- Brings p50 list-endpoint latency under 50ms and p99 under 200ms. +- Survives Redis outages without taking the app down (fail-open). +- Rolls out namespace-by-namespace with a global kill switch. +- Handles thundering-herd on hot aggregate keys via probabilistic early refresh. + +Non-goal: multi-region replication, Redis Cluster, Prometheus integration, in-process L1 cache (beyond a 5s version-key short-circuit). + +## 2. Constraints & Decisions (locked) + +| Topic | Decision | +|-------|----------| +| Scope | Full caching platform (foundation + read caching across all read-heavy routes + dogpile protection + observability + flagged rollout). | +| Hosting | Local `redis://localhost:6379` in dev; single managed Redis node in prod (DigitalOcean / Railway / Render). | +| Invalidation | **D-hybrid** — versioned keys for per-user data, short TTL for cross-user aggregates. | +| Failure mode | Fail-open. Cache errors degrade to direct DB reads. Logged. | +| Observability | Structured logs only. No Prometheus in scope. | +| Rollout | Per-namespace env flag (`CACHE_NAMESPACES`) + global `CACHE_ENABLED` kill switch. | +| Serialization | orjson. No pickle. Keyed `blake2b` hash for arg keys. | +| Stampede protection | None for per-user namespaces; XFetch (probabilistic early refresh) for aggregates, exposed via `strategy="xfetch"` decorator flag. | +| Architecture shape | Decorator-on-service-fns is primary surface. XFetch math hidden inside decorator. | +| Success metrics | Latency + hit ratio + Mongo load + 1000-concurrent load test, all zero-error. | + +## 3. Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ FastAPI Worker (×N) │ +│ │ +│ request → router → service fn (@cached) → cache facade │ +│ │ │ +│ ▼ │ +│ ┌───────────────────────┐ │ +│ │ app.core.cache │ │ +│ │ - decorator │ │ +│ │ - get_or_set │ │ +│ │ - bump_version │ │ +│ │ - serializer (orjson)│ │ +│ │ - xfetch math │ │ +│ │ - per-ns enable flag │ │ +│ │ - fail-open wrapper │ │ +│ └──────────┬────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────┐ │ +│ │ app.core.redis_client│ ← lazy async pool │ +│ └──────────┬───────────┘ │ +└────────────────────────────────┼─────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────┐ + │ Redis (single node) │ + │ - cache:* keys │ + │ - ratelimit:* (SlowAPI)│ + │ - tokver:* (auth) │ + └──────────────────────────┘ +``` + +Layers: + +- **`redis_client`** — singleton async `redis.asyncio.Redis` pool, opened in FastAPI lifespan, pinged on startup. Shared by cache + SlowAPI + token cache. +- **`cache`** — single facade module. Decorator + helpers. All routes import only from this. +- **service fns** — annotated with `@cached(ns=..., ttl=..., strategy=...)`. Writes call `await bump_version(ns, uid)` or `await cache_invalidate(ns, key)`. + +## 4. Public API + +From `app.core.cache`: + +```python +@cached(ns="bookmarks", ttl=120, scope="user") # per-user +@cached(ns="urlshort", ttl=600, scope="global", strategy="xfetch") # aggregate +await bump_version(ns="bookmarks", uid=uid) # on writes +await cache_invalidate(ns="urlshort", key="resolve:slug=gh") # explicit drop +await get_or_set(ns=..., key=..., loader=..., ttl=..., strategy=...) # escape hatch +``` + +Decorator parameters: + +| Param | Type | Default | Notes | +|-------|------|---------|-------| +| `ns` | str | required | Must be registered in namespace registry | +| `ttl` | int | required | Seconds | +| `scope` | `"user" \| "global"` | `"user"` | `"user"` requires `uid` kwarg at call time | +| `strategy` | `"simple" \| "xfetch"` | `"simple"` | XFetch requires `scope="global"` | +| `key` | callable \| None | None | Custom key builder; default = blake2b of sorted kwargs | + +## 5. Components & File Map + +| File | Status | Purpose | +|------|--------|---------| +| `apps/backend/pyproject.toml` | modify | Add `redis[asyncio]>=5.0`, `orjson>=3.10` | +| `apps/backend/app/core/config.py` | modify | Add `REDIS_URL`, `CACHE_ENABLED`, `CACHE_NAMESPACES`, `CACHE_DEFAULT_TTL`, `CACHE_LOG_LEVEL`, `CACHE_OP_TIMEOUT_MS`, `CACHE_XFETCH_BETA` | +| `apps/backend/app/core/redis_client.py` | **new** | Lazy async Redis pool, lifespan open/close, ping on startup | +| `apps/backend/app/core/cache/__init__.py` | **new** | Public API exports | +| `apps/backend/app/core/cache/decorator.py` | **new** | `@cached` + simple/xfetch read paths + fail-open wrapper | +| `apps/backend/app/core/cache/keys.py` | **new** | Key builder, version key resolver, namespace registry | +| `apps/backend/app/core/cache/serializer.py` | **new** | orjson encode/decode, Pydantic and datetime handling | +| `apps/backend/app/core/cache/xfetch.py` | **new** | Probabilistic early-expiration math | +| `apps/backend/app/core/cache/flags.py` | **new** | `is_namespace_enabled(ns)` | +| `apps/backend/app/main.py` | modify | Wire Redis pool open/close to FastAPI lifespan | +| `apps/backend/app/core/limiter.py` | modify | Switch SlowAPI to Redis storage backend | +| `apps/backend/app/api/routes/auth/services.py` | modify | Cache `verify_token` + `get_current_user`; invalidate on logout / password change | +| `apps/backend/app/api/routes/bookmarks/services.py` | modify | `@cached` on reads; `bump_version` on writes | +| `apps/backend/app/api/routes/notes/services.py` | modify | same pattern | +| `apps/backend/app/api/routes/code_snippets/services.py` | modify | same | +| `apps/backend/app/api/routes/tasks/services.py` | modify | same | +| `apps/backend/app/api/routes/passwords/services.py` | modify | same (ciphertext only) | +| `apps/backend/app/api/routes/api_client/services.py` | modify | same | +| `apps/backend/app/api/routes/user_preferences/services.py` | modify | same | +| `apps/backend/app/api/routes/url_shortener/services.py` | modify | `@cached(strategy="xfetch", scope="global")` on public resolves; `scope="user"` for owner list | +| `apps/backend/app/api/routes/analytics/services.py` | modify | `@cached(strategy="xfetch")` on aggregates | +| `apps/backend/app/api/routes/dns_lookup/services.py` | modify | `@cached(scope="global")` on pure-fn lookups | +| `tests/test_cache_core.py` | **new** | Unit: decorator, version, xfetch math, fail-open, namespace flag | +| `tests/test_cache_integration.py` | **new** | Integration with real Redis container | +| `scripts/loadtest_cache.py` | **new** | k6 / locust scenario for 1000 concurrent | + +## 6. Key Schema + +``` +cache:{ns}:{scope_part}:v{ver}:{op}:{arg_hash} +``` + +- `ns` — namespace; must match `CACHE_NAMESPACES`. +- `scope_part` — `u:{uid}` for `scope="user"`, `g` for `scope="global"`. +- `v{ver}` — version int from `cache:ver:{ns}:u:{uid}`; omitted for `scope="global"` (TTL only). +- `op` — fn name + variant (`list`, `get`, `search:tag`, ...). +- `arg_hash` — `blake2b(orjson(sorted_kwargs), key=APP_SECRET)[:16]` hex. + +Examples: + +``` +cache:bookmarks:u:abc123:v7:list:8f3a1c0d2e4b5a78 +cache:notes:u:abc123:v2:get:id=xyz +cache:urlshort:g:resolve:slug=gh +cache:analytics:g:top_tools:days=7 +cache:auth_token:g:verify:7a4b2c9d1e0f3a8b ← key = sha256(token)[:16] +cache:auth_user:u:abc123 +``` + +Version keys (no TTL, INCR-only): + +``` +cache:ver:bookmarks:u:abc123 → 7 +cache:ver:notes:u:abc123 → 2 +``` + +If Redis is flushed, INCR returns 1 — all old version-bearing keys become orphans and evict via their own TTL. + +## 7. TTL & Strategy Table + +| Namespace | Scope | TTL | Strategy | Notes | +|-----------|-------|-----|----------|-------| +| `auth_token` | global | 300s | simple | Firebase JWT verify result; key = sha256(token)[:16] | +| `auth_user` | user | 60s | simple | Mongo user-profile lookup | +| `bookmarks` | user | 120s | simple | List + get + search | +| `notes` | user | 120s | simple | List + get | +| `code_snippets` | user | 120s | simple | List + get | +| `tasks` | user | 60s | simple | List (frequent writes) | +| `passwords` | user | 60s | simple | Ciphertext only | +| `api_client` | user | 300s | simple | Collections + saved requests | +| `user_preferences` | user | 600s | simple | Rarely changes | +| `url_shortener_resolve` | global | 600s | xfetch | Hot, public, read-heavy | +| `url_shortener_owner` | user | 120s | simple | Owner's own short URLs list | +| `analytics_aggregate` | global | 300s | xfetch | Top tools, activity buckets | +| `dns_lookup` | global | 3600s | simple | Pure fn of (host, type) | + +Memory budget estimate: ~80MB at 1000 active users (8 namespaces × 5 keys × ~2KB serialized). Single node sufficient. + +Redis deployment config: `maxmemory-policy allkeys-lru`. Documented in deployment notes. + +## 8. Invalidation & Write Paths + +**Per-user pattern (90% of writes):** + +```python +async def create_bookmark(uid: str, body: BookmarkCreate) -> BookmarkOut: + doc = {...} + await db_manager.insert_one(BOOKMARKS, doc) + await bump_version(ns="bookmarks", uid=uid) # one line + return _doc_to_out(doc) +``` + +`bump_version` = `INCR cache:ver:bookmarks:u:{uid}`. Atomic, single round-trip. All cached keys for that (ns, uid) instantly orphaned. + +Bulk writes get one bump, not N. + +**Read path (decorator-internal):** + +``` +1. ver = await redis.get(f"cache:ver:{ns}:u:{uid}") or "0" + (in-process 5s LRU cache to skip this round-trip on hot loops) +2. key = f"cache:{ns}:u:{uid}:v{ver}:{op}:{arg_hash}" +3. cached = await redis.get(key) +4. if cached: deserialize + return +5. result = await fn(...) +6. await redis.setex(key, ttl, serialize(result)) +7. return result +``` + +**Global-namespace writes:** + +| Trigger | Action | +|---------|--------| +| Create/update/delete short URL | `cache_invalidate(ns="url_shortener_resolve", key=f"resolve:slug={slug}")` | +| Usage event logged (analytics) | No invalidation — TTL handles it. Stale 30-300s acceptable for charts. | +| Logout / password change | `cache_invalidate(ns="auth_token", key=token_hash)` + `cache_invalidate(ns="auth_user", key=uid)` | + +**XFetch (stale-while-revalidate):** + +Stored payload carries `(value, computed_at, ttl, delta)`. On read: + +```python +if now > computed_at + ttl - beta * delta * ln(random()): + asyncio.create_task(refresh()) # single request rebuilds +return value # all others serve stale, no wait +``` + +- `beta` = `CACHE_XFETCH_BETA` (default 1.0). +- `delta` = rolling-average measured fn runtime, stored alongside payload. + +## 9. Error Handling + +All Redis calls wrapped: + +```python +async def _safe(coro, *, op: str, ns: str): + try: + return await asyncio.wait_for(coro, timeout=CACHE_OP_TIMEOUT_MS / 1000) + except (RedisError, asyncio.TimeoutError, ConnectionError) as e: + log.warning("cache.error", op=op, ns=ns, err=type(e).__name__, msg=str(e)) + return None # treat as miss +``` + +- `CACHE_OP_TIMEOUT_MS = 50` default. Better to miss than block. +- Write-path `bump_version` also fail-open: log + continue. Worst case: one user serves stale until TTL. +- Startup `redis.ping()` failure → log error, set runtime `CACHE_ENABLED=false`, app continues booting. + +## 10. Observability + +Single logger `app.cache`. Events: + +| Event | Level | Fields | +|-------|-------|--------| +| `cache.hit` | DEBUG | ns, op, latency_us | +| `cache.miss` | DEBUG | ns, op | +| `cache.set` | DEBUG | ns, op, size_bytes, ttl | +| `cache.bump_version` | INFO | ns, uid | +| `cache.error` | WARN | ns, op, err, msg | +| `cache.xfetch.refresh` | INFO | ns, key, age_s | +| `cache.namespace_disabled` | DEBUG | ns | + +Prod default: `cache.*` at WARN+. Set `CACHE_LOG_LEVEL=DEBUG` for live hit-rate debugging. + +## 11. Security + +| Concern | Mitigation | +|---------|------------| +| Cache poisoning (cross-user) | All user-scoped keys include `u:{uid}` from authenticated context. Decorator rejects `scope="user"` without `uid` kwarg at runtime. | +| Sensitive data leak | Passwords cached as ciphertext only. JWTs never cached — only verification result. Audit-log writes never cached. | +| Hash collision | `arg_hash = blake2b(orjson(kwargs), key=APP_SECRET)[:16]`. Keyed hash, 64-bit, collision-resistant up to ~10⁹ keys. | +| Pickle RCE | orjson only. No pickle import in cache module. | +| Redis exposure | Localhost in dev. Prod: VPC-only + `requirepass` + TLS. Documented in deployment notes. | +| Auth revocation | Logout / password change must call `cache_invalidate` on `auth_token` + `auth_user`. Token-cache TTL ≤ Firebase token validity. | + +## 12. Testing + +**Unit (`tests/test_cache_core.py`):** + +- Key builder: deterministic, kwargs order-insensitive, collision check across 10k random inputs. +- Version bump: INCR semantics, missing-key → 1. +- XFetch math: monotonic refresh probability; beta=0 ≡ TTL-only; beta=1 fires ~10% before expiry on average. +- Serializer: round-trip Pydantic, datetime, bytes, None, nested. +- Fail-open: Redis stub raises → returns None, no exception bubbles. +- Namespace flag: disabled ns → zero Redis calls. + +**Integration (`tests/test_cache_integration.py`):** + +- Real Redis via `testcontainers` or local `redis:7-alpine`. +- One test per cached service fn: miss → hit → invalidate → miss. +- Write-then-read invalidation: create_bookmark → list_bookmarks reflects new doc immediately. +- Cross-user isolation: user A's bump doesn't affect user B's cache. +- Logout invalidates token + user caches. +- Concurrent reads on per-user miss: N parallel reads → N Mongo calls (no lock by design). + +**Load test (`scripts/loadtest_cache.py`):** + +- Scenario A: 1000 concurrent, 90/10 read/write. +- Scenario B: same with `CACHE_ENABLED=false` (baseline). +- Compare p50/p99 latency, Mongo `find()` count, error rate. + +## 13. Acceptance Criteria + +All must pass before declaring done: + +- [ ] p50 list-endpoint latency < 50ms. +- [ ] p99 list-endpoint latency < 200ms. +- [ ] Per-user hit ratio ≥ 80% after 5-min warm. +- [ ] Aggregate hit ratio ≥ 95% after 5-min warm. +- [ ] Mongo `find()` per minute on cached collections reduced ≥ 60%. +- [ ] 1000-concurrent load test: 0% error rate. +- [ ] Existing test suite passes (`pytest -q`). +- [ ] All routes still work with `CACHE_ENABLED=false`. + +## 14. Rollout Phases + +Deploy code with all namespaces inactive; ramp via `CACHE_NAMESPACES`: + +| Phase | Namespaces added | Notes | +|-------|------------------|-------| +| 0 | (empty) | Code shipped, decorators no-op. Verify zero regression. | +| 1 | `auth_token,auth_user` | Lowest risk, biggest Firebase saving. ≥48h soak. | +| 2 | `user_preferences,dns_lookup` | Read-mostly, low write rate. | +| 3 | `bookmarks,notes,code_snippets,api_client` | Core per-user; heavy use. | +| 4 | `tasks,passwords,url_shortener_owner` | Frequent writes — verify invalidation. | +| 5 | `url_shortener_resolve,analytics_aggregate` | XFetch namespaces — verify no stampede. | + +Each phase: ≥24h soak. Kill switch = drop namespace from env + restart workers. Global kill = `CACHE_ENABLED=false`. + +## 15. Open Risks & Mitigations + +| Risk | Mitigation | +|------|------------| +| XFetch `beta` mis-tuned per namespace | Expose `CACHE_XFETCH_BETA` env; default 1.0; revisit per ns after Phase 5 metrics. | +| Hot-key with very large payload (e.g. user with 100k bookmarks) | Pagination already bounds payload. Cache layer skips entries > 1MB (logs `cache.skip.toolarge`); falls through to Mongo. | +| Redis flush / data loss | Fail-open + INCR-returns-1 path handles it; cache rebuilds within TTL. | +| Version-key INCR overflow | int64 → millions of years at realistic rates. Non-issue. | +| Reliance on `APP_SECRET` for hash keying | Already required for JWT; rotation invalidates all cached arg-hashes — acceptable (TTL evicts stale entries). | + +## 16. Out of Scope (explicit) + +- Multi-region cache replication. +- Redis Cluster / sharding. +- Per-key encryption at rest (relies on Redis access control + TLS). +- Prometheus / Grafana dashboards. +- L1 in-process payload cache (only version-key has 5s in-proc LRU). +- Cache warming on deploy (cold-start acceptable; warms within minutes). + +## 17. Relationship to Prior Plan + +This spec subsumes the Redis-related portions of `2026-06-20-backend-scale-1000-concurrent.md`: + +- Redis client lifecycle → `app.core.redis_client`. +- SlowAPI Redis storage → still in scope (Phase 1). +- Auth token cache → `auth_token` namespace. +- User profile cache → `auth_user` namespace. + +That plan's non-Redis items (Gunicorn workers, MongoDB pool tuning, `find()` cap) remain independent and out of scope here. From 637f690f29507f7a8e16109865d631f82636e4cb Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 16:37:54 +0530 Subject: [PATCH 12/52] docs(cache): implementation plan for Redis caching masterplan 25 task-by-task TDD plan with full code, exact paths, commit boundaries. Covers foundation (deps, config, client, decorator, lifespan, slowapi), route caches per namespace, integration tests, load test, ops docs. Co-Authored-By: Claude Opus 4.7 --- .../2026-06-22-redis-caching-masterplan.md | 2263 +++++++++++++++++ 1 file changed, 2263 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-22-redis-caching-masterplan.md diff --git a/docs/superpowers/plans/2026-06-22-redis-caching-masterplan.md b/docs/superpowers/plans/2026-06-22-redis-caching-masterplan.md new file mode 100644 index 00000000..8eb5461d --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-redis-caching-masterplan.md @@ -0,0 +1,2263 @@ +# Redis Caching Masterplan — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a production-grade Redis caching layer that reduces MongoDB read load ≥60%, brings p50 list latency under 50ms, and rolls out per-namespace with a global kill-switch. + +**Architecture:** Single-node Redis behind a lazy async pool. One `app.core.cache` facade with a `@cached` decorator. Versioned per-user keys (INCR on writes) plus short TTL on cross-user aggregates. XFetch (probabilistic early refresh) protects hot global keys from stampedes. Fail-open everywhere; structured logs. + +**Tech Stack:** FastAPI, Motor/MongoDB, `redis[asyncio]>=5.0`, `orjson>=3.10`, SlowAPI, pytest + httpx, `testcontainers-python` (integration tests). + +## Global Constraints + +- Python ≥ 3.10 (already required). +- Add only `redis[asyncio]>=5.0` and `orjson>=3.10` as top-level deps. No `pickle`. No new caching libraries. +- All new env vars added to `Settings` in `apps/backend/app/core/config.py` with safe defaults. +- Existing test suite must pass after every task — run `pytest -q` from `apps/backend/`. +- Commit after every task. Conventional Commits format (`feat(cache): ...`, `test(cache): ...`). +- Cache layer is fail-open: every Redis call wrapped to log + degrade to direct DB. Never propagate `RedisError`. +- Decorator default scope = `"user"` — requires `uid` kwarg. Global namespaces opt in via `scope="global"`. +- Sensitive payloads (passwords) stored only as ciphertext, same shape as Mongo doc. Never cache decrypted secrets. +- Phase-0 ship = `CACHE_NAMESPACES=""` — every `@cached` is a runtime no-op. Code lands with zero behavior change. + +--- + +## File Map + +| File | Status | Responsibility | +|------|--------|---------------| +| `apps/backend/pyproject.toml` | modify | Add `redis[asyncio]>=5.0`, `orjson>=3.10`, `testcontainers>=4.0` (dev). | +| `apps/backend/app/core/config.py` | modify | Add cache + redis env vars. | +| `apps/backend/app/core/redis_client.py` | new | Lazy async pool, lifespan open/close, startup ping. | +| `apps/backend/app/core/cache/__init__.py` | new | Public API re-exports. | +| `apps/backend/app/core/cache/serializer.py` | new | orjson dumps/loads with Pydantic + datetime support. | +| `apps/backend/app/core/cache/keys.py` | new | Namespace registry, key + version-key builders. | +| `apps/backend/app/core/cache/flags.py` | new | `is_namespace_enabled(ns)`; global kill check. | +| `apps/backend/app/core/cache/xfetch.py` | new | Probabilistic early-expiration math. | +| `apps/backend/app/core/cache/decorator.py` | new | `@cached`, `get_or_set`, `bump_version`, `cache_invalidate`. | +| `apps/backend/app/main.py` | modify | Wire redis pool into lifespan; runtime kill-switch on ping failure. | +| `apps/backend/app/core/limiter.py` | modify | Switch SlowAPI to Redis storage when `REDIS_URL` set. | +| `apps/backend/app/api/routes/auth/services.py` | modify | Cache `verify_id_token`, wrap `get_current_user`, invalidate on logout. | +| `apps/backend/app/api/routes/bookmarks/services.py` | modify | `@cached` on reads, `bump_version` on writes. | +| `apps/backend/app/api/routes/notes/services.py` | modify | same pattern. | +| `apps/backend/app/api/routes/code_snippets/services.py` | modify | same. | +| `apps/backend/app/api/routes/tasks/services.py` | modify | same (60s TTL). | +| `apps/backend/app/api/routes/passwords/services.py` | modify | same (ciphertext only). | +| `apps/backend/app/api/routes/api_client/services.py` | modify | same. | +| `apps/backend/app/api/routes/user_preferences/services.py` | modify | same (600s TTL). | +| `apps/backend/app/api/routes/url_shortener/services.py` | modify | XFetch on public resolve, user-scoped list on owner reads. | +| `apps/backend/app/api/routes/analytics/services.py` | modify | XFetch on top-tools + activity aggregates. | +| `apps/backend/app/api/routes/dns_lookup/services.py` | modify | `scope="global"`, 1h TTL. | +| `apps/backend/tests/test_cache_serializer.py` | new | Serializer round-trip tests. | +| `apps/backend/tests/test_cache_keys.py` | new | Key builder + hash determinism. | +| `apps/backend/tests/test_cache_xfetch.py` | new | XFetch math properties. | +| `apps/backend/tests/test_cache_flags.py` | new | Namespace flag parsing. | +| `apps/backend/tests/test_cache_decorator.py` | new | Decorator hit/miss/fail-open + version bump. | +| `apps/backend/tests/test_cache_integration.py` | new | Real Redis integration (testcontainers). | +| `apps/backend/scripts/loadtest_cache.py` | new | k6/locust scenario for 1000 concurrent. | +| `apps/backend/README.md` | modify | Document cache env vars + deployment notes. | + +--- + +## Task 1: Add dependencies + +**Files:** +- Modify: `apps/backend/pyproject.toml` + +**Interfaces:** +- Produces: `redis.asyncio` importable; `orjson` importable; `testcontainers.redis.RedisContainer` importable (dev only). + +- [ ] **Step 1: Add deps to `pyproject.toml`** + +Edit `apps/backend/pyproject.toml`. The `dependencies` list becomes: + +```toml +dependencies = [ + "fastapi[standard]", + "uvicorn[standard]>=0.35.0", + "pydantic-settings>=2.10.1", + "firebase-admin>=7.1.0", + "pymongo", + "motor>=3.7.0", + "python-jose[cryptography]>=3.5.0", + "boto3>=1.38.0", + "slowapi>=0.1.9", + "redis[asyncio]>=5.0", + "orjson>=3.10", +] +``` + +The `[project.optional-dependencies]` `dev` list becomes: + +```toml +dev = [ + "pytest>=8.4.1", + "httpx>=0.28.1", + "ruff>=0.13.0", + "testcontainers[redis]>=4.0", +] +``` + +- [ ] **Step 2: Install** + +Run: +```bash +cd apps/backend && uv sync --all-extras +``` +Expected: resolves without conflicts. + +- [ ] **Step 3: Verify imports** + +Run: +```bash +cd apps/backend && python -c "import redis.asyncio, orjson; print('ok')" +``` +Expected output: `ok` + +- [ ] **Step 4: Run existing tests to verify no regression** + +Run: +```bash +cd apps/backend && pytest -q +``` +Expected: all existing tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/pyproject.toml apps/backend/uv.lock +git commit -m "feat(cache): add redis[asyncio] + orjson deps" +``` + +--- + +## Task 2: Add cache config settings + +**Files:** +- Modify: `apps/backend/app/core/config.py` + +**Interfaces:** +- Produces: `Settings.REDIS_URL: str | None`, `Settings.CACHE_ENABLED: bool`, `Settings.CACHE_NAMESPACES: str`, `Settings.CACHE_DEFAULT_TTL: int`, `Settings.CACHE_OP_TIMEOUT_MS: int`, `Settings.CACHE_XFETCH_BETA: float`, `Settings.CACHE_LOG_LEVEL: str`. + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_cache_config.py`: + +```python +import os +from app.core.config import Settings + + +def test_cache_defaults(): + os.environ.pop("CACHE_ENABLED", None) + os.environ.pop("CACHE_NAMESPACES", None) + s = Settings(ALLOWED_ORIGINS="http://localhost", ACCESS_TOKEN_EXPIRE_MINUTES=30, REFRESH_TOKEN_EXPIRE_DAYS=7) + assert s.REDIS_URL is None + assert s.CACHE_ENABLED is True + assert s.CACHE_NAMESPACES == "" + assert s.CACHE_DEFAULT_TTL == 120 + assert s.CACHE_OP_TIMEOUT_MS == 50 + assert s.CACHE_XFETCH_BETA == 1.0 + assert s.CACHE_LOG_LEVEL == "WARNING" +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: +```bash +cd apps/backend && pytest tests/test_cache_config.py -v +``` +Expected: FAIL — `AttributeError: 'Settings' object has no attribute 'REDIS_URL'`. + +- [ ] **Step 3: Add settings** + +Edit `apps/backend/app/core/config.py`. Inside the `Settings` class, after `ALLOWED_ORIGINS: str`, add: + +```python + # Redis + cache + REDIS_URL: str | None = None + CACHE_ENABLED: bool = True + CACHE_NAMESPACES: str = "" # comma-separated; empty = no-op + CACHE_DEFAULT_TTL: int = 120 # seconds + CACHE_OP_TIMEOUT_MS: int = 50 # per Redis call + CACHE_XFETCH_BETA: float = 1.0 # XFetch tuning constant + CACHE_LOG_LEVEL: str = "WARNING" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: +```bash +cd apps/backend && pytest tests/test_cache_config.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Run full suite** + +Run: +```bash +cd apps/backend && pytest -q +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add apps/backend/app/core/config.py apps/backend/tests/test_cache_config.py +git commit -m "feat(cache): add redis + cache env vars to Settings" +``` + +--- + +## Task 3: Redis client module + +**Files:** +- Create: `apps/backend/app/core/redis_client.py` +- Create: `apps/backend/tests/test_redis_client.py` + +**Interfaces:** +- Produces: + - `get_redis() -> redis.asyncio.Redis | None` — singleton, returns `None` if `REDIS_URL` unset OR pool failed startup ping. + - `async def open_redis() -> None` — called from lifespan; sets module singleton; pings. + - `async def close_redis() -> None` — closes pool. + - `async def is_redis_available() -> bool` — cheap status check. + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_redis_client.py`: + +```python +import pytest +from app.core import redis_client + + +@pytest.mark.asyncio +async def test_get_redis_returns_none_without_url(monkeypatch): + monkeypatch.setattr("app.core.redis_client._client", None) + monkeypatch.setattr("app.core.redis_client.get_settings", lambda: type("S", (), {"REDIS_URL": None})()) + assert redis_client.get_redis() is None + + +@pytest.mark.asyncio +async def test_open_redis_noop_without_url(monkeypatch): + monkeypatch.setattr("app.core.redis_client.get_settings", lambda: type("S", (), {"REDIS_URL": None, "CACHE_OP_TIMEOUT_MS": 50})()) + await redis_client.open_redis() + assert redis_client.get_redis() is None +``` + +(Note: `pytest-asyncio` already present via existing test pattern; if not, switch to `asyncio.run(...)` inside `def`.) + +- [ ] **Step 2: Run test — verify failure** + +```bash +cd apps/backend && pytest tests/test_redis_client.py -v +``` +Expected: FAIL — module `app.core.redis_client` not found. + +- [ ] **Step 3: Implement module** + +Create `apps/backend/app/core/redis_client.py`: + +```python +"""Lazy async Redis singleton + lifespan helpers.""" +from __future__ import annotations + +import asyncio +import logging +from typing import Optional + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from app.core.config import get_settings + +log = logging.getLogger("app.cache") + +_client: Optional[Redis] = None + + +def get_redis() -> Optional[Redis]: + """Return the singleton Redis client or None if unavailable.""" + settings = get_settings() + if settings.REDIS_URL is None: + return None + return _client + + +async def open_redis() -> None: + """Open the pool and ping; called from FastAPI lifespan.""" + global _client + settings = get_settings() + if settings.REDIS_URL is None: + log.info("redis.disabled url_unset") + return + try: + client = Redis.from_url( + settings.REDIS_URL, + encoding="utf-8", + decode_responses=False, + socket_timeout=settings.CACHE_OP_TIMEOUT_MS / 1000, + socket_connect_timeout=2.0, + health_check_interval=30, + ) + await asyncio.wait_for(client.ping(), timeout=2.0) + _client = client + log.info("redis.connected") + except (RedisError, asyncio.TimeoutError, OSError) as exc: + log.error("redis.connect.failed err=%s msg=%s", type(exc).__name__, exc) + _client = None # fail-open: cache layer treats all ops as miss + + +async def close_redis() -> None: + """Close the pool.""" + global _client + if _client is not None: + try: + await _client.aclose() + except Exception as exc: # noqa: BLE001 + log.warning("redis.close.failed err=%s", exc) + _client = None + + +async def is_redis_available() -> bool: + if _client is None: + return False + try: + await asyncio.wait_for(_client.ping(), timeout=0.5) + return True + except (RedisError, asyncio.TimeoutError): + return False +``` + +- [ ] **Step 4: Run test — verify pass** + +```bash +cd apps/backend && pytest tests/test_redis_client.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Run full suite** + +```bash +cd apps/backend && pytest -q +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add apps/backend/app/core/redis_client.py apps/backend/tests/test_redis_client.py +git commit -m "feat(cache): add lazy async Redis client with fail-open ping" +``` + +--- + +## Task 4: Cache serializer + +**Files:** +- Create: `apps/backend/app/core/cache/__init__.py` (empty marker for now) +- Create: `apps/backend/app/core/cache/serializer.py` +- Create: `apps/backend/tests/test_cache_serializer.py` + +**Interfaces:** +- Produces: + - `dumps(value: Any) -> bytes` — orjson; supports Pydantic v2 `BaseModel`, list/dict of them, datetime, bytes. + - `loads(payload: bytes) -> Any` — parses to dict/list/scalars. Pydantic reconstruction is caller's job. + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_cache_serializer.py`: + +```python +from datetime import datetime, timezone + +import pytest +from pydantic import BaseModel + +from app.core.cache.serializer import dumps, loads + + +class Sample(BaseModel): + id: str + when: datetime + tags: list[str] + + +def test_round_trip_dict(): + payload = {"a": 1, "b": "two", "c": [1, 2, 3]} + assert loads(dumps(payload)) == payload + + +def test_pydantic_round_trip(): + s = Sample(id="x", when=datetime(2026, 1, 1, tzinfo=timezone.utc), tags=["a", "b"]) + raw = dumps(s) + parsed = loads(raw) + assert parsed["id"] == "x" + assert parsed["tags"] == ["a", "b"] + assert parsed["when"].startswith("2026-01-01") + + +def test_list_of_pydantic(): + items = [Sample(id=str(i), when=datetime(2026, 1, 1, tzinfo=timezone.utc), tags=[]) for i in range(3)] + parsed = loads(dumps(items)) + assert isinstance(parsed, list) + assert parsed[0]["id"] == "0" + + +def test_none_round_trip(): + assert loads(dumps(None)) is None + + +def test_bytes_round_trip(): + raw = dumps({"k": b"\x00\xff"}) + # bytes auto-serialized as base64 string by orjson default + parsed = loads(raw) + assert "k" in parsed +``` + +- [ ] **Step 2: Run — verify failure** + +```bash +cd apps/backend && pytest tests/test_cache_serializer.py -v +``` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `apps/backend/app/core/cache/__init__.py`: + +```python +"""Cache facade. Public API lives in decorator.py and is re-exported here.""" +``` + +Create `apps/backend/app/core/cache/serializer.py`: + +```python +"""Cache serializer — orjson with Pydantic + datetime + bytes support.""" +from __future__ import annotations + +import base64 +from typing import Any + +import orjson +from pydantic import BaseModel + + +def _default(obj: Any) -> Any: + if isinstance(obj, BaseModel): + return obj.model_dump(mode="json") + if isinstance(obj, bytes): + return base64.b64encode(obj).decode("ascii") + raise TypeError(f"Type {type(obj).__name__} not serializable") + + +def dumps(value: Any) -> bytes: + return orjson.dumps( + value, + default=_default, + option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY, + ) + + +def loads(payload: bytes) -> Any: + return orjson.loads(payload) +``` + +- [ ] **Step 4: Run test — verify pass** + +```bash +cd apps/backend && pytest tests/test_cache_serializer.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/app/core/cache/__init__.py apps/backend/app/core/cache/serializer.py apps/backend/tests/test_cache_serializer.py +git commit -m "feat(cache): add orjson serializer with Pydantic support" +``` + +--- + +## Task 5: Cache keys + namespace registry + +**Files:** +- Create: `apps/backend/app/core/cache/keys.py` +- Create: `apps/backend/tests/test_cache_keys.py` + +**Interfaces:** +- Produces: + - `NAMESPACES: dict[str, NamespaceSpec]` — registry; `NamespaceSpec = TypedDict` with `scope`, `default_ttl`, `default_strategy`. + - `register_namespace(name, *, scope, default_ttl, default_strategy) -> None`. + - `build_key(*, ns: str, scope: str, uid: str | None, ver: int | None, op: str, args_hash: str) -> str`. + - `version_key(ns: str, uid: str) -> str`. + - `args_hash(kwargs: dict, *, secret: bytes) -> str` — keyed blake2b, returns 16-hex. + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_cache_keys.py`: + +```python +import pytest + +from app.core.cache.keys import ( + NAMESPACES, + args_hash, + build_key, + register_namespace, + version_key, +) + + +def test_register_and_lookup(): + register_namespace("bookmarks_test", scope="user", default_ttl=120, default_strategy="simple") + assert NAMESPACES["bookmarks_test"]["scope"] == "user" + + +def test_build_user_key(): + key = build_key(ns="bookmarks", scope="user", uid="u1", ver=7, op="list", args_hash="abcd1234") + assert key == "cache:bookmarks:u:u1:v7:list:abcd1234" + + +def test_build_global_key_no_version(): + key = build_key(ns="urlshort", scope="global", uid=None, ver=None, op="resolve", args_hash="ff00ee11") + assert key == "cache:urlshort:g:resolve:ff00ee11" + + +def test_version_key(): + assert version_key("bookmarks", "u1") == "cache:ver:bookmarks:u:u1" + + +def test_args_hash_deterministic(): + h1 = args_hash({"a": 1, "b": 2}, secret=b"k") + h2 = args_hash({"b": 2, "a": 1}, secret=b"k") + assert h1 == h2 + assert len(h1) == 16 + + +def test_args_hash_changes_with_input(): + h1 = args_hash({"a": 1}, secret=b"k") + h2 = args_hash({"a": 2}, secret=b"k") + assert h1 != h2 + + +def test_args_hash_changes_with_secret(): + h1 = args_hash({"a": 1}, secret=b"k1") + h2 = args_hash({"a": 1}, secret=b"k2") + assert h1 != h2 +``` + +- [ ] **Step 2: Run — verify failure** + +```bash +cd apps/backend && pytest tests/test_cache_keys.py -v +``` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `apps/backend/app/core/cache/keys.py`: + +```python +"""Cache key builders + namespace registry.""" +from __future__ import annotations + +import hashlib +from typing import Literal, TypedDict + +import orjson + + +class NamespaceSpec(TypedDict): + scope: Literal["user", "global"] + default_ttl: int + default_strategy: Literal["simple", "xfetch"] + + +NAMESPACES: dict[str, NamespaceSpec] = {} + + +def register_namespace( + name: str, + *, + scope: Literal["user", "global"], + default_ttl: int, + default_strategy: Literal["simple", "xfetch"] = "simple", +) -> None: + if scope == "user" and default_strategy == "xfetch": + raise ValueError("xfetch requires scope='global'") + NAMESPACES[name] = {"scope": scope, "default_ttl": default_ttl, "default_strategy": default_strategy} + + +def build_key( + *, + ns: str, + scope: str, + uid: str | None, + ver: int | None, + op: str, + args_hash: str, +) -> str: + if scope == "user": + if uid is None or ver is None: + raise ValueError("user-scoped key requires uid and ver") + return f"cache:{ns}:u:{uid}:v{ver}:{op}:{args_hash}" + return f"cache:{ns}:g:{op}:{args_hash}" + + +def version_key(ns: str, uid: str) -> str: + return f"cache:ver:{ns}:u:{uid}" + + +def args_hash(kwargs: dict, *, secret: bytes) -> str: + payload = orjson.dumps(kwargs, option=orjson.OPT_SORT_KEYS) + return hashlib.blake2b(payload, digest_size=8, key=secret[:64]).hexdigest() +``` + +Pre-register the namespaces from the spec. Append at end of `keys.py`: + +```python +# --- spec-locked namespaces --- +register_namespace("auth_token", scope="global", default_ttl=300) +register_namespace("auth_user", scope="user", default_ttl=60) +register_namespace("bookmarks", scope="user", default_ttl=120) +register_namespace("notes", scope="user", default_ttl=120) +register_namespace("code_snippets", scope="user", default_ttl=120) +register_namespace("tasks", scope="user", default_ttl=60) +register_namespace("passwords", scope="user", default_ttl=60) +register_namespace("api_client", scope="user", default_ttl=300) +register_namespace("user_preferences", scope="user", default_ttl=600) +register_namespace("url_shortener_resolve", scope="global", default_ttl=600, default_strategy="xfetch") +register_namespace("url_shortener_owner", scope="user", default_ttl=120) +register_namespace("analytics_aggregate", scope="global", default_ttl=300, default_strategy="xfetch") +register_namespace("dns_lookup", scope="global", default_ttl=3600) +``` + +- [ ] **Step 4: Run — verify pass** + +```bash +cd apps/backend && pytest tests/test_cache_keys.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/app/core/cache/keys.py apps/backend/tests/test_cache_keys.py +git commit -m "feat(cache): add namespace registry + key builders" +``` + +--- + +## Task 6: XFetch math + +**Files:** +- Create: `apps/backend/app/core/cache/xfetch.py` +- Create: `apps/backend/tests/test_cache_xfetch.py` + +**Interfaces:** +- Produces: + - `should_refresh(*, computed_at: float, ttl: float, delta: float, beta: float, now: float, rand: float) -> bool` + - `wrap_payload(value: Any, *, computed_at: float, delta: float) -> dict` + - `unwrap_payload(payload: dict) -> tuple[Any, float, float]` → `(value, computed_at, delta)` + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_cache_xfetch.py`: + +```python +import math + +import pytest + +from app.core.cache.xfetch import should_refresh, unwrap_payload, wrap_payload + + +def test_beta_zero_never_refreshes_early(): + # beta=0 reduces to: now > computed_at + ttl → only after TTL + assert should_refresh(computed_at=0, ttl=100, delta=5, beta=0, now=50, rand=0.001) is False + assert should_refresh(computed_at=0, ttl=100, delta=5, beta=0, now=101, rand=0.5) is True + + +def test_high_beta_refreshes_earlier(): + # With rand → 0, ln(rand) → -inf, refresh fires very early when beta > 0 + fires_at_t50 = should_refresh(computed_at=0, ttl=100, delta=5, beta=10, now=50, rand=1e-9) + assert fires_at_t50 is True + + +def test_rand_near_one_no_early_refresh(): + # rand close to 1 → ln(rand) close to 0 → only past TTL + assert should_refresh(computed_at=0, ttl=100, delta=5, beta=1, now=80, rand=0.999) is False + + +def test_wrap_unwrap_round_trip(): + p = wrap_payload({"x": 1}, computed_at=12345.0, delta=2.5) + val, ca, dt = unwrap_payload(p) + assert val == {"x": 1} + assert ca == 12345.0 + assert dt == 2.5 +``` + +- [ ] **Step 2: Run — verify failure** + +```bash +cd apps/backend && pytest tests/test_cache_xfetch.py -v +``` +Expected: FAIL — module missing. + +- [ ] **Step 3: Implement** + +Create `apps/backend/app/core/cache/xfetch.py`: + +```python +"""Probabilistic early-expiration (XFetch). + +Reference: "Optimal Probabilistic Cache Stampede Prevention" (Vattani et al., 2015). +""" +from __future__ import annotations + +import math +from typing import Any + + +def should_refresh( + *, + computed_at: float, + ttl: float, + delta: float, + beta: float, + now: float, + rand: float, +) -> bool: + """Return True if the caller should refresh the cached value now.""" + if rand <= 0.0: + rand = 1e-12 + if rand > 1.0: + rand = 1.0 + threshold = computed_at + ttl - beta * delta * math.log(rand) + return now >= threshold + + +def wrap_payload(value: Any, *, computed_at: float, delta: float) -> dict: + return {"v": value, "ca": computed_at, "dt": delta} + + +def unwrap_payload(payload: dict) -> tuple[Any, float, float]: + return payload["v"], float(payload["ca"]), float(payload["dt"]) +``` + +Note the math: `should_refresh` is `now >= ca + ttl - beta*delta*ln(rand)`. Because `ln(rand)` is negative for `rand < 1`, `-beta*delta*ln(rand)` is positive — it shifts the threshold *earlier*. Larger `beta` or `delta` ⇒ refreshes earlier. + +- [ ] **Step 4: Run — verify pass** + +```bash +cd apps/backend && pytest tests/test_cache_xfetch.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/app/core/cache/xfetch.py apps/backend/tests/test_cache_xfetch.py +git commit -m "feat(cache): add XFetch math for stampede protection" +``` + +--- + +## Task 7: Namespace flags + +**Files:** +- Create: `apps/backend/app/core/cache/flags.py` +- Create: `apps/backend/tests/test_cache_flags.py` + +**Interfaces:** +- Produces: + - `is_namespace_enabled(ns: str) -> bool` — checks `CACHE_ENABLED` AND `ns in CACHE_NAMESPACES`. + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_cache_flags.py`: + +```python +import pytest +from app.core import config +from app.core.cache.flags import is_namespace_enabled + + +def _set(monkeypatch, enabled=True, namespaces=""): + monkeypatch.setattr(config, "get_settings", lambda: type("S", (), { + "CACHE_ENABLED": enabled, + "CACHE_NAMESPACES": namespaces, + })()) + # invalidate any LRU cache on flags + from app.core.cache import flags as f + f._parsed_namespaces.cache_clear() + + +def test_disabled_globally(monkeypatch): + _set(monkeypatch, enabled=False, namespaces="bookmarks") + assert is_namespace_enabled("bookmarks") is False + + +def test_empty_namespaces(monkeypatch): + _set(monkeypatch, enabled=True, namespaces="") + assert is_namespace_enabled("bookmarks") is False + + +def test_matching_namespace(monkeypatch): + _set(monkeypatch, enabled=True, namespaces="bookmarks,notes") + assert is_namespace_enabled("bookmarks") is True + assert is_namespace_enabled("notes") is True + assert is_namespace_enabled("tasks") is False + + +def test_whitespace_tolerant(monkeypatch): + _set(monkeypatch, enabled=True, namespaces="bookmarks , notes") + assert is_namespace_enabled("bookmarks") is True + assert is_namespace_enabled("notes") is True +``` + +- [ ] **Step 2: Run — verify failure** + +```bash +cd apps/backend && pytest tests/test_cache_flags.py -v +``` +Expected: FAIL — module missing. + +- [ ] **Step 3: Implement** + +Create `apps/backend/app/core/cache/flags.py`: + +```python +"""Per-namespace cache enable flags.""" +from __future__ import annotations + +from functools import lru_cache + +from app.core.config import get_settings + + +@lru_cache(maxsize=1) +def _parsed_namespaces() -> frozenset[str]: + raw = get_settings().CACHE_NAMESPACES or "" + return frozenset(p.strip() for p in raw.split(",") if p.strip()) + + +def is_namespace_enabled(ns: str) -> bool: + s = get_settings() + if not s.CACHE_ENABLED: + return False + return ns in _parsed_namespaces() +``` + +- [ ] **Step 4: Run — verify pass** + +```bash +cd apps/backend && pytest tests/test_cache_flags.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/app/core/cache/flags.py apps/backend/tests/test_cache_flags.py +git commit -m "feat(cache): add per-namespace enable flag with global kill" +``` + +--- + +## Task 8: Cache decorator (core read path + fail-open) + +**Files:** +- Create: `apps/backend/app/core/cache/decorator.py` +- Create: `apps/backend/tests/test_cache_decorator.py` + +**Interfaces:** +- Produces: + - `cached(*, ns, ttl=None, scope=None, strategy=None, key=None)` — decorator factory for async fns. + - `async def bump_version(*, ns: str, uid: str) -> None` + - `async def cache_invalidate(*, ns: str, key: str) -> None` (where `key` is full key string OR `(op, args)` tuple — keep simple: full key) + - `async def get_or_set(*, ns, key, loader, ttl=None, strategy=None, scope=None, uid=None) -> Any` + +- [ ] **Step 1: Write failing test** + +Create `apps/backend/tests/test_cache_decorator.py`: + +```python +import asyncio +import pytest + +from app.core.cache.decorator import cached, bump_version +from app.core.cache import keys as keys_mod + + +class _FakeRedis: + def __init__(self): + self.store: dict[bytes, bytes] = {} + self.versions: dict[bytes, int] = {} + self.fail: bool = False + self.calls: list[tuple[str, str]] = [] + + async def get(self, k): + self.calls.append(("get", k.decode() if isinstance(k, bytes) else k)) + if self.fail: + raise ConnectionError("down") + return self.store.get(k if isinstance(k, bytes) else k.encode()) + + async def setex(self, k, ttl, v): + self.calls.append(("setex", k.decode() if isinstance(k, bytes) else k)) + if self.fail: + raise ConnectionError("down") + self.store[k if isinstance(k, bytes) else k.encode()] = v + + async def incr(self, k): + if self.fail: + raise ConnectionError("down") + key = k if isinstance(k, bytes) else k.encode() + self.versions[key] = self.versions.get(key, 0) + 1 + return self.versions[key] + + async def delete(self, k): + if self.fail: + raise ConnectionError("down") + self.store.pop(k if isinstance(k, bytes) else k.encode(), None) + + +@pytest.fixture +def fake_redis(monkeypatch): + r = _FakeRedis() + monkeypatch.setattr("app.core.cache.decorator.get_redis", lambda: r) + # also make ver-key lookups return string bytes + async def _get(k): + v = r.versions.get(k if isinstance(k, bytes) else k.encode()) + return str(v).encode() if v is not None else None + # override to support both reads and version reads + orig = r.get + async def patched_get(k): + # version key path + if (k if isinstance(k, str) else k.decode()).startswith("cache:ver:"): + return await _get(k) + return await orig(k) + monkeypatch.setattr(r, "get", patched_get) + return r + + +@pytest.fixture +def enable_ns(monkeypatch): + monkeypatch.setattr("app.core.cache.decorator.is_namespace_enabled", lambda ns: True) + monkeypatch.setattr("app.core.cache.decorator._secret", lambda: b"test-secret") + + +@pytest.mark.asyncio +async def test_decorator_miss_then_hit(fake_redis, enable_ns): + calls = {"n": 0} + + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(*, uid: str): + calls["n"] += 1 + return [{"id": "a"}] + + r1 = await list_bookmarks(uid="u1") + r2 = await list_bookmarks(uid="u1") + assert r1 == r2 == [{"id": "a"}] + assert calls["n"] == 1 # second call served from cache + + +@pytest.mark.asyncio +async def test_bump_version_invalidates(fake_redis, enable_ns): + calls = {"n": 0} + + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(*, uid: str): + calls["n"] += 1 + return [{"id": "a"}] + + await list_bookmarks(uid="u1") + await bump_version(ns="bookmarks", uid="u1") + await list_bookmarks(uid="u1") + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_fail_open(fake_redis, enable_ns): + fake_redis.fail = True + + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(*, uid: str): + return [{"id": "from-mongo"}] + + out = await list_bookmarks(uid="u1") + assert out == [{"id": "from-mongo"}] + + +@pytest.mark.asyncio +async def test_disabled_namespace_skips_redis(fake_redis, monkeypatch): + monkeypatch.setattr("app.core.cache.decorator.is_namespace_enabled", lambda ns: False) + monkeypatch.setattr("app.core.cache.decorator._secret", lambda: b"x") + + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(*, uid: str): + return [{"id": "a"}] + + fake_redis.calls.clear() + await list_bookmarks(uid="u1") + assert fake_redis.calls == [] + + +@pytest.mark.asyncio +async def test_user_scope_requires_uid(fake_redis, enable_ns): + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(**kw): + return [] + + with pytest.raises(ValueError, match="uid"): + await list_bookmarks(folder_id="x") +``` + +- [ ] **Step 2: Run — verify failure** + +```bash +cd apps/backend && pytest tests/test_cache_decorator.py -v +``` +Expected: FAIL — module missing. + +- [ ] **Step 3: Implement** + +Create `apps/backend/app/core/cache/decorator.py`: + +```python +"""Cache decorator + helpers — read path, fail-open, version invalidation.""" +from __future__ import annotations + +import asyncio +import functools +import inspect +import logging +import random +import time +from typing import Any, Awaitable, Callable, Literal, Optional + +from redis.exceptions import RedisError + +from app.core.cache.flags import is_namespace_enabled +from app.core.cache.keys import ( + NAMESPACES, + NamespaceSpec, + args_hash as _args_hash, + build_key, + version_key, +) +from app.core.cache.serializer import dumps, loads +from app.core.cache.xfetch import should_refresh, unwrap_payload, wrap_payload +from app.core.config import get_settings +from app.core.redis_client import get_redis + +log = logging.getLogger("app.cache") + + +def _secret() -> bytes: + s = get_settings() + return (s.JWT_SECRET_KEY or "default-cache-secret").encode("utf-8") + + +async def _safe(coro: Awaitable, *, op: str, ns: str) -> Any: + s = get_settings() + try: + return await asyncio.wait_for(coro, timeout=s.CACHE_OP_TIMEOUT_MS / 1000) + except (RedisError, asyncio.TimeoutError, ConnectionError, OSError) as exc: + log.warning("cache.error op=%s ns=%s err=%s msg=%s", op, ns, type(exc).__name__, exc) + return None + + +async def _get_version(r, ns: str, uid: str) -> int: + raw = await _safe(r.get(version_key(ns, uid)), op="ver_get", ns=ns) + if raw is None: + return 0 + try: + return int(raw) + except (TypeError, ValueError): + return 0 + + +def cached( + *, + ns: str, + ttl: Optional[int] = None, + scope: Optional[Literal["user", "global"]] = None, + strategy: Optional[Literal["simple", "xfetch"]] = None, + key: Optional[Callable[..., str]] = None, +): + spec: NamespaceSpec | None = NAMESPACES.get(ns) + if spec is None: + raise ValueError(f"Namespace not registered: {ns!r}") + eff_scope = scope or spec["scope"] + eff_ttl = ttl or spec["default_ttl"] + eff_strategy = strategy or spec["default_strategy"] + if eff_strategy == "xfetch" and eff_scope != "global": + raise ValueError("xfetch requires scope='global'") + + def decorator(fn: Callable[..., Awaitable[Any]]): + if not inspect.iscoroutinefunction(fn): + raise TypeError(f"@cached requires async fn; got {fn!r}") + op_name = fn.__name__ + + @functools.wraps(fn) + async def wrapper(*args, **kwargs): + # always allow disabled-namespace short-circuit + if not is_namespace_enabled(ns): + return await fn(*args, **kwargs) + + r = get_redis() + if r is None: + return await fn(*args, **kwargs) + + uid = kwargs.get("uid") + if eff_scope == "user" and not uid: + raise ValueError(f"{op_name}: scope='user' requires uid kwarg") + + # build args_hash from kwargs except 'uid' + hash_args = {k: v for k, v in kwargs.items() if k != "uid"} + ah = key(**kwargs) if key else _args_hash(hash_args, secret=_secret()) + + ver = await _get_version(r, ns, uid) if eff_scope == "user" else None + k = build_key(ns=ns, scope=eff_scope, uid=uid, ver=ver, op=op_name, args_hash=ah) + k_bytes = k.encode() + + raw = await _safe(r.get(k_bytes), op="get", ns=ns) + if raw is not None: + try: + payload = loads(raw) + except Exception as exc: # noqa: BLE001 + log.warning("cache.deserialize.failed ns=%s err=%s", ns, exc) + payload = None + + if eff_strategy == "xfetch" and isinstance(payload, dict) and "v" in payload: + value, computed_at, delta = unwrap_payload(payload) + if should_refresh( + computed_at=computed_at, + ttl=eff_ttl, + delta=delta, + beta=get_settings().CACHE_XFETCH_BETA, + now=time.time(), + rand=random.random(), + ): + asyncio.create_task(_refresh(fn, args, kwargs, r, k_bytes, eff_ttl, ns, eff_strategy)) + log.debug("cache.hit ns=%s op=%s", ns, op_name) + return value + + log.debug("cache.hit ns=%s op=%s", ns, op_name) + return payload + + log.debug("cache.miss ns=%s op=%s", ns, op_name) + t0 = time.time() + result = await fn(*args, **kwargs) + delta = max(time.time() - t0, 0.001) + + if eff_strategy == "xfetch": + stored = dumps(wrap_payload(result, computed_at=time.time(), delta=delta)) + else: + stored = dumps(result) + + await _safe(r.setex(k_bytes, eff_ttl, stored), op="setex", ns=ns) + return result + + return wrapper + + return decorator + + +async def _refresh(fn, args, kwargs, r, k_bytes, ttl, ns, strategy): + try: + t0 = time.time() + result = await fn(*args, **kwargs) + delta = max(time.time() - t0, 0.001) + if strategy == "xfetch": + stored = dumps(wrap_payload(result, computed_at=time.time(), delta=delta)) + else: + stored = dumps(result) + await _safe(r.setex(k_bytes, ttl, stored), op="setex_refresh", ns=ns) + log.info("cache.xfetch.refresh ns=%s", ns) + except Exception as exc: # noqa: BLE001 + log.warning("cache.xfetch.refresh.failed ns=%s err=%s", ns, exc) + + +async def bump_version(*, ns: str, uid: str) -> None: + r = get_redis() + if r is None: + return + await _safe(r.incr(version_key(ns, uid)), op="ver_incr", ns=ns) + log.info("cache.bump_version ns=%s uid=%s", ns, uid) + + +async def cache_invalidate(*, ns: str, key: str) -> None: + r = get_redis() + if r is None: + return + await _safe(r.delete(key.encode() if isinstance(key, str) else key), op="del", ns=ns) + + +async def get_or_set( + *, + ns: str, + key: str, + loader: Callable[[], Awaitable[Any]], + ttl: Optional[int] = None, + strategy: Optional[Literal["simple", "xfetch"]] = None, +) -> Any: + spec = NAMESPACES.get(ns) + if spec is None: + return await loader() + eff_ttl = ttl or spec["default_ttl"] + eff_strategy = strategy or spec["default_strategy"] + if not is_namespace_enabled(ns): + return await loader() + r = get_redis() + if r is None: + return await loader() + raw = await _safe(r.get(key.encode()), op="get", ns=ns) + if raw is not None: + return loads(raw) + result = await loader() + if eff_strategy == "xfetch": + stored = dumps(wrap_payload(result, computed_at=time.time(), delta=0.001)) + else: + stored = dumps(result) + await _safe(r.setex(key.encode(), eff_ttl, stored), op="setex", ns=ns) + return result +``` + +- [ ] **Step 4: Run — verify pass** + +```bash +cd apps/backend && pytest tests/test_cache_decorator.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Run full suite** + +```bash +cd apps/backend && pytest -q +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add apps/backend/app/core/cache/decorator.py apps/backend/tests/test_cache_decorator.py +git commit -m "feat(cache): add @cached decorator + bump_version + invalidate" +``` + +--- + +## Task 9: Cache public API re-exports + +**Files:** +- Modify: `apps/backend/app/core/cache/__init__.py` + +**Interfaces:** +- Produces: importable from `app.core.cache`: `cached`, `bump_version`, `cache_invalidate`, `get_or_set`. + +- [ ] **Step 1: Add test for the public surface** + +Append to `apps/backend/tests/test_cache_decorator.py`: + +```python +def test_public_api(): + from app.core.cache import cached, bump_version, cache_invalidate, get_or_set + assert callable(cached) + assert callable(bump_version) + assert callable(cache_invalidate) + assert callable(get_or_set) +``` + +- [ ] **Step 2: Run — verify failure** + +```bash +cd apps/backend && pytest tests/test_cache_decorator.py::test_public_api -v +``` +Expected: FAIL — `ImportError: cannot import name 'cached' from 'app.core.cache'`. + +- [ ] **Step 3: Implement re-exports** + +Replace `apps/backend/app/core/cache/__init__.py`: + +```python +"""Public cache API.""" +from app.core.cache.decorator import ( + bump_version, + cache_invalidate, + cached, + get_or_set, +) + +__all__ = ["cached", "bump_version", "cache_invalidate", "get_or_set"] +``` + +- [ ] **Step 4: Run — verify pass** + +```bash +cd apps/backend && pytest tests/test_cache_decorator.py::test_public_api -v +``` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/backend/app/core/cache/__init__.py apps/backend/tests/test_cache_decorator.py +git commit -m "feat(cache): expose public cache API from app.core.cache" +``` + +--- + +## Task 10: Wire Redis to FastAPI lifespan + +**Files:** +- Modify: `apps/backend/app/main.py` + +**Interfaces:** +- Produces: Redis pool opened on app startup, closed on shutdown; failure to connect does not stop the app. + +- [ ] **Step 1: Add lifespan integration test** + +Create `apps/backend/tests/test_lifespan_redis.py`: + +```python +import pytest +from httpx import ASGITransport, AsyncClient + + +@pytest.mark.asyncio +async def test_app_boots_without_redis(monkeypatch): + monkeypatch.delenv("REDIS_URL", raising=False) + from app.main import app + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as ac: + r = await ac.get("/health") + assert r.status_code == 200 +``` + +- [ ] **Step 2: Run — observe current state (may pass; lifespan currently doesn't open Redis)** + +```bash +cd apps/backend && pytest tests/test_lifespan_redis.py -v +``` +Expected: PASS (current lifespan doesn't touch Redis). + +- [ ] **Step 3: Add Redis open/close calls to lifespan** + +Edit `apps/backend/app/main.py`. Replace the `lifespan` function with: + +```python +@asynccontextmanager +async def lifespan(_app: FastAPI): + try: + from app.core.indexes import ensure_indexes + await ensure_indexes() + except Exception as exc: + logging.getLogger(__name__).warning("Index creation failed: %s", exc) + + from app.core.redis_client import open_redis, close_redis + await open_redis() + + try: + yield + finally: + await close_redis() +``` + +- [ ] **Step 4: Re-run lifespan test** + +```bash +cd apps/backend && pytest tests/test_lifespan_redis.py -v +``` +Expected: PASS — app still boots without Redis (fail-open). + +- [ ] **Step 5: Full suite** + +```bash +cd apps/backend && pytest -q +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add apps/backend/app/main.py apps/backend/tests/test_lifespan_redis.py +git commit -m "feat(cache): open/close Redis pool from FastAPI lifespan" +``` + +--- + +## Task 11: SlowAPI Redis storage backend + +**Files:** +- Modify: `apps/backend/app/core/limiter.py` + +**Interfaces:** +- Produces: `limiter` uses Redis storage when `REDIS_URL` is set; falls back to in-memory otherwise. + +- [ ] **Step 1: Update `limiter.py`** + +Replace `apps/backend/app/core/limiter.py`: + +```python +import logging + +from fastapi import Request +from slowapi import Limiter +from slowapi.util import get_remote_address + +from app.core.config import get_settings + +log = logging.getLogger("app.limiter") + + +def _get_client_ip(request: Request) -> str: + forwarded = request.headers.get("x-forwarded-for") + if forwarded: + return forwarded.split(",")[0].strip() + return get_remote_address(request) + + +def _build_limiter() -> Limiter: + settings = get_settings() + if settings.REDIS_URL: + try: + return Limiter( + key_func=_get_client_ip, + storage_uri=settings.REDIS_URL, + ) + except Exception as exc: # noqa: BLE001 + log.warning("limiter.redis.fallback err=%s", exc) + return Limiter(key_func=_get_client_ip) + + +limiter = _build_limiter() +``` + +- [ ] **Step 2: Run full suite** + +```bash +cd apps/backend && pytest -q +``` +Expected: all pass (no behavior change without `REDIS_URL`). + +- [ ] **Step 3: Commit** + +```bash +git add apps/backend/app/core/limiter.py +git commit -m "feat(cache): switch SlowAPI to Redis storage when REDIS_URL set" +``` + +--- + +## Task 12: Integration test against real Redis + +**Files:** +- Create: `apps/backend/tests/test_cache_integration.py` + +**Interfaces:** +- Verifies: decorator + bump_version + invalidate against a real `redis:7-alpine` container. + +- [ ] **Step 1: Write integration test** + +Create `apps/backend/tests/test_cache_integration.py`: + +```python +import asyncio + +import pytest +from testcontainers.redis import RedisContainer + +from app.core.cache import cached, bump_version + + +@pytest.fixture(scope="module") +def redis_container(): + with RedisContainer("redis:7-alpine") as c: + yield c + + +@pytest.fixture +async def real_redis(redis_container, monkeypatch): + url = f"redis://{redis_container.get_container_host_ip()}:{redis_container.get_exposed_port(6379)}" + monkeypatch.setenv("REDIS_URL", url) + monkeypatch.setenv("CACHE_NAMESPACES", "bookmarks,notes,analytics_aggregate") + # reset Settings cache + from app.core import config + config.get_settings.cache_clear() + from app.core.cache import flags + flags._parsed_namespaces.cache_clear() + from app.core.redis_client import open_redis, close_redis + await open_redis() + yield + await close_redis() + + +@pytest.mark.asyncio +async def test_real_decorator_hit(real_redis): + calls = {"n": 0} + + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(*, uid: str): + calls["n"] += 1 + return [{"id": "x"}] + + await list_bookmarks(uid="u1") + await list_bookmarks(uid="u1") + assert calls["n"] == 1 + + +@pytest.mark.asyncio +async def test_real_bump_invalidates(real_redis): + calls = {"n": 0} + + @cached(ns="notes", ttl=60, scope="user") + async def list_notes(*, uid: str): + calls["n"] += 1 + return [{"id": "y"}] + + await list_notes(uid="u1") + await bump_version(ns="notes", uid="u1") + await list_notes(uid="u1") + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_cross_user_isolation(real_redis): + calls = {"n": 0} + + @cached(ns="bookmarks", ttl=60, scope="user") + async def list_bookmarks(*, uid: str): + calls["n"] += 1 + return [{"uid": uid}] + + await list_bookmarks(uid="u1") + await bump_version(ns="bookmarks", uid="u1") # only u1 invalidated + await list_bookmarks(uid="u2") + await list_bookmarks(uid="u2") # second u2 call must hit cache + assert calls["n"] == 3 # u1, u1-after-bump, u2-first (u2-second = cache hit) +``` + +- [ ] **Step 2: Run integration tests** + +```bash +cd apps/backend && pytest tests/test_cache_integration.py -v +``` +Expected: PASS (Docker required for testcontainers). + +- [ ] **Step 3: Commit** + +```bash +git add apps/backend/tests/test_cache_integration.py +git commit -m "test(cache): add integration tests against real Redis container" +``` + +--- + +## Task 13: Auth — token + user profile cache + +**Files:** +- Modify: `apps/backend/app/api/routes/auth/services.py` + +**Interfaces:** +- Produces: `verify_id_token_cached(token: str) -> dict` (drop-in for `verify_id_token`); `get_current_user` reads user doc through cache; logout calls `cache_invalidate` on both. + +- [ ] **Step 1: Read current `verify_id_token` + `get_current_user`** + +Reference (existing, do not change unless noted): `apps/backend/app/api/routes/auth/services.py` lines 1-100. + +- [ ] **Step 2: Add helper near top of file (after imports)** + +Edit `apps/backend/app/api/routes/auth/services.py`. After existing imports, add: + +```python +import hashlib + +from app.core.cache import cached, cache_invalidate, get_or_set +from app.core.cache.keys import build_key + + +def _token_cache_key(token: str) -> str: + h = hashlib.sha256(token.encode("utf-8")).hexdigest()[:16] + return build_key(ns="auth_token", scope="global", uid=None, ver=None, op="verify", args_hash=h) +``` + +- [ ] **Step 3: Wrap token verification** + +Add a new fn alongside `verify_id_token`: + +```python +async def verify_id_token_cached(id_token: str, check_revoked: bool = False) -> dict: + key = _token_cache_key(id_token) + + async def _loader(): + return verify_id_token(id_token, check_revoked=check_revoked) + + return await get_or_set(ns="auth_token", key=key, loader=_loader) +``` + +(Keep `verify_id_token` synchronous as-is for any caller that needs uncached behavior.) + +- [ ] **Step 4: Cache user-profile fetch** + +Add (or replace existing `get_current_user` body — keep request-state memoization, but on Mongo miss go through cache): + +```python +async def _fetch_user_doc_cached(uid: str) -> dict | None: + @cached(ns="auth_user", ttl=60, scope="user") + async def _inner(*, uid: str): + return await get_user_doc(uid) + return await _inner(uid=uid) +``` + +Then modify `get_current_user` (after `doc = await get_user_doc(uid)`): + +```python + cached_doc = getattr(request.state, "current_user_doc", None) + if cached_doc is not None and cached_doc.get("_id") == uid: + doc = cached_doc + else: + doc = await _fetch_user_doc_cached(uid=uid) + if not doc: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found.") + request.state.current_user_doc = doc +``` + +- [ ] **Step 5: Add invalidation on logout / password change** + +Find the existing `logout` (or equivalent) handler and add at the end (after the existing logic): + +```python + try: + if id_token: + await cache_invalidate(ns="auth_token", key=_token_cache_key(id_token)) + if uid: + from app.core.cache.keys import build_key, version_key + await cache_invalidate(ns="auth_user", key=build_key( + ns="auth_user", scope="user", uid=uid, ver=0, op="_inner", args_hash="*" + )) + except Exception: + pass # fail-open +``` + +(If logout doesn't have `id_token` at hand, only invalidate `auth_user`; bump the user-scoped version key instead.) + +Recommended cleaner approach — call `bump_version` on logout: + +```python + from app.core.cache import bump_version + try: + await bump_version(ns="auth_user", uid=uid) + except Exception: + pass +``` + +- [ ] **Step 6: Run suite** + +```bash +cd apps/backend && pytest -q +``` +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git add apps/backend/app/api/routes/auth/services.py +git commit -m "feat(cache): cache Firebase verify + user profile; invalidate on logout" +``` + +--- + +## Task 14: Bookmarks — `@cached` reads + `bump_version` writes + +**Files:** +- Modify: `apps/backend/app/api/routes/bookmarks/services.py` + +**Interfaces:** +- Produces: reads of bookmarks and folders cached per-user; every write `bump_version`s `bookmarks`. + +- [ ] **Step 1: Add imports** + +Edit top of `apps/backend/app/api/routes/bookmarks/services.py`. Add: + +```python +from app.core.cache import cached, bump_version +``` + +- [ ] **Step 2: Decorate reads** + +Find `async def list_bookmarks(uid: str, ...)`. Add decorator above: + +```python +@cached(ns="bookmarks", ttl=120, scope="user") +async def list_bookmarks(*, uid: str, folder_id: Optional[str] = None, skip: int = 0, limit: Optional[int] = None) -> list[BookmarkOut]: + ... +``` + +(Note: signature changes — `uid` must become keyword-only. Update call sites in the router/api file to pass `uid=uid` keyword.) + +Also decorate `get_bookmark` similarly: + +```python +@cached(ns="bookmarks", ttl=120, scope="user") +async def get_bookmark(*, uid: str, bookmark_id: str) -> BookmarkOut: + ... +``` + +And folder list (`list_folders` if present): same pattern. + +- [ ] **Step 3: Update writes** + +In every write fn (`create_bookmark`, `update_bookmark`, `delete_bookmark`, `bulk_delete_bookmarks`, `move_bookmark`, `import_bookmarks`, folder writes), insert one line just before the `return`: + +```python + await bump_version(ns="bookmarks", uid=uid) +``` + +For bulk write fns, one bump after the entire batch (not per item). + +- [ ] **Step 4: Update API layer to pass `uid` as kwarg** + +Edit `apps/backend/app/api/routes/bookmarks/api.py`. Each route handler that calls a service fn must pass `uid` as kwarg. Example: + +```python +# Before: +return await list_bookmarks(uid, folder_id=folder_id, skip=skip, limit=limit) +# After: +return await list_bookmarks(uid=uid, folder_id=folder_id, skip=skip, limit=limit) +``` + +Apply to every site that calls a `@cached` fn. + +- [ ] **Step 5: Run suite** + +```bash +cd apps/backend && pytest -q +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add apps/backend/app/api/routes/bookmarks/ +git commit -m "feat(cache): cache bookmarks reads; bump version on writes" +``` + +--- + +## Task 15: Notes — same pattern + +**Files:** +- Modify: `apps/backend/app/api/routes/notes/services.py` +- Modify: `apps/backend/app/api/routes/notes/api.py` + +**Interfaces:** +- Produces: reads cached; writes bump version on `notes`. + +- [ ] **Step 1: Import** + +Add to top of `services.py`: + +```python +from app.core.cache import cached, bump_version +``` + +- [ ] **Step 2: Decorate every read fn** + +For each `async def list_notes(...)`, `get_note(...)`, `search_notes(...)`: + +```python +@cached(ns="notes", ttl=120, scope="user") +async def list_notes(*, uid: str, ...): + ... +``` + +Make `uid` keyword-only. + +- [ ] **Step 3: Bump on writes** + +In `create_note`, `update_note`, `delete_note`, bulk variants, add before return: + +```python + await bump_version(ns="notes", uid=uid) +``` + +- [ ] **Step 4: Update `api.py` to pass `uid=uid`** + +Same edit pattern as Task 14 Step 4. + +- [ ] **Step 5: Run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/notes/ +git commit -m "feat(cache): cache notes reads; bump version on writes" +``` + +--- + +## Task 16: Code snippets — same pattern + +**Files:** +- Modify: `apps/backend/app/api/routes/code_snippets/services.py` +- Modify: `apps/backend/app/api/routes/code_snippets/api.py` + +- [ ] **Step 1: Import + decorate reads + bump on writes (ns="code_snippets", ttl=120)** + +Apply the exact pattern from Task 15. Decorator: + +```python +@cached(ns="code_snippets", ttl=120, scope="user") +async def list_snippets(*, uid: str, ...): + ... +``` + +Bump on writes: + +```python + await bump_version(ns="code_snippets", uid=uid) +``` + +- [ ] **Step 2: Update api.py call sites to use `uid=uid`** + +- [ ] **Step 3: Run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/code_snippets/ +git commit -m "feat(cache): cache code snippets reads; bump version on writes" +``` + +--- + +## Task 17: Tasks (todo) — `ns="tasks"`, TTL 60s + +**Files:** +- Modify: `apps/backend/app/api/routes/tasks/services.py` +- Modify: `apps/backend/app/api/routes/tasks/api.py` + +- [ ] **Step 1: Import + decorate reads (ttl=60) + bump on writes** + +Decorator: + +```python +@cached(ns="tasks", ttl=60, scope="user") +async def list_tasks(*, uid: str, ...): + ... +``` + +Bump: + +```python + await bump_version(ns="tasks", uid=uid) +``` + +Tasks are written frequently — verify every status toggle / reorder / move bumps version. + +- [ ] **Step 2: Update api.py call sites** + +- [ ] **Step 3: Run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/tasks/ +git commit -m "feat(cache): cache tasks reads; bump version on writes" +``` + +--- + +## Task 18: Passwords — ciphertext only + +**Files:** +- Modify: `apps/backend/app/api/routes/passwords/services.py` +- Modify: `apps/backend/app/api/routes/passwords/api.py` + +**Interfaces:** +- Cached payload is the same shape as the Mongo doc (ciphertext + iv). Decryption stays outside the cache. + +- [ ] **Step 1: Decorate reads (ns="passwords", ttl=60)** + +```python +@cached(ns="passwords", ttl=60, scope="user") +async def list_passwords(*, uid: str, ...): + ... +``` + +If any read fn currently decrypts before returning, **split** it: keep a private `_fetch_password_docs(*, uid)` (decorated, returns ciphertext) and a thin caller that decrypts. + +- [ ] **Step 2: Bump on writes** + +```python + await bump_version(ns="passwords", uid=uid) +``` + +- [ ] **Step 3: Update api.py call sites + run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/passwords/ +git commit -m "feat(cache): cache passwords (ciphertext only); bump on writes" +``` + +--- + +## Task 19: API client — `ns="api_client"`, TTL 300s + +**Files:** +- Modify: `apps/backend/app/api/routes/api_client/services.py` +- Modify: `apps/backend/app/api/routes/api_client/api.py` + +- [ ] **Step 1: Decorate reads + bump on writes** + +Decorator: + +```python +@cached(ns="api_client", ttl=300, scope="user") +async def list_collections(*, uid: str, ...): + ... +``` + +Apply to: collections list/get, saved requests list/get, environments list/get (if part of api_client). Bump on every write. + +- [ ] **Step 2: api.py + run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/api_client/ +git commit -m "feat(cache): cache api_client reads; bump on writes" +``` + +--- + +## Task 20: User preferences — `ns="user_preferences"`, TTL 600s + +**Files:** +- Modify: `apps/backend/app/api/routes/user_preferences/services.py` +- Modify: `apps/backend/app/api/routes/user_preferences/api.py` + +- [ ] **Step 1: Decorate reads + bump on writes** + +```python +@cached(ns="user_preferences", ttl=600, scope="user") +async def get_user_preferences(*, uid: str): + ... +``` + +```python + await bump_version(ns="user_preferences", uid=uid) +``` + +- [ ] **Step 2: Run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/user_preferences/ +git commit -m "feat(cache): cache user preferences (10-min TTL)" +``` + +--- + +## Task 21: URL shortener — global XFetch + owner cache + +**Files:** +- Modify: `apps/backend/app/api/routes/url_shortener/services.py` +- Modify: `apps/backend/app/api/routes/url_shortener/api.py` + +**Interfaces:** +- Public `resolve_short_url(slug)` uses XFetch on `url_shortener_resolve`. Owner-facing list uses `url_shortener_owner` per-user. + +- [ ] **Step 1: Decorate public resolve** + +```python +from app.core.cache import cached, cache_invalidate, bump_version +from app.core.cache.keys import build_key + + +@cached(ns="url_shortener_resolve", ttl=600, scope="global", strategy="xfetch") +async def resolve_short_url(*, slug: str): + ... +``` + +Note: `scope="global"` means no `uid` is required, and XFetch math applies. + +- [ ] **Step 2: Invalidate on slug write** + +In `create_short_url`, `update_short_url`, `delete_short_url`, add: + +```python + key = build_key(ns="url_shortener_resolve", scope="global", uid=None, ver=None, op="resolve_short_url", args_hash=_args_hash_for_slug(slug)) + await cache_invalidate(ns="url_shortener_resolve", key=key) +``` + +Where `_args_hash_for_slug` mirrors what the decorator computed: + +```python +from app.core.cache.keys import args_hash as _ah +from app.core.config import get_settings + + +def _args_hash_for_slug(slug: str) -> str: + return _ah({"slug": slug}, secret=(get_settings().JWT_SECRET_KEY or "default-cache-secret").encode()) +``` + +- [ ] **Step 3: Decorate owner-facing list** + +```python +@cached(ns="url_shortener_owner", ttl=120, scope="user") +async def list_my_short_urls(*, uid: str, ...): + ... +``` + +Bump on writes: + +```python + await bump_version(ns="url_shortener_owner", uid=uid) +``` + +- [ ] **Step 4: api.py call sites + run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/url_shortener/ +git commit -m "feat(cache): XFetch on public URL resolve; per-user owner cache" +``` + +--- + +## Task 22: Analytics aggregates — XFetch + +**Files:** +- Modify: `apps/backend/app/api/routes/analytics/services.py` +- Modify: `apps/backend/app/api/routes/analytics/api.py` + +**Interfaces:** +- Top tools + activity buckets cached with XFetch on `analytics_aggregate`. No write-side invalidation; rely on TTL. + +- [ ] **Step 1: Decorate aggregates** + +```python +from app.core.cache import cached + + +@cached(ns="analytics_aggregate", ttl=300, scope="global", strategy="xfetch") +async def get_top_tools(*, days: int = 7, limit: int = 10): + ... + + +@cached(ns="analytics_aggregate", ttl=300, scope="global", strategy="xfetch") +async def get_activity_buckets(*, days: int = 7): + ... +``` + +**Important:** Analytics aggregates are global (cross-user) reads. Per-user analytics views (if any) should use a separate `scope="user"` namespace, not `analytics_aggregate`. If the current code mixes per-user and aggregate reads in one fn, split them before decorating. + +- [ ] **Step 2: api.py call sites + run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/analytics/ +git commit -m "feat(cache): XFetch on analytics aggregates" +``` + +--- + +## Task 23: DNS lookup — global TTL 1h + +**Files:** +- Modify: `apps/backend/app/api/routes/dns_lookup/services.py` +- Modify: `apps/backend/app/api/routes/dns_lookup/api.py` + +- [ ] **Step 1: Decorate** + +```python +from app.core.cache import cached + + +@cached(ns="dns_lookup", ttl=3600, scope="global") +async def lookup(*, host: str, record_type: str = "A"): + ... +``` + +Pure function of `(host, record_type)` — safe to cache globally for 1h. + +- [ ] **Step 2: api.py + run + commit** + +```bash +cd apps/backend && pytest -q +git add apps/backend/app/api/routes/dns_lookup/ +git commit -m "feat(cache): cache DNS lookups globally (1h TTL)" +``` + +--- + +## Task 24: Load test script + +**Files:** +- Create: `apps/backend/scripts/loadtest_cache.py` + +**Interfaces:** +- Produces: a locust-driven load test scenario; reports p50/p99 + Mongo find count delta vs `CACHE_ENABLED=false` baseline. + +- [ ] **Step 1: Write script** + +Create `apps/backend/scripts/loadtest_cache.py`: + +```python +"""Load test: 1000 concurrent users, 90/10 read/write. + +Usage: + pip install locust + CACHE_ENABLED=false locust -f scripts/loadtest_cache.py --headless -u 1000 -r 100 -t 5m \ + --host http://localhost:8000 --csv=baseline + CACHE_ENABLED=true locust -f scripts/loadtest_cache.py --headless -u 1000 -r 100 -t 5m \ + --host http://localhost:8000 --csv=with_cache + +Compare baseline_stats.csv vs with_cache_stats.csv. +""" +import random +import string + +from locust import HttpUser, between, task + + +def _rand_str(n: int = 8) -> str: + return "".join(random.choices(string.ascii_lowercase, k=n)) + + +class DevToolsUser(HttpUser): + wait_time = between(0.5, 2.0) + headers: dict[str, str] + + def on_start(self): + # Replace with a real test token issuance flow for the env. + # Sketch: hit /auth/anon-login or seed a user. + self.headers = {"Authorization": f"Bearer {self._token()}"} + + def _token(self) -> str: + # Pull from env or local fixture file. Out of scope here. + import os + return os.environ.get("LOADTEST_TOKEN", "") + + @task(45) + def list_bookmarks(self): + self.client.get("/bookmarks", headers=self.headers, name="GET /bookmarks") + + @task(20) + def list_notes(self): + self.client.get("/notes", headers=self.headers, name="GET /notes") + + @task(15) + def list_snippets(self): + self.client.get("/code-snippets", headers=self.headers, name="GET /code-snippets") + + @task(10) + def analytics_top(self): + self.client.get("/analytics/top-tools?days=7", headers=self.headers, name="GET /analytics/top-tools") + + @task(10) + def write_bookmark(self): + self.client.post( + "/bookmarks", + json={"title": _rand_str(), "url": f"https://example.com/{_rand_str()}", "tags": []}, + headers=self.headers, + name="POST /bookmarks", + ) +``` + +- [ ] **Step 2: Document run procedure (skip executing in CI)** + +Append to `apps/backend/README.md` under a new section: + +```markdown +## Load testing cache + +See `scripts/loadtest_cache.py`. Requires `locust` (install separately) and a running backend. + +1. Boot backend with `CACHE_ENABLED=false`; run a 5-min baseline. +2. Boot backend with `CACHE_ENABLED=true` + chosen `CACHE_NAMESPACES`; re-run. +3. Compare p50/p99 in `*_stats.csv`. Acceptance gates: p50 < 50ms, p99 < 200ms, error rate 0%. +``` + +- [ ] **Step 3: Commit** + +```bash +git add apps/backend/scripts/loadtest_cache.py apps/backend/README.md +git commit -m "test(cache): add locust load test for cache acceptance gates" +``` + +--- + +## Task 25: Deployment notes + env documentation + +**Files:** +- Modify: `apps/backend/README.md` + +**Interfaces:** +- Produces: clear ops doc for env vars + Redis config recommendation. + +- [ ] **Step 1: Document env vars** + +Append to `apps/backend/README.md`: + +```markdown +## Redis cache + +### Environment + +| Var | Default | Purpose | +|-----|---------|---------| +| `REDIS_URL` | unset | If unset, cache is inert. `redis://...` or `rediss://...`. | +| `CACHE_ENABLED` | `true` | Global kill switch. | +| `CACHE_NAMESPACES` | `""` | Comma-separated namespaces to activate. Empty = no caching. | +| `CACHE_DEFAULT_TTL` | `120` | Fallback TTL seconds. | +| `CACHE_OP_TIMEOUT_MS` | `50` | Per-Redis-call timeout. | +| `CACHE_XFETCH_BETA` | `1.0` | XFetch tuning constant. Higher = refreshes earlier. | +| `CACHE_LOG_LEVEL` | `WARNING` | `app.cache` logger level. | + +### Rollout phases + +Ramp `CACHE_NAMESPACES` namespace-by-namespace; restart workers each phase. + +| Phase | Add to `CACHE_NAMESPACES` | +|-------|---------------------------| +| 1 | `auth_token,auth_user` | +| 2 | `user_preferences,dns_lookup` | +| 3 | `bookmarks,notes,code_snippets,api_client` | +| 4 | `tasks,passwords,url_shortener_owner` | +| 5 | `url_shortener_resolve,analytics_aggregate` | + +Kill switch: drop a namespace from env + restart workers. Global kill: `CACHE_ENABLED=false`. + +### Redis server config (prod) + +- `maxmemory` sized to expected working set (start with 512 MB for 1k users). +- `maxmemory-policy allkeys-lru`. +- `requirepass` set; TLS enabled if exposed outside VPC. +- Bind to private network only. +``` + +- [ ] **Step 2: Commit** + +```bash +git add apps/backend/README.md +git commit -m "docs(cache): document env vars, rollout phases, prod Redis config" +``` + +--- + +## Self-Review Notes + +- **Spec coverage:** + - Sections 3 (architecture), 5 (file map), 6 (key schema) → Tasks 3-9. + - Section 4 (public API) → Task 9. + - Section 7 (TTL table) → registered in Task 5 + applied in Tasks 13-23. + - Section 8 (invalidation) → bump_version in every write task. + - Section 9 (error handling) → `_safe` wrapper in Task 8. + - Section 10 (observability) → log lines throughout Tasks 8 + 13. + - Section 11 (security) → keyed blake2b in Task 5; ciphertext-only in Task 18; logout invalidation in Task 13. + - Section 12 (testing) → Tasks 4-8 unit tests + Task 12 integration. + - Section 13 (acceptance) → Task 24 load test. + - Section 14 (rollout) → Task 25 docs. +- **No placeholders:** every step has exact code or exact command + expected output. +- **Type consistency:** `cached`, `bump_version`, `cache_invalidate`, `get_or_set` names match across Tasks 8, 9, and every route task. +- **Frequent commits:** every task ends in a commit. From 3f458fc59b008ee4c50694e6c68bc267d0ec1463 Mon Sep 17 00:00:00 2001 From: itsmeakhil Date: Mon, 22 Jun 2026 16:58:08 +0530 Subject: [PATCH 13/52] Activity --- apps/backend/pyproject.toml | 3 + apps/backend/uv.lock | 168 ++++++++++++++++++ apps/web/messages/en.json | 4 +- .../src/app/api/backend/[...path]/route.ts | 8 + apps/web/src/app/api/proxy/route.ts | 12 ++ apps/web/src/app/dashboard/page.tsx | 24 +-- .../activity/activity-log-drawer.tsx | 68 +++++++ .../dashboard/activity/activity-log-panel.tsx | 68 ++++--- .../dashboard/activity/audit-event-row.tsx | 87 ++++++++- .../components/dashboard/dashboard-hero.tsx | 10 +- .../dashboard/dashboard-pinned-section.tsx | 62 ++++++- .../dashboard/dashboard-recent-tools.tsx | 42 ++++- .../dashboard/dashboard-tool-card.tsx | 43 ++++- .../dashboard/dashboard-tool-grid.tsx | 14 +- apps/web/src/components/dashboard/types.ts | 20 +++ .../src/components/user-preferences-sync.tsx | 17 +- apps/web/src/lib/logout-user.ts | 10 ++ apps/web/src/store/pinned-tools-store.ts | 15 ++ ...06-22-dashboard-analytics-graphs-design.md | 1 + 19 files changed, 609 insertions(+), 67 deletions(-) create mode 100644 apps/web/src/components/dashboard/activity/activity-log-drawer.tsx diff --git a/apps/backend/pyproject.toml b/apps/backend/pyproject.toml index 8d1b99c4..9b611f4b 100644 --- a/apps/backend/pyproject.toml +++ b/apps/backend/pyproject.toml @@ -14,6 +14,8 @@ dependencies = [ "python-jose[cryptography]>=3.5.0", "boto3>=1.38.0", "slowapi>=0.1.9", + "redis[asyncio]>=5.0", + "orjson>=3.10", ] [project.optional-dependencies] @@ -21,6 +23,7 @@ dev = [ "pytest>=8.4.1", "httpx>=0.28.1", "ruff>=0.13.0", + "testcontainers[redis]>=4.0", ] [tool.pytest.ini_options] diff --git a/apps/backend/uv.lock b/apps/backend/uv.lock index 19e3cc3a..8bb99087 100644 --- a/apps/backend/uv.lock +++ b/apps/backend/uv.lock @@ -40,6 +40,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + [[package]] name = "boto3" version = "1.43.2" @@ -379,6 +388,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, ] +[[package]] +name = "docker" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, +] + [[package]] name = "ecdsa" version = "0.19.2" @@ -1191,9 +1214,11 @@ dependencies = [ { name = "fastapi", extra = ["standard"] }, { name = "firebase-admin" }, { name = "motor" }, + { name = "orjson" }, { name = "pydantic-settings" }, { name = "pymongo" }, { name = "python-jose", extra = ["cryptography"] }, + { name = "redis" }, { name = "slowapi" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -1203,6 +1228,7 @@ dev = [ { name = "httpx" }, { name = "pytest" }, { name = "ruff" }, + { name = "testcontainers", extra = ["redis"] }, ] [package.metadata] @@ -1212,16 +1238,100 @@ requires-dist = [ { name = "firebase-admin", specifier = ">=7.1.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28.1" }, { name = "motor", specifier = ">=3.7.0" }, + { name = "orjson", specifier = ">=3.10" }, { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "pymongo" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.1" }, { name = "python-jose", extras = ["cryptography"], specifier = ">=3.5.0" }, + { name = "redis", extras = ["asyncio"], specifier = ">=5.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.13.0" }, { name = "slowapi", specifier = ">=0.1.9" }, + { name = "testcontainers", extras = ["redis"], marker = "extra == 'dev'", specifier = ">=4.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.35.0" }, ] provides-extras = ["dev"] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/5d/b95ca542a001135cc250a49370f282f578c8f4e46cc8617d73775297eea8/orjson-3.11.9-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:135869ef917b8704ea0a94e01620e0c05021c15c52036e4663baffe75e72f8ce", size = 228986, upload-time = "2026-05-06T15:09:14.765Z" }, + { url = "https://files.pythonhosted.org/packages/80/01/be33fbff646e22f93398429ea645f20d2097aea1a6cdc1e6628e70125f83/orjson-3.11.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:115ab5f5f4a0f203cc2a5f0fb09aee503a3f771aa08392949ab5ca230c4fbdbd", size = 132558, upload-time = "2026-05-06T15:09:17.431Z" }, + { url = "https://files.pythonhosted.org/packages/4e/61/73d49333bba660a075daccca10970dc6409ce1cf42ae4046646a19468aad/orjson-3.11.9-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4da3c38a2083ca4aaf9c2a36776cce3e9328e6647b10d118948f3cfb4913ffe4", size = 128213, upload-time = "2026-05-06T15:09:18.719Z" }, + { url = "https://files.pythonhosted.org/packages/1f/7d/30e844b3dac3f74aed66b1f984daf9db3c98c0328c03d965a9e8dc06449e/orjson-3.11.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53b50b0e14084b8f7e29c5ce84c5af0f1160169b30d8a6914231d97d2fe297d4", size = 135430, upload-time = "2026-05-06T15:09:20.257Z" }, + { url = "https://files.pythonhosted.org/packages/16/64/bd815f5c610b3facc204f26ba94e87a9eb49b0d83de3d5fc1eee2402d91b/orjson-3.11.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:231742b4a11dad8d5380a435962c57e91b7c37b79be858f4ef1c0df1a259897e", size = 146178, upload-time = "2026-05-06T15:09:21.616Z" }, + { url = "https://files.pythonhosted.org/packages/c7/35/e744fd36c79b339d27beb06068b5a08a8882ef5418804d0ce545a31f718d/orjson-3.11.9-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34fd2317602587321faab75ab76c623a0117e80841a6413654f04e47f339a8fb", size = 133068, upload-time = "2026-05-06T15:09:23.228Z" }, + { url = "https://files.pythonhosted.org/packages/2a/56/d54152b67b63a0b3e556cfc549d6ce84f74d7f425ddeadc6c8a74d913da7/orjson-3.11.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71f3db16e69b667b132e0f305a833d5497da302d801508cbb051ed9a9819da47", size = 134217, upload-time = "2026-05-06T15:09:24.847Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ee/66154baf69f71c7164a268a5e888908aec5a0819d13c81d5e2755a257758/orjson-3.11.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0b34789fa0da61cf7bef0546b09c738fb195331e017e477096d129e9105ab03d", size = 141917, upload-time = "2026-05-06T15:09:26.647Z" }, + { url = "https://files.pythonhosted.org/packages/09/d3/c5824260ca8b9d7ba82648d042a3f8f4815d18c15bb98a1f30edd1bb2d83/orjson-3.11.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:87e4d4ab280b0c87424d47695bec2182caf8cfc17879ea78dab76680194abc13", size = 415356, upload-time = "2026-05-06T15:09:28.252Z" }, + { url = "https://files.pythonhosted.org/packages/64/cb/509c2e816fe4df641d93dc92f6a89adc8df3ada8ebdee2bd44aba3264c3c/orjson-3.11.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ace6c58523302d3b97b6ac5c38a5298a54b473762b6be82726b4265c41029f92", size = 148112, upload-time = "2026-05-06T15:09:29.783Z" }, + { url = "https://files.pythonhosted.org/packages/db/b5/3ceae56d2e4962979eedb023ba6a46a4bb65f333960379be0ca470686220/orjson-3.11.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:97d0d932803c1b164fde11cb542a9efcb1e0f63b184537cca65887147906ff48", size = 137112, upload-time = "2026-05-06T15:09:31.432Z" }, + { url = "https://files.pythonhosted.org/packages/d7/7a/81fa3f2c7bef79b04cf2ab7838e5ac74b1f12511ceab979759b0275d6bb4/orjson-3.11.9-cp310-cp310-win32.whl", hash = "sha256:b3afcf569c15577a9fe64627292daa3e6b3a70f4fb77a5df246a87ec21681b94", size = 131706, upload-time = "2026-05-06T15:09:32.707Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d8/b64600f9083c7f151ad39717a5877fccbeb0ef6d7efcb55f971ce00b6bee/orjson-3.11.9-cp310-cp310-win_amd64.whl", hash = "sha256:8697ab6a080a5c46edaad50e2bc5bd8c7ca5c66442d24104fa44ec74910a8244", size = 127282, upload-time = "2026-05-06T15:09:33.955Z" }, + { url = "https://files.pythonhosted.org/packages/1e/51/3fb9e65ae76ee97bd611869a503fa3fc0a6e81dd8b737cf3003f682df7ff/orjson-3.11.9-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f01c4818b3fc9b0da8e096722a84318071eaa118df35f6ed2344da0e73a5444f", size = 228522, upload-time = "2026-05-06T15:09:35.362Z" }, + { url = "https://files.pythonhosted.org/packages/16/fa/9d54b07cb3f3b0bfd57841478e42d7a0ece4a9f49f9907eecf5a45461687/orjson-3.11.9-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:3ebca4179031ee716ed076ffadc29428e900512f6fccee8614c9983157fcf19c", size = 128463, upload-time = "2026-05-06T15:09:37.063Z" }, + { url = "https://files.pythonhosted.org/packages/88/b1/6ceafc2eefd0a553e3be77ce6c49d107e772485d9568629376171c50e634/orjson-3.11.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48ee05097750de0ff69ed5b7bbcf0732182fd57a24043dcc2a1da780a5ead3a5", size = 132306, upload-time = "2026-05-06T15:09:38.299Z" }, + { url = "https://files.pythonhosted.org/packages/ea/76/f11311285324a40aab1e3031385c50b635a7cd0734fdaf60c7e89a696f60/orjson-3.11.9-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6082706765a95a6680d812e1daf1c0cfe8adec7831b3ff3b625693f3b461b1c", size = 127988, upload-time = "2026-05-06T15:09:39.597Z" }, + { url = "https://files.pythonhosted.org/packages/9e/85/0ef63bcf1337f44031ce9b91b1919563f62a37527b3ea4368bb15a22e5d7/orjson-3.11.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:277fefe9d76ee17eb14debf399e3533d4d63b5f677a4d3719eb763536af1f4bd", size = 135188, upload-time = "2026-05-06T15:09:40.957Z" }, + { url = "https://files.pythonhosted.org/packages/05/94/b0d27090ea8a2095db3c2bd1b1c96f96f19bbb494d7fef33130e846e613d/orjson-3.11.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03db380e3780fa0015ed776a90f20e8e20bb11dde13b216ce19e5718e3dfba62", size = 145937, upload-time = "2026-05-06T15:09:42.249Z" }, + { url = "https://files.pythonhosted.org/packages/09/eb/75d50c29c05b8054013e221e598820a365c8e64065312e75e202ed880709/orjson-3.11.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33d7d766701847dc6729846362dc27895d2f2d2251264f9d10e7cb9878194877", size = 132758, upload-time = "2026-05-06T15:09:43.945Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/360686f39348aa88827cb6fbf7dc606fd41c831a35235e1abf1db8e3a9e6/orjson-3.11.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:147302878da387104b66bb4a8b0227d1d487e976ce41a8501916161072ed87b1", size = 133971, upload-time = "2026-05-06T15:09:45.239Z" }, + { url = "https://files.pythonhosted.org/packages/0e/30/3178eb16f3221aeef068b6f1f1ebe05f656ea5c6dffe9f6c917329fe17a3/orjson-3.11.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3513550321f8c8c811a7c3297b8a630e82dc08e4c10216d07703c997776236cd", size = 141685, upload-time = "2026-05-06T15:09:46.858Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f1/ff2f19ed0225f9680fafa42febca3570dd59444ebf190980738d376214c2/orjson-3.11.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c5d001196b89fa9cf0a4ab79766cd835b991a166e4b621ba95089edc50c429ff", size = 415167, upload-time = "2026-05-06T15:09:48.312Z" }, + { url = "https://files.pythonhosted.org/packages/9b/61/863bddf0da6e9e586765414debd54b4e58db05f560902b6d00658cb88636/orjson-3.11.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:16969c9d369c98eb084889c6e4d2d39b77c7eb38ceccf8da2a9fff62ae908980", size = 147913, upload-time = "2026-05-06T15:09:49.733Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4081492586d75b073d60c5271a8d0f05a0955cabf1e34c8473f6fcd84235/orjson-3.11.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:63e0efbc991250c0b3143488fa57d95affcabbfc63c99c48d625dd37779aafe2", size = 136959, upload-time = "2026-05-06T15:09:51.311Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bd/70b6ab193594d7abb875320c0a7c8335e846f28968c432c31042409c3c8d/orjson-3.11.9-cp311-cp311-win32.whl", hash = "sha256:14ed654580c1ed2bc217352ec82f91b047aef82951aa71c7f64e0dcb03c0e180", size = 131533, upload-time = "2026-05-06T15:09:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/3f/17/1a1a228183d62d1b77e2c30d210f47dd4768b310ebe1607c63e3c0e3a71e/orjson-3.11.9-cp311-cp311-win_amd64.whl", hash = "sha256:57ea77fb70a448ce87d18fca050193202a3da5e54598f6501ca5476fb66cfe02", size = 127106, upload-time = "2026-05-06T15:09:54.204Z" }, + { url = "https://files.pythonhosted.org/packages/b8/95/285de5fa296d09681ee9c546cd4a8aeb773b701cf343dc125994f4d52953/orjson-3.11.9-cp311-cp311-win_arm64.whl", hash = "sha256:19b72ed11572a2ee51a67a903afbe5af504f84ed6f529c0fe44b0ab3fb5cc697", size = 126848, upload-time = "2026-05-06T15:09:55.551Z" }, + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1624,6 +1734,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/78/4126abcbdbd3c559d43e0db7f7b9173fc6befe45d39a2856cc0b8ec2a5a6/python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645", size = 29254, upload-time = "2026-04-27T10:51:24.997Z" }, ] +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/1b/9cfdeac80ee45bebbbcb31f1b7b99a0d81a1c72de48d837be984e0e88b1d/pywin32-312-cp310-cp310-win32.whl", hash = "sha256:772235332b5d1024c696f11cea1ae4be7930f0a8b894bb43db14e3f435f1ff7e", size = 6361387, upload-time = "2026-06-04T07:49:14.329Z" }, + { url = "https://files.pythonhosted.org/packages/33/b1/7afc96d041d982c27bc2df6f853d43f01fd273e3d39d04be3647ddeb533d/pywin32-312-cp310-cp310-win_amd64.whl", hash = "sha256:5dbc35d2b5320dc07f25fa31269cfb767471002b17de5eb067d03da68c7cb2db", size = 6926780, upload-time = "2026-06-04T07:49:16.881Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/4140da9ad54108e517f4a16b2d83da3033e08662144623e1239587cb7db6/pywin32-312-cp310-cp310-win_arm64.whl", hash = "sha256:3020656e34f1cf7faeb7bccd2b84653a607c6ff0c55ada85e6487d61716deabd", size = 4307203, upload-time = "2026-06-04T07:49:18.993Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f5/10a6e845a00fc5e7afd0a988b744f403d4d57162a28d160a093c4d9322f0/pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c", size = 6362659, upload-time = "2026-06-04T07:49:21.349Z" }, + { url = "https://files.pythonhosted.org/packages/35/c4/dcd2d62b5944b6d5db53413a5899016ccd57ffcb7278f3f81655d25d2027/pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a", size = 6928825, upload-time = "2026-06-04T07:49:23.934Z" }, + { url = "https://files.pythonhosted.org/packages/b7/56/3cbb433fe4501cdba2eb9040f56a4e1a8243faa4186b25295564d1a7a79d/pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47", size = 6721875, upload-time = "2026-06-04T07:49:26.416Z" }, + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1688,6 +1823,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "redis" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ae/ed461cca5780b5fc8b9fe8ca0ed98d89508645fb9d880c24cc42c087678f/redis-8.0.0.tar.gz", hash = "sha256:a00c5355432051ac14e593b8b197fc76c887ee12d55a0984f69328a1115fdc49", size = 5101591, upload-time = "2026-05-28T12:45:13.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/e3/b519734372d305bd547534a9f32e4ce9f98552af753dce72cf3483a0ff0b/redis-8.0.0-py3-none-any.whl", hash = "sha256:c938c18338585009f0bc310f4c7e4e4b4d37639356c4ac072cedf3af570c8dc7", size = 499870, upload-time = "2026-05-28T12:45:11.697Z" }, +] + [[package]] name = "requests" version = "2.33.1" @@ -1956,6 +2103,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, ] +[[package]] +name = "testcontainers" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docker" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/ac/a597c3a0e02b26cbed6dd07df68be1e57684766fd1c381dee9b170a99690/testcontainers-4.14.2.tar.gz", hash = "sha256:1340ccf16fe3acd9389a6c9e1d9ab21d9fe99a8afdf8165f89c3e69c1967d239", size = 166841, upload-time = "2026-03-18T05:19:16.696Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/2d/26b8b30067d94339afee62c3edc9b803a6eb9332f521ba77d8aaab5de873/testcontainers-4.14.2-py3-none-any.whl", hash = "sha256:0d0522c3cd8f8d9627cda41f7a6b51b639fa57bdc492923c045117933c668d68", size = 125712, upload-time = "2026-03-18T05:19:15.29Z" }, +] + +[package.optional-dependencies] +redis = [ + { name = "redis" }, +] + [[package]] name = "tomli" version = "2.4.1" diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 05f5fed4..73bec8a4 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -229,6 +229,7 @@ "welcomeBackNamed": "Welcome back, {name}", "tagline": "What would you like to build today?", "brandName": "MyDevTools", + "viewActivity": "Recent activity", "signIn": "Sign in", "stats": { "tools": "Tools", @@ -313,7 +314,8 @@ "changedFields": "What changed", "device": "Device", "before": "Before", - "after": "After" + "after": "After", + "viewAll": "View all activity" }, "toolCard": { "defaultDescription": "Explore this tool for better functionality.", diff --git a/apps/web/src/app/api/backend/[...path]/route.ts b/apps/web/src/app/api/backend/[...path]/route.ts index 62ec5399..54e7fd6d 100644 --- a/apps/web/src/app/api/backend/[...path]/route.ts +++ b/apps/web/src/app/api/backend/[...path]/route.ts @@ -44,6 +44,14 @@ async function forward(req: NextRequest, method: string, pathSegments: string[]) const contentType = req.headers.get("content-type") if (contentType) headers["content-type"] = contentType + // Forward the real client's User-Agent + IP so the backend audit log records + // the actual device, not this Next.js server's fetch agent. + const userAgent = req.headers.get("user-agent") + if (userAgent) headers["user-agent"] = userAgent + + const forwardedFor = req.headers.get("x-forwarded-for") || req.headers.get("x-real-ip") + if (forwardedFor) headers["x-forwarded-for"] = forwardedFor + let body: BodyInit | undefined = undefined if (method !== "GET" && method !== "HEAD") { body = await req.text() diff --git a/apps/web/src/app/api/proxy/route.ts b/apps/web/src/app/api/proxy/route.ts index 32b609da..53b8ecd1 100644 --- a/apps/web/src/app/api/proxy/route.ts +++ b/apps/web/src/app/api/proxy/route.ts @@ -150,6 +150,18 @@ export async function POST(req: NextRequest) { requestHeaders["cookie"] = incomingCookie } + // For trusted backend calls only, forward the real client's User-Agent + IP so the + // audit log records the actual device (not this server's fetch agent). Never leak + // these to arbitrary SSRF-checked targets. + if (isBackendRequest) { + const hasHeader = (name: string) => + Object.keys(requestHeaders).some((k) => k.toLowerCase() === name) + const userAgent = req.headers.get("user-agent") + if (userAgent && !hasHeader("user-agent")) requestHeaders["user-agent"] = userAgent + const forwardedFor = req.headers.get("x-forwarded-for") || req.headers.get("x-real-ip") + if (forwardedFor && !hasHeader("x-forwarded-for")) requestHeaders["x-forwarded-for"] = forwardedFor + } + let requestBody: BodyInit | undefined = body || undefined if (body && typeof body === "object" && body.mode === "form-data" && Array.isArray(body.entries)) { diff --git a/apps/web/src/app/dashboard/page.tsx b/apps/web/src/app/dashboard/page.tsx index 0a0e697a..d3a9d305 100644 --- a/apps/web/src/app/dashboard/page.tsx +++ b/apps/web/src/app/dashboard/page.tsx @@ -26,6 +26,7 @@ import { DashboardSearchBar } from '@/components/dashboard/dashboard-search-bar' import { DashboardPinnedSection } from '@/components/dashboard/dashboard-pinned-section' import { DashboardWhatsNew } from '@/components/dashboard/dashboard-whats-new' import { DashboardRecentTools } from '@/components/dashboard/dashboard-recent-tools' +import { ActivityLogDrawer } from '@/components/dashboard/activity/activity-log-drawer' import { DashboardLoginCta } from '@/components/dashboard/dashboard-login-cta' import { DashboardToolGrid } from '@/components/dashboard/dashboard-tool-grid' @@ -272,16 +273,19 @@ const DashboardPage: React.FC = () => { desktopOnly /> - - - - {tTabs('apps')} - - - - {tTabs('analytics')} - - +
+ + + + {tTabs('apps')} + + + + {tTabs('analytics')} + + + +
+ + + + + + + {t('title')} + {t('subtitle')} + + +
+ +
+ + + + + {t('viewAll')} + + + + + + + ) +} diff --git a/apps/web/src/components/dashboard/activity/activity-log-panel.tsx b/apps/web/src/components/dashboard/activity/activity-log-panel.tsx index f059ee3e..c761cee3 100644 --- a/apps/web/src/components/dashboard/activity/activity-log-panel.tsx +++ b/apps/web/src/components/dashboard/activity/activity-log-panel.tsx @@ -7,7 +7,7 @@ import { AuditEventRow } from './audit-event-row' const PAGE = 50 -export function ActivityLogPanel() { +export function ActivityLogPanel({ embedded = false }: { embedded?: boolean }) { const t = useTranslations('Dashboard.activity') const [events, setEvents] = useState([]) const [total, setTotal] = useState(0) @@ -47,31 +47,30 @@ export function ActivityLogPanel() { // eslint-disable-next-line react-hooks/exhaustive-deps }, [outcome, search]) - return ( -
-

{t('title')}

-

{t('subtitle')}

- -
- - setSearch(e.target.value)} - className="flex-1 rounded-md border border-border bg-background px-2 py-1 text-sm" - /> -
+ const filters = ( +
+ + setSearch(e.target.value)} + className="flex-1 rounded-md border border-border bg-background px-2 py-1 text-sm" + /> +
+ ) + const list = ( + <> {error &&
{error}
} {!error && events.length === 0 && !loading && (
{t('empty')}
@@ -93,6 +92,25 @@ export function ActivityLogPanel() { {t('loadMore')} )} + + ) + + // Drawer variant: fills the sheet, filters pinned, list scrolls. + if (embedded) { + return ( +
+
{filters}
+
{list}
+
+ ) + } + + return ( +
+

{t('title')}

+

{t('subtitle')}

+
{filters}
+ {list}
) } diff --git a/apps/web/src/components/dashboard/activity/audit-event-row.tsx b/apps/web/src/components/dashboard/activity/audit-event-row.tsx index be1befe9..0ee87c0b 100644 --- a/apps/web/src/components/dashboard/activity/audit-event-row.tsx +++ b/apps/web/src/components/dashboard/activity/audit-event-row.tsx @@ -14,6 +14,86 @@ function relativeTime(ts: number): string { return new Date(ts).toLocaleDateString() } +/** Verb segment of an action ("create", "post", "login", …) → human label. */ +const VERB_LABELS: Record = { + create: 'Created', + add: 'Added', + post: 'Created', + update: 'Updated', + patch: 'Updated', + edit: 'Updated', + put: 'Saved', + delete: 'Deleted', + remove: 'Deleted', + move: 'Moved', + import: 'Imported', + export: 'Exported', + get: 'Viewed', + login: 'Signed in', + logout: 'Signed out', + register: 'Created account', + token_refresh: 'Refreshed session', + 'clear-all': 'Cleared all', +} + +/** Module segment → friendly singular noun. */ +const MODULE_NOUNS: Record = { + 'bookmark-folders': 'folder', + 'bookmark_folders': 'folder', + 'code_snippets': 'snippet', + 'code-snippets': 'snippet', + 'api-client': 'API request', + 'api_client': 'API request', + 'user-preferences': 'preferences', + 'user_preferences': 'preferences', + 'sql-client': 'SQL connection', + 'sql_client': 'SQL connection', + 'environment-manager': 'environment', + 'environment_manager': 'environment', + 'url-shortener': 'short link', + 'game-scores': 'game score', + nosql: 'database connection', + passwords: 'password', + bookmarks: 'bookmark', + tasks: 'task', + notes: 'note', + projects: 'project', + feedback: 'feedback', +} + +function prettyModule(mod: string | null): string { + if (!mod) return 'item' + if (MODULE_NOUNS[mod]) return MODULE_NOUNS[mod] + return mod.replace(/[-_]/g, ' ').replace(/s$/, '') +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1) +} + +/** Human title for an event when no server summary is set. */ +function describeAction(event: AuditEvent): string { + const action = event.action || '' + const dot = action.indexOf('.') + const mod = dot > -1 ? action.slice(0, dot) : event.module + const verb = dot > -1 ? action.slice(dot + 1) : event.method.toLowerCase() + const verbLabel = VERB_LABELS[verb] + + if (mod === 'auth') { + return verbLabel || capitalize(verb.replace(/_/g, ' ')) + } + if (verbLabel) return `${verbLabel} ${prettyModule(mod)}` + return capitalize(prettyModule(mod)) +} + +/** Friendly category chip ("Bookmarks", "Account", …). */ +function categoryLabel(event: AuditEvent): string { + const mod = event.module + if (mod === 'auth') return 'Account' + if (!mod) return 'Activity' + return capitalize(mod.replace(/[-_]/g, ' ')) +} + export function AuditEventRow({ event }: { event: AuditEvent }) { const t = useTranslations('Dashboard.activity') const [open, setOpen] = useState(false) @@ -21,6 +101,8 @@ export function AuditEventRow({ event }: { event: AuditEvent }) { const device = event.device ? `${event.device.browser} on ${event.device.os}` : '—' + const title = event.summary?.trim() || describeAction(event) + const category = categoryLabel(event) return (
@@ -32,7 +114,7 @@ export function AuditEventRow({ event }: { event: AuditEvent }) { >
- {event.action} + {category}
-
{event.summary || event.path}
+ {/* Meaningful description: server summary, else a humanized verb+noun. Never the raw API URL. */} +
{title}
{device}
diff --git a/apps/web/src/components/dashboard/dashboard-hero.tsx b/apps/web/src/components/dashboard/dashboard-hero.tsx index a2e33a57..70146402 100644 --- a/apps/web/src/components/dashboard/dashboard-hero.tsx +++ b/apps/web/src/components/dashboard/dashboard-hero.tsx @@ -1,7 +1,8 @@ 'use client' import React from 'react' -import { Layers, Zap, Pin, Clock } from 'lucide-react' +import Link from 'next/link' +import { Layers, Zap, Pin, Clock, History } from 'lucide-react' import { useTranslations } from 'next-intl' import { dashboardGreeting } from './types' @@ -82,6 +83,13 @@ export function DashboardHero({
+ + +
{totalTools} diff --git a/apps/web/src/components/dashboard/dashboard-pinned-section.tsx b/apps/web/src/components/dashboard/dashboard-pinned-section.tsx index fa3639b0..e0b69e13 100644 --- a/apps/web/src/components/dashboard/dashboard-pinned-section.tsx +++ b/apps/web/src/components/dashboard/dashboard-pinned-section.tsx @@ -1,10 +1,12 @@ 'use client' -import React from 'react' -import { Pin } from 'lucide-react' +import React, { useState } from 'react' +import { Pin, ChevronDown } from 'lucide-react' import { useTranslations } from 'next-intl' import { type RenderToolItem, type ToolCardProps } from './types' -import { ToolCard, HScrollFade } from './dashboard-tool-card' +import { ToolCard, ToolCardSkeleton, HScrollFade } from './dashboard-tool-card' +import { usePinnedToolsHydrated } from '@/store/pinned-tools-store' +import { Button } from '@/components/ui/button' interface DashboardPinnedSectionProps { pinnedItems: RenderToolItem[] @@ -13,6 +15,11 @@ interface DashboardPinnedSectionProps { filterGroup: string | null } +const VISIBLE_CAP = 8 + +const SKELETON_GRID_CLASS = + 'hidden md:grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5 gap-3 md:gap-4' + /** * Pinned tools section. * Shows a helpful empty-state prompt when nothing is pinned, @@ -25,14 +32,35 @@ export function DashboardPinnedSection({ filterGroup, }: DashboardPinnedSectionProps) { const t = useTranslations('Dashboard') + const hydrated = usePinnedToolsHydrated() + const [expanded, setExpanded] = useState(false) // Hide entirely when searching or filtering by category if (searchQuery || filterGroup) return null + // Show skeleton until persisted state hydrates — avoids empty-state flash + if (!hydrated) { + return ( +
+
+
+ +
+

{t('sections.pinned')}

+
+
+ {Array.from({ length: 4 }).map((_, i) => ( + + ))} +
+
+ ) + } + // Empty state if (pinnedItems.length === 0) { return ( -
+
@@ -44,10 +72,13 @@ export function DashboardPinnedSection({ ) } + const overflow = pinnedItems.length - VISIBLE_CAP + const visible = expanded ? pinnedItems : pinnedItems.slice(0, VISIBLE_CAP) + return (
-
+

{t('sections.pinned')}

@@ -71,8 +102,8 @@ export function DashboardPinnedSection({
{/* Desktop: grid */} -
- {pinnedItems.map((item, index) => ( +
+ {visible.map((item, index) => ( ))}
+ {overflow > 0 && ( +
+ +
+ )}
) } diff --git a/apps/web/src/components/dashboard/dashboard-recent-tools.tsx b/apps/web/src/components/dashboard/dashboard-recent-tools.tsx index 434c477a..7009e6e5 100644 --- a/apps/web/src/components/dashboard/dashboard-recent-tools.tsx +++ b/apps/web/src/components/dashboard/dashboard-recent-tools.tsx @@ -1,10 +1,12 @@ 'use client' -import React from 'react' -import { Clock } from 'lucide-react' +import React, { useState } from 'react' +import Link from 'next/link' +import { Clock, ChevronDown, History } from 'lucide-react' import { useTranslations } from 'next-intl' import { type FavoriteItem, type ToolCardProps } from './types' import { ToolCard, HScrollFade } from './dashboard-tool-card' +import { Button } from '@/components/ui/button' interface DashboardRecentToolsProps { recentItems: FavoriteItem[] @@ -13,6 +15,8 @@ interface DashboardRecentToolsProps { searchQuery: string } +const VISIBLE_CAP = 8 + /** * Recently Used tools section. * Only shown when the user is logged in and not searching. @@ -24,14 +28,18 @@ export function DashboardRecentTools({ searchQuery, }: DashboardRecentToolsProps) { const t = useTranslations('Dashboard') + const [expanded, setExpanded] = useState(false) if (!user || recentItems.length === 0 || searchQuery) return null + const overflow = recentItems.length - VISIBLE_CAP + const visible = expanded ? recentItems : recentItems.slice(0, VISIBLE_CAP) + return (
-
+

{t('sections.recentlyUsed')}

@@ -39,6 +47,13 @@ export function DashboardRecentTools({ {recentItems.length}
+ + + {t('viewActivity')} +
{/* Mobile: horizontal scroll */}
@@ -62,8 +77,8 @@ export function DashboardRecentTools({
{/* Desktop: grid */} -
- {recentItems.map((item, index) => ( +
+ {visible.map((item, index) => ( ))}
+ {overflow > 0 && ( +
+ +
+ )}
) } diff --git a/apps/web/src/components/dashboard/dashboard-tool-card.tsx b/apps/web/src/components/dashboard/dashboard-tool-card.tsx index f6fb00dd..2ff9626d 100644 --- a/apps/web/src/components/dashboard/dashboard-tool-card.tsx +++ b/apps/web/src/components/dashboard/dashboard-tool-card.tsx @@ -14,7 +14,21 @@ import { type ToolCardProps, formatRelativeTime } from './types' export const HScrollFade = ({ children }: { children: React.ReactNode }) => (
{children} -
+
+
+) + +/** Matches ToolCard footprint so layout doesn't shift on hydration. */ +export const ToolCardSkeleton = () => ( +
+
+ + +
+
+ + +
) @@ -26,7 +40,6 @@ export const ToolCard = React.memo(function ToolCard({ togglePin, timestamp, }: ToolCardProps) { - const tCard = useTranslations('Dashboard') const tTools = useTranslations('Dashboard.tools') const pathname = item.url?.toString().split('?')[0] ?? '' const toolKey = TOOL_PATH_TO_MESSAGE_KEY[pathname] @@ -43,12 +56,19 @@ export const ToolCard = React.memo(function ToolCard({ } } + const pinned = item.url ? isPinned(item.url.toString()) : false + return (
- - + + -
+
{item.icon ? ( ) : ( @@ -67,7 +87,7 @@ export const ToolCard = React.memo(function ToolCard({ )} {timestamp && !item.badge && ( - + {formatRelativeTime(timestamp)} )} @@ -75,8 +95,13 @@ export const ToolCard = React.memo(function ToolCard({ {item.url && ( + + + + + {t('title')} + {t('subtitle')} + + +
+ +
+ + + + + {t('viewAll')} + + + + + + + ) +} diff --git a/apps/web/src/components/s3-drive/file-browser.tsx b/apps/web/src/components/s3-drive/file-browser.tsx index ef2e030a..d567ecb9 100644 --- a/apps/web/src/components/s3-drive/file-browser.tsx +++ b/apps/web/src/components/s3-drive/file-browser.tsx @@ -1,6 +1,6 @@ "use client" -import { useCallback, useEffect, useRef, useState } from "react" +import { useCallback, useEffect, useMemo, useRef, useState } from "react" import { toast } from "sonner" import { cn } from "@/lib/utils" import { Button } from "@/components/ui/button" @@ -80,7 +80,7 @@ import { IconMinus, IconChevronDown, } from "@tabler/icons-react" -import { listObjects, deleteObjects, getPresignedDownloadUrl, getPresignedUploadUrl, moveObject, configureBucketCors } from "@/lib/s3-drive-api" +import { listObjects, deleteObjects, getPresignedDownloadUrl, getPresignedUploadUrl, getPresignedBatch, moveObject, configureBucketCors } from "@/lib/s3-drive-api" import type { S3Credentials, S3ObjectItem } from "@/lib/s3-drive-api" import { useS3DriveStore } from "@/store/s3-drive-store" import { CreateFolderDialog } from "./create-folder-dialog" @@ -164,15 +164,18 @@ function FileIconComp({ type, className }: { type: string; className?: string }) // ── Checkbox ────────────────────────────────────────────────────────────────── function Checkbox({ - checked, indeterminate, onToggle, className, + checked, indeterminate, onToggle, className, ariaLabel, }: { - checked: boolean; indeterminate?: boolean; onToggle: () => void; className?: string + checked: boolean; indeterminate?: boolean; onToggle: () => void; className?: string; ariaLabel?: string }) { return (
{ if (e.key === " " || e.key === "Enter") { e.preventDefault(); onToggle() } }} onClick={(e) => { e.stopPropagation(); onToggle() }} className={cn( "size-[18px] rounded-[4px] border-2 flex items-center justify-center cursor-pointer shrink-0 transition-all duration-100", @@ -375,18 +378,24 @@ type FileUploadStatus = { name: string status: "queued" | "uploading" | "done" | "error" progress: number + file?: File + key?: string + error?: string } function UploadProgressPanel({ - queue, onClearCompleted, onDismiss, + queue, onClearCompleted, onDismiss, onRetry, onRetryAll, }: { queue: FileUploadStatus[] onClearCompleted: () => void onDismiss: () => void + onRetry: (idx: number) => void + onRetryAll: () => void }) { const [collapsed, setCollapsed] = useState(false) const activeCount = queue.filter((f) => f.status === "uploading").length const doneCount = queue.filter((f) => f.status === "done" || f.status === "error").length + const errorCount = queue.filter((f) => f.status === "error").length const allDone = doneCount === queue.length if (queue.length === 0) return null @@ -398,8 +407,18 @@ function UploadProgressPanel({ {allDone ? `${doneCount}/${queue.length} complete` : `Uploading ${activeCount} file${activeCount !== 1 ? "s" : ""}…`} + {errorCount > 0 && ( + + + + + Retry failed ({errorCount}) + + )} {allDone && ( - )} @@ -427,8 +446,22 @@ function UploadProgressPanel({ {Math.round(f.progress * 100)}% )} {f.status === "done" && Done} - {f.status === "error" && Failed} + {f.status === "error" && ( + <> + Failed + + + )}
+ {f.status === "error" && f.error && ( +
{f.error}
+ )} {f.status === "uploading" && (
void; loading: boolean }) { + const ref = useRef(null) + const onVisibleRef = useRef(onVisible) + const loadingRef = useRef(loading) + onVisibleRef.current = onVisible + loadingRef.current = loading + + useEffect(() => { + const el = ref.current + if (!el) return + const obs = new IntersectionObserver((entries) => { + if (entries[0]?.isIntersecting && !loadingRef.current) onVisibleRef.current() + }, { rootMargin: "300px 0px" }) + obs.observe(el) + return () => obs.disconnect() + }, []) + + return ( +
+ {loading ? <> Loading more… : Scroll for more} +
+ ) +} + // ── FileBrowser ─────────────────────────────────────────────────────────────── type Props = { credentials: S3Credentials; connectionName: string } @@ -779,6 +838,7 @@ export function FileBrowser({ credentials, connectionName }: Props) { const [viewMode, setViewMode] = useState("list") const [search, setSearch] = useState("") + const [debouncedSearch, setDebouncedSearch] = useState("") const [sortCol, setSortCol] = useState("name") const [sortDir, setSortDir] = useState("asc") const [focusedIndex, setFocusedIndex] = useState(null) @@ -786,6 +846,8 @@ export function FileBrowser({ credentials, connectionName }: Props) { const [deleteConfirmOpen, setDeleteConfirmOpen] = useState(false) const [deleting, setDeleting] = useState(false) const [uploadQueue, setUploadQueue] = useState([]) + const uploadQueueRef = useRef([]) + useEffect(() => { uploadQueueRef.current = uploadQueue }, [uploadQueue]) const [uploadPanelOpen, setUploadPanelOpen] = useState(false) const [zipProgress, setZipProgress] = useState<{ done: number; total: number } | null>(null) const [preview, setPreview] = useState(null) @@ -816,32 +878,47 @@ export function FileBrowser({ credentials, connectionName }: Props) { useEffect(() => { loadObjects(currentPrefix) }, [currentPrefix]) // eslint-disable-line react-hooks/exhaustive-deps - // Reset focus when navigating - useEffect(() => { setFocusedIndex(null) }, [currentPrefix, search]) - - // Sorted + filtered lists (defined early so keyboard handler can use them) - const filteredPrefixes = prefixes.filter((p) => !search || p.toLowerCase().includes(search.toLowerCase())) - const filteredObjects = objects.filter((o) => !search || o.key.toLowerCase().includes(search.toLowerCase())) + // Debounce search to avoid re-sort on every keystroke + useEffect(() => { + const t = setTimeout(() => setDebouncedSearch(search), 200) + return () => clearTimeout(t) + }, [search]) - const sortedPrefixes = [...filteredPrefixes].sort((a, b) => { - const an = a.replace(currentPrefix, "").replace(/\/$/, "").toLowerCase() - const bn = b.replace(currentPrefix, "").replace(/\/$/, "").toLowerCase() - return sortDir === "asc" ? an.localeCompare(bn) : bn.localeCompare(an) - }) - const sortedObjects = [...filteredObjects].sort((a, b) => { - let cmp = 0 - if (sortCol === "name") { - cmp = a.key.replace(currentPrefix, "").toLowerCase() - .localeCompare(b.key.replace(currentPrefix, "").toLowerCase()) - } else if (sortCol === "size") { - cmp = (a.size ?? 0) - (b.size ?? 0) - } else if (sortCol === "modified") { - cmp = (a.lastModified ?? "").localeCompare(b.lastModified ?? "") - } - return sortDir === "asc" ? cmp : -cmp - }) + // Reset focus when navigating + useEffect(() => { setFocusedIndex(null) }, [currentPrefix, debouncedSearch]) + + // Sorted + filtered lists (memoized — recompute only when inputs change) + const sortedPrefixes = useMemo(() => { + const q = debouncedSearch.toLowerCase() + const filtered = q ? prefixes.filter((p) => p.toLowerCase().includes(q)) : prefixes + return [...filtered].sort((a, b) => { + const an = a.replace(currentPrefix, "").replace(/\/$/, "").toLowerCase() + const bn = b.replace(currentPrefix, "").replace(/\/$/, "").toLowerCase() + return sortDir === "asc" ? an.localeCompare(bn) : bn.localeCompare(an) + }) + }, [prefixes, debouncedSearch, currentPrefix, sortDir]) + + const sortedObjects = useMemo(() => { + const q = debouncedSearch.toLowerCase() + const filtered = q ? objects.filter((o) => o.key.toLowerCase().includes(q)) : objects + return [...filtered].sort((a, b) => { + let cmp = 0 + if (sortCol === "name") { + cmp = a.key.replace(currentPrefix, "").toLowerCase() + .localeCompare(b.key.replace(currentPrefix, "").toLowerCase()) + } else if (sortCol === "size") { + cmp = (a.size ?? 0) - (b.size ?? 0) + } else if (sortCol === "modified") { + cmp = (a.lastModified ?? "").localeCompare(b.lastModified ?? "") + } + return sortDir === "asc" ? cmp : -cmp + }) + }, [objects, debouncedSearch, currentPrefix, sortCol, sortDir]) - const allItems = [...sortedPrefixes, ...sortedObjects.map((o) => o.key)] + const allItems = useMemo( + () => [...sortedPrefixes, ...sortedObjects.map((o) => o.key)], + [sortedPrefixes, sortedObjects], + ) // Keyboard shortcuts useEffect(() => { @@ -919,9 +996,9 @@ export function FileBrowser({ credentials, connectionName }: Props) { try { const { url } = await getPresignedDownloadUrl(credentials, key) if (fileType === "code" || fileType === "doc") { - const res = await fetch(url) + const res = await fetch(url, { headers: { Range: "bytes=0-204799" } }) const text = res.ok ? await res.text() : undefined - setPreview({ key, url, loading: false, fileType, textContent: text?.slice(0, 200_000) }) + setPreview({ key, url, loading: false, fileType, textContent: text }) } else { setPreview({ key, url, loading: false, fileType }) } @@ -947,12 +1024,42 @@ export function FileBrowser({ credentials, connectionName }: Props) { } } + const updateFileRef = useRef<(idx: number, update: Partial) => void>(() => {}) + updateFileRef.current = (idx, update) => + setUploadQueue((prev) => prev.map((f, j) => (j === idx ? { ...f, ...update } : f))) + + const uploadOneRef = useRef<(idx: number) => Promise>(async () => false) + uploadOneRef.current = async (idx: number): Promise => { + const entry = uploadQueueRef.current[idx] + if (!entry?.file || !entry.key) return false + updateFileRef.current(idx, { status: "uploading", progress: 0, error: undefined }) + try { + const { url } = await getPresignedUploadUrl(credentials, entry.key, entry.file.type || "application/octet-stream") + await uploadFileXHR(url, entry.file, (p) => updateFileRef.current(idx, { progress: p })) + updateFileRef.current(idx, { status: "done", progress: 1 }) + return true + } catch (err) { + updateFileRef.current(idx, { status: "error", error: err instanceof Error ? err.message : "Upload failed" }) + return false + } + } + async function onUploadFiles(files: FileList | File[]) { const arr = Array.from(files) if (!arr.length) return - const initial: FileUploadStatus[] = arr.map((f) => ({ name: f.name, status: "queued", progress: 0 })) - setUploadQueue(initial) + let startIdx = 0 + setUploadQueue((prev) => { + startIdx = prev.length + const additions: FileUploadStatus[] = arr.map((f) => ({ + name: f.name, + status: "queued", + progress: 0, + file: f, + key: `${currentPrefix}${f.name}`, + })) + return [...prev, ...additions] + }) setUploadPanelOpen(true) if (!corsConfiguredRef.current) { @@ -967,25 +1074,10 @@ export function FileBrowser({ credentials, connectionName }: Props) { const CONCURRENCY = 4 let successCount = 0 - const updateFile = (idx: number, update: Partial) => - setUploadQueue((prev) => prev.map((f, j) => (j === idx ? { ...f, ...update } : f))) - - async function uploadOne(idx: number, file: File) { - updateFile(idx, { status: "uploading", progress: 0 }) - try { - const key = `${currentPrefix}${file.name}` - const { url } = await getPresignedUploadUrl(credentials, key, file.type || "application/octet-stream") - await uploadFileXHR(url, file, (p) => updateFile(idx, { progress: p })) - updateFile(idx, { status: "done", progress: 1 }) - successCount++ - } catch { - updateFile(idx, { status: "error" }) - } - } - for (let i = 0; i < arr.length; i += CONCURRENCY) { - const batch = arr.slice(i, i + CONCURRENCY) - await Promise.allSettled(batch.map((file, j) => uploadOne(i + j, file))) + const indices = arr.slice(i, i + CONCURRENCY).map((_, j) => startIdx + i + j) + const results = await Promise.allSettled(indices.map((idx) => uploadOneRef.current(idx))) + successCount += results.filter((r) => r.status === "fulfilled" && r.value).length } if (successCount > 0) { @@ -993,8 +1085,28 @@ export function FileBrowser({ credentials, connectionName }: Props) { loadObjects(currentPrefix) } if (successCount < arr.length) { - toast.error(`${arr.length - successCount} file${arr.length - successCount > 1 ? "s" : ""} failed`) + toast.error(`${arr.length - successCount} file${arr.length - successCount > 1 ? "s" : ""} failed — retry from panel`) + } + } + + async function retryUpload(idx: number) { + const ok = await uploadOneRef.current(idx) + if (ok) loadObjects(currentPrefix) + } + + async function retryAllFailed() { + const failedIndices = uploadQueueRef.current + .map((f, i) => (f.status === "error" ? i : -1)) + .filter((i) => i !== -1) + if (!failedIndices.length) return + const CONCURRENCY = 4 + let ok = 0 + for (let i = 0; i < failedIndices.length; i += CONCURRENCY) { + const batch = failedIndices.slice(i, i + CONCURRENCY) + const results = await Promise.allSettled(batch.map((idx) => uploadOneRef.current(idx))) + ok += results.filter((r) => r.status === "fulfilled" && r.value).length } + if (ok > 0) loadObjects(currentPrefix) } async function onDownloadZip() { @@ -1007,9 +1119,24 @@ export function FileBrowser({ credentials, connectionName }: Props) { ]) const zip = new JSZip() let ok = 0 - for (const key of fileKeys) { + const ZIP_CONCURRENCY = 6 + // Batch presign — single request for all keys (max 100/batch) + const presigned: Record = {} + try { + for (let i = 0; i < fileKeys.length; i += 100) { + const chunk = fileKeys.slice(i, i + 100) + const { urls } = await getPresignedBatch(credentials, chunk.map((key) => ({ key, op: "get" }))) + for (const u of urls) presigned[u.key] = u.url + } + } catch (err) { + toast.error(err instanceof Error ? err.message : "Failed to presign URLs") + setZipProgress(null) + return + } + async function fetchOne(key: string) { try { - const { url } = await getPresignedDownloadUrl(credentials, key) + const url = presigned[key] + if (!url) throw new Error("missing url") const res = await fetch(url) const blob = await res.blob() zip.file(key.split("/").pop() ?? key, blob) @@ -1019,6 +1146,10 @@ export function FileBrowser({ credentials, connectionName }: Props) { toast.error(`Failed to fetch ${key.split("/").pop()}`) } } + for (let i = 0; i < fileKeys.length; i += ZIP_CONCURRENCY) { + const batch = fileKeys.slice(i, i + ZIP_CONCURRENCY) + await Promise.allSettled(batch.map(fetchOne)) + } if (ok > 0) { const blob = await zip.generateAsync({ type: "blob" }) saveAs(blob, `download-${Date.now()}.zip`) @@ -1099,7 +1230,7 @@ export function FileBrowser({ credentials, connectionName }: Props) { ) } - const allCount = filteredPrefixes.length + filteredObjects.length + const allCount = sortedPrefixes.length + sortedObjects.length const allSelected = allCount > 0 && selectedKeys.size >= allCount const someSelected = selectedKeys.size > 0 && !allSelected const hasSelection = selectedKeys.size > 0 @@ -1320,12 +1451,10 @@ export function FileBrowser({ credentials, connectionName }: Props) { )} {isTruncated && ( -
- -
+ loadObjects(currentPrefix, nextContinuationToken)} + loading={isLoading} + /> )} @@ -1384,9 +1513,9 @@ export function FileBrowser({ credentials, connectionName }: Props) { {/* Status bar */} {!hasSelection && allCount > 0 && (
- {filteredPrefixes.length + filteredObjects.length} items - {filteredPrefixes.length > 0 && · {filteredPrefixes.length} folder{filteredPrefixes.length !== 1 ? "s" : ""}} - {filteredObjects.length > 0 && · {filteredObjects.length} file{filteredObjects.length !== 1 ? "s" : ""}} + {sortedPrefixes.length + sortedObjects.length} items + {sortedPrefixes.length > 0 && · {sortedPrefixes.length} folder{sortedPrefixes.length !== 1 ? "s" : ""}} + {sortedObjects.length > 0 && · {sortedObjects.length} file{sortedObjects.length !== 1 ? "s" : ""}} {currentPrefix && /{currentPrefix.replace(/\/$/, "")}}
)} @@ -1441,6 +1570,8 @@ export function FileBrowser({ credentials, connectionName }: Props) { queue={uploadQueue} onClearCompleted={() => setUploadQueue([])} onDismiss={() => setUploadPanelOpen(false)} + onRetry={retryUpload} + onRetryAll={retryAllFailed} /> )}
diff --git a/apps/web/src/lib/s3-drive-api.ts b/apps/web/src/lib/s3-drive-api.ts index f74cb47e..dd3acde0 100644 --- a/apps/web/src/lib/s3-drive-api.ts +++ b/apps/web/src/lib/s3-drive-api.ts @@ -148,6 +148,17 @@ export async function getPresignedUploadUrl( return s3Request("POST", `${BASE}/operations/presigned-upload`, { credentials, key, contentType }) } +export type PresignedBatchItem = { key: string; op?: "get" | "put"; contentType?: string } +export type PresignedBatchResponse = { urls: PresignedUrlResponse[] } + +export async function getPresignedBatch( + credentials: S3Credentials, + items: PresignedBatchItem[], + expiresIn = 3600, +): Promise { + return s3Request("POST", `${BASE}/operations/presigned-batch`, { credentials, items, expiresIn }) +} + export async function moveObject( credentials: S3Credentials, sourceKey: string,