Akhil/optimise backend v2

apps/backend/README.md

@@ -118,6 +118,49 @@ Set one of the following:
 
 If both JSON and path are set, `FIREBASE_CREDENTIALS_JSON` is used first.
 
+## Redis cache
+
+### Environment
+
+| Var | Default | Purpose |
+|-----|---------|---------|
+| `REDIS_URL` | unset | If unset, cache is inert. `redis://...` or `rediss://...`. |
+| `CACHE_ENABLED` | `true` | Global kill switch. |
+| `CACHE_NAMESPACES` | `""` | Comma-separated namespaces to activate. Empty = no caching. |
+| `CACHE_DEFAULT_TTL` | `120` | Fallback TTL seconds. |
+| `CACHE_OP_TIMEOUT_MS` | `50` | Per-Redis-call timeout. |
+| `CACHE_XFETCH_BETA` | `1.0` | XFetch tuning constant. Higher = refreshes earlier. |
+| `CACHE_LOG_LEVEL` | `WARNING` | `app.cache` logger level. |
+
+### Rollout phases
+
+Ramp `CACHE_NAMESPACES` namespace-by-namespace; restart workers each phase.
+
+| Phase | Add to `CACHE_NAMESPACES` |
+|-------|---------------------------|
+| 1 | `auth_token,auth_user` |
+| 2 | `user_preferences,dns_lookup` |
+| 3 | `bookmarks,notes,code_snippets,api_client` |
+| 4 | `tasks,passwords,url_shortener_owner` |
+| 5 | `url_shortener_resolve,analytics_aggregate` |
+
+Kill switch: drop a namespace from env + restart workers. Global kill: `CACHE_ENABLED=false`.
+
+### Redis server config (prod)
+
+- `maxmemory` sized to expected working set (start with 512 MB for 1k users).
+- `maxmemory-policy allkeys-lru`.
+- `requirepass` set; TLS enabled if exposed outside VPC.
+- Bind to private network only.
+
+## Load testing cache
+
+See `scripts/loadtest_cache.py`. Requires `locust` (install separately) and a running backend.
+
+1. Boot backend with `CACHE_ENABLED=false`; run a 5-min baseline.
+2. Boot backend with `CACHE_ENABLED=true` + chosen `CACHE_NAMESPACES`; re-run.
+3. Compare p50/p99 in `*_stats.csv`. Acceptance gates: p50 < 50ms, p99 < 200ms, error rate 0%.
+
 ## From monorepo root
 
 - `pnpm dev:backend` - start backend server

apps/backend/app/api/router.py

@@ -21,6 +21,7 @@
 from app.api.routes.redis_commander.api import router as redis_commander_router
 from app.api.routes.url_shortener.api import router as url_shortener_router
 from app.api.routes.dns_lookup.api import router as dns_lookup_router
+from app.api.routes.audit_log.api import router as audit_log_router
 
 api_router = APIRouter()
 api_router.include_router(health_router)
@@ -44,3 +45,4 @@
 api_router.include_router(redis_commander_router)
 api_router.include_router(url_shortener_router)
 api_router.include_router(dns_lookup_router)
+api_router.include_router(audit_log_router)

apps/backend/app/api/routes/analytics/api.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Query
 
 from app.api.routes.analytics import services as analytics_svc
 from app.api.routes.auth.services import get_current_uid
@@ -9,3 +9,20 @@
 @router.get("/summary", summary="Dashboard analytics counts")
 async def dashboard_summary(uid: str = Depends(get_current_uid)):
     return await analytics_svc.get_dashboard_analytics(uid)
+
+
+@router.get("/top-tools", summary="Top tools by usage (global, cached 5 min)")
+async def top_tools(
+    days: int = Query(default=7, ge=1, le=90),
+    limit: int = Query(default=10, ge=1, le=50),
+    _uid: str = Depends(get_current_uid),
+):
+    return await analytics_svc.get_top_tools(days=days, limit=limit)
+
+
+@router.get("/activity-buckets", summary="Daily activity counts (global, cached 5 min)")
+async def activity_buckets(
+    days: int = Query(default=7, ge=1, le=90),
+    _uid: str = Depends(get_current_uid),
+):
+    return await analytics_svc.get_activity_buckets(days=days)

apps/backend/app/api/routes/analytics/services.py

@@ -1,10 +1,13 @@
 import asyncio
+import datetime
 
 from app.api.routes.tasks import services as task_svc
+from app.core.cache import cached
 from app.utils.collection_name import (
     API_CLIENT_COLLECTIONS,
     API_CLIENT_ENVIRONMENTS,
     API_CLIENT_HISTORY,
+    AUDIT_LOG,
     BOOKMARK_FOLDERS,
     BOOKMARKS,
     CODE_SNIPPETS,
@@ -18,6 +21,44 @@
 from app.database import db_manager
 
 
+@cached(ns="analytics_aggregate", ttl=300, scope="global", strategy="xfetch")
+async def get_top_tools(*, days: int = 7, limit: int = 10) -> list[dict]:
+    """Return the top N tools by event count across all users over the last ``days`` days."""
+    since = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=days)
+    pipeline = [
+        {"$match": {"module": {"$ne": None}, "ts": {"$gte": since.timestamp() * 1000}}},
+        {"$group": {"_id": "$module", "count": {"$sum": 1}}},
+        {"$sort": {"count": -1}},
+        {"$limit": limit},
+        {"$project": {"_id": 0, "tool": "$_id", "count": 1}},
+    ]
+    return await db_manager.aggregate(AUDIT_LOG, pipeline)
+
+
+@cached(ns="analytics_aggregate", ttl=300, scope="global", strategy="xfetch")
+async def get_activity_buckets(*, days: int = 7) -> list[dict]:
+    """Return daily event counts across all users over the last ``days`` days."""
+    since = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=days)
+    pipeline = [
+        {"$match": {"ts": {"$gte": since.timestamp() * 1000}}},
+        {
+            "$group": {
+                "_id": {
+                    "$dateToString": {
+                        "format": "%Y-%m-%d",
+                        "date": {"$toDate": "$ts"},
+                        "timezone": "UTC",
+                    }
+                },
+                "count": {"$sum": 1},
+            }
+        },
+        {"$sort": {"_id": 1}},
+        {"$project": {"_id": 0, "date": "$_id", "count": 1}},
+    ]
+    return await db_manager.aggregate(AUDIT_LOG, pipeline)
+
+
 async def get_dashboard_analytics(uid: str) -> DashboardAnalyticsOut:
     base = {"created_by": uid}
     nosql_filter = {
@@ -39,7 +80,7 @@ async def get_dashboard_analytics(uid: str) -> DashboardAnalyticsOut:
         json_docs,
         code_snippets,
     ) = await asyncio.gather(
-        task_svc.get_task_stats(uid),
+        task_svc.get_task_stats(uid=uid),
         db_manager.count_documents(PASSWORD_ENTRIES, base),
         db_manager.count_documents(BOOKMARKS, base),
         db_manager.count_documents(BOOKMARK_FOLDERS, base),

apps/backend/app/api/routes/api_client/api.py

@@ -20,7 +20,7 @@
 
 @router.get("/collections", response_model=list[ApiClientCollectionOut], summary="List API client collections")
 async def list_collections(uid: str = Depends(get_current_uid)) -> list[ApiClientCollectionOut]:
-    return await api_client_svc.list_collections(uid)
+    return await api_client_svc.list_collections(uid=uid)
 
 
 @router.post("/collections", response_model=ApiClientCollectionOut, summary="Create API client collection")
@@ -51,7 +51,7 @@ async def delete_collection(collection_id: str, uid: str = Depends(get_current_u
 
 @router.get("/environments", response_model=list[ApiClientEnvironmentOut], summary="List API client environments")
 async def list_environments(uid: str = Depends(get_current_uid)) -> list[ApiClientEnvironmentOut]:
-    return await api_client_svc.list_environments(uid)
+    return await api_client_svc.list_environments(uid=uid)
 
 
 @router.post("/environments", response_model=ApiClientEnvironmentOut, summary="Create API client environment")
@@ -85,7 +85,7 @@ async def list_history(
     uid: str = Depends(get_current_uid),
     limit: int = Query(default=HISTORY_MAX_ITEMS, ge=1, le=HISTORY_MAX_ITEMS),
 ) -> list[ApiClientHistoryOut]:
-    return await api_client_svc.list_history(uid, limit=limit)
+    return await api_client_svc.list_history(uid=uid, limit=limit)
 
 
 @router.post("/history", response_model=ApiClientHistoryOut, summary="Append API client history entry")

apps/backend/app/api/routes/api_client/services.py

@@ -23,6 +23,7 @@
     ApiClientHistoryOut,
     HISTORY_MAX_ITEMS,
 )
+from app.core.cache import cached, bump_version
 from app.database import db_manager
 
 HISTORY_TRIM_BATCH_SIZE = 500
@@ -56,7 +57,8 @@ def _env_to_out(doc: dict[str, Any]) -> ApiClientEnvironmentOut:
     )
 
 
-async def list_collections(uid: str) -> list[ApiClientCollectionOut]:
+@cached(ns="api_client", ttl=300, scope="user")
+async def list_collections(*, uid: str) -> list[ApiClientCollectionOut]:
     docs = await db_manager.find(
         API_CLIENT_COLLECTIONS,
         {"created_by": uid},
@@ -75,6 +77,7 @@ async def create_collection(uid: str, body: ApiClientCollectionCreate) -> ApiCli
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to create collection."
         ) from exc
     doc["_id"] = result.inserted_id
+    await bump_version(ns="api_client", uid=uid)
     return _collection_to_out(doc)
 
 
@@ -99,6 +102,7 @@ async def patch_collection(uid: str, collection_id: str, body: ApiClientCollecti
         ) from exc
     if not doc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Collection not found.")
+    await bump_version(ns="api_client", uid=uid)
     return _collection_to_out(doc)
 
 
@@ -107,9 +111,11 @@ async def delete_collection(uid: str, collection_id: str) -> None:
     result = await db_manager.delete_one(API_CLIENT_COLLECTIONS, {"_id": oid, "created_by": uid})
     if result.deleted_count == 0:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Collection not found.")
+    await bump_version(ns="api_client", uid=uid)
 
 
-async def list_environments(uid: str) -> list[ApiClientEnvironmentOut]:
+@cached(ns="api_client", ttl=300, scope="user")
+async def list_environments(*, uid: str) -> list[ApiClientEnvironmentOut]:
     docs = await db_manager.find(
         API_CLIENT_ENVIRONMENTS,
         {"created_by": uid},
@@ -128,6 +134,7 @@ async def create_environment(uid: str, body: ApiClientEnvironmentCreate) -> ApiC
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to create environment."
         ) from exc
     doc["_id"] = result.inserted_id
+    await bump_version(ns="api_client", uid=uid)
     return _env_to_out(doc)
 
 
@@ -152,6 +159,7 @@ async def patch_environment(uid: str, environment_id: str, body: ApiClientEnviro
         ) from exc
     if not doc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Environment not found.")
+    await bump_version(ns="api_client", uid=uid)
     return _env_to_out(doc)
 
 
@@ -160,6 +168,7 @@ async def delete_environment(uid: str, environment_id: str) -> None:
     result = await db_manager.delete_one(API_CLIENT_ENVIRONMENTS, {"_id": oid, "created_by": uid})
     if result.deleted_count == 0:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Environment not found.")
+    await bump_version(ns="api_client", uid=uid)
 
 
 def _history_doc_to_out(doc: dict[str, Any]) -> ApiClientHistoryOut:
@@ -195,9 +204,11 @@ async def trim_history(uid: str) -> None:
         return
     ids = [d["_id"] for d in stale_docs]
     await db_manager.delete_many(API_CLIENT_HISTORY, {"_id": {"$in": ids}, "created_by": uid})
+    await bump_version(ns="api_client", uid=uid)
 
 
-async def list_history(uid: str, *, limit: int = HISTORY_MAX_ITEMS) -> list[ApiClientHistoryOut]:
+@cached(ns="api_client", ttl=300, scope="user")
+async def list_history(*, uid: str, limit: int = HISTORY_MAX_ITEMS) -> list[ApiClientHistoryOut]:
     lim = max(1, min(limit, HISTORY_MAX_ITEMS))
     docs = await db_manager.find(API_CLIENT_HISTORY, {"created_by": uid}, sort=[("timestamp", -1)], limit=lim)
     return [_history_doc_to_out(d) for d in docs]
@@ -224,6 +235,7 @@ async def create_history(uid: str, body: ApiClientHistoryCreate) -> ApiClientHis
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to save history entry."
         ) from exc
     doc["_id"] = result.inserted_id
+    await bump_version(ns="api_client", uid=uid)
     return _history_doc_to_out(doc)
 
 
@@ -232,6 +244,7 @@ async def delete_history_entry(uid: str, entry_id: str) -> None:
     result = await db_manager.delete_one(API_CLIENT_HISTORY, {"_id": oid, "created_by": uid})
     if result.deleted_count == 0:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="History entry not found.")
+    await bump_version(ns="api_client", uid=uid)
 
 
 async def clear_history(uid: str) -> None:
@@ -241,3 +254,4 @@ async def clear_history(uid: str) -> None:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to clear history."
         ) from exc
+    await bump_version(ns="api_client", uid=uid)

apps/backend/app/api/routes/audit_log/api.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+
+from app.api.routes.auth.services import get_current_uid
+from app.api.routes.audit_log import services as svc
+from app.api.routes.audit_log.schema import AuditListOut
+
+router = APIRouter(prefix="/audit-log", tags=["audit-log"])
+
+
+@router.get("", response_model=AuditListOut, summary="List the current user's audit events")
+async def list_events(
+    uid: str = Depends(get_current_uid),
+    skip: int = Query(default=0, ge=0),
+    limit: int = Query(default=50, ge=1, le=100),
+    module: Optional[str] = Query(default=None),
+    action: Optional[str] = Query(default=None),
+    outcome: Optional[str] = Query(default=None),
+    ts_from: Optional[int] = Query(default=None, alias="from"),
+    ts_to: Optional[int] = Query(default=None, alias="to"),
+    search: Optional[str] = Query(default=None),
+) -> AuditListOut:
+    return await svc.list_audit_events(
+        uid, skip=skip, limit=limit, module=module, action=action,
+        outcome=outcome, ts_from=ts_from, ts_to=ts_to, search=search,
+    )

apps/backend/app/api/routes/audit_log/schema.py

@@ -0,0 +1,42 @@
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+class AuditChange(BaseModel):
+    field: str
+    before: Any | None = None
+    after: Any | None = None
+
+
+class AuditDevice(BaseModel):
+    browser: str
+    os: str
+    device_type: str
+
+
+class AuditEventOut(BaseModel):
+    id: str
+    uid: Optional[str] = None
+    action: str
+    module: Optional[str] = None
+    entity_type: Optional[str] = None
+    entity_id: Optional[str] = None
+    method: str
+    path: str
+    status: int
+    outcome: str
+    changes: Optional[list[AuditChange]] = None
+    summary: Optional[str] = None
+    ip: Optional[str] = None
+    ua_raw: Optional[str] = None
+    device: Optional[AuditDevice] = None
+    latency_ms: int
+    ts: int
+
+
+class AuditListOut(BaseModel):
+    items: list[AuditEventOut]
+    total: int
+    skip: int
+    limit: int