Refactor database handling, enhance metrics tracking, and improve UI

README.md

@@ -100,7 +100,17 @@ workspace:
 > [!CAUTION]
 > **Database Schema Consistency**: Edmate's `database_service.py` currently expects tables to have specific columns (e.g., `title`, `options`, `correct_options`). If your database uses different column names, you must update the SQL queries in `content_gen/scripts/processing/database_service.py` to match your schema.
 
-### 3. Understanding Pipeline Settings
+### 3. Auth & Storage Configuration
+Edmate supports both local self-hosted deployment and cloud SaaS operation. Configure these settings in `content_gen/.env`:
+
+- **Authentication (`EDMATE_AUTH_REQUIRED`)**: 
+  - `false` (Default): Operates in "Self-Hosted Mode". All requests bypass auth checks and are treated as unrestricted "pro" users. Perfect for local development.
+  - `true`: Operates in "Cloud Mode". Requires a valid JWT token via `Authorization: Bearer <token>`. Anonymous users have restricted access (e.g., cannot export without hitting a paywall).
+- **Storage Backend (`EDMATE_STORAGE_BACKEND`)**:
+  - `local` (Default): Saves all drafts and extracted assets locally to the `.drafts` directory.
+  - `s3`: Placeholder for cloud deployments to store files in AWS S3 or compatible blob storage.
+
+### 4. Understanding Pipeline Settings
 The **Automation Hub** provides several "Admin" settings to handle diverse document formats:
 - **Extraction Guardrails**: Adjust "Detection Mode" to **Strict** for standard papers or **Open** for noisy documents.
 - **Model Routing**: Strategies to balance cost and quality. Edmate can use cheaper models (like Gemini Flash) for extraction and switch to high-precision models (like GPT-4o) for final content generation.
@@ -152,6 +162,32 @@ If a partner platform wants end-users to provide their own API key in the platfo
 3. Partner backend sends requests to Edmate with `X-API-Key`.
 4. Edmate processes the file and returns job status/results.
 
+## Pricing & Limits
+
+- **Free / Open Source** – No upload limits, but you cannot download output files.
+- **Basic Solutions Plan ($10/month)** – Up to **30 uploads per month**, each **≤ 10 MB** (optimal for documents up to **50 pages**). Unlimited exports in CSV/JSON/MD/Docx.
+- **Pro Solutions Plan ($50/month)** – Unlimited uploads, each ≤ **25 MB** (up to **200 pages**). Advanced exports (Word & Image ZIPs), high‑concurrency ingestion, dedicated schema slots, and **priority support**.
+
+These limits are now documented in the **FAQ** section of the marketing page and reflected in the UI.
+
+## FAQ
+
+We added a comprehensive FAQ section on the landing page covering:
+- File size limits
+- Page count recommendations
+- BYOK vs hosted Gemini usage
+- Pedagogical techniques applied
+- Data security and API key handling
+
+The FAQ is styled consistently for both dark and light themes.
+
+## Development Notes
+
+- The large `marketing.css` file has been split into modular sub‑files (`nav.css`, `hero.css`, `features.css`, `pricing.css`, `footer.css`, `faq.css`, `responsive.css`, `theme.css`).
+- Base font size is now **15 px** for a tighter, more professional SaaS feel across all pages.
+- Mobile responsiveness has been overhauled with a slide‑over sidebar, full‑width layout on mobile, and a simplified navigation bar.
+- Theme toggle button styles are now encapsulated in `faq.css` and removed from inline HTML.
+
 ### Supported API key headers
 
 - Preferred: `X-API-Key`

content_gen/.env.example

@@ -31,3 +31,18 @@ DATABASE_URL=postgresql://user:password@host:5432/edmate
 
 # Path to Google Service Account JSON (optional, for Vertex AI/Google Cloud mode)
 # GOOGLE_APPLICATION_CREDENTIALS=credentials/gemini_creds.json
+
+# ===== Auth (Optional — omit for self-hosted mode) =====
+EDMATE_AUTH_REQUIRED=false          # Set true for SaaS deployment
+# AUTH_PROVIDER=postgres            # "postgres" | "supabase" | "custom"
+# AUTH_DATABASE_URL=...             # If different from main DATABASE_URL
+
+# ===== Storage =====
+EDMATE_STORAGE_BACKEND=local        # "local" | "s3"
+# S3_ENDPOINT=...
+# S3_BUCKET=edmate-uploads
+# S3_ACCESS_KEY=...
+# S3_SECRET_KEY=...
+
+# ===== Encryption (for server-side BYOK storage) =====
+# EDMATE_ENCRYPTION_SECRET=...

content_gen/adapters/postgres_adapter.py

@@ -1,10 +1,10 @@
 import re
-import psycopg2
 import uuid
 from datetime import datetime, timezone
 from typing import List, Dict, Optional, Any
 
 from .base import BaseStorageAdapter
+from ..db.session import connect_real_dict
 from ..core.schemas import ProcessedQuestion, Flashcard
 from ..core.config_loader import ConfigLoader
 from ..core.config_schema import EdmateConfig
@@ -20,7 +20,7 @@ class PostgresStorageAdapter(BaseStorageAdapter):
 
     def __init__(self, connection_string: str, edmate_config: Optional[EdmateConfig] = None):
         self.conn_str = connection_string
-        self.conn = psycopg2.connect(connection_string)
+        self.conn = connect_real_dict(connection_string, connect_timeout=30)
         self.cur = self.conn.cursor()
         self._workspace = (edmate_config or ConfigLoader.load_config()).workspace
 
@@ -31,7 +31,7 @@ def initialize_schema(connection_string: str, edmate_config: Optional[EdmateConf
         Table names come from edmate_config workspace.target_tables when set;
         otherwise a legacy Cambridge-style multi-table set is created for backward compatibility.
         """
-        conn = psycopg2.connect(connection_string)
+        conn = connect_real_dict(connection_string, connect_timeout=30)
         cur = conn.cursor()
         try:
             # Shared tables

content_gen/core/config_schema.py

@@ -66,7 +66,7 @@ class StorageSettings(BaseModel):
 
 
 class ExtractionSettings(BaseModel):
-    engine: ExtractionEngine = ExtractionEngine.PDF_EXTRACT_KIT
+    engine: ExtractionEngine = ExtractionEngine.VISION
     min_question_number: int = 1
     max_question_number: Optional[int] = None
     question_detection_mode: DetectionMode = DetectionMode.BALANCED

content_gen/core/media_encoding.py

@@ -0,0 +1,15 @@
+"""Shared binary → data URI helpers (PNG)."""
+
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+
+
+def png_bytes_to_data_uri(png_bytes: bytes) -> str:
+    return "data:image/png;base64," + base64.b64encode(png_bytes).decode("utf-8")
+
+
+def png_file_to_data_uri(path: Path) -> str:
+    with open(path, "rb") as f:
+        return png_bytes_to_data_uri(f.read())

content_gen/core/metrics.py

@@ -1,59 +1,184 @@
+import importlib
 import json
+import os
+import threading
+from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Dict, Any
+from typing import Any, Dict, Optional
 from datetime import datetime
 
 
-class MetricsTracker:
-    """
-    Tracks LLM usage, tokens, and costs across Edmate sessions.
-    Persists data to a local JSON file for UI consumption.
-    """
+class MetricsTrackerBase(ABC):
+    """LLM usage / cost tracking (file, memory, or Redis)."""
+
+    @property
+    @abstractmethod
+    def metrics(self) -> Dict[str, Any]:
+        """Aggregate counters (mutable dict for file/memory; snapshot for redis)."""
+        ...
+
+    @abstractmethod
+    def log_usage(self, response_obj: Any) -> None:
+        ...
+
+    @abstractmethod
+    def get_current_cost(self) -> float:
+        ...
+
+    def reset_if_new_day(self) -> None:
+        """Placeholder for daily reset logic if needed."""
+        pass
+
+
+class FileMetricsTracker(MetricsTrackerBase):
+    """Persists aggregate metrics to a JSON file (default; single-process friendly)."""
 
     def __init__(self, storage_path: str = "content_gen/data/session_metrics.json"):
         self.storage_path = Path(storage_path)
         self.storage_path.parent.mkdir(parents=True, exist_ok=True)
-        self.metrics = self._load_metrics()
+        self._lock = threading.Lock()
+        self._metrics = self._load_metrics()
+
+    @property
+    def metrics(self) -> Dict[str, Any]:
+        return self._metrics
 
     def _load_metrics(self) -> Dict[str, Any]:
         if not self.storage_path.exists():
             return {"total_cost": 0.0, "total_tokens": 0, "last_updated": None, "calls": 0}
-
         try:
-            with open(self.storage_path, 'r') as f:
+            with open(self.storage_path, "r", encoding="utf-8") as f:
                 return json.load(f)
         except Exception:
             return {"total_cost": 0.0, "total_tokens": 0, "last_updated": None, "calls": 0}
 
-    def log_usage(self, response_obj: Any):
-        """
-        Extracts usage data from a LiteLLM response object and updates local metrics.
-        """
-        # LiteLLM provides usage and cost in the response
-        usage = getattr(response_obj, 'usage', {})
+    def log_usage(self, response_obj: Any) -> None:
+        usage = getattr(response_obj, "usage", {})
+        try:
+            from litellm import completion_cost
+
+            actual_cost = completion_cost(completion_response=response_obj)
+        except Exception:
+            actual_cost = 0.0
+
+        with self._lock:
+            self._metrics["total_cost"] += actual_cost
+            self._metrics["total_tokens"] += usage.get("total_tokens", 0)
+            self._metrics["calls"] = int(self._metrics.get("calls", 0)) + 1
+            self._metrics["last_updated"] = datetime.now().isoformat()
+            self._save_metrics_unlocked()
+
+    def _save_metrics_unlocked(self) -> None:
+        with open(self.storage_path, "w", encoding="utf-8") as f:
+            json.dump(self._metrics, f, indent=2)
+
+    def get_current_cost(self) -> float:
+        with self._lock:
+            return float(self._metrics.get("total_cost", 0.0))
+
+
+class MemoryMetricsTracker(MetricsTrackerBase):
+    """In-process metrics only (safe default under multi-worker: no shared file corruption)."""
 
-        # Real cost from LiteLLM is usually in response._hidden_params or similar
-        # but LiteLLM also provides it in a more direct way if litellm.ModelResponse is used
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._metrics: Dict[str, Any] = {
+            "total_cost": 0.0,
+            "total_tokens": 0,
+            "last_updated": None,
+            "calls": 0,
+        }
+
+    @property
+    def metrics(self) -> Dict[str, Any]:
+        return self._metrics
+
+    def log_usage(self, response_obj: Any) -> None:
+        usage = getattr(response_obj, "usage", {})
         try:
             from litellm import completion_cost
+
             actual_cost = completion_cost(completion_response=response_obj)
         except Exception:
             actual_cost = 0.0
 
-        self.metrics["total_cost"] += actual_cost
-        self.metrics["total_tokens"] += usage.get("total_tokens", 0)
-        self.metrics["calls"] += 1
-        self.metrics["last_updated"] = datetime.now().isoformat()
+        with self._lock:
+            self._metrics["total_cost"] += actual_cost
+            self._metrics["total_tokens"] += usage.get("total_tokens", 0)
+            self._metrics["calls"] = int(self._metrics.get("calls", 0)) + 1
+            self._metrics["last_updated"] = datetime.now().isoformat()
 
-        self._save_metrics()
+    def get_current_cost(self) -> float:
+        with self._lock:
+            return float(self._metrics.get("total_cost", 0.0))
 
-    def _save_metrics(self):
-        with open(self.storage_path, 'w') as f:
-            json.dump(self.metrics, f, indent=2)
+
+class RedisMetricsTracker(MetricsTrackerBase):
+    """Optional Redis-backed aggregates (hash: atomic HINCRBY*)."""
+
+    def __init__(self, url: str, key: str = "edmate:metrics:session"):
+        try:
+            redis_mod = importlib.import_module("redis")
+        except ImportError as e:
+            raise RuntimeError("redis package required for Redis metrics backend") from e
+        self._r = redis_mod.from_url(url, decode_responses=True)
+        self._key = key
+
+    @property
+    def metrics(self) -> Dict[str, Any]:
+        raw = self._r.hgetall(self._key)
+        if not raw:
+            return {"total_cost": 0.0, "total_tokens": 0, "last_updated": None, "calls": 0}
+        return {
+            "total_cost": float(raw.get("total_cost") or 0.0),
+            "total_tokens": int(raw.get("total_tokens") or 0),
+            "last_updated": raw.get("last_updated"),
+            "calls": int(raw.get("calls") or 0),
+        }
+
+    def log_usage(self, response_obj: Any) -> None:
+        usage = getattr(response_obj, "usage", {})
+        try:
+            from litellm import completion_cost
+
+            actual_cost = completion_cost(completion_response=response_obj)
+        except Exception:
+            actual_cost = 0.0
+
+        tokens = int(usage.get("total_tokens", 0) or 0)
+        pipe = self._r.pipeline()
+        pipe.hincrbyfloat(self._key, "total_cost", float(actual_cost))
+        pipe.hincrby(self._key, "total_tokens", tokens)
+        pipe.hincrby(self._key, "calls", 1)
+        pipe.hset(self._key, "last_updated", datetime.now().isoformat())
+        pipe.execute()
 
     def get_current_cost(self) -> float:
-        return self.metrics.get("total_cost", 0.0)
+        v = self._r.hget(self._key, "total_cost")
+        return float(v or 0.0)
 
-    def reset_if_new_day(self):
-        """Placeholder for daily reset logic if needed."""
-        pass
+
+def create_metrics_tracker(
+    storage_path: Optional[str] = None,
+) -> MetricsTrackerBase:
+    """
+    Factory: EDMATE_METRICS_BACKEND=file|memory|redis (default: file).
+
+    - ``memory``: no disk; per-process (recommended for multi-worker until Redis is used).
+    - ``file``: legacy JSON file (single-worker or dev).
+    - ``redis``: requires EDMATE_METRICS_REDIS_URL and ``redis`` package.
+    """
+    mode = (os.environ.get("EDMATE_METRICS_BACKEND") or "file").strip().lower()
+    if mode == "memory":
+        return MemoryMetricsTracker()
+    if mode == "redis":
+        url = (os.environ.get("EDMATE_METRICS_REDIS_URL") or "").strip()
+        if not url:
+            return MemoryMetricsTracker()
+        return RedisMetricsTracker(url)
+    path = storage_path or os.environ.get("EDMATE_METRICS_FILE") or "content_gen/data/session_metrics.json"
+    return FileMetricsTracker(path)
+
+
+# Backward-compatible name used by ModelRoutingEngine and tests.
+MetricsTracker = FileMetricsTracker