Migrate observability to Langfuse v4 SDK

.env.example

@@ -70,6 +70,8 @@ CLOUDFRONT_URL=
 # ============================================================
 # PART 8: Observability (to be added)
 # ============================================================
-# LANGFUSE_PUBLIC_KEY=...
-# LANGFUSE_SECRET_KEY=...
-# LANGFUSE_HOST=https://cloud.langfuse.com
+# LANGFUSE_PUBLIC_KEY=pk-lf-...
+# LANGFUSE_SECRET_KEY=sk-lf-...
+# LANGFUSE_BASE_URL=https://us.cloud.langfuse.com
+# LANGFUSE_HOST=https://us.cloud.langfuse.com
+# LANGFUSE_TRACING_ENVIRONMENT=development

.gitignore

@@ -220,7 +220,7 @@ __marimo__/
 .terraform/
 terraform.tfstate.d/
 *.tfstate
-*.tfstate.backup
+*.tfstate.*
 
 # Lambda deployment packages
 lambda_function.zip

backend/charter/lambda_handler.py

@@ -134,31 +134,44 @@ def lambda_handler(event, context):
         "portfolio_data": {...}
     }
     """
-    # Wrap entire handler with observability context
-    with observe():
-        try:
-            logger.info(f"Charter Lambda invoked with event keys: {list(event.keys()) if isinstance(event, dict) else 'not a dict'}")
+    if isinstance(event, str):
+        event = json.loads(event)
 
-            # Parse event
-            if isinstance(event, str):
-                event = json.loads(event)
+    logger.info(f"Charter Lambda invoked with event keys: {list(event.keys()) if isinstance(event, dict) else 'not a dict'}")
 
-            job_id = event.get('job_id')
-            if not job_id:
-                return {
-                    'statusCode': 400,
-                    'body': json.dumps({'error': 'job_id is required'})
-                }
+    job_id = event.get('job_id')
+    if not job_id:
+        return {
+            'statusCode': 400,
+            'body': json.dumps({'error': 'job_id is required'})
+        }
 
-            # Initialize database first
-            db = Database()
+    db = Database()
+    user_id = None
+    job = None
+    try:
+        job = db.jobs.find_by_id(job_id)
+        if job:
+            user_id = job.get('clerk_user_id')
+    except Exception as e:
+        logger.warning(f"Charter: Could not load job owner for tracing: {e}")
+
+    with observe(
+        name="chart-portfolio",
+        user_id=user_id,
+        session_id=job_id,
+        tags=["charter", "portfolio-analysis"],
+        metadata={"agent": "charter"},
+        input={"job_id": job_id},
+    ) as obs:
+        try:
 
             portfolio_data = event.get('portfolio_data')
             if not portfolio_data:
                 # Load portfolio data from database (like Reporter does)
                 logger.info(f"Charter: Loading portfolio data for job {job_id}")
                 try:
-                    job = db.jobs.find_by_id(job_id)
+                    job = job or db.jobs.find_by_id(job_id)
                     if job:
                         user_id = job['clerk_user_id']
                         user = db.users.find_by_clerk_id(user_id)
@@ -212,13 +225,17 @@ def lambda_handler(event, context):
             result = asyncio.run(run_charter_agent(job_id, portfolio_data, db))
 
             logger.info(f"Charter completed for job {job_id}: {result}")
-
+            obs.update(output={
+                "status": "completed" if result.get("success") else "failed",
+                "charts_generated": result.get("charts_generated", 0),
+            })
             return {
                 'statusCode': 200,
                 'body': json.dumps(result)
             }
 
         except Exception as e:
+            obs.update(output={"status": "failed", "error": str(e)})
             logger.error(f"Error in charter: {e}", exc_info=True)
             return {
                 'statusCode': 500,