diff --git a/gcp-deployment/.env.example b/gcp-deployment/.env.example new file mode 100644 index 00000000..947352fe --- /dev/null +++ b/gcp-deployment/.env.example @@ -0,0 +1,93 @@ +# Alex (GCP) - Environment Variables +# Copy to .env and fill with your deployment values + +# ============================================================ +# Core GCP project settings +# ============================================================ +PROJECT_ID=alex-multi-agent-saas-xxxxxx +GCP_PROJECT_ID=alex-multi-agent-saas-xxxxxx +GCP_REGION=us-central1 + +# ============================================================ +# LLM configuration (Gemini 2.0 Flash by default) +# ============================================================ +LLM_PROVIDER=vertex_ai # vertex_ai | openai +VERTEX_AI_MODEL=vertex_ai/gemini-2.0-flash-exp +OPENAI_MODEL=openai/gpt-4o-mini +OPENAI_API_KEY= +OPENAI_API_BASE= + +# Optional per-agent overrides (leave blank to use defaults) +PLANNER_MODEL= +REPORTER_MODEL= +REPORTER_JUDGE_MODEL= +RETIREMENT_MODEL= +CHARTER_MODEL= +TAGGER_MODEL= +RESEARCHER_MODEL= + +# ============================================================ +# Vertex AI training artifacts (Guide 2 outputs) +# ============================================================ +MODEL_ARTIFACTS_BUCKET= +TRAINING_DATA_BUCKET= +TENSORBOARD_NAME= +ANTHROPIC_API_KEY_SECRET_ID= +OPENAI_API_KEY_SECRET_ID= + +# ============================================================ +# Ingestion / vectors (Guide 3) +# ============================================================ +VECTOR_BUCKET= +ALEX_API_ENDPOINT= +ALEX_API_KEY= + +# ============================================================ +# Researcher service (Guide 4) +# ============================================================ +POLYGON_API_KEY= +POLYGON_PLAN=free + +# ============================================================ +# Database (Guide 5 - Cloud SQL) +# ============================================================ +DATABASE_NAME=alex +DATABASE_USER=alex_app +INSTANCE_CONNECTION_NAME= +DB_PASSWORD_SECRET_ID= +DB_CONNECTION_STRING_SECRET_ID= +PRIVATE_IP_ADDRESS= +VPC_NETWORK_ID= +VPC_SUBNET_ID= + +# ============================================================ +# Agents (Guide 6) +# ============================================================ +TAGGER_FUNCTION=alex-tagger +REPORTER_FUNCTION=alex-reporter +CHARTER_FUNCTION=alex-charter +RETIREMENT_FUNCTION=alex-retirement +MOCK_LAMBDAS=false +CLOUD_RUN_SERVICE_ACCOUNT_EMAIL= +SQS_QUEUE_URL= + +# ============================================================ +# Frontend & API (Guide 7) +# ============================================================ +FRONTEND_URL=http://localhost:3000 +CLOUDFRONT_URL= +CLERK_JWKS_URL= +CLERK_ISSUER= + +# ============================================================ +# Observability / misc +# ============================================================ +LANGFUSE_PUBLIC_KEY= +LANGFUSE_SECRET_KEY= +LANGFUSE_HOST=https://cloud.langfuse.com + +# ============================================================ +# Legacy AWS compatibility (remove once not needed) +# ============================================================ +DEFAULT_AWS_REGION=us-east-1 +SAGEMAKER_ENDPOINT= diff --git a/gcp-deployment/.gitignore b/gcp-deployment/.gitignore new file mode 100644 index 00000000..ac16b027 --- /dev/null +++ b/gcp-deployment/.gitignore @@ -0,0 +1,71 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +.venv +*.egg-info/ +dist/ +build/ +*.egg +# UV +uv.lock + +# Environment variables +.env +.env.local +.env.*.local + +# Terraform +*.tfstate +*.tfstate.* +.terraform/ +.terraform.lock.hcl +terraform.tfvars +*.tfvars +!*.tfvars.example + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +logs/ + +# Secrets and credentials +*.json +!package.json +!tsconfig.json +!eslint.config.json +*-terraform-key.json +application_default_credentials.json +service-account-*.json + +# Node +node_modules/ +.next/ +out/ + +# Docker +*.dockerignore + +# Temporary files +*.tmp +*.bak +*.backup + +# Local development +Local Postgresql Kill and Start Commands.txt + diff --git a/gcp-deployment/CHANGELOG.md b/gcp-deployment/CHANGELOG.md new file mode 100644 index 00000000..e3dcfe26 --- /dev/null +++ b/gcp-deployment/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +## [Unreleased] + +### Fixed +- **Missing pg8000 dependency in agent containers**: Fixed `ModuleNotFoundError: No module named 'pg8000'` in Reporter, Charter, and Retirement agents by explicitly installing the database package with all dependencies before syncing main project dependencies. See [guides/FIX_PG8000_DEPENDENCY.md](guides/FIX_PG8000_DEPENDENCY.md) for details. + +### Changed +- Updated Dockerfiles for Reporter, Charter, and Retirement agents to explicitly install database package dependencies +- Added troubleshooting section for pg8000 dependency issue + +### Files Modified +- `backend/reporter/Dockerfile` +- `backend/charter/Dockerfile` +- `backend/retirement/Dockerfile` +- `guides/TROUBLESHOOTING.md` +- `guides/FIX_PG8000_DEPENDENCY.md` (new) + diff --git a/gcp-deployment/FIXES_APPLIED.md b/gcp-deployment/FIXES_APPLIED.md new file mode 100644 index 00000000..4406f7d6 --- /dev/null +++ b/gcp-deployment/FIXES_APPLIED.md @@ -0,0 +1,49 @@ +# Fixes Applied to GCP Deployment + +This document summarizes the fixes applied to resolve issues encountered during deployment. + +## Fix: Missing pg8000 Dependency + +### Issue +Reporter, Charter, and Retirement agents were failing with: +``` +ModuleNotFoundError: No module named 'pg8000' +``` + +This prevented agents from connecting to the Cloud SQL database, resulting in 500 Internal Server Error responses. + +### Root Cause +The `pg8000` package is a transitive dependency of the `alex-database` package (required by Cloud SQL connector). When using `uv sync --no-install-project` with local path dependencies, transitive dependencies may not be installed correctly. + +### Solution +Updated Dockerfiles to explicitly install the database package with all dependencies before syncing main project dependencies: + +```dockerfile +# First install database package with all its dependencies (including pg8000) +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project +``` + +### Files Modified +- `backend/reporter/Dockerfile` +- `backend/charter/Dockerfile` +- `backend/retirement/Dockerfile` + +### Documentation Added +- `guides/FIX_PG8000_DEPENDENCY.md` - Detailed fix documentation +- Updated `guides/TROUBLESHOOTING.md` - Added troubleshooting section +- Updated `guides/6_agents.md` - Added Dockerfile example with fix +- Updated `README.md` - Added reference to fix documentation + +### Verification +After applying the fix: +1. Rebuild Docker images +2. Push to Artifact Registry +3. Update Cloud Run services +4. Verify no more `pg8000` errors in logs +5. Test agent workflow end-to-end + +### Status +โœ… **Fixed and Verified** - All three agents now work correctly and can connect to the database. + diff --git a/gcp-deployment/README.md b/gcp-deployment/README.md new file mode 100644 index 00000000..917054e2 --- /dev/null +++ b/gcp-deployment/README.md @@ -0,0 +1,390 @@ +# Alex Multi-Agent SaaS - GCP Deployment + +This repository contains the GCP deployment configuration for the Alex Multi-Agent SaaS application, translated from the original AWS deployment. + +## ๐Ÿ“‹ Prerequisites + +1. **GCP Account** with billing enabled +2. **gcloud CLI** installed and authenticated +3. **Terraform** >= 1.5.0 +4. **Docker** installed +5. **Git** installed + +## ๐Ÿš€ Quick Start + +### 1. Clone and Configure + +```bash +# Set your project ID +export PROJECT_ID="your-gcp-project-id" +export REGION="us-central1" + +# Authenticate with GCP +gcloud auth login +gcloud config set project $PROJECT_ID +``` + +### 2. Enable APIs + +```bash +gcloud services enable \ + compute.googleapis.com \ + run.googleapis.com \ + cloudfunctions.googleapis.com \ + sqladmin.googleapis.com \ + aiplatform.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iam.googleapis.com \ + storage.googleapis.com \ + servicenetworking.googleapis.com \ + cloudbuild.googleapis.com +``` + +### 3. Deploy + +```bash +# Make deploy script executable +chmod +x scripts/deploy.sh + +# Run full deployment +./scripts/deploy.sh full +``` + +## ๐Ÿ“ Project Structure + +``` +gcp-deployment/ +โ”œโ”€โ”€ README.md # This file +โ”œโ”€โ”€ guides/ +โ”‚ โ”œโ”€โ”€ 0_AWS_TO_GCP_MAPPING.md # AWS to GCP service mapping +โ”‚ โ”œโ”€โ”€ 1_permissions.md # IAM setup guide +โ”‚ โ”œโ”€โ”€ 2_vertex_ai.md # Vertex AI setup guide +โ”‚ โ”œโ”€โ”€ 5_database.md # Cloud SQL setup guide +โ”‚ โ”œโ”€โ”€ 6_agents.md # Agent deployment guide +โ”‚ โ”œโ”€โ”€ 7_frontend.md # Frontend deployment guide +โ”‚ โ”œโ”€โ”€ ARCHITECTURE_COMPARISON.md # Detailed architecture comparison +โ”‚ โ”œโ”€โ”€ TROUBLESHOOTING.md # Combined troubleshooting guide +โ”‚ โ”œโ”€โ”€ GEMINI_SETUP.md # Gemini model setup guide +โ”‚ โ””โ”€โ”€ WINDOWS_SETUP.md # Windows-specific setup +โ”œโ”€โ”€ terraform/ +โ”‚ โ”œโ”€โ”€ 1_permissions/ # IAM and service accounts +โ”‚ โ”œโ”€โ”€ 2_vertex_ai/ # Vertex AI configuration +โ”‚ โ”œโ”€โ”€ 3_pubsub/ # Pub/Sub topic and subscription +โ”‚ โ”œโ”€โ”€ 5_database/ # Cloud SQL PostgreSQL +โ”‚ โ”œโ”€โ”€ 6_agents/ # Cloud Run services (agents) +โ”‚ โ””โ”€โ”€ 7_frontend/ # Frontend with Cloud CDN +โ”œโ”€โ”€ backend/ # Agent code and API +โ”‚ โ”œโ”€โ”€ planner/ # Orchestrator agent +โ”‚ โ”œโ”€โ”€ tagger/ # Instrument classification +โ”‚ โ”œโ”€โ”€ reporter/ # Portfolio analysis +โ”‚ โ”œโ”€โ”€ charter/ # Visualization agent +โ”‚ โ”œโ”€โ”€ retirement/ # Retirement projection +โ”‚ โ”œโ”€โ”€ researcher/ # Market research agent +โ”‚ โ”œโ”€โ”€ api/ # FastAPI backend +โ”‚ โ”œโ”€โ”€ database/ # Shared database library +โ”‚ โ””โ”€โ”€ common/ # Shared utilities (LLM config) +โ”œโ”€โ”€ frontend/ # NextJS React application +โ””โ”€โ”€ scripts/ # Deployment scripts +``` + +## ๐Ÿ”„ AWS to GCP Service Mapping + +This deployment uses GCP-native services as alternatives to the AWS services used in the original course. Below is a detailed comparison of the resources used: + +| AWS Service | GCP Equivalent | Key Differences | +|-------------|----------------|------------------| +| **IAM** | **Cloud IAM** | Uses service accounts instead of IAM roles. Workload Identity Federation for CI/CD. | +| **SageMaker** | **Vertex AI** | Vertex AI provides embeddings via API (no endpoint deployment needed). Model Garden for foundation models. | +| **Bedrock** | **Vertex AI Model Garden** | Access to Gemini 2.0 Flash (recommended, cost-effective) and Claude via Anthropic API. No inference profiles needed. | +| **Lambda** | **Cloud Run** | Container-based serverless. Better for multi-file applications. Scales to zero automatically. | +| **App Runner** | **Cloud Run** | Same service as Lambda alternative. Supports any containerized application. | +| **ECR** | **Artifact Registry** | Multi-format registry (Docker, npm, Python, Maven). Better integration with Cloud Build. | +| **RDS Aurora Serverless v2** | **Cloud SQL PostgreSQL** | Managed PostgreSQL with automatic backups. Uses Cloud SQL Proxy or Unix socket for connections. No Data API needed. | +| **S3** | **Cloud Storage** | Object storage with similar API. Used for static frontend hosting. | +| **S3 Vectors** | **Vertex AI Vector Search** (TODO) | Vector search functionality needs GCP implementation. Currently placeholder in code. | +| **API Gateway** | **Cloud Run** (built-in) | Cloud Run provides HTTPS endpoints automatically. No separate API Gateway needed. | +| **SQS** | **Cloud Pub/Sub** | Message queuing with push/pull subscriptions. Better integration with Cloud Run. | +| **CloudFront** | **Cloud CDN** | Content delivery network. Optional for frontend deployment. | +| **Route 53** | **Cloud DNS** | DNS management. Optional for custom domains. | +| **Secrets Manager** | **Secret Manager** | Similar functionality. Integrated with Cloud Run via environment variables. | +| **CloudWatch** | **Cloud Monitoring & Logging** | Unified observability platform. Better integration with GCP services. | + +### Key Architectural Differences + +#### 1. **Serverless Compute** +- **AWS**: Lambda functions (zip-based, 50MB limit, requires packaging) +- **GCP**: Cloud Run (container-based, no size limit, easier deployment) +- **Benefit**: No need for `package_docker.py` scripts. Direct container deployment. + +#### 2. **AI/ML Services** +- **AWS**: Bedrock (requires model access requests, inference profiles for cross-region) +- **GCP**: Vertex AI Model Garden (Gemini 2.0 Flash recommended, no API keys needed with ADC) +- **Benefit**: Lower costs with Gemini, native GCP integration, simpler setup. + +#### 3. **Database** +- **AWS**: Aurora Serverless v2 with Data API (HTTP-based, no VPC needed) +- **GCP**: Cloud SQL PostgreSQL with Unix socket or Cloud SQL Proxy +- **Benefit**: Standard PostgreSQL connections, better performance, automatic backups. + +#### 4. **Message Queuing** +- **AWS**: SQS (simple queue service) +- **GCP**: Pub/Sub (publish/subscribe with topics and subscriptions) +- **Benefit**: Better integration with Cloud Run, push subscriptions available. + +#### 5. **Vector Storage** (Future Implementation) +- **AWS**: S3 Vectors (90% cost savings vs OpenSearch) +- **GCP**: Vertex AI Vector Search or Matching Engine (to be implemented) +- **Status**: Currently placeholder in code. See `backend/reporter/agent.py` for TODO. + +#### 6. **Container Registry** +- **AWS**: ECR (Elastic Container Registry) +- **GCP**: Artifact Registry (multi-format, better CI/CD integration) +- **Benefit**: Supports Docker, npm, Python packages in one registry. + +### Cost Comparison + +| Service Category | AWS | GCP | Notes | +|-----------------|-----|-----|-------| +| **Serverless Compute** | Lambda: $0.20 per 1M requests | Cloud Run: Pay per request, scales to zero | GCP often cheaper for low traffic | +| **Database** | Aurora: ~$50-100/month | Cloud SQL: ~$30-100/month | Similar pricing, GCP slightly cheaper | +| **AI/ML** | Bedrock: Varies by model | Vertex AI: Gemini 2.0 Flash is cost-effective | Gemini significantly cheaper than Claude | +| **Container Registry** | ECR: $0.10/GB/month | Artifact Registry: $0.10/GB/month | Similar pricing | +| **Message Queuing** | SQS: $0.40 per 1M requests | Pub/Sub: $0.40 per 1M requests | Similar pricing | + +**Recommendation**: Use Gemini 2.0 Flash for most tasks to reduce AI costs significantly compared to Claude models. + +## ๐Ÿ“– Deployment Phases + +### Phase 1: Permissions +Sets up service accounts and IAM bindings. + +```bash +cd terraform/1_permissions +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your values +terraform init && terraform apply +``` + +### Phase 2: Vertex AI +Configures Vertex AI for ML/LLM access. + +```bash +cd terraform/2_vertex_ai +terraform init && terraform apply +``` + +### Phase 5: Database +Deploys Cloud SQL PostgreSQL. + +```bash +cd terraform/5_database +terraform init && terraform apply +``` + +### Phase 6: Agents +Deploys multi-agent backend on Cloud Run. + +```bash +cd terraform/6_agents +terraform init && terraform apply +``` + +### Phase 7: Frontend +Deploys NextJS frontend with optional Cloud CDN. + +```bash +cd terraform/7_frontend +terraform init && terraform apply +``` + +## ๐Ÿ” Setting Up Secrets + +Store your API keys in Secret Manager: + +```bash +# Set your project ID +export PROJECT_ID="your-gcp-project-id" + +# OpenAI API key (optional, if using OpenAI alongside Gemini) +echo -n "sk-proj-xxx" | gcloud secrets create openai-api-key --data-file=- --project=$PROJECT_ID + +# Clerk keys (required for frontend authentication) +echo -n "pk_live_xxx" | gcloud secrets create clerk-publishable-key --data-file=- --project=$PROJECT_ID +echo -n "sk_live_xxx" | gcloud secrets create clerk-secret-key --data-file=- --project=$PROJECT_ID + +# Database password (created automatically by terraform/5_database) +# You can update it manually if needed: +echo -n "your-db-password" | gcloud secrets versions add alex-db-password --data-file=- --project=$PROJECT_ID +``` + +**Note**: Gemini 2.0 Flash (recommended) doesn't require API keys - it uses Application Default Credentials (ADC) which are automatically configured when you run `gcloud auth application-default login`. + +## ๐ŸŒ Using AI Models on GCP + +### Recommended: Gemini 2.0 Flash (Cost-Effective) + +Gemini 2.0 Flash is the recommended model for this deployment. It's significantly more cost-effective than Claude and provides excellent performance. + +```python +from agents import Agent, Runner +from litellm import LitellmModel +import os + +# Set GCP project and region +os.environ["GCP_PROJECT_ID"] = "your-gcp-project-id" +os.environ["GCP_REGION"] = "us-central1" + +# Create model (uses Application Default Credentials, no API key needed) +model = LitellmModel(model="vertex_ai/gemini-2.0-flash-exp") + +# Use with Agent +agent = Agent( + name="My Agent", + instructions="You are a helpful assistant.", + model=model +) + +result = await Runner.run(agent, input="Hello!") +``` + +### Alternative: OpenAI API (via Secret Manager) + +If you need OpenAI models (GPT-4o, GPT-4o-mini), store the API key in Secret Manager: + +```python +from google.cloud import secretmanager +import os + +def get_secret(secret_id: str) -> str: + client = secretmanager.SecretManagerServiceClient() + project_id = os.getenv("GCP_PROJECT_ID") + name = f"projects/{project_id}/secrets/{secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + return response.payload.data.decode("UTF-8") + +# Get API key from Secret Manager +api_key = get_secret("openai-api-key") + +# Use with LiteLLM +from litellm import LitellmModel +model = LitellmModel(model="openai/gpt-4o-mini", api_key=api_key) +``` + +### Optional: Claude via Anthropic API + +Claude is available but more expensive. Store API key in Secret Manager if needed. + +See [guides/GEMINI_SETUP.md](guides/GEMINI_SETUP.md) for detailed setup instructions. + +## ๐Ÿ’ฐ Cost Estimation + +| Service | Estimated Monthly Cost (Dev) | +|---------|------------------------------| +| Cloud Run | $20-50 (scale to zero) | +| Cloud SQL | $30-100 (db-custom-2-4096) | +| Artifact Registry | $5-10 | +| Cloud Storage | $5-10 | +| Vertex AI | Pay-per-use | +| Cloud CDN | Pay-per-GB | + +## ๐Ÿงน Cleanup + +### Using the Destroy Script (Recommended) + +The easiest way to destroy resources is using the PowerShell destroy script: + +```powershell +# Destroy everything (with confirmation) +.\scripts\destroy.ps1 + +# Destroy everything including secrets (no confirmation) +.\scripts\destroy.ps1 -DestroySecrets -SkipConfirmation + +# Destroy only database (biggest cost savings) +.\scripts\destroy.ps1 -DestroyDatabase -DestroyAll:$false + +# Destroy specific phases +.\scripts\destroy.ps1 -DestroyFrontend -DestroyAgents -DestroyAll:$false +``` + +See [scripts/DESTROY_README.md](scripts/DESTROY_README.md) for full documentation. + +### Manual Cleanup + +Or destroy manually in reverse order: + +```bash +for phase in 7_frontend 6_agents 5_database 3_pubsub 2_vertex_ai 1_permissions; do + cd terraform/$phase + terraform destroy -auto-approve + cd ../.. +done +``` + +**Note**: Destroy in reverse order (7 โ†’ 1) to handle dependencies correctly. + +## โ“ Troubleshooting + +For detailed troubleshooting information, see [guides/TROUBLESHOOTING.md](guides/TROUBLESHOOTING.md). + +### Common Issues + +1. **API not enabled**: Run the API enable command above +2. **Permission denied**: Check IAM bindings in `terraform/1_permissions` +3. **Quota exceeded**: Request quota increase in GCP Console +4. **Environment variables not loading**: Check `.env` file in root directory +5. **Pub/Sub errors**: Verify topic exists and service account has permissions +6. **Missing pg8000 dependency**: See [guides/FIX_PG8000_DEPENDENCY.md](guides/FIX_PG8000_DEPENDENCY.md) - Fixed in Dockerfiles + +### Useful Commands + +```bash +# Check Cloud Run services +gcloud run services list --project=your-gcp-project-id + +# View logs +gcloud logging read "resource.type=cloud_run_revision" --limit=50 --project=your-gcp-project-id + +# Check database status +gcloud sql instances describe alex-postgres --project=your-gcp-project-id + +# List Pub/Sub topics +gcloud pubsub topics list --project=your-gcp-project-id + +# Check service accounts +gcloud iam service-accounts list --project=your-gcp-project-id +``` + +## ๐Ÿ“š Additional Resources + +### GCP Documentation +- [GCP Cloud Run Documentation](https://cloud.google.com/run/docs) +- [Vertex AI Documentation](https://cloud.google.com/vertex-ai/docs) +- [Cloud SQL Documentation](https://cloud.google.com/sql/docs) +- [Cloud Pub/Sub Documentation](https://cloud.google.com/pubsub/docs) +- [Secret Manager Documentation](https://cloud.google.com/secret-manager/docs) + +### Terraform +- [Terraform GCP Provider](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +### AI/ML +- [Vertex AI Model Garden](https://cloud.google.com/vertex-ai/docs/model-garden/overview) +- [Gemini API Documentation](https://ai.google.dev/docs) +- [Anthropic on Vertex AI](https://docs.anthropic.com/en/api/vertex) + +### Guides in This Repository +- [AWS to GCP Mapping](guides/0_AWS_TO_GCP_MAPPING.md) - Service comparison +- [Architecture Comparison](guides/ARCHITECTURE_COMPARISON.md) - Detailed architecture differences +- [Troubleshooting Guide](guides/TROUBLESHOOTING.md) - Common issues and solutions +- [Gemini Setup](guides/GEMINI_SETUP.md) - Configuring Gemini models +- [Fix: pg8000 Dependency](guides/FIX_PG8000_DEPENDENCY.md) - Solution for missing database dependencies in agent containers + +## ๐Ÿค Contributing + +Contributions are welcome! Please submit a PR with your GCP deployment improvements. + +## ๐Ÿ“ License + +This project is licensed under the MIT License - see the original course repository for details. diff --git a/gcp-deployment/assets/alex.png b/gcp-deployment/assets/alex.png new file mode 100644 index 00000000..323a66df Binary files /dev/null and b/gcp-deployment/assets/alex.png differ diff --git a/gcp-deployment/backend/.python-version b/gcp-deployment/backend/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/api/.python-version b/gcp-deployment/backend/api/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/api/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/api/Dockerfile b/gcp-deployment/backend/api/Dockerfile new file mode 100644 index 00000000..49673a5c --- /dev/null +++ b/gcp-deployment/backend/api/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Install uv +RUN pip install uv + +# Copy shared modules (from backend/ directory) +COPY database ./database +COPY common ./common + +# Copy API dependencies +COPY api/pyproject.toml ./ + +# Fix path in pyproject.toml for alex-database (change from workspace to path dependency) +RUN sed -i.bak 's|workspace = true|path = "./database"|g' pyproject.toml && \ + sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && \ + rm pyproject.toml.bak || true + +# Install dependencies +RUN uv sync + +# Copy API code +COPY api/*.py ./ + +# Expose port +EXPOSE 8080 + +# Run server +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8080"] + diff --git a/gcp-deployment/backend/api/lambda_handler.py b/gcp-deployment/backend/api/lambda_handler.py new file mode 100644 index 00000000..4e8ecdfe --- /dev/null +++ b/gcp-deployment/backend/api/lambda_handler.py @@ -0,0 +1,8 @@ +"""Lambda handler for the FastAPI application.""" + +from mangum import Mangum +from api.main import app + +# Create the Lambda handler +# API Gateway passes the full path including /api/ prefix +handler = Mangum(app, lifespan="off") \ No newline at end of file diff --git a/gcp-deployment/backend/api/main.py b/gcp-deployment/backend/api/main.py new file mode 100644 index 00000000..2d1851c0 --- /dev/null +++ b/gcp-deployment/backend/api/main.py @@ -0,0 +1,809 @@ +""" +FastAPI backend for Alex Financial Advisor +Handles all API routes with Clerk JWT authentication +""" + +import os +import json +import logging +from typing import Optional, List, Dict, Any +from datetime import datetime +from decimal import Decimal +import uuid + +from fastapi import FastAPI, HTTPException, Depends, status, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field, ValidationError +from mangum import Mangum +from dotenv import load_dotenv +from pathlib import Path +from fastapi_clerk_auth import ClerkConfig, ClerkHTTPBearer, HTTPAuthorizationCredentials + +from src import Database +from src.schemas import ( + UserCreate, + AccountCreate, + PositionCreate, + JobCreate, JobUpdate, + JobType, JobStatus +) + +# Load .env file from project root (alex-gcp/.env) +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize FastAPI app +app = FastAPI( + title="Alex Financial Advisor API", + description="Backend API for AI-powered financial planning", + version="1.0.0" +) + +# CORS configuration +# Get frontend URL and additional origins +frontend_url = os.getenv("FRONTEND_URL", "") +cors_origins = [] +if frontend_url: + cors_origins.append(frontend_url) +# Add additional origins from env var +additional_origins = os.getenv("CORS_ORIGINS", "").split(",") +cors_origins.extend([origin.strip() for origin in additional_origins if origin.strip()]) +# Fallback to localhost for development +if not cors_origins: + cors_origins = ["http://localhost:3000"] +app.add_middleware( + CORSMiddleware, + allow_origins=cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Custom exception handlers for better error messages +@app.exception_handler(ValidationError) +async def validation_exception_handler(request: Request, exc: ValidationError): + """Handle Pydantic validation errors with user-friendly messages""" + return JSONResponse( + status_code=422, + content={"detail": "Invalid input data. Please check your request and try again."} + ) + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + """Handle HTTP exceptions with improved messages""" + # Map technical errors to user-friendly messages + user_friendly_messages = { + 401: "Your session has expired. Please sign in again.", + 403: "You don't have permission to access this resource.", + 404: "The requested resource was not found.", + 429: "Too many requests. Please slow down and try again later.", + 500: "An internal error occurred. Please try again later.", + 503: "The service is temporarily unavailable. Please try again later." + } + + message = user_friendly_messages.get(exc.status_code, exc.detail) + return JSONResponse( + status_code=exc.status_code, + content={"detail": message} + ) + +@app.exception_handler(Exception) +async def general_exception_handler(request: Request, exc: Exception): + """Handle unexpected errors gracefully""" + logger.error(f"Unexpected error: {exc}", exc_info=True) + return JSONResponse( + status_code=500, + content={"detail": "An unexpected error occurred. Our team has been notified."} + ) + +# Initialize services +db = Database() + +# Pub/Sub client for job queueing +from google.cloud import pubsub_v1 + +publisher = pubsub_v1.PublisherClient() + +# Get project ID from environment or gcloud config +PROJECT_ID = os.getenv('GCP_PROJECT_ID') or os.getenv('PROJECT_ID') +if not PROJECT_ID: + # Try to get from gcloud config as fallback + try: + import subprocess + result = subprocess.run( + ['gcloud', 'config', 'get-value', 'project'], + capture_output=True, + text=True, + check=True, + timeout=5 + ) + PROJECT_ID = result.stdout.strip() + logger.info(f"Using gcloud default project: {PROJECT_ID}") + except Exception: + logger.warning("GCP_PROJECT_ID not set and could not get from gcloud config") + +PUBSUB_TOPIC = os.getenv('PUBSUB_TOPIC', 'alex-job-queue') + +# Clerk authentication setup (exactly like saas reference) +clerk_config = ClerkConfig(jwks_url=os.getenv("CLERK_JWKS_URL")) +clerk_guard = ClerkHTTPBearer(clerk_config) + +async def get_current_user_id(creds: HTTPAuthorizationCredentials = Depends(clerk_guard)) -> str: + """Extract user ID from validated Clerk token""" + # The clerk_guard dependency already validated the token + # creds.decoded contains the JWT payload + user_id = creds.decoded["sub"] + logger.info(f"Authenticated user: {user_id}") + return user_id + +# Request/Response models +class UserResponse(BaseModel): + user: Dict[str, Any] + created: bool + +class UserUpdate(BaseModel): + """Update user settings""" + display_name: Optional[str] = None + years_until_retirement: Optional[int] = None + target_retirement_income: Optional[float] = None + asset_class_targets: Optional[Dict[str, float]] = None + region_targets: Optional[Dict[str, float]] = None + +class AccountUpdate(BaseModel): + """Update account""" + account_name: Optional[str] = None + account_purpose: Optional[str] = None + cash_balance: Optional[float] = None + +class PositionUpdate(BaseModel): + """Update position""" + quantity: Optional[float] = None + +class AnalyzeRequest(BaseModel): + analysis_type: str = Field(default="portfolio", description="Type of analysis to perform") + options: Dict[str, Any] = Field(default_factory=dict, description="Analysis options") + +class AnalyzeResponse(BaseModel): + job_id: str + message: str + +# API Routes + +@app.get("/health") +async def health_check(): + """Health check endpoint""" + return {"status": "healthy", "timestamp": datetime.now().isoformat()} + +@app.get("/api/user", response_model=UserResponse) +async def get_or_create_user( + clerk_user_id: str = Depends(get_current_user_id), + creds: HTTPAuthorizationCredentials = Depends(clerk_guard) +): + """Get user or create if first time""" + + try: + # Check if user exists + user = db.users.find_by_clerk_id(clerk_user_id) + + if user: + return UserResponse(user=user, created=False) + + # Create new user with defaults from JWT token + token_data = creds.decoded + display_name = token_data.get('name') or token_data.get('email', '').split('@')[0] or "New User" + + # Create user with ALL defaults in one operation + user_data = { + 'clerk_user_id': clerk_user_id, + 'display_name': display_name, + 'years_until_retirement': 20, + 'target_retirement_income': 60000, + 'asset_class_targets': {"equity": 70, "fixed_income": 30}, + 'region_targets': {"north_america": 50, "international": 50} + } + + # Insert directly with all data + created_clerk_id = db.users.db.insert('users', user_data, returning='clerk_user_id') + + # Fetch the created user + created_user = db.users.find_by_clerk_id(clerk_user_id) + logger.info(f"Created new user: {clerk_user_id}") + + return UserResponse(user=created_user, created=True) + + except Exception as e: + logger.error(f"Error in get_or_create_user: {e}") + raise HTTPException(status_code=500, detail="Failed to load user profile") + +@app.put("/api/user") +async def update_user(user_update: UserUpdate, clerk_user_id: str = Depends(get_current_user_id)): + """Update user settings""" + + try: + # Get user + user = db.users.find_by_clerk_id(clerk_user_id) + + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Update user - users table uses clerk_user_id as primary key + update_data = user_update.model_dump(exclude_unset=True) + + # Use the database client directly since users table has clerk_user_id as PK + db.users.db.update( + 'users', + update_data, + "clerk_user_id = :clerk_user_id", + {'clerk_user_id': clerk_user_id} + ) + + # Return updated user + updated_user = db.users.find_by_clerk_id(clerk_user_id) + return updated_user + + except Exception as e: + logger.error(f"Error updating user: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/api/accounts") +async def list_accounts(clerk_user_id: str = Depends(get_current_user_id)): + """List user's accounts""" + + try: + # Get accounts for user + accounts = db.accounts.find_by_user(clerk_user_id) + return accounts + + except Exception as e: + logger.error(f"Error listing accounts: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.post("/api/accounts") +async def create_account(account: AccountCreate, clerk_user_id: str = Depends(get_current_user_id)): + """Create new account""" + + try: + # Verify user exists + user = db.users.find_by_clerk_id(clerk_user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Create account + account_id = db.accounts.create_account( + clerk_user_id=clerk_user_id, + account_name=account.account_name, + account_purpose=account.account_purpose, + cash_balance=getattr(account, 'cash_balance', Decimal('0')) + ) + + # Return created account + created_account = db.accounts.find_by_id(account_id) + return created_account + + except Exception as e: + logger.error(f"Error creating account: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.put("/api/accounts/{account_id}") +async def update_account(account_id: str, account_update: AccountUpdate, clerk_user_id: str = Depends(get_current_user_id)): + """Update account""" + + try: + # Verify account belongs to user + account = db.accounts.find_by_id(account_id) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + + # Verify ownership - accounts table stores clerk_user_id directly + if account.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + # Update account + update_data = account_update.model_dump(exclude_unset=True) + db.accounts.update(account_id, update_data) + + # Return updated account + updated_account = db.accounts.find_by_id(account_id) + return updated_account + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating account: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.delete("/api/accounts/{account_id}") +async def delete_account(account_id: str, clerk_user_id: str = Depends(get_current_user_id)): + """Delete an account and all its positions""" + + try: + # Verify account belongs to user + account = db.accounts.find_by_id(account_id) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + + # Verify ownership - accounts table stores clerk_user_id directly + if account.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + # Delete all positions first (due to foreign key constraint) + positions = db.positions.find_by_account(account_id) + for position in positions: + db.positions.delete(position['id']) + + # Delete the account + db.accounts.delete(account_id) + + return {"message": "Account deleted successfully"} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting account: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/api/accounts/{account_id}/positions") +async def list_positions(account_id: str, clerk_user_id: str = Depends(get_current_user_id)): + """Get positions for account""" + + try: + # Verify account belongs to user + account = db.accounts.find_by_id(account_id) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + + # Verify ownership - accounts table stores clerk_user_id directly + if account.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + positions = db.positions.find_by_account(account_id) + + # Format positions with instrument data for frontend + formatted_positions = [] + for pos in positions: + # Get full instrument data + instrument = db.instruments.find_by_symbol(pos['symbol']) + formatted_positions.append({ + **pos, + 'instrument': instrument + }) + + return {"positions": formatted_positions} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing positions: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.post("/api/positions") +async def create_position(position: PositionCreate, clerk_user_id: str = Depends(get_current_user_id)): + """Create position""" + + try: + # Verify account belongs to user + account = db.accounts.find_by_id(position.account_id) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + + # Verify ownership - accounts table stores clerk_user_id directly + if account.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + # Check if instrument exists, if not create it + instrument = db.instruments.find_by_symbol(position.symbol.upper()) + if not instrument: + logger.info(f"Creating new instrument: {position.symbol.upper()}") + # Create a basic instrument entry with default allocations + # Import the schema from database + from src.schemas import InstrumentCreate + + # Determine type based on common patterns + symbol_upper = position.symbol.upper() + if len(symbol_upper) <= 5 and symbol_upper.isalpha(): + instrument_type = "stock" + else: + instrument_type = "etf" + + # Create instrument with basic default allocations + # These can be updated later by the tagger agent + new_instrument = InstrumentCreate( + symbol=symbol_upper, + name=f"{symbol_upper} - User Added", # Basic name, can be updated later + instrument_type=instrument_type, + current_price=Decimal("0.00"), # Price will be updated by background processes + allocation_regions={"north_america": 100.0}, # Default to 100% NA + allocation_sectors={"other": 100.0}, # Default to 100% other + allocation_asset_class={"equity": 100.0} if instrument_type == "stock" else {"fixed_income": 100.0} + ) + + db.instruments.create_instrument(new_instrument) + + # Add position + position_id = db.positions.add_position( + account_id=position.account_id, + symbol=position.symbol.upper(), + quantity=position.quantity + ) + + # Return created position + created_position = db.positions.find_by_id(position_id) + return created_position + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating position: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.put("/api/positions/{position_id}") +async def update_position(position_id: str, position_update: PositionUpdate, clerk_user_id: str = Depends(get_current_user_id)): + """Update position""" + + try: + # Get position and verify ownership + position = db.positions.find_by_id(position_id) + if not position: + raise HTTPException(status_code=404, detail="Position not found") + + account = db.accounts.find_by_id(position['account_id']) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + + # Verify ownership - accounts table stores clerk_user_id directly + if account.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + # Update position + update_data = position_update.model_dump(exclude_unset=True) + db.positions.update(position_id, update_data) + + # Return updated position + updated_position = db.positions.find_by_id(position_id) + return updated_position + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating position: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.delete("/api/positions/{position_id}") +async def delete_position(position_id: str, clerk_user_id: str = Depends(get_current_user_id)): + """Delete position""" + + try: + # Get position and verify ownership + position = db.positions.find_by_id(position_id) + if not position: + raise HTTPException(status_code=404, detail="Position not found") + + account = db.accounts.find_by_id(position['account_id']) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + + # Verify ownership - accounts table stores clerk_user_id directly + if account.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + db.positions.delete(position_id) + return {"message": "Position deleted"} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting position: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/api/instruments") +async def list_instruments(clerk_user_id: str = Depends(get_current_user_id)): + """Get all available instruments for autocomplete""" + + try: + instruments = db.instruments.find_all() + # Return simplified list for autocomplete + return [ + { + "symbol": inst["symbol"], + "name": inst["name"], + "instrument_type": inst["instrument_type"], + "current_price": float(inst["current_price"]) if inst.get("current_price") else None + } + for inst in instruments + ] + except Exception as e: + logger.error(f"Error fetching instruments: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.post("/api/analyze", response_model=AnalyzeResponse) +async def trigger_analysis(request: AnalyzeRequest, clerk_user_id: str = Depends(get_current_user_id)): + """Trigger portfolio analysis""" + + try: + # Get user + user = db.users.find_by_clerk_id(clerk_user_id) + + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Create job + job_id = db.jobs.create_job( + clerk_user_id=clerk_user_id, + job_type="portfolio_analysis", + request_payload=request.model_dump() + ) + + # Get the created job + job = db.jobs.find_by_id(job_id) + + # Send to Pub/Sub + if PROJECT_ID and PUBSUB_TOPIC: + message_data = { + 'job_id': str(job_id), + 'clerk_user_id': clerk_user_id, + 'analysis_type': request.analysis_type, + 'options': request.options + } + + try: + topic_path = publisher.topic_path(PROJECT_ID, PUBSUB_TOPIC) + future = publisher.publish( + topic_path, + json.dumps(message_data).encode('utf-8') + ) + message_id = future.result() # Wait for publish to complete + logger.info(f"Sent analysis job to Pub/Sub: {job_id} (message_id: {message_id})") + except Exception as e: + logger.error(f"Error publishing to Pub/Sub: {e}") + # Don't fail the request, job is already created in DB + logger.warning(f"Job {job_id} created but not queued to Pub/Sub") + else: + logger.warning("GCP_PROJECT_ID or PUBSUB_TOPIC not configured, job created but not queued") + + return AnalyzeResponse( + job_id=str(job_id), + message="Analysis started. Check job status for results." + ) + + except Exception as e: + logger.error(f"Error triggering analysis: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/api/jobs/{job_id}") +async def get_job_status(job_id: str, clerk_user_id: str = Depends(get_current_user_id)): + """Get job status and results""" + + try: + # Get job + job = db.jobs.find_by_id(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + # Verify job belongs to user - jobs table stores clerk_user_id directly + if job.get('clerk_user_id') != clerk_user_id: + raise HTTPException(status_code=403, detail="Not authorized") + + return job + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting job status: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/api/jobs") +async def list_jobs(clerk_user_id: str = Depends(get_current_user_id)): + """List user's analysis jobs""" + + try: + # Get jobs for this user (with higher limit to avoid missing recent jobs) + user_jobs = db.jobs.find_by_user(clerk_user_id, limit=100) + # Sort by created_at descending (most recent first) + user_jobs.sort(key=lambda x: x.get('created_at', ''), reverse=True) + return {"jobs": user_jobs} + + except Exception as e: + logger.error(f"Error listing jobs: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.delete("/api/reset-accounts") +async def reset_accounts(clerk_user_id: str = Depends(get_current_user_id)): + """Delete all accounts for the current user""" + + try: + # Get user + user = db.users.find_by_clerk_id(clerk_user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Get all accounts for user + accounts = db.accounts.find_by_user(clerk_user_id) + + # Delete each account (positions will cascade delete) + deleted_count = 0 + for account in accounts: + try: + # Positions are deleted automatically via CASCADE + db.accounts.delete(account['id']) + deleted_count += 1 + except Exception as e: + logger.warning(f"Could not delete account {account['id']}: {e}") + + return { + "message": f"Deleted {deleted_count} account(s)", + "accounts_deleted": deleted_count + } + + except Exception as e: + logger.error(f"Error resetting accounts: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@app.post("/api/populate-test-data") +async def populate_test_data(clerk_user_id: str = Depends(get_current_user_id)): + """Populate test data for the current user""" + + try: + # Get user + user = db.users.find_by_clerk_id(clerk_user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Define missing instruments that might not be in the database + missing_instruments = { + "AAPL": { + "name": "Apple Inc.", + "type": "stock", + "current_price": 195.89, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"technology": 100}, + "allocation_asset_class": {"equity": 100} + }, + "AMZN": { + "name": "Amazon.com Inc.", + "type": "stock", + "current_price": 178.35, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"consumer_discretionary": 100}, + "allocation_asset_class": {"equity": 100} + }, + "NVDA": { + "name": "NVIDIA Corporation", + "type": "stock", + "current_price": 522.74, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"technology": 100}, + "allocation_asset_class": {"equity": 100} + }, + "MSFT": { + "name": "Microsoft Corporation", + "type": "stock", + "current_price": 430.82, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"technology": 100}, + "allocation_asset_class": {"equity": 100} + }, + "GOOGL": { + "name": "Alphabet Inc. Class A", + "type": "stock", + "current_price": 173.69, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"technology": 100}, + "allocation_asset_class": {"equity": 100} + }, + } + + # Check and add missing instruments + for symbol, info in missing_instruments.items(): + existing = db.instruments.find_by_symbol(symbol) + if not existing: + try: + from src.schemas import InstrumentCreate + + instrument_data = InstrumentCreate( + symbol=symbol, + name=info["name"], + instrument_type=info["type"], + current_price=Decimal(str(info["current_price"])), + allocation_regions=info["allocation_regions"], + allocation_sectors=info["allocation_sectors"], + allocation_asset_class=info["allocation_asset_class"] + ) + db.instruments.create_instrument(instrument_data) + logger.info(f"Added missing instrument: {symbol}") + except Exception as e: + logger.warning(f"Could not add instrument {symbol}: {e}") + + # Create accounts with test data + accounts_data = [ + { + "name": "401k Long-term", + "purpose": "Primary retirement savings account with employer match", + "cash": 5000.00, + "positions": [ + ("SPY", 150), # S&P 500 ETF + ("VTI", 100), # Total Stock Market ETF + ("BND", 200), # Bond ETF + ("QQQ", 75), # Nasdaq ETF + ("IWM", 50), # Small Cap ETF + ] + }, + { + "name": "Roth IRA", + "purpose": "Tax-free retirement growth account", + "cash": 2500.00, + "positions": [ + ("VTI", 80), # Total Stock Market ETF + ("VXUS", 60), # International Stock ETF + ("VNQ", 40), # Real Estate ETF + ("GLD", 25), # Gold ETF + ("TLT", 30), # Long-term Treasury ETF + ("VIG", 45), # Dividend Growth ETF + ] + }, + { + "name": "Brokerage Account", + "purpose": "Taxable investment account for individual stocks", + "cash": 10000.00, + "positions": [ + ("TSLA", 15), # Tesla + ("AAPL", 50), # Apple + ("AMZN", 10), # Amazon + ("NVDA", 25), # Nvidia + ("MSFT", 30), # Microsoft + ("GOOGL", 20), # Google + ] + } + ] + + created_accounts = [] + for account_data in accounts_data: + # Create account + account_id = db.accounts.create_account( + clerk_user_id=clerk_user_id, + account_name=account_data["name"], + account_purpose=account_data["purpose"], + cash_balance=Decimal(str(account_data["cash"])) + ) + + # Add positions + for symbol, quantity in account_data["positions"]: + try: + db.positions.add_position( + account_id=account_id, + symbol=symbol, + quantity=Decimal(str(quantity)) + ) + except Exception as e: + logger.warning(f"Could not add position {symbol}: {e}") + + created_accounts.append(account_id) + + # Get all accounts with their positions for summary + all_accounts = [] + for account_id in created_accounts: + account = db.accounts.find_by_id(account_id) + positions = db.positions.find_by_account(account_id) + account['positions'] = positions + all_accounts.append(account) + + return { + "message": "Test data populated successfully", + "accounts_created": len(created_accounts), + "accounts": all_accounts + } + + except Exception as e: + logger.error(f"Error populating test data: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +# Lambda handler +handler = Mangum(app) + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/gcp-deployment/backend/api/package_docker.py b/gcp-deployment/backend/api/package_docker.py new file mode 100644 index 00000000..4bb1acb8 --- /dev/null +++ b/gcp-deployment/backend/api/package_docker.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +""" +Package the FastAPI API for Lambda deployment using Docker. +This ensures binary compatibility with Lambda's runtime environment. +""" + +import os +import sys +import shutil +import subprocess +from pathlib import Path +import tempfile +import zipfile + +def run_command(cmd, cwd=None): + """Run a shell command and handle errors.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + +def main(): + # Get the API directory + api_dir = Path(__file__).parent.absolute() + backend_dir = api_dir.parent + project_root = backend_dir.parent + + print(f"API directory: {api_dir}") + print(f"Backend directory: {backend_dir}") + + # Check if Docker is running + try: + run_command(["docker", "info"]) + except Exception as e: + print("Error: Docker is not running or not installed") + print("Please ensure Docker Desktop is running and try again") + sys.exit(1) + + # Create temp directory for packaging + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + package_dir = temp_path / "package" + package_dir.mkdir() + + print(f"Packaging in: {package_dir}") + + # Copy API code + api_package = package_dir / "api" + shutil.copytree(api_dir, api_package, ignore=shutil.ignore_patterns( + "__pycache__", "*.pyc", ".env*", "*.zip", "package_docker.py", "test_*.py" + )) + + # Copy lambda_handler.py to root level for Lambda to find it + shutil.copy2(api_dir / "lambda_handler.py", package_dir / "lambda_handler.py") + + # Copy database package + database_src = backend_dir / "database" / "src" + database_dst = package_dir / "src" + if database_src.exists(): + shutil.copytree(database_src, database_dst, ignore=shutil.ignore_patterns( + "__pycache__", "*.pyc" + )) + print(f"Copied database package from {database_src}") + else: + print(f"Warning: Database package not found at {database_src}") + + # Create requirements.txt from pyproject.toml + requirements_file = package_dir / "requirements.txt" + with open(requirements_file, "w") as f: + # Core dependencies + f.write("fastapi>=0.116.0\n") + f.write("uvicorn>=0.35.0\n") + f.write("mangum>=0.19.0\n") + f.write("boto3>=1.26.0\n") + f.write("fastapi-clerk-auth>=0.0.7\n") + f.write("pydantic>=2.0.0\n") + f.write("python-dotenv>=1.0.0\n") + + # Create Dockerfile + dockerfile_content = """ +FROM public.ecr.aws/lambda/python:3.12 + +# Copy requirements and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt -t /var/task + +# Copy application code +COPY . /var/task/ + +# Set the handler +CMD ["api.main.handler"] +""" + + dockerfile = package_dir / "Dockerfile" + with open(dockerfile, "w") as f: + f.write(dockerfile_content) + + # Build Docker image for x86_64 architecture (Lambda runtime) + print("Building Docker image for x86_64 architecture...") + run_command([ + "docker", "build", + "--platform", "linux/amd64", + "-t", "alex-api-packager", + "." + ], cwd=package_dir) + + # Create container and extract files + print("Extracting Lambda package...") + container_name = "alex-api-extract" + + # Remove container if it exists + run_command(["docker", "rm", "-f", container_name], cwd=package_dir) + + # Create container + run_command([ + "docker", "create", + "--name", container_name, + "alex-api-packager" + ], cwd=package_dir) + + # Extract /var/task contents + extract_dir = temp_path / "lambda" + extract_dir.mkdir() + + run_command([ + "docker", "cp", + f"{container_name}:/var/task/.", + str(extract_dir) + ]) + + # Clean up container + run_command(["docker", "rm", "-f", container_name]) + + # Create the final zip + zip_path = api_dir / "api_lambda.zip" + print(f"Creating zip file: {zip_path}") + + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for root, dirs, files in os.walk(extract_dir): + # Skip __pycache__ directories + dirs[:] = [d for d in dirs if d != '__pycache__'] + + for file in files: + # Skip .pyc files + if file.endswith('.pyc'): + continue + + file_path = Path(root) / file + arcname = file_path.relative_to(extract_dir) + zipf.write(file_path, arcname) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"โœ… Lambda package created: {zip_path} ({size_mb:.2f} MB)") + + # Verify the package + print("\nPackage contents (first 20 files):") + with zipfile.ZipFile(zip_path, 'r') as zipf: + files = zipf.namelist()[:20] + for f in files: + print(f" - {f}") + if len(zipf.namelist()) > 20: + print(f" ... and {len(zipf.namelist()) - 20} more files") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/api/pyproject.toml b/gcp-deployment/backend/api/pyproject.toml new file mode 100644 index 00000000..f0cef9a4 --- /dev/null +++ b/gcp-deployment/backend/api/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "api" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "google-cloud-pubsub>=2.18.0", + "fastapi>=0.116.1", + "fastapi-clerk-auth>=0.0.7", + "httpx>=0.28.1", + "mangum>=0.19.0", + "pydantic>=2.11.7", + "python-dotenv>=1.1.1", + "python-jose>=3.5.0", + "uvicorn>=0.35.0", +] + +[tool.uv.sources] +alex-database = { workspace = true } diff --git a/gcp-deployment/backend/api/server.py b/gcp-deployment/backend/api/server.py new file mode 100644 index 00000000..5612dee1 --- /dev/null +++ b/gcp-deployment/backend/api/server.py @@ -0,0 +1,11 @@ +""" +FastAPI server for Cloud Run deployment +""" +import os +import uvicorn +from main import app + +if __name__ == "__main__": + port = int(os.getenv("PORT", "8080")) + uvicorn.run(app, host="0.0.0.0", port=port) + diff --git a/gcp-deployment/backend/api/test_pubsub.py b/gcp-deployment/backend/api/test_pubsub.py new file mode 100644 index 00000000..b9eb8667 --- /dev/null +++ b/gcp-deployment/backend/api/test_pubsub.py @@ -0,0 +1,63 @@ +import os +import json +from pathlib import Path +from google.cloud import pubsub_v1 +from dotenv import load_dotenv + +# Load .env file from project root (alex-gcp/.env) +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Get project ID from environment or gcloud config +project_id = os.getenv('GCP_PROJECT_ID') or os.getenv('PROJECT_ID') + +if not project_id: + # Try to get from gcloud config + import subprocess + try: + result = subprocess.run( + ['gcloud', 'config', 'get-value', 'project'], + capture_output=True, + text=True, + check=True + ) + project_id = result.stdout.strip() + print(f"[WARNING] GCP_PROJECT_ID not set, using gcloud default: {project_id}") + except (subprocess.CalledProcessError, FileNotFoundError): + print("[ERROR] GCP_PROJECT_ID not set and gcloud not available") + print(" Set GCP_PROJECT_ID in .env file or run: gcloud config set project YOUR_PROJECT_ID") + exit(1) + +topic_name = os.getenv('PUBSUB_TOPIC', 'alex-job-queue') + +print(f"Using project: {project_id}") +print(f"Using topic: {topic_name}") + +publisher = pubsub_v1.PublisherClient() +topic_path = publisher.topic_path(project_id, topic_name) + +# Publish a test message +message_data = { + 'job_id': 'test-123', + 'clerk_user_id': 'test-user', + 'analysis_type': 'portfolio', + 'options': {} +} + +try: + future = publisher.publish( + topic_path, + json.dumps(message_data).encode('utf-8') + ) + + message_id = future.result() + print(f"[SUCCESS] Published message: {message_id}") + print(f" Topic: {topic_path}") + print(f" Message: {message_data}") +except Exception as e: + print(f"[ERROR] Error publishing message: {e}") + print(f"\nTroubleshooting:") + print(f" 1. Verify topic exists: gcloud pubsub topics list --project={project_id}") + print(f" 2. Check IAM permissions: gcloud pubsub topics get-iam-policy {topic_name} --project={project_id}") + print(f" 3. Verify GCP_PROJECT_ID in .env file: {project_id}") + raise \ No newline at end of file diff --git a/gcp-deployment/backend/charter/.python-version b/gcp-deployment/backend/charter/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/charter/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/charter/Dockerfile b/gcp-deployment/backend/charter/Dockerfile new file mode 100644 index 00000000..b9c03599 --- /dev/null +++ b/gcp-deployment/backend/charter/Dockerfile @@ -0,0 +1,41 @@ +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Python package manager +RUN pip install uv + +# Copy database package (required dependency) +# Build context should be from backend/ directory +COPY database ./database + +# Copy charter-specific files +COPY charter/pyproject.toml charter/uv.lock ./ + + +# Copy shared modules +COPY common ./common + +# Update pyproject.toml to use ./database instead of ../database +RUN sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && rm pyproject.toml.bak + +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +# First install database package with all its dependencies (including pg8000) +# This ensures transitive dependencies from the local path dependency are installed +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project + +# Copy charter application code +COPY charter/*.py ./ + +# Expose port +EXPOSE 8000 + +# Set environment variable for Cloud Run +ENV PORT=8000 + +# Run the application +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/gcp-deployment/backend/charter/agent.py b/gcp-deployment/backend/charter/agent.py new file mode 100644 index 00000000..cf0b7fbc --- /dev/null +++ b/gcp-deployment/backend/charter/agent.py @@ -0,0 +1,157 @@ +""" +Chart Maker Agent - creates visualization data for portfolio analysis. +""" + +import os +import logging +from typing import Dict, Any + +from common.llm import get_litellm_model + +from templates import CHARTER_INSTRUCTIONS, create_charter_task + +logger = logging.getLogger() + + +def analyze_portfolio(portfolio_data: Dict[str, Any]) -> str: + """ + Analyze the portfolio to understand its composition and calculate key metrics. + Returns detailed breakdown of positions, accounts, and calculated allocations. + """ + result = [] + total_value = 0.0 + position_values = {} + account_totals = {} + + # Calculate position values and totals + for account in portfolio_data.get("accounts", []): + account_name = account.get("name", "Unknown") + account_type = account.get("type", "unknown") + # Handle None or missing cash_balance + cash_balance = account.get("cash_balance") + if cash_balance is None or cash_balance == "": + cash = 0.0 + else: + cash = float(cash_balance) + + if account_name not in account_totals: + account_totals[account_name] = {"value": 0, "type": account_type, "positions": []} + + account_totals[account_name]["value"] += cash + total_value += cash + + for position in account.get("positions", []): + symbol = position.get("symbol") + quantity = float(position.get("quantity", 0)) + instrument = position.get("instrument", {}) + # Handle None or missing current_price + current_price = instrument.get("current_price") + if current_price is None or current_price == "": + price = 1.0 # Default price if not available + logger.warning(f"Charter: No price for {symbol}, using default of 1.0") + else: + price = float(current_price) + value = quantity * price + + position_values[symbol] = position_values.get(symbol, 0) + value + account_totals[account_name]["value"] += value + account_totals[account_name]["positions"].append( + {"symbol": symbol, "value": value, "instrument": instrument} + ) + total_value += value + + # Build analysis summary + result.append("Portfolio Analysis:") + result.append(f"Total Value: ${total_value:,.2f}") + result.append(f"Number of Accounts: {len(account_totals)}") + result.append(f"Number of Positions: {len(position_values)}") + + result.append("\nAccount Breakdown:") + for name, data in account_totals.items(): + pct = (data["value"] / total_value * 100) if total_value > 0 else 0 + result.append(f" {name} ({data['type']}): ${data['value']:,.2f} ({pct:.1f}%)") + + result.append("\nTop Holdings by Value:") + sorted_positions = sorted(position_values.items(), key=lambda x: x[1], reverse=True)[:10] + for symbol, value in sorted_positions: + pct = (value / total_value * 100) if total_value > 0 else 0 + result.append(f" {symbol}: ${value:,.2f} ({pct:.1f}%)") + + # Calculate aggregated allocations for the agent + result.append("\nCalculated Allocations:") + + # Asset class aggregation + asset_classes = {} + regions = {} + sectors = {} + + for account in portfolio_data.get("accounts", []): + for position in account.get("positions", []): + symbol = position.get("symbol") + quantity = float(position.get("quantity", 0)) + instrument = position.get("instrument", {}) + # Handle None or missing current_price + current_price = instrument.get("current_price") + if current_price is None or current_price == "": + price = 1.0 # Default price if not available + logger.warning(f"Charter: No price for {symbol}, using default of 1.0") + else: + price = float(current_price) + value = quantity * price + + # Aggregate asset classes + for asset_class, pct in instrument.get("allocation_asset_class", {}).items(): + asset_value = value * (pct / 100) + asset_classes[asset_class] = asset_classes.get(asset_class, 0) + asset_value + + # Aggregate regions + for region, pct in instrument.get("allocation_regions", {}).items(): + region_value = value * (pct / 100) + regions[region] = regions.get(region, 0) + region_value + + # Aggregate sectors + for sector, pct in instrument.get("allocation_sectors", {}).items(): + sector_value = value * (pct / 100) + sectors[sector] = sectors.get(sector, 0) + sector_value + + # Add cash to asset classes + total_cash = sum( + float(acc.get("cash_balance")) if acc.get("cash_balance") is not None else 0 + for acc in portfolio_data.get("accounts", []) + ) + if total_cash > 0: + asset_classes["cash"] = asset_classes.get("cash", 0) + total_cash + + result.append("\nAsset Classes:") + for asset_class, value in sorted(asset_classes.items(), key=lambda x: x[1], reverse=True): + result.append(f" {asset_class}: ${value:,.2f}") + + result.append("\nGeographic Regions:") + for region, value in sorted(regions.items(), key=lambda x: x[1], reverse=True): + result.append(f" {region}: ${value:,.2f}") + + result.append("\nSectors:") + for sector, value in sorted(sectors.items(), key=lambda x: x[1], reverse=True)[:10]: + result.append(f" {sector}: ${value:,.2f}") + + return "\n".join(result) + + +def create_agent(job_id: str, portfolio_data: Dict[str, Any], db=None): + """Create the charter agent without tools - will output JSON directly.""" + + model_override = os.getenv("CHARTER_MODEL") + model = get_litellm_model(model_override) + logger.info("Charter: Job ID: %s", job_id) + + # Analyze the portfolio upfront + portfolio_analysis = analyze_portfolio(portfolio_data) + logger.info(f"Charter: Portfolio analysis generated, length: {len(portfolio_analysis)}") + + # Create the task using template + task = create_charter_task(portfolio_analysis, portfolio_data) + + logger.info(f"Charter: Task created, length: {len(task)} characters") + + # Return model and task (no tools or context needed) + return model, task \ No newline at end of file diff --git a/gcp-deployment/backend/charter/lambda_handler.py b/gcp-deployment/backend/charter/lambda_handler.py new file mode 100644 index 00000000..1db93b70 --- /dev/null +++ b/gcp-deployment/backend/charter/lambda_handler.py @@ -0,0 +1,261 @@ +""" +Chart Maker Agent Lambda Handler +""" + +import os +import json +import asyncio +import logging +from typing import Dict, Any + +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +try: + from dotenv import load_dotenv + load_dotenv(override=True) +except ImportError: + pass + +# Import database package +from src import Database + +from templates import CHARTER_INSTRUCTIONS +from agent import create_agent +from observability import observe + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +@retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info(f"Charter: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds...") +) +async def run_charter_agent(job_id: str, portfolio_data: Dict[str, Any], db=None) -> Dict[str, Any]: + """Run the charter agent to generate visualization data.""" + + # Create agent without tools - will output JSON + model, task = create_agent(job_id, portfolio_data, db) + + # Run agent - no tools, no context + with trace("Charter Agent"): + agent = Agent( + name="Chart Maker", + instructions=CHARTER_INSTRUCTIONS, + model=model + ) + + result = await Runner.run( + agent, + input=task, + max_turns=5 # Reduced since we expect one-shot JSON response + ) + + # Extract and parse JSON from the output + output = result.final_output + logger.info(f"Charter: Agent completed, output length: {len(output) if output else 0}") + + # Log the actual output for debugging + if output: + logger.info(f"Charter: Output preview (first 1000 chars): {output[:1000]}") + else: + logger.warning("Charter: Agent returned empty output!") + # Check if there were any messages + if hasattr(result, 'messages') and result.messages: + logger.info(f"Charter: Number of messages: {len(result.messages)}") + for i, msg in enumerate(result.messages): + logger.info(f"Charter: Message {i}: {str(msg)[:500]}") + + # Parse the JSON output + charts_data = None + charts_saved = False + + if output: + # Try to find JSON in the output + # Look for the opening and closing braces of the JSON object + start_idx = output.find('{') + end_idx = output.rfind('}') + + if start_idx >= 0 and end_idx > start_idx: + json_str = output[start_idx:end_idx + 1] + logger.info(f"Charter: Extracted JSON substring, length: {len(json_str)}") + + try: + parsed_data = json.loads(json_str) + charts = parsed_data.get('charts', []) + logger.info(f"Charter: Successfully parsed JSON, found {len(charts)} charts") + + if charts: + # Build the charts_payload with chart keys as top-level keys + charts_data = {} + for chart in charts: + chart_key = chart.get('key', f"chart_{len(charts_data) + 1}") + # Remove the 'key' from the chart data since it's now the dict key + chart_copy = {k: v for k, v in chart.items() if k != 'key'} + charts_data[chart_key] = chart_copy + + logger.info(f"Charter: Created charts_data with keys: {list(charts_data.keys())}") + + # Save to database + if db and charts_data: + try: + success = db.jobs.update_charts(job_id, charts_data) + charts_saved = bool(success) + logger.info(f"Charter: Database update returned: {success}") + except Exception as e: + logger.error(f"Charter: Database error: {e}") + else: + logger.warning("Charter: No charts found in parsed JSON") + + except json.JSONDecodeError as e: + logger.error(f"Charter: Failed to parse JSON: {e}") + logger.error(f"Charter: JSON string attempted: {json_str[:500]}...") + else: + logger.error(f"Charter: No JSON structure found in output") + logger.error(f"Charter: Output preview: {output[:500]}...") + + return { + 'success': charts_saved, + 'message': f'Generated {len(charts_data) if charts_data else 0} charts' if charts_saved else 'Failed to generate charts', + 'charts_generated': len(charts_data) if charts_data else 0, + 'chart_keys': list(charts_data.keys()) if charts_data else [] + } + +def lambda_handler(event, context): + """ + Lambda handler expecting job_id and portfolio_data in event. + + Expected event: + { + "job_id": "uuid", + "portfolio_data": {...} + } + """ + # Wrap entire handler with observability context + with observe(): + try: + logger.info(f"Charter Lambda invoked with event keys: {list(event.keys()) if isinstance(event, dict) else 'not a dict'}") + + # Parse event + if isinstance(event, str): + event = json.loads(event) + + job_id = event.get('job_id') + if not job_id: + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'job_id is required'}) + } + + # Initialize database first + db = Database() + + portfolio_data = event.get('portfolio_data') + if not portfolio_data: + # Load portfolio data from database (like Reporter does) + logger.info(f"Charter: Loading portfolio data for job {job_id}") + try: + job = db.jobs.find_by_id(job_id) + if job: + user_id = job['clerk_user_id'] + user = db.users.find_by_clerk_id(user_id) + accounts = db.accounts.find_by_user(user_id) + + portfolio_data = { + 'user_id': user_id, + 'job_id': job_id, + 'years_until_retirement': user.get('years_until_retirement', 30) if user else 30, + 'accounts': [] + } + + for account in accounts: + account_data = { + 'id': account['id'], + 'name': account['account_name'], + 'type': account.get('account_type', 'investment'), + 'cash_balance': float(account.get('cash_balance', 0)), + 'positions': [] + } + + positions = db.positions.find_by_account(account['id']) + for position in positions: + instrument = db.instruments.find_by_symbol(position['symbol']) + if instrument: + account_data['positions'].append({ + 'symbol': position['symbol'], + 'quantity': float(position['quantity']), + 'instrument': instrument + }) + + portfolio_data['accounts'].append(account_data) + + logger.info(f"Charter: Loaded {len(portfolio_data['accounts'])} accounts with positions") + else: + logger.error(f"Charter: Job {job_id} not found") + return { + 'statusCode': 404, + 'body': json.dumps({'error': 'Job not found'}) + } + except Exception as e: + logger.error(f"Charter: Error loading portfolio data: {e}") + return { + 'statusCode': 500, + 'body': json.dumps({'error': f'Failed to load portfolio data: {str(e)}'}) + } + + logger.info(f"Charter: Processing job {job_id}") + + # Run the agent + result = asyncio.run(run_charter_agent(job_id, portfolio_data, db)) + + logger.info(f"Charter completed for job {job_id}: {result}") + + return { + 'statusCode': 200, + 'body': json.dumps(result) + } + + except Exception as e: + logger.error(f"Error in charter: {e}", exc_info=True) + return { + 'statusCode': 500, + 'body': json.dumps({ + 'success': False, + 'error': str(e) + }) + } + +# For local testing +if __name__ == "__main__": + test_event = { + "job_id": "550e8400-e29b-41d4-a716-446655440001", + "portfolio_data": { + "accounts": [ + { + "id": "acc1", + "name": "401(k)", + "type": "401k", + "cash_balance": 5000, + "positions": [ + { + "symbol": "SPY", + "quantity": 100, + "instrument": { + "name": "SPDR S&P 500 ETF", + "current_price": 450, + "allocation_asset_class": {"equity": 100}, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"technology": 30, "healthcare": 15, "financials": 15, "consumer_discretionary": 20, "industrials": 20} + } + } + ] + } + ] + } + } + + result = lambda_handler(test_event, None) + print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/gcp-deployment/backend/charter/observability.py b/gcp-deployment/backend/charter/observability.py new file mode 100644 index 00000000..c694860b --- /dev/null +++ b/gcp-deployment/backend/charter/observability.py @@ -0,0 +1,112 @@ +""" +Observability module for LangFuse integration. +Provides a simple context manager for setting up and flushing traces. +""" + +import os +import logging +from contextlib import contextmanager + +# Use root logger for Lambda compatibility +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@contextmanager +def observe(): + """ + Context manager for observability with LangFuse. + + Sets up LangFuse observability if environment variables are configured, + and ensures traces are flushed on exit. + + Usage: + from observability import observe + + with observe(): + # Your code that uses OpenAI Agents SDK + result = await agent.run(...) + """ + logger.info("๐Ÿ” Observability: Checking configuration...") + + # Check if required environment variables exist + has_langfuse = bool(os.getenv("LANGFUSE_SECRET_KEY")) + has_openai = bool(os.getenv("OPENAI_API_KEY")) + + logger.info(f"๐Ÿ” Observability: LANGFUSE_SECRET_KEY exists: {has_langfuse}") + logger.info(f"๐Ÿ” Observability: OPENAI_API_KEY exists: {has_openai}") + + if not has_langfuse: + logger.info("๐Ÿ” Observability: LangFuse not configured, skipping setup") + yield + return + + if not has_openai: + logger.warning("โš ๏ธ Observability: OPENAI_API_KEY not set, traces may not export") + + # Local variable for the client (no global needed) + langfuse_client = None + + # Try to set up LangFuse + try: + logger.info("๐Ÿ” Observability: Setting up LangFuse...") + + import logfire + from langfuse import get_client + + # Configure logfire to instrument OpenAI Agents SDK + logfire.configure( + service_name="alex_charter_agent", + send_to_logfire=False, # Don't send to Logfire cloud + ) + logger.info("โœ… Observability: Logfire configured") + + # Instrument OpenAI Agents SDK + logfire.instrument_openai_agents() + logger.info("โœ… Observability: OpenAI Agents SDK instrumented") + + # Initialize LangFuse client + langfuse_client = get_client() + logger.info("โœ… Observability: LangFuse client initialized") + + # Optional: Check authentication (blocking call, use sparingly) + try: + auth_result = langfuse_client.auth_check() + logger.info( + f"โœ… Observability: LangFuse authentication check passed (result: {auth_result})" + ) + except Exception as auth_error: + logger.warning(f"โš ๏ธ Observability: Auth check failed but continuing: {auth_error}") + + logger.info("๐ŸŽฏ Observability: Setup complete - traces will be sent to LangFuse") + + except ImportError as e: + logger.error(f"โŒ Observability: Missing required package: {e}") + langfuse_client = None + except Exception as e: + logger.error(f"โŒ Observability: Setup failed: {e}") + langfuse_client = None + + try: + # Yield control back to the calling code + yield + finally: + # Flush traces on exit + if langfuse_client: + try: + logger.info("๐Ÿ” Observability: Flushing traces to LangFuse...") + langfuse_client.flush() + langfuse_client.shutdown() + + # Add a 10 second delay to ensure network requests complete + # This is a workaround for Lambda's immediate termination + import time + + logger.info("๐Ÿ” Observability: Waiting 10 seconds for flush to complete...") + time.sleep(10) + + logger.info("โœ… Observability: Traces flushed successfully") + except Exception as e: + logger.error(f"โŒ Observability: Failed to flush traces: {e}") + else: + logger.debug("๐Ÿ” Observability: No client to flush") diff --git a/gcp-deployment/backend/charter/package_docker.py b/gcp-deployment/backend/charter/package_docker.py new file mode 100644 index 00000000..1adf5651 --- /dev/null +++ b/gcp-deployment/backend/charter/package_docker.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Package the Charter Lambda function using Docker for AWS compatibility. +""" + +import os +import sys +import shutil +import tempfile +import subprocess +import argparse +from pathlib import Path + +def run_command(cmd, cwd=None): + """Run a command and capture output.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + +def package_lambda(): + """Package the Lambda function with all dependencies.""" + + # Get the directory containing this script + charter_dir = Path(__file__).parent.absolute() + backend_dir = charter_dir.parent + + # Create a temporary directory for packaging + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + package_dir = temp_path / "package" + package_dir.mkdir() + + print("Creating Lambda package using Docker...") + + # Export exact requirements from uv.lock (excluding the editable database package) + print("Exporting requirements from uv.lock...") + requirements_result = run_command( + ["uv", "export", "--no-hashes", "--no-emit-project"], + cwd=str(charter_dir) + ) + + # Filter out packages that don't work in Lambda + filtered_requirements = [] + for line in requirements_result.splitlines(): + # Skip pyperclip (clipboard library not needed in Lambda) + if line.startswith("pyperclip"): + print(f"Excluding from Lambda: {line}") + continue + filtered_requirements.append(line) + + req_file = temp_path / "requirements.txt" + req_file.write_text("\n".join(filtered_requirements)) + + # Use Docker to install dependencies for Lambda's architecture + docker_cmd = [ + "docker", "run", "--rm", + "--platform", "linux/amd64", + "-v", f"{temp_path}:/build", + "-v", f"{backend_dir}/database:/database", + "--entrypoint", "/bin/bash", + "public.ecr.aws/lambda/python:3.12", + "-c", + """cd /build && pip install --target ./package -r requirements.txt && pip install --target ./package --no-deps /database""" + ] + + run_command(docker_cmd) + + # Copy Lambda handler, agent, templates, and observability + shutil.copy(charter_dir / "lambda_handler.py", package_dir) + shutil.copy(charter_dir / "agent.py", package_dir) + shutil.copy(charter_dir / "templates.py", package_dir) + shutil.copy(charter_dir / "observability.py", package_dir) + + # Create the zip file + zip_path = charter_dir / "charter_lambda.zip" + + # Remove old zip if it exists + if zip_path.exists(): + zip_path.unlink() + + # Create new zip + print(f"Creating zip file: {zip_path}") + run_command( + ["zip", "-r", str(zip_path), "."], + cwd=str(package_dir) + ) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"Package created: {zip_path} ({size_mb:.1f} MB)") + + return zip_path + +def deploy_lambda(zip_path): + """Deploy the Lambda function to AWS.""" + import boto3 + + lambda_client = boto3.client('lambda') + function_name = 'alex-charter' + + print(f"Deploying to Lambda function: {function_name}") + + try: + # Try to update existing function + with open(zip_path, 'rb') as f: + response = lambda_client.update_function_code( + FunctionName=function_name, + ZipFile=f.read() + ) + print(f"Successfully updated Lambda function: {function_name}") + print(f"Function ARN: {response['FunctionArn']}") + except lambda_client.exceptions.ResourceNotFoundException: + print(f"Lambda function {function_name} not found. Please deploy via Terraform first.") + sys.exit(1) + except Exception as e: + print(f"Error deploying Lambda: {e}") + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser(description='Package Charter Lambda for deployment') + parser.add_argument('--deploy', action='store_true', help='Deploy to AWS after packaging') + args = parser.parse_args() + + # Check if Docker is available + try: + run_command(["docker", "--version"]) + except FileNotFoundError: + print("Error: Docker is not installed or not in PATH") + sys.exit(1) + + # Package the Lambda + zip_path = package_lambda() + + # Deploy if requested + if args.deploy: + deploy_lambda(zip_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/charter/pyproject.toml b/gcp-deployment/backend/charter/pyproject.toml new file mode 100644 index 00000000..772fb749 --- /dev/null +++ b/gcp-deployment/backend/charter/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "charter" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "boto3>=1.40.9", # Keep for backward compatibility + "fastapi>=0.116.1", + "uvicorn>=0.35.0", + "langfuse>=3.3.4", + "openai-agents[litellm]>=0.2.6", + "pydantic>=2.11.7", + "pydantic-ai>=1.0.6", + "python-dotenv>=1.1.1", + "tenacity>=9.1.2", +] + +[tool.uv.sources] +alex-database = { path = "../database", editable = true } diff --git a/gcp-deployment/backend/charter/server.py b/gcp-deployment/backend/charter/server.py new file mode 100644 index 00000000..03f7e6a7 --- /dev/null +++ b/gcp-deployment/backend/charter/server.py @@ -0,0 +1,229 @@ +""" +Charter Agent - Cloud Run HTTP Server +Generates portfolio visualization data +""" + +import os +import sys +import json +import asyncio +import logging +from pathlib import Path +from typing import Dict, Any +from datetime import datetime, UTC + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from dotenv import load_dotenv +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +# Add parent directories to Python path for imports +backend_dir = Path(__file__).parent.parent +if str(backend_dir) not in sys.path: + sys.path.insert(0, str(backend_dir)) + +from src import Database +from templates import CHARTER_INSTRUCTIONS +from agent import create_agent +from observability import observe + +# Load .env file from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize FastAPI app +app = FastAPI( + title="Alex Charter Service", + description="Portfolio visualization generation agent", + version="1.0.0" +) + +# Initialize database +db = Database() + + +@retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info(f"Charter: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds...") +) +async def run_charter_agent(job_id: str, portfolio_data: Dict[str, Any], db=None) -> Dict[str, Any]: + """Run the charter agent to generate visualization data.""" + + # Create agent without tools - will output JSON + model, task = create_agent(job_id, portfolio_data, db) + + # Run agent - no tools, no context + with trace("Charter Agent"): + agent = Agent( + name="Chart Maker", + instructions=CHARTER_INSTRUCTIONS, + model=model + ) + + result = await Runner.run( + agent, + input=task, + max_turns=5 # Reduced since we expect one-shot JSON response + ) + + # Extract and parse JSON from the output + output = result.final_output + logger.info(f"Charter: Agent completed, output length: {len(output) if output else 0}") + + # Parse the JSON output + charts_data = None + charts_saved = False + + if output: + # Try to find JSON in the output + start_idx = output.find('{') + end_idx = output.rfind('}') + + if start_idx >= 0 and end_idx > start_idx: + json_str = output[start_idx:end_idx + 1] + logger.info(f"Charter: Extracted JSON substring, length: {len(json_str)}") + + try: + parsed_data = json.loads(json_str) + charts = parsed_data.get('charts', []) + logger.info(f"Charter: Successfully parsed JSON, found {len(charts)} charts") + + if charts: + # Build the charts_payload with chart keys as top-level keys + charts_data = {} + for chart in charts: + chart_key = chart.get('key', f"chart_{len(charts_data) + 1}") + chart_copy = {k: v for k, v in chart.items() if k != 'key'} + charts_data[chart_key] = chart_copy + + logger.info(f"Charter: Created charts_data with keys: {list(charts_data.keys())}") + + # Save to database + if db and charts_data: + try: + success = db.jobs.update_charts(job_id, charts_data) + charts_saved = bool(success) + logger.info(f"Charter: Database update returned: {success}") + except Exception as e: + logger.error(f"Charter: Database error: {e}") + else: + logger.warning("Charter: No charts found in parsed JSON") + + except json.JSONDecodeError as e: + logger.error(f"Charter: Failed to parse JSON: {e}") + else: + logger.error(f"Charter: No JSON structure found in output") + + return { + 'success': charts_saved, + 'message': f'Generated {len(charts_data) if charts_data else 0} charts' if charts_saved else 'Failed to generate charts', + 'charts_generated': len(charts_data) if charts_data else 0, + 'chart_keys': list(charts_data.keys()) if charts_data else [] + } + + +# Request/Response models +class JobRequest(BaseModel): + """Request to process a job""" + job_id: str + portfolio_data: Dict[str, Any] = None + + +@app.get("/") +async def root(): + """Health check endpoint""" + return { + "service": "Alex Charter", + "status": "healthy", + "timestamp": datetime.now(UTC).isoformat(), + } + + +@app.get("/health") +async def health(): + """Health check endpoint (alternative)""" + return {"status": "healthy"} + + +@app.post("/") +async def handle_job(request: JobRequest): + """ + Handle job processing request. + + Request body: + { + "job_id": "uuid", + "portfolio_data": {...} # Optional, will load from DB if not provided + } + """ + try: + logger.info(f"Charter: Received job request: {request.job_id}") + + # Load portfolio_data from database if not provided + portfolio_data = request.portfolio_data + if not portfolio_data: + job = db.jobs.find_by_id(request.job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job {request.job_id} not found") + + user_id = job['clerk_user_id'] + user = db.users.find_by_clerk_id(user_id) + accounts = db.accounts.find_by_user(user_id) + + portfolio_data = { + 'user_id': user_id, + 'job_id': request.job_id, + 'years_until_retirement': user.get('years_until_retirement', 30) if user else 30, + 'accounts': [] + } + + for account in accounts: + account_data = { + 'id': account['id'], + 'name': account['account_name'], + 'type': account.get('account_type', 'investment'), + 'cash_balance': float(account.get('cash_balance', 0)), + 'positions': [] + } + + positions = db.positions.find_by_account(account['id']) + for position in positions: + instrument = db.instruments.find_by_symbol(position['symbol']) + if instrument: + account_data['positions'].append({ + 'symbol': position['symbol'], + 'quantity': float(position['quantity']), + 'instrument': instrument + }) + + portfolio_data['accounts'].append(account_data) + + logger.info(f"Charter: Loaded {len(portfolio_data['accounts'])} accounts with positions") + + # Run the agent + with observe(): + result = await run_charter_agent(request.job_id, portfolio_data, db) + + logger.info(f"Charter completed for job {request.job_id}: {result}") + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Charter: Error processing job: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# For local testing +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) + diff --git a/gcp-deployment/backend/charter/templates.py b/gcp-deployment/backend/charter/templates.py new file mode 100644 index 00000000..6be4da50 --- /dev/null +++ b/gcp-deployment/backend/charter/templates.py @@ -0,0 +1,128 @@ +""" +Prompt templates for the Chart Maker Agent. +""" + +import json + +CHARTER_INSTRUCTIONS = """You are a Chart Maker Agent that creates visualization data for investment portfolios. + +Your task is to analyze the portfolio and output a JSON object containing 4-6 charts that tell a compelling story about the portfolio. + +You must output ONLY valid JSON in the exact format shown below. Do not include any text before or after the JSON. + +REQUIRED JSON FORMAT: +{ + "charts": [ + { + "key": "asset_class_distribution", + "title": "Asset Class Distribution", + "type": "pie", + "description": "Shows the distribution of asset classes in the portfolio", + "data": [ + {"name": "Equity", "value": 146365.00, "color": "#3B82F6"}, + {"name": "Fixed Income", "value": 29000.00, "color": "#10B981"}, + {"name": "Real Estate", "value": 14500.00, "color": "#F59E0B"}, + {"name": "Cash", "value": 5000.00, "color": "#EF4444"} + ] + } + ] +} + +IMPORTANT RULES: +1. Output ONLY the JSON object, nothing else +2. Each chart must have: key, title, type, description, and data array +3. Chart types: 'pie', 'bar', 'donut', or 'horizontalBar' +4. Values must be dollar amounts (not percentages - Recharts calculates those) +5. Colors must be hex format like '#3B82F6' +6. Create 4-6 different charts from different perspectives + +CHART IDEAS TO IMPLEMENT: +- Asset class distribution (equity vs bonds vs alternatives) +- Geographic exposure (North America, Europe, Asia, etc.) +- Sector breakdown (Technology, Healthcare, Financials, etc.) +- Account type allocation (401k, IRA, Taxable, etc.) +- Top holdings concentration (largest 5-10 positions) +- Tax efficiency (tax-advantaged vs taxable accounts) + +EXAMPLE OUTPUT (this is what you should generate): +{ + "charts": [ + { + "key": "asset_allocation", + "title": "Asset Class Distribution", + "type": "pie", + "description": "Portfolio allocation across major asset classes", + "data": [ + {"name": "Equities", "value": 65900.50, "color": "#3B82F6"}, + {"name": "Bonds", "value": 14100.25, "color": "#10B981"}, + {"name": "Real Estate", "value": 9400.00, "color": "#F59E0B"}, + {"name": "Cash", "value": 4600.00, "color": "#6B7280"} + ] + }, + { + "key": "geographic_exposure", + "title": "Geographic Distribution", + "type": "bar", + "description": "Investment allocation by region", + "data": [ + {"name": "North America", "value": 56340.00, "color": "#6366F1"}, + {"name": "Europe", "value": 18780.00, "color": "#14B8A6"}, + {"name": "Asia Pacific", "value": 14100.00, "color": "#F97316"}, + {"name": "Emerging Markets", "value": 4700.00, "color": "#EC4899"} + ] + }, + { + "key": "sector_breakdown", + "title": "Sector Allocation", + "type": "donut", + "description": "Distribution across industry sectors", + "data": [ + {"name": "Technology", "value": 28200.00, "color": "#8B5CF6"}, + {"name": "Healthcare", "value": 14100.00, "color": "#059669"}, + {"name": "Financials", "value": 14100.00, "color": "#0891B2"}, + {"name": "Consumer", "value": 18800.00, "color": "#DC2626"}, + {"name": "Industrials", "value": 18800.00, "color": "#7C3AED"} + ] + }, + { + "key": "account_types", + "title": "Account Distribution", + "type": "pie", + "description": "Allocation across different account types", + "data": [ + {"name": "401(k)", "value": 45000.00, "color": "#10B981"}, + {"name": "Roth IRA", "value": 28000.00, "color": "#3B82F6"}, + {"name": "Taxable", "value": 20920.75, "color": "#F59E0B"} + ] + }, + { + "key": "top_holdings", + "title": "Top 5 Holdings", + "type": "horizontalBar", + "description": "Largest positions in the portfolio", + "data": [ + {"name": "SPY", "value": 23500.00, "color": "#3B82F6"}, + {"name": "QQQ", "value": 14100.00, "color": "#60A5FA"}, + {"name": "BND", "value": 9400.00, "color": "#93C5FD"}, + {"name": "VTI", "value": 7050.00, "color": "#BFDBFE"}, + {"name": "VXUS", "value": 4700.00, "color": "#DBEAFE"} + ] + } + ] +} + +Remember: Output ONLY the JSON object. No explanations, no text before or after.""" + + +def create_charter_task(portfolio_analysis: str, portfolio_data: dict) -> str: + """Generate the task prompt for the Charter agent.""" + # Don't include the full raw portfolio data - just the analysis + # This reduces context size significantly + + return f"""Analyze this investment portfolio and create 4-6 visualization charts. + +{portfolio_analysis} + +Create charts based on this portfolio data. Calculate aggregated values from the positions shown above. + +OUTPUT ONLY THE JSON OBJECT with 4-6 charts - no other text.""" \ No newline at end of file diff --git a/gcp-deployment/backend/charter/test_full.py b/gcp-deployment/backend/charter/test_full.py new file mode 100644 index 00000000..3a465f0d --- /dev/null +++ b/gcp-deployment/backend/charter/test_full.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +Full test for Charter agent via Lambda +""" + +import json +import boto3 +import time +from dotenv import load_dotenv + +from src import Database +from src.schemas import JobCreate + +load_dotenv(override=True) + + +def test_charter_lambda(): + """Test the Charter agent via Lambda invocation""" + + db = Database() + lambda_client = boto3.client("lambda") + + # Create test job + test_user_id = "test_user_001" + + job_create = JobCreate( + clerk_user_id=test_user_id, + job_type="portfolio_analysis", + request_payload={"analysis_type": "test", "test": True}, + ) + job_id = db.jobs.create(job_create.model_dump()) + + # Load portfolio data for the test + user = db.users.find_by_clerk_id(test_user_id) + accounts = db.accounts.find_by_user(test_user_id) + + portfolio_data = { + "user_id": test_user_id, + "job_id": job_id, + "years_until_retirement": user.get("years_until_retirement", 30), + "accounts": [], + } + + for account in accounts: + positions = db.positions.find_by_account(account["id"]) + account_data = { + "id": account["id"], + "name": account["account_name"], + "cash_balance": float(account.get("cash_balance", 0)), + "positions": [], + } + + for position in positions: + instrument = db.instruments.find_by_symbol(position["symbol"]) + if instrument: + account_data["positions"].append( + { + "symbol": position["symbol"], + "quantity": float(position["quantity"]), + "instrument": instrument, + } + ) + + portfolio_data["accounts"].append(account_data) + + print(f"Testing Charter Lambda with job {job_id}") + print("=" * 60) + + # Invoke Lambda + try: + response = lambda_client.invoke( + FunctionName="alex-charter", + InvocationType="RequestResponse", + Payload=json.dumps({"job_id": job_id, "portfolio_data": portfolio_data}), + ) + + result = json.loads(response["Payload"].read()) + print(f"Lambda Response: {json.dumps(result, indent=2)}") + + # Check database for results + time.sleep(2) # Give it a moment + job = db.jobs.find_by_id(job_id) + + if job and job.get("charts_payload"): + print(f"\n๐Ÿ“Š Charts Created ({len(job['charts_payload'])} total):") + print("=" * 50) + for chart_key, chart_data in job["charts_payload"].items(): + print(f"\n๐ŸŽฏ Chart: {chart_key}") + print(f" Title: {chart_data.get('title', 'N/A')}") + print(f" Type: {chart_data.get('type', 'N/A')}") + print(f" Description: {chart_data.get('description', 'N/A')}") + + data_points = chart_data.get("data", []) + print(f" Data Points ({len(data_points)}):") + for i, point in enumerate(data_points): + name = point.get("name", "N/A") + value = point.get("value", 0) + color = point.get("color", "N/A") + print(f" {i+1}. {name}: ${value:,.2f} {color}") + + else: + print("\nโŒ No charts found in database") + + except Exception as e: + print(f"Error invoking Lambda: {e}") + + print("=" * 60) + + +if __name__ == "__main__": + test_charter_lambda() diff --git a/gcp-deployment/backend/charter/test_simple.py b/gcp-deployment/backend/charter/test_simple.py new file mode 100644 index 00000000..9e38a132 --- /dev/null +++ b/gcp-deployment/backend/charter/test_simple.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +Simple test for Charter agent +""" + +import asyncio +import json +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database +from src.schemas import JobCreate +from lambda_handler import lambda_handler + + +def test_charter(): + """Test the charter agent with simple portfolio data""" + + # Create a real job in the database + db = Database() + job_create = JobCreate( + clerk_user_id="test_user_001", job_type="portfolio_analysis", request_payload={"test": True} + ) + job_id = db.jobs.create(job_create.model_dump()) + print(f"Created test job: {job_id}") + + test_event = { + "job_id": job_id, + "portfolio_data": { + "accounts": [ + { + "name": "401(k)", + "type": "401k", + "cash_balance": 5000, + "positions": [ + { + "symbol": "SPY", + "quantity": 100, + "instrument": { + "name": "SPDR S&P 500 ETF", + "current_price": 450, + "allocation_asset_class": {"equity": 100}, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "technology": 30, + "healthcare": 15, + "financials": 15, + }, + }, + } + ], + } + ] + }, + } + + print("Testing Charter Agent...") + print("=" * 60) + + import sys + + print("About to call lambda_handler...", flush=True) + sys.stdout.flush() + result = lambda_handler(test_event, None) + print("lambda_handler returned", flush=True) + + print(f"Status Code: {result['statusCode']}") + + if result["statusCode"] == 200: + body = json.loads(result["body"]) + print(f"Success: {body.get('success', False)}") + print(f"Message: {body.get('message', 'N/A')}") + + # Check what charts were created + job = db.jobs.find_by_id(job_id) + if job and job.get("charts_payload"): + print(f"\n๐Ÿ“Š Charts Created ({len(job['charts_payload'])} total):") + print("=" * 50) + for chart_key, chart_data in job["charts_payload"].items(): + print(f"\n๐ŸŽฏ Chart: {chart_key}") + print(f" Title: {chart_data.get('title', 'N/A')}") + print(f" Type: {chart_data.get('type', 'N/A')}") + print(f" Description: {chart_data.get('description', 'N/A')}") + + data_points = chart_data.get("data", []) + print(f" Data Points ({len(data_points)}):") + for i, point in enumerate(data_points): + name = point.get("name", "N/A") + value = point.get("value", 0) + color = point.get("color", "N/A") + print(f" {i+1}. {name}: ${value:,.2f} {color}") + + else: + print("\nโŒ No charts found in database") + else: + print(f"Error: {result['body']}") + + # Clean up - delete the test job + db.jobs.delete(job_id) + print(f"Deleted test job: {job_id}") + + print("=" * 60) + + +if __name__ == "__main__": + test_charter() diff --git a/gcp-deployment/backend/check_db.py b/gcp-deployment/backend/check_db.py new file mode 100644 index 00000000..8bee1a6a --- /dev/null +++ b/gcp-deployment/backend/check_db.py @@ -0,0 +1,35 @@ +from database.src import Database + +db = Database() +print("Checking instrument prices...") +instruments = db.instruments.find_all() +print(f"Found {len(instruments)} instruments") +for inst in instruments: + price = inst.get("current_price") + symbol = inst.get("symbol") + if price: + price_val = float(price) if isinstance(price, str) else price + print(f" {symbol}: ${price_val:.2f}") + else: + print(f" {symbol}: N/A") + +print("\nChecking recent jobs...") +jobs = db.jobs.find_all() +print(f"Found {len(jobs)} total jobs") + +# Sort jobs by created_at and show last 5 +sorted_jobs = sorted(jobs, key=lambda x: x['created_at'], reverse=True)[:5] +for job in sorted_jobs: + print(f" Job {job['id'][:8]}...: {job['status']} - {job['created_at']}") + if job.get('results'): + print(f" Has results: Yes (length: {len(str(job['results']))} chars)") + # Check if it's JSON data + import json + try: + results = json.loads(job['results']) if isinstance(job['results'], str) else job['results'] + if 'charter' in results: + print(f" Charter data: {len(results['charter'])} charts") + except: + pass + else: + print(f" Has results: No") \ No newline at end of file diff --git a/gcp-deployment/backend/check_job_details.py b/gcp-deployment/backend/check_job_details.py new file mode 100644 index 00000000..fe975784 --- /dev/null +++ b/gcp-deployment/backend/check_job_details.py @@ -0,0 +1,52 @@ +from database.src import Database +import json + +db = Database() + +# Get the most recent completed job +jobs = db.jobs.find_all() +sorted_jobs = sorted(jobs, key=lambda x: x['created_at'], reverse=True) + +# Find first completed job +completed_job = None +for job in sorted_jobs: + if job['status'] == 'completed': + completed_job = job + break + +if completed_job: + print(f"Examining job: {completed_job['id']}") + print(f"Status: {completed_job['status']}") + print(f"Created: {completed_job['created_at']}") + print(f"Updated: {completed_job.get('updated_at', 'N/A')}") + + # Check all fields + for key, value in completed_job.items(): + if key == 'results': + if value: + print(f"\n{key}: Present") + try: + results = json.loads(value) if isinstance(value, str) else value + print(f" Keys in results: {list(results.keys())}") + for r_key in results: + if isinstance(results[r_key], str): + print(f" {r_key}: {len(results[r_key])} chars") + elif isinstance(results[r_key], list): + print(f" {r_key}: {len(results[r_key])} items") + elif isinstance(results[r_key], dict): + print(f" {r_key}: dict with keys {list(results[r_key].keys())}") + except Exception as e: + print(f" Error parsing results: {e}") + print(f" Raw value type: {type(value)}") + print(f" Raw value (first 500 chars): {str(value)[:500]}") + else: + print(f"\n{key}: None/Empty") + elif key not in ['id', 'status', 'created_at', 'updated_at']: + if value: + value_str = str(value) + if len(value_str) > 100: + print(f"{key}: {value_str[:100]}...") + else: + print(f"{key}: {value_str}") +else: + print("No completed jobs found") \ No newline at end of file diff --git a/gcp-deployment/backend/common/__init__.py b/gcp-deployment/backend/common/__init__.py new file mode 100644 index 00000000..dec7547b --- /dev/null +++ b/gcp-deployment/backend/common/__init__.py @@ -0,0 +1,2 @@ +# Shared utilities for backend services. + diff --git a/gcp-deployment/backend/common/llm.py b/gcp-deployment/backend/common/llm.py new file mode 100644 index 00000000..38712907 --- /dev/null +++ b/gcp-deployment/backend/common/llm.py @@ -0,0 +1,113 @@ +""" +Centralized helpers for configuring LiteLLM models. +""" + +from __future__ import annotations + +import logging +import os +from typing import Optional + +from agents.extensions.models.litellm_model import LitellmModel + +LOGGER = logging.getLogger(__name__) + +DEFAULT_VERTEX_MODEL = "vertex_ai/gemini-2.0-flash-exp" +DEFAULT_OPENAI_MODEL = "openai/gpt-4o-mini" + +# Default GCP configuration (override with environment variables) +# These are defaults only - actual values should come from environment variables +GCP_PROJECT_ID = "your-gcp-project-id" # Set via GCP_PROJECT_ID env var +GCP_REGION = "us-central1" # Set via GCP_REGION env var +# Gemini model (default) +VERTEX_AI_MODEL = "vertex_ai/gemini-2.0-flash-exp" +# Optional: OpenAI API key (if using OpenAI alongside Gemini) +# Should be retrieved from Secret Manager, not hardcoded +OPENAI_API_KEY_SECRET = None # Use Secret Manager instead +CLOUD_RUN_SERVICE_ACCOUNT_EMAIL = "cloud-run-sa@your-gcp-project-id.iam.gserviceaccount.com" # Set via env var + + +def _get_project_id() -> Optional[str]: + """Return the GCP project id from environment.""" + return os.getenv("GCP_PROJECT_ID") or os.getenv("PROJECT_ID") + + +def build_vertex_model(model_name: str) -> LitellmModel: + """Create a LitellmModel configured for Vertex AI.""" + project_id = _get_project_id() + if not project_id: + raise ValueError( + "GCP_PROJECT_ID (or PROJECT_ID) must be set to use Vertex AI models." + ) + + region = os.getenv("GCP_REGION", "us-central1") + + # Set environment variables for LiteLLM to use with Vertex AI + # LiteLLM reads these automatically when using vertex_ai/ model prefix + + # Use environment variable if set, otherwise use default (for local dev only) + if not os.getenv("GCP_PROJECT_ID"): + os.environ["GCP_PROJECT_ID"] = project_id + if not os.getenv("GCP_REGION"): + os.environ["GCP_REGION"] = region + # LiteLLM also needs AWS_REGION_NAME for some providers, but for Vertex AI it uses GCP_PROJECT_ID + # However, some LiteLLM versions expect VERTEX_PROJECT and VERTEX_LOCATION + os.environ["VERTEX_PROJECT"] = project_id + os.environ["VERTEX_LOCATION"] = region + + LOGGER.info("Using Vertex AI model '%s' in project '%s' (%s)", model_name, project_id, region) + # LitellmModel only accepts model and api_key - project/location come from env vars + return LitellmModel(model=model_name) + + +def build_openai_model(model_name: str) -> LitellmModel: + """Create a LitellmModel configured for OpenAI models.""" + # Get API key from environment variable or Secret Manager + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + # Try to get from Secret Manager if secret ID is provided + secret_id = os.getenv("OPENAI_API_KEY_SECRET_ID") + if secret_id: + try: + from google.cloud import secretmanager + project_id = _get_project_id() + if project_id: + client = secretmanager.SecretManagerServiceClient() + name = f"projects/{project_id}/secrets/{secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + api_key = response.payload.data.decode("UTF-8") + except Exception as e: + LOGGER.warning(f"Could not retrieve OpenAI API key from Secret Manager: {e}") + + if not api_key: + raise ValueError( + "OPENAI_API_KEY must be set (via environment variable or Secret Manager) when using OPENAI provider." + ) + + api_base = os.getenv("OPENAI_API_BASE") + extra_kwargs = {"api_base": api_base} if api_base else {} + + LOGGER.info("Using OpenAI model '%s'", model_name) + return LitellmModel(model=model_name, api_key=api_key, **extra_kwargs) + + +def get_litellm_model(model_override: Optional[str] = None) -> LitellmModel: + """ + Build a LitellmModel using environment configuration. + + Priority: + 1. Respect explicit override (e.g., per-agent needs). + 2. Use OPENAI provider if LLM_PROVIDER=openai. + 3. Default to Vertex AI Gemini 2.0 Flash. + """ + + provider = os.getenv("LLM_PROVIDER", "vertex_ai").lower() + + if provider == "openai": + model_name = model_override or os.getenv("OPENAI_MODEL", DEFAULT_OPENAI_MODEL) + return build_openai_model(model_name) + + # Default to Vertex AI + model_name = model_override or os.getenv("VERTEX_AI_MODEL", DEFAULT_VERTEX_MODEL) + return build_vertex_model(model_name) + diff --git a/gcp-deployment/backend/database/.python-version b/gcp-deployment/backend/database/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/database/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/database/migrations/001_schema.sql b/gcp-deployment/backend/database/migrations/001_schema.sql new file mode 100644 index 00000000..abef076b --- /dev/null +++ b/gcp-deployment/backend/database/migrations/001_schema.sql @@ -0,0 +1,119 @@ +-- Alex Financial Planner Database Schema +-- Version: 001 +-- Description: Initial schema for multi-user financial planning platform + +-- Enable UUID extension for gen_random_uuid() +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- Minimal users table (Clerk handles auth) +CREATE TABLE IF NOT EXISTS users ( + clerk_user_id VARCHAR(255) PRIMARY KEY, + display_name VARCHAR(255), + years_until_retirement INTEGER, + target_retirement_income DECIMAL(12,2), -- Annual income goal + + -- Allocation targets for rebalancing (stored as JSON) + asset_class_targets JSONB DEFAULT '{"equity": 70, "fixed_income": 30}', + region_targets JSONB DEFAULT '{"north_america": 50, "international": 50}', + + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +-- Reference data for instruments +CREATE TABLE IF NOT EXISTS instruments ( + symbol VARCHAR(20) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + instrument_type VARCHAR(50), -- 'equity', 'etf', 'mutual_fund', 'bond_fund' + current_price DECIMAL(12,4), -- Current price for portfolio calculations + + -- Allocation percentages (0-100, stored as JSON) + allocation_regions JSONB DEFAULT '{}', -- {"north_america": 60, "europe": 20, "asia": 20} + allocation_sectors JSONB DEFAULT '{}', -- {"technology": 30, "healthcare": 20, ...} + allocation_asset_class JSONB DEFAULT '{}', -- {"equity": 80, "fixed_income": 20} + + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +-- User's investment accounts +CREATE TABLE IF NOT EXISTS accounts ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + clerk_user_id VARCHAR(255) REFERENCES users(clerk_user_id) ON DELETE CASCADE, + account_name VARCHAR(255) NOT NULL, -- "401k", "Roth IRA" + account_purpose TEXT, -- "Long-term retirement savings" + cash_balance DECIMAL(12,2) DEFAULT 0, -- Uninvested cash + cash_interest DECIMAL(5,4) DEFAULT 0, -- Annual interest rate (0.045 = 4.5%) + + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +-- Current positions in each account +CREATE TABLE IF NOT EXISTS positions ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, + symbol VARCHAR(20) REFERENCES instruments(symbol), + quantity DECIMAL(20,8) NOT NULL, -- Supports fractional shares + as_of_date DATE DEFAULT CURRENT_DATE, + + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + + -- Ensure no duplicate positions per account + UNIQUE(account_id, symbol) +); + +-- Jobs tracking for async analysis +CREATE TABLE IF NOT EXISTS jobs ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + clerk_user_id VARCHAR(255) REFERENCES users(clerk_user_id) ON DELETE CASCADE, + job_type VARCHAR(50) NOT NULL, -- 'portfolio_analysis', 'rebalance', 'projection' + status VARCHAR(20) DEFAULT 'pending', -- 'pending', 'running', 'completed', 'failed' + request_payload JSONB, -- Input parameters + + -- Separate fields for each agent's results (no merging needed) + report_payload JSONB, -- Reporter agent's markdown analysis + charts_payload JSONB, -- Charter agent's visualization data + retirement_payload JSONB, -- Retirement agent's projections + summary_payload JSONB, -- Planner's final summary/metadata + + error_message TEXT, + + created_at TIMESTAMP DEFAULT NOW(), + started_at TIMESTAMP, + completed_at TIMESTAMP, + updated_at TIMESTAMP DEFAULT NOW() +); + +-- Create indexes for common queries +CREATE INDEX IF NOT EXISTS idx_accounts_user ON accounts(clerk_user_id); +CREATE INDEX IF NOT EXISTS idx_positions_account ON positions(account_id); +CREATE INDEX IF NOT EXISTS idx_positions_symbol ON positions(symbol); +CREATE INDEX IF NOT EXISTS idx_jobs_user ON jobs(clerk_user_id); +CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status); + +-- Create update timestamp trigger function +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Add update triggers to tables with updated_at +CREATE TRIGGER update_users_updated_at BEFORE UPDATE ON users + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_instruments_updated_at BEFORE UPDATE ON instruments + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_accounts_updated_at BEFORE UPDATE ON accounts + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_positions_updated_at BEFORE UPDATE ON positions + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_jobs_updated_at BEFORE UPDATE ON jobs + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); \ No newline at end of file diff --git a/gcp-deployment/backend/database/pyproject.toml b/gcp-deployment/backend/database/pyproject.toml new file mode 100644 index 00000000..27a2c27d --- /dev/null +++ b/gcp-deployment/backend/database/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "alex-database" +version = "0.1.0" +description = "Database package for Alex Financial Planner with Aurora Data API" +requires-python = ">=3.12" +dependencies = [ + "boto3>=1.40.8", # For Aurora Data API (backward compatibility) + "psycopg2-binary>=2.9.9", # For Cloud SQL PostgreSQL + "cloud-sql-python-connector>=1.11.0", # For Cloud SQL on Cloud Run + "pg8000>=1.31.2", # Required by Cloud SQL connector for PostgreSQL + "google-cloud-secret-manager>=2.18.0", # For retrieving passwords + "pydantic>=2.11.7", + "python-dotenv>=1.1.1", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src"] diff --git a/gcp-deployment/backend/database/reset_db.py b/gcp-deployment/backend/database/reset_db.py new file mode 100644 index 00000000..8dc83c87 --- /dev/null +++ b/gcp-deployment/backend/database/reset_db.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Database Reset Script +Drops all tables, recreates schema, and loads seed data +""" + +import sys +import argparse +from pathlib import Path +from src.client import DataAPIClient +from src.models import Database +from src.schemas import UserCreate, AccountCreate, PositionCreate +from decimal import Decimal + + +def drop_all_tables(db: DataAPIClient): + """Drop all tables in correct order (respecting foreign keys)""" + print("๐Ÿ—‘๏ธ Dropping existing tables...") + + # Order matters due to foreign key constraints + tables_to_drop = [ + 'positions', + 'accounts', + 'jobs', + 'instruments', + 'users' + ] + + for table in tables_to_drop: + try: + db.execute(f"DROP TABLE IF EXISTS {table} CASCADE") + print(f" โœ… Dropped {table}") + except Exception as e: + print(f" โš ๏ธ Error dropping {table}: {e}") + + # Also drop the function + try: + db.execute("DROP FUNCTION IF EXISTS update_updated_at_column() CASCADE") + print(f" โœ… Dropped update_updated_at_column function") + except Exception as e: + print(f" โš ๏ธ Error dropping function: {e}") + + +def create_test_data(db_models: Database): + """Create test user with sample portfolio""" + print("\n๐Ÿ‘ค Creating test user and portfolio...") + + # Create test user with Pydantic validation + user_data = UserCreate( + clerk_user_id='test_user_001', + display_name='Test User', + years_until_retirement=25, + target_retirement_income=Decimal('100000') + ) + + # Check if user exists + existing = db_models.users.find_by_clerk_id('test_user_001') + if existing: + print(" โ„น๏ธ Test user already exists") + else: + # Use validated data from Pydantic model + validated = user_data.model_dump() + db_models.users.create_user( + clerk_user_id=validated['clerk_user_id'], + display_name=validated['display_name'], + years_until_retirement=validated['years_until_retirement'], + target_retirement_income=validated['target_retirement_income'] + ) + print(" โœ… Created test user") + + # Create test accounts with Pydantic validation + accounts = [ + AccountCreate( + account_name='401(k)', + account_purpose='Primary retirement savings', + cash_balance=Decimal('5000'), + cash_interest=Decimal('0.045') + ), + AccountCreate( + account_name='Roth IRA', + account_purpose='Tax-free retirement savings', + cash_balance=Decimal('1000'), + cash_interest=Decimal('0.04') + ), + AccountCreate( + account_name='Taxable Brokerage', + account_purpose='General investment account', + cash_balance=Decimal('2500'), + cash_interest=Decimal('0.035') + ) + ] + + user_accounts = db_models.accounts.find_by_user('test_user_001') + + if user_accounts: + print(f" โ„น๏ธ User already has {len(user_accounts)} accounts") + account_ids = [acc['id'] for acc in user_accounts] + else: + account_ids = [] + for acc_data in accounts: + validated = acc_data.model_dump() + acc_id = db_models.accounts.create_account( + 'test_user_001', + account_name=validated['account_name'], + account_purpose=validated['account_purpose'], + cash_balance=validated['cash_balance'], + cash_interest=validated['cash_interest'] + ) + account_ids.append(acc_id) + print(f" โœ… Created account: {validated['account_name']}") + + # Create test positions in first account (401k) + if account_ids: + positions = [ + ('SPY', Decimal('100')), # $45,000 approx + ('QQQ', Decimal('50')), # $20,000 approx + ('BND', Decimal('200')), # $16,000 approx + ('VEA', Decimal('150')), # $7,500 approx + ('GLD', Decimal('25')), # $5,000 approx + ] + + account_id = account_ids[0] + existing_positions = db_models.positions.find_by_account(account_id) + + if existing_positions: + print(f" โ„น๏ธ Account already has {len(existing_positions)} positions") + else: + for symbol, quantity in positions: + # Validate position with Pydantic + position = PositionCreate( + account_id=account_id, + symbol=symbol, + quantity=quantity + ) + validated = position.model_dump() + db_models.positions.add_position( + validated['account_id'], + validated['symbol'], + validated['quantity'] + ) + print(f" โœ… Added position: {quantity} shares of {symbol}") + + +def main(): + parser = argparse.ArgumentParser(description='Reset Alex database') + parser.add_argument('--with-test-data', action='store_true', + help='Create test user with sample portfolio') + parser.add_argument('--skip-drop', action='store_true', + help='Skip dropping tables (just reload data)') + args = parser.parse_args() + + print("๐Ÿš€ Database Reset Script") + print("=" * 50) + + # Initialize database + db = DataAPIClient() + db_models = Database() + + if not args.skip_drop: + # Drop all tables + drop_all_tables(db) + + # Run migrations + print("\n๐Ÿ“ Running migrations...") + import subprocess + result = subprocess.run(['uv', 'run', 'run_migrations.py'], + capture_output=True, text=True) + + if result.returncode != 0: + print("โŒ Migration failed!") + print(result.stderr) + sys.exit(1) + else: + print("โœ… Migrations completed") + + # Load seed data + print("\n๐ŸŒฑ Loading seed data...") + import subprocess + result = subprocess.run(['uv', 'run', 'seed_data.py'], + capture_output=True, text=True) + + if result.returncode != 0: + print("โŒ Seed data failed!") + print(result.stderr) + sys.exit(1) + else: + # Extract instrument count from output + if '22/22 instruments loaded' in result.stdout: + print("โœ… Loaded 22 instruments") + else: + print("โœ… Seed data loaded") + + # Create test data if requested + if args.with_test_data: + create_test_data(db_models) + + # Final verification + print("\n๐Ÿ” Final verification...") + + # Count records + tables = ['users', 'instruments', 'accounts', 'positions', 'jobs'] + for table in tables: + result = db.query(f"SELECT COUNT(*) as count FROM {table}") + count = result[0]['count'] if result else 0 + print(f" โ€ข {table}: {count} records") + + print("\n" + "=" * 50) + print("โœ… Database reset complete!") + + if args.with_test_data: + print("\n๐Ÿ“ Test user created:") + print(" โ€ข User ID: test_user_001") + print(" โ€ข 3 accounts (401k, Roth IRA, Taxable)") + print(" โ€ข 5 positions in 401k account") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/database/run_migrations.py b/gcp-deployment/backend/database/run_migrations.py new file mode 100644 index 00000000..05da95bf --- /dev/null +++ b/gcp-deployment/backend/database/run_migrations.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Simple migration runner that executes statements one by one +""" + +import os +from pathlib import Path + +from dotenv import load_dotenv + +# Ensure we can import the shared Cloud SQL client +BASE_DIR = Path(__file__).resolve().parent +SRC_DIR = BASE_DIR / "src" +import sys +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + +from cloudsql_client import CloudSQLClient + +# Load environment variables +load_dotenv(override=True) + +# Initialize Cloud SQL client (reads env vars for host/instance/user/password) +client = CloudSQLClient( + instance_connection_name=os.environ.get("INSTANCE_CONNECTION_NAME"), + database=os.environ.get("DATABASE_NAME", "alex"), + user=os.environ.get("DATABASE_USER", "alex_app"), + password=os.environ.get("DB_PASSWORD"), + host=os.environ.get("DB_HOST"), + port=int(os.environ.get("DB_PORT", "5432")), +) + +# Read migration file +with open("migrations/001_schema.sql") as f: + sql = f.read() + +# Define statements in order (since splitting is complex) +statements = [ + # Extension + 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"', + # Tables + """CREATE TABLE IF NOT EXISTS users ( + clerk_user_id VARCHAR(255) PRIMARY KEY, + display_name VARCHAR(255), + years_until_retirement INTEGER, + target_retirement_income DECIMAL(12,2), + asset_class_targets JSONB DEFAULT '{"equity": 70, "fixed_income": 30}', + region_targets JSONB DEFAULT '{"north_america": 50, "international": 50}', + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + )""", + """CREATE TABLE IF NOT EXISTS instruments ( + symbol VARCHAR(20) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + instrument_type VARCHAR(50), + current_price DECIMAL(12,4), + allocation_regions JSONB DEFAULT '{}', + allocation_sectors JSONB DEFAULT '{}', + allocation_asset_class JSONB DEFAULT '{}', + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + )""", + """CREATE TABLE IF NOT EXISTS accounts ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + clerk_user_id VARCHAR(255) REFERENCES users(clerk_user_id) ON DELETE CASCADE, + account_name VARCHAR(255) NOT NULL, + account_purpose TEXT, + cash_balance DECIMAL(12,2) DEFAULT 0, + cash_interest DECIMAL(5,4) DEFAULT 0, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + )""", + """CREATE TABLE IF NOT EXISTS positions ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, + symbol VARCHAR(20) REFERENCES instruments(symbol), + quantity DECIMAL(20,8) NOT NULL, + as_of_date DATE DEFAULT CURRENT_DATE, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + UNIQUE(account_id, symbol) + )""", + """CREATE TABLE IF NOT EXISTS jobs ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + clerk_user_id VARCHAR(255) REFERENCES users(clerk_user_id) ON DELETE CASCADE, + job_type VARCHAR(50) NOT NULL, + status VARCHAR(20) DEFAULT 'pending', + request_payload JSONB, + report_payload JSONB, + charts_payload JSONB, + retirement_payload JSONB, + summary_payload JSONB, + error_message TEXT, + created_at TIMESTAMP DEFAULT NOW(), + started_at TIMESTAMP, + completed_at TIMESTAMP, + updated_at TIMESTAMP DEFAULT NOW() + )""", + # Indexes + "CREATE INDEX IF NOT EXISTS idx_accounts_user ON accounts(clerk_user_id)", + "CREATE INDEX IF NOT EXISTS idx_positions_account ON positions(account_id)", + "CREATE INDEX IF NOT EXISTS idx_positions_symbol ON positions(symbol)", + "CREATE INDEX IF NOT EXISTS idx_jobs_user ON jobs(clerk_user_id)", + "CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)", + # Function for timestamps + """CREATE OR REPLACE FUNCTION update_updated_at_column() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = NOW(); + RETURN NEW; + END; + $$ LANGUAGE plpgsql""", + # Triggers + """CREATE TRIGGER update_users_updated_at BEFORE UPDATE ON users + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()""", + """CREATE TRIGGER update_instruments_updated_at BEFORE UPDATE ON instruments + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()""", + """CREATE TRIGGER update_accounts_updated_at BEFORE UPDATE ON accounts + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()""", + """CREATE TRIGGER update_positions_updated_at BEFORE UPDATE ON positions + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()""", + """CREATE TRIGGER update_jobs_updated_at BEFORE UPDATE ON jobs + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()""", +] + +print("๐Ÿš€ Running database migrations...") +print("=" * 50) + +success_count = 0 +error_count = 0 + +for i, stmt in enumerate(statements, 1): + # Get a description of what we're creating + stmt_type = "statement" + if "CREATE TABLE" in stmt.upper(): + stmt_type = "table" + elif "CREATE INDEX" in stmt.upper(): + stmt_type = "index" + elif "CREATE TRIGGER" in stmt.upper(): + stmt_type = "trigger" + elif "CREATE FUNCTION" in stmt.upper(): + stmt_type = "function" + elif "CREATE EXTENSION" in stmt.upper(): + stmt_type = "extension" + + # First non-empty line for display + first_line = next(l for l in stmt.split("\n") if l.strip())[:60] + print(f"\n[{i}/{len(statements)}] Creating {stmt_type}...") + print(f" {first_line}...") + + try: + client.execute(sql=stmt) + print(f" โœ… Success") + success_count += 1 + + except Exception as e: + error_msg = str(e) + if "already exists" in error_msg.lower(): + print(f" โš ๏ธ Already exists (skipping)") + success_count += 1 + else: + print(f" โŒ Error: {error_msg[:100]}") + error_count += 1 + +print("\n" + "=" * 50) +print(f"Migration complete: {success_count} successful, {error_count} errors") + +if error_count == 0: + print("\nโœ… All migrations completed successfully!") + print("\n๐Ÿ“ Next steps:") + print("1. Load seed data: uv run seed_data.py") + print("2. Test database operations: uv run test_db.py") +else: + print(f"\nโš ๏ธ Some statements failed. Check errors above.") diff --git a/gcp-deployment/backend/database/seed_data.py b/gcp-deployment/backend/database/seed_data.py new file mode 100644 index 00000000..20b85092 --- /dev/null +++ b/gcp-deployment/backend/database/seed_data.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +""" +Seed data for Alex Financial Planner +Loads 20+ popular ETF instruments with allocation data +""" + +import os +import json +from pathlib import Path + +from pydantic import ValidationError +from dotenv import load_dotenv + +from src.schemas import InstrumentCreate +import sys + +# Make sure src directory is importable +BASE_DIR = Path(__file__).resolve().parent +SRC_DIR = BASE_DIR / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + +from cloudsql_client import CloudSQLClient + +# Load environment variables +load_dotenv(override=True) + +# Initialize Cloud SQL client +client = CloudSQLClient( + instance_connection_name=os.environ.get("INSTANCE_CONNECTION_NAME"), + database=os.environ.get("DATABASE_NAME", "alex"), + user=os.environ.get("DATABASE_USER", "alex_app"), + password=os.environ.get("DB_PASSWORD"), + host=os.environ.get("DB_HOST"), + port=int(os.environ.get("DB_PORT", "5432")), +) + +# Define popular ETF instruments with realistic allocation data +# All percentages should sum to 100 for each allocation type +INSTRUMENTS = [ + # Core US Equity + { + "symbol": "SPY", + "name": "SPDR S&P 500 ETF Trust", + "instrument_type": "etf", + "current_price": 450.25, # Approximate prices as of 2024 + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "technology": 28, + "healthcare": 13, + "financials": 13, + "consumer_discretionary": 12, + "industrials": 9, + "communication": 9, + "consumer_staples": 6, + "energy": 4, + "utilities": 3, + "real_estate": 2, + "materials": 1, + }, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "QQQ", + "name": "Invesco QQQ Trust", + "instrument_type": "etf", + "current_price": 385.50, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "technology": 50, + "communication": 17, + "consumer_discretionary": 15, + "healthcare": 8, + "consumer_staples": 5, + "industrials": 3, + "other": 2, + }, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "IWM", + "name": "iShares Russell 2000 ETF", + "instrument_type": "etf", + "current_price": 205.75, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "healthcare": 18, + "financials": 17, + "industrials": 16, + "technology": 14, + "consumer_discretionary": 12, + "real_estate": 7, + "energy": 6, + "materials": 4, + "consumer_staples": 3, + "utilities": 2, + "communication": 1, + }, + "allocation_asset_class": {"equity": 100}, + }, + # International Equity + { + "symbol": "VEA", + "name": "Vanguard FTSE Developed Markets ETF", + "instrument_type": "etf", + "current_price": 48.30, + "allocation_regions": {"europe": 60, "asia": 35, "oceania": 5}, + "allocation_sectors": { + "financials": 18, + "industrials": 14, + "healthcare": 12, + "consumer_discretionary": 11, + "technology": 10, + "consumer_staples": 9, + "materials": 8, + "energy": 6, + "communication": 5, + "utilities": 4, + "real_estate": 3, + }, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "VWO", + "name": "Vanguard FTSE Emerging Markets ETF", + "instrument_type": "etf", + "current_price": 42.15, + "allocation_regions": {"asia": 75, "latin_america": 10, "africa": 8, "europe": 7}, + "allocation_sectors": { + "technology": 22, + "financials": 20, + "consumer_discretionary": 15, + "communication": 10, + "energy": 8, + "materials": 7, + "industrials": 6, + "consumer_staples": 5, + "healthcare": 4, + "utilities": 2, + "real_estate": 1, + }, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "EFA", + "name": "iShares MSCI EAFE ETF", + "instrument_type": "etf", + "current_price": 75.80, + "allocation_regions": {"europe": 65, "asia": 35}, + "allocation_sectors": { + "financials": 17, + "industrials": 15, + "healthcare": 13, + "consumer_discretionary": 12, + "consumer_staples": 10, + "technology": 9, + "materials": 8, + "energy": 5, + "communication": 5, + "utilities": 3, + "real_estate": 3, + }, + "allocation_asset_class": {"equity": 100}, + }, + # Fixed Income + { + "symbol": "AGG", + "name": "iShares Core U.S. Aggregate Bond ETF", + "instrument_type": "bond_fund", + "current_price": 98.20, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "treasury": 40, + "corporate": 25, + "mortgage": 28, + "government_related": 7, + }, + "allocation_asset_class": {"fixed_income": 100}, + }, + { + "symbol": "BND", + "name": "Vanguard Total Bond Market ETF", + "instrument_type": "bond_fund", + "current_price": 72.50, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "treasury": 42, + "corporate": 24, + "mortgage": 27, + "government_related": 7, + }, + "allocation_asset_class": {"fixed_income": 100}, + }, + { + "symbol": "TLT", + "name": "iShares 20+ Year Treasury Bond ETF", + "instrument_type": "bond_fund", + "current_price": 92.30, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"treasury": 100}, + "allocation_asset_class": {"fixed_income": 100}, + }, + { + "symbol": "HYG", + "name": "iShares iBoxx High Yield Corporate Bond ETF", + "instrument_type": "bond_fund", + "current_price": 76.85, + "allocation_regions": {"north_america": 95, "international": 5}, + "allocation_sectors": {"corporate": 100}, + "allocation_asset_class": {"fixed_income": 100}, + }, + # Sector ETFs + { + "symbol": "XLK", + "name": "Technology Select Sector SPDR Fund", + "instrument_type": "etf", + "current_price": 175.40, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"technology": 100}, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "XLV", + "name": "Health Care Select Sector SPDR Fund", + "instrument_type": "etf", + "current_price": 135.60, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"healthcare": 100}, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "XLF", + "name": "Financial Select Sector SPDR Fund", + "instrument_type": "etf", + "current_price": 38.25, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"financials": 100}, + "allocation_asset_class": {"equity": 100}, + }, + { + "symbol": "XLE", + "name": "Energy Select Sector SPDR Fund", + "instrument_type": "etf", + "current_price": 85.90, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"energy": 100}, + "allocation_asset_class": {"equity": 100}, + }, + # Real Estate + { + "symbol": "VNQ", + "name": "Vanguard Real Estate ETF", + "instrument_type": "etf", + "current_price": 82.45, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": {"real_estate": 100}, + "allocation_asset_class": {"real_estate": 100}, + }, + # Commodities + { + "symbol": "GLD", + "name": "SPDR Gold Shares", + "instrument_type": "etf", + "current_price": 195.70, + "allocation_regions": {"global": 100}, + "allocation_sectors": {"commodities": 100}, + "allocation_asset_class": {"commodities": 100}, + }, + { + "symbol": "SLV", + "name": "iShares Silver Trust", + "instrument_type": "etf", + "current_price": 22.40, + "allocation_regions": {"global": 100}, + "allocation_sectors": {"commodities": 100}, + "allocation_asset_class": {"commodities": 100}, + }, + # Mixed/Balanced + { + "symbol": "AOR", + "name": "iShares Core Growth Allocation ETF", + "instrument_type": "etf", + "current_price": 48.90, + "allocation_regions": {"north_america": 60, "international": 40}, + "allocation_sectors": {"diversified": 100}, + "allocation_asset_class": {"equity": 60, "fixed_income": 40}, + }, + { + "symbol": "AOA", + "name": "iShares Core Aggressive Allocation ETF", + "instrument_type": "etf", + "current_price": 65.15, + "allocation_regions": {"north_america": 55, "international": 45}, + "allocation_sectors": {"diversified": 100}, + "allocation_asset_class": {"equity": 80, "fixed_income": 20}, + }, + # Growth ETFs + { + "symbol": "VUG", + "name": "Vanguard Growth ETF", + "instrument_type": "etf", + "current_price": 312.80, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "technology": 45, + "consumer_discretionary": 18, + "healthcare": 12, + "industrials": 10, + "communication": 8, + "financials": 4, + "other": 3, + }, + "allocation_asset_class": {"equity": 100}, + }, + # Value ETFs + { + "symbol": "VTV", + "name": "Vanguard Value ETF", + "instrument_type": "etf", + "current_price": 152.60, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "financials": 20, + "healthcare": 18, + "industrials": 12, + "consumer_staples": 11, + "energy": 10, + "utilities": 8, + "communication": 7, + "materials": 6, + "technology": 5, + "other": 3, + }, + "allocation_asset_class": {"equity": 100}, + }, + # Dividend ETFs + { + "symbol": "VIG", + "name": "Vanguard Dividend Appreciation ETF", + "instrument_type": "etf", + "current_price": 168.90, + "allocation_regions": {"north_america": 100}, + "allocation_sectors": { + "technology": 22, + "healthcare": 16, + "financials": 14, + "consumer_staples": 13, + "industrials": 12, + "consumer_discretionary": 10, + "utilities": 5, + "materials": 4, + "other": 4, + }, + "allocation_asset_class": {"equity": 100}, + }, +] + + +def insert_instrument(instrument_data): + """Insert a single instrument into the database with Pydantic validation""" + # Validate with Pydantic first + try: + instrument = InstrumentCreate(**instrument_data) + except ValidationError as e: + print(f" โŒ Validation error: {e}") + return False + + # Get validated data + validated = instrument.model_dump() + + sql = """ + INSERT INTO instruments ( + symbol, name, instrument_type, current_price, + allocation_regions, allocation_sectors, allocation_asset_class + ) VALUES ( + :symbol, :name, :instrument_type, :current_price::numeric, + :allocation_regions::jsonb, :allocation_sectors::jsonb, :allocation_asset_class::jsonb + ) + ON CONFLICT (symbol) DO UPDATE SET + name = EXCLUDED.name, + instrument_type = EXCLUDED.instrument_type, + current_price = EXCLUDED.current_price, + allocation_regions = EXCLUDED.allocation_regions, + allocation_sectors = EXCLUDED.allocation_sectors, + allocation_asset_class = EXCLUDED.allocation_asset_class, + updated_at = NOW() + """ + + parameters = [ + {"name": "symbol", "value": {"stringValue": validated["symbol"]}}, + {"name": "name", "value": {"stringValue": validated["name"]}}, + {"name": "instrument_type", "value": {"stringValue": validated["instrument_type"]}}, + { + "name": "current_price", + "value": {"stringValue": str(validated.get("current_price") or 0)}, + }, + { + "name": "allocation_regions", + "value": {"stringValue": json.dumps(validated["allocation_regions"]), "skip_json_decode": True}, + }, + { + "name": "allocation_sectors", + "value": {"stringValue": json.dumps(validated["allocation_sectors"]), "skip_json_decode": True}, + }, + { + "name": "allocation_asset_class", + "value": {"stringValue": json.dumps(validated["allocation_asset_class"]), "skip_json_decode": True}, + }, + ] + + try: + client.execute(sql=sql, parameters=parameters) + return True + except Exception as e: + print(f" โŒ Error inserting: {str(e)[:100]}") + return False + + +def verify_allocations(instrument): + """Verify instrument using Pydantic validation""" + try: + InstrumentCreate(**instrument) + return [] # No errors + except ValidationError as e: + # Extract error messages + errors = [] + for error in e.errors(): + field = ".".join(str(x) for x in error["loc"]) + msg = error["msg"] + errors.append(f"{field}: {msg}") + return errors + + +def main(): + print("๐Ÿš€ Seeding Instrument Data") + print("=" * 50) + print(f"Loading {len(INSTRUMENTS)} instruments...") + + # First verify all allocations + print("\n๐Ÿ“Š Verifying allocation data...") + all_valid = True + for inst in INSTRUMENTS: + errors = verify_allocations(inst) + if errors: + print(f" โŒ {inst['symbol']}: {', '.join(errors)}") + all_valid = False + + if not all_valid: + print("\nโŒ Some instruments have invalid allocations. Please fix before continuing.") + exit(1) + + print(" โœ… All allocations valid!") + + # Insert instruments + print("\n๐Ÿ’พ Inserting instruments...") + success_count = 0 + + for inst in INSTRUMENTS: + print( + f" [{success_count + 1}/{len(INSTRUMENTS)}] {inst['symbol']}: {inst['name'][:40]}..." + ) + if insert_instrument(inst): + print(f" โœ… Success") + success_count += 1 + else: + print(f" โŒ Failed") + + print("\n" + "=" * 50) + print(f"Seeding complete: {success_count}/{len(INSTRUMENTS)} instruments loaded") + + # Verify by querying + print("\n๐Ÿ” Verifying data...") + try: + response = client.query("SELECT COUNT(*) as count FROM instruments") + count = response[0]["count"] if response else 0 + print(f" Database now contains {count} instruments") + + # Show a sample + response = client.query("SELECT symbol, name FROM instruments ORDER BY symbol LIMIT 5") + + print("\n Sample instruments:") + for record in response: + symbol = record["symbol"] + name = record["name"] + print(f" - {symbol}: {name}") + + except Exception as e: + print(f" โŒ Error verifying: {e}") + + print("\nโœ… Seed data loaded successfully!") + print("\n๐Ÿ“ Next steps:") + print("1. Create test user and portfolio: uv run create_test_data.py") + print("2. Test database operations: uv run test_db.py") + + +if __name__ == "__main__": + main() diff --git a/gcp-deployment/backend/database/src/__init__.py b/gcp-deployment/backend/database/src/__init__.py new file mode 100644 index 00000000..5bc75e95 --- /dev/null +++ b/gcp-deployment/backend/database/src/__init__.py @@ -0,0 +1,51 @@ +""" +Database package for Alex Financial Planner +Provides database models, schemas, and Data API client +""" + +from .client import DataAPIClient +from .models import Database +from .schemas import ( + # Types + RegionType, + AssetClassType, + SectorType, + InstrumentType, + JobType, + JobStatus, + AccountType, + + # Create schemas (for inputs) + InstrumentCreate, + UserCreate, + AccountCreate, + PositionCreate, + JobCreate, + JobUpdate, + + # Response schemas (for outputs) + InstrumentResponse, + PortfolioAnalysis, + RebalanceRecommendation, +) + +__all__ = [ + 'Database', + 'DataAPIClient', + 'InstrumentCreate', + 'UserCreate', + 'AccountCreate', + 'PositionCreate', + 'JobCreate', + 'JobUpdate', + 'InstrumentResponse', + 'PortfolioAnalysis', + 'RebalanceRecommendation', + 'RegionType', + 'AssetClassType', + 'SectorType', + 'InstrumentType', + 'JobType', + 'JobStatus', + 'AccountType', +] \ No newline at end of file diff --git a/gcp-deployment/backend/database/src/client.py b/gcp-deployment/backend/database/src/client.py new file mode 100644 index 00000000..f91994e9 --- /dev/null +++ b/gcp-deployment/backend/database/src/client.py @@ -0,0 +1,310 @@ +""" +Aurora Data API Client Wrapper +Provides a simple interface for database operations +""" + +import boto3 +import json +import os +from typing import List, Dict, Any, Optional, Tuple +from datetime import date, datetime +from decimal import Decimal +from botocore.exceptions import ClientError +import logging + +# Try to load .env file if it exists +try: + from dotenv import load_dotenv + + load_dotenv(override=True) +except ImportError: + pass # dotenv not installed, continue without it + +logger = logging.getLogger(__name__) + + +class DataAPIClient: + """Wrapper for AWS RDS Data API to simplify database operations""" + + def __init__( + self, + cluster_arn: str = None, + secret_arn: str = None, + database: str = None, + region: str = None, + ): + """ + Initialize Data API client + + Args: + cluster_arn: Aurora cluster ARN (or from env AURORA_CLUSTER_ARN) + secret_arn: Secrets Manager ARN (or from env AURORA_SECRET_ARN) + database: Database name (or from env AURORA_DATABASE) + region: AWS region (or from env AWS_REGION) + """ + self.cluster_arn = cluster_arn or os.environ.get("AURORA_CLUSTER_ARN") + self.secret_arn = secret_arn or os.environ.get("AURORA_SECRET_ARN") + self.database = database or os.environ.get("AURORA_DATABASE", "alex") + + if not self.cluster_arn or not self.secret_arn: + raise ValueError( + "Missing required Aurora configuration. " + "Set AURORA_CLUSTER_ARN and AURORA_SECRET_ARN environment variables." + ) + + self.region = os.environ.get("DEFAULT_AWS_REGION", "us-east-1") + self.client = boto3.client("rds-data", region_name=self.region) + + def execute(self, sql: str, parameters: List[Dict] = None) -> Dict: + """ + Execute a SQL statement + + Args: + sql: SQL statement to execute + parameters: Optional list of parameters for prepared statement + + Returns: + Response from Data API + """ + try: + kwargs = { + "resourceArn": self.cluster_arn, + "secretArn": self.secret_arn, + "database": self.database, + "sql": sql, + "includeResultMetadata": True, # Include column names + } + + if parameters: + kwargs["parameters"] = parameters + + response = self.client.execute_statement(**kwargs) + return response + + except ClientError as e: + logger.error(f"Database error: {e}") + raise + + def query(self, sql: str, parameters: List[Dict] = None) -> List[Dict]: + """ + Execute a SELECT query and return results as list of dicts + + Args: + sql: SELECT statement + parameters: Optional parameters + + Returns: + List of dictionaries with column names as keys + """ + response = self.execute(sql, parameters) + + if "records" not in response: + return [] + + # Extract column names + columns = [col["name"] for col in response.get("columnMetadata", [])] + + # Convert records to dictionaries + results = [] + for record in response["records"]: + row = {} + for i, col in enumerate(columns): + value = self._extract_value(record[i]) + row[col] = value + results.append(row) + + return results + + def query_one(self, sql: str, parameters: List[Dict] = None) -> Optional[Dict]: + """ + Execute a SELECT query and return first result + + Args: + sql: SELECT statement + parameters: Optional parameters + + Returns: + Dictionary with column names as keys, or None if no results + """ + results = self.query(sql, parameters) + return results[0] if results else None + + def insert(self, table: str, data: Dict, returning: str = None) -> str: + """ + Insert a record into a table + + Args: + table: Table name + data: Dictionary of column names and values + returning: Column to return (e.g., 'id', 'clerk_user_id') + + Returns: + Value of returning column if specified + """ + columns = list(data.keys()) + placeholders = [] + + # Check if columns need type casting + for col in columns: + if isinstance(data[col], (dict, list)): + placeholders.append(f":{col}::jsonb") + elif isinstance(data[col], Decimal): + placeholders.append(f":{col}::numeric") + elif isinstance(data[col], date) and not isinstance(data[col], datetime): + placeholders.append(f":{col}::date") + elif isinstance(data[col], datetime): + placeholders.append(f":{col}::timestamp") + else: + placeholders.append(f":{col}") + + sql = f""" + INSERT INTO {table} ({", ".join(columns)}) + VALUES ({", ".join(placeholders)}) + """ + + # Add RETURNING clause if specified + if returning: + sql += f" RETURNING {returning}" + + parameters = self._build_parameters(data) + response = self.execute(sql, parameters) + + # Return value if RETURNING was used + if returning and response.get("records"): + return self._extract_value(response["records"][0][0]) + return None + + def update(self, table: str, data: Dict, where: str, where_params: Dict = None) -> int: + """ + Update records in a table + + Args: + table: Table name + data: Dictionary of columns to update + where: WHERE clause (without WHERE keyword) + where_params: Parameters for WHERE clause + + Returns: + Number of affected rows + """ + # Build SET clause with type casting where needed + set_parts = [] + for col, val in data.items(): + if isinstance(val, (dict, list)): + set_parts.append(f"{col} = :{col}::jsonb") + elif isinstance(val, Decimal): + set_parts.append(f"{col} = :{col}::numeric") + elif isinstance(val, date) and not isinstance(val, datetime): + set_parts.append(f"{col} = :{col}::date") + elif isinstance(val, datetime): + set_parts.append(f"{col} = :{col}::timestamp") + else: + set_parts.append(f"{col} = :{col}") + + set_clause = ", ".join(set_parts) + + sql = f""" + UPDATE {table} + SET {set_clause} + WHERE {where} + """ + + # Combine data and where parameters + all_params = {**data, **(where_params or {})} + parameters = self._build_parameters(all_params) + + response = self.execute(sql, parameters) + return response.get("numberOfRecordsUpdated", 0) + + def delete(self, table: str, where: str, where_params: Dict = None) -> int: + """ + Delete records from a table + + Args: + table: Table name + where: WHERE clause (without WHERE keyword) + where_params: Parameters for WHERE clause + + Returns: + Number of deleted rows + """ + sql = f"DELETE FROM {table} WHERE {where}" + parameters = self._build_parameters(where_params) if where_params else None + + response = self.execute(sql, parameters) + return response.get("numberOfRecordsUpdated", 0) + + def begin_transaction(self) -> str: + """Begin a database transaction""" + response = self.client.begin_transaction( + resourceArn=self.cluster_arn, secretArn=self.secret_arn, database=self.database + ) + return response["transactionId"] + + def commit_transaction(self, transaction_id: str): + """Commit a database transaction""" + self.client.commit_transaction( + resourceArn=self.cluster_arn, secretArn=self.secret_arn, transactionId=transaction_id + ) + + def rollback_transaction(self, transaction_id: str): + """Rollback a database transaction""" + self.client.rollback_transaction( + resourceArn=self.cluster_arn, secretArn=self.secret_arn, transactionId=transaction_id + ) + + def _build_parameters(self, data: Dict) -> List[Dict]: + """Convert dictionary to Data API parameter format""" + if not data: + return [] + + parameters = [] + for key, value in data.items(): + param = {"name": key} + + if value is None: + param["value"] = {"isNull": True} + elif isinstance(value, bool): + param["value"] = {"booleanValue": value} + elif isinstance(value, int): + param["value"] = {"longValue": value} + elif isinstance(value, float): + param["value"] = {"doubleValue": value} + elif isinstance(value, Decimal): + param["value"] = {"stringValue": str(value)} + elif isinstance(value, (date, datetime)): + param["value"] = {"stringValue": value.isoformat()} + elif isinstance(value, dict): + param["value"] = {"stringValue": json.dumps(value)} + elif isinstance(value, list): + param["value"] = {"stringValue": json.dumps(value)} + else: + param["value"] = {"stringValue": str(value)} + + parameters.append(param) + + return parameters + + def _extract_value(self, field: Dict) -> Any: + """Extract value from Data API field response""" + if field.get("isNull"): + return None + elif "booleanValue" in field: + return field["booleanValue"] + elif "longValue" in field: + return field["longValue"] + elif "doubleValue" in field: + return field["doubleValue"] + elif "stringValue" in field: + value = field["stringValue"] + # Try to parse JSON if it looks like JSON + if value and value[0] in ["{", "["]: + try: + return json.loads(value) + except json.JSONDecodeError: + pass + return value + elif "blobValue" in field: + return field["blobValue"] + else: + return None diff --git a/gcp-deployment/backend/database/src/cloudsql_client.py b/gcp-deployment/backend/database/src/cloudsql_client.py new file mode 100644 index 00000000..95775a77 --- /dev/null +++ b/gcp-deployment/backend/database/src/cloudsql_client.py @@ -0,0 +1,506 @@ +""" +Cloud SQL PostgreSQL Client Wrapper +Provides a simple interface for database operations compatible with DataAPIClient +""" + +import os +import json +import logging +from typing import List, Dict, Any, Optional +from datetime import date, datetime +from decimal import Decimal + +# Try to load .env file if it exists +try: + from dotenv import load_dotenv + load_dotenv(override=True) +except ImportError: + pass + +logger = logging.getLogger(__name__) + +# Try to import PostgreSQL libraries +try: + import psycopg2 + from psycopg2.extras import RealDictCursor, Json + PSYCOPG2_AVAILABLE = True +except ImportError: + PSYCOPG2_AVAILABLE = False + logger.warning("psycopg2 not available, falling back to pg8000") + try: + import pg8000 + PG8000_AVAILABLE = True + except ImportError: + PG8000_AVAILABLE = False + logger.error("Neither psycopg2 nor pg8000 available. Install one: uv add psycopg2-binary") + +# For Cloud Run, use Cloud SQL connector +try: + from google.cloud.sql.connector import Connector + CLOUD_SQL_CONNECTOR_AVAILABLE = True +except ImportError: + try: + # Try alternative import path + from cloud_sql_python_connector import Connector + CLOUD_SQL_CONNECTOR_AVAILABLE = True + except ImportError: + CLOUD_SQL_CONNECTOR_AVAILABLE = False + + +class CloudSQLClient: + """Wrapper for Cloud SQL PostgreSQL to simplify database operations""" + + def __init__( + self, + instance_connection_name: str = None, + database: str = None, + user: str = None, + password: str = None, + host: str = None, + port: int = None, + ): + """ + Initialize Cloud SQL client + + Args: + instance_connection_name: Cloud SQL connection name (for Cloud Run) + database: Database name (or from env DATABASE_NAME) + user: Database user (or from env DATABASE_USER) + password: Database password (or from env, or from Secret Manager) + host: Database host (for local development via proxy) + port: Database port (for local development via proxy) + """ + # Get configuration from environment or parameters + self.instance_connection_name = instance_connection_name or os.environ.get("INSTANCE_CONNECTION_NAME") + self.database = database or os.environ.get("DATABASE_NAME", "alex") + self.user = user or os.environ.get("DATABASE_USER", "alex_app") + + # Get password from Secret Manager if secret ID is provided + password_secret_id = os.environ.get("DB_PASSWORD_SECRET_ID") + if password_secret_id and not password: + try: + from google.cloud import secretmanager + client = secretmanager.SecretManagerServiceClient() + project_id = os.environ.get("GCP_PROJECT_ID") + name = f"projects/{project_id}/secrets/{password_secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + password = response.payload.data.decode("UTF-8") + except Exception as e: + logger.warning(f"Could not get password from Secret Manager: {e}") + + self.password = password or os.environ.get("DB_PASSWORD") + + # For local development via Cloud SQL Proxy + # Only use DB_HOST if explicitly set (for local dev with proxy) + # In Cloud Run, DB_HOST should NOT be set, allowing Cloud SQL connector to be used + db_host_env = os.environ.get("DB_HOST") + if host: + self.host = host + elif db_host_env: + self.host = db_host_env + else: + self.host = None # No host set = use Cloud SQL connector + + if self.host: + self.port = port or int(os.environ.get("DB_PORT", "5432")) + else: + self.port = None # Port not needed for Cloud SQL connector + + # Determine connection mode + # Use Cloud SQL connector if: + # 1. Connector is available + # 2. Instance connection name is set + # 3. Host is NOT explicitly set (meaning we're in Cloud Run, not local dev) + self.use_cloud_sql_connector = ( + CLOUD_SQL_CONNECTOR_AVAILABLE and + self.instance_connection_name and + not self.host # If host is explicitly set, use direct connection (local dev) + ) + + # Initialize connection pool + self._connection = None + self._connector = None + + if not self.password: + raise ValueError( + "Missing required database password. " + "Set DB_PASSWORD environment variable or DB_PASSWORD_SECRET_ID for Secret Manager." + ) + + def _get_connection(self): + """Get database connection""" + if self._connection: + try: + # Test if connection is still alive + if hasattr(self._connection, 'closed') and not self._connection.closed: + return self._connection + except: + pass + + if self.use_cloud_sql_connector: + # Use Cloud SQL connector for Cloud Run + if not self._connector: + self._connector = Connector() + + import pg8000 + conn = self._connector.connect( + self.instance_connection_name, + "pg8000", + user=self.user, + password=self.password, + db=self.database, + ) + self._connection = conn + return conn + else: + # Use direct connection (local development via proxy or private IP) + if PSYCOPG2_AVAILABLE: + conn = psycopg2.connect( + host=self.host, + port=self.port, + database=self.database, + user=self.user, + password=self.password, + ) + self._connection = conn + return conn + elif PG8000_AVAILABLE: + conn = pg8000.connect( + host=self.host, + port=self.port, + database=self.database, + user=self.user, + password=self.password, + ) + self._connection = conn + return conn + else: + raise ImportError("Neither psycopg2 nor pg8000 is available. Install one: uv add psycopg2-binary") + + def execute(self, sql: str, parameters: List[Dict] = None) -> Dict: + """ + Execute a SQL statement + + Args: + sql: SQL statement to execute + parameters: Optional list of parameters (DataAPIClient format) + + Returns: + Response dict with 'records' and 'columnMetadata' (compatible with DataAPIClient) + """ + conn = self._get_connection() + cursor = conn.cursor() + + try: + # Convert DataAPIClient parameter format to PostgreSQL format + if parameters: + # Extract parameter values and names + param_dict = {} + param_order = [] # Track parameter order for positional params + for param in parameters: + name = param.get('name', '') + value_obj = param.get('value', {}) + value = self._extract_param_value(value_obj) + # Remove : prefix if present + param_name = name.lstrip(':') + param_dict[param_name] = value + param_order.append(param_name) + + # Determine which driver is being used + # If using Cloud SQL connector, it uses pg8000 which only supports %s + # If using direct connection, check if psycopg2 is available + use_pg8000 = self.use_cloud_sql_connector or not PSYCOPG2_AVAILABLE + + if use_pg8000: + # pg8000 only supports %s positional parameters + # Convert :param_name to %s and build positional parameter list + sql_adapted = sql + param_list = [] + for param_name in param_order: + # Replace :param_name with %s + sql_adapted = sql_adapted.replace(f":{param_name}", "%s", 1) + param_list.append(param_dict[param_name]) + cursor.execute(sql_adapted, param_list) + else: + # psycopg2 supports named parameters %(name)s + sql_adapted = sql + for param_name in param_dict.keys(): + sql_adapted = sql_adapted.replace(f":{param_name}", f"%({param_name})s") + cursor.execute(sql_adapted, param_dict) + else: + cursor.execute(sql) + + # Get results + if cursor.description: + columns = [desc[0] for desc in cursor.description] + records = cursor.fetchall() + + # Convert to DataAPIClient format + result_records = [] + for record in records: + row = [] + for i, col in enumerate(columns): + value = record[i] + # Convert to DataAPIClient field format + field = self._value_to_field(value) + row.append(field) + result_records.append(row) + + column_metadata = [{'name': col} for col in columns] + + return { + 'records': result_records, + 'columnMetadata': column_metadata, + 'numberOfRecordsUpdated': cursor.rowcount if cursor.rowcount else 0 + } + else: + # No results (INSERT, UPDATE, DELETE) + return { + 'records': [], + 'columnMetadata': [], + 'numberOfRecordsUpdated': cursor.rowcount if cursor.rowcount else 0 + } + except Exception as e: + # Rollback on error to clear failed transaction state + try: + conn.rollback() + except: + pass + raise + finally: + cursor.close() + # Only commit if no exception occurred + try: + conn.commit() + except Exception as e: + # If commit fails (e.g., already rolled back), that's okay + logger.warning(f"Commit failed (may have been rolled back): {e}") + + def query(self, sql: str, parameters: List[Dict] = None) -> List[Dict]: + """ + Execute a SELECT query and return results as list of dicts + + Args: + sql: SELECT statement + parameters: Optional parameters + + Returns: + List of dictionaries with column names as keys + """ + response = self.execute(sql, parameters) + + if "records" not in response: + return [] + + # Extract column names + columns = [col["name"] for col in response.get("columnMetadata", [])] + + # Convert records to dictionaries + results = [] + for record in response["records"]: + row = {} + for i, col in enumerate(columns): + value = self._extract_value(record[i]) + row[col] = value + results.append(row) + + return results + + def query_one(self, sql: str, parameters: List[Dict] = None) -> Optional[Dict]: + """ + Execute a SELECT query and return first result + + Args: + sql: SELECT statement + parameters: Optional parameters + + Returns: + Dictionary with column names as keys, or None if no results + """ + results = self.query(sql, parameters) + return results[0] if results else None + + def insert(self, table: str, data: Dict, returning: str = None) -> str: + """ + Insert a record into a table + + Args: + table: Table name + data: Dictionary of column names and values + returning: Column to return (e.g., 'id', 'clerk_user_id') + + Returns: + Value of returning column if specified + """ + columns = list(data.keys()) + placeholders = [] + + # Build placeholders with type casting where needed + for col in columns: + if isinstance(data[col], (dict, list)): + placeholders.append(f":{col}::jsonb") + elif isinstance(data[col], Decimal): + placeholders.append(f":{col}::numeric") + elif isinstance(data[col], date) and not isinstance(data[col], datetime): + placeholders.append(f":{col}::date") + elif isinstance(data[col], datetime): + placeholders.append(f":{col}::timestamp") + else: + placeholders.append(f":{col}") + + sql = f""" + INSERT INTO {table} ({", ".join(columns)}) + VALUES ({", ".join(placeholders)}) + """ + + # Add RETURNING clause if specified + if returning: + sql += f" RETURNING {returning}" + + parameters = self._build_parameters(data) + response = self.execute(sql, parameters) + + # Return value if RETURNING was used + if returning and response.get("records"): + return self._extract_value(response["records"][0][0]) + return None + + def update(self, table: str, data: Dict, where: str, where_params: Dict = None) -> int: + """ + Update records in a table + + Args: + table: Table name + data: Dictionary of columns to update + where: WHERE clause (without WHERE keyword) + where_params: Parameters for WHERE clause + + Returns: + Number of affected rows + """ + # Build SET clause with type casting where needed + set_parts = [] + for col, val in data.items(): + if isinstance(val, (dict, list)): + set_parts.append(f"{col} = :{col}::jsonb") + elif isinstance(val, Decimal): + set_parts.append(f"{col} = :{val}::numeric") + elif isinstance(val, date) and not isinstance(val, datetime): + set_parts.append(f"{col} = :{col}::date") + elif isinstance(val, datetime): + set_parts.append(f"{col} = :{col}::timestamp") + else: + set_parts.append(f"{col} = :{col}") + + set_clause = ", ".join(set_parts) + + sql = f""" + UPDATE {table} + SET {set_clause} + WHERE {where} + """ + + # Combine data and where parameters + all_params = {**data, **(where_params or {})} + parameters = self._build_parameters(all_params) + + response = self.execute(sql, parameters) + return response.get("numberOfRecordsUpdated", 0) + + def delete(self, table: str, where: str, where_params: Dict = None) -> int: + """ + Delete records from a table + + Args: + table: Table name + where: WHERE clause (without WHERE keyword) + where_params: Parameters for WHERE clause + + Returns: + Number of deleted rows + """ + sql = f"DELETE FROM {table} WHERE {where}" + parameters = self._build_parameters(where_params) if where_params else None + + response = self.execute(sql, parameters) + return response.get("numberOfRecordsUpdated", 0) + + def begin_transaction(self) -> str: + """Begin a database transaction (returns transaction ID for compatibility)""" + conn = self._get_connection() + # For PostgreSQL, we use savepoints or just track the connection + # Return a dummy ID for compatibility + return "txn_1" + + def commit_transaction(self, transaction_id: str): + """Commit a database transaction""" + conn = self._get_connection() + conn.commit() + + def rollback_transaction(self, transaction_id: str): + """Rollback a database transaction""" + conn = self._get_connection() + conn.rollback() + + def _build_parameters(self, data: Dict) -> List[Dict]: + """Convert dictionary to DataAPIClient parameter format""" + if not data: + return [] + + parameters = [] + for key, value in data.items(): + param = {"name": key} + param["value"] = self._value_to_field(value) + parameters.append(param) + + return parameters + + def _value_to_field(self, value: Any) -> Dict: + """Convert Python value to DataAPIClient field format""" + if value is None: + return {"isNull": True} + elif isinstance(value, bool): + return {"booleanValue": value} + elif isinstance(value, int): + return {"longValue": value} + elif isinstance(value, float): + return {"doubleValue": value} + elif isinstance(value, Decimal): + return {"stringValue": str(value)} + elif isinstance(value, (date, datetime)): + return {"stringValue": value.isoformat()} + elif isinstance(value, dict): + return {"stringValue": json.dumps(value)} + elif isinstance(value, list): + return {"stringValue": json.dumps(value)} + else: + return {"stringValue": str(value)} + + def _extract_param_value(self, value_obj: Dict) -> Any: + """Extract Python value from DataAPIClient parameter format""" + if value_obj.get("skip_json_decode"): + return value_obj.get("stringValue") + if value_obj.get("isNull"): + return None + elif "booleanValue" in value_obj: + return value_obj["booleanValue"] + elif "longValue" in value_obj: + return value_obj["longValue"] + elif "doubleValue" in value_obj: + return value_obj["doubleValue"] + elif "stringValue" in value_obj: + value = value_obj["stringValue"] + # Try to parse JSON if it looks like JSON + if value and value[0] in ["{", "["]: + try: + return json.loads(value) + except json.JSONDecodeError: + pass + return value + else: + return None + + def _extract_value(self, field: Dict) -> Any: + """Extract value from DataAPIClient field response""" + return self._extract_param_value(field) + diff --git a/gcp-deployment/backend/database/src/models.py b/gcp-deployment/backend/database/src/models.py new file mode 100644 index 00000000..d2041902 --- /dev/null +++ b/gcp-deployment/backend/database/src/models.py @@ -0,0 +1,340 @@ +""" +Database models and query builders +""" + +import logging +from typing import Dict, List, Optional, Any +from datetime import datetime, date +from decimal import Decimal +from .cloudsql_client import CloudSQLClient + +logger = logging.getLogger(__name__) +from .schemas import ( + InstrumentCreate, UserCreate, AccountCreate, + PositionCreate, JobCreate, JobUpdate +) + + +class BaseModel: + """Base class for database models""" + + table_name = None + + def __init__(self, db: CloudSQLClient): + self.db = db + if not self.table_name: + raise ValueError("table_name must be defined") + + def find_by_id(self, id: Any) -> Optional[Dict]: + """Find a record by ID""" + sql = f"SELECT * FROM {self.table_name} WHERE id = :id::uuid" + return self.db.query_one(sql, [{'name': 'id', 'value': {'stringValue': str(id)}}]) + + def find_all(self, limit: int = 100, offset: int = 0) -> List[Dict]: + """Find all records with pagination""" + sql = f"SELECT * FROM {self.table_name} LIMIT :limit OFFSET :offset" + params = [ + {'name': 'limit', 'value': {'longValue': limit}}, + {'name': 'offset', 'value': {'longValue': offset}} + ] + return self.db.query(sql, params) + + def create(self, data: Dict, returning: str = 'id') -> str: + """Create a new record""" + return self.db.insert(self.table_name, data, returning=returning) + + def update(self, id: Any, data: Dict) -> int: + """Update a record by ID""" + return self.db.update(self.table_name, data, "id = :id::uuid", {'id': str(id)}) + + def delete(self, id: Any) -> int: + """Delete a record by ID""" + return self.db.delete(self.table_name, "id = :id::uuid", {'id': str(id)}) + + +class Users(BaseModel): + """Users table operations""" + table_name = 'users' + + def find_by_clerk_id(self, clerk_user_id: str) -> Optional[Dict]: + """Find user by Clerk ID""" + sql = f"SELECT * FROM {self.table_name} WHERE clerk_user_id = :clerk_id" + params = [{'name': 'clerk_id', 'value': {'stringValue': clerk_user_id}}] + return self.db.query_one(sql, params) + + def create_user(self, clerk_user_id: str, display_name: str = None, + years_until_retirement: int = None, + target_retirement_income: Decimal = None) -> str: + """Create a new user""" + data = { + 'clerk_user_id': clerk_user_id, + 'display_name': display_name, + 'years_until_retirement': years_until_retirement, + 'target_retirement_income': target_retirement_income + } + # Remove None values + data = {k: v for k, v in data.items() if v is not None} + return self.db.insert(self.table_name, data, returning='clerk_user_id') + + +class Instruments(BaseModel): + """Instruments table operations""" + table_name = 'instruments' + + def find_all(self, limit: int = None, offset: int = 0) -> List[Dict]: + """Find all instruments - no limit by default for autocomplete""" + sql = f"SELECT * FROM {self.table_name} ORDER BY symbol" + return self.db.query(sql, []) + + def find_by_symbol(self, symbol: str) -> Optional[Dict]: + """Find instrument by symbol""" + sql = f"SELECT * FROM {self.table_name} WHERE symbol = :symbol" + params = [{'name': 'symbol', 'value': {'stringValue': symbol}}] + return self.db.query_one(sql, params) + + def create_instrument(self, instrument: InstrumentCreate) -> str: + """Create a new instrument with validation""" + # Validate using Pydantic + validated = instrument.model_dump() + + # Convert allocations to JSON strings for storage + data = { + 'symbol': validated['symbol'], + 'name': validated['name'], + 'instrument_type': validated['instrument_type'], + 'allocation_regions': validated['allocation_regions'], + 'allocation_sectors': validated['allocation_sectors'], + 'allocation_asset_class': validated['allocation_asset_class'] + } + + return self.db.insert(self.table_name, data, returning='symbol') + + def find_by_type(self, instrument_type: str) -> List[Dict]: + """Find all instruments of a specific type""" + sql = f"SELECT * FROM {self.table_name} WHERE instrument_type = :type ORDER BY symbol" + params = [{'name': 'type', 'value': {'stringValue': instrument_type}}] + return self.db.query(sql, params) + + def search(self, query: str) -> List[Dict]: + """Search instruments by symbol or name""" + sql = f""" + SELECT * FROM {self.table_name} + WHERE LOWER(symbol) LIKE LOWER(:query) + OR LOWER(name) LIKE LOWER(:query) + ORDER BY symbol + LIMIT 20 + """ + params = [{'name': 'query', 'value': {'stringValue': f'%{query}%'}}] + return self.db.query(sql, params) + + +class Accounts(BaseModel): + """Accounts table operations""" + table_name = 'accounts' + + def find_by_user(self, clerk_user_id: str) -> List[Dict]: + """Find all accounts for a user""" + sql = f""" + SELECT * FROM {self.table_name} + WHERE clerk_user_id = :user_id + ORDER BY created_at DESC + """ + params = [{'name': 'user_id', 'value': {'stringValue': clerk_user_id}}] + return self.db.query(sql, params) + + def create_account(self, clerk_user_id: str, account_name: str, + account_purpose: str = None, cash_balance: Decimal = Decimal('0'), + cash_interest: Decimal = Decimal('0')) -> str: + """Create a new account""" + data = { + 'clerk_user_id': clerk_user_id, + 'account_name': account_name, + 'account_purpose': account_purpose, + 'cash_balance': cash_balance, + 'cash_interest': cash_interest + } + return self.db.insert(self.table_name, data, returning='id') + + +class Positions(BaseModel): + """Positions table operations""" + table_name = 'positions' + + def find_by_account(self, account_id: str) -> List[Dict]: + """Find all positions in an account""" + sql = f""" + SELECT p.*, i.name as instrument_name, i.instrument_type, i.current_price + FROM {self.table_name} p + JOIN instruments i ON p.symbol = i.symbol + WHERE p.account_id = :account_id::uuid + ORDER BY p.symbol + """ + params = [{'name': 'account_id', 'value': {'stringValue': account_id}}] + return self.db.query(sql, params) + + def get_portfolio_value(self, account_id: str) -> Dict: + """Calculate total portfolio value using current prices from instruments table""" + sql = """ + SELECT + COUNT(DISTINCT p.symbol) as num_positions, + SUM(p.quantity * i.current_price) as total_value, + SUM(p.quantity) as total_shares + FROM positions p + JOIN instruments i ON p.symbol = i.symbol + WHERE p.account_id = :account_id::uuid + """ + params = [ + {'name': 'account_id', 'value': {'stringValue': account_id}} + ] + result = self.db.query_one(sql, params) + if result: + return { + 'num_positions': result.get('num_positions', 0), + 'total_value': float(result.get('total_value', 0)) if result.get('total_value') else 0, + 'total_shares': float(result.get('total_shares', 0)) if result.get('total_shares') else 0 + } + return {'num_positions': 0, 'total_value': 0, 'total_shares': 0} + + def add_position(self, account_id: str, symbol: str, quantity: Decimal) -> str: + """Add or update a position""" + # Use UPSERT to handle existing positions + sql = """ + INSERT INTO positions (account_id, symbol, quantity, as_of_date) + VALUES (:account_id::uuid, :symbol, :quantity::numeric, :as_of_date::date) + ON CONFLICT (account_id, symbol) + DO UPDATE SET + quantity = EXCLUDED.quantity, + as_of_date = EXCLUDED.as_of_date, + updated_at = NOW() + RETURNING id + """ + params = [ + {'name': 'account_id', 'value': {'stringValue': account_id}}, + {'name': 'symbol', 'value': {'stringValue': symbol}}, + {'name': 'quantity', 'value': {'stringValue': str(quantity)}}, + {'name': 'as_of_date', 'value': {'stringValue': date.today().isoformat()}} + ] + response = self.db.execute(sql, params) + if response.get('records'): + return response['records'][0][0].get('stringValue') + return None + + +class Jobs(BaseModel): + """Jobs table operations""" + table_name = 'jobs' + + def create_job(self, clerk_user_id: str, job_type: str, + request_payload: Dict = None) -> str: + """Create a new job""" + data = { + 'clerk_user_id': clerk_user_id, + 'job_type': job_type, + 'status': 'pending', + 'request_payload': request_payload + } + return self.db.insert(self.table_name, data, returning='id') + + def update_status(self, job_id: str, status: str, error_message: str = None) -> int: + """Update job status""" + data = {'status': status} + + if status == 'running': + data['started_at'] = datetime.utcnow() + elif status in ['completed', 'failed']: + data['completed_at'] = datetime.utcnow() + + if error_message: + data['error_message'] = error_message + + return self.db.update(self.table_name, data, "id = :id::uuid", {'id': job_id}) + + def update_report(self, job_id: str, report_payload: Dict) -> int: + """Update job with Reporter agent's analysis""" + data = {'report_payload': report_payload} + return self.db.update(self.table_name, data, "id = :id::uuid", {'id': job_id}) + + def update_charts(self, job_id: str, charts_payload: Dict) -> int: + """Update job with Charter agent's visualization data""" + data = {'charts_payload': charts_payload} + return self.db.update(self.table_name, data, "id = :id::uuid", {'id': job_id}) + + def update_retirement(self, job_id: str, retirement_payload: Dict) -> int: + """Update job with Retirement agent's projections""" + data = {'retirement_payload': retirement_payload} + return self.db.update(self.table_name, data, "id = :id::uuid", {'id': job_id}) + + def update_summary(self, job_id: str, summary_payload: Dict) -> int: + """Update job with Planner's final summary""" + data = {'summary_payload': summary_payload} + return self.db.update(self.table_name, data, "id = :id::uuid", {'id': job_id}) + + def find_by_user(self, clerk_user_id: str, status: str = None, + limit: int = 20) -> List[Dict]: + """Find jobs for a user""" + if status: + sql = f""" + SELECT * FROM {self.table_name} + WHERE clerk_user_id = :user_id AND status = :status + ORDER BY created_at DESC + LIMIT :limit + """ + params = [ + {'name': 'user_id', 'value': {'stringValue': clerk_user_id}}, + {'name': 'status', 'value': {'stringValue': status}}, + {'name': 'limit', 'value': {'longValue': limit}} + ] + else: + sql = f""" + SELECT * FROM {self.table_name} + WHERE clerk_user_id = :user_id + ORDER BY created_at DESC + LIMIT :limit + """ + params = [ + {'name': 'user_id', 'value': {'stringValue': clerk_user_id}}, + {'name': 'limit', 'value': {'longValue': limit}} + ] + + return self.db.query(sql, params) + + +class Database: + """Main database interface providing access to all models""" + + def __init__(self, instance_connection_name: str = None, + database: str = None): + """Initialize database with all model classes (GCP Cloud SQL only)""" + import os + + # GCP Cloud SQL configuration + instance_connection_name = instance_connection_name or os.environ.get("INSTANCE_CONNECTION_NAME") + db_password_secret_id = os.environ.get("DB_PASSWORD_SECRET_ID") + + if not instance_connection_name and not db_password_secret_id: + raise ValueError( + "INSTANCE_CONNECTION_NAME or DB_PASSWORD_SECRET_ID must be set for Cloud SQL connection. " + "This is a GCP-only deployment." + ) + + self.client = CloudSQLClient( + instance_connection_name=instance_connection_name, + database=database or os.environ.get("DATABASE_NAME", "alex"), + user=os.environ.get("DATABASE_USER", "alex_app"), + password=os.environ.get("DB_PASSWORD"), + ) + + # Initialize all models + self.users = Users(self.client) + self.instruments = Instruments(self.client) + self.accounts = Accounts(self.client) + self.positions = Positions(self.client) + self.jobs = Jobs(self.client) + + def execute_raw(self, sql: str, parameters: List[Dict] = None) -> Dict: + """Execute raw SQL for complex queries""" + return self.client.execute(sql, parameters) + + def query_raw(self, sql: str, parameters: List[Dict] = None) -> List[Dict]: + """Execute raw SELECT query""" + return self.client.query(sql, parameters) \ No newline at end of file diff --git a/gcp-deployment/backend/database/src/schemas.py b/gcp-deployment/backend/database/src/schemas.py new file mode 100644 index 00000000..30952398 --- /dev/null +++ b/gcp-deployment/backend/database/src/schemas.py @@ -0,0 +1,284 @@ +""" +Pydantic schemas for data validation and LLM tool interfaces +These models serve as both database validation and LLM structured output schemas +""" + +from typing import Dict, Literal, Optional, List +from pydantic import BaseModel, Field, field_validator +from decimal import Decimal +from datetime import date, datetime + + +# Define allowed values as Literals for LLM compatibility +RegionType = Literal[ + "north_america", + "europe", + "asia", + "latin_america", + "africa", + "middle_east", + "oceania", + "global", + "international", # For mixed non-US +] + +AssetClassType = Literal[ + "equity", "fixed_income", "real_estate", "commodities", "cash", "alternatives" +] + +SectorType = Literal[ + "technology", + "healthcare", + "financials", + "consumer_discretionary", + "consumer_staples", + "industrials", + "energy", + "materials", + "utilities", + "real_estate", + "communication", + "treasury", + "corporate", + "mortgage", + "government_related", + "commodities", + "diversified", + "other", +] + +InstrumentType = Literal["etf", "mutual_fund", "stock", "bond", "bond_fund", "commodity", "reit"] + +JobType = Literal[ + "portfolio_analysis", + "rebalance_recommendation", + "retirement_projection", + "risk_assessment", + "tax_optimization", + "instrument_research", +] + +JobStatus = Literal["pending", "running", "completed", "failed"] + +AccountType = Literal[ + "401k", "roth_ira", "traditional_ira", "taxable", "529", "hsa", "pension", "other" +] + + +class AllocationDict(BaseModel): + """Base class for allocation dictionaries ensuring they sum to 100""" + + @field_validator("*", mode="after") + def validate_sum(cls, v, info): + """Ensure allocation percentages sum to 100""" + if isinstance(v, dict): + total = sum(v.values()) + if abs(total - 100) > 3: # Allow small floating point errors + raise ValueError(f"Allocations must sum to 100, got {total}") + return v + + +class RegionAllocation(BaseModel): + """Geographic allocation of an instrument""" + + allocations: Dict[RegionType, float] = Field( + description="Percentage allocation by geographic region. Must sum to 100.", + example={"north_america": 60, "europe": 25, "asia": 15}, + ) + + @field_validator("allocations") + def validate_sum(cls, v): + total = sum(v.values()) + if abs(total - 100) > 3: + raise ValueError(f"Region allocations must sum to 100, got {total}") + return v + + +class AssetClassAllocation(BaseModel): + """Asset class allocation of an instrument""" + + allocations: Dict[AssetClassType, float] = Field( + description="Percentage allocation by asset class. Must sum to 100.", + example={"equity": 80, "fixed_income": 20}, + ) + + @field_validator("allocations") + def validate_sum(cls, v): + total = sum(v.values()) + if abs(total - 100) > 3: + raise ValueError(f"Asset class allocations must sum to 100, got {total}") + return v + + +class SectorAllocation(BaseModel): + """Sector allocation of an instrument""" + + allocations: Dict[SectorType, float] = Field( + description="Percentage allocation by market sector. Must sum to 100.", + example={"technology": 30, "healthcare": 25, "financials": 20, "other": 25}, + ) + + @field_validator("allocations") + def validate_sum(cls, v): + total = sum(v.values()) + if abs(total - 100) > 3: + raise ValueError(f"Sector allocations must sum to 100, got {total}") + return v + + +class InstrumentCreate(BaseModel): + """Schema for creating a new instrument - suitable for LLM tool input""" + + symbol: str = Field( + description="The ticker symbol of the instrument (e.g., 'SPY', 'BND')", + min_length=1, + max_length=20, + ) + name: str = Field(description="Full name of the instrument", min_length=1, max_length=255) + instrument_type: InstrumentType = Field(description="The type of financial instrument") + current_price: Optional[Decimal] = Field( + None, + description="Current price of the instrument for portfolio calculations", + ge=0, + le=999999, + ) + allocation_regions: Dict[RegionType, float] = Field( + description="Geographic allocation percentages. Must sum to 100.", + example={"north_america": 100}, + ) + allocation_sectors: Dict[SectorType, float] = Field( + description="Sector allocation percentages. Must sum to 100.", + example={"technology": 40, "healthcare": 30, "financials": 30}, + ) + allocation_asset_class: Dict[AssetClassType, float] = Field( + description="Asset class allocation percentages. Must sum to 100.", example={"equity": 100} + ) + + @field_validator("allocation_regions", "allocation_sectors", "allocation_asset_class") + def validate_allocations(cls, v): + """Ensure all allocations sum to 100""" + if not v: + raise ValueError("Allocation cannot be empty") + total = sum(v.values()) + if abs(total - 100) > 3: + raise ValueError(f"Allocations must sum to 100, got {total}") + return v + + +class InstrumentResponse(InstrumentCreate): + """Schema for instrument responses from database""" + + created_at: datetime + updated_at: datetime + + +class UserCreate(BaseModel): + """Schema for creating a user - suitable for LLM tool input""" + + clerk_user_id: str = Field(description="Unique identifier from Clerk authentication system") + display_name: Optional[str] = Field(None, description="User's display name", max_length=255) + years_until_retirement: Optional[int] = Field( + None, description="Number of years until the user plans to retire", ge=0, le=100 + ) + target_retirement_income: Optional[Decimal] = Field( + None, description="Annual income goal in retirement (in dollars)", ge=0, decimal_places=2 + ) + asset_class_targets: Optional[Dict[AssetClassType, float]] = Field( + default={"equity": 70, "fixed_income": 30}, + description="Target allocation percentages for rebalancing. Must sum to 100.", + ) + region_targets: Optional[Dict[RegionType, float]] = Field( + default={"north_america": 50, "international": 50}, + description="Target geographic allocation for rebalancing. Must sum to 100.", + ) + + +class AccountCreate(BaseModel): + """Schema for creating an account - suitable for LLM tool input""" + + account_name: str = Field( + description="Name of the account (e.g., '401k', 'Roth IRA')", min_length=1, max_length=255 + ) + account_purpose: Optional[str] = Field(None, description="Purpose or goal of this account") + cash_balance: Decimal = Field( + default=Decimal("0"), + description="Uninvested cash balance in the account", + ge=0, + decimal_places=2, + ) + cash_interest: Decimal = Field( + default=Decimal("0"), + description="Annual interest rate on cash (e.g., 0.045 for 4.5%)", + ge=0, + le=1, + decimal_places=4, + ) + + +class PositionCreate(BaseModel): + """Schema for creating a position - suitable for LLM tool input""" + + account_id: str = Field(description="UUID of the account holding this position") + symbol: str = Field(description="Ticker symbol of the instrument", min_length=1, max_length=20) + quantity: Decimal = Field( + description="Number of shares (supports fractional shares)", gt=0, decimal_places=8 + ) + as_of_date: Optional[date] = Field( + default_factory=date.today, description="Date of this position snapshot" + ) + + +class JobCreate(BaseModel): + """Schema for creating a job - suitable for LLM tool input""" + + clerk_user_id: str = Field(description="User requesting this job") + job_type: JobType = Field(description="Type of analysis or operation to perform") + request_payload: Optional[Dict] = Field(None, description="Input parameters for the job") + + +class JobUpdate(BaseModel): + """Schema for updating job status - suitable for LLM tool output""" + + status: JobStatus = Field(description="Current status of the job") + result_payload: Optional[Dict] = Field(None, description="Results of the completed job") + error_message: Optional[str] = Field(None, description="Error details if job failed") + + +class PortfolioAnalysis(BaseModel): + """Schema for portfolio analysis results - LLM structured output""" + + total_value: Decimal = Field(description="Total portfolio value in dollars", decimal_places=2) + asset_allocation: Dict[AssetClassType, float] = Field( + description="Current asset class allocation percentages" + ) + region_allocation: Dict[RegionType, float] = Field( + description="Current geographic allocation percentages" + ) + sector_allocation: Dict[SectorType, float] = Field( + description="Current sector allocation percentages" + ) + risk_score: int = Field( + description="Risk score from 1 (conservative) to 10 (aggressive)", ge=1, le=10 + ) + recommendations: List[str] = Field( + description="List of actionable recommendations for the portfolio" + ) + + +class RebalanceRecommendation(BaseModel): + """Schema for rebalancing recommendations - LLM structured output""" + + current_allocation: Dict[str, float] = Field( + description="Current allocation by instrument symbol" + ) + target_allocation: Dict[str, float] = Field( + description="Recommended target allocation by symbol" + ) + trades: List[Dict] = Field( + description="List of trades needed to rebalance", + example=[ + {"symbol": "SPY", "action": "sell", "quantity": 10}, + {"symbol": "BND", "action": "buy", "quantity": 50}, + ], + ) + rationale: str = Field(description="Explanation of why these changes are recommended") diff --git a/gcp-deployment/backend/database/test_data_api.py b/gcp-deployment/backend/database/test_data_api.py new file mode 100644 index 00000000..d5454e1e --- /dev/null +++ b/gcp-deployment/backend/database/test_data_api.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +""" +Test Aurora Data API Connection +This script verifies that Aurora Serverless v2 is properly configured with Data API enabled. +""" + +import boto3 +import json +import os +import sys +from botocore.exceptions import ClientError +from dotenv import load_dotenv + +# Load environment variables +load_dotenv(override=True) + +def get_current_region(): + """Get the current AWS region from the session""" + session = boto3.Session() + return session.region_name or os.getenv('DEFAULT_AWS_REGION', 'us-east-1') + +def get_cluster_details(region): + """Get Aurora cluster ARN and secret ARN from environment variables or verify they exist""" + + # First try to get from environment variables + cluster_arn = os.getenv('AURORA_CLUSTER_ARN') + secret_arn = os.getenv('AURORA_SECRET_ARN') + + if cluster_arn and secret_arn: + print(f"๐Ÿ“‹ Using configuration from .env file") + + # Verify the cluster exists and Data API is enabled + rds_client = boto3.client('rds', region_name=region) + try: + cluster_id = cluster_arn.split(':')[-1] + response = rds_client.describe_db_clusters( + DBClusterIdentifier=cluster_id + ) + + if response['DBClusters']: + cluster = response['DBClusters'][0] + if not cluster.get('HttpEndpointEnabled', False): + print("โŒ Data API is not enabled on the Aurora cluster") + print("๐Ÿ’ก Run: aws rds modify-db-cluster --db-cluster-identifier alex-aurora-cluster --enable-http-endpoint --apply-immediately") + return None, None + else: + print(f"โŒ Aurora cluster '{cluster_id}' not found") + return None, None + + except ClientError as e: + print(f"โš ๏ธ Could not verify cluster status: {e}") + # Continue anyway - the cluster might exist but we can't describe it + + return cluster_arn, secret_arn + + # Fallback to auto-discovery if not in .env + print("โš ๏ธ AURORA_CLUSTER_ARN or AURORA_SECRET_ARN not found in .env file") + print("๐Ÿ’ก After running 'terraform apply', add these to your .env file:") + print(" AURORA_CLUSTER_ARN=") + print(" AURORA_SECRET_ARN=") + print("\nAttempting to auto-discover Aurora resources...") + + rds_client = boto3.client('rds', region_name=region) + secrets_client = boto3.client('secretsmanager', region_name=region) + + try: + # Get cluster ARN + response = rds_client.describe_db_clusters( + DBClusterIdentifier='alex-aurora-cluster' + ) + + if not response['DBClusters']: + print("โŒ Aurora cluster 'alex-aurora-cluster' not found") + return None, None + + cluster = response['DBClusters'][0] + cluster_arn = cluster['DBClusterArn'] + + # Check if Data API is enabled + if not cluster.get('HttpEndpointEnabled', False): + print("โŒ Data API is not enabled on the Aurora cluster") + print("๐Ÿ’ก Run: aws rds modify-db-cluster --db-cluster-identifier alex-aurora-cluster --enable-http-endpoint --apply-immediately") + return None, None + + # Find the most recently created aurora secret for alex + secrets = secrets_client.list_secrets() + aurora_secrets = [] + + for secret in secrets['SecretList']: + if 'aurora' in secret['Name'].lower() and 'alex' in secret['Name'].lower(): + aurora_secrets.append(secret) + + if not aurora_secrets: + print("โŒ Could not find Aurora credentials in Secrets Manager") + print("๐Ÿ’ก Look for a secret containing 'aurora' in the name") + return None, None + + # Sort by creation date and pick the most recent + aurora_secrets.sort(key=lambda x: x.get('CreatedDate', ''), reverse=True) + secret_arn = aurora_secrets[0]['ARN'] + + print(f"\n๐Ÿ“ Found Aurora resources. Add these to your .env file:") + print(f"AURORA_CLUSTER_ARN={cluster_arn}") + print(f"AURORA_SECRET_ARN={secret_arn}") + + return cluster_arn, secret_arn + + except ClientError as e: + print(f"โŒ Error accessing AWS resources: {e}") + return None, None + +def test_data_api(cluster_arn, secret_arn, region): + """Test the Data API connection""" + client = boto3.client('rds-data', region_name=region) + + print(f"\n๐Ÿ” Testing Data API Connection") + print(f" Region: {region}") + print(f" Cluster ARN: {cluster_arn}") + print(f" Secret ARN: {secret_arn}") + print("-" * 50) + + # Test 1: Simple SELECT + print("\n1๏ธโƒฃ Testing basic SELECT...") + try: + response = client.execute_statement( + resourceArn=cluster_arn, + secretArn=secret_arn, + database='alex', + sql='SELECT 1 as test_connection, current_timestamp as server_time' + ) + + if response['records']: + test_val = response['records'][0][0].get('longValue') + server_time = response['records'][0][1].get('stringValue') + print(f" โœ… Connection successful!") + print(f" Server time: {server_time}") + else: + print(" โŒ Query executed but returned no results") + + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == 'BadRequestException': + # This might mean the database doesn't exist yet + print(f" โš ๏ธ Database 'alex' might not exist or credentials are incorrect") + print(f" Error: {e.response['Error']['Message']}") + + # Try without specifying database + print("\n Retrying without database parameter...") + try: + response = client.execute_statement( + resourceArn=cluster_arn, + secretArn=secret_arn, + sql='SELECT current_database()' + ) + print(f" โœ… Connection successful (but 'alex' database may not exist)") + return True + except: + pass + else: + print(f" โŒ Error: {e}") + return False + + # Test 2: Check for tables + print("\n2๏ธโƒฃ Checking for existing tables...") + try: + response = client.execute_statement( + resourceArn=cluster_arn, + secretArn=secret_arn, + database='alex', + sql=""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name + """ + ) + + tables = [record[0].get('stringValue') for record in response.get('records', [])] + + if tables: + print(f" โœ… Found {len(tables)} tables:") + for table in tables: + print(f" - {table}") + else: + print(" โ„น๏ธ No tables found (database is empty)") + print(" ๐Ÿ’ก Run the migration script to create tables") + + except ClientError as e: + print(f" โš ๏ธ Could not list tables: {e}") + + # Test 3: Check database size + print("\n3๏ธโƒฃ Checking database info...") + try: + response = client.execute_statement( + resourceArn=cluster_arn, + secretArn=secret_arn, + database='alex', + sql="SELECT pg_database_size('alex') as size_bytes" + ) + + if response['records']: + size_bytes = response['records'][0][0].get('longValue', 0) + size_mb = size_bytes / (1024 * 1024) + print(f" โœ… Database size: {size_mb:.2f} MB") + + except: + pass + + print("\n" + "=" * 50) + print("โœ… Data API is working correctly!") + print("\n๐Ÿ“ Next steps:") + print("1. Run migrations to create tables: uv run migrate.py") + print("2. Load seed data: uv run seed_data.py") + print("3. Test the database package: uv run test_db.py") + + return True + +def main(): + """Main function""" + print("๐Ÿš€ Aurora Data API Connection Test") + print("=" * 50) + + # Get current region + region = get_current_region() + print(f"๐Ÿ“ Using AWS Region: {region}") + + # Get cluster and secret ARNs + cluster_arn, secret_arn = get_cluster_details(region) + + if not cluster_arn or not secret_arn: + print("\nโŒ Could not find Aurora cluster or credentials") + print("\n๐Ÿ’ก Make sure you have:") + print(" 1. Created the Aurora cluster with 'terraform apply'") + print(" 2. Enabled Data API on the cluster") + print(" 3. Created credentials in Secrets Manager") + sys.exit(1) + + # Test the Data API + success = test_data_api(cluster_arn, secret_arn, region) + + if not success: + print("\nโŒ Data API test failed") + print("\n๐Ÿ’ก Troubleshooting:") + print(" 1. Check if the Aurora instance is 'available'") + print(" 2. Verify Data API is enabled") + print(" 3. Check IAM permissions for rds-data:ExecuteStatement") + sys.exit(1) + + # Save connection details for other scripts + print(f"\nโœ… Data API test successful!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/database/verify_database.py b/gcp-deployment/backend/database/verify_database.py new file mode 100644 index 00000000..4d2b2d1d --- /dev/null +++ b/gcp-deployment/backend/database/verify_database.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +""" +Comprehensive database verification script +Shows that all tables exist and are properly populated + +This script verifies: +- All tables are created +- Record counts for each table +- Sample instruments with allocations +- Allocation percentages sum to 100% +- Asset class distribution +- Database indexes and triggers + +Note: JSONB values are stored as floats (100.0) not strings ('100') +""" + +import os +import boto3 +import json +from pathlib import Path +from botocore.exceptions import ClientError +from dotenv import load_dotenv + +# Load environment variables +load_dotenv(override=True) + +# Get config from environment +cluster_arn = os.environ.get('AURORA_CLUSTER_ARN') +secret_arn = os.environ.get('AURORA_SECRET_ARN') +database = os.environ.get('AURORA_DATABASE', 'alex') +region = os.environ.get('DEFAULT_AWS_REGION', 'us-east-1') + +if not cluster_arn or not secret_arn: + print("โŒ Missing AURORA_CLUSTER_ARN or AURORA_SECRET_ARN in .env file") + exit(1) + +client = boto3.client('rds-data', region_name=region) + +def execute_query(sql, description): + """Execute a query and return results""" + print(f"\n{description}") + print("-" * 50) + + try: + response = client.execute_statement( + resourceArn=cluster_arn, + secretArn=secret_arn, + database=database, + sql=sql + ) + return response + except ClientError as e: + print(f"โŒ Error: {e.response['Error']['Message']}") + return None + +def main(): + print("๐Ÿ” DATABASE VERIFICATION REPORT") + print("=" * 70) + print(f"๐Ÿ“ Region: {region}") + print(f"๐Ÿ“ฆ Database: {database}") + print("=" * 70) + + # 1. Show all tables + response = execute_query( + """ + SELECT table_name, + pg_size_pretty(pg_total_relation_size(quote_ident(table_name)::regclass)) as size + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """, + "๐Ÿ“Š ALL TABLES IN DATABASE" + ) + + if response and response['records']: + print(f"โœ… Found {len(response['records'])} tables:\n") + for record in response['records']: + table_name = record[0]['stringValue'] + size = record[1]['stringValue'] + print(f" โ€ข {table_name:<20} Size: {size}") + + # 2. Count records in each table + response = execute_query( + """ + SELECT + 'users' as table_name, COUNT(*) as count FROM users + UNION ALL + SELECT 'instruments', COUNT(*) FROM instruments + UNION ALL + SELECT 'accounts', COUNT(*) FROM accounts + UNION ALL + SELECT 'positions', COUNT(*) FROM positions + UNION ALL + SELECT 'jobs', COUNT(*) FROM jobs + ORDER BY table_name + """, + "๐Ÿ“ˆ RECORD COUNTS PER TABLE" + ) + + if response and response['records']: + print("\nTable record counts:\n") + for record in response['records']: + table_name = record[0]['stringValue'] + count = record[1]['longValue'] + status = "โœ…" if (table_name == 'instruments' and count > 0) else "๐Ÿ“ญ" + print(f" {status} {table_name:<20} {count:,} records") + + # 3. Show instruments with allocation data + response = execute_query( + """ + SELECT symbol, name, instrument_type, + allocation_asset_class::text as asset_class + FROM instruments + ORDER BY symbol + LIMIT 10 + """, + "๐ŸŽฏ SAMPLE INSTRUMENTS (First 10)" + ) + + if response and response['records']: + print("\nSymbol | Name | Type | Asset Class Allocation") + print("-" * 70) + for record in response['records']: + symbol = record[0]['stringValue'] + name = record[1]['stringValue'][:35] + inst_type = record[2]['stringValue'] + asset_class = record[3]['stringValue'] + print(f"{symbol:<6} | {name:<35} | {inst_type:<10} | {asset_class}") + + # 4. Verify allocation sums + response = execute_query( + """ + SELECT symbol, + (SELECT SUM(value::numeric) FROM jsonb_each_text(allocation_regions)) as regions_sum, + (SELECT SUM(value::numeric) FROM jsonb_each_text(allocation_sectors)) as sectors_sum, + (SELECT SUM(value::numeric) FROM jsonb_each_text(allocation_asset_class)) as asset_sum + FROM instruments + WHERE symbol IN ('SPY', 'QQQ', 'BND', 'VEA', 'GLD') + """, + "โœ… ALLOCATION VALIDATION (Sample ETFs)" + ) + + if response and response['records']: + print("\nVerifying allocations sum to 100%:\n") + print("Symbol | Regions | Sectors | Assets | Status") + print("-" * 50) + for record in response['records']: + symbol = record[0]['stringValue'] + # Handle numeric values from SUM() + regions = float(record[1].get('stringValue', '0')) if record[1] and 'stringValue' in record[1] else 0 + sectors = float(record[2].get('stringValue', '0')) if record[2] and 'stringValue' in record[2] else 0 + assets = float(record[3].get('stringValue', '0')) if record[3] and 'stringValue' in record[3] else 0 + + all_valid = regions == 100 and sectors == 100 and assets == 100 + status = "โœ… Valid" if all_valid else "โŒ Invalid" + + print(f"{symbol:<6} | {regions:>7}% | {sectors:>7}% | {assets:>6}% | {status}") + + # 5. Show asset class distribution + response = execute_query( + """ + SELECT + COUNT(*) FILTER (WHERE (allocation_asset_class->>'equity')::numeric = 100) as pure_equity, + COUNT(*) FILTER (WHERE (allocation_asset_class->>'fixed_income')::numeric = 100) as pure_bonds, + COUNT(*) FILTER (WHERE (allocation_asset_class->>'real_estate')::numeric = 100) as real_estate, + COUNT(*) FILTER (WHERE (allocation_asset_class->>'commodities')::numeric = 100) as commodities, + COUNT(*) FILTER (WHERE jsonb_typeof(allocation_asset_class) = 'object' + AND (SELECT COUNT(*) FROM jsonb_object_keys(allocation_asset_class)) > 1) as mixed, + COUNT(*) as total + FROM instruments + """, + "๐Ÿ“Š ASSET CLASS DISTRIBUTION" + ) + + if response and response['records']: + record = response['records'][0] + print("\nInstrument breakdown by asset class:\n") + print(f" โ€ข Pure Equity ETFs: {record[0]['longValue']:>3}") + print(f" โ€ข Pure Bond Funds: {record[1]['longValue']:>3}") + print(f" โ€ข Real Estate ETFs: {record[2]['longValue']:>3}") + print(f" โ€ข Commodity ETFs: {record[3]['longValue']:>3}") + print(f" โ€ข Mixed Allocation ETFs: {record[4]['longValue']:>3}") + print(f" " + "-" * 25) + print(f" โ€ข TOTAL INSTRUMENTS: {record[5]['longValue']:>3}") + + # 6. Check indexes exist + response = execute_query( + """ + SELECT schemaname, tablename, indexname + FROM pg_indexes + WHERE schemaname = 'public' + AND indexname LIKE 'idx_%' + ORDER BY tablename, indexname + """, + "๐Ÿ” DATABASE INDEXES" + ) + + if response and response['records']: + print(f"\nโœ… Found {len(response['records'])} custom indexes") + + # 7. Check triggers exist + response = execute_query( + """ + SELECT trigger_name, event_object_table + FROM information_schema.triggers + WHERE trigger_schema = 'public' + ORDER BY event_object_table + """, + "โšก DATABASE TRIGGERS" + ) + + if response and response['records']: + print(f"\nโœ… Found {len(response['records'])} update triggers for timestamp management") + + # Final summary + print("\n" + "=" * 70) + print("๐ŸŽ‰ DATABASE VERIFICATION COMPLETE") + print("=" * 70) + print("\nโœ… All tables created successfully") + print("โœ… 22 instruments loaded with complete allocation data") + print("โœ… All allocation percentages sum to 100%") + print("โœ… Indexes and triggers are in place") + print("โœ… Database is ready for Part 6: Agent Orchestra!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/deploy_all_lambdas.py b/gcp-deployment/backend/deploy_all_lambdas.py new file mode 100644 index 00000000..b608576d --- /dev/null +++ b/gcp-deployment/backend/deploy_all_lambdas.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +""" +Deploy all Part 6 Lambda functions to AWS using Terraform. +This script ensures Lambda functions are properly updated by: +1. Optionally packaging the Lambda functions +2. Tainting Lambda resources in Terraform to force recreation +3. Running terraform apply to deploy with the latest code + +Usage: + cd backend + uv run deploy_all_lambdas.py [--package] + +Options: + --package Force re-packaging of all Lambda functions before deployment +""" + +import boto3 +import sys +import subprocess +import os +from pathlib import Path +from typing import List, Tuple + +def taint_and_deploy_via_terraform() -> bool: + """ + Deploy Lambda functions using Terraform with forced recreation. + + Returns: + True if successful, False otherwise + """ + # Change to terraform directory + terraform_dir = Path(__file__).parent.parent / "terraform" / "6_agents" + if not terraform_dir.exists(): + print(f"โŒ Terraform directory not found: {terraform_dir}") + return False + + # Lambda function names to taint + lambda_functions = ['planner', 'tagger', 'reporter', 'charter', 'retirement'] + + print("๐Ÿ“Œ Step 1: Tainting Lambda functions to force recreation...") + print("-" * 50) + + # Taint each Lambda function + for func in lambda_functions: + print(f" Tainting aws_lambda_function.{func}...") + result = subprocess.run( + ['terraform', 'taint', f'aws_lambda_function.{func}'], + cwd=terraform_dir, + capture_output=True, + text=True + ) + + if result.returncode == 0 or "already" in result.stderr: + print(f" โœ“ {func} marked for recreation") + elif "No such resource instance" in result.stderr: + print(f" โš ๏ธ {func} doesn't exist (will be created)") + else: + print(f" โš ๏ธ Warning: {result.stderr[:100]}") + + print() + print("๐Ÿš€ Step 2: Running terraform apply...") + print("-" * 50) + + # Run terraform apply + result = subprocess.run( + ['terraform', 'apply', '-auto-approve'], + cwd=terraform_dir, + capture_output=False, # Show output directly + text=True + ) + + if result.returncode == 0: + print() + print("โœ… Terraform deployment completed successfully!") + return True + else: + print() + print("โŒ Terraform deployment failed!") + return False + +def package_lambda(service_name: str, service_dir: Path) -> bool: + """ + Package a Lambda function using package_docker.py. + + Args: + service_name: Name of the service (e.g., 'planner') + service_dir: Path to the service directory + + Returns: + True if successful, False otherwise + """ + print(f" ๐Ÿ“ฆ Packaging {service_name}...") + + package_script = service_dir / 'package_docker.py' + if not package_script.exists(): + print(f" โœ— package_docker.py not found in {service_dir}") + return False + + try: + # Run uv run package_docker.py in the service directory + result = subprocess.run( + ['uv', 'run', 'package_docker.py'], + cwd=service_dir, + capture_output=True, + text=True + ) + + if result.returncode == 0: + # Check if zip was created + zip_path = service_dir / f'{service_name}_lambda.zip' + if zip_path.exists(): + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f" โœ“ Created {size_mb:.1f} MB package") + return True + else: + print(f" โœ— Package not created") + return False + else: + print(f" โœ— Packaging failed: {result.stderr}") + return False + + except Exception as e: + print(f" โœ— Error running package_docker.py: {e}") + return False + +def main(): + """Main deployment function.""" + # Check for --package flag + force_package = '--package' in sys.argv + + print("๐ŸŽฏ Deploying Alex Agent Lambda Functions (via Terraform)") + print("=" * 50) + + # Get AWS account ID + try: + sts_client = boto3.client('sts') + account_id = sts_client.get_caller_identity()['Account'] + region = boto3.Session().region_name + print(f"AWS Account: {account_id}") + print(f"AWS Region: {region}") + except Exception as e: + print(f"โŒ Failed to get AWS account info: {e}") + print(" Make sure your AWS credentials are configured") + sys.exit(1) + + print() + + # Define Lambda functions to check/package + backend_dir = Path(__file__).parent + services = [ + ('planner', backend_dir / 'planner' / 'planner_lambda.zip'), + ('tagger', backend_dir / 'tagger' / 'tagger_lambda.zip'), + ('reporter', backend_dir / 'reporter' / 'reporter_lambda.zip'), + ('charter', backend_dir / 'charter' / 'charter_lambda.zip'), + ('retirement', backend_dir / 'retirement' / 'retirement_lambda.zip'), + ] + + # Check if packages exist and optionally package them + print("๐Ÿ“‹ Checking deployment packages...") + services_to_package = [] + + for service_name, zip_path in services: + service_dir = backend_dir / service_name + + if force_package: + # Force re-packaging all services + services_to_package.append((service_name, service_dir)) + print(f" โŸณ {service_name}: Will re-package") + elif zip_path.exists(): + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f" โœ“ {service_name}: {size_mb:.1f} MB") + else: + print(f" โœ— {service_name}: Not found") + services_to_package.append((service_name, service_dir)) + + # Package missing or all services if requested + if services_to_package: + print() + print("๐Ÿ“ฆ Packaging Lambda functions...") + failed_packages = [] + + for service_name, service_dir in services_to_package: + if not package_lambda(service_name, service_dir): + failed_packages.append(service_name) + + if failed_packages: + print() + print(f"โŒ Failed to package: {', '.join(failed_packages)}") + print(" Make sure Docker is running and package_docker.py exists") + response = input("Continue anyway? (y/N): ") + if response.lower() != 'y': + sys.exit(1) + + print() + + # Deploy via Terraform with forced recreation + if taint_and_deploy_via_terraform(): + print() + print("๐ŸŽ‰ All Lambda functions deployed successfully!") + print() + print("โš ๏ธ IMPORTANT: Lambda functions were FORCE RECREATED") + print(" This ensures your latest code is running in AWS") + print() + print("Next steps:") + print(" 1. Test locally: cd && uv run test_simple.py") + print(" 2. Run integration test: cd backend && uv run test_full.py") + print(" 3. Monitor CloudWatch Logs for each function") + sys.exit(0) + else: + print() + print("โŒ Deployment failed!") + print() + print("๐Ÿ’ก Troubleshooting tips:") + print(" 1. Check terraform output for errors") + print(" 2. Ensure all packages exist (use --package flag)") + print(" 3. Verify AWS credentials and permissions") + print(" 4. Check terraform state: cd terraform/6_agents && terraform plan") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/ingest/.python-version b/gcp-deployment/backend/ingest/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/ingest/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/ingest/cleanup_s3vectors.py b/gcp-deployment/backend/ingest/cleanup_s3vectors.py new file mode 100644 index 00000000..f809e789 --- /dev/null +++ b/gcp-deployment/backend/ingest/cleanup_s3vectors.py @@ -0,0 +1,119 @@ +""" +Clean up S3 Vectors database by removing all test data. +This script directly accesses S3 Vectors without going through API Gateway. +""" + +import os +import json +import boto3 +from dotenv import load_dotenv +from pathlib import Path + +# Load environment variables from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Get configuration +VECTOR_BUCKET = os.getenv('VECTOR_BUCKET') +INDEX_NAME = 'financial-research' + +if not VECTOR_BUCKET: + print("Error: VECTOR_BUCKET not found in .env") + exit(1) + +# Initialize S3 Vectors client +s3_vectors = boto3.client('s3vectors') + +def delete_all_vectors(): + """Delete all vectors from the index.""" + print("Cleaning S3 Vectors database...") + print(f"Bucket: {VECTOR_BUCKET}") + print(f"Index: {INDEX_NAME}") + print() + + deleted_count = 0 + + try: + # S3 Vectors doesn't have a list operation, so we need to search broadly + print("Searching for vectors to delete...") + + # Get a real embedding for a generic search term + sagemaker_runtime = boto3.client('sagemaker-runtime') + SAGEMAKER_ENDPOINT = os.getenv('SAGEMAKER_ENDPOINT', 'alex-embedding-endpoint') + + response = sagemaker_runtime.invoke_endpoint( + EndpointName=SAGEMAKER_ENDPOINT, + ContentType='application/json', + Body='{"inputs": "document"}' + ) + + result = json.loads(response['Body'].read().decode()) + # Extract from nested array [[[embedding]]] + dummy_vector = result[0][0] + + # S3 Vectors limits topK to 30, so we need to loop + all_vectors = [] + batch_size = 30 + + while True: + response = s3_vectors.query_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + queryVector={"float32": dummy_vector}, + topK=batch_size, + returnMetadata=True + ) + + vectors = response.get('vectors', []) + if not vectors: + break + + all_vectors.extend(vectors) + + # Delete this batch before getting more + print(f" Found batch of {len(vectors)} vectors...") + for vector in vectors: + try: + s3_vectors.delete_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + keys=[vector['key']] + ) + deleted_count += 1 + except Exception as e: + print(f" Error deleting {vector['key']}: {e}") + + # If we got less than batch_size, we're done + if len(vectors) < batch_size: + break + + if deleted_count > 0: + print(f"\nโœ… Successfully deleted {deleted_count} vectors") + else: + print("โœ… No vectors found - database is already empty") + + except Exception as e: + print(f"โŒ Error during cleanup: {e}") + if deleted_count > 0: + print(f" (Partially successful - deleted {deleted_count} vectors)") + +def main(): + """Clean up the S3 Vectors database.""" + print("=" * 60) + print("S3 Vectors Database Cleanup") + print("=" * 60) + print() + + # Confirm before deleting + response = input("โš ๏ธ This will DELETE ALL vectors. Continue? (yes/no): ") + if response.lower() != 'yes': + print("Cleanup cancelled.") + return + + print() + delete_all_vectors() + + print("\n๐Ÿ’ก Tip: Run test_api.py to add new test data") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/ingest/ingest_s3vectors.py b/gcp-deployment/backend/ingest/ingest_s3vectors.py new file mode 100644 index 00000000..e88a9a72 --- /dev/null +++ b/gcp-deployment/backend/ingest/ingest_s3vectors.py @@ -0,0 +1,102 @@ +""" +Lambda function for ingesting text into S3 Vectors with embeddings. +""" + +import json +import os +import boto3 +import datetime +import uuid + +# Environment variables +VECTOR_BUCKET = os.environ.get('VECTOR_BUCKET', 'alex-vectors') +SAGEMAKER_ENDPOINT = os.environ.get('SAGEMAKER_ENDPOINT') +INDEX_NAME = os.environ.get('INDEX_NAME', 'financial-research') + +# Initialize AWS clients +sagemaker_runtime = boto3.client('sagemaker-runtime') +s3_vectors = boto3.client('s3vectors') + + +def get_embedding(text): + """Get embedding vector from SageMaker endpoint.""" + response = sagemaker_runtime.invoke_endpoint( + EndpointName=SAGEMAKER_ENDPOINT, + ContentType='application/json', + Body=json.dumps({'inputs': text}) + ) + + result = json.loads(response['Body'].read().decode()) + # HuggingFace returns nested array [[[embedding]]], extract the actual embedding + if isinstance(result, list) and len(result) > 0: + if isinstance(result[0], list) and len(result[0]) > 0: + if isinstance(result[0][0], list): + return result[0][0] # Extract from [[[embedding]]] + return result[0] # Extract from [[embedding]] + return result # Return as-is if not nested + + +def lambda_handler(event, context): + """ + Main Lambda handler. + Expects JSON body with: + { + "text": "Text to ingest", + "metadata": { + "source": "optional source", + "category": "optional category" + } + } + """ + try: + # Parse the request body + if isinstance(event.get('body'), str): + body = json.loads(event['body']) + else: + body = event.get('body', {}) + + text = body.get('text') + metadata = body.get('metadata', {}) + + if not text: + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'Missing required field: text'}) + } + + # Get embedding from SageMaker + print(f"Getting embedding for text: {text[:100]}...") + embedding = get_embedding(text) + + # Generate unique ID for the vector + vector_id = str(uuid.uuid4()) + + # Store in S3 Vectors + print(f"Storing vector in bucket: {VECTOR_BUCKET}, index: {INDEX_NAME}") + s3_vectors.put_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + vectors=[{ + "key": vector_id, + "data": {"float32": embedding}, + "metadata": { + "text": text, + "timestamp": datetime.datetime.utcnow().isoformat(), + **metadata # Include any additional metadata + } + }] + ) + + return { + 'statusCode': 200, + 'body': json.dumps({ + 'message': 'Document indexed successfully', + 'document_id': vector_id + }) + } + except Exception as e: + print(f"Error: {str(e)}") + return { + 'statusCode': 500, + 'body': json.dumps({'error': str(e)}) + } \ No newline at end of file diff --git a/gcp-deployment/backend/ingest/package.py b/gcp-deployment/backend/ingest/package.py new file mode 100644 index 00000000..a1ea36b0 --- /dev/null +++ b/gcp-deployment/backend/ingest/package.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +Cross-platform Lambda deployment package creator using uv. +Works on Windows, Mac, and Linux. +""" + +import os +import sys +import shutil +import zipfile +from pathlib import Path + + +def create_deployment_package(): + """Create a Lambda deployment package with dependencies from uv.""" + + # Paths + current_dir = Path(__file__).parent + build_dir = current_dir / 'build' + package_dir = build_dir / 'package' + zip_path = current_dir / 'lambda_function.zip' + venv_site_packages = current_dir / '.venv' / 'lib' + + # Clean up previous builds + if build_dir.exists(): + shutil.rmtree(build_dir) + if zip_path.exists(): + os.remove(zip_path) + + # Create build directory + package_dir.mkdir(parents=True, exist_ok=True) + + # Find the site-packages directory (cross-platform) + site_packages = None + for path in venv_site_packages.rglob('site-packages'): + site_packages = path + break + + if not site_packages or not site_packages.exists(): + print("Error: Could not find site-packages. Make sure you've run 'uv init' and 'uv add' for dependencies.") + sys.exit(1) + + print(f"Copying dependencies from {site_packages}...") + # Copy all dependencies to package directory + for item in site_packages.iterdir(): + if item.name.endswith('.dist-info') or item.name == '__pycache__': + continue + if item.is_dir(): + shutil.copytree(item, package_dir / item.name, dirs_exist_ok=True) + else: + shutil.copy2(item, package_dir) + + # Copy Lambda function code + print("Copying Lambda function code...") + + # Copy S3 Vectors Lambda handlers + if (current_dir / 'ingest_s3vectors.py').exists(): + shutil.copy(current_dir / 'ingest_s3vectors.py', package_dir) + if (current_dir / 'search_s3vectors.py').exists(): + shutil.copy(current_dir / 'search_s3vectors.py', package_dir) + + # Create ZIP file + print("Creating deployment package...") + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for root, dirs, files in os.walk(package_dir): + # Skip __pycache__ directories + dirs[:] = [d for d in dirs if d != '__pycache__'] + for file in files: + if file.endswith('.pyc'): + continue + file_path = Path(root) / file + arcname = file_path.relative_to(package_dir) + zipf.write(file_path, arcname) + + # Clean up build directory + shutil.rmtree(build_dir) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"\nโœ… Deployment package created: {zip_path}") + print(f" Size: {size_mb:.2f} MB") + + if size_mb > 50: + print("โš ๏ธ Warning: Package exceeds 50MB. Consider using Lambda Layers.") + + return str(zip_path) + + +if __name__ == '__main__': + create_deployment_package() \ No newline at end of file diff --git a/gcp-deployment/backend/ingest/pyproject.toml b/gcp-deployment/backend/ingest/pyproject.toml new file mode 100644 index 00000000..3753df34 --- /dev/null +++ b/gcp-deployment/backend/ingest/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "lambda" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "boto3>=1.40.1", + "opensearch-py>=3.0.0", + "requests-aws4auth>=1.3.1", + "requests>=2.31.0", + "python-dotenv>=1.0.0", + "tenacity>=9.1.2", +] diff --git a/gcp-deployment/backend/ingest/search_s3vectors.py b/gcp-deployment/backend/ingest/search_s3vectors.py new file mode 100644 index 00000000..63e421fd --- /dev/null +++ b/gcp-deployment/backend/ingest/search_s3vectors.py @@ -0,0 +1,92 @@ +""" +Lambda function for searching S3 Vectors. +""" + +import json +import os +import boto3 + +# Environment variables +VECTOR_BUCKET = os.environ.get('VECTOR_BUCKET', 'alex-vectors') +SAGEMAKER_ENDPOINT = os.environ.get('SAGEMAKER_ENDPOINT') +INDEX_NAME = os.environ.get('INDEX_NAME', 'financial-research') + +# Initialize AWS clients +sagemaker_runtime = boto3.client('sagemaker-runtime') +s3_vectors = boto3.client('s3vectors') + + +def get_embedding(text): + """Get embedding vector from SageMaker endpoint.""" + response = sagemaker_runtime.invoke_endpoint( + EndpointName=SAGEMAKER_ENDPOINT, + ContentType='application/json', + Body=json.dumps({'inputs': text}) + ) + + result = json.loads(response['Body'].read().decode()) + # HuggingFace returns nested array [[[embedding]]], extract the actual embedding + if isinstance(result, list) and len(result) > 0: + if isinstance(result[0], list) and len(result[0]) > 0: + if isinstance(result[0][0], list): + return result[0][0] # Extract from [[[embedding]]] + return result[0] # Extract from [[embedding]] + return result # Return as-is if not nested + + +def lambda_handler(event, context): + """ + Search handler. + Expects JSON body with: + { + "query": "Search query text", + "k": 5 # Optional, defaults to 5 + } + """ + # Parse the request body + if isinstance(event.get('body'), str): + body = json.loads(event['body']) + else: + body = event.get('body', {}) + + query_text = body.get('query') + k = body.get('k', 5) + + if not query_text: + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'Missing required field: query'}) + } + + # Get embedding for query + print(f"Getting embedding for query: {query_text}") + query_embedding = get_embedding(query_text) + + # Search S3 Vectors + print(f"Searching in bucket: {VECTOR_BUCKET}, index: {INDEX_NAME}") + response = s3_vectors.query_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + queryVector={"float32": query_embedding}, + topK=k, + returnDistance=True, + returnMetadata=True + ) + + # Format results + results = [] + for vector in response.get('vectors', []): + results.append({ + 'id': vector['key'], + 'score': vector.get('distance', 0), + 'text': vector.get('metadata', {}).get('text', ''), + 'metadata': vector.get('metadata', {}) + }) + + return { + 'statusCode': 200, + 'body': json.dumps({ + 'results': results, + 'count': len(results) + }) + } \ No newline at end of file diff --git a/gcp-deployment/backend/ingest/test_ingest_s3vectors.py b/gcp-deployment/backend/ingest/test_ingest_s3vectors.py new file mode 100644 index 00000000..3c71bb09 --- /dev/null +++ b/gcp-deployment/backend/ingest/test_ingest_s3vectors.py @@ -0,0 +1,135 @@ +""" +Test script for ingesting documents directly to S3 Vectors. +This bypasses API Gateway and tests the S3 Vectors service directly. +""" + +import os +import json +import boto3 +import uuid +import datetime +from dotenv import load_dotenv +from pathlib import Path + +# Load environment variables from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Get configuration +VECTOR_BUCKET = os.getenv('VECTOR_BUCKET') +SAGEMAKER_ENDPOINT = os.getenv('SAGEMAKER_ENDPOINT', 'alex-embedding-endpoint') +INDEX_NAME = 'financial-research' + +if not VECTOR_BUCKET: + print("Error: Please run Guide 3 Step 4 to save VECTOR_BUCKET to .env") + exit(1) + +# Initialize AWS clients +s3_vectors = boto3.client('s3vectors') +sagemaker_runtime = boto3.client('sagemaker-runtime') + +def get_embedding(text): + """Get embedding vector from SageMaker endpoint.""" + response = sagemaker_runtime.invoke_endpoint( + EndpointName=SAGEMAKER_ENDPOINT, + ContentType='application/json', + Body=json.dumps({'inputs': text}) + ) + + result = json.loads(response['Body'].read().decode()) + # HuggingFace returns nested array [[[embedding]]], extract the actual embedding + if isinstance(result, list) and len(result) > 0: + if isinstance(result[0], list) and len(result[0]) > 0: + if isinstance(result[0][0], list): + return result[0][0] # Extract from [[[embedding]]] + return result[0] # Extract from [[embedding]] + return result # Return as-is if not nested + +def ingest_document(text, metadata=None): + """Ingest a document directly to S3 Vectors.""" + # Get embedding from SageMaker + print(f"Getting embedding for text: {text[:100]}...") + embedding = get_embedding(text) + + # Generate unique ID for the vector + vector_id = str(uuid.uuid4()) + + # Store in S3 Vectors + print(f"Storing vector in bucket: {VECTOR_BUCKET}, index: {INDEX_NAME}") + s3_vectors.put_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + vectors=[{ + "key": vector_id, + "data": {"float32": embedding}, + "metadata": { + "text": text, + "timestamp": datetime.datetime.utcnow().isoformat(), + **(metadata or {}) # Include any additional metadata + } + }] + ) + + return vector_id + +def main(): + """Test direct ingestion to S3 Vectors.""" + + print("Testing S3 Vectors Direct Ingestion") + print("=" * 60) + print(f"Bucket: {VECTOR_BUCKET}") + print(f"Index: {INDEX_NAME}") + print(f"Embedding Model: {SAGEMAKER_ENDPOINT}") + print() + + # Test documents + test_docs = [ + { + 'text': "Tesla Inc. (TSLA) is an electric vehicle and clean energy company. It designs, manufactures, and sells electric vehicles, energy storage systems, and solar panels.", + 'metadata': { + 'ticker': 'TSLA', + 'company_name': 'Tesla Inc.', + 'sector': 'Automotive/Energy', + 'source': 'portfolio' + } + }, + { + 'text': "Amazon.com Inc. (AMZN) is a multinational technology company focusing on e-commerce, cloud computing (AWS), digital streaming, and artificial intelligence.", + 'metadata': { + 'ticker': 'AMZN', + 'company_name': 'Amazon.com Inc.', + 'sector': 'Technology/Retail', + 'source': 'portfolio' + } + }, + { + 'text': "NVIDIA Corporation (NVDA) designs graphics processing units (GPUs) for gaming and professional markets, as well as system on chip units for mobile computing and automotive.", + 'metadata': { + 'ticker': 'NVDA', + 'company_name': 'NVIDIA Corporation', + 'sector': 'Technology/Semiconductors', + 'source': 'portfolio' + } + } + ] + + # Ingest each document + for i, doc in enumerate(test_docs, 1): + print(f"Ingesting document {i}: {doc['metadata'].get('ticker', 'Unknown')}") + try: + doc_id = ingest_document(doc['text'], doc['metadata']) + print(f" โœ“ Success! Document ID: {doc_id}") + except Exception as e: + print(f" โœ— Error: {e}") + print() + + print("Testing complete!") + print("\nYour S3 Vectors knowledge base now contains information about:") + for doc in test_docs: + print(f" - {doc['metadata']['company_name']} ({doc['metadata']['ticker']})") + + print("\nโฑ๏ธ Note: S3 Vectors updates are available immediately.") + print(" You can run test_search_s3vectors.py right away to search!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/ingest/test_search_s3vectors.py b/gcp-deployment/backend/ingest/test_search_s3vectors.py new file mode 100644 index 00000000..728adebd --- /dev/null +++ b/gcp-deployment/backend/ingest/test_search_s3vectors.py @@ -0,0 +1,151 @@ +""" +Test script for searching S3 Vectors. +This demonstrates how to search the indexed documents. +""" + +import os +import json +import boto3 +from dotenv import load_dotenv +from pathlib import Path + +# Load environment variables from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Get configuration +VECTOR_BUCKET = os.getenv('VECTOR_BUCKET') +SAGEMAKER_ENDPOINT = os.getenv('SAGEMAKER_ENDPOINT', 'alex-embedding-endpoint') +INDEX_NAME = 'financial-research' + +if not VECTOR_BUCKET: + print("Error: Please run Guide 3 Step 4 to save VECTOR_BUCKET to .env") + exit(1) + +# Initialize AWS clients +s3_vectors = boto3.client('s3vectors') +sagemaker_runtime = boto3.client('sagemaker-runtime') + +def get_embedding(text): + """Get embedding vector from SageMaker endpoint.""" + response = sagemaker_runtime.invoke_endpoint( + EndpointName=SAGEMAKER_ENDPOINT, + ContentType='application/json', + Body=json.dumps({'inputs': text}) + ) + + result = json.loads(response['Body'].read().decode()) + # HuggingFace returns nested array [[[embedding]]], extract the actual embedding + if isinstance(result, list) and len(result) > 0: + if isinstance(result[0], list) and len(result[0]) > 0: + if isinstance(result[0][0], list): + return result[0][0] # Extract from [[[embedding]]] + return result[0] # Extract from [[embedding]] + return result # Return as-is if not nested + +def list_all_vectors(): + """List all vectors in the index.""" + print(f"Listing vectors in bucket: {VECTOR_BUCKET}, index: {INDEX_NAME}") + print("=" * 60) + + try: + # S3 Vectors doesn't have a direct list operation, so we'll do a broad search + # Search for a common term to get some results + test_embedding = get_embedding("company") + + response = s3_vectors.query_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + queryVector={"float32": test_embedding}, + topK=10, + returnDistance=True, + returnMetadata=True + ) + + vectors = response.get('vectors', []) + print(f"\nFound {len(vectors)} vectors in the index:\n") + + for i, vector in enumerate(vectors, 1): + metadata = vector.get('metadata', {}) + text_preview = metadata.get('text', '')[:100] + '...' if len(metadata.get('text', '')) > 100 else metadata.get('text', '') + + print(f"{i}. Vector ID: {vector['key']}") + if metadata.get('ticker'): + print(f" Ticker: {metadata['ticker']}") + if metadata.get('company_name'): + print(f" Company: {metadata['company_name']}") + if metadata.get('sector'): + print(f" Sector: {metadata['sector']}") + print(f" Text: {text_preview}") + print() + + except Exception as e: + print(f"Error listing vectors: {e}") + +def search_vectors(query_text, k=5): + """Search for vectors by query text.""" + print(f"\nSearching for: '{query_text}'") + print("-" * 40) + + try: + # Get embedding for query + query_embedding = get_embedding(query_text) + + # Search S3 Vectors + response = s3_vectors.query_vectors( + vectorBucketName=VECTOR_BUCKET, + indexName=INDEX_NAME, + queryVector={"float32": query_embedding}, + topK=k, + returnDistance=True, + returnMetadata=True + ) + + vectors = response.get('vectors', []) + print(f"Found {len(vectors)} results:\n") + + for vector in vectors: + metadata = vector.get('metadata', {}) + distance = vector.get('distance', 0) + + print(f"Score: {1 - distance:.3f}") # Convert distance to similarity score + if metadata.get('company_name'): + print(f"Company: {metadata['company_name']} ({metadata.get('ticker', 'N/A')})") + print(f"Text: {metadata.get('text', '')[:200]}...") + print() + + except Exception as e: + print(f"Error searching: {e}") + +def main(): + """Explore the S3 Vectors database.""" + print("=" * 60) + print("Alex S3 Vectors Database Explorer") + print("=" * 60) + print(f"Bucket: {VECTOR_BUCKET}") + print(f"Index: {INDEX_NAME}") + print() + + # List all vectors + list_all_vectors() + + # Example searches + print("=" * 60) + print("Example Semantic Searches") + print("=" * 60) + + # Search for specific concepts + search_queries = [ + "electric vehicles and sustainable transportation", + "cloud computing and AWS services", + "artificial intelligence and GPU computing" + ] + + for query in search_queries: + search_vectors(query, k=3) + + print("\nโœจ S3 Vectors provides semantic search - notice how it finds") + print(" conceptually related documents even with different wording!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/package_docker.py b/gcp-deployment/backend/package_docker.py new file mode 100644 index 00000000..311bc013 --- /dev/null +++ b/gcp-deployment/backend/package_docker.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +""" +Package all Lambda functions using Docker for AWS compatibility. +Runs each agent's package_docker.py script. +""" + +import os +import sys +import subprocess +from pathlib import Path + + +def run_packaging(agent_name): + """Run packaging for a specific agent.""" + agent_dir = Path(__file__).parent / agent_name + package_script = agent_dir / "package_docker.py" + + if not package_script.exists(): + print(f" โŒ {agent_name}: Missing package_docker.py") + return False + + print(f"\n๐Ÿ“ฆ Packaging {agent_name.upper()} agent...") + print(f" Running: cd {agent_dir} && uv run package_docker.py") + + try: + result = subprocess.run( + ["uv", "run", "package_docker.py"], cwd=str(agent_dir), capture_output=True, text=True + ) + + if result.returncode == 0: + # Look for the created zip file + zip_files = list(agent_dir.glob("*.zip")) + if zip_files: + zip_file = zip_files[0] + size_mb = zip_file.stat().st_size / (1024 * 1024) + print(f" โœ… Created: {zip_file.name} ({size_mb:.1f} MB)") + return True + else: + print(f" โš ๏ธ Warning: No zip file found after packaging") + return True + else: + print( + f" โŒ Error with {agent_name.upper()}:\nPlease note that warnings about uv environment can be ignored:\n{result.stderr}\nOutput from script is:\n{result.stdout}" + ) + return False + + except Exception as e: + print(f" โŒ Error: {e}") + return False + + +def main(): + """Package all Lambda functions.""" + print("=" * 60) + print("PACKAGING ALL LAMBDA FUNCTIONS") + print("=" * 60) + + agents = ["tagger", "reporter", "charter", "retirement", "planner"] + results = {} + + for agent in agents: + success = run_packaging(agent) + results[agent] = success + + print("\n" + "=" * 60) + print("PACKAGING SUMMARY") + print("=" * 60) + + success_count = sum(1 for s in results.values() if s) + total_count = len(results) + + for agent, success in results.items(): + status = "โœ… Success" if success else "โŒ Failed" + print(f"{agent.ljust(12)}: {status}") + + print("\n" + "=" * 60) + print(f"Packaged: {success_count}/{total_count}") + + if success_count == total_count: + print("\nโœ… ALL LAMBDA FUNCTIONS PACKAGED SUCCESSFULLY!") + print("\nNext steps:") + print("1. Deploy infrastructure: cd terraform/6_agents && terraform apply") + print("2. Deploy Lambda functions: cd backend && uv run deploy_all_lambdas.py") + return 0 + else: + print(f"\nโš ๏ธ {total_count - success_count} agents failed to package") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gcp-deployment/backend/planner/.python-version b/gcp-deployment/backend/planner/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/planner/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/planner/Dockerfile b/gcp-deployment/backend/planner/Dockerfile new file mode 100644 index 00000000..5b45587e --- /dev/null +++ b/gcp-deployment/backend/planner/Dockerfile @@ -0,0 +1,40 @@ +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Python package manager +RUN pip install uv + +# Copy database package (required dependency) +# Build context should be from backend/ directory +COPY database ./database + +# Copy common module (required for imports) +COPY common ./common + +# Copy planner-specific files +COPY planner/pyproject.toml planner/uv.lock ./ + +# Update pyproject.toml to use ./database instead of ../database +RUN sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && rm pyproject.toml.bak + +# Verify the update +RUN echo "=== Updated path in pyproject.toml ===" && grep -A 1 "alex-database" pyproject.toml + +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +# This will regenerate the lock file with the correct path +RUN uv sync --no-install-project + +# Copy planner application code +COPY planner/*.py ./ + +# Expose port +EXPOSE 8000 + +# Set environment variable for Cloud Run +ENV PORT=8000 + +# Run the application +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/gcp-deployment/backend/planner/agent.py b/gcp-deployment/backend/planner/agent.py new file mode 100644 index 00000000..d3cf6e42 --- /dev/null +++ b/gcp-deployment/backend/planner/agent.py @@ -0,0 +1,294 @@ +""" +Financial Planner Orchestrator Agent - coordinates portfolio analysis across specialized agents. +""" + +import os +import sys +import json +import logging +from pathlib import Path +from typing import Dict, List, Any, Optional +from datetime import datetime +from dataclasses import dataclass + +import httpx +from google.auth.transport.requests import Request +from google.oauth2 import id_token + +# Add parent directories to Python path for imports +backend_dir = Path(__file__).parent.parent +if str(backend_dir) not in sys.path: + sys.path.insert(0, str(backend_dir)) + +from agents import function_tool, RunContextWrapper +from common.llm import get_litellm_model + +logger = logging.getLogger() + +# Cloud Run service URLs from environment (required) +TAGGER_SERVICE_URL = os.getenv("TAGGER_SERVICE_URL", "") +REPORTER_SERVICE_URL = os.getenv("REPORTER_SERVICE_URL", "") +CHARTER_SERVICE_URL = os.getenv("CHARTER_SERVICE_URL", "") +RETIREMENT_SERVICE_URL = os.getenv("RETIREMENT_SERVICE_URL", "") + + +@dataclass +class PlannerContext: + """Context for planner agent tools.""" + job_id: str + + +async def invoke_cloud_run_agent( + agent_name: str, service_url: str, payload: Dict[str, Any] +) -> Dict[str, Any]: + """Invoke a Cloud Run agent service via HTTP.""" + + if not service_url: + raise ValueError(f"{agent_name} service URL not configured. Set {agent_name.upper()}_SERVICE_URL environment variable.") + + try: + logger.info(f"Invoking {agent_name} Cloud Run service: {service_url}") + + # Get ID token for authentication (required for Cloud Run) + auth_req = Request() + token = id_token.fetch_id_token(auth_req, service_url) + + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + # Make HTTP POST request + async with httpx.AsyncClient(timeout=300.0) as client: + response = await client.post( + service_url, + json=payload, + headers=headers + ) + response.raise_for_status() + result = response.json() + + logger.info(f"{agent_name} completed successfully") + return result + + except Exception as e: + logger.error(f"Error invoking {agent_name} Cloud Run service: {e}") + return {"error": str(e)} + + +async def invoke_agent(agent_name: str, service_url: str, payload: Dict[str, Any]) -> Dict[str, Any]: + """Invoke an agent service via Cloud Run.""" + if not service_url: + raise ValueError(f"{agent_name} service URL not configured. Set {agent_name.upper()}_SERVICE_URL.") + return await invoke_cloud_run_agent(agent_name, service_url, payload) + + +async def handle_missing_instruments(job_id: str, db) -> None: + """ + Check for and tag any instruments missing allocation data. + This is done automatically before the agent runs. + """ + logger.info("Planner: Checking for instruments missing allocation data...") + + # Get job and portfolio data + job = db.jobs.find_by_id(job_id) + if not job: + logger.error(f"Job {job_id} not found") + return + + user_id = job["clerk_user_id"] + accounts = db.accounts.find_by_user(user_id) + + missing = [] + for account in accounts: + positions = db.positions.find_by_account(account["id"]) + for position in positions: + instrument = db.instruments.find_by_symbol(position["symbol"]) + if instrument: + has_allocations = bool( + instrument.get("allocation_regions") + and instrument.get("allocation_sectors") + and instrument.get("allocation_asset_class") + ) + if not has_allocations: + missing.append( + {"symbol": position["symbol"], "name": instrument.get("name", "")} + ) + else: + missing.append({"symbol": position["symbol"], "name": ""}) + + if missing: + logger.info( + f"Planner: Found {len(missing)} instruments needing classification: {[m['symbol'] for m in missing]}" + ) + + try: + # Invoke Tagger agent via Cloud Run + result = await invoke_agent("Tagger", TAGGER_SERVICE_URL, {"instruments": missing}) + + if isinstance(result, dict): + if result.get("success") or "error" not in result: + logger.info( + f"Planner: InstrumentTagger completed - Tagged {len(missing)} instruments" + ) + else: + logger.error( + f"Planner: InstrumentTagger failed: {result.get('error')}" + ) + + except Exception as e: + logger.error(f"Planner: Error tagging instruments: {e}") + else: + logger.info("Planner: All instruments have allocation data") + + +def load_portfolio_summary(job_id: str, db) -> Dict[str, Any]: + """Load basic portfolio summary statistics only.""" + try: + job = db.jobs.find_by_id(job_id) + if not job: + raise ValueError(f"Job {job_id} not found") + + user_id = job["clerk_user_id"] + user = db.users.find_by_clerk_id(user_id) + if not user: + raise ValueError(f"User {user_id} not found") + + accounts = db.accounts.find_by_user(user_id) + + # Calculate simple summary statistics + total_value = 0.0 + total_positions = 0 + total_cash = 0.0 + + for account in accounts: + total_cash += float(account.get("cash_balance", 0)) + positions = db.positions.find_by_account(account["id"]) + total_positions += len(positions) + + # Add position values + for position in positions: + instrument = db.instruments.find_by_symbol(position["symbol"]) + if instrument and instrument.get("current_price"): + price = float(instrument["current_price"]) + quantity = float(position["quantity"]) + total_value += price * quantity + + total_value += total_cash + + # Return only summary statistics + # Handle None values - .get() only returns default if key doesn't exist, not if value is None + years_until_retirement = user.get("years_until_retirement") + if years_until_retirement is None: + years_until_retirement = 30 + + target_retirement_income = user.get("target_retirement_income") + if target_retirement_income is None: + target_retirement_income = 80000 + + return { + "total_value": total_value, + "num_accounts": len(accounts), + "num_positions": total_positions, + "years_until_retirement": years_until_retirement, + "target_retirement_income": float(target_retirement_income) + } + + except Exception as e: + logger.error(f"Error loading portfolio summary: {e}") + raise + + +async def invoke_reporter_internal(job_id: str) -> str: + """ + Invoke the Report Writer agent to generate portfolio analysis narrative. + + Args: + job_id: The job ID for the analysis + + Returns: + Confirmation message + """ + result = await invoke_agent("Reporter", REPORTER_SERVICE_URL, {"job_id": job_id}) + + if "error" in result: + return f"Reporter agent failed: {result['error']}" + + return "Reporter agent completed successfully. Portfolio analysis narrative has been generated and saved." + + +async def invoke_charter_internal(job_id: str) -> str: + """ + Invoke the Chart Maker agent to create portfolio visualizations. + + Args: + job_id: The job ID for the analysis + + Returns: + Confirmation message + """ + result = await invoke_agent("Charter", CHARTER_SERVICE_URL, {"job_id": job_id}) + + if "error" in result: + return f"Charter agent failed: {result['error']}" + + return "Charter agent completed successfully. Portfolio visualizations have been created and saved." + + +async def invoke_retirement_internal(job_id: str) -> str: + """ + Invoke the Retirement Specialist agent for retirement projections. + + Args: + job_id: The job ID for the analysis + + Returns: + Confirmation message + """ + result = await invoke_agent("Retirement", RETIREMENT_SERVICE_URL, {"job_id": job_id}) + + if "error" in result: + return f"Retirement agent failed: {result['error']}" + + return "Retirement agent completed successfully. Retirement projections have been calculated and saved." + + + +@function_tool +async def invoke_reporter(wrapper: RunContextWrapper[PlannerContext]) -> str: + """Invoke the Report Writer agent to generate portfolio analysis narrative.""" + return await invoke_reporter_internal(wrapper.context.job_id) + +@function_tool +async def invoke_charter(wrapper: RunContextWrapper[PlannerContext]) -> str: + """Invoke the Chart Maker agent to create portfolio visualizations.""" + return await invoke_charter_internal(wrapper.context.job_id) + +@function_tool +async def invoke_retirement(wrapper: RunContextWrapper[PlannerContext]) -> str: + """Invoke the Retirement Specialist agent for retirement projections.""" + return await invoke_retirement_internal(wrapper.context.job_id) + + +def create_agent(job_id: str, portfolio_summary: Dict[str, Any], db): + """Create the orchestrator agent with tools.""" + + # Create context for tools + context = PlannerContext(job_id=job_id) + + model_override = os.getenv("PLANNER_MODEL") + model = get_litellm_model(model_override) + + tools = [ + invoke_reporter, + invoke_charter, + invoke_retirement, + ] + + # Create minimal task context + task = f"""Job {job_id} has {portfolio_summary['num_positions']} positions. +Retirement: {portfolio_summary['years_until_retirement']} years. + +Call the appropriate agents.""" + + return model, tools, task, context diff --git a/gcp-deployment/backend/planner/lambda_handler.py b/gcp-deployment/backend/planner/lambda_handler.py new file mode 100644 index 00000000..701e72cf --- /dev/null +++ b/gcp-deployment/backend/planner/lambda_handler.py @@ -0,0 +1,195 @@ +""" +Financial Planner Orchestrator Handler for Cloud Run (Pub/Sub triggered) +""" + +import os +import json +import base64 +import asyncio +import logging +from typing import Dict, Any + +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +try: + from dotenv import load_dotenv + load_dotenv(override=True) +except ImportError: + pass + +# Import database package +from src import Database + +from templates import ORCHESTRATOR_INSTRUCTIONS +from agent import create_agent, handle_missing_instruments, load_portfolio_summary +from market import update_instrument_prices +from observability import observe + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# Initialize database +db = Database() + +@retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info(f"Planner: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds...") +) +async def run_orchestrator(job_id: str) -> None: + """Run the orchestrator agent to coordinate portfolio analysis.""" + try: + # Update job status to running + db.jobs.update_status(job_id, 'running') + + # Handle missing instruments first (non-agent pre-processing) + await handle_missing_instruments(job_id, db) + + # Update instrument prices after tagging + logger.info("Planner: Updating instrument prices from market data") + await asyncio.to_thread(update_instrument_prices, job_id, db) + + # Load portfolio summary (just statistics, not full data) + portfolio_summary = await asyncio.to_thread(load_portfolio_summary, job_id, db) + + # Create agent with tools and context + model, tools, task, context = create_agent(job_id, portfolio_summary, db) + + # Run the orchestrator + with trace("Planner Orchestrator"): + from agent import PlannerContext + agent = Agent[PlannerContext]( + name="Financial Planner", + instructions=ORCHESTRATOR_INSTRUCTIONS, + model=model, + tools=tools + ) + + result = await Runner.run( + agent, + input=task, + context=context, + max_turns=20 + ) + + # Mark job as completed after all agents finish + db.jobs.update_status(job_id, "completed") + logger.info(f"Planner: Job {job_id} completed successfully") + + except Exception as e: + logger.error(f"Planner: Error in orchestration: {e}", exc_info=True) + db.jobs.update_status(job_id, 'failed', error_message=str(e)) + raise + +def lambda_handler(event, context): + """ + Cloud Run handler for Pub/Sub-triggered orchestration. + + Expected event from Pub/Sub (push subscription): + { + "message": { + "data": "base64_encoded_json", + "attributes": {} + } + } + + Or direct invocation: + { + "job_id": "..." + } + """ + # Wrap entire handler with observability context + with observe(): + try: + logger.info(f"Planner Cloud Run handler invoked with event: {json.dumps(event)[:500]}") + + job_id = None + + # Try Pub/Sub format (GCP Cloud Run) + if 'message' in event: + # Pub/Sub push subscription format + message_data = event['message'].get('data', '') + if message_data: + try: + decoded_data = base64.b64decode(message_data).decode('utf-8') + body = json.loads(decoded_data) + job_id = body.get('job_id') + logger.info(f"Extracted job_id from Pub/Sub message: {job_id}") + except (base64.binascii.Error, json.JSONDecodeError) as e: + logger.error(f"Error decoding Pub/Sub message: {e}") + + # Try direct invocation format + if not job_id and 'job_id' in event: + job_id = event['job_id'] + logger.info(f"Extracted job_id from direct invocation: {job_id}") + + if not job_id: + logger.error("No job_id found in event") + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'No job_id provided'}) + } + + logger.info(f"Planner: Starting orchestration for job {job_id}") + + # Run the orchestrator + asyncio.run(run_orchestrator(job_id)) + + return { + 'statusCode': 200, + 'body': json.dumps({ + 'success': True, + 'message': f'Analysis completed for job {job_id}' + }) + } + + except Exception as e: + logger.error(f"Planner: Error in Cloud Run handler: {e}", exc_info=True) + return { + 'statusCode': 500, + 'body': json.dumps({ + 'success': False, + 'error': str(e) + }) + } + +# For local testing +if __name__ == "__main__": + # Define a test user + test_user_id = "test_user_planner_local" + + # Ensure the test user exists before creating a job + from src.schemas import UserCreate, JobCreate + + user = db.users.find_by_clerk_id(test_user_id) + if not user: + print(f"Creating test user: {test_user_id}") + user_create = UserCreate(clerk_user_id=test_user_id, display_name="Test Planner User") + db.users.create(user_create.model_dump(), returning='clerk_user_id') + + # Create a test job + print("Creating test job...") + job_create = JobCreate( + clerk_user_id=test_user_id, + job_type='portfolio_analysis', + request_payload={ + 'analysis_type': 'comprehensive', + 'test': True + } + ) + + job = db.jobs.create(job_create.model_dump()) + job_id = job + + print(f"Created test job: {job_id}") + + # Test the handler + test_event = { + 'job_id': job_id + } + + result = lambda_handler(test_event, None) + print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/gcp-deployment/backend/planner/market.py b/gcp-deployment/backend/planner/market.py new file mode 100644 index 00000000..05403719 --- /dev/null +++ b/gcp-deployment/backend/planner/market.py @@ -0,0 +1,139 @@ +""" +Market data functions using polygon.io for fetching real-time prices. +""" + +import logging +from typing import Set +from prices import get_share_price + +logger = logging.getLogger() + + +def update_instrument_prices(job_id: str, db) -> None: + """ + Fetch current prices for all instruments in the user's portfolio using polygon.io. + Updates the instruments table with current prices. + + Args: + job_id: The job ID to identify the user's portfolio + db: Database instance + """ + try: + logger.info(f"Market: Fetching current prices for job {job_id}") + + # Get the job to find the user + job = db.jobs.find_by_id(job_id) + if not job: + logger.error(f"Market: Job {job_id} not found") + return + + user_id = job['clerk_user_id'] + + # Get all unique symbols from user's positions + accounts = db.accounts.find_by_user(user_id) + symbols = set() + + for account in accounts: + positions = db.positions.find_by_account(account['id']) + for position in positions: + symbols.add(position['symbol']) + + if not symbols: + logger.info("Market: No symbols to update prices for") + return + + logger.info(f"Market: Fetching prices for {len(symbols)} symbols: {symbols}") + + # Update prices for each symbol + update_prices_for_symbols(symbols, db) + + logger.info("Market: Price update complete") + + except Exception as e: + logger.error(f"Market: Error updating instrument prices: {e}") + # Non-critical error, continue with analysis + + +def update_prices_for_symbols(symbols: Set[str], db) -> None: + """ + Fetch and update prices for a set of symbols using polygon.io. + + Args: + symbols: Set of ticker symbols to update + db: Database instance + """ + if not symbols: + logger.info("Market: No symbols to update") + return + + symbols_list = list(symbols) + price_map = {} + + # Fetch price for each symbol using polygon.io + for symbol in symbols_list: + try: + price = get_share_price(symbol) + if price > 0: + price_map[symbol] = price + logger.info(f"Market: Retrieved {symbol} price: ${price:.2f}") + else: + logger.warning(f"Market: No price available for {symbol}") + except Exception as e: + logger.warning(f"Market: Could not fetch price for {symbol}: {e}") + + logger.info(f"Market: Retrieved prices for {len(price_map)}/{len(symbols_list)} symbols") + + # Update database with fetched prices + for symbol, price in price_map.items(): + try: + instrument = db.instruments.find_by_symbol(symbol) + if instrument: + update_data = {'current_price': price} + success = db.client.update( + 'instruments', + update_data, + "symbol = :symbol", + {'symbol': symbol} + ) + if success: + logger.info(f"Market: Updated {symbol} price to ${price:.2f}") + else: + logger.warning(f"Market: Failed to update price for {symbol}") + else: + logger.warning(f"Market: Instrument {symbol} not found in database") + except Exception as e: + logger.error(f"Market: Error updating {symbol} in database: {e}") + + # Log symbols that didn't get prices + missing = set(symbols_list) - set(price_map.keys()) + if missing: + logger.warning(f"Market: No prices found for: {missing}") + + +def get_all_portfolio_symbols(db) -> Set[str]: + """ + Get all unique symbols across all users' portfolios. + Useful for pre-fetching prices in batch operations. + + Args: + db: Database instance + + Returns: + Set of unique ticker symbols + """ + symbols = set() + + try: + # Get all positions (this might need pagination for large datasets) + all_positions = db.db.execute( + "SELECT DISTINCT symbol FROM positions" + ) + + for position in all_positions: + if position['symbol']: + symbols.add(position['symbol']) + + except Exception as e: + logger.error(f"Market: Error fetching all symbols: {e}") + + return symbols \ No newline at end of file diff --git a/gcp-deployment/backend/planner/observability.py b/gcp-deployment/backend/planner/observability.py new file mode 100644 index 00000000..bd2c0c7e --- /dev/null +++ b/gcp-deployment/backend/planner/observability.py @@ -0,0 +1,112 @@ +""" +Observability module for LangFuse integration. +Provides a simple context manager for setting up and flushing traces. +""" + +import os +import logging +from contextlib import contextmanager + +# Use root logger for Lambda compatibility +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@contextmanager +def observe(): + """ + Context manager for observability with LangFuse. + + Sets up LangFuse observability if environment variables are configured, + and ensures traces are flushed on exit. + + Usage: + from observability import observe + + with observe(): + # Your code that uses OpenAI Agents SDK + result = await agent.run(...) + """ + logger.info("๐Ÿ” Observability: Checking configuration...") + + # Check if required environment variables exist + has_langfuse = bool(os.getenv("LANGFUSE_SECRET_KEY")) + has_openai = bool(os.getenv("OPENAI_API_KEY")) + + logger.info(f"๐Ÿ” Observability: LANGFUSE_SECRET_KEY exists: {has_langfuse}") + logger.info(f"๐Ÿ” Observability: OPENAI_API_KEY exists: {has_openai}") + + if not has_langfuse: + logger.info("๐Ÿ” Observability: LangFuse not configured, skipping setup") + yield + return + + if not has_openai: + logger.warning("โš ๏ธ Observability: OPENAI_API_KEY not set, traces may not export") + + # Local variable for the client (no global needed) + langfuse_client = None + + # Try to set up LangFuse + try: + logger.info("๐Ÿ” Observability: Setting up LangFuse...") + + import logfire + from langfuse import get_client + + # Configure logfire to instrument OpenAI Agents SDK + logfire.configure( + service_name="alex_planner_agent", + send_to_logfire=False, # Don't send to Logfire cloud + ) + logger.info("โœ… Observability: Logfire configured") + + # Instrument OpenAI Agents SDK + logfire.instrument_openai_agents() + logger.info("โœ… Observability: OpenAI Agents SDK instrumented") + + # Initialize LangFuse client + langfuse_client = get_client() + logger.info("โœ… Observability: LangFuse client initialized") + + # Optional: Check authentication (blocking call, use sparingly) + try: + auth_result = langfuse_client.auth_check() + logger.info( + f"โœ… Observability: LangFuse authentication check passed (result: {auth_result})" + ) + except Exception as auth_error: + logger.warning(f"โš ๏ธ Observability: Auth check failed but continuing: {auth_error}") + + logger.info("๐ŸŽฏ Observability: Setup complete - traces will be sent to LangFuse") + + except ImportError as e: + logger.error(f"โŒ Observability: Missing required package: {e}") + langfuse_client = None + except Exception as e: + logger.error(f"โŒ Observability: Setup failed: {e}") + langfuse_client = None + + try: + # Yield control back to the calling code + yield + finally: + # Flush traces on exit + if langfuse_client: + try: + logger.info("๐Ÿ” Observability: Flushing traces to LangFuse...") + langfuse_client.flush() + langfuse_client.shutdown() + + # Add a 10 second delay to ensure network requests complete + # This is a workaround for Lambda's immediate termination + import time + + logger.info("๐Ÿ” Observability: Waiting 15 seconds for flush to complete...") + time.sleep(15) + + logger.info("โœ… Observability: Traces flushed successfully") + except Exception as e: + logger.error(f"โŒ Observability: Failed to flush traces: {e}") + else: + logger.debug("๐Ÿ” Observability: No client to flush") diff --git a/gcp-deployment/backend/planner/package_docker.py b/gcp-deployment/backend/planner/package_docker.py new file mode 100644 index 00000000..7c694978 --- /dev/null +++ b/gcp-deployment/backend/planner/package_docker.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Package the Planner Lambda function using Docker for AWS compatibility. +Uses the official AWS Lambda Python runtime image to ensure binary compatibility. +""" + +import os +import sys +import shutil +import tempfile +import subprocess +import argparse +from pathlib import Path + +def run_command(cmd, cwd=None): + """Run a command and capture output.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + +def package_lambda(): + """Package the Lambda function with all dependencies.""" + + # Get the directory containing this script + planner_dir = Path(__file__).parent.absolute() + backend_dir = planner_dir.parent + project_root = backend_dir.parent + + # Create a temporary directory for packaging + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + package_dir = temp_path / "package" + package_dir.mkdir() + + print("Creating Lambda package using Docker...") + + # Export exact requirements from uv.lock (excluding the editable database package) + print("Exporting requirements from uv.lock...") + requirements_result = run_command( + ["uv", "export", "--no-hashes", "--no-emit-project"], + cwd=str(planner_dir) + ) + + # Filter out packages that don't work in Lambda + filtered_requirements = [] + for line in requirements_result.splitlines(): + # Skip pyperclip (clipboard library not needed in Lambda) + if line.startswith("pyperclip"): + print(f"Excluding from Lambda: {line}") + continue + filtered_requirements.append(line) + + req_file = temp_path / "requirements.txt" + req_file.write_text("\n".join(filtered_requirements)) + + # Use Docker to install dependencies for Lambda's architecture + # The --no-emit-project excludes the current project from requirements + # We still need to manually install the database package + docker_cmd = [ + "docker", "run", "--rm", + "--platform", "linux/amd64", + "-v", f"{temp_path}:/build", + "-v", f"{backend_dir}/database:/database", + "--entrypoint", "/bin/bash", + "public.ecr.aws/lambda/python:3.12", + "-c", + """cd /build && pip install --target ./package -r requirements.txt && pip install --target ./package --no-deps /database""" + ] + + run_command(docker_cmd) + + # Copy Lambda handler and Python modules + shutil.copy(planner_dir / "lambda_handler.py", package_dir) + shutil.copy(planner_dir / "agent.py", package_dir) + shutil.copy(planner_dir / "templates.py", package_dir) + shutil.copy(planner_dir / "market.py", package_dir) + shutil.copy(planner_dir / "prices.py", package_dir) + shutil.copy(planner_dir / "observability.py", package_dir) + + # Create the zip file + zip_path = planner_dir / "planner_lambda.zip" + + # Remove old zip if it exists + if zip_path.exists(): + zip_path.unlink() + + # Create new zip + print(f"Creating zip file: {zip_path}") + run_command( + ["zip", "-r", str(zip_path), "."], + cwd=str(package_dir) + ) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"Package created: {zip_path} ({size_mb:.1f} MB)") + + return zip_path + +def deploy_lambda(zip_path): + """Deploy the Lambda function to AWS.""" + import boto3 + + lambda_client = boto3.client('lambda') + function_name = 'alex-planner' + + print(f"Deploying to Lambda function: {function_name}") + + try: + # Try to update existing function + with open(zip_path, 'rb') as f: + response = lambda_client.update_function_code( + FunctionName=function_name, + ZipFile=f.read() + ) + print(f"Successfully updated Lambda function: {function_name}") + print(f"Function ARN: {response['FunctionArn']}") + except lambda_client.exceptions.ResourceNotFoundException: + print(f"Lambda function {function_name} not found. Please deploy via Terraform first.") + sys.exit(1) + except Exception as e: + print(f"Error deploying Lambda: {e}") + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser(description='Package Planner Lambda for deployment') + parser.add_argument('--deploy', action='store_true', help='Deploy to AWS after packaging') + args = parser.parse_args() + + # Check if Docker is available + try: + run_command(["docker", "--version"]) + except FileNotFoundError: + print("Error: Docker is not installed or not in PATH") + sys.exit(1) + + # Package the Lambda + zip_path = package_lambda() + + # Deploy if requested + if args.deploy: + deploy_lambda(zip_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/planner/prices.py b/gcp-deployment/backend/planner/prices.py new file mode 100644 index 00000000..7eaacff8 --- /dev/null +++ b/gcp-deployment/backend/planner/prices.py @@ -0,0 +1,65 @@ +from polygon import RESTClient +from dotenv import load_dotenv +import os +from datetime import datetime +import random +from functools import lru_cache +from datetime import timezone + +load_dotenv(override=True) + +polygon_api_key = os.getenv("POLYGON_API_KEY") +polygon_plan = os.getenv("POLYGON_PLAN") + +is_paid_polygon = polygon_plan == "paid" + + +def is_market_open() -> bool: + client = RESTClient(polygon_api_key) + market_status = client.get_market_status() + return market_status.market == "open" + + +def get_all_share_prices_polygon_eod() -> dict[str, float]: + """With much thanks to student Reema R. for fixing the timezone issue with this!""" + client = RESTClient(polygon_api_key) + + probe = client.get_previous_close_agg("SPY")[0] + last_close = datetime.fromtimestamp(probe.timestamp / 1000, tz=timezone.utc).date() + + results = client.get_grouped_daily_aggs(last_close, adjusted=True, include_otc=False) + return {result.ticker: result.close for result in results} + + +@lru_cache(maxsize=2) +def get_market_for_prior_date(today): + market_data = get_all_share_prices_polygon_eod() + return market_data + + +def get_share_price_polygon_eod(symbol) -> float: + today = datetime.now().date().strftime("%Y-%m-%d") + market_data = get_market_for_prior_date(today) + return market_data.get(symbol, 0.0) + + +def get_share_price_polygon_min(symbol) -> float: + client = RESTClient(polygon_api_key) + result = client.get_snapshot_ticker("stocks", symbol) + return result.min.close or result.prev_day.close + + +def get_share_price_polygon(symbol) -> float: + if is_paid_polygon: + return get_share_price_polygon_min(symbol) + else: + return get_share_price_polygon_eod(symbol) + + +def get_share_price(symbol) -> float: + if polygon_api_key: + try: + return get_share_price_polygon(symbol) + except Exception as e: + print(f"Was not able to use the polygon API due to {e}; using a random number") + return float(random.randint(1, 100)) diff --git a/gcp-deployment/backend/planner/pyproject.toml b/gcp-deployment/backend/planner/pyproject.toml new file mode 100644 index 00000000..b5067ea6 --- /dev/null +++ b/gcp-deployment/backend/planner/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "planner" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "boto3>=1.40.8", # Keep for backward compatibility + "fastapi>=0.116.1", + "uvicorn>=0.35.0", + "httpx>=0.28.1", + "langfuse>=3.3.4", + "openai-agents[litellm]>=0.3.0", + "polygon-api-client>=1.15.3", + "pydantic>=2.11.7", + "pydantic-ai>=1.0.6", + "python-dotenv>=1.1.1", + "tenacity>=9.1.2", +] + +[tool.uv.sources] +alex-database = { path = "../database", editable = true } diff --git a/gcp-deployment/backend/planner/server.py b/gcp-deployment/backend/planner/server.py new file mode 100644 index 00000000..e4ea0c7d --- /dev/null +++ b/gcp-deployment/backend/planner/server.py @@ -0,0 +1,224 @@ +""" +Planner Agent - Cloud Run HTTP Server +Orchestrates portfolio analysis across specialized agents +""" + +import os +import sys +import json +import base64 +import asyncio +import logging +from pathlib import Path +from typing import Dict, Any +from datetime import datetime, UTC + +# Add parent directories to Python path for imports +backend_dir = Path(__file__).parent.parent +if str(backend_dir) not in sys.path: + sys.path.insert(0, str(backend_dir)) + +from fastapi import FastAPI, HTTPException, Request +from pydantic import BaseModel +from dotenv import load_dotenv +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +# Load .env file from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Import database package +from src import Database + +# Import from common module (now in path) +from common.llm import get_litellm_model + +from templates import ORCHESTRATOR_INSTRUCTIONS +from agent import create_agent, handle_missing_instruments, load_portfolio_summary +from market import update_instrument_prices +from observability import observe + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize FastAPI app +app = FastAPI( + title="Alex Planner Service", + description="Orchestrator agent for portfolio analysis", + version="1.0.0" +) + +# Initialize database +db = Database() + + +@retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info(f"Planner: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds...") +) +async def run_orchestrator(job_id: str) -> None: + """Run the orchestrator agent to coordinate portfolio analysis.""" + try: + # Update job status to running + db.jobs.update_status(job_id, 'running') + + # Handle missing instruments first (non-agent pre-processing) + await asyncio.to_thread(handle_missing_instruments, job_id, db) + + # Update instrument prices after tagging + await asyncio.to_thread(update_instrument_prices, job_id, db) + + # Load portfolio summary + portfolio_summary = await asyncio.to_thread(load_portfolio_summary, job_id, db) + + # Create and run the orchestrator agent + model, tools, task, context = create_agent(job_id, portfolio_summary, db) + + with trace("Planner Orchestrator"): + agent = Agent[type(context)]( + name="Financial Planner Orchestrator", + instructions=ORCHESTRATOR_INSTRUCTIONS, + model=model, + tools=tools + ) + + result = await Runner.run( + agent, + input=task, + context=context, + max_turns=20 + ) + + # Update job status to completed + db.jobs.update_status(job_id, 'completed') + logger.info(f"Planner: Orchestration completed for job {job_id}") + + except Exception as e: + logger.error(f"Planner: Error in orchestration: {e}", exc_info=True) + db.jobs.update_status(job_id, 'failed') + raise + + +# Request/Response models +class JobRequest(BaseModel): + """Request to process a job""" + job_id: str + + +class PubSubMessage(BaseModel): + """Pub/Sub push message format""" + message: Dict[str, Any] + subscription: str + + +@app.get("/") +async def root(): + """Health check endpoint""" + return { + "service": "Alex Planner", + "status": "healthy", + "timestamp": datetime.now(UTC).isoformat(), + } + + +@app.get("/health") +async def health(): + """Health check endpoint (alternative)""" + return {"status": "healthy"} + + +@app.post("/") +async def handle_job(request: JobRequest): + """ + Handle job processing request (direct invocation). + + Used for: + - Direct HTTP calls from other services + - Testing + + Request body: {"job_id": "uuid-string"} + + Note: job_id must be a valid UUID and the job must exist in the database. + """ + try: + logger.info(f"Planner: Received direct job request: {request.job_id}") + + # Validate that job exists before processing + job = db.jobs.find_by_id(request.job_id) + if not job: + raise HTTPException( + status_code=404, + detail=f"Job {request.job_id} not found in database. Create the job first via the API." + ) + + await run_orchestrator(request.job_id) + return { + "success": True, + "message": f"Analysis completed for job {request.job_id}", + "job_id": request.job_id + } + except HTTPException: + raise + except Exception as e: + logger.error(f"Planner: Error processing job: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/pubsub") +async def handle_pubsub_push(request: Request): + """ + Handle Pub/Sub push subscription. + + Expected format: + { + "message": { + "data": "base64_encoded_json", + "attributes": {} + }, + "subscription": "projects/.../subscriptions/..." + } + """ + try: + body = await request.json() + message = body.get('message', {}) + + # Decode base64 message data + message_data = message.get('data', '') + if not message_data: + raise HTTPException(status_code=400, detail="No data in Pub/Sub message") + + decoded_data = base64.b64decode(message_data).decode('utf-8') + payload = json.loads(decoded_data) + job_id = payload.get('job_id') + + if not job_id: + raise HTTPException(status_code=400, detail="No job_id in message payload") + + logger.info(f"Planner: Received Pub/Sub message for job: {job_id}") + + # Run orchestrator + await run_orchestrator(job_id) + + return { + "success": True, + "message": f"Analysis completed for job {job_id}", + "job_id": job_id + } + except json.JSONDecodeError as e: + logger.error(f"Planner: Error decoding Pub/Sub message: {e}") + raise HTTPException(status_code=400, detail=f"Invalid JSON in message: {e}") + except Exception as e: + logger.error(f"Planner: Error processing Pub/Sub message: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# For local testing +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) + diff --git a/gcp-deployment/backend/planner/templates.py b/gcp-deployment/backend/planner/templates.py new file mode 100644 index 00000000..afdbad01 --- /dev/null +++ b/gcp-deployment/backend/planner/templates.py @@ -0,0 +1,19 @@ +""" +Instruction templates for the Financial Planner orchestrator agent. +""" + +ORCHESTRATOR_INSTRUCTIONS = """You coordinate portfolio analysis by calling other agents. + +Tools (use ONLY these three): +- invoke_reporter: Generates analysis text +- invoke_charter: Creates charts +- invoke_retirement: Calculates retirement projections + +Steps: +1. Call invoke_reporter if positions > 0 +2. Call invoke_charter if positions >= 2 +3. Call invoke_retirement if retirement goals exist +4. Respond with "Done" + +Use ONLY the three tools above. +""" \ No newline at end of file diff --git a/gcp-deployment/backend/planner/test_full.py b/gcp-deployment/backend/planner/test_full.py new file mode 100644 index 00000000..c70b02c9 --- /dev/null +++ b/gcp-deployment/backend/planner/test_full.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +Run a full end-to-end test of the Alex agent orchestration. +This creates a test job and monitors it through completion. + +Usage: + cd backend/planner + uv run run_full_test.py +""" + +import os +import json +import boto3 +import time +import logging +from datetime import datetime, timezone +from dotenv import load_dotenv + +# Load environment +load_dotenv(override=True) + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Import database +from src import Database + +db = Database() +sqs = boto3.client('sqs') +sts = boto3.client('sts') + +# Get configuration +QUEUE_NAME = os.getenv('SQS_QUEUE_NAME', 'alex-analysis-jobs') + + +def get_queue_url(): + """Get the SQS queue URL.""" + response = sqs.list_queues(QueueNamePrefix=QUEUE_NAME) + queues = response.get('QueueUrls', []) + + for queue_url in queues: + if QUEUE_NAME in queue_url: + return queue_url + + raise ValueError(f"Queue {QUEUE_NAME} not found") + + +def main(): + """Run the full test.""" + print("=" * 70) + print("๐ŸŽฏ Alex Agent Orchestration - Full Test") + print("=" * 70) + + # Display AWS info + account_id = sts.get_caller_identity()['Account'] + region = boto3.Session().region_name + print(f"AWS Account: {account_id}") + print(f"AWS Region: {region}") + print(f"Bedrock Region: {os.getenv('BEDROCK_REGION', 'us-west-2')}") + print(f"Bedrock Model: {os.getenv('BEDROCK_MODEL_ID', 'Not set')}") + print() + + # Check for test user + print("๐Ÿ“Š Checking test data...") + test_user_id = 'test_user_001' + user = db.users.find_by_clerk_id(test_user_id) + + if not user: + print("โŒ Test user not found. Please run database setup first:") + print(" cd ../database && uv run reset_db.py --with-test-data") + return 1 + + print(f"โœ“ Test user: {user.get('display_name', test_user_id)}") + + # Check accounts and positions + accounts = db.accounts.find_by_user(test_user_id) + total_positions = 0 + for account in accounts: + positions = db.positions.find_by_account(account['id']) + total_positions += len(positions) + + print(f"โœ“ Portfolio: {len(accounts)} accounts, {total_positions} positions") + + # Create test job + print("\n๐Ÿš€ Creating test job...") + job_data = { + 'clerk_user_id': test_user_id, + 'job_type': 'portfolio_analysis', + 'status': 'pending', + 'request_payload': { + 'analysis_type': 'full', + 'requested_at': datetime.now(timezone.utc).isoformat(), + 'test_run': True + } + } + + job_id = db.jobs.create(job_data) + print(f"โœ“ Created job: {job_id}") + + # Send to SQS + print("\n๐Ÿ“ค Sending job to SQS queue...") + try: + queue_url = get_queue_url() + response = sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps({'job_id': job_id}) + ) + print(f"โœ“ Message sent: {response['MessageId']}") + except Exception as e: + print(f"โŒ Failed to send to SQS: {e}") + return 1 + + # Monitor job + print("\nโณ Monitoring job progress (timeout: 3 minutes)...") + print("-" * 50) + + start_time = time.time() + timeout = 180 # 3 minutes + last_status = None + + while time.time() - start_time < timeout: + job = db.jobs.find_by_id(job_id) + status = job['status'] + + if status != last_status: + elapsed = int(time.time() - start_time) + print(f"[{elapsed:3d}s] Status: {status}") + last_status = status + + if status == 'completed': + print("-" * 50) + print("โœ… Job completed successfully!") + break + elif status == 'failed': + print("-" * 50) + print(f"โŒ Job failed: {job.get('error_message', 'Unknown error')}") + return 1 + + time.sleep(2) + else: + print("-" * 50) + print("โŒ Job timed out after 3 minutes") + return 1 + + # Display results + print("\n" + "=" * 70) + print("๐Ÿ“‹ ANALYSIS RESULTS") + print("=" * 70) + + # Orchestrator summary + if job.get('summary_payload'): + print("\n๐ŸŽฏ Orchestrator Summary:") + summary = job['summary_payload'] + print(f"Summary: {summary.get('summary', 'N/A')}") + + if summary.get('key_findings'): + print("\nKey Findings:") + for finding in summary['key_findings']: + print(f" โ€ข {finding}") + + if summary.get('recommendations'): + print("\nRecommendations:") + for rec in summary['recommendations']: + print(f" โ€ข {rec}") + + # Report analysis + if job.get('report_payload'): + print("\n๐Ÿ“ Portfolio Report:") + report = job['report_payload'] + analysis = report.get('analysis', '') + print(f" Length: {len(analysis)} characters") + if analysis: + preview = analysis[:300] + if len(analysis) > 300: + preview += "..." + print(f" Preview: {preview}") + + # Charts + if job.get('charts_payload'): + print(f"\n๐Ÿ“Š Visualizations: {len(job['charts_payload'])} charts") + for chart_key, chart_data in job['charts_payload'].items(): + print(f" โ€ข {chart_key}: {chart_data.get('title', 'Untitled')}") + if chart_data.get('data'): + print(f" Data points: {len(chart_data['data'])}") + + # Retirement projections + if job.get('retirement_payload'): + print("\n๐ŸŽฏ Retirement Analysis:") + ret = job['retirement_payload'] + print(f" Success Rate: {ret.get('success_rate', 'N/A')}%") + print(f" Projected Value: ${ret.get('projected_value', 0):,.0f}") + print(f" Years to Retirement: {ret.get('years_to_retirement', 'N/A')}") + + print("\n" + "=" * 70) + print("โœ… Full test completed successfully!") + print("=" * 70) + + return 0 + + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/gcp-deployment/backend/planner/test_market.py b/gcp-deployment/backend/planner/test_market.py new file mode 100644 index 00000000..0ae647af --- /dev/null +++ b/gcp-deployment/backend/planner/test_market.py @@ -0,0 +1,51 @@ +""" +Test market data fetching +""" + +from src import Database +from market import update_instrument_prices + +def test_market(): + db = Database() + + # Find a user with positions + user_id = 'user_30BmVRQvPMVcGt9kWAH4BOy5Cjy' + + # Create a test job + job_id = db.jobs.create_job( + clerk_user_id=user_id, + job_type='test_market', + request_payload={'test': True} + ) + + print(f"Testing market data fetch for job {job_id}") + + # Get initial prices + accounts = db.accounts.find_by_user(user_id) + symbols = set() + for account in accounts: + positions = db.positions.find_by_account(account['id']) + for position in positions: + symbols.add(position['symbol']) + instrument = db.instruments.find_by_symbol(position['symbol']) + if instrument: + print(f" {position['symbol']}: Current price = ${instrument.get('current_price')}") + + print(f"\nFetching prices for {len(symbols)} symbols...") + + # Update prices + update_instrument_prices(job_id, db) + + print("\nAfter update:") + # Check updated prices + for symbol in symbols: + instrument = db.instruments.find_by_symbol(symbol) + if instrument: + print(f" {symbol}: Current price = ${instrument.get('current_price')}") + + # Clean up + db.jobs.delete(job_id) + print(f"\nDeleted test job {job_id}") + +if __name__ == "__main__": + test_market() \ No newline at end of file diff --git a/gcp-deployment/backend/planner/test_server.py b/gcp-deployment/backend/planner/test_server.py new file mode 100644 index 00000000..50cc2cfe --- /dev/null +++ b/gcp-deployment/backend/planner/test_server.py @@ -0,0 +1,79 @@ +""" +Test script for Planner server +Creates a test job and processes it +""" + +import requests +import uuid +import json + +# Test server URL +BASE_URL = "http://localhost:8000" + +def test_health_check(): + """Test health check endpoint""" + print("Testing health check...") + response = requests.get(f"{BASE_URL}/") + print(f"Status: {response.status_code}") + print(f"Response: {response.json()}") + print() + +def test_job_processing(): + """Test job processing with a valid UUID""" + print("Testing job processing...") + + # Generate a valid UUID for testing + test_job_id = str(uuid.uuid4()) + print(f"Using job ID: {test_job_id}") + + # First, we need to create a job in the database + # For now, let's just test if the endpoint accepts the request + # In a real scenario, the job would be created via the API first + + payload = { + "job_id": test_job_id + } + + try: + response = requests.post( + f"{BASE_URL}/", + json=payload, + timeout=30 + ) + print(f"Status: {response.status_code}") + print(f"Response: {response.json()}") + except requests.exceptions.RequestException as e: + print(f"Error: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Response: {e.response.text}") + +def test_with_existing_job(): + """Test with a job that exists in the database""" + print("Testing with existing job...") + print("Note: This requires a job to exist in the database.") + print("Create a job via the API first, then use its ID here.") + print() + + # You can get a real job ID from the database or API + # For example, from a previous API call: + # job_id = "your-actual-job-uuid-here" + # payload = {"job_id": job_id} + # response = requests.post(f"{BASE_URL}/", json=payload) + +if __name__ == "__main__": + print("=" * 50) + print("Planner Server Test") + print("=" * 50) + print() + + # Test 1: Health check + test_health_check() + + # Test 2: Job processing (will fail if job doesn't exist) + print("Note: Job processing test requires a job to exist in the database.") + print("The job_id must be a valid UUID and exist in the jobs table.") + print() + + # Uncomment to test (will fail if job doesn't exist): + # test_job_processing() + diff --git a/gcp-deployment/backend/planner/test_simple.py b/gcp-deployment/backend/planner/test_simple.py new file mode 100644 index 00000000..8768d055 --- /dev/null +++ b/gcp-deployment/backend/planner/test_simple.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +Simple test for Planner orchestrator +""" + +import asyncio +import json +import os +import subprocess +from dotenv import load_dotenv + +load_dotenv(override=True) + +# Mock lambdas for testing +os.environ['MOCK_LAMBDAS'] = 'true' + +from src import Database +from src.schemas import JobCreate + +def setup_test_data(): + """Ensure test data exists and create a test job""" + # Run reset_db with test data to ensure we have a test user and portfolio + print("Ensuring test data exists...") + result = subprocess.run( + ["uv", "run", "reset_db.py", "--with-test-data", "--skip-drop"], + cwd="../database", + capture_output=True, + text=True + ) + if result.returncode != 0: + print(f"Warning: Could not ensure test data: {result.stderr}") + + db = Database() + + # The reset_db script creates test_user_001 + test_user_id = "test_user_001" + + # Check if user exists + user = db.users.find_by_clerk_id(test_user_id) + if not user: + raise ValueError(f"Test user {test_user_id} not found. Please run: cd ../database && uv run reset_db.py --with-test-data") + + # Create test job + job_create = JobCreate( + clerk_user_id=test_user_id, + job_type="portfolio_analysis", + request_payload={"analysis_type": "comprehensive", "test": True} + ) + job_id = db.jobs.create(job_create.model_dump()) + + return job_id + +def test_planner(): + """Test the planner orchestrator""" + + # Setup test data + job_id = setup_test_data() + + test_event = { + "job_id": job_id + } + + print("Testing Planner Orchestrator...") + print(f"Job ID: {job_id}") + print("=" * 60) + + from lambda_handler import lambda_handler + + result = lambda_handler(test_event, None) + + print(f"Status Code: {result['statusCode']}") + + if result['statusCode'] == 200: + body = json.loads(result['body']) + print(f"Success: {body.get('success', False)}") + print(f"Message: {body.get('message', 'N/A')}") + else: + print(f"Error: {result['body']}") + + print("=" * 60) + +if __name__ == "__main__": + test_planner() \ No newline at end of file diff --git a/gcp-deployment/backend/planner/test_workflow.ps1 b/gcp-deployment/backend/planner/test_workflow.ps1 new file mode 100644 index 00000000..d3f87900 --- /dev/null +++ b/gcp-deployment/backend/planner/test_workflow.ps1 @@ -0,0 +1,450 @@ +# Test script for agent workflow +# This script creates a job and triggers the planner agent + +param( + [string]$PlannerUrl = "", + [string]$ClerkUserId = "user_test123", + [string]$DbPassword = "", + [string]$DbHost = "", + [int]$DbPort = 0 +) + +# Function to load .env file +function Load-EnvFile { + param([string]$EnvPath) + + if (-not (Test-Path $EnvPath)) { + Write-Host "Warning: .env file not found at $EnvPath" -ForegroundColor Yellow + return @{} + } + + $envVars = @{} + Get-Content $EnvPath | ForEach-Object { + if ($_ -match '^\s*([^#][^=]*)\s*=\s*(.*)$') { + $key = $matches[1].Trim() + $value = $matches[2].Trim() + # Remove quotes if present + if ($value -match '^["''](.*)["'']$') { + $value = $matches[1] + } + $envVars[$key] = $value + } + } + return $envVars +} + +# Load .env file from project root +$projectRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$envPath = Join-Path $projectRoot ".env" +$envVars = Load-EnvFile -EnvPath $envPath + +# Set environment variables from .env +foreach ($key in $envVars.Keys) { + if (-not [string]::IsNullOrEmpty($envVars[$key])) { + Set-Item -Path "env:$key" -Value $envVars[$key] + } +} + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Agent Workflow Test" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +# Step 1: Get Planner URL +if ([string]::IsNullOrEmpty($PlannerUrl)) { + if ($env:PLANNER_URL) { + $PlannerUrl = $env:PLANNER_URL + } else { + Write-Host "Getting Planner URL from Terraform..." -ForegroundColor Yellow + Push-Location "$PSScriptRoot/../../terraform/6_agents" + $PlannerUrl = terraform output -raw planner_service_url 2>$null + Pop-Location + if ([string]::IsNullOrEmpty($PlannerUrl)) { + Write-Host "ERROR: Could not get Planner URL. Set it manually with -PlannerUrl or PLANNER_URL in .env" -ForegroundColor Red + exit 1 + } + } +} +Write-Host "Planner URL: $PlannerUrl" -ForegroundColor Green +Write-Host "" + +# Step 2: Get database connection info +if ([string]::IsNullOrEmpty($DbHost)) { + $DbHost = $env:DB_HOST + if ([string]::IsNullOrEmpty($DbHost)) { + $DbHost = "127.0.0.1" + } +} + +if ($DbPort -eq 0) { + $DbPort = $env:DB_PORT + if ([string]::IsNullOrEmpty($DbPort)) { + $DbPort = 5432 + } else { + $DbPort = [int]$DbPort + } +} + +# Get database user and name +$DbUser = $env:DATABASE_USER +if ([string]::IsNullOrEmpty($DbUser)) { + $DbUser = $env:DB_USER + if ([string]::IsNullOrEmpty($DbUser)) { + $DbUser = "alex_app" # Default from codebase + } +} + +$DbName = $env:DATABASE_NAME +if ([string]::IsNullOrEmpty($DbName)) { + $DbName = $env:DB_NAME + if ([string]::IsNullOrEmpty($DbName)) { + $DbName = "alex" # Default from codebase + } +} + +if ([string]::IsNullOrEmpty($DbPassword)) { + # Try to get password from Secret Manager if DB_PASSWORD_SECRET_ID is set + if ($env:DB_PASSWORD_SECRET_ID -and $env:GCP_PROJECT_ID) { + Write-Host "Retrieving database password from Secret Manager..." -ForegroundColor Yellow + try { + $DbPassword = gcloud secrets versions access latest --secret=$env:DB_PASSWORD_SECRET_ID --project=$env:GCP_PROJECT_ID 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Host "Warning: Failed to get password from Secret Manager. Will prompt for password." -ForegroundColor Yellow + $DbPassword = "" + } + } catch { + Write-Host "Warning: Failed to get password from Secret Manager. Will prompt for password." -ForegroundColor Yellow + $DbPassword = "" + } + } else { + # Try direct DB_PASSWORD from .env + $DbPassword = $env:DB_PASSWORD + } + + if ([string]::IsNullOrEmpty($DbPassword)) { + $DbPassword = Read-Host "Enter database password" + } +} + +$env:PGPASSWORD = $DbPassword + +Write-Host "Database connection:" -ForegroundColor Cyan +Write-Host " Host: $DbHost" -ForegroundColor Gray +Write-Host " Port: $DbPort" -ForegroundColor Gray +Write-Host " User: $DbUser" -ForegroundColor Gray +Write-Host " Database: $DbName" -ForegroundColor Gray +Write-Host "" + +# Helper function to run psql and handle errors +function Invoke-PSQL { + param( + [string]$Command, + [hashtable]$Variables = @{}, + [switch]$Silent + ) + + # Build psql command with variables if provided + $psqlArgs = @("-h", $DbHost, "-p", $DbPort, "-U", $DbUser, "-d", $DbName, "-t") + + # Add variables using -v option + foreach ($key in $Variables.Keys) { + $psqlArgs += "-v" + $psqlArgs += "${key}=$($Variables[$key])" + } + + $psqlArgs += "-c" + $psqlArgs += $Command + + $output = & psql $psqlArgs 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + if (-not $Silent) { + Write-Host "ERROR: Database connection failed" -ForegroundColor Red + Write-Host $output -ForegroundColor Red + } + return $null + } + + if ($output) { + return $output.Trim() + } + return "" +} + +# Step 3: Create test user and account (if needed) +Write-Host "Step 1: Setting up test user and account..." -ForegroundColor Yellow + +# Check if user exists +$userExists = Invoke-PSQL -Command "SELECT COUNT(*) FROM users WHERE clerk_user_id = '$ClerkUserId';" +if ($null -eq $userExists) { + Write-Host "ERROR: Cannot connect to database. Is Cloud SQL proxy running?" -ForegroundColor Red + Write-Host " Start proxy: cloud-sql-proxy --port=$DbPort $($env:INSTANCE_CONNECTION_NAME)" -ForegroundColor Yellow + exit 1 +} + +if ($userExists -eq "0") { + Write-Host "Creating test user..." -ForegroundColor Yellow + $null = Invoke-PSQL -Command "INSERT INTO users (clerk_user_id, display_name) VALUES ('$ClerkUserId', 'Test User') ON CONFLICT (clerk_user_id) DO NOTHING;" -Silent +} + +# Get or create account +$accountId = Invoke-PSQL -Command "SELECT id FROM accounts WHERE clerk_user_id = '$ClerkUserId' LIMIT 1;" + +if ([string]::IsNullOrEmpty($accountId)) { + Write-Host "Creating test account..." -ForegroundColor Yellow + $accountId = Invoke-PSQL -Command "INSERT INTO accounts (id, clerk_user_id, account_name, account_purpose, cash_balance) VALUES (gen_random_uuid(), '$ClerkUserId', 'Test 401k', '401k', 0.00) RETURNING id;" +} + +if ([string]::IsNullOrEmpty($accountId)) { + Write-Host "ERROR: Failed to create or retrieve account" -ForegroundColor Red + exit 1 +} + +Write-Host "Account ID: $accountId" -ForegroundColor Green + +# Add test positions if needed +$positionCount = Invoke-PSQL -Command "SELECT COUNT(*) FROM positions p JOIN accounts a ON p.account_id = a.id WHERE a.clerk_user_id = '$ClerkUserId';" + +if ($positionCount -eq "0") { + Write-Host "Adding test positions (SPY, QQQ, BND)..." -ForegroundColor Yellow + $sqlPositions = "INSERT INTO positions (id, account_id, symbol, quantity) VALUES (gen_random_uuid(), '$accountId', 'SPY', 10), (gen_random_uuid(), '$accountId', 'QQQ', 5), (gen_random_uuid(), '$accountId', 'BND', 20) ON CONFLICT DO NOTHING;" + $null = Invoke-PSQL -Command $sqlPositions -Silent +} + +Write-Host "User and account ready" -ForegroundColor Green +Write-Host "" + +# Step 4: Create job +Write-Host "Step 2: Creating analysis job..." -ForegroundColor Yellow +$payloadJson = (@{ + analysis_type = "portfolio_analysis" + options = @{} +} | ConvertTo-Json -Compress) + +# Write SQL to temporary file to avoid quote escaping issues +$tempFile = [System.IO.Path]::GetTempFileName() +# Build SQL content - use single quotes around JSON, JSON itself has double quotes which is fine +$sqlContent = "INSERT INTO jobs (id, clerk_user_id, job_type, status, request_payload) VALUES (gen_random_uuid(), '$ClerkUserId', 'portfolio_analysis', 'pending', '$payloadJson'::jsonb) RETURNING id;" +# Write to file using UTF8 encoding +[System.IO.File]::WriteAllText($tempFile, $sqlContent, [System.Text.Encoding]::UTF8) + +try { + # Execute SQL from file + $output = psql -h $DbHost -p $DbPort -U $DbUser -d $DbName -t -f $tempFile 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -eq 0 -and $output) { + $rawOutput = $output.Trim() + Write-Host "Raw psql output: $rawOutput" -ForegroundColor Gray + + # Extract UUID from output (may include "INSERT 0 1" text) + # Pattern: UUID format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (case-insensitive) + $uuidPattern = '([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' + + $jobId = $null + if ($rawOutput -match $uuidPattern) { + if ($matches -and $matches.Count -gt 1) { + $jobId = $matches[1] + Write-Host "Extracted job ID: $jobId" -ForegroundColor Gray + } + } + + # If regex didn't work, try simpler extraction + if ([string]::IsNullOrEmpty($jobId)) { + # Split by whitespace and take first token (should be UUID) + $tokens = $rawOutput -split '\s+' + if ($tokens.Count -gt 0) { + $firstToken = $tokens[0].Trim() + # Verify it looks like a UUID + if ($firstToken -match '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') { + $jobId = $firstToken + Write-Host "Extracted job ID from first token: $jobId" -ForegroundColor Gray + } + } + } + + if ([string]::IsNullOrEmpty($jobId)) { + Write-Host "WARNING: Could not extract job ID from output: $rawOutput" -ForegroundColor Yellow + } + } else { + $jobId = $null + Write-Host "ERROR: Database connection failed" -ForegroundColor Red + Write-Host $output -ForegroundColor Red + } +} finally { + # Clean up temp file + if (Test-Path $tempFile) { + Remove-Item $tempFile -Force + } +} + +if ([string]::IsNullOrEmpty($jobId)) { + # Fallback: try using psql with -v variable (avoids quote escaping) + Write-Host "Trying alternative method with psql variables..." -ForegroundColor Yellow + + # Create a new temp file - try with dollar-quoting + $tempFile2 = [System.IO.Path]::GetTempFileName() + # Use dollar-quoting: $tag$content$tag$ to avoid quote escaping + # Build the SQL by concatenation to avoid PowerShell variable expansion + $dollarStart = '$json$' + $dollarEnd = '$json$' + $sqlContent2 = "INSERT INTO jobs (id, clerk_user_id, job_type, status, request_payload) VALUES (gen_random_uuid(), '$ClerkUserId', 'portfolio_analysis', 'pending', " + $dollarStart + $payloadJson + $dollarEnd + "::jsonb) RETURNING id;" + [System.IO.File]::WriteAllText($tempFile2, $sqlContent2, [System.Text.Encoding]::UTF8) + + try { + $output2 = psql -h $DbHost -p $DbPort -U $DbUser -d $DbName -t -f $tempFile2 2>&1 + $exitCode2 = $LASTEXITCODE + + if ($exitCode2 -eq 0 -and $output2) { + $rawOutput2 = $output2.Trim() + Write-Host "Alternative method output: $rawOutput2" -ForegroundColor Gray + + # Try regex first + $uuidPattern2 = '([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' + if ($rawOutput2 -match $uuidPattern2) { + if ($matches -and $matches.Count -gt 1) { + $jobId = $matches[1] + Write-Host "Successfully created job with alternative method: $jobId" -ForegroundColor Green + } + } + + # Fallback to first token + if ([string]::IsNullOrEmpty($jobId)) { + $tokens2 = $rawOutput2 -split '\s+' + if ($tokens2.Count -gt 0) { + $firstToken2 = $tokens2[0].Trim() + if ($firstToken2 -match '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') { + $jobId = $firstToken2 + Write-Host "Extracted job ID from first token: $jobId" -ForegroundColor Green + } + } + } + + if ([string]::IsNullOrEmpty($jobId)) { + Write-Host "Alternative method also failed to extract job ID" -ForegroundColor Red + } + } else { + Write-Host "Alternative method also failed:" -ForegroundColor Red + Write-Host $output2 -ForegroundColor Red + } + } finally { + if (Test-Path $tempFile2) { + Remove-Item $tempFile2 -Force + } + } +} + +if ([string]::IsNullOrEmpty($jobId)) { + Write-Host "ERROR: Failed to create job" -ForegroundColor Red + exit 1 +} + +# Ensure job ID is a clean UUID (remove any extra text) +$jobId = $jobId.Trim() +if ($jobId -match '([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})') { + $jobId = $matches[1] +} else { + Write-Host "ERROR: Invalid job ID format: $jobId" -ForegroundColor Red + exit 1 +} + +Write-Host "Created job: $jobId" -ForegroundColor Green +Write-Host "" + +# Step 5: Get authentication token +Write-Host "Step 3: Getting authentication token..." -ForegroundColor Yellow +try { + $token = gcloud auth print-identity-token 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Failed to get identity token. Run 'gcloud auth login' first." -ForegroundColor Red + exit 1 + } + Write-Host "Token obtained" -ForegroundColor Green +} catch { + Write-Host "ERROR: Failed to get identity token: $_" -ForegroundColor Red + exit 1 +} +Write-Host "" + +# Step 6: Trigger planner +Write-Host "Step 4: Triggering planner agent..." -ForegroundColor Yellow +$headers = @{ + "Authorization" = "Bearer $token" + "Content-Type" = "application/json" +} +$body = @{ + job_id = $jobId +} | ConvertTo-Json + +try { + Write-Host "Sending POST request to $PlannerUrl..." -ForegroundColor Cyan + Write-Host "Job ID being sent: $jobId" -ForegroundColor Gray + Write-Host "Request body: $body" -ForegroundColor Gray + $response = Invoke-WebRequest -Uri "$PlannerUrl/" -Method POST -Headers $headers -Body $body -TimeoutSec 300 + Write-Host "Planner triggered successfully!" -ForegroundColor Green + Write-Host "" + Write-Host "Response:" -ForegroundColor Cyan + Write-Host $response.Content +} catch { + Write-Host "ERROR: Failed to trigger planner" -ForegroundColor Red + Write-Host $_.Exception.Message -ForegroundColor Red + if ($_.Exception.Response) { + $reader = New-Object System.IO.StreamReader($_.Exception.Response.GetResponseStream()) + $responseBody = $reader.ReadToEnd() + Write-Host "Response body: $responseBody" -ForegroundColor Red + } + Write-Host "" + Write-Host "Debugging info:" -ForegroundColor Yellow + Write-Host " Job ID: $jobId" -ForegroundColor Gray + Write-Host " Job ID length: $($jobId.Length)" -ForegroundColor Gray + Write-Host " Is valid UUID format: $($jobId -match '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$')" -ForegroundColor Gray + + # Verify job exists in database + Write-Host " Verifying job exists in database..." -ForegroundColor Yellow + $jobCheck = Invoke-PSQL -Command "SELECT id, status FROM jobs WHERE id = '$jobId';" -Silent + if ($jobCheck) { + Write-Host " Job found in database: $jobCheck" -ForegroundColor Green + } else { + Write-Host " Job NOT found in database!" -ForegroundColor Red + } + + exit 1 +} +Write-Host "" + +# Step 7: Wait and check status +Write-Host "Step 5: Waiting for job to complete (30 seconds)..." -ForegroundColor Yellow +Start-Sleep -Seconds 30 + +Write-Host "Checking job status..." -ForegroundColor Yellow +$sqlCheckStatus = "SELECT id, status, CASE WHEN report IS NOT NULL THEN 'Yes' ELSE 'No' END as has_report, CASE WHEN charts IS NOT NULL THEN 'Yes' ELSE 'No' END as has_charts, CASE WHEN retirement IS NOT NULL THEN 'Yes' ELSE 'No' END as has_retirement, updated_at FROM jobs WHERE id = '$jobId';" +$jobStatus = Invoke-PSQL -Command $sqlCheckStatus +if ($jobStatus) { + Write-Host $jobStatus +} else { + Write-Host "Could not retrieve job status" -ForegroundColor Yellow +} + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Test Complete!" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan + Write-Host "" + Write-Host "To view Cloud Run logs (most recent errors):" -ForegroundColor Yellow + $projectId = $env:GCP_PROJECT_ID + if ([string]::IsNullOrEmpty($projectId)) { + $projectId = (gcloud config get-value project 2>$null) + } + if (-not [string]::IsNullOrEmpty($projectId)) { + Write-Host " gcloud logging read 'resource.type=cloud_run_revision AND resource.labels.service_name=alex-planner' --limit 20 --project $projectId --format json" -ForegroundColor Gray + Write-Host "" + Write-Host "Or view in console:" -ForegroundColor Yellow + Write-Host " https://console.cloud.google.com/run/detail/$($PlannerUrl.Split('/')[2].Split('.')[0])/alex-planner/logs?project=$projectId" -ForegroundColor Gray + } else { + Write-Host " gcloud logging read 'resource.type=cloud_run_revision AND resource.labels.service_name=alex-planner' --limit 20 --format json" -ForegroundColor Gray + } + Write-Host "" + Write-Host "To check job details in database:" -ForegroundColor Yellow + $checkCmd = "psql -h $DbHost -p $DbPort -U $DbUser -d $DbName -c `"SELECT id, status, error_message, updated_at FROM jobs WHERE id = '$jobId';`"" + Write-Host " $checkCmd" -ForegroundColor Gray diff --git a/gcp-deployment/backend/pyproject.toml b/gcp-deployment/backend/pyproject.toml new file mode 100644 index 00000000..db25a81d --- /dev/null +++ b/gcp-deployment/backend/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "backend" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "boto3>=1.40.29", + "langfuse>=3.3.4", + "openai-agents>=0.3.0", + "pydantic-ai>=1.0.6", + "python-dotenv>=1.1.1", +] + +[tool.uv.workspace] +members = [ + "database", + "api", + "scheduler", +] + +[tool.uv.sources] +alex-database = { workspace = true } diff --git a/gcp-deployment/backend/reporter/.python-version b/gcp-deployment/backend/reporter/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/reporter/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/reporter/Dockerfile b/gcp-deployment/backend/reporter/Dockerfile new file mode 100644 index 00000000..9b99dc8f --- /dev/null +++ b/gcp-deployment/backend/reporter/Dockerfile @@ -0,0 +1,39 @@ +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Python package manager +RUN pip install uv + +# Copy database package (required dependency) +# Build context should be from backend/ directory +COPY database ./database + +# Copy shared modules +COPY common ./common +# Copy reporter-specific files +COPY reporter/pyproject.toml reporter/uv.lock ./ + +# Update pyproject.toml to use ./database instead of ../database +RUN sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && rm pyproject.toml.bak + +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +# First install database package with all its dependencies (including pg8000) +# This ensures transitive dependencies from the local path dependency are installed +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project + +# Copy reporter application code +COPY reporter/*.py ./ + +# Expose port +EXPOSE 8000 + +# Set environment variable for Cloud Run +ENV PORT=8000 + +# Run the application +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/gcp-deployment/backend/reporter/agent.py b/gcp-deployment/backend/reporter/agent.py new file mode 100644 index 00000000..12210e49 --- /dev/null +++ b/gcp-deployment/backend/reporter/agent.py @@ -0,0 +1,192 @@ +""" +Report Writer Agent - generates portfolio analysis narratives. +""" + +import os +import json +import logging +from typing import Dict, Any, List, Optional +from dataclasses import dataclass + +from agents import function_tool, RunContextWrapper +from common.llm import get_litellm_model + +logger = logging.getLogger() + + +@dataclass +class ReporterContext: + """Context for the Reporter agent""" + + job_id: str + portfolio_data: Dict[str, Any] + user_data: Dict[str, Any] + db: Optional[Any] = None # Database connection (optional for testing) + + +def calculate_portfolio_metrics(portfolio_data: Dict[str, Any]) -> Dict[str, Any]: + """Calculate basic portfolio metrics.""" + metrics = { + "total_value": 0, + "cash_balance": 0, + "num_accounts": len(portfolio_data.get("accounts", [])), + "num_positions": 0, + "unique_symbols": set(), + } + + for account in portfolio_data.get("accounts", []): + metrics["cash_balance"] += float(account.get("cash_balance", 0)) + positions = account.get("positions", []) + metrics["num_positions"] += len(positions) + + for position in positions: + symbol = position.get("symbol") + if symbol: + metrics["unique_symbols"].add(symbol) + + # Calculate value if we have price + instrument = position.get("instrument", {}) + if instrument.get("current_price"): + value = float(position.get("quantity", 0)) * float(instrument["current_price"]) + metrics["total_value"] += value + + metrics["total_value"] += metrics["cash_balance"] + metrics["unique_symbols"] = len(metrics["unique_symbols"]) + + return metrics + + +def format_portfolio_for_analysis(portfolio_data: Dict[str, Any], user_data: Dict[str, Any]) -> str: + """Format portfolio data for agent analysis.""" + metrics = calculate_portfolio_metrics(portfolio_data) + + lines = [ + f"Portfolio Overview:", + f"- {metrics['num_accounts']} accounts", + f"- {metrics['num_positions']} total positions", + f"- {metrics['unique_symbols']} unique holdings", + f"- ${metrics['cash_balance']:,.2f} in cash", + f"- ${metrics['total_value']:,.2f} total value" if metrics["total_value"] > 0 else "", + "", + "Account Details:", + ] + + for account in portfolio_data.get("accounts", []): + name = account.get("name", "Unknown") + cash = float(account.get("cash_balance", 0)) + lines.append(f"\n{name} (${cash:,.2f} cash):") + + for position in account.get("positions", []): + symbol = position.get("symbol") + quantity = float(position.get("quantity", 0)) + instrument = position.get("instrument", {}) + name = instrument.get("name", "") + + # Include allocation info if available + allocations = [] + if instrument.get("asset_class"): + allocations.append(f"Asset: {instrument['asset_class']}") + if instrument.get("regions"): + regions = ", ".join( + [f"{r['name']} {r['percentage']}%" for r in instrument["regions"][:2]] + ) + allocations.append(f"Regions: {regions}") + + alloc_str = f" ({', '.join(allocations)})" if allocations else "" + lines.append(f" - {symbol}: {quantity:,.2f} shares{alloc_str}") + + # Add user context + lines.extend( + [ + "", + "User Profile:", + f"- Years to retirement: {user_data.get('years_until_retirement', 'Not specified')}", + f"- Target retirement income: ${user_data.get('target_retirement_income', 0):,.0f}/year", + ] + ) + + return "\n".join(lines) + + +# update_report tool removed - report is now saved directly in lambda_handler + + +@function_tool +async def get_market_insights( + wrapper: RunContextWrapper[ReporterContext], symbols: List[str] +) -> str: + """ + Retrieve market insights from vector knowledge base (GCP implementation needed). + + Args: + wrapper: Context wrapper with job_id and database + symbols: List of symbols to get insights for + + Returns: + Relevant market context and insights + + Note: + This function requires GCP vector search implementation using Vertex AI Vector Search + or similar GCP service. Currently returns placeholder message. + """ + try: + # TODO: Implement GCP vector search using Vertex AI Vector Search or Vertex AI Matching Engine + # This would replace the AWS S3 Vectors implementation with: + # 1. Vertex AI embeddings API for generating embeddings + # 2. Vertex AI Vector Search or Matching Engine for similarity search + # 3. Cloud Storage for storing vector data + + symbols_str = ", ".join(symbols[:5]) if symbols else "general market" + logger.info(f"Reporter: Market insights requested for {symbols_str} (GCP vector search not yet implemented)") + + return ( + f"Market insights for {symbols_str} are currently unavailable. " + "Vector search functionality needs to be implemented using GCP services " + "(Vertex AI Vector Search or Matching Engine). Proceeding with standard analysis." + ) + + except Exception as e: + logger.warning(f"Reporter: Could not retrieve market insights: {e}") + return "Market insights unavailable - proceeding with standard analysis." + + +def create_agent(job_id: str, portfolio_data: Dict[str, Any], user_data: Dict[str, Any], db=None): + """Create the reporter agent with tools and context.""" + + model_override = os.getenv("REPORTER_MODEL") + model = get_litellm_model(model_override) + + # Create context + context = ReporterContext( + job_id=job_id, portfolio_data=portfolio_data, user_data=user_data, db=db + ) + + # Tools - only get_market_insights now, report saved in lambda_handler + tools = [get_market_insights] + + # Format portfolio for analysis + portfolio_summary = format_portfolio_for_analysis(portfolio_data, user_data) + + # Create task + task = f"""Analyze this investment portfolio and write a comprehensive report. + +{portfolio_summary} + +Your task: +1. First, get market insights for the top holdings using get_market_insights() +2. Analyze the portfolio's current state, strengths, and weaknesses +3. Generate a detailed, professional analysis report in markdown format + +The report should include: +- Executive Summary +- Portfolio Composition Analysis +- Risk Assessment +- Diversification Analysis +- Retirement Readiness (based on user goals) +- Recommendations +- Market Context (from insights) + +Provide your complete analysis as the final output in clear markdown format. +Make the report informative yet accessible to a retail investor.""" + + return model, tools, task, context diff --git a/gcp-deployment/backend/reporter/judge.py b/gcp-deployment/backend/reporter/judge.py new file mode 100644 index 00000000..f046b960 --- /dev/null +++ b/gcp-deployment/backend/reporter/judge.py @@ -0,0 +1,54 @@ +from agents import Agent, Runner +from pydantic import BaseModel, Field +import os +import logging +from common.llm import get_litellm_model + +logger = logging.getLogger() + + +class Evaluation(BaseModel): + feedback: str = Field( + description="Your feedback on the financial report and rationale for your score" + ) + score: float = Field( + description="Score from 0 to 100 where 0 represents a terrible quality financial report and 100 represents an outstanding financial report" + ) + + +async def evaluate(original_instructions, original_task, original_output) -> Evaluation: + model_override = os.getenv("REPORTER_JUDGE_MODEL") or os.getenv("REPORTER_MODEL") + model = get_litellm_model(model_override) + + instructions = """ +You are an Evaluation Agent that evaluates the quality of a financial report from a financial planning agent. +You will be provided with the instructions that were sent to the analyst, and its output, and you must evaluate the quality of the output. +""" + + # Create task + task = f""" +The financial planning agent was given the following instructions: + +{original_instructions} + +And it was assigned this task: + +{original_task} + +The financial planning agent's output was: + +{original_output} + +Evaluate this output and respond with your comments and score. +""" + + try: + logger.info("Judging financial report") + agent = Agent( + name="Judge Agent", instructions=instructions, model=model, output_type=Evaluation + ) + result = await Runner.run(agent, input=task, max_turns=5) + return result.final_output_as(Evaluation) + except Exception as e: + logger.error(f"Error evaluating financial report: {e}") + return Evaluation(feedback=f"Error evaluating financial report: {e}", score=80) diff --git a/gcp-deployment/backend/reporter/lambda_handler.py b/gcp-deployment/backend/reporter/lambda_handler.py new file mode 100644 index 00000000..2232d56f --- /dev/null +++ b/gcp-deployment/backend/reporter/lambda_handler.py @@ -0,0 +1,252 @@ +""" +Report Writer Agent Lambda Handler +""" + +import os +import json +import asyncio +import logging +from typing import Dict, Any +from datetime import datetime + +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError +from judge import evaluate + +GUARD_AGAINST_SCORE = 0.3 # Guard against score being too low + +try: + from dotenv import load_dotenv + + load_dotenv(override=True) +except ImportError: + pass + +# Import database package +from src import Database + +from templates import REPORTER_INSTRUCTIONS +from agent import create_agent, ReporterContext +from observability import observe + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info( + f"Reporter: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds..." + ), +) +async def run_reporter_agent( + job_id: str, + portfolio_data: Dict[str, Any], + user_data: Dict[str, Any], + db=None, + observability=None, +) -> Dict[str, Any]: + """Run the reporter agent to generate analysis.""" + + # Create agent with tools and context + model, tools, task, context = create_agent(job_id, portfolio_data, user_data, db) + + # Run agent with context + with trace("Reporter Agent"): + agent = Agent[ReporterContext]( # Specify the context type + name="Report Writer", instructions=REPORTER_INSTRUCTIONS, model=model, tools=tools + ) + + result = await Runner.run( + agent, + input=task, + context=context, # Pass the context + max_turns=10, + ) + + response = result.final_output + + if observability: + with observability.start_as_current_span(name="judge") as span: + evaluation = await evaluate(REPORTER_INSTRUCTIONS, task, response) + score = evaluation.score / 100 + comment = evaluation.feedback + span.score(name="Judge", value=score, data_type="NUMERIC", comment=comment) + observation = f"Score: {score} - Feedback: {comment}" + observability.create_event(name="Judge Event", status_message=observation) + if score < GUARD_AGAINST_SCORE: + logger.error(f"Reporter score is too low: {score}") + response = "I'm sorry, I'm not able to generate a report for you. Please try again later." + + # Save the report to database + report_payload = { + "content": response, + "generated_at": datetime.utcnow().isoformat(), + "agent": "reporter", + } + + success = db.jobs.update_report(job_id, report_payload) + + if not success: + logger.error(f"Failed to save report for job {job_id}") + + return { + "success": success, + "message": "Report generated and stored" + if success + else "Report generated but failed to save", + "final_output": result.final_output, + } + + +def lambda_handler(event, context): + """ + Lambda handler expecting job_id, portfolio_data, and user_data in event. + + Expected event: + { + "job_id": "uuid", + "portfolio_data": {...}, + "user_data": {...} + } + """ + # Wrap entire handler with observability context + with observe() as observability: + try: + logger.info(f"Reporter Lambda invoked with event: {json.dumps(event)[:500]}") + + # Parse event + if isinstance(event, str): + event = json.loads(event) + + job_id = event.get("job_id") + if not job_id: + return {"statusCode": 400, "body": json.dumps({"error": "job_id is required"})} + + # Initialize database + db = Database() + + portfolio_data = event.get("portfolio_data") + if not portfolio_data: + # Try to load from database + try: + job = db.jobs.find_by_id(job_id) + if job: + user_id = job["clerk_user_id"] + + if observability: + observability.create_event( + name="Reporter Started!", status_message="OK" + ) + user = db.users.find_by_clerk_id(user_id) + accounts = db.accounts.find_by_user(user_id) + + portfolio_data = {"user_id": user_id, "job_id": job_id, "accounts": []} + + for account in accounts: + positions = db.positions.find_by_account(account["id"]) + account_data = { + "id": account["id"], + "name": account["account_name"], + "type": account.get("account_type", "investment"), + "cash_balance": float(account.get("cash_balance", 0)), + "positions": [], + } + + for position in positions: + instrument = db.instruments.find_by_symbol(position["symbol"]) + if instrument: + account_data["positions"].append( + { + "symbol": position["symbol"], + "quantity": float(position["quantity"]), + "instrument": instrument, + } + ) + + portfolio_data["accounts"].append(account_data) + else: + return { + "statusCode": 404, + "body": json.dumps({"error": f"Job {job_id} not found"}), + } + except Exception as e: + logger.error(f"Could not load portfolio from database: {e}") + return { + "statusCode": 400, + "body": json.dumps({"error": "No portfolio data provided"}), + } + + user_data = event.get("user_data", {}) + if not user_data: + # Try to load from database + try: + job = db.jobs.find_by_id(job_id) + if job and job.get("clerk_user_id"): + status = f"Job ID: {job_id} Clerk User ID: {job['clerk_user_id']}" + if observability: + observability.create_event( + name="Reporter about to run", status_message=status + ) + user = db.users.find_by_clerk_id(job["clerk_user_id"]) + if user: + user_data = { + "years_until_retirement": user.get("years_until_retirement", 30), + "target_retirement_income": float( + user.get("target_retirement_income", 80000) + ), + } + else: + user_data = { + "years_until_retirement": 30, + "target_retirement_income": 80000, + } + except Exception as e: + logger.warning(f"Could not load user data: {e}. Using defaults.") + user_data = {"years_until_retirement": 30, "target_retirement_income": 80000} + + # Run the agent + result = asyncio.run( + run_reporter_agent(job_id, portfolio_data, user_data, db, observability) + ) + + logger.info(f"Reporter completed for job {job_id}") + + return {"statusCode": 200, "body": json.dumps(result)} + + except Exception as e: + logger.error(f"Error in reporter: {e}", exc_info=True) + return {"statusCode": 500, "body": json.dumps({"success": False, "error": str(e)})} + + +# For local testing +if __name__ == "__main__": + test_event = { + "job_id": "550e8400-e29b-41d4-a716-446655440002", + "portfolio_data": { + "accounts": [ + { + "name": "401(k)", + "cash_balance": 5000, + "positions": [ + { + "symbol": "SPY", + "quantity": 100, + "instrument": { + "name": "SPDR S&P 500 ETF", + "current_price": 450, + "asset_class": "equity", + }, + } + ], + } + ] + }, + "user_data": {"years_until_retirement": 25, "target_retirement_income": 75000}, + } + + result = lambda_handler(test_event, None) + print(json.dumps(result, indent=2)) diff --git a/gcp-deployment/backend/reporter/observability.py b/gcp-deployment/backend/reporter/observability.py new file mode 100644 index 00000000..5709320c --- /dev/null +++ b/gcp-deployment/backend/reporter/observability.py @@ -0,0 +1,112 @@ +""" +Observability module for LangFuse integration. +Provides a simple context manager for setting up and flushing traces. +""" + +import os +import logging +from contextlib import contextmanager + +# Use root logger for Lambda compatibility +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@contextmanager +def observe(): + """ + Context manager for observability with LangFuse. + + Sets up LangFuse observability if environment variables are configured, + and ensures traces are flushed on exit. + + Usage: + from observability import observe + + with observe(): + # Your code that uses OpenAI Agents SDK + result = await agent.run(...) + """ + logger.info("๐Ÿ” Observability: Checking configuration...") + + # Check if required environment variables exist + has_langfuse = bool(os.getenv("LANGFUSE_SECRET_KEY")) + has_openai = bool(os.getenv("OPENAI_API_KEY")) + + logger.info(f"๐Ÿ” Observability: LANGFUSE_SECRET_KEY exists: {has_langfuse}") + logger.info(f"๐Ÿ” Observability: OPENAI_API_KEY exists: {has_openai}") + + if not has_langfuse: + logger.info("๐Ÿ” Observability: LangFuse not configured, skipping setup") + yield None + return + + if not has_openai: + logger.warning("โš ๏ธ Observability: OPENAI_API_KEY not set, traces may not export") + + # Local variable for the client (no global needed) + langfuse_client = None + + # Try to set up LangFuse + try: + logger.info("๐Ÿ” Observability: Setting up LangFuse...") + + import logfire + from langfuse import get_client + + # Configure logfire to instrument OpenAI Agents SDK + logfire.configure( + service_name="alex_reporter_agent", + send_to_logfire=False, # Don't send to Logfire cloud + ) + logger.info("โœ… Observability: Logfire configured") + + # Instrument OpenAI Agents SDK + logfire.instrument_openai_agents() + logger.info("โœ… Observability: OpenAI Agents SDK instrumented") + + # Initialize LangFuse client + langfuse_client = get_client() + logger.info("โœ… Observability: LangFuse client initialized") + + # Optional: Check authentication (blocking call, use sparingly) + try: + auth_result = langfuse_client.auth_check() + logger.info( + f"โœ… Observability: LangFuse authentication check passed (result: {auth_result})" + ) + except Exception as auth_error: + logger.warning(f"โš ๏ธ Observability: Auth check failed but continuing: {auth_error}") + + logger.info("๐ŸŽฏ Observability: Setup complete - traces will be sent to LangFuse") + + except ImportError as e: + logger.error(f"โŒ Observability: Missing required package: {e}") + langfuse_client = None + except Exception as e: + logger.error(f"โŒ Observability: Setup failed: {e}") + langfuse_client = None + + try: + # Yield control back to the calling code + yield langfuse_client + finally: + # Flush traces on exit + if langfuse_client: + try: + logger.info("๐Ÿ” Observability: Flushing traces to LangFuse...") + langfuse_client.flush() + langfuse_client.shutdown() + + # Add a 10 second delay to ensure network requests complete + # This is a workaround for Lambda's immediate termination + import time + + logger.info("๐Ÿ” Observability: Waiting 10 seconds for flush to complete...") + time.sleep(10) + + logger.info("โœ… Observability: Traces flushed successfully") + except Exception as e: + logger.error(f"โŒ Observability: Failed to flush traces: {e}") + else: + logger.debug("๐Ÿ” Observability: No client to flush") diff --git a/gcp-deployment/backend/reporter/package_docker.py b/gcp-deployment/backend/reporter/package_docker.py new file mode 100644 index 00000000..4d823241 --- /dev/null +++ b/gcp-deployment/backend/reporter/package_docker.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Package the Reporter Lambda function using Docker for AWS compatibility. +""" + +import os +import sys +import shutil +import tempfile +import subprocess +import argparse +from pathlib import Path + + +def run_command(cmd, cwd=None): + """Run a command and capture output.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + + +def package_lambda(): + """Package the Lambda function with all dependencies.""" + + # Get the directory containing this script + reporter_dir = Path(__file__).parent.absolute() + backend_dir = reporter_dir.parent + + # Create a temporary directory for packaging + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + package_dir = temp_path / "package" + package_dir.mkdir() + + print("Creating Lambda package using Docker...") + + # Export exact requirements from uv.lock (excluding the editable database package) + print("Exporting requirements from uv.lock...") + requirements_result = run_command( + ["uv", "export", "--no-hashes", "--no-emit-project"], cwd=str(reporter_dir) + ) + + # Filter out packages that don't work in Lambda + filtered_requirements = [] + for line in requirements_result.splitlines(): + # Skip pyperclip (clipboard library not needed in Lambda) + if line.startswith("pyperclip"): + print(f"Excluding from Lambda: {line}") + continue + filtered_requirements.append(line) + + req_file = temp_path / "requirements.txt" + req_file.write_text("\n".join(filtered_requirements)) + + # Use Docker to install dependencies for Lambda's architecture + docker_cmd = [ + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "-v", + f"{temp_path}:/build", + "-v", + f"{backend_dir}/database:/database", + "--entrypoint", + "/bin/bash", + "public.ecr.aws/lambda/python:3.12", + "-c", + """cd /build && pip install --target ./package -r requirements.txt && pip install --target ./package --no-deps /database""", + ] + + run_command(docker_cmd) + + # Copy Lambda handler, agent, templates, and observability + shutil.copy(reporter_dir / "lambda_handler.py", package_dir) + shutil.copy(reporter_dir / "agent.py", package_dir) + shutil.copy(reporter_dir / "templates.py", package_dir) + shutil.copy(reporter_dir / "observability.py", package_dir) + shutil.copy(reporter_dir / "judge.py", package_dir) + + # Create the zip file + zip_path = reporter_dir / "reporter_lambda.zip" + + # Remove old zip if it exists + if zip_path.exists(): + zip_path.unlink() + + # Create new zip + print(f"Creating zip file: {zip_path}") + run_command(["zip", "-r", str(zip_path), "."], cwd=str(package_dir)) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"Package created: {zip_path} ({size_mb:.1f} MB)") + + return zip_path + + +def deploy_lambda(zip_path): + """Deploy the Lambda function to AWS.""" + import boto3 + + lambda_client = boto3.client("lambda") + function_name = "alex-reporter" + + print(f"Deploying to Lambda function: {function_name}") + + try: + # Try to update existing function + with open(zip_path, "rb") as f: + response = lambda_client.update_function_code( + FunctionName=function_name, ZipFile=f.read() + ) + print(f"Successfully updated Lambda function: {function_name}") + print(f"Function ARN: {response['FunctionArn']}") + except lambda_client.exceptions.ResourceNotFoundException: + print(f"Lambda function {function_name} not found. Please deploy via Terraform first.") + sys.exit(1) + except Exception as e: + print(f"Error deploying Lambda: {e}") + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser(description="Package Reporter Lambda for deployment") + parser.add_argument("--deploy", action="store_true", help="Deploy to AWS after packaging") + args = parser.parse_args() + + # Check if Docker is available + try: + run_command(["docker", "--version"]) + except FileNotFoundError: + print("Error: Docker is not installed or not in PATH") + sys.exit(1) + + # Package the Lambda + zip_path = package_lambda() + + # Deploy if requested + if args.deploy: + deploy_lambda(zip_path) + + +if __name__ == "__main__": + main() diff --git a/gcp-deployment/backend/reporter/pyproject.toml b/gcp-deployment/backend/reporter/pyproject.toml new file mode 100644 index 00000000..395d3c0b --- /dev/null +++ b/gcp-deployment/backend/reporter/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "reporter" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "boto3>=1.40.9", # Keep for backward compatibility + "fastapi>=0.116.1", + "uvicorn>=0.35.0", + "langfuse>=3.3.4", + "openai-agents[litellm]>=0.2.6", + "pydantic>=2.11.7", + "pydantic-ai>=1.0.6", + "python-dotenv>=1.1.1", + "tenacity>=9.1.2", +] + +[tool.uv.sources] +alex-database = { path = "../database", editable = true } diff --git a/gcp-deployment/backend/reporter/server.py b/gcp-deployment/backend/reporter/server.py new file mode 100644 index 00000000..fe2f80f6 --- /dev/null +++ b/gcp-deployment/backend/reporter/server.py @@ -0,0 +1,236 @@ +""" +Reporter Agent - Cloud Run HTTP Server +Generates portfolio analysis reports +""" + +import os +import sys +import json +import asyncio +import logging +from pathlib import Path +from typing import Dict, Any +from datetime import datetime, UTC + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from dotenv import load_dotenv +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +# Add parent directories to Python path for imports +backend_dir = Path(__file__).parent.parent +if str(backend_dir) not in sys.path: + sys.path.insert(0, str(backend_dir)) + +from judge import evaluate +from src import Database +from templates import REPORTER_INSTRUCTIONS +from agent import create_agent, ReporterContext +from observability import observe + +# Load .env file from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +GUARD_AGAINST_SCORE = 0.3 # Guard against score being too low + +# Initialize FastAPI app +app = FastAPI( + title="Alex Reporter Service", + description="Portfolio analysis report generation agent", + version="1.0.0" +) + +# Initialize database +db = Database() + + +@retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info( + f"Reporter: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds..." + ), +) +async def run_reporter_agent( + job_id: str, + portfolio_data: Dict[str, Any], + user_data: Dict[str, Any], + db=None, + observability=None, +) -> Dict[str, Any]: + """Run the reporter agent to generate analysis.""" + + # Create agent with tools and context + model, tools, task, context = create_agent(job_id, portfolio_data, user_data, db) + + # Run agent with context + with trace("Reporter Agent"): + agent = Agent[ReporterContext]( + name="Report Writer", instructions=REPORTER_INSTRUCTIONS, model=model, tools=tools + ) + + result = await Runner.run( + agent, + input=task, + context=context, + max_turns=10, + ) + + response = result.final_output + + if observability: + with observability.start_as_current_span(name="judge") as span: + evaluation = await evaluate(REPORTER_INSTRUCTIONS, task, response) + score = evaluation.score / 100 + comment = evaluation.feedback + span.score(name="Judge", value=score, data_type="NUMERIC", comment=comment) + observation = f"Score: {score} - Feedback: {comment}" + observability.create_event(name="Judge Event", status_message=observation) + if score < GUARD_AGAINST_SCORE: + logger.error(f"Reporter score is too low: {score}") + response = "I'm sorry, I'm not able to generate a report for you. Please try again later." + + # Save the report to database + report_payload = { + "content": response, + "generated_at": datetime.utcnow().isoformat(), + "agent": "reporter", + } + + success = db.jobs.update_report(job_id, report_payload) + + if not success: + logger.error(f"Failed to save report for job {job_id}") + + return { + "success": success, + "message": "Report generated and stored" + if success + else "Report generated but failed to save", + "final_output": result.final_output, + } + + +# Request/Response models +class JobRequest(BaseModel): + """Request to process a job""" + job_id: str + portfolio_data: Dict[str, Any] = None + user_data: Dict[str, Any] = None + + +@app.get("/") +async def root(): + """Health check endpoint""" + return { + "service": "Alex Reporter", + "status": "healthy", + "timestamp": datetime.now(UTC).isoformat(), + } + + +@app.get("/health") +async def health(): + """Health check endpoint (alternative)""" + return {"status": "healthy"} + + +@app.post("/") +async def handle_job(request: JobRequest): + """ + Handle job processing request. + + Request body: + { + "job_id": "uuid", + "portfolio_data": {...}, # Optional, will load from DB if not provided + "user_data": {...} # Optional, will load from DB if not provided + } + """ + try: + logger.info(f"Reporter: Received job request: {request.job_id}") + + # Load portfolio_data from database if not provided + portfolio_data = request.portfolio_data + if not portfolio_data: + job = db.jobs.find_by_id(request.job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job {request.job_id} not found") + + user_id = job["clerk_user_id"] + user = db.users.find_by_clerk_id(user_id) + accounts = db.accounts.find_by_user(user_id) + + portfolio_data = {"user_id": user_id, "job_id": request.job_id, "accounts": []} + + for account in accounts: + positions = db.positions.find_by_account(account["id"]) + account_data = { + "id": account["id"], + "name": account["account_name"], + "type": account.get("account_type", "investment"), + "cash_balance": float(account.get("cash_balance", 0)), + "positions": [], + } + + for position in positions: + instrument = db.instruments.find_by_symbol(position["symbol"]) + if instrument: + account_data["positions"].append( + { + "symbol": position["symbol"], + "quantity": float(position["quantity"]), + "instrument": instrument, + } + ) + + portfolio_data["accounts"].append(account_data) + + # Load user_data from database if not provided + user_data = request.user_data + if not user_data: + job = db.jobs.find_by_id(request.job_id) + if job and job.get("clerk_user_id"): + user = db.users.find_by_clerk_id(job["clerk_user_id"]) + if user: + user_data = { + "years_until_retirement": user.get("years_until_retirement", 30), + "target_retirement_income": float( + user.get("target_retirement_income", 80000) + ), + } + else: + user_data = { + "years_until_retirement": 30, + "target_retirement_income": 80000, + } + else: + user_data = {"years_until_retirement": 30, "target_retirement_income": 80000} + + # Run the agent + with observe() as observability: + result = await run_reporter_agent(request.job_id, portfolio_data, user_data, db, observability) + + logger.info(f"Reporter completed for job {request.job_id}") + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Reporter: Error processing job: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# For local testing +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) + diff --git a/gcp-deployment/backend/reporter/templates.py b/gcp-deployment/backend/reporter/templates.py new file mode 100644 index 00000000..62536203 --- /dev/null +++ b/gcp-deployment/backend/reporter/templates.py @@ -0,0 +1,34 @@ +""" +Prompt templates for the Report Writer Agent. +""" + +REPORTER_INSTRUCTIONS = """You are a Report Writer Agent specializing in portfolio analysis and financial narrative generation. + +Your primary task is to analyze the provided portfolio and generate a comprehensive markdown report. + +You have access to this tool: +1. get_market_insights - Retrieve relevant market context for specific symbols + +Your workflow: +1. First, analyze the portfolio data provided +2. Use get_market_insights to get relevant market context for the holdings +3. Generate a comprehensive analysis report in markdown format covering: + - Executive Summary (3-4 key points) + - Portfolio Composition Analysis + - Diversification Assessment + - Risk Profile Evaluation + - Retirement Readiness + - Specific Recommendations (5-7 actionable items) + - Conclusion + +4. Respond with your complete analysis in clear markdown format. + +Report Guidelines: +- Write in clear, professional language accessible to retail investors +- Use markdown formatting with headers, bullets, and emphasis +- Include specific percentages and numbers where relevant +- Focus on actionable insights, not just observations +- Prioritize recommendations by impact +- Keep sections concise but comprehensive + +""" diff --git a/gcp-deployment/backend/reporter/test_full.py b/gcp-deployment/backend/reporter/test_full.py new file mode 100644 index 00000000..ef25a34c --- /dev/null +++ b/gcp-deployment/backend/reporter/test_full.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Full test for Reporter agent via Lambda +""" + +import os +import json +import boto3 +import time +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database +from src.schemas import JobCreate + +def test_reporter_lambda(): + """Test the Reporter agent via Lambda invocation""" + + db = Database() + lambda_client = boto3.client('lambda') + + # Create test job + test_user_id = "test_user_001" + + job_create = JobCreate( + clerk_user_id=test_user_id, + job_type="portfolio_analysis", + request_payload={"analysis_type": "test", "test": True} + ) + job_id = db.jobs.create(job_create.model_dump()) + + print(f"Testing Reporter Lambda with job {job_id}") + print("=" * 60) + + # Invoke Lambda + try: + response = lambda_client.invoke( + FunctionName='alex-reporter', + InvocationType='RequestResponse', + Payload=json.dumps({'job_id': job_id}) + ) + + result = json.loads(response['Payload'].read()) + print(f"Lambda Response: {json.dumps(result, indent=2)}") + + # Check database for results + time.sleep(2) # Give it a moment + job = db.jobs.find_by_id(job_id) + + if job and job.get('report_payload'): + print("\nโœ… Report generated successfully!") + print(f"Report preview: {job['report_payload'][:500]}...") + else: + print("\nโŒ No report found in database") + + except Exception as e: + print(f"Error invoking Lambda: {e}") + + print("=" * 60) + +if __name__ == "__main__": + test_reporter_lambda() \ No newline at end of file diff --git a/gcp-deployment/backend/reporter/test_simple.py b/gcp-deployment/backend/reporter/test_simple.py new file mode 100644 index 00000000..2d4ab431 --- /dev/null +++ b/gcp-deployment/backend/reporter/test_simple.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +Simple test for Reporter agent +""" + +import asyncio +import json +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database +from src.schemas import JobCreate +from lambda_handler import lambda_handler + +def test_reporter(): + """Test the reporter agent with simple portfolio data""" + + # Create a real job in the database + db = Database() + job_create = JobCreate( + clerk_user_id="test_user_001", + job_type="portfolio_analysis", + request_payload={"test": True} + ) + job_id = db.jobs.create(job_create.model_dump()) + print(f"Created test job: {job_id}") + + test_event = { + "job_id": job_id, + "portfolio_data": { + "accounts": [ + { + "name": "401(k)", + "cash_balance": 5000, + "positions": [ + { + "symbol": "SPY", + "quantity": 100, + "instrument": { + "name": "SPDR S&P 500 ETF", + "current_price": 450, + "asset_class": "equity" + } + } + ] + } + ] + }, + "user_data": { + "years_until_retirement": 25, + "target_retirement_income": 75000 + } + } + + print("Testing Reporter Agent...") + print("=" * 60) + + result = lambda_handler(test_event, None) + + print(f"Status Code: {result['statusCode']}") + + if result['statusCode'] == 200: + body = json.loads(result['body']) + print(f"Success: {body.get('success', False)}") + print(f"Message: {body.get('message', 'N/A')}") + + # Check what was actually saved in the database + print("\n" + "=" * 60) + print("CHECKING DATABASE CONTENT") + print("=" * 60) + + job = db.jobs.find_by_id(job_id) + if job and job.get('report_payload'): + payload = job['report_payload'] + print(f"โœ… Report data found in database") + print(f"Payload keys: {list(payload.keys())}") + + if 'content' in payload: + content = payload['content'] + print(f"\nContent type: {type(content).__name__}") + + if isinstance(content, str): + print(f"Report length: {len(content)} characters") + + # Check if it contains reasoning artifacts + reasoning_indicators = [ + "I need to", + "I will", + "Let me", + "First,", + "I should", + "I'll", + "Now I", + "Next,", + ] + + contains_reasoning = any(indicator.lower() in content.lower() for indicator in reasoning_indicators) + + if contains_reasoning: + print("โš ๏ธ WARNING: Report may contain reasoning/thinking text") + else: + print("โœ… Report appears to be final output only (no reasoning detected)") + + # Show first 500 characters and last 200 characters + print(f"\nFirst 500 characters:") + print("-" * 40) + print(content[:500]) + print("-" * 40) + + if len(content) > 700: + print(f"\nLast 200 characters:") + print("-" * 40) + print(content[-200:]) + print("-" * 40) + else: + print(f"โš ๏ธ Content is not a string: {type(content)}") + print(f"Content: {str(content)[:200]}") + + print(f"\nGenerated at: {payload.get('generated_at', 'N/A')}") + print(f"Agent: {payload.get('agent', 'N/A')}") + else: + print("โŒ No report data found in database") + else: + print(f"Error: {result['body']}") + + # Clean up - delete the test job + db.jobs.delete(job_id) + print(f"\nDeleted test job: {job_id}") + + print("=" * 60) + +if __name__ == "__main__": + test_reporter() \ No newline at end of file diff --git a/gcp-deployment/backend/researcher/.python-version b/gcp-deployment/backend/researcher/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/researcher/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/researcher/Dockerfile b/gcp-deployment/backend/researcher/Dockerfile new file mode 100644 index 00000000..0e9a1505 --- /dev/null +++ b/gcp-deployment/backend/researcher/Dockerfile @@ -0,0 +1,36 @@ +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Node.js +RUN apt-get update && apt-get install -y \ + curl \ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Install Playwright dependencies (system-level dependencies for Chromium) +RUN npx -y playwright install --with-deps chromium + +# Install Python package manager +RUN pip install uv + +# Copy project files +COPY pyproject.toml uv.lock ./ + + +# Copy shared modules +COPY common ./common + +# Install Python dependencies during build +# uv will automatically install packages for the container's architecture (linux/amd64) +RUN uv sync --frozen --no-install-project + +# Copy application code +COPY *.py ./ + +# Expose port +EXPOSE 8000 + +# Run the application (dependencies already installed) +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/gcp-deployment/backend/researcher/context.py b/gcp-deployment/backend/researcher/context.py new file mode 100644 index 00000000..e52b2468 --- /dev/null +++ b/gcp-deployment/backend/researcher/context.py @@ -0,0 +1,43 @@ +""" +Agent instructions and prompts for the Alex Researcher +""" +from datetime import datetime + + +def get_agent_instructions(): + """Get agent instructions with current date.""" + today = datetime.now().strftime("%B %d, %Y") + + return f"""You are Alex, a concise investment researcher. Today is {today}. + +CRITICAL: Work quickly and efficiently. You have limited time. + +Your THREE steps (BE CONCISE): + +1. WEB RESEARCH (1-2 pages MAX): + - Navigate to ONE main source (Yahoo Finance or MarketWatch) + - Use browser_snapshot to read content + - If needed, visit ONE more page for verification + - DO NOT browse extensively - 2 pages maximum + +2. BRIEF ANALYSIS (Keep it short): + - Key facts and numbers only + - 3-5 bullet points maximum + - One clear recommendation + - Be extremely concise + +3. SAVE TO DATABASE: + - Use ingest_financial_document immediately + - Topic: "[Asset] Analysis {datetime.now().strftime('%b %d')}" + - Save your brief analysis + +SPEED IS CRITICAL: +- Maximum 2 web pages +- Brief, bullet-point analysis +- No lengthy explanations +- Work as quickly as possible +""" + +DEFAULT_RESEARCH_PROMPT = """Please research a current, interesting investment topic from today's financial news. +Pick something trending or significant happening in the markets right now. +Follow all three steps: browse, analyze, and store your findings.""" \ No newline at end of file diff --git a/gcp-deployment/backend/researcher/deploy.py b/gcp-deployment/backend/researcher/deploy.py new file mode 100644 index 00000000..f997234a --- /dev/null +++ b/gcp-deployment/backend/researcher/deploy.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +""" +Deploy researcher service to AWS App Runner +Cross-platform deployment script for Mac/Windows/Linux +""" + +import subprocess +import sys +import os +import json +from pathlib import Path +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv(override=True) + + +def run_command(cmd, capture_output=False, shell=False): + """Run a command and handle errors.""" + try: + result = subprocess.run( + cmd, shell=shell, capture_output=capture_output, text=True, check=True + ) + if capture_output: + return result.stdout.strip() + return None + except subprocess.CalledProcessError as e: + print(f"Error running command: {e}") + if e.stderr: + print(f"Error details: {e.stderr}") + sys.exit(1) + + +def main(): + print("Alex Researcher Service - Docker Deployment") + print("===========================================") + + # Get AWS account ID + print("\nGetting AWS account details...") + account_id = run_command( + ["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"], + capture_output=True, + ) + + region = os.environ.get("DEFAULT_AWS_REGION") + if not region: + print("Error: DEFAULT_AWS_REGION not found in your .env file.") + sys.exit(1) + + ecr_repository = "alex-researcher" + + print(f"AWS Account: {account_id}") + print(f"Region: {region}") + + # Get ECR repository URL from Terraform + print("\nGetting ECR repository URL...") + terraform_dir = Path(__file__).parent.parent.parent / "terraform" / "4_researcher" + original_dir = os.getcwd() + + try: + os.chdir(terraform_dir) + ecr_url = run_command( + ["terraform", "output", "-raw", "ecr_repository_url"], capture_output=True + ) + finally: + os.chdir(original_dir) + + if not ecr_url: + print("Error: ECR repository not found. Run 'terraform apply' first.") + sys.exit(1) + + print(f"ECR Repository: {ecr_url}") + + # Login to ECR + print("\nLogging in to ECR...") + password = run_command( + ["aws", "ecr", "get-login-password", "--region", region], capture_output=True + ) + + login_cmd = ["docker", "login", "--username", "AWS", "--password-stdin", ecr_url] + login_process = subprocess.Popen( + login_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + stdout, stderr = login_process.communicate(input=password) + + if login_process.returncode != 0: + print(f"Error logging into ECR: {stderr}") + sys.exit(1) + + print("Login successful!") + + # Generate a unique tag using timestamp + import time + + timestamp = int(time.time()) + image_tag = f"deploy-{timestamp}" + + # Build Docker image + print(f"\nBuilding Docker image for linux/amd64 with tag: {image_tag}") + print("(This ensures compatibility with AWS App Runner)") + run_command( + [ + "docker", + "build", + "--platform", + "linux/amd64", + "-t", + f"{ecr_repository}:{image_tag}", + # Removed --no-cache to use Docker layer caching for faster builds + ".", + ] + ) + + # Tag for ECR with both unique tag and latest + print("\nTagging image for ECR...") + run_command(["docker", "tag", f"{ecr_repository}:{image_tag}", f"{ecr_url}:{image_tag}"]) + run_command(["docker", "tag", f"{ecr_repository}:{image_tag}", f"{ecr_url}:latest"]) + + # Push to ECR + print("\nPushing image to ECR...") + run_command(["docker", "push", f"{ecr_url}:{image_tag}"]) + run_command(["docker", "push", f"{ecr_url}:latest"]) + + print("\nโœ… Docker image pushed successfully!") + print( + "\nNext step: Run 'terraform apply' in terraform/4_researcher to create the App Runner service." + ) + + # Get App Runner service ARN + print("\nGetting App Runner service details...") + try: + services = run_command( + [ + "aws", + "apprunner", + "list-services", + "--region", + region, + "--query", + "ServiceSummaryList[?ServiceName=='alex-researcher'].ServiceArn", + "--output", + "json", + ], + capture_output=True, + ) + + if services: + service_arns = json.loads(services) + if service_arns: + service_arn = service_arns[0] + print(f"Found service: {service_arn}") + + # Get the current service configuration to preserve the access role + print("\nGetting current service configuration...") + service_details = run_command( + [ + "aws", + "apprunner", + "describe-service", + "--service-arn", + service_arn, + "--region", + region, + "--query", + "Service.SourceConfiguration.AuthenticationConfiguration.AccessRoleArn", + "--output", + "text", + ], + capture_output=True, + ) + + # Update the service to use the new image with unique tag + print(f"\nUpdating service to use new image: {ecr_url}:{image_tag}") + run_command( + [ + "aws", + "apprunner", + "update-service", + "--service-arn", + service_arn, + "--region", + region, + "--source-configuration", + json.dumps( + { + "ImageRepository": { + "ImageIdentifier": f"{ecr_url}:{image_tag}", + "ImageConfiguration": { + "Port": "8000", + "RuntimeEnvironmentVariables": { + "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY", ""), + "ALEX_API_KEY": os.environ.get("ALEX_API_KEY", ""), + "ALEX_API_ENDPOINT": os.environ.get( + "ALEX_API_ENDPOINT", "" + ), + }, + }, + "ImageRepositoryType": "ECR", + }, + "AuthenticationConfiguration": {"AccessRoleArn": service_details}, + "AutoDeploymentsEnabled": False, + } + ), + ], + capture_output=True, + ) + print("โœ… Service updated with new image!") + + # Wait for deployment to complete + print("\nWaiting for deployment to complete (this may take 5-10 minutes)...") + import time + + max_attempts = 120 # 10 minutes with 5-second intervals + attempts = 0 + + while attempts < max_attempts: + status = run_command( + [ + "aws", + "apprunner", + "describe-service", + "--service-arn", + service_arn, + "--region", + region, + "--query", + "Service.Status", + "--output", + "text", + ], + capture_output=True, + ) + + # Strip any whitespace that might be causing comparison issues + status = status.strip() + + if status == "RUNNING": + print("\nโœ… Deployment complete! Service is running.") + + # Get and display the service URL + service_url = run_command( + [ + "aws", + "apprunner", + "describe-service", + "--service-arn", + service_arn, + "--region", + region, + "--query", + "Service.ServiceUrl", + "--output", + "text", + ], + capture_output=True, + ) + + print(f"\n๐Ÿš€ Your service is available at:") + print(f" https://{service_url}") + print(f"\nTest it with:") + print(f" curl https://{service_url}/health") + break + elif status == "OPERATION_IN_PROGRESS": + # Check operation status for more details + operation_status = run_command( + [ + "aws", + "apprunner", + "list-operations", + "--service-arn", + service_arn, + "--region", + region, + "--query", + "OperationSummaryList[0].Status", + "--output", + "text", + ], + capture_output=True, + ).strip() + + if operation_status == "SUCCEEDED": + # Operation completed but service status might not be updated yet + print("\nโณ Operation succeeded, checking service status...") + time.sleep(2) + continue + elif operation_status == "FAILED": + print(f"\nโŒ Deployment failed!") + print("Check the AWS Console for error details.") + break + else: + print(".", end="", flush=True) + # Show progress every 30 seconds + if attempts > 0 and attempts % 6 == 0: + elapsed_minutes = (attempts * 5) / 60 + print( + f" ({elapsed_minutes:.1f} minutes elapsed)", end="", flush=True + ) + time.sleep(5) + attempts += 1 + else: + print(f"\nโš ๏ธ Unexpected status: {status}") + print("Check the AWS Console for more details.") + break + else: + print("\nโš ๏ธ Deployment is taking longer than expected.") + print("Check the status in the AWS Console.") + else: + print( + "\nApp Runner service not found. You may need to run 'terraform apply' first." + ) + print("\nTo manually deploy:") + print(" 1. Go to AWS Console > App Runner") + print(" 2. Select 'alex-researcher' service") + print(" 3. Click 'Deploy' to pull the latest image") + except Exception as e: + print(f"\nCouldn't automatically start deployment: {e}") + print("\nTo manually deploy:") + print(" 1. Go to AWS Console > App Runner") + print(" 2. Select 'alex-researcher' service") + print(" 3. Click 'Deploy' to pull the latest image") + + +if __name__ == "__main__": + main() diff --git a/gcp-deployment/backend/researcher/mcp_servers.py b/gcp-deployment/backend/researcher/mcp_servers.py new file mode 100644 index 00000000..f056f84a --- /dev/null +++ b/gcp-deployment/backend/researcher/mcp_servers.py @@ -0,0 +1,47 @@ +""" +MCP server configurations for the Alex Researcher +""" +from agents.mcp import MCPServerStdio + + +def create_playwright_mcp_server(timeout_seconds=60): + """Create a Playwright MCP server instance for web browsing. + + Args: + timeout_seconds: Client session timeout in seconds (default: 60) + + Returns: + MCPServerStdio instance configured for Playwright + """ + # Base arguments + args = [ + "@playwright/mcp@latest", + "--headless", + "--isolated", + "--no-sandbox", + "--ignore-https-errors", + "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" + ] + + # Add executable path in Docker environment + import os + import glob + if os.path.exists("/.dockerenv") or os.environ.get("AWS_EXECUTION_ENV"): + # Find the installed Chrome executable dynamically + chrome_paths = glob.glob("/root/.cache/ms-playwright/chromium-*/chrome-linux/chrome") + if chrome_paths: + # Use the first (should be only one) Chrome installation found + chrome_path = chrome_paths[0] + print(f"DEBUG: Found Chrome at: {chrome_path}") + args.extend(["--executable-path", chrome_path]) + else: + # Fallback to a known path if glob doesn't find it + print("DEBUG: Chrome not found via glob, using fallback path") + args.extend(["--executable-path", "/root/.cache/ms-playwright/chromium-1187/chrome-linux/chrome"]) + + params = { + "command": "npx", + "args": args + } + + return MCPServerStdio(params=params, client_session_timeout_seconds=timeout_seconds) \ No newline at end of file diff --git a/gcp-deployment/backend/researcher/pyproject.toml b/gcp-deployment/backend/researcher/pyproject.toml new file mode 100644 index 00000000..066adf63 --- /dev/null +++ b/gcp-deployment/backend/researcher/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "researcher" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "boto3>=1.40.6", + "fastapi>=0.116.1", + "httpx>=0.28.1", + "openai-agents[litellm]>=0.2.4", + "playwright>=1.54.0", + "pydantic>=2.11.7", + "python-dotenv>=1.1.1", + "requests>=2.32.4", + "tenacity>=9.1.2", + "uvicorn>=0.35.0", +] diff --git a/gcp-deployment/backend/researcher/server.py b/gcp-deployment/backend/researcher/server.py new file mode 100644 index 00000000..d3427f95 --- /dev/null +++ b/gcp-deployment/backend/researcher/server.py @@ -0,0 +1,145 @@ +""" +Alex Researcher Service - Investment Advice Agent +""" + +import os +import logging +from datetime import datetime, UTC +from typing import Optional + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from dotenv import load_dotenv +from agents import Agent, Runner, trace +from common.llm import get_litellm_model + +# Suppress LiteLLM warnings about optional dependencies +logging.getLogger("LiteLLM").setLevel(logging.CRITICAL) + +# Import from our modules +from context import get_agent_instructions, DEFAULT_RESEARCH_PROMPT +from mcp_servers import create_playwright_mcp_server +from tools import ingest_financial_document + +# Load environment +load_dotenv(override=True) + +app = FastAPI(title="Alex Researcher Service") + + +# Request model +class ResearchRequest(BaseModel): + topic: Optional[str] = None # Optional - if not provided, agent picks a topic + + +async def run_research_agent(topic: str = None) -> str: + """Run the research agent to generate investment advice.""" + + # Prepare the user query + if topic: + query = f"Research this investment topic: {topic}" + else: + query = DEFAULT_RESEARCH_PROMPT + + model_override = os.getenv("RESEARCHER_MODEL") + model = get_litellm_model(model_override) + + # Create and run the agent with MCP server + with trace("Researcher"): + async with create_playwright_mcp_server(timeout_seconds=60) as playwright_mcp: + agent = Agent( + name="Alex Investment Researcher", + instructions=get_agent_instructions(), + model=model, + tools=[ingest_financial_document], + mcp_servers=[playwright_mcp], + ) + + result = await Runner.run(agent, input=query, max_turns=15) + + return result.final_output + + +@app.get("/") +async def root(): + """Health check endpoint.""" + return { + "service": "Alex Researcher", + "status": "healthy", + "timestamp": datetime.now(UTC).isoformat(), + } + + +@app.post("/research") +async def research(request: ResearchRequest) -> str: + """ + Generate investment research and advice. + + The agent will: + 1. Browse current financial websites for data + 2. Analyze the information found + 3. Store the analysis in the knowledge base + + If no topic is provided, the agent will pick a trending topic. + """ + try: + response = await run_research_agent(request.topic) + return response + except Exception as e: + print(f"Error in research endpoint: {e}") + import traceback + + traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +@app.get("/research/auto") +async def research_auto(): + """ + Automated research endpoint for scheduled runs. + Picks a trending topic automatically and generates research. + Used by EventBridge Scheduler for periodic research updates. + """ + try: + # Always use agent's choice for automated runs + response = await run_research_agent(topic=None) + return { + "status": "success", + "timestamp": datetime.now(UTC).isoformat(), + "message": "Automated research completed", + "preview": response[:200] + "..." if len(response) > 200 else response, + } + except Exception as e: + print(f"Error in automated research: {e}") + return {"status": "error", "timestamp": datetime.now(UTC).isoformat(), "error": str(e)} + + +@app.get("/health") +async def health(): + """Detailed health check.""" + # Debug container detection + container_indicators = { + "dockerenv": os.path.exists("/.dockerenv"), + "containerenv": os.path.exists("/run/.containerenv"), + "aws_execution_env": os.environ.get("AWS_EXECUTION_ENV", ""), + "ecs_container_metadata": os.environ.get("ECS_CONTAINER_METADATA_URI", ""), + "kubernetes_service": os.environ.get("KUBERNETES_SERVICE_HOST", ""), + } + + return { + "service": "Alex Researcher", + "status": "healthy", + "alex_api_configured": bool(os.getenv("ALEX_API_ENDPOINT") and os.getenv("ALEX_API_KEY")), + "timestamp": datetime.now(UTC).isoformat(), + "debug_container": container_indicators, + "llm_provider": os.getenv("LLM_PROVIDER", "vertex_ai"), + "llm_model": model_override + or os.getenv("VERTEX_AI_MODEL") + or os.getenv("OPENAI_MODEL"), + } + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/gcp-deployment/backend/researcher/test_local.py b/gcp-deployment/backend/researcher/test_local.py new file mode 100644 index 00000000..f0252706 --- /dev/null +++ b/gcp-deployment/backend/researcher/test_local.py @@ -0,0 +1,50 @@ +# \!/usr/bin/env python3 +""" +Test the researcher locally before deployment +""" + +import asyncio +from context import get_agent_instructions, DEFAULT_RESEARCH_PROMPT +from mcp_servers import create_playwright_mcp_server +from tools import ingest_financial_document +from agents import Agent, Runner +from dotenv import load_dotenv + +load_dotenv(override=True) + + +async def test_local(): + """Test the researcher agent locally.""" + print("Testing researcher agent locally...") + print("=" * 60) + + # Test with no topic (agent picks) + query = DEFAULT_RESEARCH_PROMPT + + try: + async with create_playwright_mcp_server() as playwright_mcp: + agent = Agent( + name="Alex Investment Researcher", + instructions=get_agent_instructions(), + model="gpt-4.1-mini", + tools=[ingest_financial_document], + mcp_servers=[playwright_mcp], + ) + + result = await Runner.run(agent, input=query) + + print("\nRESULT:") + print("=" * 60) + print(result.final_output) + print("=" * 60) + print("\nโœ… Test completed successfully!") + + except Exception as e: + print(f"\nโŒ Error: {e}") + import traceback + + traceback.print_exc() + + +if __name__ == "__main__": + asyncio.run(test_local()) diff --git a/gcp-deployment/backend/researcher/test_research.py b/gcp-deployment/backend/researcher/test_research.py new file mode 100644 index 00000000..20c9a6d5 --- /dev/null +++ b/gcp-deployment/backend/researcher/test_research.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +Test the researcher service by generating investment research. +Cross-platform script for Mac/Windows/Linux. +""" + +import subprocess +import sys +import json +import requests +import argparse + + +def get_service_url(): + """Get the App Runner service URL from AWS.""" + try: + # Get service ARN first + result = subprocess.run([ + "aws", "apprunner", "list-services", + "--query", "ServiceSummaryList[?ServiceName=='alex-researcher'].ServiceArn", + "--output", "json" + ], capture_output=True, text=True, check=True) + + service_arns = json.loads(result.stdout) + if not service_arns: + print("โŒ App Runner service 'alex-researcher' not found.") + print(" Have you deployed it yet? Run: python deploy.py") + sys.exit(1) + + service_arn = service_arns[0] + + # Get service URL + result = subprocess.run([ + "aws", "apprunner", "describe-service", + "--service-arn", service_arn, + "--query", "Service.ServiceUrl", + "--output", "text" + ], capture_output=True, text=True, check=True) + + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"โŒ Error getting service URL: {e}") + print(" Make sure AWS CLI is configured and you have the right permissions.") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"โŒ Error parsing AWS response: {e}") + sys.exit(1) + + +def test_research(topic=None): + """Test the researcher service with a topic.""" + # If no topic, let the agent pick one + display_topic = topic if topic else "Agent's choice (trending topic)" + + # Get service URL + print("Getting App Runner service URL...") + service_url = get_service_url() + + if not service_url: + print("โŒ Could not get service URL") + sys.exit(1) + + print(f"โœ… Found service at: https://{service_url}") + + # Test health endpoint first + print("\nChecking service health...") + try: + health_url = f"https://{service_url}/health" + response = requests.get(health_url, timeout=10) + response.raise_for_status() + print("โœ… Service is healthy") + except requests.exceptions.RequestException as e: + print(f"โŒ Health check failed: {e}") + print(" The service may still be starting. Try again in a minute.") + sys.exit(1) + + # Call research endpoint + print(f"\n๐Ÿ”ฌ Generating research for: {display_topic}") + print(" This will take 20-30 seconds as the agent researches and analyzes...") + + try: + research_url = f"https://{service_url}/research" + # Only include topic in payload if it's provided + payload = {"topic": topic} if topic else {} + response = requests.post( + research_url, + json=payload, + timeout=180 # Give it 3 minutes for research + ) + response.raise_for_status() + + # Parse and display the result + result = response.json() + + print("\nโœ… Research generated successfully!") + print("\n" + "="*60) + print("RESEARCH RESULT:") + print("="*60) + print(result) + print("="*60) + + print("\nโœ… The research has been automatically stored in your knowledge base.") + print(" To verify, run:") + print(" cd ../ingest") + print(" uv run test_search_s3vectors.py") + + except requests.exceptions.Timeout: + print("โŒ Request timed out. The service might be under heavy load.") + print(" Try again in a moment.") + sys.exit(1) + except requests.exceptions.RequestException as e: + print(f"โŒ Error calling research endpoint: {e}") + if hasattr(e, 'response') and e.response is not None: + try: + error_detail = e.response.json() + print(f" Error details: {error_detail}") + except (json.JSONDecodeError, AttributeError): + print(f" Response: {e.response.text}") + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="Test the Alex Researcher service", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Let agent pick a trending topic + uv run test_research.py + + # Research specific topic + uv run test_research.py "Tesla competitive advantages" + + # Research another topic + uv run test_research.py "Microsoft cloud revenue growth" + """ + ) + parser.add_argument( + "topic", + nargs="?", + default=None, + help="Investment topic to research (optional - agent will pick trending topic if not provided)" + ) + + args = parser.parse_args() + test_research(args.topic) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/researcher/tools.py b/gcp-deployment/backend/researcher/tools.py new file mode 100644 index 00000000..aa78e9c0 --- /dev/null +++ b/gcp-deployment/backend/researcher/tools.py @@ -0,0 +1,75 @@ +""" +Tools for the Alex Researcher agent +""" +import os +from typing import Dict, Any +from datetime import datetime, UTC +import httpx +from agents import function_tool +from tenacity import retry, stop_after_attempt, wait_exponential + +# Configuration from environment +ALEX_API_ENDPOINT = os.getenv("ALEX_API_ENDPOINT") +ALEX_API_KEY = os.getenv("ALEX_API_KEY") + + +def _ingest(document: Dict[str, Any]) -> Dict[str, Any]: + """Internal function to make the actual API call.""" + with httpx.Client() as client: + response = client.post( + ALEX_API_ENDPOINT, + json=document, + headers={"x-api-key": ALEX_API_KEY}, + timeout=30.0 + ) + response.raise_for_status() + return response.json() + + +@retry( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=1, max=10) +) +def ingest_with_retries(document: Dict[str, Any]) -> Dict[str, Any]: + """Ingest with retry logic for SageMaker cold starts.""" + return _ingest(document) + + +@function_tool +def ingest_financial_document(topic: str, analysis: str) -> Dict[str, Any]: + """ + Ingest a financial document into the Alex knowledge base. + + Args: + topic: The topic or subject of the analysis (e.g., "AAPL Stock Analysis", "Retirement Planning Guide") + analysis: Detailed analysis or advice with specific data and insights + + Returns: + Dictionary with success status and document ID + """ + if not ALEX_API_ENDPOINT or not ALEX_API_KEY: + return { + "success": False, + "error": "Alex API not configured. Running in local mode." + } + + document = { + "text": analysis, + "metadata": { + "topic": topic, + "timestamp": datetime.now(UTC).isoformat() + } + } + + try: + result = ingest_with_retries(document) + return { + "success": True, + "document_id": result.get("document_id"), # Changed from documentId + "message": f"Successfully ingested analysis for {topic}" + } + except Exception as e: + return { + "success": False, + "error": str(e) + } \ No newline at end of file diff --git a/gcp-deployment/backend/retirement/.python-version b/gcp-deployment/backend/retirement/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/retirement/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/retirement/Dockerfile b/gcp-deployment/backend/retirement/Dockerfile new file mode 100644 index 00000000..f8bc7e3e --- /dev/null +++ b/gcp-deployment/backend/retirement/Dockerfile @@ -0,0 +1,40 @@ +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Python package manager +RUN pip install uv + +# Copy database package (required dependency) +# Build context should be from backend/ directory +COPY database ./database + +# Copy shared modules +COPY common ./common + +# Copy retirement-specific files +COPY retirement/pyproject.toml retirement/uv.lock ./ + +# Update pyproject.toml to use ./database instead of ../database +RUN sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && rm pyproject.toml.bak + +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +# First install database package with all its dependencies (including pg8000) +# This ensures transitive dependencies from the local path dependency are installed +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project + +# Copy retirement application code +COPY retirement/*.py ./ + +# Expose port +EXPOSE 8000 + +# Set environment variable for Cloud Run +ENV PORT=8000 + +# Run the application +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/gcp-deployment/backend/retirement/agent.py b/gcp-deployment/backend/retirement/agent.py new file mode 100644 index 00000000..6f738629 --- /dev/null +++ b/gcp-deployment/backend/retirement/agent.py @@ -0,0 +1,319 @@ +""" +Retirement Specialist Agent - provides retirement planning analysis and projections. +""" + +import os +import json +import logging +import random +from typing import Dict, Any +from datetime import datetime + +# No tools needed - simplified agent +from common.llm import get_litellm_model + +logger = logging.getLogger() + +# Context removed - no longer needed without tools + + +def calculate_portfolio_value(portfolio_data: Dict[str, Any]) -> float: + """Calculate current portfolio value.""" + total_value = 0.0 + + for account in portfolio_data.get("accounts", []): + cash = float(account.get("cash_balance", 0)) + total_value += cash + + for position in account.get("positions", []): + quantity = float(position.get("quantity", 0)) + instrument = position.get("instrument", {}) + price = float(instrument.get("current_price", 100)) + total_value += quantity * price + + return total_value + + +def calculate_asset_allocation(portfolio_data: Dict[str, Any]) -> Dict[str, float]: + """Calculate asset allocation percentages.""" + total_equity = 0.0 + total_bonds = 0.0 + total_real_estate = 0.0 + total_commodities = 0.0 + total_cash = 0.0 + total_value = 0.0 + + for account in portfolio_data.get("accounts", []): + cash = float(account.get("cash_balance", 0)) + total_cash += cash + total_value += cash + + for position in account.get("positions", []): + quantity = float(position.get("quantity", 0)) + instrument = position.get("instrument", {}) + price = float(instrument.get("current_price", 100)) + value = quantity * price + total_value += value + + # Get asset class allocation + asset_allocation = instrument.get("allocation_asset_class", {}) + if asset_allocation: + total_equity += value * asset_allocation.get("equity", 0) / 100 + total_bonds += value * asset_allocation.get("fixed_income", 0) / 100 + total_real_estate += value * asset_allocation.get("real_estate", 0) / 100 + total_commodities += value * asset_allocation.get("commodities", 0) / 100 + + if total_value == 0: + return {"equity": 0, "bonds": 0, "real_estate": 0, "commodities": 0, "cash": 0} + + return { + "equity": total_equity / total_value, + "bonds": total_bonds / total_value, + "real_estate": total_real_estate / total_value, + "commodities": total_commodities / total_value, + "cash": total_cash / total_value, + } + + +def run_monte_carlo_simulation( + current_value: float, + years_until_retirement: int, + target_annual_income: float, + asset_allocation: Dict[str, float], + num_simulations: int = 500, +) -> Dict[str, Any]: + """Run Monte Carlo simulation for retirement planning.""" + + # Historical return parameters (annualized) + equity_return_mean = 0.07 + equity_return_std = 0.18 + bond_return_mean = 0.04 + bond_return_std = 0.05 + real_estate_return_mean = 0.06 + real_estate_return_std = 0.12 + + successful_scenarios = 0 + final_values = [] + years_lasted = [] + + for _ in range(num_simulations): + portfolio_value = current_value + + # Accumulation phase + for _ in range(years_until_retirement): + equity_return = random.gauss(equity_return_mean, equity_return_std) + bond_return = random.gauss(bond_return_mean, bond_return_std) + real_estate_return = random.gauss(real_estate_return_mean, real_estate_return_std) + + portfolio_return = ( + asset_allocation["equity"] * equity_return + + asset_allocation["bonds"] * bond_return + + asset_allocation["real_estate"] * real_estate_return + + asset_allocation["cash"] * 0.02 + ) + + portfolio_value = portfolio_value * (1 + portfolio_return) + portfolio_value += 10000 # Annual contribution + + # Retirement phase + retirement_years = 30 + annual_withdrawal = target_annual_income + years_income_lasted = 0 + + for year in range(retirement_years): + if portfolio_value <= 0: + break + + # Inflation adjustment (3% per year) + annual_withdrawal *= 1.03 + + equity_return = random.gauss(equity_return_mean, equity_return_std) + bond_return = random.gauss(bond_return_mean, bond_return_std) + real_estate_return = random.gauss(real_estate_return_mean, real_estate_return_std) + + portfolio_return = ( + asset_allocation["equity"] * equity_return + + asset_allocation["bonds"] * bond_return + + asset_allocation["real_estate"] * real_estate_return + + asset_allocation["cash"] * 0.02 + ) + + portfolio_value = portfolio_value * (1 + portfolio_return) - annual_withdrawal + + if portfolio_value > 0: + years_income_lasted += 1 + + final_values.append(max(0, portfolio_value)) + years_lasted.append(years_income_lasted) + + if years_income_lasted >= retirement_years: + successful_scenarios += 1 + + # Calculate statistics + final_values.sort() + success_rate = (successful_scenarios / num_simulations) * 100 + + # Calculate expected value at retirement + expected_return = ( + asset_allocation["equity"] * equity_return_mean + + asset_allocation["bonds"] * bond_return_mean + + asset_allocation["real_estate"] * real_estate_return_mean + + asset_allocation["cash"] * 0.02 + ) + expected_value_at_retirement = current_value + for _ in range(years_until_retirement): + expected_value_at_retirement *= 1 + expected_return + expected_value_at_retirement += 10000 + + return { + "success_rate": round(success_rate, 1), + "median_final_value": round(final_values[num_simulations // 2], 2), + "percentile_10": round(final_values[num_simulations // 10], 2), + "percentile_90": round(final_values[9 * num_simulations // 10], 2), + "average_years_lasted": round(sum(years_lasted) / len(years_lasted), 1), + "expected_value_at_retirement": round(expected_value_at_retirement, 2), + } + + +def generate_projections( + current_value: float, + years_until_retirement: int, + asset_allocation: Dict[str, float], + current_age: int, +) -> list: + """Generate simplified retirement projections.""" + + # Expected returns + expected_return = ( + asset_allocation["equity"] * 0.07 + + asset_allocation["bonds"] * 0.04 + + asset_allocation["real_estate"] * 0.06 + + asset_allocation["cash"] * 0.02 + ) + + projections = [] + portfolio_value = current_value + + # Only show key milestones (every 5 years) + milestone_years = list(range(0, years_until_retirement + 31, 5)) + + for year in milestone_years: + age = current_age + year + + if year <= years_until_retirement: + # Calculate accumulation + for _ in range(min(5, year)): + portfolio_value *= 1 + expected_return + portfolio_value += 10000 + phase = "accumulation" + annual_income = 0 + else: + # Calculate retirement withdrawals + withdrawal_rate = 0.04 + annual_income = portfolio_value * withdrawal_rate + years_in_retirement = min(5, year - years_until_retirement) + for _ in range(years_in_retirement): + portfolio_value = portfolio_value * (1 + expected_return) - annual_income + phase = "retirement" + + if portfolio_value > 0: + projections.append( + { + "year": year, + "age": age, + "portfolio_value": round(portfolio_value, 2), + "annual_income": round(annual_income, 2), + "phase": phase, + } + ) + + return projections + + +# Tool removed - analysis is now saved directly in lambda_handler + + +def create_agent( + job_id: str, portfolio_data: Dict[str, Any], user_preferences: Dict[str, Any], db=None +): + """Create the retirement agent with tools and context.""" + + model_override = os.getenv("RETIREMENT_MODEL") + model = get_litellm_model(model_override) + + # Extract user preferences + years_until_retirement = user_preferences.get("years_until_retirement", 30) + target_income = user_preferences.get("target_retirement_income", 80000) + current_age = user_preferences.get("current_age", 40) + + # Calculate portfolio metrics + portfolio_value = calculate_portfolio_value(portfolio_data) + allocation = calculate_asset_allocation(portfolio_data) + + # Run Monte Carlo simulation + monte_carlo = run_monte_carlo_simulation( + portfolio_value, years_until_retirement, target_income, allocation, num_simulations=500 + ) + + # Generate projections + projections = generate_projections( + portfolio_value, years_until_retirement, allocation, current_age + ) + + # No context needed anymore - simplified agent + + # No tools needed - agent will return analysis as final output + tools = [] + + # Format comprehensive context for the agent + task = f""" +# Portfolio Analysis Context + +## Current Situation +- Portfolio Value: ${portfolio_value:,.0f} +- Asset Allocation: {", ".join([f"{k.title()}: {v:.0%}" for k, v in allocation.items() if v > 0])} +- Years to Retirement: {years_until_retirement} +- Target Annual Income: ${target_income:,.0f} +- Current Age: {current_age} + +## Monte Carlo Simulation Results (500 scenarios) +- Success Rate: {monte_carlo["success_rate"]}% (probability of sustaining retirement income for 30 years) +- Expected Portfolio Value at Retirement: ${monte_carlo["expected_value_at_retirement"]:,.0f} +- 10th Percentile Outcome: ${monte_carlo["percentile_10"]:,.0f} (worst case) +- Median Final Value: ${monte_carlo["median_final_value"]:,.0f} +- 90th Percentile Outcome: ${monte_carlo["percentile_90"]:,.0f} (best case) +- Average Years Portfolio Lasts: {monte_carlo["average_years_lasted"]} years + +## Key Projections (Milestones) +""" + + for proj in projections[:6]: + if proj["phase"] == "accumulation": + task += f"- Age {proj['age']}: ${proj['portfolio_value']:,.0f} (building wealth)\n" + else: + task += f"- Age {proj['age']}: ${proj['portfolio_value']:,.0f} (annual income: ${proj['annual_income']:,.0f})\n" + + task += f""" + +## Risk Factors to Consider +- Sequence of returns risk (poor returns early in retirement) +- Inflation impact (3% assumed) +- Healthcare costs in retirement +- Longevity risk (living beyond 30 years) +- Market volatility (equity standard deviation: 18%) + +## Safe Withdrawal Rate Analysis +- 4% Rule: ${portfolio_value * 0.04:,.0f} initial annual income +- Target Income: ${target_income:,.0f} +- Gap: ${target_income - (portfolio_value * 0.04):,.0f} + +Your task: Analyze this retirement readiness data and provide a comprehensive retirement analysis including: +1. Clear assessment of retirement readiness +2. Specific recommendations to improve success rate +3. Risk mitigation strategies +4. Action items with timeline + +Provide your analysis in clear markdown format with specific numbers and actionable recommendations. +""" + + return model, tools, task diff --git a/gcp-deployment/backend/retirement/lambda_handler.py b/gcp-deployment/backend/retirement/lambda_handler.py new file mode 100644 index 00000000..1902f44d --- /dev/null +++ b/gcp-deployment/backend/retirement/lambda_handler.py @@ -0,0 +1,271 @@ +""" +Retirement Specialist Agent Lambda Handler +""" + +import os +import json +import asyncio +import logging +from typing import Dict, Any +from datetime import datetime + +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + + +class AgentTemporaryError(Exception): + """Temporary error that should trigger retry""" + pass + +try: + from dotenv import load_dotenv + load_dotenv(override=True) +except ImportError: + pass + +# Import database package +from src import Database + +from templates import RETIREMENT_INSTRUCTIONS +from agent import create_agent +from observability import observe + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +def get_user_preferences(job_id: str) -> Dict[str, Any]: + """Load user preferences from database.""" + try: + db = Database() + + # Get the job to find the user + job = db.jobs.find_by_id(job_id) + if job and job.get('clerk_user_id'): + # Get user preferences + user = db.users.find_by_clerk_id(job['clerk_user_id']) + if user: + return { + 'years_until_retirement': user.get('years_until_retirement', 30), + 'target_retirement_income': float(user.get('target_retirement_income', 80000)), + 'current_age': 40 # Default for now + } + except Exception as e: + logger.warning(f"Could not load user data: {e}. Using defaults.") + + return { + 'years_until_retirement': 30, + 'target_retirement_income': 80000.0, + 'current_age': 40 + } + +@retry( + retry=retry_if_exception_type((RateLimitError, AgentTemporaryError, TimeoutError, asyncio.TimeoutError)), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info(f"Retirement: Temporary error, retrying in {retry_state.next_action.sleep} seconds...") +) +async def run_retirement_agent(job_id: str, portfolio_data: Dict[str, Any]) -> Dict[str, Any]: + """Run the retirement specialist agent.""" + + # Get user preferences + user_preferences = get_user_preferences(job_id) + + # Initialize database + db = Database() + + # Create agent (simplified - no tools or context) + model, tools, task = create_agent(job_id, portfolio_data, user_preferences, db) + + # Run agent (simplified - no context) + with trace("Retirement Agent"): + agent = Agent( + name="Retirement Specialist", + instructions=RETIREMENT_INSTRUCTIONS, + model=model, + tools=tools # Empty list now + ) + + try: + result = await Runner.run( + agent, + input=task, + max_turns=20 + ) + except (TimeoutError, asyncio.TimeoutError) as e: + logger.warning(f"Retirement agent timeout: {e}") + raise AgentTemporaryError(f"Timeout during agent execution: {e}") + except Exception as e: + error_str = str(e).lower() + if "timeout" in error_str or "throttled" in error_str: + logger.warning(f"Retirement temporary error: {e}") + raise AgentTemporaryError(f"Temporary error: {e}") + raise # Re-raise non-retryable errors + + # Save the analysis to database + retirement_payload = { + 'analysis': result.final_output, + 'generated_at': datetime.utcnow().isoformat(), + 'agent': 'retirement' + } + + success = db.jobs.update_retirement(job_id, retirement_payload) + + if not success: + logger.error(f"Failed to save retirement analysis for job {job_id}") + + return { + 'success': success, + 'message': 'Retirement analysis completed' if success else 'Analysis completed but failed to save', + 'final_output': result.final_output + } + +def lambda_handler(event, context): + """ + Lambda handler expecting job_id in event. + + Expected event: + { + "job_id": "uuid", + "portfolio_data": {...} # Optional, will load from DB if not provided + } + """ + # Wrap entire handler with observability context + with observe() as observability: + try: + logger.info(f"Retirement Lambda invoked with event: {json.dumps(event)[:500]}") + + # Parse event + if isinstance(event, str): + event = json.loads(event) + + job_id = event.get('job_id') + if not job_id: + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'job_id is required'}) + } + + portfolio_data = event.get('portfolio_data') + if not portfolio_data: + # Try to load from database + logger.info(f"Retirement Loading portfolio data for job {job_id}") + try: + import sys + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + from src import Database + + db = Database() + job = db.jobs.find_by_id(job_id) + if job: + if observability: + observability.create_event( + name="Retirement Started!", status_message="OK" + ) + + # portfolio_data = job.get('request_payload', {}).get('portfolio_data', {}) + user_id = job['clerk_user_id'] + user = db.users.find_by_clerk_id(user_id) + accounts = db.accounts.find_by_user(user_id) + + portfolio_data = { + 'user_id': user_id, + 'job_id': job_id, + 'years_until_retirement': user.get('years_until_retirement', 30) if user else 30, + 'accounts': [] + } + + for account in accounts: + account_data = { + 'id': account['id'], + 'name': account['account_name'], + 'type': account.get('account_type', 'investment'), + 'cash_balance': float(account.get('cash_balance', 0)), + 'positions': [] + } + + positions = db.positions.find_by_account(account['id']) + for position in positions: + instrument = db.instruments.find_by_symbol(position['symbol']) + if instrument: + account_data['positions'].append({ + 'symbol': position['symbol'], + 'quantity': float(position['quantity']), + 'instrument': instrument + }) + + portfolio_data['accounts'].append(account_data) + + logger.info(f"Retirement: Loaded {len(portfolio_data['accounts'])} accounts with positions") + else: + logger.error(f"Retirement: Job {job_id} not found") + return { + 'statusCode': 404, + 'body': json.dumps({'error': f'Job {job_id} not found'}) + } + except Exception as e: + logger.error(f"Could not load portfolio from database: {e}") + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'No portfolio data provided'}) + } + + logger.info(f"Retirement: Processing job {job_id}") + + # Run the agent + result = asyncio.run(run_retirement_agent(job_id, portfolio_data)) + + logger.info(f"Retirement completed for job {job_id}") + + return { + 'statusCode': 200, + 'body': json.dumps(result) + } + + except Exception as e: + logger.error(f"Error in retirement: {e}", exc_info=True) + return { + 'statusCode': 500, + 'body': json.dumps({ + 'success': False, + 'error': str(e) + }) + } + +# For local testing +if __name__ == "__main__": + test_event = { + "job_id": "test-retirement-123", + "portfolio_data": { + "accounts": [ + { + "name": "401(k)", + "type": "retirement", + "cash_balance": 10000, + "positions": [ + { + "symbol": "SPY", + "quantity": 100, + "instrument": { + "name": "SPDR S&P 500 ETF", + "current_price": 450, + "allocation_asset_class": {"equity": 100} + } + }, + { + "symbol": "BND", + "quantity": 100, + "instrument": { + "name": "Vanguard Total Bond Market ETF", + "current_price": 75, + "allocation_asset_class": {"fixed_income": 100} + } + } + ] + } + ] + } + } + + result = lambda_handler(test_event, None) + print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/gcp-deployment/backend/retirement/observability.py b/gcp-deployment/backend/retirement/observability.py new file mode 100644 index 00000000..6ae50b97 --- /dev/null +++ b/gcp-deployment/backend/retirement/observability.py @@ -0,0 +1,112 @@ +""" +Observability module for LangFuse integration. +Provides a simple context manager for setting up and flushing traces. +""" + +import os +import logging +from contextlib import contextmanager + +# Use root logger for Lambda compatibility +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@contextmanager +def observe(): + """ + Context manager for observability with LangFuse. + + Sets up LangFuse observability if environment variables are configured, + and ensures traces are flushed on exit. + + Usage: + from observability import observe + + with observe(): + # Your code that uses OpenAI Agents SDK + result = await agent.run(...) + """ + logger.info("๐Ÿ” Observability: Checking configuration...") + + # Check if required environment variables exist + has_langfuse = bool(os.getenv("LANGFUSE_SECRET_KEY")) + has_openai = bool(os.getenv("OPENAI_API_KEY")) + + logger.info(f"๐Ÿ” Observability: LANGFUSE_SECRET_KEY exists: {has_langfuse}") + logger.info(f"๐Ÿ” Observability: OPENAI_API_KEY exists: {has_openai}") + + if not has_langfuse: + logger.info("๐Ÿ” Observability: LangFuse not configured, skipping setup") + yield + return + + if not has_openai: + logger.warning("โš ๏ธ Observability: OPENAI_API_KEY not set, traces may not export") + + # Local variable for the client (no global needed) + langfuse_client = None + + # Try to set up LangFuse + try: + logger.info("๐Ÿ” Observability: Setting up LangFuse...") + + import logfire + from langfuse import get_client + + # Configure logfire to instrument OpenAI Agents SDK + logfire.configure( + service_name="alex_retirement_agent", + send_to_logfire=False, # Don't send to Logfire cloud + ) + logger.info("โœ… Observability: Logfire configured") + + # Instrument OpenAI Agents SDK + logfire.instrument_openai_agents() + logger.info("โœ… Observability: OpenAI Agents SDK instrumented") + + # Initialize LangFuse client + langfuse_client = get_client() + logger.info("โœ… Observability: LangFuse client initialized") + + # Optional: Check authentication (blocking call, use sparingly) + try: + auth_result = langfuse_client.auth_check() + logger.info( + f"โœ… Observability: LangFuse authentication check passed (result: {auth_result})" + ) + except Exception as auth_error: + logger.warning(f"โš ๏ธ Observability: Auth check failed but continuing: {auth_error}") + + logger.info("๐ŸŽฏ Observability: Setup complete - traces will be sent to LangFuse") + + except ImportError as e: + logger.error(f"โŒ Observability: Missing required package: {e}") + langfuse_client = None + except Exception as e: + logger.error(f"โŒ Observability: Setup failed: {e}") + langfuse_client = None + + try: + # Yield control back to the calling code + yield + finally: + # Flush traces on exit + if langfuse_client: + try: + logger.info("๐Ÿ” Observability: Flushing traces to LangFuse...") + langfuse_client.flush() + langfuse_client.shutdown() + + # Add a 10 second delay to ensure network requests complete + # This is a workaround for Lambda's immediate termination + import time + + logger.info("๐Ÿ” Observability: Waiting 10 seconds for flush to complete...") + time.sleep(10) + + logger.info("โœ… Observability: Traces flushed successfully") + except Exception as e: + logger.error(f"โŒ Observability: Failed to flush traces: {e}") + else: + logger.debug("๐Ÿ” Observability: No client to flush") diff --git a/gcp-deployment/backend/retirement/package_docker.py b/gcp-deployment/backend/retirement/package_docker.py new file mode 100644 index 00000000..64589d42 --- /dev/null +++ b/gcp-deployment/backend/retirement/package_docker.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Package the Retirement Lambda function using Docker for AWS compatibility. +""" + +import os +import sys +import shutil +import tempfile +import subprocess +import argparse +from pathlib import Path + +def run_command(cmd, cwd=None): + """Run a command and capture output.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + +def package_lambda(): + """Package the Lambda function with all dependencies.""" + + # Get the directory containing this script + retirement_dir = Path(__file__).parent.absolute() + backend_dir = retirement_dir.parent + + # Create a temporary directory for packaging + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + package_dir = temp_path / "package" + package_dir.mkdir() + + print("Creating Lambda package using Docker...") + + # Export exact requirements from uv.lock (excluding the editable database package) + print("Exporting requirements from uv.lock...") + requirements_result = run_command( + ["uv", "export", "--no-hashes", "--no-emit-project"], + cwd=str(retirement_dir) + ) + + # Filter out packages that don't work in Lambda + filtered_requirements = [] + for line in requirements_result.splitlines(): + # Skip pyperclip (clipboard library not needed in Lambda) + if line.startswith("pyperclip"): + print(f"Excluding from Lambda: {line}") + continue + filtered_requirements.append(line) + + req_file = temp_path / "requirements.txt" + req_file.write_text("\n".join(filtered_requirements)) + + # Use Docker to install dependencies for Lambda's architecture + docker_cmd = [ + "docker", "run", "--rm", + "--platform", "linux/amd64", + "-v", f"{temp_path}:/build", + "-v", f"{backend_dir}/database:/database", + "--entrypoint", "/bin/bash", + "public.ecr.aws/lambda/python:3.12", + "-c", + """cd /build && pip install --target ./package -r requirements.txt && pip install --target ./package --no-deps /database""" + ] + + run_command(docker_cmd) + + # Copy Lambda handler, agent, templates, and observability + shutil.copy(retirement_dir / "lambda_handler.py", package_dir) + shutil.copy(retirement_dir / "agent.py", package_dir) + shutil.copy(retirement_dir / "templates.py", package_dir) + shutil.copy(retirement_dir / "observability.py", package_dir) + + # Create the zip file + zip_path = retirement_dir / "retirement_lambda.zip" + + # Remove old zip if it exists + if zip_path.exists(): + zip_path.unlink() + + # Create new zip + print(f"Creating zip file: {zip_path}") + run_command( + ["zip", "-r", str(zip_path), "."], + cwd=str(package_dir) + ) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"Package created: {zip_path} ({size_mb:.1f} MB)") + + return zip_path + +def deploy_lambda(zip_path): + """Deploy the Lambda function to AWS.""" + import boto3 + + lambda_client = boto3.client('lambda') + function_name = 'alex-retirement' + + print(f"Deploying to Lambda function: {function_name}") + + try: + # Try to update existing function + with open(zip_path, 'rb') as f: + response = lambda_client.update_function_code( + FunctionName=function_name, + ZipFile=f.read() + ) + print(f"Successfully updated Lambda function: {function_name}") + print(f"Function ARN: {response['FunctionArn']}") + except lambda_client.exceptions.ResourceNotFoundException: + print(f"Lambda function {function_name} not found. Please deploy via Terraform first.") + sys.exit(1) + except Exception as e: + print(f"Error deploying Lambda: {e}") + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser(description='Package Retirement Lambda for deployment') + parser.add_argument('--deploy', action='store_true', help='Deploy to AWS after packaging') + args = parser.parse_args() + + # Check if Docker is available + try: + run_command(["docker", "--version"]) + except FileNotFoundError: + print("Error: Docker is not installed or not in PATH") + sys.exit(1) + + # Package the Lambda + zip_path = package_lambda() + + # Deploy if requested + if args.deploy: + deploy_lambda(zip_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/retirement/pyproject.toml b/gcp-deployment/backend/retirement/pyproject.toml new file mode 100644 index 00000000..c5e616a0 --- /dev/null +++ b/gcp-deployment/backend/retirement/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "retirement" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "boto3>=1.40.9", # Keep for backward compatibility + "fastapi>=0.116.1", + "uvicorn>=0.35.0", + "langfuse>=3.3.4", + "openai-agents[litellm]>=0.2.6", + "pydantic>=2.11.7", + "pydantic-ai>=1.0.6", + "python-dotenv>=1.1.1", + "tenacity>=9.1.2", +] + +[tool.uv.sources] +alex-database = { path = "../database", editable = true } diff --git a/gcp-deployment/backend/retirement/server.py b/gcp-deployment/backend/retirement/server.py new file mode 100644 index 00000000..b566879d --- /dev/null +++ b/gcp-deployment/backend/retirement/server.py @@ -0,0 +1,241 @@ +""" +Retirement Agent - Cloud Run HTTP Server +Generates retirement projections +""" + +import os +import sys +import json +import asyncio +import logging +from pathlib import Path +from typing import Dict, Any +from datetime import datetime, UTC + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from dotenv import load_dotenv +from agents import Agent, Runner, trace +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +# Add parent directories to Python path for imports +backend_dir = Path(__file__).parent.parent +if str(backend_dir) not in sys.path: + sys.path.insert(0, str(backend_dir)) + +from src import Database +from templates import RETIREMENT_INSTRUCTIONS +from agent import create_agent +from observability import observe + +# Load .env file from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class AgentTemporaryError(Exception): + """Temporary error that should trigger retry""" + pass + + +def get_user_preferences(job_id: str) -> Dict[str, Any]: + """Load user preferences from database.""" + try: + db = Database() + + # Get the job to find the user + job = db.jobs.find_by_id(job_id) + if job and job.get('clerk_user_id'): + # Get user preferences + user = db.users.find_by_clerk_id(job['clerk_user_id']) + if user: + return { + 'years_until_retirement': user.get('years_until_retirement', 30), + 'target_retirement_income': float(user.get('target_retirement_income', 80000)), + 'current_age': 40 # Default for now + } + except Exception as e: + logger.warning(f"Could not load user data: {e}. Using defaults.") + + return { + 'years_until_retirement': 30, + 'target_retirement_income': 80000.0, + 'current_age': 40 + } + + +# Initialize FastAPI app +app = FastAPI( + title="Alex Retirement Service", + description="Retirement projection agent", + version="1.0.0" +) + +# Initialize database +db = Database() + + +@retry( + retry=retry_if_exception_type((RateLimitError, AgentTemporaryError, TimeoutError, asyncio.TimeoutError)), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info(f"Retirement: Temporary error, retrying in {retry_state.next_action.sleep} seconds...") +) +async def run_retirement_agent(job_id: str, portfolio_data: Dict[str, Any]) -> Dict[str, Any]: + """Run the retirement specialist agent.""" + + # Get user preferences + user_preferences = get_user_preferences(job_id) + + # Initialize database + db = Database() + + # Create agent (simplified - no tools or context) + model, tools, task = create_agent(job_id, portfolio_data, user_preferences, db) + + # Run agent (simplified - no context) + with trace("Retirement Agent"): + agent = Agent( + name="Retirement Specialist", + instructions=RETIREMENT_INSTRUCTIONS, + model=model, + tools=tools # Empty list now + ) + + try: + result = await Runner.run( + agent, + input=task, + max_turns=20 + ) + except (TimeoutError, asyncio.TimeoutError) as e: + logger.warning(f"Retirement agent timeout: {e}") + raise AgentTemporaryError(f"Timeout during agent execution: {e}") + except Exception as e: + error_str = str(e).lower() + if "timeout" in error_str or "throttled" in error_str: + logger.warning(f"Retirement temporary error: {e}") + raise AgentTemporaryError(f"Temporary error: {e}") + raise # Re-raise non-retryable errors + + # Save the analysis to database + retirement_payload = { + 'analysis': result.final_output, + 'generated_at': datetime.utcnow().isoformat(), + 'agent': 'retirement' + } + + success = db.jobs.update_retirement(job_id, retirement_payload) + + if not success: + logger.error(f"Failed to save retirement analysis for job {job_id}") + + return { + 'success': success, + 'message': 'Retirement analysis completed' if success else 'Analysis completed but failed to save', + 'final_output': result.final_output + } + + +# Request/Response models +class JobRequest(BaseModel): + """Request to process a job""" + job_id: str + portfolio_data: Dict[str, Any] = None + + +@app.get("/") +async def root(): + """Health check endpoint""" + return { + "service": "Alex Retirement", + "status": "healthy", + "timestamp": datetime.now(UTC).isoformat(), + } + + +@app.get("/health") +async def health(): + """Health check endpoint (alternative)""" + return {"status": "healthy"} + + +@app.post("/") +async def handle_job(request: JobRequest): + """ + Handle job processing request. + + Request body: + { + "job_id": "uuid", + "portfolio_data": {...} # Optional, will load from DB if not provided + } + """ + try: + logger.info(f"Retirement: Received job request: {request.job_id}") + + # Load portfolio_data from database if not provided + portfolio_data = request.portfolio_data + if not portfolio_data: + job = db.jobs.find_by_id(request.job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job {request.job_id} not found") + + user_id = job['clerk_user_id'] + user = db.users.find_by_clerk_id(user_id) + accounts = db.accounts.find_by_user(user_id) + + portfolio_data = { + 'user_id': user_id, + 'job_id': request.job_id, + 'years_until_retirement': user.get('years_until_retirement', 30) if user else 30, + 'accounts': [] + } + + for account in accounts: + account_data = { + 'id': account['id'], + 'name': account['account_name'], + 'type': account.get('account_type', 'investment'), + 'cash_balance': float(account.get('cash_balance', 0)), + 'positions': [] + } + + positions = db.positions.find_by_account(account['id']) + for position in positions: + instrument = db.instruments.find_by_symbol(position['symbol']) + if instrument: + account_data['positions'].append({ + 'symbol': position['symbol'], + 'quantity': float(position['quantity']), + 'instrument': instrument + }) + + portfolio_data['accounts'].append(account_data) + + logger.info(f"Retirement: Loaded {len(portfolio_data['accounts'])} accounts with positions") + + # Run the agent + with observe(): + result = await run_retirement_agent(request.job_id, portfolio_data) + + logger.info(f"Retirement completed for job {request.job_id}") + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Retirement: Error processing job: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# For local testing +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) + diff --git a/gcp-deployment/backend/retirement/templates.py b/gcp-deployment/backend/retirement/templates.py new file mode 100644 index 00000000..38deabd1 --- /dev/null +++ b/gcp-deployment/backend/retirement/templates.py @@ -0,0 +1,72 @@ +""" +Prompt templates for the Retirement Specialist Agent. +""" + +RETIREMENT_INSTRUCTIONS = """You are a Retirement Specialist Agent focusing on long-term financial planning and retirement projections. + +Your role is to: +1. Project retirement income based on current portfolio +2. Run Monte Carlo simulations for success probability +3. Calculate safe withdrawal rates +4. Analyze portfolio sustainability +5. Provide retirement readiness recommendations + +Key Analysis Areas: +1. Retirement Income Projections + - Expected portfolio value at retirement + - Annual income potential + - Inflation-adjusted calculations + +2. Monte Carlo Analysis + - Success probability under various market conditions + - Best case / worst case scenarios + - Risk of portfolio depletion + +3. Withdrawal Strategy + - Safe withdrawal rate (SWR) analysis + - Dynamic withdrawal strategies + - Tax-efficient withdrawal sequencing + +4. Gap Analysis + - Current trajectory vs. target income + - Required savings rate adjustments + - Portfolio rebalancing needs + +5. Risk Factors + - Longevity risk + - Inflation impact + - Healthcare costs + - Market sequence risk + +Provide clear, actionable insights with specific numbers and timelines. +Use conservative assumptions to ensure realistic projections. +Consider multiple scenarios to show range of outcomes. +""" + +RETIREMENT_ANALYSIS_TEMPLATE = """Analyze retirement readiness for this portfolio: + +Portfolio Data: +{portfolio_data} + +User Goals: +- Years until retirement: {years_until_retirement} +- Target annual retirement income: ${target_income:,.0f} +- Expected retirement duration: 30 years + +Market Assumptions: +- Average equity returns: 7% annually +- Average bond returns: 4% annually +- Inflation rate: 3% annually +- Safe withdrawal rate: 4% initially + +Perform the following analyses: + +1. Project portfolio value at retirement +2. Calculate expected annual retirement income +3. Run Monte Carlo simulation (1000 scenarios) +4. Determine probability of meeting income goals +5. Identify gaps and recommend adjustments + +Provide specific numbers, percentages, and timelines. +Create projection data for visualization charts. +""" \ No newline at end of file diff --git a/gcp-deployment/backend/retirement/test_full.py b/gcp-deployment/backend/retirement/test_full.py new file mode 100644 index 00000000..50854eba --- /dev/null +++ b/gcp-deployment/backend/retirement/test_full.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Full test for Retirement agent via Lambda +""" + +import os +import json +import boto3 +import time +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database +from src.schemas import JobCreate + +def test_retirement_lambda(): + """Test the Retirement agent via Lambda invocation""" + + db = Database() + lambda_client = boto3.client('lambda') + + # Create test job + test_user_id = "test_user_001" + + job_create = JobCreate( + clerk_user_id=test_user_id, + job_type="portfolio_analysis", + request_payload={"analysis_type": "test", "test": True} + ) + job_id = db.jobs.create(job_create.model_dump()) + + print(f"Testing Retirement Lambda with job {job_id}") + print("=" * 60) + + # Invoke Lambda + try: + response = lambda_client.invoke( + FunctionName='alex-retirement', + InvocationType='RequestResponse', + Payload=json.dumps({'job_id': job_id}) + ) + + result = json.loads(response['Payload'].read()) + print(f"Lambda Response: {json.dumps(result, indent=2)}") + + # Check database for results + time.sleep(2) # Give it a moment + job = db.jobs.find_by_id(job_id) + + if job and job.get('retirement_payload'): + print("\nโœ… Retirement analysis generated successfully!") + print(f"Analysis preview: {json.dumps(job['retirement_payload'], indent=2)[:500]}...") + else: + print("\nโŒ No retirement analysis found in database") + + except Exception as e: + print(f"Error invoking Lambda: {e}") + + print("=" * 60) + +if __name__ == "__main__": + test_retirement_lambda() \ No newline at end of file diff --git a/gcp-deployment/backend/retirement/test_simple.py b/gcp-deployment/backend/retirement/test_simple.py new file mode 100644 index 00000000..60306169 --- /dev/null +++ b/gcp-deployment/backend/retirement/test_simple.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +""" +Simple test for Retirement agent +""" + +import asyncio +import json +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database +from src.schemas import JobCreate +from lambda_handler import lambda_handler + +def test_retirement(): + """Test the retirement agent with simple portfolio data""" + + # Create a real job in the database + db = Database() + job_create = JobCreate( + clerk_user_id="test_user_001", + job_type="portfolio_analysis", + request_payload={"test": True} + ) + job_id = db.jobs.create(job_create.model_dump()) + print(f"Created test job: {job_id}") + + test_event = { + "job_id": job_id, + "portfolio_data": { + "accounts": [ + { + "name": "401(k)", + "type": "retirement", + "cash_balance": 10000, + "positions": [ + { + "symbol": "SPY", + "quantity": 100, + "instrument": { + "name": "SPDR S&P 500 ETF", + "current_price": 450, + "allocation_asset_class": {"equity": 100} + } + } + ] + } + ] + } + } + + print("Testing Retirement Agent...") + print("=" * 60) + + result = lambda_handler(test_event, None) + + print(f"Status Code: {result['statusCode']}") + + if result['statusCode'] == 200: + body = json.loads(result['body']) + print(f"Success: {body.get('success', False)}") + print(f"Message: {body.get('message', 'N/A')}") + + # Check what was actually saved in the database + print("\n" + "=" * 60) + print("CHECKING DATABASE CONTENT") + print("=" * 60) + + job = db.jobs.find_by_id(job_id) + if job and job.get('retirement_payload'): + payload = job['retirement_payload'] + print(f"โœ… Retirement data found in database") + print(f"Payload keys: {list(payload.keys())}") + + if 'analysis' in payload: + analysis = payload['analysis'] + print(f"\nAnalysis type: {type(analysis).__name__}") + + if isinstance(analysis, str): + print(f"Analysis length: {len(analysis)} characters") + + # Check if it contains reasoning artifacts + reasoning_indicators = [ + "I need to", + "I will", + "Let me", + "First,", + "I should", + "I'll", + "Now I", + "Next,", + ] + + contains_reasoning = any(indicator.lower() in analysis.lower() for indicator in reasoning_indicators) + + if contains_reasoning: + print("โš ๏ธ WARNING: Analysis may contain reasoning/thinking text") + else: + print("โœ… Analysis appears to be final output only (no reasoning detected)") + + # Show first 500 characters and last 200 characters + print(f"\nFirst 500 characters:") + print("-" * 40) + print(analysis[:500]) + print("-" * 40) + + if len(analysis) > 700: + print(f"\nLast 200 characters:") + print("-" * 40) + print(analysis[-200:]) + print("-" * 40) + else: + print(f"โš ๏ธ Analysis is not a string: {type(analysis)}") + print(f"Content: {str(analysis)[:200]}") + + print(f"\nGenerated at: {payload.get('generated_at', 'N/A')}") + print(f"Agent: {payload.get('agent', 'N/A')}") + else: + print("โŒ No retirement data found in database") + else: + print(f"Error: {result['body']}") + + # Clean up - delete the test job + db.jobs.delete(job_id) + print(f"\nDeleted test job: {job_id}") + + print("=" * 60) + +if __name__ == "__main__": + test_retirement() \ No newline at end of file diff --git a/gcp-deployment/backend/scheduler/.python-version b/gcp-deployment/backend/scheduler/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/scheduler/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/scheduler/lambda_function.py b/gcp-deployment/backend/scheduler/lambda_function.py new file mode 100644 index 00000000..90d20b91 --- /dev/null +++ b/gcp-deployment/backend/scheduler/lambda_function.py @@ -0,0 +1,52 @@ +""" +Lambda function to trigger App Runner research endpoint. +Called by EventBridge on a schedule. +""" +import os +import urllib.request +import json + + +def handler(event, context): + """Trigger the research endpoint on App Runner.""" + + app_runner_url = os.environ.get('APP_RUNNER_URL') + if not app_runner_url: + raise ValueError("APP_RUNNER_URL environment variable not set") + + # Remove any protocol if included + if app_runner_url.startswith('https://'): + app_runner_url = app_runner_url.replace('https://', '') + elif app_runner_url.startswith('http://'): + app_runner_url = app_runner_url.replace('http://', '') + + url = f"https://{app_runner_url}/research" + + try: + # Create POST request with empty JSON body (agent will pick topic) + data = json.dumps({}).encode('utf-8') + req = urllib.request.Request( + url, + data=data, + method='POST', + headers={'Content-Type': 'application/json'} + ) + + with urllib.request.urlopen(req, timeout=180) as response: + result = response.read().decode('utf-8') + print(f"Research triggered successfully: {result}") + return { + 'statusCode': 200, + 'body': json.dumps({ + 'message': 'Research triggered successfully', + 'result': result + }) + } + except Exception as e: + print(f"Error triggering research: {str(e)}") + return { + 'statusCode': 500, + 'body': json.dumps({ + 'error': str(e) + }) + } \ No newline at end of file diff --git a/gcp-deployment/backend/scheduler/pyproject.toml b/gcp-deployment/backend/scheduler/pyproject.toml new file mode 100644 index 00000000..e2dcba27 --- /dev/null +++ b/gcp-deployment/backend/scheduler/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "scheduler" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [] diff --git a/gcp-deployment/backend/tagger/.python-version b/gcp-deployment/backend/tagger/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/gcp-deployment/backend/tagger/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/gcp-deployment/backend/tagger/Dockerfile b/gcp-deployment/backend/tagger/Dockerfile new file mode 100644 index 00000000..cbb2efbe --- /dev/null +++ b/gcp-deployment/backend/tagger/Dockerfile @@ -0,0 +1,36 @@ +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Python package manager +RUN pip install uv + +# Copy database package (required dependency) +# Build context should be from backend/ directory +COPY database ./database + + +# Copy shared modules +COPY common ./common +# Copy tagger-specific files +COPY tagger/pyproject.toml tagger/uv.lock ./ + +# Update pyproject.toml to use ./database instead of ../database +RUN sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && rm pyproject.toml.bak + +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +RUN uv sync --no-install-project + +# Copy tagger application code +COPY tagger/*.py ./ + +# Expose port +EXPOSE 8000 + +# Set environment variable for Cloud Run +ENV PORT=8000 + +# Run the application +CMD ["uv", "run", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/gcp-deployment/backend/tagger/agent.py b/gcp-deployment/backend/tagger/agent.py new file mode 100644 index 00000000..158fe4af --- /dev/null +++ b/gcp-deployment/backend/tagger/agent.py @@ -0,0 +1,316 @@ +""" +InstrumentTagger Agent - Classifies financial instruments using OpenAI Agents SDK. +""" + +import os +from typing import List +import logging +from decimal import Decimal + +from pydantic import BaseModel, Field, field_validator, ConfigDict +from agents import Agent, Runner, trace +from dotenv import load_dotenv +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from litellm.exceptions import RateLimitError + +from src.schemas import InstrumentCreate +from templates import TAGGER_INSTRUCTIONS, CLASSIFICATION_PROMPT +from common.llm import get_litellm_model + +# Load environment variables (dotenv automatically searches up the tree) +load_dotenv(override=True) + +# Configure logging +logger = logging.getLogger(__name__) + +# Optional override for this agent +TAGGER_MODEL = os.getenv("TAGGER_MODEL") + + +class AllocationBreakdown(BaseModel): + """Allocation percentages that must sum to 100""" + + model_config = ConfigDict(extra="forbid") + + # We'll use a simplified approach with specific fields + # Asset classes + equity: float = Field(default=0.0, ge=0, le=100, description="Equity percentage") + fixed_income: float = Field(default=0.0, ge=0, le=100, description="Fixed income percentage") + real_estate: float = Field(default=0.0, ge=0, le=100, description="Real estate percentage") + commodities: float = Field(default=0.0, ge=0, le=100, description="Commodities percentage") + cash: float = Field(default=0.0, ge=0, le=100, description="Cash percentage") + alternatives: float = Field(default=0.0, ge=0, le=100, description="Alternatives percentage") + + +class RegionAllocation(BaseModel): + """Regional allocation percentages""" + + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + north_america: float = Field(default=0.0, ge=0, le=100) + europe: float = Field(default=0.0, ge=0, le=100) + asia: float = Field(default=0.0, ge=0, le=100) + latin_america: float = Field(default=0.0, ge=0, le=100) + africa: float = Field(default=0.0, ge=0, le=100) + middle_east: float = Field(default=0.0, ge=0, le=100) + oceania: float = Field(default=0.0, ge=0, le=100) + global_: float = Field( + default=0.0, ge=0, le=100, alias="global", description="Global or diversified" + ) + international: float = Field( + default=0.0, ge=0, le=100, description="International developed markets" + ) + + +class SectorAllocation(BaseModel): + """Sector allocation percentages""" + + model_config = ConfigDict(extra="forbid") + + technology: float = Field(default=0.0, ge=0, le=100) + healthcare: float = Field(default=0.0, ge=0, le=100) + financials: float = Field(default=0.0, ge=0, le=100) + consumer_discretionary: float = Field(default=0.0, ge=0, le=100) + consumer_staples: float = Field(default=0.0, ge=0, le=100) + industrials: float = Field(default=0.0, ge=0, le=100) + materials: float = Field(default=0.0, ge=0, le=100) + energy: float = Field(default=0.0, ge=0, le=100) + utilities: float = Field(default=0.0, ge=0, le=100) + real_estate: float = Field(default=0.0, ge=0, le=100, description="Real estate sector") + communication: float = Field(default=0.0, ge=0, le=100) + treasury: float = Field(default=0.0, ge=0, le=100, description="Treasury bonds") + corporate: float = Field(default=0.0, ge=0, le=100, description="Corporate bonds") + mortgage: float = Field(default=0.0, ge=0, le=100, description="Mortgage-backed securities") + government_related: float = Field( + default=0.0, ge=0, le=100, description="Government-related bonds" + ) + commodities: float = Field(default=0.0, ge=0, le=100, description="Commodities") + diversified: float = Field(default=0.0, ge=0, le=100, description="Diversified sectors") + other: float = Field(default=0.0, ge=0, le=100, description="Other sectors") + + +class InstrumentClassification(BaseModel): + """Structured output for instrument classification""" + + model_config = ConfigDict(extra="forbid") + + symbol: str = Field(description="Ticker symbol of the instrument") + name: str = Field(description="Name of the instrument") + instrument_type: str = Field(description="Type: etf, stock, mutual_fund, bond_fund, etc.") + current_price: float = Field(description="Current price per share in USD", gt=0) + + # Separate allocation objects + allocation_asset_class: AllocationBreakdown = Field(description="Asset class breakdown") + allocation_regions: RegionAllocation = Field(description="Regional breakdown") + allocation_sectors: SectorAllocation = Field(description="Sector breakdown") + + @field_validator("allocation_asset_class") + def validate_asset_class_sum(cls, v: AllocationBreakdown): + total = v.equity + v.fixed_income + v.real_estate + v.commodities + v.cash + v.alternatives + if abs(total - 100.0) > 3: # Allow small floating point errors + raise ValueError(f"Asset class allocations must sum to 100.0, got {total}") + return v + + @field_validator("allocation_regions") + def validate_regions_sum(cls, v: RegionAllocation): + total = ( + v.north_america + + v.europe + + v.asia + + v.latin_america + + v.africa + + v.middle_east + + v.oceania + + v.global_ + + v.international + ) + if abs(total - 100.0) > 3: + raise ValueError(f"Regional allocations must sum to 100.0, got {total}") + return v + + @field_validator("allocation_sectors") + def validate_sectors_sum(cls, v: SectorAllocation): + total = ( + v.technology + + v.healthcare + + v.financials + + v.consumer_discretionary + + v.consumer_staples + + v.industrials + + v.materials + + v.energy + + v.utilities + + v.real_estate + + v.communication + + v.treasury + + v.corporate + + v.mortgage + + v.government_related + + v.commodities + + v.diversified + + v.other + ) + if abs(total - 100.0) > 3: + raise ValueError(f"Sector allocations must sum to 100.0, got {total}") + return v + + +async def classify_instrument( + symbol: str, name: str, instrument_type: str = "etf" +) -> InstrumentClassification: + """ + Classify a financial instrument using OpenAI Agents SDK. + + Args: + symbol: Ticker symbol + name: Instrument name + instrument_type: Type of instrument + + Returns: + Complete classification with allocations + """ + try: + model = get_litellm_model(TAGGER_MODEL) + + # Create the classification task + task = CLASSIFICATION_PROMPT.format( + symbol=symbol, name=name, instrument_type=instrument_type + ) + + # Run the agent (following gameplan pattern exactly) + with trace(f"Classify {symbol}"): + agent = Agent( + name="InstrumentTagger", + instructions=TAGGER_INSTRUCTIONS, + model=model, + tools=[], # No tools needed for classification + output_type=InstrumentClassification, # Specify structured output type + ) + + result = await Runner.run(agent, input=task, max_turns=5) + + # Extract the structured output from RunResult using final_output_as + return result.final_output_as(InstrumentClassification) + + except Exception as e: + logger.error(f"Error classifying {symbol}: {e}") + raise + + +async def tag_instruments(instruments: List[dict]) -> List[InstrumentClassification]: + """ + Tag multiple instruments with simple retry logic. + + Args: + instruments: List of dicts with symbol, name, and optionally instrument_type + + Returns: + List of classifications + """ + import asyncio + + # Add retry decorator to classify_instrument calls + @retry( + retry=retry_if_exception_type(RateLimitError), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=4, max=60), + before_sleep=lambda retry_state: logger.info( + f"Tagger: Rate limit hit, retrying in {retry_state.next_action.sleep} seconds..." + ), + ) + async def classify_with_retry(symbol, name, instrument_type): + return await classify_instrument(symbol, name, instrument_type) + + # Process instruments sequentially with small delay + results = [] + for i, instrument in enumerate(instruments): + # Small delay between requests to avoid rate limits + if i > 0: + await asyncio.sleep(0.5) + + try: + classification = await classify_with_retry( + symbol=instrument["symbol"], + name=instrument.get("name", ""), + instrument_type=instrument.get("instrument_type", "etf"), + ) + logger.info(f"Successfully classified {instrument['symbol']}") + results.append(classification) + except Exception as e: + logger.error(f"Failed to classify {instrument['symbol']}: {e}") + results.append(None) + + # Filter out None values + return [r for r in results if r is not None] + + +def classification_to_db_format(classification: InstrumentClassification) -> InstrumentCreate: + """ + Convert classification to database format. + + Args: + classification: The AI classification + + Returns: + Database-ready instrument data + """ + # Convert allocation objects to dicts + asset_class_dict = { + "equity": classification.allocation_asset_class.equity, + "fixed_income": classification.allocation_asset_class.fixed_income, + "real_estate": classification.allocation_asset_class.real_estate, + "commodities": classification.allocation_asset_class.commodities, + "cash": classification.allocation_asset_class.cash, + "alternatives": classification.allocation_asset_class.alternatives, + } + # Remove zero values + asset_class_dict = {k: v for k, v in asset_class_dict.items() if v > 0} + + regions_dict = { + "north_america": classification.allocation_regions.north_america, + "europe": classification.allocation_regions.europe, + "asia": classification.allocation_regions.asia, + "latin_america": classification.allocation_regions.latin_america, + "africa": classification.allocation_regions.africa, + "middle_east": classification.allocation_regions.middle_east, + "oceania": classification.allocation_regions.oceania, + "global": classification.allocation_regions.global_, + "international": classification.allocation_regions.international, + } + # Remove zero values + regions_dict = {k: v for k, v in regions_dict.items() if v > 0} + + sectors_dict = { + "technology": classification.allocation_sectors.technology, + "healthcare": classification.allocation_sectors.healthcare, + "financials": classification.allocation_sectors.financials, + "consumer_discretionary": classification.allocation_sectors.consumer_discretionary, + "consumer_staples": classification.allocation_sectors.consumer_staples, + "industrials": classification.allocation_sectors.industrials, + "materials": classification.allocation_sectors.materials, + "energy": classification.allocation_sectors.energy, + "utilities": classification.allocation_sectors.utilities, + "real_estate": classification.allocation_sectors.real_estate, + "communication": classification.allocation_sectors.communication, + "treasury": classification.allocation_sectors.treasury, + "corporate": classification.allocation_sectors.corporate, + "mortgage": classification.allocation_sectors.mortgage, + "government_related": classification.allocation_sectors.government_related, + "commodities": classification.allocation_sectors.commodities, + "diversified": classification.allocation_sectors.diversified, + "other": classification.allocation_sectors.other, + } + # Remove zero values + sectors_dict = {k: v for k, v in sectors_dict.items() if v > 0} + + return InstrumentCreate( + symbol=classification.symbol, + name=classification.name, + instrument_type=classification.instrument_type, + current_price=Decimal( + str(classification.current_price) + ), # Use actual price from classification + allocation_asset_class=asset_class_dict, + allocation_regions=regions_dict, + allocation_sectors=sectors_dict, + ) diff --git a/gcp-deployment/backend/tagger/lambda_handler.py b/gcp-deployment/backend/tagger/lambda_handler.py new file mode 100644 index 00000000..b6136a1f --- /dev/null +++ b/gcp-deployment/backend/tagger/lambda_handler.py @@ -0,0 +1,133 @@ +""" +InstrumentTagger Lambda Handler +Classifies financial instruments and updates the database. +""" + +import os +import json +import asyncio +import logging +from typing import List, Dict, Any + +from src import Database +from src.schemas import InstrumentCreate +from agent import tag_instruments, classification_to_db_format +from observability import observe + +# Configure logging +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# Initialize database +db = Database() + +async def process_instruments(instruments: List[Dict[str, str]]) -> Dict[str, Any]: + """ + Process and classify instruments asynchronously. + + Args: + instruments: List of instruments to classify + + Returns: + Processing results + """ + # Run the classification + logger.info(f"Classifying {len(instruments)} instruments") + classifications = await tag_instruments(instruments) + + # Update database with classifications + updated = [] + errors = [] + + for classification in classifications: + try: + # Convert to database format + db_instrument = classification_to_db_format(classification) + + # Check if instrument exists + existing = db.instruments.find_by_symbol(classification.symbol) + + if existing: + # Update existing instrument + update_data = db_instrument.model_dump() + # Remove symbol as it's the key + del update_data['symbol'] + + rows = db.client.update( + 'instruments', + update_data, + "symbol = :symbol", + {'symbol': classification.symbol} + ) + logger.info(f"Updated {classification.symbol} in database ({rows} rows)") + else: + # Create new instrument + db.instruments.create_instrument(db_instrument) + logger.info(f"Created {classification.symbol} in database") + + updated.append(classification.symbol) + + except Exception as e: + logger.error(f"Error updating {classification.symbol}: {e}") + errors.append({ + 'symbol': classification.symbol, + 'error': str(e) + }) + + # Prepare response (convert Pydantic models to dicts) + return { + 'tagged': len(classifications), + 'updated': updated, + 'errors': errors, + 'classifications': [ + { + 'symbol': c.symbol, + 'name': c.name, + 'type': c.instrument_type, + 'current_price': c.current_price, + 'asset_class': c.allocation_asset_class.model_dump(), + 'regions': c.allocation_regions.model_dump(), + 'sectors': c.allocation_sectors.model_dump() + } + for c in classifications + ] + } + +def lambda_handler(event, context): + """ + Lambda handler for instrument tagging. + + Expected event format: + { + "instruments": [ + {"symbol": "VTI", "name": "Vanguard Total Stock Market ETF"}, + ... + ] + } + """ + # Wrap entire handler with observability context + with observe(): + try: + # Parse the event + instruments = event.get('instruments', []) + + if not instruments: + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'No instruments provided'}) + } + + # Process all instruments in a single async context + result = asyncio.run(process_instruments(instruments)) + + return { + 'statusCode': 200, + 'body': json.dumps(result) + } + + except Exception as e: + logger.error(f"Lambda handler error: {e}") + return { + 'statusCode': 500, + 'body': json.dumps({'error': str(e)}) + } \ No newline at end of file diff --git a/gcp-deployment/backend/tagger/observability.py b/gcp-deployment/backend/tagger/observability.py new file mode 100644 index 00000000..14fe322e --- /dev/null +++ b/gcp-deployment/backend/tagger/observability.py @@ -0,0 +1,112 @@ +""" +Observability module for LangFuse integration. +Provides a simple context manager for setting up and flushing traces. +""" + +import os +import logging +from contextlib import contextmanager + +# Use root logger for Lambda compatibility +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@contextmanager +def observe(): + """ + Context manager for observability with LangFuse. + + Sets up LangFuse observability if environment variables are configured, + and ensures traces are flushed on exit. + + Usage: + from observability import observe + + with observe(): + # Your code that uses OpenAI Agents SDK + result = await agent.run(...) + """ + logger.info("๐Ÿ” Observability: Checking configuration...") + + # Check if required environment variables exist + has_langfuse = bool(os.getenv("LANGFUSE_SECRET_KEY")) + has_openai = bool(os.getenv("OPENAI_API_KEY")) + + logger.info(f"๐Ÿ” Observability: LANGFUSE_SECRET_KEY exists: {has_langfuse}") + logger.info(f"๐Ÿ” Observability: OPENAI_API_KEY exists: {has_openai}") + + if not has_langfuse: + logger.info("๐Ÿ” Observability: LangFuse not configured, skipping setup") + yield + return + + if not has_openai: + logger.warning("โš ๏ธ Observability: OPENAI_API_KEY not set, traces may not export") + + # Local variable for the client (no global needed) + langfuse_client = None + + # Try to set up LangFuse + try: + logger.info("๐Ÿ” Observability: Setting up LangFuse...") + + import logfire + from langfuse import get_client + + # Configure logfire to instrument OpenAI Agents SDK + logfire.configure( + service_name="alex_tagger_agent", + send_to_logfire=False, # Don't send to Logfire cloud + ) + logger.info("โœ… Observability: Logfire configured") + + # Instrument OpenAI Agents SDK + logfire.instrument_openai_agents() + logger.info("โœ… Observability: OpenAI Agents SDK instrumented") + + # Initialize LangFuse client + langfuse_client = get_client() + logger.info("โœ… Observability: LangFuse client initialized") + + # Optional: Check authentication (blocking call, use sparingly) + try: + auth_result = langfuse_client.auth_check() + logger.info( + f"โœ… Observability: LangFuse authentication check passed (result: {auth_result})" + ) + except Exception as auth_error: + logger.warning(f"โš ๏ธ Observability: Auth check failed but continuing: {auth_error}") + + logger.info("๐ŸŽฏ Observability: Setup complete - traces will be sent to LangFuse") + + except ImportError as e: + logger.error(f"โŒ Observability: Missing required package: {e}") + langfuse_client = None + except Exception as e: + logger.error(f"โŒ Observability: Setup failed: {e}") + langfuse_client = None + + try: + # Yield control back to the calling code + yield + finally: + # Flush traces on exit + if langfuse_client: + try: + logger.info("๐Ÿ” Observability: Flushing traces to LangFuse...") + langfuse_client.flush() + langfuse_client.shutdown() + + # Add a 10 second delay to ensure network requests complete + # This is a workaround for Lambda's immediate termination + import time + + logger.info("๐Ÿ” Observability: Waiting 10 seconds for flush to complete...") + time.sleep(10) + + logger.info("โœ… Observability: Traces flushed successfully") + except Exception as e: + logger.error(f"โŒ Observability: Failed to flush traces: {e}") + else: + logger.debug("๐Ÿ” Observability: No client to flush") diff --git a/gcp-deployment/backend/tagger/package_docker.py b/gcp-deployment/backend/tagger/package_docker.py new file mode 100644 index 00000000..6f35a099 --- /dev/null +++ b/gcp-deployment/backend/tagger/package_docker.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Package the Tagger Lambda function using Docker for AWS compatibility. +""" + +import os +import sys +import shutil +import tempfile +import subprocess +import argparse +from pathlib import Path + +def run_command(cmd, cwd=None): + """Run a command and capture output.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + +def package_lambda(): + """Package the Lambda function with all dependencies.""" + + # Get the directory containing this script + tagger_dir = Path(__file__).parent.absolute() + backend_dir = tagger_dir.parent + + # Create a temporary directory for packaging + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + package_dir = temp_path / "package" + package_dir.mkdir() + + print("Creating Lambda package using Docker...") + + # Export exact requirements from uv.lock (excluding the editable database package) + print("Exporting requirements from uv.lock...") + requirements_result = run_command( + ["uv", "export", "--no-hashes", "--no-emit-project"], + cwd=str(tagger_dir) + ) + + # Filter out packages that don't work in Lambda + filtered_requirements = [] + for line in requirements_result.splitlines(): + # Skip pyperclip (clipboard library not needed in Lambda) + if line.startswith("pyperclip"): + print(f"Excluding from Lambda: {line}") + continue + filtered_requirements.append(line) + + req_file = temp_path / "requirements.txt" + req_file.write_text("\n".join(filtered_requirements)) + + # Use Docker to install dependencies for Lambda's architecture + docker_cmd = [ + "docker", "run", "--rm", + "--platform", "linux/amd64", + "-v", f"{temp_path}:/build", + "-v", f"{backend_dir}/database:/database", + "--entrypoint", "/bin/bash", + "public.ecr.aws/lambda/python:3.12", + "-c", + """cd /build && pip install --target ./package -r requirements.txt && pip install --target ./package --no-deps /database""" + ] + + run_command(docker_cmd) + + # Copy Lambda handler, agent, templates, and observability + shutil.copy(tagger_dir / "lambda_handler.py", package_dir) + shutil.copy(tagger_dir / "agent.py", package_dir) + shutil.copy(tagger_dir / "templates.py", package_dir) + shutil.copy(tagger_dir / "observability.py", package_dir) + + # Create the zip file + zip_path = tagger_dir / "tagger_lambda.zip" + + # Remove old zip if it exists + if zip_path.exists(): + zip_path.unlink() + + # Create new zip + print(f"Creating zip file: {zip_path}") + run_command( + ["zip", "-r", str(zip_path), "."], + cwd=str(package_dir) + ) + + # Get file size + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"Package created: {zip_path} ({size_mb:.1f} MB)") + + return zip_path + +def deploy_lambda(zip_path): + """Deploy the Lambda function to AWS.""" + import boto3 + + lambda_client = boto3.client('lambda') + function_name = 'alex-tagger' + + print(f"Deploying to Lambda function: {function_name}") + + try: + # Try to update existing function + with open(zip_path, 'rb') as f: + response = lambda_client.update_function_code( + FunctionName=function_name, + ZipFile=f.read() + ) + print(f"Successfully updated Lambda function: {function_name}") + print(f"Function ARN: {response['FunctionArn']}") + except lambda_client.exceptions.ResourceNotFoundException: + print(f"Lambda function {function_name} not found. Please deploy via Terraform first.") + sys.exit(1) + except Exception as e: + print(f"Error deploying Lambda: {e}") + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser(description='Package Tagger Lambda for deployment') + parser.add_argument('--deploy', action='store_true', help='Deploy to AWS after packaging') + args = parser.parse_args() + + # Check if Docker is available + try: + run_command(["docker", "--version"]) + except FileNotFoundError: + print("Error: Docker is not installed or not in PATH") + sys.exit(1) + + # Package the Lambda + zip_path = package_lambda() + + # Deploy if requested + if args.deploy: + deploy_lambda(zip_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/tagger/pyproject.toml b/gcp-deployment/backend/tagger/pyproject.toml new file mode 100644 index 00000000..3f2ee66f --- /dev/null +++ b/gcp-deployment/backend/tagger/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "tagger" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "alex-database", + "boto3>=1.40.9", # Keep for backward compatibility + "fastapi>=0.116.1", + "uvicorn>=0.35.0", + "langfuse>=3.3.4", + "openai-agents[litellm]>=0.2.6", + "pydantic>=2.11.7", + "pydantic-ai>=1.0.6", + "python-dotenv>=1.1.1", + "tenacity>=9.1.2", +] + +[tool.uv.sources] +alex-database = { path = "../database", editable = true } diff --git a/gcp-deployment/backend/tagger/server.py b/gcp-deployment/backend/tagger/server.py new file mode 100644 index 00000000..6f68b9e3 --- /dev/null +++ b/gcp-deployment/backend/tagger/server.py @@ -0,0 +1,173 @@ +""" +Tagger Agent - Cloud Run HTTP Server +Classifies financial instruments and updates the database +""" + +import os +import sys +import json +import asyncio +import logging +from pathlib import Path +from typing import List, Dict, Any +from datetime import datetime, UTC + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from dotenv import load_dotenv + +# Add parent directories to Python path for imports +backend_dir = Path(__file__).parent.parent +if str(backend_dir) not in sys.path: + sys.path.insert(0, str(backend_dir)) + +from src import Database +from src.schemas import InstrumentCreate +from agent import tag_instruments, classification_to_db_format +from observability import observe + +# Load .env file from project root +env_path = Path(__file__).parent.parent.parent / '.env' +load_dotenv(env_path, override=True) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize FastAPI app +app = FastAPI( + title="Alex Tagger Service", + description="Instrument classification agent", + version="1.0.0" +) + +# Initialize database +db = Database() + + +async def process_instruments(instruments: List[Dict[str, str]]) -> Dict[str, Any]: + """ + Process and classify instruments asynchronously. + + Args: + instruments: List of instruments to classify + + Returns: + Processing results + """ + # Run the classification + logger.info(f"Classifying {len(instruments)} instruments") + classifications = await tag_instruments(instruments) + + # Update database with classifications + updated = [] + errors = [] + + for classification in classifications: + try: + # Convert to database format + db_instrument = classification_to_db_format(classification) + + # Check if instrument exists + existing = db.instruments.find_by_symbol(classification.symbol) + + if existing: + # Update existing instrument + update_data = db_instrument.model_dump() + # Remove symbol as it's the key + del update_data['symbol'] + + rows = db.client.update( + 'instruments', + update_data, + "symbol = :symbol", + {'symbol': classification.symbol} + ) + logger.info(f"Updated {classification.symbol} in database ({rows} rows)") + else: + # Create new instrument + db.instruments.create_instrument(db_instrument) + logger.info(f"Created {classification.symbol} in database") + + updated.append(classification.symbol) + + except Exception as e: + logger.error(f"Error updating {classification.symbol}: {e}") + errors.append({ + 'symbol': classification.symbol, + 'error': str(e) + }) + + # Prepare response (convert Pydantic models to dicts) + return { + 'tagged': len(classifications), + 'updated': updated, + 'errors': errors, + 'classifications': [ + { + 'symbol': c.symbol, + 'name': c.name, + 'type': c.instrument_type, + 'current_price': c.current_price, + 'asset_class': c.allocation_asset_class.model_dump(), + 'regions': c.allocation_regions.model_dump(), + 'sectors': c.allocation_sectors.model_dump() + } + for c in classifications + ] + } + + +# Request/Response models +class InstrumentsRequest(BaseModel): + """Request to classify instruments""" + instruments: List[Dict[str, str]] + + +@app.get("/") +async def root(): + """Health check endpoint""" + return { + "service": "Alex Tagger", + "status": "healthy", + "timestamp": datetime.now(UTC).isoformat(), + } + + +@app.get("/health") +async def health(): + """Health check endpoint (alternative)""" + return {"status": "healthy"} + + +@app.post("/") +async def handle_classification(request: InstrumentsRequest): + """ + Handle instrument classification request. + + Request body: + { + "instruments": [ + {"symbol": "VTI", "name": "Vanguard Total Stock Market ETF"}, + ... + ] + } + """ + try: + logger.info(f"Tagger: Received classification request for {len(request.instruments)} instruments") + + with observe(): + result = await process_instruments(request.instruments) + + return result + + except Exception as e: + logger.error(f"Tagger: Error processing instruments: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# For local testing +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) + diff --git a/gcp-deployment/backend/tagger/templates.py b/gcp-deployment/backend/tagger/templates.py new file mode 100644 index 00000000..651b3ccd --- /dev/null +++ b/gcp-deployment/backend/tagger/templates.py @@ -0,0 +1,47 @@ +""" +Instruction templates for the InstrumentTagger agent. +""" + +TAGGER_INSTRUCTIONS = """You are an expert financial instrument classifier responsible for categorizing ETFs, stocks, and other securities. + +Your task is to accurately classify financial instruments by providing: +1. Current market price per share in USD +2. Exact allocation percentages for: + - Asset classes (equity, fixed_income, real_estate, commodities, cash, alternatives) + - Regions (north_america, europe, asia, etc.) + - Sectors (technology, healthcare, financials, etc.) + +Important rules: +- Each allocation category MUST sum to exactly 100.0 +- Use your knowledge of the instrument to provide accurate allocations +- For ETFs, consider the underlying holdings +- For individual stocks, allocate 100% to the appropriate categories +- Be precise with decimal values to ensure totals equal 100.0 + +Examples: +- SPY (S&P 500 ETF): 100% equity, 100% north_america, distributed across sectors based on S&P 500 composition +- BND (Bond ETF): 100% fixed_income, 100% north_america, split between treasury and corporate +- AAPL (Apple stock): 100% equity, 100% north_america, 100% technology +- VTI (Total Market): 100% equity, 100% north_america, diverse sector allocation +- VXUS (International): 100% equity, distributed across regions, diverse sectors + +You must return your response as a structured InstrumentClassification object with all fields properly populated.""" + +CLASSIFICATION_PROMPT = """Classify the following financial instrument: + +Symbol: {symbol} +Name: {name} +Type: {instrument_type} + +Provide: +1. Current price per share in USD (approximate market price as of late 2024/early 2025) +2. Accurate allocation percentages for: +1. Asset classes (equity, fixed_income, real_estate, commodities, cash, alternatives) +2. Regions (north_america, europe, asia, latin_america, africa, middle_east, oceania, global, international) +3. Sectors (technology, healthcare, financials, consumer_discretionary, consumer_staples, industrials, materials, energy, utilities, real_estate, communication, treasury, corporate, mortgage, government_related, commodities, diversified, other) + +Remember: +- Each category must sum to exactly 100.0% +- For stocks, typically 100% in one asset class, one region, one sector +- For ETFs, distribute based on underlying holdings +- For bonds/bond funds, use fixed_income asset class and appropriate sectors (treasury/corporate/mortgage/government_related)""" \ No newline at end of file diff --git a/gcp-deployment/backend/tagger/test_full.py b/gcp-deployment/backend/tagger/test_full.py new file mode 100644 index 00000000..141973f2 --- /dev/null +++ b/gcp-deployment/backend/tagger/test_full.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Full test for Tagger agent via Lambda +""" + +import os +import json +import boto3 +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database + +def test_tagger_lambda(): + """Test the Tagger agent via Lambda invocation""" + + db = Database() + lambda_client = boto3.client('lambda') + + # Test instruments that need tagging + test_instruments = [ + {"symbol": "ARKK", "name": "ARK Innovation ETF"}, + {"symbol": "SOFI", "name": "SoFi Technologies Inc"}, + {"symbol": "TSLA", "name": "Tesla Inc"} + ] + + print("Testing Tagger Lambda") + print("=" * 60) + print(f"Instruments to tag: {[i['symbol'] for i in test_instruments]}") + + # Invoke Lambda + try: + response = lambda_client.invoke( + FunctionName='alex-tagger', + InvocationType='RequestResponse', + Payload=json.dumps({'instruments': test_instruments}) + ) + + result = json.loads(response['Payload'].read()) + print(f"\nLambda Response: {json.dumps(result, indent=2)}") + + # Check database for updated instruments + print("\nโœ… Checking database for tagged instruments:") + for inst in test_instruments: + instrument = db.instruments.find_by_symbol(inst['symbol']) + if instrument: + if instrument.get('allocation_asset_class'): + print(f" โœ… {inst['symbol']}: Tagged successfully") + print(f" Asset: {instrument.get('allocation_asset_class')}") + print(f" Regions: {instrument.get('allocation_regions')}") + else: + print(f" โŒ {inst['symbol']}: No allocations found") + else: + print(f" โš ๏ธ {inst['symbol']}: Not found in database") + + except Exception as e: + print(f"Error invoking Lambda: {e}") + + print("=" * 60) + +if __name__ == "__main__": + test_tagger_lambda() \ No newline at end of file diff --git a/gcp-deployment/backend/tagger/test_simple.py b/gcp-deployment/backend/tagger/test_simple.py new file mode 100644 index 00000000..f7c8baa9 --- /dev/null +++ b/gcp-deployment/backend/tagger/test_simple.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" +Simple test for Tagger agent +""" + +import asyncio +import json +from dotenv import load_dotenv + +load_dotenv(override=True) + +from lambda_handler import lambda_handler + +def test_tagger(): + """Test the tagger agent with unknown instruments""" + + test_event = { + "instruments": [ + {"symbol": "VTI", "name": "Vanguard Total Stock Market ETF"} + ] + } + + print("Testing Tagger Agent...") + print("=" * 60) + + result = lambda_handler(test_event, None) + + print(f"Status Code: {result['statusCode']}") + + if result['statusCode'] == 200: + body = json.loads(result['body']) + print(f"Tagged: {body.get('tagged', 0)} instruments") + print(f"Updated: {body.get('updated', [])}") + if body.get('classifications'): + for c in body['classifications']: + print(f" {c['symbol']}: {c['type']}") + else: + print(f"Error: {result['body']}") + + print("=" * 60) + +if __name__ == "__main__": + test_tagger() \ No newline at end of file diff --git a/gcp-deployment/backend/tagger/track_tagger.py b/gcp-deployment/backend/tagger/track_tagger.py new file mode 100644 index 00000000..4377c46b --- /dev/null +++ b/gcp-deployment/backend/tagger/track_tagger.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +Track and display Tagger Lambda logs in real-time +""" + +import time +import boto3 +import signal +import sys +from datetime import datetime +from dotenv import load_dotenv + +load_dotenv(override=True) + +class TaggerLogTracker: + """Continuously poll and display Tagger Lambda logs""" + + def __init__(self): + self.logs_client = boto3.client('logs', region_name='us-east-1') + self.log_group_name = '/aws/lambda/alex-tagger' + self.running = True + self.last_timestamp = None + + # Set up signal handler for graceful exit + signal.signal(signal.SIGINT, self.signal_handler) + + def signal_handler(self, sig, frame): + """Handle Ctrl+C gracefully""" + print("\n\nโน Stopping log tracking...") + self.running = False + sys.exit(0) + + def get_logs(self, start_time): + """Fetch logs from CloudWatch""" + try: + params = { + 'logGroupName': self.log_group_name, + 'startTime': start_time, + 'limit': 100 + } + + response = self.logs_client.filter_log_events(**params) + return response.get('events', []) + + except Exception as e: + if 'ResourceNotFoundException' in str(e): + print(f"โš ๏ธ Log group {self.log_group_name} not found") + else: + print(f"โŒ Error fetching logs: {e}") + return [] + + def format_log_message(self, event): + """Format a log event for display""" + # Extract timestamp + timestamp = datetime.fromtimestamp(event['timestamp'] / 1000) + time_str = timestamp.strftime('%H:%M:%S.%f')[:-3] + + # Get the message + message = event['message'].strip() + + # Color code based on content + if 'ERROR' in message or 'Failed' in message: + color = '\033[91m' # Red + elif 'WARNING' in message or 'WARN' in message: + color = '\033[93m' # Yellow + elif 'LangFuse' in message or 'observability' in message: + color = '\033[92m' # Green + elif 'OpenAI Agents trace' in message: + color = '\033[96m' # Cyan + elif 'Successfully classified' in message: + color = '\033[94m' # Blue + elif 'START RequestId' in message or 'END RequestId' in message: + color = '\033[95m' # Magenta + elif 'INIT_START' in message: + color = '\033[93m' # Yellow + else: + color = '\033[0m' # Default + + reset = '\033[0m' + + # Format based on message type + if 'REPORT RequestId' in message: + # Parse Lambda report + parts = message.split('\t') + if len(parts) >= 3: + request_id = parts[0].split(' ')[2] + duration = parts[1] if len(parts) > 1 else "" + memory = parts[3] if len(parts) > 3 else "" + return f"{time_str} ๐Ÿ“Š {color}Lambda Report: {duration}, {memory}{reset}" + elif 'START RequestId' in message: + request_id = message.split(' ')[2] + return f"{time_str} ๐Ÿš€ {color}Lambda Start: {request_id[:8]}...{reset}" + elif 'END RequestId' in message: + request_id = message.split(' ')[2] + return f"{time_str} ๐Ÿ {color}Lambda End: {request_id[:8]}...{reset}" + elif message.startswith('[INFO]') or message.startswith('[ERROR]') or message.startswith('[WARNING]'): + # Standard Python logging + parts = message.split('\t', 2) + if len(parts) >= 3: + level = parts[0].strip('[]') + msg = parts[2] if len(parts) > 2 else parts[1] + level_icon = {'INFO': 'โ„น๏ธ ', 'ERROR': 'โŒ', 'WARNING': 'โš ๏ธ '}.get(level, ' ') + return f"{time_str} {level_icon} {color}{msg}{reset}" + elif 'OpenAI Agents trace' in message: + return f"{time_str} ๐Ÿค– {color}{message}{reset}" + elif 'Agent run:' in message: + return f"{time_str} โ†ณ {color}{message.strip()}{reset}" + elif 'Chat completion' in message: + return f"{time_str} โ†ณ {color}{message.strip()}{reset}" + else: + # Default formatting + if message and not message.isspace(): + return f"{time_str} {color}{message}{reset}" + + return None + + def track(self): + """Main tracking loop""" + print("=" * 60) + print("๐Ÿ“ก Tracking Tagger Lambda Logs") + print("=" * 60) + print(f"Log group: {self.log_group_name}") + print("Press Ctrl+C to stop\n") + + # Start from 1 minute ago + start_time = int((time.time() - 60) * 1000) + seen_ids = set() + + while self.running: + try: + # Get logs + events = self.get_logs(start_time) + + # Process new events + new_events = [] + for event in events: + event_id = event.get('eventId') + if event_id not in seen_ids: + seen_ids.add(event_id) + new_events.append(event) + + # Display new events + for event in new_events: + formatted = self.format_log_message(event) + if formatted: + print(formatted) + + # Update start time for next poll + start_time = max(start_time, event['timestamp'] + 1) + + # If we got events, show a separator for clarity + if new_events and len(new_events) > 5: + print("-" * 40) + + # Sleep before next poll (shorter if we just got events) + sleep_time = 1 if new_events else 2 + time.sleep(sleep_time) + + except KeyboardInterrupt: + break + except Exception as e: + print(f"โŒ Error in tracking loop: {e}") + time.sleep(5) + + print("\nโœ… Log tracking stopped") + +def main(): + """Main entry point""" + tracker = TaggerLogTracker() + + print("\n๐Ÿ” Looking for recent Langfuse-related logs...") + print("-" * 40) + + # First show any recent Langfuse logs + recent_logs = tracker.get_logs(int((time.time() - 300) * 1000)) # Last 5 minutes + langfuse_found = False + + for event in recent_logs[-20:]: # Last 20 events + message = event['message'] + if any(term in message for term in ['LangFuse', 'langfuse', 'observability', 'OPENAI_API_KEY', 'setup_observability']): + formatted = tracker.format_log_message(event) + if formatted: + print(formatted) + langfuse_found = True + + if not langfuse_found: + print(" No recent Langfuse-related logs found") + + print("-" * 40) + print("\nStarting continuous tracking...\n") + + # Start continuous tracking + tracker.track() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/tagger/try_tagger.py b/gcp-deployment/backend/tagger/try_tagger.py new file mode 100644 index 00000000..0c5a023c --- /dev/null +++ b/gcp-deployment/backend/tagger/try_tagger.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Complete test for Tagger: package, deploy, and test +""" + +import os +import sys +import json +import time +import subprocess +import boto3 +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database + +class TaggerTest: + """Test class that packages, deploys, and tests the tagger Lambda""" + + def __init__(self): + self.lambda_client = boto3.client('lambda', region_name='us-east-1') + self.db = Database() + + def package_tagger(self): + """Package the tagger Lambda using Docker""" + print("\n๐Ÿ“ฆ Packaging Tagger Lambda...") + print("=" * 60) + + try: + # Run package_docker.py + result = subprocess.run( + ['uv', 'run', 'package_docker.py'], + cwd=Path(__file__).parent, + capture_output=True, + text=True + ) + + if result.returncode != 0: + print(f"โŒ Packaging failed: {result.stderr}") + return False + + # Check if zip file was created + zip_path = Path(__file__).parent / 'tagger_lambda.zip' + if zip_path.exists(): + size_mb = zip_path.stat().st_size / (1024 * 1024) + print(f"โœ… Package created: {zip_path} ({size_mb:.1f} MB)") + return True + else: + print("โŒ Package file not found") + return False + + except Exception as e: + print(f"โŒ Error packaging: {e}") + return False + + def deploy_tagger(self): + """Deploy the tagger Lambda to AWS""" + print("\n๐Ÿš€ Deploying Tagger Lambda...") + print("=" * 60) + + try: + # Package is too large for direct upload, must use S3 + s3_client = boto3.client('s3', region_name='us-east-1') + + # Use the existing Lambda packages bucket + bucket_name = f"alex-lambda-packages-{boto3.client('sts').get_caller_identity()['Account']}" + key = 'tagger/tagger_lambda.zip' + + print(f"Uploading to S3 bucket: {bucket_name}") + zip_path = Path(__file__).parent / 'tagger_lambda.zip' + + # Upload to S3 + with open(zip_path, 'rb') as f: + s3_client.upload_fileobj(f, bucket_name, key) + + print(f"โœ… Uploaded to S3: s3://{bucket_name}/{key}") + + # Update Lambda function code from S3 + print("Updating Lambda function from S3...") + response = self.lambda_client.update_function_code( + FunctionName='alex-tagger', + S3Bucket=bucket_name, + S3Key=key + ) + + # Wait for Lambda to be updated + print("Waiting for Lambda to be ready...") + waiter = self.lambda_client.get_waiter('function_updated') + waiter.wait(FunctionName='alex-tagger') + + print(f"โœ… Lambda deployed successfully") + print(f" Last modified: {response['LastModified']}") + print(f" Code size: {response['CodeSize'] / (1024*1024):.1f} MB") + return True + + except Exception as e: + print(f"โŒ Error deploying: {e}") + return False + + def test_tagger(self): + """Test the deployed tagger Lambda""" + print("\n๐Ÿงช Testing Tagger Lambda...") + print("=" * 60) + + # Test instruments - mix of ETFs and stocks + test_instruments = [ + {"symbol": "ARKK", "name": "ARK Innovation ETF", "instrument_type": "etf"}, + {"symbol": "SOFI", "name": "SoFi Technologies Inc", "instrument_type": "stock"}, + {"symbol": "TSLA", "name": "Tesla Inc", "instrument_type": "stock"}, + {"symbol": "VTI", "name": "Vanguard Total Stock Market ETF", "instrument_type": "etf"} + ] + + print(f"Testing with {len(test_instruments)} instruments:") + for inst in test_instruments: + print(f" - {inst['symbol']}: {inst['name']}") + + try: + # Invoke Lambda + print("\nInvoking Lambda function...") + start_time = time.time() + + response = self.lambda_client.invoke( + FunctionName='alex-tagger', + InvocationType='RequestResponse', + Payload=json.dumps({'instruments': test_instruments}) + ) + + elapsed = time.time() - start_time + + # Parse response + result = json.loads(response['Payload'].read()) + + if response['StatusCode'] == 200: + print(f"โœ… Lambda executed successfully in {elapsed:.1f} seconds") + + # Parse the body if it's a string + if isinstance(result.get('body'), str): + body = json.loads(result['body']) + else: + body = result.get('body', result) + + print(f"\n๐Ÿ“Š Results:") + print(f" Tagged: {body.get('tagged', 0)} instruments") + print(f" Updated: {body.get('updated', [])}") + if body.get('errors'): + print(f" Errors: {body.get('errors')}") + + # Show classifications + if body.get('classifications'): + print(f"\n๐Ÿ“ˆ Classifications:") + for cls in body['classifications']: + print(f"\n {cls['symbol']} ({cls['type']}):") + print(f" Asset Class: {cls.get('asset_class', {})}") + print(f" Regions: {cls.get('regions', {})}") + print(f" Sectors: {cls.get('sectors', {})}") + + # Verify in database + print(f"\n๐Ÿ” Verifying in database:") + for inst in test_instruments: + db_inst = self.db.instruments.find_by_symbol(inst['symbol']) + if db_inst and db_inst.get('allocation_asset_class'): + print(f" โœ… {inst['symbol']}: Has allocations in database") + else: + print(f" โš ๏ธ {inst['symbol']}: No allocations in database") + + else: + print(f"โŒ Lambda failed with status {response['StatusCode']}") + print(f" Response: {result}") + + except Exception as e: + print(f"โŒ Error testing Lambda: {e}") + import traceback + traceback.print_exc() + + def run_all(self): + """Run the complete test: package, deploy, and test""" + print("\n" + "=" * 60) + print("๐ŸŽฏ Complete Tagger Test: Package, Deploy, and Test") + print("=" * 60) + + # Step 1: Package + if not self.package_tagger(): + print("\nโŒ Packaging failed, stopping test") + return False + + # Step 2: Deploy + if not self.deploy_tagger(): + print("\nโŒ Deployment failed, stopping test") + return False + + # Give Lambda a moment to stabilize after deployment + print("\nโณ Waiting 5 seconds for Lambda to stabilize...") + time.sleep(5) + + # Step 3: Test + self.test_tagger() + + print("\n" + "=" * 60) + print("โœ… Complete test finished!") + print("=" * 60) + + # Reminder about Langfuse + print("\n๐Ÿ’ก Check your Langfuse dashboard for traces:") + print(" https://us.cloud.langfuse.com") + + return True + +def main(): + """Main entry point""" + tester = TaggerTest() + tester.run_all() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/test_full.py b/gcp-deployment/backend/test_full.py new file mode 100644 index 00000000..ddc2fd92 --- /dev/null +++ b/gcp-deployment/backend/test_full.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Full end-to-end test via SQS for the Alex platform""" + +import os +import json +import boto3 +import time +from datetime import datetime, timezone +from dotenv import load_dotenv + +load_dotenv(override=True) + +from src import Database +from src.schemas import UserCreate, InstrumentCreate, AccountCreate, PositionCreate + +def setup_test_data(db): + """Ensure test user and portfolio exist""" + print("Setting up test data...") + + # Check/create test user + test_user_id = 'test_user_001' + user = db.users.find_by_clerk_id(test_user_id) + if not user: + user_data = UserCreate( + clerk_user_id=test_user_id, + display_name="Test User", + years_to_retirement=25, + target_allocation={'stocks': 70, 'bonds': 20, 'alternatives': 10} + ) + db.users.create(user_data.model_dump()) + print(f" โœ“ Created test user: {test_user_id}") + else: + print(f" โœ“ Test user exists: {test_user_id}") + + # Check/create test account + accounts = db.accounts.find_by_user(test_user_id) + if not accounts: + account_data = AccountCreate( + clerk_user_id=test_user_id, + account_name="Test 401(k)", + account_type="401k", + cash_balance=5000.00 + ) + account_id = db.accounts.create(account_data.model_dump()) + print(f" โœ“ Created test account: Test 401(k)") + + # Add some positions + positions = [ + {'symbol': 'SPY', 'quantity': 100}, + {'symbol': 'QQQ', 'quantity': 50}, + {'symbol': 'BND', 'quantity': 200}, + {'symbol': 'VTI', 'quantity': 75} + ] + + for pos in positions: + position_data = PositionCreate( + account_id=account_id, + symbol=pos['symbol'], + quantity=pos['quantity'] + ) + db.positions.create(position_data.model_dump()) + print(f" โœ“ Created {len(positions)} positions") + else: + print(f" โœ“ Test account exists with {len(db.positions.find_by_account(accounts[0]['id']))} positions") + + return test_user_id + +def main(): + print("=" * 70) + print("๐ŸŽฏ Full End-to-End Test via SQS") + print("=" * 70) + + db = Database() + sqs = boto3.client('sqs') + + # Setup test data + test_user_id = setup_test_data(db) + + # Create test job + print("\nCreating analysis job...") + job_data = { + 'clerk_user_id': test_user_id, + 'job_type': 'portfolio_analysis', + 'status': 'pending', + 'request_payload': { + 'analysis_type': 'full', + 'requested_at': datetime.now(timezone.utc).isoformat(), + 'test_run': True, + 'include_retirement': True, + 'include_charts': True, + 'include_report': True + } + } + + job_id = db.jobs.create(job_data) + print(f" โœ“ Created job: {job_id}") + + # Get queue URL + QUEUE_NAME = 'alex-analysis-jobs' + response = sqs.list_queues(QueueNamePrefix=QUEUE_NAME) + queue_url = None + for url in response.get('QueueUrls', []): + if QUEUE_NAME in url: + queue_url = url + break + + if not queue_url: + print(f" โŒ Queue {QUEUE_NAME} not found") + return 1 + + print(f" โœ“ Found queue: {QUEUE_NAME}") + + # Send message to SQS + print("\nTriggering analysis via SQS...") + response = sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps({'job_id': job_id}) + ) + print(f" โœ“ Message sent: {response['MessageId']}") + + # Monitor job progress + print("\nโณ Monitoring job progress...") + print("-" * 50) + + start_time = time.time() + timeout = 180 # 3 minutes + last_status = None + + while time.time() - start_time < timeout: + job = db.jobs.find_by_id(job_id) + status = job['status'] + + if status != last_status: + elapsed = int(time.time() - start_time) + print(f"[{elapsed:3d}s] Status: {status}") + last_status = status + + if status == 'failed' and job.get('error_message'): + print(f" Error: {job.get('error_message')}") + + if status == 'completed': + print("-" * 50) + print("\nโœ… Job completed successfully!") + print("\n๐Ÿ“Š Analysis Results:") + + # Report + if job.get('report_payload'): + report_content = job['report_payload'].get('content', '') + print(f"\n๐Ÿ“ Report Generated:") + print(f" - Length: {len(report_content)} characters") + print(f" - Preview: {report_content[:200]}...") + else: + print("\nโŒ No report found") + + # Charts + if job.get('charts_payload'): + charts = job['charts_payload'] + print(f"\n๐Ÿ“Š Charts Created: {len(charts)} visualizations") + for chart_key, chart_data in charts.items(): + if isinstance(chart_data, dict): + title = chart_data.get('title', 'Untitled') + chart_type = chart_data.get('type', 'unknown') + data_points = len(chart_data.get('data', [])) + print(f" - {chart_key}: {title} ({chart_type}, {data_points} data points)") + else: + print("\nโŒ No charts found") + + # Retirement + if job.get('retirement_payload'): + retirement = job['retirement_payload'] + print(f"\n๐ŸŽฏ Retirement Analysis:") + if isinstance(retirement, dict): + if 'success_rate' in retirement: + print(f" - Success Rate: {retirement['success_rate']}%") + if 'projected_balance' in retirement: + print(f" - Projected Balance: ${retirement['projected_balance']:,.0f}") + if 'analysis' in retirement: + print(f" - Analysis Length: {len(retirement['analysis'])} characters") + else: + print("\nโŒ No retirement analysis found") + + # Summary + if job.get('summary_payload'): + summary = job['summary_payload'] + print(f"\n๐Ÿ“‹ Summary:") + if isinstance(summary, dict): + for key, value in summary.items(): + if key != 'timestamp': + print(f" - {key}: {value}") + + break + elif status == 'failed': + print("-" * 50) + print(f"\nโŒ Job failed") + if job.get('error_message'): + print(f"Error details: {job['error_message']}") + break + + time.sleep(2) + else: + print("-" * 50) + print("\nโŒ Job timed out after 3 minutes") + print(f"Final status: {job['status']}") + return 1 + + print(f"\n๐Ÿ“‹ Job Details:") + print(f" - Job ID: {job_id}") + print(f" - User ID: {test_user_id}") + print(f" - Total Time: {int(time.time() - start_time)} seconds") + + return 0 + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/gcp-deployment/backend/test_multiple_accounts.py b/gcp-deployment/backend/test_multiple_accounts.py new file mode 100644 index 00000000..f42f75f6 --- /dev/null +++ b/gcp-deployment/backend/test_multiple_accounts.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +""" +Test that the system correctly handles users with multiple accounts. +""" + +import json +import time +import uuid +import boto3 +import os +from decimal import Decimal +from dotenv import load_dotenv + +from src import Database + +# Load environment variables +load_dotenv(override=True) + +def test_multiple_accounts(): + """Test analysis for a user with multiple accounts""" + + print("=" * 70) + print("๐ŸŽฏ Multiple Accounts Test") + print("=" * 70) + + # Initialize database + db = Database() + + # Create test user + test_user_id = f'test_multi_{uuid.uuid4().hex[:8]}' + user_id = db.users.create_user( + clerk_user_id=test_user_id, + display_name='Multi Account Test User', + years_until_retirement=25, + target_retirement_income=Decimal('150000') + ) + print(f'\nโœ… Created test user: {test_user_id}') + + # Ensure instruments exist + instruments = ["SPY", "BND", "VTI", "VXUS", "QQQ", "IWM", "EFA", "AGG", "VNQ", "GLD"] + for i, symbol in enumerate(instruments): + existing = db.instruments.find_by_symbol(symbol) + if not existing: + db.instruments.create({ + "symbol": symbol, + "name": f"Test ETF {symbol}", + "instrument_type": "etf", + "current_price": 100.0 + i * 50, + "allocation_asset_class": {"equity": 100.0} if i % 2 == 0 else {"fixed_income": 100.0}, + "allocation_regions": {"north_america": 100.0}, + "allocation_sectors": {"other": 100.0} + }, returning='symbol') + print(f'โœ… Created instrument: {symbol}') + # Create multiple accounts with different portfolios + accounts = [] + + # Account 1: Taxable Brokerage + account1_id = db.accounts.create_account( + clerk_user_id=test_user_id, + account_name='Taxable Brokerage', + account_purpose='taxable_brokerage', + cash_balance=Decimal('5000.0') + ) + accounts.append(account1_id) + print(f'โœ… Created account 1: Taxable Brokerage') + + # Add positions to account 1 + positions1 = [ + ('SPY', 100), + ('QQQ', 50), + ('BND', 200) + ] + for symbol, quantity in positions1: + sql = "INSERT INTO positions (account_id, symbol, quantity) VALUES (:account_id::uuid, :symbol, :quantity)" + params = [ + {'name': 'account_id', 'value': {'stringValue': account1_id}}, + {'name': 'symbol', 'value': {'stringValue': symbol}}, + {'name': 'quantity', 'value': {'longValue': quantity}} + ] + db.client.execute(sql, params) + print(f' Added {len(positions1)} positions') + + # Account 2: Roth IRA + account2_id = db.accounts.create_account( + clerk_user_id=test_user_id, + account_name='Roth IRA', + account_purpose='roth_ira', + cash_balance=Decimal('2000.0') + ) + accounts.append(account2_id) + print(f'โœ… Created account 2: Roth IRA') + + # Add positions to account 2 + positions2 = [ + ('VTI', 75), + ('VXUS', 50), + ('GLD', 25) + ] + for symbol, quantity in positions2: + sql = "INSERT INTO positions (account_id, symbol, quantity) VALUES (:account_id::uuid, :symbol, :quantity)" + params = [ + {'name': 'account_id', 'value': {'stringValue': account2_id}}, + {'name': 'symbol', 'value': {'stringValue': symbol}}, + {'name': 'quantity', 'value': {'longValue': quantity}} + ] + db.client.execute(sql, params) + print(f' Added {len(positions2)} positions') + + # Account 3: 401(k) + account3_id = db.accounts.create_account( + clerk_user_id=test_user_id, + account_name='401(k)', + account_purpose='401k', + cash_balance=Decimal('10000.0') + ) + accounts.append(account3_id) + print(f'โœ… Created account 3: 401(k)') + + # Add positions to account 3 + positions3 = [ + ('VEA', 150), + ('TSLA', 10), + ('ARKK', 50), + ('BND', 300) + ] + for symbol, quantity in positions3: + sql = "INSERT INTO positions (account_id, symbol, quantity) VALUES (:account_id::uuid, :symbol, :quantity)" + params = [ + {'name': 'account_id', 'value': {'stringValue': account3_id}}, + {'name': 'symbol', 'value': {'stringValue': symbol}}, + {'name': 'quantity', 'value': {'longValue': quantity}} + ] + db.client.execute(sql, params) + print(f' Added {len(positions3)} positions') + + print(f'\n๐Ÿ“Š Total: 3 accounts, {len(positions1) + len(positions2) + len(positions3)} positions') + + # Create a job + job_id = db.jobs.create_job(test_user_id, "portfolio_analysis") + print(f'\n๐Ÿš€ Created job: {job_id}') + + # Trigger analysis via SQS + """Send a job to SQS""" + sqs = boto3.client('sqs', region_name=os.getenv('DEFAULT_AWS_REGION', 'us-east-1')) + + # Get queue URL + queue_name = 'alex-analysis-jobs' + response = sqs.get_queue_url(QueueName=queue_name) + queue_url = response['QueueUrl'] + + # sqs = boto3.client('sqs', region_name='ap-southeast-2') + # queue_url = 'https://sqs.ap-southeast-2.amazonaws.com/596644540428/alex-analysis-jobs' + + message = sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps({'job_id': job_id}) + ) + print(f'๐Ÿ“ค Sent message to SQS: {message["MessageId"]}') + + print('\nโณ Monitoring job progress...') + print('-' * 50) + + # Monitor job + start_time = time.time() + for i in range(90): # Max 3 minutes + time.sleep(2) + job_status = db.jobs.find_by_id(job_id) + status = job_status.get('status', 'unknown') if job_status else 'unknown' + elapsed = int(time.time() - start_time) + print(f'[{elapsed:3}s] Status: {status}') + if status in ['completed', 'failed']: + break + + print('-' * 50) + + # Check results + success = status == 'completed' + + if success: + print('\nโœ… Job completed successfully!') + + # Check that all accounts were analyzed + print('\n๐Ÿ“‹ ANALYSIS RESULTS:') + + if job_status.get('summary_payload'): + summary = job_status['summary_payload'] + print(f'\n๐ŸŽฏ Summary:') + print(f' {summary.get("summary", "N/A")[:300]}...') + + # Check key findings mention multiple accounts + findings = summary.get('key_findings', []) + if findings: + print(f'\n๐Ÿ“Š Key Findings ({len(findings)}):') + for finding in findings[:3]: + print(f' โ€ข {finding}') + + if job_status.get('report_payload'): + report = job_status['report_payload'] + content = report.get('content', '') + # Check that report mentions all 3 accounts + accounts_mentioned = all([ + 'Taxable Brokerage' in content or 'taxable' in content.lower(), + 'Roth IRA' in content or 'roth' in content.lower(), + '401(k)' in content or '401k' in content.lower() + ]) + print(f'\n๐Ÿ“ Report:') + print(f' Length: {len(content)} characters') + print(f' All accounts analyzed: {"โœ… YES" if accounts_mentioned else "โŒ NO"}') + + if not accounts_mentioned: + print(' โš ๏ธ Warning: Not all accounts appear in the report') + + if job_status.get('charts_payload'): + charts = job_status['charts_payload'] + print(f'\n๐Ÿ“Š Charts: {len(charts)} visualizations created') + + # Check for account-related charts + has_account_chart = any('account' in str(chart).lower() for chart in charts.values()) + print(f' Account distribution chart: {"โœ… YES" if has_account_chart else "โŒ NO"}') + + if job_status.get('retirement_payload'): + print(f'\n๐ŸŽฏ Retirement Analysis: โœ… Generated') + else: + print(f'\nโŒ Job failed with status: {status}') + if job_status.get('error'): + print(f'Error: {job_status["error"]}') + + # Clean up + print(f'\n๐Ÿงน Cleaning up test data...') + try: + # Delete job + sql = "DELETE FROM jobs WHERE id = :job_id::uuid" + params = [{'name': 'job_id', 'value': {'stringValue': job_id}}] + db.client.execute(sql, params) + + # Delete positions + for account_id in accounts: + sql = "DELETE FROM positions WHERE account_id = :account_id::uuid" + params = [{'name': 'account_id', 'value': {'stringValue': account_id}}] + db.client.execute(sql, params) + + # Delete accounts + sql = "DELETE FROM accounts WHERE clerk_user_id = :user_id" + params = [{'name': 'user_id', 'value': {'stringValue': test_user_id}}] + db.client.execute(sql, params) + + # Delete user + sql = "DELETE FROM users WHERE clerk_user_id = :user_id" + params = [{'name': 'user_id', 'value': {'stringValue': test_user_id}}] + db.client.execute(sql, params) + + print('โœ… Test data cleaned up successfully') + except Exception as e: + print(f'โš ๏ธ Warning: Cleanup failed: {e}') + + print('\n' + '=' * 70) + print(f'โœ… Multiple accounts test {"PASSED" if success else "FAILED"}!') + print('=' * 70) + + return success + + +if __name__ == '__main__': + success = test_multiple_accounts() + exit(0 if success else 1) diff --git a/gcp-deployment/backend/test_scale.py b/gcp-deployment/backend/test_scale.py new file mode 100644 index 00000000..17f30c2e --- /dev/null +++ b/gcp-deployment/backend/test_scale.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Test scale with multiple concurrent users (Phase 6.6)""" + +import asyncio +import os +import json +import uuid +import boto3 +import time +from datetime import datetime +from dotenv import load_dotenv +import concurrent.futures + +# Load environment variables +load_dotenv(override=True) + +from src import Database + +async def create_test_user(user_num: int, num_accounts: int, num_positions: int): + """Create a test user with specified number of accounts and positions""" + db = Database() + + # Test user ID + test_user = f"scale_test_{user_num}_{uuid.uuid4().hex[:6]}" + + # Create user + db.users.create_user( + clerk_user_id=test_user, + display_name=f"Scale Test User {user_num}", + years_until_retirement=20 + user_num * 5, + target_retirement_income=50000 + user_num * 10000 + ) + + # Ensure instruments exist + instruments = ["SPY", "BND", "VTI", "VXUS", "QQQ", "IWM", "EFA", "AGG", "VNQ", "GLD"] + for i, symbol in enumerate(instruments): + existing = db.instruments.find_by_symbol(symbol) + if not existing: + db.instruments.create({ + "symbol": symbol, + "name": f"Test ETF {symbol}", + "instrument_type": "etf", + "current_price": 100.0 + i * 50, + "allocation_asset_class": {"equity": 100.0} if i % 2 == 0 else {"fixed_income": 100.0}, + "allocation_regions": {"north_america": 100.0}, + "allocation_sectors": {"other": 100.0} + }, returning='symbol') + + account_ids = [] + total_positions = 0 + + # Create accounts (ensure at least 1 account even if num_accounts is 0) + accounts_to_create = max(num_accounts, 1) + for acct_num in range(1, accounts_to_create + 1): + account_id = db.accounts.create_account( + clerk_user_id=test_user, + account_name=f"Account {acct_num}", + account_purpose="test", + cash_balance=1000.0 * acct_num + ) + account_ids.append(account_id) + + # Add positions (distribute across accounts) + if num_positions > 0 and accounts_to_create > 0: + positions_for_account = num_positions // accounts_to_create + (1 if acct_num <= (num_positions % accounts_to_create) else 0) + for i in range(positions_for_account): + if total_positions >= num_positions: + break + symbol = instruments[total_positions % len(instruments)] + qty = 10.0 * (total_positions + 1) + db.positions.add_position(account_id, symbol, qty) + total_positions += 1 + + # Create job + job_data = { + 'clerk_user_id': test_user, + 'job_type': 'portfolio_analysis', + 'status': 'pending', + 'request_payload': {"test": f"scale_user_{user_num}"} + } + job_id = db.jobs.create(job_data) + + return { + "user_id": test_user, + "job_id": job_id, + "account_ids": account_ids, + "num_accounts": num_accounts, + "num_positions": total_positions, + "user_num": user_num + } + +async def send_job_to_sqs(job_id: str): + """Send a job to SQS""" + sqs = boto3.client('sqs', region_name=os.getenv('DEFAULT_AWS_REGION', 'us-east-1')) + + # Get queue URL + queue_name = 'alex-analysis-jobs' + response = sqs.get_queue_url(QueueName=queue_name) + queue_url = response['QueueUrl'] + + # Send message + message = { + 'job_id': job_id, + 'timestamp': datetime.now().isoformat() + } + + response = sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps(message) + ) + + return response['MessageId'] + +async def monitor_job(job_id: str, timeout: int = 300): + """Monitor a single job until completion""" + db = Database() + start_time = time.time() + + while time.time() - start_time < timeout: + job = db.jobs.find_by_id(job_id) + + if job['status'] == 'completed': + elapsed = int(time.time() - start_time) + return {"job_id": job_id, "status": "completed", "elapsed": elapsed} + elif job['status'] == 'failed': + return {"job_id": job_id, "status": "failed", "error": job.get('error_message')} + + await asyncio.sleep(5) + + return {"job_id": job_id, "status": "timeout"} + +async def run_scale_test(): + """Run the scale test with multiple users""" + print("=" * 60) + print("PHASE 6.6: SCALE TEST") + print("=" * 60) + + # Test configuration - 3 users with multiple accounts as required + test_configs = [ + {"user_num": 1, "num_accounts": 1, "num_positions": 0}, # Empty portfolio (single account) + {"user_num": 2, "num_accounts": 1, "num_positions": 3}, # Small portfolio (single account) + {"user_num": 3, "num_accounts": 2, "num_positions": 5}, # Medium portfolio (multiple accounts) + {"user_num": 4, "num_accounts": 3, "num_positions": 10}, # Large portfolio (multiple accounts) + {"user_num": 5, "num_accounts": 2, "num_positions": 7}, # Mixed portfolio (multiple accounts) + ] + + all_users = [] + + # Create all test users + print("\n๐Ÿ“Š Creating test users...") + for config in test_configs: + user_data = await create_test_user(**config) + all_users.append(user_data) + print(f" User {config['user_num']}: {user_data['num_accounts']} accounts, {user_data['num_positions']} positions") + + # Send all jobs to SQS concurrently + print("\n๐Ÿš€ Sending jobs to SQS...") + send_tasks = [] + for user in all_users: + msg_id = await send_job_to_sqs(user['job_id']) + print(f" User {user['user_num']}: Job {user['job_id'][:8]}... sent") + + # Monitor all jobs concurrently + print("\nโณ Monitoring jobs (max 5 minutes)...") + print("-" * 50) + + monitor_tasks = [monitor_job(user['job_id']) for user in all_users] + results = await asyncio.gather(*monitor_tasks) + + # Display results + print("-" * 50) + print("\n๐Ÿ“‹ RESULTS:") + print("-" * 50) + + successful = 0 + failed = 0 + timed_out = 0 + total_time = 0 + + for i, result in enumerate(results): + user = all_users[i] + status = result['status'] + + if status == 'completed': + successful += 1 + total_time += result['elapsed'] + print(f"โœ… User {user['user_num']}: Completed in {result['elapsed']}s") + elif status == 'failed': + failed += 1 + print(f"โŒ User {user['user_num']}: Failed - {result.get('error', 'Unknown')}") + else: + timed_out += 1 + print(f"โฑ๏ธ User {user['user_num']}: Timed out") + + # Summary + print("\n" + "=" * 60) + print("SUMMARY") + print("=" * 60) + print(f"Total users: {len(all_users)}") + print(f"Successful: {successful}") + print(f"Failed: {failed}") + print(f"Timed out: {timed_out}") + if successful > 0: + print(f"Average completion time: {total_time/successful:.1f}s") + + # Verify job details + print("\n๐Ÿ“Š Detailed Results:") + db = Database() + for user in all_users: + job = db.jobs.find_by_id(user['job_id']) + if job['status'] == 'completed': + report_size = 0 + if job.get('report_payload'): + report_data = job['report_payload'] + if isinstance(report_data, dict): + report_size = len(report_data.get('content', '')) + else: + report_size = len(str(report_data)) + + charts_payload = job.get('charts_payload') + num_charts = len(charts_payload) if charts_payload else 0 + has_retirement = job.get('retirement_payload') is not None + + print(f" User {user['user_num']}: Report {report_size:,} chars, {num_charts} charts, Retirement: {has_retirement}") + + # Cleanup + print("\n๐Ÿงน Cleaning up test data...") + for user in all_users: + # Delete positions + for account_id in user['account_ids']: + db.execute_raw( + "DELETE FROM positions WHERE account_id = :account_id::uuid", + [{"name": "account_id", "value": {"stringValue": account_id}}] + ) + + # Delete accounts + db.execute_raw( + "DELETE FROM accounts WHERE clerk_user_id = :user_id", + [{"name": "user_id", "value": {"stringValue": user['user_id']}}] + ) + + # Delete jobs + db.execute_raw( + "DELETE FROM jobs WHERE clerk_user_id = :user_id", + [{"name": "user_id", "value": {"stringValue": user['user_id']}}] + ) + + # Delete user + db.execute_raw( + "DELETE FROM users WHERE clerk_user_id = :user_id", + [{"name": "user_id", "value": {"stringValue": user['user_id']}}] + ) + + print("Cleanup completed") + + # Final result + if successful == len(all_users): + print("\nโœ… PHASE 6.6 TEST PASSED: All users processed successfully") + return True + else: + print(f"\nโŒ PHASE 6.6 TEST FAILED: {failed + timed_out} users did not complete") + return False + +async def main(): + """Main entry point""" + try: + success = await run_scale_test() + exit(0 if success else 1) + except Exception as e: + print(f"\nโŒ ERROR during test: {e}") + import traceback + traceback.print_exc() + exit(1) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/gcp-deployment/backend/test_simple.py b/gcp-deployment/backend/test_simple.py new file mode 100644 index 00000000..40da5ced --- /dev/null +++ b/gcp-deployment/backend/test_simple.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Test all agents by running their individual test_simple.py files in their own directories. +This ensures each agent runs with its own dependencies and environment. +""" + +import os +import subprocess +import sys +from pathlib import Path + +def run_command(cmd, cwd): + """Run a command and capture output.""" + print(f"Running in {cwd}: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + return result.returncode == 0, result.stdout, result.stderr + +def test_agent(agent_name, test_file="test_simple.py"): + """Test an individual agent in its directory.""" + backend_dir = Path(__file__).parent + agent_dir = backend_dir / agent_name + + if not agent_dir.exists(): + print(f" โŒ {agent_name}: Directory not found") + return False + + test_path = agent_dir / test_file + if not test_path.exists(): + print(f" โš ๏ธ {agent_name}: No {test_file} found, skipping") + return True # Not a failure, just skip + + # Set environment for mocked lambdas + env = os.environ.copy() + env['MOCK_LAMBDAS'] = 'true' + + # Run the test with uv + success, stdout, stderr = run_command( + ['uv', 'run', test_file], + cwd=str(agent_dir) + ) + + if success: + print(f" โœ… {agent_name}: Test passed") + if stdout and "Status Code: 200" in stdout: + # Extract key info from successful runs + for line in stdout.split('\n'): + if 'Tagged:' in line or 'Success:' in line or 'Message:' in line: + print(f" {line.strip()}") + else: + print(f" โŒ {agent_name}: Test failed") + if stderr: + # Show first error line + error_lines = [l for l in stderr.split('\n') if l.strip()] + if error_lines: + print(f" Error: {error_lines[0][:100]}") + + return success + +def main(): + """Run all agent tests.""" + print("="*60) + print("TESTING ALL AGENTS") + print("Running individual test_simple.py in each agent directory") + print("="*60) + + # List of agents to test + agents = [ + 'tagger', + 'reporter', + 'charter', + 'retirement', + 'planner' + ] + + results = {} + + for agent in agents: + print(f"\n{agent.upper()} Agent:") + results[agent] = test_agent(agent) + + # Summary + print("\n" + "="*60) + print("TEST SUMMARY") + print("="*60) + + passed = sum(1 for r in results.values() if r) + failed = sum(1 for r in results.values() if not r) + + print(f"Passed: {passed}/{len(agents)}") + print(f"Failed: {failed}/{len(agents)}") + + if failed > 0: + print("\nFailed agents:") + for agent, success in results.items(): + if not success: + print(f" - {agent}") + + print("="*60) + + if failed > 0: + print("\nโš ๏ธ SOME TESTS FAILED") + sys.exit(1) + else: + print("\nโœ… ALL TESTS PASSED!") + sys.exit(0) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/backend/watch_agents.py b/gcp-deployment/backend/watch_agents.py new file mode 100644 index 00000000..12356810 --- /dev/null +++ b/gcp-deployment/backend/watch_agents.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Watch CloudWatch logs from all Alex agents in real-time. +Polls all 5 agent log groups simultaneously and displays output with color coding. +""" + +import boto3 +import time +import sys +from datetime import datetime, timedelta +from typing import Dict, List, Optional +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed + +# ANSI color codes for terminal output +COLORS = { + 'PLANNER': '\033[94m', # Blue + 'TAGGER': '\033[93m', # Yellow + 'REPORTER': '\033[92m', # Green + 'CHARTER': '\033[96m', # Cyan + 'RETIREMENT': '\033[95m', # Magenta + 'ERROR': '\033[91m', # Red + 'LANGFUSE': '\033[35m', # Purple (for LangFuse-related logs) + 'RESET': '\033[0m', # Reset to default + 'BOLD': '\033[1m', # Bold text +} + +# Agent log groups +LOG_GROUPS = { + 'PLANNER': '/aws/lambda/alex-planner', + 'TAGGER': '/aws/lambda/alex-tagger', + 'REPORTER': '/aws/lambda/alex-reporter', + 'CHARTER': '/aws/lambda/alex-charter', + 'RETIREMENT': '/aws/lambda/alex-retirement', +} + + +class AgentLogWatcher: + """Watches CloudWatch logs for all agents.""" + + def __init__(self, region: str = 'us-east-1', lookback_minutes: int = 5): + """Initialize the log watcher.""" + self.logs_client = boto3.client('logs', region_name=region) + self.lookback_minutes = lookback_minutes + self.last_timestamps = {agent: 0 for agent in LOG_GROUPS} + + def get_log_events(self, agent: str, start_time: int) -> List[Dict]: + """Get log events for a specific agent.""" + log_group = LOG_GROUPS[agent] + + try: + # Get all log streams in the log group + response = self.logs_client.describe_log_streams( + logGroupName=log_group, + orderBy='LastEventTime', + descending=True, + limit=5 # Get the 5 most recent streams + ) + + if not response.get('logStreams'): + return [] + + # Collect events from all recent streams + all_events = [] + for stream in response['logStreams']: + stream_name = stream['logStreamName'] + + # Get events from this stream + try: + events_response = self.logs_client.filter_log_events( + logGroupName=log_group, + logStreamNames=[stream_name], + startTime=start_time, + limit=100 + ) + + events = events_response.get('events', []) + all_events.extend(events) + + except Exception as e: + # Stream might have been deleted or have no events + continue + + # Sort events by timestamp + all_events.sort(key=lambda x: x['timestamp']) + + # Update last timestamp for this agent + if all_events: + self.last_timestamps[agent] = all_events[-1]['timestamp'] + 1 + + return all_events + + except self.logs_client.exceptions.ResourceNotFoundException: + print(f"{COLORS['ERROR']}Log group {log_group} not found{COLORS['RESET']}") + return [] + except Exception as e: + print(f"{COLORS['ERROR']}Error fetching logs for {agent}: {e}{COLORS['RESET']}") + return [] + + def format_message(self, agent: str, event: Dict) -> str: + """Format a log message with color coding.""" + timestamp = datetime.fromtimestamp(event['timestamp'] / 1000).strftime('%H:%M:%S.%f')[:-3] + message = event['message'].rstrip() + + # Color the agent name + agent_color = COLORS[agent] + agent_label = f"{agent_color}[{agent:10}]{COLORS['RESET']}" + + # Highlight specific message types + if 'ERROR' in message or 'Exception' in message: + message_color = COLORS['ERROR'] + elif 'LangFuse' in message or 'Observability' in message: + message_color = COLORS['LANGFUSE'] + else: + message_color = '' + + if message_color: + message = f"{message_color}{message}{COLORS['RESET']}" + + return f"{timestamp} {agent_label} {message}" + + def poll_agent(self, agent: str, start_time: int) -> List[str]: + """Poll a single agent for new log events.""" + events = self.get_log_events(agent, start_time) + formatted_messages = [] + + for event in events: + formatted_messages.append(self.format_message(agent, event)) + + return formatted_messages + + def watch(self, poll_interval: int = 2): + """Watch all agent logs continuously.""" + print(f"{COLORS['BOLD']}Watching CloudWatch logs for all Alex agents...{COLORS['RESET']}") + print(f"Looking back {self.lookback_minutes} minutes initially") + print(f"Polling every {poll_interval} seconds") + print(f"Press Ctrl+C to stop\n") + + # Initial start time (lookback period) + initial_start = int((datetime.now() - timedelta(minutes=self.lookback_minutes)).timestamp() * 1000) + + # Set initial timestamps + for agent in LOG_GROUPS: + self.last_timestamps[agent] = initial_start + + try: + while True: + # Poll all agents in parallel + with ThreadPoolExecutor(max_workers=5) as executor: + futures = { + executor.submit(self.poll_agent, agent, self.last_timestamps[agent]): agent + for agent in LOG_GROUPS + } + + # Collect and display results + all_messages = [] + for future in as_completed(futures): + messages = future.result() + all_messages.extend(messages) + + # Sort messages by timestamp and display + all_messages.sort() + for message in all_messages: + print(message) + + # Wait before next poll + time.sleep(poll_interval) + + except KeyboardInterrupt: + print(f"\n{COLORS['BOLD']}Stopped watching logs{COLORS['RESET']}") + sys.exit(0) + except Exception as e: + print(f"{COLORS['ERROR']}Error: {e}{COLORS['RESET']}") + sys.exit(1) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description='Watch CloudWatch logs from all Alex agents') + parser.add_argument( + '--region', + default='us-east-1', + help='AWS region (default: us-east-1)' + ) + parser.add_argument( + '--lookback', + type=int, + default=5, + help='Minutes to look back initially (default: 5)' + ) + parser.add_argument( + '--interval', + type=int, + default=2, + help='Polling interval in seconds (default: 2)' + ) + + args = parser.parse_args() + + watcher = AgentLogWatcher(region=args.region, lookback_minutes=args.lookback) + watcher.watch(poll_interval=args.interval) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/gcp-deployment/frontend/.gitignore b/gcp-deployment/frontend/.gitignore new file mode 100644 index 00000000..5ef6a520 --- /dev/null +++ b/gcp-deployment/frontend/.gitignore @@ -0,0 +1,41 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/gcp-deployment/frontend/Dockerfile b/gcp-deployment/frontend/Dockerfile new file mode 100644 index 00000000..c284356b --- /dev/null +++ b/gcp-deployment/frontend/Dockerfile @@ -0,0 +1,34 @@ +FROM node:20-alpine AS builder +WORKDIR /app + +# Copy package files +COPY package*.json ./ +RUN npm ci + +# Copy source +COPY . . + + # Build NextJS app + # Build arguments for Next.js public env vars + ARG NEXT_PUBLIC_API_URL + ARG NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY + ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} + ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY} + RUN npm run build + +# Production stage +FROM node:20-alpine AS runner +WORKDIR /app + +ENV NODE_ENV=production +ENV PORT=8080 + +# Copy built application +COPY --from=builder /app/public ./public +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/.next/static ./.next/static + +EXPOSE 8080 + +CMD ["node", "server.js"] + diff --git a/gcp-deployment/frontend/README.md b/gcp-deployment/frontend/README.md new file mode 100644 index 00000000..ef0e47e3 --- /dev/null +++ b/gcp-deployment/frontend/README.md @@ -0,0 +1,40 @@ +This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/pages/api-reference/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file. + +[API routes](https://nextjs.org/docs/pages/building-your-application/routing/api-routes) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`. + +The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/pages/building-your-application/routing/api-routes) instead of React pages. + +This project uses [`next/font`](https://nextjs.org/docs/pages/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn-pages-router) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/pages/building-your-application/deploying) for more details. diff --git a/gcp-deployment/frontend/components/ConfirmModal.tsx b/gcp-deployment/frontend/components/ConfirmModal.tsx new file mode 100644 index 00000000..3d230559 --- /dev/null +++ b/gcp-deployment/frontend/components/ConfirmModal.tsx @@ -0,0 +1,58 @@ +import { ReactNode } from 'react'; + +interface ConfirmModalProps { + isOpen: boolean; + title: string; + message: string | ReactNode; + confirmText?: string; + cancelText?: string; + confirmButtonClass?: string; + onConfirm: () => void; + onCancel: () => void; + isProcessing?: boolean; +} + +export default function ConfirmModal({ + isOpen, + title, + message, + confirmText = 'Confirm', + cancelText = 'Cancel', + confirmButtonClass = 'bg-red-600 hover:bg-red-700', + onConfirm, + onCancel, + isProcessing = false, +}: ConfirmModalProps) { + if (!isOpen) return null; + + return ( +
+
+
+

{title}

+
+ +
+ {message} +
+ +
+ + +
+
+
+ ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/components/ErrorBoundary.tsx b/gcp-deployment/frontend/components/ErrorBoundary.tsx new file mode 100644 index 00000000..9a4963f6 --- /dev/null +++ b/gcp-deployment/frontend/components/ErrorBoundary.tsx @@ -0,0 +1,60 @@ +import React, { Component, ErrorInfo, ReactNode } from 'react'; +import Link from 'next/link'; + +interface Props { + children: ReactNode; +} + +interface State { + hasError: boolean; + error: Error | null; +} + +export default class ErrorBoundary extends Component { + public state: State = { + hasError: false, + error: null + }; + + public static getDerivedStateFromError(error: Error): State { + return { hasError: true, error }; + } + + public componentDidCatch(error: Error, errorInfo: ErrorInfo) { + console.error('Error boundary caught:', error, errorInfo); + } + + private handleReset = () => { + this.setState({ hasError: false, error: null }); + window.location.href = '/dashboard'; + }; + + public render() { + if (this.state.hasError) { + return ( +
+
+

Something went wrong

+

+ An unexpected error occurred. The error has been logged and we'll look into it. +

+ {this.state.error && ( +
+ Error details + 
{this.state.error.toString()}
+
+ )} + +
+
+ ); + } + + return this.props.children; + } +} \ No newline at end of file diff --git a/gcp-deployment/frontend/components/Layout.tsx b/gcp-deployment/frontend/components/Layout.tsx new file mode 100644 index 00000000..e9786172 --- /dev/null +++ b/gcp-deployment/frontend/components/Layout.tsx @@ -0,0 +1,168 @@ +import { useUser, UserButton, Protect } from "@clerk/nextjs"; +import Link from "next/link"; +import { useRouter } from "next/router"; +import { ReactNode } from "react"; +import PageTransition from "./PageTransition"; + +interface LayoutProps { + children: ReactNode; +} + +export default function Layout({ children }: LayoutProps) { + const { user } = useUser(); + const router = useRouter(); + + // Helper to determine if a link is active + const isActive = (path: string) => router.pathname === path; + + return ( + +
+

Redirecting to sign in...

+
+ + }> +
+ {/* Navigation */} + + + {/* Main Content */} +
+ + {children} + +
+ + {/* Footer */} +
+
+
+

+ Important Disclaimer +

+

+ This AI-generated advice has not been vetted by a qualified financial advisor and should not be used for trading decisions. + For informational purposes only. Always consult with a licensed financial professional before making investment decisions. +

+
+
+

+ ยฉ 2025 Alex AI Financial Advisor. Powered by AI agents and built with care. +

+
+
+
+
+ + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/components/PageTransition.tsx b/gcp-deployment/frontend/components/PageTransition.tsx new file mode 100644 index 00000000..eb77dc99 --- /dev/null +++ b/gcp-deployment/frontend/components/PageTransition.tsx @@ -0,0 +1,28 @@ +import { useRouter } from 'next/router'; +import { useEffect, useState, ReactNode } from 'react'; + +export default function PageTransition({ children }: { children: ReactNode }) { + const router = useRouter(); + const [isTransitioning, setIsTransitioning] = useState(false); + + useEffect(() => { + const handleStart = () => setIsTransitioning(true); + const handleComplete = () => setIsTransitioning(false); + + router.events.on('routeChangeStart', handleStart); + router.events.on('routeChangeComplete', handleComplete); + router.events.on('routeChangeError', handleComplete); + + return () => { + router.events.off('routeChangeStart', handleStart); + router.events.off('routeChangeComplete', handleComplete); + router.events.off('routeChangeError', handleComplete); + }; + }, [router]); + + return ( +
+ {children} +
+ ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/components/Skeleton.tsx b/gcp-deployment/frontend/components/Skeleton.tsx new file mode 100644 index 00000000..6d2cb241 --- /dev/null +++ b/gcp-deployment/frontend/components/Skeleton.tsx @@ -0,0 +1,34 @@ +export const Skeleton = ({ className = "" }: { className?: string }) => ( +
+); + +export const SkeletonText = ({ lines = 1 }: { lines?: number }) => ( +
+ {Array.from({ length: lines }).map((_, i) => ( + + ))} +
+); + +export const SkeletonCard = () => ( +
+ + +
+); + +export const SkeletonTable = ({ rows = 3 }: { rows?: number }) => ( +
+
+ +
+ {Array.from({ length: rows }).map((_, i) => ( +
+ + + + +
+ ))} +
+); \ No newline at end of file diff --git a/gcp-deployment/frontend/components/Toast.tsx b/gcp-deployment/frontend/components/Toast.tsx new file mode 100644 index 00000000..17dcc508 --- /dev/null +++ b/gcp-deployment/frontend/components/Toast.tsx @@ -0,0 +1,86 @@ +import { useEffect, useState } from 'react'; + +export interface ToastMessage { + id: string; + type: 'success' | 'error' | 'info'; + message: string; + duration?: number; +} + +interface ToastProps { + toast: ToastMessage; + onClose: (id: string) => void; +} + +const Toast = ({ toast, onClose }: ToastProps) => { + useEffect(() => { + const timer = setTimeout(() => { + onClose(toast.id); + }, toast.duration || 3000); + + return () => clearTimeout(timer); + }, [toast, onClose]); + + const bgColor = { + success: 'bg-green-500', + error: 'bg-red-500', + info: 'bg-blue-500' + }[toast.type]; + + const icon = { + success: 'โœ“', + error: 'โœ•', + info: 'โ„น' + }[toast.type]; + + return ( +
+ {icon} +

{toast.message}

+ +
+ ); +}; + +export const ToastContainer = () => { + const [toasts, setToasts] = useState([]); + + useEffect(() => { + const handleToast = (event: CustomEvent>) => { + const newToast: ToastMessage = { + ...event.detail, + id: Date.now().toString() + }; + setToasts(prev => [...prev, newToast]); + }; + + window.addEventListener('toast', handleToast as EventListener); + return () => window.removeEventListener('toast', handleToast as EventListener); + }, []); + + const removeToast = (id: string) => { + setToasts(prev => prev.filter(t => t.id !== id)); + }; + + if (toasts.length === 0) return null; + + return ( +
+ {toasts.map(toast => ( + + ))} +
+ ); +}; + +// Helper function to show toast +export const showToast = (type: 'success' | 'error' | 'info', message: string, duration?: number) => { + window.dispatchEvent(new CustomEvent('toast', { + detail: { type, message, duration } + })); +}; \ No newline at end of file diff --git a/gcp-deployment/frontend/eslint.config.mjs b/gcp-deployment/frontend/eslint.config.mjs new file mode 100644 index 00000000..719cea2b --- /dev/null +++ b/gcp-deployment/frontend/eslint.config.mjs @@ -0,0 +1,25 @@ +import { dirname } from "path"; +import { fileURLToPath } from "url"; +import { FlatCompat } from "@eslint/eslintrc"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const compat = new FlatCompat({ + baseDirectory: __dirname, +}); + +const eslintConfig = [ + ...compat.extends("next/core-web-vitals", "next/typescript"), + { + ignores: [ + "node_modules/**", + ".next/**", + "out/**", + "build/**", + "next-env.d.ts", + ], + }, +]; + +export default eslintConfig; diff --git a/gcp-deployment/frontend/next.config.ts b/gcp-deployment/frontend/next.config.ts new file mode 100644 index 00000000..57b3e23e --- /dev/null +++ b/gcp-deployment/frontend/next.config.ts @@ -0,0 +1,13 @@ +import type { NextConfig } from "next"; + +const nextConfig: NextConfig = { + reactStrictMode: true, + output: 'standalone', // Changed from 'export' for Cloud Run + images: { + unoptimized: true + }, + // Disable automatic trailing slash redirect for API routes + trailingSlash: false, +}; + +export default nextConfig; diff --git a/gcp-deployment/frontend/package.json b/gcp-deployment/frontend/package.json new file mode 100644 index 00000000..f20ef299 --- /dev/null +++ b/gcp-deployment/frontend/package.json @@ -0,0 +1,33 @@ +{ + "name": "frontend", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "eslint" + }, + "dependencies": { + "@clerk/nextjs": "^6.32.0", + "@microsoft/fetch-event-source": "^2.0.1", + "next": "15.5.3", + "react": "19.1.0", + "react-dom": "19.1.0", + "react-markdown": "^10.1.0", + "recharts": "^3.2.0", + "remark-breaks": "^4.0.0", + "remark-gfm": "^4.0.1" + }, + "devDependencies": { + "@eslint/eslintrc": "^3", + "@tailwindcss/postcss": "^4", + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "eslint": "^9", + "eslint-config-next": "15.5.3", + "tailwindcss": "^4", + "typescript": "^5" + } +} diff --git a/gcp-deployment/frontend/pages/404.tsx b/gcp-deployment/frontend/pages/404.tsx new file mode 100644 index 00000000..b4845321 --- /dev/null +++ b/gcp-deployment/frontend/pages/404.tsx @@ -0,0 +1,26 @@ +import Link from 'next/link'; +import Head from 'next/head'; + +export default function Custom404() { + return ( + <> + + 404 - Page Not Found | Alex AI Financial Advisor + +
+
+

404

+

Page Not Found

+

+ The page you're looking for doesn't exist or has been moved. +

+ + + +
+
+ + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/500.tsx b/gcp-deployment/frontend/pages/500.tsx new file mode 100644 index 00000000..68e507a3 --- /dev/null +++ b/gcp-deployment/frontend/pages/500.tsx @@ -0,0 +1,26 @@ +import Link from 'next/link'; +import Head from 'next/head'; + +export default function Custom500() { + return ( + <> + + 500 - Server Error | Alex AI Financial Advisor + +
+
+

500

+

Internal Server Error

+

+ Something went wrong on our end. Please try again later. +

+ + + +
+
+ + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/_app.tsx b/gcp-deployment/frontend/pages/_app.tsx new file mode 100644 index 00000000..9cce692d --- /dev/null +++ b/gcp-deployment/frontend/pages/_app.tsx @@ -0,0 +1,16 @@ +import "@/styles/globals.css"; +import type { AppProps } from "next/app"; +import { ClerkProvider } from "@clerk/nextjs"; +import { ToastContainer } from "@/components/Toast"; +import ErrorBoundary from "@/components/ErrorBoundary"; + +export default function App({ Component, pageProps }: AppProps) { + return ( + + + + + + + ); +} diff --git a/gcp-deployment/frontend/pages/_document.tsx b/gcp-deployment/frontend/pages/_document.tsx new file mode 100644 index 00000000..5724928f --- /dev/null +++ b/gcp-deployment/frontend/pages/_document.tsx @@ -0,0 +1,20 @@ +import { Html, Head, Main, NextScript } from "next/document"; + +export default function Document() { + return ( + + + + + + + + + + +
+ + + + ); +} diff --git a/gcp-deployment/frontend/pages/accounts.tsx b/gcp-deployment/frontend/pages/accounts.tsx new file mode 100644 index 00000000..111568a7 --- /dev/null +++ b/gcp-deployment/frontend/pages/accounts.tsx @@ -0,0 +1,560 @@ +import { useAuth } from "@clerk/nextjs"; +import { useState, useEffect, useCallback } from "react"; +import { useRouter } from "next/router"; +import Layout from "../components/Layout"; +import ConfirmModal from "../components/ConfirmModal"; +import { API_URL } from "../lib/config"; +import { SkeletonTable } from "../components/Skeleton"; +import Head from "next/head"; + +interface Position { + id: string; + symbol: string; + quantity: number; + current_price?: number; +} + +interface Account { + id: string; + account_name: string; + account_purpose: string; + cash_balance: number; + positions?: Position[]; +} + +export default function Accounts() { + const { getToken } = useAuth(); + const router = useRouter(); + const [accounts, setAccounts] = useState([]); + const [loading, setLoading] = useState(true); + const [populatingData, setPopulatingData] = useState(false); + const [resettingAccounts, setResettingAccounts] = useState(false); + const [message, setMessage] = useState<{ type: 'success' | 'error', text: string } | null>(null); + const [showAddModal, setShowAddModal] = useState(false); + const [newAccount, setNewAccount] = useState({ name: '', purpose: '', cash_balance: '' }); + const [savingAccount, setSavingAccount] = useState(false); + const [deletingAccountId, setDeletingAccountId] = useState(null); + const [confirmModal, setConfirmModal] = useState<{ + isOpen: boolean; + type: 'reset' | 'delete'; + accountId?: string; + accountName?: string; + }>({ isOpen: false, type: 'reset' }); + + const loadAccounts = useCallback(async () => { + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/accounts`, { + headers: { + 'Authorization': `Bearer ${token}`, + }, + }); + + if (response.ok) { + const data = await response.json(); + console.log('Accounts received from API:', data); + // For each account, load positions + const accountsWithPositions = await Promise.all( + data.map(async (account: Account) => { + console.log('Processing account:', account.id, account.account_name); + // Skip if account has no ID + if (!account.id) { + console.warn('Account missing ID:', account); + return { ...account, positions: [] }; + } + + try { + const positionsResponse = await fetch( + `${API_URL}/api/accounts/${account.id}/positions`, + { + headers: { + 'Authorization': `Bearer ${token}`, + }, + } + ); + if (positionsResponse.ok) { + const data = await positionsResponse.json(); + const positions = data.positions || []; + console.log(`Loaded ${positions.length} positions for account ${account.id}`); + return { ...account, positions }; + } + } catch (err) { + console.error(`Error loading positions for account ${account.id}:`, err); + } + return { ...account, positions: [] }; + }) + ); + console.log('Final accounts with positions:', accountsWithPositions); + setAccounts(accountsWithPositions); + } + } catch (error) { + console.error('Error loading accounts:', error); + setMessage({ type: 'error', text: 'Failed to load accounts' }); + } finally { + setLoading(false); + } + }, [getToken]); + + useEffect(() => { + loadAccounts(); + }, [loadAccounts]); + + // Listen for analysis completion events to refresh data + useEffect(() => { + const handleAnalysisCompleted = () => { + // Refresh accounts to get updated prices after analysis + console.log('Analysis completed - refreshing accounts...'); + loadAccounts(); + }; + + // Listen for the completion event + window.addEventListener('analysis:completed', handleAnalysisCompleted); + + return () => { + window.removeEventListener('analysis:completed', handleAnalysisCompleted); + }; + }, [loadAccounts]); + + const populateTestData = async () => { + setPopulatingData(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/populate-test-data`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + }); + + if (response.ok) { + const data = await response.json(); + setMessage({ type: 'success', text: data.message }); + await loadAccounts(); // Reload accounts after population + } else { + setMessage({ type: 'error', text: 'Failed to populate test data' }); + } + } catch (error) { + console.error('Error populating test data:', error); + setMessage({ type: 'error', text: 'Error populating test data' }); + } finally { + setPopulatingData(false); + } + }; + + const resetAccounts = async () => { + setResettingAccounts(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/reset-accounts`, { + method: 'DELETE', + headers: { + 'Authorization': `Bearer ${token}`, + }, + }); + + if (response.ok) { + const data = await response.json(); + setMessage({ type: 'success', text: data.message }); + // Clear accounts immediately after successful reset + setAccounts([]); + // Then reload to confirm empty state + await loadAccounts(); + } else { + setMessage({ type: 'error', text: 'Failed to reset accounts' }); + } + } catch (error) { + console.error('Error resetting accounts:', error); + setMessage({ type: 'error', text: 'Error resetting accounts' }); + } finally { + setResettingAccounts(false); + } + }; + + const calculateAccountTotal = (account: Account) => { + const positionsValue = account.positions?.reduce((sum, position) => { + const value = Number(position.quantity) * (Number(position.current_price) || 0); + return sum + value; + }, 0) || 0; + return Number(account.cash_balance) + positionsValue; + }; + + const calculatePortfolioTotal = () => { + return accounts.reduce((sum, account) => sum + calculateAccountTotal(account), 0); + }; + + const handleAddAccount = async () => { + if (!newAccount.name.trim()) { + setMessage({ type: 'error', text: 'Please enter an account name' }); + return; + } + + setSavingAccount(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/accounts`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + account_name: newAccount.name, + account_purpose: newAccount.purpose || 'Investment Account', + cash_balance: parseFloat(newAccount.cash_balance.replace(/,/g, '')) || 0, + }), + }); + + if (response.ok) { + setMessage({ type: 'success', text: 'Account created successfully' }); + setShowAddModal(false); + setNewAccount({ name: '', purpose: '', cash_balance: '' }); + await loadAccounts(); + } else { + const error = await response.json(); + setMessage({ type: 'error', text: error.detail || 'Failed to create account' }); + } + } catch (error) { + console.error('Error creating account:', error); + setMessage({ type: 'error', text: 'Error creating account' }); + } finally { + setSavingAccount(false); + } + }; + + const handleDeleteAccount = async (accountId: string) => { + setDeletingAccountId(accountId); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/accounts/${accountId}`, { + method: 'DELETE', + headers: { + 'Authorization': `Bearer ${token}`, + }, + }); + + if (response.ok) { + setMessage({ type: 'success', text: 'Account deleted successfully' }); + await loadAccounts(); + } else { + setMessage({ type: 'error', text: 'Failed to delete account' }); + } + } catch (error) { + console.error('Error deleting account:', error); + setMessage({ type: 'error', text: 'Error deleting account' }); + } finally { + setDeletingAccountId(null); + } + }; + + const formatCurrencyInput = (value: string) => { + // Remove non-numeric characters except decimal + const cleaned = value.replace(/[^0-9.]/g, ''); + // Format with commas + const parts = cleaned.split('.'); + parts[0] = parts[0].replace(/\B(?=(\d{3})+(?!\d))/g, ','); + return parts.join('.'); + }; + + return ( + <> + + Accounts - Alex AI Financial Advisor + + +
+
+
+
+

Investment Accounts

+

Manage your investment accounts and portfolios

+
+
+ + {accounts.length === 0 && !loading && ( + + )} + {accounts.length > 0 && ( + + )} +
+
+ + {message && ( +
+ {message.text} +
+ )} + + {loading ? ( + + ) : accounts.length === 0 ? ( +
+

+ No accounts found +

+

+ Click the "Populate Test Data" button above to create sample accounts with positions +

+
+ ) : ( + <> + {/* Portfolio Summary */} +
+
+
+

Total Portfolio Value

+

+ ${calculatePortfolioTotal().toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} +

+
+
+

Number of Accounts

+

{accounts.length}

+
+
+

Total Positions

+

+ {accounts.reduce((sum, acc) => sum + (acc.positions?.length || 0), 0)} +

+
+
+
+ + {/* Accounts Table */} +
+ + + + + + + + + + + + + {accounts.map((account) => { + const positionsValue = calculateAccountTotal(account) - Number(account.cash_balance); + return ( + + + + + + + + + ); + })} + +
Account NameTypePositionsCashTotal ValueActions
+
+

{account.account_name}

+

{account.account_purpose}

+
+ 		+ {account.account_purpose} + +
+

{account.positions?.length || 0}

+ {positionsValue > 0 && ( +

+ ${positionsValue.toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })} +

+ )} +
+ 		+ ${Number(account.cash_balance).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} + +

+ ${calculateAccountTotal(account).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} +

+ 		+
+ + +
+
+
+ + )} +
+ + {/* Add Account Modal */} + {showAddModal && ( +
+
+

Add New Account

+ +
+
+ + setNewAccount({ ...newAccount, name: e.target.value })} + className="w-full border border-gray-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + placeholder="e.g., 401k, Roth IRA, Brokerage" + /> +
+ +
+ + setNewAccount({ ...newAccount, purpose: e.target.value })} + className="w-full border border-gray-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + placeholder="e.g., Long-term Growth, Retirement" + /> +
+ +
+ +
+ $ + setNewAccount({ ...newAccount, cash_balance: formatCurrencyInput(e.target.value) })} + className="w-full border border-gray-300 rounded-lg pl-8 pr-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + placeholder="0.00" + /> +
+
+
+ + {message && message.type === 'error' && ( +
+ {message.text} +
+ )} + +
+ + +
+
+
+ )} + + {/* Confirmation Modal */} + +

Are you sure you want to delete all your accounts?

+

This will permanently remove:

+
    +
  • All {accounts.length} account{accounts.length !== 1 ? 's' : ''}
    • +
  • All positions in those accounts
    • +
  • All transaction history
    • +
+

This action cannot be undone.

+
+ ) : ( +
+

Are you sure you want to delete “{confirmModal.accountName}”?

+

This will also delete all positions in this account.

+

This action cannot be undone.

+
+ ) + } + confirmText={confirmModal.type === 'reset' ? 'Delete All Accounts' : 'Delete Account'} + cancelText="Cancel" + confirmButtonClass="bg-red-600 hover:bg-red-700" + onConfirm={() => { + if (confirmModal.type === 'reset') { + resetAccounts(); + } else if (confirmModal.accountId) { + handleDeleteAccount(confirmModal.accountId); + } + setConfirmModal({ isOpen: false, type: 'reset' }); + }} + onCancel={() => setConfirmModal({ isOpen: false, type: 'reset' })} + isProcessing={resettingAccounts || deletingAccountId !== null} + /> +
+ + + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/accounts/[id].tsx b/gcp-deployment/frontend/pages/accounts/[id].tsx new file mode 100644 index 00000000..9926438d --- /dev/null +++ b/gcp-deployment/frontend/pages/accounts/[id].tsx @@ -0,0 +1,705 @@ +import { useAuth } from "@clerk/nextjs"; +import { useState, useEffect, useCallback } from "react"; +import { useRouter } from "next/router"; +import Layout from "../../components/Layout"; +import ConfirmModal from "../../components/ConfirmModal"; +import { API_URL } from "../../lib/config"; + +interface Instrument { + symbol: string; + name: string; + instrument_type: string; + current_price: number; +} + +interface Position { + id: string; + symbol: string; + quantity: number; + current_price?: number; +} + +interface Account { + id: string; + account_name: string; + account_purpose: string; + cash_balance: number; + positions?: Position[]; +} + +export default function AccountDetail() { + const { getToken } = useAuth(); + const router = useRouter(); + const { id } = router.query; + const [account, setAccount] = useState(null); + const [positions, setPositions] = useState([]); + const [instruments, setInstruments] = useState([]); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [message, setMessage] = useState<{ type: 'success' | 'error', text: string } | null>(null); + const [editingAccount, setEditingAccount] = useState(false); + const [editedAccount, setEditedAccount] = useState({ name: '', purpose: '', cash_balance: '' }); + const [editingPosition, setEditingPosition] = useState(null); + const [editedQuantity, setEditedQuantity] = useState(''); + const [showAddPosition, setShowAddPosition] = useState(false); + const [newPosition, setNewPosition] = useState({ symbol: '', quantity: '' }); + const [searchTerm, setSearchTerm] = useState(''); + const [showSymbolSuggestions, setShowSymbolSuggestions] = useState(false); + const [confirmModal, setConfirmModal] = useState<{ + isOpen: boolean; + positionId: string; + symbol: string; + }>({ isOpen: false, positionId: '', symbol: '' }); + + const loadAccount = useCallback(async () => { + if (!id) return; + + try { + const token = await getToken(); + + // Load account details + const accountResponse = await fetch(`${API_URL}/api/accounts`, { + headers: { + 'Authorization': `Bearer ${token}`, + }, + }); + + if (accountResponse.ok) { + const accounts = await accountResponse.json(); + const foundAccount = accounts.find((acc: Account) => acc.id === id); + + if (foundAccount) { + setAccount(foundAccount); + setEditedAccount({ + name: foundAccount.account_name, + purpose: foundAccount.account_purpose, + cash_balance: Number(foundAccount.cash_balance).toLocaleString('en-US'), + }); + } else { + setMessage({ type: 'error', text: 'Account not found' }); + setTimeout(() => router.push('/accounts'), 2000); + return; + } + } + + // Load positions + const positionsResponse = await fetch( + `${API_URL}/api/accounts/${id}/positions`, + { + headers: { + 'Authorization': `Bearer ${token}`, + }, + } + ); + + if (positionsResponse.ok) { + const data = await positionsResponse.json(); + setPositions(data.positions || []); + } + + // Load instruments for autocomplete + const instrumentsResponse = await fetch( + `${API_URL}/api/instruments`, + { + headers: { + 'Authorization': `Bearer ${token}`, + }, + } + ); + + if (instrumentsResponse.ok) { + const instrumentsData = await instrumentsResponse.json(); + setInstruments(instrumentsData); + } + + } catch (error) { + console.error('Error loading account:', error); + setMessage({ type: 'error', text: 'Failed to load account details' }); + } finally { + setLoading(false); + } + }, [id, getToken, router]); + + useEffect(() => { + loadAccount(); + }, [loadAccount]); + + const handleSaveAccount = async () => { + setSaving(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/accounts/${id}`, { + method: 'PUT', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + account_name: editedAccount.name, + account_purpose: editedAccount.purpose, + cash_balance: parseFloat(editedAccount.cash_balance.replace(/,/g, '')), + }), + }); + + if (response.ok) { + const updatedAccount = await response.json(); + setAccount(updatedAccount); + setEditingAccount(false); + setMessage({ type: 'success', text: 'Account updated successfully' }); + } else { + setMessage({ type: 'error', text: 'Failed to update account' }); + } + } catch (error) { + console.error('Error updating account:', error); + setMessage({ type: 'error', text: 'Error updating account' }); + } finally { + setSaving(false); + } + }; + + const handleUpdatePosition = async (positionId: string) => { + const quantity = parseFloat(editedQuantity); + if (isNaN(quantity) || quantity < 0) { + setMessage({ type: 'error', text: 'Please enter a valid quantity' }); + return; + } + + setSaving(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/positions/${positionId}`, { + method: 'PUT', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + quantity: quantity, + }), + }); + + if (response.ok) { + setMessage({ type: 'success', text: 'Position updated successfully' }); + setEditingPosition(null); + await loadAccount(); + } else { + setMessage({ type: 'error', text: 'Failed to update position' }); + } + } catch (error) { + console.error('Error updating position:', error); + setMessage({ type: 'error', text: 'Error updating position' }); + } finally { + setSaving(false); + } + }; + + const handleDeletePosition = async (positionId: string) => { + setSaving(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/positions/${positionId}`, { + method: 'DELETE', + headers: { + 'Authorization': `Bearer ${token}`, + }, + }); + + if (response.ok) { + setMessage({ type: 'success', text: 'Position deleted successfully' }); + await loadAccount(); + } else { + setMessage({ type: 'error', text: 'Failed to delete position' }); + } + } catch (error) { + console.error('Error deleting position:', error); + setMessage({ type: 'error', text: 'Error deleting position' }); + } finally { + setSaving(false); + } + }; + + const handleAddPosition = async () => { + if (!newPosition.symbol.trim() || !newPosition.quantity.trim()) { + setMessage({ type: 'error', text: 'Please enter symbol and quantity' }); + return; + } + + const quantity = parseFloat(newPosition.quantity); + if (isNaN(quantity) || quantity <= 0) { + setMessage({ type: 'error', text: 'Please enter a valid quantity' }); + return; + } + + setSaving(true); + setMessage(null); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/positions`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + account_id: id, + symbol: newPosition.symbol.toUpperCase(), + quantity: quantity, + }), + }); + + if (response.ok) { + setMessage({ type: 'success', text: 'Position added successfully' }); + setShowAddPosition(false); + setNewPosition({ symbol: '', quantity: '' }); + setSearchTerm(''); + await loadAccount(); + } else { + const error = await response.json(); + setMessage({ type: 'error', text: error.detail || 'Failed to add position' }); + } + } catch (error) { + console.error('Error adding position:', error); + setMessage({ type: 'error', text: 'Error adding position' }); + } finally { + setSaving(false); + } + }; + + const formatCurrencyInput = (value: string) => { + const cleaned = value.replace(/[^0-9.]/g, ''); + const parts = cleaned.split('.'); + parts[0] = parts[0].replace(/\B(?=(\d{3})+(?!\d))/g, ','); + return parts.join('.'); + }; + + const calculatePositionsValue = () => { + return positions.reduce((sum, position) => { + return sum + (Number(position.quantity) * (position.current_price || 0)); + }, 0); + }; + + const calculateTotalValue = () => { + return (account ? Number(account.cash_balance) : 0) + calculatePositionsValue(); + }; + + const filteredInstruments = instruments.filter(inst => + inst.symbol.toLowerCase().includes(searchTerm.toLowerCase()) || + inst.name.toLowerCase().includes(searchTerm.toLowerCase()) + ).slice(0, 5); + + if (loading) { + return ( + +
+
+

Loading account details...

+
+
+ + ); + } + + if (!account) { + return ( + +
+
+

Account not found

+
+
+ + ); + } + + return ( + +
+ {/* Breadcrumb */} +
+ +
+ + {/* Account Details */} +
+
+
+ {editingAccount ? ( +
+
+ + setEditedAccount({ ...editedAccount, name: e.target.value })} + className="w-full border border-gray-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + setEditedAccount({ ...editedAccount, purpose: e.target.value })} + className="w-full border border-gray-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ +
+ $ + setEditedAccount({ ...editedAccount, cash_balance: formatCurrencyInput(e.target.value) })} + className="w-full border border-gray-300 rounded-lg pl-8 pr-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+
+ + +
+
+ ) : ( + <> +

{account.account_name}

+

{account.account_purpose}

+ + )} +
+ {!editingAccount && ( + + )} +
+ + {message && ( +
+ {message.text} +
+ )} + + {/* Account Summary */} +
+
+

Cash Balance

+

+ ${Number(account.cash_balance).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} +

+
+
+

Positions Value

+

+ ${calculatePositionsValue().toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} +

+
+
+

Total Value

+

+ ${calculateTotalValue().toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} +

+
+
+

Positions

+

{positions.length}

+
+
+
+ + {/* Positions */} +
+
+

Positions

+ +
+ + {positions.length === 0 ? ( +
+

No positions in this account yet

+

Click “Add Position” to start building your portfolio

+
+ ) : ( +
+ + + + + + + + + + + + {positions.map((position) => ( + + + + + + + + ))} + +
SymbolQuantityPriceValueActions
{position.symbol} + {editingPosition === position.id ? ( + setEditedQuantity(e.target.value)} + className="w-24 border border-gray-300 rounded px-2 py-1 text-right focus:outline-none focus:ring-2 focus:ring-primary" + step="0.01" + min="0" + /> + ) : ( + Number(position.quantity).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 2 }) + )} + + ${position.current_price?.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }) || 'N/A'} + + ${((position.current_price || 0) * Number(position.quantity)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} + +
+ {editingPosition === position.id ? ( + <> + + + + ) : ( + <> + + + + )} +
+
+
+ )} +
+ + {/* Add Position Modal */} + {showAddPosition && ( +
+
+

Add New Position

+ +
+
+ +
+ { + const value = e.target.value.toUpperCase(); + setSearchTerm(value); + setNewPosition({ ...newPosition, symbol: value }); + setShowSymbolSuggestions(value.length > 0); + }} + onFocus={() => setShowSymbolSuggestions(searchTerm.length > 0)} + className="w-full border border-gray-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary uppercase" + placeholder="Enter ticker symbol (e.g., SPY, AAPL)" + /> + + {showSymbolSuggestions && filteredInstruments.length > 0 && ( +
+ {filteredInstruments.map((inst) => ( + + ))} +
+ )} +
+

+ If the symbol is not in our database, it will be added automatically +

+
+ +
+ + setNewPosition({ ...newPosition, quantity: e.target.value })} + className="w-full border border-gray-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-primary" + placeholder="0" + step="0.01" + min="0" + /> +
+
+ + {message && message.type === 'error' && ( +
+ {message.text} +
+ )} + +
+ + +
+
+
+ )} + + {/* Delete Position Confirmation Modal */} + +

Are you sure you want to delete your {confirmModal.symbol} position?

+

This will remove this holding from your account.

+

This action cannot be undone.

+
+ } + confirmText="Delete Position" + cancelText="Cancel" + confirmButtonClass="bg-red-600 hover:bg-red-700" + onConfirm={() => { + handleDeletePosition(confirmModal.positionId); + setConfirmModal({ isOpen: false, positionId: '', symbol: '' }); + }} + onCancel={() => setConfirmModal({ isOpen: false, positionId: '', symbol: '' })} + isProcessing={saving} + /> + + + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/advisor-team.tsx b/gcp-deployment/frontend/pages/advisor-team.tsx new file mode 100644 index 00000000..7dc01600 --- /dev/null +++ b/gcp-deployment/frontend/pages/advisor-team.tsx @@ -0,0 +1,419 @@ +import { useState, useEffect } from 'react'; +import { useRouter } from 'next/router'; +import { useAuth } from '@clerk/nextjs'; +import Layout from '../components/Layout'; +import { API_URL } from '../lib/config'; +import { emitAnalysisCompleted, emitAnalysisFailed, emitAnalysisStarted } from '../lib/events'; +import Head from 'next/head'; + +interface Agent { + icon: string; + name: string; + role: string; + description: string; + color: string; + bgColor: string; +} + +interface Job { + id: string; + created_at: string; + status: string; + job_type: string; +} + +interface AnalysisProgress { + stage: 'idle' | 'starting' | 'planner' | 'parallel' | 'completing' | 'complete' | 'error'; + message: string; + activeAgents: string[]; + error?: string; +} + +const agents: Agent[] = [ + { + icon: '๐ŸŽฏ', + name: 'Financial Planner', + role: 'Orchestrator', + description: 'Coordinates your financial analysis', + color: 'text-ai-accent', + bgColor: 'bg-ai-accent' + }, + { + icon: '๐Ÿ“Š', + name: 'Portfolio Analyst', + role: 'Reporter', + description: 'Analyzes your holdings and performance', + color: 'text-primary', + bgColor: 'bg-primary' + }, + { + icon: '๐Ÿ“ˆ', + name: 'Chart Specialist', + role: 'Charter', + description: 'Visualizes your portfolio composition', + color: 'text-green-600', + bgColor: 'bg-green-600' + }, + { + icon: '๐ŸŽฏ', + name: 'Retirement Planner', + role: 'Retirement', + description: 'Projects your retirement readiness', + color: 'text-accent', + bgColor: 'bg-accent' + } +]; + +export default function AdvisorTeam() { + const router = useRouter(); + const { getToken } = useAuth(); + const [jobs, setJobs] = useState([]); + const [isAnalyzing, setIsAnalyzing] = useState(false); + const [currentJobId, setCurrentJobId] = useState(null); + const [progress, setProgress] = useState({ + stage: 'idle', + message: '', + activeAgents: [] + }); + const [pollInterval, setPollInterval] = useState(null); + + useEffect(() => { + fetchJobs(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + const checkJobStatusLocal = async (jobId: string) => { + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/jobs/${jobId}`, { + headers: { + 'Authorization': `Bearer ${token}` + } + }); + + if (response.ok) { + const job = await response.json(); + + if (job.status === 'completed') { + setProgress({ + stage: 'complete', + message: 'Analysis complete!', + activeAgents: [] + }); + + if (pollInterval) { + clearInterval(pollInterval); + setPollInterval(null); + } + + // Emit completion event so other components can refresh + emitAnalysisCompleted(jobId); + + // Also refresh our own jobs list + fetchJobs(); + + setTimeout(() => { + router.push(`/analysis?job_id=${jobId}`); + }, 1500); + } else if (job.status === 'failed') { + setProgress({ + stage: 'error', + message: 'Analysis failed', + activeAgents: [], + error: job.error || 'Analysis encountered an error' + }); + + if (pollInterval) { + clearInterval(pollInterval); + setPollInterval(null); + } + + // Emit failure event + emitAnalysisFailed(jobId, job.error); + + setIsAnalyzing(false); + setCurrentJobId(null); + } + } + } catch (error) { + console.error('Error checking job status:', error); + } + }; + + if (currentJobId && !pollInterval) { + const interval = setInterval(() => { + checkJobStatusLocal(currentJobId); + }, 2000); + setPollInterval(interval); + } + + return () => { + if (pollInterval) { + clearInterval(pollInterval); + setPollInterval(null); + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentJobId, pollInterval, router]); + + const fetchJobs = async () => { + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/jobs`, { + headers: { + 'Authorization': `Bearer ${token}` + } + }); + + if (response.ok) { + const data = await response.json(); + setJobs(data.jobs || []); + } + } catch (error) { + console.error('Error fetching jobs:', error); + } + }; + + const startAnalysis = async () => { + setIsAnalyzing(true); + setProgress({ + stage: 'starting', + message: 'Initializing analysis...', + activeAgents: [] + }); + + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/analyze`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }, + body: JSON.stringify({ + analysis_type: 'portfolio', + options: {} + }) + }); + + if (response.ok) { + const data = await response.json(); + setCurrentJobId(data.job_id); + + // Emit start event + emitAnalysisStarted(data.job_id); + + setProgress({ + stage: 'planner', + message: 'Financial Planner coordinating analysis...', + activeAgents: ['Financial Planner'] + }); + + setTimeout(() => { + setProgress({ + stage: 'parallel', + message: 'Agents working in parallel...', + activeAgents: ['Portfolio Analyst', 'Chart Specialist', 'Retirement Planner'] + }); + }, 5000); + } else { + throw new Error('Failed to start analysis'); + } + } catch (error) { + console.error('Error starting analysis:', error); + setProgress({ + stage: 'error', + message: 'Failed to start analysis', + activeAgents: [], + error: error instanceof Error ? error.message : 'Unknown error' + }); + setIsAnalyzing(false); + setCurrentJobId(null); + } + }; + + + const formatDate = (dateString: string) => { + return new Date(dateString).toLocaleString('en-US', { + month: 'short', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit' + }); + }; + + const getStatusColor = (status: string) => { + switch (status) { + case 'completed': + return 'text-green-600'; + case 'failed': + return 'text-red-500'; + case 'running': + return 'text-blue-600'; + default: + return 'text-gray-500'; + } + }; + + const isAgentActive = (agentName: string) => { + return progress.activeAgents.includes(agentName); + }; + + return ( + <> + + Advisor Team - Alex AI Financial Advisor + + +
+
+
+

Your AI Advisory Team

+

+ Meet your team of specialized AI agents that work together to provide comprehensive financial analysis. +

+
+ +
+ {agents.map((agent) => ( +
+ {isAgentActive(agent.name) && ( +
+ )} +
+
{agent.icon}
+

+ {agent.name} +

+

{agent.role}

+

{agent.description}

+ {isAgentActive(agent.name) && ( +
+ โ— + Active +
+ )} +
+
+ ))} +
+ +
+
+

Analysis Center

+ +
+ + {isAnalyzing && ( +
+
+

Analysis Progress

+ {progress.stage !== 'error' && progress.stage !== 'complete' && ( +
+
+
+
+
+ )} +
+ +

+ {progress.message} +

+ + {progress.stage === 'error' && progress.error && ( +
+

{progress.error}

+ +
+ )} + + {progress.stage !== 'idle' && progress.stage !== 'error' && ( +
+
+
+ )} +
+ )} + +
+

Previous Analyses

+ {jobs.length === 0 ? ( +

No previous analyses found. Start your first analysis above!

+ ) : ( +
+ {jobs.slice(0, 5).map((job) => ( +
+
+

+ Analysis #{job.id.slice(0, 8)} +

+

+ {formatDate(job.created_at)} +

+
+
+ + {job.status.charAt(0).toUpperCase() + job.status.slice(1)} + + {job.status === 'completed' && ( + + )} +
+
+ ))} +
+ )} +
+
+
+
+ + + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/analysis.tsx b/gcp-deployment/frontend/pages/analysis.tsx new file mode 100644 index 00000000..2c08198d --- /dev/null +++ b/gcp-deployment/frontend/pages/analysis.tsx @@ -0,0 +1,559 @@ +import { useState, useEffect } from 'react'; +import { useRouter } from 'next/router'; +import { useAuth } from '@clerk/nextjs'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import remarkBreaks from 'remark-breaks'; +import { + PieChart, Pie, Cell, BarChart, Bar, LineChart, Line, + XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer +} from 'recharts'; +import Layout from '../components/Layout'; +import { API_URL } from '../lib/config'; +import Head from 'next/head'; + +interface Job { + id: string; + created_at: string; + status: string; + job_type: string; + report_payload?: { + agent: string; + content: string; + generated_at: string; + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + charts_payload?: Record | null; // Charter stores charts with dynamic keys + retirement_payload?: { + agent: string; + analysis: string; + generated_at: string; + }; + error_message?: string; +} + +interface JobListItem { + id: string; + created_at: string; + status: string; + job_type: string; +} + +type TabType = 'overview' | 'charts' | 'retirement'; + +// Color palette for charts +const COLORS = [ + '#209DD7', // primary + '#753991', // AI accent + '#FFB707', // accent + '#062147', // dark + '#60A5FA', // light blue + '#A78BFA', // light purple + '#FBBF24', // yellow + '#34D399', // green + '#F87171', // red + '#94A3B8', // gray +]; + +export default function Analysis() { + const router = useRouter(); + const { getToken } = useAuth(); + const { job_id } = router.query; + const [job, setJob] = useState(null); + const [loading, setLoading] = useState(true); + const [activeTab, setActiveTab] = useState('overview'); + const [fetchingLatest, setFetchingLatest] = useState(false); + + useEffect(() => { + const loadJob = async (jobId: string) => { + try { + const token = await getToken(); + const response = await fetch(`${API_URL}/api/jobs/${jobId}`, { + headers: { + 'Authorization': `Bearer ${token}` + } + }); + + if (response.ok) { + const jobData = await response.json(); + setJob(jobData); + } else { + console.error('Failed to fetch job'); + } + } catch (error) { + console.error('Error fetching job:', error); + } finally { + setLoading(false); + } + }; + + const loadLatestJob = async () => { + setFetchingLatest(true); + try { + const token = await getToken(); + // First, get the list of jobs to find the latest completed one + const response = await fetch(`${API_URL}/api/jobs`, { + headers: { + 'Authorization': `Bearer ${token}` + } + }); + + if (response.ok) { + const data = await response.json(); + const jobs: JobListItem[] = data.jobs || []; + // Find the latest completed job + const latestCompletedJob = jobs + .filter(j => j.status === 'completed') + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())[0]; + + if (latestCompletedJob) { + // Load the full job details + await loadJob(latestCompletedJob.id); + // Update the URL to include the job_id without causing a page reload + router.replace(`/analysis?job_id=${latestCompletedJob.id}`, undefined, { shallow: true }); + } else { + setLoading(false); + } + } else { + setLoading(false); + } + } catch (error) { + console.error('Error fetching latest job:', error); + setLoading(false); + } finally { + setFetchingLatest(false); + } + }; + + if (job_id) { + loadJob(job_id as string); + } else if (router.isReady) { + // Router is ready but no job_id provided - fetch the latest analysis + loadLatestJob(); + } + }, [job_id, router.isReady, getToken, router]); + + + const formatDate = (dateString: string) => { + return new Date(dateString).toLocaleString('en-US', { + month: 'long', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit' + }); + }; + + if (loading) { + return ( + +
+
+
+
+
+
+
+
+
+
+ + ); + } + + if (!job) { + return ( + +
+
+
+

+ {fetchingLatest ? 'Loading Latest Analysis...' : 'No Analysis Available'} +

+

+ {fetchingLatest + ? 'Please wait while we load your latest analysis.' + : 'You have not completed any analyses yet. Start a new analysis to see results here.'} +

+ {!fetchingLatest && ( + + )} +
+
+
+ + ); + } + + if (job.status === 'running' || job.status === 'pending') { + return ( + +
+
+
+

Analysis In Progress

+

Your analysis is still being processed. Please check back in a few moments.

+
+
+
+
+
+ +
+
+
+ + ); + } + + if (job.status === 'failed') { + return ( + +
+
+
+

Analysis Failed

+

The analysis encountered an error and could not be completed.

+ {job.error_message && ( +
+

{job.error_message}

+
+ )} + +
+
+
+ + ); + } + + + // Tab content renderers + const renderOverview = () => { + const report = job?.report_payload?.content; + if (!report) { + return ( +
+ No portfolio report available. +
+ ); + } + + return ( +
+

{children}

, + h2: ({children}) =>

{children}

, + h3: ({children}) =>

{children}

, + ul: ({children}) =>
    {children}
, + ol: ({children}) =>
    {children}
, + li: ({children}) =>
  • {children}
    • , + p: ({children}) =>

    {children}

    , + table: ({children}) => ( +
    + {children}
    +
    + ), + thead: ({children}) => {children}, + th: ({children}) => {children}, + td: ({children}) => {children}, + strong: ({children}) => {children}, + blockquote: ({children}) => ( +
    + {children} +
    + ), + }} + > + {report} +     +
    + ); + }; + + const renderCharts = () => { + const chartsPayload = job?.charts_payload; + if (!chartsPayload || Object.keys(chartsPayload).length === 0) { + return ( +
    + No chart data available. +
    + ); + } + + // Helper function to format chart title from key + const formatTitle = (key: string): string => { + return key + .split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + }; + + // Helper function to determine chart type based on data structure or chart metadata + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const getChartType = (chartData: any): 'pie' | 'donut' | 'bar' | 'horizontalBar' | 'line' => { + // If the charter agent specifies a type, use it directly if supported + if (chartData.type) { + const supportedTypes = ['pie', 'donut', 'bar', 'horizontalBar', 'line']; + if (supportedTypes.includes(chartData.type)) { + return chartData.type; + } + // Map variations to supported types + const typeMap: Record = { + 'column': 'bar', + 'area': 'line' + }; + if (typeMap[chartData.type]) { + return typeMap[chartData.type]; + } + } + + // Otherwise, make an intelligent guess based on the data + // If data has dates/time series, use line chart + if (chartData.data?.[0]?.date || chartData.data?.[0]?.year) return 'line'; + + // If data represents parts of a whole (has percentages or small dataset), use pie + if (chartData.data?.length <= 10 && chartData.data?.[0]?.value) return 'pie'; + + // Default to bar chart for other cases + return 'bar'; + }; + + // Dynamically render all charts provided by the charter agent + const chartEntries = Object.entries(chartsPayload); + + return ( +
    + {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */} + {chartEntries.map(([key, chartData]: [string, any]) => { + // Skip if no data + if (!chartData?.data || chartData.data.length === 0) return null; + + const chartType = getChartType(chartData); + const title = chartData.title || formatTitle(key); + + return ( +
    +

    {title}

    + + {chartType === 'pie' || chartType === 'donut' ? ( + + + {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */} + {chartData.data.map((entry: any, idx: number) => ( + + ))} + + `$${value.toLocaleString('en-US')}`} /> + + ) : chartType === 'horizontalBar' ? ( + // For horizontal bars, just use regular vertical bars with rotated labels + // Recharts horizontal layout can be problematic + + + + `$${(value/1000).toFixed(0)}k`} + /> + `$${value.toLocaleString('en-US')}`} /> + + {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */} + {chartData.data?.map((entry: any, index: number) => ( + + ))} + + + ) : chartType === 'bar' ? ( + + + + `$${(value/1000).toFixed(0)}k`} /> + `$${value.toLocaleString('en-US')}`} /> + + + ) : ( + // Line chart for time series data + + + + `$${(value/1000).toFixed(0)}k`} /> + `$${value.toLocaleString('en-US')}`} /> + + + )} + + + {/* Add legend for pie/donut charts with many items */} + {(chartType === 'pie' || chartType === 'donut') && chartData.data.length > 6 && ( +
    + {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */} + {chartData.data.map((entry: any, idx: number) => ( +
    +
    + {entry.name} +
    + ))} +
    + )} +
    + ); + })} +
    + ); + }; + + const renderRetirement = () => { + const retirement = job?.retirement_payload; + if (!retirement) { + return ( +
    + No retirement projection available. +
    + ); + } + + // Backend provides 'analysis' as markdown text + const retirementAnalysis = retirement.analysis; + + return ( +
    + {/* Analysis Section */} + {retirementAnalysis && ( +
    +
    +

    {children}

    , + h3: ({children}) =>

    {children}

    , + p: ({children}) =>

    {children}

    , + strong: ({children}) => {children}, + ul: ({children}) =>
      {children}
    , + li: ({children}) =>
  • {children}
    • , + }} + > + {retirementAnalysis} +     +
    +
    + )} + +
    + ); + }; + + return ( + <> + + Analysis - Alex AI Financial Advisor + + +
    +
    + {/* Header */} +
    +
    +
    +

    Portfolio Analysis Results

    +

    + Completed on {formatDate(job.created_at)} +

    +
    + +
    +
    + + {/* Tabs */} +
    +
    + +
    +
    + + {/* Tab Content */} +
    + {activeTab === 'overview' && renderOverview()} + {activeTab === 'charts' && renderCharts()} + {activeTab === 'retirement' && renderRetirement()} +
    +
    +
    +     + + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/dashboard.tsx b/gcp-deployment/frontend/pages/dashboard.tsx new file mode 100644 index 00000000..ba8c361f --- /dev/null +++ b/gcp-deployment/frontend/pages/dashboard.tsx @@ -0,0 +1,660 @@ +import { useUser, useAuth } from "@clerk/nextjs"; +import { useEffect, useState, useCallback } from "react"; +import { API_URL } from "../lib/config"; +import Layout from "../components/Layout"; +import { PieChart, Pie, Cell, ResponsiveContainer, Tooltip, Legend } from "recharts"; +import { Skeleton, SkeletonCard } from "../components/Skeleton"; +import { showToast } from "../components/Toast"; +import Head from "next/head"; + +interface UserData { + clerk_user_id: string; + display_name: string; + years_until_retirement: number; + target_retirement_income: number; + asset_class_targets: Record; + region_targets: Record; +} + +interface Account { + account_id: string; + clerk_user_id: string; + account_name: string; + account_type: string; + account_purpose: string; + cash_balance: number; + created_at: string; + updated_at: string; +} + +interface Position { + position_id: string; + account_id: string; + symbol: string; + quantity: number; + created_at: string; + updated_at: string; +} + +interface Instrument { + symbol: string; + name: string; + instrument_type: string; + current_price?: number; + asset_class_allocation?: Record; + region_allocation?: Record; + sector_allocation?: Record; +} + +export default function Dashboard() { + const { user, isLoaded: userLoaded } = useUser(); + const { getToken } = useAuth(); + const [userData, setUserData] = useState(null); + const [accounts, setAccounts] = useState([]); + const [positions, setPositions] = useState>({}); + const [instruments, setInstruments] = useState>({}); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + const [lastAnalysisDate, setLastAnalysisDate] = useState(null); + + // Form state for editable fields - start empty to avoid flicker + const [displayName, setDisplayName] = useState(""); + const [yearsUntilRetirement, setYearsUntilRetirement] = useState(0); + const [targetRetirementIncome, setTargetRetirementIncome] = useState(0); + const [equityTarget, setEquityTarget] = useState(0); + const [fixedIncomeTarget, setFixedIncomeTarget] = useState(0); + const [northAmericaTarget, setNorthAmericaTarget] = useState(0); + const [internationalTarget, setInternationalTarget] = useState(0); + + // Calculate portfolio summary + const calculatePortfolioSummary = useCallback(() => { + let totalValue = 0; + const assetClassBreakdown: Record = { + equity: 0, + fixed_income: 0, + alternatives: 0, + cash: 0 + }; + + // Add cash balances + accounts.forEach(account => { + const cashBalance = Number(account.cash_balance); + totalValue += cashBalance; + assetClassBreakdown.cash += cashBalance; + }); + + // Add position values + Object.entries(positions).forEach(([, accountPositions]) => { + accountPositions.forEach(position => { + const instrument = instruments[position.symbol]; + if (instrument?.current_price) { + const positionValue = Number(position.quantity) * Number(instrument.current_price); + totalValue += positionValue; + + // Add to asset class breakdown + if (instrument.asset_class_allocation) { + Object.entries(instrument.asset_class_allocation).forEach(([assetClass, percentage]) => { + assetClassBreakdown[assetClass] = (assetClassBreakdown[assetClass] || 0) + (positionValue * percentage / 100); + }); + } + } + }); + }); + + return { totalValue, assetClassBreakdown }; + }, [accounts, positions, instruments]); + + // Load user data and accounts + useEffect(() => { + async function loadData() { + if (!userLoaded || !user) return; + + try { + const token = await getToken(); + if (!token) { + setError("Not authenticated"); + setLoading(false); + return; + } + + // Get/create user + const userResponse = await fetch(`${API_URL}/api/user`, { + headers: { + "Authorization": `Bearer ${token}`, + }, + }); + + if (!userResponse.ok) { + throw new Error(`Failed to sync user: ${userResponse.status}`); + } + + const response = await userResponse.json(); + const userData = response.user; // Extract user from response + setUserData(userData); + setDisplayName(userData.display_name || ""); + setYearsUntilRetirement(userData.years_until_retirement || 0); + // Ensure target_retirement_income is a number + const income = userData.target_retirement_income + ? (typeof userData.target_retirement_income === 'string' + ? parseFloat(userData.target_retirement_income) + : userData.target_retirement_income) + : 0; + setTargetRetirementIncome(income); + setEquityTarget(userData.asset_class_targets?.equity || 0); + setFixedIncomeTarget(userData.asset_class_targets?.fixed_income || 0); + setNorthAmericaTarget(userData.region_targets?.north_america || 0); + setInternationalTarget(userData.region_targets?.international || 0); + + // Get accounts + const accountsResponse = await fetch(`${API_URL}/api/accounts`, { + headers: { + "Authorization": `Bearer ${token}`, + }, + }); + + if (accountsResponse.ok) { + const accountsData = await accountsResponse.json(); + setAccounts(accountsData); + + // Get positions for each account + const positionsMap: Record = {}; + const instrumentsMap: Record = {}; + + for (const account of accountsData) { + // Skip if account has no ID + if (!account.id) { + console.warn('Account missing ID in dashboard:', account); + continue; + } + + const positionsResponse = await fetch(`${API_URL}/api/accounts/${account.id}/positions`, { + headers: { + "Authorization": `Bearer ${token}`, + }, + }); + + if (positionsResponse.ok) { + const positionsData = await positionsResponse.json(); + // API returns positions in a positions key + positionsMap[account.id] = positionsData.positions || []; + + // Store instrument data from each position + for (const position of positionsData.positions || []) { + if (position.instrument) { + instrumentsMap[position.symbol] = position.instrument as Instrument; + } + } + } + } + + setPositions(positionsMap); + setInstruments(instrumentsMap); + } + + // Get last analysis date + // This would come from the jobs endpoint in a real implementation + setLastAnalysisDate(null); + + } catch (err) { + console.error("Error loading data:", err); + setError(err instanceof Error ? err.message : "Failed to load data"); + } finally { + setLoading(false); + } + } + + loadData(); + }, [userLoaded, user, getToken]); + + // Listen for analysis completion events to refresh data + useEffect(() => { + if (!userLoaded || !user) return; + + const handleAnalysisCompleted = async () => { + try { + const token = await getToken(); + if (!token) return; + + console.log('Analysis completed - refreshing dashboard data...'); + + // Refresh accounts to get latest prices + const accountsResponse = await fetch(`${API_URL}/api/accounts`, { + headers: { + "Authorization": `Bearer ${token}`, + }, + }); + + if (accountsResponse.ok) { + const accountsData = await accountsResponse.json(); + setAccounts(accountsData.accounts || []); + + // Load positions for each account + const positionsData: Record = {}; + const instrumentsData: Record = {}; + + for (const account of accountsData.accounts || []) { + const positionsResponse = await fetch( + `${API_URL}/api/accounts/${account.id}/positions`, + { + headers: { + "Authorization": `Bearer ${token}`, + }, + } + ); + + if (positionsResponse.ok) { + const data = await positionsResponse.json(); + positionsData[account.id] = data.positions || []; + + // Extract instruments from positions + for (const position of data.positions || []) { + if (position.instrument) { + instrumentsData[position.symbol] = position.instrument; + } + } + } + } + + setPositions(positionsData); + setInstruments(instrumentsData); + + // Portfolio will be recalculated on render + } + } catch (err) { + console.error("Error refreshing dashboard data:", err); + } + }; + + // Listen for the completion event + window.addEventListener('analysis:completed', handleAnalysisCompleted); + + return () => { + window.removeEventListener('analysis:completed', handleAnalysisCompleted); + }; + }, [userLoaded, user, getToken, calculatePortfolioSummary]); + + // Save user settings + const handleSaveSettings = async () => { + if (!userData) return; + + // Input validation + if (!displayName || displayName.trim().length === 0) { + showToast('error', 'Display name is required'); + return; + } + + if (yearsUntilRetirement < 0 || yearsUntilRetirement > 50) { + showToast('error', 'Years until retirement must be between 0 and 50'); + return; + } + + if (targetRetirementIncome < 0) { + showToast('error', 'Target retirement income must be positive'); + return; + } + + // Validate allocation percentages + const equityFixed = equityTarget + fixedIncomeTarget; + if (Math.abs(equityFixed - 100) > 0.01) { + showToast('error', 'Equity and Fixed Income must sum to 100%'); + return; + } + + const regionTotal = northAmericaTarget + internationalTarget; + if (Math.abs(regionTotal - 100) > 0.01) { + showToast('error', 'North America and International must sum to 100%'); + return; + } + + setSaving(true); + setError(null); + + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + + const updateData = { + display_name: displayName.trim(), + years_until_retirement: yearsUntilRetirement, + target_retirement_income: targetRetirementIncome, + asset_class_targets: { + equity: equityTarget, + fixed_income: fixedIncomeTarget + }, + region_targets: { + north_america: northAmericaTarget, + international: internationalTarget + } + }; + + const response = await fetch(`${API_URL}/api/user`, { + method: "PUT", + headers: { + "Authorization": `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(updateData), + }); + + if (!response.ok) { + throw new Error(`Failed to save settings: ${response.status}`); + } + + const updatedUser = await response.json(); + setUserData(updatedUser); + + // Show success toast + showToast('success', 'Settings saved successfully!'); + + } catch (err) { + console.error("Error saving settings:", err); + showToast('error', err instanceof Error ? err.message : "Failed to save settings"); + } finally { + setSaving(false); + } + }; + + const { totalValue, assetClassBreakdown } = calculatePortfolioSummary(); + + // Prepare data for pie chart + const pieChartData = Object.entries(assetClassBreakdown) + .filter(([, value]) => value > 0) + .map(([key, value]) => ({ + name: key.charAt(0).toUpperCase() + key.slice(1).replace('_', ' '), + value: Math.round(value), + percentage: totalValue > 0 ? Math.round(value / totalValue * 100) : 0 + })); + + const COLORS = ['#209DD7', '#753991', '#FFB707', '#062147', '#10B981']; + + return ( + <> + + Dashboard - Alex AI Financial Advisor + + +
    +

    Dashboard

    + + {loading ? ( + // Loading skeleton +
    +
    + {Array.from({ length: 4 }).map((_, i) => ( +
    + + +
    + ))} +
    + + +
    + ) : ( + <> + {/* Portfolio Summary Cards */} +
    +
    +

    Total Portfolio Value

    +

    + ${totalValue % 1 === 0 + ? totalValue.toLocaleString('en-US') + : totalValue.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })} +

    +
    + +
    +

    Number of Accounts

    +

    {accounts.length}

    +
    + +
    +

    Asset Allocation

    + {pieChartData.length > 0 ? ( +
    + + + + {pieChartData.map((entry, index) => ( + + ))} + + `$${value.toLocaleString()}`} /> + + +
    + ) : ( +

    No positions yet

    + )} +
    + +
    +

    Last Analysis

    +

    + {lastAnalysisDate ? new Date(lastAnalysisDate).toLocaleDateString() : "Never"} +

    +
    +
    + + {/* User Settings Section */} +
    +

    User Settings

    + + {loading ? ( +

    Loading...

    + ) : error && !error.includes("success") ? ( +
    +

    {error}

    +
    + ) : error && error.includes("success") ? ( +
    +

    โœ… {error}

    +
    + ) : null} + +
    + {/* Basic Info */} +
    + + setDisplayName(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
    + +
    + + { + // Remove commas and parse as number + const value = e.target.value.replace(/,/g, ''); + const num = parseInt(value) || 0; + if (!isNaN(num)) { + setTargetRetirementIncome(num); + } + }} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
    + + {/* Retirement Slider */} +
    + + setYearsUntilRetirement(Number(e.target.value))} + className="w-full" + /> +
    + 0 + 10 + 20 + 30 + 40 + 50 +
    +
    + + {/* Target Allocations */} +
    +

    Target Asset Class Allocation

    +
    +
    + + { + const val = Number(e.target.value); + setEquityTarget(val); + setFixedIncomeTarget(100 - val); + }} + className="w-full" + /> +
    +
    + + { + const val = Number(e.target.value); + setFixedIncomeTarget(val); + setEquityTarget(100 - val); + }} + className="w-full" + /> +
    +
    + + {/* Mini pie chart for asset allocation */} +
    + + + + + + + `${value}%`} /> + + + +
    +
    + +
    +

    Target Regional Allocation

    +
    +
    + + { + const val = Number(e.target.value); + setNorthAmericaTarget(val); + setInternationalTarget(100 - val); + }} + className="w-full" + /> +
    +
    + + { + const val = Number(e.target.value); + setInternationalTarget(val); + setNorthAmericaTarget(100 - val); + }} + className="w-full" + /> +
    +
    + + {/* Mini pie chart for regional allocation */} +
    + + + + + + + `${value}%`} /> + + + +
    +
    +
    + +
    + +
    +
    + + )} +
    +     + + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/pages/index.tsx b/gcp-deployment/frontend/pages/index.tsx new file mode 100644 index 00000000..6827139e --- /dev/null +++ b/gcp-deployment/frontend/pages/index.tsx @@ -0,0 +1,162 @@ +import { SignInButton, SignUpButton, SignedIn, SignedOut, UserButton } from "@clerk/nextjs"; +import Link from "next/link"; +import Head from "next/head"; + +export default function Home() { + return ( + <> + + Alex AI Financial Advisor - Intelligent Portfolio Management + +
    + {/* Navigation */} + + + {/* Hero Section */} +
    +
    +

    + Your AI-Powered Financial Future +

    +

    + Experience the power of autonomous AI agents working together to analyze your portfolio, + plan your retirement, and optimize your investments. +

    +
    + + + + + + + + + + + +
    +
    +
    + + {/* Features Section */} +
    +
    +

    + Meet Your AI Advisory Team +

    +
    +
    +
    ๐ŸŽฏ
    +

    Financial Planner

    +

    Coordinates your complete financial analysis with intelligent orchestration

    +
    +
    +
    ๐Ÿ“Š
    +

    Portfolio Analyst

    +

    Deep analysis of holdings, performance metrics, and risk assessment

    +
    +
    +
    ๐Ÿ“ˆ
    +

    Chart Specialist

    +

    Visualizes your portfolio composition with interactive charts

    +
    +
    +
    ๐ŸŽฏ
    +

    Retirement Planner

    +

    Projects your retirement readiness with Monte Carlo simulations

    +
    +
    +
    +
    + + {/* Benefits Section */} +
    +
    +

    + Enterprise-Grade AI Advisory +

    +
    +
    +
    โšก
    +

    Real-Time Analysis

    +

    Watch AI agents collaborate in parallel to analyze your complete financial picture

    +
    +
    +
    ๐Ÿ”’
    +

    Bank-Level Security

    +

    Your data is protected with enterprise security and row-level access controls

    +
    +
    +
    ๐Ÿ“Š
    +

    Comprehensive Reports

    +

    Detailed markdown reports with interactive charts and retirement projections

    +
    +
    +
    +
    + + {/* CTA Section */} +
    +
    +

    + Ready to Transform Your Financial Future? +

    +

    + Join thousands of investors using AI to optimize their portfolios +

    + + + +
    +
    + + {/* Footer */} +
    +

    ยฉ 2025 Alex AI Financial Advisor. All rights reserved.

    +

    + This AI-generated advice has not been vetted by a qualified financial advisor and should not be used for trading decisions. + For informational purposes only. +

    +
    +
    + + ); +} \ No newline at end of file diff --git a/gcp-deployment/frontend/postcss.config.mjs b/gcp-deployment/frontend/postcss.config.mjs new file mode 100644 index 00000000..c7bcb4b1 --- /dev/null +++ b/gcp-deployment/frontend/postcss.config.mjs @@ -0,0 +1,5 @@ +const config = { + plugins: ["@tailwindcss/postcss"], +}; + +export default config; diff --git a/gcp-deployment/frontend/public/favicon.ico b/gcp-deployment/frontend/public/favicon.ico new file mode 100644 index 00000000..26dee47d Binary files /dev/null and b/gcp-deployment/frontend/public/favicon.ico differ diff --git a/gcp-deployment/frontend/public/favicon.svg b/gcp-deployment/frontend/public/favicon.svg new file mode 100644 index 00000000..b9e03d33 --- /dev/null +++ b/gcp-deployment/frontend/public/favicon.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/gcp-deployment/frontend/public/file.svg b/gcp-deployment/frontend/public/file.svg new file mode 100644 index 00000000..004145cd --- /dev/null +++ b/gcp-deployment/frontend/public/file.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/gcp-deployment/frontend/public/globe.svg b/gcp-deployment/frontend/public/globe.svg new file mode 100644 index 00000000..567f17b0 --- /dev/null +++ b/gcp-deployment/frontend/public/globe.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/gcp-deployment/frontend/public/next.svg b/gcp-deployment/frontend/public/next.svg new file mode 100644 index 00000000..5174b28c --- /dev/null +++ b/gcp-deployment/frontend/public/next.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/gcp-deployment/frontend/public/vercel.svg b/gcp-deployment/frontend/public/vercel.svg new file mode 100644 index 00000000..77053960 --- /dev/null +++ b/gcp-deployment/frontend/public/vercel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/gcp-deployment/frontend/public/window.svg b/gcp-deployment/frontend/public/window.svg new file mode 100644 index 00000000..b2b2a44f --- /dev/null +++ b/gcp-deployment/frontend/public/window.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/gcp-deployment/frontend/styles/globals.css b/gcp-deployment/frontend/styles/globals.css new file mode 100644 index 00000000..2ff1b9fd --- /dev/null +++ b/gcp-deployment/frontend/styles/globals.css @@ -0,0 +1,77 @@ +@import "tailwindcss"; + +:root { + --background: #ffffff; + --foreground: #171717; +} + +@theme inline { + /* Custom colors for Alex Financial Advisor */ + --color-primary: #209DD7; /* Primary blue */ + --color-ai-accent: #753991; /* Purple for AI/agent features */ + --color-accent: #FFB707; /* Yellow/gold accent */ + --color-dark: #062147; /* Dark navy */ + --color-success: #10b981; /* Green for positive */ + --color-error: #ef4444; /* Red for errors */ + + /* Preserve defaults */ + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); +} + +/* Light mode only - no dark mode for enterprise look */ +body { + background: var(--background); + color: var(--foreground); + font-family: system-ui, -apple-system, sans-serif; +} + +/* Stronger pulsing animation for agent activity */ +@keyframes strong-pulse { + 0%, 100% { + opacity: 1; + transform: scale(1); + } + 50% { + opacity: 0.4; + transform: scale(0.95); + } +} + +.animate-strong-pulse { + animation: strong-pulse 1.5s cubic-bezier(0.4, 0, 0.6, 1) infinite; +} + +/* Even stronger glow pulse for active agents */ +@keyframes glow-pulse { + 0%, 100% { + opacity: 1; + box-shadow: 0 0 20px rgba(117, 57, 145, 0.5), 0 0 40px rgba(117, 57, 145, 0.3); + } + 50% { + opacity: 0.7; + box-shadow: 0 0 30px rgba(117, 57, 145, 0.8), 0 0 60px rgba(117, 57, 145, 0.5); + } +} + +.animate-glow-pulse { + animation: glow-pulse 1.5s ease-in-out infinite; +} + +/* Toast slide-in animation */ +@keyframes slide-in { + from { + transform: translateX(100%); + opacity: 0; + } + to { + transform: translateX(0); + opacity: 1; + } +} + +.animate-slide-in { + animation: slide-in 0.3s ease-out; +} diff --git a/gcp-deployment/frontend/tsconfig.json b/gcp-deployment/frontend/tsconfig.json new file mode 100644 index 00000000..957e71fe --- /dev/null +++ b/gcp-deployment/frontend/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] +} diff --git a/gcp-deployment/guides/0_AWS_TO_GCP_MAPPING.md b/gcp-deployment/guides/0_AWS_TO_GCP_MAPPING.md new file mode 100644 index 00000000..ccf7842b --- /dev/null +++ b/gcp-deployment/guides/0_AWS_TO_GCP_MAPPING.md @@ -0,0 +1,126 @@ +# AWS to GCP Service Mapping Guide + +## Overview + +This guide translates the AWS deployment for the Multi-Agent SaaS App (Alex) to Google Cloud Platform (GCP). The original course deploys using AWS services; this translation provides equivalent GCP services and Terraform configurations. + +## Service Mapping Reference + +| AWS Service | GCP Equivalent | Purpose | +|-------------|----------------|---------| +| **IAM** | Cloud IAM | Identity and Access Management | +| **SageMaker** | Vertex AI | ML model hosting and training | +| **Bedrock** | Vertex AI (Model Garden) | LLM/AI services (Claude, Gemini) | +| **Lambda** | Cloud Functions / Cloud Run | Serverless compute | +| **App Runner** | Cloud Run | Container hosting | +| **ECR** | Artifact Registry | Container registry | +| **RDS (PostgreSQL)** | Cloud SQL | Managed database | +| **S3** | Cloud Storage | Object storage | +| **API Gateway** | API Gateway / Cloud Endpoints | HTTP API management | +| **VPC** | VPC | Virtual Private Cloud | +| **CloudWatch** | Cloud Monitoring & Logging | Observability | +| **Secrets Manager** | Secret Manager | Secrets management | +| **CloudFront** | Cloud CDN | Content delivery | +| **Route 53** | Cloud DNS | DNS management | +| **ECS/Fargate** | Cloud Run / GKE Autopilot | Container orchestration | +| **SQS** | Cloud Pub/Sub | Message queuing | +| **EventBridge** | Cloud Scheduler / Eventarc | Event-driven architecture | + +## Deployment Phase Mapping + +### Week 3 Day 3: 1_permissions and 2_sagemaker +- **AWS**: IAM roles, policies, SageMaker endpoints +- **GCP**: Service accounts, IAM bindings, Vertex AI endpoints + +### Week 3 Day 4: 3_ingest +- **AWS**: S3 buckets, Lambda functions for data ingestion +- **GCP**: Cloud Storage buckets, Cloud Functions + +### Week 3 Day 5: 4_researcher +- **AWS**: Lambda/App Runner for research agents, Bedrock +- **GCP**: Cloud Run services, Vertex AI API + +### Week 4 Day 1: 5_database +- **AWS**: RDS PostgreSQL +- **GCP**: Cloud SQL PostgreSQL + +### Week 4 Day 2: 6_agents +- **AWS**: Lambda functions, Bedrock AgentCore +- **GCP**: Cloud Functions/Run, Vertex AI Agents + +### Week 4 Day 3: 7_frontend +- **AWS**: App Runner, CloudFront, Route 53 +- **GCP**: Cloud Run, Cloud CDN, Cloud DNS + +### Week 4 Day 4: 8_enterprise +- **AWS**: Enterprise features, monitoring, scaling +- **GCP**: Identity Platform, Cloud Monitoring, Autoscaling + +## Key Differences + +### Authentication +- **AWS**: IAM users, roles, access keys +- **GCP**: Service accounts, Workload Identity, OAuth + +### AI/ML Services +- **AWS Bedrock**: Direct API access to Claude, Titan models +- **GCP Vertex AI**: Access to Gemini, Claude (via Model Garden), PaLM + +### Serverless +- **AWS Lambda**: Function-based, cold starts +- **Cloud Functions**: Similar to Lambda (Gen 2 uses Cloud Run under the hood) +- **Cloud Run**: Better for containerized apps, request-based scaling + +### Container Registry +- **AWS ECR**: Docker registry +- **GCP Artifact Registry**: Multi-format (Docker, npm, Python, Maven) + +## Prerequisites for GCP Deployment + +1. **GCP Account** with billing enabled +2. **gcloud CLI** installed and configured +3. **Terraform** >= 1.5.0 +4. **Docker** for container builds +5. **Enable required APIs**: + ```bash + gcloud services enable \ + compute.googleapis.com \ + run.googleapis.com \ + cloudfunctions.googleapis.com \ + sqladmin.googleapis.com \ + aiplatform.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iam.googleapis.com \ + storage.googleapis.com \ + dns.googleapis.com \ + certificatemanager.googleapis.com + ``` + +## Project Structure + +``` +alex-gcp/ +โ”œโ”€โ”€ guides/ +โ”‚ โ”œโ”€โ”€ 0_AWS_TO_GCP_MAPPING.md +โ”‚ โ”œโ”€โ”€ 1_permissions.md +โ”‚ โ”œโ”€โ”€ 2_vertex_ai.md +โ”‚ โ”œโ”€โ”€ 3_ingest.md +โ”‚ โ”œโ”€โ”€ 4_researcher.md +โ”‚ โ”œโ”€โ”€ 5_database.md +โ”‚ โ”œโ”€โ”€ 6_agents.md +โ”‚ โ”œโ”€โ”€ 7_frontend.md +โ”‚ โ””โ”€โ”€ 8_enterprise.md +โ”œโ”€โ”€ terraform/ +โ”‚ โ”œโ”€โ”€ 1_permissions/ +โ”‚ โ”œโ”€โ”€ 2_vertex_ai/ +โ”‚ โ”œโ”€โ”€ 3_ingest/ +โ”‚ โ”œโ”€โ”€ 4_researcher/ +โ”‚ โ”œโ”€โ”€ 5_database/ +โ”‚ โ”œโ”€โ”€ 6_agents/ +โ”‚ โ”œโ”€โ”€ 7_frontend/ +โ”‚ โ””โ”€โ”€ 8_enterprise/ +โ””โ”€โ”€ scripts/ + โ””โ”€โ”€ deploy.sh +``` diff --git a/gcp-deployment/guides/1_permissions.md b/gcp-deployment/guides/1_permissions.md new file mode 100644 index 00000000..fb904314 --- /dev/null +++ b/gcp-deployment/guides/1_permissions.md @@ -0,0 +1,286 @@ +# Phase 1: GCP Permissions Setup + +## Overview + +This guide sets up the foundational IAM permissions on GCP, equivalent to AWS's IAM setup in the original course. + +## AWS vs GCP Comparison + +| AWS Concept | GCP Equivalent | +|-------------|----------------| +| IAM User | Service Account / User | +| IAM Role | IAM Role (predefined or custom) | +| IAM Policy | IAM Policy Binding | +| Trust Policy | Service Account Impersonation | +| Access Keys | Service Account Keys (avoid) / Workload Identity | + +## Steps + +### Step 1: Create a GCP Project + +**For Linux/Mac (Bash):** +```bash +# Set your project ID +export PROJECT_ID="alex-multiagent-saas" +export REGION="us-central1" + +# Create project (or use existing) +gcloud projects create $PROJECT_ID --name="Alex Multi-Agent SaaS" + +# Set as default +gcloud config set project $PROJECT_ID + +# Link billing account +gcloud billing accounts list +gcloud billing projects link $PROJECT_ID --billing-account=YOUR_BILLING_ACCOUNT_ID +``` + +**For Windows (PowerShell):** +```powershell +# Set your project ID +$PROJECT_ID = "alex-multiagent-saas" +$REGION = "us-central1" + +# Create project (or use existing) +gcloud projects create $PROJECT_ID --name="Alex Multi-Agent SaaS" + +# Set as default +gcloud config set project $PROJECT_ID + +# Link billing account +gcloud billing accounts list +gcloud billing projects link $PROJECT_ID --billing-account=YOUR_BILLING_ACCOUNT_ID +``` + +**Note:** If you see a quota project warning, run: +```powershell +gcloud auth application-default set-quota-project $PROJECT_ID +``` + +### Step 2: Enable Required APIs + +**For Linux/Mac (Bash):** +```bash +gcloud services enable \ + compute.googleapis.com \ + run.googleapis.com \ + cloudfunctions.googleapis.com \ + sqladmin.googleapis.com \ + aiplatform.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iam.googleapis.com \ + storage.googleapis.com \ + dns.googleapis.com \ + certificatemanager.googleapis.com \ + cloudbuild.googleapis.com \ + logging.googleapis.com \ + monitoring.googleapis.com +``` + +**For Windows (PowerShell):** +```powershell +gcloud services enable ` + compute.googleapis.com ` + run.googleapis.com ` + cloudfunctions.googleapis.com ` + sqladmin.googleapis.com ` + aiplatform.googleapis.com ` + artifactregistry.googleapis.com ` + secretmanager.googleapis.com ` + cloudresourcemanager.googleapis.com ` + iam.googleapis.com ` + storage.googleapis.com ` + dns.googleapis.com ` + certificatemanager.googleapis.com ` + cloudbuild.googleapis.com ` + logging.googleapis.com ` + monitoring.googleapis.com +``` + +**Alternative (PowerShell - single line):** +```powershell +gcloud services enable compute.googleapis.com run.googleapis.com cloudfunctions.googleapis.com sqladmin.googleapis.com aiplatform.googleapis.com artifactregistry.googleapis.com secretmanager.googleapis.com cloudresourcemanager.googleapis.com iam.googleapis.com storage.googleapis.com dns.googleapis.com certificatemanager.googleapis.com cloudbuild.googleapis.com logging.googleapis.com monitoring.googleapis.com +``` + +### Step 3: Initialize Terraform + +Navigate to `terraform/1_permissions/` and run: + +```bash +cd terraform/1_permissions/ +terraform init +terraform plan +terraform apply +``` + +### Step 4: Verify Setup + +**For Linux/Mac (Bash):** +```bash +# List service accounts +gcloud iam service-accounts list + +# Check IAM bindings +gcloud projects get-iam-policy $PROJECT_ID +``` + +**For Windows (PowerShell):** +```powershell +# List service accounts +gcloud iam service-accounts list + +# Check IAM bindings +gcloud projects get-iam-policy $PROJECT_ID +``` + +**Note:** Make sure `$PROJECT_ID` is set in your current PowerShell session. If you opened a new terminal, set it again with `$PROJECT_ID = "alex-multiagent-saas"`. + +## Service Accounts Created + +| Service Account | Purpose | +|-----------------|---------| +| `vertex-ai-sa` | Vertex AI model access | +| `cloud-run-sa` | Cloud Run service execution | +| `cloud-functions-sa` | Cloud Functions execution | +| `cloud-sql-sa` | Cloud SQL access | +| `storage-sa` | Cloud Storage access | +| `deploy-sa` | CI/CD deployment | + +## Best Practices + +1. **Use Workload Identity** instead of service account keys +2. **Principle of Least Privilege** - grant minimum required permissions +3. **Use predefined roles** where possible +4. **Audit IAM regularly** using Cloud Audit Logs + +## Troubleshooting + +### Permission Denied Errors + +**For Linux/Mac (Bash):** +```bash +# Check current permissions +gcloud auth list +gcloud projects get-iam-policy $PROJECT_ID --format=json | jq '.bindings[] | select(.members[] | contains("YOUR_EMAIL"))' +``` + +**For Windows (PowerShell):** +```powershell +# Check current permissions +gcloud auth list +gcloud projects get-iam-policy $PROJECT_ID --format=json | ConvertFrom-Json | Select-Object -ExpandProperty bindings | Where-Object { $_.members -contains "YOUR_EMAIL" } +``` + +### Application Default Credentials (ADC) Issues + +**Common Error:** `error getting credentials using GOOGLE_APPLICATION_CREDENTIALS environment variable: open : The system cannot find the file specified` + +This occurs when the `GOOGLE_APPLICATION_CREDENTIALS` environment variable points to a non-existent file, preventing Terraform from using the default ADC location. + +**Solution:** + +**For Linux/Mac (Bash):** +```bash +# Unset the environment variable to use default ADC location +unset GOOGLE_APPLICATION_CREDENTIALS + +# Or set it to the default location +export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.config/gcloud/application_default_credentials.json" +``` + +**For Windows (PowerShell):** +```powershell +# Unset the environment variable to use default ADC location +$env:GOOGLE_APPLICATION_CREDENTIALS = $null +# Or use: +Remove-Item Env:GOOGLE_APPLICATION_CREDENTIALS -ErrorAction SilentlyContinue + +# Or set it to the default location +$env:GOOGLE_APPLICATION_CREDENTIALS = "$env:APPDATA\gcloud\application_default_credentials.json" +``` + +**After setting up ADC:** +```powershell +# Login to set up Application Default Credentials +gcloud auth application-default login + +# Set quota project (if you see a warning) +gcloud auth application-default set-quota-project $PROJECT_ID +``` + +### API Not Enabled + +**Common Error:** `Error 403: API has not been used in project before or it is disabled` + +**For Linux/Mac (Bash):** +```bash +# List all enabled APIs +gcloud services list --enabled + +# Check if a specific API is enabled +gcloud services list --enabled --filter="name:artifactregistry.googleapis.com" + +# Enable a specific API +gcloud services enable artifactregistry.googleapis.com --project=$PROJECT_ID + +# Enable all required APIs (if you missed any) +gcloud services enable \ + compute.googleapis.com \ + run.googleapis.com \ + cloudfunctions.googleapis.com \ + sqladmin.googleapis.com \ + aiplatform.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iam.googleapis.com \ + storage.googleapis.com \ + dns.googleapis.com \ + certificatemanager.googleapis.com \ + cloudbuild.googleapis.com \ + logging.googleapis.com \ + monitoring.googleapis.com \ + --project=$PROJECT_ID +``` + +**For Windows (PowerShell):** +```powershell +# List all enabled APIs +gcloud services list --enabled + +# Check if a specific API is enabled +gcloud services list --enabled --filter="name:artifactregistry.googleapis.com" --project=$PROJECT_ID + +# Enable a specific API +gcloud services enable artifactregistry.googleapis.com --project=$PROJECT_ID + +# Enable all required APIs (if you missed any) +gcloud services enable ` + compute.googleapis.com ` + run.googleapis.com ` + cloudfunctions.googleapis.com ` + sqladmin.googleapis.com ` + aiplatform.googleapis.com ` + artifactregistry.googleapis.com ` + secretmanager.googleapis.com ` + cloudresourcemanager.googleapis.com ` + iam.googleapis.com ` + storage.googleapis.com ` + dns.googleapis.com ` + certificatemanager.googleapis.com ` + cloudbuild.googleapis.com ` + logging.googleapis.com ` + monitoring.googleapis.com ` + --project=$PROJECT_ID +``` + +**Important Notes:** +- **API Propagation Delay**: After enabling an API, it may take 2-5 minutes to fully propagate across GCP systems. If you get an error immediately after enabling, wait a few minutes and retry. +- **Verify API Status**: Always verify an API is enabled before running Terraform. The command above will show `STATE: ENABLED` when ready. +- **Artifact Registry API**: This API is commonly missed and required for container image storage. If Terraform fails with "Artifact Registry API has not been used", enable it explicitly and wait 2-3 minutes before retrying. + +## Next Steps + +After completing permissions setup, proceed to [2_vertex_ai.md](2_vertex_ai.md) for Vertex AI setup (equivalent to AWS SageMaker). diff --git a/gcp-deployment/guides/2_vertex_ai.md b/gcp-deployment/guides/2_vertex_ai.md new file mode 100644 index 00000000..801d640c --- /dev/null +++ b/gcp-deployment/guides/2_vertex_ai.md @@ -0,0 +1,407 @@ +# Phase 2: Vertex AI Setup (Equivalent to AWS SageMaker) + +## Overview + +This guide sets up Vertex AI on GCP, which is the equivalent of AWS SageMaker. Vertex AI provides: +- Model training and deployment +- Access to foundation models (Gemini 2.0 Flash recommended, Claude optional) +- MLOps pipelines +- Custom model hosting + +**Recommended Model: Gemini 2.0 Flash** +- **Cost-effective**: Significantly cheaper than Claude models in Vertex AI +- **Fast**: Optimized for speed and efficiency +- **Native GCP integration**: Uses Application Default Credentials (no API keys needed) +- **Full feature support**: Supports all OpenAI Agents SDK features via LiteLLM + +## AWS vs GCP Comparison + +| AWS SageMaker | GCP Vertex AI | +|---------------|---------------| +| SageMaker Endpoints | Vertex AI Endpoints | +| SageMaker Notebooks | Vertex AI Workbench | +| SageMaker Training Jobs | Vertex AI Training | +| Bedrock (LLMs) | Model Garden / Generative AI | +| SageMaker Pipelines | Vertex AI Pipelines | +| SageMaker Feature Store | Vertex AI Feature Store | + +## Model Options on GCP + +**Recommended: Gemini 2.0 Flash (Default)** +- Native Vertex AI integration +- Cost-effective pricing +- No API keys required (uses ADC) +- Full LiteLLM support + +**Alternative: OpenAI API (via Secret Manager)** +- Use OpenAI models (GPT-4o, GPT-4o-mini, etc.) alongside Gemini +- API keys stored securely in Secret Manager +- Good for specific use cases requiring GPT models + +**Optional: Claude (Higher Cost)** +- Available via Anthropic API (requires API key) +- More expensive than Gemini in Vertex AI +- Enable with `enable_anthropic_api = true` in terraform + +## Steps + +### Step 1: Enable Required APIs + +**Important:** Ensure all required APIs from [Phase 1](1_permissions.md) are enabled, plus the Vertex AI API. + +**For Linux/Mac (Bash):** +```bash +# Enable Vertex AI API (required for this phase) +gcloud services enable aiplatform.googleapis.com --project=YOUR_PROJECT_ID + +# Verify all required APIs are enabled +gcloud services list --enabled --project=YOUR_PROJECT_ID +``` + +**For Windows (PowerShell):** +```powershell +# Enable Vertex AI API (required for this phase) +gcloud services enable aiplatform.googleapis.com --project=alex-multi-agent-saas-479504 + +# Verify all required APIs are enabled +gcloud services list --enabled --project=alex-multi-agent-saas-479504 +``` + +**Required APIs for this phase:** +- `aiplatform.googleapis.com` (Vertex AI API) +- `secretmanager.googleapis.com` (Secret Manager API - for API keys) +- All APIs from Phase 1 (compute, run, cloudfunctions, sqladmin, artifactregistry, etc.) + +**If you get API errors during terraform apply:** +1. Enable the missing API: `gcloud services enable --project=YOUR_PROJECT_ID` +2. Wait 2-3 minutes for propagation +3. Retry `terraform apply` + +### Step 2: Deploy Terraform + +**Prerequisites:** Complete [Phase 1: Permissions Setup](1_permissions.md) first, as this step requires the Vertex AI service account. + +**Get the Vertex AI Service Account Email:** + +You'll need the service account email from Phase 1. Get it using one of these methods: + +**Method 1: From Terraform Output (Recommended)** +```bash +cd terraform/1_permissions/ +terraform output vertex_ai_service_account_email +``` + +**Method 2: Using gcloud Command** +```bash +gcloud iam service-accounts list \ + --project=YOUR_PROJECT_ID \ + --filter="email:vertex-ai-sa" \ + --format="value(email)" +``` + +**Method 3: Construct Manually** +The format is: `@.iam.gserviceaccount.com` +- Account ID: `vertex-ai-sa` +- Example: `vertex-ai-sa@alex-multi-agent-saas-479504.iam.gserviceaccount.com` + +**For Windows (PowerShell):** +```powershell +# Method 1: Terraform output +cd "alex-gcp\terraform\1_permissions" +terraform output vertex_ai_service_account_email + +# Method 2: gcloud command +gcloud iam service-accounts list --project=alex-multi-agent-saas-479504 --filter="email:vertex-ai-sa" --format="value(email)" +``` + +**Deploy Terraform:** + +```bash +cd terraform/2_vertex_ai/ +terraform init +terraform plan # You'll be prompted for vertex_ai_service_account - use the email from above +terraform apply +``` + +**Note:** When prompted for `vertex_ai_service_account`, enter the full service account email (e.g., `vertex-ai-sa@alex-multi-agent-saas-479504.iam.gserviceaccount.com`). + +**Terraform Variables:** +- `enable_anthropic_api` (default: `false`) - Set to `true` only if you want to use Claude via Anthropic API. For cost savings, keep it `false` and use Gemini 2.0 Flash instead. + +**After Terraform Apply - Set Up OpenAI API Key:** + +The terraform creates a secret for OpenAI API key, but you need to add the actual key value: + +**For Linux/Mac (Bash):** +```bash +# Create a file with your OpenAI API key +echo -n "your-openai-api-key-here" > /tmp/openai-key.txt + +# Add the secret version +gcloud secrets versions add openai-api-key \ + --data-file=/tmp/openai-key.txt \ + --project=YOUR_PROJECT_ID + +# Clean up +rm /tmp/openai-key.txt +``` + +**For Windows (PowerShell):** +```powershell +# Create a file with your OpenAI API key +"your-openai-api-key-here" | Out-File -FilePath "$env:TEMP\openai-key.txt" -NoNewline -Encoding utf8 + +# Add the secret version +gcloud secrets versions add openai-api-key ` + --data-file="$env:TEMP\openai-key.txt" ` + --project=alex-multi-agent-saas-479504 + +# Clean up +Remove-Item "$env:TEMP\openai-key.txt" +``` + +**Note:** Gemini 2.0 Flash doesn't require an API key - it uses Application Default Credentials (ADC) which you already set up in Phase 1. + +### Step 3: Configure Model Selection + +**Recommended: Use Gemini 2.0 Flash (Cost-Effective)** + +Gemini 2.0 Flash is significantly more cost-effective than Claude models in Vertex AI. The terraform configuration defaults to using Gemini (Anthropic API disabled by default). + +**When deploying terraform, you can optionally enable Anthropic API:** +- Set `enable_anthropic_api = false` (default) to use Gemini 2.0 Flash +- Set `enable_anthropic_api = true` if you want to use Claude via Anthropic API + +### Step 4: Using Gemini 2.0 Flash with LiteLLM (Recommended) + +**For use with OpenAI Agents SDK (as used in Alex backend):** + +```python +from litellm import completion +import os +from google.cloud import secretmanager + +# Get project ID from environment +PROJECT_ID = os.getenv("GCP_PROJECT_ID", "your-project-id") +REGION = os.getenv("GCP_REGION", "us-central1") + +# Initialize Vertex AI (happens automatically with LiteLLM) +# No API key needed - uses Application Default Credentials + +# Use Gemini 2.0 Flash via LiteLLM +response = completion( + model="vertex_ai/gemini-2.0-flash-exp", # or "gemini-2.0-flash-exp" + messages=[ + {"role": "user", "content": "Hello, Gemini!"} + ], + project=PROJECT_ID, + location=REGION +) + +print(response.choices[0].message.content) +``` + +**For use with OpenAI Agents SDK (as in agent.py files):** + +```python +from agents import Agent, Runner +from litellm import LitellmModel +import os + +# Set GCP project and region +os.environ["GCP_PROJECT_ID"] = "your-project-id" +os.environ["GCP_REGION"] = "us-central1" + +# Create model using LiteLLM +model = LitellmModel(model="vertex_ai/gemini-2.0-flash-exp") + +# Use with Agent +agent = Agent( + name="My Agent", + instructions="You are a helpful assistant.", + model=model +) + +result = await Runner.run(agent, input="Hello!") +print(result.final_output) +``` + +**Available Gemini Models:** +- `vertex_ai/gemini-2.0-flash-exp` - Gemini 2.0 Flash (recommended, cost-effective) +- `vertex_ai/gemini-1.5-pro` - Gemini 1.5 Pro (more capable, higher cost) +- `vertex_ai/gemini-1.5-flash` - Gemini 1.5 Flash (faster, lower cost) + +### Step 5: Using OpenAI API Keys (Alongside Gemini) + +**OpenAI API keys are stored in Secret Manager and can be used alongside Gemini:** + +```python +from litellm import completion +from google.cloud import secretmanager +import os + +def get_secret(secret_id: str) -> str: + """Get secret from Secret Manager.""" + client = secretmanager.SecretManagerServiceClient() + project_id = os.getenv("GCP_PROJECT_ID") + name = f"projects/{project_id}/secrets/{secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + return response.payload.data.decode("UTF-8") + +# Get OpenAI API key +openai_api_key = get_secret("openai-api-key") + +# Use OpenAI via LiteLLM +response = completion( + model="gpt-4o-mini", # or "gpt-4o", "gpt-3.5-turbo", etc. + messages=[ + {"role": "user", "content": "Hello, OpenAI!"} + ], + api_key=openai_api_key +) + +print(response.choices[0].message.content) +``` + +**Mixed Usage Example (Gemini for most tasks, OpenAI for specific needs):** + +```python +# Use Gemini 2.0 Flash for general tasks (cost-effective) +gemini_response = completion( + model="vertex_ai/gemini-2.0-flash-exp", + messages=[{"role": "user", "content": "General question"}], + project=PROJECT_ID, + location=REGION +) + +# Use OpenAI for specific tasks requiring GPT-4 +openai_response = completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Complex reasoning task"}], + api_key=openai_api_key +) +``` + +### Step 6: Using Claude (Optional - Higher Cost) + +**If you enabled Anthropic API in terraform (`enable_anthropic_api = true`):** + +```python +import anthropic +from google.cloud import secretmanager + +def get_secret(secret_id: str) -> str: + client = secretmanager.SecretManagerServiceClient() + project_id = os.getenv("GCP_PROJECT_ID") + name = f"projects/{project_id}/secrets/{secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + return response.payload.data.decode("UTF-8") + +# Get Anthropic API key +api_key = get_secret("anthropic-api-key") +client = anthropic.Anthropic(api_key=api_key) + +# Make a request +message = client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=1024, + messages=[ + {"role": "user", "content": "Hello, Claude!"} + ] +) +print(message.content[0].text) +``` + +**Note:** Claude models are more expensive in Vertex AI. Consider using Gemini 2.0 Flash for cost savings. + +## Custom Model Deployment + +For deploying custom models (like fine-tuned models): + +```python +from google.cloud import aiplatform + +# Initialize +aiplatform.init(project="your-project", location="us-central1") + +# Upload model +model = aiplatform.Model.upload( + display_name="my-custom-model", + artifact_uri="gs://your-bucket/model-artifacts/", + serving_container_image_uri="us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest", +) + +# Deploy to endpoint +endpoint = model.deploy( + machine_type="n1-standard-4", + min_replica_count=1, + max_replica_count=3, +) + +# Get predictions +response = endpoint.predict(instances=[{"input": "test"}]) +``` + +## Cost Optimization Tips + +1. **Use preemptible/spot VMs** for training jobs +2. **Auto-scale endpoints** based on traffic +3. **Use Model Garden** for foundation models (pay-per-use) +4. **Batch predictions** for non-real-time workloads + +## Troubleshooting + +### API Not Enabled Errors + +**Common Errors:** +- `Error 403: Secret Manager API has not been used in project before or it is disabled` +- `Error 403: Vertex AI API has not been used in project before or it is disabled` + +**Solution:** + +**For Linux/Mac (Bash):** +```bash +# Enable missing APIs +gcloud services enable aiplatform.googleapis.com secretmanager.googleapis.com --project=YOUR_PROJECT_ID + +# Verify APIs are enabled +gcloud services list --enabled --project=YOUR_PROJECT_ID --filter="name:(aiplatform.googleapis.com OR secretmanager.googleapis.com)" + +# Wait 2-3 minutes for propagation, then retry terraform apply +``` + +**For Windows (PowerShell):** +```powershell +# Enable missing APIs +gcloud services enable aiplatform.googleapis.com secretmanager.googleapis.com --project=alex-multi-agent-saas-479504 + +# Verify APIs are enabled +gcloud services list --enabled --project=alex-multi-agent-saas-479504 --filter="name:(aiplatform.googleapis.com OR secretmanager.googleapis.com)" + +# Wait 2-3 minutes for propagation, then retry terraform apply +``` + +**Note:** GCP APIs can take 2-5 minutes to fully propagate after enabling. If you still get errors, wait a few more minutes and retry. + +### Quota Issues +```bash +# Check quotas +gcloud compute project-info describe --project=YOUR_PROJECT + +# Request quota increase via Cloud Console +``` + +### Permission Denied +```bash +# Verify service account has aiplatform.user role +gcloud projects get-iam-policy YOUR_PROJECT --format=json | \ + jq '.bindings[] | select(.role | contains("aiplatform"))' +``` + +## Next Steps + +After setting up Vertex AI, proceed to [3_ingest.md](3_ingest.md) for data ingestion setup. + +**For Backend Integration:** +- See [GEMINI_SETUP.md](GEMINI_SETUP.md) for detailed instructions on updating your backend agent code to use Gemini 2.0 Flash +- Includes code examples, environment variable setup, and troubleshooting diff --git a/gcp-deployment/guides/5_database.md b/gcp-deployment/guides/5_database.md new file mode 100644 index 00000000..f36220c8 --- /dev/null +++ b/gcp-deployment/guides/5_database.md @@ -0,0 +1,300 @@ +# Phase 5: Database Setup (Cloud SQL - Equivalent to AWS RDS) + +## Overview + +This guide sets up Cloud SQL PostgreSQL on GCP, which is the equivalent of AWS RDS PostgreSQL. + +## AWS vs GCP Comparison + +| AWS RDS | GCP Cloud SQL | +|---------|---------------| +| RDS PostgreSQL | Cloud SQL PostgreSQL | +| Multi-AZ | High Availability Configuration | +| Read Replicas | Read Replicas | +| Parameter Groups | Database Flags | +| Security Groups | Firewall Rules / Authorized Networks | +| RDS Proxy | Cloud SQL Auth Proxy | +| IAM Authentication | IAM Database Authentication | + +## Steps + +### Step 1: Deploy Terraform + +**Enable required APIs (if not already enabled):** + +**For Linux/Mac (Bash):** +```bash +gcloud services enable sqladmin.googleapis.com servicenetworking.googleapis.com --project=YOUR_PROJECT_ID +``` + +**For Windows (PowerShell):** +```powershell +gcloud services enable sqladmin.googleapis.com servicenetworking.googleapis.com --project=YOUR_PROJECT_ID +``` + +**For Linux/Mac (Bash):** +```bash +cd terraform/5_database/ +terraform init +terraform plan +terraform apply +``` + +**For Windows (PowerShell):** +```powershell +cd "alex-gcp\terraform\5_database" +terraform init +terraform plan +terraform apply +``` + +### Step 2: Connect to Database + +**Option A: Using Cloud SQL Auth Proxy (Recommended)** + +**For Linux/Mac (Bash):** +```bash +# Install Cloud SQL Auth Proxy +curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.8.0/cloud-sql-proxy.linux.amd64 +chmod +x cloud-sql-proxy + +# Start proxy (public IP enabled on the instance) +./cloud-sql-proxy --port 5432 PROJECT_ID:REGION:INSTANCE_NAME + +# Connect via psql (use the same port you set above) +psql "host=127.0.0.1 port=5432 user=postgres dbname=alex" +``` + +**For Windows (PowerShell):** +```powershell +# Download Cloud SQL Auth Proxy (Windows x64) +Invoke-WebRequest ` + -Uri "https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.8.0/cloud-sql-proxy.x64.exe" ` + -OutFile "cloud-sql-proxy.exe" + +# Make sure port 5432 is free. Check with: +# netstat -ano | findstr 5432 +# If another process is using it, either stop that process or pick a different local port (e.g. 5433) + +# Start proxy (change --port if 5432 is taken) +.\cloud-sql-proxy.exe --port 5432 PROJECT_ID:REGION:INSTANCE_NAME + +# If 127.0.0.1 binding fails due to security policies, bind to 0.0.0.0: +# .\cloud-sql-proxy.exe --address 0.0.0.0 --port 5433 PROJECT_ID:REGION:INSTANCE_NAME + +# Connect via psql (use the same host/port you set above) +Use gcloud secrets versions access latest --secret=alex-db-password to retrieve the password. +psql "host=127.0.0.1 port=5432 user=postgres dbname=alex" +# If you used --address 0.0.0.0 and port 5433: +# psql "host=127.0.0.1 port=5433 user=postgres dbname=alex" + +# Important: Keep the proxy window open. Pressing Ctrl+C stops the proxy and any client connections will immediately drop. +# +# Tip: Run the proxy in a separate PowerShell window so you can keep working: +# Start-Process powershell -ArgumentList '-NoExit','-Command','cd "C:\path\to\project\alex-gcp\terraform\5_database"; .\cloud-sql-proxy.exe --address 0.0.0.0 --port 5433 PROJECT_ID:REGION:INSTANCE_NAME' +``` + +**Option B: Using Private IP (within VPC)** + +```bash +psql "host=PRIVATE_IP port=5432 user=postgres dbname=alex" +``` + +> **Important:** Option B only works for clients running **inside the same VPC** (Cloud Run, GCE, etc.). For local development, either enable a public IP (set `enable_public_ip = true` in `terraform.tfvars` and add your home IP to `authorized_networks`) or run the proxy from a VM inside the VPC. + +### Step 3: Database Initialization + +Run the initialization script located at `backend/database/migrations/001_schema.sql`: + +**For Linux/Mac (Bash):** +```bash +psql -h 127.0.0.1 -U postgres -d alex -f backend/database/migrations/001_schema.sql +``` + +**For Windows (PowerShell):** +```powershell +psql -h 127.0.0.1 -U postgres -d alex -f ".\backend\database\migrations\001_schema.sql" +``` + +### Step 4: Connection from Cloud Run + +In your Cloud Run service, use the Cloud SQL connector: + +```python +import os +from google.cloud.sql.connector import Connector +import pg8000 +import sqlalchemy + +def get_connection(): + connector = Connector() + + def getconn(): + conn = connector.connect( + os.environ["INSTANCE_CONNECTION_NAME"], + "pg8000", + user=os.environ["DB_USER"], + password=os.environ["DB_PASS"], + db=os.environ["DB_NAME"], + ) + return conn + + pool = sqlalchemy.create_engine( + "postgresql+pg8000://", + creator=getconn, + ) + return pool +``` + +### Step 5: Set Up Secrets + +**Get the Cloud Run service account email (needed for secret access):** + +**Method 1 โ€“ Terraform output (recommended)** +```bash +cd terraform/1_permissions/ +terraform output cloud_run_service_account_email +``` + +**Method 2 โ€“ gcloud command** +```bash +gcloud iam service-accounts list \ + --project=YOUR_PROJECT_ID \ + --filter="email:cloud-run-sa" \ + --format="value(email)" +``` + +**Method 3 โ€“ Construct manually** +- Pattern: `cloud-run-sa@.iam.gserviceaccount.com` +- Example: `cloud-run-sa@alex-multi-agent-saas-479504.iam.gserviceaccount.com` + +Store database credentials in Secret Manager: + +**For Linux/Mac (Bash):** +```bash +# Create secrets +echo -n "your_password" | gcloud secrets create db-password --data-file=- +echo -n "postgres" | gcloud secrets create db-user --data-file=- + +# Grant access to Cloud Run service account +gcloud secrets add-iam-policy-binding db-password \ + --member="serviceAccount:cloud-run-sa@PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +**For Windows (PowerShell):** +```powershell +# Create secrets +"your_password" | Set-Content -Path "$env:TEMP\db-password.txt" -NoNewline +"postgres" | Set-Content -Path "$env:TEMP\db-user.txt" -NoNewline + +gcloud secrets create db-password --data-file="$env:TEMP\db-password.txt" +gcloud secrets create db-user --data-file="$env:TEMP\db-user.txt" + +Remove-Item "$env:TEMP\db-password.txt","$env:TEMP\db-user.txt" + +# Grant access to Cloud Run service account +gcloud secrets add-iam-policy-binding db-password ` + --member="serviceAccount:cloud-run-sa@PROJECT_ID.iam.gserviceaccount.com" ` + --role="roles/secretmanager.secretAccessor" +``` + +## Database Schema Example + +```sql +-- Users table +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email VARCHAR(255) UNIQUE NOT NULL, + clerk_id VARCHAR(255) UNIQUE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +-- Conversations table +CREATE TABLE conversations ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + title VARCHAR(500), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +-- Messages table +CREATE TABLE messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + conversation_id UUID REFERENCES conversations(id) ON DELETE CASCADE, + role VARCHAR(50) NOT NULL, + content TEXT NOT NULL, + tokens_used INTEGER, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +-- Create indexes +CREATE INDEX idx_conversations_user_id ON conversations(user_id); +CREATE INDEX idx_messages_conversation_id ON messages(conversation_id); +CREATE INDEX idx_messages_created_at ON messages(created_at); +``` + +## Backup and Recovery + +```bash +# Create on-demand backup +gcloud sql backups create --instance=alex-postgres --project=YOUR_PROJECT_ID + +# List backups +gcloud sql backups list --instance=alex-postgres --project=YOUR_PROJECT_ID + +# Restore from backup +gcloud sql backups restore BACKUP_ID --restore-instance=alex-postgres --project=YOUR_PROJECT_ID +``` + +> **Tip:** If you see `HTTPError 404` when creating backups, run `gcloud config set project YOUR_PROJECT_ID` and make sure the authenticated account has Cloud SQL permissions for that project. + +## Troubleshooting + +### Service Networking API not enabled +```bash +# Enable API (required for private service access) +gcloud services enable servicenetworking.googleapis.com --project=YOUR_PROJECT_ID + +# Wait 2-3 minutes for propagation, then rerun terraform apply +``` + +### Cloud SQL Proxy can't bind to port 5432 (Windows) +```powershell +# Check which processes are using the port +netstat -ano | findstr 5432 + +# Option A: Stop the process that's using 5432 (e.g., local Postgres service) +# Option B: Start the proxy on a different port (e.g., 5433) +.\cloud-sql-proxy.exe --port 5433 PROJECT_ID:REGION:INSTANCE_NAME + +# Then connect with psql using the same port +psql "host=127.0.0.1 port=5433 user=postgres dbname=alex" +``` + +### Connection Issues +```bash +# Check instance status +gcloud sql instances describe alex-postgres + +# Check authorized networks +gcloud sql instances describe alex-postgres --format="value(settings.ipConfiguration.authorizedNetworks)" + +# Test connectivity +pg_isready -h INSTANCE_IP -p 5432 +``` + +### Performance Issues +```bash +# Check slow queries +SELECT * FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10; + +# Check connections +SELECT count(*) FROM pg_stat_activity; +``` + +## Next Steps + +After setting up the database, proceed to [6_agents.md](6_agents.md) for agent deployment. diff --git a/gcp-deployment/guides/6_agents.md b/gcp-deployment/guides/6_agents.md new file mode 100644 index 00000000..ab1342ab --- /dev/null +++ b/gcp-deployment/guides/6_agents.md @@ -0,0 +1,277 @@ +# Phase 6: Agent Deployment (Cloud Run - Equivalent to AWS Lambda/App Runner) + +## Overview + +This guide deploys the multi-agent backend on GCP using Cloud Run, which is equivalent to AWS Lambda/App Runner. + +## AWS vs GCP Comparison + +| AWS Service | GCP Equivalent | Use Case | +|-------------|----------------|----------| +| Lambda | Cloud Functions (Gen 2) | Event-driven, short tasks | +| App Runner | Cloud Run | Containerized web services | +| Bedrock AgentCore | Vertex AI Agents | Agent orchestration | +| API Gateway | Cloud Run (built-in) / API Gateway | HTTP endpoints | +| SQS | Cloud Pub/Sub | Message queuing | + +## Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Cloud Run Services โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ Researcher โ”‚ Ingestor โ”‚ Planner โ”‚ Executor โ”‚ +โ”‚ Agent โ”‚ Agent โ”‚ Agent โ”‚ Agent โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ Vertex AI (Claude/Gemini) โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ Cloud SQL (PostgreSQL) โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Steps + +### Step 1: Build Container Images + +```bash +# Set variables +export PROJECT_ID="your-project-id" +export REGION="us-central1" +export REPO="alex-containers" + +# Configure Docker for Artifact Registry +gcloud auth configure-docker ${REGION}-docker.pkg.dev + +# Build and push agent images +cd backend/agents + +# Build researcher agent +docker build -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/researcher:latest -f Dockerfile.researcher . +docker push ${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/researcher:latest + +# Build other agents similarly... +``` + +### Step 2: Deploy Terraform + +```bash +cd terraform/6_agents/ +terraform init +terraform plan +terraform apply +``` + +### Step 3: Configure Environment Variables + +Cloud Run services need these environment variables: + +```bash +# Set via Terraform or gcloud +gcloud run services update researcher-agent \ + --set-env-vars="PROJECT_ID=${PROJECT_ID}" \ + --set-env-vars="REGION=${REGION}" \ + --set-secrets="ANTHROPIC_API_KEY=anthropic-api-key:latest" \ + --set-secrets="DB_PASSWORD=alex-db-password:latest" +``` + +### Step 4: Inter-Service Communication + +For agents to communicate with each other: + +```python +import os +import requests +from google.auth.transport.requests import Request +from google.oauth2 import id_token + +def call_agent(agent_url: str, payload: dict) -> dict: + """Call another Cloud Run agent service.""" + # Get ID token for authentication + auth_req = Request() + token = id_token.fetch_id_token(auth_req, agent_url) + + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + response = requests.post(agent_url, json=payload, headers=headers) + response.raise_for_status() + return response.json() + +# Example usage +result = call_agent( + os.environ["RESEARCHER_AGENT_URL"], + {"query": "Research AI trends"} +) +``` + +## Agent Code Structure + +### Base Agent Class + +```python +# backend/agents/base.py +from abc import ABC, abstractmethod +from typing import Any, Dict +import anthropic +from google.cloud import secretmanager + +class BaseAgent(ABC): + def __init__(self): + self.client = self._init_anthropic_client() + + def _init_anthropic_client(self) -> anthropic.Anthropic: + """Initialize Anthropic client with API key from Secret Manager.""" + api_key = self._get_secret("anthropic-api-key") + return anthropic.Anthropic(api_key=api_key) + + def _get_secret(self, secret_id: str) -> str: + """Get secret from Secret Manager.""" + client = secretmanager.SecretManagerServiceClient() + project_id = os.environ["PROJECT_ID"] + name = f"projects/{project_id}/secrets/{secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + return response.payload.data.decode("UTF-8") + + @abstractmethod + async def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]: + """Process input and return result.""" + pass +``` + +### FastAPI Endpoint + +```python +# backend/agents/researcher/main.py +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from base import BaseAgent + +app = FastAPI() + +class ResearchRequest(BaseModel): + query: str + context: dict = {} + +class ResearchAgent(BaseAgent): + async def process(self, input_data: dict) -> dict: + message = self.client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=4096, + system="You are a research assistant...", + messages=[ + {"role": "user", "content": input_data["query"]} + ] + ) + return {"result": message.content[0].text} + +agent = ResearchAgent() + +@app.post("/research") +async def research(request: ResearchRequest): + try: + result = await agent.process(request.dict()) + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/health") +async def health(): + return {"status": "healthy"} +``` + +### Dockerfile + +**Important**: When using local path dependencies (like `alex-database`), you must explicitly install the database package to ensure all transitive dependencies (including `pg8000`) are installed. See [FIX_PG8000_DEPENDENCY.md](FIX_PG8000_DEPENDENCY.md) for details. + +```dockerfile +# backend/reporter/Dockerfile (example) +FROM --platform=linux/amd64 python:3.12-slim + +WORKDIR /app + +# Install Python package manager +RUN pip install uv + +# Copy database package (required dependency) +COPY database ./database + +# Copy shared modules +COPY common ./common + +# Copy agent-specific files +COPY reporter/pyproject.toml reporter/uv.lock ./ + +# Update pyproject.toml to use ./database instead of ../database +RUN sed -i.bak 's|path = "../database"|path = "./database"|g' pyproject.toml && rm pyproject.toml.bak + +# Install Python dependencies +# First install database package with all its dependencies (including pg8000) +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project + +# Copy agent application code +COPY reporter/*.py ./ + +# Cloud Run expects port 8000 +ENV PORT=8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] +``` + +## Scaling Configuration + +Cloud Run automatically scales based on traffic: + +```hcl +# In terraform +resource "google_cloud_run_v2_service" "agent" { + template { + scaling { + min_instance_count = 0 # Scale to zero + max_instance_count = 10 # Max instances + } + + containers { + resources { + limits = { + cpu = "2" + memory = "2Gi" + } + cpu_idle = true # Don't charge for idle CPU + } + } + } +} +``` + +## Monitoring + +```bash +# View logs +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=researcher-agent" --limit=50 + +# View metrics +gcloud monitoring dashboards list +``` + +## Troubleshooting + +### Cold Start Issues +- Increase `min_instance_count` to 1 +- Use smaller container images +- Optimize initialization code + +### Memory Errors +- Increase memory limits in Cloud Run +- Profile memory usage with Cloud Profiler + +### Timeout Issues +- Increase request timeout (max 60 min) +- Use async processing for long tasks + +## Next Steps + +After deploying agents, proceed to [7_frontend.md](7_frontend.md) for frontend deployment. diff --git a/gcp-deployment/guides/7_frontend.md b/gcp-deployment/guides/7_frontend.md new file mode 100644 index 00000000..ac61cf61 --- /dev/null +++ b/gcp-deployment/guides/7_frontend.md @@ -0,0 +1,246 @@ +# Phase 7: Frontend Deployment (Cloud Run + Cloud CDN) + +## Overview + +This guide deploys the NextJS React frontend on GCP using Cloud Run, which is equivalent to AWS App Runner + CloudFront. + +## AWS vs GCP Comparison + +| AWS Service | GCP Equivalent | +|-------------|----------------| +| App Runner | Cloud Run | +| CloudFront | Cloud CDN + Cloud Load Balancing | +| Route 53 | Cloud DNS | +| ACM (SSL Certs) | Certificate Manager | +| S3 (Static Assets) | Cloud Storage | + +## Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Internet โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Cloud Load Balancer (HTTPS) โ”‚ +โ”‚ + Cloud CDN โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Cloud Run (Frontend) โ”‚ +โ”‚ NextJS App โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Cloud Run (Backend API) โ”‚ +โ”‚ Orchestrator โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Steps + +### Step 1: Build Frontend Container + +```bash +# Navigate to frontend directory +cd frontend + +# Create Dockerfile +cat > Dockerfile << 'EOF' +FROM node:20-alpine AS builder +WORKDIR /app +COPY package*.json ./ +RUN npm ci +COPY . . +RUN npm run build + +FROM node:20-alpine AS runner +WORKDIR /app +ENV NODE_ENV=production +COPY --from=builder /app/public ./public +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/.next/static ./.next/static + +ENV PORT=8080 +EXPOSE 8080 + +CMD ["node", "server.js"] +EOF + +# Build and push +export PROJECT_ID="your-project-id" +export REGION="us-central1" + +docker build -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/alex-containers/frontend:latest . +docker push ${REGION}-docker.pkg.dev/${PROJECT_ID}/alex-containers/frontend:latest +``` + +### Step 2: Configure NextJS for Standalone + +Update `next.config.js`: + +```javascript +/** @type {import('next').NextConfig} */ +const nextConfig = { + output: 'standalone', + env: { + NEXT_PUBLIC_API_URL: process.env.NEXT_PUBLIC_API_URL, + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, + }, +} + +module.exports = nextConfig +``` + +### Step 3: Deploy Terraform + +```bash +cd terraform/7_frontend/ +terraform init +terraform plan +terraform apply +``` + +### Step 4: Configure Clerk Authentication + +1. Get your Clerk keys from dashboard.clerk.com +2. Store in Secret Manager: + +```bash +gcloud secrets create clerk-publishable-key --data-file=- << EOF +pk_live_xxx +EOF + +gcloud secrets create clerk-secret-key --data-file=- << EOF +sk_live_xxx +EOF +``` + +### Step 5: Set Up Custom Domain (Optional) + +1. Verify domain ownership in Cloud DNS +2. Create managed SSL certificate +3. Configure load balancer with domain + +```bash +# Add DNS record +gcloud dns record-sets create yourdomain.com \ + --zone=your-zone \ + --type=A \ + --ttl=300 \ + --rrdatas=LOAD_BALANCER_IP +``` + +## Environment Variables + +The frontend needs these environment variables: + +| Variable | Description | +|----------|-------------| +| `NEXT_PUBLIC_API_URL` | Backend orchestrator URL | +| `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` | Clerk public key | +| `CLERK_SECRET_KEY` | Clerk secret key | + +## Vercel Alternative + +If you prefer Vercel for frontend (as mentioned in the course): + +```bash +# Install Vercel CLI +npm i -g vercel + +# Deploy +vercel --prod + +# Set environment variables +vercel env add NEXT_PUBLIC_API_URL production +vercel env add NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY production +``` + +For Vercel + GCP backend: +1. Deploy frontend to Vercel +2. Keep backend on GCP Cloud Run +3. Configure CORS on backend + +```python +# backend/main.py +from fastapi.middleware.cors import CORSMiddleware + +app.add_middleware( + CORSMiddleware, + allow_origins=["https://yourdomain.vercel.app"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) +``` + +## Performance Optimization + +### Enable Cloud CDN + +```hcl +# In terraform +resource "google_compute_backend_service" "frontend" { + enable_cdn = true + + cdn_policy { + cache_mode = "CACHE_ALL_STATIC" + default_ttl = 3600 + max_ttl = 86400 + } +} +``` + +### Configure Cache Headers in NextJS + +```javascript +// next.config.js +async headers() { + return [ + { + source: '/_next/static/:path*', + headers: [ + { + key: 'Cache-Control', + value: 'public, max-age=31536000, immutable', + }, + ], + }, + ] +} +``` + +## Monitoring + +```bash +# View frontend logs +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=frontend" --limit=50 + +# Check latency metrics +gcloud monitoring metrics list --filter="metric.type:cloudrun" +``` + +## Troubleshooting + +### Build Failures +- Check Node version compatibility +- Verify all dependencies are in package.json +- Check for missing environment variables + +### Runtime Errors +- Check Cloud Run logs +- Verify API URL is correct +- Check Clerk configuration + +### Performance Issues +- Enable Cloud CDN +- Optimize images with `next/image` +- Use React Server Components where possible + +## Next Steps + +After deploying frontend, proceed to [8_enterprise.md](8_enterprise.md) for enterprise features. diff --git a/gcp-deployment/guides/ARCHITECTURE_COMPARISON.md b/gcp-deployment/guides/ARCHITECTURE_COMPARISON.md new file mode 100644 index 00000000..86d36795 --- /dev/null +++ b/gcp-deployment/guides/ARCHITECTURE_COMPARISON.md @@ -0,0 +1,295 @@ +# AWS vs GCP Architecture Comparison + +## Current State (AWS) vs Target State (GCP) + +### High-Level Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ AWS ARCHITECTURE โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Frontend โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”‚ API Gateway โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”‚ Lambda โ”‚ โ”‚ +โ”‚ โ”‚ (Next.js) โ”‚ โ”‚ โ”‚ โ”‚ (API) โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ SQS โ”‚ โ”‚ +โ”‚ โ”‚ (Job Queue) โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Planner โ”‚ โ”‚ Tagger โ”‚ โ”‚ Reporter โ”‚ โ”‚ +โ”‚ โ”‚ (Lambda) โ”‚ โ”‚ (Lambda) โ”‚ โ”‚ (Lambda) โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Charter โ”‚ โ”‚Retirementโ”‚ โ”‚Researcherโ”‚ โ”‚ +โ”‚ โ”‚ (Lambda) โ”‚ โ”‚ (Lambda) โ”‚ โ”‚(App Run)โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Bedrock โ”‚ โ”‚ SageMakerโ”‚ โ”‚S3 Vectorsโ”‚ โ”‚ +โ”‚ โ”‚ (LLM) โ”‚ โ”‚(Embedding)โ”‚ โ”‚ (Search)โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Aurora โ”‚ โ”‚ +โ”‚ โ”‚ (PostgreSQL)โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ GCP ARCHITECTURE โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Frontend โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”‚ Cloud Run โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”‚ Cloud Run โ”‚ โ”‚ +โ”‚ โ”‚ (Next.js) โ”‚ โ”‚ (API) โ”‚ โ”‚ (API) โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Pub/Sub โ”‚ โ”‚ +โ”‚ โ”‚ (Job Queue) โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Planner โ”‚ โ”‚ Tagger โ”‚ โ”‚ Reporter โ”‚ โ”‚ +โ”‚ โ”‚(Cloud Run)โ”‚ โ”‚(Cloud Run)โ”‚ โ”‚(Cloud Run)โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Charter โ”‚ โ”‚Retirementโ”‚ โ”‚Researcherโ”‚ โ”‚ +โ”‚ โ”‚(Cloud Run)โ”‚ โ”‚(Cloud Run)โ”‚ โ”‚(Cloud Run)โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Vertex AIโ”‚ โ”‚ Vertex AIโ”‚ โ”‚ Vertex AIโ”‚ โ”‚ +โ”‚ โ”‚ (Gemini) โ”‚ โ”‚(Embedding)โ”‚ โ”‚(Vector โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ Search) โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Cloud SQL โ”‚ โ”‚ +โ”‚ โ”‚ (PostgreSQL)โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Service-by-Service Mapping + +### Compute Services + +| Component | AWS | GCP | Migration Status | +|-----------|-----|-----|------------------| +| **API Backend** | Lambda | Cloud Run | ๐Ÿ”„ To Do | +| **Planner Agent** | Lambda | Cloud Run | ๐Ÿ”„ To Do | +| **Tagger Agent** | Lambda | Cloud Run | ๐Ÿ”„ To Do | +| **Reporter Agent** | Lambda | Cloud Run | ๐Ÿ”„ To Do | +| **Charter Agent** | Lambda | Cloud Run | ๐Ÿ”„ To Do | +| **Retirement Agent** | Lambda | Cloud Run | ๐Ÿ”„ To Do | +| **Researcher Agent** | App Runner | Cloud Run | ๐Ÿ”„ To Do | + +### Messaging & Orchestration + +| Component | AWS | GCP | Migration Status | +|-----------|-----|-----|------------------| +| **Job Queue** | SQS | Pub/Sub | ๐Ÿ”„ To Do | +| **Event Scheduling** | EventBridge | Cloud Scheduler | ๐Ÿ”„ To Do | + +### AI/ML Services + +| Component | AWS | GCP | Migration Status | +|-----------|-----|-----|------------------| +| **LLM Inference** | Bedrock (Nova Pro) | Vertex AI (Gemini 2.0 Flash) | โœ… Done | +| **Embeddings** | SageMaker Endpoint | Vertex AI Embeddings API | ๐Ÿ”„ To Do | +| **Vector Search** | S3 Vectors | Vertex AI Vector Search | ๐Ÿ”„ To Do | + +### Storage & Database + +| Component | AWS | GCP | Migration Status | +|-----------|-----|-----|------------------| +| **Database** | Aurora Serverless v2 | Cloud SQL PostgreSQL | โœ… Done | +| **Object Storage** | S3 | Cloud Storage | ๐Ÿ”„ To Do (if needed) | +| **Secrets** | Secrets Manager | Secret Manager | โœ… Done | + +### Networking & API + +| Component | AWS | GCP | Migration Status | +|-----------|-----|-----|------------------| +| **API Gateway** | API Gateway | Cloud Run + API Gateway | ๐Ÿ”„ To Do | +| **CDN** | CloudFront | Cloud CDN | ๐Ÿ”„ To Do | +| **DNS** | Route 53 | Cloud DNS | ๐Ÿ”„ To Do (if needed) | + +## Code Pattern Changes + +### Invoking Agents + +**AWS Pattern (Current)**: +```python +import boto3 + +lambda_client = boto3.client('lambda') +response = lambda_client.invoke( + FunctionName='alex-reporter', + InvocationType='RequestResponse', + Payload=json.dumps({'job_id': job_id}) +) +result = json.loads(response['Payload'].read()) +``` + +**GCP Pattern (Target)**: +```python +import requests +from google.auth.transport.requests import Request +from google.oauth2 import id_token + +# Get authentication token +auth_req = Request() +token = id_token.fetch_id_token(auth_req, 'https://reporter-xxxxx.run.app') + +# Make HTTP request +response = requests.post( + 'https://reporter-xxxxx.run.app', + json={'job_id': job_id}, + headers={'Authorization': f'Bearer {token}'} +) +result = response.json() +``` + +### Job Queueing + +**AWS Pattern (Current)**: +```python +import boto3 + +sqs = boto3.client('sqs') +sqs.send_message( + QueueUrl='https://sqs.us-east-1.amazonaws.com/.../alex-queue', + MessageBody=json.dumps({'job_id': job_id}) +) +``` + +**GCP Pattern (Target)**: +```python +from google.cloud import pubsub_v1 + +publisher = pubsub_v1.PublisherClient() +topic_path = publisher.topic_path(project_id, 'alex-job-queue') +future = publisher.publish( + topic_path, + json.dumps({'job_id': job_id}).encode('utf-8') +) +message_id = future.result() +``` + +### Vector Search + +**AWS Pattern (Current)**: +```python +import boto3 + +s3v = boto3.client('s3vectors') +response = s3v.query_vectors( + vectorBucketName='alex-vectors-123456', + indexName='financial-research', + queryVector={'float32': embedding}, + topK=3 +) +``` + +**GCP Pattern (Target)**: +```python +from google.cloud import aiplatform + +aiplatform.init(project=project_id, location=region) +index = aiplatform.MatchingEngineIndex(index_id=index_id) +results = index.find_neighbors( + deployed_index_id=deployed_index_id, + queries=[embedding], + num_neighbors=3 +) +``` + +### Embeddings + +**AWS Pattern (Current)**: +```python +import boto3 + +sagemaker = boto3.client('sagemaker-runtime') +response = sagemaker.invoke_endpoint( + EndpointName='alex-embedding-endpoint', + ContentType='application/json', + Body=json.dumps({'inputs': text}) +) +embedding = json.loads(response['Body'].read()) +``` + +**GCP Pattern (Target)**: +```python +from vertexai.preview.language_models import TextEmbeddingModel + +model = TextEmbeddingModel.from_pretrained("textembedding-gecko@003") +embeddings = model.get_embeddings([text]) +embedding = embeddings[0].values +``` + +## Migration Priority Matrix + +### ๐Ÿ”ด High Priority (Blocking) +1. **Pub/Sub Migration** - Required for job orchestration +2. **Agent Compute Migration** - Core functionality +3. **API Backend Migration** - Frontend integration + +### ๐ŸŸก Medium Priority (Important) +4. **Vector Storage Migration** - Market insights feature +5. **Embedding Service Migration** - Vector search dependency + +### ๐ŸŸข Low Priority (Nice to Have) +6. **Monitoring Migration** - Observability improvements +7. **CDN Migration** - Performance optimization + +## Cost Comparison (Estimated) + +| Service | AWS Cost | GCP Cost | Notes | +|---------|----------|----------|-------| +| **Compute** | Lambda: $0.20/1M requests | Cloud Run: $0.40/1M requests | GCP slightly more expensive | +| **LLM** | Bedrock Nova Pro: $0.003/1K tokens | Vertex AI Gemini: $0.000125/1K tokens | **GCP 24x cheaper!** | +| **Database** | Aurora: ~$0.10/hour | Cloud SQL: ~$0.10/hour | Similar pricing | +| **Vector Search** | S3 Vectors: ~$0.10/GB/month | Vertex AI: ~$0.50/GB/month | GCP more expensive but more features | +| **Message Queue** | SQS: $0.40/1M requests | Pub/Sub: $0.40/1M requests | Similar pricing | + +**Key Insight**: Gemini 2.0 Flash is significantly cheaper than Nova Pro, which may offset other cost differences. + +## Next Steps + +1. โœ… **Read**: `GCP_MIGRATION_PLAN.md` for detailed migration steps +2. โœ… **Start**: `MIGRATION_QUICKSTART.md` for immediate action items +3. ๐Ÿ”„ **Execute**: Phase 2 - Pub/Sub migration (2-3 hours) +4. ๐Ÿ”„ **Execute**: Phase 3 - Agent compute migration (5-7 days) +5. ๐Ÿ”„ **Execute**: Phase 4 - Vector storage migration (3-4 days) + diff --git a/gcp-deployment/guides/FIX_PG8000_DEPENDENCY.md b/gcp-deployment/guides/FIX_PG8000_DEPENDENCY.md new file mode 100644 index 00000000..91e94a22 --- /dev/null +++ b/gcp-deployment/guides/FIX_PG8000_DEPENDENCY.md @@ -0,0 +1,153 @@ +# Fix: Missing pg8000 Dependency in Agent Containers + +## Problem + +When deploying Reporter, Charter, and Retirement agents to Cloud Run, they were failing with: + +``` +ModuleNotFoundError: No module named 'pg8000' +``` + +This error occurred when agents tried to connect to the Cloud SQL database. The planner agent worked fine, but the other three agents returned 500 Internal Server Error. + +## Root Cause + +The `pg8000` package is a dependency of the `alex-database` package (required by the Cloud SQL connector). However, when using `uv sync --no-install-project` in Docker builds, transitive dependencies from local path dependencies may not be installed correctly. + +The database package's `pyproject.toml` includes: +```toml +dependencies = [ + "psycopg2-binary>=2.9.9", + "cloud-sql-python-connector>=1.11.0", + "pg8000>=1.31.2", # Required by Cloud SQL connector + ... +] +``` + +But when the database package is referenced as a local path dependency (`path = "./database"`), `uv sync` may not install all transitive dependencies. + +## Solution + +Explicitly install the database package with all its dependencies before syncing the main project dependencies. This ensures that `pg8000` and all other database package dependencies are installed in the container. + +### Updated Dockerfile Pattern + +For Reporter, Charter, and Retirement agents, update the Dockerfile: + +```dockerfile +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +# First install database package with all its dependencies (including pg8000) +# This ensures transitive dependencies from the local path dependency are installed +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project +``` + +### Files Modified + +- `backend/reporter/Dockerfile` +- `backend/charter/Dockerfile` +- `backend/retirement/Dockerfile` + +## Deployment Steps + +1. **Rebuild Docker images:** + ```bash + cd backend + + # Build each agent + docker build --platform linux/amd64 \ + -f reporter/Dockerfile \ + -t us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/reporter:latest . + + docker build --platform linux/amd64 \ + -f charter/Dockerfile \ + -t us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/charter:latest . + + docker build --platform linux/amd64 \ + -f retirement/Dockerfile \ + -t us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/retirement:latest . + ``` + +2. **Push to Artifact Registry:** + ```bash + gcloud auth configure-docker us-central1-docker.pkg.dev + + docker push us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/reporter:latest + docker push us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/charter:latest + docker push us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/retirement:latest + ``` + +3. **Update Cloud Run services:** + ```bash + gcloud run services update alex-reporter \ + --image us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/reporter:latest \ + --region us-central1 \ + --project PROJECT_ID + + gcloud run services update alex-charter \ + --image us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/charter:latest \ + --region us-central1 \ + --project PROJECT_ID + + gcloud run services update alex-retirement \ + --image us-central1-docker.pkg.dev/PROJECT_ID/alex-agents/retirement:latest \ + --region us-central1 \ + --project PROJECT_ID + ``` + +## Verification + +After deployment, verify the fix: + +1. **Check logs for errors:** + ```bash + gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=~'alex-(reporter|charter|retirement)' AND severity>=ERROR" \ + --limit 20 \ + --format="table(timestamp,resource.labels.service_name,severity,textPayload)" \ + --project=PROJECT_ID \ + --freshness=10m + ``` + + Should see no more `ModuleNotFoundError: No module named 'pg8000'` errors. + +2. **Trigger a test analysis:** + - Go to frontend โ†’ Advisor Team page + - Click "Run Analysis" + - Verify results appear (report, charts, retirement analysis) + +3. **Check agent logs for successful database connections:** + ```bash + gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=alex-reporter AND textPayload=~'saved|completed'" \ + --limit 10 \ + --format="table(timestamp,textPayload)" \ + --project=PROJECT_ID \ + --freshness=10m + ``` + +## Why This Works + +- `uv pip install --system -e .` installs the database package in editable mode with all its dependencies +- The `--system` flag installs to the system Python (required in Docker containers without a virtual environment) +- Installing the database package first ensures `pg8000` is available before `uv sync` runs +- `uv sync` then installs the remaining project dependencies, which can now find `pg8000` if needed + +## Alternative Solutions Considered + +1. **Adding pg8000 directly to agent dependencies:** This would work but duplicates the dependency and could lead to version conflicts. + +2. **Using `uv sync` without `--no-install-project`:** This would install the project itself, but we want to keep the build minimal. + +3. **Installing pg8000 separately:** This works but doesn't address the root cause - other transitive dependencies might also be missing. + +The chosen solution (explicitly installing the database package) is the most robust as it ensures all database package dependencies are installed correctly. + +## Related Files + +- `backend/database/pyproject.toml` - Database package dependencies +- `backend/database/src/cloudsql_client.py` - Database client that requires pg8000 +- `backend/reporter/Dockerfile` - Reporter agent Dockerfile +- `backend/charter/Dockerfile` - Charter agent Dockerfile +- `backend/retirement/Dockerfile` - Retirement agent Dockerfile + diff --git a/gcp-deployment/guides/GEMINI_SETUP.md b/gcp-deployment/guides/GEMINI_SETUP.md new file mode 100644 index 00000000..a84edf53 --- /dev/null +++ b/gcp-deployment/guides/GEMINI_SETUP.md @@ -0,0 +1,97 @@ +# Using Gemini 2.0 Flash with Alex Backend + +## Quick Reference + +All backend agents now use a shared helper (`common/llm.py`) to configure LiteLLM models. The helper defaults to Vertex AI Gemini 2.0 Flash but can switch to OpenAI by changing a single environment variable. + +## Environment Variables + +Add these to `.env` (and your deployment environments): + +```bash +# Core project settings +PROJECT_ID=alex-multi-agent-saas-479504 +GCP_PROJECT_ID=alex-multi-agent-saas-479504 +GCP_REGION=us-central1 + +# LLM configuration +LLM_PROVIDER=vertex_ai # or "openai" +VERTEX_AI_MODEL=vertex_ai/gemini-2.0-flash-exp +OPENAI_MODEL=openai/gpt-4o-mini +OPENAI_API_KEY= # only required if LLM_PROVIDER=openai +OPENAI_API_BASE= # optional custom endpoint + +# Optional per-agent overrides +PLANNER_MODEL= +REPORTER_MODEL= +REPORTER_JUDGE_MODEL= +RETIREMENT_MODEL= +CHARTER_MODEL= +TAGGER_MODEL= +RESEARCHER_MODEL= +``` + +If you need to store secrets in Secret Manager, set the env var at runtime by reading the secret before launching the container. + +## Shared Helper + +All agents import the helper: + +```python +from common.llm import get_litellm_model + +model = get_litellm_model(os.getenv("PLANNER_MODEL")) +``` + +The helper automatically: + +- Builds a Vertex AI LiteLLM model when `LLM_PROVIDER=vertex_ai` (using `GCP_PROJECT_ID` + `GCP_REGION`) +- Builds an OpenAI LiteLLM model when `LLM_PROVIDER=openai` (requires `OPENAI_API_KEY`) +- Accepts optional overrides, so each agent can specify a different model if needed + +## When You Need OpenAI + +Set: + +```bash +LLM_PROVIDER=openai +OPENAI_API_KEY=sk-... +OPENAI_MODEL=openai/gpt-4o-mini +``` + +Or override per agent: + +```python +model = get_litellm_model(os.getenv("REPORTER_MODEL", "openai/gpt-4o")) +``` + +## Why a Shared Helper? + +- Keeps all providers in one place +- Ensures consistent validation (project/region/API keys) +- Makes switching providers as simple as editing `.env` + +## Testing Locally + +```powershell +gcloud auth application-default login +gcloud auth application-default set-quota-project alex-multi-agent-saas-479504 +``` + +Then run your agent test as normal; the helper will pick up the credentials. + +## Cost Reminder + +- Gemini 2.0 Flash ~95% cheaper than Claude Sonnet +- Keep `LLM_PROVIDER=vertex_ai` for day-to-day work, only switch to OpenAI for experiments that truly need GPT-4 + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| `ValueError: GCP_PROJECT_ID must be set` | Populate `GCP_PROJECT_ID` (or `PROJECT_ID`) in `.env` | +| `OPENAI_API_KEY must be set` | Set `OPENAI_API_KEY` when `LLM_PROVIDER=openai` | +| `Model not found` | Check `VERTEX_AI_MODEL` or `OPENAI_MODEL` spelling and regional availability | +| `Permission denied` | Ensure `aiplatform.googleapis.com` is enabled and Cloud Run service account has `roles/aiplatform.user` | + +Gemini remains the recommended default for cost and latency. Use OpenAI only when absolutely required. diff --git a/gcp-deployment/guides/TESTING_AGENT_WORKFLOW.md b/gcp-deployment/guides/TESTING_AGENT_WORKFLOW.md new file mode 100644 index 00000000..ed575afd --- /dev/null +++ b/gcp-deployment/guides/TESTING_AGENT_WORKFLOW.md @@ -0,0 +1,343 @@ +# Testing Agent-to-Agent Communication + +This guide walks you through testing the complete agent workflow: creating a job, triggering the planner via Pub/Sub, and verifying that agents communicate via HTTP and write results to the database. + +## Prerequisites + +โœ… Cloud Run services deployed for all agents +โœ… Database seeded with 22 instruments +โœ… Cloud SQL proxy running (if testing locally) +โœ… Environment variables configured in `.env` + +## Step 1: Get Cloud Run Service URLs + +First, get the URLs of your deployed Cloud Run services: + +```powershell +# Navigate to terraform directory +cd alex-gcp/terraform/6_agents + +# Get all service URLs +terraform output +``` + +You should see URLs like: +- `planner_url = "https://alex-planner-xxxxx-uc.a.run.app"` +- `reporter_url = "https://alex-reporter-xxxxx-uc.a.run.app"` +- `tagger_url = "https://alex-tagger-xxxxx-uc.a.run.app"` +- etc. + +**Add these to your `.env` file:** +```bash +PLANNER_URL=https://alex-planner-xxxxx-uc.a.run.app +TAGGER_URL=https://alex-tagger-xxxxx-uc.a.run.app +REPORTER_URL=https://alex-reporter-xxxxx-uc.a.run.app +CHARTER_URL=https://alex-charter-xxxxx-uc.a.run.app +RETIREMENT_URL=https://alex-retirement-xxxxx-uc.a.run.app +``` + +## Step 2: Create a Test User and Account + +Before creating a job, you need a user and account in the database. You can do this via the API or directly in the database. + +### Option A: Via API (if API is running) + +```powershell +# Set your test Clerk user ID (or use a real one from Clerk) +$CLERK_USER_ID = "user_test123" + +# Create user (replace with your API endpoint) +$headers = @{ + "Authorization" = "Bearer YOUR_CLERK_TOKEN" + "Content-Type" = "application/json" +} + +$body = @{ + clerk_user_id = $CLERK_USER_ID + display_name = "Test User" +} | ConvertTo-Json + +Invoke-WebRequest -Uri "http://localhost:8000/api/user" -Method POST -Headers $headers -Body $body +``` + +### Option B: Direct Database Insert (for testing) + +```powershell +# Connect to database via proxy +$env:PGPASSWORD = "YOUR_DB_PASSWORD" +psql -h 127.0.0.1 -p 5432 -U postgres -d alex + +# In psql: +INSERT INTO users (clerk_user_id, display_name) +VALUES ('user_test123', 'Test User') +ON CONFLICT (clerk_user_id) DO NOTHING; + +INSERT INTO accounts (id, clerk_user_id, account_name, account_purpose, cash_balance) +VALUES (gen_random_uuid(), 'user_test123', 'Test 401k', '401k', 0.00) +ON CONFLICT (id) DO NOTHING; + +-- Get the account ID +SELECT id, account_name FROM accounts WHERE clerk_user_id = 'user_test123'; +\q +``` + +## Step 3: Add Test Positions + +Add some positions to the account so the planner has data to analyze: + +```powershell +# In psql (replace ACCOUNT_ID with actual UUID from Step 2): +INSERT INTO positions (id, account_id, symbol, quantity) +VALUES + (gen_random_uuid(), 'ACCOUNT_ID', 'SPY', 10), + (gen_random_uuid(), 'ACCOUNT_ID', 'QQQ', 5), + (gen_random_uuid(), 'ACCOUNT_ID', 'BND', 20) +ON CONFLICT (id) DO NOTHING; + +-- Verify positions +SELECT p.symbol, p.quantity, a.account_name +FROM positions p +JOIN accounts a ON p.account_id = a.id +WHERE a.clerk_user_id = 'user_test123'; +``` + +## Step 4: Create a Job + +### Option A: Via API (Recommended) + +```powershell +# Create analysis job via API +$headers = @{ + "Authorization" = "Bearer YOUR_CLERK_TOKEN" + "Content-Type" = "application/json" +} + +$body = @{ + analysis_type = "portfolio_analysis" + options = @{ + include_retirement_projection = $true + include_charts = $true + } +} | ConvertTo-Json + +$response = Invoke-WebRequest -Uri "http://localhost:8000/api/analyze" -Method POST -Headers $headers -Body $body +$jobData = $response.Content | ConvertFrom-Json +$jobId = $jobData.job_id +Write-Host "Created job: $jobId" +``` + +### Option B: Direct Database Insert + +```powershell +# In psql: +INSERT INTO jobs (id, clerk_user_id, job_type, status, request_payload) +VALUES ( + gen_random_uuid(), + 'user_test123', + 'portfolio_analysis', + 'pending', + '{"analysis_type": "portfolio_analysis", "options": {}}'::jsonb +) +RETURNING id; + +-- Copy the returned UUID +\q +``` + +## Step 5: Trigger Planner Agent + +You can trigger the planner in two ways: + +### Option A: Via Pub/Sub (Production Flow) + +The job should already be queued if you created it via the API. If not, publish manually: + +```powershell +cd alex-gcp/backend/api + +# Load environment variables +$env:GOOGLE_APPLICATION_CREDENTIALS = "" # Use ADC +gcloud auth application-default login + +# Run the Pub/Sub test script (modify it to use your job_id) +uv run python test_pubsub.py +``` + +Or publish directly: + +```powershell +# Get project ID and topic name from .env +$PROJECT_ID = (Get-Content .env | Select-String "GCP_PROJECT_ID").ToString().Split("=")[1].Trim() +$TOPIC = (Get-Content .env | Select-String "PUBSUB_TOPIC").ToString().Split("=")[1].Trim() + +# Publish message +$message = @{ + job_id = "YOUR_JOB_ID_HERE" + clerk_user_id = "user_test123" + analysis_type = "portfolio_analysis" +} | ConvertTo-Json + +gcloud pubsub topics publish $TOPIC --message $message --project $PROJECT_ID +``` + +### Option B: Direct HTTP POST (Testing) + +```powershell +# Get ID token for Cloud Run authentication +$token = gcloud auth print-identity-token + +# Get planner URL from terraform output or .env +$PLANNER_URL = "https://alex-planner-xxxxx-uc.a.run.app" + +# Trigger planner directly +$headers = @{ + "Authorization" = "Bearer $token" + "Content-Type" = "application/json" +} + +$body = @{ + job_id = "YOUR_JOB_ID_HERE" +} | ConvertTo-Json + +Invoke-WebRequest -Uri "$PLANNER_URL/" -Method POST -Headers $headers -Body $body +``` + +## Step 6: Monitor Agent Execution + +### Check Cloud Run Logs + +```powershell +# Planner logs +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=alex-planner" --limit 50 --format json --project $PROJECT_ID + +# Reporter logs +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=alex-reporter" --limit 50 --format json --project $PROJECT_ID + +# Or use the Cloud Console: https://console.cloud.google.com/run +``` + +### Check Job Status in Database + +```powershell +# In psql: +SELECT id, status, created_at, updated_at, error_message +FROM jobs +WHERE id = 'YOUR_JOB_ID_HERE'; + +-- Check if report was generated +SELECT id, status, report IS NOT NULL as has_report, charts IS NOT NULL as has_charts +FROM jobs +WHERE id = 'YOUR_JOB_ID_HERE'; +``` + +## Step 7: Verify Results + +### Check Job Completion + +```powershell +# In psql: +SELECT + id, + status, + CASE WHEN report IS NOT NULL THEN 'Yes' ELSE 'No' END as has_report, + CASE WHEN charts IS NOT NULL THEN 'Yes' ELSE 'No' END as has_charts, + CASE WHEN retirement IS NOT NULL THEN 'Yes' ELSE 'No' END as has_retirement, + updated_at +FROM jobs +WHERE clerk_user_id = 'user_test123' +ORDER BY created_at DESC +LIMIT 5; +``` + +### View Report Content + +```powershell +# In psql: +SELECT + id, + status, + LEFT(report::text, 200) as report_preview, + LEFT(charts::text, 200) as charts_preview +FROM jobs +WHERE id = 'YOUR_JOB_ID_HERE' AND status = 'completed'; +``` + +## Troubleshooting + +### Planner Returns 404 "Job not found" + +- **Cause**: Job ID doesn't exist in database +- **Fix**: Create the job first (Step 4), then use that exact UUID + +### Planner Returns 500 Error + +- **Cause**: Check Cloud Run logs for the actual error +- **Common issues**: + - Missing environment variables (Cloud Run service URLs, database connection) + - Database connection failure + - Missing instruments in database (run `seed_data.py`) + +### Agents Not Being Called + +- **Cause**: Planner can't reach other agents +- **Fix**: + 1. Verify `PLANNER_URL`, `TAGGER_URL`, etc. are set in Cloud Run environment variables + 2. Check IAM permissions: Planner service account needs `roles/run.invoker` on other services + 3. Verify agents are deployed and healthy: `gcloud run services list` + +### Database Connection Errors + +- **Cause**: Cloud SQL connection issues +- **Fix**: + 1. Verify `INSTANCE_CONNECTION_NAME` is set correctly + 2. Check VPC connector is attached to Cloud Run services + 3. Verify service account has `roles/cloudsql.client` permission + +## Next Steps + +Once the workflow is working: + +1. **Test with real Clerk authentication** - Use actual user tokens from your frontend +2. **Monitor costs** - Check Cloud Run billing and Pub/Sub usage +3. **Add error handling** - Implement retries and dead-letter queues +4. **Performance testing** - Test with multiple concurrent jobs +5. **Frontend integration** - Connect the NextJS frontend to display results + +## Quick Test Script + +Here's a complete PowerShell script to test the workflow: + +```powershell +# Set variables +$CLERK_USER_ID = "user_test123" +$PROJECT_ID = "alex-multi-agent-saas-479504" # Replace with your project +$PLANNER_URL = "https://alex-planner-xxxxx-uc.a.run.app" # Replace with actual URL + +# 1. Create job in database +$env:PGPASSWORD = "YOUR_DB_PASSWORD" +$jobId = psql -h 127.0.0.1 -p 5432 -U postgres -d alex -t -c "INSERT INTO jobs (id, clerk_user_id, job_type, status, request_payload) VALUES (gen_random_uuid(), '$CLERK_USER_ID', 'portfolio_analysis', 'pending', '{}'::jsonb) RETURNING id;" +$jobId = $jobId.Trim() +Write-Host "Created job: $jobId" + +# 2. Trigger planner +$token = gcloud auth print-identity-token +$headers = @{ + "Authorization" = "Bearer $token" + "Content-Type" = "application/json" +} +$body = @{ job_id = $jobId } | ConvertTo-Json + +try { + $response = Invoke-WebRequest -Uri "$PLANNER_URL/" -Method POST -Headers $headers -Body $body + Write-Host "Planner triggered successfully!" + Write-Host $response.Content +} catch { + Write-Host "Error: $_" + Write-Host $_.Exception.Response +} + +# 3. Check status (wait a few seconds first) +Start-Sleep -Seconds 10 +psql -h 127.0.0.1 -p 5432 -U postgres -d alex -c "SELECT id, status, updated_at FROM jobs WHERE id = '$jobId';" +``` + diff --git a/gcp-deployment/guides/TROUBLESHOOTING.md b/gcp-deployment/guides/TROUBLESHOOTING.md new file mode 100644 index 00000000..d779c6d3 --- /dev/null +++ b/gcp-deployment/guides/TROUBLESHOOTING.md @@ -0,0 +1,358 @@ +# Troubleshooting Guide + +This guide covers common issues and solutions when deploying and running Alex on GCP. + +## Table of Contents + +1. [Environment Variables](#environment-variables) +2. [Pub/Sub Issues](#pubsub-issues) +3. [Cloud Run Logs](#cloud-run-logs) +4. [Docker Build Issues](#docker-build-issues) +5. [Database Connection Issues](#database-connection-issues) +6. [Authentication Issues](#authentication-issues) +7. [API Not Enabled Errors](#api-not-enabled-errors) + +## Environment Variables + +### Issue: Environment Variables Not Loading + +**Symptoms:** +- `GCP_PROJECT_ID` is `None` or wrong value +- Configuration errors in application + +**Solutions:** + +1. **Verify `.env` file exists in root directory:** + ```bash + # Required variables + GCP_PROJECT_ID=your-gcp-project-id + GCP_REGION=us-central1 + PUBSUB_TOPIC=alex-job-queue + ``` + +2. **Check environment variable loading:** + ```bash + cd backend/api + uv run python -c "import os; from dotenv import load_dotenv; load_dotenv(); print('GCP_PROJECT_ID:', os.getenv('GCP_PROJECT_ID'))" + ``` + +3. **For Cloud Run, set environment variables in Terraform:** + - Check `terraform/6_agents/main.tf` for environment variable configuration + - Ensure all required variables are set in `terraform.tfvars` + +## Pub/Sub Issues + +### Error: "404 Requested project not found" + +**Symptoms:** +``` +google.api_core.exceptions.NotFound: 404 Requested project not found +``` + +**Solutions:** + +1. **Set GCP_PROJECT_ID in .env file:** + ```bash + GCP_PROJECT_ID=your-gcp-project-id + ``` + +2. **Verify current gcloud project:** + ```bash + gcloud config get-value project + ``` + +3. **Set gcloud default project:** + ```bash + gcloud config set project your-gcp-project-id + ``` + +### Error: "Permission denied" or "403 Forbidden" + +**Symptoms:** +``` +google.api_core.exceptions.PermissionDenied: 403 User does not have permission +``` + +**Solutions:** + +1. **Grant Pub/Sub Publisher role:** + ```bash + gcloud pubsub topics add-iam-policy-binding alex-job-queue \ + --member="serviceAccount:cloud-run-sa@your-gcp-project-id.iam.gserviceaccount.com" \ + --role="roles/pubsub.publisher" \ + --project=your-gcp-project-id + ``` + +2. **Verify topic exists:** + ```bash + gcloud pubsub topics list --project=your-gcp-project-id + ``` + +### Error: "Topic not found" + +**Solutions:** + +1. **Deploy Pub/Sub Terraform:** + ```bash + cd terraform/3_pubsub + terraform apply + ``` + +2. **Or create topic manually:** + ```bash + gcloud pubsub topics create alex-job-queue --project=your-gcp-project-id + ``` + +### Issue: Old Pub/Sub Messages with Invalid Job IDs + +**Symptoms:** +- Errors like `invalid input syntax for type uuid: "test-123"` +- Old test messages in queue + +**Solution: Purge the Pub/Sub subscription** +```bash +gcloud pubsub subscriptions seek alex-planner-subscription \ + --time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --project=your-gcp-project-id +``` + +## Cloud Run Logs + +### View Recent Logs + +```bash +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=alex-planner" \ + --limit 20 \ + --format="table(timestamp,severity,textPayload)" \ + --project=your-gcp-project-id \ + --freshness=10m +``` + +### View Only Errors + +```bash +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=alex-planner AND severity>=ERROR" \ + --limit 10 \ + --format="value(textPayload)" \ + --project=your-gcp-project-id \ + --freshness=10m +``` + +### Follow Logs in Real-Time + +```bash +gcloud logging tail "resource.type=cloud_run_revision AND resource.labels.service_name=alex-planner" \ + --project=your-gcp-project-id +``` + +### View Logs for All Agents + +```bash +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=~'alex-.*'" \ + --limit 50 \ + --format="table(timestamp,resource.labels.service_name,severity,textPayload)" \ + --project=your-gcp-project-id \ + --freshness=10m +``` + +## Docker Build Issues + +### Error: "ModuleNotFoundError: No module named 'pg8000'" + +**Symptoms:** +- Agents (Reporter, Charter, Retirement) fail with `ModuleNotFoundError: No module named 'pg8000'` +- Error occurs when agents try to connect to Cloud SQL database +- Planner works fine, but other agents return 500 errors + +**Root Cause:** +The `pg8000` package is a dependency of the `alex-database` package, but when using `uv sync --no-install-project`, transitive dependencies from local path dependencies may not be installed correctly. + +**Solution:** +Explicitly install the database package before syncing main project dependencies. Update the Dockerfiles for Reporter, Charter, and Retirement: + +```dockerfile +# Install Python dependencies +# Don't use --frozen because the lock file has the old path +# First install database package with all its dependencies (including pg8000) +# This ensures transitive dependencies from the local path dependency are installed +RUN cd database && uv pip install --system -e . && cd .. +# Then sync the main project dependencies +RUN uv sync --no-install-project +``` + +**Verification:** +After rebuilding and redeploying, check logs to confirm no more `pg8000` errors: +```bash +gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=~'alex-(reporter|charter|retirement)' AND severity>=ERROR" \ + --limit 20 \ + --format="table(timestamp,resource.labels.service_name,severity,textPayload)" \ + --project=your-gcp-project-id \ + --freshness=10m +``` + +### Error: "Distribution not found at: file:///database" + +**Cause:** Docker build context doesn't include the `database` directory. + +**Solution:** Build from `backend/` directory, not individual agent directories: + +```bash +cd backend + +# Build planner +docker build -f planner/Dockerfile -t us-central1-docker.pkg.dev/your-gcp-project-id/alex-agents/planner:latest . + +# Build other agents similarly +docker build -f tagger/Dockerfile -t us-central1-docker.pkg.dev/your-gcp-project-id/alex-agents/tagger:latest . +``` + +**Key Points:** +- Build context must be `backend/` directory +- Use `-f agent/Dockerfile` to specify Dockerfile location +- Dockerfiles copy `database` and `common` directories into image + +### Error: "sed command fails" + +If `sed -i` doesn't work in Dockerfile, use Python instead: + +```dockerfile +RUN python3 -c "import re; content = open('pyproject.toml').read(); open('pyproject.toml', 'w').write(re.sub(r'path = \"\.\./database\"', 'path = \"./database\"', content))" +``` + +## Database Connection Issues + +### Error: "Instance connection name not found" + +**Solutions:** + +1. **Verify Cloud SQL instance exists:** + ```bash + gcloud sql instances list --project=your-gcp-project-id + ``` + +2. **Get connection name:** + ```bash + gcloud sql instances describe alex-postgres --project=your-gcp-project-id --format="value(connectionName)" + ``` + +3. **Set environment variable:** + ```bash + INSTANCE_CONNECTION_NAME=your-gcp-project-id:us-central1:alex-postgres + ``` + +### Error: "Permission denied" for database + +**Solutions:** + +1. **Grant Cloud SQL Client role to service account:** + ```bash + gcloud projects add-iam-policy-binding your-gcp-project-id \ + --member="serviceAccount:cloud-run-sa@your-gcp-project-id.iam.gserviceaccount.com" \ + --role="roles/cloudsql.client" + ``` + +2. **Verify database password secret exists:** + ```bash + gcloud secrets versions access latest --secret="alex-db-password" --project=your-gcp-project-id + ``` + +## Authentication Issues + +### Error: "Application Default Credentials not found" + +**Solutions:** + +1. **Authenticate with gcloud:** + ```bash + gcloud auth application-default login + ``` + +2. **Set quota project:** + ```bash + gcloud auth application-default set-quota-project your-gcp-project-id + ``` + +3. **For Cloud Run, use service account:** + - Service accounts are automatically used by Cloud Run + - Verify service account has required IAM roles + +## API Not Enabled Errors + +### Error: "API has not been used in project before or it is disabled" + +**Symptoms:** +``` +Error 403: API has not been used in project before or it is disabled +``` + +**Solutions:** + +1. **Enable required APIs:** + ```bash + gcloud services enable \ + compute.googleapis.com \ + run.googleapis.com \ + cloudfunctions.googleapis.com \ + sqladmin.googleapis.com \ + aiplatform.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iam.googleapis.com \ + storage.googleapis.com \ + pubsub.googleapis.com \ + cloudbuild.googleapis.com \ + logging.googleapis.com \ + monitoring.googleapis.com \ + --project=your-gcp-project-id + ``` + +2. **Wait 2-5 minutes for API propagation** + +3. **Verify API is enabled:** + ```bash + gcloud services list --enabled --project=your-gcp-project-id + ``` + +## Quick Reference + +### Common Commands + +| Task | Command | +|------|---------| +| Check project | `gcloud config get-value project` | +| List Cloud Run services | `gcloud run services list --project=your-gcp-project-id` | +| View logs | `gcloud logging read "resource.type=cloud_run_revision" --limit=20` | +| Check Pub/Sub topics | `gcloud pubsub topics list --project=your-gcp-project-id` | +| List secrets | `gcloud secrets list --project=your-gcp-project-id` | +| Check Cloud SQL instances | `gcloud sql instances list --project=your-gcp-project-id` | + +### Environment Variables Checklist + +- [ ] `GCP_PROJECT_ID` set in `.env` +- [ ] `GCP_REGION` set in `.env` +- [ ] `PUBSUB_TOPIC` set in `.env` +- [ ] `INSTANCE_CONNECTION_NAME` set (for database) +- [ ] `DB_PASSWORD_SECRET_ID` set (for database) +- [ ] All required APIs enabled +- [ ] Service accounts have required IAM roles + +## Getting More Help + +1. **Check Terraform outputs:** + ```bash + cd terraform/[phase] + terraform output + ``` + +2. **Check Cloud Console:** + - Cloud Run: https://console.cloud.google.com/run + - Pub/Sub: https://console.cloud.google.com/cloudpubsub + - Cloud SQL: https://console.cloud.google.com/sql + - Logs: https://console.cloud.google.com/logs + +3. **Review guide files:** + - `guides/1_permissions.md` - IAM setup + - `guides/5_database.md` - Database setup + - `guides/6_agents.md` - Agent deployment + diff --git a/gcp-deployment/guides/WINDOWS_SETUP.md b/gcp-deployment/guides/WINDOWS_SETUP.md new file mode 100644 index 00000000..d399191f --- /dev/null +++ b/gcp-deployment/guides/WINDOWS_SETUP.md @@ -0,0 +1,181 @@ +# Windows Setup Guide for Alex GCP Deployment + +This guide covers setting up the GCP deployment environment on Windows. + +## Prerequisites Installation + +### 1. Install Google Cloud SDK + +Download and install from: https://cloud.google.com/sdk/docs/install + +After installation, open **PowerShell** and run: +```powershell +gcloud init +gcloud auth login +``` + +### 2. Install Terraform + +**Option A: Using Chocolatey (recommended)** +```powershell +# Install Chocolatey first if you don't have it +Set-ExecutionPolicy Bypass -Scope Process -Force +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 +iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + +# Install Terraform +choco install terraform +``` + +**Option B: Manual Installation** +1. Download from: https://developer.hashicorp.com/terraform/downloads +2. Extract to `C:\terraform` +3. Add `C:\terraform` to your PATH environment variable + +Verify installation: +```powershell +terraform --version +``` + +### 3. Install Docker Desktop + +1. Download from: https://www.docker.com/products/docker-desktop +2. Install and restart your computer +3. Start Docker Desktop + +Verify installation: +```powershell +docker --version +``` + +### 4. Install Git (if not already installed) + +Download from: https://git-scm.com/download/win + +## Project Setup + +### 1. Extract the Project + +Extract `alex-gcp-deployment.zip` to a folder, e.g., `C:\Projects\alex-gcp` + +### 2. Set Environment Variables + +Open PowerShell and set your project ID: +```powershell +$env:PROJECT_ID = "your-gcp-project-id" +$env:REGION = "us-central1" +``` + +To make these permanent, add to your PowerShell profile: +```powershell +notepad $PROFILE +# Add these lines: +# $env:PROJECT_ID = "your-gcp-project-id" +# $env:REGION = "us-central1" +``` + +### 3. Authenticate with GCP + +```powershell +gcloud auth login +gcloud auth application-default login +gcloud config set project $env:PROJECT_ID +``` + +## Deployment + +### Option 1: Using PowerShell Script + +```powershell +cd C:\Projects\alex-gcp + +# Allow script execution (one-time) +Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +# Run full deployment +.\scripts\deploy.ps1 full + +# Or deploy phase by phase +.\scripts\deploy.ps1 phase 1_permissions +.\scripts\deploy.ps1 phase 2_vertex_ai +.\scripts\deploy.ps1 phase 5_database +.\scripts\deploy.ps1 phase 6_agents +.\scripts\deploy.ps1 phase 7_frontend +``` + +### Option 2: Manual Terraform Commands + +```powershell +cd C:\Projects\alex-gcp\terraform\1_permissions + +# Copy and edit variables +Copy-Item terraform.tfvars.example terraform.tfvars +notepad terraform.tfvars # Edit with your values + +# Deploy +terraform init +terraform plan +terraform apply +``` + +## Windows-Specific Notes + +### Path Separators +Terraform on Windows handles both `/` and `\` path separators, so the terraform files work without modification. + +### Line Endings +If you encounter issues with scripts, ensure files have Windows line endings (CRLF). In VS Code: +- Click "LF" in the bottom right +- Select "CRLF" + +### Docker on Windows +- Ensure Docker Desktop is running before building images +- WSL 2 backend is recommended for better performance +- If you get permission errors, run PowerShell as Administrator + +### Long Path Support +If you encounter path length issues: +```powershell +# Run as Administrator +New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force +``` + +## Troubleshooting + +### "gcloud is not recognized" +Add Google Cloud SDK to PATH: +1. Open System Properties โ†’ Environment Variables +2. Add `C:\Users\\AppData\Local\Google\Cloud SDK\google-cloud-sdk\bin` to PATH + +### "terraform is not recognized" +Add Terraform to PATH or use full path: +```powershell +C:\terraform\terraform.exe init +``` + +### Docker Connection Errors +1. Ensure Docker Desktop is running +2. Check Docker is set to use Linux containers (right-click Docker icon โ†’ Switch to Linux containers) + +### Permission Denied on Scripts +```powershell +Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser +``` + +### SSL/TLS Errors +```powershell +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +``` + +## VS Code Setup (Recommended) + +Install these extensions for better experience: +- HashiCorp Terraform +- Google Cloud Code +- Docker + +## Next Steps + +1. Follow the guides in the `guides/` folder in order +2. Start with `1_permissions.md` +3. Edit `terraform.tfvars` files with your project ID before each phase diff --git a/gcp-deployment/scripts/DESTROY_README.md b/gcp-deployment/scripts/DESTROY_README.md new file mode 100644 index 00000000..94fb1100 --- /dev/null +++ b/gcp-deployment/scripts/DESTROY_README.md @@ -0,0 +1,143 @@ +# Destroy Script Usage + +The `destroy.ps1` script safely destroys all or selected parts of the Alex Multi-Agent SaaS deployment on GCP. + +## Basic Usage + +### Destroy Everything (Full Teardown) + +```powershell +.\scripts\destroy.ps1 +``` + +This will destroy all resources in reverse order: +1. Frontend (7_frontend) +2. Agents (6_agents) +3. Database (5_database) +4. Pub/Sub (3_pubsub) +5. Vertex AI (2_vertex_ai) +6. Permissions (1_permissions) + +### Destroy Specific Phases + +```powershell +# Destroy only frontend +.\scripts\destroy.ps1 -DestroyFrontend -DestroyAll:$false + +# Destroy only agents +.\scripts\destroy.ps1 -DestroyAgents -DestroyAll:$false + +# Destroy database (biggest cost savings) +.\scripts\destroy.ps1 -DestroyDatabase -DestroyAll:$false + +# Destroy multiple phases +.\scripts\destroy.ps1 -DestroyFrontend -DestroyAgents -DestroyAll:$false +``` + +### Skip Confirmation Prompt + +```powershell +.\scripts\destroy.ps1 -SkipConfirmation +``` + +### Destroy Secrets Too + +```powershell +.\scripts\destroy.ps1 -DestroySecrets +``` + +This will also delete: +- `alex-db-password` +- `polygon-api-key` +- `openai-api-key` +- `clerk-publishable-key` +- `clerk-secret-key` + +**Warning**: Only use this if you're completely tearing down the deployment. You'll need to recreate secrets if you redeploy. + +## Examples + +### Quick Cost Savings (Destroy Database) + +```powershell +# Destroy database when not actively working (saves ~$30-100/month) +.\scripts\destroy.ps1 -DestroyDatabase -DestroyAll:$false +``` + +### Complete Cleanup + +```powershell +# Destroy everything including secrets +.\scripts\destroy.ps1 -DestroySecrets -SkipConfirmation +``` + +### Partial Cleanup (Keep Database) + +```powershell +# Destroy frontend and agents, keep database +.\scripts\destroy.ps1 -DestroyFrontend -DestroyAgents -DestroyAll:$false +``` + +## Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `-ProjectId` | GCP Project ID (auto-detected from .env or gcloud config) | Auto | +| `-Region` | GCP Region | `us-central1` | +| `-SkipConfirmation` | Skip the confirmation prompt | `$false` | +| `-DestroySecrets` | Also destroy Secret Manager secrets | `$false` | +| `-DestroyAll` | Destroy all phases | `$true` | +| `-DestroyFrontend` | Destroy frontend only | `$false` | +| `-DestroyAgents` | Destroy agents only | `$false` | +| `-DestroyDatabase` | Destroy database only | `$false` | +| `-DestroyPubSub` | Destroy Pub/Sub only | `$false` | +| `-DestroyVertexAI` | Destroy Vertex AI only | `$false` | +| `-DestroyPermissions` | Destroy permissions only | `$false` | + +## Important Notes + +1. **Database Destruction**: The database is the most expensive resource. Destroy it when not actively working to save costs. + +2. **Dependencies**: The script destroys in reverse order to handle dependencies. If you destroy a phase manually, you may need to destroy dependent phases first. + +3. **State Files**: Terraform state files are kept locally. If you lose them, you may need to manually clean up resources in the GCP Console. + +4. **Secrets**: Only destroy secrets if you're completely removing the deployment. You'll need to recreate them for redeployment. + +5. **Deletion Protection**: Production databases may have deletion protection enabled. You'll need to disable it first in the GCP Console. + +## Troubleshooting + +### Terraform State Lock + +If you get a state lock error: +```powershell +# Manually unlock (use with caution) +cd terraform/[phase] +terraform force-unlock [LOCK_ID] +``` + +### Resources Not Destroying + +Some resources may have dependencies. Check the GCP Console for: +- Cloud Run services that depend on the database +- Pub/Sub subscriptions that depend on Cloud Run services +- IAM bindings that depend on service accounts + +### Manual Cleanup + +If the script fails, you can manually destroy resources: +```powershell +cd terraform/[phase] +terraform destroy +``` + +Or use gcloud commands: +```powershell +# List Cloud Run services +gcloud run services list --project=PROJECT_ID + +# Delete a service +gcloud run services delete SERVICE_NAME --region=REGION --project=PROJECT_ID +``` + diff --git a/gcp-deployment/scripts/auto_deploy_frontend.ps1 b/gcp-deployment/scripts/auto_deploy_frontend.ps1 new file mode 100644 index 00000000..70d8594f --- /dev/null +++ b/gcp-deployment/scripts/auto_deploy_frontend.ps1 @@ -0,0 +1,154 @@ +# Automatic Frontend Deployment +$ErrorActionPreference = "Stop" + +# Unset invalid GOOGLE_APPLICATION_CREDENTIALS if pointing to non-existent file +if ($env:GOOGLE_APPLICATION_CREDENTIALS -and -not (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS)) { + Remove-Item Env:GOOGLE_APPLICATION_CREDENTIALS +} + +# Load .env +$projectRoot = Split-Path -Parent $PSScriptRoot +$envPath = Join-Path $projectRoot ".env" +$envVars = @{} +if (Test-Path $envPath) { + Get-Content $envPath | ForEach-Object { + if ($_ -match '^\s*([^#][^=]*)\s*=\s*(.*)$') { + $key = $matches[1].Trim() + $value = $matches[2].Trim() -replace '^["'']|["'']$', '' + if ($value) { + $envVars[$key] = $value + } + } + } +} +Write-Host "Loaded $($envVars.Count) environment variables from .env" -ForegroundColor Cyan + +$PROJECT_ID = if ($envVars.ContainsKey("GCP_PROJECT_ID")) { $envVars["GCP_PROJECT_ID"] } else { "alex-multi-agent-saas-479504" } +$REGION = if ($envVars.ContainsKey("GCP_REGION")) { $envVars["GCP_REGION"] } else { "us-central1" } +$REPO = "alex-agents" +$ARTIFACT_REGISTRY = "${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}" + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Automatic Frontend Deployment" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +# Step 1: Build & Push API +Write-Host "[1/5] Building API image..." -ForegroundColor Yellow +docker build --platform linux/amd64 -f backend/api/Dockerfile -t ${ARTIFACT_REGISTRY}/api:latest backend/ +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Build failed" -ForegroundColor Red; exit 1 } +docker push ${ARTIFACT_REGISTRY}/api:latest +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Push failed" -ForegroundColor Red; exit 1 } +Write-Host "โœ“ API image ready" -ForegroundColor Green + +# Step 2: Create/Update Clerk Secrets +Write-Host "[2/5] Setting up Clerk secrets..." -ForegroundColor Yellow +$clerkPub = if ($envVars.ContainsKey("NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY") -and $envVars["NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"]) { $envVars["NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"] } else { "" } +$clerkSec = if ($envVars.ContainsKey("CLERK_SECRET_KEY") -and $envVars["CLERK_SECRET_KEY"]) { $envVars["CLERK_SECRET_KEY"] } else { "" } +Write-Host "Clerk Pub Key length: $($clerkPub.Length), Clerk Sec Key length: $($clerkSec.Length)" -ForegroundColor Gray +if ($clerkPub) { + echo $clerkPub | gcloud secrets create clerk-publishable-key --data-file=- --project=$PROJECT_ID 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + echo $clerkPub | gcloud secrets versions add clerk-publishable-key --data-file=- --project=$PROJECT_ID 2>&1 | Out-Null + } +} +if ($clerkSec) { + echo $clerkSec | gcloud secrets create clerk-secret-key --data-file=- --project=$PROJECT_ID 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + echo $clerkSec | gcloud secrets versions add clerk-secret-key --data-file=- --project=$PROJECT_ID 2>&1 | Out-Null + } +} +Write-Host "โœ“ Clerk secrets ready" -ForegroundColor Green + +# Step 3: Deploy API +Write-Host "[3/5] Deploying API service..." -ForegroundColor Yellow +Push-Location "$projectRoot/terraform/6_agents" +$clerkJwks = if ($envVars.ContainsKey("CLERK_JWKS_URL")) { $envVars["CLERK_JWKS_URL"] } else { "https://modern-sparrow-23.clerk.accounts.dev/.well-known/jwks.json" } +$clerkIssuer = if ($envVars.ContainsKey("CLERK_ISSUER")) { $envVars["CLERK_ISSUER"] } else { "https://modern-sparrow-23.clerk.accounts.dev" } + +# Update tfvars using simple string replacement +$tfvarsContent = Get-Content terraform.tfvars +$newTfvars = @() +foreach ($line in $tfvarsContent) { + if ($line -match '^\s*clerk_jwks_url\s*=') { + $newTfvars += "clerk_jwks_url = `"$clerkJwks`"" + } elseif ($line -match '^\s*clerk_issuer\s*=') { + $newTfvars += "clerk_issuer = `"$clerkIssuer`"" + } else { + $newTfvars += $line + } +} +Set-Content terraform.tfvars $newTfvars + +terraform apply -auto-approve +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Terraform apply failed" -ForegroundColor Red; Pop-Location; exit 1 } +$API_URL = terraform output -raw api_service_url +Pop-Location +Write-Host "โœ“ API deployed: $API_URL" -ForegroundColor Green + +# Step 4: Build & Push Frontend +Write-Host "[4/5] Building Frontend image..." -ForegroundColor Yellow +$clerkPubKey = if ($envVars.ContainsKey("NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY") -and $envVars["NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"]) { $envVars["NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"] } else { "" } +if (-not $clerkPubKey) { + Write-Host "WARNING: NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY not found in .env" -ForegroundColor Yellow +} +docker build --platform linux/amd64 ` + --build-arg NEXT_PUBLIC_API_URL=$API_URL ` + --build-arg NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$clerkPubKey ` + -f frontend/Dockerfile ` + -t ${ARTIFACT_REGISTRY}/frontend:latest ` + frontend/ +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Frontend build failed" -ForegroundColor Red; exit 1 } +docker push ${ARTIFACT_REGISTRY}/frontend:latest +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Frontend push failed" -ForegroundColor Red; exit 1 } +Write-Host "โœ“ Frontend image ready" -ForegroundColor Green + +# Step 5: Deploy Frontend +Write-Host "[5/5] Deploying Frontend..." -ForegroundColor Yellow +Push-Location "$projectRoot/terraform/7_frontend" +$tfvarsContent = Get-Content terraform.tfvars +$newTfvars = @() +foreach ($line in $tfvarsContent) { + if ($line -match '^\s*backend_api_url\s*=') { + $newTfvars += "backend_api_url = `"$API_URL`"" + } else { + $newTfvars += $line + } +} +Set-Content terraform.tfvars $newTfvars + +# Initialize Terraform if needed +Write-Host "Initializing Terraform..." -ForegroundColor Gray +terraform init +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Terraform init failed" -ForegroundColor Red; Pop-Location; exit 1 } + +terraform apply -auto-approve +if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Frontend terraform apply failed" -ForegroundColor Red; Pop-Location; exit 1 } +$FRONTEND_URL = terraform output -raw frontend_url +Pop-Location +Write-Host "โœ“ Frontend deployed: $FRONTEND_URL" -ForegroundColor Green + +# Step 6: Update API CORS +Write-Host "Updating API CORS..." -ForegroundColor Yellow +Push-Location "$projectRoot/terraform/6_agents" +$tfvarsContent = Get-Content terraform.tfvars +$newTfvars = @() +foreach ($line in $tfvarsContent) { + if ($line -match '^\s*frontend_url\s*=') { + $newTfvars += "frontend_url = `"$FRONTEND_URL`"" + } else { + $newTfvars += $line + } +} +Set-Content terraform.tfvars $newTfvars + +terraform apply -auto-approve +Pop-Location + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "โœ“ Deployment Complete!" -ForegroundColor Green +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Frontend: $FRONTEND_URL" -ForegroundColor Green +Write-Host "API: $API_URL" -ForegroundColor Green +Write-Host "" diff --git a/gcp-deployment/scripts/deploy.ps1 b/gcp-deployment/scripts/deploy.ps1 new file mode 100644 index 00000000..e0f86641 --- /dev/null +++ b/gcp-deployment/scripts/deploy.ps1 @@ -0,0 +1,235 @@ +# ============================================================================= +# GCP Deployment Script for Alex Multi-Agent SaaS (Windows PowerShell) +# ============================================================================= + +param( + [Parameter(Position=0)] + [ValidateSet("full", "phase", "build", "destroy", "help")] + [string]$Command = "help", + + [Parameter(Position=1)] + [string]$Phase = "" +) + +# Configuration - Set these or use environment variables +$PROJECT_ID = if ($env:PROJECT_ID) { $env:PROJECT_ID } else { "" } +$REGION = if ($env:REGION) { $env:REGION } else { "us-central1" } +$ENVIRONMENT = if ($env:ENVIRONMENT) { $env:ENVIRONMENT } else { "dev" } + +# Colors +function Write-Status { param($Message) Write-Host "[INFO] $Message" -ForegroundColor Green } +function Write-Warning { param($Message) Write-Host "[WARN] $Message" -ForegroundColor Yellow } +function Write-Error { param($Message) Write-Host "[ERROR] $Message" -ForegroundColor Red } + +# Check prerequisites +function Test-Prerequisites { + Write-Status "Checking prerequisites..." + + # Check gcloud + if (-not (Get-Command "gcloud" -ErrorAction SilentlyContinue)) { + Write-Error "gcloud CLI is not installed. Download from: https://cloud.google.com/sdk/docs/install" + exit 1 + } + + # Check terraform + if (-not (Get-Command "terraform" -ErrorAction SilentlyContinue)) { + Write-Error "Terraform is not installed. Download from: https://developer.hashicorp.com/terraform/downloads" + exit 1 + } + + # Check docker + if (-not (Get-Command "docker" -ErrorAction SilentlyContinue)) { + Write-Error "Docker is not installed. Download Docker Desktop from: https://www.docker.com/products/docker-desktop" + exit 1 + } + + # Check PROJECT_ID + if ([string]::IsNullOrEmpty($PROJECT_ID)) { + Write-Error "PROJECT_ID is not set. Run: `$env:PROJECT_ID = 'your-project-id'" + exit 1 + } + + Write-Status "All prerequisites met!" +} + +# Setup GCP project +function Set-GCPProject { + Write-Status "Setting up GCP project..." + + gcloud config set project $PROJECT_ID + gcloud config set compute/region $REGION + + Write-Status "Enabling required APIs..." + $apis = @( + "compute.googleapis.com", + "run.googleapis.com", + "cloudfunctions.googleapis.com", + "sqladmin.googleapis.com", + "aiplatform.googleapis.com", + "artifactregistry.googleapis.com", + "secretmanager.googleapis.com", + "cloudresourcemanager.googleapis.com", + "iam.googleapis.com", + "storage.googleapis.com", + "servicenetworking.googleapis.com", + "cloudbuild.googleapis.com" + ) + + gcloud services enable $apis +} + +# Deploy a phase +function Deploy-Phase { + param([string]$PhaseName) + + Write-Status "Deploying phase: $PhaseName" + + $originalLocation = Get-Location + Set-Location "terraform\$PhaseName" + + # Check for tfvars + if (-not (Test-Path "terraform.tfvars")) { + Write-Warning "terraform.tfvars not found, copying from example..." + if (Test-Path "terraform.tfvars.example") { + Copy-Item "terraform.tfvars.example" "terraform.tfvars" + Write-Warning "Please edit terraform\$PhaseName\terraform.tfvars with your values" + } + } + + # Run terraform + terraform init + if ($LASTEXITCODE -ne 0) { Set-Location $originalLocation; exit 1 } + + terraform plan -out=tfplan + if ($LASTEXITCODE -ne 0) { Set-Location $originalLocation; exit 1 } + + terraform apply tfplan + if ($LASTEXITCODE -ne 0) { Set-Location $originalLocation; exit 1 } + + Set-Location $originalLocation +} + +# Build and push container images +function Build-Images { + Write-Status "Building and pushing container images..." + + # Configure Docker for Artifact Registry + gcloud auth configure-docker "$REGION-docker.pkg.dev" + + $REPO = "$REGION-docker.pkg.dev/$PROJECT_ID/alex-containers" + + # Build each agent + $agents = @("researcher", "planner", "executor", "orchestrator") + foreach ($agent in $agents) { + $agentPath = "backend\agents\$agent" + if (Test-Path $agentPath) { + Write-Status "Building $agent agent..." + docker build -t "$REPO/${agent}:latest" -f "$agentPath\Dockerfile" $agentPath + docker push "$REPO/${agent}:latest" + } + } + + # Build frontend + if (Test-Path "frontend") { + Write-Status "Building frontend..." + docker build -t "$REPO/frontend:latest" frontend + docker push "$REPO/frontend:latest" + } +} + +# Full deployment +function Invoke-FullDeploy { + Test-Prerequisites + Set-GCPProject + + Write-Status "Starting full deployment..." + + Deploy-Phase "1_permissions" + Deploy-Phase "2_vertex_ai" + Deploy-Phase "5_database" + + Build-Images + + Deploy-Phase "6_agents" + Deploy-Phase "7_frontend" + + Write-Status "Deployment complete!" + + Set-Location "terraform\7_frontend" + $frontendUrl = terraform output -raw frontend_url + Set-Location "..\6_agents" + $backendUrl = terraform output -raw orchestrator_url + Set-Location "..\.." + + Write-Status "Frontend URL: $frontendUrl" + Write-Status "Backend API URL: $backendUrl" +} + +# Destroy all resources +function Invoke-Destroy { + Write-Warning "This will destroy all resources. Continue? (y/N)" + $confirm = Read-Host + + if ($confirm -ne "y") { + Write-Status "Aborted" + exit 0 + } + + $phases = @("7_frontend", "6_agents", "5_database", "2_vertex_ai", "1_permissions") + $originalLocation = Get-Location + + foreach ($phase in $phases) { + Write-Status "Destroying $phase..." + Set-Location "terraform\$phase" + terraform destroy -auto-approve + Set-Location $originalLocation + } + + Write-Status "All resources destroyed" +} + +# Show help +function Show-Help { + Write-Host @" +GCP Deployment Script for Alex Multi-Agent SaaS + +Usage: .\deploy.ps1 [options] + +Commands: + full Run full deployment + phase Deploy specific phase (e.g., 1_permissions) + build Build and push container images + destroy Destroy all resources + help Show this help message + +Before running, set environment variables: + `$env:PROJECT_ID = "your-gcp-project-id" + `$env:REGION = "us-central1" # optional, defaults to us-central1 + +Examples: + .\deploy.ps1 full + .\deploy.ps1 phase 1_permissions + .\deploy.ps1 build + .\deploy.ps1 destroy +"@ +} + +# Main +switch ($Command) { + "full" { Invoke-FullDeploy } + "phase" { + if ([string]::IsNullOrEmpty($Phase)) { + Write-Error "Please specify a phase (e.g., .\deploy.ps1 phase 1_permissions)" + exit 1 + } + Test-Prerequisites + Deploy-Phase $Phase + } + "build" { + Test-Prerequisites + Build-Images + } + "destroy" { Invoke-Destroy } + "help" { Show-Help } + default { Show-Help } +} diff --git a/gcp-deployment/scripts/deploy.sh b/gcp-deployment/scripts/deploy.sh new file mode 100644 index 00000000..b550577f --- /dev/null +++ b/gcp-deployment/scripts/deploy.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# ============================================================================= +# GCP Deployment Script for Alex Multi-Agent SaaS +# ============================================================================= + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Configuration +PROJECT_ID="${PROJECT_ID:-}" +REGION="${REGION:-us-central1}" +ENVIRONMENT="${ENVIRONMENT:-dev}" + +# Function to print colored output +print_status() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check prerequisites +check_prerequisites() { + print_status "Checking prerequisites..." + + if ! command -v gcloud &> /dev/null; then + print_error "gcloud CLI is not installed" + exit 1 + fi + + if ! command -v terraform &> /dev/null; then + print_error "Terraform is not installed" + exit 1 + fi + + if ! command -v docker &> /dev/null; then + print_error "Docker is not installed" + exit 1 + fi + + if [ -z "$PROJECT_ID" ]; then + print_error "PROJECT_ID environment variable is not set" + exit 1 + fi + + print_status "All prerequisites met!" +} + +# Set up GCP project +setup_project() { + print_status "Setting up GCP project..." + + gcloud config set project $PROJECT_ID + gcloud config set compute/region $REGION + + # Enable APIs + print_status "Enabling required APIs..." + gcloud services enable \ + compute.googleapis.com \ + run.googleapis.com \ + cloudfunctions.googleapis.com \ + sqladmin.googleapis.com \ + aiplatform.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iam.googleapis.com \ + storage.googleapis.com \ + servicenetworking.googleapis.com \ + cloudbuild.googleapis.com +} + +# Deploy phase +deploy_phase() { + local phase=$1 + print_status "Deploying phase: $phase" + + cd "terraform/$phase" + + if [ ! -f "terraform.tfvars" ]; then + print_warning "terraform.tfvars not found, using defaults" + cp terraform.tfvars.example terraform.tfvars 2>/dev/null || true + fi + + terraform init + terraform plan -out=tfplan + terraform apply tfplan + + cd ../.. +} + +# Build and push container images +build_images() { + print_status "Building and pushing container images..." + + # Configure Docker for Artifact Registry + gcloud auth configure-docker ${REGION}-docker.pkg.dev + + REPO="${REGION}-docker.pkg.dev/${PROJECT_ID}/alex-containers" + + # Build each agent + for agent in researcher planner executor orchestrator; do + if [ -d "backend/agents/$agent" ]; then + print_status "Building $agent agent..." + docker build -t ${REPO}/${agent}:latest -f backend/agents/$agent/Dockerfile backend/agents/$agent + docker push ${REPO}/${agent}:latest + fi + done + + # Build frontend + if [ -d "frontend" ]; then + print_status "Building frontend..." + docker build -t ${REPO}/frontend:latest frontend + docker push ${REPO}/frontend:latest + fi +} + +# Full deployment +full_deploy() { + check_prerequisites + setup_project + + print_status "Starting full deployment..." + + # Phase 1: Permissions + deploy_phase "1_permissions" + + # Phase 2: Vertex AI + deploy_phase "2_vertex_ai" + + # Phase 5: Database (skipping 3 and 4 as they depend on actual code) + deploy_phase "5_database" + + # Build images before deploying agents + build_images + + # Phase 6: Agents + deploy_phase "6_agents" + + # Phase 7: Frontend + deploy_phase "7_frontend" + + print_status "Deployment complete!" + print_status "Frontend URL: $(terraform -chdir=terraform/7_frontend output -raw frontend_url)" + print_status "Backend API URL: $(terraform -chdir=terraform/6_agents output -raw orchestrator_url)" +} + +# Destroy all resources +destroy_all() { + print_warning "This will destroy all resources. Continue? (y/N)" + read -r confirm + + if [ "$confirm" != "y" ]; then + print_status "Aborted" + exit 0 + fi + + for phase in 7_frontend 6_agents 5_database 2_vertex_ai 1_permissions; do + print_status "Destroying $phase..." + cd "terraform/$phase" + terraform destroy -auto-approve || true + cd ../.. + done + + print_status "All resources destroyed" +} + +# Main +case "${1:-}" in + "full") + full_deploy + ;; + "phase") + if [ -z "${2:-}" ]; then + print_error "Please specify a phase (e.g., 1_permissions)" + exit 1 + fi + deploy_phase "$2" + ;; + "build") + build_images + ;; + "destroy") + destroy_all + ;; + *) + echo "Usage: $0 {full|phase |build|destroy}" + echo "" + echo "Commands:" + echo " full - Run full deployment" + echo " phase - Deploy specific phase (e.g., 1_permissions)" + echo " build - Build and push container images" + echo " destroy - Destroy all resources" + exit 1 + ;; +esac diff --git a/gcp-deployment/scripts/deploy_frontend.ps1 b/gcp-deployment/scripts/deploy_frontend.ps1 new file mode 100644 index 00000000..c8ef7a28 --- /dev/null +++ b/gcp-deployment/scripts/deploy_frontend.ps1 @@ -0,0 +1,196 @@ +# Quick Frontend Deployment Script +# This script deploys both backend API and frontend + +param( + [string]$ProjectId = "", + [string]$Region = "us-central1" +) + +# Load .env +$projectRoot = Split-Path -Parent $PSScriptRoot +$envPath = Join-Path $projectRoot ".env" +if (Test-Path $envPath) { + Get-Content $envPath | ForEach-Object { + if ($_ -match '^\s*([^#][^=]*)\s*=\s*(.*)$') { + $key = $matches[1].Trim() + $value = $matches[2].Trim() + if ($value -match '^["''](.*)["'']$') { + $value = $matches[1] + } + Set-Item -Path "env:$key" -Value $value + } + } +} + +# Get project ID +if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = $env:GCP_PROJECT_ID + if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = gcloud config get-value project 2>$null + } +} + +$REGION = $Region +$REPO = "alex-agents" +$ARTIFACT_REGISTRY = "${REGION}-docker.pkg.dev/${ProjectId}/${REPO}" + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Frontend Deployment" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Project: $ProjectId" -ForegroundColor Gray +Write-Host "Region: $REGION" -ForegroundColor Gray +Write-Host "" + +# Step 1: Build and push API +Write-Host "Step 1: Building API Docker image..." -ForegroundColor Yellow +docker build --platform linux/amd64 -f backend/api/Dockerfile -t ${ARTIFACT_REGISTRY}/api:latest backend/ 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: API build failed" -ForegroundColor Red + exit 1 +} + +Write-Host "Pushing API image..." -ForegroundColor Yellow +docker push ${ARTIFACT_REGISTRY}/api:latest 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: API push failed" -ForegroundColor Red + exit 1 +} +Write-Host "โœ“ API image pushed" -ForegroundColor Green +Write-Host "" + +# Step 2: Create Clerk secrets +Write-Host "Step 2: Creating Clerk secrets..." -ForegroundColor Yellow +$clerkPublishable = $env:NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY +$clerkSecret = $env:CLERK_SECRET_KEY + +if ([string]::IsNullOrEmpty($clerkPublishable) -or [string]::IsNullOrEmpty($clerkSecret)) { + Write-Host "WARNING: Clerk keys not found in .env. Creating empty secrets..." -ForegroundColor Yellow + echo "placeholder" | gcloud secrets create clerk-publishable-key --data-file=- --project=$ProjectId 2>&1 | Out-Null + echo "placeholder" | gcloud secrets create clerk-secret-key --data-file=- --project=$ProjectId 2>&1 | Out-Null +} else { + echo $clerkPublishable | gcloud secrets create clerk-publishable-key --data-file=- --project=$ProjectId 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + # Secret might exist, update it + echo $clerkPublishable | gcloud secrets versions add clerk-publishable-key --data-file=- --project=$ProjectId 2>&1 | Out-Null + } + + echo $clerkSecret | gcloud secrets create clerk-secret-key --data-file=- --project=$ProjectId 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + echo $clerkSecret | gcloud secrets versions add clerk-secret-key --data-file=- --project=$ProjectId 2>&1 | Out-Null + } +} +Write-Host "โœ“ Clerk secrets ready" -ForegroundColor Green +Write-Host "" + +# Step 3: Deploy API via Terraform +Write-Host "Step 3: Deploying API service..." -ForegroundColor Yellow +Push-Location "$projectRoot/terraform/6_agents" + +# Get values from existing tfvars +$tfvarsPath = Join-Path (Get-Location) "terraform.tfvars" +if (-not (Test-Path $tfvarsPath)) { + Write-Host "ERROR: terraform.tfvars not found. Copy from terraform.tfvars.example" -ForegroundColor Red + Pop-Location + exit 1 +} + +# Add Clerk and CORS variables to tfvars if not present +$tfvarsContent = Get-Content $tfvarsPath -Raw +if ($tfvarsContent -notmatch "clerk_jwks_url") { + Add-Content $tfvarsPath "`nclerk_jwks_url = `"$($env:CLERK_JWKS_URL)`"" + Add-Content $tfvarsPath "clerk_issuer = `"$($env:CLERK_ISSUER)`"" + Add-Content $tfvarsPath "frontend_url = `"`" # Will be updated after frontend deployment" + Add-Content $tfvarsPath "cors_origins = `"`"" +} + +terraform apply -auto-approve 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Terraform apply failed" -ForegroundColor Red + Pop-Location + exit 1 +} + +# Get API URL +$apiUrl = terraform output -raw api_service_url 2>$null +Write-Host "โœ“ API deployed: $apiUrl" -ForegroundColor Green +Pop-Location +Write-Host "" + +# Step 4: Build and push Frontend +Write-Host "Step 4: Building Frontend Docker image..." -ForegroundColor Yellow +Push-Location "$projectRoot/frontend" + +# Set API URL for build +$env:NEXT_PUBLIC_API_URL = $apiUrl + +docker build --platform linux/amd64 -t ${ARTIFACT_REGISTRY}/frontend:latest . 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Frontend build failed" -ForegroundColor Red + Pop-Location + exit 1 +} + +Write-Host "Pushing Frontend image..." -ForegroundColor Yellow +docker push ${ARTIFACT_REGISTRY}/frontend:latest 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Frontend push failed" -ForegroundColor Red + Pop-Location + exit 1 +} +Write-Host "โœ“ Frontend image pushed" -ForegroundColor Green +Pop-Location +Write-Host "" + +# Step 5: Deploy Frontend +Write-Host "Step 5: Deploying Frontend..." -ForegroundColor Yellow +Push-Location "$projectRoot/terraform/7_frontend" + +# Create tfvars if not exists +$tfvarsPath = Join-Path (Get-Location) "terraform.tfvars" +if (-not (Test-Path $tfvarsPath)) { + Copy-Item terraform.tfvars.example terraform.tfvars +} + +# Update tfvars with API URL +$tfvarsContent = Get-Content $tfvarsPath -Raw +$tfvarsContent = $tfvarsContent -replace 'backend_api_url\s*=\s*"[^"]*"', "backend_api_url = `"$apiUrl`"" +Set-Content $tfvarsPath $tfvarsContent + +terraform apply -auto-approve 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Frontend terraform apply failed" -ForegroundColor Red + Pop-Location + exit 1 +} + +$frontendUrl = terraform output -raw frontend_service_url 2>$null +Write-Host "โœ“ Frontend deployed: $frontendUrl" -ForegroundColor Green +Pop-Location +Write-Host "" + +# Step 6: Update API CORS with frontend URL +Write-Host "Step 6: Updating API CORS configuration..." -ForegroundColor Yellow +Push-Location "$projectRoot/terraform/6_agents" + +# Update tfvars with frontend URL +$tfvarsContent = Get-Content terraform.tfvars -Raw +$tfvarsContent = $tfvarsContent -replace 'frontend_url\s*=\s*"[^"]*"', "frontend_url = `"$frontendUrl`"" +Set-Content terraform.tfvars $tfvarsContent + +terraform apply -auto-approve 2>&1 | Out-Null +Write-Host "โœ“ CORS updated" -ForegroundColor Green +Pop-Location + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Deployment Complete!" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Frontend URL: $frontendUrl" -ForegroundColor Green +Write-Host "API URL: $apiUrl" -ForegroundColor Green +Write-Host "" +Write-Host "Test the deployment:" -ForegroundColor Yellow +Write-Host " curl $apiUrl/health" -ForegroundColor Gray +Write-Host " curl $frontendUrl" -ForegroundColor Gray +Write-Host "" + diff --git a/gcp-deployment/scripts/destroy.ps1 b/gcp-deployment/scripts/destroy.ps1 new file mode 100644 index 00000000..d769fc57 --- /dev/null +++ b/gcp-deployment/scripts/destroy.ps1 @@ -0,0 +1,204 @@ +# ============================================================================= +# Destroy Alex Multi-Agent SaaS Deployment on GCP +# ============================================================================= +# This script destroys all GCP resources created by the Alex deployment +# Resources are destroyed in reverse order of deployment to handle dependencies +# ============================================================================= + +param( + [string]$ProjectId = "", + [string]$Region = "us-central1", + [switch]$SkipConfirmation = $false, + [switch]$DestroySecrets = $false, + [switch]$DestroyAll = $true, + [switch]$DestroyFrontend = $false, + [switch]$DestroyAgents = $false, + [switch]$DestroyDatabase = $false, + [switch]$DestroyPubSub = $false, + [switch]$DestroyVertexAI = $false, + [switch]$DestroyPermissions = $false +) + +# Load .env file if it exists +$projectRoot = Split-Path -Parent $PSScriptRoot +$envPath = Join-Path $projectRoot ".env" +if (Test-Path $envPath) { + Get-Content $envPath | ForEach-Object { + if ($_ -match '^\s*([^#][^=]*)\s*=\s*(.*)$') { + $key = $matches[1].Trim() + $value = $matches[2].Trim() + if ($value -match '^["''](.*)["'']$') { + $value = $matches[1] + } + Set-Item -Path "env:$key" -Value $value + } + } +} + +# Get project ID +if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = $env:GCP_PROJECT_ID + if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = gcloud config get-value project 2>$null + if ([string]::IsNullOrEmpty($ProjectId)) { + Write-Host "ERROR: Could not determine project ID. Set GCP_PROJECT_ID in .env or use -ProjectId parameter" -ForegroundColor Red + exit 1 + } + } +} + +# Determine what to destroy +$destroyPhases = @() + +if ($DestroyAll) { + $destroyPhases = @("7_frontend", "6_agents", "5_database", "3_pubsub", "2_vertex_ai", "1_permissions") +} else { + if ($DestroyFrontend) { $destroyPhases += "7_frontend" } + if ($DestroyAgents) { $destroyPhases += "6_agents" } + if ($DestroyDatabase) { $destroyPhases += "5_database" } + if ($DestroyPubSub) { $destroyPhases += "3_pubsub" } + if ($DestroyVertexAI) { $destroyPhases += "2_vertex_ai" } + if ($DestroyPermissions) { $destroyPhases += "1_permissions" } +} + +if ($destroyPhases.Count -eq 0) { + Write-Host "No phases selected for destruction. Use -DestroyAll or specific phase flags." -ForegroundColor Yellow + exit 0 +} + +Write-Host "" +Write-Host "========================================" -ForegroundColor Red +Write-Host " DESTROY ALEX DEPLOYMENT" -ForegroundColor Red +Write-Host "========================================" -ForegroundColor Red +Write-Host "" +Write-Host "Project ID: $ProjectId" -ForegroundColor Yellow +Write-Host "Region: $Region" -ForegroundColor Yellow +Write-Host "" +Write-Host "Phases to destroy:" -ForegroundColor Yellow +foreach ($phase in $destroyPhases) { + Write-Host " - $phase" -ForegroundColor Yellow +} +Write-Host "" + +if ($DestroySecrets) { + Write-Host "โš ๏ธ WARNING: Secrets will also be destroyed!" -ForegroundColor Red + Write-Host "" +} + +if (-not $SkipConfirmation) { + $confirmation = Read-Host "Are you sure you want to destroy these resources? (yes/no)" + if ($confirmation -ne "yes") { + Write-Host "Destruction cancelled." -ForegroundColor Green + exit 0 + } +} + +Write-Host "" +Write-Host "Starting destruction..." -ForegroundColor Cyan +Write-Host "" + +$terraformDir = Join-Path $projectRoot "terraform" +$errors = @() + +foreach ($phase in $destroyPhases) { + $phaseDir = Join-Path $terraformDir $phase + + if (-not (Test-Path $phaseDir)) { + Write-Host "โš ๏ธ Phase directory not found: $phase" -ForegroundColor Yellow + continue + } + + Write-Host "----------------------------------------" -ForegroundColor Cyan + Write-Host "Destroying: $phase" -ForegroundColor Cyan + Write-Host "----------------------------------------" -ForegroundColor Cyan + + Push-Location $phaseDir + + try { + # Check if terraform is initialized + if (-not (Test-Path ".terraform")) { + Write-Host "Initializing Terraform..." -ForegroundColor Gray + terraform init -upgrade 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Host "โš ๏ธ Failed to initialize Terraform for $phase" -ForegroundColor Yellow + Pop-Location + continue + } + } + + # Destroy resources + Write-Host "Running terraform destroy..." -ForegroundColor Gray + terraform destroy -auto-approve + + if ($LASTEXITCODE -eq 0) { + Write-Host "โœ… Successfully destroyed $phase" -ForegroundColor Green + } else { + Write-Host "โŒ Error destroying $phase" -ForegroundColor Red + $errors += $phase + } + } + catch { + Write-Host "โŒ Exception destroying $phase : $_" -ForegroundColor Red + $errors += $phase + } + finally { + Pop-Location + } + + Write-Host "" +} + +# Destroy secrets if requested +if ($DestroySecrets) { + Write-Host "----------------------------------------" -ForegroundColor Cyan + Write-Host "Destroying Secrets" -ForegroundColor Cyan + Write-Host "----------------------------------------" -ForegroundColor Cyan + + $secrets = @( + "alex-db-password", + "polygon-api-key", + "openai-api-key", + "clerk-publishable-key", + "clerk-secret-key" + ) + + foreach ($secret in $secrets) { + Write-Host "Checking secret: $secret..." -ForegroundColor Gray + $exists = gcloud secrets describe $secret --project=$ProjectId 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Destroying secret: $secret..." -ForegroundColor Yellow + gcloud secrets delete $secret --project=$ProjectId --quiet + if ($LASTEXITCODE -eq 0) { + Write-Host "โœ… Destroyed secret: $secret" -ForegroundColor Green + } else { + Write-Host "โš ๏ธ Failed to destroy secret: $secret" -ForegroundColor Yellow + } + } else { + Write-Host "Secret $secret does not exist, skipping..." -ForegroundColor Gray + } + } + Write-Host "" +} + +# Summary +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " DESTRUCTION SUMMARY" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +if ($errors.Count -eq 0) { + Write-Host "โœ… All selected phases destroyed successfully!" -ForegroundColor Green +} else { + Write-Host "โš ๏ธ Some phases had errors:" -ForegroundColor Yellow + foreach ($error in $errors) { + Write-Host " - $error" -ForegroundColor Yellow + } + Write-Host "" + Write-Host "You may need to manually clean up these resources in the GCP Console." -ForegroundColor Yellow +} + +Write-Host "" +Write-Host "Note: Some resources may take a few minutes to fully delete." -ForegroundColor Gray +Write-Host "Check the GCP Console to verify all resources are removed." -ForegroundColor Gray +Write-Host "" + diff --git a/gcp-deployment/scripts/quick_deploy.ps1 b/gcp-deployment/scripts/quick_deploy.ps1 new file mode 100644 index 00000000..f484156c --- /dev/null +++ b/gcp-deployment/scripts/quick_deploy.ps1 @@ -0,0 +1,57 @@ +# Quick Deployment Script - Deploys API and Frontend +# Run this from the alex-gcp directory + +$PROJECT_ID = "alex-multi-agent-saas-479504" +$REGION = "us-central1" +$REPO = "alex-agents" +$ARTIFACT_REGISTRY = "${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}" + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Quick Frontend Deployment" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +# Step 1: Build API +Write-Host "Building API..." -ForegroundColor Yellow +docker build --platform linux/amd64 -f backend/api/Dockerfile -t ${ARTIFACT_REGISTRY}/api:latest backend/ +docker push ${ARTIFACT_REGISTRY}/api:latest +Write-Host "โœ“ API built" -ForegroundColor Green + +# Step 2: Update Terraform with API +Write-Host "Deploying API..." -ForegroundColor Yellow +cd terraform/6_agents +terraform apply -auto-approve +$API_URL = terraform output -raw api_service_url +cd ../.. +Write-Host "โœ“ API deployed: $API_URL" -ForegroundColor Green + +# Step 3: Build Frontend +Write-Host "Building Frontend..." -ForegroundColor Yellow +$env:NEXT_PUBLIC_API_URL = $API_URL +docker build --platform linux/amd64 -f frontend/Dockerfile -t ${ARTIFACT_REGISTRY}/frontend:latest frontend/ +docker push ${ARTIFACT_REGISTRY}/frontend:latest +Write-Host "โœ“ Frontend built" -ForegroundColor Green + +# Step 4: Deploy Frontend +Write-Host "Deploying Frontend..." -ForegroundColor Yellow +cd terraform/7_frontend +# Update backend_api_url in tfvars +(Get-Content terraform.tfvars) -replace 'backend_api_url\s*=\s*"[^"]*"', "backend_api_url = `"$API_URL`"" | Set-Content terraform.tfvars +terraform apply -auto-approve +$FRONTEND_URL = terraform output -raw frontend_url +cd ../.. + +# Step 5: Update API CORS +Write-Host "Updating API CORS..." -ForegroundColor Yellow +cd terraform/6_agents +(Get-Content terraform.tfvars) -replace 'frontend_url\s*=\s*"[^"]*"', "frontend_url = `"$FRONTEND_URL`"" | Set-Content terraform.tfvars +terraform apply -auto-approve +cd ../.. + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Deployment Complete!" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Frontend: $FRONTEND_URL" -ForegroundColor Green +Write-Host "API: $API_URL" -ForegroundColor Green + diff --git a/gcp-deployment/scripts/rebuild_agents.ps1 b/gcp-deployment/scripts/rebuild_agents.ps1 new file mode 100644 index 00000000..eb8fe7a7 --- /dev/null +++ b/gcp-deployment/scripts/rebuild_agents.ps1 @@ -0,0 +1,111 @@ +# Rebuild and push all agent Docker images +# This script rebuilds images after code changes (e.g., database client fixes) + +param( + [string]$ProjectId = "", + [string]$Region = "us-central1", + [string]$Repo = "alex-agents", + [string]$ImageTag = "latest" +) + +# Load .env file if it exists +$projectRoot = Split-Path -Parent $PSScriptRoot +$envPath = Join-Path $projectRoot ".env" +if (Test-Path $envPath) { + Get-Content $envPath | ForEach-Object { + if ($_ -match '^\s*([^#][^=]*)\s*=\s*(.*)$') { + $key = $matches[1].Trim() + $value = $matches[2].Trim() + if ($value -match '^["''](.*)["'']$') { + $value = $matches[1] + } + Set-Item -Path "env:$key" -Value $value + } + } +} + +# Get project ID +if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = $env:GCP_PROJECT_ID + if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = gcloud config get-value project 2>$null + if ([string]::IsNullOrEmpty($ProjectId)) { + Write-Host "ERROR: Could not determine project ID. Set GCP_PROJECT_ID in .env or use -ProjectId parameter" -ForegroundColor Red + exit 1 + } + } +} + +$ARTIFACT_REGISTRY = "${Region}-docker.pkg.dev/${ProjectId}/${Repo}" + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Rebuilding Agent Docker Images" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Project: $ProjectId" -ForegroundColor Gray +Write-Host "Region: $Region" -ForegroundColor Gray +Write-Host "Repository: $Repo" -ForegroundColor Gray +Write-Host "Tag: $ImageTag" -ForegroundColor Gray +Write-Host "" + +# Authenticate Docker +Write-Host "Authenticating Docker with Artifact Registry..." -ForegroundColor Yellow +gcloud auth configure-docker "${Region}-docker.pkg.dev" 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Failed to authenticate Docker" -ForegroundColor Red + exit 1 +} +Write-Host "โœ“ Docker authenticated" -ForegroundColor Green +Write-Host "" + +# Build and push each agent +$agents = @("planner", "tagger", "reporter", "charter", "retirement") +$backendDir = Join-Path $projectRoot "backend" + +foreach ($agent in $agents) { + Write-Host "----------------------------------------" -ForegroundColor Cyan + Write-Host "Building $agent..." -ForegroundColor Cyan + Write-Host "----------------------------------------" -ForegroundColor Cyan + + $imageName = "${ARTIFACT_REGISTRY}/${agent}:${ImageTag}" + + # Build the image + Write-Host "Building Docker image..." -ForegroundColor Yellow + docker build --platform linux/amd64 ` + -f "$backendDir/$agent/Dockerfile" ` + -t $imageName ` + $backendDir + + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Failed to build $agent image" -ForegroundColor Red + continue + } + + Write-Host "โœ“ Image built successfully" -ForegroundColor Green + + # Push the image + Write-Host "Pushing to Artifact Registry..." -ForegroundColor Yellow + docker push $imageName + + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Failed to push $agent image" -ForegroundColor Red + continue + } + + Write-Host "โœ“ $agent pushed successfully" -ForegroundColor Green + Write-Host "" +} + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Rebuild Complete!" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Next steps:" -ForegroundColor Yellow +Write-Host "1. Apply Terraform to update Cloud Run services:" -ForegroundColor Gray +Write-Host " cd terraform/6_agents" -ForegroundColor Gray +Write-Host " terraform apply" -ForegroundColor Gray +Write-Host "" +Write-Host "2. Or force Cloud Run to use new image (if using 'latest' tag):" -ForegroundColor Gray +Write-Host " gcloud run services update alex-planner --region=$Region --image=$ARTIFACT_REGISTRY/planner:$ImageTag" -ForegroundColor Gray +Write-Host "" + diff --git a/gcp-deployment/scripts/update_cloud_run_images.ps1 b/gcp-deployment/scripts/update_cloud_run_images.ps1 new file mode 100644 index 00000000..0772f9dc --- /dev/null +++ b/gcp-deployment/scripts/update_cloud_run_images.ps1 @@ -0,0 +1,76 @@ +# Update Cloud Run services to use new Docker images +# Run this after rebuilding Docker images + +param( + [string]$ProjectId = "", + [string]$Region = "us-central1", + [string]$Repo = "alex-agents", + [string]$ImageTag = "latest" +) + +# Load .env if available +$projectRoot = Split-Path -Parent $PSScriptRoot +$envPath = Join-Path $projectRoot ".env" +if (Test-Path $envPath) { + Get-Content $envPath | ForEach-Object { + if ($_ -match '^\s*([^#][^=]*)\s*=\s*(.*)$') { + $key = $matches[1].Trim() + $value = $matches[2].Trim() + if ($value -match '^["''](.*)["'']$') { + $value = $matches[1] + } + Set-Item -Path "env:$key" -Value $value + } + } +} + +# Get project ID +if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = $env:GCP_PROJECT_ID + if ([string]::IsNullOrEmpty($ProjectId)) { + $ProjectId = gcloud config get-value project 2>$null + if ([string]::IsNullOrEmpty($ProjectId)) { + Write-Host "ERROR: Could not determine project ID" -ForegroundColor Red + exit 1 + } + } +} + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Updating Cloud Run Services" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Project: $ProjectId" -ForegroundColor Gray +Write-Host "Region: $Region" -ForegroundColor Gray +Write-Host "" + +$agents = @("planner", "tagger", "reporter", "charter", "retirement") +$ARTIFACT_REGISTRY = "${Region}-docker.pkg.dev/${ProjectId}/${Repo}" + +foreach ($agent in $agents) { + Write-Host "Updating $agent..." -ForegroundColor Yellow + $imageUrl = "${ARTIFACT_REGISTRY}/${agent}:${ImageTag}" + + gcloud run services update alex-$agent ` + --region=$Region ` + --image=$imageUrl ` + --project=$ProjectId ` + --quiet + + if ($LASTEXITCODE -eq 0) { + Write-Host "โœ“ $agent updated successfully" -ForegroundColor Green + } else { + Write-Host "โœ— Failed to update $agent" -ForegroundColor Red + } + Write-Host "" +} + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Update Complete!" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Next: Apply Terraform to update environment variables:" -ForegroundColor Yellow +Write-Host " cd terraform/6_agents" -ForegroundColor Gray +Write-Host " terraform apply" -ForegroundColor Gray +Write-Host "" + diff --git a/gcp-deployment/terraform/1_permissions/main.tf b/gcp-deployment/terraform/1_permissions/main.tf new file mode 100644 index 00000000..ca42dd5e --- /dev/null +++ b/gcp-deployment/terraform/1_permissions/main.tf @@ -0,0 +1,308 @@ +# ============================================================================= +# GCP Permissions Setup - Equivalent to AWS IAM +# ============================================================================= +# This Terraform configuration sets up service accounts and IAM bindings +# for the Multi-Agent SaaS application on GCP. +# ============================================================================= + +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 5.0" + } + } + + # Uncomment to use GCS backend for state storage + # backend "gcs" { + # bucket = "your-terraform-state-bucket" + # prefix = "alex-multiagent/1_permissions" + # } +} + +provider "google" { + project = var.project_id + region = var.region +} + +provider "google-beta" { + project = var.project_id + region = var.region +} + +# ============================================================================= +# DATA SOURCES +# ============================================================================= + +data "google_project" "current" { + project_id = var.project_id +} + +# ============================================================================= +# SERVICE ACCOUNTS +# ============================================================================= + +# Vertex AI Service Account (equivalent to SageMaker execution role) +resource "google_service_account" "vertex_ai" { + account_id = "vertex-ai-sa" + display_name = "Vertex AI Service Account" + description = "Service account for Vertex AI model training and inference" + project = var.project_id +} + +# Cloud Run Service Account (equivalent to App Runner/Lambda execution role) +resource "google_service_account" "cloud_run" { + account_id = "cloud-run-sa" + display_name = "Cloud Run Service Account" + description = "Service account for Cloud Run services (agents, backend)" + project = var.project_id +} + +# Cloud Functions Service Account +resource "google_service_account" "cloud_functions" { + account_id = "cloud-functions-sa" + display_name = "Cloud Functions Service Account" + description = "Service account for Cloud Functions (ingest, processing)" + project = var.project_id +} + +# Cloud SQL Proxy Service Account +resource "google_service_account" "cloud_sql" { + account_id = "cloud-sql-sa" + display_name = "Cloud SQL Service Account" + description = "Service account for Cloud SQL access" + project = var.project_id +} + +# Storage Service Account +resource "google_service_account" "storage" { + account_id = "storage-sa" + display_name = "Storage Service Account" + description = "Service account for Cloud Storage access" + project = var.project_id +} + +# Deployment Service Account (for CI/CD) +resource "google_service_account" "deploy" { + account_id = "deploy-sa" + display_name = "Deployment Service Account" + description = "Service account for CI/CD deployments (GitHub Actions)" + project = var.project_id +} + +# ============================================================================= +# IAM BINDINGS - Vertex AI Service Account +# ============================================================================= + +resource "google_project_iam_member" "vertex_ai_user" { + project = var.project_id + role = "roles/aiplatform.user" + member = "serviceAccount:${google_service_account.vertex_ai.email}" +} + +resource "google_project_iam_member" "vertex_ai_admin" { + project = var.project_id + role = "roles/aiplatform.admin" + member = "serviceAccount:${google_service_account.vertex_ai.email}" +} + +resource "google_project_iam_member" "vertex_ai_storage" { + project = var.project_id + role = "roles/storage.objectViewer" + member = "serviceAccount:${google_service_account.vertex_ai.email}" +} + +# ============================================================================= +# IAM BINDINGS - Cloud Run Service Account +# ============================================================================= + +resource "google_project_iam_member" "cloud_run_invoker" { + project = var.project_id + role = "roles/run.invoker" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_project_iam_member" "cloud_run_vertex_ai" { + project = var.project_id + role = "roles/aiplatform.user" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_project_iam_member" "cloud_run_secretmanager" { + project = var.project_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_project_iam_member" "cloud_run_cloudsql" { + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_project_iam_member" "cloud_run_storage" { + project = var.project_id + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +# ============================================================================= +# IAM BINDINGS - Cloud Functions Service Account +# ============================================================================= + +resource "google_project_iam_member" "functions_invoker" { + project = var.project_id + role = "roles/cloudfunctions.invoker" + member = "serviceAccount:${google_service_account.cloud_functions.email}" +} + +resource "google_project_iam_member" "functions_vertex_ai" { + project = var.project_id + role = "roles/aiplatform.user" + member = "serviceAccount:${google_service_account.cloud_functions.email}" +} + +resource "google_project_iam_member" "functions_storage" { + project = var.project_id + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.cloud_functions.email}" +} + +resource "google_project_iam_member" "functions_secretmanager" { + project = var.project_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.cloud_functions.email}" +} + +# ============================================================================= +# IAM BINDINGS - Cloud SQL Service Account +# ============================================================================= + +resource "google_project_iam_member" "sql_client" { + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.cloud_sql.email}" +} + +# ============================================================================= +# IAM BINDINGS - Storage Service Account +# ============================================================================= + +resource "google_project_iam_member" "storage_admin" { + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.storage.email}" +} + +# ============================================================================= +# IAM BINDINGS - Deploy Service Account +# ============================================================================= + +resource "google_project_iam_member" "deploy_run_admin" { + project = var.project_id + role = "roles/run.admin" + member = "serviceAccount:${google_service_account.deploy.email}" +} + +resource "google_project_iam_member" "deploy_functions_admin" { + project = var.project_id + role = "roles/cloudfunctions.admin" + member = "serviceAccount:${google_service_account.deploy.email}" +} + +resource "google_project_iam_member" "deploy_storage_admin" { + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.deploy.email}" +} + +resource "google_project_iam_member" "deploy_artifact_admin" { + project = var.project_id + role = "roles/artifactregistry.admin" + member = "serviceAccount:${google_service_account.deploy.email}" +} + +resource "google_project_iam_member" "deploy_cloudbuild" { + project = var.project_id + role = "roles/cloudbuild.builds.builder" + member = "serviceAccount:${google_service_account.deploy.email}" +} + +resource "google_project_iam_member" "deploy_service_account_user" { + project = var.project_id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${google_service_account.deploy.email}" +} + +# ============================================================================= +# WORKLOAD IDENTITY FOR GITHUB ACTIONS (Optional but recommended) +# ============================================================================= + +resource "google_iam_workload_identity_pool" "github" { + count = var.enable_github_workload_identity ? 1 : 0 + project = var.project_id + workload_identity_pool_id = "github-pool" + display_name = "GitHub Actions Pool" + description = "Workload Identity Pool for GitHub Actions" +} + +resource "google_iam_workload_identity_pool_provider" "github" { + count = var.enable_github_workload_identity ? 1 : 0 + project = var.project_id + workload_identity_pool_id = google_iam_workload_identity_pool.github[0].workload_identity_pool_id + workload_identity_pool_provider_id = "github-provider" + display_name = "GitHub Provider" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.actor" = "assertion.actor" + "attribute.repository" = "assertion.repository" + } + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account_iam_member" "github_workload_identity" { + count = var.enable_github_workload_identity ? 1 : 0 + service_account_id = google_service_account.deploy.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github[0].name}/attribute.repository/${var.github_repo}" +} + +# ============================================================================= +# ARTIFACT REGISTRY REPOSITORY +# ============================================================================= + +resource "google_artifact_registry_repository" "main" { + location = var.region + repository_id = "alex-containers" + description = "Docker repository for Alex Multi-Agent SaaS" + format = "DOCKER" + project = var.project_id +} + +# Grant Cloud Run service account access to pull images +resource "google_artifact_registry_repository_iam_member" "cloud_run_reader" { + project = var.project_id + location = var.region + repository = google_artifact_registry_repository.main.name + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +# Grant deploy service account access to push images +resource "google_artifact_registry_repository_iam_member" "deploy_writer" { + project = var.project_id + location = var.region + repository = google_artifact_registry_repository.main.name + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.deploy.email}" +} diff --git a/gcp-deployment/terraform/1_permissions/outputs.tf b/gcp-deployment/terraform/1_permissions/outputs.tf new file mode 100644 index 00000000..6ac110ae --- /dev/null +++ b/gcp-deployment/terraform/1_permissions/outputs.tf @@ -0,0 +1,48 @@ +# ============================================================================= +# OUTPUTS - GCP Permissions +# ============================================================================= + +output "vertex_ai_service_account_email" { + description = "Vertex AI Service Account email" + value = google_service_account.vertex_ai.email +} + +output "cloud_run_service_account_email" { + description = "Cloud Run Service Account email" + value = google_service_account.cloud_run.email +} + +output "cloud_functions_service_account_email" { + description = "Cloud Functions Service Account email" + value = google_service_account.cloud_functions.email +} + +output "cloud_sql_service_account_email" { + description = "Cloud SQL Service Account email" + value = google_service_account.cloud_sql.email +} + +output "storage_service_account_email" { + description = "Storage Service Account email" + value = google_service_account.storage.email +} + +output "deploy_service_account_email" { + description = "Deployment Service Account email" + value = google_service_account.deploy.email +} + +output "artifact_registry_repository" { + description = "Artifact Registry repository URL" + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.main.name}" +} + +output "workload_identity_pool_provider" { + description = "Workload Identity Pool Provider for GitHub Actions" + value = var.enable_github_workload_identity ? google_iam_workload_identity_pool_provider.github[0].name : null +} + +output "project_number" { + description = "GCP Project Number" + value = data.google_project.current.number +} diff --git a/gcp-deployment/terraform/1_permissions/terraform.tfvars.example b/gcp-deployment/terraform/1_permissions/terraform.tfvars.example new file mode 100644 index 00000000..e181d389 --- /dev/null +++ b/gcp-deployment/terraform/1_permissions/terraform.tfvars.example @@ -0,0 +1,13 @@ +# ============================================================================= +# EXAMPLE TERRAFORM VARIABLES +# Copy this file to terraform.tfvars and fill in your values +# ============================================================================= + +project_id = "your-gcp-project-id" +region = "us-central1" + +# Optional: Enable GitHub Actions Workload Identity Federation +enable_github_workload_identity = false +github_repo = "your-username/alex" + +environment = "dev" diff --git a/gcp-deployment/terraform/1_permissions/variables.tf b/gcp-deployment/terraform/1_permissions/variables.tf new file mode 100644 index 00000000..c6e417b0 --- /dev/null +++ b/gcp-deployment/terraform/1_permissions/variables.tf @@ -0,0 +1,32 @@ +# ============================================================================= +# VARIABLES - GCP Permissions +# ============================================================================= + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "enable_github_workload_identity" { + description = "Enable Workload Identity Federation for GitHub Actions" + type = bool + default = false +} + +variable "github_repo" { + description = "GitHub repository in format 'owner/repo' for Workload Identity" + type = string + default = "" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} diff --git a/gcp-deployment/terraform/2_vertex_ai/main.tf b/gcp-deployment/terraform/2_vertex_ai/main.tf new file mode 100644 index 00000000..a7d96af3 --- /dev/null +++ b/gcp-deployment/terraform/2_vertex_ai/main.tf @@ -0,0 +1,235 @@ +# ============================================================================= +# GCP Vertex AI Setup - Equivalent to AWS SageMaker/Bedrock +# ============================================================================= + +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# ============================================================================= +# STORAGE FOR MODEL ARTIFACTS +# ============================================================================= + +resource "google_storage_bucket" "model_artifacts" { + name = "${var.project_id}-model-artifacts" + location = var.region + force_destroy = var.environment != "prod" + + uniform_bucket_level_access = true + + versioning { + enabled = true + } + + lifecycle_rule { + condition { + age = 90 + } + action { + type = "Delete" + } + } + + labels = { + environment = var.environment + purpose = "model-artifacts" + } +} + +resource "google_storage_bucket" "training_data" { + name = "${var.project_id}-training-data" + location = var.region + force_destroy = var.environment != "prod" + + uniform_bucket_level_access = true + + labels = { + environment = var.environment + purpose = "training-data" + } +} + +# ============================================================================= +# SECRET MANAGER FOR API KEYS +# ============================================================================= + +# Anthropic API Key (Optional - only needed if using Claude via Anthropic API) +# For cost-effective option, use Gemini 2.0 Flash via Vertex AI instead +resource "google_secret_manager_secret" "anthropic_api_key" { + count = var.enable_anthropic_api ? 1 : 0 + secret_id = "anthropic-api-key" + + replication { + auto {} + } + + labels = { + environment = var.environment + service = "vertex-ai" + } +} + +# Note: You'll need to add the secret version manually or via CLI: +# gcloud secrets versions add anthropic-api-key --data-file=./api-key.txt + +resource "google_secret_manager_secret" "openai_api_key" { + secret_id = "openai-api-key" + + replication { + auto {} + } + + labels = { + environment = var.environment + service = "vertex-ai" + } +} + +# ============================================================================= +# VERTEX AI WORKBENCH (Optional - for development) +# ============================================================================= + +resource "google_notebooks_instance" "workbench" { + count = var.create_workbench ? 1 : 0 + name = "alex-workbench" + location = "${var.region}-a" + machine_type = var.workbench_machine_type + + vm_image { + project = "deeplearning-platform-release" + image_family = "common-cpu-notebooks" + } + + install_gpu_driver = var.workbench_gpu + + service_account = var.vertex_ai_service_account + + metadata = { + proxy-mode = "service_account" + } + + labels = { + environment = var.environment + } +} + +# ============================================================================= +# VERTEX AI TENSORBOARD (for experiment tracking) +# ============================================================================= + +resource "google_vertex_ai_tensorboard" "main" { + display_name = "alex-tensorboard" + description = "TensorBoard instance for Alex Multi-Agent SaaS" + region = var.region + + labels = { + environment = var.environment + } +} + +# ============================================================================= +# VPC NETWORK FOR VERTEX AI (Optional - for private endpoints) +# ============================================================================= + +resource "google_compute_network" "vertex_ai_vpc" { + count = var.create_private_network ? 1 : 0 + name = "vertex-ai-vpc" + auto_create_subnetworks = false + project = var.project_id +} + +resource "google_compute_subnetwork" "vertex_ai_subnet" { + count = var.create_private_network ? 1 : 0 + name = "vertex-ai-subnet" + ip_cidr_range = "10.0.0.0/24" + region = var.region + network = google_compute_network.vertex_ai_vpc[0].id + + private_ip_google_access = true +} + +# VPC Peering for Vertex AI +resource "google_compute_global_address" "vertex_ai_peering" { + count = var.create_private_network ? 1 : 0 + name = "vertex-ai-peering-range" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = google_compute_network.vertex_ai_vpc[0].id +} + +resource "google_service_networking_connection" "vertex_ai_peering" { + count = var.create_private_network ? 1 : 0 + network = google_compute_network.vertex_ai_vpc[0].id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.vertex_ai_peering[0].name] +} + +# ============================================================================= +# VERTEX AI FEATURE STORE (Optional) +# ============================================================================= + +resource "google_vertex_ai_featurestore" "main" { + count = var.create_feature_store ? 1 : 0 + name = "alex_featurestore" + region = var.region + + online_serving_config { + fixed_node_count = 1 + } + + labels = { + environment = var.environment + } +} + +# ============================================================================= +# IAM FOR SECRETS ACCESS +# ============================================================================= + +resource "google_secret_manager_secret_iam_member" "anthropic_accessor" { + count = var.enable_anthropic_api ? 1 : 0 + secret_id = google_secret_manager_secret.anthropic_api_key[0].secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${var.vertex_ai_service_account}" +} + +resource "google_secret_manager_secret_iam_member" "openai_accessor" { + secret_id = google_secret_manager_secret.openai_api_key.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${var.vertex_ai_service_account}" +} + +# ============================================================================= +# CLOUD SCHEDULER FOR MODEL MONITORING (Optional) +# ============================================================================= + +resource "google_cloud_scheduler_job" "model_monitoring" { + count = var.enable_model_monitoring ? 1 : 0 + name = "model-monitoring-job" + description = "Trigger model monitoring checks" + schedule = "0 */6 * * *" # Every 6 hours + time_zone = "UTC" + region = var.region + + http_target { + http_method = "POST" + uri = var.monitoring_endpoint_url + + oidc_token { + service_account_email = var.vertex_ai_service_account + } + } +} diff --git a/gcp-deployment/terraform/2_vertex_ai/outputs.tf b/gcp-deployment/terraform/2_vertex_ai/outputs.tf new file mode 100644 index 00000000..5a1a70da --- /dev/null +++ b/gcp-deployment/terraform/2_vertex_ai/outputs.tf @@ -0,0 +1,43 @@ +# ============================================================================= +# OUTPUTS - Vertex AI +# ============================================================================= + +output "model_artifacts_bucket" { + description = "GCS bucket for model artifacts" + value = google_storage_bucket.model_artifacts.name +} + +output "training_data_bucket" { + description = "GCS bucket for training data" + value = google_storage_bucket.training_data.name +} + +output "anthropic_api_key_secret_id" { + description = "Secret Manager ID for Anthropic API key" + value = google_secret_manager_secret.anthropic_api_key.secret_id +} + +output "openai_api_key_secret_id" { + description = "Secret Manager ID for OpenAI API key" + value = google_secret_manager_secret.openai_api_key.secret_id +} + +output "tensorboard_name" { + description = "Vertex AI TensorBoard instance name" + value = google_vertex_ai_tensorboard.main.name +} + +output "workbench_url" { + description = "Vertex AI Workbench URL" + value = var.create_workbench ? google_notebooks_instance.workbench[0].proxy_uri : null +} + +output "vpc_network_id" { + description = "VPC network ID for Vertex AI" + value = var.create_private_network ? google_compute_network.vertex_ai_vpc[0].id : null +} + +output "feature_store_id" { + description = "Vertex AI Feature Store ID" + value = var.create_feature_store ? google_vertex_ai_featurestore.main[0].id : null +} diff --git a/gcp-deployment/terraform/2_vertex_ai/variables.tf b/gcp-deployment/terraform/2_vertex_ai/variables.tf new file mode 100644 index 00000000..729818bf --- /dev/null +++ b/gcp-deployment/terraform/2_vertex_ai/variables.tf @@ -0,0 +1,73 @@ +# ============================================================================= +# VARIABLES - Vertex AI +# ============================================================================= + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "vertex_ai_service_account" { + description = "Service account email for Vertex AI" + type = string +} + +variable "create_workbench" { + description = "Create Vertex AI Workbench instance" + type = bool + default = false +} + +variable "workbench_machine_type" { + description = "Machine type for Workbench instance" + type = string + default = "n1-standard-4" +} + +variable "workbench_gpu" { + description = "Install GPU driver on Workbench" + type = bool + default = false +} + +variable "create_private_network" { + description = "Create private VPC network for Vertex AI" + type = bool + default = false +} + +variable "create_feature_store" { + description = "Create Vertex AI Feature Store" + type = bool + default = false +} + +variable "enable_model_monitoring" { + description = "Enable model monitoring scheduler" + type = bool + default = false +} + +variable "monitoring_endpoint_url" { + description = "URL for model monitoring endpoint" + type = string + default = "" +} + +variable "enable_anthropic_api" { + description = "Enable Anthropic API key secret (set to false to use Gemini 2.0 Flash instead)" + type = bool + default = false +} diff --git a/gcp-deployment/terraform/3_pubsub/main.tf b/gcp-deployment/terraform/3_pubsub/main.tf new file mode 100644 index 00000000..90353f9a --- /dev/null +++ b/gcp-deployment/terraform/3_pubsub/main.tf @@ -0,0 +1,75 @@ +# ============================================================================= +# PUB/SUB - Job Queue for Agent Orchestration +# ============================================================================= + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# Pub/Sub Topic for job queue +resource "google_pubsub_topic" "job_queue" { + name = "alex-job-queue" + + labels = { + environment = var.environment + managed_by = "terraform" + } +} + +# Subscription for Planner agent (push subscription) +# Note: Push endpoint will be set after Cloud Run service is deployed in Phase 3 +# For now, we'll create a pull subscription that can be converted to push later +resource "google_pubsub_subscription" "planner_subscription" { + name = "alex-planner-subscription" + topic = google_pubsub_topic.job_queue.name + + # Message retention: 24 hours + message_retention_duration = "86400s" + + # Acknowledge deadline: 60 seconds + ack_deadline_seconds = 60 + + # Enable message ordering (optional, but good for job processing) + enable_message_ordering = false + + # Retry policy + retry_policy { + minimum_backoff = "10s" + maximum_backoff = "600s" + } + + # Expiration policy: never expire + expiration_policy { + ttl = "" + } + + labels = { + environment = var.environment + managed_by = "terraform" + } +} + +# IAM binding: Allow Cloud Run service account to publish messages +resource "google_pubsub_topic_iam_member" "publisher" { + topic = google_pubsub_topic.job_queue.name + role = "roles/pubsub.publisher" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +# IAM binding: Allow Cloud Run service account to subscribe +resource "google_pubsub_subscription_iam_member" "subscriber" { + subscription = google_pubsub_subscription.planner_subscription.name + role = "roles/pubsub.subscriber" + member = "serviceAccount:${var.cloud_run_service_account}" +} + diff --git a/gcp-deployment/terraform/3_pubsub/outputs.tf b/gcp-deployment/terraform/3_pubsub/outputs.tf new file mode 100644 index 00000000..3577c78e --- /dev/null +++ b/gcp-deployment/terraform/3_pubsub/outputs.tf @@ -0,0 +1,34 @@ +# ============================================================================= +# OUTPUTS - Pub/Sub Job Queue +# ============================================================================= + +output "topic_name" { + description = "Pub/Sub topic name" + value = google_pubsub_topic.job_queue.name +} + +output "topic_id" { + description = "Pub/Sub topic ID" + value = google_pubsub_topic.job_queue.id +} + +output "topic_path" { + description = "Full Pub/Sub topic path" + value = google_pubsub_topic.job_queue.id +} + +output "subscription_name" { + description = "Pub/Sub subscription name" + value = google_pubsub_subscription.planner_subscription.name +} + +output "subscription_id" { + description = "Pub/Sub subscription ID" + value = google_pubsub_subscription.planner_subscription.id +} + +output "subscription_path" { + description = "Full Pub/Sub subscription path" + value = google_pubsub_subscription.planner_subscription.id +} + diff --git a/gcp-deployment/terraform/3_pubsub/terraform.tfvars.example b/gcp-deployment/terraform/3_pubsub/terraform.tfvars.example new file mode 100644 index 00000000..8ba33695 --- /dev/null +++ b/gcp-deployment/terraform/3_pubsub/terraform.tfvars.example @@ -0,0 +1,12 @@ +# ============================================================================= +# Pub/Sub Configuration +# ============================================================================= + +project_id = "your-gcp-project-id" +region = "us-central1" +environment = "dev" + +# Service account for Cloud Run (must have Pub/Sub permissions) +# Update this value after running terraform/1_permissions +cloud_run_service_account = "cloud-run-sa@your-gcp-project-id.iam.gserviceaccount.com" + diff --git a/gcp-deployment/terraform/3_pubsub/variables.tf b/gcp-deployment/terraform/3_pubsub/variables.tf new file mode 100644 index 00000000..4fc7161c --- /dev/null +++ b/gcp-deployment/terraform/3_pubsub/variables.tf @@ -0,0 +1,26 @@ +# ============================================================================= +# VARIABLES - Pub/Sub Job Queue +# ============================================================================= + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "cloud_run_service_account" { + description = "Cloud Run service account email for Pub/Sub permissions" + type = string +} + diff --git a/gcp-deployment/terraform/5_database/cloud-sql-proxy.exe b/gcp-deployment/terraform/5_database/cloud-sql-proxy.exe new file mode 100644 index 00000000..cbea5a56 Binary files /dev/null and b/gcp-deployment/terraform/5_database/cloud-sql-proxy.exe differ diff --git a/gcp-deployment/terraform/5_database/main.tf b/gcp-deployment/terraform/5_database/main.tf new file mode 100644 index 00000000..3e61d10b --- /dev/null +++ b/gcp-deployment/terraform/5_database/main.tf @@ -0,0 +1,259 @@ +# ============================================================================= +# GCP Cloud SQL Setup - Equivalent to AWS RDS PostgreSQL +# ============================================================================= + +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# ============================================================================= +# RANDOM PASSWORD GENERATION +# ============================================================================= + +resource "random_password" "db_password" { + length = 32 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + +# ============================================================================= +# VPC NETWORK (if not using existing) +# ============================================================================= + +resource "google_compute_network" "main" { + count = var.create_vpc ? 1 : 0 + name = "alex-vpc" + auto_create_subnetworks = false + project = var.project_id +} + +resource "google_compute_subnetwork" "main" { + count = var.create_vpc ? 1 : 0 + name = "alex-subnet" + ip_cidr_range = "10.0.0.0/24" + region = var.region + network = google_compute_network.main[0].id + + private_ip_google_access = true +} + +# ============================================================================= +# PRIVATE SERVICE CONNECTION FOR CLOUD SQL +# ============================================================================= + +resource "google_compute_global_address" "private_ip_address" { + name = "alex-db-private-ip" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = var.create_vpc ? google_compute_network.main[0].id : var.vpc_network_id +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = var.create_vpc ? google_compute_network.main[0].id : var.vpc_network_id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip_address.name] +} + +# ============================================================================= +# CLOUD SQL POSTGRESQL INSTANCE +# ============================================================================= + +resource "google_sql_database_instance" "main" { + name = "alex-postgres" + database_version = "POSTGRES_15" + region = var.region + project = var.project_id + + deletion_protection = var.environment == "prod" + + depends_on = [google_service_networking_connection.private_vpc_connection] + + settings { + tier = var.db_tier + availability_type = var.high_availability ? "REGIONAL" : "ZONAL" + disk_size = var.disk_size_gb + disk_type = "PD_SSD" + disk_autoresize = true + + backup_configuration { + enabled = true + start_time = "03:00" + point_in_time_recovery_enabled = var.environment == "prod" + backup_retention_settings { + retained_backups = var.environment == "prod" ? 30 : 7 + } + } + + ip_configuration { + ipv4_enabled = var.enable_public_ip + private_network = var.create_vpc ? google_compute_network.main[0].id : var.vpc_network_id + require_ssl = true + + dynamic "authorized_networks" { + for_each = var.authorized_networks + content { + name = authorized_networks.value.name + value = authorized_networks.value.cidr + } + } + } + + database_flags { + name = "max_connections" + value = var.max_connections + } + + database_flags { + name = "log_min_duration_statement" + value = "1000" # Log queries > 1 second + } + + maintenance_window { + day = 7 # Sunday + hour = 4 # 4 AM + update_track = "stable" + } + + insights_config { + query_insights_enabled = true + query_string_length = 1024 + record_application_tags = true + record_client_address = true + } + + user_labels = { + environment = var.environment + app = "alex" + } + } +} + +# ============================================================================= +# DATABASE +# ============================================================================= + +resource "google_sql_database" "alex" { + name = "alex" + instance = google_sql_database_instance.main.name + project = var.project_id +} + +# ============================================================================= +# DATABASE USER +# ============================================================================= + +resource "google_sql_user" "app_user" { + name = "alex_app" + instance = google_sql_database_instance.main.name + password = random_password.db_password.result + project = var.project_id +} + +# ============================================================================= +# READ REPLICA (Optional for production) +# ============================================================================= + +resource "google_sql_database_instance" "read_replica" { + count = var.create_read_replica ? 1 : 0 + name = "alex-postgres-replica" + master_instance_name = google_sql_database_instance.main.name + database_version = "POSTGRES_15" + region = var.region + project = var.project_id + + replica_configuration { + failover_target = false + } + + settings { + tier = var.replica_tier + disk_size = var.disk_size_gb + disk_type = "PD_SSD" + disk_autoresize = true + + ip_configuration { + ipv4_enabled = var.enable_public_ip + private_network = var.create_vpc ? google_compute_network.main[0].id : var.vpc_network_id + require_ssl = true + } + + user_labels = { + environment = var.environment + app = "alex" + role = "read-replica" + } + } +} + +# ============================================================================= +# SECRET MANAGER FOR DATABASE CREDENTIALS +# ============================================================================= + +resource "google_secret_manager_secret" "db_password" { + secret_id = "alex-db-password" + + replication { + auto {} + } + + labels = { + environment = var.environment + service = "database" + } +} + +resource "google_secret_manager_secret_version" "db_password" { + secret = google_secret_manager_secret.db_password.id + secret_data = random_password.db_password.result +} + +resource "google_secret_manager_secret" "db_connection_string" { + secret_id = "alex-db-connection-string" + + replication { + auto {} + } + + labels = { + environment = var.environment + service = "database" + } +} + +resource "google_secret_manager_secret_version" "db_connection_string" { + secret = google_secret_manager_secret.db_connection_string.id + secret_data = "postgresql://alex_app:${random_password.db_password.result}@/${google_sql_database.alex.name}?host=/cloudsql/${google_sql_database_instance.main.connection_name}" +} + +# ============================================================================= +# IAM FOR SECRET ACCESS +# ============================================================================= + +resource "google_secret_manager_secret_iam_member" "db_password_accessor" { + secret_id = google_secret_manager_secret.db_password.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +resource "google_secret_manager_secret_iam_member" "db_connection_accessor" { + secret_id = google_secret_manager_secret.db_connection_string.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${var.cloud_run_service_account}" +} diff --git a/gcp-deployment/terraform/5_database/outputs.tf b/gcp-deployment/terraform/5_database/outputs.tf new file mode 100644 index 00000000..f1761346 --- /dev/null +++ b/gcp-deployment/terraform/5_database/outputs.tf @@ -0,0 +1,58 @@ +# ============================================================================= +# OUTPUTS - Cloud SQL Database +# ============================================================================= + +output "instance_name" { + description = "Cloud SQL instance name" + value = google_sql_database_instance.main.name +} + +output "instance_connection_name" { + description = "Cloud SQL connection name for Cloud SQL Proxy" + value = google_sql_database_instance.main.connection_name +} + +output "private_ip_address" { + description = "Private IP address of the database" + value = google_sql_database_instance.main.private_ip_address +} + +output "public_ip_address" { + description = "Public IP address of the database (if enabled)" + value = var.enable_public_ip ? google_sql_database_instance.main.public_ip_address : null +} + +output "database_name" { + description = "Database name" + value = google_sql_database.alex.name +} + +output "database_user" { + description = "Database user" + value = google_sql_user.app_user.name +} + +output "db_password_secret_id" { + description = "Secret Manager ID for database password" + value = google_secret_manager_secret.db_password.secret_id +} + +output "db_connection_string_secret_id" { + description = "Secret Manager ID for database connection string" + value = google_secret_manager_secret.db_connection_string.secret_id +} + +output "read_replica_ip" { + description = "Private IP of read replica" + value = var.create_read_replica ? google_sql_database_instance.read_replica[0].private_ip_address : null +} + +output "vpc_network_id" { + description = "VPC network ID" + value = var.create_vpc ? google_compute_network.main[0].id : var.vpc_network_id +} + +output "vpc_subnet_id" { + description = "VPC subnet ID" + value = var.create_vpc ? google_compute_subnetwork.main[0].id : null +} diff --git a/gcp-deployment/terraform/5_database/restart-local-postgres.ps1 b/gcp-deployment/terraform/5_database/restart-local-postgres.ps1 new file mode 100644 index 00000000..f108f861 --- /dev/null +++ b/gcp-deployment/terraform/5_database/restart-local-postgres.ps1 @@ -0,0 +1,50 @@ + Set-Content -Path ".\restart-local-postgres.ps1" -Value @' + param( + [ValidateSet("stop","start","restart")] + [string]$Action = "restart", + [string]$ServiceName = "postgresql-x64-14" + ) + + function Ensure-ServiceExists { + param([string]$Name) + $svc = Get-Service -Name $Name -ErrorAction SilentlyContinue + if (-not $svc) { + Write-Error "Service '$Name' not found. Run 'Get-Service *postgres*' to see available names." + exit 1 + } + return $svc + } + + function Stop-Postgres { + param([string]$Name) + $svc = Ensure-ServiceExists -Name $Name + if ($svc.Status -eq "Stopped") { + Write-Host "Service '$Name' is already stopped." + return + } + Write-Host "Stopping PostgreSQL service '$Name'..." + Stop-Service -Name $Name -Force -ErrorAction Stop + Write-Host "Service stopped." + } + + function Start-Postgres { + param([string]$Name) + $svc = Ensure-ServiceExists -Name $Name + if ($svc.Status -eq "Running") { + Write-Host "Service '$Name' is already running." + return + } + Write-Host "Starting PostgreSQL service '$Name'..." + Start-Service -Name $Name -ErrorAction Stop + Write-Host "Service started." + } + + switch ($Action) { + "stop" { Stop-Postgres -Name $ServiceName } + "start" { Start-Postgres -Name $ServiceName } + "restart" { + Stop-Postgres -Name $ServiceName + Start-Postgres -Name $ServiceName + } + } + '@ \ No newline at end of file diff --git a/gcp-deployment/terraform/5_database/terraform.tfvars.example b/gcp-deployment/terraform/5_database/terraform.tfvars.example new file mode 100644 index 00000000..e9af45f8 --- /dev/null +++ b/gcp-deployment/terraform/5_database/terraform.tfvars.example @@ -0,0 +1,30 @@ +project_id = "your-gcp-project-id" +region = "us-central1" +environment = "dev" + +# Network configuration +create_vpc = true +vpc_network_id = "" # Only set if create_vpc = false + +# Database sizing +db_tier = "db-custom-2-4096" +disk_size_gb = 20 +max_connections = "100" +high_availability = false + +# Connectivity +enable_public_ip = false +authorized_networks = [ + # { + # name = "office" + # cidr = "203.0.113.0/24" + # } +] + +# Read replica (optional) +create_read_replica = false +replica_tier = "db-custom-1-2048" + +# Service account for Cloud Run (grants Secret Manager access) +cloud_run_service_account = "cloud-run-sa@your-gcp-project-id.iam.gserviceaccount.com" + diff --git a/gcp-deployment/terraform/5_database/variables.tf b/gcp-deployment/terraform/5_database/variables.tf new file mode 100644 index 00000000..f3cb22d1 --- /dev/null +++ b/gcp-deployment/terraform/5_database/variables.tf @@ -0,0 +1,88 @@ +# ============================================================================= +# VARIABLES - Cloud SQL Database +# ============================================================================= + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "create_vpc" { + description = "Create new VPC for database" + type = bool + default = true +} + +variable "vpc_network_id" { + description = "Existing VPC network ID (if create_vpc is false)" + type = string + default = "" +} + +variable "db_tier" { + description = "Cloud SQL instance tier" + type = string + default = "db-custom-2-4096" # 2 vCPU, 4GB RAM +} + +variable "disk_size_gb" { + description = "Database disk size in GB" + type = number + default = 20 +} + +variable "max_connections" { + description = "Maximum database connections" + type = string + default = "100" +} + +variable "high_availability" { + description = "Enable high availability (multi-zone)" + type = bool + default = false +} + +variable "enable_public_ip" { + description = "Enable public IP for database" + type = bool + default = false +} + +variable "authorized_networks" { + description = "List of authorized networks for public access" + type = list(object({ + name = string + cidr = string + })) + default = [] +} + +variable "create_read_replica" { + description = "Create read replica" + type = bool + default = false +} + +variable "replica_tier" { + description = "Cloud SQL read replica tier" + type = string + default = "db-custom-1-2048" +} + +variable "cloud_run_service_account" { + description = "Cloud Run service account email for secret access" + type = string +} diff --git a/gcp-deployment/terraform/6_agents/db_password.txt b/gcp-deployment/terraform/6_agents/db_password.txt new file mode 100644 index 00000000..8f1542fa --- /dev/null +++ b/gcp-deployment/terraform/6_agents/db_password.txt @@ -0,0 +1 @@ +O*JDU*Bmd!eXftLv${vxOG2*Si5ZxXL# \ No newline at end of file diff --git a/gcp-deployment/terraform/6_agents/main.tf b/gcp-deployment/terraform/6_agents/main.tf new file mode 100644 index 00000000..c8c31ddf --- /dev/null +++ b/gcp-deployment/terraform/6_agents/main.tf @@ -0,0 +1,882 @@ +# ============================================================================= +# GCP Cloud Run Agents - Equivalent to AWS Lambda +# ============================================================================= + +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# ============================================================================= +# DATA SOURCES +# ============================================================================= + +data "google_project" "current" { + project_id = var.project_id +} + +# ============================================================================= +# ARTIFACT REGISTRY REPOSITORY +# ============================================================================= + +resource "google_artifact_registry_repository" "agents" { + location = var.region + repository_id = "alex-agents" + description = "Container images for Alex agents" + format = "DOCKER" + + labels = { + environment = var.environment + purpose = "agents" + } +} + +# ============================================================================= +# CLOUD RUN SERVICE - PLANNER AGENT +# ============================================================================= + +resource "google_cloud_run_v2_service" "planner" { + name = "alex-planner" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}/planner:${var.image_tag}" + + ports { + container_port = 8000 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + # Environment variables + env { + name = "GCP_PROJECT_ID" + value = var.project_id + } + + env { + name = "GCP_REGION" + value = var.region + } + + env { + name = "INSTANCE_CONNECTION_NAME" + value = var.db_connection_name + } + + env { + name = "DATABASE_NAME" + value = var.database_name + } + + env { + name = "DATABASE_USER" + value = var.database_user + } + + env { + name = "DB_PASSWORD_SECRET_ID" + value = var.db_password_secret_id + } + + env { + name = "PUBSUB_TOPIC" + value = var.pubsub_topic + } + + # Service URLs for other agents (set after deployment) + env { + name = "TAGGER_SERVICE_URL" + value = google_cloud_run_v2_service.tagger.uri + } + + env { + name = "REPORTER_SERVICE_URL" + value = google_cloud_run_v2_service.reporter.uri + } + + env { + name = "CHARTER_SERVICE_URL" + value = google_cloud_run_v2_service.charter.uri + } + + env { + name = "RETIREMENT_SERVICE_URL" + value = google_cloud_run_v2_service.retirement.uri + } + + # LLM Configuration + env { + name = "VERTEX_AI_MODEL" + value = var.vertex_ai_model + } + + env { + name = "LLM_PROVIDER" + value = var.llm_provider + } + + # Secrets from Secret Manager + env { + name = "DB_PASSWORD" + value_source { + secret_key_ref { + secret = var.db_password_secret_id + version = "latest" + } + } + } + + # OpenAI API Key (only if secret ID is provided) + dynamic "env" { + for_each = var.openai_api_key_secret_id != "" ? [1] : [] + content { + name = "OPENAI_API_KEY" + value_source { + secret_key_ref { + secret = var.openai_api_key_secret_id + version = "latest" + } + } + } + } + + # Polygon.io API Key (only if secret ID is provided) + dynamic "env" { + for_each = var.polygon_api_key_secret_id != "" ? [1] : [] + content { + name = "POLYGON_API_KEY" + value_source { + secret_key_ref { + secret = var.polygon_api_key_secret_id + version = "latest" + } + } + } + } + + # Polygon plan type + env { + name = "POLYGON_PLAN" + value = var.polygon_plan + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 3 + } + + liveness_probe { + http_get { + path = "/health" + } + period_seconds = 30 + failure_threshold = 3 + } + } + + timeout = "900s" # 15 minutes for orchestration + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + agent = "planner" + } +} + +# ============================================================================= +# CLOUD RUN SERVICE - TAGGER AGENT +# ============================================================================= + +resource "google_cloud_run_v2_service" "tagger" { + name = "alex-tagger" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}/tagger:${var.image_tag}" + + ports { + container_port = 8000 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + env { + name = "GCP_PROJECT_ID" + value = var.project_id + } + + env { + name = "GCP_REGION" + value = var.region + } + + env { + name = "INSTANCE_CONNECTION_NAME" + value = var.db_connection_name + } + + env { + name = "DATABASE_NAME" + value = var.database_name + } + + env { + name = "DATABASE_USER" + value = var.database_user + } + + env { + name = "DB_PASSWORD" + value_source { + secret_key_ref { + secret = var.db_password_secret_id + version = "latest" + } + } + } + + # LLM Configuration + env { + name = "VERTEX_AI_MODEL" + value = var.vertex_ai_model + } + + env { + name = "LLM_PROVIDER" + value = var.llm_provider + } + + # OpenAI API Key (only if secret ID is provided) + dynamic "env" { + for_each = var.openai_api_key_secret_id != "" ? [1] : [] + content { + name = "OPENAI_API_KEY" + value_source { + secret_key_ref { + secret = var.openai_api_key_secret_id + version = "latest" + } + } + } + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 3 + } + } + + timeout = "300s" + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + agent = "tagger" + } +} + +# ============================================================================= +# CLOUD RUN SERVICE - REPORTER AGENT +# ============================================================================= + +resource "google_cloud_run_v2_service" "reporter" { + name = "alex-reporter" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}/reporter:${var.image_tag}" + + ports { + container_port = 8000 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + env { + name = "GCP_PROJECT_ID" + value = var.project_id + } + + env { + name = "GCP_REGION" + value = var.region + } + + env { + name = "INSTANCE_CONNECTION_NAME" + value = var.db_connection_name + } + + env { + name = "DATABASE_NAME" + value = var.database_name + } + + env { + name = "DATABASE_USER" + value = var.database_user + } + + env { + name = "DB_PASSWORD" + value_source { + secret_key_ref { + secret = var.db_password_secret_id + version = "latest" + } + } + } + + # LLM Configuration + env { + name = "VERTEX_AI_MODEL" + value = var.vertex_ai_model + } + + env { + name = "LLM_PROVIDER" + value = var.llm_provider + } + + # OpenAI API Key (only if secret ID is provided) + dynamic "env" { + for_each = var.openai_api_key_secret_id != "" ? [1] : [] + content { + name = "OPENAI_API_KEY" + value_source { + secret_key_ref { + secret = var.openai_api_key_secret_id + version = "latest" + } + } + } + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 3 + } + } + + timeout = "600s" + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + agent = "reporter" + } +} + +# ============================================================================= +# CLOUD RUN SERVICE - CHARTER AGENT +# ============================================================================= + +resource "google_cloud_run_v2_service" "charter" { + name = "alex-charter" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}/charter:${var.image_tag}" + + ports { + container_port = 8000 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + env { + name = "GCP_PROJECT_ID" + value = var.project_id + } + + env { + name = "GCP_REGION" + value = var.region + } + + env { + name = "INSTANCE_CONNECTION_NAME" + value = var.db_connection_name + } + + env { + name = "DATABASE_NAME" + value = var.database_name + } + + env { + name = "DATABASE_USER" + value = var.database_user + } + + env { + name = "DB_PASSWORD" + value_source { + secret_key_ref { + secret = var.db_password_secret_id + version = "latest" + } + } + } + + # LLM Configuration + env { + name = "VERTEX_AI_MODEL" + value = var.vertex_ai_model + } + + env { + name = "LLM_PROVIDER" + value = var.llm_provider + } + + # OpenAI API Key (only if secret ID is provided) + dynamic "env" { + for_each = var.openai_api_key_secret_id != "" ? [1] : [] + content { + name = "OPENAI_API_KEY" + value_source { + secret_key_ref { + secret = var.openai_api_key_secret_id + version = "latest" + } + } + } + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 3 + } + } + + timeout = "600s" + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + agent = "charter" + } +} + +# ============================================================================= +# CLOUD RUN SERVICE - RETIREMENT AGENT +# ============================================================================= + +resource "google_cloud_run_v2_service" "retirement" { + name = "alex-retirement" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}/retirement:${var.image_tag}" + + ports { + container_port = 8000 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + env { + name = "GCP_PROJECT_ID" + value = var.project_id + } + + env { + name = "GCP_REGION" + value = var.region + } + + env { + name = "INSTANCE_CONNECTION_NAME" + value = var.db_connection_name + } + + env { + name = "DATABASE_NAME" + value = var.database_name + } + + env { + name = "DATABASE_USER" + value = var.database_user + } + + env { + name = "DB_PASSWORD" + value_source { + secret_key_ref { + secret = var.db_password_secret_id + version = "latest" + } + } + } + + # LLM Configuration + env { + name = "VERTEX_AI_MODEL" + value = var.vertex_ai_model + } + + env { + name = "LLM_PROVIDER" + value = var.llm_provider + } + + # OpenAI API Key (only if secret ID is provided) + dynamic "env" { + for_each = var.openai_api_key_secret_id != "" ? [1] : [] + content { + name = "OPENAI_API_KEY" + value_source { + secret_key_ref { + secret = var.openai_api_key_secret_id + version = "latest" + } + } + } + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 3 + } + } + + timeout = "600s" + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + agent = "retirement" + } +} + +# ============================================================================= +# IAM - Allow Inter-Service Communication +# ============================================================================= + +resource "google_cloud_run_v2_service_iam_member" "planner_invoker" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.planner.name + role = "roles/run.invoker" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +resource "google_cloud_run_v2_service_iam_member" "tagger_invoker" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.tagger.name + role = "roles/run.invoker" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +resource "google_cloud_run_v2_service_iam_member" "reporter_invoker" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.reporter.name + role = "roles/run.invoker" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +resource "google_cloud_run_v2_service_iam_member" "charter_invoker" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.charter.name + role = "roles/run.invoker" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +resource "google_cloud_run_v2_service_iam_member" "retirement_invoker" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.retirement.name + role = "roles/run.invoker" + member = "serviceAccount:${var.cloud_run_service_account}" +} + +# ============================================================================= +# CLOUD RUN SERVICE - API (Backend for Frontend) +# ============================================================================= + +resource "google_cloud_run_v2_service" "api" { + name = "alex-api" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}/api:${var.image_tag}" + + ports { + container_port = 8080 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + env { + name = "GCP_PROJECT_ID" + value = var.project_id + } + + env { + name = "GCP_REGION" + value = var.region + } + + env { + name = "INSTANCE_CONNECTION_NAME" + value = var.db_connection_name + } + + env { + name = "DATABASE_NAME" + value = var.database_name + } + + env { + name = "DATABASE_USER" + value = var.database_user + } + + env { + name = "DB_PASSWORD" + value_source { + secret_key_ref { + secret = var.db_password_secret_id + version = "latest" + } + } + } + + env { + name = "PUBSUB_TOPIC" + value = var.pubsub_topic + } + + env { + name = "CLERK_JWKS_URL" + value = var.clerk_jwks_url + } + + env { + name = "CLERK_ISSUER" + value = var.clerk_issuer + } + + env { + name = "FRONTEND_URL" + value = var.frontend_url + } + + env { + name = "CORS_ORIGINS" + value = var.cors_origins + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 3 + } + } + + timeout = "60s" + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + component = "api" + } +} + +# IAM - Allow public access to API +resource "google_cloud_run_v2_service_iam_member" "api_public" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.api.name + role = "roles/run.invoker" + member = "allUsers" +} + +# ============================================================================= +# UPDATE PUB/SUB SUBSCRIPTION TO PUSH TO PLANNER +# ============================================================================= +# Note: The subscription is created in terraform/3_pubsub/ +# We update it here to add the push endpoint after Cloud Run is deployed +# This requires the subscription to exist first (deploy terraform/3_pubsub first) + +data "google_pubsub_subscription" "planner_subscription" { + name = "alex-planner-subscription" +} + +resource "google_pubsub_subscription" "planner_subscription_push" { + name = data.google_pubsub_subscription.planner_subscription.name + topic = var.pubsub_topic + + push_config { + push_endpoint = "${google_cloud_run_v2_service.planner.uri}/pubsub" + + oidc_token { + service_account_email = var.cloud_run_service_account + } + } + + ack_deadline_seconds = 60 + message_retention_duration = "86400s" + + labels = { + environment = var.environment + agent = "planner" + managed_by = "terraform" + } + + depends_on = [google_cloud_run_v2_service.planner] +} diff --git a/gcp-deployment/terraform/6_agents/outputs.tf b/gcp-deployment/terraform/6_agents/outputs.tf new file mode 100644 index 00000000..43cdab62 --- /dev/null +++ b/gcp-deployment/terraform/6_agents/outputs.tf @@ -0,0 +1,60 @@ +# ============================================================================= +# OUTPUTS - Cloud Run Agents +# ============================================================================= + +output "artifact_registry_repository" { + description = "Artifact Registry repository name" + value = google_artifact_registry_repository.agents.repository_id +} + +output "artifact_registry_url" { + description = "Full Artifact Registry URL" + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.agents.repository_id}" +} + +output "planner_service_url" { + description = "Planner agent Cloud Run URL" + value = google_cloud_run_v2_service.planner.uri +} + +output "tagger_service_url" { + description = "Tagger agent Cloud Run URL" + value = google_cloud_run_v2_service.tagger.uri +} + +output "reporter_service_url" { + description = "Reporter agent Cloud Run URL" + value = google_cloud_run_v2_service.reporter.uri +} + +output "charter_service_url" { + description = "Charter agent Cloud Run URL" + value = google_cloud_run_v2_service.charter.uri +} + +output "retirement_service_url" { + description = "Retirement agent Cloud Run URL" + value = google_cloud_run_v2_service.retirement.uri +} + +output "api_service_url" { + description = "Backend API Cloud Run URL" + value = google_cloud_run_v2_service.api.uri +} + +output "all_service_urls" { + description = "All agent service URLs" + value = { + api = google_cloud_run_v2_service.api.uri + planner = google_cloud_run_v2_service.planner.uri + tagger = google_cloud_run_v2_service.tagger.uri + reporter = google_cloud_run_v2_service.reporter.uri + charter = google_cloud_run_v2_service.charter.uri + retirement = google_cloud_run_v2_service.retirement.uri + } +} + +output "pubsub_subscription_name" { + description = "Pub/Sub subscription name for planner" + value = data.google_pubsub_subscription.planner_subscription.name +} diff --git a/gcp-deployment/terraform/6_agents/terraform.tfvars.example b/gcp-deployment/terraform/6_agents/terraform.tfvars.example new file mode 100644 index 00000000..b7b1e119 --- /dev/null +++ b/gcp-deployment/terraform/6_agents/terraform.tfvars.example @@ -0,0 +1,51 @@ +# ============================================================================= +# Cloud Run Agents Configuration +# ============================================================================= + +project_id = "your-gcp-project-id" +region = "us-central1" +environment = "dev" + +# Container image tag +image_tag = "latest" + +# Service account for Cloud Run +cloud_run_service_account = "cloud-run-sa@your-gcp-project-id.iam.gserviceaccount.com" + +# Database configuration (from terraform/5_database outputs) +# Update these values after running terraform/5_database +db_connection_name = "your-gcp-project-id:us-central1:alex-postgres" +database_name = "alex" +database_user = "alex_app" +db_password_secret_id = "alex-db-password" + +# Pub/Sub configuration (from terraform/3_pubsub outputs) +# Update this value after running terraform/3_pubsub +pubsub_topic = "alex-job-queue" + +# LLM Configuration +vertex_ai_model = "vertex_ai/gemini-2.0-flash-exp" +llm_provider = "vertex_ai" +openai_api_key_secret_id = "openai-api-key" # Secret Manager ID (not the actual key) + +# Clerk Configuration +# Update these values with your Clerk configuration +clerk_jwks_url = "https://your-clerk-instance.clerk.accounts.dev/.well-known/jwks.json" +clerk_issuer = "https://your-clerk-instance.clerk.accounts.dev" +frontend_url = "https://your-frontend-url.run.app" +cors_origins = "" + +# Scaling configuration +min_instances = 0 # Scale to zero when not in use +max_instances = 10 + +# Resource limits +cpu_limit = "2" +memory_limit = "2Gi" + +# Polygon.io API configuration (for market data) +# First, create the secret in Secret Manager: +# echo -n "your-polygon-api-key" | gcloud secrets create polygon-api-key --data-file=- --project=PROJECT_ID +polygon_api_key_secret_id = "polygon-api-key" # Leave empty "" if not using Polygon +polygon_plan = "free" # "free" or "paid" + diff --git a/gcp-deployment/terraform/6_agents/variables.tf b/gcp-deployment/terraform/6_agents/variables.tf new file mode 100644 index 00000000..73b9aa27 --- /dev/null +++ b/gcp-deployment/terraform/6_agents/variables.tf @@ -0,0 +1,138 @@ +# ============================================================================= +# VARIABLES - Cloud Run Agents +# ============================================================================= + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "image_tag" { + description = "Container image tag" + type = string + default = "latest" +} + +variable "cloud_run_service_account" { + description = "Service account email for Cloud Run services" + type = string +} + +variable "db_connection_name" { + description = "Cloud SQL instance connection name (from database terraform output)" + type = string +} + +variable "database_name" { + description = "Database name" + type = string + default = "alex" +} + +variable "database_user" { + description = "Database user" + type = string + default = "alex_app" +} + +variable "db_password_secret_id" { + description = "Secret Manager ID for database password" + type = string + default = "alex-db-password" +} + +variable "pubsub_topic" { + description = "Pub/Sub topic name for job queue" + type = string + default = "alex-job-queue" +} + +variable "min_instances" { + description = "Minimum number of instances (0 for scale-to-zero)" + type = number + default = 0 +} + +variable "max_instances" { + description = "Maximum number of instances" + type = number + default = 10 +} + +variable "cpu_limit" { + description = "CPU limit for containers" + type = string + default = "2" +} + +variable "memory_limit" { + description = "Memory limit for containers" + type = string + default = "2Gi" +} + +variable "vertex_ai_model" { + description = "Vertex AI model name" + type = string + default = "vertex_ai/gemini-2.0-flash-exp" +} + +variable "llm_provider" { + description = "LLM provider (vertex_ai or openai)" + type = string + default = "vertex_ai" +} + +variable "openai_api_key_secret_id" { + description = "Secret Manager ID for OpenAI API key (optional, leave empty if not using OpenAI)" + type = string + default = "" +} + +variable "clerk_jwks_url" { + description = "Clerk JWKS URL for JWT verification" + type = string + default = "" +} + +variable "clerk_issuer" { + description = "Clerk issuer URL" + type = string + default = "" +} + +variable "frontend_url" { + description = "Frontend Cloud Run URL (for CORS)" + type = string + default = "" +} + +variable "cors_origins" { + description = "Additional CORS origins (comma-separated)" + type = string + default = "" +} + +variable "polygon_api_key_secret_id" { + description = "Secret Manager ID for Polygon.io API key (optional, leave empty if not using Polygon)" + type = string + default = "" +} + +variable "polygon_plan" { + description = "Polygon.io plan type (free or paid) - affects which API endpoints are used" + type = string + default = "free" +} diff --git a/gcp-deployment/terraform/7_frontend/main.tf b/gcp-deployment/terraform/7_frontend/main.tf new file mode 100644 index 00000000..34abd3bd --- /dev/null +++ b/gcp-deployment/terraform/7_frontend/main.tf @@ -0,0 +1,250 @@ +# ============================================================================= +# GCP Frontend Deployment - Equivalent to AWS App Runner + CloudFront +# ============================================================================= + +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# ============================================================================= +# CLOUD RUN SERVICE - FRONTEND +# ============================================================================= + +resource "google_cloud_run_v2_service" "frontend" { + name = "frontend" + location = var.region + + template { + service_account = var.cloud_run_service_account + + scaling { + min_instance_count = var.min_instances + max_instance_count = var.max_instances + } + + containers { + image = "${var.artifact_registry_url}/frontend:${var.image_tag}" + + ports { + container_port = 8080 + } + + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + cpu_idle = true + } + + env { + name = "NEXT_PUBLIC_API_URL" + value = var.backend_api_url + } + + env { + name = "NODE_ENV" + value = "production" + } + + # Clerk keys are baked into the build image at build time + # NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY is already in the image + # CLERK_SECRET_KEY is only needed by backend API, not frontend + + startup_probe { + http_get { + path = "/" + port = 8080 + } + initial_delay_seconds = 10 + period_seconds = 10 + failure_threshold = 5 + } + + liveness_probe { + http_get { + path = "/" + port = 8080 + } + period_seconds = 30 + failure_threshold = 3 + } + } + + timeout = "60s" + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + labels = { + environment = var.environment + component = "frontend" + } +} + +# ============================================================================= +# IAM - Allow Public Access +# ============================================================================= + +resource "google_cloud_run_v2_service_iam_member" "frontend_public" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.frontend.name + role = "roles/run.invoker" + member = "allUsers" +} + +# ============================================================================= +# SECRET MANAGER FOR CLERK KEYS +# ============================================================================= +# Secrets are created by the deployment script and referenced by name +# IAM bindings are managed via gcloud commands in the deployment script + +# ============================================================================= +# LOAD BALANCER WITH CLOUD CDN (Optional - for custom domain) +# ============================================================================= + +resource "google_compute_region_network_endpoint_group" "frontend_neg" { + count = var.enable_load_balancer ? 1 : 0 + name = "frontend-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + + cloud_run { + service = google_cloud_run_v2_service.frontend.name + } +} + +resource "google_compute_backend_service" "frontend" { + count = var.enable_load_balancer ? 1 : 0 + name = "frontend-backend" + protocol = "HTTP" + port_name = "http" + timeout_sec = 60 + + enable_cdn = var.enable_cdn + + dynamic "cdn_policy" { + for_each = var.enable_cdn ? [1] : [] + content { + cache_mode = "CACHE_ALL_STATIC" + default_ttl = 3600 + max_ttl = 86400 + client_ttl = 3600 + negative_caching = true + signed_url_cache_max_age_sec = 0 + + cache_key_policy { + include_host = true + include_protocol = true + include_query_string = true + } + } + } + + backend { + group = google_compute_region_network_endpoint_group.frontend_neg[0].id + } +} + +resource "google_compute_url_map" "frontend" { + count = var.enable_load_balancer ? 1 : 0 + name = "frontend-url-map" + default_service = google_compute_backend_service.frontend[0].id +} + +# ============================================================================= +# SSL CERTIFICATE (for custom domain) +# ============================================================================= + +resource "google_compute_managed_ssl_certificate" "frontend" { + count = var.enable_load_balancer && var.custom_domain != "" ? 1 : 0 + name = "frontend-ssl-cert" + + managed { + domains = [var.custom_domain] + } +} + +resource "google_compute_target_https_proxy" "frontend" { + count = var.enable_load_balancer && var.custom_domain != "" ? 1 : 0 + name = "frontend-https-proxy" + url_map = google_compute_url_map.frontend[0].id + ssl_certificates = [google_compute_managed_ssl_certificate.frontend[0].id] +} + +resource "google_compute_global_forwarding_rule" "frontend_https" { + count = var.enable_load_balancer && var.custom_domain != "" ? 1 : 0 + name = "frontend-https-forwarding" + target = google_compute_target_https_proxy.frontend[0].id + port_range = "443" + ip_address = google_compute_global_address.frontend[0].address + load_balancing_scheme = "EXTERNAL_MANAGED" +} + +resource "google_compute_global_address" "frontend" { + count = var.enable_load_balancer ? 1 : 0 + name = "frontend-ip" +} + +# HTTP to HTTPS redirect +resource "google_compute_url_map" "frontend_redirect" { + count = var.enable_load_balancer && var.custom_domain != "" ? 1 : 0 + name = "frontend-http-redirect" + + default_url_redirect { + https_redirect = true + redirect_response_code = "MOVED_PERMANENTLY_DEFAULT" + strip_query = false + } +} + +resource "google_compute_target_http_proxy" "frontend_redirect" { + count = var.enable_load_balancer && var.custom_domain != "" ? 1 : 0 + name = "frontend-http-proxy" + url_map = google_compute_url_map.frontend_redirect[0].id +} + +resource "google_compute_global_forwarding_rule" "frontend_http" { + count = var.enable_load_balancer && var.custom_domain != "" ? 1 : 0 + name = "frontend-http-forwarding" + target = google_compute_target_http_proxy.frontend_redirect[0].id + port_range = "80" + ip_address = google_compute_global_address.frontend[0].address + load_balancing_scheme = "EXTERNAL_MANAGED" +} + +# ============================================================================= +# CLOUD DNS (Optional) +# ============================================================================= + +resource "google_dns_managed_zone" "main" { + count = var.create_dns_zone ? 1 : 0 + name = "alex-zone" + dns_name = "${var.custom_domain}." + description = "DNS zone for Alex frontend" +} + +resource "google_dns_record_set" "frontend" { + count = var.enable_load_balancer && var.create_dns_zone ? 1 : 0 + name = "${var.custom_domain}." + managed_zone = google_dns_managed_zone.main[0].name + type = "A" + ttl = 300 + rrdatas = [google_compute_global_address.frontend[0].address] +} diff --git a/gcp-deployment/terraform/7_frontend/outputs.tf b/gcp-deployment/terraform/7_frontend/outputs.tf new file mode 100644 index 00000000..cd6e83da --- /dev/null +++ b/gcp-deployment/terraform/7_frontend/outputs.tf @@ -0,0 +1,33 @@ +# ============================================================================= +# OUTPUTS - Frontend +# ============================================================================= + +output "frontend_url" { + description = "Frontend Cloud Run URL" + value = google_cloud_run_v2_service.frontend.uri +} + +output "load_balancer_ip" { + description = "Load balancer IP address" + value = var.enable_load_balancer ? google_compute_global_address.frontend[0].address : null +} + +output "custom_domain_url" { + description = "Custom domain URL" + value = var.custom_domain != "" ? "https://${var.custom_domain}" : null +} + +output "clerk_publishable_key_secret_id" { + description = "Secret Manager ID for Clerk publishable key" + value = "clerk-publishable-key" +} + +output "clerk_secret_key_secret_id" { + description = "Secret Manager ID for Clerk secret key" + value = "clerk-secret-key" +} + +output "dns_nameservers" { + description = "DNS nameservers (if zone created)" + value = var.create_dns_zone ? google_dns_managed_zone.main[0].name_servers : null +} diff --git a/gcp-deployment/terraform/7_frontend/variables.tf b/gcp-deployment/terraform/7_frontend/variables.tf new file mode 100644 index 00000000..3248fdde --- /dev/null +++ b/gcp-deployment/terraform/7_frontend/variables.tf @@ -0,0 +1,89 @@ +# ============================================================================= +# VARIABLES - Frontend +# ============================================================================= + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "artifact_registry_url" { + description = "Artifact Registry URL" + type = string +} + +variable "image_tag" { + description = "Container image tag" + type = string + default = "latest" +} + +variable "cloud_run_service_account" { + description = "Service account email for Cloud Run" + type = string +} + +variable "backend_api_url" { + description = "Backend API URL (orchestrator)" + type = string +} + +variable "min_instances" { + description = "Minimum number of instances" + type = number + default = 0 +} + +variable "max_instances" { + description = "Maximum number of instances" + type = number + default = 10 +} + +variable "cpu_limit" { + description = "CPU limit" + type = string + default = "1" +} + +variable "memory_limit" { + description = "Memory limit" + type = string + default = "1Gi" +} + +variable "enable_load_balancer" { + description = "Enable Cloud Load Balancer" + type = bool + default = false +} + +variable "enable_cdn" { + description = "Enable Cloud CDN" + type = bool + default = false +} + +variable "custom_domain" { + description = "Custom domain name" + type = string + default = "" +} + +variable "create_dns_zone" { + description = "Create Cloud DNS zone" + type = bool + default = false +}