Bootcamp - Dmitry: Fixing bugs

backend/charter/test_simple.py

@@ -5,6 +5,7 @@
 
 import asyncio
 import json
+from decimal import Decimal
 from dotenv import load_dotenv
 
 load_dotenv(override=True)
@@ -19,8 +20,21 @@ def test_charter():
 
     # Create a real job in the database
     db = Database()
+
+    # Ensure test user exists
+    test_user_id = "test_user_001"
+    existing_user = db.users.find_by_clerk_id(test_user_id)
+    if not existing_user:
+        db.users.create_user(
+            clerk_user_id=test_user_id,
+            display_name="Test User",
+            years_until_retirement=25,
+            target_retirement_income=Decimal('75000')
+        )
+        print(f"Created test user: {test_user_id}")
+
     job_create = JobCreate(
-        clerk_user_id="test_user_001", job_type="portfolio_analysis", request_payload={"test": True}
+        clerk_user_id=test_user_id, job_type="portfolio_analysis", request_payload={"test": True}
     )
     job_id = db.jobs.create(job_create.model_dump())
     print(f"Created test job: {job_id}")

backend/reporter/test_simple.py

@@ -5,6 +5,7 @@
 
 import asyncio
 import json
+from decimal import Decimal
 from dotenv import load_dotenv
 
 load_dotenv(override=True)
@@ -18,8 +19,21 @@ def test_reporter():
 
     # Create a real job in the database
     db = Database()
+
+    # Ensure test user exists
+    test_user_id = "test_user_001"
+    existing_user = db.users.find_by_clerk_id(test_user_id)
+    if not existing_user:
+        db.users.create_user(
+            clerk_user_id=test_user_id,
+            display_name="Test User",
+            years_until_retirement=25,
+            target_retirement_income=Decimal('75000')
+        )
+        print(f"Created test user: {test_user_id}")
+
     job_create = JobCreate(
-        clerk_user_id="test_user_001",
+        clerk_user_id=test_user_id,
         job_type="portfolio_analysis",
         request_payload={"test": True}
     )

backend/researcher/server.py

@@ -54,7 +54,7 @@ async def run_research_agent(topic: str = None) -> str:
     # bedrock/openai.gpt-oss-120b-1:0 for OpenAI OSS models
     # bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 for Claude Sonnet 4
     # NOTE that nova-pro is needed to support tools and MCP servers; nova-lite is not enough - thank you Yuelin L.!
-    MODEL = "bedrock/us.amazon.nova-pro-v1:0"
+    MODEL = "bedrock/amazon.nova-pro-v1:0"
     model = LitellmModel(model=MODEL)
 
     # Create and run the agent with MCP server

backend/retirement/agent.py

@@ -93,7 +93,8 @@ def run_monte_carlo_simulation(
     real_estate_return_std = 0.12
 
     successful_scenarios = 0
-    final_values = []
+    final_values = []  # After retirement withdrawals
+    values_at_retirement = []  # At retirement (before withdrawals start)
     years_lasted = []
 
     for _ in range(num_simulations):
@@ -115,6 +116,10 @@ def run_monte_carlo_simulation(
             portfolio_value = portfolio_value * (1 + portfolio_return)
             portfolio_value += 10000  # Annual contribution
 
+        # Store value at retirement (before withdrawals start)
+        value_at_retirement = portfolio_value
+        values_at_retirement.append(value_at_retirement)
+
         # Retirement phase
         retirement_years = 30
         annual_withdrawal = target_annual_income
@@ -151,6 +156,7 @@ def run_monte_carlo_simulation(
 
     # Calculate statistics
     final_values.sort()
+    values_at_retirement.sort()  # Sort for percentile calculation
     success_rate = (successful_scenarios / num_simulations) * 100
 
     # Calculate expected value at retirement
@@ -168,8 +174,8 @@ def run_monte_carlo_simulation(
     return {
         "success_rate": round(success_rate, 1),
         "median_final_value": round(final_values[num_simulations // 2], 2),
-        "percentile_10": round(final_values[num_simulations // 10], 2),
-        "percentile_90": round(final_values[9 * num_simulations // 10], 2),
+        "percentile_10": round(values_at_retirement[num_simulations // 10], 2),  # Value at retirement
+        "percentile_90": round(values_at_retirement[9 * num_simulations // 10], 2),  # Value at retirement
         "average_years_lasted": round(sum(years_lasted) / len(years_lasted), 1),
         "expected_value_at_retirement": round(expected_value_at_retirement, 2),
     }
@@ -284,9 +290,9 @@ def create_agent(
 ## Monte Carlo Simulation Results (500 scenarios)
 - Success Rate: {monte_carlo["success_rate"]}% (probability of sustaining retirement income for 30 years)
 - Expected Portfolio Value at Retirement: ${monte_carlo["expected_value_at_retirement"]:,.0f}
-- 10th Percentile Outcome: ${monte_carlo["percentile_10"]:,.0f} (worst case)
-- Median Final Value: ${monte_carlo["median_final_value"]:,.0f}
-- 90th Percentile Outcome: ${monte_carlo["percentile_90"]:,.0f} (best case)
+- 10th Percentile Value at Retirement: ${monte_carlo["percentile_10"]:,.0f} (worst-case scenario at retirement)
+- Median Final Value (after 30 years): ${monte_carlo["median_final_value"]:,.0f}
+- 90th Percentile Value at Retirement: ${monte_carlo["percentile_90"]:,.0f} (best-case scenario at retirement)
 - Average Years Portfolio Lasts: {monte_carlo["average_years_lasted"]} years
 
 ## Key Projections (Milestones)

backend/retirement/test_simple.py

@@ -5,6 +5,7 @@
 
 import asyncio
 import json
+from decimal import Decimal
 from dotenv import load_dotenv
 
 load_dotenv(override=True)
@@ -18,8 +19,21 @@ def test_retirement():
 
     # Create a real job in the database
     db = Database()
+
+    # Ensure test user exists
+    test_user_id = "test_user_001"
+    existing_user = db.users.find_by_clerk_id(test_user_id)
+    if not existing_user:
+        db.users.create_user(
+            clerk_user_id=test_user_id,
+            display_name="Test User",
+            years_until_retirement=25,
+            target_retirement_income=Decimal('75000')
+        )
+        print(f"Created test user: {test_user_id}")
+
     job_create = JobCreate(
-        clerk_user_id="test_user_001",
+        clerk_user_id=test_user_id,
         job_type="portfolio_analysis",
         request_payload={"test": True}
     )

backend/tagger/agent.py

@@ -3,7 +3,7 @@
 """
 
 import os
-from typing import List
+from typing import List, Dict
 import logging
 from decimal import Decimal
 
@@ -108,8 +108,20 @@ class InstrumentClassification(BaseModel):
     @field_validator("allocation_asset_class")
     def validate_asset_class_sum(cls, v: AllocationBreakdown):
         total = v.equity + v.fixed_income + v.real_estate + v.commodities + v.cash + v.alternatives
-        if abs(total - 100.0) > 3:  # Allow small floating point errors
-            raise ValueError(f"Asset class allocations must sum to 100.0, got {total}")
+        # Normalize if sum is close to 100 (within reasonable range)
+        if abs(total - 100.0) > 0.1:  # Only normalize if not exactly 100
+            if 50.0 <= total <= 150.0:  # Reasonable range for normalization
+                # Scale all values proportionally to sum to 100
+                scale_factor = 100.0 / total
+                v.equity *= scale_factor
+                v.fixed_income *= scale_factor
+                v.real_estate *= scale_factor
+                v.commodities *= scale_factor
+                v.cash *= scale_factor
+                v.alternatives *= scale_factor
+            else:
+                # Sum is way off - reject it
+                raise ValueError(f"Asset class allocations must sum to approximately 100.0, got {total}")
         return v
 
     @field_validator("allocation_regions")
@@ -125,8 +137,23 @@ def validate_regions_sum(cls, v: RegionAllocation):
             + v.global_
             + v.international
         )
-        if abs(total - 100.0) > 3:
-            raise ValueError(f"Regional allocations must sum to 100.0, got {total}")
+        # Normalize if sum is close to 100 (within reasonable range)
+        if abs(total - 100.0) > 0.1:  # Only normalize if not exactly 100
+            if 50.0 <= total <= 150.0:  # Reasonable range for normalization
+                # Scale all values proportionally to sum to 100
+                scale_factor = 100.0 / total
+                v.north_america *= scale_factor
+                v.europe *= scale_factor
+                v.asia *= scale_factor
+                v.latin_america *= scale_factor
+                v.africa *= scale_factor
+                v.middle_east *= scale_factor
+                v.oceania *= scale_factor
+                v.global_ *= scale_factor
+                v.international *= scale_factor
+            else:
+                # Sum is way off - reject it
+                raise ValueError(f"Regional allocations must sum to approximately 100.0, got {total}")
         return v
 
     @field_validator("allocation_sectors")
@@ -151,8 +178,32 @@ def validate_sectors_sum(cls, v: SectorAllocation):
             + v.diversified
             + v.other
         )
-        if abs(total - 100.0) > 3:
-            raise ValueError(f"Sector allocations must sum to 100.0, got {total}")
+        # Normalize if sum is close to 100 (within reasonable range)
+        if abs(total - 100.0) > 0.1:  # Only normalize if not exactly 100
+            if 50.0 <= total <= 150.0:  # Reasonable range for normalization
+                # Scale all values proportionally to sum to 100
+                scale_factor = 100.0 / total
+                v.technology *= scale_factor
+                v.healthcare *= scale_factor
+                v.financials *= scale_factor
+                v.consumer_discretionary *= scale_factor
+                v.consumer_staples *= scale_factor
+                v.industrials *= scale_factor
+                v.materials *= scale_factor
+                v.energy *= scale_factor
+                v.utilities *= scale_factor
+                v.real_estate *= scale_factor
+                v.communication *= scale_factor
+                v.treasury *= scale_factor
+                v.corporate *= scale_factor
+                v.mortgage *= scale_factor
+                v.government_related *= scale_factor
+                v.commodities *= scale_factor
+                v.diversified *= scale_factor
+                v.other *= scale_factor
+            else:
+                # Sum is way off - reject it
+                raise ValueError(f"Sector allocations must sum to approximately 100.0, got {total}")
         return v
 
 
@@ -205,15 +256,15 @@ async def classify_instrument(
         raise
 
 
-async def tag_instruments(instruments: List[dict]) -> List[InstrumentClassification]:
+async def tag_instruments(instruments: List[dict]) -> tuple[List[InstrumentClassification], List[Dict[str, str]]]:
     """
     Tag multiple instruments with simple retry logic.
 
     Args:
         instruments: List of dicts with symbol, name, and optionally instrument_type
 
     Returns:
-        List of classifications
+        Tuple of (list of successful classifications, list of errors with symbol and error message)
     """
     import asyncio
 
@@ -230,26 +281,32 @@ async def classify_with_retry(symbol, name, instrument_type):
         return await classify_instrument(symbol, name, instrument_type)
 
     # Process instruments sequentially with small delay
-    results = []
+    classifications = []
+    errors = []
+
     for i, instrument in enumerate(instruments):
         # Small delay between requests to avoid rate limits
         if i > 0:
             await asyncio.sleep(0.5)
 
+        symbol = instrument["symbol"]
         try:
             classification = await classify_with_retry(
-                symbol=instrument["symbol"],
+                symbol=symbol,
                 name=instrument.get("name", ""),
                 instrument_type=instrument.get("instrument_type", "etf"),
             )
-            logger.info(f"Successfully classified {instrument['symbol']}")
-            results.append(classification)
+            logger.info(f"Successfully classified {symbol}")
+            classifications.append(classification)
         except Exception as e:
-            logger.error(f"Failed to classify {instrument['symbol']}: {e}")
-            results.append(None)
-
-    # Filter out None values
-    return [r for r in results if r is not None]
+            error_msg = str(e)
+            logger.error(f"Failed to classify {symbol}: {error_msg}")
+            errors.append({
+                'symbol': symbol,
+                'error': error_msg
+            })
+
+    return classifications, errors
 
 
 def classification_to_db_format(classification: InstrumentClassification) -> InstrumentCreate:

backend/tagger/lambda_handler.py

@@ -33,11 +33,13 @@ async def process_instruments(instruments: List[Dict[str, str]]) -> Dict[str, An
     """
     # Run the classification
     logger.info(f"Classifying {len(instruments)} instruments")
-    classifications = await tag_instruments(instruments)
+    classifications, classification_errors = await tag_instruments(instruments)
+
+    # Start with classification errors
+    errors = classification_errors.copy()
 
     # Update database with classifications
     updated = []
-    errors = []
 
     for classification in classifications:
         try:

backend/tagger/package_docker.py

@@ -99,24 +99,58 @@ def deploy_lambda(zip_path):
     import boto3
 
     lambda_client = boto3.client('lambda')
+    s3_client = boto3.client('s3')
     function_name = 'alex-tagger'
 
     print(f"Deploying to Lambda function: {function_name}")
 
+    # Check file size (Lambda direct upload limit is ~50MB, but we use 70MB as threshold)
+    file_size = zip_path.stat().st_size
+    size_mb = file_size / (1024 * 1024)
+    max_direct_upload = 70 * 1024 * 1024  # 70 MB
+
     try:
-        # Try to update existing function
-        with open(zip_path, 'rb') as f:
+        if file_size > max_direct_upload:
+            # Package is too large, must use S3
+            print(f"Package size ({size_mb:.1f} MB) exceeds direct upload limit, using S3...")
+
+            # Get AWS account ID
+            sts_client = boto3.client('sts')
+            account_id = sts_client.get_caller_identity()['Account']
+            bucket_name = f"alex-lambda-packages-{account_id}"
+            key = 'tagger/tagger_lambda.zip'
+
+            # Upload to S3
+            print(f"Uploading to S3: s3://{bucket_name}/{key}")
+            with open(zip_path, 'rb') as f:
+                s3_client.upload_fileobj(f, bucket_name, key)
+            print(f"✅ Uploaded to S3")
+
+            # Update Lambda from S3
+            print("Updating Lambda function from S3...")
             response = lambda_client.update_function_code(
                 FunctionName=function_name,
-                ZipFile=f.read()
+                S3Bucket=bucket_name,
+                S3Key=key
             )
-        print(f"Successfully updated Lambda function: {function_name}")
-        print(f"Function ARN: {response['FunctionArn']}")
+        else:
+            # Small enough for direct upload
+            print(f"Uploading directly ({size_mb:.1f} MB)...")
+            with open(zip_path, 'rb') as f:
+                response = lambda_client.update_function_code(
+                    FunctionName=function_name,
+                    ZipFile=f.read()
+                )
+
+        print(f"✅ Successfully updated Lambda function: {function_name}")
+        print(f"   Function ARN: {response['FunctionArn']}")
+        print(f"   Last modified: {response['LastModified']}")
+
     except lambda_client.exceptions.ResourceNotFoundException:
-        print(f"Lambda function {function_name} not found. Please deploy via Terraform first.")
+        print(f"❌ Lambda function {function_name} not found. Please deploy via Terraform first.")
         sys.exit(1)
     except Exception as e:
-        print(f"Error deploying Lambda: {e}")
+        print(f"❌ Error deploying Lambda: {e}")
         sys.exit(1)
 
 def main():