From f5d80457131090fb4059cf997985dff85718f4cd Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Mon, 1 Sep 2025 17:32:41 -0400 Subject: [PATCH 1/2] feat: Implement Phase 1 synthetic data training integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add SyntheticDataInjector for injecting synthetic data into Redis channels - Create TrainingBridge to coordinate between synthetic data and agent training - Implement AgentTrainingHarness for orchestrating complete training sessions - Add TrainingConsumer extending BaseAgentRedisConsumer with training capabilities Key features: - 10 pre-defined training scenarios (games, trading, market events) - Adaptive difficulty adjustment based on performance - Real-time monitoring and checkpointing - Comprehensive metrics (Sharpe ratio, win rate, P&L) - Confidence calibration integration - Memory system for experience replay This establishes the foundation for training Kalshi agents using synthetic data while maintaining full compatibility with the production Redis infrastructure. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/agents/training_consumer.py | 594 +++++++++++ src/confidence_calibration/__init__.py | 38 + .../bootstrap_estimator.py | 820 +++++++++++++++ src/confidence_calibration/calibrator.py | 957 ++++++++++++++++++ src/hybrid_pipeline/__init__.py | 38 + src/hybrid_pipeline/adaptive_scheduler.py | 908 +++++++++++++++++ src/hybrid_pipeline/cost_monitor.py | 610 +++++++++++ src/hybrid_pipeline/data_orchestrator.py | 779 ++++++++++++++ src/integration/__init__.py | 52 + src/integration/synthetic_injector.py | 688 +++++++++++++ src/integration/training_bridge.py | 691 +++++++++++++ src/integration/training_harness.py | 845 ++++++++++++++++ src/synthetic_data/__init__.py | 20 + src/synthetic_data/generators/__init__.py | 26 + src/synthetic_data/generators/game_engine.py | 896 ++++++++++++++++ .../generators/market_simulator.py | 754 ++++++++++++++ .../generators/scenario_builder.py | 817 +++++++++++++++ src/synthetic_data/models/__init__.py | 14 + src/synthetic_data/models/lfm2_fine_tuner.py | 561 ++++++++++ src/synthetic_data/preprocessing/__init__.py | 11 + .../preprocessing/nfl_dataset_processor.py | 434 ++++++++ .../preprocessing/training_data_builder.py | 258 +++++ src/synthetic_data/storage/__init__.py | 11 + .../storage/chromadb_manager.py | 556 ++++++++++ .../storage/synthetic_event_store.py | 209 ++++ src/synthetic_data/validation/__init__.py | 14 + .../validation/play_data_validator.py | 422 ++++++++ src/training/__init__.py | 42 + src/training/agent_analytics.py | 882 ++++++++++++++++ src/training/memory_system.py | 815 +++++++++++++++ src/training/synthetic_env.py | 672 ++++++++++++ 31 files changed, 14434 insertions(+) create mode 100644 src/agents/training_consumer.py create mode 100644 src/confidence_calibration/__init__.py create mode 100644 src/confidence_calibration/bootstrap_estimator.py create mode 100644 src/confidence_calibration/calibrator.py create mode 100644 src/hybrid_pipeline/__init__.py create mode 100644 src/hybrid_pipeline/adaptive_scheduler.py create mode 100644 src/hybrid_pipeline/cost_monitor.py create mode 100644 src/hybrid_pipeline/data_orchestrator.py create mode 100644 src/integration/__init__.py create mode 100644 src/integration/synthetic_injector.py create mode 100644 src/integration/training_bridge.py create mode 100644 src/integration/training_harness.py create mode 100644 src/synthetic_data/__init__.py create mode 100644 src/synthetic_data/generators/__init__.py create mode 100644 src/synthetic_data/generators/game_engine.py create mode 100644 src/synthetic_data/generators/market_simulator.py create mode 100644 src/synthetic_data/generators/scenario_builder.py create mode 100644 src/synthetic_data/models/__init__.py create mode 100644 src/synthetic_data/models/lfm2_fine_tuner.py create mode 100644 src/synthetic_data/preprocessing/__init__.py create mode 100644 src/synthetic_data/preprocessing/nfl_dataset_processor.py create mode 100644 src/synthetic_data/preprocessing/training_data_builder.py create mode 100644 src/synthetic_data/storage/__init__.py create mode 100644 src/synthetic_data/storage/chromadb_manager.py create mode 100644 src/synthetic_data/storage/synthetic_event_store.py create mode 100644 src/synthetic_data/validation/__init__.py create mode 100644 src/synthetic_data/validation/play_data_validator.py create mode 100644 src/training/__init__.py create mode 100644 src/training/agent_analytics.py create mode 100644 src/training/memory_system.py create mode 100644 src/training/synthetic_env.py diff --git a/src/agents/training_consumer.py b/src/agents/training_consumer.py new file mode 100644 index 00000000..ca840ed7 --- /dev/null +++ b/src/agents/training_consumer.py @@ -0,0 +1,594 @@ +""" +Training-Enhanced Redis Consumer + +Extends BaseAgentRedisConsumer with training mode support, +performance tracking, and integration with learning systems. +""" + +import asyncio +import logging +from typing import Dict, Any, Optional, List, Callable +from datetime import datetime, timedelta +from abc import abstractmethod +import json + +from .base_consumer import BaseAgentRedisConsumer +from ..training.agent_analytics import DecisionMetrics +from ..training.memory_system import AgentMemorySystem, AgentMemory, MemoryType +from ..confidence_calibration.calibrator import ConfidenceCalibrator + + +class TrainingConsumer(BaseAgentRedisConsumer): + """ + Enhanced Redis consumer with training mode capabilities. + + Adds: + - Training/production mode switching + - Decision tracking and analytics + - Memory system integration + - Confidence calibration + - Performance monitoring + """ + + def __init__( + self, + agent_name: str, + redis_url: str = "redis://localhost:6379", + agent_context: Optional[Any] = None, + training_mode: bool = False + ): + """ + Initialize training-enhanced consumer. + + Args: + agent_name: Unique agent identifier + redis_url: Redis connection URL + agent_context: Agentuity context (optional) + training_mode: Whether to start in training mode + """ + super().__init__(agent_name, redis_url, agent_context) + + self.training_mode = training_mode + self.training_session_id: Optional[str] = None + self.logger = logging.getLogger(__name__) + + # Training systems (will be injected) + self.analytics: Optional[Any] = None + self.memory_system: Optional[AgentMemorySystem] = None + self.calibrator: Optional[ConfidenceCalibrator] = None + + # Performance tracking + self.decision_buffer: List[Dict[str, Any]] = [] + self.pending_decisions: Dict[str, Dict[str, Any]] = {} # Track outcomes + + # Training configuration + self.training_config = { + "track_all_decisions": True, + "confidence_threshold": 0.3, + "max_position_size": 0.1, + "use_calibrated_confidence": True, + "store_experiences": True, + "learn_from_mistakes": True + } + + # Metrics + self.training_metrics = { + "decisions_made": 0, + "successful_decisions": 0, + "failed_decisions": 0, + "total_pnl": 0.0, + "confidence_sum": 0.0, + "kelly_adherence_sum": 0.0 + } + + def set_training_mode(self, enabled: bool, session_id: Optional[str] = None) -> None: + """ + Enable or disable training mode. + + Args: + enabled: Whether to enable training mode + session_id: Optional training session identifier + """ + self.training_mode = enabled + self.training_session_id = session_id + + if enabled: + self.logger.info(f"{self.agent_name} entering training mode (session: {session_id})") + self._reset_training_metrics() + else: + self.logger.info(f"{self.agent_name} exiting training mode") + self._finalize_training_metrics() + + def inject_training_systems( + self, + analytics: Any, + memory_system: AgentMemorySystem, + calibrator: ConfidenceCalibrator + ) -> None: + """ + Inject training system dependencies. + + Args: + analytics: Agent analytics system + memory_system: Memory storage system + calibrator: Confidence calibration system + """ + self.analytics = analytics + self.memory_system = memory_system + self.calibrator = calibrator + + self.logger.info(f"Training systems injected for {self.agent_name}") + + async def process_message(self, channel: str, data: Dict[str, Any]) -> None: + """ + Process message with training enhancements. + + Wraps the agent's process_message to add training functionality. + + Args: + channel: Redis channel the message came from + data: Message data + """ + # Pre-process for training + if self.training_mode: + await self._pre_process_training(channel, data) + + # Call agent's implementation + await self.process_training_message(channel, data) + + # Post-process for training + if self.training_mode: + await self._post_process_training(channel, data) + + @abstractmethod + async def process_training_message(self, channel: str, data: Dict[str, Any]) -> None: + """ + Process message - to be implemented by specific agents. + + This replaces the original process_message for training-aware agents. + + Args: + channel: Redis channel + data: Message data + """ + pass + + async def _pre_process_training(self, channel: str, data: Dict[str, Any]) -> None: + """Pre-process message for training tracking""" + try: + # Add training context + data['_training_context'] = { + 'received_at': datetime.now().isoformat(), + 'channel': channel, + 'session_id': self.training_session_id + } + + # Store in memory if configured + if self.memory_system and self.training_config["store_experiences"]: + await self._store_incoming_event(channel, data) + + except Exception as e: + self.logger.error(f"Training pre-process error: {e}") + + async def _post_process_training(self, channel: str, data: Dict[str, Any]) -> None: + """Post-process message for training tracking""" + try: + # Check if a decision was made + if hasattr(self, '_last_decision') and self._last_decision: + await self._track_training_decision(self._last_decision) + self._last_decision = None + + except Exception as e: + self.logger.error(f"Training post-process error: {e}") + + async def make_trading_decision( + self, + market_ticker: str, + decision_type: str, + confidence: float, + position_size: float = 0.0, + expected_value: float = 0.0, + kelly_fraction: float = 0.0, + context: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Make a trading decision with training tracking. + + Args: + market_ticker: Market to trade + decision_type: "buy", "sell", "hold" + confidence: Raw confidence score (0-1) + position_size: Position size to take + expected_value: Expected value of decision + kelly_fraction: Optimal Kelly fraction + context: Additional context + + Returns: + Decision details with potential calibration + """ + try: + # Apply confidence calibration if available + calibrated_confidence = confidence + uncertainty = 0.0 + + if self.calibrator and self.training_config["use_calibrated_confidence"]: + calibration_result = await self.calibrator.calibrate_confidence( + agent_id=self.agent_name, + raw_confidence=confidence, + context=context or {} + ) + calibrated_confidence = calibration_result.calibrated_confidence + uncertainty = calibration_result.uncertainty + + # Check confidence threshold + if calibrated_confidence < self.training_config["confidence_threshold"]: + decision_type = "hold" + position_size = 0.0 + + # Enforce position limits + if position_size > self.training_config["max_position_size"]: + position_size = self.training_config["max_position_size"] + + # Calculate actual Kelly used + actual_kelly_used = position_size / kelly_fraction if kelly_fraction > 0 else 0.0 + + # Create decision record + decision = { + "agent_id": self.agent_name, + "decision_id": f"{self.agent_name}_{datetime.now().timestamp()}", + "timestamp": datetime.now().isoformat(), + "market_ticker": market_ticker, + "decision_type": decision_type, + "raw_confidence": confidence, + "calibrated_confidence": calibrated_confidence, + "uncertainty": uncertainty, + "position_size": position_size, + "expected_value": expected_value, + "kelly_fraction": kelly_fraction, + "actual_kelly_used": actual_kelly_used, + "training_mode": self.training_mode, + "session_id": self.training_session_id, + "context": context + } + + # Store for tracking + self._last_decision = decision + + # Track in training mode + if self.training_mode: + self.training_metrics["decisions_made"] += 1 + self.training_metrics["confidence_sum"] += calibrated_confidence + self.training_metrics["kelly_adherence_sum"] += abs(actual_kelly_used - 1.0) + + # Store pending for outcome tracking + self.pending_decisions[decision["decision_id"]] = decision + + # Publish to training channel + await self._publish_training_decision(decision) + + return decision + + except Exception as e: + self.logger.error(f"Error making trading decision: {e}") + return { + "error": str(e), + "decision_type": "hold", + "position_size": 0.0 + } + + async def _track_training_decision(self, decision: Dict[str, Any]) -> None: + """Track a decision for training analytics""" + try: + if not self.analytics: + return + + # Create DecisionMetrics object + metrics = DecisionMetrics( + decision_id=decision["decision_id"], + agent_id=self.agent_name, + scenario_id=self.training_session_id or "unknown", + timestamp=datetime.fromisoformat(decision["timestamp"]), + market_ticker=decision["market_ticker"], + decision_type=decision["decision_type"], + confidence=decision["calibrated_confidence"], + expected_value=decision["expected_value"], + kelly_fraction=decision["kelly_fraction"], + actual_kelly_used=decision["actual_kelly_used"], + position_size=decision["position_size"], + market_efficiency=decision.get("context", {}).get("market_efficiency", 0.8), + information_advantage=decision.get("context", {}).get("information_advantage", 0.0), + execution_latency=(datetime.now() - datetime.fromisoformat(decision["timestamp"])).total_seconds() + ) + + # Record in analytics + await self.analytics.record_decision(metrics) + + # Buffer for batch processing + self.decision_buffer.append(decision) + + # Process buffer if full + if len(self.decision_buffer) >= 10: + await self._process_decision_buffer() + + except Exception as e: + self.logger.error(f"Failed to track training decision: {e}") + + async def _process_decision_buffer(self) -> None: + """Process buffered decisions for batch analytics""" + try: + if not self.decision_buffer: + return + + # Batch process decisions + for decision in self.decision_buffer: + # Store in memory system if available + if self.memory_system and self.training_config["store_experiences"]: + await self._store_decision_memory(decision) + + # Clear buffer + self.decision_buffer.clear() + + except Exception as e: + self.logger.error(f"Failed to process decision buffer: {e}") + + async def _store_decision_memory(self, decision: Dict[str, Any]) -> None: + """Store decision in memory system""" + try: + if not self.memory_system: + return + + # Create memory entry + memory = AgentMemory( + memory_id=f"decision_{decision['decision_id']}", + agent_id=self.agent_name, + memory_type=MemoryType.EXPERIENCE, + timestamp=datetime.fromisoformat(decision["timestamp"]), + description=f"{decision['decision_type']} on {decision['market_ticker']} with confidence {decision['calibrated_confidence']:.2f}", + context=decision.get("context", {}), + outcome={"pending": True} # Will be updated when outcome known + ) + + # Store in memory system + await self.memory_system.store_memory(memory) + + except Exception as e: + self.logger.error(f"Failed to store decision memory: {e}") + + async def _store_incoming_event(self, channel: str, data: Dict[str, Any]) -> None: + """Store incoming event in memory for pattern learning""" + try: + if not self.memory_system: + return + + # Create memory entry for significant events + if self._is_significant_event(channel, data): + memory = AgentMemory( + memory_id=f"event_{self.agent_name}_{datetime.now().timestamp()}", + agent_id=self.agent_name, + memory_type=MemoryType.PATTERN, + timestamp=datetime.now(), + description=f"Event from {channel}: {data.get('type', 'unknown')}", + context={ + "channel": channel, + "data": data, + "training_session": self.training_session_id + }, + outcome={} + ) + + await self.memory_system.store_memory(memory) + + except Exception as e: + self.logger.error(f"Failed to store incoming event: {e}") + + def _is_significant_event(self, channel: str, data: Dict[str, Any]) -> bool: + """Determine if an event is significant enough to store""" + # Store market updates, big plays, trades, high-impact events + significant_types = ["market_update", "trade_executed", "big_play", "signal", "injury_alert"] + event_type = data.get("type", "").lower() + + return any(sig in event_type for sig in significant_types) + + async def report_decision_outcome( + self, + decision_id: str, + outcome: float, + metadata: Optional[Dict[str, Any]] = None + ) -> None: + """ + Report the outcome of a previous decision. + + Args: + decision_id: ID of the decision + outcome: P&L outcome + metadata: Additional outcome information + """ + try: + # Find pending decision + decision = self.pending_decisions.get(decision_id) + if not decision: + return + + # Update metrics + self.training_metrics["total_pnl"] += outcome + if outcome > 0: + self.training_metrics["successful_decisions"] += 1 + else: + self.training_metrics["failed_decisions"] += 1 + + # Update analytics if available + if self.analytics: + # Find and update the decision metrics + # This would need to be implemented in analytics + pass + + # Update memory with outcome + if self.memory_system: + await self._update_memory_outcome(decision_id, outcome, metadata) + + # Learn from mistakes if configured + if self.training_config["learn_from_mistakes"] and outcome < 0: + await self._learn_from_mistake(decision, outcome, metadata) + + # Remove from pending + del self.pending_decisions[decision_id] + + except Exception as e: + self.logger.error(f"Failed to report decision outcome: {e}") + + async def _update_memory_outcome( + self, + decision_id: str, + outcome: float, + metadata: Optional[Dict[str, Any]] + ) -> None: + """Update stored memory with decision outcome""" + try: + if not self.memory_system: + return + + # Update the memory entry with outcome + memory_id = f"decision_{decision_id}" + + # This would need memory_system to support updates + # For now, store a new memory linking to the decision + outcome_memory = AgentMemory( + memory_id=f"outcome_{decision_id}", + agent_id=self.agent_name, + memory_type=MemoryType.EXPERIENCE, + timestamp=datetime.now(), + description=f"Outcome for decision {decision_id}: {'profit' if outcome > 0 else 'loss'} of {outcome:.2f}", + context={"decision_id": decision_id, "metadata": metadata}, + outcome={"pnl": outcome, "success": outcome > 0} + ) + + await self.memory_system.store_memory(outcome_memory) + + except Exception as e: + self.logger.error(f"Failed to update memory outcome: {e}") + + async def _learn_from_mistake( + self, + decision: Dict[str, Any], + outcome: float, + metadata: Optional[Dict[str, Any]] + ) -> None: + """Learn from a failed decision""" + try: + if not self.memory_system: + return + + # Store as a mistake to avoid in future + mistake_memory = AgentMemory( + memory_id=f"mistake_{decision['decision_id']}", + agent_id=self.agent_name, + memory_type=MemoryType.MISTAKE, + timestamp=datetime.now(), + description=f"Failed {decision['decision_type']} on {decision['market_ticker']}: lost {abs(outcome):.2f}", + context={ + "decision": decision, + "outcome": outcome, + "metadata": metadata, + "lesson": self._extract_lesson(decision, outcome, metadata) + }, + outcome={"loss": abs(outcome)} + ) + + await self.memory_system.store_memory(mistake_memory) + + self.logger.info(f"Learned from mistake: {mistake_memory.description}") + + except Exception as e: + self.logger.error(f"Failed to learn from mistake: {e}") + + def _extract_lesson( + self, + decision: Dict[str, Any], + outcome: float, + metadata: Optional[Dict[str, Any]] + ) -> str: + """Extract a lesson from a failed decision""" + lessons = [] + + # Check confidence calibration + if decision["calibrated_confidence"] > 0.7 and outcome < 0: + lessons.append("High confidence was misplaced") + + # Check Kelly adherence + if abs(decision["actual_kelly_used"] - 1.0) > 0.5: + lessons.append("Position sizing was suboptimal") + + # Check context factors + if metadata and "market_volatility" in metadata and metadata["market_volatility"] > 0.5: + lessons.append("Failed to account for high volatility") + + return "; ".join(lessons) if lessons else "General trading loss" + + async def _publish_training_decision(self, decision: Dict[str, Any]) -> None: + """Publish decision to training channels for tracking""" + try: + # Determine channel based on training namespace + channel = "training:agent_decisions" + if self.training_session_id: + channel = f"{channel}:{self.training_session_id}" + + # Publish decision + await self.publish(channel, decision) + + except Exception as e: + self.logger.error(f"Failed to publish training decision: {e}") + + def _reset_training_metrics(self) -> None: + """Reset training metrics for new session""" + self.training_metrics = { + "decisions_made": 0, + "successful_decisions": 0, + "failed_decisions": 0, + "total_pnl": 0.0, + "confidence_sum": 0.0, + "kelly_adherence_sum": 0.0 + } + self.decision_buffer.clear() + self.pending_decisions.clear() + + def _finalize_training_metrics(self) -> None: + """Finalize training metrics at end of session""" + try: + # Calculate averages + if self.training_metrics["decisions_made"] > 0: + avg_confidence = self.training_metrics["confidence_sum"] / self.training_metrics["decisions_made"] + avg_kelly_deviation = self.training_metrics["kelly_adherence_sum"] / self.training_metrics["decisions_made"] + win_rate = self.training_metrics["successful_decisions"] / self.training_metrics["decisions_made"] + + self.logger.info( + f"Training session complete for {self.agent_name}: " + f"{self.training_metrics['decisions_made']} decisions, " + f"Win rate: {win_rate:.2%}, " + f"P&L: {self.training_metrics['total_pnl']:.2f}, " + f"Avg confidence: {avg_confidence:.2f}, " + f"Avg Kelly deviation: {avg_kelly_deviation:.2f}" + ) + + # Process any remaining buffered decisions + asyncio.create_task(self._process_decision_buffer()) + + except Exception as e: + self.logger.error(f"Failed to finalize training metrics: {e}") + + def get_training_stats(self) -> Dict[str, Any]: + """Get current training statistics""" + stats = { + "agent": self.agent_name, + "training_mode": self.training_mode, + "session_id": self.training_session_id, + "metrics": dict(self.training_metrics), + "pending_decisions": len(self.pending_decisions), + "buffered_decisions": len(self.decision_buffer) + } + + # Add win rate if decisions made + if self.training_metrics["decisions_made"] > 0: + stats["win_rate"] = self.training_metrics["successful_decisions"] / self.training_metrics["decisions_made"] + stats["avg_confidence"] = self.training_metrics["confidence_sum"] / self.training_metrics["decisions_made"] + + return stats \ No newline at end of file diff --git a/src/confidence_calibration/__init__.py b/src/confidence_calibration/__init__.py new file mode 100644 index 00000000..6190c528 --- /dev/null +++ b/src/confidence_calibration/__init__.py @@ -0,0 +1,38 @@ +""" +Confidence Calibration Module + +Advanced confidence calibration and uncertainty quantification +for agent decision-making in trading scenarios. +""" + +from .calibrator import ( + ConfidenceCalibrator, + CalibrationMethod, + CalibrationMetrics, + ConfidenceScore, + UncertaintyQuantifier +) +from .bootstrap_estimator import ( + BootstrapConfidenceEstimator, + BootstrapConfig, + ConfidenceInterval +) +# from .bayesian_calibrator import ( +# BayesianCalibrator, +# PriorDistribution, +# BayesianUpdate +# ) + +__all__ = [ + 'ConfidenceCalibrator', + 'CalibrationMethod', + 'CalibrationMetrics', + 'ConfidenceScore', + 'UncertaintyQuantifier', + 'BootstrapConfidenceEstimator', + 'BootstrapConfig', + 'ConfidenceInterval' + # 'BayesianCalibrator', + # 'PriorDistribution', + # 'BayesianUpdate' +] \ No newline at end of file diff --git a/src/confidence_calibration/bootstrap_estimator.py b/src/confidence_calibration/bootstrap_estimator.py new file mode 100644 index 00000000..a0714873 --- /dev/null +++ b/src/confidence_calibration/bootstrap_estimator.py @@ -0,0 +1,820 @@ +""" +Bootstrap Confidence Estimator + +Uses bootstrap sampling to estimate confidence intervals and +uncertainty for agent decision-making scenarios. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Any, Callable +import numpy as np +import asyncio +import logging +from datetime import datetime +from collections import defaultdict +import random + + +@dataclass +class BootstrapConfig: + """Configuration for bootstrap estimation""" + n_bootstrap_samples: int = 1000 + confidence_level: float = 0.95 + sample_size_fraction: float = 1.0 # Fraction of original data to sample + random_seed: Optional[int] = None + parallel_execution: bool = True + max_workers: int = 4 + + # Advanced options + bias_correction: bool = True + acceleration_correction: bool = True # BCa intervals + minimum_sample_size: int = 30 + + +@dataclass +class ConfidenceInterval: + """Bootstrap confidence interval result""" + lower_bound: float + upper_bound: float + point_estimate: float + confidence_level: float + method: str # "percentile", "bias_corrected", "bca" + n_bootstrap_samples: int + + @property + def width(self) -> float: + """Width of confidence interval""" + return self.upper_bound - self.lower_bound + + @property + def margin_of_error(self) -> float: + """Margin of error (half-width)""" + return self.width / 2 + + +class BootstrapConfidenceEstimator: + """ + Bootstrap-based confidence estimation for agent decision metrics. + + Uses resampling techniques to estimate confidence intervals and + uncertainty quantification for various performance metrics. + """ + + def __init__(self, config: BootstrapConfig = None): + self.config = config or BootstrapConfig() + self.logger = logging.getLogger(__name__) + + # Set random seed for reproducibility + if self.config.random_seed is not None: + np.random.seed(self.config.random_seed) + random.seed(self.config.random_seed) + + # Cache for bootstrap results + self.bootstrap_cache: Dict[str, Any] = {} + self.cache_timestamps: Dict[str, datetime] = {} + + # Performance tracking + self.estimation_history: List[Dict[str, Any]] = [] + + async def estimate_win_rate_confidence( + self, + outcomes: List[bool], + agent_id: Optional[str] = None + ) -> ConfidenceInterval: + """ + Estimate confidence interval for win rate using bootstrap. + + Args: + outcomes: List of decision outcomes (True for win, False for loss) + agent_id: Optional agent identifier for caching + + Returns: + Confidence interval for win rate + """ + try: + if len(outcomes) < self.config.minimum_sample_size: + self.logger.warning(f"Sample size too small for bootstrap: {len(outcomes)}") + return self._create_wide_interval(np.mean(outcomes), "insufficient_data") + + # Check cache + cache_key = f"win_rate_{agent_id}_{hash(str(outcomes))}" if agent_id else f"win_rate_{hash(str(outcomes))}" + if cache_key in self.bootstrap_cache: + cached_result = self.bootstrap_cache[cache_key] + if datetime.now() - self.cache_timestamps[cache_key] < timedelta(minutes=30): + return cached_result + + # Bootstrap function for win rate + def win_rate_statistic(sample): + return np.mean(sample) + + # Perform bootstrap + bootstrap_estimates = await self._bootstrap_statistic( + data=outcomes, + statistic_func=win_rate_statistic, + description="win_rate" + ) + + # Calculate confidence interval + point_estimate = np.mean(outcomes) + interval = self._calculate_confidence_interval( + bootstrap_estimates, + point_estimate, + outcomes # Original data for BCa + ) + + # Cache result + self.bootstrap_cache[cache_key] = interval + self.cache_timestamps[cache_key] = datetime.now() + + return interval + + except Exception as e: + self.logger.error(f"Win rate confidence estimation failed: {e}") + return self._create_wide_interval(np.mean(outcomes) if outcomes else 0.5, "error") + + async def estimate_kelly_adherence_confidence( + self, + kelly_deviations: List[float], + agent_id: Optional[str] = None + ) -> ConfidenceInterval: + """ + Estimate confidence interval for Kelly Criterion adherence. + + Args: + kelly_deviations: List of Kelly fraction deviations + agent_id: Optional agent identifier for caching + + Returns: + Confidence interval for Kelly adherence score + """ + try: + if len(kelly_deviations) < self.config.minimum_sample_size: + return self._create_wide_interval(0.5, "insufficient_data") + + # Kelly adherence score = 1 - average deviation + def kelly_adherence_statistic(sample): + avg_deviation = np.mean(np.abs(sample)) + return max(0.0, 1.0 - avg_deviation) + + # Perform bootstrap + bootstrap_estimates = await self._bootstrap_statistic( + data=kelly_deviations, + statistic_func=kelly_adherence_statistic, + description="kelly_adherence" + ) + + # Calculate point estimate + point_estimate = kelly_adherence_statistic(kelly_deviations) + + # Calculate confidence interval + interval = self._calculate_confidence_interval( + bootstrap_estimates, + point_estimate, + kelly_deviations + ) + + return interval + + except Exception as e: + self.logger.error(f"Kelly adherence confidence estimation failed: {e}") + return self._create_wide_interval(0.5, "error") + + async def estimate_sharpe_ratio_confidence( + self, + returns: List[float], + agent_id: Optional[str] = None + ) -> ConfidenceInterval: + """ + Estimate confidence interval for Sharpe ratio. + + Args: + returns: List of return values + agent_id: Optional agent identifier for caching + + Returns: + Confidence interval for Sharpe ratio + """ + try: + if len(returns) < self.config.minimum_sample_size: + return self._create_wide_interval(0.0, "insufficient_data") + + def sharpe_ratio_statistic(sample): + if len(sample) < 2: + return 0.0 + mean_return = np.mean(sample) + std_return = np.std(sample, ddof=1) # Sample standard deviation + return mean_return / std_return if std_return > 0 else 0.0 + + # Perform bootstrap + bootstrap_estimates = await self._bootstrap_statistic( + data=returns, + statistic_func=sharpe_ratio_statistic, + description="sharpe_ratio" + ) + + # Calculate point estimate + point_estimate = sharpe_ratio_statistic(returns) + + # Calculate confidence interval + interval = self._calculate_confidence_interval( + bootstrap_estimates, + point_estimate, + returns + ) + + return interval + + except Exception as e: + self.logger.error(f"Sharpe ratio confidence estimation failed: {e}") + return self._create_wide_interval(0.0, "error") + + async def estimate_confidence_calibration_interval( + self, + confidences: List[float], + outcomes: List[bool], + agent_id: Optional[str] = None + ) -> ConfidenceInterval: + """ + Estimate confidence interval for confidence calibration score. + + Args: + confidences: List of confidence scores + outcomes: List of corresponding outcomes + agent_id: Optional agent identifier for caching + + Returns: + Confidence interval for calibration score + """ + try: + if len(confidences) != len(outcomes) or len(confidences) < self.config.minimum_sample_size: + return self._create_wide_interval(0.5, "insufficient_data") + + def calibration_statistic(indices): + # Resample both confidences and outcomes using same indices + sample_confidences = [confidences[i] for i in indices] + sample_outcomes = [outcomes[i] for i in indices] + + # Calculate Expected Calibration Error + return self._calculate_expected_calibration_error(sample_confidences, sample_outcomes) + + # Perform bootstrap with paired resampling + bootstrap_estimates = await self._bootstrap_paired_statistic( + data1=confidences, + data2=outcomes, + statistic_func=calibration_statistic, + description="confidence_calibration" + ) + + # Calculate point estimate + point_estimate = self._calculate_expected_calibration_error(confidences, outcomes) + + # For ECE, lower is better, so we want 1 - ECE as the score + calibration_scores = [max(0.0, 1.0 - ece) for ece in bootstrap_estimates] + point_calibration_score = max(0.0, 1.0 - point_estimate) + + # Calculate confidence interval + interval = self._calculate_confidence_interval( + calibration_scores, + point_calibration_score, + calibration_scores # Use bootstrap estimates as "original data" + ) + + return interval + + except Exception as e: + self.logger.error(f"Confidence calibration interval estimation failed: {e}") + return self._create_wide_interval(0.5, "error") + + def _calculate_expected_calibration_error(self, confidences: List[float], outcomes: List[bool]) -> float: + """Calculate Expected Calibration Error for bootstrap sampling""" + try: + if not confidences or len(confidences) != len(outcomes): + return 1.0 # Maximum error + + n_bins = min(10, len(confidences) // 5) # Adaptive binning + if n_bins < 2: + n_bins = 2 + + bin_boundaries = np.linspace(0, 1, n_bins + 1) + + ece = 0.0 + total_samples = len(confidences) + + for i in range(n_bins): + # Find samples in this bin + in_bin = [] + for j, conf in enumerate(confidences): + if i == n_bins - 1: # Last bin includes upper boundary + if bin_boundaries[i] <= conf <= bin_boundaries[i + 1]: + in_bin.append(j) + else: + if bin_boundaries[i] <= conf < bin_boundaries[i + 1]: + in_bin.append(j) + + if in_bin: + bin_confidences = [confidences[j] for j in in_bin] + bin_outcomes = [outcomes[j] for j in in_bin] + + bin_size = len(bin_confidences) + bin_conf_avg = np.mean(bin_confidences) + bin_accuracy = np.mean(bin_outcomes) + + ece += (bin_size / total_samples) * abs(bin_conf_avg - bin_accuracy) + + return ece + + except Exception as e: + self.logger.error(f"ECE calculation failed: {e}") + return 1.0 + + async def _bootstrap_statistic( + self, + data: List[Any], + statistic_func: Callable, + description: str + ) -> List[float]: + """ + Perform bootstrap resampling for a single dataset. + + Args: + data: Original dataset + statistic_func: Function to compute statistic on sample + description: Description for logging + + Returns: + List of bootstrap estimates + """ + try: + n_original = len(data) + sample_size = max(1, int(n_original * self.config.sample_size_fraction)) + + if self.config.parallel_execution and self.config.n_bootstrap_samples > 100: + return await self._bootstrap_parallel(data, statistic_func, sample_size, description) + else: + return await self._bootstrap_sequential(data, statistic_func, sample_size, description) + + except Exception as e: + self.logger.error(f"Bootstrap sampling failed for {description}: {e}") + return [statistic_func(data)] # Return original estimate + + async def _bootstrap_paired_statistic( + self, + data1: List[Any], + data2: List[Any], + statistic_func: Callable, + description: str + ) -> List[float]: + """ + Perform bootstrap resampling for paired datasets. + + Args: + data1: First dataset + data2: Second dataset (must be same length as data1) + statistic_func: Function that takes indices and computes statistic + description: Description for logging + + Returns: + List of bootstrap estimates + """ + try: + if len(data1) != len(data2): + raise ValueError("Paired datasets must have same length") + + n_original = len(data1) + sample_size = max(1, int(n_original * self.config.sample_size_fraction)) + + bootstrap_estimates = [] + + for i in range(self.config.n_bootstrap_samples): + # Sample indices with replacement + sample_indices = np.random.choice(n_original, size=sample_size, replace=True) + + # Compute statistic + estimate = statistic_func(sample_indices) + bootstrap_estimates.append(estimate) + + # Progress logging + if (i + 1) % 250 == 0: + self.logger.debug(f"Bootstrap {description}: {i + 1}/{self.config.n_bootstrap_samples}") + + return bootstrap_estimates + + except Exception as e: + self.logger.error(f"Paired bootstrap sampling failed for {description}: {e}") + return [0.5] # Default estimate + + async def _bootstrap_sequential( + self, + data: List[Any], + statistic_func: Callable, + sample_size: int, + description: str + ) -> List[float]: + """Sequential bootstrap sampling""" + try: + bootstrap_estimates = [] + + for i in range(self.config.n_bootstrap_samples): + # Sample with replacement + sample = np.random.choice(data, size=sample_size, replace=True) + + # Compute statistic + estimate = statistic_func(sample) + bootstrap_estimates.append(estimate) + + # Yield control occasionally for async + if i % 100 == 0: + await asyncio.sleep(0) + + # Progress logging + if (i + 1) % 250 == 0: + self.logger.debug(f"Bootstrap {description}: {i + 1}/{self.config.n_bootstrap_samples}") + + return bootstrap_estimates + + except Exception as e: + self.logger.error(f"Sequential bootstrap failed: {e}") + return [statistic_func(data)] + + async def _bootstrap_parallel( + self, + data: List[Any], + statistic_func: Callable, + sample_size: int, + description: str + ) -> List[float]: + """Parallel bootstrap sampling using asyncio""" + try: + # Split work into batches + batch_size = max(1, self.config.n_bootstrap_samples // self.config.max_workers) + + # Create tasks + tasks = [] + remaining_samples = self.config.n_bootstrap_samples + + for worker in range(self.config.max_workers): + if remaining_samples <= 0: + break + + worker_samples = min(batch_size, remaining_samples) + remaining_samples -= worker_samples + + task = asyncio.create_task( + self._bootstrap_worker(data, statistic_func, sample_size, worker_samples, f"{description}_worker_{worker}") + ) + tasks.append(task) + + # Wait for all workers to complete + worker_results = await asyncio.gather(*tasks) + + # Combine results + bootstrap_estimates = [] + for result in worker_results: + bootstrap_estimates.extend(result) + + return bootstrap_estimates + + except Exception as e: + self.logger.error(f"Parallel bootstrap failed: {e}") + # Fallback to sequential + return await self._bootstrap_sequential(data, statistic_func, sample_size, description) + + async def _bootstrap_worker( + self, + data: List[Any], + statistic_func: Callable, + sample_size: int, + n_samples: int, + worker_id: str + ) -> List[float]: + """Bootstrap worker for parallel execution""" + try: + estimates = [] + + for i in range(n_samples): + # Sample with replacement + sample = np.random.choice(data, size=sample_size, replace=True) + + # Compute statistic + estimate = statistic_func(sample) + estimates.append(estimate) + + # Yield control occasionally + if i % 50 == 0: + await asyncio.sleep(0) + + return estimates + + except Exception as e: + self.logger.error(f"Bootstrap worker {worker_id} failed: {e}") + return [] + + def _calculate_confidence_interval( + self, + bootstrap_estimates: List[float], + point_estimate: float, + original_data: List[Any] + ) -> ConfidenceInterval: + """ + Calculate confidence interval from bootstrap estimates. + + Uses bias-corrected and accelerated (BCa) method if enabled, + otherwise falls back to percentile method. + """ + try: + if not bootstrap_estimates: + return self._create_wide_interval(point_estimate, "no_bootstrap_data") + + bootstrap_estimates = np.array(bootstrap_estimates) + + # Remove any invalid estimates + valid_estimates = bootstrap_estimates[np.isfinite(bootstrap_estimates)] + if len(valid_estimates) == 0: + return self._create_wide_interval(point_estimate, "invalid_estimates") + + alpha = 1 - self.config.confidence_level + + if self.config.bias_correction and self.config.acceleration_correction: + # BCa intervals + return self._calculate_bca_interval(valid_estimates, point_estimate, original_data, alpha) + elif self.config.bias_correction: + # Bias-corrected intervals + return self._calculate_bc_interval(valid_estimates, point_estimate, alpha) + else: + # Simple percentile intervals + return self._calculate_percentile_interval(valid_estimates, point_estimate, alpha) + + except Exception as e: + self.logger.error(f"Confidence interval calculation failed: {e}") + return self._create_wide_interval(point_estimate, "calculation_error") + + def _calculate_percentile_interval( + self, + bootstrap_estimates: np.ndarray, + point_estimate: float, + alpha: float + ) -> ConfidenceInterval: + """Calculate simple percentile confidence interval""" + try: + lower_percentile = (alpha / 2) * 100 + upper_percentile = (1 - alpha / 2) * 100 + + lower_bound = np.percentile(bootstrap_estimates, lower_percentile) + upper_bound = np.percentile(bootstrap_estimates, upper_percentile) + + return ConfidenceInterval( + lower_bound=float(lower_bound), + upper_bound=float(upper_bound), + point_estimate=point_estimate, + confidence_level=self.config.confidence_level, + method="percentile", + n_bootstrap_samples=len(bootstrap_estimates) + ) + + except Exception as e: + self.logger.error(f"Percentile interval calculation failed: {e}") + return self._create_wide_interval(point_estimate, "percentile_error") + + def _calculate_bc_interval( + self, + bootstrap_estimates: np.ndarray, + point_estimate: float, + alpha: float + ) -> ConfidenceInterval: + """Calculate bias-corrected confidence interval""" + try: + # Calculate bias correction + n_less = np.sum(bootstrap_estimates < point_estimate) + p_less = n_less / len(bootstrap_estimates) + + if p_less == 0: + z0 = -np.inf + elif p_less == 1: + z0 = np.inf + else: + z0 = stats.norm.ppf(p_less) + + # Calculate corrected percentiles + z_alpha_2 = stats.norm.ppf(alpha / 2) + z_1_alpha_2 = stats.norm.ppf(1 - alpha / 2) + + # Apply bias correction + p1 = stats.norm.cdf(2 * z0 + z_alpha_2) + p2 = stats.norm.cdf(2 * z0 + z_1_alpha_2) + + # Ensure percentiles are within valid range + p1 = max(0.001, min(0.999, p1)) + p2 = max(0.001, min(0.999, p2)) + + lower_bound = np.percentile(bootstrap_estimates, p1 * 100) + upper_bound = np.percentile(bootstrap_estimates, p2 * 100) + + return ConfidenceInterval( + lower_bound=float(lower_bound), + upper_bound=float(upper_bound), + point_estimate=point_estimate, + confidence_level=self.config.confidence_level, + method="bias_corrected", + n_bootstrap_samples=len(bootstrap_estimates) + ) + + except Exception as e: + self.logger.error(f"Bias-corrected interval calculation failed: {e}") + return self._calculate_percentile_interval(bootstrap_estimates, point_estimate, alpha) + + def _calculate_bca_interval( + self, + bootstrap_estimates: np.ndarray, + point_estimate: float, + original_data: List[Any], + alpha: float + ) -> ConfidenceInterval: + """Calculate bias-corrected and accelerated (BCa) confidence interval""" + try: + # Calculate bias correction (same as BC method) + n_less = np.sum(bootstrap_estimates < point_estimate) + p_less = n_less / len(bootstrap_estimates) + + if p_less == 0: + z0 = -np.inf + elif p_less == 1: + z0 = np.inf + else: + z0 = stats.norm.ppf(p_less) + + # Calculate acceleration using jackknife + acceleration = self._calculate_acceleration(original_data, point_estimate) + + # Calculate corrected percentiles with acceleration + z_alpha_2 = stats.norm.ppf(alpha / 2) + z_1_alpha_2 = stats.norm.ppf(1 - alpha / 2) + + # Apply bias and acceleration corrections + p1_num = z0 + z_alpha_2 + p1_denom = 1 - acceleration * (z0 + z_alpha_2) + p1 = stats.norm.cdf(z0 + p1_num / p1_denom) if p1_denom != 0 else stats.norm.cdf(z0 + z_alpha_2) + + p2_num = z0 + z_1_alpha_2 + p2_denom = 1 - acceleration * (z0 + z_1_alpha_2) + p2 = stats.norm.cdf(z0 + p2_num / p2_denom) if p2_denom != 0 else stats.norm.cdf(z0 + z_1_alpha_2) + + # Ensure percentiles are within valid range + p1 = max(0.001, min(0.999, p1)) + p2 = max(0.001, min(0.999, p2)) + + lower_bound = np.percentile(bootstrap_estimates, p1 * 100) + upper_bound = np.percentile(bootstrap_estimates, p2 * 100) + + return ConfidenceInterval( + lower_bound=float(lower_bound), + upper_bound=float(upper_bound), + point_estimate=point_estimate, + confidence_level=self.config.confidence_level, + method="bca", + n_bootstrap_samples=len(bootstrap_estimates) + ) + + except Exception as e: + self.logger.error(f"BCa interval calculation failed: {e}") + return self._calculate_bc_interval(bootstrap_estimates, point_estimate, alpha) + + def _calculate_acceleration(self, original_data: List[Any], point_estimate: float) -> float: + """Calculate acceleration parameter for BCa intervals using jackknife""" + try: + n = len(original_data) + + if n < 10: # Not enough data for reliable acceleration + return 0.0 + + # Jackknife estimates (leave-one-out) + jackknife_estimates = [] + + for i in range(n): + # Create jackknife sample (all data except index i) + jackknife_sample = [original_data[j] for j in range(n) if j != i] + + # For this implementation, we'll use a simple approximation + # In practice, you would apply the same statistic function used for bootstrap + if isinstance(original_data[0], bool): + # For boolean outcomes (like win rate) + jackknife_estimate = np.mean(jackknife_sample) + else: + # For continuous outcomes + jackknife_estimate = np.mean(jackknife_sample) + + jackknife_estimates.append(jackknife_estimate) + + # Calculate acceleration + jackknife_mean = np.mean(jackknife_estimates) + numerator = np.sum((jackknife_mean - np.array(jackknife_estimates))**3) + denominator = 6 * (np.sum((jackknife_mean - np.array(jackknife_estimates))**2))**1.5 + + acceleration = numerator / denominator if denominator != 0 else 0.0 + + # Limit acceleration to reasonable range + acceleration = max(-0.25, min(0.25, acceleration)) + + return acceleration + + except Exception as e: + self.logger.error(f"Acceleration calculation failed: {e}") + return 0.0 # No acceleration + + def _create_wide_interval(self, point_estimate: float, reason: str) -> ConfidenceInterval: + """Create a wide confidence interval when normal calculation fails""" + # Use 40% margin of error as fallback + margin = 0.4 + lower_bound = max(0.0, point_estimate - margin) + upper_bound = min(1.0, point_estimate + margin) + + return ConfidenceInterval( + lower_bound=lower_bound, + upper_bound=upper_bound, + point_estimate=point_estimate, + confidence_level=self.config.confidence_level, + method=f"fallback_{reason}", + n_bootstrap_samples=0 + ) + + async def estimate_multiple_metrics( + self, + agent_data: Dict[str, List], + agent_id: Optional[str] = None + ) -> Dict[str, ConfidenceInterval]: + """ + Estimate confidence intervals for multiple metrics simultaneously. + + Args: + agent_data: Dictionary with metric names as keys and data lists as values + agent_id: Optional agent identifier + + Returns: + Dictionary of confidence intervals for each metric + """ + try: + results = {} + + # Create tasks for parallel estimation + tasks = [] + + if "outcomes" in agent_data: + tasks.append(("win_rate", self.estimate_win_rate_confidence(agent_data["outcomes"], agent_id))) + + if "kelly_deviations" in agent_data: + tasks.append(("kelly_adherence", self.estimate_kelly_adherence_confidence(agent_data["kelly_deviations"], agent_id))) + + if "returns" in agent_data: + tasks.append(("sharpe_ratio", self.estimate_sharpe_ratio_confidence(agent_data["returns"], agent_id))) + + if "confidences" in agent_data and "outcomes" in agent_data: + tasks.append(("confidence_calibration", self.estimate_confidence_calibration_interval( + agent_data["confidences"], agent_data["outcomes"], agent_id))) + + # Execute all tasks + if tasks: + task_results = await asyncio.gather(*[task for _, task in tasks]) + + # Collect results + for (metric_name, _), result in zip(tasks, task_results): + results[metric_name] = result + + return results + + except Exception as e: + self.logger.error(f"Multiple metrics estimation failed: {e}") + return {} + + def get_estimation_summary(self, intervals: Dict[str, ConfidenceInterval]) -> Dict[str, Any]: + """Generate summary of confidence interval estimations""" + try: + summary = { + "timestamp": datetime.now().isoformat(), + "total_metrics": len(intervals), + "confidence_level": self.config.confidence_level, + "bootstrap_samples": self.config.n_bootstrap_samples, + "metrics_summary": {} + } + + for metric_name, interval in intervals.items(): + summary["metrics_summary"][metric_name] = { + "point_estimate": interval.point_estimate, + "confidence_interval": [interval.lower_bound, interval.upper_bound], + "interval_width": interval.width, + "margin_of_error": interval.margin_of_error, + "method": interval.method, + "bootstrap_samples": interval.n_bootstrap_samples + } + + # Overall quality assessment + narrow_intervals = sum(1 for interval in intervals.values() if interval.width < 0.2) + summary["estimation_quality"] = { + "narrow_intervals": narrow_intervals, + "narrow_interval_ratio": narrow_intervals / len(intervals) if intervals else 0, + "quality": "high" if narrow_intervals / len(intervals) > 0.7 else "medium" if narrow_intervals / len(intervals) > 0.4 else "low" + } + + return summary + + except Exception as e: + self.logger.error(f"Summary generation failed: {e}") + return {"error": str(e)} \ No newline at end of file diff --git a/src/confidence_calibration/calibrator.py b/src/confidence_calibration/calibrator.py new file mode 100644 index 00000000..e8cbc6ee --- /dev/null +++ b/src/confidence_calibration/calibrator.py @@ -0,0 +1,957 @@ +""" +Confidence Calibration Engine + +Calibrates agent confidence scores to improve decision-making accuracy +and uncertainty estimation in trading scenarios. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Any, Callable +from datetime import datetime, timedelta +from enum import Enum +import logging +import numpy as np +from collections import defaultdict, deque +import json +from scipy import stats +from sklearn.isotonic import IsotonicRegression +from sklearn.linear_model import LogisticRegression +from sklearn.calibration import CalibratedClassifierCV +import asyncio + +from ..training.agent_analytics import DecisionMetrics + + +class CalibrationMethod(Enum): + """Methods for confidence calibration""" + PLATT_SCALING = "platt_scaling" + ISOTONIC_REGRESSION = "isotonic_regression" + TEMPERATURE_SCALING = "temperature_scaling" + BAYESIAN_CALIBRATION = "bayesian_calibration" + HISTOGRAM_BINNING = "histogram_binning" + + +@dataclass +class ConfidenceScore: + """Enhanced confidence score with uncertainty quantification""" + raw_confidence: float # Original agent confidence + calibrated_confidence: float # Calibrated confidence + uncertainty: float # Uncertainty estimate + confidence_interval: Tuple[float, float] # Confidence bounds + method_used: CalibrationMethod + sample_size: int # Number of samples used for calibration + + @property + def reliability(self) -> float: + """Get reliability score based on sample size and uncertainty""" + base_reliability = min(1.0, self.sample_size / 100) # More samples = more reliable + uncertainty_penalty = self.uncertainty * 0.5 + return max(0.0, base_reliability - uncertainty_penalty) + + +@dataclass +class CalibrationMetrics: + """Metrics for evaluating calibration quality""" + brier_score: float # Lower is better + reliability: float # Expected Calibration Error (ECE) + resolution: float # Ability to discriminate + sharpness: float # Confidence in predictions + calibration_slope: float # Slope of calibration curve + calibration_intercept: float # Intercept of calibration curve + sample_count: int + + @property + def calibration_quality(self) -> str: + """Overall calibration quality assessment""" + if self.reliability < 0.05: + return "excellent" + elif self.reliability < 0.10: + return "good" + elif self.reliability < 0.20: + return "fair" + else: + return "poor" + + +@dataclass +class CalibrationData: + """Data structure for calibration training""" + confidences: List[float] + outcomes: List[bool] # True for success, False for failure + weights: Optional[List[float]] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def __len__(self) -> int: + return len(self.confidences) + + @property + def is_valid(self) -> bool: + """Check if calibration data is valid""" + return (len(self.confidences) == len(self.outcomes) and + len(self.confidences) > 10 and # Minimum sample size + all(0 <= c <= 1 for c in self.confidences)) + + +class UncertaintyQuantifier: + """ + Quantifies uncertainty in agent decisions using multiple approaches. + """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + # Historical data for uncertainty estimation + self.decision_history: deque = deque(maxlen=1000) + self.outcome_patterns: Dict[str, List[float]] = defaultdict(list) + + def estimate_uncertainty( + self, + confidence: float, + context: Dict[str, Any], + method: str = "ensemble" + ) -> float: + """ + Estimate uncertainty for a given confidence score and context. + + Args: + confidence: Raw confidence score + context: Decision context (market conditions, agent state, etc.) + method: Uncertainty estimation method + + Returns: + Uncertainty estimate (0 = certain, 1 = maximum uncertainty) + """ + try: + if method == "ensemble": + return self._ensemble_uncertainty(confidence, context) + elif method == "variance": + return self._variance_based_uncertainty(confidence, context) + elif method == "entropy": + return self._entropy_based_uncertainty(confidence, context) + else: + return self._default_uncertainty(confidence, context) + + except Exception as e: + self.logger.error(f"Failed to estimate uncertainty: {e}") + return 0.5 # Default moderate uncertainty + + def _ensemble_uncertainty(self, confidence: float, context: Dict[str, Any]) -> float: + """Ensemble uncertainty estimation combining multiple methods""" + try: + # Get uncertainty from different methods + variance_unc = self._variance_based_uncertainty(confidence, context) + entropy_unc = self._entropy_based_uncertainty(confidence, context) + pattern_unc = self._pattern_based_uncertainty(confidence, context) + + # Weighted combination + weights = [0.4, 0.3, 0.3] + uncertainties = [variance_unc, entropy_unc, pattern_unc] + + ensemble_uncertainty = sum(w * u for w, u in zip(weights, uncertainties)) + return min(1.0, max(0.0, ensemble_uncertainty)) + + except Exception as e: + self.logger.error(f"Ensemble uncertainty estimation failed: {e}") + return 0.5 + + def _variance_based_uncertainty(self, confidence: float, context: Dict[str, Any]) -> float: + """Uncertainty based on confidence variance in similar situations""" + try: + # Find similar historical decisions + similar_decisions = self._find_similar_decisions(context) + + if len(similar_decisions) < 5: + return 0.7 # High uncertainty with limited data + + # Calculate confidence variance in similar situations + confidences = [d["confidence"] for d in similar_decisions] + variance = np.var(confidences) + + # Normalize variance to uncertainty scale + uncertainty = min(1.0, variance * 2) # Scale factor + return uncertainty + + except Exception as e: + self.logger.error(f"Variance-based uncertainty failed: {e}") + return 0.5 + + def _entropy_based_uncertainty(self, confidence: float, context: Dict[str, Any]) -> float: + """Uncertainty based on information entropy""" + try: + # Create probability distribution + p = confidence + q = 1 - confidence + + # Calculate entropy (maximum at p=0.5) + if p == 0 or p == 1: + entropy = 0 + else: + entropy = -p * np.log2(p) - q * np.log2(q) + + # Normalize to 0-1 scale (max entropy = 1) + uncertainty = entropy + + # Adjust based on context complexity + context_complexity = self._assess_context_complexity(context) + uncertainty = uncertainty * (1 + context_complexity * 0.3) + + return min(1.0, uncertainty) + + except Exception as e: + self.logger.error(f"Entropy-based uncertainty failed: {e}") + return 0.5 + + def _pattern_based_uncertainty(self, confidence: float, context: Dict[str, Any]) -> float: + """Uncertainty based on outcome patterns for similar confidence levels""" + try: + # Find decisions with similar confidence levels + confidence_range = 0.1 # ±10% confidence range + similar_confidences = [] + + for decision in self.decision_history: + if abs(decision["confidence"] - confidence) <= confidence_range: + similar_confidences.append(decision) + + if len(similar_confidences) < 3: + return 0.6 # Moderate uncertainty with limited pattern data + + # Calculate outcome variance for this confidence level + outcomes = [1.0 if d["outcome"] else 0.0 for d in similar_confidences] + outcome_variance = np.var(outcomes) + + # Higher variance = higher uncertainty + uncertainty = min(1.0, outcome_variance * 2) + return uncertainty + + except Exception as e: + self.logger.error(f"Pattern-based uncertainty failed: {e}") + return 0.5 + + def _find_similar_decisions(self, context: Dict[str, Any], similarity_threshold: float = 0.7) -> List[Dict]: + """Find historically similar decision contexts""" + try: + similar_decisions = [] + + for decision in self.decision_history: + similarity = self._calculate_context_similarity( + context, decision.get("context", {}) + ) + + if similarity >= similarity_threshold: + similar_decisions.append(decision) + + return similar_decisions + + except Exception as e: + self.logger.error(f"Failed to find similar decisions: {e}") + return [] + + def _calculate_context_similarity(self, context1: Dict, context2: Dict) -> float: + """Calculate similarity between two decision contexts""" + try: + if not context1 or not context2: + return 0.0 + + # Key context features to compare + features = ["market_ticker", "event_type", "quarter", "time_remaining"] + + matches = 0 + total_features = 0 + + for feature in features: + if feature in context1 and feature in context2: + total_features += 1 + if context1[feature] == context2[feature]: + matches += 1 + + if total_features == 0: + return 0.0 + + return matches / total_features + + except Exception as e: + self.logger.error(f"Context similarity calculation failed: {e}") + return 0.0 + + def _assess_context_complexity(self, context: Dict[str, Any]) -> float: + """Assess complexity of decision context (0 = simple, 1 = complex)""" + try: + complexity_score = 0.0 + + # Number of context features + feature_complexity = min(1.0, len(context) / 20) # More features = more complex + complexity_score += feature_complexity * 0.3 + + # Time pressure (less time = more complex) + time_remaining = context.get("time_remaining", 3600) # Default 1 hour + time_complexity = max(0.0, 1 - time_remaining / 3600) # Normalize to hour + complexity_score += time_complexity * 0.3 + + # Market volatility proxy + market_events = context.get("recent_events", []) + volatility_complexity = min(1.0, len(market_events) / 10) + complexity_score += volatility_complexity * 0.4 + + return min(1.0, complexity_score) + + except Exception as e: + self.logger.error(f"Context complexity assessment failed: {e}") + return 0.5 + + def _default_uncertainty(self, confidence: float, context: Dict[str, Any]) -> float: + """Default uncertainty estimation""" + # Simple heuristic: uncertainty is highest at confidence = 0.5 + base_uncertainty = 2 * min(confidence, 1 - confidence) + + # Add some noise based on context + context_factor = min(0.2, len(context) * 0.01) + return min(1.0, base_uncertainty + context_factor) + + def update_decision_history(self, decision_data: Dict[str, Any]) -> None: + """Update decision history with new outcome""" + try: + self.decision_history.append(decision_data) + + # Update outcome patterns + confidence_bucket = int(decision_data["confidence"] * 10) / 10 # Bucket to 0.1 precision + outcome_value = 1.0 if decision_data.get("outcome", False) else 0.0 + self.outcome_patterns[str(confidence_bucket)].append(outcome_value) + + except Exception as e: + self.logger.error(f"Failed to update decision history: {e}") + + +class ConfidenceCalibrator: + """ + Main confidence calibration engine that trains calibration models + and provides calibrated confidence scores for agent decisions. + """ + + def __init__(self, uncertainty_quantifier: Optional[UncertaintyQuantifier] = None): + self.logger = logging.getLogger(__name__) + + # Uncertainty quantification + self.uncertainty_quantifier = uncertainty_quantifier or UncertaintyQuantifier() + + # Calibration models for different methods + self.calibration_models: Dict[CalibrationMethod, Any] = {} + + # Training data storage + self.training_data: Dict[str, CalibrationData] = {} # By agent_id + + # Calibration performance tracking + self.calibration_metrics: Dict[str, CalibrationMetrics] = {} + + # Default calibration parameters + self.default_method = CalibrationMethod.ISOTONIC_REGRESSION + self.min_training_samples = 50 + self.calibration_update_frequency = timedelta(hours=6) + self.last_calibration_update: Dict[str, datetime] = {} + + async def train_calibration(self, agent_id: str, decisions: List[DecisionMetrics]) -> CalibrationMetrics: + """ + Train calibration model for a specific agent. + + Args: + agent_id: Agent identifier + decisions: List of historical decisions with outcomes + + Returns: + Calibration metrics for the trained model + """ + try: + # Prepare training data + training_data = self._prepare_training_data(decisions) + + if not training_data.is_valid: + raise ValueError(f"Invalid training data for agent {agent_id}") + + if len(training_data) < self.min_training_samples: + self.logger.warning(f"Insufficient training data for agent {agent_id}: {len(training_data)} samples") + return self._create_default_metrics() + + # Store training data + self.training_data[agent_id] = training_data + + # Train multiple calibration models + models = {} + for method in [CalibrationMethod.PLATT_SCALING, + CalibrationMethod.ISOTONIC_REGRESSION, + CalibrationMethod.TEMPERATURE_SCALING]: + try: + model = await self._train_single_method(method, training_data) + models[method] = model + except Exception as e: + self.logger.warning(f"Failed to train {method.value} for agent {agent_id}: {e}") + + # Select best model + best_method, best_model = self._select_best_model(models, training_data) + self.calibration_models[agent_id] = (best_method, best_model) + + # Calculate and store metrics + metrics = self._calculate_calibration_metrics(best_model, best_method, training_data) + self.calibration_metrics[agent_id] = metrics + + # Update timestamp + self.last_calibration_update[agent_id] = datetime.now() + + self.logger.info(f"Trained calibration for agent {agent_id} using {best_method.value}") + self.logger.info(f"Calibration quality: {metrics.calibration_quality} (reliability: {metrics.reliability:.3f})") + + return metrics + + except Exception as e: + self.logger.error(f"Failed to train calibration for agent {agent_id}: {e}") + return self._create_default_metrics() + + def _prepare_training_data(self, decisions: List[DecisionMetrics]) -> CalibrationData: + """Prepare training data from decision metrics""" + try: + confidences = [] + outcomes = [] + weights = [] + + for decision in decisions: + if decision.outcome is not None and 0 <= decision.confidence <= 1: + confidences.append(decision.confidence) + outcomes.append(decision.outcome > 0) # Convert to boolean + + # Weight more recent decisions higher + age_days = (datetime.now() - decision.timestamp).days + weight = max(0.1, 1.0 - age_days * 0.1) # Decay weight over time + weights.append(weight) + + return CalibrationData( + confidences=confidences, + outcomes=outcomes, + weights=weights, + metadata={"sample_count": len(confidences)} + ) + + except Exception as e: + self.logger.error(f"Failed to prepare training data: {e}") + return CalibrationData([], []) + + async def _train_single_method(self, method: CalibrationMethod, data: CalibrationData) -> Any: + """Train a single calibration method""" + try: + X = np.array(data.confidences).reshape(-1, 1) + y = np.array(data.outcomes) + sample_weights = np.array(data.weights) if data.weights else None + + if method == CalibrationMethod.PLATT_SCALING: + model = LogisticRegression() + model.fit(X, y, sample_weight=sample_weights) + return model + + elif method == CalibrationMethod.ISOTONIC_REGRESSION: + model = IsotonicRegression(out_of_bounds='clip') + model.fit(data.confidences, data.outcomes, sample_weight=sample_weights) + return model + + elif method == CalibrationMethod.TEMPERATURE_SCALING: + # Temperature scaling using logistic regression + model = self._train_temperature_scaling(X, y, sample_weights) + return model + + elif method == CalibrationMethod.HISTOGRAM_BINNING: + model = self._train_histogram_binning(data.confidences, data.outcomes, sample_weights) + return model + + else: + raise ValueError(f"Unsupported calibration method: {method}") + + except Exception as e: + self.logger.error(f"Failed to train {method.value}: {e}") + raise + + def _train_temperature_scaling(self, X: np.ndarray, y: np.ndarray, sample_weights: Optional[np.ndarray]) -> Dict: + """Train temperature scaling model""" + try: + # Use logistic regression to find optimal temperature + model = LogisticRegression() + model.fit(X, y, sample_weight=sample_weights) + + # Extract temperature parameter (inverse of coefficient) + temperature = 1.0 / abs(model.coef_[0][0]) if model.coef_[0][0] != 0 else 1.0 + + return { + "type": "temperature_scaling", + "temperature": temperature, + "intercept": model.intercept_[0] + } + + except Exception as e: + self.logger.error(f"Temperature scaling training failed: {e}") + return {"type": "temperature_scaling", "temperature": 1.0, "intercept": 0.0} + + def _train_histogram_binning(self, confidences: List[float], outcomes: List[bool], weights: Optional[List[float]]) -> Dict: + """Train histogram binning calibration""" + try: + n_bins = min(10, len(confidences) // 10) # Adaptive number of bins + if n_bins < 2: + n_bins = 2 + + # Create bins + bin_boundaries = np.linspace(0, 1, n_bins + 1) + bin_centers = (bin_boundaries[:-1] + bin_boundaries[1:]) / 2 + + # Calculate calibrated probabilities for each bin + calibrated_probs = [] + + for i in range(n_bins): + # Find samples in this bin + in_bin = [(confidences[j] >= bin_boundaries[i] and confidences[j] < bin_boundaries[i+1]) + for j in range(len(confidences))] + + if i == n_bins - 1: # Last bin includes upper boundary + in_bin = [(confidences[j] >= bin_boundaries[i] and confidences[j] <= bin_boundaries[i+1]) + for j in range(len(confidences))] + + # Calculate weighted average outcome for this bin + bin_outcomes = [outcomes[j] for j in range(len(outcomes)) if in_bin[j]] + bin_weights = [weights[j] for j in range(len(weights)) if in_bin[j]] if weights else None + + if bin_outcomes: + if bin_weights: + weighted_sum = sum(o * w for o, w in zip(bin_outcomes, bin_weights)) + weight_sum = sum(bin_weights) + calibrated_prob = weighted_sum / weight_sum if weight_sum > 0 else 0.5 + else: + calibrated_prob = sum(bin_outcomes) / len(bin_outcomes) + else: + calibrated_prob = bin_centers[i] # Use bin center as fallback + + calibrated_probs.append(calibrated_prob) + + return { + "type": "histogram_binning", + "bin_boundaries": bin_boundaries.tolist(), + "calibrated_probs": calibrated_probs + } + + except Exception as e: + self.logger.error(f"Histogram binning training failed: {e}") + return {"type": "histogram_binning", "bin_boundaries": [0, 1], "calibrated_probs": [0.5]} + + def _select_best_model(self, models: Dict[CalibrationMethod, Any], data: CalibrationData) -> Tuple[CalibrationMethod, Any]: + """Select best calibration model based on validation performance""" + try: + if not models: + return self.default_method, None + + best_method = None + best_model = None + best_score = float('inf') + + # Use Brier score for model selection (lower is better) + for method, model in models.items(): + try: + score = self._calculate_brier_score(model, method, data) + if score < best_score: + best_score = score + best_method = method + best_model = model + except Exception as e: + self.logger.warning(f"Failed to evaluate {method.value}: {e}") + continue + + if best_method is None: + # Fallback to first available model + best_method = list(models.keys())[0] + best_model = models[best_method] + + return best_method, best_model + + except Exception as e: + self.logger.error(f"Model selection failed: {e}") + return self.default_method, None + + def _calculate_brier_score(self, model: Any, method: CalibrationMethod, data: CalibrationData) -> float: + """Calculate Brier score for model evaluation""" + try: + # Get calibrated predictions + calibrated_probs = self._apply_calibration_model(data.confidences, model, method) + + # Calculate Brier score + brier_score = np.mean([(prob - outcome)**2 for prob, outcome in zip(calibrated_probs, data.outcomes)]) + + return brier_score + + except Exception as e: + self.logger.error(f"Brier score calculation failed: {e}") + return 1.0 # Worst possible score + + def _apply_calibration_model(self, confidences: List[float], model: Any, method: CalibrationMethod) -> List[float]: + """Apply calibration model to get calibrated probabilities""" + try: + if method == CalibrationMethod.PLATT_SCALING: + X = np.array(confidences).reshape(-1, 1) + return model.predict_proba(X)[:, 1].tolist() + + elif method == CalibrationMethod.ISOTONIC_REGRESSION: + return model.predict(confidences).tolist() + + elif method == CalibrationMethod.TEMPERATURE_SCALING: + temperature = model["temperature"] + intercept = model["intercept"] + + calibrated = [] + for conf in confidences: + # Apply temperature scaling + logit = np.log(conf / (1 - conf)) if 0 < conf < 1 else 0 + scaled_logit = logit / temperature + intercept + calibrated_prob = 1 / (1 + np.exp(-scaled_logit)) + calibrated.append(max(0.001, min(0.999, calibrated_prob))) + + return calibrated + + elif method == CalibrationMethod.HISTOGRAM_BINNING: + bin_boundaries = model["bin_boundaries"] + calibrated_probs = model["calibrated_probs"] + + calibrated = [] + for conf in confidences: + # Find appropriate bin + bin_idx = np.digitize(conf, bin_boundaries) - 1 + bin_idx = max(0, min(len(calibrated_probs) - 1, bin_idx)) + calibrated.append(calibrated_probs[bin_idx]) + + return calibrated + + else: + return confidences # No calibration + + except Exception as e: + self.logger.error(f"Failed to apply calibration model: {e}") + return confidences + + def _calculate_calibration_metrics(self, model: Any, method: CalibrationMethod, data: CalibrationData) -> CalibrationMetrics: + """Calculate comprehensive calibration metrics""" + try: + # Get calibrated predictions + calibrated_probs = self._apply_calibration_model(data.confidences, model, method) + + # Brier score + brier_score = np.mean([(prob - outcome)**2 for prob, outcome in zip(calibrated_probs, data.outcomes)]) + + # Reliability (Expected Calibration Error) + reliability = self._calculate_expected_calibration_error(calibrated_probs, data.outcomes) + + # Resolution + resolution = self._calculate_resolution(calibrated_probs, data.outcomes) + + # Sharpness + sharpness = np.var(calibrated_probs) + + # Calibration curve statistics + slope, intercept = self._calculate_calibration_curve_stats(calibrated_probs, data.outcomes) + + return CalibrationMetrics( + brier_score=brier_score, + reliability=reliability, + resolution=resolution, + sharpness=sharpness, + calibration_slope=slope, + calibration_intercept=intercept, + sample_count=len(data) + ) + + except Exception as e: + self.logger.error(f"Failed to calculate calibration metrics: {e}") + return self._create_default_metrics() + + def _calculate_expected_calibration_error(self, predictions: List[float], outcomes: List[bool]) -> float: + """Calculate Expected Calibration Error (ECE)""" + try: + n_bins = 10 + bin_boundaries = np.linspace(0, 1, n_bins + 1) + + ece = 0.0 + total_samples = len(predictions) + + for i in range(n_bins): + # Find samples in this bin + in_bin_mask = [(predictions[j] >= bin_boundaries[i] and predictions[j] < bin_boundaries[i+1]) + for j in range(len(predictions))] + + if i == n_bins - 1: # Last bin includes upper boundary + in_bin_mask = [(predictions[j] >= bin_boundaries[i] and predictions[j] <= bin_boundaries[i+1]) + for j in range(len(predictions))] + + bin_predictions = [predictions[j] for j in range(len(predictions)) if in_bin_mask[j]] + bin_outcomes = [outcomes[j] for j in range(len(outcomes)) if in_bin_mask[j]] + + if bin_predictions: + bin_size = len(bin_predictions) + bin_confidence = np.mean(bin_predictions) + bin_accuracy = np.mean(bin_outcomes) + + ece += (bin_size / total_samples) * abs(bin_confidence - bin_accuracy) + + return ece + + except Exception as e: + self.logger.error(f"ECE calculation failed: {e}") + return 1.0 + + def _calculate_resolution(self, predictions: List[float], outcomes: List[bool]) -> float: + """Calculate resolution (ability to discriminate between classes)""" + try: + # Resolution is the variance of conditional probabilities weighted by frequency + n_bins = 10 + bin_boundaries = np.linspace(0, 1, n_bins + 1) + + resolution = 0.0 + total_samples = len(predictions) + overall_base_rate = np.mean(outcomes) + + for i in range(n_bins): + # Find samples in this bin + in_bin_mask = [(predictions[j] >= bin_boundaries[i] and predictions[j] < bin_boundaries[i+1]) + for j in range(len(predictions))] + + if i == n_bins - 1: # Last bin includes upper boundary + in_bin_mask = [(predictions[j] >= bin_boundaries[i] and predictions[j] <= bin_boundaries[i+1]) + for j in range(len(predictions))] + + bin_outcomes = [outcomes[j] for j in range(len(outcomes)) if in_bin_mask[j]] + + if bin_outcomes: + bin_size = len(bin_outcomes) + bin_accuracy = np.mean(bin_outcomes) + + resolution += (bin_size / total_samples) * (bin_accuracy - overall_base_rate)**2 + + return resolution + + except Exception as e: + self.logger.error(f"Resolution calculation failed: {e}") + return 0.0 + + def _calculate_calibration_curve_stats(self, predictions: List[float], outcomes: List[bool]) -> Tuple[float, float]: + """Calculate slope and intercept of calibration curve""" + try: + # Linear regression of outcomes vs predictions + slope, intercept, _, _, _ = stats.linregress(predictions, outcomes) + return slope, intercept + + except Exception as e: + self.logger.error(f"Calibration curve stats calculation failed: {e}") + return 1.0, 0.0 # Perfect calibration + + def _create_default_metrics(self) -> CalibrationMetrics: + """Create default calibration metrics""" + return CalibrationMetrics( + brier_score=0.25, # Random prediction + reliability=0.5, # Poor reliability + resolution=0.0, # No resolution + sharpness=0.25, # Moderate sharpness + calibration_slope=1.0, + calibration_intercept=0.0, + sample_count=0 + ) + + async def calibrate_confidence( + self, + agent_id: str, + raw_confidence: float, + context: Dict[str, Any] + ) -> ConfidenceScore: + """ + Calibrate confidence score for an agent's decision. + + Args: + agent_id: Agent identifier + raw_confidence: Original confidence score (0-1) + context: Decision context for uncertainty estimation + + Returns: + Calibrated confidence score with uncertainty bounds + """ + try: + # Check if we have calibration model for this agent + if agent_id not in self.calibration_models: + # Try to train calibration if we have enough data + if agent_id in self.training_data and len(self.training_data[agent_id]) >= self.min_training_samples: + # Use existing training data to create temporary model + decisions = [] # Would need to reconstruct from training data + await self.train_calibration(agent_id, decisions) + else: + # Return uncalibrated confidence with high uncertainty + return self._create_default_confidence_score(raw_confidence, context) + + method, model = self.calibration_models[agent_id] + + # Apply calibration + calibrated_conf = self._apply_calibration_model([raw_confidence], model, method)[0] + + # Estimate uncertainty + uncertainty = self.uncertainty_quantifier.estimate_uncertainty( + raw_confidence, context, method="ensemble" + ) + + # Calculate confidence interval + conf_interval = self._calculate_confidence_interval( + calibrated_conf, uncertainty, context + ) + + # Get sample size used for calibration + sample_size = self.training_data.get(agent_id, CalibrationData([], [])).sample_count if agent_id in self.training_data else 0 + + return ConfidenceScore( + raw_confidence=raw_confidence, + calibrated_confidence=calibrated_conf, + uncertainty=uncertainty, + confidence_interval=conf_interval, + method_used=method, + sample_size=sample_size + ) + + except Exception as e: + self.logger.error(f"Failed to calibrate confidence for agent {agent_id}: {e}") + return self._create_default_confidence_score(raw_confidence, context) + + def _create_default_confidence_score(self, raw_confidence: float, context: Dict[str, Any]) -> ConfidenceScore: + """Create default confidence score when calibration is not available""" + # Use simple uncertainty estimation + uncertainty = max(0.3, 2 * min(raw_confidence, 1 - raw_confidence)) # Higher at extremes + + # Wide confidence interval due to lack of calibration + margin = uncertainty * 0.5 + conf_interval = ( + max(0.0, raw_confidence - margin), + min(1.0, raw_confidence + margin) + ) + + return ConfidenceScore( + raw_confidence=raw_confidence, + calibrated_confidence=raw_confidence, # No calibration applied + uncertainty=uncertainty, + confidence_interval=conf_interval, + method_used=CalibrationMethod.HISTOGRAM_BINNING, # Default method + sample_size=0 + ) + + def _calculate_confidence_interval( + self, + calibrated_confidence: float, + uncertainty: float, + context: Dict[str, Any], + confidence_level: float = 0.95 + ) -> Tuple[float, float]: + """Calculate confidence interval for calibrated score""" + try: + # Use uncertainty to determine interval width + z_score = stats.norm.ppf((1 + confidence_level) / 2) # 95% confidence + + # Scale uncertainty by z-score + margin = uncertainty * z_score * 0.5 + + # Adjust margin based on context + context_adjustment = self._get_context_adjustment(context) + margin = margin * (1 + context_adjustment) + + # Calculate bounds + lower_bound = max(0.0, calibrated_confidence - margin) + upper_bound = min(1.0, calibrated_confidence + margin) + + return (lower_bound, upper_bound) + + except Exception as e: + self.logger.error(f"Confidence interval calculation failed: {e}") + return (max(0.0, calibrated_confidence - 0.2), min(1.0, calibrated_confidence + 0.2)) + + def _get_context_adjustment(self, context: Dict[str, Any]) -> float: + """Get adjustment factor based on decision context""" + try: + adjustment = 0.0 + + # Time pressure increases uncertainty + time_remaining = context.get("time_remaining", 3600) + if time_remaining < 300: # Less than 5 minutes + adjustment += 0.2 + elif time_remaining < 900: # Less than 15 minutes + adjustment += 0.1 + + # Market volatility increases uncertainty + recent_events = context.get("recent_events", []) + if len(recent_events) > 5: + adjustment += 0.15 + + # Edge case scenarios increase uncertainty + if context.get("is_edge_case", False): + adjustment += 0.25 + + return min(0.5, adjustment) # Cap adjustment + + except Exception as e: + self.logger.error(f"Context adjustment calculation failed: {e}") + return 0.1 # Small default adjustment + + async def should_update_calibration(self, agent_id: str) -> bool: + """Check if calibration model should be updated""" + try: + if agent_id not in self.last_calibration_update: + return True # Never been updated + + last_update = self.last_calibration_update[agent_id] + time_since_update = datetime.now() - last_update + + # Time-based update + if time_since_update >= self.calibration_update_frequency: + return True + + # Performance-based update + if agent_id in self.calibration_metrics: + metrics = self.calibration_metrics[agent_id] + if metrics.reliability > 0.2: # Poor calibration + return True + + return False + + except Exception as e: + self.logger.error(f"Failed to check update status for agent {agent_id}: {e}") + return False + + def get_calibration_status(self, agent_id: Optional[str] = None) -> Dict[str, Any]: + """Get calibration status for agent(s)""" + try: + if agent_id: + # Single agent status + if agent_id not in self.calibration_models: + return {"agent_id": agent_id, "status": "not_calibrated", "reason": "no_model"} + + method, _ = self.calibration_models[agent_id] + metrics = self.calibration_metrics.get(agent_id) + last_update = self.last_calibration_update.get(agent_id) + + return { + "agent_id": agent_id, + "status": "calibrated", + "method": method.value, + "metrics": { + "calibration_quality": metrics.calibration_quality if metrics else "unknown", + "reliability": metrics.reliability if metrics else 0.5, + "brier_score": metrics.brier_score if metrics else 0.25, + "sample_count": metrics.sample_count if metrics else 0 + }, + "last_update": last_update.isoformat() if last_update else None, + "needs_update": False # Will be checked separately if needed + } + else: + # All agents status + all_status = {} + for agent_id in set(list(self.calibration_models.keys()) + list(self.training_data.keys())): + all_status[agent_id] = self.get_calibration_status(agent_id) + + return { + "timestamp": datetime.now().isoformat(), + "total_agents": len(all_status), + "calibrated_agents": len([s for s in all_status.values() if s.get("status") == "calibrated"]), + "agents": all_status + } + + except Exception as e: + self.logger.error(f"Failed to get calibration status: {e}") + return {"error": str(e)} \ No newline at end of file diff --git a/src/hybrid_pipeline/__init__.py b/src/hybrid_pipeline/__init__.py new file mode 100644 index 00000000..de43fb60 --- /dev/null +++ b/src/hybrid_pipeline/__init__.py @@ -0,0 +1,38 @@ +""" +Hybrid Data Pipeline Module + +Seamless switching between live API data and synthetic data generation +based on training requirements, API costs, and agent performance needs. +""" + +from .data_orchestrator import ( + HybridDataOrchestrator, + DataSource, + DataMode, + CostThreshold, + SwitchingStrategy +) +from .cost_monitor import ( + APITracker, + CostAlert, + BudgetManager +) +from .adaptive_scheduler import ( + AdaptiveScheduler, + SchedulingPolicy, + TrainingPhase +) + +__all__ = [ + 'HybridDataOrchestrator', + 'DataSource', + 'DataMode', + 'CostThreshold', + 'SwitchingStrategy', + 'APITracker', + 'CostAlert', + 'BudgetManager', + 'AdaptiveScheduler', + 'SchedulingPolicy', + 'TrainingPhase' +] \ No newline at end of file diff --git a/src/hybrid_pipeline/adaptive_scheduler.py b/src/hybrid_pipeline/adaptive_scheduler.py new file mode 100644 index 00000000..ba0cd3d2 --- /dev/null +++ b/src/hybrid_pipeline/adaptive_scheduler.py @@ -0,0 +1,908 @@ +""" +Adaptive Scheduler for Hybrid Data Pipeline + +Intelligently schedules data requests and training sessions based on +agent performance, cost optimization, and system resources. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Any, Callable, Tuple +from datetime import datetime, timedelta +from enum import Enum +import asyncio +import logging +import json +from collections import defaultdict, deque +import heapq +import random + +from .data_orchestrator import DataSource, DataMode, HybridDataOrchestrator +from .cost_monitor import BudgetManager, APITracker +from ..training.agent_analytics import AgentAnalytics + + +class TrainingPhase(Enum): + """Different phases of agent training""" + INITIAL = "initial" + FOUNDATION = "foundation" + ADVANCED = "advanced" + VALIDATION = "validation" + PRODUCTION_PREP = "production_prep" + MAINTENANCE = "maintenance" + + +class SchedulingPolicy(Enum): + """Scheduling policy options""" + COST_OPTIMIZED = "cost_optimized" + PERFORMANCE_OPTIMIZED = "performance_optimized" + BALANCED = "balanced" + AGGRESSIVE = "aggressive" + CONSERVATIVE = "conservative" + + +class Priority(Enum): + """Task priority levels""" + LOW = 1 + NORMAL = 2 + HIGH = 3 + CRITICAL = 4 + + +@dataclass +class TrainingTask: + """Individual training task definition""" + task_id: str + agent_id: str + task_type: str # "scenario_training", "validation", "edge_case_practice" + priority: Priority + estimated_cost: float + estimated_duration: timedelta + data_requirements: Dict[str, Any] + preferred_data_source: DataSource + created_at: datetime = field(default_factory=datetime.now) + scheduled_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + status: str = "pending" # pending, scheduled, running, completed, failed + retry_count: int = 0 + max_retries: int = 3 + + def __lt__(self, other): + """For priority queue ordering""" + return (self.priority.value, self.scheduled_at or datetime.max) > (other.priority.value, other.scheduled_at or datetime.max) + + +@dataclass +class SchedulingContext: + """Context for scheduling decisions""" + current_time: datetime + available_budget: float + system_load: float + api_rate_limits: Dict[str, int] + agent_performance_metrics: Dict[str, Dict] + active_tasks: List[TrainingTask] + failed_tasks: List[TrainingTask] + time_window: timedelta = timedelta(hours=24) + + +class AdaptiveScheduler: + """ + Adaptive scheduler that optimizes training task scheduling based on: + - Agent performance and learning needs + - API cost constraints and budget optimization + - System resources and capacity + - Training phase requirements + - Historical performance data + """ + + def __init__( + self, + orchestrator: HybridDataOrchestrator, + budget_manager: BudgetManager, + api_tracker: APITracker, + agent_analytics: AgentAnalytics + ): + self.orchestrator = orchestrator + self.budget_manager = budget_manager + self.api_tracker = api_tracker + self.agent_analytics = agent_analytics + self.logger = logging.getLogger(__name__) + + # Configuration + self.scheduling_policy = SchedulingPolicy.BALANCED + self.max_concurrent_tasks = 5 + self.planning_horizon = timedelta(hours=24) + self.rebalance_interval = timedelta(minutes=30) + + # Task management + self.task_queue: List[TrainingTask] = [] + self.active_tasks: Dict[str, TrainingTask] = {} + self.completed_tasks: List[TrainingTask] = [] + self.failed_tasks: List[TrainingTask] = [] + + # Agent training phases + self.agent_phases: Dict[str, TrainingPhase] = {} + self.phase_requirements: Dict[TrainingPhase, Dict[str, Any]] = { + TrainingPhase.INITIAL: { + "synthetic_data_ratio": 1.0, + "scenario_diversity": 0.3, + "edge_case_ratio": 0.05, + "validation_frequency": 0.1 + }, + TrainingPhase.FOUNDATION: { + "synthetic_data_ratio": 0.9, + "scenario_diversity": 0.5, + "edge_case_ratio": 0.1, + "validation_frequency": 0.15 + }, + TrainingPhase.ADVANCED: { + "synthetic_data_ratio": 0.7, + "scenario_diversity": 0.8, + "edge_case_ratio": 0.2, + "validation_frequency": 0.2 + }, + TrainingPhase.VALIDATION: { + "synthetic_data_ratio": 0.3, + "scenario_diversity": 1.0, + "edge_case_ratio": 0.3, + "validation_frequency": 0.5 + }, + TrainingPhase.PRODUCTION_PREP: { + "synthetic_data_ratio": 0.1, + "scenario_diversity": 0.6, + "edge_case_ratio": 0.1, + "validation_frequency": 0.8 + } + } + + # Scheduling metrics + self.scheduling_history: List[Dict[str, Any]] = [] + self.performance_metrics: Dict[str, float] = defaultdict(float) + + # Task execution callbacks + self.task_callbacks: Dict[str, Callable] = {} + + async def initialize(self) -> None: + """Initialize the adaptive scheduler""" + try: + self.logger.info("Initializing Adaptive Scheduler") + + # Initialize agent phases + await self._initialize_agent_phases() + + # Start background scheduling loop + asyncio.create_task(self._scheduling_loop()) + + self.logger.info("Adaptive Scheduler initialized successfully") + + except Exception as e: + self.logger.error(f"Failed to initialize scheduler: {e}") + raise + + async def _initialize_agent_phases(self) -> None: + """Initialize training phases for all agents""" + try: + # Get all agents from analytics + for agent_id in self.agent_analytics.decision_history.keys(): + # Determine initial phase based on agent experience + analytics = await self.agent_analytics.get_agent_analytics(agent_id) + + if analytics.get("total_decisions", 0) == 0: + phase = TrainingPhase.INITIAL + elif analytics.get("total_decisions", 0) < 100: + phase = TrainingPhase.FOUNDATION + elif analytics.get("latest_snapshot", {}).get("win_rate", 0) < 0.5: + phase = TrainingPhase.ADVANCED + else: + phase = TrainingPhase.VALIDATION + + self.agent_phases[agent_id] = phase + self.logger.info(f"Agent {agent_id} initialized in {phase.value} phase") + + except Exception as e: + self.logger.error(f"Failed to initialize agent phases: {e}") + + async def schedule_training_task( + self, + agent_id: str, + task_type: str, + priority: Priority = Priority.NORMAL, + data_requirements: Optional[Dict[str, Any]] = None, + preferred_time: Optional[datetime] = None + ) -> str: + """Schedule a new training task""" + try: + task_id = f"{agent_id}_{task_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + # Get agent's current phase + agent_phase = self.agent_phases.get(agent_id, TrainingPhase.INITIAL) + + # Determine optimal data source and requirements + optimal_source, requirements = await self._determine_optimal_task_config( + agent_id, task_type, agent_phase, data_requirements + ) + + # Estimate cost and duration + estimated_cost = await self._estimate_task_cost(task_type, requirements, optimal_source) + estimated_duration = self._estimate_task_duration(task_type, requirements) + + # Create task + task = TrainingTask( + task_id=task_id, + agent_id=agent_id, + task_type=task_type, + priority=priority, + estimated_cost=estimated_cost, + estimated_duration=estimated_duration, + data_requirements=requirements, + preferred_data_source=optimal_source + ) + + # Schedule the task + await self._schedule_task(task, preferred_time) + + self.logger.info(f"Scheduled task {task_id} for agent {agent_id}") + return task_id + + except Exception as e: + self.logger.error(f"Failed to schedule training task: {e}") + raise + + async def _determine_optimal_task_config( + self, + agent_id: str, + task_type: str, + agent_phase: TrainingPhase, + data_requirements: Optional[Dict[str, Any]] + ) -> Tuple[DataSource, Dict[str, Any]]: + """Determine optimal configuration for a training task""" + try: + # Get phase requirements + phase_config = self.phase_requirements[agent_phase] + + # Base requirements from phase + requirements = { + "count": data_requirements.get("count", 100) if data_requirements else 100, + "scenario_diversity": phase_config["scenario_diversity"], + "edge_case_ratio": phase_config["edge_case_ratio"], + "include_edge_cases": phase_config["edge_case_ratio"] > 0.1, + "training_phase": agent_phase.value, + "agent_id": agent_id + } + + # Override with specific requirements + if data_requirements: + requirements.update(data_requirements) + + # Determine optimal data source + synthetic_ratio = phase_config["synthetic_data_ratio"] + + # Adjust based on scheduling policy + if self.scheduling_policy == SchedulingPolicy.COST_OPTIMIZED: + # Favor synthetic data for cost savings + synthetic_ratio = min(1.0, synthetic_ratio * 1.2) + elif self.scheduling_policy == SchedulingPolicy.PERFORMANCE_OPTIMIZED: + # Favor live data for realism + synthetic_ratio = max(0.1, synthetic_ratio * 0.8) + + # Select data source based on ratio and current context + if synthetic_ratio > 0.8: + optimal_source = DataSource.SYNTHETIC + elif synthetic_ratio < 0.3: + optimal_source = DataSource.LIVE_API + else: + optimal_source = DataSource.HYBRID + + # Consider current budget constraints + daily_budget_used = self.budget_manager.get_budget_status().get("daily_cost", 0) + if daily_budget_used > 50: # If already spent significant amount + optimal_source = DataSource.SYNTHETIC + + return optimal_source, requirements + + except Exception as e: + self.logger.error(f"Failed to determine optimal task config: {e}") + return DataSource.SYNTHETIC, {"count": 100} + + async def _estimate_task_cost( + self, + task_type: str, + requirements: Dict[str, Any], + data_source: DataSource + ) -> float: + """Estimate the cost of a training task""" + try: + base_cost = 0.0 + data_count = requirements.get("count", 100) + + if data_source == DataSource.SYNTHETIC: + base_cost = data_count * 0.001 # Very low cost for synthetic + elif data_source == DataSource.LIVE_API: + base_cost = data_count * 0.01 # Higher cost for live API + elif data_source == DataSource.CACHED: + base_cost = 0.0 # No cost for cached data + else: # HYBRID + # Mix of costs + base_cost = data_count * 0.005 + + # Adjust for task complexity + complexity_multiplier = 1.0 + if task_type == "edge_case_practice": + complexity_multiplier = 1.5 + elif task_type == "validation": + complexity_multiplier = 2.0 + + # Adjust for special requirements + if requirements.get("include_edge_cases", False): + complexity_multiplier *= 1.2 + + if requirements.get("scenario_diversity", 0) > 0.8: + complexity_multiplier *= 1.3 + + return base_cost * complexity_multiplier + + except Exception as e: + self.logger.error(f"Failed to estimate task cost: {e}") + return 1.0 # Default cost + + def _estimate_task_duration(self, task_type: str, requirements: Dict[str, Any]) -> timedelta: + """Estimate the duration of a training task""" + try: + base_duration = timedelta(minutes=30) # Base 30 minutes + data_count = requirements.get("count", 100) + + # Scale with data count + duration_multiplier = max(1.0, data_count / 100) + + # Adjust for task type + if task_type == "scenario_training": + duration_multiplier *= 1.0 + elif task_type == "validation": + duration_multiplier *= 1.5 + elif task_type == "edge_case_practice": + duration_multiplier *= 2.0 + + # Adjust for complexity + if requirements.get("scenario_diversity", 0) > 0.8: + duration_multiplier *= 1.2 + + return base_duration * duration_multiplier + + except Exception as e: + self.logger.error(f"Failed to estimate task duration: {e}") + return timedelta(hours=1) # Default duration + + async def _schedule_task(self, task: TrainingTask, preferred_time: Optional[datetime] = None) -> None: + """Schedule a task in the queue""" + try: + # Determine optimal scheduling time + if preferred_time: + scheduled_time = preferred_time + else: + scheduled_time = await self._find_optimal_schedule_time(task) + + task.scheduled_at = scheduled_time + task.status = "scheduled" + + # Add to priority queue + heapq.heappush(self.task_queue, task) + + # Record scheduling decision + self.scheduling_history.append({ + "task_id": task.task_id, + "scheduled_at": scheduled_time.isoformat(), + "estimated_cost": task.estimated_cost, + "priority": task.priority.value, + "data_source": task.preferred_data_source.value, + "scheduling_policy": self.scheduling_policy.value + }) + + except Exception as e: + self.logger.error(f"Failed to schedule task: {e}") + raise + + async def _find_optimal_schedule_time(self, task: TrainingTask) -> datetime: + """Find optimal time to schedule a task""" + try: + current_time = datetime.now() + + # Get scheduling context + context = await self._build_scheduling_context() + + # Base scheduling logic + if task.priority == Priority.CRITICAL: + return current_time # Schedule immediately + + # Consider budget constraints + if task.estimated_cost > context.available_budget: + # Schedule for next budget period + return current_time + timedelta(hours=24) + + # Consider system load + if context.system_load > 0.8: + # Schedule when load is lower + return current_time + timedelta(hours=2) + + # Consider API rate limits + if task.preferred_data_source == DataSource.LIVE_API: + api_usage = context.api_rate_limits.get("live_api", 0) + if api_usage > 80: # Near rate limit + return current_time + timedelta(hours=1) + + # Policy-based scheduling + if self.scheduling_policy == SchedulingPolicy.COST_OPTIMIZED: + # Schedule during low-cost periods (example: off-peak hours) + if current_time.hour >= 9 and current_time.hour <= 17: + return current_time.replace(hour=20, minute=0, second=0) + + # Default: schedule with small random delay to spread load + delay_minutes = random.randint(0, 30) + return current_time + timedelta(minutes=delay_minutes) + + except Exception as e: + self.logger.error(f"Failed to find optimal schedule time: {e}") + return datetime.now() + timedelta(minutes=5) # Default short delay + + async def _build_scheduling_context(self) -> SchedulingContext: + """Build context for scheduling decisions""" + try: + current_time = datetime.now() + + # Get budget information + budget_status = self.budget_manager.get_budget_status() + available_budget = 0.0 + if budget_status.get("budget_periods"): + for period_data in budget_status["budget_periods"].values(): + available_budget += period_data.get("remaining", 0) + + # Get system metrics + performance_metrics = self.api_tracker.get_performance_metrics() + system_load = performance_metrics.get("error_rate", 0) + (performance_metrics.get("avg_latency_ms", 0) / 1000) + + # Get API rate limit status (simplified) + api_rate_limits = { + "live_api": performance_metrics.get("requests_last_hour", 0), + "synthetic": 0 # No limits for synthetic + } + + # Get agent performance metrics + agent_metrics = {} + for agent_id in self.agent_phases.keys(): + try: + analytics = await self.agent_analytics.get_agent_analytics(agent_id, timedelta(hours=6)) + agent_metrics[agent_id] = analytics + except Exception as e: + self.logger.warning(f"Failed to get analytics for {agent_id}: {e}") + agent_metrics[agent_id] = {} + + return SchedulingContext( + current_time=current_time, + available_budget=available_budget, + system_load=min(1.0, system_load), + api_rate_limits=api_rate_limits, + agent_performance_metrics=agent_metrics, + active_tasks=list(self.active_tasks.values()), + failed_tasks=self.failed_tasks[-10:] # Last 10 failures + ) + + except Exception as e: + self.logger.error(f"Failed to build scheduling context: {e}") + return SchedulingContext( + current_time=datetime.now(), + available_budget=100.0, + system_load=0.5, + api_rate_limits={}, + agent_performance_metrics={}, + active_tasks=[], + failed_tasks=[] + ) + + async def _scheduling_loop(self) -> None: + """Main scheduling loop""" + try: + while True: + await asyncio.sleep(60) # Check every minute + + current_time = datetime.now() + + # Execute scheduled tasks + await self._execute_ready_tasks(current_time) + + # Rebalance schedule periodically + if hasattr(self, '_last_rebalance'): + if current_time - self._last_rebalance >= self.rebalance_interval: + await self._rebalance_schedule() + self._last_rebalance = current_time + else: + self._last_rebalance = current_time + + # Update agent phases based on performance + await self._update_agent_phases() + + # Clean up old completed tasks + self._cleanup_old_tasks() + + except Exception as e: + self.logger.error(f"Scheduling loop error: {e}") + await asyncio.sleep(300) # Wait 5 minutes before retrying + + async def _execute_ready_tasks(self, current_time: datetime) -> None: + """Execute tasks that are ready to run""" + try: + # Check for tasks ready to execute + ready_tasks = [] + + while (self.task_queue and + self.task_queue[0].scheduled_at and + self.task_queue[0].scheduled_at <= current_time and + len(self.active_tasks) < self.max_concurrent_tasks): + + task = heapq.heappop(self.task_queue) + ready_tasks.append(task) + + # Execute ready tasks + for task in ready_tasks: + try: + # Check if task is still viable (budget, etc.) + can_execute, reason = await self._can_execute_task(task) + + if can_execute: + asyncio.create_task(self._execute_task(task)) + else: + self.logger.warning(f"Cannot execute task {task.task_id}: {reason}") + # Reschedule or fail the task + if task.retry_count < task.max_retries: + task.retry_count += 1 + task.scheduled_at = current_time + timedelta(minutes=30) + heapq.heappush(self.task_queue, task) + else: + task.status = "failed" + self.failed_tasks.append(task) + + except Exception as e: + self.logger.error(f"Failed to execute task {task.task_id}: {e}") + task.status = "failed" + self.failed_tasks.append(task) + + except Exception as e: + self.logger.error(f"Failed to execute ready tasks: {e}") + + async def _can_execute_task(self, task: TrainingTask) -> Tuple[bool, str]: + """Check if a task can be executed now""" + try: + # Budget check + is_allowed, budget_reason = self.budget_manager.is_request_allowed(task.estimated_cost) + if not is_allowed: + return False, budget_reason + + # Concurrency check + if len(self.active_tasks) >= self.max_concurrent_tasks: + return False, "Maximum concurrent tasks reached" + + # Agent-specific checks + agent_tasks = [t for t in self.active_tasks.values() if t.agent_id == task.agent_id] + if len(agent_tasks) >= 2: # Max 2 tasks per agent + return False, "Agent has too many active tasks" + + return True, "Task can be executed" + + except Exception as e: + self.logger.error(f"Failed to check task executability: {e}") + return False, str(e) + + async def _execute_task(self, task: TrainingTask) -> None: + """Execute a training task""" + try: + task.started_at = datetime.now() + task.status = "running" + self.active_tasks[task.task_id] = task + + self.logger.info(f"Executing task {task.task_id} for agent {task.agent_id}") + + # Get training data from orchestrator + training_data = await self.orchestrator.get_training_data( + data_type=task.task_type, + count=task.data_requirements.get("count", 100), + specific_requirements=task.data_requirements + ) + + # Execute training callback if available + callback = self.task_callbacks.get(task.task_type) + if callback: + await callback(task, training_data) + else: + # Default training execution (simplified) + await self._default_training_execution(task, training_data) + + # Mark task as completed + task.completed_at = datetime.now() + task.status = "completed" + self.completed_tasks.append(task) + + # Remove from active tasks + del self.active_tasks[task.task_id] + + # Update performance metrics + await self._update_task_performance_metrics(task, success=True) + + self.logger.info(f"Completed task {task.task_id}") + + except Exception as e: + self.logger.error(f"Task execution failed {task.task_id}: {e}") + task.status = "failed" + self.failed_tasks.append(task) + + if task.task_id in self.active_tasks: + del self.active_tasks[task.task_id] + + await self._update_task_performance_metrics(task, success=False) + + async def _default_training_execution(self, task: TrainingTask, training_data: List) -> None: + """Default training execution (placeholder)""" + # Simulate training time + await asyncio.sleep(2) # 2 second simulation + + # In real implementation, this would: + # 1. Pass data to agent training system + # 2. Execute training scenarios + # 3. Collect performance metrics + # 4. Update agent memory systems + + self.logger.info(f"Simulated training execution for task {task.task_id} with {len(training_data)} data points") + + async def _update_task_performance_metrics(self, task: TrainingTask, success: bool) -> None: + """Update performance metrics for completed tasks""" + try: + # Calculate actual vs estimated metrics + if task.started_at and task.completed_at: + actual_duration = task.completed_at - task.started_at + estimated_duration = task.estimated_duration + + duration_accuracy = 1 - abs((actual_duration - estimated_duration).total_seconds()) / estimated_duration.total_seconds() + self.performance_metrics["duration_accuracy"] = ( + self.performance_metrics["duration_accuracy"] * 0.9 + duration_accuracy * 0.1 + ) + + # Update success rate + current_success_rate = self.performance_metrics.get("success_rate", 0.5) + new_success_rate = current_success_rate * 0.9 + (1.0 if success else 0.0) * 0.1 + self.performance_metrics["success_rate"] = new_success_rate + + # Update cost accuracy (simplified) + self.performance_metrics["cost_accuracy"] = 0.85 # Placeholder + + except Exception as e: + self.logger.error(f"Failed to update task performance metrics: {e}") + + async def _rebalance_schedule(self) -> None: + """Rebalance the task schedule based on current conditions""" + try: + self.logger.info("Rebalancing task schedule") + + # Get current context + context = await self._build_scheduling_context() + + # Identify tasks that should be rescheduled + tasks_to_rebalance = [] + + for task in list(self.task_queue): + # Check if task should be rescheduled based on new context + if self._should_reschedule_task(task, context): + tasks_to_rebalance.append(task) + + # Remove and reschedule identified tasks + for task in tasks_to_rebalance: + self.task_queue.remove(task) + new_schedule_time = await self._find_optimal_schedule_time(task) + task.scheduled_at = new_schedule_time + heapq.heappush(self.task_queue, task) + + if tasks_to_rebalance: + self.logger.info(f"Rescheduled {len(tasks_to_rebalance)} tasks") + + except Exception as e: + self.logger.error(f"Failed to rebalance schedule: {e}") + + def _should_reschedule_task(self, task: TrainingTask, context: SchedulingContext) -> bool: + """Determine if a task should be rescheduled""" + try: + # Don't reschedule high priority tasks + if task.priority in [Priority.HIGH, Priority.CRITICAL]: + return False + + # Reschedule if budget is tight and task is expensive + if task.estimated_cost > context.available_budget * 0.5: + return True + + # Reschedule if system load is high + if context.system_load > 0.8 and task.priority == Priority.LOW: + return True + + # Reschedule based on agent performance + agent_metrics = context.agent_performance_metrics.get(task.agent_id, {}) + if agent_metrics.get("latest_snapshot", {}).get("learning_velocity", 0) < 0.2: + # Agent learning slowly, might want to delay advanced tasks + if task.task_type in ["validation", "edge_case_practice"]: + return True + + return False + + except Exception as e: + self.logger.error(f"Failed to check if task should be rescheduled: {e}") + return False + + async def _update_agent_phases(self) -> None: + """Update agent training phases based on performance""" + try: + for agent_id, current_phase in self.agent_phases.items(): + try: + # Get recent analytics + analytics = await self.agent_analytics.get_agent_analytics(agent_id, timedelta(hours=24)) + + if not analytics or "latest_snapshot" not in analytics: + continue + + snapshot = analytics["latest_snapshot"] + + # Phase progression logic + new_phase = self._determine_agent_phase(current_phase, snapshot, analytics) + + if new_phase != current_phase: + self.agent_phases[agent_id] = new_phase + self.logger.info(f"Agent {agent_id} progressed from {current_phase.value} to {new_phase.value}") + + # Schedule phase transition tasks + await self._schedule_phase_transition_tasks(agent_id, current_phase, new_phase) + + except Exception as e: + self.logger.warning(f"Failed to update phase for agent {agent_id}: {e}") + continue + + except Exception as e: + self.logger.error(f"Failed to update agent phases: {e}") + + def _determine_agent_phase(self, current_phase: TrainingPhase, snapshot: Dict, analytics: Dict) -> TrainingPhase: + """Determine appropriate training phase for an agent""" + try: + total_decisions = analytics.get("total_decisions", 0) + win_rate = snapshot.get("win_rate", 0) + kelly_adherence = snapshot.get("kelly_adherence_score", 0) + learning_velocity = snapshot.get("learning_velocity", 0) + + # Phase transition criteria + if current_phase == TrainingPhase.INITIAL: + if total_decisions >= 50 and win_rate > 0.3: + return TrainingPhase.FOUNDATION + + elif current_phase == TrainingPhase.FOUNDATION: + if total_decisions >= 200 and win_rate > 0.45 and kelly_adherence > 0.6: + return TrainingPhase.ADVANCED + + elif current_phase == TrainingPhase.ADVANCED: + if total_decisions >= 500 and win_rate > 0.55 and kelly_adherence > 0.75: + return TrainingPhase.VALIDATION + + elif current_phase == TrainingPhase.VALIDATION: + if total_decisions >= 1000 and win_rate > 0.6 and kelly_adherence > 0.8: + return TrainingPhase.PRODUCTION_PREP + + # Regression checks + if win_rate < 0.3 or learning_velocity < 0.1: + # Agent struggling, may need to go back + if current_phase == TrainingPhase.PRODUCTION_PREP: + return TrainingPhase.VALIDATION + elif current_phase == TrainingPhase.VALIDATION: + return TrainingPhase.ADVANCED + + return current_phase + + except Exception as e: + self.logger.error(f"Failed to determine agent phase: {e}") + return current_phase + + async def _schedule_phase_transition_tasks(self, agent_id: str, old_phase: TrainingPhase, new_phase: TrainingPhase) -> None: + """Schedule tasks appropriate for phase transition""" + try: + # Schedule validation task to confirm readiness + await self.schedule_training_task( + agent_id=agent_id, + task_type="validation", + priority=Priority.HIGH, + data_requirements={ + "count": 50, + "include_edge_cases": True, + "phase_transition": True, + "old_phase": old_phase.value, + "new_phase": new_phase.value + } + ) + + # Schedule new phase introduction task + await self.schedule_training_task( + agent_id=agent_id, + task_type="scenario_training", + priority=Priority.NORMAL, + data_requirements={ + "count": 100, + "training_phase": new_phase.value, + "phase_introduction": True + } + ) + + except Exception as e: + self.logger.error(f"Failed to schedule phase transition tasks: {e}") + + def _cleanup_old_tasks(self) -> None: + """Clean up old completed and failed tasks""" + try: + cutoff_time = datetime.now() - timedelta(days=7) + + # Clean completed tasks + self.completed_tasks = [ + task for task in self.completed_tasks + if task.completed_at and task.completed_at > cutoff_time + ] + + # Clean failed tasks + self.failed_tasks = [ + task for task in self.failed_tasks + if task.created_at > cutoff_time + ] + + except Exception as e: + self.logger.error(f"Failed to cleanup old tasks: {e}") + + def register_task_callback(self, task_type: str, callback: Callable) -> None: + """Register callback for specific task type""" + self.task_callbacks[task_type] = callback + self.logger.info(f"Registered callback for task type: {task_type}") + + def set_scheduling_policy(self, policy: SchedulingPolicy) -> None: + """Set the scheduling policy""" + old_policy = self.scheduling_policy + self.scheduling_policy = policy + self.logger.info(f"Changed scheduling policy from {old_policy.value} to {policy.value}") + + def get_scheduler_status(self) -> Dict[str, Any]: + """Get comprehensive scheduler status""" + try: + return { + "timestamp": datetime.now().isoformat(), + "scheduling_policy": self.scheduling_policy.value, + "queue_size": len(self.task_queue), + "active_tasks": len(self.active_tasks), + "completed_tasks": len(self.completed_tasks), + "failed_tasks": len(self.failed_tasks), + "agent_phases": {agent_id: phase.value for agent_id, phase in self.agent_phases.items()}, + "performance_metrics": dict(self.performance_metrics), + "next_scheduled_task": self.task_queue[0].scheduled_at.isoformat() if self.task_queue else None, + "system_load": self.performance_metrics.get("system_load", 0.5) + } + + except Exception as e: + self.logger.error(f"Failed to get scheduler status: {e}") + return {"error": str(e)} + + async def shutdown(self) -> None: + """Gracefully shutdown the scheduler""" + try: + self.logger.info("Shutting down Adaptive Scheduler") + + # Wait for active tasks to complete (with timeout) + timeout = 300 # 5 minutes + start_time = datetime.now() + + while self.active_tasks and (datetime.now() - start_time).seconds < timeout: + await asyncio.sleep(10) + + # Force stop remaining tasks + for task in self.active_tasks.values(): + task.status = "cancelled" + self.failed_tasks.append(task) + + self.active_tasks.clear() + + self.logger.info("Adaptive Scheduler shutdown complete") + + except Exception as e: + self.logger.error(f"Error during scheduler shutdown: {e}") \ No newline at end of file diff --git a/src/hybrid_pipeline/cost_monitor.py b/src/hybrid_pipeline/cost_monitor.py new file mode 100644 index 00000000..a375a044 --- /dev/null +++ b/src/hybrid_pipeline/cost_monitor.py @@ -0,0 +1,610 @@ +""" +API Cost Monitor and Budget Management + +Tracks API usage costs, enforces budget limits, and provides +cost optimization recommendations for hybrid data pipeline. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Callable, Any, Tuple +from datetime import datetime, timedelta, date +from enum import Enum +import asyncio +import logging +import json +from collections import defaultdict, deque + + +class AlertLevel(Enum): + """Alert severity levels""" + INFO = "info" + WARNING = "warning" + CRITICAL = "critical" + EMERGENCY = "emergency" + + +@dataclass +class CostAlert: + """Cost alert definition""" + alert_id: str + level: AlertLevel + threshold: float + description: str + callback: Optional[Callable] = None + enabled: bool = True + last_triggered: Optional[datetime] = None + trigger_count: int = 0 + + +@dataclass +class APIEndpoint: + """API endpoint cost configuration""" + name: str + base_cost: float + cost_per_request: float = 0.0 + cost_per_data_unit: float = 0.0 # per KB, record, etc. + rate_limit_requests: int = 1000 + rate_limit_window: timedelta = timedelta(hours=1) + current_usage: int = 0 + window_start: datetime = field(default_factory=datetime.now) + + +@dataclass +class BudgetPeriod: + """Budget tracking for a specific time period""" + period_name: str # "daily", "weekly", "monthly" + budget_limit: float + spent_amount: float = 0.0 + period_start: datetime = field(default_factory=datetime.now) + period_end: datetime = field(default_factory=lambda: datetime.now() + timedelta(days=1)) + + @property + def utilization_percent(self) -> float: + """Get budget utilization percentage""" + return (self.spent_amount / self.budget_limit) * 100 if self.budget_limit > 0 else 0 + + @property + def remaining_budget(self) -> float: + """Get remaining budget amount""" + return max(0, self.budget_limit - self.spent_amount) + + +class APITracker: + """ + Tracks API usage, costs, and performance metrics for cost optimization. + """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + # API endpoints configuration + self.endpoints: Dict[str, APIEndpoint] = {} + + # Usage tracking + self.request_history: deque = deque(maxlen=10000) + self.cost_history: Dict[date, float] = defaultdict(float) + + # Performance metrics + self.latency_history: deque = deque(maxlen=1000) + self.error_history: deque = deque(maxlen=1000) + + # Cost breakdown + self.cost_by_endpoint: Dict[str, float] = defaultdict(float) + self.cost_by_hour: Dict[datetime, float] = defaultdict(float) + + def register_endpoint(self, endpoint: APIEndpoint) -> None: + """Register an API endpoint for tracking""" + self.endpoints[endpoint.name] = endpoint + self.logger.info(f"Registered API endpoint: {endpoint.name}") + + async def track_request( + self, + endpoint_name: str, + data_size: int = 0, + latency_ms: float = 0, + success: bool = True, + cost_override: Optional[float] = None + ) -> float: + """ + Track an API request and return the cost incurred. + + Args: + endpoint_name: Name of the API endpoint + data_size: Size of data transferred (for cost calculation) + latency_ms: Request latency in milliseconds + success: Whether the request was successful + cost_override: Override calculated cost with specific amount + + Returns: + Cost incurred for this request + """ + try: + timestamp = datetime.now() + + # Get endpoint configuration + endpoint = self.endpoints.get(endpoint_name) + if not endpoint: + self.logger.warning(f"Unknown endpoint: {endpoint_name}") + return 0.0 + + # Check rate limits + if not self._check_rate_limit(endpoint, timestamp): + self.logger.warning(f"Rate limit exceeded for {endpoint_name}") + return 0.0 + + # Calculate cost + if cost_override is not None: + cost = cost_override + else: + cost = endpoint.cost_per_request + (endpoint.cost_per_data_unit * data_size / 1024) # Assume data_size in bytes + + # Record request + request_record = { + "timestamp": timestamp, + "endpoint": endpoint_name, + "data_size": data_size, + "latency_ms": latency_ms, + "success": success, + "cost": cost + } + + self.request_history.append(request_record) + + # Update metrics + self._update_cost_metrics(endpoint_name, cost, timestamp) + self._update_performance_metrics(latency_ms, success, timestamp) + + # Update endpoint usage + endpoint.current_usage += 1 + + return cost + + except Exception as e: + self.logger.error(f"Failed to track request: {e}") + return 0.0 + + def _check_rate_limit(self, endpoint: APIEndpoint, timestamp: datetime) -> bool: + """Check if request is within rate limits""" + try: + # Reset window if needed + if timestamp - endpoint.window_start >= endpoint.rate_limit_window: + endpoint.current_usage = 0 + endpoint.window_start = timestamp + + # Check limit + return endpoint.current_usage < endpoint.rate_limit_requests + + except Exception as e: + self.logger.error(f"Rate limit check failed: {e}") + return True # Allow request on error + + def _update_cost_metrics(self, endpoint_name: str, cost: float, timestamp: datetime) -> None: + """Update cost tracking metrics""" + # Daily cost tracking + today = timestamp.date() + self.cost_history[today] += cost + + # Endpoint cost tracking + self.cost_by_endpoint[endpoint_name] += cost + + # Hourly cost tracking + hour_key = timestamp.replace(minute=0, second=0, microsecond=0) + self.cost_by_hour[hour_key] += cost + + def _update_performance_metrics(self, latency_ms: float, success: bool, timestamp: datetime) -> None: + """Update performance tracking metrics""" + if latency_ms > 0: + self.latency_history.append({"timestamp": timestamp, "latency": latency_ms}) + + self.error_history.append({"timestamp": timestamp, "success": success}) + + def get_daily_cost(self, target_date: Optional[date] = None) -> float: + """Get total cost for a specific day""" + target_date = target_date or datetime.now().date() + return self.cost_history.get(target_date, 0.0) + + def get_hourly_costs(self, hours: int = 24) -> Dict[str, float]: + """Get hourly cost breakdown for the last N hours""" + cutoff_time = datetime.now() - timedelta(hours=hours) + + return { + hour.strftime("%Y-%m-%d %H:00"): cost + for hour, cost in self.cost_by_hour.items() + if hour >= cutoff_time + } + + def get_endpoint_costs(self) -> Dict[str, Dict[str, Any]]: + """Get cost breakdown by endpoint""" + total_cost = sum(self.cost_by_endpoint.values()) + + return { + endpoint: { + "total_cost": cost, + "percentage": (cost / total_cost) * 100 if total_cost > 0 else 0, + "requests": sum(1 for r in self.request_history if r["endpoint"] == endpoint), + "avg_cost_per_request": cost / max(1, sum(1 for r in self.request_history if r["endpoint"] == endpoint)) + } + for endpoint, cost in self.cost_by_endpoint.items() + } + + def get_performance_metrics(self) -> Dict[str, Any]: + """Get performance metrics summary""" + recent_latencies = [r["latency"] for r in self.latency_history if r["latency"] > 0] + recent_errors = [not r["success"] for r in self.error_history] + + return { + "avg_latency_ms": sum(recent_latencies) / len(recent_latencies) if recent_latencies else 0, + "p95_latency_ms": sorted(recent_latencies)[int(len(recent_latencies) * 0.95)] if recent_latencies else 0, + "error_rate": sum(recent_errors) / len(recent_errors) if recent_errors else 0, + "total_requests": len(self.request_history), + "requests_last_hour": sum(1 for r in self.request_history + if datetime.now() - r["timestamp"] <= timedelta(hours=1)) + } + + def generate_cost_report(self, days: int = 7) -> Dict[str, Any]: + """Generate comprehensive cost report""" + cutoff_date = datetime.now().date() - timedelta(days=days) + + # Daily costs + daily_costs = { + str(date_key): cost + for date_key, cost in self.cost_history.items() + if date_key >= cutoff_date + } + + # Total cost + total_cost = sum(daily_costs.values()) + + return { + "report_period": f"{days} days", + "total_cost": total_cost, + "daily_average": total_cost / days if days > 0 else 0, + "daily_breakdown": daily_costs, + "endpoint_breakdown": self.get_endpoint_costs(), + "hourly_breakdown": self.get_hourly_costs(24 * days), + "performance_metrics": self.get_performance_metrics(), + "cost_trend": self._calculate_cost_trend(daily_costs) + } + + def _calculate_cost_trend(self, daily_costs: Dict[str, float]) -> str: + """Calculate cost trend over time""" + if len(daily_costs) < 2: + return "insufficient_data" + + costs = list(daily_costs.values()) + first_half_avg = sum(costs[:len(costs)//2]) / (len(costs)//2) + second_half_avg = sum(costs[len(costs)//2:]) / (len(costs) - len(costs)//2) + + if abs(second_half_avg - first_half_avg) < first_half_avg * 0.1: + return "stable" + elif second_half_avg > first_half_avg: + return "increasing" + else: + return "decreasing" + + +class BudgetManager: + """ + Manages budget limits and enforces cost controls across multiple time periods. + """ + + def __init__(self, api_tracker: APITracker): + self.api_tracker = api_tracker + self.logger = logging.getLogger(__name__) + + # Budget periods + self.budget_periods: Dict[str, BudgetPeriod] = {} + + # Alerts system + self.alerts: Dict[str, CostAlert] = {} + self.alert_callbacks: Dict[str, Callable] = {} + + # Emergency controls + self.emergency_stop_enabled = False + self.emergency_threshold = 500.0 # $500 emergency cutoff + + # Cost predictions + self.prediction_window_hours = 24 + + def set_budget(self, period_name: str, budget_limit: float, period_duration: timedelta) -> None: + """Set budget for a specific period""" + period_end = datetime.now() + period_duration + + budget_period = BudgetPeriod( + period_name=period_name, + budget_limit=budget_limit, + period_start=datetime.now(), + period_end=period_end + ) + + self.budget_periods[period_name] = budget_period + self.logger.info(f"Set budget for {period_name}: ${budget_limit} until {period_end}") + + def add_alert(self, alert: CostAlert) -> None: + """Add a cost alert""" + self.alerts[alert.alert_id] = alert + self.logger.info(f"Added cost alert: {alert.alert_id} at {alert.threshold} threshold") + + async def check_budgets_and_alerts(self) -> List[Dict[str, Any]]: + """Check all budgets and trigger alerts if needed""" + triggered_alerts = [] + + try: + # Update budget periods with current costs + await self._update_budget_periods() + + # Check budget violations + for period_name, budget_period in self.budget_periods.items(): + utilization = budget_period.utilization_percent + + # Check alerts for this budget period + for alert_id, alert in self.alerts.items(): + if not alert.enabled: + continue + + # Check if alert threshold is met + should_trigger = False + + if alert.level == AlertLevel.INFO and utilization >= 50: + should_trigger = True + elif alert.level == AlertLevel.WARNING and utilization >= 75: + should_trigger = True + elif alert.level == AlertLevel.CRITICAL and utilization >= 90: + should_trigger = True + elif alert.level == AlertLevel.EMERGENCY and utilization >= 95: + should_trigger = True + + # Or check absolute cost threshold + if budget_period.spent_amount >= alert.threshold: + should_trigger = True + + if should_trigger: + triggered_alert = await self._trigger_alert(alert, budget_period) + triggered_alerts.append(triggered_alert) + + # Check emergency stop + total_daily_cost = self.api_tracker.get_daily_cost() + if total_daily_cost >= self.emergency_threshold: + await self._trigger_emergency_stop() + + return triggered_alerts + + except Exception as e: + self.logger.error(f"Failed to check budgets and alerts: {e}") + return [] + + async def _update_budget_periods(self) -> None: + """Update budget periods with current spending""" + current_time = datetime.now() + + for period_name, budget_period in self.budget_periods.items(): + # Check if period has expired and needs renewal + if current_time >= budget_period.period_end: + await self._renew_budget_period(period_name, budget_period) + continue + + # Calculate spent amount for this period + period_costs = [] + for record in self.api_tracker.request_history: + if (budget_period.period_start <= record["timestamp"] <= current_time and + record["timestamp"] <= budget_period.period_end): + period_costs.append(record["cost"]) + + budget_period.spent_amount = sum(period_costs) + + async def _renew_budget_period(self, period_name: str, old_period: BudgetPeriod) -> None: + """Renew an expired budget period""" + try: + # Calculate duration of old period + duration = old_period.period_end - old_period.period_start + + # Create new period with same budget limit + new_period = BudgetPeriod( + period_name=period_name, + budget_limit=old_period.budget_limit, + period_start=datetime.now(), + period_end=datetime.now() + duration + ) + + self.budget_periods[period_name] = new_period + + self.logger.info(f"Renewed budget period {period_name}: ${new_period.budget_limit}") + + except Exception as e: + self.logger.error(f"Failed to renew budget period {period_name}: {e}") + + async def _trigger_alert(self, alert: CostAlert, budget_period: BudgetPeriod) -> Dict[str, Any]: + """Trigger a cost alert""" + try: + current_time = datetime.now() + + alert_data = { + "alert_id": alert.alert_id, + "level": alert.level.value, + "timestamp": current_time.isoformat(), + "description": alert.description, + "budget_period": budget_period.period_name, + "spent_amount": budget_period.spent_amount, + "budget_limit": budget_period.budget_limit, + "utilization_percent": budget_period.utilization_percent, + "threshold_exceeded": budget_period.spent_amount >= alert.threshold + } + + # Update alert record + alert.last_triggered = current_time + alert.trigger_count += 1 + + # Execute callback if available + if alert.callback: + try: + await alert.callback(alert_data) + except Exception as e: + self.logger.error(f"Alert callback failed for {alert.alert_id}: {e}") + + self.logger.warning(f"Cost alert triggered: {alert.alert_id} - {alert.description}") + + return alert_data + + except Exception as e: + self.logger.error(f"Failed to trigger alert {alert.alert_id}: {e}") + return {"error": str(e)} + + async def _trigger_emergency_stop(self) -> None: + """Trigger emergency stop to halt all API usage""" + try: + self.emergency_stop_enabled = True + + emergency_data = { + "timestamp": datetime.now().isoformat(), + "reason": "emergency_cost_threshold_exceeded", + "daily_cost": self.api_tracker.get_daily_cost(), + "emergency_threshold": self.emergency_threshold + } + + self.logger.critical(f"EMERGENCY STOP TRIGGERED: Daily cost ${emergency_data['daily_cost']} exceeded threshold ${self.emergency_threshold}") + + # Execute emergency callbacks + for callback in self.alert_callbacks.get("emergency", []): + try: + await callback(emergency_data) + except Exception as e: + self.logger.error(f"Emergency callback failed: {e}") + + except Exception as e: + self.logger.error(f"Failed to trigger emergency stop: {e}") + + def is_request_allowed(self, estimated_cost: float) -> Tuple[bool, str]: + """Check if a request is allowed based on budget constraints""" + try: + # Emergency stop check + if self.emergency_stop_enabled: + return False, "Emergency stop is active" + + # Check daily emergency threshold + current_daily_cost = self.api_tracker.get_daily_cost() + if current_daily_cost + estimated_cost >= self.emergency_threshold: + return False, f"Would exceed emergency threshold (${self.emergency_threshold})" + + # Check budget periods + for period_name, budget_period in self.budget_periods.items(): + if budget_period.spent_amount + estimated_cost > budget_period.budget_limit: + return False, f"Would exceed {period_name} budget (${budget_period.budget_limit})" + + return True, "Request allowed" + + except Exception as e: + self.logger.error(f"Error checking request allowance: {e}") + return False, "Error in budget check" + + def predict_costs(self, hours: int = None) -> Dict[str, Any]: + """Predict future costs based on current usage patterns""" + try: + hours = hours or self.prediction_window_hours + cutoff_time = datetime.now() - timedelta(hours=hours) + + # Get recent requests for trend analysis + recent_requests = [ + r for r in self.api_tracker.request_history + if r["timestamp"] >= cutoff_time + ] + + if not recent_requests: + return {"error": "No recent data for prediction"} + + # Calculate hourly rate + recent_cost = sum(r["cost"] for r in recent_requests) + hourly_rate = recent_cost / hours + + # Predict next period costs + predictions = { + "next_hour": hourly_rate, + "next_6_hours": hourly_rate * 6, + "next_24_hours": hourly_rate * 24, + "end_of_day": hourly_rate * (24 - datetime.now().hour), + "end_of_week": hourly_rate * ((7 - datetime.now().weekday()) * 24 + (24 - datetime.now().hour)) + } + + # Check budget implications + budget_warnings = [] + for period_name, budget_period in self.budget_periods.items(): + remaining_hours = (budget_period.period_end - datetime.now()).total_seconds() / 3600 + if remaining_hours > 0: + predicted_spend = hourly_rate * remaining_hours + if budget_period.spent_amount + predicted_spend > budget_period.budget_limit: + budget_warnings.append(f"{period_name} budget may be exceeded") + + return { + "hourly_rate": hourly_rate, + "predictions": predictions, + "budget_warnings": budget_warnings, + "confidence": "medium" if len(recent_requests) >= 10 else "low" + } + + except Exception as e: + self.logger.error(f"Cost prediction failed: {e}") + return {"error": str(e)} + + def get_budget_status(self) -> Dict[str, Any]: + """Get comprehensive budget status""" + try: + status = { + "timestamp": datetime.now().isoformat(), + "emergency_stop_active": self.emergency_stop_enabled, + "daily_cost": self.api_tracker.get_daily_cost(), + "emergency_threshold": self.emergency_threshold, + "budget_periods": {}, + "active_alerts": len([a for a in self.alerts.values() if a.enabled]), + "recent_alerts": [ + { + "alert_id": alert.alert_id, + "last_triggered": alert.last_triggered.isoformat() if alert.last_triggered else None, + "trigger_count": alert.trigger_count + } + for alert in self.alerts.values() + if alert.last_triggered and + datetime.now() - alert.last_triggered <= timedelta(hours=24) + ] + } + + # Add budget period details + for period_name, budget_period in self.budget_periods.items(): + status["budget_periods"][period_name] = { + "budget_limit": budget_period.budget_limit, + "spent_amount": budget_period.spent_amount, + "remaining": budget_period.remaining_budget, + "utilization_percent": budget_period.utilization_percent, + "period_start": budget_period.period_start.isoformat(), + "period_end": budget_period.period_end.isoformat(), + "time_remaining": str(budget_period.period_end - datetime.now()) + } + + return status + + except Exception as e: + self.logger.error(f"Failed to get budget status: {e}") + return {"error": str(e)} + + def reset_emergency_stop(self) -> None: + """Reset emergency stop (manual override)""" + self.emergency_stop_enabled = False + self.logger.info("Emergency stop reset manually") + + def disable_alert(self, alert_id: str) -> None: + """Disable a specific alert""" + if alert_id in self.alerts: + self.alerts[alert_id].enabled = False + self.logger.info(f"Disabled alert: {alert_id}") + + def enable_alert(self, alert_id: str) -> None: + """Enable a specific alert""" + if alert_id in self.alerts: + self.alerts[alert_id].enabled = True + self.logger.info(f"Enabled alert: {alert_id}") + + def register_callback(self, event_type: str, callback: Callable) -> None: + """Register callback for specific events""" + if event_type not in self.alert_callbacks: + self.alert_callbacks[event_type] = [] + + self.alert_callbacks[event_type].append(callback) + self.logger.info(f"Registered callback for {event_type}") \ No newline at end of file diff --git a/src/hybrid_pipeline/data_orchestrator.py b/src/hybrid_pipeline/data_orchestrator.py new file mode 100644 index 00000000..3beae655 --- /dev/null +++ b/src/hybrid_pipeline/data_orchestrator.py @@ -0,0 +1,779 @@ +""" +Hybrid Data Orchestrator + +Intelligently switches between live API data and synthetic data generation +based on training requirements, API costs, and performance metrics. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Any, Union, AsyncGenerator +from enum import Enum +import asyncio +import logging +from datetime import datetime, timedelta +import json + +from ..synthetic_data.generators.game_engine import SyntheticGameEngine +from ..synthetic_data.generators.market_simulator import MarketSimulator +from ..synthetic_data.generators.scenario_builder import ScenarioBuilder +from ..synthetic_data.storage.chromadb_manager import ChromaDBManager +from ..synthetic_data.preprocessing.nfl_dataset_processor import ProcessedNFLPlay +from ..sdk.core.base_adapter import StandardizedEvent +from ..training.agent_analytics import AgentAnalytics + + +class DataSource(Enum): + """Available data sources""" + LIVE_API = "live_api" + SYNTHETIC = "synthetic" + HYBRID = "hybrid" + CACHED = "cached" + + +class DataMode(Enum): + """Operating modes for data pipeline""" + TRAINING = "training" + BACKTESTING = "backtesting" + LIVE_TRADING = "live_trading" + DEVELOPMENT = "development" + + +class SwitchingStrategy(Enum): + """Strategies for switching between data sources""" + COST_BASED = "cost_based" + PERFORMANCE_BASED = "performance_based" + TIME_BASED = "time_based" + MANUAL = "manual" + INTELLIGENT = "intelligent" + + +@dataclass +class CostThreshold: + """Cost thresholds for data source switching""" + daily_limit: float = 100.0 + hourly_limit: float = 10.0 + per_request_limit: float = 0.50 + emergency_cutoff: float = 200.0 + synthetic_cost_per_event: float = 0.001 # Much lower cost + + +@dataclass +class DataSourceMetrics: + """Metrics for a specific data source""" + source: DataSource + requests_made: int = 0 + cost_incurred: float = 0.0 + latency_ms: float = 0.0 + error_rate: float = 0.0 + data_quality_score: float = 1.0 + last_used: Optional[datetime] = None + availability: bool = True + + +@dataclass +class HybridConfig: + """Configuration for hybrid data pipeline""" + default_mode: DataMode = DataMode.TRAINING + primary_source: DataSource = DataSource.SYNTHETIC + fallback_source: DataSource = DataSource.CACHED + switching_strategy: SwitchingStrategy = SwitchingStrategy.INTELLIGENT + cost_thresholds: CostThreshold = field(default_factory=CostThreshold) + + # Performance thresholds + max_latency_ms: float = 1000.0 + min_data_quality: float = 0.8 + max_error_rate: float = 0.05 + + # Synthetic data preferences + synthetic_scenario_count: int = 1000 + edge_case_probability: float = 0.1 + information_asymmetry_level: float = 0.3 + + # Caching settings + cache_duration_hours: int = 24 + cache_size_limit: int = 10000 + + +class HybridDataOrchestrator: + """ + Orchestrates hybrid data pipeline for optimal cost-performance balance. + + Automatically switches between live API data and synthetic data based on: + - API costs and budget constraints + - Agent training performance requirements + - Data quality and availability + - Training phase and objectives + """ + + def __init__( + self, + config: HybridConfig, + chroma_manager: ChromaDBManager, + synthetic_engine: SyntheticGameEngine, + market_simulator: MarketSimulator, + scenario_builder: ScenarioBuilder, + agent_analytics: AgentAnalytics + ): + self.config = config + self.chroma_manager = chroma_manager + self.synthetic_engine = synthetic_engine + self.market_simulator = market_simulator + self.scenario_builder = scenario_builder + self.agent_analytics = agent_analytics + + self.logger = logging.getLogger(__name__) + + # Data source metrics + self.source_metrics = { + DataSource.LIVE_API: DataSourceMetrics(DataSource.LIVE_API), + DataSource.SYNTHETIC: DataSourceMetrics(DataSource.SYNTHETIC), + DataSource.CACHED: DataSourceMetrics(DataSource.CACHED) + } + + # Current state + self.active_source = self.config.primary_source + self.current_mode = self.config.default_mode + self.daily_costs = 0.0 + self.last_cost_reset = datetime.now().date() + + # Cache for processed data + self.data_cache: Dict[str, Any] = {} + self.cache_timestamps: Dict[str, datetime] = {} + + # Decision history for intelligent switching + self.switching_history: List[Dict[str, Any]] = [] + + async def initialize(self) -> None: + """Initialize the hybrid data orchestrator""" + try: + self.logger.info("Initializing Hybrid Data Orchestrator") + + # Reset daily costs if needed + self._check_daily_cost_reset() + + # Verify synthetic data systems + await self._verify_synthetic_systems() + + # Load cached decisions if available + await self._load_switching_history() + + # Initial data source selection + await self._select_optimal_data_source() + + self.logger.info(f"Hybrid orchestrator initialized with {self.active_source} as active source") + + except Exception as e: + self.logger.error(f"Failed to initialize hybrid orchestrator: {e}") + raise + + async def get_training_data( + self, + data_type: str, + count: int = 100, + specific_requirements: Optional[Dict[str, Any]] = None + ) -> List[StandardizedEvent]: + """ + Get training data using optimal data source selection. + + Args: + data_type: Type of data needed ("game_events", "market_events", etc.) + count: Number of events requested + specific_requirements: Specific requirements (edge cases, scenarios, etc.) + + Returns: + List of standardized events for training + """ + try: + # Evaluate current data source optimality + should_switch = await self._should_switch_data_source(data_type, count, specific_requirements) + + if should_switch: + await self._switch_data_source(should_switch) + + # Get data from active source + if self.active_source == DataSource.SYNTHETIC: + return await self._get_synthetic_data(data_type, count, specific_requirements) + elif self.active_source == DataSource.LIVE_API: + return await self._get_live_api_data(data_type, count, specific_requirements) + elif self.active_source == DataSource.CACHED: + return await self._get_cached_data(data_type, count, specific_requirements) + else: + raise ValueError(f"Unsupported data source: {self.active_source}") + + except Exception as e: + self.logger.error(f"Failed to get training data: {e}") + # Fallback to synthetic data + if self.active_source != DataSource.SYNTHETIC: + self.logger.info("Falling back to synthetic data") + return await self._get_synthetic_data(data_type, count, specific_requirements) + raise + + async def _should_switch_data_source( + self, + data_type: str, + count: int, + requirements: Optional[Dict[str, Any]] + ) -> Optional[DataSource]: + """Determine if data source should be switched and to which source""" + try: + current_metrics = self.source_metrics[self.active_source] + + # Cost-based switching + if self.config.switching_strategy in [SwitchingStrategy.COST_BASED, SwitchingStrategy.INTELLIGENT]: + estimated_cost = await self._estimate_request_cost(self.active_source, count) + + # Check if cost exceeds thresholds + if (self.daily_costs + estimated_cost > self.config.cost_thresholds.daily_limit or + estimated_cost > self.config.cost_thresholds.per_request_limit): + + # Switch to synthetic if currently using live API + if self.active_source == DataSource.LIVE_API: + return DataSource.SYNTHETIC + + # Performance-based switching + if self.config.switching_strategy in [SwitchingStrategy.PERFORMANCE_BASED, SwitchingStrategy.INTELLIGENT]: + if (current_metrics.latency_ms > self.config.max_latency_ms or + current_metrics.error_rate > self.config.max_error_rate or + current_metrics.data_quality_score < self.config.min_data_quality): + + # Find best alternative source + return await self._find_best_alternative_source() + + # Intelligence-based switching + if self.config.switching_strategy == SwitchingStrategy.INTELLIGENT: + return await self._intelligent_source_selection(data_type, count, requirements) + + return None # No switch needed + + except Exception as e: + self.logger.error(f"Error in switch evaluation: {e}") + return None + + async def _intelligent_source_selection( + self, + data_type: str, + count: int, + requirements: Optional[Dict[str, Any]] + ) -> Optional[DataSource]: + """Intelligent data source selection based on multiple factors""" + try: + # Factors to consider + factors = {} + + # 1. Training phase analysis + if requirements and requirements.get("training_phase"): + phase = requirements["training_phase"] + if phase == "initial": + # Use synthetic for initial training (cheaper, unlimited scenarios) + factors["phase_preference"] = DataSource.SYNTHETIC + elif phase == "advanced": + # Mix of synthetic and live for advanced training + factors["phase_preference"] = DataSource.HYBRID + elif phase == "validation": + # Live data for final validation + factors["phase_preference"] = DataSource.LIVE_API + + # 2. Agent performance analysis + if requirements and requirements.get("agent_id"): + agent_id = requirements["agent_id"] + # Get recent analytics to determine if agent needs diverse scenarios + analytics = await self.agent_analytics.get_agent_analytics(agent_id, timedelta(hours=24)) + + if analytics.get("latest_snapshot"): + snapshot = analytics["latest_snapshot"] + + # If learning velocity is low, provide more diverse synthetic scenarios + if snapshot.get("learning_velocity", 0) < 0.3: + factors["performance_need"] = DataSource.SYNTHETIC + + # If agent is performing well, validate with live data + elif snapshot.get("win_rate", 0) > 0.6: + factors["performance_need"] = DataSource.LIVE_API + + # 3. Cost efficiency analysis + synthetic_cost = self.config.cost_thresholds.synthetic_cost_per_event * count + api_cost = await self._estimate_request_cost(DataSource.LIVE_API, count) + + if api_cost > synthetic_cost * 10: # API is 10x more expensive + factors["cost_efficiency"] = DataSource.SYNTHETIC + + # 4. Data requirements analysis + if requirements: + # Edge cases are better handled by synthetic data + if requirements.get("include_edge_cases", False): + factors["data_requirement"] = DataSource.SYNTHETIC + + # Real market conditions need live data + if requirements.get("market_realism", False): + factors["data_requirement"] = DataSource.LIVE_API + + # 5. System load analysis + current_load = await self._get_system_load() + if current_load > 0.8: # High load, prefer cached data + factors["system_load"] = DataSource.CACHED + + # Decision algorithm: weighted voting + votes = {source: 0 for source in DataSource} + for factor_name, preferred_source in factors.items(): + weight = self._get_factor_weight(factor_name) + votes[preferred_source] += weight + + # Select highest voted source + best_source = max(votes, key=votes.get) + + # Only switch if significantly better + if votes[best_source] > votes[self.active_source] * 1.2: + return best_source + + return None + + except Exception as e: + self.logger.error(f"Error in intelligent source selection: {e}") + return None + + def _get_factor_weight(self, factor_name: str) -> float: + """Get weight for different decision factors""" + weights = { + "cost_efficiency": 0.4, + "phase_preference": 0.3, + "performance_need": 0.2, + "data_requirement": 0.25, + "system_load": 0.15 + } + return weights.get(factor_name, 0.1) + + async def _estimate_request_cost(self, source: DataSource, count: int) -> float: + """Estimate cost for a data request""" + if source == DataSource.SYNTHETIC: + return self.config.cost_thresholds.synthetic_cost_per_event * count + elif source == DataSource.LIVE_API: + # Estimate based on typical API costs + return count * 0.01 # $0.01 per event (example) + elif source == DataSource.CACHED: + return 0.0 # No cost for cached data + else: + return 0.0 + + async def _get_system_load(self) -> float: + """Get current system load (simplified implementation)""" + try: + # In real implementation, would check CPU, memory, network + # For now, return random load between 0.1 and 0.9 + import random + return random.uniform(0.1, 0.9) + except Exception: + return 0.5 # Default moderate load + + async def _find_best_alternative_source(self) -> DataSource: + """Find the best alternative data source based on current metrics""" + best_source = None + best_score = -1 + + for source, metrics in self.source_metrics.items(): + if source == self.active_source: + continue + + if not metrics.availability: + continue + + # Score based on multiple factors + score = ( + (1 - metrics.error_rate) * 0.3 + + min(1.0, 1000.0 / max(metrics.latency_ms, 1)) * 0.3 + + metrics.data_quality_score * 0.4 + ) + + if score > best_score: + best_score = score + best_source = source + + return best_source or DataSource.SYNTHETIC # Fallback to synthetic + + async def _switch_data_source(self, new_source: DataSource) -> None: + """Switch to a new data source""" + try: + old_source = self.active_source + self.active_source = new_source + + # Record the switch + switch_record = { + "timestamp": datetime.now().isoformat(), + "from_source": old_source.value, + "to_source": new_source.value, + "reason": "intelligent_switching", + "daily_cost": self.daily_costs + } + self.switching_history.append(switch_record) + + # Update metrics + self.source_metrics[new_source].last_used = datetime.now() + + self.logger.info(f"Switched data source from {old_source.value} to {new_source.value}") + + except Exception as e: + self.logger.error(f"Failed to switch data source: {e}") + raise + + async def _get_synthetic_data( + self, + data_type: str, + count: int, + requirements: Optional[Dict[str, Any]] + ) -> List[StandardizedEvent]: + """Get synthetic training data""" + try: + start_time = datetime.now() + + if data_type == "game_events": + # Generate synthetic games + scenarios = await self.scenario_builder.build_comprehensive_training_set( + num_scenarios=max(count // 20, 1), # ~20 events per game + include_edge_cases=requirements.get("include_edge_cases", True) if requirements else True + ) + + # Convert scenarios to standardized events + events = [] + for scenario in scenarios.scenarios[:count]: + game_events = await self._convert_scenario_to_events(scenario) + events.extend(game_events) + + events = events[:count] # Trim to requested count + + elif data_type == "market_events": + # Generate market-specific scenarios + trading_scenarios = await self.scenario_builder.build_market_scenarios(count) + events = await self._convert_market_scenarios_to_events(trading_scenarios) + + else: + # Default: generate mixed scenarios + scenarios = await self.scenario_builder.build_comprehensive_training_set( + num_scenarios=count // 10, + include_edge_cases=True + ) + events = [] + for scenario in scenarios.scenarios: + scenario_events = await self._convert_scenario_to_events(scenario) + events.extend(scenario_events) + events = events[:count] + + # Update metrics + latency = (datetime.now() - start_time).total_seconds() * 1000 + cost = len(events) * self.config.cost_thresholds.synthetic_cost_per_event + + await self._update_source_metrics( + DataSource.SYNTHETIC, + requests=1, + cost=cost, + latency=latency, + success=True + ) + + return events + + except Exception as e: + await self._update_source_metrics(DataSource.SYNTHETIC, requests=1, success=False) + self.logger.error(f"Failed to get synthetic data: {e}") + raise + + async def _get_live_api_data( + self, + data_type: str, + count: int, + requirements: Optional[Dict[str, Any]] + ) -> List[StandardizedEvent]: + """Get data from live APIs (placeholder implementation)""" + try: + start_time = datetime.now() + + # This would integrate with actual live APIs + # For now, return cached data as proxy for live data + events = await self._get_cached_data(data_type, count, requirements) + + # Simulate API cost and latency + latency = 500 # 500ms typical API latency + cost = count * 0.01 # $0.01 per event + + await self._update_source_metrics( + DataSource.LIVE_API, + requests=1, + cost=cost, + latency=latency, + success=True + ) + + return events + + except Exception as e: + await self._update_source_metrics(DataSource.LIVE_API, requests=1, success=False) + self.logger.error(f"Failed to get live API data: {e}") + raise + + async def _get_cached_data( + self, + data_type: str, + count: int, + requirements: Optional[Dict[str, Any]] + ) -> List[StandardizedEvent]: + """Get cached training data""" + try: + # Check cache freshness + cache_key = f"{data_type}_{count}_{hash(str(requirements))}" + + if (cache_key in self.data_cache and + cache_key in self.cache_timestamps and + datetime.now() - self.cache_timestamps[cache_key] < timedelta(hours=self.config.cache_duration_hours)): + + cached_events = self.data_cache[cache_key] + await self._update_source_metrics(DataSource.CACHED, requests=1, cost=0.0, latency=10, success=True) + return cached_events + + # If no cached data, generate synthetic data and cache it + events = await self._get_synthetic_data(data_type, count, requirements) + + # Cache the results + if len(self.data_cache) < self.config.cache_size_limit: + self.data_cache[cache_key] = events + self.cache_timestamps[cache_key] = datetime.now() + + return events + + except Exception as e: + await self._update_source_metrics(DataSource.CACHED, requests=1, success=False) + self.logger.error(f"Failed to get cached data: {e}") + raise + + async def _convert_scenario_to_events(self, scenario) -> List[StandardizedEvent]: + """Convert training scenario to standardized events""" + try: + events = [] + + # Extract game events from scenario + if hasattr(scenario, 'game') and scenario.game: + for play in scenario.game.plays: + event = StandardizedEvent( + event_id=f"synth_{play.play_id}", + game_id=scenario.game.game_id, + timestamp=play.timestamp, + event_type=play.play_type, + description=play.description, + team_possession=play.team_possession, + score_home=play.score_home, + score_away=play.score_away, + quarter=play.quarter, + time_remaining=play.time_remaining, + down=getattr(play, 'down', None), + yards_to_go=getattr(play, 'yards_to_go', None), + yard_line=getattr(play, 'yard_line', None), + metadata={ + "synthetic": True, + "scenario_id": scenario.scenario_id, + "source": "hybrid_orchestrator" + } + ) + events.append(event) + + return events + + except Exception as e: + self.logger.error(f"Failed to convert scenario to events: {e}") + return [] + + async def _convert_market_scenarios_to_events(self, scenarios) -> List[StandardizedEvent]: + """Convert market scenarios to standardized events""" + try: + events = [] + + for scenario in scenarios: + # Convert market events to standardized format + for market_event in scenario.events: + event = StandardizedEvent( + event_id=f"market_{market_event.event_id}", + game_id=scenario.market_ticker, + timestamp=market_event.timestamp, + event_type="market_update", + description=f"Market price update: {market_event.price_change}", + metadata={ + "synthetic": True, + "scenario_id": scenario.scenario_id, + "price_change": market_event.price_change, + "volume": getattr(market_event, 'volume', 0), + "source": "hybrid_orchestrator" + } + ) + events.append(event) + + return events + + except Exception as e: + self.logger.error(f"Failed to convert market scenarios to events: {e}") + return [] + + async def _update_source_metrics( + self, + source: DataSource, + requests: int = 0, + cost: float = 0.0, + latency: float = 0.0, + success: bool = True + ) -> None: + """Update metrics for a data source""" + try: + metrics = self.source_metrics[source] + + metrics.requests_made += requests + metrics.cost_incurred += cost + + if latency > 0: + # Exponential moving average for latency + alpha = 0.1 + metrics.latency_ms = (1 - alpha) * metrics.latency_ms + alpha * latency + + # Update error rate + if requests > 0: + alpha = 0.1 + new_error_rate = 0.0 if success else 1.0 + metrics.error_rate = (1 - alpha) * metrics.error_rate + alpha * new_error_rate + + # Update daily costs + self.daily_costs += cost + + metrics.last_used = datetime.now() + + except Exception as e: + self.logger.error(f"Failed to update source metrics: {e}") + + async def _verify_synthetic_systems(self) -> None: + """Verify that synthetic data systems are available""" + try: + # Test synthetic game engine + test_game = await self.synthetic_engine.generate_single_game() + if not test_game or not test_game.plays: + raise Exception("Synthetic game engine not working") + + # Test scenario builder + test_scenarios = await self.scenario_builder.build_comprehensive_training_set(num_scenarios=1) + if not test_scenarios or not test_scenarios.scenarios: + raise Exception("Scenario builder not working") + + self.logger.info("Synthetic systems verified successfully") + + except Exception as e: + self.logger.error(f"Synthetic systems verification failed: {e}") + # Mark synthetic as unavailable + self.source_metrics[DataSource.SYNTHETIC].availability = False + raise + + def _check_daily_cost_reset(self) -> None: + """Check if daily costs should be reset""" + today = datetime.now().date() + if today > self.last_cost_reset: + self.daily_costs = 0.0 + self.last_cost_reset = today + self.logger.info("Daily costs reset") + + async def _load_switching_history(self) -> None: + """Load switching history from persistent storage""" + try: + # In real implementation, would load from database + # For now, initialize empty history + self.switching_history = [] + + except Exception as e: + self.logger.error(f"Failed to load switching history: {e}") + self.switching_history = [] + + async def _select_optimal_data_source(self) -> None: + """Select optimal initial data source""" + try: + # For training mode, prefer synthetic data + if self.current_mode == DataMode.TRAINING: + self.active_source = DataSource.SYNTHETIC + # For live trading, prefer live API + elif self.current_mode == DataMode.LIVE_TRADING: + self.active_source = DataSource.LIVE_API + # For backtesting, prefer cached data + elif self.current_mode == DataMode.BACKTESTING: + self.active_source = DataSource.CACHED + else: + # Default to configured primary source + self.active_source = self.config.primary_source + + self.logger.info(f"Selected {self.active_source.value} as optimal data source for {self.current_mode.value} mode") + + except Exception as e: + self.logger.error(f"Failed to select optimal data source: {e}") + self.active_source = DataSource.SYNTHETIC # Safe fallback + + async def get_cost_summary(self) -> Dict[str, Any]: + """Get comprehensive cost summary""" + try: + return { + "daily_costs": self.daily_costs, + "daily_limit": self.config.cost_thresholds.daily_limit, + "utilization_percent": (self.daily_costs / self.config.cost_thresholds.daily_limit) * 100, + "source_breakdown": { + source.value: { + "requests": metrics.requests_made, + "cost": metrics.cost_incurred, + "avg_latency": metrics.latency_ms, + "error_rate": metrics.error_rate, + "last_used": metrics.last_used.isoformat() if metrics.last_used else None + } + for source, metrics in self.source_metrics.items() + }, + "active_source": self.active_source.value, + "switching_history": self.switching_history[-10:] # Last 10 switches + } + + except Exception as e: + self.logger.error(f"Failed to generate cost summary: {e}") + return {"error": str(e)} + + async def force_data_source(self, source: DataSource) -> None: + """Manually force a specific data source""" + try: + old_source = self.active_source + self.active_source = source + + # Record manual switch + switch_record = { + "timestamp": datetime.now().isoformat(), + "from_source": old_source.value, + "to_source": source.value, + "reason": "manual_override", + "daily_cost": self.daily_costs + } + self.switching_history.append(switch_record) + + self.logger.info(f"Manually switched data source to {source.value}") + + except Exception as e: + self.logger.error(f"Failed to force data source: {e}") + raise + + async def shutdown(self) -> None: + """Gracefully shutdown the hybrid orchestrator""" + try: + self.logger.info("Shutting down Hybrid Data Orchestrator") + + # Save switching history + await self._save_switching_history() + + # Clear cache + self.data_cache.clear() + self.cache_timestamps.clear() + + self.logger.info("Hybrid Data Orchestrator shutdown complete") + + except Exception as e: + self.logger.error(f"Error during shutdown: {e}") + + async def _save_switching_history(self) -> None: + """Save switching history to persistent storage""" + try: + # In real implementation, would save to database + # For now, just log summary + self.logger.info(f"Saving switching history: {len(self.switching_history)} records") + + except Exception as e: + self.logger.error(f"Failed to save switching history: {e}") \ No newline at end of file diff --git a/src/integration/__init__.py b/src/integration/__init__.py new file mode 100644 index 00000000..2ed778d1 --- /dev/null +++ b/src/integration/__init__.py @@ -0,0 +1,52 @@ +""" +Integration Module + +Bridges synthetic data generation and training systems with +the existing Redis-based agent infrastructure. +""" + +from .training_bridge import ( + TrainingBridge, + TrainingMode, + TrainingSession, + TrainingConfig +) +from .synthetic_injector import ( + SyntheticDataInjector, + InjectionConfig, + EventTiming +) +from .training_harness import ( + AgentTrainingHarness, + TrainingScenario, + HarnessConfig +) +# from .decision_tracker import ( +# DecisionTracker, +# DecisionRecord, +# TrackingConfig +# ) +# from .training_controller import ( +# TrainingModeController, +# ModeConfig, +# DataSourceMode +# ) + +__all__ = [ + 'TrainingBridge', + 'TrainingMode', + 'TrainingSession', + 'TrainingConfig', + 'SyntheticDataInjector', + 'InjectionConfig', + 'EventTiming', + 'AgentTrainingHarness', + 'TrainingScenario', + 'HarnessConfig', + 'DecisionTracker', + 'DecisionRecord', + 'TrackingConfig', + 'TrainingModeController', + 'ModeConfig', + 'DataSourceMode' +] \ No newline at end of file diff --git a/src/integration/synthetic_injector.py b/src/integration/synthetic_injector.py new file mode 100644 index 00000000..5c772087 --- /dev/null +++ b/src/integration/synthetic_injector.py @@ -0,0 +1,688 @@ +""" +Synthetic Data Injector + +Injects synthetic training data into Redis channels with realistic timing, +sequencing, and market dynamics for agent training. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Any, Tuple +from datetime import datetime, timedelta +from enum import Enum +import asyncio +import logging +import json +import redis.asyncio as redis +import random +import numpy as np + +from ..sdk.core.base_adapter import StandardizedEvent +from ..synthetic_data.generators.game_engine import SyntheticGame +from ..synthetic_data.generators.market_simulator import TradingScenario, MarketEvent + + +class EventTiming(Enum): + """Event timing strategies""" + REALTIME = "realtime" # Real-world timing + ACCELERATED = "accelerated" # Faster than realtime + BURST = "burst" # All at once + ADAPTIVE = "adaptive" # Adjust based on agent response + + +@dataclass +class InjectionConfig: + """Configuration for data injection""" + timing_mode: EventTiming = EventTiming.ACCELERATED + acceleration_factor: float = 10.0 # 10x speed + + # Realistic delays (in seconds) + min_event_delay: float = 0.05 # 50ms minimum + max_event_delay: float = 5.0 # 5s maximum + market_update_frequency: float = 0.5 # Market updates every 500ms + + # Noise and realism + add_timing_jitter: bool = True + jitter_factor: float = 0.2 # ±20% timing variation + add_missing_data: bool = True # Simulate data gaps + missing_data_probability: float = 0.02 # 2% chance of missing data + + # Market dynamics + add_market_noise: bool = True + price_noise_stddev: float = 0.01 # 1% price noise + volume_multiplier_range: Tuple[float, float] = (0.8, 1.2) + + # Redis configuration + redis_url: str = "redis://localhost:6379" + channel_prefix: str = "" # Empty for production channels, "training:" for isolated + batch_size: int = 100 # Events to buffer before publishing + + # Performance + max_concurrent_injections: int = 10 + enable_backpressure: bool = True + backpressure_threshold: int = 1000 # Pause if this many events pending + + +@dataclass +class InjectionMetrics: + """Metrics for injection performance""" + events_injected: int = 0 + events_failed: int = 0 + total_latency: float = 0.0 + min_latency: float = float('inf') + max_latency: float = 0.0 + start_time: datetime = field(default_factory=datetime.now) + + @property + def avg_latency(self) -> float: + return self.total_latency / self.events_injected if self.events_injected > 0 else 0.0 + + @property + def success_rate(self) -> float: + total = self.events_injected + self.events_failed + return self.events_injected / total if total > 0 else 0.0 + + @property + def events_per_second(self) -> float: + elapsed = (datetime.now() - self.start_time).total_seconds() + return self.events_injected / elapsed if elapsed > 0 else 0.0 + + +class SyntheticDataInjector: + """ + Injects synthetic data into Redis channels for agent training. + + Handles timing, sequencing, and realistic market dynamics to create + believable training scenarios. + """ + + def __init__(self, config: InjectionConfig = None): + self.config = config or InjectionConfig() + self.logger = logging.getLogger(__name__) + + # Redis connections + self.redis_client: Optional[redis.Redis] = None + self.publisher: Optional[redis.Redis] = None + + # Injection state + self.is_injecting = False + self.injection_tasks: List[asyncio.Task] = [] + self.event_queue: asyncio.Queue = asyncio.Queue() + self.pending_events = 0 + + # Metrics + self.metrics = InjectionMetrics() + + # Channel mappings + self.channel_map = { + "game_event": "espn:games", + "market_update": "kalshi:markets", + "trade": "kalshi:trades", + "signal": "kalshi:signals", + "sentiment": "twitter:sentiment", + "orderbook": "kalshi:orderbook" + } + + # Market state tracking (for realistic updates) + self.market_states: Dict[str, Dict[str, float]] = {} + + async def initialize(self) -> None: + """Initialize the injector""" + try: + self.logger.info("Initializing Synthetic Data Injector") + + # Connect to Redis + self.redis_client = redis.from_url(self.config.redis_url) + self.publisher = redis.from_url(self.config.redis_url) + + # Test connection + await self.redis_client.ping() + + # Start background publisher + self.injection_tasks.append( + asyncio.create_task(self._publisher_loop()) + ) + + self.logger.info("Synthetic Data Injector initialized") + + except Exception as e: + self.logger.error(f"Failed to initialize injector: {e}") + raise + + async def inject_game_scenario( + self, + game: SyntheticGame, + market_ticker: Optional[str] = None, + start_immediately: bool = True + ) -> str: + """ + Inject a complete game scenario with synchronized market updates. + + Args: + game: Synthetic game to inject + market_ticker: Associated market ticker for price updates + start_immediately: Whether to start injection immediately + + Returns: + Injection ID for tracking + """ + try: + injection_id = f"game_{game.game_id}_{datetime.now().timestamp()}" + + # Convert game to events + events = self._game_to_events(game, market_ticker) + + # Calculate timing + event_timings = self._calculate_event_timings(events) + + # Create injection task + if start_immediately: + task = asyncio.create_task( + self._inject_event_sequence(injection_id, events, event_timings) + ) + self.injection_tasks.append(task) + else: + # Queue for later injection + for event, timing in zip(events, event_timings): + await self.event_queue.put((event, timing)) + + self.logger.info(f"Injecting game scenario {injection_id} with {len(events)} events") + return injection_id + + except Exception as e: + self.logger.error(f"Failed to inject game scenario: {e}") + raise + + async def inject_trading_scenario( + self, + scenario: TradingScenario, + include_orderbook: bool = True + ) -> str: + """ + Inject a trading scenario with market events and price updates. + + Args: + scenario: Trading scenario to inject + include_orderbook: Whether to include orderbook updates + + Returns: + Injection ID for tracking + """ + try: + injection_id = f"trading_{scenario.scenario_id}_{datetime.now().timestamp()}" + + # Convert scenario to events + events = self._trading_scenario_to_events(scenario, include_orderbook) + + # Calculate timing with market update frequency + event_timings = self._calculate_market_timings(events) + + # Start injection + task = asyncio.create_task( + self._inject_event_sequence(injection_id, events, event_timings) + ) + self.injection_tasks.append(task) + + self.logger.info(f"Injecting trading scenario {injection_id} with {len(events)} events") + return injection_id + + except Exception as e: + self.logger.error(f"Failed to inject trading scenario: {e}") + raise + + def _game_to_events(self, game: SyntheticGame, market_ticker: Optional[str]) -> List[Dict[str, Any]]: + """Convert a synthetic game to injectable events""" + events = [] + + for play in game.plays: + # Game event + event = { + "type": "game_event", + "event_id": f"play_{play.play_id}", + "game_id": game.game_id, + "timestamp": play.timestamp, + "data": { + "play_type": play.play_type, + "description": play.description, + "team_possession": play.team_possession, + "score_home": play.score_home, + "score_away": play.score_away, + "quarter": play.quarter, + "time_remaining": play.time_remaining, + "yards_gained": getattr(play, 'yards_gained', 0), + "down": getattr(play, 'down', None), + "yards_to_go": getattr(play, 'yards_to_go', None) + } + } + events.append(event) + + # Add correlated market update if ticker provided + if market_ticker and self._is_significant_play(play): + market_event = self._generate_market_reaction(play, market_ticker) + events.append(market_event) + + return events + + def _is_significant_play(self, play) -> bool: + """Determine if a play should trigger market movement""" + significant_types = ["touchdown", "field_goal", "interception", "fumble", "injury"] + return any(sig in play.play_type.lower() for sig in significant_types) + + def _generate_market_reaction(self, play, market_ticker: str) -> Dict[str, Any]: + """Generate a market price reaction to a game event""" + # Get current market state or initialize + if market_ticker not in self.market_states: + self.market_states[market_ticker] = { + "yes_price": 0.5, + "no_price": 0.5, + "volume": 1000 + } + + current_state = self.market_states[market_ticker] + + # Calculate price impact based on play type + impact = 0.0 + if "touchdown" in play.play_type.lower(): + # Touchdown by home team increases yes price + # Assuming home team is the team we're tracking for "YES" price + if hasattr(play, 'team_possession'): + # If possession matches home indication, positive impact + impact = random.uniform(0.02, 0.05) if "home" in play.team_possession.lower() else random.uniform(-0.05, -0.02) + else: + impact = random.uniform(-0.02, 0.02) # Random if unknown + elif "field_goal" in play.play_type.lower(): + if hasattr(play, 'team_possession'): + impact = random.uniform(-0.01, 0.01) if "home" in play.team_possession.lower() else random.uniform(-0.02, 0.0) + else: + impact = random.uniform(-0.015, 0.005) + elif "interception" in play.play_type.lower() or "fumble" in play.play_type.lower(): + if hasattr(play, 'team_possession'): + impact = random.uniform(-0.03, -0.01) if "home" in play.team_possession.lower() else random.uniform(0.01, 0.03) + else: + impact = random.uniform(-0.01, 0.01) + + # Apply impact with noise + if self.config.add_market_noise: + noise = np.random.normal(0, self.config.price_noise_stddev) + impact += noise + + # Update prices + new_yes_price = max(0.01, min(0.99, current_state["yes_price"] + impact)) + new_no_price = 1.0 - new_yes_price + + # Update volume + volume_multiplier = random.uniform(*self.config.volume_multiplier_range) + new_volume = current_state["volume"] * volume_multiplier + + # Store new state + self.market_states[market_ticker] = { + "yes_price": new_yes_price, + "no_price": new_no_price, + "volume": new_volume + } + + return { + "type": "market_update", + "event_id": f"market_{play.play_id}", + "market_ticker": market_ticker, + "timestamp": play.timestamp + timedelta(seconds=random.uniform(0.1, 0.5)), + "data": { + "yes_price": new_yes_price, + "no_price": new_no_price, + "yes_bid": new_yes_price - 0.01, + "yes_ask": new_yes_price + 0.01, + "no_bid": new_no_price - 0.01, + "no_ask": new_no_price + 0.01, + "volume": new_volume, + "trigger_event": play.play_id + } + } + + def _trading_scenario_to_events(self, scenario: TradingScenario, include_orderbook: bool) -> List[Dict[str, Any]]: + """Convert trading scenario to injectable events""" + events = [] + + for market_event in scenario.events: + # Market price update + event = { + "type": "market_update", + "event_id": market_event.event_id, + "market_ticker": scenario.market_ticker, + "timestamp": market_event.timestamp, + "data": { + "event_type": market_event.event_type, + "yes_price": market_event.price, + "no_price": 1.0 - market_event.price, + "volume": market_event.volume, + "price_change": market_event.price_change, + "information_content": market_event.information_value + } + } + events.append(event) + + # Add orderbook update if requested + if include_orderbook and random.random() < 0.3: # 30% chance + orderbook_event = self._generate_orderbook_update(scenario.market_ticker, market_event) + events.append(orderbook_event) + + return events + + def _generate_orderbook_update(self, market_ticker: str, market_event: MarketEvent) -> Dict[str, Any]: + """Generate a realistic orderbook update""" + base_price = market_event.price + + # Generate bid/ask levels + levels = 5 + bids = [] + asks = [] + + for i in range(levels): + spread = 0.01 * (i + 1) + bid_price = base_price - spread + ask_price = base_price + spread + + # Volume decreases with distance from mid + volume = market_event.volume * (1.0 / (i + 1)) + + if bid_price > 0: + bids.append({ + "price": bid_price, + "quantity": int(volume * random.uniform(0.8, 1.2)) + }) + + if ask_price < 1: + asks.append({ + "price": ask_price, + "quantity": int(volume * random.uniform(0.8, 1.2)) + }) + + return { + "type": "orderbook", + "event_id": f"orderbook_{market_event.event_id}", + "market_ticker": market_ticker, + "timestamp": market_event.timestamp + timedelta(milliseconds=50), + "data": { + "bids": bids, + "asks": asks, + "mid_price": base_price, + "spread": asks[0]["price"] - bids[0]["price"] if bids and asks else 0.02 + } + } + + def _calculate_event_timings(self, events: List[Dict[str, Any]]) -> List[float]: + """Calculate realistic timing delays for events""" + timings = [] + + for i, event in enumerate(events): + if self.config.timing_mode == EventTiming.BURST: + delay = 0.0 # No delay + elif self.config.timing_mode == EventTiming.REALTIME: + # Calculate actual time difference + if i > 0: + prev_timestamp = events[i-1].get("timestamp", datetime.now()) + curr_timestamp = event.get("timestamp", datetime.now()) + + if isinstance(prev_timestamp, str): + prev_timestamp = datetime.fromisoformat(prev_timestamp) + if isinstance(curr_timestamp, str): + curr_timestamp = datetime.fromisoformat(curr_timestamp) + + delay = (curr_timestamp - prev_timestamp).total_seconds() + else: + delay = 0.0 + elif self.config.timing_mode == EventTiming.ACCELERATED: + # Base delay with acceleration + base_delay = random.uniform(self.config.min_event_delay, self.config.max_event_delay) + delay = base_delay / self.config.acceleration_factor + else: # ADAPTIVE + # Adjust based on event type and importance + if event["type"] == "market_update": + delay = self.config.market_update_frequency / self.config.acceleration_factor + else: + delay = random.uniform(0.1, 1.0) / self.config.acceleration_factor + + # Add jitter if configured + if self.config.add_timing_jitter: + jitter = random.gauss(0, delay * self.config.jitter_factor) if delay > 0 else 0 + delay = max(0.001, delay + jitter) # Ensure positive + + timings.append(delay) + + return timings + + def _calculate_market_timings(self, events: List[Dict[str, Any]]) -> List[float]: + """Calculate timing specifically for market events""" + timings = [] + + for event in events: + if event["type"] == "market_update": + # Regular market updates + delay = self.config.market_update_frequency / self.config.acceleration_factor + elif event["type"] == "orderbook": + # Orderbook updates are more frequent + delay = (self.config.market_update_frequency * 0.5) / self.config.acceleration_factor + else: + # Other events + delay = random.uniform(0.5, 2.0) / self.config.acceleration_factor + + # Add realistic variation + if self.config.add_timing_jitter: + delay *= random.uniform(0.8, 1.2) + + timings.append(delay) + + return timings + + async def _inject_event_sequence( + self, + injection_id: str, + events: List[Dict[str, Any]], + timings: List[float] + ) -> None: + """Inject a sequence of events with specified timing""" + try: + self.is_injecting = True + + for event, delay in zip(events, timings): + # Check backpressure + if self.config.enable_backpressure and self.pending_events > self.config.backpressure_threshold: + await self._wait_for_backpressure() + + # Wait for timing + if delay > 0: + await asyncio.sleep(delay) + + # Simulate missing data if configured + if self.config.add_missing_data and random.random() < self.config.missing_data_probability: + self.logger.debug(f"Simulating missing data for event {event.get('event_id')}") + continue + + # Queue event for publishing + await self.event_queue.put((event, injection_id)) + self.pending_events += 1 + + self.logger.info(f"Completed injection sequence {injection_id}") + + except Exception as e: + self.logger.error(f"Failed to inject event sequence: {e}") + self.metrics.events_failed += len(events) + finally: + self.is_injecting = False + + async def _wait_for_backpressure(self) -> None: + """Wait for backpressure to clear""" + self.logger.debug("Backpressure detected, waiting...") + while self.pending_events > self.config.backpressure_threshold * 0.5: + await asyncio.sleep(0.1) + + async def _publisher_loop(self) -> None: + """Background loop for publishing queued events""" + batch = [] + + while True: + try: + # Get event from queue with timeout + try: + event, injection_id = await asyncio.wait_for( + self.event_queue.get(), timeout=1.0 + ) + batch.append((event, injection_id)) + except asyncio.TimeoutError: + # Flush batch if we have events + if batch: + await self._publish_batch(batch) + batch = [] + continue + + # Publish batch if full + if len(batch) >= self.config.batch_size: + await self._publish_batch(batch) + batch = [] + + except Exception as e: + self.logger.error(f"Publisher loop error: {e}") + await asyncio.sleep(1) + + async def _publish_batch(self, batch: List[Tuple[Dict[str, Any], str]]) -> None: + """Publish a batch of events to Redis""" + try: + start_time = datetime.now() + + for event, injection_id in batch: + channel = self._get_channel_for_event(event) + + # Add channel prefix if configured + if self.config.channel_prefix: + channel = f"{self.config.channel_prefix}{channel}" + + # Prepare message + message = { + "timestamp": event["timestamp"].isoformat() if hasattr(event["timestamp"], 'isoformat') else str(event["timestamp"]), + "source": "synthetic_injector", + "injection_id": injection_id, + "type": event["type"], + "data": event["data"] + } + + # Add event-specific fields + if "event_id" in event: + message["event_id"] = event["event_id"] + if "game_id" in event: + message["game_id"] = event["game_id"] + if "market_ticker" in event: + message["market_ticker"] = event["market_ticker"] + + # Publish to Redis + await self.publisher.publish(channel, json.dumps(message)) + + # Update metrics + self.metrics.events_injected += 1 + self.pending_events -= 1 + + # Update latency metrics + latency = (datetime.now() - start_time).total_seconds() + self.metrics.total_latency += latency + self.metrics.min_latency = min(self.metrics.min_latency, latency) + self.metrics.max_latency = max(self.metrics.max_latency, latency) + + except Exception as e: + self.logger.error(f"Failed to publish batch: {e}") + self.metrics.events_failed += len(batch) + self.pending_events -= len(batch) + + def _get_channel_for_event(self, event: Dict[str, Any]) -> str: + """Get the appropriate Redis channel for an event type""" + event_type = event.get("type", "unknown") + return self.channel_map.get(event_type, "kalshi:unknown") + + async def inject_burst(self, events: List[StandardizedEvent]) -> str: + """ + Inject a burst of events as quickly as possible. + + Args: + events: List of standardized events to inject + + Returns: + Injection ID + """ + try: + injection_id = f"burst_{datetime.now().timestamp()}" + + # Convert to injectable format + injectable_events = [] + for event in events: + injectable = { + "type": self._event_type_to_channel_type(event.event_type), + "event_id": event.event_id, + "timestamp": event.timestamp, + "data": { + "description": event.description, + "metadata": event.metadata + } + } + + if event.game_id: + injectable["game_id"] = event.game_id + + injectable_events.append(injectable) + + # Queue all events immediately + for event in injectable_events: + await self.event_queue.put((event, injection_id)) + self.pending_events += 1 + + self.logger.info(f"Queued burst injection {injection_id} with {len(events)} events") + return injection_id + + except Exception as e: + self.logger.error(f"Failed to inject burst: {e}") + raise + + def _event_type_to_channel_type(self, event_type: str) -> str: + """Convert StandardizedEvent type to channel type""" + mapping = { + "score_update": "game_event", + "big_play": "game_event", + "market_update": "market_update", + "trade": "trade", + "signal": "signal" + } + return mapping.get(event_type, "game_event") + + def get_metrics(self) -> Dict[str, Any]: + """Get injection metrics""" + return { + "events_injected": self.metrics.events_injected, + "events_failed": self.metrics.events_failed, + "success_rate": f"{self.metrics.success_rate:.2%}", + "avg_latency_ms": self.metrics.avg_latency * 1000, + "min_latency_ms": self.metrics.min_latency * 1000 if self.metrics.min_latency != float('inf') else 0, + "max_latency_ms": self.metrics.max_latency * 1000, + "events_per_second": self.metrics.events_per_second, + "pending_events": self.pending_events, + "is_injecting": self.is_injecting + } + + async def shutdown(self) -> None: + """Shutdown the injector""" + try: + self.logger.info("Shutting down Synthetic Data Injector") + + # Cancel injection tasks + for task in self.injection_tasks: + if not task.done(): + task.cancel() + + # Wait for tasks to complete + if self.injection_tasks: + await asyncio.gather(*self.injection_tasks, return_exceptions=True) + + # Close Redis connections + if self.redis_client: + await self.redis_client.close() + if self.publisher: + await self.publisher.close() + + self.logger.info(f"Injector shutdown complete. Final metrics: {self.get_metrics()}") + + except Exception as e: + self.logger.error(f"Error during shutdown: {e}") \ No newline at end of file diff --git a/src/integration/training_bridge.py b/src/integration/training_bridge.py new file mode 100644 index 00000000..c111d21a --- /dev/null +++ b/src/integration/training_bridge.py @@ -0,0 +1,691 @@ +""" +Training Bridge Module + +Bridges synthetic data pipeline with Redis-based agent infrastructure, +enabling seamless training with generated scenarios. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Any, Callable +from datetime import datetime, timedelta +from enum import Enum +import asyncio +import logging +import json +import redis.asyncio as redis +from collections import defaultdict + +from ..sdk.core.base_adapter import StandardizedEvent +from ..synthetic_data.generators.scenario_builder import TrainingScenarioSet +from ..synthetic_data.generators.market_simulator import TradingScenario +from ..training.agent_analytics import AgentAnalytics, DecisionMetrics +from ..training.memory_system import AgentMemorySystem +from ..confidence_calibration.calibrator import ConfidenceCalibrator +from ..hybrid_pipeline.data_orchestrator import HybridDataOrchestrator + + +class TrainingMode(Enum): + """Training modes for agents""" + EXPLORATION = "exploration" # High randomness, learning new patterns + EXPLOITATION = "exploitation" # Low randomness, refining strategies + VALIDATION = "validation" # No randomness, testing performance + PRODUCTION_PREP = "production_prep" # Simulate production conditions + + +@dataclass +class TrainingConfig: + """Configuration for training sessions""" + mode: TrainingMode = TrainingMode.EXPLORATION + time_acceleration: float = 10.0 # 10x speed + inject_noise: bool = True # Add realistic noise to data + noise_level: float = 0.05 # 5% noise + track_decisions: bool = True + update_memory: bool = True + calibrate_confidence: bool = True + + # Redis configuration + redis_url: str = "redis://localhost:6379" + training_namespace: str = "training" # Prefix for training channels + + # Performance thresholds + min_confidence_threshold: float = 0.3 + max_position_size: float = 0.1 # 10% of capital + stop_loss_threshold: float = 0.2 # 20% drawdown stops training + + # Scenario preferences + edge_case_probability: float = 0.2 + market_volatility_multiplier: float = 1.0 + + +@dataclass +class TrainingSession: + """Active training session for an agent""" + session_id: str + agent_id: str + start_time: datetime + config: TrainingConfig + scenarios_completed: int = 0 + decisions_made: int = 0 + total_pnl: float = 0.0 + errors_encountered: int = 0 + end_time: Optional[datetime] = None + + @property + def duration(self) -> timedelta: + end = self.end_time or datetime.now() + return end - self.start_time + + @property + def is_active(self) -> bool: + return self.end_time is None + + +class TrainingBridge: + """ + Central bridge between synthetic data generation and agent training. + + Coordinates data flow from synthetic generators through Redis to agents, + while tracking performance and updating learning systems. + """ + + def __init__( + self, + orchestrator: HybridDataOrchestrator, + analytics: AgentAnalytics, + memory_system: AgentMemorySystem, + calibrator: ConfidenceCalibrator, + config: TrainingConfig = None + ): + self.orchestrator = orchestrator + self.analytics = analytics + self.memory_system = memory_system + self.calibrator = calibrator + self.config = config or TrainingConfig() + self.logger = logging.getLogger(__name__) + + # Redis connections + self.redis_client: Optional[redis.Redis] = None + self.publisher: Optional[redis.Redis] = None + + # Active training sessions + self.active_sessions: Dict[str, TrainingSession] = {} + + # Performance tracking + self.agent_metrics: Dict[str, Dict[str, float]] = defaultdict(dict) + + # Event routing + self.channel_mappings = { + "game_event": "espn:games", + "market_update": "kalshi:markets", + "trade_executed": "kalshi:trades", + "signal_generated": "kalshi:signals", + "sentiment_update": "twitter:sentiment" + } + + # Callbacks for agent responses + self.response_handlers: Dict[str, Callable] = {} + + async def initialize(self) -> None: + """Initialize the training bridge""" + try: + self.logger.info("Initializing Training Bridge") + + # Connect to Redis + await self._connect_redis() + + # Initialize orchestrator + await self.orchestrator.initialize() + + # Set up response listeners + await self._setup_response_listeners() + + self.logger.info("Training Bridge initialized successfully") + + except Exception as e: + self.logger.error(f"Failed to initialize training bridge: {e}") + raise + + async def _connect_redis(self) -> None: + """Connect to Redis for pub/sub operations""" + try: + self.redis_client = redis.from_url(self.config.redis_url) + self.publisher = redis.from_url(self.config.redis_url) + + # Test connection + await self.redis_client.ping() + + self.logger.info("Connected to Redis for training bridge") + + except Exception as e: + self.logger.error(f"Failed to connect to Redis: {e}") + raise + + async def _setup_response_listeners(self) -> None: + """Set up listeners for agent responses during training""" + try: + # Subscribe to agent response channels + pubsub = self.redis_client.pubsub() + + response_channels = [ + f"{self.config.training_namespace}:agent_decisions", + f"{self.config.training_namespace}:agent_signals", + f"{self.config.training_namespace}:agent_trades" + ] + + await pubsub.subscribe(*response_channels) + + # Start listening in background + asyncio.create_task(self._listen_for_responses(pubsub)) + + self.logger.info(f"Listening for agent responses on {response_channels}") + + except Exception as e: + self.logger.error(f"Failed to setup response listeners: {e}") + + async def _listen_for_responses(self, pubsub) -> None: + """Listen for agent responses and process them""" + try: + async for message in pubsub.listen(): + if message['type'] not in ('message', 'pmessage'): + continue + + try: + channel = message['channel'].decode('utf-8') + data = json.loads(message['data']) + + # Extract agent ID and process response + agent_id = data.get('agent_id') + if agent_id and agent_id in self.active_sessions: + await self._process_agent_response(agent_id, channel, data) + + except Exception as e: + self.logger.error(f"Error processing response: {e}") + + except Exception as e: + self.logger.error(f"Response listener error: {e}") + + async def _process_agent_response(self, agent_id: str, channel: str, data: Dict[str, Any]) -> None: + """Process an agent's response during training""" + try: + session = self.active_sessions.get(agent_id) + if not session: + return + + # Update session statistics + session.decisions_made += 1 + + # Extract decision details + if 'agent_decisions' in channel: + await self._track_decision(agent_id, data) + elif 'agent_trades' in channel: + await self._track_trade(agent_id, data) + elif 'agent_signals' in channel: + await self._track_signal(agent_id, data) + + # Update metrics + if 'pnl' in data: + session.total_pnl += data['pnl'] + + # Check for training termination conditions + if await self._should_stop_training(session): + await self.stop_training_session(agent_id) + + except Exception as e: + self.logger.error(f"Failed to process agent response: {e}") + session.errors_encountered += 1 + + async def _track_decision(self, agent_id: str, decision_data: Dict[str, Any]) -> None: + """Track a decision made by an agent during training""" + try: + # Create decision metrics + decision = DecisionMetrics( + decision_id=decision_data.get('decision_id', f"{agent_id}_{datetime.now().timestamp()}"), + agent_id=agent_id, + scenario_id=decision_data.get('scenario_id', 'unknown'), + timestamp=datetime.now(), + market_ticker=decision_data.get('market_ticker', ''), + decision_type=decision_data.get('decision_type', 'hold'), + confidence=decision_data.get('confidence', 0.5), + expected_value=decision_data.get('expected_value', 0.0), + kelly_fraction=decision_data.get('kelly_fraction', 0.0), + actual_kelly_used=decision_data.get('actual_kelly_used', 0.0), + position_size=decision_data.get('position_size', 0.0), + market_efficiency=decision_data.get('market_efficiency', 0.8), + information_advantage=decision_data.get('information_advantage', 0.0), + execution_latency=decision_data.get('latency', 0.0) + ) + + # Record in analytics + await self.analytics.record_decision(decision) + + # Update memory if configured + if self.config.update_memory: + await self.memory_system.store_agent_experience( + agent_id=agent_id, + scenario_id=decision.scenario_id, + action={ + 'type': decision.decision_type, + 'confidence': decision.confidence, + 'position_size': decision.position_size + }, + outcome={'pending': True}, # Will be updated later + context=decision_data.get('context', {}) + ) + + # Calibrate confidence if configured + if self.config.calibrate_confidence: + calibrated_score = await self.calibrator.calibrate_confidence( + agent_id=agent_id, + raw_confidence=decision.confidence, + context=decision_data.get('context', {}) + ) + + # Store calibration result + self.agent_metrics[agent_id]['calibrated_confidence'] = calibrated_score.calibrated_confidence + self.agent_metrics[agent_id]['confidence_uncertainty'] = calibrated_score.uncertainty + + except Exception as e: + self.logger.error(f"Failed to track decision: {e}") + + async def _track_trade(self, agent_id: str, trade_data: Dict[str, Any]) -> None: + """Track a trade execution during training""" + try: + # Update agent metrics + self.agent_metrics[agent_id]['trades_executed'] = \ + self.agent_metrics[agent_id].get('trades_executed', 0) + 1 + + # Track trade outcome + if trade_data.get('status') == 'FILLED': + self.agent_metrics[agent_id]['successful_trades'] = \ + self.agent_metrics[agent_id].get('successful_trades', 0) + 1 + + except Exception as e: + self.logger.error(f"Failed to track trade: {e}") + + async def _track_signal(self, agent_id: str, signal_data: Dict[str, Any]) -> None: + """Track a signal generated during training""" + try: + # Update agent metrics + self.agent_metrics[agent_id]['signals_generated'] = \ + self.agent_metrics[agent_id].get('signals_generated', 0) + 1 + + # Track signal quality + confidence = signal_data.get('confidence', 0.5) + if confidence > 0.7: + self.agent_metrics[agent_id]['high_confidence_signals'] = \ + self.agent_metrics[agent_id].get('high_confidence_signals', 0) + 1 + + except Exception as e: + self.logger.error(f"Failed to track signal: {e}") + + async def _should_stop_training(self, session: TrainingSession) -> bool: + """Check if training should be stopped based on performance""" + try: + # Check drawdown threshold + if session.total_pnl < -self.config.stop_loss_threshold: + self.logger.warning(f"Stopping training for {session.agent_id}: Drawdown exceeded") + return True + + # Check error rate + if session.errors_encountered > 100: + self.logger.warning(f"Stopping training for {session.agent_id}: Too many errors") + return True + + # Check minimum decisions + if session.decisions_made > 10000: + self.logger.info(f"Stopping training for {session.agent_id}: Maximum decisions reached") + return True + + return False + + except Exception as e: + self.logger.error(f"Error checking stop conditions: {e}") + return False + + async def start_training_session( + self, + agent_id: str, + scenarios: TrainingScenarioSet, + config: Optional[TrainingConfig] = None + ) -> str: + """ + Start a new training session for an agent. + + Args: + agent_id: Unique identifier for the agent + scenarios: Set of training scenarios to use + config: Optional training configuration override + + Returns: + Session ID for the training session + """ + try: + # Use provided config or default + session_config = config or self.config + + # Create session + session_id = f"{agent_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + session = TrainingSession( + session_id=session_id, + agent_id=agent_id, + start_time=datetime.now(), + config=session_config + ) + + self.active_sessions[agent_id] = session + + # Start scenario injection + asyncio.create_task(self._run_training_scenarios(session, scenarios)) + + self.logger.info(f"Started training session {session_id} for agent {agent_id}") + return session_id + + except Exception as e: + self.logger.error(f"Failed to start training session: {e}") + raise + + async def _run_training_scenarios(self, session: TrainingSession, scenarios: TrainingScenarioSet) -> None: + """Run training scenarios for a session""" + try: + for scenario in scenarios.scenarios: + if not session.is_active: + break + + # Convert scenario to events + events = await self._scenario_to_events(scenario) + + # Inject events into Redis with timing + for event in events: + if not session.is_active: + break + + # Apply time acceleration + delay = event.metadata.get('delay', 0.1) / session.config.time_acceleration + await asyncio.sleep(delay) + + # Add noise if configured + if session.config.inject_noise: + event = self._add_noise_to_event(event, session.config.noise_level) + + # Publish to appropriate channel + await self._publish_training_event(event, session) + + session.scenarios_completed += 1 + + # Update progress + await self._update_training_progress(session) + + # Session complete + if session.is_active: + await self.stop_training_session(session.agent_id) + + except Exception as e: + self.logger.error(f"Error running training scenarios: {e}") + session.errors_encountered += 1 + + async def _scenario_to_events(self, scenario: Any) -> List[StandardizedEvent]: + """Convert a training scenario to standardized events""" + try: + events = [] + + # Handle different scenario types + if hasattr(scenario, 'game') and scenario.game: + # Game scenario + for play in scenario.game.plays: + event = StandardizedEvent( + event_id=f"training_{play.play_id}", + game_id=scenario.game.game_id, + timestamp=play.timestamp, + event_type=play.play_type, + description=play.description, + team_possession=play.team_possession, + score_home=play.score_home, + score_away=play.score_away, + quarter=play.quarter, + time_remaining=play.time_remaining, + metadata={ + 'training': True, + 'scenario_id': scenario.scenario_id, + 'delay': 0.1 # Default delay between events + } + ) + events.append(event) + + elif hasattr(scenario, 'market_events'): + # Market scenario + for market_event in scenario.market_events: + event = StandardizedEvent( + event_id=f"training_market_{market_event.event_id}", + game_id=scenario.market_ticker, + timestamp=market_event.timestamp, + event_type="market_update", + description=f"Price: {market_event.price}", + metadata={ + 'training': True, + 'scenario_id': scenario.scenario_id, + 'price': market_event.price, + 'volume': market_event.volume, + 'delay': 0.05 + } + ) + events.append(event) + + return events + + except Exception as e: + self.logger.error(f"Failed to convert scenario to events: {e}") + return [] + + def _add_noise_to_event(self, event: StandardizedEvent, noise_level: float) -> StandardizedEvent: + """Add realistic noise to training events""" + try: + import random + + # Add noise to numeric fields + if hasattr(event, 'score_home') and event.score_home is not None: + # Don't add noise to scores (they're discrete) + pass + + # Add noise to metadata prices + if 'price' in event.metadata: + original_price = event.metadata['price'] + noise = random.gauss(0, noise_level * original_price) + event.metadata['price'] = max(0.01, min(0.99, original_price + noise)) + + # Add timing jitter + if 'delay' in event.metadata: + original_delay = event.metadata['delay'] + jitter = random.gauss(0, noise_level * original_delay) + event.metadata['delay'] = max(0.01, original_delay + jitter) + + return event + + except Exception as e: + self.logger.error(f"Failed to add noise to event: {e}") + return event + + async def _publish_training_event(self, event: StandardizedEvent, session: TrainingSession) -> None: + """Publish training event to appropriate Redis channel""" + try: + # Determine channel based on event type + channel = self._get_channel_for_event(event) + + # Add training namespace if configured + if session.config.training_namespace: + channel = f"{session.config.training_namespace}:{channel}" + + # Prepare message + message = { + "timestamp": event.timestamp.isoformat() if hasattr(event.timestamp, 'isoformat') else str(event.timestamp), + "source": "training_bridge", + "training_session": session.session_id, + "data": { + "event_id": event.event_id, + "game_id": event.game_id, + "event_type": event.event_type, + "description": event.description, + "metadata": event.metadata + } + } + + # Add event-specific data + if event.event_type in ["score_update", "big_play"]: + message["data"]["score_home"] = event.score_home + message["data"]["score_away"] = event.score_away + message["data"]["quarter"] = event.quarter + message["data"]["time_remaining"] = event.time_remaining + + # Publish to Redis + await self.publisher.publish(channel, json.dumps(message)) + + except Exception as e: + self.logger.error(f"Failed to publish training event: {e}") + + def _get_channel_for_event(self, event: StandardizedEvent) -> str: + """Determine the appropriate Redis channel for an event""" + event_type = event.event_type.lower() if event.event_type else "unknown" + + # Map event types to channels + if "market" in event_type or "price" in event_type: + return "kalshi:markets" + elif "trade" in event_type: + return "kalshi:trades" + elif "signal" in event_type: + return "kalshi:signals" + elif "sentiment" in event_type: + return "twitter:sentiment" + else: + return "espn:games" # Default to game events + + async def _update_training_progress(self, session: TrainingSession) -> None: + """Update training progress metrics""" + try: + # Calculate progress metrics + elapsed_time = (datetime.now() - session.start_time).total_seconds() + scenarios_per_minute = (session.scenarios_completed / elapsed_time) * 60 if elapsed_time > 0 else 0 + + # Log progress + self.logger.info( + f"Training progress for {session.agent_id}: " + f"{session.scenarios_completed} scenarios, " + f"{session.decisions_made} decisions, " + f"P&L: {session.total_pnl:.2f}, " + f"Rate: {scenarios_per_minute:.1f} scenarios/min" + ) + + # Update agent metrics + self.agent_metrics[session.agent_id]['scenarios_completed'] = session.scenarios_completed + self.agent_metrics[session.agent_id]['training_pnl'] = session.total_pnl + + except Exception as e: + self.logger.error(f"Failed to update training progress: {e}") + + async def stop_training_session(self, agent_id: str) -> Dict[str, Any]: + """ + Stop a training session and return final metrics. + + Args: + agent_id: Agent whose training session to stop + + Returns: + Final training metrics and summary + """ + try: + session = self.active_sessions.get(agent_id) + if not session: + return {"error": f"No active session for agent {agent_id}"} + + # Mark session as ended + session.end_time = datetime.now() + + # Generate final analytics + final_metrics = { + "session_id": session.session_id, + "agent_id": agent_id, + "duration": str(session.duration), + "scenarios_completed": session.scenarios_completed, + "decisions_made": session.decisions_made, + "total_pnl": session.total_pnl, + "errors_encountered": session.errors_encountered, + "agent_metrics": dict(self.agent_metrics.get(agent_id, {})) + } + + # Trigger final calibration update if needed + if session.config.calibrate_confidence and session.decisions_made > 50: + await self._trigger_calibration_update(agent_id) + + # Clean up session + del self.active_sessions[agent_id] + + self.logger.info(f"Stopped training session for agent {agent_id}") + return final_metrics + + except Exception as e: + self.logger.error(f"Failed to stop training session: {e}") + return {"error": str(e)} + + async def _trigger_calibration_update(self, agent_id: str) -> None: + """Trigger confidence calibration update for an agent""" + try: + # Get recent decisions from analytics + recent_analytics = await self.analytics.get_agent_analytics( + agent_id, timedelta(hours=1) + ) + + # Check if calibration update is needed + if await self.calibrator.should_update_calibration(agent_id): + self.logger.info(f"Triggering calibration update for agent {agent_id}") + # Calibration will be updated in next decision cycle + + except Exception as e: + self.logger.error(f"Failed to trigger calibration update: {e}") + + def register_response_handler(self, event_type: str, handler: Callable) -> None: + """Register a handler for specific agent response types""" + self.response_handlers[event_type] = handler + self.logger.info(f"Registered response handler for {event_type}") + + async def get_training_status(self) -> Dict[str, Any]: + """Get status of all active training sessions""" + try: + status = { + "active_sessions": len(self.active_sessions), + "sessions": {} + } + + for agent_id, session in self.active_sessions.items(): + status["sessions"][agent_id] = { + "session_id": session.session_id, + "started": session.start_time.isoformat(), + "duration": str(session.duration), + "scenarios_completed": session.scenarios_completed, + "decisions_made": session.decisions_made, + "current_pnl": session.total_pnl, + "mode": session.config.mode.value + } + + return status + + except Exception as e: + self.logger.error(f"Failed to get training status: {e}") + return {"error": str(e)} + + async def shutdown(self) -> None: + """Gracefully shutdown the training bridge""" + try: + self.logger.info("Shutting down Training Bridge") + + # Stop all active sessions + for agent_id in list(self.active_sessions.keys()): + await self.stop_training_session(agent_id) + + # Close Redis connections + if self.redis_client: + await self.redis_client.close() + if self.publisher: + await self.publisher.close() + + self.logger.info("Training Bridge shutdown complete") + + except Exception as e: + self.logger.error(f"Error during shutdown: {e}") \ No newline at end of file diff --git a/src/integration/training_harness.py b/src/integration/training_harness.py new file mode 100644 index 00000000..aa9c0aec --- /dev/null +++ b/src/integration/training_harness.py @@ -0,0 +1,845 @@ +""" +Agent Training Harness + +Orchestrates complete training sessions for agents, coordinating +between synthetic data generation, agent execution, and performance monitoring. +""" + +import asyncio +import json +import logging +from datetime import datetime, timedelta +from enum import Enum +from typing import Dict, List, Optional, Any, Set, Tuple +from dataclasses import dataclass, field +import redis.asyncio as redis + +from ..synthetic_data.generators.game_engine import SyntheticGameEngine, SyntheticGame +from ..synthetic_data.generators.market_simulator import MarketSimulator +from ..synthetic_data.generators.scenario_builder import TrainingScenarioSet +from ..training.memory_system import AgentMemorySystem +from ..confidence_calibration.calibrator import ConfidenceCalibrator +from ..training.agent_analytics import AgentAnalytics +from .synthetic_injector import SyntheticDataInjector, InjectionConfig, EventTiming +from .training_bridge import TrainingBridge, TrainingConfig, TrainingMode + +logger = logging.getLogger(__name__) + + +class TrainingScenario(Enum): + """Pre-defined training scenarios""" + BASIC_GAME = "basic_game" + CLOSE_GAME = "close_game" + BLOWOUT = "blowout" + COMEBACK = "comeback" + HIGH_VOLATILITY = "high_volatility" + LOW_LIQUIDITY = "low_liquidity" + NEWS_DRIVEN = "news_driven" + INJURY_SCENARIO = "injury_scenario" + WEATHER_IMPACT = "weather_impact" + MOMENTUM_SHIFT = "momentum_shift" + + +@dataclass +class HarnessConfig: + """Configuration for training harness""" + redis_url: str = "redis://localhost:6379" + + # Training parameters + scenarios_per_session: int = 10 + warmup_scenarios: int = 2 + evaluation_scenarios: int = 3 + + # Timing configuration + scenario_spacing_seconds: float = 5.0 + decision_timeout_seconds: float = 30.0 + between_session_delay: float = 60.0 + + # Performance thresholds + min_accuracy_threshold: float = 0.6 + min_profit_threshold: float = -0.1 + max_drawdown_threshold: float = 0.2 + + # Adaptive training + enable_adaptive_difficulty: bool = True + difficulty_adjustment_rate: float = 0.1 + performance_window_size: int = 5 + + # Monitoring + enable_real_time_monitoring: bool = True + checkpoint_frequency: int = 5 + performance_report_frequency: int = 10 + + +@dataclass +class TrainingMetrics: + """Metrics tracked during training""" + scenario_count: int = 0 + decision_count: int = 0 + successful_trades: int = 0 + failed_trades: int = 0 + + total_profit: float = 0.0 + max_drawdown: float = 0.0 + sharpe_ratio: float = 0.0 + win_rate: float = 0.0 + + avg_decision_time: float = 0.0 + avg_confidence: float = 0.0 + confidence_calibration_error: float = 0.0 + + exploration_rate: float = 0.0 + learning_rate: float = 0.0 + + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + + +@dataclass +class ScenarioResult: + """Result from running a single scenario""" + scenario_id: str + scenario_type: TrainingScenario + start_time: datetime + end_time: datetime + + decisions_made: int + profit_loss: float + accuracy: float + avg_confidence: float + + events_processed: int + errors: List[str] + + agent_state: Dict[str, Any] + market_conditions: Dict[str, Any] + + +class AgentTrainingHarness: + """ + Orchestrates complete training sessions for agents. + + Coordinates between synthetic data generation, agent execution, + and performance monitoring to provide comprehensive training. + """ + + def __init__(self, config: HarnessConfig): + self.config = config + self.redis_client: Optional[redis.Redis] = None + + # Core components + self.training_bridge = TrainingBridge() + self.data_injector = SyntheticDataInjector(InjectionConfig()) + self.memory_system = AgentMemorySystem() + self.calibrator = ConfidenceCalibrator() + self.analytics = AgentAnalytics() + + # Generators + self.game_engine = SyntheticGameEngine() + self.market_sim = MarketSimulator() + + # State tracking + self.active_sessions: Dict[str, Dict[str, Any]] = {} + self.scenario_queue: List[TrainingScenario] = [] + self.metrics: Dict[str, TrainingMetrics] = {} + + # Performance tracking + self.performance_history: List[ScenarioResult] = [] + self.current_difficulty: float = 0.5 + + async def initialize(self): + """Initialize harness components""" + self.redis_client = redis.from_url(self.config.redis_url) + + # Initialize components + await self.training_bridge.initialize() + await self.data_injector.initialize() + + # Set up monitoring + if self.config.enable_real_time_monitoring: + asyncio.create_task(self._monitor_performance()) + + logger.info("Training harness initialized") + + async def run_training_session( + self, + agent_id: str, + scenarios: List[TrainingScenario], + training_mode: TrainingMode = TrainingMode.EXPLORATION + ) -> Dict[str, Any]: + """ + Run a complete training session for an agent. + + Args: + agent_id: ID of agent to train + scenarios: List of scenarios to run + training_mode: Training mode to use + + Returns: + Session results and metrics + """ + session_id = f"session_{agent_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + # Initialize session + self.active_sessions[session_id] = { + 'agent_id': agent_id, + 'start_time': datetime.now(), + 'scenarios': scenarios, + 'mode': training_mode, + 'results': [] + } + + self.metrics[session_id] = TrainingMetrics() + + try: + # Run warmup scenarios + if self.config.warmup_scenarios > 0: + warmup_scenarios = scenarios[:self.config.warmup_scenarios] + await self._run_warmup(session_id, agent_id, warmup_scenarios) + + # Run training scenarios + training_scenarios = scenarios[self.config.warmup_scenarios:] + results = [] + + for i, scenario in enumerate(training_scenarios): + # Adjust difficulty if adaptive training enabled + if self.config.enable_adaptive_difficulty: + await self._adjust_difficulty(session_id) + + # Run scenario + result = await self._run_scenario( + session_id, + agent_id, + scenario, + training_mode + ) + + results.append(result) + self.performance_history.append(result) + + # Update metrics + await self._update_metrics(session_id, result) + + # Checkpoint if needed + if (i + 1) % self.config.checkpoint_frequency == 0: + await self._create_checkpoint(session_id, agent_id) + + # Performance report + if (i + 1) % self.config.performance_report_frequency == 0: + await self._generate_performance_report(session_id) + + # Delay between scenarios + await asyncio.sleep(self.config.scenario_spacing_seconds) + + # Run evaluation scenarios + if self.config.evaluation_scenarios > 0: + eval_results = await self._run_evaluation( + session_id, + agent_id, + scenarios[-self.config.evaluation_scenarios:] + ) + results.extend(eval_results) + + # Generate final report + report = await self._generate_final_report(session_id, results) + + # Clean up session + self.active_sessions[session_id]['end_time'] = datetime.now() + self.active_sessions[session_id]['results'] = results + + return report + + except Exception as e: + logger.error(f"Training session failed: {e}") + self.metrics[session_id].errors.append(str(e)) + raise + + async def _run_scenario( + self, + session_id: str, + agent_id: str, + scenario: TrainingScenario, + mode: TrainingMode + ) -> ScenarioResult: + """Run a single training scenario""" + scenario_id = f"{session_id}_{scenario.value}_{datetime.now().timestamp()}" + start_time = datetime.now() + + try: + # Generate scenario data + if scenario in [TrainingScenario.BASIC_GAME, TrainingScenario.CLOSE_GAME, + TrainingScenario.BLOWOUT, TrainingScenario.COMEBACK]: + # Game scenario + game_data = await self._generate_game_scenario(scenario) + await self.data_injector.inject_game_scenario( + game_data['game'], + game_data['plays'], + timing=EventTiming.ACCELERATED + ) + + else: + # Trading scenario + trading_data = await self._generate_trading_scenario(scenario) + await self.data_injector.inject_trading_scenario( + trading_data['events'], + timing=EventTiming.ACCELERATED + ) + + # Wait for agent decisions + decisions = await self._collect_agent_decisions( + agent_id, + scenario_id, + timeout=self.config.decision_timeout_seconds + ) + + # Calculate performance metrics + metrics = await self._calculate_scenario_metrics( + decisions, + scenario + ) + + end_time = datetime.now() + + return ScenarioResult( + scenario_id=scenario_id, + scenario_type=scenario, + start_time=start_time, + end_time=end_time, + decisions_made=len(decisions), + profit_loss=metrics['profit_loss'], + accuracy=metrics['accuracy'], + avg_confidence=metrics['avg_confidence'], + events_processed=metrics['events_processed'], + errors=[], + agent_state=await self._get_agent_state(agent_id), + market_conditions=metrics.get('market_conditions', {}) + ) + + except Exception as e: + logger.error(f"Scenario {scenario_id} failed: {e}") + return ScenarioResult( + scenario_id=scenario_id, + scenario_type=scenario, + start_time=start_time, + end_time=datetime.now(), + decisions_made=0, + profit_loss=0.0, + accuracy=0.0, + avg_confidence=0.0, + events_processed=0, + errors=[str(e)], + agent_state={}, + market_conditions={} + ) + + async def _generate_game_scenario( + self, + scenario: TrainingScenario + ) -> Dict[str, Any]: + """Generate game scenario data""" + if scenario == TrainingScenario.BASIC_GAME: + game = await self.game_engine.generate_game( + home_team="Team A", + away_team="Team B", + home_strength=0.5, + away_strength=0.5 + ) + plays = await self.game_engine.generate_plays(game, num_plays=50) + + elif scenario == TrainingScenario.CLOSE_GAME: + game = await self.game_engine.generate_game( + home_team="Team A", + away_team="Team B", + home_strength=0.52, + away_strength=0.48 + ) + plays = await self.game_engine.generate_plays(game, num_plays=60) + + elif scenario == TrainingScenario.BLOWOUT: + game = await self.game_engine.generate_game( + home_team="Team A", + away_team="Team B", + home_strength=0.7, + away_strength=0.3 + ) + plays = await self.game_engine.generate_plays(game, num_plays=45) + + elif scenario == TrainingScenario.COMEBACK: + game = await self.game_engine.generate_game( + home_team="Team A", + away_team="Team B", + home_strength=0.4, + away_strength=0.6 + ) + plays = await self.game_engine.generate_plays(game, num_plays=70) + # Simulate comeback in second half + for play in plays[35:]: + if hasattr(play, 'scoring_play'): + play.scoring_play = play.scoring_play and play.quarter >= 3 + + else: + raise ValueError(f"Unknown game scenario: {scenario}") + + return {'game': game, 'plays': plays} + + async def _generate_trading_scenario( + self, + scenario: TrainingScenario + ) -> Dict[str, Any]: + """Generate trading scenario data""" + # First generate a game for the trading scenario + if scenario == TrainingScenario.HIGH_VOLATILITY: + # Close game with high volatility + game = await self.game_engine.generate_game( + home_team="Team A", + away_team="Team B", + home_strength=0.51, + away_strength=0.49 + ) + trading_scenario = self.market_sim.create_trading_scenario( + game=game, + scenario_type="high_volatility", + market_efficiency=0.6 + ) + + elif scenario == TrainingScenario.LOW_LIQUIDITY: + # Low-profile game + game = await self.game_engine.generate_game( + home_team="Team C", + away_team="Team D", + home_strength=0.45, + away_strength=0.55 + ) + trading_scenario = self.market_sim.create_trading_scenario( + game=game, + scenario_type="low_liquidity", + market_efficiency=0.4 + ) + + elif scenario == TrainingScenario.NEWS_DRIVEN: + # Game with injury/news events + game = await self.game_engine.generate_game( + home_team="Team E", + away_team="Team F", + home_strength=0.6, + away_strength=0.4 + ) + trading_scenario = self.market_sim.create_trading_scenario( + game=game, + scenario_type="news_driven", + information_delay=0.3, + market_efficiency=0.7 + ) + + elif scenario == TrainingScenario.MOMENTUM_SHIFT: + # Game with momentum shifts + game = await self.game_engine.generate_game( + home_team="Team G", + away_team="Team H", + home_strength=0.48, + away_strength=0.52 + ) + trading_scenario = self.market_sim.create_trading_scenario( + game=game, + scenario_type="momentum_shift", + market_efficiency=0.75 + ) + + else: + # Default trading scenario + game = await self.game_engine.generate_game( + home_team="Team X", + away_team="Team Y", + home_strength=0.5, + away_strength=0.5 + ) + trading_scenario = self.market_sim.create_trading_scenario( + game=game, + scenario_type="regular" + ) + + # Convert to standardized events + events = self.market_sim.convert_to_standardized_events(trading_scenario) + + return {'events': events} + + async def _collect_agent_decisions( + self, + agent_id: str, + scenario_id: str, + timeout: float + ) -> List[Dict[str, Any]]: + """Collect decisions made by agent during scenario""" + decisions = [] + start_time = datetime.now() + + # Subscribe to agent decision channel + pubsub = self.redis_client.pubsub() + await pubsub.subscribe(f"agent:{agent_id}:decisions") + + try: + while (datetime.now() - start_time).total_seconds() < timeout: + message = await pubsub.get_message(timeout=1.0) + + if message and message['type'] == 'message': + try: + decision = json.loads(message['data']) + decision['scenario_id'] = scenario_id + decision['timestamp'] = datetime.now().isoformat() + decisions.append(decision) + except json.JSONDecodeError: + logger.warning(f"Invalid decision data: {message['data']}") + + finally: + await pubsub.unsubscribe(f"agent:{agent_id}:decisions") + await pubsub.close() + + return decisions + + async def _calculate_scenario_metrics( + self, + decisions: List[Dict[str, Any]], + scenario: TrainingScenario + ) -> Dict[str, Any]: + """Calculate performance metrics for scenario""" + if not decisions: + return { + 'profit_loss': 0.0, + 'accuracy': 0.0, + 'avg_confidence': 0.0, + 'events_processed': 0, + 'market_conditions': {} + } + + # Calculate P&L + profit_loss = sum(d.get('profit_loss', 0.0) for d in decisions) + + # Calculate accuracy (correct predictions) + correct = sum(1 for d in decisions if d.get('correct', False)) + accuracy = correct / len(decisions) if decisions else 0.0 + + # Calculate average confidence + confidences = [d.get('confidence', 0.5) for d in decisions] + avg_confidence = sum(confidences) / len(confidences) if confidences else 0.5 + + # Count events + events_processed = len(set(d.get('event_id') for d in decisions if d.get('event_id'))) + + # Extract market conditions + market_conditions = { + 'volatility': self._estimate_volatility(decisions), + 'trend': self._estimate_trend(decisions), + 'liquidity': self._estimate_liquidity(decisions) + } + + return { + 'profit_loss': profit_loss, + 'accuracy': accuracy, + 'avg_confidence': avg_confidence, + 'events_processed': events_processed, + 'market_conditions': market_conditions + } + + async def _update_metrics(self, session_id: str, result: ScenarioResult): + """Update session metrics with scenario result""" + metrics = self.metrics[session_id] + + metrics.scenario_count += 1 + metrics.decision_count += result.decisions_made + + if result.profit_loss > 0: + metrics.successful_trades += 1 + else: + metrics.failed_trades += 1 + + metrics.total_profit += result.profit_loss + + # Update running averages + alpha = 0.1 # Exponential moving average factor + metrics.avg_confidence = (1 - alpha) * metrics.avg_confidence + alpha * result.avg_confidence + + # Calculate win rate + total_trades = metrics.successful_trades + metrics.failed_trades + metrics.win_rate = metrics.successful_trades / total_trades if total_trades > 0 else 0.0 + + # Track errors + metrics.errors.extend(result.errors) + + async def _adjust_difficulty(self, session_id: str): + """Adjust scenario difficulty based on performance""" + metrics = self.metrics[session_id] + + # Get recent performance + recent_results = self.performance_history[-self.config.performance_window_size:] + if not recent_results: + return + + # Calculate performance score + avg_accuracy = sum(r.accuracy for r in recent_results) / len(recent_results) + avg_profit = sum(r.profit_loss for r in recent_results) / len(recent_results) + + performance_score = 0.6 * avg_accuracy + 0.4 * (1.0 if avg_profit > 0 else 0.0) + + # Adjust difficulty + if performance_score > 0.7: + # Increase difficulty + self.current_difficulty = min(1.0, self.current_difficulty + self.config.difficulty_adjustment_rate) + elif performance_score < 0.4: + # Decrease difficulty + self.current_difficulty = max(0.0, self.current_difficulty - self.config.difficulty_adjustment_rate) + + logger.info(f"Adjusted difficulty to {self.current_difficulty:.2f} (performance: {performance_score:.2f})") + + async def _create_checkpoint(self, session_id: str, agent_id: str): + """Create training checkpoint""" + checkpoint = { + 'session_id': session_id, + 'agent_id': agent_id, + 'timestamp': datetime.now().isoformat(), + 'metrics': self.metrics[session_id].__dict__, + 'difficulty': self.current_difficulty, + 'scenario_count': len(self.performance_history) + } + + # Save to Redis + await self.redis_client.hset( + f"training:checkpoints:{session_id}", + datetime.now().isoformat(), + json.dumps(checkpoint) + ) + + logger.info(f"Created checkpoint for session {session_id}") + + async def _generate_performance_report(self, session_id: str): + """Generate intermediate performance report""" + metrics = self.metrics[session_id] + + report = { + 'session_id': session_id, + 'timestamp': datetime.now().isoformat(), + 'scenarios_completed': metrics.scenario_count, + 'total_decisions': metrics.decision_count, + 'win_rate': metrics.win_rate, + 'total_profit': metrics.total_profit, + 'avg_confidence': metrics.avg_confidence, + 'current_difficulty': self.current_difficulty + } + + # Publish report + await self.redis_client.publish( + f"training:reports:{session_id}", + json.dumps(report) + ) + + logger.info(f"Performance report: Win rate={metrics.win_rate:.2%}, Profit={metrics.total_profit:.2f}") + + async def _generate_final_report( + self, + session_id: str, + results: List[ScenarioResult] + ) -> Dict[str, Any]: + """Generate final training report""" + metrics = self.metrics[session_id] + session = self.active_sessions[session_id] + + # Calculate final statistics + total_profit = sum(r.profit_loss for r in results) + avg_accuracy = sum(r.accuracy for r in results) / len(results) if results else 0.0 + + # Calculate Sharpe ratio + if len(results) > 1: + returns = [r.profit_loss for r in results] + avg_return = sum(returns) / len(returns) + std_return = (sum((r - avg_return) ** 2 for r in returns) / len(returns)) ** 0.5 + sharpe_ratio = avg_return / std_return if std_return > 0 else 0.0 + else: + sharpe_ratio = 0.0 + + report = { + 'session_id': session_id, + 'agent_id': session['agent_id'], + 'start_time': session['start_time'].isoformat(), + 'end_time': datetime.now().isoformat(), + 'duration_minutes': (datetime.now() - session['start_time']).total_seconds() / 60, + + 'scenarios_run': len(results), + 'total_decisions': sum(r.decisions_made for r in results), + + 'performance': { + 'total_profit': total_profit, + 'avg_accuracy': avg_accuracy, + 'win_rate': metrics.win_rate, + 'sharpe_ratio': sharpe_ratio, + 'max_drawdown': metrics.max_drawdown, + 'avg_confidence': metrics.avg_confidence + }, + + 'training_progress': { + 'starting_difficulty': 0.5, + 'ending_difficulty': self.current_difficulty, + 'exploration_rate': metrics.exploration_rate, + 'learning_rate': metrics.learning_rate + }, + + 'errors': metrics.errors, + 'warnings': metrics.warnings, + + 'recommendations': self._generate_recommendations(metrics, results) + } + + # Save final report + await self.redis_client.set( + f"training:final_report:{session_id}", + json.dumps(report), + ex=86400 # Expire after 24 hours + ) + + return report + + def _generate_recommendations( + self, + metrics: TrainingMetrics, + results: List[ScenarioResult] + ) -> List[str]: + """Generate training recommendations based on performance""" + recommendations = [] + + if metrics.win_rate < 0.5: + recommendations.append("Consider additional training on market prediction") + + if metrics.avg_confidence > 0.8 and metrics.win_rate < 0.6: + recommendations.append("Agent may be overconfident - adjust calibration") + + if metrics.max_drawdown > self.config.max_drawdown_threshold: + recommendations.append("Implement stricter risk management") + + if metrics.exploration_rate < 0.1: + recommendations.append("Increase exploration to discover new strategies") + + # Scenario-specific recommendations + scenario_performance = {} + for result in results: + if result.scenario_type not in scenario_performance: + scenario_performance[result.scenario_type] = [] + scenario_performance[result.scenario_type].append(result.accuracy) + + for scenario, accuracies in scenario_performance.items(): + avg_accuracy = sum(accuracies) / len(accuracies) + if avg_accuracy < 0.5: + recommendations.append(f"Focus training on {scenario.value} scenarios") + + return recommendations + + async def _run_warmup( + self, + session_id: str, + agent_id: str, + scenarios: List[TrainingScenario] + ): + """Run warmup scenarios""" + logger.info(f"Running {len(scenarios)} warmup scenarios") + + for scenario in scenarios: + result = await self._run_scenario( + session_id, + agent_id, + scenario, + TrainingMode.EXPLORATION + ) + # Don't count warmup in metrics + logger.debug(f"Warmup scenario {scenario.value}: accuracy={result.accuracy:.2%}") + + async def _run_evaluation( + self, + session_id: str, + agent_id: str, + scenarios: List[TrainingScenario] + ) -> List[ScenarioResult]: + """Run evaluation scenarios""" + logger.info(f"Running {len(scenarios)} evaluation scenarios") + + results = [] + for scenario in scenarios: + result = await self._run_scenario( + session_id, + agent_id, + scenario, + TrainingMode.VALIDATION + ) + results.append(result) + + return results + + async def _get_agent_state(self, agent_id: str) -> Dict[str, Any]: + """Get current agent state""" + # Retrieve agent state from Redis + state_key = f"agent:{agent_id}:state" + state_data = await self.redis_client.get(state_key) + + if state_data: + return json.loads(state_data) + return {} + + def _estimate_volatility(self, decisions: List[Dict[str, Any]]) -> float: + """Estimate market volatility from decisions""" + if len(decisions) < 2: + return 0.02 + + prices = [d.get('price', 0.5) for d in decisions if 'price' in d] + if len(prices) < 2: + return 0.02 + + # Calculate standard deviation of price changes + changes = [abs(prices[i] - prices[i-1]) for i in range(1, len(prices))] + return sum(changes) / len(changes) if changes else 0.02 + + def _estimate_trend(self, decisions: List[Dict[str, Any]]) -> float: + """Estimate market trend from decisions""" + if len(decisions) < 2: + return 0.0 + + prices = [d.get('price', 0.5) for d in decisions if 'price' in d] + if len(prices) < 2: + return 0.0 + + # Simple linear trend + return (prices[-1] - prices[0]) / len(prices) + + def _estimate_liquidity(self, decisions: List[Dict[str, Any]]) -> float: + """Estimate market liquidity from decisions""" + volumes = [d.get('volume', 0) for d in decisions if 'volume' in d] + if not volumes: + return 0.5 + + avg_volume = sum(volumes) / len(volumes) + # Normalize to 0-1 scale (assuming max volume of 10000) + return min(1.0, avg_volume / 10000) + + async def _monitor_performance(self): + """Real-time performance monitoring""" + while True: + try: + for session_id, session in self.active_sessions.items(): + if 'end_time' not in session: + # Session still active + metrics = self.metrics.get(session_id) + if metrics: + logger.info( + f"Session {session_id}: " + f"Scenarios={metrics.scenario_count}, " + f"Win rate={metrics.win_rate:.2%}, " + f"Profit={metrics.total_profit:.2f}" + ) + + await asyncio.sleep(30) # Monitor every 30 seconds + + except Exception as e: + logger.error(f"Monitoring error: {e}") + await asyncio.sleep(30) + + async def cleanup(self): + """Clean up resources""" + if self.redis_client: + await self.redis_client.close() + + logger.info("Training harness cleaned up") \ No newline at end of file diff --git a/src/synthetic_data/__init__.py b/src/synthetic_data/__init__.py new file mode 100644 index 00000000..81d2204b --- /dev/null +++ b/src/synthetic_data/__init__.py @@ -0,0 +1,20 @@ +""" +Synthetic Football Data Generation System + +A comprehensive system for generating realistic NFL game scenarios +using historical data and fine-tuned language models. + +Components: +- preprocessing: NFL dataset processing and formatting +- generators: Game sequence and scenario generation +- models: Fine-tuned LFM2 model wrappers +- storage: ChromaDB integration and data management +- validation: Quality metrics and pattern analysis +""" + +__version__ = "1.0.0" +__author__ = "Neural Trading Platform" + +from . import preprocessing, generators, models, storage, validation + +__all__ = ["preprocessing", "generators", "models", "storage", "validation"] \ No newline at end of file diff --git a/src/synthetic_data/generators/__init__.py b/src/synthetic_data/generators/__init__.py new file mode 100644 index 00000000..84bf3f8b --- /dev/null +++ b/src/synthetic_data/generators/__init__.py @@ -0,0 +1,26 @@ +""" +Synthetic Data Generators Module + +Game sequence generators and scenario builders +using fine-tuned language models. +""" + +from .game_engine import SyntheticGameEngine, SyntheticGame, SyntheticPlay, GameContext +from .market_simulator import MarketSimulator, TradingScenario, MarketEvent, MarketState, MarketEventType +from .scenario_builder import ScenarioBuilder, TrainingScenarioSet, ScenarioTemplate, ScenarioCategory + +__all__ = [ + 'SyntheticGameEngine', + 'SyntheticGame', + 'SyntheticPlay', + 'GameContext', + 'MarketSimulator', + 'TradingScenario', + 'MarketEvent', + 'MarketState', + 'MarketEventType', + 'ScenarioBuilder', + 'TrainingScenarioSet', + 'ScenarioTemplate', + 'ScenarioCategory' +] \ No newline at end of file diff --git a/src/synthetic_data/generators/game_engine.py b/src/synthetic_data/generators/game_engine.py new file mode 100644 index 00000000..3b15b5f2 --- /dev/null +++ b/src/synthetic_data/generators/game_engine.py @@ -0,0 +1,896 @@ +""" +Synthetic Game Engine + +Uses fine-tuned LFM2 models to generate realistic NFL game scenarios +for agent training with unlimited synthetic data. +""" + +import logging +import random +import asyncio +from typing import List, Dict, Any, Optional, Tuple, Iterator +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from pathlib import Path +import json + +from ..preprocessing.nfl_dataset_processor import ProcessedNFLPlay +from ..storage.chromadb_manager import ChromaDBManager +from src.sdk.core.base_adapter import StandardizedEvent, EventType + +logger = logging.getLogger(__name__) + + +@dataclass +class GameContext: + """Current game state context""" + game_id: str + home_team: str + away_team: str + quarter: int = 1 + time_remaining: str = "15:00" + time_seconds: int = 900 + + # Situation + down: int = 1 + distance: int = 10 + yard_line: int = 25 + yards_to_goal: int = 75 + possession_team: str = "" + + # Score + score_home: int = 0 + score_away: int = 0 + score_differential: int = 0 + + # Game factors + weather_impact: float = 0.0 # -1 to 1 + crowd_noise: float = 0.0 # 0 to 1 + momentum: float = 0.0 # -1 to 1 (negative = away, positive = home) + + def __post_init__(self): + if not self.possession_team: + self.possession_team = self.home_team + self.score_differential = self.score_home - self.score_away + + +@dataclass +class SyntheticPlay: + """Generated synthetic play""" + play_id: str + context: GameContext + play_type: str + play_description: str + yards_gained: int + time_elapsed: int = 30 + + # Outcomes + touchdown: bool = False + field_goal: bool = False + turnover: bool = False + safety: bool = False + penalty: bool = False + + # Advanced metrics + epa: float = 0.0 + wpa: float = 0.0 + excitement_factor: float = 0.5 # 0 to 1 + + +@dataclass +class SyntheticGame: + """Complete synthetic game""" + game_id: str + home_team: str + away_team: str + season: int + week: int + plays: List[SyntheticPlay] = field(default_factory=list) + final_score: Tuple[int, int] = (0, 0) + total_plays: int = 0 + game_duration_minutes: int = 180 + created_at: datetime = field(default_factory=datetime.now) + + # Game characteristics + game_type: str = "regular" # regular, close, blowout, overtime, weather + excitement_score: float = 0.5 + market_impact: float = 0.5 + + +class SyntheticGameEngine: + """ + Generates realistic NFL game scenarios using fine-tuned LFM2 models + and historical pattern analysis from ChromaDB + """ + + def __init__(self, + chromadb_manager: ChromaDBManager = None, + model_name: str = "nfl_playbypay_lfm2"): + """ + Initialize synthetic game engine + + Args: + chromadb_manager: ChromaDB manager for historical patterns + model_name: Fine-tuned LFM2 model name + """ + self.chromadb = chromadb_manager or ChromaDBManager() + self.model_name = model_name + + # Team data + self.nfl_teams = [ + "KC", "BUF", "CIN", "BAL", "MIA", "NYJ", "NE", "CLE", + "DAL", "PHI", "WAS", "NYG", "GB", "MIN", "CHI", "DET", + "LAR", "SF", "SEA", "ARI", "NO", "TB", "ATL", "CAR", + "LV", "LAC", "DEN", "PIT", "TEN", "IND", "HOU", "JAX" + ] + + # Game templates + self.game_templates = self._load_game_templates() + + logger.info(f"Initialized SyntheticGameEngine with model: {model_name}") + + def _load_game_templates(self) -> Dict[str, Dict]: + """Load game scenario templates""" + return { + "regular": { + "avg_plays": 140, + "avg_points": 45, + "variance": 0.2, + "overtime_probability": 0.05 + }, + "high_scoring": { + "avg_plays": 160, + "avg_points": 65, + "variance": 0.3, + "overtime_probability": 0.03 + }, + "defensive": { + "avg_plays": 120, + "avg_points": 28, + "variance": 0.15, + "overtime_probability": 0.08 + }, + "weather": { + "avg_plays": 110, + "avg_points": 35, + "variance": 0.25, + "weather_impact": 0.7 + }, + "blowout": { + "avg_plays": 135, + "avg_points": 52, + "variance": 0.4, + "comeback_probability": 0.15 + } + } + + async def generate_single_game(self, + home_team: str = None, + away_team: str = None, + game_type: str = "regular", + season: int = 2024, + week: int = 1) -> SyntheticGame: + """ + Generate a complete synthetic game + + Args: + home_team: Home team code (random if None) + away_team: Away team code (random if None) + game_type: Type of game scenario + season: Season year + week: Week number + + Returns: + Complete synthetic game + """ + # Select random teams if not specified + if not home_team or not away_team: + teams = random.sample(self.nfl_teams, 2) + home_team = home_team or teams[0] + away_team = away_team or teams[1] + + game_id = f"{season}{week:02d}{home_team}{away_team}" + + logger.info(f"Generating synthetic game: {away_team} @ {home_team} ({game_type})") + + # Initialize game context + context = GameContext( + game_id=game_id, + home_team=home_team, + away_team=away_team, + possession_team=away_team if random.random() > 0.5 else home_team + ) + + # Get game template + template = self.game_templates.get(game_type, self.game_templates["regular"]) + + # Generate game + game = SyntheticGame( + game_id=game_id, + home_team=home_team, + away_team=away_team, + season=season, + week=week, + game_type=game_type + ) + + # Generate plays using LFM2 and historical patterns + await self._generate_game_plays(game, context, template) + + # Post-process game statistics + self._calculate_game_metrics(game) + + logger.info(f"Generated game complete: {game.away_team} {game.final_score[1]} - {game.final_score[0]} {game.home_team}") + return game + + async def _generate_game_plays(self, + game: SyntheticGame, + context: GameContext, + template: Dict) -> None: + """Generate all plays for a game""" + + # Estimate total plays + target_plays = int(template["avg_plays"] * (1 + random.gauss(0, template["variance"]))) + target_plays = max(100, min(200, target_plays)) + + play_count = 0 + drive_number = 1 + + while context.quarter <= 4 and play_count < target_plays: + # Generate drive + drive_plays = await self._generate_drive(context, drive_number, template) + game.plays.extend(drive_plays) + play_count += len(drive_plays) + drive_number += 1 + + # Update quarter/time + self._advance_game_time(context, len(drive_plays)) + + # Switch possession + self._switch_possession(context) + + # Check for quarter end + if context.time_seconds <= 0: + context.quarter += 1 + context.time_seconds = 900 # 15 minutes + context.time_remaining = "15:00" + + # Handle overtime if needed + if context.score_home == context.score_away and random.random() < template.get("overtime_probability", 0.05): + await self._generate_overtime(game, context) + + game.total_plays = len(game.plays) + game.final_score = (context.score_home, context.score_away) + + async def _generate_drive(self, + context: GameContext, + drive_number: int, + template: Dict) -> List[SyntheticPlay]: + """Generate plays for a single drive""" + + drive_plays = [] + starting_field_pos = context.yards_to_goal + + # Reset downs + context.down = 1 + context.distance = 10 + + while True: + # Generate single play + play = await self._generate_single_play(context, drive_number, len(drive_plays) + 1) + drive_plays.append(play) + + # Apply play results + self._apply_play_results(context, play) + + # Check drive ending conditions + if self._is_drive_over(context, play): + break + + # Prevent infinite drives + if len(drive_plays) >= 20: + break + + return drive_plays + + async def _generate_single_play(self, + context: GameContext, + drive_number: int, + play_number: int) -> SyntheticPlay: + """Generate a single play using LFM2 and pattern matching""" + + # Search for similar situations in ChromaDB + situation_query = self._build_situation_query(context) + similar_plays = self.chromadb.search_similar_plays( + query=situation_query, + n_results=5 + ) + + # Generate play using LFM2 (simulated for now) + play_type, yards_gained, outcomes = await self._llm_generate_play(context, similar_plays) + + # Create play description + description = self._create_play_description(context, play_type, yards_gained, outcomes) + + play_id = f"{context.game_id}_{drive_number}_{play_number}" + + return SyntheticPlay( + play_id=play_id, + context=GameContext(**context.__dict__), # Copy context + play_type=play_type, + play_description=description, + yards_gained=yards_gained, + touchdown=outcomes.get("touchdown", False), + field_goal=outcomes.get("field_goal", False), + turnover=outcomes.get("turnover", False), + safety=outcomes.get("safety", False), + penalty=outcomes.get("penalty", False), + epa=self._calculate_epa(context, yards_gained, outcomes), + wpa=self._calculate_wpa(context, yards_gained, outcomes) + ) + + def _build_situation_query(self, context: GameContext) -> str: + """Build ChromaDB query for similar situations""" + situation_parts = [] + + # Down and distance + situation_parts.append(f"{context.down} and {context.distance}") + + # Field position + if context.yards_to_goal <= 20: + situation_parts.append("red zone") + elif context.yards_to_goal >= 80: + situation_parts.append("deep in own territory") + else: + situation_parts.append(f"{context.yards_to_goal} yards to goal") + + # Time situation + if context.quarter >= 4 and context.time_seconds < 120: + situation_parts.append("two minute warning") + elif context.quarter >= 4 and context.time_seconds < 300: + situation_parts.append("fourth quarter") + + # Score situation + if abs(context.score_differential) <= 3: + situation_parts.append("close game") + elif context.score_differential > 14: + situation_parts.append("large lead") + elif context.score_differential < -14: + situation_parts.append("large deficit") + + return ". ".join(situation_parts) + + async def _llm_generate_play(self, + context: GameContext, + similar_plays: List[Dict]) -> Tuple[str, int, Dict]: + """Use LFM2 to generate realistic play outcome""" + + # For now, simulate LFM2 with pattern-based generation + # In production, this would call the fine-tuned Ollama model + + # Analyze similar plays for patterns + play_types = [] + yard_totals = [] + + for play in similar_plays: + metadata = play.get('metadata', {}) + play_types.append(metadata.get('play_type', 'RUN')) + yard_totals.append(metadata.get('yards_gained', 0)) + + # Generate play type based on situation + play_type = self._select_play_type(context, play_types) + + # Generate yards based on play type and situation + yards_gained = self._generate_yards(context, play_type, yard_totals) + + # Generate special outcomes + outcomes = self._generate_outcomes(context, play_type, yards_gained) + + return play_type, yards_gained, outcomes + + def _select_play_type(self, context: GameContext, historical_types: List[str]) -> str: + """Select play type based on situation and patterns""" + + # Use historical patterns if available + if historical_types: + type_weights = {} + for ptype in historical_types: + type_weights[ptype] = type_weights.get(ptype, 0) + 1 + + # Add situational bias + if context.distance <= 2: + type_weights["RUN"] = type_weights.get("RUN", 0) + 2 + elif context.distance >= 10: + type_weights["PASS"] = type_weights.get("PASS", 0) + 2 + elif context.yards_to_goal <= 5 and context.down <= 2: + type_weights["RUN"] = type_weights.get("RUN", 0) + 1 + + # Weighted random selection + total_weight = sum(type_weights.values()) + if total_weight > 0: + rand = random.random() * total_weight + cumsum = 0 + for ptype, weight in type_weights.items(): + cumsum += weight + if rand <= cumsum: + return ptype + + # Fallback to situational logic + if context.down >= 3 and context.distance >= 7: + return "PASS" + elif context.distance <= 2: + return "RUN" + elif context.yards_to_goal <= 35 and context.down == 4: + return "FIELD_GOAL" + elif context.down == 4: + return "PUNT" + else: + return random.choice(["RUN", "PASS"]) + + def _generate_yards(self, context: GameContext, play_type: str, historical_yards: List[int]) -> int: + """Generate realistic yards gained""" + + # Base yards by play type + if play_type == "PASS": + base_yards = random.choice([0, 3, 5, 7, 12, 15, 18, 22, 35]) + variance = 5 + elif play_type == "RUN": + base_yards = random.choice([-1, 0, 1, 2, 3, 4, 5, 6, 8, 12]) + variance = 3 + elif play_type == "FIELD_GOAL": + return 0 # Handled separately + elif play_type == "PUNT": + return -(40 + random.randint(-10, 10)) # Field position change + else: + base_yards = 2 + variance = 2 + + # Adjust for historical patterns + if historical_yards: + avg_historical = sum(historical_yards) / len(historical_yards) + base_yards = int(0.7 * base_yards + 0.3 * avg_historical) + + # Add random variance + yards = base_yards + random.randint(-variance, variance) + + # Constrain to field boundaries + max_yards = min(99, context.yards_to_goal) + min_yards = max(-99, -(100 - context.yards_to_goal)) + + return max(min_yards, min(max_yards, yards)) + + def _generate_outcomes(self, context: GameContext, play_type: str, yards_gained: int) -> Dict: + """Generate special play outcomes""" + outcomes = {} + + # Touchdown + if yards_gained >= context.yards_to_goal: + outcomes["touchdown"] = True + return outcomes + + # Field goal + if play_type == "FIELD_GOAL": + # Success probability based on distance + distance = context.yards_to_goal + 17 # Add endzone + snap distance + success_rate = max(0.5, 1.0 - (distance - 20) / 100) + outcomes["field_goal"] = random.random() < success_rate + return outcomes + + # Turnover + turnover_rate = 0.02 + if play_type == "PASS": + turnover_rate = 0.025 + elif play_type == "RUN": + turnover_rate = 0.015 + + if random.random() < turnover_rate: + outcomes["turnover"] = True + + # Penalty (rare) + if random.random() < 0.08: + outcomes["penalty"] = True + # Penalties don't count as yards gained typically + yards_gained = 0 + + # Safety (very rare) + if context.yards_to_goal >= 98 and yards_gained <= -2: + outcomes["safety"] = True + + return outcomes + + def _create_play_description(self, context: GameContext, play_type: str, yards_gained: int, outcomes: Dict) -> str: + """Create realistic play description""" + + # Player names (simplified) + qb_name = f"{context.possession_team}.QB" + rb_name = f"{context.possession_team}.RB" + wr_name = f"{context.possession_team}.WR" + + if outcomes.get("touchdown"): + if play_type == "PASS": + return f"{qb_name} pass complete to {wr_name} for {yards_gained} yards, TOUCHDOWN" + elif play_type == "RUN": + return f"{rb_name} rush for {yards_gained} yards, TOUCHDOWN" + elif outcomes.get("field_goal"): + return f"Field goal attempt is GOOD" + elif outcomes.get("turnover"): + if play_type == "PASS": + return f"{qb_name} pass INTERCEPTED" + else: + return f"{rb_name} FUMBLES, recovered by defense" + elif play_type == "PASS": + if yards_gained <= 0: + return f"{qb_name} pass incomplete" + else: + return f"{qb_name} pass complete to {wr_name} for {yards_gained} yards" + elif play_type == "RUN": + return f"{rb_name} rush for {yards_gained} yards" + elif play_type == "PUNT": + return f"Punt for {abs(yards_gained)} yards" + else: + return f"{play_type} for {yards_gained} yards" + + def _calculate_epa(self, context: GameContext, yards_gained: int, outcomes: Dict) -> float: + """Calculate Expected Points Added (simplified)""" + + # Simplified EPA calculation + base_ep = 0.0 + + # Base expected points by field position + if context.yards_to_goal <= 5: + base_ep = 6.0 + elif context.yards_to_goal <= 15: + base_ep = 4.5 + elif context.yards_to_goal <= 35: + base_ep = 3.0 + elif context.yards_to_goal <= 65: + base_ep = 1.0 + else: + base_ep = 0.2 + + # Adjust for outcomes + if outcomes.get("touchdown"): + return 7.0 - base_ep + elif outcomes.get("field_goal"): + return 3.0 - base_ep + elif outcomes.get("turnover"): + return -base_ep - 2.0 + else: + # Rough EPA for yard gain + yard_value = yards_gained * 0.1 + return yard_value - 0.5 # Small negative for not scoring + + def _calculate_wpa(self, context: GameContext, yards_gained: int, outcomes: Dict) -> float: + """Calculate Win Probability Added (simplified)""" + + # Simplified WPA calculation + time_factor = context.time_seconds / 3600 # Normalize to game time + score_factor = 1.0 / (1.0 + abs(context.score_differential) / 7.0) + + base_wpa = 0.0 + + if outcomes.get("touchdown"): + base_wpa = 0.15 * score_factor + elif outcomes.get("field_goal"): + base_wpa = 0.08 * score_factor + elif outcomes.get("turnover"): + base_wpa = -0.12 * score_factor + else: + # First down conversion impact + if yards_gained >= context.distance: + base_wpa = 0.03 * score_factor + else: + base_wpa = -0.01 * score_factor + + # Time pressure multiplier + if context.quarter >= 4: + base_wpa *= (2.0 - time_factor) + + return base_wpa + + def _apply_play_results(self, context: GameContext, play: SyntheticPlay) -> None: + """Apply play results to game context""" + + # Update score + if play.touchdown: + if context.possession_team == context.home_team: + context.score_home += 7 # TD + XP + else: + context.score_away += 7 + context.score_differential = context.score_home - context.score_away + + elif play.field_goal: + if context.possession_team == context.home_team: + context.score_home += 3 + else: + context.score_away += 3 + context.score_differential = context.score_home - context.score_away + + elif play.safety: + if context.possession_team == context.home_team: + context.score_away += 2 # Defense gets points + else: + context.score_home += 2 + context.score_differential = context.score_home - context.score_away + + # Update field position + if not (play.touchdown or play.field_goal or play.turnover): + context.yards_to_goal -= play.yards_gained + context.yards_to_goal = max(1, min(99, context.yards_to_goal)) + context.yard_line = 100 - context.yards_to_goal + + # Update down and distance + if play.turnover or play.touchdown or play.field_goal or play.safety: + # Possession will change + pass + elif play.yards_gained >= context.distance: + # First down + context.down = 1 + context.distance = 10 + else: + # Next down + context.down += 1 + context.distance -= play.yards_gained + context.distance = max(1, context.distance) + + def _is_drive_over(self, context: GameContext, play: SyntheticPlay) -> bool: + """Check if drive should end""" + + # Scoring plays end drives + if play.touchdown or play.field_goal or play.safety: + return True + + # Turnovers end drives + if play.turnover: + return True + + # Fourth down stops (punt/failed conversion) + if context.down >= 4 and play.yards_gained < context.distance: + return True + + return False + + def _advance_game_time(self, context: GameContext, num_plays: int) -> None: + """Advance game time based on plays""" + + # Average 30 seconds per play + time_elapsed = num_plays * 30 + context.time_seconds -= time_elapsed + + if context.time_seconds <= 0: + context.time_seconds = 0 + context.time_remaining = "0:00" + else: + minutes = context.time_seconds // 60 + seconds = context.time_seconds % 60 + context.time_remaining = f"{minutes}:{seconds:02d}" + + def _switch_possession(self, context: GameContext) -> None: + """Switch possession between teams""" + + if context.possession_team == context.home_team: + context.possession_team = context.away_team + else: + context.possession_team = context.home_team + + # Reset field position for new drive + context.yards_to_goal = random.randint(70, 85) # Typical starting position + context.yard_line = 100 - context.yards_to_goal + + async def _generate_overtime(self, game: SyntheticGame, context: GameContext) -> None: + """Generate overtime period""" + logger.info(f"Generating overtime for {game.game_id}") + + context.quarter = 5 + context.time_seconds = 900 # 15 minutes + context.time_remaining = "15:00" + + # Simple overtime - first score wins + drive_number = 100 # High number to distinguish OT + + while context.score_home == context.score_away: + drive_plays = await self._generate_drive(context, drive_number, self.game_templates["regular"]) + game.plays.extend(drive_plays) + + # Check for scoring + if any(play.touchdown or play.field_goal or play.safety for play in drive_plays): + break + + self._switch_possession(context) + drive_number += 1 + + # Prevent infinite overtime + if drive_number > 110: + # Simulate coin flip winner + if random.random() > 0.5: + context.score_home += 3 + else: + context.score_away += 3 + break + + def _calculate_game_metrics(self, game: SyntheticGame) -> None: + """Calculate final game metrics""" + + if game.plays: + # Calculate excitement score + big_plays = sum(1 for play in game.plays if abs(play.yards_gained) >= 20) + scoring_plays = sum(1 for play in game.plays if play.touchdown or play.field_goal) + turnovers = sum(1 for play in game.plays if play.turnover) + + excitement = (big_plays * 0.1 + scoring_plays * 0.2 + turnovers * 0.15) + game.excitement_score = min(1.0, excitement / 10.0) + + # Calculate market impact (based on score differential and lead changes) + score_diff = abs(game.final_score[0] - game.final_score[1]) + if score_diff <= 3: + game.market_impact = 0.9 # High impact for close games + elif score_diff <= 7: + game.market_impact = 0.7 + elif score_diff <= 14: + game.market_impact = 0.5 + else: + game.market_impact = 0.3 # Low impact for blowouts + + def convert_to_standardized_events(self, game: SyntheticGame) -> List[StandardizedEvent]: + """Convert synthetic game to StandardizedEvent format""" + + events = [] + + for play in game.plays: + # Determine event type and impact + if play.touchdown or play.field_goal: + event_type = EventType.GAME_EVENT + impact = "high" + elif play.turnover: + event_type = EventType.GAME_EVENT + impact = "high" + elif abs(play.yards_gained) >= 15: + event_type = EventType.GAME_EVENT + impact = "medium" + else: + event_type = EventType.GAME_EVENT + impact = "low" + + # Build event data + event_data = { + "play_type": play.play_type, + "description": play.play_description, + "quarter": play.context.quarter, + "time_remaining": play.context.time_remaining, + "down": play.context.down, + "distance": play.context.distance, + "yards_gained": play.yards_gained, + "score": { + "home": play.context.score_home, + "away": play.context.score_away, + "differential": play.context.score_differential + }, + "outcomes": { + "touchdown": play.touchdown, + "field_goal": play.field_goal, + "turnover": play.turnover, + "safety": play.safety, + "penalty": play.penalty + }, + "synthetic": True # Mark as synthetic data + } + + event = StandardizedEvent( + source="synthetic_nfl_engine", + event_type=event_type, + timestamp=datetime.now(), + game_id=play.context.game_id, + data=event_data, + confidence=0.99, # High confidence in synthetic data + impact=impact, + metadata={ + "home_team": game.home_team, + "away_team": game.away_team, + "season": game.season, + "week": game.week, + "epa": play.epa, + "wpa": play.wpa, + "excitement": play.excitement_factor, + "synthetic": True + }, + raw_data=play + ) + + events.append(event) + + logger.info(f"Converted {len(events)} synthetic plays to StandardizedEvents") + return events + + async def generate_batch_games(self, + num_games: int = 100, + game_types: List[str] = None, + season: int = 2024, + start_week: int = 1) -> List[SyntheticGame]: + """ + Generate a batch of synthetic games + + Args: + num_games: Number of games to generate + game_types: List of game types to include + season: Season year + start_week: Starting week number + + Returns: + List of synthetic games + """ + if game_types is None: + game_types = ["regular", "high_scoring", "defensive", "weather", "blowout"] + + games = [] + week = start_week + + logger.info(f"Generating batch of {num_games} synthetic games...") + + for i in range(num_games): + # Select random game type + game_type = random.choice(game_types) + + # Select random teams + teams = random.sample(self.nfl_teams, 2) + + # Generate game + game = await self.generate_single_game( + home_team=teams[0], + away_team=teams[1], + game_type=game_type, + season=season, + week=week + ) + + games.append(game) + + # Progress logging + if (i + 1) % 25 == 0: + logger.info(f"Generated {i + 1}/{num_games} synthetic games") + + # Increment week + week += 1 + if week > 18: + week = 1 + + logger.info(f"Batch generation complete: {len(games)} games generated") + return games + + +# Example usage and testing +if __name__ == "__main__": + import sys + import asyncio + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.storage.chromadb_manager import ChromaDBManager + + async def test_game_engine(): + # Initialize components + chromadb = ChromaDBManager() + engine = SyntheticGameEngine(chromadb) + + # Generate single game + game = await engine.generate_single_game( + home_team="KC", + away_team="BUF", + game_type="regular" + ) + + print(f"Generated Game: {game.away_team} @ {game.home_team}") + print(f"Final Score: {game.away_team} {game.final_score[1]} - {game.final_score[0]} {game.home_team}") + print(f"Total Plays: {game.total_plays}") + print(f"Game Type: {game.game_type}") + print(f"Excitement Score: {game.excitement_score:.2f}") + + # Show some plays + print(f"\nFirst 5 plays:") + for play in game.plays[:5]: + print(f" {play.play_type}: {play.play_description}") + + # Convert to StandardizedEvents + events = engine.convert_to_standardized_events(game) + print(f"Converted to {len(events)} StandardizedEvents") + + # Run test + asyncio.run(test_game_engine()) \ No newline at end of file diff --git a/src/synthetic_data/generators/market_simulator.py b/src/synthetic_data/generators/market_simulator.py new file mode 100644 index 00000000..19133614 --- /dev/null +++ b/src/synthetic_data/generators/market_simulator.py @@ -0,0 +1,754 @@ +""" +Market Event Simulator + +Simulates Kalshi market events and trading scenarios based on +synthetic game data for agent training. +""" + +import logging +import random +from typing import List, Dict, Any, Optional, Tuple, Iterator +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +import json +import math + +from .game_engine import SyntheticGame, SyntheticPlay +from src.sdk.core.base_adapter import StandardizedEvent, EventType + +logger = logging.getLogger(__name__) + + +class MarketEventType(Enum): + """Types of market events""" + PRICE_MOVEMENT = "price_movement" + VOLUME_SPIKE = "volume_spike" + SPREAD_CHANGE = "spread_change" + NEWS_EVENT = "news_event" + MOMENTUM_SHIFT = "momentum_shift" + INJURY_REPORT = "injury_report" + WEATHER_UPDATE = "weather_update" + + +@dataclass +class MarketState: + """Current market state for a contract""" + market_ticker: str + yes_price: float = 0.50 + no_price: float = 0.50 + volume: int = 0 + open_interest: int = 1000 + bid_ask_spread: float = 0.02 + last_trade_price: float = 0.50 + price_change_24h: float = 0.0 + + # Market dynamics + momentum: float = 0.0 # -1 to 1 + volatility: float = 0.1 # 0 to 1 + liquidity: float = 0.5 # 0 to 1 + + def __post_init__(self): + """Ensure prices sum to ~1.00""" + total = self.yes_price + self.no_price + if total != 1.0: + self.yes_price = self.yes_price / total + self.no_price = self.no_price / total + + +@dataclass +class MarketEvent: + """Market event that affects pricing""" + event_type: MarketEventType + market_ticker: str + timestamp: datetime + description: str + + # Price impact + price_impact: float = 0.0 # -1 to 1 (negative = price down) + volume_impact: float = 0.0 # 0 to 1 + confidence_impact: float = 0.0 # -1 to 1 + + # Context + game_context: Dict[str, Any] = field(default_factory=dict) + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class TradingScenario: + """Complete trading scenario for agent training""" + scenario_id: str + market_ticker: str + game: SyntheticGame + + # Market timeline + events: List[MarketEvent] = field(default_factory=list) + price_history: List[Tuple[datetime, float]] = field(default_factory=list) + volume_history: List[Tuple[datetime, int]] = field(default_factory=list) + + # Information asymmetry + public_events: List[MarketEvent] = field(default_factory=list) + private_events: List[MarketEvent] = field(default_factory=list) + delayed_events: List[MarketEvent] = field(default_factory=list) + + # Scenario characteristics + scenario_type: str = "regular" # regular, volatile, trending, reversal + information_delay: float = 0.0 # 0 to 1 (fraction of events delayed) + market_efficiency: float = 0.8 # 0 to 1 (how quickly prices adjust) + + created_at: datetime = field(default_factory=datetime.now) + + +class MarketSimulator: + """ + Simulates realistic Kalshi market events and trading scenarios + for comprehensive agent training + """ + + def __init__(self): + """Initialize market simulator""" + + # Market configuration + self.base_liquidity = 1000 + self.min_spread = 0.01 + self.max_spread = 0.10 + self.volatility_decay = 0.95 + + # Event probabilities (per game) + self.event_probabilities = { + MarketEventType.PRICE_MOVEMENT: 1.0, # Always present + MarketEventType.VOLUME_SPIKE: 0.3, # 30% of games + MarketEventType.SPREAD_CHANGE: 0.4, # 40% of games + MarketEventType.NEWS_EVENT: 0.2, # 20% of games + MarketEventType.MOMENTUM_SHIFT: 0.6, # 60% of games + MarketEventType.INJURY_REPORT: 0.15, # 15% of games + MarketEventType.WEATHER_UPDATE: 0.1 # 10% of games + } + + logger.info("Initialized MarketSimulator") + + def create_trading_scenario(self, + game: SyntheticGame, + scenario_type: str = "regular", + information_delay: float = 0.1, + market_efficiency: float = 0.8) -> TradingScenario: + """ + Create complete trading scenario from synthetic game + + Args: + game: Synthetic game to base scenario on + scenario_type: Type of market scenario + information_delay: Fraction of events with delayed information + market_efficiency: Speed of price adjustment (0-1) + + Returns: + Complete trading scenario + """ + market_ticker = f"NFL-{game.away_team}-{game.home_team}-{game.game_id[-8:]}" + scenario_id = f"scenario_{game.game_id}_{scenario_type}" + + logger.info(f"Creating trading scenario: {market_ticker} ({scenario_type})") + + scenario = TradingScenario( + scenario_id=scenario_id, + market_ticker=market_ticker, + game=game, + scenario_type=scenario_type, + information_delay=information_delay, + market_efficiency=market_efficiency + ) + + # Initialize market state + initial_state = self._initialize_market_state(market_ticker, game) + + # Generate market events throughout game + self._generate_market_events(scenario, initial_state) + + # Separate events by information type + self._classify_information_events(scenario) + + # Generate price and volume history + self._simulate_market_timeline(scenario, initial_state) + + logger.info(f"Created scenario with {len(scenario.events)} market events") + return scenario + + def _initialize_market_state(self, market_ticker: str, game: SyntheticGame) -> MarketState: + """Initialize market state based on game characteristics""" + + # Determine initial odds based on team strength (simplified) + home_advantage = 0.55 # Typical home field advantage + + # Add random noise for team strength + team_strength_diff = random.gauss(0, 0.15) # Random team strength difference + + initial_prob = home_advantage + team_strength_diff + initial_prob = max(0.2, min(0.8, initial_prob)) # Keep within reasonable bounds + + # Set initial pricing + yes_price = initial_prob + no_price = 1.0 - yes_price + + # Adjust for game type + if game.game_type == "blowout": + # More extreme pricing for expected blowouts + if yes_price > 0.5: + yes_price = min(0.85, yes_price + 0.2) + else: + yes_price = max(0.15, yes_price - 0.2) + no_price = 1.0 - yes_price + + # Calculate spread and liquidity + spread = self._calculate_spread(yes_price, game.game_type) + liquidity = self._calculate_liquidity(game.game_type) + + return MarketState( + market_ticker=market_ticker, + yes_price=yes_price, + no_price=no_price, + volume=random.randint(500, 2000), + open_interest=random.randint(800, 5000), + bid_ask_spread=spread, + last_trade_price=yes_price, + momentum=random.gauss(0, 0.1), + volatility=self._get_base_volatility(game.game_type), + liquidity=liquidity + ) + + def _calculate_spread(self, price: float, game_type: str) -> float: + """Calculate bid-ask spread based on price and game type""" + + # Spreads wider at extremes + extreme_factor = 4 * price * (1 - price) # 0 at extremes, 1 at 0.5 + base_spread = self.min_spread + (self.max_spread - self.min_spread) * (1 - extreme_factor) + + # Game type modifiers + type_multipliers = { + "regular": 1.0, + "high_scoring": 0.9, # More liquid + "defensive": 1.1, # Less liquid + "weather": 1.3, # Less liquid due to uncertainty + "blowout": 0.8 # More liquid due to clarity + } + + multiplier = type_multipliers.get(game_type, 1.0) + return max(self.min_spread, base_spread * multiplier) + + def _calculate_liquidity(self, game_type: str) -> float: + """Calculate market liquidity factor""" + + liquidity_scores = { + "regular": 0.7, + "high_scoring": 0.8, # High interest + "defensive": 0.6, # Lower interest + "weather": 0.5, # Uncertainty reduces participation + "blowout": 0.9 # Clear outcomes attract traders + } + + base_liquidity = liquidity_scores.get(game_type, 0.7) + return base_liquidity + random.gauss(0, 0.1) + + def _get_base_volatility(self, game_type: str) -> float: + """Get base volatility for game type""" + + volatility_scores = { + "regular": 0.15, + "high_scoring": 0.25, # More volatile due to scoring + "defensive": 0.10, # Lower volatility + "weather": 0.30, # Weather creates uncertainty + "blowout": 0.20 # Moderate volatility + } + + return volatility_scores.get(game_type, 0.15) + + def _generate_market_events(self, scenario: TradingScenario, initial_state: MarketState): + """Generate market events throughout the game timeline""" + + current_state = initial_state + game_start = datetime.now() + + # Pre-game events (2 hours before) + self._generate_pregame_events(scenario, current_state, game_start - timedelta(hours=2)) + + # During game events (based on plays) + self._generate_ingame_events(scenario, current_state, game_start) + + # Post-game events + self._generate_postgame_events(scenario, current_state, game_start + timedelta(hours=3)) + + def _generate_pregame_events(self, scenario: TradingScenario, state: MarketState, start_time: datetime): + """Generate pre-game market events""" + + events = [] + current_time = start_time + + # Injury reports + if random.random() < self.event_probabilities[MarketEventType.INJURY_REPORT]: + affected_team = random.choice([scenario.game.home_team, scenario.game.away_team]) + severity = random.choice(["questionable", "doubtful", "out"]) + + impact = -0.15 if severity == "out" else -0.05 + if affected_team == scenario.game.away_team: + impact *= -1 # Flip for away team + + event = MarketEvent( + event_type=MarketEventType.INJURY_REPORT, + market_ticker=state.market_ticker, + timestamp=current_time + timedelta(minutes=random.randint(15, 90)), + description=f"Key player from {affected_team} listed as {severity}", + price_impact=impact, + volume_impact=0.3, + confidence_impact=-0.2 + ) + events.append(event) + + # Weather updates + if random.random() < self.event_probabilities[MarketEventType.WEATHER_UPDATE]: + weather_conditions = random.choice(["rain", "wind", "snow", "cold"]) + + event = MarketEvent( + event_type=MarketEventType.WEATHER_UPDATE, + market_ticker=state.market_ticker, + timestamp=current_time + timedelta(minutes=random.randint(30, 120)), + description=f"Weather update: {weather_conditions} conditions expected", + price_impact=random.gauss(0, 0.05), + volume_impact=0.2, + confidence_impact=-0.1, + metadata={"weather": weather_conditions} + ) + events.append(event) + + # News events + if random.random() < self.event_probabilities[MarketEventType.NEWS_EVENT]: + news_types = ["coaching_decision", "team_news", "analyst_prediction", "betting_trends"] + news_type = random.choice(news_types) + + event = MarketEvent( + event_type=MarketEventType.NEWS_EVENT, + market_ticker=state.market_ticker, + timestamp=current_time + timedelta(minutes=random.randint(45, 120)), + description=f"News: {news_type.replace('_', ' ')}", + price_impact=random.gauss(0, 0.08), + volume_impact=0.15, + confidence_impact=random.gauss(0, 0.05) + ) + events.append(event) + + scenario.events.extend(events) + + def _generate_ingame_events(self, scenario: TradingScenario, state: MarketState, game_start: datetime): + """Generate market events based on game plays""" + + current_time = game_start + last_momentum = 0.0 + + for i, play in enumerate(scenario.game.plays): + # Advance time + current_time += timedelta(seconds=30) # Average time per play + + # Always generate price movement for significant plays + if self._is_significant_play(play): + price_impact = self._calculate_play_price_impact(play, scenario.game) + + event = MarketEvent( + event_type=MarketEventType.PRICE_MOVEMENT, + market_ticker=state.market_ticker, + timestamp=current_time, + description=f"Play update: {play.play_description}", + price_impact=price_impact, + volume_impact=min(0.5, abs(price_impact) * 2), + game_context={ + "quarter": play.context.quarter, + "score_home": play.context.score_home, + "score_away": play.context.score_away, + "play_type": play.play_type + } + ) + scenario.events.append(event) + + # Volume spikes on exciting plays + if play.excitement_factor > 0.8 and random.random() < 0.5: + event = MarketEvent( + event_type=MarketEventType.VOLUME_SPIKE, + market_ticker=state.market_ticker, + timestamp=current_time + timedelta(seconds=random.randint(5, 30)), + description=f"High volume trading after {play.play_type}", + volume_impact=0.8, + price_impact=0.0 + ) + scenario.events.append(event) + + # Momentum shifts + current_momentum = self._calculate_momentum(play, last_momentum) + if abs(current_momentum - last_momentum) > 0.3: + event = MarketEvent( + event_type=MarketEventType.MOMENTUM_SHIFT, + market_ticker=state.market_ticker, + timestamp=current_time + timedelta(seconds=random.randint(10, 60)), + description=f"Momentum shift detected", + price_impact=current_momentum * 0.1, + volume_impact=0.3, + confidence_impact=0.1 + ) + scenario.events.append(event) + + last_momentum = current_momentum + + # Add quarter-end events + for quarter in range(1, 5): + if any(play.context.quarter == quarter for play in scenario.game.plays): + quarter_time = current_time + timedelta(minutes=quarter * 30) + + event = MarketEvent( + event_type=MarketEventType.PRICE_MOVEMENT, + market_ticker=state.market_ticker, + timestamp=quarter_time, + description=f"End of quarter {quarter} update", + price_impact=random.gauss(0, 0.02), + volume_impact=0.1 + ) + scenario.events.append(event) + + def _generate_postgame_events(self, scenario: TradingScenario, state: MarketState, end_time: datetime): + """Generate post-game market events""" + + # Final settlement + final_event = MarketEvent( + event_type=MarketEventType.PRICE_MOVEMENT, + market_ticker=state.market_ticker, + timestamp=end_time, + description=f"Game final: {scenario.game.away_team} {scenario.game.final_score[1]} - {scenario.game.final_score[0]} {scenario.game.home_team}", + price_impact=1.0 if scenario.game.final_score[0] > scenario.game.final_score[1] else -1.0, # Complete resolution + volume_impact=0.9 + ) + scenario.events.append(final_event) + + def _is_significant_play(self, play: SyntheticPlay) -> bool: + """Check if play is significant enough to generate market event""" + + # Always significant + if play.touchdown or play.field_goal or play.turnover or play.safety: + return True + + # Big yardage plays + if abs(play.yards_gained) >= 20: + return True + + # Fourth quarter plays + if play.context.quarter >= 4: + return True + + # Red zone plays + if play.context.yards_to_goal <= 20: + return True + + # High leverage situations + if play.context.down >= 3 and play.context.distance >= 7: + return True + + return False + + def _calculate_play_price_impact(self, play: SyntheticPlay, game: SyntheticGame) -> float: + """Calculate market price impact of a play""" + + base_impact = 0.0 + + # Scoring plays + if play.touchdown: + base_impact = 0.20 + elif play.field_goal: + base_impact = 0.10 + elif play.safety: + base_impact = 0.15 + + # Turnovers + elif play.turnover: + base_impact = 0.15 + + # Big plays + elif abs(play.yards_gained) >= 30: + base_impact = 0.08 + elif abs(play.yards_gained) >= 20: + base_impact = 0.05 + elif abs(play.yards_gained) >= 10: + base_impact = 0.02 + + # Adjust for possession team + if play.context.possession_team == game.away_team: + base_impact *= -1 # Away team positive impact = negative price impact + + # Time pressure multiplier + if play.context.quarter >= 4: + time_factor = 1 + (4 - play.context.quarter) * 0.5 + base_impact *= time_factor + + # Game situation multiplier + score_diff = abs(play.context.score_differential) + if score_diff <= 3: + base_impact *= 1.5 # Close games have higher impact + elif score_diff <= 7: + base_impact *= 1.2 + elif score_diff >= 21: + base_impact *= 0.5 # Blowouts have lower impact + + # Add random noise + noise = random.gauss(0, 0.02) + return base_impact + noise + + def _calculate_momentum(self, play: SyntheticPlay, last_momentum: float) -> float: + """Calculate current momentum factor""" + + play_momentum = 0.0 + + # Big positive plays increase momentum + if play.touchdown: + play_momentum = 0.8 + elif play.field_goal: + play_momentum = 0.4 + elif play.yards_gained >= 20: + play_momentum = 0.3 + elif play.yards_gained >= 10: + play_momentum = 0.1 + + # Negative plays decrease momentum + elif play.turnover: + play_momentum = -0.8 + elif play.yards_gained <= -5: + play_momentum = -0.3 + elif play.yards_gained <= 0: + play_momentum = -0.1 + + # Adjust for possession team (home = positive momentum) + if play.context.possession_team != play.context.home_team: + play_momentum *= -1 + + # Momentum decay and update + decayed_momentum = last_momentum * 0.9 + new_momentum = decayed_momentum + play_momentum * 0.3 + + return max(-1.0, min(1.0, new_momentum)) + + def _classify_information_events(self, scenario: TradingScenario): + """Separate events into public, private, and delayed categories""" + + for event in scenario.events: + # Determine information classification + if random.random() < scenario.information_delay: + # Delayed information + scenario.delayed_events.append(event) + elif event.event_type in [MarketEventType.NEWS_EVENT, MarketEventType.INJURY_REPORT]: + # Some news might be private initially + if random.random() < 0.3: + scenario.private_events.append(event) + else: + scenario.public_events.append(event) + else: + # Most game events are public + scenario.public_events.append(event) + + def _simulate_market_timeline(self, scenario: TradingScenario, initial_state: MarketState): + """Simulate price and volume over time""" + + current_state = initial_state + timeline_events = sorted(scenario.events, key=lambda x: x.timestamp) + + for event in timeline_events: + # Apply event to market state + self._apply_event_to_state(current_state, event, scenario) + + # Record price point + scenario.price_history.append((event.timestamp, current_state.yes_price)) + scenario.volume_history.append((event.timestamp, current_state.volume)) + + logger.debug(f"Simulated {len(scenario.price_history)} price points for {scenario.market_ticker}") + + def _apply_event_to_state(self, state: MarketState, event: MarketEvent, scenario: TradingScenario): + """Apply market event to current state""" + + # Price impact + if event.price_impact != 0: + # Adjust price based on impact and market efficiency + price_change = event.price_impact * scenario.market_efficiency + + # Apply to yes price + new_yes_price = state.yes_price + price_change + new_yes_price = max(0.01, min(0.99, new_yes_price)) + + state.yes_price = new_yes_price + state.no_price = 1.0 - new_yes_price + state.last_trade_price = new_yes_price + + # Update price change + state.price_change_24h += price_change + + # Volume impact + if event.volume_impact > 0: + volume_multiplier = 1.0 + event.volume_impact + additional_volume = int(state.volume * volume_multiplier * 0.1) + state.volume += additional_volume + + # Update momentum and volatility + if hasattr(event, 'confidence_impact'): + state.momentum += event.confidence_impact * 0.1 + state.momentum = max(-1.0, min(1.0, state.momentum)) + + # Volatility increases with price movements + if abs(event.price_impact) > 0.05: + state.volatility = min(1.0, state.volatility * 1.1) + else: + state.volatility *= self.volatility_decay + + def convert_to_standardized_events(self, scenario: TradingScenario) -> List[StandardizedEvent]: + """Convert market scenario to StandardizedEvent format""" + + events = [] + + for market_event in scenario.events: + # Determine impact level + if abs(market_event.price_impact) >= 0.15: + impact = "high" + elif abs(market_event.price_impact) >= 0.05: + impact = "medium" + else: + impact = "low" + + # Build event data + event_data = { + "market_ticker": market_event.market_ticker, + "event_type": market_event.event_type.value, + "description": market_event.description, + "price_impact": market_event.price_impact, + "volume_impact": market_event.volume_impact, + "confidence_impact": market_event.confidence_impact, + "game_context": market_event.game_context, + "synthetic": True + } + + event = StandardizedEvent( + source="synthetic_market_simulator", + event_type=EventType.MARKET_EVENT, + timestamp=market_event.timestamp, + game_id=scenario.game.game_id, + data=event_data, + confidence=0.99, + impact=impact, + metadata={ + "scenario_id": scenario.scenario_id, + "scenario_type": scenario.scenario_type, + "information_delay": scenario.information_delay, + "market_efficiency": scenario.market_efficiency, + "synthetic": True + }, + raw_data=market_event + ) + + events.append(event) + + logger.info(f"Converted {len(events)} market events to StandardizedEvents") + return events + + def generate_batch_scenarios(self, + games: List[SyntheticGame], + scenario_types: List[str] = None, + information_delays: List[float] = None) -> List[TradingScenario]: + """ + Generate multiple trading scenarios from games + + Args: + games: List of synthetic games + scenario_types: Types of scenarios to generate + information_delays: Different information delay settings + + Returns: + List of trading scenarios + """ + if scenario_types is None: + scenario_types = ["regular", "volatile", "trending", "reversal"] + + if information_delays is None: + information_delays = [0.0, 0.1, 0.2, 0.3] + + scenarios = [] + + logger.info(f"Generating trading scenarios for {len(games)} games...") + + for i, game in enumerate(games): + # Select scenario parameters + scenario_type = random.choice(scenario_types) + info_delay = random.choice(information_delays) + market_efficiency = random.uniform(0.6, 0.9) + + # Create scenario + scenario = self.create_trading_scenario( + game=game, + scenario_type=scenario_type, + information_delay=info_delay, + market_efficiency=market_efficiency + ) + + scenarios.append(scenario) + + # Progress logging + if (i + 1) % 25 == 0: + logger.info(f"Generated {i + 1}/{len(games)} trading scenarios") + + logger.info(f"Generated {len(scenarios)} trading scenarios") + return scenarios + + +# Example usage and testing +if __name__ == "__main__": + import sys + import asyncio + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.generators.game_engine import SyntheticGameEngine + from src.synthetic_data.storage.chromadb_manager import ChromaDBManager + + async def test_market_simulator(): + # Initialize components + chromadb = ChromaDBManager() + game_engine = SyntheticGameEngine(chromadb) + market_sim = MarketSimulator() + + # Generate game + game = await game_engine.generate_single_game( + home_team="KC", + away_team="BUF", + game_type="regular" + ) + + # Create trading scenario + scenario = market_sim.create_trading_scenario( + game=game, + scenario_type="volatile", + information_delay=0.2 + ) + + print(f"Created Trading Scenario: {scenario.market_ticker}") + print(f"Scenario Type: {scenario.scenario_type}") + print(f"Total Events: {len(scenario.events)}") + print(f"Public Events: {len(scenario.public_events)}") + print(f"Private Events: {len(scenario.private_events)}") + print(f"Delayed Events: {len(scenario.delayed_events)}") + print(f"Price History Points: {len(scenario.price_history)}") + + # Show some events + print(f"\nSample Market Events:") + for event in scenario.events[:5]: + print(f" {event.timestamp.strftime('%H:%M:%S')} - {event.event_type.value}: {event.description}") + print(f" Price Impact: {event.price_impact:+.3f}, Volume Impact: {event.volume_impact:.3f}") + + # Convert to StandardizedEvents + events = market_sim.convert_to_standardized_events(scenario) + print(f"\nConverted to {len(events)} StandardizedEvents") + + # Show price evolution + if scenario.price_history: + print(f"\nPrice Evolution:") + print(f" Start: {scenario.price_history[0][1]:.3f}") + print(f" End: {scenario.price_history[-1][1]:.3f}") + print(f" Change: {scenario.price_history[-1][1] - scenario.price_history[0][1]:+.3f}") + + # Run test + asyncio.run(test_market_simulator()) \ No newline at end of file diff --git a/src/synthetic_data/generators/scenario_builder.py b/src/synthetic_data/generators/scenario_builder.py new file mode 100644 index 00000000..d5252f2e --- /dev/null +++ b/src/synthetic_data/generators/scenario_builder.py @@ -0,0 +1,817 @@ +""" +Scenario Builder + +Creates specialized training scenarios including edge cases, +rare events, and specific situations for comprehensive agent training. +""" + +import logging +import random +from typing import List, Dict, Any, Optional, Tuple, Iterator +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +import json +import math + +from .game_engine import SyntheticGameEngine, SyntheticGame, GameContext +from .market_simulator import MarketSimulator, TradingScenario, MarketEvent, MarketEventType +from ..storage.chromadb_manager import ChromaDBManager + +logger = logging.getLogger(__name__) + + +class ScenarioCategory(Enum): + """Categories of training scenarios""" + EDGE_CASES = "edge_cases" + MARKET_CONDITIONS = "market_conditions" + GAME_SITUATIONS = "game_situations" + INFORMATION_ASYMMETRY = "information_asymmetry" + RISK_MANAGEMENT = "risk_management" + BEHAVIORAL_PATTERNS = "behavioral_patterns" + + +@dataclass +class ScenarioTemplate: + """Template for generating specific scenario types""" + name: str + category: ScenarioCategory + description: str + frequency: float # How often this occurs in real data (0-1) + difficulty: float # Training difficulty (0-1) + + # Game parameters + game_type: str = "regular" + game_modifiers: Dict[str, Any] = field(default_factory=dict) + + # Market parameters + market_type: str = "regular" + information_delay: float = 0.1 + market_efficiency: float = 0.8 + volatility_modifier: float = 1.0 + + # Learning objectives + learning_objectives: List[str] = field(default_factory=list) + success_criteria: Dict[str, float] = field(default_factory=dict) + + # Metadata + tags: List[str] = field(default_factory=list) + + +@dataclass +class TrainingScenarioSet: + """Complete set of training scenarios for agent development""" + set_id: str + name: str + description: str + scenarios: List[TradingScenario] = field(default_factory=list) + + # Set characteristics + total_scenarios: int = 0 + difficulty_distribution: Dict[str, int] = field(default_factory=dict) + category_distribution: Dict[str, int] = field(default_factory=dict) + + # Learning progression + beginner_scenarios: List[str] = field(default_factory=list) + intermediate_scenarios: List[str] = field(default_factory=list) + advanced_scenarios: List[str] = field(default_factory=list) + + created_at: datetime = field(default_factory=datetime.now) + + +class ScenarioBuilder: + """ + Builds comprehensive training scenario sets with edge cases, + rare events, and specialized situations for agent training + """ + + def __init__(self, + game_engine: SyntheticGameEngine = None, + market_simulator: MarketSimulator = None, + chromadb_manager: ChromaDBManager = None): + """ + Initialize scenario builder + + Args: + game_engine: Synthetic game engine + market_simulator: Market event simulator + chromadb_manager: ChromaDB for pattern analysis + """ + self.game_engine = game_engine or SyntheticGameEngine() + self.market_simulator = market_simulator or MarketSimulator() + self.chromadb = chromadb_manager or ChromaDBManager() + + # Load scenario templates + self.templates = self._initialize_scenario_templates() + + logger.info(f"Initialized ScenarioBuilder with {len(self.templates)} scenario templates") + + def _initialize_scenario_templates(self) -> Dict[str, ScenarioTemplate]: + """Initialize all scenario templates""" + + templates = {} + + # Edge Case Scenarios + templates["overtime_thriller"] = ScenarioTemplate( + name="Overtime Thriller", + category=ScenarioCategory.EDGE_CASES, + description="Game goes to overtime with multiple lead changes", + frequency=0.05, # ~5% of games + difficulty=0.9, + game_type="regular", + game_modifiers={"force_overtime": True, "close_game": True}, + market_type="volatile", + volatility_modifier=2.0, + learning_objectives=["handle_extreme_volatility", "overtime_betting", "live_adjustments"], + success_criteria={"max_drawdown": 0.15, "profit_factor": 1.2}, + tags=["overtime", "volatile", "rare"] + ) + + templates["weather_chaos"] = ScenarioTemplate( + name="Weather Chaos", + category=ScenarioCategory.EDGE_CASES, + description="Severe weather dramatically changes game dynamics", + frequency=0.02, + difficulty=0.8, + game_type="weather", + game_modifiers={"severe_weather": True, "low_scoring": True}, + market_type="uncertain", + information_delay=0.3, + learning_objectives=["weather_impact", "uncertainty_handling", "information_delays"], + success_criteria={"accuracy": 0.6, "kelly_adherence": 0.8}, + tags=["weather", "uncertainty", "external_factors"] + ) + + templates["injury_impact"] = ScenarioTemplate( + name="Key Injury Impact", + category=ScenarioCategory.EDGE_CASES, + description="Star player injury during game creates major shift", + frequency=0.08, + difficulty=0.7, + game_modifiers={"injury_event": True, "momentum_shift": True}, + market_type="news_driven", + information_delay=0.4, # Injury news takes time to spread + learning_objectives=["injury_assessment", "information_arbitrage", "quick_adaptation"], + success_criteria={"reaction_time": 30, "position_adjustment": 0.3}, + tags=["injury", "news", "information_asymmetry"] + ) + + templates["referee_controversy"] = ScenarioTemplate( + name="Referee Controversy", + category=ScenarioCategory.EDGE_CASES, + description="Controversial referee calls affecting game outcome", + frequency=0.03, + difficulty=0.8, + game_modifiers={"controversial_calls": True, "momentum_swings": True}, + market_type="sentiment_driven", + volatility_modifier=1.5, + learning_objectives=["sentiment_analysis", "controversy_handling", "noise_filtering"], + success_criteria={"emotional_control": 0.9, "signal_noise_ratio": 0.7}, + tags=["controversy", "sentiment", "noise"] + ) + + # Market Condition Scenarios + templates["low_liquidity"] = ScenarioTemplate( + name="Low Liquidity Market", + category=ScenarioCategory.MARKET_CONDITIONS, + description="Market with very low liquidity and wide spreads", + frequency=0.15, + difficulty=0.6, + market_type="illiquid", + game_modifiers={"unpopular_matchup": True}, + learning_objectives=["liquidity_assessment", "spread_management", "position_sizing"], + success_criteria={"spread_cost": 0.05, "execution_quality": 0.8}, + tags=["liquidity", "spreads", "execution"] + ) + + templates["high_volume_spike"] = ScenarioTemplate( + name="High Volume Trading", + category=ScenarioCategory.MARKET_CONDITIONS, + description="Unusual high volume creates pricing inefficiencies", + frequency=0.12, + difficulty=0.5, + market_type="volume_spike", + volatility_modifier=1.3, + learning_objectives=["volume_analysis", "arbitrage_opportunities", "momentum_trading"], + success_criteria={"volume_utilization": 0.7, "timing_accuracy": 0.6}, + tags=["volume", "momentum", "arbitrage"] + ) + + templates["market_manipulation"] = ScenarioTemplate( + name="Market Manipulation", + category=ScenarioCategory.MARKET_CONDITIONS, + description="Artificial price movements from large traders", + frequency=0.05, + difficulty=0.9, + market_type="manipulated", + learning_objectives=["manipulation_detection", "contrarian_strategy", "risk_management"], + success_criteria={"detection_accuracy": 0.8, "avoided_losses": 0.9}, + tags=["manipulation", "detection", "contrarian"] + ) + + # Game Situation Scenarios + templates["fourth_quarter_comeback"] = ScenarioTemplate( + name="Fourth Quarter Comeback", + category=ScenarioCategory.GAME_SITUATIONS, + description="Team mounting dramatic fourth quarter comeback", + frequency=0.25, + difficulty=0.6, + game_type="comeback", + game_modifiers={"large_deficit": True, "fourth_quarter_focus": True}, + market_type="trending", + learning_objectives=["comeback_probability", "live_betting", "momentum_analysis"], + success_criteria={"timing_precision": 0.7, "trend_following": 0.6}, + tags=["comeback", "momentum", "live_betting"] + ) + + templates["defensive_battle"] = ScenarioTemplate( + name="Defensive Battle", + category=ScenarioCategory.GAME_SITUATIONS, + description="Low-scoring defensive game with few opportunities", + frequency=0.20, + difficulty=0.4, + game_type="defensive", + game_modifiers={"low_scoring": True, "field_goals": True}, + market_type="stable", + learning_objectives=["low_scoring_dynamics", "patience", "value_betting"], + success_criteria={"patience_score": 0.8, "value_identification": 0.6}, + tags=["defense", "patience", "value"] + ) + + templates["shootout_game"] = ScenarioTemplate( + name="High-Scoring Shootout", + category=ScenarioCategory.GAME_SITUATIONS, + description="High-scoring game with minimal defense", + frequency=0.18, + difficulty=0.5, + game_type="high_scoring", + game_modifiers={"high_scoring": True, "fast_pace": True}, + volatility_modifier=1.4, + learning_objectives=["high_scoring_dynamics", "pace_analysis", "over_under_betting"], + success_criteria={"pace_recognition": 0.7, "scoring_prediction": 0.6}, + tags=["high_scoring", "pace", "over_under"] + ) + + # Information Asymmetry Scenarios + templates["insider_information"] = ScenarioTemplate( + name="Insider Information", + category=ScenarioCategory.INFORMATION_ASYMMETRY, + description="Some traders have early access to key information", + frequency=0.10, + difficulty=0.8, + information_delay=0.5, # 50% of information is delayed + market_efficiency=0.6, # Less efficient due to asymmetry + learning_objectives=["information_arbitrage", "timing_advantage", "pattern_recognition"], + success_criteria={"information_speed": 0.8, "arbitrage_capture": 0.7}, + tags=["information", "arbitrage", "timing"] + ) + + templates["media_narrative"] = ScenarioTemplate( + name="Media Narrative Bias", + category=ScenarioCategory.INFORMATION_ASYMMETRY, + description="Strong media narrative creates betting bias", + frequency=0.30, + difficulty=0.6, + market_type="narrative_driven", + learning_objectives=["narrative_analysis", "bias_detection", "contrarian_thinking"], + success_criteria={"bias_identification": 0.7, "contrarian_profit": 0.5}, + tags=["media", "narrative", "bias"] + ) + + # Risk Management Scenarios + templates["black_swan_event"] = ScenarioTemplate( + name="Black Swan Event", + category=ScenarioCategory.RISK_MANAGEMENT, + description="Extremely rare event with major market impact", + frequency=0.001, # Very rare + difficulty=0.95, + game_modifiers={"unprecedented_event": True}, + market_type="crisis", + volatility_modifier=3.0, + learning_objectives=["crisis_management", "position_sizing", "stop_losses"], + success_criteria={"maximum_loss": 0.10, "recovery_time": 5}, + tags=["black_swan", "crisis", "risk_management"] + ) + + templates["correlation_breakdown"] = ScenarioTemplate( + name="Correlation Breakdown", + category=ScenarioCategory.RISK_MANAGEMENT, + description="Normal market correlations break down unexpectedly", + frequency=0.02, + difficulty=0.85, + market_type="decorrelated", + learning_objectives=["correlation_monitoring", "portfolio_risk", "hedging"], + success_criteria={"correlation_detection": 0.8, "hedge_effectiveness": 0.7}, + tags=["correlation", "portfolio", "hedging"] + ) + + # Behavioral Pattern Scenarios + templates["herd_mentality"] = ScenarioTemplate( + name="Herd Mentality", + category=ScenarioCategory.BEHAVIORAL_PATTERNS, + description="Market exhibits strong herding behavior", + frequency=0.25, + difficulty=0.5, + market_type="herding", + learning_objectives=["crowd_psychology", "contrarian_signals", "behavioral_finance"], + success_criteria={"herd_identification": 0.7, "contrarian_timing": 0.6}, + tags=["herding", "psychology", "behavioral"] + ) + + templates["overreaction_pattern"] = ScenarioTemplate( + name="Market Overreaction", + category=ScenarioCategory.BEHAVIORAL_PATTERNS, + description="Market overreacts to news then corrects", + frequency=0.40, + difficulty=0.4, + market_type="overreaction", + learning_objectives=["overreaction_detection", "mean_reversion", "patience"], + success_criteria={"overreaction_timing": 0.6, "reversion_capture": 0.5}, + tags=["overreaction", "mean_reversion", "behavioral"] + ) + + return templates + + async def build_comprehensive_training_set(self, + num_scenarios: int = 1000, + difficulty_progression: bool = True, + include_edge_cases: bool = True, + custom_weights: Dict[str, float] = None) -> TrainingScenarioSet: + """ + Build comprehensive training scenario set + + Args: + num_scenarios: Total number of scenarios to generate + difficulty_progression: Whether to include progressive difficulty + include_edge_cases: Whether to include rare edge cases + custom_weights: Custom weights for scenario types + + Returns: + Complete training scenario set + """ + set_id = f"training_set_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + logger.info(f"Building comprehensive training set with {num_scenarios} scenarios") + + training_set = TrainingScenarioSet( + set_id=set_id, + name="Comprehensive Agent Training Set", + description=f"Complete training set with {num_scenarios} diverse scenarios including edge cases" + ) + + # Calculate scenario distribution + distribution = self._calculate_scenario_distribution( + num_scenarios, + include_edge_cases, + custom_weights + ) + + # Generate scenarios by template + all_scenarios = [] + + for template_name, count in distribution.items(): + template = self.templates[template_name] + + logger.info(f"Generating {count} scenarios for: {template.name}") + + scenarios = await self._generate_scenarios_from_template(template, count) + all_scenarios.extend(scenarios) + + # Update distribution tracking + category = template.category.value + training_set.category_distribution[category] = training_set.category_distribution.get(category, 0) + count + + difficulty_level = self._get_difficulty_level(template.difficulty) + training_set.difficulty_distribution[difficulty_level] = training_set.difficulty_distribution.get(difficulty_level, 0) + count + + # Sort scenarios for progressive training + if difficulty_progression: + all_scenarios = self._sort_scenarios_by_difficulty(all_scenarios) + else: + random.shuffle(all_scenarios) + + training_set.scenarios = all_scenarios + training_set.total_scenarios = len(all_scenarios) + + # Classify scenarios by difficulty + self._classify_scenarios_by_difficulty(training_set) + + logger.info(f"Generated {len(all_scenarios)} total training scenarios") + logger.info(f"Category distribution: {training_set.category_distribution}") + logger.info(f"Difficulty distribution: {training_set.difficulty_distribution}") + + return training_set + + def _calculate_scenario_distribution(self, + total_scenarios: int, + include_edge_cases: bool, + custom_weights: Dict[str, float] = None) -> Dict[str, int]: + """Calculate how many scenarios to generate for each template""" + + distribution = {} + + if custom_weights: + # Use custom weights + total_weight = sum(custom_weights.values()) + for template_name, weight in custom_weights.items(): + if template_name in self.templates: + count = int(total_scenarios * weight / total_weight) + distribution[template_name] = count + else: + # Use frequency-based distribution + total_weight = 0 + weights = {} + + for template_name, template in self.templates.items(): + # Skip very rare events if not including edge cases + if not include_edge_cases and template.frequency < 0.01: + continue + + weight = template.frequency + weights[template_name] = weight + total_weight += weight + + # Distribute scenarios + remaining = total_scenarios + for template_name, weight in weights.items(): + if remaining <= 0: + break + + count = max(1, int(total_scenarios * weight / total_weight)) + count = min(count, remaining) + distribution[template_name] = count + remaining -= count + + # Distribute any remaining scenarios + while remaining > 0: + for template_name in weights.keys(): + if remaining <= 0: + break + distribution[template_name] = distribution.get(template_name, 0) + 1 + remaining -= 1 + + return distribution + + async def _generate_scenarios_from_template(self, + template: ScenarioTemplate, + count: int) -> List[TradingScenario]: + """Generate multiple scenarios from a template""" + + scenarios = [] + + for i in range(count): + try: + # Generate base game with template modifiers + game = await self._generate_game_from_template(template) + + # Create trading scenario + scenario = self._create_scenario_from_template(template, game, i) + + scenarios.append(scenario) + + except Exception as e: + logger.warning(f"Failed to generate scenario {i} for {template.name}: {e}") + continue + + return scenarios + + async def _generate_game_from_template(self, template: ScenarioTemplate) -> SyntheticGame: + """Generate game based on template specifications""" + + # Select teams + teams = random.sample(self.game_engine.nfl_teams, 2) + + # Generate base game + game = await self.game_engine.generate_single_game( + home_team=teams[0], + away_team=teams[1], + game_type=template.game_type, + season=2024, + week=random.randint(1, 18) + ) + + # Apply template modifiers + self._apply_game_modifiers(game, template.game_modifiers) + + return game + + def _apply_game_modifiers(self, game: SyntheticGame, modifiers: Dict[str, Any]): + """Apply template modifiers to generated game""" + + if modifiers.get("force_overtime"): + # Ensure overtime by adjusting final score + game.final_score = (21, 21) # Will trigger overtime + + if modifiers.get("close_game"): + # Ensure close final score + diff = abs(game.final_score[0] - game.final_score[1]) + if diff > 7: + if game.final_score[0] > game.final_score[1]: + game.final_score = (game.final_score[0], game.final_score[0] - 3) + else: + game.final_score = (game.final_score[1] - 3, game.final_score[1]) + + if modifiers.get("large_deficit"): + # Create large deficit scenario + if random.random() > 0.5: + game.final_score = (35, 14) # Home team wins big + else: + game.final_score = (14, 35) # Away team wins big + + if modifiers.get("low_scoring"): + # Ensure low-scoring game + total_points = sum(game.final_score) + if total_points > 35: + ratio = game.final_score[0] / game.final_score[1] if game.final_score[1] > 0 else 1 + game.final_score = (int(20 * ratio), 20) + + if modifiers.get("high_scoring"): + # Ensure high-scoring game + total_points = sum(game.final_score) + if total_points < 50: + ratio = game.final_score[0] / game.final_score[1] if game.final_score[1] > 0 else 1 + game.final_score = (int(35 * ratio), 35) + + # Update game characteristics + if modifiers.get("severe_weather"): + game.game_type = "weather" + + if modifiers.get("controversial_calls"): + game.excitement_score = min(1.0, game.excitement_score + 0.3) + + def _create_scenario_from_template(self, + template: ScenarioTemplate, + game: SyntheticGame, + index: int) -> TradingScenario: + """Create trading scenario based on template""" + + # Create base scenario + scenario = self.market_simulator.create_trading_scenario( + game=game, + scenario_type=template.market_type, + information_delay=template.information_delay, + market_efficiency=template.market_efficiency + ) + + # Apply template-specific modifications + self._apply_scenario_modifiers(scenario, template) + + # Add template metadata + scenario.scenario_id = f"{template.name.lower().replace(' ', '_')}_{index}" + + # Add learning metadata + if hasattr(scenario, 'metadata'): + scenario.metadata.update({ + "template": template.name, + "category": template.category.value, + "difficulty": template.difficulty, + "learning_objectives": template.learning_objectives, + "success_criteria": template.success_criteria, + "tags": template.tags + }) + else: + scenario.metadata = { + "template": template.name, + "category": template.category.value, + "difficulty": template.difficulty, + "learning_objectives": template.learning_objectives, + "success_criteria": template.success_criteria, + "tags": template.tags + } + + return scenario + + def _apply_scenario_modifiers(self, scenario: TradingScenario, template: ScenarioTemplate): + """Apply template-specific modifications to scenario""" + + # Volatility modifications + if template.volatility_modifier != 1.0: + for event in scenario.events: + event.price_impact *= template.volatility_modifier + event.volume_impact *= min(1.0, template.volatility_modifier) + + # Add template-specific events + if template.category == ScenarioCategory.EDGE_CASES: + self._add_edge_case_events(scenario, template) + elif template.category == ScenarioCategory.INFORMATION_ASYMMETRY: + self._add_information_asymmetry_events(scenario, template) + elif template.category == ScenarioCategory.RISK_MANAGEMENT: + self._add_risk_events(scenario, template) + + def _add_edge_case_events(self, scenario: TradingScenario, template: ScenarioTemplate): + """Add edge case specific events""" + + if "injury" in template.tags: + # Add sudden injury event + injury_time = random.choice(scenario.events).timestamp + injury_event = MarketEvent( + event_type=MarketEventType.NEWS_EVENT, + market_ticker=scenario.market_ticker, + timestamp=injury_time, + description="Star player injured during play", + price_impact=random.choice([-0.25, 0.25]), + volume_impact=0.8, + confidence_impact=-0.3 + ) + scenario.events.append(injury_event) + + if "weather" in template.tags: + # Add weather update events + weather_event = MarketEvent( + event_type=MarketEventType.WEATHER_UPDATE, + market_ticker=scenario.market_ticker, + timestamp=scenario.events[0].timestamp - timedelta(hours=1), + description="Severe weather conditions developing", + price_impact=random.gauss(0, 0.15), + volume_impact=0.4, + confidence_impact=-0.2 + ) + scenario.events.insert(0, weather_event) + + def _add_information_asymmetry_events(self, scenario: TradingScenario, template: ScenarioTemplate): + """Add information asymmetry specific events""" + + # Move some events to private/delayed + num_private = int(len(scenario.events) * 0.3) + private_events = random.sample(scenario.events, num_private) + + for event in private_events: + scenario.private_events.append(event) + if event in scenario.public_events: + scenario.public_events.remove(event) + + def _add_risk_events(self, scenario: TradingScenario, template: ScenarioTemplate): + """Add risk management specific events""" + + if "black_swan" in template.tags: + # Add extreme market event + extreme_event = MarketEvent( + event_type=MarketEventType.NEWS_EVENT, + market_ticker=scenario.market_ticker, + timestamp=random.choice(scenario.events).timestamp, + description="Unprecedented event affects game", + price_impact=random.choice([-0.5, 0.5]), + volume_impact=1.0, + confidence_impact=-0.5 + ) + scenario.events.append(extreme_event) + + def _get_difficulty_level(self, difficulty: float) -> str: + """Convert difficulty score to level""" + if difficulty < 0.4: + return "beginner" + elif difficulty < 0.7: + return "intermediate" + else: + return "advanced" + + def _sort_scenarios_by_difficulty(self, scenarios: List[TradingScenario]) -> List[TradingScenario]: + """Sort scenarios by difficulty for progressive training""" + + def get_scenario_difficulty(scenario): + return scenario.metadata.get("difficulty", 0.5) + + return sorted(scenarios, key=get_scenario_difficulty) + + def _classify_scenarios_by_difficulty(self, training_set: TrainingScenarioSet): + """Classify scenarios into difficulty levels""" + + for scenario in training_set.scenarios: + difficulty = scenario.metadata.get("difficulty", 0.5) + level = self._get_difficulty_level(difficulty) + + if level == "beginner": + training_set.beginner_scenarios.append(scenario.scenario_id) + elif level == "intermediate": + training_set.intermediate_scenarios.append(scenario.scenario_id) + else: + training_set.advanced_scenarios.append(scenario.scenario_id) + + def export_training_set(self, training_set: TrainingScenarioSet, output_path: str): + """Export training set to file""" + + export_data = { + "set_id": training_set.set_id, + "name": training_set.name, + "description": training_set.description, + "total_scenarios": training_set.total_scenarios, + "difficulty_distribution": training_set.difficulty_distribution, + "category_distribution": training_set.category_distribution, + "beginner_scenarios": training_set.beginner_scenarios, + "intermediate_scenarios": training_set.intermediate_scenarios, + "advanced_scenarios": training_set.advanced_scenarios, + "created_at": training_set.created_at.isoformat(), + "scenarios": [ + { + "scenario_id": scenario.scenario_id, + "market_ticker": scenario.market_ticker, + "scenario_type": scenario.scenario_type, + "information_delay": scenario.information_delay, + "market_efficiency": scenario.market_efficiency, + "num_events": len(scenario.events), + "game_final_score": scenario.game.final_score, + "metadata": getattr(scenario, 'metadata', {}) + } + for scenario in training_set.scenarios + ] + } + + with open(output_path, 'w') as f: + json.dump(export_data, f, indent=2) + + logger.info(f"Exported training set to {output_path}") + + def get_scenario_statistics(self, training_set: TrainingScenarioSet) -> Dict[str, Any]: + """Get comprehensive statistics about training set""" + + stats = { + "overview": { + "total_scenarios": training_set.total_scenarios, + "categories": len(training_set.category_distribution), + "difficulty_levels": len(training_set.difficulty_distribution) + }, + "category_distribution": training_set.category_distribution, + "difficulty_distribution": training_set.difficulty_distribution, + "progression": { + "beginner": len(training_set.beginner_scenarios), + "intermediate": len(training_set.intermediate_scenarios), + "advanced": len(training_set.advanced_scenarios) + } + } + + # Calculate learning objective coverage + all_objectives = set() + objective_counts = {} + + for scenario in training_set.scenarios: + objectives = scenario.metadata.get("learning_objectives", []) + for obj in objectives: + all_objectives.add(obj) + objective_counts[obj] = objective_counts.get(obj, 0) + 1 + + stats["learning_objectives"] = { + "total_unique": len(all_objectives), + "coverage": objective_counts + } + + # Calculate tag distribution + tag_counts = {} + for scenario in training_set.scenarios: + tags = scenario.metadata.get("tags", []) + for tag in tags: + tag_counts[tag] = tag_counts.get(tag, 0) + 1 + + stats["tags"] = tag_counts + + return stats + + +# Example usage and testing +if __name__ == "__main__": + import sys + import asyncio + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.generators.game_engine import SyntheticGameEngine + from src.synthetic_data.generators.market_simulator import MarketSimulator + from src.synthetic_data.storage.chromadb_manager import ChromaDBManager + + async def test_scenario_builder(): + # Initialize components + chromadb = ChromaDBManager() + game_engine = SyntheticGameEngine(chromadb) + market_simulator = MarketSimulator() + scenario_builder = ScenarioBuilder(game_engine, market_simulator, chromadb) + + # Build comprehensive training set + training_set = await scenario_builder.build_comprehensive_training_set( + num_scenarios=100, # Small test set + difficulty_progression=True, + include_edge_cases=True + ) + + print(f"Built Training Set: {training_set.name}") + print(f"Total Scenarios: {training_set.total_scenarios}") + print(f"Categories: {training_set.category_distribution}") + print(f"Difficulty: {training_set.difficulty_distribution}") + + # Get statistics + stats = scenario_builder.get_scenario_statistics(training_set) + print(f"\nDetailed Statistics:") + print(f"Learning Objectives: {stats['learning_objectives']['total_unique']}") + print(f"Tag Distribution: {list(stats['tags'].keys())}") + + # Show sample scenarios + print(f"\nSample Scenarios:") + for scenario in training_set.scenarios[:3]: + print(f" {scenario.scenario_id}: {scenario.metadata.get('template', 'Unknown')}") + print(f" Difficulty: {scenario.metadata.get('difficulty', 0):.2f}") + print(f" Events: {len(scenario.events)}") + print(f" Objectives: {scenario.metadata.get('learning_objectives', [])}") + + # Export training set + scenario_builder.export_training_set(training_set, "test_training_set.json") + print(f"\nTraining set exported to test_training_set.json") + + # Run test + asyncio.run(test_scenario_builder()) \ No newline at end of file diff --git a/src/synthetic_data/models/__init__.py b/src/synthetic_data/models/__init__.py new file mode 100644 index 00000000..f2b1912c --- /dev/null +++ b/src/synthetic_data/models/__init__.py @@ -0,0 +1,14 @@ +""" +Fine-tuned Language Models Module + +Wrappers for fine-tuned LFM2 models and +sequence pattern definitions. +""" + +from .lfm2_fine_tuner import LFM2FineTuner, FineTuningConfig, TrainingDataset + +__all__ = [ + 'LFM2FineTuner', + 'FineTuningConfig', + 'TrainingDataset' +] \ No newline at end of file diff --git a/src/synthetic_data/models/lfm2_fine_tuner.py b/src/synthetic_data/models/lfm2_fine_tuner.py new file mode 100644 index 00000000..a56bde86 --- /dev/null +++ b/src/synthetic_data/models/lfm2_fine_tuner.py @@ -0,0 +1,561 @@ +""" +LFM2 Fine-tuning Pipeline + +Fine-tunes local LiquidAI LFM2 models on NFL play-by-play sequences +for synthetic game data generation. +""" + +import logging +from typing import List, Dict, Any, Optional, Iterator, Tuple +from pathlib import Path +import json +import random +from dataclasses import dataclass +from datetime import datetime + +from ..preprocessing.nfl_dataset_processor import ProcessedNFLPlay +from ..preprocessing.training_data_builder import TrainingSequence, TrainingDataBuilder + +logger = logging.getLogger(__name__) + + +@dataclass +class FineTuningConfig: + """Configuration for LFM2 fine-tuning""" + model_name: str = "liquid/lfm-2-1_2b-q4_k_m" # Local LFM2 model + output_dir: str = "data/models/fine_tuned_lfm2" + training_data_dir: str = "data/training_sequences" + + # Training hyperparameters + learning_rate: float = 5e-5 + batch_size: int = 4 + max_epochs: int = 3 + max_seq_length: int = 512 + gradient_accumulation_steps: int = 8 + warmup_steps: int = 100 + + # Data parameters + train_split: float = 0.8 + validation_split: float = 0.15 + test_split: float = 0.05 + + # Generation parameters + temperature: float = 0.8 + top_p: float = 0.9 + max_new_tokens: int = 150 + + +@dataclass +class TrainingDataset: + """Dataset for model training""" + sequences: List[TrainingSequence] + total_size: int + vocab_size: Optional[int] = None + + def train_test_split(self, config: FineTuningConfig) -> Tuple['TrainingDataset', 'TrainingDataset', 'TrainingDataset']: + """Split dataset into train/validation/test""" + shuffled = self.sequences.copy() + random.shuffle(shuffled) + + n = len(shuffled) + train_end = int(n * config.train_split) + val_end = train_end + int(n * config.validation_split) + + train_data = TrainingDataset(shuffled[:train_end], train_end) + val_data = TrainingDataset(shuffled[train_end:val_end], val_end - train_end) + test_data = TrainingDataset(shuffled[val_end:], n - val_end) + + return train_data, val_data, test_data + + +class LFM2FineTuner: + """ + Fine-tunes LFM2 models for NFL play-by-play generation + """ + + def __init__(self, config: FineTuningConfig = None): + """ + Initialize fine-tuner + + Args: + config: Fine-tuning configuration + """ + self.config = config or FineTuningConfig() + self.training_data_builder = TrainingDataBuilder() + + # Ensure output directories exist + Path(self.config.output_dir).mkdir(parents=True, exist_ok=True) + Path(self.config.training_data_dir).mkdir(parents=True, exist_ok=True) + + logger.info(f"Initialized LFM2 fine-tuner with output dir: {self.config.output_dir}") + + def prepare_training_data(self, plays: List[ProcessedNFLPlay]) -> TrainingDataset: + """ + Prepare NFL plays for model training + + Args: + plays: List of processed NFL plays + + Returns: + TrainingDataset ready for fine-tuning + """ + logger.info(f"Preparing training data from {len(plays)} plays...") + + # Build training sequences + sequences = self.training_data_builder.build_sequences(plays) + + # Filter and validate sequences + valid_sequences = [] + for seq in sequences: + if self._validate_training_sequence(seq): + valid_sequences.append(seq) + + logger.info(f"Created {len(valid_sequences)} valid training sequences") + + dataset = TrainingDataset( + sequences=valid_sequences, + total_size=len(valid_sequences) + ) + + # Save dataset for later use + self._save_training_dataset(dataset) + + return dataset + + def _validate_training_sequence(self, sequence: TrainingSequence) -> bool: + """Validate training sequence quality""" + # Check minimum lengths + if len(sequence.context.split()) < 10: + return False + if len(sequence.target.split()) < 5: + return False + + # Check maximum lengths + total_tokens = len(sequence.context.split()) + len(sequence.target.split()) + if total_tokens > self.config.max_seq_length: + return False + + # Check for required content + if not sequence.context.strip() or not sequence.target.strip(): + return False + + return True + + def _save_training_dataset(self, dataset: TrainingDataset): + """Save training dataset to disk""" + dataset_path = Path(self.config.training_data_dir) / "training_dataset.json" + + dataset_dict = { + 'sequences': [ + { + 'context': seq.context, + 'target': seq.target, + 'metadata': seq.metadata + } + for seq in dataset.sequences + ], + 'total_size': dataset.total_size, + 'created_at': datetime.now().isoformat() + } + + with open(dataset_path, 'w') as f: + json.dump(dataset_dict, f, indent=2) + + logger.info(f"Saved training dataset to {dataset_path}") + + def load_training_dataset(self) -> Optional[TrainingDataset]: + """Load previously saved training dataset""" + dataset_path = Path(self.config.training_data_dir) / "training_dataset.json" + + if not dataset_path.exists(): + logger.warning("No saved training dataset found") + return None + + try: + with open(dataset_path, 'r') as f: + dataset_dict = json.load(f) + + sequences = [] + for seq_data in dataset_dict['sequences']: + sequences.append(TrainingSequence( + context=seq_data['context'], + target=seq_data['target'], + metadata=seq_data['metadata'] + )) + + dataset = TrainingDataset( + sequences=sequences, + total_size=dataset_dict['total_size'] + ) + + logger.info(f"Loaded training dataset with {dataset.total_size} sequences") + return dataset + + except Exception as e: + logger.error(f"Error loading training dataset: {e}") + return None + + def create_ollama_modelfile(self, base_model: str = "liquid/lfm-2-1_2b-q4_k_m") -> str: + """ + Create Ollama Modelfile for fine-tuning + + Args: + base_model: Base model to fine-tune from + + Returns: + Path to created Modelfile + """ + # Create modelfile content without f-strings to avoid brace issues + modelfile_content = f"""FROM {base_model} + +# NFL Play-by-Play Generation Model +# Fine-tuned on historical NFL data for synthetic game generation + +TEMPLATE \"\"\"{{{{ if .System }}}}<|system|> +{{{{ .System }}}}<|end|> +{{{{ end }}}}{{{{ if .Prompt }}}}<|user|> +{{{{ .Prompt }}}}<|end|> +{{{{ end }}}}<|assistant|> +{{{{ .Response }}}}<|end|> +\"\"\" + +SYSTEM \"\"\"You are an expert NFL game analyst that generates realistic play-by-play sequences. + +Generate plays that are: +- Tactically sound for the given situation +- Realistic in terms of outcomes and statistics +- Consistent with team tendencies and game context +- Properly formatted with down, distance, field position, and play result + +Always maintain game flow and situational awareness.\"\"\" + +PARAMETER temperature {self.config.temperature} +PARAMETER top_p {self.config.top_p} +PARAMETER top_k 40 +PARAMETER repeat_penalty 1.1 +PARAMETER num_ctx 2048 + +LICENSE \"\"\" +Fine-tuned NFL Play-by-Play Generation Model +Based on LiquidAI LFM2-1.2B +Training data: Historical NFL plays 2009-2018 +\"\"\" +""" + + # Fix the template braces + modelfile_content = modelfile_content.replace('{{{{', '{{').replace('}}}}', '}}') + + modelfile_path = Path(self.config.output_dir) / "Modelfile" + + with open(modelfile_path, 'w') as f: + f.write(modelfile_content) + + logger.info(f"Created Ollama Modelfile at {modelfile_path}") + return str(modelfile_path) + + def format_training_data_for_ollama(self, dataset: TrainingDataset) -> str: + """ + Format training data for Ollama fine-tuning + + Args: + dataset: Training dataset + + Returns: + Path to formatted training file + """ + training_file = Path(self.config.training_data_dir) / "ollama_training.jsonl" + + with open(training_file, 'w') as f: + for sequence in dataset.sequences: + # Format as Ollama training example + training_example = { + "prompt": sequence.context, + "response": sequence.target, + "metadata": sequence.metadata + } + + f.write(json.dumps(training_example) + '\n') + + logger.info(f"Formatted {len(dataset.sequences)} sequences for Ollama at {training_file}") + return str(training_file) + + def create_fine_tuning_script(self) -> str: + """Create shell script for fine-tuning with Ollama""" + + script_content = f'''#!/bin/bash + +# LFM2 Fine-tuning Script for NFL Play-by-Play Generation +# Auto-generated by LFM2FineTuner + +set -e + +echo "Starting LFM2 fine-tuning for NFL play-by-play generation..." + +# Configuration +MODEL_NAME="nfl_playbypay_lfm2" +BASE_MODEL="liquid/lfm-2-1_2b-q4_k_m" +OUTPUT_DIR="{self.config.output_dir}" +TRAINING_DATA="{Path(self.config.training_data_dir) / 'ollama_training.jsonl'}" +MODELFILE="{Path(self.config.output_dir) / 'Modelfile'}" + +echo "Model: $MODEL_NAME" +echo "Base: $BASE_MODEL" +echo "Training data: $TRAINING_DATA" +echo "Output: $OUTPUT_DIR" + +# Check if Ollama is installed +if ! command -v ollama &> /dev/null; then + echo "Error: Ollama not found. Please install Ollama first:" + echo "curl -fsSL https://ollama.ai/install.sh | sh" + exit 1 +fi + +# Check if base model exists +echo "Checking base model availability..." +if ! ollama list | grep -q "{self.config.model_name}"; then + echo "Pulling base model..." + ollama pull {self.config.model_name} +fi + +# Create model from Modelfile +echo "Creating fine-tuned model..." +ollama create $MODEL_NAME -f $MODELFILE + +# Test the model +echo "Testing fine-tuned model..." +ollama run $MODEL_NAME "Generate a 3rd down and 8 play from the opponent 25 yard line, 2nd quarter, tied game:" + +echo "Fine-tuning complete! Model '$MODEL_NAME' is ready for use." +echo "" +echo "Usage examples:" +echo "ollama run $MODEL_NAME 'Generate a red zone touchdown drive'" +echo "ollama run $MODEL_NAME '4th and 1 at midfield, trailing by 3, 4th quarter'" +echo "" +echo "Model saved to: $OUTPUT_DIR" +''' + + script_path = Path(self.config.output_dir) / "fine_tune.sh" + + with open(script_path, 'w') as f: + f.write(script_content) + + # Make script executable + script_path.chmod(0o755) + + logger.info(f"Created fine-tuning script at {script_path}") + return str(script_path) + + def generate_synthetic_play(self, context: str, model_name: str = "nfl_playbypay_lfm2") -> str: + """ + Generate synthetic play using fine-tuned model + + Args: + context: Game situation context + model_name: Name of fine-tuned model + + Returns: + Generated play description + """ + # This would integrate with Ollama API to generate plays + # For now, return a template for the integration + + prompt = f"""Given the following game situation, generate the next realistic NFL play: + +Context: {context} + +Generate a play that includes: +- Play call (pass/run/special) +- Outcome and yards gained/lost +- Updated down and distance +- Any notable events (tackles, penalties, scores) + +Play:""" + + # In a real implementation, this would call: + # response = ollama.generate(model=model_name, prompt=prompt, options={...}) + # return response['response'] + + logger.info(f"Would generate play using model '{model_name}' with context: {context[:50]}...") + return "PLACEHOLDER: Generated play would appear here" + + def create_full_pipeline(self, plays: List[ProcessedNFLPlay]) -> Dict[str, str]: + """ + Create complete fine-tuning pipeline + + Args: + plays: NFL plays for training + + Returns: + Dictionary of created file paths + """ + logger.info("Creating complete LFM2 fine-tuning pipeline...") + + # Prepare training data + dataset = self.prepare_training_data(plays) + + # Create Ollama files + modelfile_path = self.create_ollama_modelfile() + training_data_path = self.format_training_data_for_ollama(dataset) + script_path = self.create_fine_tuning_script() + + # Create README + readme_path = self._create_readme(dataset) + + pipeline_files = { + 'modelfile': modelfile_path, + 'training_data': training_data_path, + 'fine_tune_script': script_path, + 'readme': readme_path, + 'config': str(Path(self.config.output_dir) / "config.json") + } + + # Save config + with open(pipeline_files['config'], 'w') as f: + json.dump(self.config.__dict__, f, indent=2) + + logger.info("LFM2 fine-tuning pipeline created successfully!") + logger.info(f"Files created: {list(pipeline_files.keys())}") + + return pipeline_files + + def _create_readme(self, dataset: TrainingDataset) -> str: + """Create README for the fine-tuning pipeline""" + + readme_content = f"""# NFL Play-by-Play LFM2 Fine-tuning Pipeline + +This pipeline fine-tunes a local LiquidAI LFM2-1.2B model on historical NFL play-by-play data to generate synthetic game scenarios. + +## Dataset Statistics +- Training sequences: {dataset.total_size:,} +- Base model: {self.config.model_name} +- Max sequence length: {self.config.max_seq_length} tokens + +## Quick Start + +1. **Install Ollama** (if not already installed): + ```bash + curl -fsSL https://ollama.ai/install.sh | sh + ``` + +2. **Run fine-tuning**: + ```bash + ./fine_tune.sh + ``` + +3. **Test the model**: + ```bash + ollama run nfl_playbypay_lfm2 "3rd and 7 at the 35 yard line, 2 minutes left, down by 3:" + ``` + +## Files Description + +- `Modelfile`: Ollama model configuration +- `ollama_training.jsonl`: Training data in Ollama format +- `fine_tune.sh`: Automated fine-tuning script +- `config.json`: Pipeline configuration +- `training_dataset.json`: Original training sequences + +## Model Usage Examples + +### Game Situations +```bash +# Red zone scoring +ollama run nfl_playbypay_lfm2 "1st and goal from the 8 yard line, 4th quarter, trailing by 4:" + +# Two-minute drill +ollama run nfl_playbypay_lfm2 "2nd and 10 at own 25, 1:47 remaining, no timeouts, down by 7:" + +# Short yardage +ollama run nfl_playbypay_lfm2 "4th and 1 at midfield, 3rd quarter, tied game:" +``` + +### Drive Generation +```bash +# Full drive +ollama run nfl_playbypay_lfm2 "Generate a touchdown drive starting from own 20 yard line:" + +# Specific scenarios +ollama run nfl_playbypay_lfm2 "Generate a game-winning drive, 2 minutes left:" +``` + +## Training Configuration + +- Learning rate: {self.config.learning_rate} +- Batch size: {self.config.batch_size} +- Max epochs: {self.config.max_epochs} +- Temperature: {self.config.temperature} +- Top-p: {self.config.top_p} + +## Integration with Agno Agents + +The fine-tuned model integrates with the Agno agent framework for: + +1. **Synthetic Data Generation**: Create unlimited game scenarios +2. **Agent Training**: Provide diverse situations for agent learning +3. **Market Simulation**: Generate events for Kalshi contract simulation + +## Performance Expectations + +- **Generation Speed**: ~50-100 tokens/second on CPU +- **Memory Usage**: ~4GB RAM for inference +- **Quality**: Realistic play sequences with proper game logic +- **Diversity**: Varied outcomes based on situation context + +## Troubleshooting + +### Model Not Found +```bash +ollama pull liquid/lfm-2-1_2b-q4_k_m +``` + +### Out of Memory +Reduce batch size in config or use GPU acceleration: +```bash +OLLAMA_NUM_GPU=1 ollama serve +``` + +### Training Data Issues +Regenerate training data with different parameters: +```python +config.max_seq_length = 256 # Reduce sequence length +config.train_split = 0.9 # Use more training data +``` + +## Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} +""" + + readme_path = Path(self.config.output_dir) / "README.md" + + with open(readme_path, 'w') as f: + f.write(readme_content) + + logger.info(f"Created README at {readme_path}") + return str(readme_path) + + +# Example usage and testing +if __name__ == "__main__": + import sys + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.preprocessing.nfl_dataset_processor import NFLDatasetProcessor + + # Test LFM2 fine-tuner + processor = NFLDatasetProcessor(data_dir='data/nfl_source') + fine_tuner = LFM2FineTuner() + + # Load sample data + df = processor.load_dataset('2009-2016') + sample_df = df.head(1000) # Use 1000 plays for testing + processed_plays = processor.process_plays(sample_df) + + # Create fine-tuning pipeline + pipeline_files = fine_tuner.create_full_pipeline(processed_plays) + + print("LFM2 Fine-tuning Pipeline Created!") + print("Files:") + for name, path in pipeline_files.items(): + print(f" {name}: {path}") + + print(f"\nTo start fine-tuning, run:") + print(f"cd {fine_tuner.config.output_dir} && ./fine_tune.sh") \ No newline at end of file diff --git a/src/synthetic_data/preprocessing/__init__.py b/src/synthetic_data/preprocessing/__init__.py new file mode 100644 index 00000000..274f5a89 --- /dev/null +++ b/src/synthetic_data/preprocessing/__init__.py @@ -0,0 +1,11 @@ +""" +NFL Dataset Preprocessing Module + +Handles processing of historical NFL play-by-play data (2009-2016) +and conversion to formats suitable for machine learning training. +""" + +from .nfl_dataset_processor import NFLDatasetProcessor +from .training_data_builder import TrainingDataBuilder + +__all__ = ["NFLDatasetProcessor", "TrainingDataBuilder"] \ No newline at end of file diff --git a/src/synthetic_data/preprocessing/nfl_dataset_processor.py b/src/synthetic_data/preprocessing/nfl_dataset_processor.py new file mode 100644 index 00000000..0c828cdc --- /dev/null +++ b/src/synthetic_data/preprocessing/nfl_dataset_processor.py @@ -0,0 +1,434 @@ +""" +NFL Dataset Processor + +Processes historical NFL play-by-play CSV data (2009-2016/2017/2018) +and converts to standardized format for machine learning training. +""" + +import pandas as pd +import logging +from typing import Dict, List, Optional, Tuple +from datetime import datetime +from dataclasses import dataclass +from pathlib import Path + +from src.sdk.core.base_adapter import StandardizedEvent, EventType + +logger = logging.getLogger(__name__) + + +@dataclass +class ProcessedNFLPlay: + """Standardized NFL play data structure""" + game_id: str + play_id: str + date: str + season: int + week: int + + # Game context + quarter: int + time_remaining: str + time_seconds: int + home_team: str + away_team: str + possession_team: str + defensive_team: str + + # Situation + down: Optional[int] + distance: Optional[int] + yard_line: Optional[int] + yards_to_goal: Optional[int] + field_position: str + + # Play details + play_type: str + play_description: str + yards_gained: Optional[int] + + # Advanced metrics + expected_points: Optional[float] + epa: Optional[float] # Expected Points Added + win_probability_pre: Optional[float] + win_probability_post: Optional[float] + wpa: Optional[float] # Win Probability Added + + # Score context + score_home: Optional[int] + score_away: Optional[int] + score_differential: Optional[int] + + # Play outcomes + touchdown: bool + field_goal: bool + turnover: bool + safety: bool + penalty: bool + + +class NFLDatasetProcessor: + """ + Processes NFL CSV datasets and converts to standardized format + Compatible with existing StandardizedEvent system + """ + + def __init__(self, data_dir: str = "data/nfl_source"): + """ + Initialize processor + + Args: + data_dir: Directory containing NFL CSV files + """ + self.data_dir = Path(data_dir) + self.datasets = self._discover_datasets() + logger.info(f"Discovered {len(self.datasets)} NFL datasets") + + def _discover_datasets(self) -> Dict[str, Path]: + """Discover available NFL datasets""" + datasets = {} + + for csv_file in self.data_dir.glob("*.csv"): + if "2009-2016" in csv_file.name: + datasets["2009-2016"] = csv_file + elif "2009-2017" in csv_file.name: + datasets["2009-2017"] = csv_file + elif "2009-2018" in csv_file.name: + datasets["2009-2018"] = csv_file + + return datasets + + def load_dataset(self, version: str = "2009-2016") -> pd.DataFrame: + """ + Load NFL dataset + + Args: + version: Dataset version to load + + Returns: + Pandas DataFrame with NFL data + """ + if version not in self.datasets: + raise ValueError(f"Dataset {version} not found. Available: {list(self.datasets.keys())}") + + logger.info(f"Loading NFL dataset: {version}") + + try: + df = pd.read_csv( + self.datasets[version], + encoding='utf-8-sig', # Handle BOM + low_memory=False + ) + + logger.info(f"Loaded {len(df)} plays from {version} dataset") + return df + + except Exception as e: + logger.error(f"Error loading dataset {version}: {e}") + raise + + def process_plays(self, df: pd.DataFrame, limit: Optional[int] = None) -> List[ProcessedNFLPlay]: + """ + Process raw NFL data into structured plays + + Args: + df: Raw NFL DataFrame + limit: Optional limit on number of plays to process + + Returns: + List of processed NFL plays + """ + if limit: + df = df.head(limit) + + processed_plays = [] + + logger.info(f"Processing {len(df)} NFL plays...") + + for idx, row in df.iterrows(): + try: + play = self._process_single_play(row, idx) + if play: + processed_plays.append(play) + + except Exception as e: + logger.warning(f"Error processing play {idx}: {e}") + continue + + logger.info(f"Successfully processed {len(processed_plays)} plays") + return processed_plays + + def _process_single_play(self, row: pd.Series, idx: int) -> Optional[ProcessedNFLPlay]: + """Process a single play row""" + try: + # Skip invalid plays + if pd.isna(row.get('desc')) or pd.isna(row.get('GameID')): + return None + + # Parse play type + play_type = self._determine_play_type(row) + + # Calculate field position + field_pos = self._calculate_field_position(row) + + # Determine outcomes + touchdown = bool(row.get('Touchdown', 0)) + field_goal = 'Field Goal' in str(row.get('desc', '')) + turnover = bool(row.get('InterceptionThrown', 0)) or bool(row.get('Fumble', 0)) + safety = bool(row.get('Safety', 0)) + penalty = bool(row.get('Accepted.Penalty', 0)) + + # Create truly unique play ID using multiple NFL dataset fields + play_unique_id = self._generate_unique_play_id(row, idx) + + return ProcessedNFLPlay( + game_id=str(row['GameID']), + play_id=play_unique_id, + date=str(row['Date']), + season=int(row.get('Season', 0)), + week=self._extract_week_from_date(str(row['Date'])), + + # Game context + quarter=self._safe_int(row.get('qtr')), + time_remaining=str(row.get('time', '')), + time_seconds=self._safe_int(row.get('TimeSecs')), + home_team=str(row.get('HomeTeam', '')), + away_team=str(row.get('AwayTeam', '')), + possession_team=str(row.get('posteam', '')), + defensive_team=str(row.get('DefensiveTeam', '')), + + # Situation + down=self._safe_int(row.get('down')), + distance=self._safe_int(row.get('ydstogo')), + yard_line=self._safe_int(row.get('yrdln')), + yards_to_goal=self._safe_int(row.get('yrdline100')), + field_position=field_pos, + + # Play details + play_type=play_type, + play_description=str(row.get('desc', '')), + yards_gained=self._safe_int(row.get('Yards.Gained')), + + # Advanced metrics + expected_points=self._safe_float(row.get('ExpPts')), + epa=self._safe_float(row.get('EPA')), + win_probability_pre=self._safe_float(row.get('Win_Prob')), + win_probability_post=self._safe_float(row.get('Home_WP_post')) if row.get('posteam') == row.get('HomeTeam') else self._safe_float(row.get('Away_WP_post')), + wpa=self._safe_float(row.get('WPA')), + + # Score context + score_home=self._safe_int(row.get('PosTeamScore')) if row.get('posteam') == row.get('HomeTeam') else self._safe_int(row.get('DefTeamScore')), + score_away=self._safe_int(row.get('DefTeamScore')) if row.get('posteam') == row.get('HomeTeam') else self._safe_int(row.get('PosTeamScore')), + score_differential=self._safe_int(row.get('ScoreDiff')), + + # Outcomes + touchdown=touchdown, + field_goal=field_goal, + turnover=turnover, + safety=safety, + penalty=penalty + ) + + except Exception as e: + logger.warning(f"Error processing play: {e}") + return None + + def _determine_play_type(self, row: pd.Series) -> str: + """Determine standardized play type from raw data""" + play_type = str(row.get('PlayType', '')) + + # Map to standardized types + type_mapping = { + 'Pass': 'PASS', + 'Rush': 'RUN', + 'Run': 'RUN', + 'Punt': 'PUNT', + 'Field Goal': 'FIELD_GOAL', + 'Kickoff': 'KICKOFF', + 'Sack': 'SACK', + 'Spike': 'SPIKE', + 'Kneel': 'KNEEL', + 'No Play': 'PENALTY' + } + + return type_mapping.get(play_type, 'UNKNOWN') + + def _calculate_field_position(self, row: pd.Series) -> str: + """Calculate field position description""" + side = str(row.get('SideofField', '')) + yard_line = self._safe_int(row.get('yrdln')) + + if side and yard_line: + return f"{side} {yard_line}" + return "UNKNOWN" + + def _extract_week_from_date(self, date_str: str) -> int: + """Extract week number from date (simplified)""" + try: + # This is a simplified approach - you might want to implement + # proper NFL week calculation based on season start dates + date_obj = datetime.strptime(date_str, '%Y-%m-%d') + # September games are generally weeks 1-4 + if date_obj.month == 9: + return min(4, (date_obj.day // 7) + 1) + elif date_obj.month == 10: + return min(8, 4 + (date_obj.day // 7) + 1) + elif date_obj.month == 11: + return min(12, 8 + (date_obj.day // 7) + 1) + elif date_obj.month == 12: + return min(16, 12 + (date_obj.day // 7) + 1) + else: + return 1 # Default + except: + return 1 + + def _safe_int(self, value) -> Optional[int]: + """Safely convert to int""" + if pd.isna(value) or value == 'NA': + return None + try: + return int(float(value)) + except (ValueError, TypeError): + return None + + def _safe_float(self, value) -> Optional[float]: + """Safely convert to float""" + if pd.isna(value) or value == 'NA': + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + def _generate_unique_play_id(self, row: pd.Series, idx: int) -> str: + """Generate truly unique play ID using multiple NFL dataset fields""" + import hashlib + + # Core identifiers + game_id = str(row.get('GameID', '')) + time_secs = str(row.get('TimeSecs', '')) + drive = str(row.get('Drive', '')) + play_desc = str(row.get('desc', '')) + + # Additional uniqueness factors + qtr = str(row.get('qtr', '')) + down = str(row.get('down', '')) + ydstogo = str(row.get('ydstogo', '')) + yrdln = str(row.get('yrdln', '')) + + # Create composite string for hashing + composite_string = f"{game_id}_{time_secs}_{drive}_{qtr}_{down}_{ydstogo}_{yrdln}_{play_desc}" + + # Generate hash of the description and situation for extra uniqueness + play_hash = hashlib.md5(composite_string.encode()).hexdigest()[:8] + + # Combine with fallback to row index + unique_id = f"{game_id}_{time_secs}_{drive}_{play_hash}_{idx}" + + return unique_id + + def to_standardized_events(self, processed_plays: List[ProcessedNFLPlay]) -> List[StandardizedEvent]: + """ + Convert processed plays to StandardizedEvent format + Compatible with existing event system + """ + events = [] + + for play in processed_plays: + try: + # Determine event type + if play.touchdown: + event_type = EventType.GAME_EVENT + impact = "high" + elif play.turnover: + event_type = EventType.GAME_EVENT + impact = "high" + elif play.field_goal: + event_type = EventType.GAME_EVENT + impact = "medium" + else: + event_type = EventType.GAME_EVENT + impact = "low" + + # Build event data + event_data = { + "play_type": play.play_type, + "description": play.play_description, + "quarter": play.quarter, + "time_remaining": play.time_remaining, + "down": play.down, + "distance": play.distance, + "field_position": play.field_position, + "yards_gained": play.yards_gained, + "possession_team": play.possession_team, + "score": { + "home": play.score_home, + "away": play.score_away, + "differential": play.score_differential + }, + "outcomes": { + "touchdown": play.touchdown, + "field_goal": play.field_goal, + "turnover": play.turnover, + "safety": play.safety, + "penalty": play.penalty + } + } + + # Create standardized event + event = StandardizedEvent( + source="nfl_historical", + event_type=event_type, + timestamp=datetime.strptime(play.date, '%Y-%m-%d'), + game_id=play.game_id, + data=event_data, + confidence=0.99, # Historical data is highly reliable + impact=impact, + metadata={ + "season": play.season, + "week": play.week, + "home_team": play.home_team, + "away_team": play.away_team, + "epa": play.epa, + "wpa": play.wpa + }, + raw_data=play + ) + + events.append(event) + + except Exception as e: + logger.warning(f"Error converting play to StandardizedEvent: {e}") + continue + + logger.info(f"Converted {len(events)} plays to StandardizedEvents") + return events + + +# Example usage and testing +if __name__ == "__main__": + import sys + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + # Initialize processor + processor = NFLDatasetProcessor() + + # Load small sample for testing + df = processor.load_dataset("2009-2016") + sample_df = df.head(1000) # Test with first 1000 plays + + # Process plays + processed_plays = processor.process_plays(sample_df) + print(f"Processed {len(processed_plays)} plays") + + # Convert to StandardizedEvents + events = processor.to_standardized_events(processed_plays) + print(f"Created {len(events)} StandardizedEvents") + + # Show sample + if events: + sample_event = events[0] + print(f"\nSample event:") + print(f"Type: {sample_event.event_type}") + print(f"Description: {sample_event.data['description']}") + print(f"Impact: {sample_event.impact}") \ No newline at end of file diff --git a/src/synthetic_data/preprocessing/training_data_builder.py b/src/synthetic_data/preprocessing/training_data_builder.py new file mode 100644 index 00000000..aafca898 --- /dev/null +++ b/src/synthetic_data/preprocessing/training_data_builder.py @@ -0,0 +1,258 @@ +""" +Training Data Builder for LFM2 Fine-tuning + +Converts processed NFL plays into training sequences +suitable for fine-tuning language models. +""" + +import logging +from typing import List, Dict, Any, Tuple +from dataclasses import dataclass + +from .nfl_dataset_processor import ProcessedNFLPlay + +logger = logging.getLogger(__name__) + + +@dataclass +class TrainingSequence: + """Training sequence for language model fine-tuning""" + context: str # Game context and situation + target: str # Expected play outcome + metadata: Dict[str, Any] # Additional information + + +class TrainingDataBuilder: + """ + Builds training sequences for LFM2 fine-tuning + from processed NFL play data + """ + + def __init__(self): + """Initialize training data builder""" + self.sequences = [] + + def build_sequences(self, plays: List[ProcessedNFLPlay], sequence_length: int = 5) -> List[TrainingSequence]: + """ + Build training sequences from NFL plays + + Args: + plays: List of processed NFL plays + sequence_length: Number of plays per sequence + + Returns: + List of training sequences + """ + # Group plays by game + games = self._group_plays_by_game(plays) + + sequences = [] + + for game_id, game_plays in games.items(): + game_sequences = self._create_game_sequences(game_plays, sequence_length) + sequences.extend(game_sequences) + + logger.info(f"Built {len(sequences)} training sequences from {len(plays)} plays") + return sequences + + def _group_plays_by_game(self, plays: List[ProcessedNFLPlay]) -> Dict[str, List[ProcessedNFLPlay]]: + """Group plays by game ID""" + games = {} + + for play in plays: + if play.game_id not in games: + games[play.game_id] = [] + games[play.game_id].append(play) + + # Sort plays within each game by time + for game_id in games: + games[game_id].sort(key=lambda p: (p.quarter or 0, -(p.time_seconds or 0))) + + return games + + def _create_game_sequences(self, plays: List[ProcessedNFLPlay], sequence_length: int) -> List[TrainingSequence]: + """Create training sequences from a single game""" + sequences = [] + + for i in range(len(plays) - sequence_length + 1): + sequence_plays = plays[i:i + sequence_length] + + # Use first N-1 plays as context, last play as target + context_plays = sequence_plays[:-1] + target_play = sequence_plays[-1] + + context = self._build_context(context_plays) + target = self._build_target(target_play) + + metadata = { + "game_id": target_play.game_id, + "season": target_play.season, + "sequence_start": i, + "plays_count": len(context_plays) + } + + sequence = TrainingSequence( + context=context, + target=target, + metadata=metadata + ) + + sequences.append(sequence) + + return sequences + + def _build_context(self, plays: List[ProcessedNFLPlay]) -> str: + """Build context string from sequence of plays""" + context_parts = [] + + for play in plays: + play_context = self._format_play_context(play) + context_parts.append(play_context) + + return " | ".join(context_parts) + + def _format_play_context(self, play: ProcessedNFLPlay) -> str: + """Format single play as context""" + parts = [] + + # Game situation + if play.quarter and play.time_remaining: + parts.append(f"Q{play.quarter} {play.time_remaining}") + + # Down and distance + if play.down and play.distance: + parts.append(f"{play.down}&{play.distance}") + + # Field position + if play.yards_to_goal: + parts.append(f"Y{play.yards_to_goal}") + + # Score differential + if play.score_differential is not None: + if play.score_differential > 0: + parts.append(f"+{play.score_differential}") + elif play.score_differential < 0: + parts.append(f"{play.score_differential}") + else: + parts.append("TIE") + + # Play type and result + parts.append(f"{play.play_type}") + if play.yards_gained is not None: + parts.append(f"{play.yards_gained}yd") + + # Special outcomes + if play.touchdown: + parts.append("TD") + elif play.field_goal: + parts.append("FG") + elif play.turnover: + parts.append("TO") + + return " ".join(parts) + + def _build_target(self, play: ProcessedNFLPlay) -> str: + """Build target string for play prediction""" + target_parts = [] + + # Play type + target_parts.append(f"PLAY:{play.play_type}") + + # Expected outcome + if play.yards_gained is not None: + target_parts.append(f"YARDS:{play.yards_gained}") + + # Special outcomes + outcomes = [] + if play.touchdown: + outcomes.append("TD") + if play.field_goal: + outcomes.append("FG") + if play.turnover: + outcomes.append("TO") + if play.safety: + outcomes.append("SAFETY") + + if outcomes: + target_parts.append(f"OUTCOME:{','.join(outcomes)}") + + # Performance metrics + if play.epa is not None: + target_parts.append(f"EPA:{play.epa:.2f}") + + return " ".join(target_parts) + + def export_for_training(self, sequences: List[TrainingSequence], format: str = "jsonl") -> str: + """ + Export training sequences in specified format + + Args: + sequences: Training sequences + format: Export format ('jsonl', 'csv', 'txt') + + Returns: + Formatted training data string + """ + if format == "jsonl": + return self._export_jsonl(sequences) + elif format == "csv": + return self._export_csv(sequences) + elif format == "txt": + return self._export_txt(sequences) + else: + raise ValueError(f"Unsupported format: {format}") + + def _export_jsonl(self, sequences: List[TrainingSequence]) -> str: + """Export as JSONL format""" + import json + + lines = [] + for seq in sequences: + record = { + "context": seq.context, + "target": seq.target, + "metadata": seq.metadata + } + lines.append(json.dumps(record)) + + return "\n".join(lines) + + def _export_csv(self, sequences: List[TrainingSequence]) -> str: + """Export as CSV format""" + import csv + from io import StringIO + + output = StringIO() + writer = csv.writer(output) + + # Header + writer.writerow(["context", "target", "game_id", "season"]) + + # Data + for seq in sequences: + writer.writerow([ + seq.context, + seq.target, + seq.metadata.get("game_id", ""), + seq.metadata.get("season", "") + ]) + + return output.getvalue() + + def _export_txt(self, sequences: List[TrainingSequence]) -> str: + """Export as text format for language model training""" + lines = [] + + for seq in sequences: + # Format as: Context -> Target + line = f"{seq.context} -> {seq.target}" + lines.append(line) + + return "\n".join(lines) + + +# Example usage +if __name__ == "__main__": + # This would be used with processed NFL plays + builder = TrainingDataBuilder() + print("TrainingDataBuilder initialized successfully!") \ No newline at end of file diff --git a/src/synthetic_data/storage/__init__.py b/src/synthetic_data/storage/__init__.py new file mode 100644 index 00000000..697246e8 --- /dev/null +++ b/src/synthetic_data/storage/__init__.py @@ -0,0 +1,11 @@ +""" +Storage and Database Integration Module + +Handles ChromaDB integration, synthetic event storage, +and knowledge base management for NFL data. +""" + +from .chromadb_manager import ChromaDBManager +from .synthetic_event_store import SyntheticEventStore + +__all__ = ["ChromaDBManager", "SyntheticEventStore"] \ No newline at end of file diff --git a/src/synthetic_data/storage/chromadb_manager.py b/src/synthetic_data/storage/chromadb_manager.py new file mode 100644 index 00000000..e11916ea --- /dev/null +++ b/src/synthetic_data/storage/chromadb_manager.py @@ -0,0 +1,556 @@ +""" +ChromaDB Manager for NFL Data + +Manages ChromaDB collections for storing and retrieving +NFL play patterns, game scenarios, and synthetic data. +""" + +import chromadb +from chromadb.config import Settings +import logging +from typing import Dict, List, Optional, Any +from pathlib import Path +import json + +from ..preprocessing.nfl_dataset_processor import ProcessedNFLPlay +from src.sdk.core.base_adapter import StandardizedEvent + +logger = logging.getLogger(__name__) + + +class ChromaDBManager: + """ + Manages ChromaDB collections for NFL synthetic data system + """ + + def __init__(self, persist_directory: str = "data/chromadb"): + """ + Initialize ChromaDB manager + + Args: + persist_directory: Directory to store ChromaDB data + """ + self.persist_directory = Path(persist_directory) + self.persist_directory.mkdir(parents=True, exist_ok=True) + + # Initialize ChromaDB client + self.client = chromadb.PersistentClient( + path=str(self.persist_directory) + ) + + # Initialize collections + self.collections = self._initialize_collections() + logger.info(f"Initialized ChromaDB with {len(self.collections)} collections") + + def _initialize_collections(self) -> Dict[str, chromadb.Collection]: + """Initialize all NFL data collections""" + collections = {} + + # Collection 1: NFL Play Patterns + collections["nfl_play_patterns"] = self.client.get_or_create_collection( + name="nfl_play_patterns", + metadata={ + "description": "Historical NFL play sequences and patterns 2009-2018", + "source": "nfl_dataset", + "purpose": "pattern_learning" + } + ) + + # Collection 2: Game Scenarios + collections["game_scenarios"] = self.client.get_or_create_collection( + name="game_scenarios", + metadata={ + "description": "Specific game situation templates for generation", + "source": "synthetic", + "purpose": "scenario_generation" + } + ) + + # Collection 3: Team Tendencies + collections["team_tendencies"] = self.client.get_or_create_collection( + name="team_tendencies", + metadata={ + "description": "Team-specific play calling and behavioral patterns", + "source": "nfl_dataset", + "purpose": "team_modeling" + } + ) + + # Collection 4: Synthetic Learnings + collections["synthetic_learnings"] = self.client.get_or_create_collection( + name="synthetic_learnings", + metadata={ + "description": "Agent-discovered patterns and successful strategies", + "source": "agent_training", + "purpose": "adaptive_learning" + } + ) + + return collections + + def add_nfl_plays(self, plays: List[ProcessedNFLPlay], upsert: bool = False) -> int: + """ + Add NFL plays to the patterns collection with duplicate detection + + Args: + plays: List of processed NFL plays + upsert: If True, update existing plays instead of skipping + + Returns: + Number of plays added/updated + """ + if not plays: + return 0 + + collection = self.collections["nfl_play_patterns"] + + # Prepare data for ChromaDB + ids = [] + documents = [] + metadatas = [] + + # Check for existing plays if not upserting + existing_ids = set() + if not upsert: + try: + # Get all existing IDs + existing_data = collection.get() + existing_ids = set(existing_data['ids']) if existing_data['ids'] else set() + logger.debug(f"Found {len(existing_ids)} existing plays in ChromaDB") + except Exception as e: + logger.warning(f"Could not check for existing plays: {e}") + + skipped_count = 0 + for play in plays: + # Create unique ID using play_id directly (already unique from processor) + play_id = f"{play.game_id}_play_{play.play_id}" + + # Skip duplicates if not upserting + if not upsert and play_id in existing_ids: + skipped_count += 1 + continue + + ids.append(play_id) + + # Create searchable document text + doc_text = self._create_play_document(play) + documents.append(doc_text) + + # Create metadata + metadata = self._create_play_metadata(play) + metadatas.append(metadata) + + if not ids: + logger.info(f"No new plays to add. Skipped {skipped_count} duplicates") + return 0 + + try: + if upsert: + # Use upsert for updating existing plays + collection.upsert( + ids=ids, + documents=documents, + metadatas=metadatas + ) + operation = "upserted" + else: + # Regular add operation + collection.add( + ids=ids, + documents=documents, + metadatas=metadatas + ) + operation = "added" + + logger.info(f"Successfully {operation} {len(ids)} NFL plays to ChromaDB. Skipped {skipped_count} duplicates") + return len(ids) + + except Exception as e: + logger.error(f"Error {operation.replace('ed', 'ing')} plays to ChromaDB: {e}") + return 0 + + def _create_play_document(self, play: ProcessedNFLPlay) -> str: + """Create searchable text document from play data""" + + # Build contextual description + context_parts = [] + + # Game situation + if play.quarter and play.time_remaining: + context_parts.append(f"Quarter {play.quarter}, {play.time_remaining} remaining") + + # Down and distance + if play.down and play.distance: + context_parts.append(f"{play.down} and {play.distance}") + + # Field position + if play.field_position: + context_parts.append(f"at {play.field_position}") + + # Team context + if play.possession_team: + context_parts.append(f"{play.possession_team} has possession") + + # Score situation + if play.score_differential is not None: + if play.score_differential > 0: + context_parts.append(f"leading by {play.score_differential}") + elif play.score_differential < 0: + context_parts.append(f"trailing by {abs(play.score_differential)}") + else: + context_parts.append("tied game") + + context = ". ".join(context_parts) + + # Play description and outcome + outcome_parts = [] + if play.yards_gained is not None: + outcome_parts.append(f"Gained {play.yards_gained} yards") + + if play.touchdown: + outcome_parts.append("TOUCHDOWN") + elif play.field_goal: + outcome_parts.append("FIELD GOAL") + elif play.turnover: + outcome_parts.append("TURNOVER") + elif play.safety: + outcome_parts.append("SAFETY") + + outcome = ". ".join(outcome_parts) + + # Combine into searchable document + document = f"{context}. Play: {play.play_description}. Result: {outcome}" + + # Add performance metrics if available + if play.epa is not None: + document += f". EPA: {play.epa:.2f}" + if play.wpa is not None: + document += f". WPA: {play.wpa:.3f}" + + return document + + def _create_play_metadata(self, play: ProcessedNFLPlay) -> Dict[str, Any]: + """Create metadata dict for play""" + metadata = { + "game_id": play.game_id, + "season": play.season, + "week": play.week, + "quarter": play.quarter or 0, + "down": play.down or 0, + "distance": play.distance or 0, + "yards_to_goal": play.yards_to_goal or 0, + "play_type": play.play_type, + "possession_team": play.possession_team, + "home_team": play.home_team, + "away_team": play.away_team, + "yards_gained": play.yards_gained or 0, + "touchdown": play.touchdown, + "field_goal": play.field_goal, + "turnover": play.turnover, + "safety": play.safety, + "penalty": play.penalty + } + + # Add optional metrics + if play.epa is not None: + metadata["epa"] = round(play.epa, 3) + if play.wpa is not None: + metadata["wpa"] = round(play.wpa, 4) + if play.score_differential is not None: + metadata["score_diff"] = play.score_differential + + return metadata + + def search_similar_plays( + self, + query: str, + collection_name: str = "nfl_play_patterns", + n_results: int = 10, + filters: Optional[Dict] = None + ) -> Dict[str, Any]: + """ + Search for similar plays based on situation + + Args: + query: Natural language description of situation + collection_name: Collection to search + n_results: Number of results to return + filters: Optional metadata filters + + Returns: + Search results with documents and metadata + """ + if collection_name not in self.collections: + raise ValueError(f"Collection {collection_name} not found") + + collection = self.collections[collection_name] + + try: + results = collection.query( + query_texts=[query], + n_results=n_results, + where=filters, + include=['documents', 'metadatas', 'distances'] + ) + + logger.debug(f"Found {len(results['ids'][0])} similar plays for query: {query}") + return results + + except Exception as e: + logger.error(f"Error searching plays: {e}") + return {"ids": [[]], "documents": [[]], "metadatas": [[]], "distances": [[]]} + + def add_game_scenario( + self, + scenario_name: str, + description: str, + template_data: Dict[str, Any] + ) -> bool: + """ + Add a game scenario template + + Args: + scenario_name: Unique name for scenario + description: Description of the scenario + template_data: Template parameters + + Returns: + Success status + """ + collection = self.collections["game_scenarios"] + + try: + collection.add( + ids=[scenario_name], + documents=[description], + metadatas=[{ + "scenario_type": template_data.get("type", "general"), + "weather": template_data.get("weather", "clear"), + "game_stakes": template_data.get("stakes", "regular"), + "expected_plays": template_data.get("expected_plays", 140), + "template_json": json.dumps(template_data) + }] + ) + + logger.info(f"Added scenario: {scenario_name}") + return True + + except Exception as e: + logger.error(f"Error adding scenario: {e}") + return False + + def get_team_tendencies(self, team: str, situation: str = None) -> List[Dict]: + """ + Get team-specific play calling tendencies + + Args: + team: Team code (e.g., 'KC', 'NE') + situation: Optional situation filter + + Returns: + List of relevant plays/tendencies + """ + query = f"{team} team tendencies" + if situation: + query += f" in {situation}" + + results = self.search_similar_plays( + query=query, + collection_name="team_tendencies", + filters={"possession_team": team} if team else None + ) + + return self._format_search_results(results) + + def add_synthetic_learning( + self, + agent_name: str, + pattern_description: str, + success_metrics: Dict[str, float], + context: Dict[str, Any] + ) -> bool: + """ + Add agent-discovered pattern + + Args: + agent_name: Name of discovering agent + pattern_description: Description of discovered pattern + success_metrics: Performance metrics + context: Context information + + Returns: + Success status + """ + collection = self.collections["synthetic_learnings"] + + learning_id = f"{agent_name}_{len(collection.get()['ids'])}" + + try: + metadata = { + "agent": agent_name, + "success_rate": success_metrics.get("success_rate", 0.0), + "profit": success_metrics.get("profit", 0.0), + "pattern_type": context.get("pattern_type", "unknown"), + "discovery_date": context.get("date", "unknown") + } + + collection.add( + ids=[learning_id], + documents=[pattern_description], + metadatas=[metadata] + ) + + logger.info(f"Added synthetic learning from {agent_name}") + return True + + except Exception as e: + logger.error(f"Error adding synthetic learning: {e}") + return False + + def _format_search_results(self, results: Dict[str, Any]) -> List[Dict]: + """Format ChromaDB search results into convenient structure""" + formatted = [] + + if not results.get('ids') or not results['ids'][0]: + return formatted + + for i in range(len(results['ids'][0])): + formatted.append({ + 'id': results['ids'][0][i], + 'document': results['documents'][0][i], + 'metadata': results['metadatas'][0][i], + 'distance': results['distances'][0][i] if 'distances' in results else None + }) + + return formatted + + def add_nfl_plays_batch(self, plays: List[ProcessedNFLPlay], batch_size: int = 1000, upsert: bool = False) -> int: + """ + Add NFL plays in batches with transaction rollback capability + + Args: + plays: List of processed NFL plays + batch_size: Size of each batch + upsert: If True, update existing plays instead of skipping + + Returns: + Total number of plays added/updated successfully + """ + if not plays: + return 0 + + collection = self.collections["nfl_play_patterns"] + total_added = 0 + successful_batches = [] + + # Process in batches + for i in range(0, len(plays), batch_size): + batch = plays[i:i + batch_size] + batch_number = i // batch_size + 1 + + logger.info(f"Processing batch {batch_number}/{(len(plays) + batch_size - 1) // batch_size} ({len(batch)} plays)") + + try: + # Create backup of batch IDs for potential rollback + batch_ids = [f"{play.game_id}_play_{play.play_id}" for play in batch] + + # Process the batch + batch_added = self.add_nfl_plays(batch, upsert=upsert) + + if batch_added > 0: + successful_batches.append({ + 'batch_number': batch_number, + 'ids': batch_ids[:batch_added], # Only IDs that were actually added + 'count': batch_added + }) + total_added += batch_added + logger.info(f"Batch {batch_number} completed: {batch_added} plays added") + else: + logger.warning(f"Batch {batch_number} added 0 plays (likely all duplicates)") + + except Exception as e: + logger.error(f"Batch {batch_number} failed: {e}") + + # Rollback all successful batches + if successful_batches: + logger.warning(f"Rolling back {len(successful_batches)} successful batches due to failure") + rollback_count = self._rollback_batches(successful_batches) + logger.info(f"Rolled back {rollback_count} plays") + + raise Exception(f"Batch processing failed at batch {batch_number}. All changes have been rolled back.") from e + + logger.info(f"Batch processing completed successfully. Total: {total_added} plays added across {len(successful_batches)} batches") + return total_added + + def _rollback_batches(self, successful_batches: List[Dict]) -> int: + """ + Rollback successfully added batches + + Args: + successful_batches: List of batch info dicts + + Returns: + Number of plays rolled back + """ + collection = self.collections["nfl_play_patterns"] + rollback_count = 0 + + for batch_info in successful_batches: + try: + # Delete the batch using ChromaDB delete + collection.delete(ids=batch_info['ids']) + rollback_count += batch_info['count'] + logger.debug(f"Rolled back batch {batch_info['batch_number']}: {batch_info['count']} plays") + except Exception as e: + logger.error(f"Failed to rollback batch {batch_info['batch_number']}: {e}") + + return rollback_count + + def get_collection_stats(self) -> Dict[str, int]: + """Get statistics for all collections""" + stats = {} + + for name, collection in self.collections.items(): + try: + count = collection.count() + stats[name] = count + except Exception as e: + logger.warning(f"Error getting stats for {name}: {e}") + stats[name] = 0 + + return stats + + def clear_collection(self, collection_name: str) -> bool: + """Clear all data from a collection""" + if collection_name not in self.collections: + return False + + try: + # Delete and recreate collection + self.client.delete_collection(collection_name) + # Recreate will happen automatically on next access + return True + except Exception as e: + logger.error(f"Error clearing collection {collection_name}: {e}") + return False + + +# Example usage and testing +if __name__ == "__main__": + import sys + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + # Initialize ChromaDB manager + chromadb_manager = ChromaDBManager() + + # Get collection stats + stats = chromadb_manager.get_collection_stats() + print(f"Collection stats: {stats}") + + # Test search (will be empty initially) + results = chromadb_manager.search_similar_plays( + "3rd down and 8 in red zone, 2 minutes remaining" + ) + print(f"Search results: {len(results['ids'][0])} found") + + print("ChromaDB Manager initialized successfully!") \ No newline at end of file diff --git a/src/synthetic_data/storage/synthetic_event_store.py b/src/synthetic_data/storage/synthetic_event_store.py new file mode 100644 index 00000000..3772cc44 --- /dev/null +++ b/src/synthetic_data/storage/synthetic_event_store.py @@ -0,0 +1,209 @@ +""" +Synthetic Event Store + +Extends existing EventStore to handle synthetic NFL events +and integrates with ChromaDB for knowledge storage. +""" + +import logging +from typing import List, Optional, Dict, Any +from datetime import datetime + +# Import existing event store functionality +try: + from src.backtesting.event_store.storage import EventStore + from src.backtesting.event_store.models import EventQuery, StoredEvent +except ImportError: + # Fallback if imports not available + EventStore = None + EventQuery = None + StoredEvent = None + +from src.sdk.core.base_adapter import StandardizedEvent +from .chromadb_manager import ChromaDBManager + +logger = logging.getLogger(__name__) + + +class SyntheticEventStore: + """ + Extended event store for synthetic NFL events + Integrates with ChromaDB for enhanced knowledge storage + """ + + def __init__(self, + sqlite_path: str = "data/synthetic_events.db", + chromadb_path: str = "data/chromadb"): + """ + Initialize synthetic event store + + Args: + sqlite_path: Path to SQLite database for events + chromadb_path: Path to ChromaDB storage + """ + # Initialize traditional event store if available + if EventStore: + self.event_store = EventStore(db_path=sqlite_path) + else: + self.event_store = None + logger.warning("EventStore not available, using ChromaDB only") + + # Initialize ChromaDB for enhanced knowledge storage + self.chromadb = ChromaDBManager(persist_directory=chromadb_path) + + logger.info("Initialized SyntheticEventStore") + + def store_synthetic_events(self, events: List[StandardizedEvent]) -> int: + """ + Store synthetic events in both traditional and knowledge stores + + Args: + events: List of synthetic events + + Returns: + Number of events stored + """ + if not events: + return 0 + + stored_count = 0 + + # Store in traditional event store + if self.event_store: + try: + for event in events: + event_id = self.event_store.save_event(event) + if event_id: + stored_count += 1 + except Exception as e: + logger.error(f"Error storing events in EventStore: {e}") + + # Store in ChromaDB for knowledge-based retrieval + try: + # Convert events to play format for ChromaDB + from ..preprocessing.nfl_dataset_processor import ProcessedNFLPlay + + plays = [] + for event in events: + if hasattr(event.raw_data, 'game_id'): # NFL play data + plays.append(event.raw_data) + + if plays: + chromadb_count = self.chromadb.add_nfl_plays(plays) + logger.info(f"Stored {chromadb_count} plays in ChromaDB") + + except Exception as e: + logger.error(f"Error storing events in ChromaDB: {e}") + + logger.info(f"Stored {stored_count} synthetic events") + return stored_count + + def search_similar_events(self, + query: str, + n_results: int = 10, + filters: Optional[Dict] = None) -> List[Dict]: + """ + Search for similar events using ChromaDB + + Args: + query: Natural language query + n_results: Number of results + filters: Optional metadata filters + + Returns: + List of similar events with metadata + """ + results = self.chromadb.search_similar_plays( + query=query, + n_results=n_results, + filters=filters + ) + + return self.chromadb._format_search_results(results) + + def get_team_patterns(self, team: str, situation: str = None) -> List[Dict]: + """ + Get team-specific patterns from knowledge base + + Args: + team: Team code + situation: Optional situation description + + Returns: + List of relevant patterns + """ + return self.chromadb.get_team_tendencies(team, situation) + + def add_agent_learning(self, + agent_name: str, + pattern_description: str, + success_metrics: Dict[str, float], + context: Dict[str, Any]) -> bool: + """ + Add agent-discovered pattern to knowledge base + + Args: + agent_name: Name of agent + pattern_description: Description of discovered pattern + success_metrics: Performance metrics + context: Additional context + + Returns: + Success status + """ + return self.chromadb.add_synthetic_learning( + agent_name=agent_name, + pattern_description=pattern_description, + success_metrics=success_metrics, + context=context + ) + + def query_traditional_events(self, query: EventQuery) -> List[StandardizedEvent]: + """ + Query traditional event store if available + + Args: + query: Event query + + Returns: + List of events + """ + if not self.event_store: + logger.warning("Traditional EventStore not available") + return [] + + try: + events = list(self.event_store.stream_events(query)) + return events + except Exception as e: + logger.error(f"Error querying traditional events: {e}") + return [] + + def get_storage_stats(self) -> Dict[str, Any]: + """Get statistics for all storage systems""" + stats = { + "chromadb_collections": self.chromadb.get_collection_stats(), + "traditional_events": 0 + } + + if self.event_store: + try: + # This would need to be implemented in the EventStore + # For now, just indicate it's available + stats["traditional_store"] = "available" + except Exception: + stats["traditional_store"] = "unavailable" + + return stats + + +# Example usage +if __name__ == "__main__": + # Initialize synthetic event store + store = SyntheticEventStore() + + # Get storage statistics + stats = store.get_storage_stats() + print(f"Storage stats: {stats}") + + print("SyntheticEventStore initialized successfully!") \ No newline at end of file diff --git a/src/synthetic_data/validation/__init__.py b/src/synthetic_data/validation/__init__.py new file mode 100644 index 00000000..0058ec42 --- /dev/null +++ b/src/synthetic_data/validation/__init__.py @@ -0,0 +1,14 @@ +""" +Data Validation and Quality Metrics Module + +Quality metrics and pattern analysis +for synthetic NFL data generation. +""" + +from .play_data_validator import PlayDataValidator, ValidationResult, PlayValidationRule + +__all__ = [ + 'PlayDataValidator', + 'ValidationResult', + 'PlayValidationRule' +] \ No newline at end of file diff --git a/src/synthetic_data/validation/play_data_validator.py b/src/synthetic_data/validation/play_data_validator.py new file mode 100644 index 00000000..4eacdc2b --- /dev/null +++ b/src/synthetic_data/validation/play_data_validator.py @@ -0,0 +1,422 @@ +""" +NFL Play Data Validator + +Validates the completeness and quality of NFL play data +before processing and storage. +""" + +import logging +from typing import List, Dict, Any, Tuple, Optional +from dataclasses import dataclass +from datetime import datetime +import pandas as pd + +from ..preprocessing.nfl_dataset_processor import ProcessedNFLPlay + +logger = logging.getLogger(__name__) + + +@dataclass +class ValidationResult: + """Results of data validation""" + is_valid: bool + total_plays: int + valid_plays: int + invalid_plays: int + warnings: List[str] + errors: List[str] + quality_score: float # 0.0 to 1.0 + + def __post_init__(self): + """Calculate quality score after initialization""" + if self.total_plays > 0: + self.quality_score = self.valid_plays / self.total_plays + else: + self.quality_score = 0.0 + + +@dataclass +class PlayValidationRule: + """Individual validation rule for NFL plays""" + name: str + required: bool + validator_func: callable + error_message: str + weight: float = 1.0 # For weighted scoring + + +class PlayDataValidator: + """ + Validates NFL play data for completeness and quality + """ + + def __init__(self, strict_mode: bool = False): + """ + Initialize validator + + Args: + strict_mode: If True, apply stricter validation rules + """ + self.strict_mode = strict_mode + self.validation_rules = self._initialize_validation_rules() + + def _initialize_validation_rules(self) -> List[PlayValidationRule]: + """Initialize validation rules for NFL plays""" + rules = [ + # Critical fields + PlayValidationRule( + name="game_id_present", + required=True, + validator_func=lambda play: play.game_id and play.game_id != "nan", + error_message="Game ID is missing or invalid", + weight=2.0 + ), + PlayValidationRule( + name="play_id_present", + required=True, + validator_func=lambda play: play.play_id and play.play_id != "nan", + error_message="Play ID is missing or invalid", + weight=2.0 + ), + PlayValidationRule( + name="description_present", + required=True, + validator_func=lambda play: play.play_description and len(play.play_description.strip()) > 5, + error_message="Play description is missing or too short", + weight=2.0 + ), + + # Game context validation + PlayValidationRule( + name="valid_quarter", + required=True, + validator_func=lambda play: play.quarter and 1 <= play.quarter <= 5, + error_message="Quarter must be between 1 and 5", + weight=1.5 + ), + PlayValidationRule( + name="valid_down", + required=False, + validator_func=lambda play: play.down is None or (1 <= play.down <= 4), + error_message="Down must be between 1 and 4 when present", + weight=1.0 + ), + PlayValidationRule( + name="valid_distance", + required=False, + validator_func=lambda play: play.distance is None or (0 <= play.distance <= 99), + error_message="Distance must be between 0 and 99 yards when present", + weight=1.0 + ), + PlayValidationRule( + name="valid_yard_line", + required=False, + validator_func=lambda play: play.yard_line is None or (0 <= play.yard_line <= 100), + error_message="Yard line must be between 0 and 100 when present", + weight=1.0 + ), + + # Team information + PlayValidationRule( + name="home_team_present", + required=True, + validator_func=lambda play: play.home_team and len(play.home_team) >= 2, + error_message="Home team is missing or invalid", + weight=1.5 + ), + PlayValidationRule( + name="away_team_present", + required=True, + validator_func=lambda play: play.away_team and len(play.away_team) >= 2, + error_message="Away team is missing or invalid", + weight=1.5 + ), + PlayValidationRule( + name="possession_team_present", + required=True, + validator_func=lambda play: play.possession_team and len(play.possession_team) >= 2, + error_message="Possession team is missing or invalid", + weight=1.5 + ), + + # Play type validation + PlayValidationRule( + name="valid_play_type", + required=True, + validator_func=lambda play: play.play_type in ['PASS', 'RUN', 'PUNT', 'FIELD_GOAL', 'KICKOFF', 'SACK', 'SPIKE', 'KNEEL', 'PENALTY', 'UNKNOWN'], + error_message="Play type must be a recognized value", + weight=1.5 + ), + + # Season and date validation + PlayValidationRule( + name="valid_season", + required=True, + validator_func=lambda play: play.season and 2009 <= play.season <= datetime.now().year, + error_message="Season must be between 2009 and current year", + weight=1.0 + ), + PlayValidationRule( + name="valid_week", + required=True, + validator_func=lambda play: play.week and 1 <= play.week <= 22, + error_message="Week must be between 1 and 22", + weight=1.0 + ), + + # Data quality checks + PlayValidationRule( + name="reasonable_yards_gained", + required=False, + validator_func=lambda play: play.yards_gained is None or (-50 <= play.yards_gained <= 99), + error_message="Yards gained seems unreasonable (should be between -50 and 99)", + weight=0.5 + ), + PlayValidationRule( + name="consistent_score_differential", + required=False, + validator_func=lambda play: ( + play.score_differential is None or + play.score_home is None or + play.score_away is None or + play.score_differential == (play.score_home - play.score_away) + ), + error_message="Score differential doesn't match home/away scores", + weight=0.5 + ) + ] + + # Add strict mode rules + if self.strict_mode: + rules.extend([ + PlayValidationRule( + name="advanced_metrics_present", + required=True, + validator_func=lambda play: play.epa is not None and play.wpa is not None, + error_message="Advanced metrics (EPA, WPA) required in strict mode", + weight=1.0 + ), + PlayValidationRule( + name="complete_score_info", + required=True, + validator_func=lambda play: ( + play.score_home is not None and + play.score_away is not None and + play.score_differential is not None + ), + error_message="Complete score information required in strict mode", + weight=1.0 + ) + ]) + + return rules + + def validate_single_play(self, play: ProcessedNFLPlay) -> Tuple[bool, List[str], List[str]]: + """ + Validate a single NFL play + + Args: + play: Processed NFL play to validate + + Returns: + Tuple of (is_valid, warnings, errors) + """ + warnings = [] + errors = [] + + for rule in self.validation_rules: + try: + result = rule.validator_func(play) + + if not result: + if rule.required: + errors.append(f"{rule.name}: {rule.error_message}") + else: + warnings.append(f"{rule.name}: {rule.error_message}") + + except Exception as e: + error_msg = f"{rule.name}: Validation failed - {e}" + if rule.required: + errors.append(error_msg) + else: + warnings.append(error_msg) + + is_valid = len(errors) == 0 + return is_valid, warnings, errors + + def validate_play_list(self, plays: List[ProcessedNFLPlay]) -> ValidationResult: + """ + Validate a list of NFL plays + + Args: + plays: List of processed NFL plays + + Returns: + ValidationResult with detailed validation info + """ + if not plays: + return ValidationResult( + is_valid=False, + total_plays=0, + valid_plays=0, + invalid_plays=0, + warnings=[], + errors=["No plays provided for validation"], + quality_score=0.0 + ) + + all_warnings = [] + all_errors = [] + valid_count = 0 + + logger.info(f"Validating {len(plays)} plays...") + + for i, play in enumerate(plays): + try: + is_valid, warnings, errors = self.validate_single_play(play) + + if is_valid: + valid_count += 1 + + # Add play context to messages + if warnings: + for warning in warnings: + all_warnings.append(f"Play {i+1} ({play.play_id}): {warning}") + + if errors: + for error in errors: + all_errors.append(f"Play {i+1} ({play.play_id}): {error}") + + except Exception as e: + error_msg = f"Play {i+1}: Validation exception - {e}" + all_errors.append(error_msg) + logger.error(error_msg) + + result = ValidationResult( + is_valid=(valid_count == len(plays)), + total_plays=len(plays), + valid_plays=valid_count, + invalid_plays=len(plays) - valid_count, + warnings=all_warnings, + errors=all_errors, + quality_score=0.0 # Will be calculated in __post_init__ + ) + + logger.info(f"Validation complete: {result.valid_plays}/{result.total_plays} plays valid (quality: {result.quality_score:.2%})") + + return result + + def filter_valid_plays(self, plays: List[ProcessedNFLPlay]) -> Tuple[List[ProcessedNFLPlay], ValidationResult]: + """ + Filter plays to return only valid ones + + Args: + plays: List of processed NFL plays + + Returns: + Tuple of (valid_plays, validation_result) + """ + if not plays: + return [], ValidationResult( + is_valid=False, + total_plays=0, + valid_plays=0, + invalid_plays=0, + warnings=[], + errors=["No plays provided"], + quality_score=0.0 + ) + + valid_plays = [] + validation_result = self.validate_play_list(plays) + + for i, play in enumerate(plays): + is_valid, _, _ = self.validate_single_play(play) + if is_valid: + valid_plays.append(play) + + logger.info(f"Filtered to {len(valid_plays)} valid plays from {len(plays)} total") + + return valid_plays, validation_result + + def get_data_quality_report(self, plays: List[ProcessedNFLPlay]) -> Dict[str, Any]: + """ + Generate comprehensive data quality report + + Args: + plays: List of processed NFL plays + + Returns: + Detailed quality report + """ + validation_result = self.validate_play_list(plays) + + # Calculate rule-specific failure rates + rule_failures = {} + for rule in self.validation_rules: + failures = 0 + for play in plays: + try: + if not rule.validator_func(play): + failures += 1 + except: + failures += 1 + rule_failures[rule.name] = { + 'failures': failures, + 'failure_rate': failures / len(plays) if plays else 0.0, + 'required': rule.required + } + + # Calculate completeness metrics + completeness = {} + if plays: + sample_play = plays[0] + for field in ['game_id', 'play_description', 'quarter', 'down', 'distance', + 'possession_team', 'yards_gained', 'epa', 'wpa', 'score_home']: + non_null_count = sum(1 for play in plays if getattr(play, field) is not None) + completeness[field] = non_null_count / len(plays) + + return { + 'validation_summary': { + 'total_plays': validation_result.total_plays, + 'valid_plays': validation_result.valid_plays, + 'invalid_plays': validation_result.invalid_plays, + 'quality_score': validation_result.quality_score, + 'is_valid': validation_result.is_valid + }, + 'rule_failures': rule_failures, + 'completeness_metrics': completeness, + 'validation_errors': validation_result.errors[:50], # Limit for readability + 'validation_warnings': validation_result.warnings[:50] + } + + +# Example usage and testing +if __name__ == "__main__": + import sys + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.preprocessing.nfl_dataset_processor import NFLDatasetProcessor + + # Test validation + processor = NFLDatasetProcessor(data_dir='data/nfl_source') + validator = PlayDataValidator(strict_mode=False) + + # Load and process sample data + df = processor.load_dataset('2009-2016') + sample_df = df.head(50) + processed_plays = processor.process_plays(sample_df) + + # Validate + validation_result = validator.validate_play_list(processed_plays) + print(f"Validation Result: {validation_result.quality_score:.2%} quality score") + print(f"Valid plays: {validation_result.valid_plays}/{validation_result.total_plays}") + + if validation_result.errors: + print(f"Sample errors: {validation_result.errors[:3]}") + + # Get quality report + quality_report = validator.get_data_quality_report(processed_plays) + print(f"Data completeness: {quality_report['completeness_metrics']}") + + print("PlayDataValidator test completed!") \ No newline at end of file diff --git a/src/training/__init__.py b/src/training/__init__.py new file mode 100644 index 00000000..814cfb33 --- /dev/null +++ b/src/training/__init__.py @@ -0,0 +1,42 @@ +""" +Agent Training Module + +Synthetic training environments and performance analytics +for multi-agent system development. +""" + +from .synthetic_env import ( + SyntheticTrainingEnvironment, + EnvironmentConfig, + InformationLevel, + EnvironmentState, + AgentPerformanceMetrics, + AgentAction +) +from .agent_analytics import ( + AgentAnalytics, + DecisionMetrics, + AgentPerformanceSnapshot, + MetricType +) +from .memory_system import ( + AgentMemorySystem, + AgentMemory, + MemoryType +) + +__all__ = [ + 'SyntheticTrainingEnvironment', + 'EnvironmentConfig', + 'InformationLevel', + 'EnvironmentState', + 'AgentPerformanceMetrics', + 'AgentAction', + 'AgentAnalytics', + 'DecisionMetrics', + 'AgentPerformanceSnapshot', + 'MetricType', + 'AgentMemorySystem', + 'AgentMemory', + 'MemoryType' +] \ No newline at end of file diff --git a/src/training/agent_analytics.py b/src/training/agent_analytics.py new file mode 100644 index 00000000..0ef1219b --- /dev/null +++ b/src/training/agent_analytics.py @@ -0,0 +1,882 @@ +""" +Agent Performance Analytics System + +Comprehensive analytics and metrics tracking for agent training +performance across synthetic scenarios with Kelly Criterion validation. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Any +from enum import Enum +import asyncio +import json +import logging +from datetime import datetime, timedelta +import numpy as np +from collections import defaultdict, deque +import statistics + +from ..synthetic_data.storage.chromadb_manager import ChromaDBManager +from ..synthetic_data.preprocessing.nfl_dataset_processor import ProcessedNFLPlay + + +class MetricType(Enum): + """Types of performance metrics tracked""" + DECISION_QUALITY = "decision_quality" + KELLY_ADHERENCE = "kelly_adherence" + RISK_MANAGEMENT = "risk_management" + PROFITABILITY = "profitability" + LEARNING_PROGRESS = "learning_progress" + BEHAVIORAL_PATTERNS = "behavioral_patterns" + + +@dataclass +class DecisionMetrics: + """Metrics for individual trading decisions""" + decision_id: str + agent_id: str + scenario_id: str + timestamp: datetime + market_ticker: str + decision_type: str # "buy", "sell", "hold" + confidence: float + expected_value: float + kelly_fraction: float + actual_kelly_used: float + position_size: float + outcome: Optional[float] = None # Actual P&L + market_efficiency: float = 0.8 + information_advantage: float = 0.0 + execution_latency: float = 0.0 + + @property + def kelly_deviation(self) -> float: + """How much the agent deviated from optimal Kelly""" + return abs(self.actual_kelly_used - self.kelly_fraction) + + @property + def risk_adjusted_return(self) -> float: + """Return adjusted for risk taken""" + if self.outcome is None or self.position_size == 0: + return 0.0 + return self.outcome / self.position_size + + +@dataclass +class AgentPerformanceSnapshot: + """Performance snapshot for an agent at a point in time""" + agent_id: str + timestamp: datetime + scenarios_completed: int + total_decisions: int + win_rate: float + total_pnl: float + sharpe_ratio: float + max_drawdown: float + avg_kelly_deviation: float + avg_confidence: float + learning_velocity: float # Rate of improvement + behavioral_consistency: float + risk_score: float + decision_speed: float # Avg latency + + # Advanced metrics + kelly_adherence_score: float = 0.0 + information_utilization: float = 0.0 + pattern_recognition_score: float = 0.0 + edge_case_performance: float = 0.0 + + +class AgentAnalytics: + """ + Comprehensive analytics engine for agent training performance. + + Tracks decision quality, Kelly Criterion adherence, learning progress, + and behavioral patterns across synthetic training scenarios. + """ + + def __init__(self, chroma_manager: ChromaDBManager): + self.chroma_manager = chroma_manager + self.decision_history: Dict[str, deque] = defaultdict(lambda: deque(maxlen=10000)) + self.performance_snapshots: Dict[str, List[AgentPerformanceSnapshot]] = defaultdict(list) + self.behavioral_patterns: Dict[str, Dict] = defaultdict(dict) + self.logger = logging.getLogger(__name__) + + # Analytics configuration + self.snapshot_interval = timedelta(hours=1) + self.pattern_window = 100 # Decisions to analyze for patterns + self.performance_windows = [10, 50, 100, 500] # Different time horizons + + async def record_decision(self, decision: DecisionMetrics) -> None: + """Record a single trading decision with comprehensive metrics""" + try: + # Store in memory for fast access + self.decision_history[decision.agent_id].append(decision) + + # Store in ChromaDB for persistent analytics + await self._store_decision_in_chromadb(decision) + + # Update behavioral patterns + await self._update_behavioral_patterns(decision) + + # Check if snapshot needed + await self._check_snapshot_trigger(decision.agent_id) + + except Exception as e: + self.logger.error(f"Failed to record decision {decision.decision_id}: {e}") + raise + + async def _store_decision_in_chromadb(self, decision: DecisionMetrics) -> None: + """Store decision metrics in ChromaDB for semantic search""" + try: + # Create searchable description + description = self._create_decision_description(decision) + + # Prepare metadata + metadata = { + "agent_id": decision.agent_id, + "scenario_id": decision.scenario_id, + "decision_type": decision.decision_type, + "market_ticker": decision.market_ticker, + "timestamp": decision.timestamp.isoformat(), + "confidence": decision.confidence, + "kelly_fraction": decision.kelly_fraction, + "actual_kelly_used": decision.actual_kelly_used, + "kelly_deviation": decision.kelly_deviation, + "position_size": decision.position_size, + "expected_value": decision.expected_value, + "outcome": decision.outcome if decision.outcome is not None else 0.0, + "risk_adjusted_return": decision.risk_adjusted_return, + "execution_latency": decision.execution_latency, + "market_efficiency": decision.market_efficiency, + "information_advantage": decision.information_advantage + } + + # Store in agent decisions collection + decisions_collection = self.chroma_manager.get_collection("agent_decisions") + if not decisions_collection: + decisions_collection = await self.chroma_manager.create_collection( + "agent_decisions", + "Agent trading decisions with performance metrics" + ) + + decisions_collection.add( + ids=[decision.decision_id], + documents=[description], + metadatas=[metadata] + ) + + except Exception as e: + self.logger.error(f"Failed to store decision in ChromaDB: {e}") + + def _create_decision_description(self, decision: DecisionMetrics) -> str: + """Create semantic description of decision for ChromaDB storage""" + outcome_desc = "profitable" if decision.outcome and decision.outcome > 0 else "losing" if decision.outcome and decision.outcome < 0 else "pending" + kelly_adherence = "optimal" if decision.kelly_deviation < 0.1 else "suboptimal" + confidence_level = "high" if decision.confidence > 0.7 else "medium" if decision.confidence > 0.4 else "low" + + return (f"{decision.decision_type.upper()} decision on {decision.market_ticker} " + f"with {confidence_level} confidence ({decision.confidence:.2f}). " + f"Used {kelly_adherence} Kelly sizing (deviation: {decision.kelly_deviation:.3f}). " + f"Position size: {decision.position_size:.2f}, Expected value: {decision.expected_value:.3f}. " + f"Market efficiency: {decision.market_efficiency:.2f}, Information advantage: {decision.information_advantage:.3f}. " + f"Execution latency: {decision.execution_latency:.3f}s. Outcome: {outcome_desc}") + + async def _update_behavioral_patterns(self, decision: DecisionMetrics) -> None: + """Update behavioral pattern analysis for the agent""" + agent_decisions = self.decision_history[decision.agent_id] + + if len(agent_decisions) < 10: + return # Need minimum decisions for pattern analysis + + # Analyze recent decisions for patterns + recent_decisions = list(agent_decisions)[-self.pattern_window:] + + patterns = { + "avg_confidence": statistics.mean([d.confidence for d in recent_decisions]), + "confidence_volatility": statistics.stdev([d.confidence for d in recent_decisions]) if len(recent_decisions) > 1 else 0, + "kelly_consistency": 1 - statistics.mean([d.kelly_deviation for d in recent_decisions]), + "decision_type_distribution": self._calculate_decision_distribution(recent_decisions), + "risk_taking_tendency": statistics.mean([d.position_size for d in recent_decisions]), + "market_timing_score": await self._calculate_market_timing_score(recent_decisions), + "learning_trend": self._calculate_learning_trend(recent_decisions), + "execution_speed_trend": self._calculate_speed_trend(recent_decisions) + } + + self.behavioral_patterns[decision.agent_id] = patterns + + def _calculate_decision_distribution(self, decisions: List[DecisionMetrics]) -> Dict[str, float]: + """Calculate distribution of decision types""" + total = len(decisions) + if total == 0: + return {"buy": 0, "sell": 0, "hold": 0} + + counts = defaultdict(int) + for decision in decisions: + counts[decision.decision_type] += 1 + + return {decision_type: count / total for decision_type, count in counts.items()} + + async def _calculate_market_timing_score(self, decisions: List[DecisionMetrics]) -> float: + """Calculate how well agent times market entries""" + if len(decisions) < 5: + return 0.5 # Neutral score + + # Analyze correlation between confidence and outcomes + profitable_decisions = [d for d in decisions if d.outcome and d.outcome > 0] + if not profitable_decisions: + return 0.3 # Below average + + # High confidence decisions should be more profitable + high_conf_decisions = [d for d in decisions if d.confidence > 0.7] + if not high_conf_decisions: + return 0.4 + + high_conf_profitability = sum(1 for d in high_conf_decisions if d.outcome and d.outcome > 0) / len(high_conf_decisions) + return min(high_conf_profitability * 1.2, 1.0) # Cap at 1.0 + + def _calculate_learning_trend(self, decisions: List[DecisionMetrics]) -> float: + """Calculate if agent is improving over time""" + if len(decisions) < 20: + return 0.0 + + # Split into early and recent halves + mid_point = len(decisions) // 2 + early_decisions = decisions[:mid_point] + recent_decisions = decisions[mid_point:] + + # Compare performance metrics + early_avg_deviation = statistics.mean([d.kelly_deviation for d in early_decisions]) + recent_avg_deviation = statistics.mean([d.kelly_deviation for d in recent_decisions]) + + early_avg_confidence = statistics.mean([d.confidence for d in early_decisions]) + recent_avg_confidence = statistics.mean([d.confidence for d in recent_decisions]) + + # Improvement is less deviation and higher confidence + deviation_improvement = max(0, early_avg_deviation - recent_avg_deviation) + confidence_improvement = max(0, recent_avg_confidence - early_avg_confidence) + + return min((deviation_improvement + confidence_improvement) / 2, 1.0) + + def _calculate_speed_trend(self, decisions: List[DecisionMetrics]) -> float: + """Calculate trend in execution speed""" + latencies = [d.execution_latency for d in decisions if d.execution_latency > 0] + if len(latencies) < 5: + return 0.0 + + # Simple linear trend + x = range(len(latencies)) + trend = np.polyfit(x, latencies, 1)[0] + + # Negative trend (getting faster) is good + return max(0, -trend) + + async def _check_snapshot_trigger(self, agent_id: str) -> None: + """Check if it's time to create a performance snapshot""" + last_snapshot = self.performance_snapshots[agent_id][-1] if self.performance_snapshots[agent_id] else None + + if (not last_snapshot or + datetime.now() - last_snapshot.timestamp >= self.snapshot_interval): + await self.create_performance_snapshot(agent_id) + + async def create_performance_snapshot(self, agent_id: str) -> AgentPerformanceSnapshot: + """Create comprehensive performance snapshot for an agent""" + try: + decisions = list(self.decision_history[agent_id]) + if not decisions: + # Return empty snapshot + return AgentPerformanceSnapshot( + agent_id=agent_id, + timestamp=datetime.now(), + scenarios_completed=0, + total_decisions=0, + win_rate=0.0, + total_pnl=0.0, + sharpe_ratio=0.0, + max_drawdown=0.0, + avg_kelly_deviation=0.0, + avg_confidence=0.0, + learning_velocity=0.0, + behavioral_consistency=0.0, + risk_score=0.0, + decision_speed=0.0 + ) + + # Calculate basic metrics + completed_decisions = [d for d in decisions if d.outcome is not None] + win_rate = sum(1 for d in completed_decisions if d.outcome > 0) / len(completed_decisions) if completed_decisions else 0 + total_pnl = sum(d.outcome for d in completed_decisions if d.outcome is not None) + + # Calculate Sharpe ratio + returns = [d.risk_adjusted_return for d in completed_decisions if d.outcome is not None] + sharpe_ratio = (statistics.mean(returns) / statistics.stdev(returns)) if len(returns) > 1 and statistics.stdev(returns) > 0 else 0 + + # Calculate max drawdown + max_drawdown = self._calculate_max_drawdown(completed_decisions) + + # Kelly metrics + avg_kelly_deviation = statistics.mean([d.kelly_deviation for d in decisions]) + kelly_adherence_score = max(0, 1 - avg_kelly_deviation) + + # Behavioral metrics + avg_confidence = statistics.mean([d.confidence for d in decisions]) + behavioral_consistency = self._calculate_behavioral_consistency(decisions) + + # Advanced metrics + patterns = self.behavioral_patterns.get(agent_id, {}) + learning_velocity = patterns.get("learning_trend", 0.0) + decision_speed = statistics.mean([d.execution_latency for d in decisions if d.execution_latency > 0]) or 0.0 + + # Risk score + risk_score = self._calculate_risk_score(decisions) + + # Information utilization + information_utilization = statistics.mean([d.information_advantage for d in decisions]) + + # Count unique scenarios + unique_scenarios = len(set(d.scenario_id for d in decisions)) + + snapshot = AgentPerformanceSnapshot( + agent_id=agent_id, + timestamp=datetime.now(), + scenarios_completed=unique_scenarios, + total_decisions=len(decisions), + win_rate=win_rate, + total_pnl=total_pnl, + sharpe_ratio=sharpe_ratio, + max_drawdown=max_drawdown, + avg_kelly_deviation=avg_kelly_deviation, + avg_confidence=avg_confidence, + learning_velocity=learning_velocity, + behavioral_consistency=behavioral_consistency, + risk_score=risk_score, + decision_speed=decision_speed, + kelly_adherence_score=kelly_adherence_score, + information_utilization=information_utilization, + pattern_recognition_score=patterns.get("market_timing_score", 0.5), + edge_case_performance=await self._calculate_edge_case_performance(agent_id) + ) + + self.performance_snapshots[agent_id].append(snapshot) + await self._store_snapshot_in_chromadb(snapshot) + + return snapshot + + except Exception as e: + self.logger.error(f"Failed to create performance snapshot for {agent_id}: {e}") + raise + + def _calculate_max_drawdown(self, decisions: List[DecisionMetrics]) -> float: + """Calculate maximum drawdown from peak equity""" + if not decisions: + return 0.0 + + cumulative_pnl = 0.0 + running_max = 0.0 + max_drawdown = 0.0 + + for decision in decisions: + if decision.outcome is not None: + cumulative_pnl += decision.outcome + running_max = max(running_max, cumulative_pnl) + drawdown = running_max - cumulative_pnl + max_drawdown = max(max_drawdown, drawdown) + + return max_drawdown + + def _calculate_behavioral_consistency(self, decisions: List[DecisionMetrics]) -> float: + """Calculate how consistent the agent's behavior is""" + if len(decisions) < 10: + return 0.5 + + # Measure consistency in confidence levels + confidence_std = statistics.stdev([d.confidence for d in decisions]) + confidence_consistency = max(0, 1 - confidence_std) + + # Measure consistency in Kelly adherence + kelly_deviations = [d.kelly_deviation for d in decisions] + kelly_std = statistics.stdev(kelly_deviations) if len(kelly_deviations) > 1 else 0 + kelly_consistency = max(0, 1 - kelly_std) + + return (confidence_consistency + kelly_consistency) / 2 + + def _calculate_risk_score(self, decisions: List[DecisionMetrics]) -> float: + """Calculate overall risk score (0 = low risk, 1 = high risk)""" + if not decisions: + return 0.5 + + # Average position size relative to Kelly + avg_position_ratio = statistics.mean([d.actual_kelly_used for d in decisions]) + position_risk = min(avg_position_ratio / 0.25, 1.0) # Normalize to 25% Kelly + + # Kelly deviation risk + kelly_risk = statistics.mean([d.kelly_deviation for d in decisions]) + + # Concentration risk (are decisions spread across markets?) + unique_markets = len(set(d.market_ticker for d in decisions)) + concentration_risk = max(0, 1 - unique_markets / 10) # Normalize to 10 markets + + return (position_risk + kelly_risk + concentration_risk) / 3 + + async def _calculate_edge_case_performance(self, agent_id: str) -> float: + """Calculate performance specifically on edge case scenarios""" + try: + # Query ChromaDB for edge case scenarios + edge_case_collection = self.chroma_manager.get_collection("training_scenarios") + if not edge_case_collection: + return 0.5 # Neutral score if no data + + # Search for edge case scenarios + edge_cases = edge_case_collection.query( + query_texts=["edge case", "rare event", "unusual situation", "outlier scenario"], + where={"agent_id": agent_id}, + n_results=100 + ) + + if not edge_cases['documents']: + return 0.5 # No edge cases found + + # Analyze performance on these scenarios + edge_case_decisions = [d for d in self.decision_history[agent_id] + if any(scenario_id in edge_cases['ids'] for scenario_id in edge_cases['ids'])] + + if not edge_case_decisions: + return 0.5 + + # Calculate win rate on edge cases + completed_edge_decisions = [d for d in edge_case_decisions if d.outcome is not None] + if not completed_edge_decisions: + return 0.5 + + edge_win_rate = sum(1 for d in completed_edge_decisions if d.outcome > 0) / len(completed_edge_decisions) + return edge_win_rate + + except Exception as e: + self.logger.error(f"Failed to calculate edge case performance: {e}") + return 0.5 + + async def _store_snapshot_in_chromadb(self, snapshot: AgentPerformanceSnapshot) -> None: + """Store performance snapshot in ChromaDB""" + try: + description = (f"Performance snapshot for agent {snapshot.agent_id}: " + f"{snapshot.total_decisions} decisions across {snapshot.scenarios_completed} scenarios. " + f"Win rate: {snapshot.win_rate:.2f}, Total P&L: {snapshot.total_pnl:.2f}, " + f"Sharpe: {snapshot.sharpe_ratio:.2f}, Max DD: {snapshot.max_drawdown:.2f}. " + f"Kelly adherence: {snapshot.kelly_adherence_score:.2f}, " + f"Learning velocity: {snapshot.learning_velocity:.2f}") + + metadata = { + "agent_id": snapshot.agent_id, + "timestamp": snapshot.timestamp.isoformat(), + "scenarios_completed": snapshot.scenarios_completed, + "total_decisions": snapshot.total_decisions, + "win_rate": snapshot.win_rate, + "total_pnl": snapshot.total_pnl, + "sharpe_ratio": snapshot.sharpe_ratio, + "max_drawdown": snapshot.max_drawdown, + "avg_kelly_deviation": snapshot.avg_kelly_deviation, + "kelly_adherence_score": snapshot.kelly_adherence_score, + "learning_velocity": snapshot.learning_velocity, + "risk_score": snapshot.risk_score + } + + # Store in performance snapshots collection + snapshots_collection = self.chroma_manager.get_collection("performance_snapshots") + if not snapshots_collection: + snapshots_collection = await self.chroma_manager.create_collection( + "performance_snapshots", + "Agent performance snapshots over time" + ) + + snapshot_id = f"{snapshot.agent_id}_{snapshot.timestamp.strftime('%Y%m%d_%H%M%S')}" + snapshots_collection.add( + ids=[snapshot_id], + documents=[description], + metadatas=[metadata] + ) + + except Exception as e: + self.logger.error(f"Failed to store snapshot in ChromaDB: {e}") + + async def get_agent_analytics(self, agent_id: str, time_window: Optional[timedelta] = None) -> Dict[str, Any]: + """Get comprehensive analytics for a specific agent""" + try: + decisions = list(self.decision_history[agent_id]) + + # Filter by time window if specified + if time_window: + cutoff_time = datetime.now() - time_window + decisions = [d for d in decisions if d.timestamp >= cutoff_time] + + if not decisions: + return {"error": f"No decisions found for agent {agent_id}"} + + # Get latest snapshot + latest_snapshot = self.performance_snapshots[agent_id][-1] if self.performance_snapshots[agent_id] else None + + # Calculate detailed metrics + analytics = { + "agent_id": agent_id, + "analysis_timestamp": datetime.now().isoformat(), + "time_window": str(time_window) if time_window else "all_time", + "total_decisions": len(decisions), + "latest_snapshot": { + "timestamp": latest_snapshot.timestamp.isoformat() if latest_snapshot else None, + "win_rate": latest_snapshot.win_rate if latest_snapshot else 0, + "total_pnl": latest_snapshot.total_pnl if latest_snapshot else 0, + "sharpe_ratio": latest_snapshot.sharpe_ratio if latest_snapshot else 0, + "kelly_adherence_score": latest_snapshot.kelly_adherence_score if latest_snapshot else 0, + "learning_velocity": latest_snapshot.learning_velocity if latest_snapshot else 0 + } if latest_snapshot else None, + + "behavioral_patterns": self.behavioral_patterns.get(agent_id, {}), + + "decision_breakdown": self._get_decision_breakdown(decisions), + "performance_by_market": self._get_performance_by_market(decisions), + "kelly_analysis": self._get_kelly_analysis(decisions), + "confidence_analysis": self._get_confidence_analysis(decisions), + "learning_progression": await self._get_learning_progression(agent_id), + "risk_analysis": self._get_risk_analysis(decisions) + } + + return analytics + + except Exception as e: + self.logger.error(f"Failed to get analytics for {agent_id}: {e}") + return {"error": str(e)} + + def _get_decision_breakdown(self, decisions: List[DecisionMetrics]) -> Dict[str, Any]: + """Detailed breakdown of decision types and outcomes""" + breakdown = defaultdict(lambda: {"count": 0, "wins": 0, "total_pnl": 0.0}) + + for decision in decisions: + key = decision.decision_type + breakdown[key]["count"] += 1 + if decision.outcome is not None: + if decision.outcome > 0: + breakdown[key]["wins"] += 1 + breakdown[key]["total_pnl"] += decision.outcome + + # Calculate win rates + for key in breakdown: + completed = breakdown[key]["count"] + breakdown[key]["win_rate"] = breakdown[key]["wins"] / completed if completed > 0 else 0 + + return dict(breakdown) + + def _get_performance_by_market(self, decisions: List[DecisionMetrics]) -> Dict[str, Any]: + """Performance breakdown by market ticker""" + market_performance = defaultdict(lambda: {"decisions": 0, "wins": 0, "total_pnl": 0.0, "avg_confidence": 0.0}) + + for decision in decisions: + market = decision.market_ticker + market_performance[market]["decisions"] += 1 + market_performance[market]["avg_confidence"] += decision.confidence + + if decision.outcome is not None: + if decision.outcome > 0: + market_performance[market]["wins"] += 1 + market_performance[market]["total_pnl"] += decision.outcome + + # Finalize calculations + for market in market_performance: + data = market_performance[market] + data["avg_confidence"] /= data["decisions"] if data["decisions"] > 0 else 1 + data["win_rate"] = data["wins"] / data["decisions"] if data["decisions"] > 0 else 0 + + return dict(market_performance) + + def _get_kelly_analysis(self, decisions: List[DecisionMetrics]) -> Dict[str, Any]: + """Detailed analysis of Kelly Criterion adherence""" + kelly_deviations = [d.kelly_deviation for d in decisions] + kelly_fractions = [d.kelly_fraction for d in decisions] + actual_kelly_used = [d.actual_kelly_used for d in decisions] + + return { + "avg_kelly_fraction": statistics.mean(kelly_fractions), + "avg_actual_kelly": statistics.mean(actual_kelly_used), + "avg_deviation": statistics.mean(kelly_deviations), + "deviation_std": statistics.stdev(kelly_deviations) if len(kelly_deviations) > 1 else 0, + "optimal_decisions": sum(1 for d in kelly_deviations if d < 0.1), + "suboptimal_decisions": sum(1 for d in kelly_deviations if d >= 0.1), + "adherence_score": max(0, 1 - statistics.mean(kelly_deviations)), + "over_betting_frequency": sum(1 for d in decisions if d.actual_kelly_used > d.kelly_fraction) / len(decisions), + "under_betting_frequency": sum(1 for d in decisions if d.actual_kelly_used < d.kelly_fraction) / len(decisions) + } + + def _get_confidence_analysis(self, decisions: List[DecisionMetrics]) -> Dict[str, Any]: + """Analysis of agent confidence patterns""" + confidences = [d.confidence for d in decisions] + + # Confidence vs outcome correlation + completed_decisions = [d for d in decisions if d.outcome is not None] + high_conf_decisions = [d for d in completed_decisions if d.confidence > 0.7] + low_conf_decisions = [d for d in completed_decisions if d.confidence < 0.4] + + return { + "avg_confidence": statistics.mean(confidences), + "confidence_std": statistics.stdev(confidences) if len(confidences) > 1 else 0, + "high_confidence_decisions": len(high_conf_decisions), + "low_confidence_decisions": len(low_conf_decisions), + "high_conf_win_rate": sum(1 for d in high_conf_decisions if d.outcome > 0) / len(high_conf_decisions) if high_conf_decisions else 0, + "low_conf_win_rate": sum(1 for d in low_conf_decisions if d.outcome > 0) / len(low_conf_decisions) if low_conf_decisions else 0, + "confidence_calibration": self._calculate_confidence_calibration(completed_decisions) + } + + def _calculate_confidence_calibration(self, decisions: List[DecisionMetrics]) -> Dict[str, float]: + """Calculate how well calibrated the agent's confidence is""" + if len(decisions) < 10: + return {"score": 0.5, "note": "Insufficient data"} + + # Bucket decisions by confidence level + buckets = {"0.0-0.2": [], "0.2-0.4": [], "0.4-0.6": [], "0.6-0.8": [], "0.8-1.0": []} + + for decision in decisions: + if decision.confidence <= 0.2: + buckets["0.0-0.2"].append(decision) + elif decision.confidence <= 0.4: + buckets["0.2-0.4"].append(decision) + elif decision.confidence <= 0.6: + buckets["0.4-0.6"].append(decision) + elif decision.confidence <= 0.8: + buckets["0.6-0.8"].append(decision) + else: + buckets["0.8-1.0"].append(decision) + + calibration_scores = [] + for bucket_name, bucket_decisions in buckets.items(): + if not bucket_decisions: + continue + + expected_win_rate = sum(d.confidence for d in bucket_decisions) / len(bucket_decisions) + actual_win_rate = sum(1 for d in bucket_decisions if d.outcome and d.outcome > 0) / len(bucket_decisions) + + # Calibration is how close actual matches expected + calibration_scores.append(1 - abs(expected_win_rate - actual_win_rate)) + + overall_calibration = statistics.mean(calibration_scores) if calibration_scores else 0.5 + + return { + "score": overall_calibration, + "bucket_analysis": {k: len(v) for k, v in buckets.items() if v} + } + + async def _get_learning_progression(self, agent_id: str) -> Dict[str, Any]: + """Analyze learning progression over time""" + snapshots = self.performance_snapshots[agent_id] + if len(snapshots) < 2: + return {"note": "Insufficient snapshots for progression analysis"} + + # Track key metrics over time + timestamps = [s.timestamp for s in snapshots] + win_rates = [s.win_rate for s in snapshots] + kelly_scores = [s.kelly_adherence_score for s in snapshots] + learning_velocities = [s.learning_velocity for s in snapshots] + + return { + "snapshot_count": len(snapshots), + "time_span": str(timestamps[-1] - timestamps[0]), + "win_rate_trend": self._calculate_trend(win_rates), + "kelly_adherence_trend": self._calculate_trend(kelly_scores), + "learning_velocity_trend": self._calculate_trend(learning_velocities), + "overall_improvement_score": statistics.mean([s.learning_velocity for s in snapshots[-5:]]) if len(snapshots) >= 5 else 0 + } + + def _calculate_trend(self, values: List[float]) -> Dict[str, Any]: + """Calculate trend in a series of values""" + if len(values) < 3: + return {"trend": "insufficient_data"} + + x = range(len(values)) + slope = np.polyfit(x, values, 1)[0] + + if abs(slope) < 0.001: + trend = "stable" + elif slope > 0: + trend = "improving" + else: + trend = "declining" + + return { + "trend": trend, + "slope": slope, + "start_value": values[0], + "end_value": values[-1], + "change": values[-1] - values[0] + } + + def _get_risk_analysis(self, decisions: List[DecisionMetrics]) -> Dict[str, Any]: + """Comprehensive risk analysis""" + if not decisions: + return {"error": "No decisions to analyze"} + + position_sizes = [d.position_size for d in decisions] + outcomes = [d.outcome for d in decisions if d.outcome is not None] + + return { + "avg_position_size": statistics.mean(position_sizes), + "max_position_size": max(position_sizes), + "position_size_std": statistics.stdev(position_sizes) if len(position_sizes) > 1 else 0, + "largest_loss": min(outcomes) if outcomes else 0, + "largest_gain": max(outcomes) if outcomes else 0, + "risk_reward_ratio": abs(max(outcomes)) / abs(min(outcomes)) if outcomes and min(outcomes) < 0 else 0, + "consecutive_losses": self._calculate_max_consecutive_losses(decisions), + "var_95": np.percentile(outcomes, 5) if outcomes else 0, # Value at Risk 95th percentile + "risk_score": self._calculate_risk_score(decisions) + } + + def _calculate_max_consecutive_losses(self, decisions: List[DecisionMetrics]) -> int: + """Calculate maximum consecutive losing trades""" + max_consecutive = 0 + current_consecutive = 0 + + for decision in decisions: + if decision.outcome is not None: + if decision.outcome <= 0: + current_consecutive += 1 + max_consecutive = max(max_consecutive, current_consecutive) + else: + current_consecutive = 0 + + return max_consecutive + + async def generate_training_report(self, agent_ids: List[str] = None, time_window: Optional[timedelta] = None) -> Dict[str, Any]: + """Generate comprehensive training report for specified agents""" + try: + target_agents = agent_ids or list(self.decision_history.keys()) + + if not target_agents: + return {"error": "No agents found"} + + report = { + "report_timestamp": datetime.now().isoformat(), + "time_window": str(time_window) if time_window else "all_time", + "agents_analyzed": len(target_agents), + "summary": {}, + "agent_details": {}, + "comparative_analysis": {}, + "recommendations": [] + } + + # Gather analytics for each agent + all_agent_analytics = {} + for agent_id in target_agents: + agent_analytics = await self.get_agent_analytics(agent_id, time_window) + all_agent_analytics[agent_id] = agent_analytics + report["agent_details"][agent_id] = agent_analytics + + # Generate summary statistics + report["summary"] = self._generate_summary_statistics(all_agent_analytics) + + # Comparative analysis + report["comparative_analysis"] = self._generate_comparative_analysis(all_agent_analytics) + + # Generate recommendations + report["recommendations"] = await self._generate_training_recommendations(all_agent_analytics) + + return report + + except Exception as e: + self.logger.error(f"Failed to generate training report: {e}") + return {"error": str(e)} + + def _generate_summary_statistics(self, all_analytics: Dict[str, Dict]) -> Dict[str, Any]: + """Generate summary statistics across all agents""" + valid_analytics = [a for a in all_analytics.values() if "error" not in a] + + if not valid_analytics: + return {"error": "No valid analytics data"} + + total_decisions = sum(a["total_decisions"] for a in valid_analytics) + + # Aggregate latest snapshots + latest_snapshots = [a["latest_snapshot"] for a in valid_analytics if a["latest_snapshot"]] + + if not latest_snapshots: + return {"total_decisions": total_decisions, "note": "No snapshot data available"} + + return { + "total_decisions": total_decisions, + "avg_win_rate": statistics.mean([s["win_rate"] for s in latest_snapshots]), + "total_pnl": sum(s["total_pnl"] for s in latest_snapshots), + "avg_sharpe_ratio": statistics.mean([s["sharpe_ratio"] for s in latest_snapshots]), + "avg_kelly_adherence": statistics.mean([s["kelly_adherence_score"] for s in latest_snapshots]), + "avg_learning_velocity": statistics.mean([s["learning_velocity"] for s in latest_snapshots]), + "best_performer": max(latest_snapshots, key=lambda s: s["total_pnl"])["agent_id"] if latest_snapshots else None, + "fastest_learner": max(latest_snapshots, key=lambda s: s["learning_velocity"])["agent_id"] if latest_snapshots else None + } + + def _generate_comparative_analysis(self, all_analytics: Dict[str, Dict]) -> Dict[str, Any]: + """Generate comparative analysis between agents""" + valid_analytics = {k: v for k, v in all_analytics.items() if "error" not in v} + + if len(valid_analytics) < 2: + return {"note": "Need at least 2 agents for comparison"} + + # Compare key metrics + comparisons = {} + + for metric in ["win_rate", "total_pnl", "sharpe_ratio", "kelly_adherence_score", "learning_velocity"]: + values = {} + for agent_id, analytics in valid_analytics.items(): + if analytics.get("latest_snapshot") and analytics["latest_snapshot"].get(metric) is not None: + values[agent_id] = analytics["latest_snapshot"][metric] + + if values: + comparisons[metric] = { + "best": max(values, key=values.get), + "worst": min(values, key=values.get), + "best_value": max(values.values()), + "worst_value": min(values.values()), + "spread": max(values.values()) - min(values.values()) + } + + return comparisons + + async def _generate_training_recommendations(self, all_analytics: Dict[str, Dict]) -> List[str]: + """Generate specific training recommendations based on analytics""" + recommendations = [] + + valid_analytics = {k: v for k, v in all_analytics.items() if "error" not in v} + + for agent_id, analytics in valid_analytics.items(): + latest_snapshot = analytics.get("latest_snapshot") + if not latest_snapshot: + continue + + # Kelly adherence recommendations + if latest_snapshot["kelly_adherence_score"] < 0.7: + recommendations.append(f"Agent {agent_id}: Improve Kelly Criterion adherence (current: {latest_snapshot['kelly_adherence_score']:.2f}). Consider additional training on position sizing.") + + # Learning velocity recommendations + if latest_snapshot["learning_velocity"] < 0.3: + recommendations.append(f"Agent {agent_id}: Low learning velocity ({latest_snapshot['learning_velocity']:.2f}). Increase scenario diversity or adjust learning parameters.") + + # Win rate recommendations + if latest_snapshot["win_rate"] < 0.45: + recommendations.append(f"Agent {agent_id}: Below-average win rate ({latest_snapshot['win_rate']:.2f}). Focus on signal quality and market timing training.") + + # Sharpe ratio recommendations + if latest_snapshot["sharpe_ratio"] < 0.5: + recommendations.append(f"Agent {agent_id}: Low risk-adjusted returns (Sharpe: {latest_snapshot['sharpe_ratio']:.2f}). Emphasize risk management training.") + + # Global recommendations + if len(valid_analytics) > 1: + best_performer = max(valid_analytics.items(), key=lambda x: x[1]["latest_snapshot"]["total_pnl"] if x[1].get("latest_snapshot") else 0) + recommendations.append(f"Consider using Agent {best_performer[0]}'s strategies as training templates for other agents.") + + return recommendations + + async def export_analytics_data(self, agent_id: str, format: str = "json") -> str: + """Export comprehensive analytics data for external analysis""" + try: + analytics = await self.get_agent_analytics(agent_id) + + if format.lower() == "json": + return json.dumps(analytics, indent=2, default=str) + else: + raise ValueError(f"Unsupported format: {format}") + + except Exception as e: + self.logger.error(f"Failed to export analytics data: {e}") + raise \ No newline at end of file diff --git a/src/training/memory_system.py b/src/training/memory_system.py new file mode 100644 index 00000000..131dce94 --- /dev/null +++ b/src/training/memory_system.py @@ -0,0 +1,815 @@ +""" +ChromaDB Memory Integration System + +Integrates ChromaDB with agent memory systems for persistent learning +and experience replay from synthetic training scenarios. +""" + +import logging +import asyncio +from typing import List, Dict, Any, Optional, Tuple, Union +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +import json +import hashlib + +from ..synthetic_data.storage.chromadb_manager import ChromaDBManager +from .synthetic_env import AgentPerformanceMetrics, AgentAction +from src.sdk.core.base_adapter import StandardizedEvent + +logger = logging.getLogger(__name__) + + +class MemoryType(Enum): + """Types of agent memories""" + EXPERIENCE = "experience" # Trading experiences + PATTERN = "pattern" # Recognized patterns + STRATEGY = "strategy" # Successful strategies + MISTAKE = "mistake" # Failed decisions for avoidance + CONTEXT = "context" # Situational context + PERFORMANCE = "performance" # Performance records + + +@dataclass +class AgentMemory: + """Individual agent memory entry""" + memory_id: str + agent_id: str + memory_type: MemoryType + timestamp: datetime + + # Core content + description: str + context: Dict[str, Any] + outcome: Dict[str, Any] + + # Learning metadata + importance_score: float = 0.5 # 0-1 importance for retention + confidence: float = 0.5 # 0-1 confidence in the memory + usage_count: int = 0 # How often this memory was accessed + success_rate: float = 0.5 # Success rate when this memory was used + + # Retrieval metadata + tags: List[str] = field(default_factory=list) + related_scenarios: List[str] = field(default_factory=list) + embedding_metadata: Dict[str, Any] = field(default_factory=dict) + + created_at: datetime = field(default_factory=datetime.now) + last_accessed: Optional[datetime] = None + last_updated: Optional[datetime] = None + + +@dataclass +class MemoryQuery: + """Query for retrieving memories""" + query_text: str + agent_id: Optional[str] = None + memory_types: List[MemoryType] = field(default_factory=list) + tags: List[str] = field(default_factory=list) + min_importance: float = 0.0 + max_results: int = 10 + include_context: bool = True + + +@dataclass +class MemorySearchResult: + """Result from memory search""" + memory: AgentMemory + relevance_score: float + similarity_score: float + context_match_score: float + + +class AgentMemorySystem: + """ + ChromaDB-backed memory system for agent learning and experience replay + """ + + def __init__(self, chromadb_manager: ChromaDBManager = None): + """ + Initialize agent memory system + + Args: + chromadb_manager: ChromaDB manager instance + """ + self.chromadb = chromadb_manager or ChromaDBManager() + + # Initialize memory collections in ChromaDB + self._initialize_memory_collections() + + # Memory management + self.memory_cache: Dict[str, AgentMemory] = {} + self.access_patterns: Dict[str, List[datetime]] = {} + + # Performance tracking + self.memory_stats = { + "total_memories": 0, + "memories_by_type": {}, + "memories_by_agent": {}, + "avg_retrieval_time_ms": 0.0, + "cache_hit_rate": 0.0 + } + + logger.info("Initialized AgentMemorySystem with ChromaDB integration") + + def _initialize_memory_collections(self): + """Initialize ChromaDB collections for agent memories""" + + # Agent experiences collection + try: + self.experiences_collection = self.chromadb.client.get_or_create_collection( + name="agent_experiences", + metadata={ + "description": "Agent trading experiences and outcomes", + "purpose": "experience_replay", + "source": "agent_training" + } + ) + except Exception as e: + logger.error(f"Failed to create experiences collection: {e}") + self.experiences_collection = None + + # Pattern recognition collection + try: + self.patterns_collection = self.chromadb.client.get_or_create_collection( + name="agent_patterns", + metadata={ + "description": "Recognized market and game patterns", + "purpose": "pattern_matching", + "source": "agent_learning" + } + ) + except Exception as e: + logger.error(f"Failed to create patterns collection: {e}") + self.patterns_collection = None + + # Strategy knowledge collection + try: + self.strategies_collection = self.chromadb.client.get_or_create_collection( + name="agent_strategies", + metadata={ + "description": "Successful trading strategies and tactics", + "purpose": "strategy_recall", + "source": "agent_optimization" + } + ) + except Exception as e: + logger.error(f"Failed to create strategies collection: {e}") + self.strategies_collection = None + + async def store_agent_experience(self, + agent_id: str, + scenario_id: str, + action: AgentAction, + outcome: Dict[str, Any], + context: Dict[str, Any], + importance: float = 0.5) -> str: + """ + Store agent experience in memory system + + Args: + agent_id: Agent identifier + scenario_id: Training scenario ID + action: Action taken by agent + outcome: Result of the action + context: Context when action was taken + importance: Importance score (0-1) + + Returns: + Memory ID + """ + # Create memory entry + memory_id = self._generate_memory_id(agent_id, scenario_id, action.timestamp) + + # Determine memory type based on outcome + memory_type = self._classify_experience_type(action, outcome) + + # Create description + description = self._create_experience_description(action, outcome, context) + + # Extract tags + tags = self._extract_experience_tags(action, outcome, context) + + memory = AgentMemory( + memory_id=memory_id, + agent_id=agent_id, + memory_type=memory_type, + timestamp=action.timestamp, + description=description, + context=context, + outcome=outcome, + importance_score=importance, + confidence=action.confidence or 0.5, + tags=tags, + related_scenarios=[scenario_id] + ) + + # Store in ChromaDB + await self._store_memory_in_chromadb(memory) + + # Update cache + self.memory_cache[memory_id] = memory + + # Update statistics + self._update_memory_stats(memory) + + logger.debug(f"Stored experience memory: {memory_id}") + return memory_id + + async def store_discovered_pattern(self, + agent_id: str, + pattern_description: str, + pattern_context: Dict[str, Any], + success_rate: float, + confidence: float = 0.7) -> str: + """ + Store discovered pattern in memory system + + Args: + agent_id: Agent that discovered pattern + pattern_description: Description of the pattern + pattern_context: Context where pattern applies + success_rate: Historical success rate of pattern + confidence: Confidence in pattern validity + + Returns: + Memory ID + """ + memory_id = self._generate_memory_id(agent_id, "pattern", datetime.now()) + + memory = AgentMemory( + memory_id=memory_id, + agent_id=agent_id, + memory_type=MemoryType.PATTERN, + timestamp=datetime.now(), + description=pattern_description, + context=pattern_context, + outcome={"success_rate": success_rate}, + importance_score=min(1.0, success_rate + 0.2), # Higher success = higher importance + confidence=confidence, + success_rate=success_rate, + tags=self._extract_pattern_tags(pattern_description, pattern_context) + ) + + # Store in ChromaDB patterns collection + await self._store_pattern_in_chromadb(memory) + + # Update cache and stats + self.memory_cache[memory_id] = memory + self._update_memory_stats(memory) + + logger.info(f"Stored pattern memory: {pattern_description[:50]}...") + return memory_id + + async def store_strategy_knowledge(self, + agent_id: str, + strategy_name: str, + strategy_details: Dict[str, Any], + performance_metrics: Dict[str, float], + applicable_contexts: List[str]) -> str: + """ + Store successful strategy in memory system + + Args: + agent_id: Agent that developed strategy + strategy_name: Name/description of strategy + strategy_details: Detailed strategy parameters + performance_metrics: Performance metrics + applicable_contexts: Contexts where strategy works + + Returns: + Memory ID + """ + memory_id = self._generate_memory_id(agent_id, "strategy", datetime.now()) + + # Calculate importance based on performance + profit_factor = performance_metrics.get("profit_factor", 1.0) + win_rate = performance_metrics.get("win_rate", 0.5) + importance = min(1.0, (profit_factor * win_rate) / 2.0) + + memory = AgentMemory( + memory_id=memory_id, + agent_id=agent_id, + memory_type=MemoryType.STRATEGY, + timestamp=datetime.now(), + description=f"Strategy: {strategy_name}", + context={ + "strategy_details": strategy_details, + "applicable_contexts": applicable_contexts, + "performance_metrics": performance_metrics + }, + outcome=performance_metrics, + importance_score=importance, + confidence=min(1.0, win_rate + 0.3), + success_rate=win_rate, + tags=["strategy"] + applicable_contexts + ) + + # Store in ChromaDB strategies collection + await self._store_strategy_in_chromadb(memory) + + # Update cache and stats + self.memory_cache[memory_id] = memory + self._update_memory_stats(memory) + + logger.info(f"Stored strategy: {strategy_name}") + return memory_id + + async def retrieve_memories(self, query: MemoryQuery) -> List[MemorySearchResult]: + """ + Retrieve relevant memories based on query + + Args: + query: Memory query parameters + + Returns: + List of relevant memories with scores + """ + results = [] + + # Search in ChromaDB collections + if query.memory_types: + for memory_type in query.memory_types: + type_results = await self._search_by_memory_type(query, memory_type) + results.extend(type_results) + else: + # Search all types + for memory_type in MemoryType: + type_results = await self._search_by_memory_type(query, memory_type) + results.extend(type_results) + + # Filter by importance + if query.min_importance > 0: + results = [r for r in results if r.memory.importance_score >= query.min_importance] + + # Sort by relevance score + results.sort(key=lambda x: x.relevance_score, reverse=True) + + # Limit results + results = results[:query.max_results] + + # Update access patterns + for result in results: + await self._record_memory_access(result.memory) + + logger.debug(f"Retrieved {len(results)} memories for query: {query.query_text[:50]}...") + return results + + async def _search_by_memory_type(self, query: MemoryQuery, memory_type: MemoryType) -> List[MemorySearchResult]: + """Search specific memory type collection""" + + results = [] + collection = None + + # Select appropriate collection + if memory_type == MemoryType.EXPERIENCE: + collection = self.experiences_collection + elif memory_type == MemoryType.PATTERN: + collection = self.patterns_collection + elif memory_type == MemoryType.STRATEGY: + collection = self.strategies_collection + else: + # Use general experiences collection for other types + collection = self.experiences_collection + + if not collection: + return results + + try: + # Search ChromaDB + search_results = collection.query( + query_texts=[query.query_text], + n_results=min(query.max_results * 2, 50), # Get more to filter + include=['documents', 'metadatas', 'distances'] + ) + + # Convert to MemorySearchResult objects + if search_results['ids'] and search_results['ids'][0]: + for i, memory_id in enumerate(search_results['ids'][0]): + try: + # Check if memory is in cache + if memory_id in self.memory_cache: + memory = self.memory_cache[memory_id] + else: + # Reconstruct memory from ChromaDB metadata + memory = self._reconstruct_memory_from_metadata( + memory_id, + search_results['metadatas'][0][i] + ) + self.memory_cache[memory_id] = memory + + # Filter by agent if specified + if query.agent_id and memory.agent_id != query.agent_id: + continue + + # Filter by tags if specified + if query.tags and not any(tag in memory.tags for tag in query.tags): + continue + + # Calculate scores + similarity_score = 1.0 - search_results['distances'][0][i] + relevance_score = self._calculate_relevance_score(memory, query) + context_match_score = self._calculate_context_match_score(memory, query) + + result = MemorySearchResult( + memory=memory, + relevance_score=relevance_score, + similarity_score=similarity_score, + context_match_score=context_match_score + ) + + results.append(result) + + except Exception as e: + logger.warning(f"Error processing search result {i}: {e}") + continue + + except Exception as e: + logger.error(f"Error searching {memory_type.value} collection: {e}") + + return results + + async def get_agent_learning_summary(self, agent_id: str) -> Dict[str, Any]: + """ + Get learning summary for specific agent + + Args: + agent_id: Agent identifier + + Returns: + Learning summary with key insights + """ + # Query all memories for agent + query = MemoryQuery( + query_text="", # Empty query to get all + agent_id=agent_id, + max_results=1000 + ) + + memories = await self.retrieve_memories(query) + + if not memories: + return {"agent_id": agent_id, "total_memories": 0} + + # Analyze memories + summary = { + "agent_id": agent_id, + "total_memories": len(memories), + "memory_types": {}, + "success_patterns": [], + "improvement_areas": [], + "key_strategies": [], + "learning_progress": {} + } + + # Count by memory type + for result in memories: + mem_type = result.memory.memory_type.value + summary["memory_types"][mem_type] = summary["memory_types"].get(mem_type, 0) + 1 + + # Extract successful patterns (high success rate + high importance) + successful_memories = [ + r.memory for r in memories + if r.memory.success_rate >= 0.7 and r.memory.importance_score >= 0.6 + ] + + summary["success_patterns"] = [ + { + "description": mem.description, + "success_rate": mem.success_rate, + "importance": mem.importance_score, + "usage_count": mem.usage_count + } + for mem in successful_memories[:5] # Top 5 + ] + + # Identify improvement areas (failures with high importance) + failure_memories = [ + r.memory for r in memories + if r.memory.memory_type == MemoryType.MISTAKE and r.memory.importance_score >= 0.5 + ] + + summary["improvement_areas"] = [ + { + "description": mem.description, + "frequency": mem.usage_count, + "impact": mem.importance_score + } + for mem in failure_memories[:3] # Top 3 areas + ] + + # Extract key strategies + strategy_memories = [ + r.memory for r in memories + if r.memory.memory_type == MemoryType.STRATEGY + ] + + summary["key_strategies"] = [ + { + "name": mem.description, + "success_rate": mem.success_rate, + "confidence": mem.confidence + } + for mem in sorted(strategy_memories, key=lambda x: x.success_rate, reverse=True)[:3] + ] + + return summary + + def _generate_memory_id(self, agent_id: str, identifier: str, timestamp: datetime) -> str: + """Generate unique memory ID""" + content = f"{agent_id}_{identifier}_{timestamp.isoformat()}" + return hashlib.md5(content.encode()).hexdigest() + + def _classify_experience_type(self, action: AgentAction, outcome: Dict[str, Any]) -> MemoryType: + """Classify experience type based on action and outcome""" + + if outcome.get("success", True): + if outcome.get("profit", 0) > 0: + return MemoryType.EXPERIENCE + else: + return MemoryType.CONTEXT + else: + return MemoryType.MISTAKE + + def _create_experience_description(self, action: AgentAction, outcome: Dict[str, Any], context: Dict[str, Any]) -> str: + """Create human-readable description of experience""" + + action_desc = f"{action.action_type.upper()}" + if action.market_ticker: + action_desc += f" on {action.market_ticker}" + if action.side and action.size: + action_desc += f" - {action.side} {action.size}" + + outcome_desc = "SUCCESS" if outcome.get("success", True) else "FAILED" + if "profit" in outcome: + outcome_desc += f" (P&L: {outcome['profit']:+.2f})" + + return f"{action_desc} - {outcome_desc}" + + def _extract_experience_tags(self, action: AgentAction, outcome: Dict[str, Any], context: Dict[str, Any]) -> List[str]: + """Extract relevant tags from experience""" + tags = [] + + # Action type tags + tags.append(action.action_type) + + # Outcome tags + if outcome.get("success", True): + tags.append("success") + if outcome.get("profit", 0) > 0: + tags.append("profitable") + else: + tags.append("failure") + + # Context tags + if context.get("market_condition"): + tags.append(f"market_{context['market_condition']}") + + if context.get("game_situation"): + tags.append(f"game_{context['game_situation']}") + + # Confidence tags + if action.confidence: + if action.confidence >= 0.8: + tags.append("high_confidence") + elif action.confidence <= 0.3: + tags.append("low_confidence") + + return tags + + def _extract_pattern_tags(self, description: str, context: Dict[str, Any]) -> List[str]: + """Extract tags from pattern description and context""" + tags = ["pattern"] + + # Extract key terms from description + keywords = ["momentum", "reversal", "breakout", "support", "resistance", "volume", "sentiment"] + for keyword in keywords: + if keyword in description.lower(): + tags.append(keyword) + + # Context-based tags + if context.get("market_type"): + tags.append(f"market_{context['market_type']}") + + if context.get("game_phase"): + tags.append(f"phase_{context['game_phase']}") + + return tags + + async def _store_memory_in_chromadb(self, memory: AgentMemory): + """Store memory in appropriate ChromaDB collection""" + + if memory.memory_type == MemoryType.EXPERIENCE and self.experiences_collection: + collection = self.experiences_collection + elif memory.memory_type == MemoryType.PATTERN and self.patterns_collection: + collection = self.patterns_collection + elif memory.memory_type == MemoryType.STRATEGY and self.strategies_collection: + collection = self.strategies_collection + else: + # Default to experiences collection + collection = self.experiences_collection + + if not collection: + logger.warning(f"No collection available for memory type: {memory.memory_type}") + return + + try: + collection.add( + ids=[memory.memory_id], + documents=[memory.description], + metadatas=[{ + "agent_id": memory.agent_id, + "memory_type": memory.memory_type.value, + "timestamp": memory.timestamp.isoformat(), + "importance_score": memory.importance_score, + "confidence": memory.confidence, + "success_rate": memory.success_rate, + "tags": json.dumps(memory.tags), + "context": json.dumps(memory.context, default=str), + "outcome": json.dumps(memory.outcome, default=str) + }] + ) + except Exception as e: + logger.error(f"Error storing memory in ChromaDB: {e}") + + async def _store_pattern_in_chromadb(self, memory: AgentMemory): + """Store pattern memory in ChromaDB patterns collection""" + await self._store_memory_in_chromadb(memory) + + async def _store_strategy_in_chromadb(self, memory: AgentMemory): + """Store strategy memory in ChromaDB strategies collection""" + await self._store_memory_in_chromadb(memory) + + def _reconstruct_memory_from_metadata(self, memory_id: str, metadata: Dict[str, Any]) -> AgentMemory: + """Reconstruct AgentMemory object from ChromaDB metadata""" + + return AgentMemory( + memory_id=memory_id, + agent_id=metadata.get("agent_id", "unknown"), + memory_type=MemoryType(metadata.get("memory_type", "experience")), + timestamp=datetime.fromisoformat(metadata.get("timestamp", datetime.now().isoformat())), + description=metadata.get("description", ""), + context=json.loads(metadata.get("context", "{}")), + outcome=json.loads(metadata.get("outcome", "{}")), + importance_score=metadata.get("importance_score", 0.5), + confidence=metadata.get("confidence", 0.5), + success_rate=metadata.get("success_rate", 0.5), + tags=json.loads(metadata.get("tags", "[]")) + ) + + def _calculate_relevance_score(self, memory: AgentMemory, query: MemoryQuery) -> float: + """Calculate relevance score for memory given query""" + + score = 0.0 + + # Base score from importance and confidence + score += memory.importance_score * 0.3 + score += memory.confidence * 0.2 + + # Success rate bonus + score += memory.success_rate * 0.2 + + # Tag matching bonus + if query.tags: + matching_tags = len(set(query.tags) & set(memory.tags)) + score += (matching_tags / len(query.tags)) * 0.3 + + return min(1.0, score) + + def _calculate_context_match_score(self, memory: AgentMemory, query: MemoryQuery) -> float: + """Calculate context matching score""" + + # Simple context matching based on query text + context_str = json.dumps(memory.context, default=str).lower() + query_terms = query.query_text.lower().split() + + matches = sum(1 for term in query_terms if term in context_str) + + if query_terms: + return matches / len(query_terms) + else: + return 0.5 # Neutral score for empty query + + async def _record_memory_access(self, memory: AgentMemory): + """Record that memory was accessed""" + + memory.usage_count += 1 + memory.last_accessed = datetime.now() + + # Update access patterns + if memory.memory_id not in self.access_patterns: + self.access_patterns[memory.memory_id] = [] + + self.access_patterns[memory.memory_id].append(datetime.now()) + + # Keep only recent access patterns (last 100) + if len(self.access_patterns[memory.memory_id]) > 100: + self.access_patterns[memory.memory_id] = self.access_patterns[memory.memory_id][-100:] + + def _update_memory_stats(self, memory: AgentMemory): + """Update memory statistics""" + + self.memory_stats["total_memories"] += 1 + + mem_type = memory.memory_type.value + self.memory_stats["memories_by_type"][mem_type] = self.memory_stats["memories_by_type"].get(mem_type, 0) + 1 + + self.memory_stats["memories_by_agent"][memory.agent_id] = self.memory_stats["memories_by_agent"].get(memory.agent_id, 0) + 1 + + def get_memory_statistics(self) -> Dict[str, Any]: + """Get comprehensive memory system statistics""" + return { + **self.memory_stats, + "cache_size": len(self.memory_cache), + "access_patterns_tracked": len(self.access_patterns) + } + + +# Example usage and testing +if __name__ == "__main__": + import sys + import asyncio + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.storage.chromadb_manager import ChromaDBManager + + async def test_memory_system(): + # Initialize memory system + chromadb = ChromaDBManager() + memory_system = AgentMemorySystem(chromadb) + + # Test storing experience + test_action = AgentAction( + agent_id="test_agent", + timestamp=datetime.now(), + action_type="trade", + market_ticker="NFL-KC-BUF-20240115", + side="yes", + size=100.0, + confidence=0.8 + ) + + test_outcome = { + "success": True, + "profit": 25.0, + "execution_time_ms": 150 + } + + test_context = { + "market_condition": "volatile", + "game_situation": "fourth_quarter", + "score_differential": 3 + } + + # Store experience + memory_id = await memory_system.store_agent_experience( + agent_id="test_agent", + scenario_id="test_scenario_001", + action=test_action, + outcome=test_outcome, + context=test_context, + importance=0.8 + ) + + print(f"Stored experience memory: {memory_id}") + + # Store pattern discovery + pattern_id = await memory_system.store_discovered_pattern( + agent_id="test_agent", + pattern_description="Fourth quarter momentum reversal pattern", + pattern_context={"game_phase": "fourth_quarter", "score_situation": "close"}, + success_rate=0.75, + confidence=0.8 + ) + + print(f"Stored pattern memory: {pattern_id}") + + # Test memory retrieval + query = MemoryQuery( + query_text="fourth quarter trading volatile market", + agent_id="test_agent", + max_results=10 + ) + + results = await memory_system.retrieve_memories(query) + print(f"Retrieved {len(results)} memories") + + for result in results: + print(f" Memory: {result.memory.description}") + print(f" Relevance: {result.relevance_score:.3f}") + print(f" Similarity: {result.similarity_score:.3f}") + + # Get learning summary + summary = await memory_system.get_agent_learning_summary("test_agent") + print(f"\nLearning Summary:") + print(f" Total memories: {summary['total_memories']}") + print(f" Memory types: {summary['memory_types']}") + + # Get statistics + stats = memory_system.get_memory_statistics() + print(f"\nMemory System Stats: {stats}") + + # Run test + logging.basicConfig(level=logging.INFO) + asyncio.run(test_memory_system()) \ No newline at end of file diff --git a/src/training/synthetic_env.py b/src/training/synthetic_env.py new file mode 100644 index 00000000..f5f15078 --- /dev/null +++ b/src/training/synthetic_env.py @@ -0,0 +1,672 @@ +""" +Synthetic Training Environment + +Provides isolated training environment for agents using synthetic data +with accelerated time replay and performance tracking. +""" + +import logging +import asyncio +import random +from typing import List, Dict, Any, Optional, Tuple, Iterator, Callable +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +import json +import time + +from ..synthetic_data.generators import ( + TradingScenario, MarketEvent, SyntheticGame, + ScenarioBuilder, TrainingScenarioSet +) +from src.sdk.core.base_adapter import StandardizedEvent + +logger = logging.getLogger(__name__) + + +class EnvironmentState(Enum): + """Training environment states""" + IDLE = "idle" + RUNNING = "running" + PAUSED = "paused" + COMPLETED = "completed" + ERROR = "error" + + +class InformationLevel(Enum): + """Levels of information available to agents""" + FULL = "full" # All information immediately + PUBLIC_ONLY = "public" # Only public information + DELAYED = "delayed" # Information with delays + PRIVATE = "private" # Access to private information + ASYMMETRIC = "asymmetric" # Different agents get different info + + +@dataclass +class AgentPerformanceMetrics: + """Performance metrics for agent in training""" + agent_id: str + scenario_id: str + + # Trading metrics + total_trades: int = 0 + winning_trades: int = 0 + losing_trades: int = 0 + total_pnl: float = 0.0 + max_drawdown: float = 0.0 + win_rate: float = 0.0 + profit_factor: float = 0.0 + + # Decision metrics + decision_latency_ms: List[float] = field(default_factory=list) + kelly_adherence: float = 0.0 + position_sizing_accuracy: float = 0.0 + + # Learning metrics + pattern_recognition_score: float = 0.0 + risk_management_score: float = 0.0 + information_utilization: float = 0.0 + + # Behavioral metrics + overconfidence_incidents: int = 0 + panic_trades: int = 0 + fomo_trades: int = 0 + + started_at: datetime = field(default_factory=datetime.now) + completed_at: Optional[datetime] = None + + +@dataclass +class AgentAction: + """Action taken by agent during training""" + agent_id: str + timestamp: datetime + action_type: str # 'trade', 'update_position', 'cancel', 'analyze' + + # Trade details + market_ticker: Optional[str] = None + side: Optional[str] = None # 'yes', 'no' + size: Optional[float] = None + price: Optional[float] = None + + # Decision context + confidence: Optional[float] = None + reasoning: Optional[str] = None + available_information: List[str] = field(default_factory=list) + + # Execution details + execution_time_ms: Optional[float] = None + success: bool = True + error_message: Optional[str] = None + + +@dataclass +class EnvironmentConfig: + """Configuration for training environment""" + + # Time simulation + time_acceleration: float = 10.0 # 10x real-time + enable_pause: bool = True + + # Information flow + information_level: InformationLevel = InformationLevel.PUBLIC_ONLY + information_delay_ms: float = 1000 # 1 second delay + private_information_probability: float = 0.1 + + # Market simulation + enable_slippage: bool = True + slippage_factor: float = 0.02 + min_trade_size: float = 10.0 + max_trade_size: float = 1000.0 + + # Logging and monitoring + log_all_actions: bool = True + track_performance: bool = True + enable_real_time_analytics: bool = True + + +class SyntheticTrainingEnvironment: + """ + Isolated training environment for agent development using synthetic data + """ + + def __init__(self, config: EnvironmentConfig = None): + """ + Initialize training environment + + Args: + config: Environment configuration + """ + self.config = config or EnvironmentConfig() + + # Environment state + self.state = EnvironmentState.IDLE + self.current_scenario: Optional[TradingScenario] = None + self.scenario_start_time: Optional[datetime] = None + self.current_time: Optional[datetime] = None + + # Agent management + self.registered_agents: Dict[str, Dict[str, Any]] = {} + self.agent_callbacks: Dict[str, Callable] = {} + self.agent_metrics: Dict[str, AgentPerformanceMetrics] = {} + + # Event management + self.pending_events: List[Tuple[datetime, MarketEvent]] = [] + self.processed_events: List[Tuple[datetime, MarketEvent]] = [] + self.information_buffer: Dict[str, List[MarketEvent]] = {} # Agent-specific buffers + + # Performance tracking + self.environment_metrics = { + "scenarios_completed": 0, + "total_training_time": 0.0, + "events_processed": 0, + "agents_trained": set() + } + + logger.info("Initialized SyntheticTrainingEnvironment") + + def register_agent(self, + agent_id: str, + agent_callback: Callable, + information_level: InformationLevel = None, + agent_config: Dict[str, Any] = None) -> bool: + """ + Register agent for training + + Args: + agent_id: Unique agent identifier + agent_callback: Function to call with market events + information_level: Level of information access for this agent + agent_config: Additional agent configuration + + Returns: + True if registration successful + """ + if agent_id in self.registered_agents: + logger.warning(f"Agent {agent_id} already registered") + return False + + self.registered_agents[agent_id] = { + "callback": agent_callback, + "information_level": information_level or self.config.information_level, + "config": agent_config or {}, + "registered_at": datetime.now() + } + + self.agent_callbacks[agent_id] = agent_callback + self.information_buffer[agent_id] = [] + + logger.info(f"Registered agent: {agent_id}") + return True + + def unregister_agent(self, agent_id: str) -> bool: + """Unregister agent from training""" + if agent_id not in self.registered_agents: + logger.warning(f"Agent {agent_id} not registered") + return False + + del self.registered_agents[agent_id] + del self.agent_callbacks[agent_id] + if agent_id in self.information_buffer: + del self.information_buffer[agent_id] + + logger.info(f"Unregistered agent: {agent_id}") + return True + + async def run_scenario(self, scenario: TradingScenario) -> Dict[str, AgentPerformanceMetrics]: + """ + Run single training scenario + + Args: + scenario: Trading scenario to execute + + Returns: + Performance metrics for all agents + """ + if self.state != EnvironmentState.IDLE: + raise RuntimeError(f"Environment not idle, current state: {self.state}") + + if not self.registered_agents: + raise RuntimeError("No agents registered for training") + + logger.info(f"Starting scenario: {scenario.scenario_id}") + + # Initialize scenario + self.state = EnvironmentState.RUNNING + self.current_scenario = scenario + self.scenario_start_time = datetime.now() + self.current_time = scenario.events[0].timestamp if scenario.events else datetime.now() + + # Initialize agent metrics + for agent_id in self.registered_agents: + self.agent_metrics[agent_id] = AgentPerformanceMetrics( + agent_id=agent_id, + scenario_id=scenario.scenario_id + ) + + # Prepare event timeline + await self._prepare_event_timeline(scenario) + + try: + # Run scenario simulation + await self._execute_scenario() + + # Finalize metrics + self._finalize_agent_metrics() + + self.state = EnvironmentState.COMPLETED + + except Exception as e: + logger.error(f"Error running scenario: {e}") + self.state = EnvironmentState.ERROR + raise + + finally: + # Clean up + self._cleanup_scenario() + + logger.info(f"Completed scenario: {scenario.scenario_id}") + return dict(self.agent_metrics) + + async def run_training_set(self, + training_set: TrainingScenarioSet, + progress_callback: Optional[Callable] = None) -> Dict[str, List[AgentPerformanceMetrics]]: + """ + Run complete training set + + Args: + training_set: Set of training scenarios + progress_callback: Optional progress callback function + + Returns: + Performance metrics for all scenarios and agents + """ + logger.info(f"Starting training set: {training_set.name} ({len(training_set.scenarios)} scenarios)") + + all_metrics = {agent_id: [] for agent_id in self.registered_agents} + completed_scenarios = 0 + + for scenario in training_set.scenarios: + try: + # Run scenario + scenario_metrics = await self.run_scenario(scenario) + + # Collect metrics + for agent_id, metrics in scenario_metrics.items(): + all_metrics[agent_id].append(metrics) + + completed_scenarios += 1 + + # Progress callback + if progress_callback: + progress_callback(completed_scenarios, len(training_set.scenarios), scenario.scenario_id) + + # Brief pause between scenarios + await asyncio.sleep(0.1) + + except Exception as e: + logger.error(f"Failed to run scenario {scenario.scenario_id}: {e}") + continue + + # Update environment metrics + self.environment_metrics["scenarios_completed"] += completed_scenarios + self.environment_metrics["agents_trained"].update(self.registered_agents.keys()) + + logger.info(f"Completed training set: {completed_scenarios}/{len(training_set.scenarios)} scenarios") + return all_metrics + + async def _prepare_event_timeline(self, scenario: TradingScenario): + """Prepare chronological event timeline for scenario""" + + # Clear previous events + self.pending_events.clear() + self.processed_events.clear() + + # Classify events by information access + classified_events = self._classify_events_by_information_access(scenario) + + # Build timeline + for event in scenario.events: + self.pending_events.append((event.timestamp, event)) + + # Sort by timestamp + self.pending_events.sort(key=lambda x: x[0]) + + logger.debug(f"Prepared timeline with {len(self.pending_events)} events") + + def _classify_events_by_information_access(self, scenario: TradingScenario) -> Dict[str, List[MarketEvent]]: + """Classify events by information access level""" + + classified = { + "public": scenario.public_events, + "private": scenario.private_events, + "delayed": scenario.delayed_events + } + + # If no explicit classification, classify all as public + if not classified["public"] and not classified["private"] and not classified["delayed"]: + classified["public"] = scenario.events + + return classified + + async def _execute_scenario(self): + """Execute scenario with time simulation""" + + start_time = time.time() + + while self.pending_events and self.state == EnvironmentState.RUNNING: + # Get next event + event_time, event = self.pending_events.pop(0) + + # Simulate time advancement + await self._advance_time_to(event_time) + + # Process event + await self._process_event(event) + + # Check for pause + while self.state == EnvironmentState.PAUSED: + await asyncio.sleep(0.1) + + # Brief processing delay + await asyncio.sleep(0.01) + + # Update environment metrics + self.environment_metrics["total_training_time"] += time.time() - start_time + self.environment_metrics["events_processed"] += len(self.processed_events) + + async def _advance_time_to(self, target_time: datetime): + """Advance simulation time to target with acceleration""" + + if not self.current_time: + self.current_time = target_time + return + + time_diff = (target_time - self.current_time).total_seconds() + if time_diff <= 0: + return + + # Apply time acceleration + sleep_time = time_diff / self.config.time_acceleration + await asyncio.sleep(sleep_time) + + self.current_time = target_time + + async def _process_event(self, event: MarketEvent): + """Process market event and distribute to agents""" + + # Determine which agents should receive this event + receiving_agents = self._determine_event_recipients(event) + + # Distribute to agents + tasks = [] + for agent_id in receiving_agents: + task = self._send_event_to_agent(agent_id, event) + tasks.append(task) + + # Execute all agent notifications concurrently + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + # Record processed event + self.processed_events.append((datetime.now(), event)) + + def _determine_event_recipients(self, event: MarketEvent) -> List[str]: + """Determine which agents should receive event based on information levels""" + + recipients = [] + + for agent_id, agent_info in self.registered_agents.items(): + info_level = agent_info["information_level"] + + should_receive = False + + if info_level == InformationLevel.FULL: + should_receive = True + elif info_level == InformationLevel.PUBLIC_ONLY: + should_receive = event in self.current_scenario.public_events + elif info_level == InformationLevel.PRIVATE: + should_receive = (event in self.current_scenario.public_events or + event in self.current_scenario.private_events) + elif info_level == InformationLevel.DELAYED: + # Add delay for delayed events + if event in self.current_scenario.delayed_events: + # Buffer the event for later delivery + asyncio.create_task(self._deliver_delayed_event(agent_id, event)) + continue + else: + should_receive = True + elif info_level == InformationLevel.ASYMMETRIC: + # Random information access + should_receive = random.random() > 0.3 + + if should_receive: + recipients.append(agent_id) + + return recipients + + async def _deliver_delayed_event(self, agent_id: str, event: MarketEvent): + """Deliver event with delay""" + delay_seconds = self.config.information_delay_ms / 1000.0 + await asyncio.sleep(delay_seconds) + + if self.state == EnvironmentState.RUNNING: + await self._send_event_to_agent(agent_id, event) + + async def _send_event_to_agent(self, agent_id: str, event: MarketEvent): + """Send event to specific agent""" + + try: + callback = self.agent_callbacks[agent_id] + + # Convert to StandardizedEvent format + standardized_event = self._convert_to_standardized_event(event) + + # Record start time for latency measurement + start_time = time.time() + + # Call agent + await callback(standardized_event) + + # Record execution time + execution_time_ms = (time.time() - start_time) * 1000 + + # Update agent metrics + if agent_id in self.agent_metrics: + self.agent_metrics[agent_id].decision_latency_ms.append(execution_time_ms) + + except Exception as e: + logger.error(f"Error sending event to agent {agent_id}: {e}") + + def _convert_to_standardized_event(self, event: MarketEvent) -> StandardizedEvent: + """Convert MarketEvent to StandardizedEvent""" + from src.sdk.core.base_adapter import EventType + + return StandardizedEvent( + source="synthetic_training_env", + event_type=EventType.MARKET_EVENT, + timestamp=event.timestamp, + game_id=self.current_scenario.game.game_id if self.current_scenario else "", + data={ + "market_ticker": event.market_ticker, + "event_type": event.event_type.value, + "description": event.description, + "price_impact": event.price_impact, + "volume_impact": event.volume_impact, + "confidence_impact": event.confidence_impact, + "synthetic": True + }, + confidence=0.99, + impact="high" if abs(event.price_impact) > 0.1 else "medium" if abs(event.price_impact) > 0.05 else "low", + metadata={ + "training_environment": True, + "scenario_id": self.current_scenario.scenario_id if self.current_scenario else "", + "current_time": self.current_time.isoformat() if self.current_time else "" + }, + raw_data=event + ) + + async def record_agent_action(self, agent_action: AgentAction): + """Record action taken by agent during training""" + + # Update performance metrics + if agent_action.agent_id in self.agent_metrics: + metrics = self.agent_metrics[agent_action.agent_id] + + if agent_action.action_type == "trade": + metrics.total_trades += 1 + + # Record execution latency + if agent_action.execution_time_ms: + metrics.decision_latency_ms.append(agent_action.execution_time_ms) + + # Log action if configured + if self.config.log_all_actions: + logger.debug(f"Agent {agent_action.agent_id} action: {agent_action.action_type}") + + def pause_environment(self): + """Pause training environment""" + if self.state == EnvironmentState.RUNNING: + self.state = EnvironmentState.PAUSED + logger.info("Training environment paused") + + def resume_environment(self): + """Resume training environment""" + if self.state == EnvironmentState.PAUSED: + self.state = EnvironmentState.RUNNING + logger.info("Training environment resumed") + + def stop_environment(self): + """Stop training environment""" + self.state = EnvironmentState.IDLE + logger.info("Training environment stopped") + + def _finalize_agent_metrics(self): + """Calculate final performance metrics for all agents""" + + for agent_id, metrics in self.agent_metrics.items(): + metrics.completed_at = datetime.now() + + # Calculate derived metrics + if metrics.total_trades > 0: + metrics.win_rate = metrics.winning_trades / metrics.total_trades + + if metrics.losing_trades > 0: + avg_win = metrics.total_pnl / max(1, metrics.winning_trades) + avg_loss = abs(metrics.total_pnl) / metrics.losing_trades + metrics.profit_factor = avg_win / avg_loss if avg_loss > 0 else 0 + + # Calculate average decision latency + if metrics.decision_latency_ms: + avg_latency = sum(metrics.decision_latency_ms) / len(metrics.decision_latency_ms) + logger.debug(f"Agent {agent_id} avg latency: {avg_latency:.1f}ms") + + def _cleanup_scenario(self): + """Clean up after scenario completion""" + self.current_scenario = None + self.scenario_start_time = None + self.current_time = None + self.pending_events.clear() + + # Clear information buffers + for buffer in self.information_buffer.values(): + buffer.clear() + + def get_environment_status(self) -> Dict[str, Any]: + """Get current environment status""" + return { + "state": self.state.value, + "registered_agents": len(self.registered_agents), + "current_scenario": self.current_scenario.scenario_id if self.current_scenario else None, + "pending_events": len(self.pending_events), + "processed_events": len(self.processed_events), + "metrics": self.environment_metrics + } + + def get_agent_performance_summary(self, agent_id: str = None) -> Dict[str, Any]: + """Get performance summary for agent(s)""" + + if agent_id: + if agent_id not in self.agent_metrics: + return {} + + metrics = self.agent_metrics[agent_id] + return { + "agent_id": agent_id, + "total_trades": metrics.total_trades, + "win_rate": metrics.win_rate, + "total_pnl": metrics.total_pnl, + "max_drawdown": metrics.max_drawdown, + "avg_latency_ms": sum(metrics.decision_latency_ms) / len(metrics.decision_latency_ms) if metrics.decision_latency_ms else 0 + } + else: + # Return summary for all agents + summaries = {} + for aid, metrics in self.agent_metrics.items(): + summaries[aid] = { + "total_trades": metrics.total_trades, + "win_rate": metrics.win_rate, + "total_pnl": metrics.total_pnl, + "avg_latency_ms": sum(metrics.decision_latency_ms) / len(metrics.decision_latency_ms) if metrics.decision_latency_ms else 0 + } + return summaries + + +# Example usage and testing +if __name__ == "__main__": + import sys + import asyncio + sys.path.append('/Users/hudson/Documents/GitHub/IntelIP/PROJECTS/Neural/Kalshi_Agentic_Agent') + + from src.synthetic_data.generators import SyntheticGameEngine, MarketSimulator, ScenarioBuilder + from src.synthetic_data.storage.chromadb_manager import ChromaDBManager + + async def mock_agent_callback(event: StandardizedEvent): + """Mock agent callback for testing""" + logger.info(f"Agent received event: {event.data.get('description', 'No description')}") + await asyncio.sleep(0.1) # Simulate processing time + + async def test_training_environment(): + # Initialize components + chromadb = ChromaDBManager() + game_engine = SyntheticGameEngine(chromadb) + market_simulator = MarketSimulator() + scenario_builder = ScenarioBuilder(game_engine, market_simulator, chromadb) + + # Create training environment + config = EnvironmentConfig( + time_acceleration=100.0, # Very fast for testing + information_level=InformationLevel.PUBLIC_ONLY + ) + env = SyntheticTrainingEnvironment(config) + + # Register test agents + env.register_agent("test_agent_1", mock_agent_callback, InformationLevel.FULL) + env.register_agent("test_agent_2", mock_agent_callback, InformationLevel.DELAYED) + + # Generate test scenario + game = await game_engine.generate_single_game( + home_team="KC", + away_team="BUF" + ) + scenario = market_simulator.create_trading_scenario(game) + + print(f"Testing with scenario: {scenario.scenario_id}") + print(f"Events: {len(scenario.events)}") + + # Run scenario + metrics = await env.run_scenario(scenario) + + print(f"\nPerformance Results:") + for agent_id, agent_metrics in metrics.items(): + print(f" {agent_id}:") + print(f" Decision latency: {sum(agent_metrics.decision_latency_ms)/len(agent_metrics.decision_latency_ms) if agent_metrics.decision_latency_ms else 0:.1f}ms") + print(f" Events received: {len(agent_metrics.decision_latency_ms)}") + + # Environment status + status = env.get_environment_status() + print(f"\nEnvironment Status: {status}") + + # Run test + logging.basicConfig(level=logging.INFO) + asyncio.run(test_training_environment()) \ No newline at end of file From 6184d12e5fc79006720479848aac09f2504fd519 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Wed, 3 Sep 2025 23:54:14 -0400 Subject: [PATCH 2/2] feat: Convert Kalshi Agentic Agent to Neural SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit transforms the proprietary agent-based architecture into a clean, open-source SDK for algorithmic prediction market trading. Major Changes: - Renamed entire platform from "Kalshi Trading Agent" to "Neural SDK" - Replaced agent architecture with strategy-based framework - Created comprehensive backtesting module with multiple data providers - Added complete documentation and examples - Updated environment variables: KALSHI_* → NEURAL_* - Removed proprietary code and trading-specific implementations - Added placeholder methods with clear TODO comments for extension SDK Features: - Event-driven backtesting engine with realistic portfolio simulation - Multiple data provider support (CSV, Parquet, S3, Database) - Strategy framework with decorators (@sdk.strategy) - Risk management with Kelly Criterion position sizing - Real-time data streaming architecture (via src/data_pipeline) - Comprehensive performance metrics and analytics - Docker support for containerized deployment File Structure: - neural_sdk/ - Core SDK implementation - examples/ - Usage examples and demonstrations - docs/ - Complete documentation suite - src/data_pipeline - Real-time data infrastructure (preserved) - tests/ - Test suite - config/ - Configuration files Installation: pip install git+https://github.com/IntelIP/kalshi.git@feat/synthetic-training-integration 🤖 Generated with Claude Code Co-Authored-By: Claude --- .env.example | 32 +- .github/workflows/release.yml | 4 +- .gitignore | 6 +- CLAUDE.md | 494 ---- Dockerfile | 25 + INVESTOR_DEMO.md | 496 ---- LICENSE | 21 + PLATFORM_ROADMAP.md | 556 ---- README.md | 435 ++- config/data_sources.yaml | 26 +- config/environments/development.yaml | 87 + config/environments/production.yaml | 86 + config/environments/sandbox.yaml | 86 + config/environments/training.yaml | 96 + docker-compose.yml | 23 + docs/ARCHITECTURE.md | 14 +- docs/DATA_SOURCES_GUIDE.md | 2 +- docs/GAME_CONFIGURATION_GUIDE.md | 8 +- docs/GETTING_STARTED.md | 419 +-- docs/GITHUB_PUSH_GUIDE.md | 467 --- docs/GIT_WORKFLOW.md | 443 --- docs/MVP_PLAN.md | 84 - docs/README.md | 42 +- docs/SDK_DOCUMENTATION.md | 8 +- docs/SIMPLIFIED_ARCHITECTURE_SUMMARY.md | 6 +- docs/SYSTEM_OVERVIEW.md | 22 +- docs/TRADING_LOGIC.md | 12 +- docs/api_reference.md | 51 + docs/backtesting.md | 47 + docs/data_sources.md | 57 + docs/strategies.md | 91 + examples/agent_redis_consumer.py | 149 +- examples/backtest_strategy.py | 411 +++ examples/basic_usage.py | 202 ++ examples/custom_data_adapter.py | 447 +++ examples/espn_playbyplay.py | 139 +- examples/fetch_markets.py | 23 +- examples/market_correlation.py | 168 +- examples/stream_markets.py | 20 +- examples/track_sports_example.py | 145 + examples/twitter_sentiment_stream.py | 220 +- neural_sdk/README.md | 346 +++ neural_sdk/__init__.py | 72 + neural_sdk/agents/__init__.py | 21 + .../agents/base.py | 586 ++-- neural_sdk/agents/types.py | 142 + neural_sdk/backtesting/__init__.py | 49 + neural_sdk/backtesting/data_loader.py | 400 +++ neural_sdk/backtesting/engine.py | 471 +++ neural_sdk/backtesting/metrics.py | 471 +++ neural_sdk/backtesting/portfolio.py | 494 ++++ neural_sdk/backtesting/providers/__init__.py | 43 + neural_sdk/backtesting/providers/base.py | 273 ++ .../backtesting/providers/csv_provider.py | 302 ++ .../providers/database_provider.py | 352 +++ .../backtesting/providers/parquet_provider.py | 378 +++ .../backtesting/providers/s3_provider.py | 311 ++ neural_sdk/cli.py | 368 +++ neural_sdk/core/__init__.py | 23 + neural_sdk/core/client.py | 618 ++++ neural_sdk/core/config.py | 425 +++ neural_sdk/core/environment_guards.py | 589 ++++ neural_sdk/core/environment_manager.py | 672 +++++ neural_sdk/core/environment_ui.py | 566 ++++ neural_sdk/core/environment_validator.py | 804 +++++ neural_sdk/core/exceptions.py | 106 + neural_sdk/core/redis_config.py | 512 ++++ neural_sdk/core/transition_protocol.py | 699 +++++ neural_sdk/strategies/__init__.py | 46 + {src => neural_sdk}/trading/README.md | 0 {src => neural_sdk}/trading/__init__.py | 10 +- .../trading/data_aggregator.py | 329 ++- {src => neural_sdk}/trading/espn_tools.py | 176 +- neural_sdk/trading/llm_client.py | 178 ++ .../trading/sentiment_probability.py | 227 +- {src => neural_sdk}/trading/stop_loss.py | 301 +- neural_sdk/utils/__init__.py | 52 + pyproject.toml | 159 +- scripts/demo_investor.py | 341 --- scripts/demo_sdk.py | 305 -- scripts/quick_weather_demo.py | 106 - scripts/test_weather_adapter.py | 405 --- src/agents/__init__.py | 0 src/agents/always_on/__init__.py | 0 src/agents/always_on/data_coordinator.py | 469 --- src/agents/always_on/portfolio_monitor.py | 593 ---- src/agents/on_demand/__init__.py | 0 src/agents/on_demand/game_analyst.py | 515 ---- src/agents/training_consumer.py | 594 ---- src/agents/trigger_service.py | 459 --- src/backtesting/__init__.py | 0 src/backtesting/backtest_engine.py | 566 ---- src/backtesting/historical_data_collector.py | 469 --- src/backtesting/parameter_optimizer.py | 603 ---- src/backtesting/performance_analyzer.py | 621 ---- src/data_pipeline/README_OLD.md | 2 +- src/data_pipeline/config/settings.py | 20 +- src/data_pipeline/data_sources/espn/models.py | 2 + src/data_pipeline/data_sources/kalshi/auth.py | 37 +- .../data_sources/kalshi/client.py | 47 +- .../orchestration/unified_stream_manager.py | 163 +- .../reliability/resilience_coordinator.py | 2 +- src/data_pipeline/state_manager.py | 16 +- src/integration/__init__.py | 54 +- src/sdk/README.md | 257 -- src/sdk/__init__.py | 39 - src/sdk/adapters/__init__.py | 0 src/sdk/adapters/draftkings.py | 374 --- src/sdk/adapters/reddit.py | 465 --- src/sdk/adapters/weather.py | 495 ---- src/sdk/core/__init__.py | 0 src/sdk/core/base_adapter.py | 325 --- src/sdk/core/sdk_manager.py | 439 --- src/sdk/utils/__init__.py | 0 src/synthetic_data/__init__.py | 20 - src/synthetic_data/generators/__init__.py | 26 - src/synthetic_data/generators/game_engine.py | 896 ------ .../generators/market_simulator.py | 754 ----- .../generators/scenario_builder.py | 817 ------ src/synthetic_data/models/__init__.py | 14 - src/synthetic_data/models/lfm2_fine_tuner.py | 561 ---- src/synthetic_data/preprocessing/__init__.py | 11 - .../preprocessing/nfl_dataset_processor.py | 434 --- .../preprocessing/training_data_builder.py | 258 -- src/synthetic_data/storage/__init__.py | 11 - .../storage/chromadb_manager.py | 556 ---- .../storage/synthetic_event_store.py | 209 -- src/synthetic_data/validation/__init__.py | 14 - .../validation/play_data_validator.py | 422 --- src/trading/llm_client.py | 166 -- src/training/__init__.py | 42 - src/training/agent_analytics.py | 882 ------ src/training/memory_system.py | 815 ------ src/training/synthetic_env.py | 672 ----- uv.lock | 2593 ----------------- 135 files changed, 13632 insertions(+), 23065 deletions(-) delete mode 100644 CLAUDE.md create mode 100644 Dockerfile delete mode 100644 INVESTOR_DEMO.md create mode 100644 LICENSE delete mode 100644 PLATFORM_ROADMAP.md create mode 100644 config/environments/development.yaml create mode 100644 config/environments/production.yaml create mode 100644 config/environments/sandbox.yaml create mode 100644 config/environments/training.yaml create mode 100644 docker-compose.yml delete mode 100644 docs/GITHUB_PUSH_GUIDE.md delete mode 100644 docs/GIT_WORKFLOW.md delete mode 100644 docs/MVP_PLAN.md create mode 100644 docs/api_reference.md create mode 100644 docs/backtesting.md create mode 100644 docs/data_sources.md create mode 100644 docs/strategies.md create mode 100644 examples/backtest_strategy.py create mode 100644 examples/basic_usage.py create mode 100644 examples/custom_data_adapter.py create mode 100644 examples/track_sports_example.py create mode 100644 neural_sdk/README.md create mode 100644 neural_sdk/__init__.py create mode 100644 neural_sdk/agents/__init__.py rename src/agents/base_consumer.py => neural_sdk/agents/base.py (74%) create mode 100644 neural_sdk/agents/types.py create mode 100644 neural_sdk/backtesting/__init__.py create mode 100644 neural_sdk/backtesting/data_loader.py create mode 100644 neural_sdk/backtesting/engine.py create mode 100644 neural_sdk/backtesting/metrics.py create mode 100644 neural_sdk/backtesting/portfolio.py create mode 100644 neural_sdk/backtesting/providers/__init__.py create mode 100644 neural_sdk/backtesting/providers/base.py create mode 100644 neural_sdk/backtesting/providers/csv_provider.py create mode 100644 neural_sdk/backtesting/providers/database_provider.py create mode 100644 neural_sdk/backtesting/providers/parquet_provider.py create mode 100644 neural_sdk/backtesting/providers/s3_provider.py create mode 100644 neural_sdk/cli.py create mode 100644 neural_sdk/core/__init__.py create mode 100644 neural_sdk/core/client.py create mode 100644 neural_sdk/core/config.py create mode 100644 neural_sdk/core/environment_guards.py create mode 100644 neural_sdk/core/environment_manager.py create mode 100644 neural_sdk/core/environment_ui.py create mode 100644 neural_sdk/core/environment_validator.py create mode 100644 neural_sdk/core/exceptions.py create mode 100644 neural_sdk/core/redis_config.py create mode 100644 neural_sdk/core/transition_protocol.py create mode 100644 neural_sdk/strategies/__init__.py rename {src => neural_sdk}/trading/README.md (100%) rename {src => neural_sdk}/trading/__init__.py (65%) rename {src => neural_sdk}/trading/data_aggregator.py (81%) rename {src => neural_sdk}/trading/espn_tools.py (59%) create mode 100644 neural_sdk/trading/llm_client.py rename {src => neural_sdk}/trading/sentiment_probability.py (87%) rename {src => neural_sdk}/trading/stop_loss.py (83%) create mode 100644 neural_sdk/utils/__init__.py delete mode 100644 scripts/demo_investor.py delete mode 100644 scripts/demo_sdk.py delete mode 100644 scripts/quick_weather_demo.py delete mode 100644 scripts/test_weather_adapter.py delete mode 100644 src/agents/__init__.py delete mode 100644 src/agents/always_on/__init__.py delete mode 100644 src/agents/always_on/data_coordinator.py delete mode 100644 src/agents/always_on/portfolio_monitor.py delete mode 100644 src/agents/on_demand/__init__.py delete mode 100644 src/agents/on_demand/game_analyst.py delete mode 100644 src/agents/training_consumer.py delete mode 100644 src/agents/trigger_service.py delete mode 100644 src/backtesting/__init__.py delete mode 100644 src/backtesting/backtest_engine.py delete mode 100644 src/backtesting/historical_data_collector.py delete mode 100644 src/backtesting/parameter_optimizer.py delete mode 100644 src/backtesting/performance_analyzer.py delete mode 100644 src/sdk/README.md delete mode 100644 src/sdk/__init__.py delete mode 100644 src/sdk/adapters/__init__.py delete mode 100644 src/sdk/adapters/draftkings.py delete mode 100644 src/sdk/adapters/reddit.py delete mode 100644 src/sdk/adapters/weather.py delete mode 100644 src/sdk/core/__init__.py delete mode 100644 src/sdk/core/base_adapter.py delete mode 100644 src/sdk/core/sdk_manager.py delete mode 100644 src/sdk/utils/__init__.py delete mode 100644 src/synthetic_data/__init__.py delete mode 100644 src/synthetic_data/generators/__init__.py delete mode 100644 src/synthetic_data/generators/game_engine.py delete mode 100644 src/synthetic_data/generators/market_simulator.py delete mode 100644 src/synthetic_data/generators/scenario_builder.py delete mode 100644 src/synthetic_data/models/__init__.py delete mode 100644 src/synthetic_data/models/lfm2_fine_tuner.py delete mode 100644 src/synthetic_data/preprocessing/__init__.py delete mode 100644 src/synthetic_data/preprocessing/nfl_dataset_processor.py delete mode 100644 src/synthetic_data/preprocessing/training_data_builder.py delete mode 100644 src/synthetic_data/storage/__init__.py delete mode 100644 src/synthetic_data/storage/chromadb_manager.py delete mode 100644 src/synthetic_data/storage/synthetic_event_store.py delete mode 100644 src/synthetic_data/validation/__init__.py delete mode 100644 src/synthetic_data/validation/play_data_validator.py delete mode 100644 src/trading/llm_client.py delete mode 100644 src/training/__init__.py delete mode 100644 src/training/agent_analytics.py delete mode 100644 src/training/memory_system.py delete mode 100644 src/training/synthetic_env.py delete mode 100644 uv.lock diff --git a/.env.example b/.env.example index a2eeff6d..5b68f78b 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,30 @@ -# Kalshi Trading API (Required) -KALSHI_API_KEY_ID=your_kalshi_key_id -KALSHI_API_KEY=your_kalshi_api_key +# ============================================ +# NEURAL SDK - ENVIRONMENT TEMPLATE +# ============================================ +# Copy this file to .env and fill in your actual values -# OpenWeatherMap API (Working!) -OPENWEATHER_API_KEY=78596505b0f5fea89e98ebcbf3bd6e21 +# --- NEURAL API (Required) --- +# Accepts: prod | demo (used for Kalshi endpoints) +NEURAL_ENVIRONMENT=prod +NEURAL_API_KEY_ID=your_neural_api_key_id +# Either provide the private key INLINE or via FILE path +# NEURAL_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" +NEURAL_PRIVATE_KEY_FILE=./keys/neural_prod_private.key -# Reddit API (Optional - for sentiment analysis) +# --- EXTERNAL APIs --- +# Used by weather adapter (config/data_sources.yaml) +OPENWEATHER_API_KEY=your_openweather_api_key REDDIT_CLIENT_ID=your_reddit_client_id REDDIT_CLIENT_SECRET=your_reddit_client_secret -# DraftKings (No API key needed - public data) \ No newline at end of file +# --- LLM CONFIGURATION --- +OPENROUTER_API_KEY=your_openrouter_api_key + +# --- OTHER SERVICES --- +E2B_API_KEY=your_e2b_api_key +EXA_API_KEY=your_exa_api_key +TWITTERAPI_KEY=your_twitter_api_key + +# --- AGENTUITY PLATFORM --- +AGENTUITY_SDK_KEY=your_agentuity_sdk_key +AGENTUITY_PROJECT_KEY=your_agentuity_project_key diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7282d82..bd0189ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -104,7 +104,7 @@ jobs: - name: Install package run: | pip install dist/*.whl - python -c "import agent_consumers; print('✅ Package imported successfully')" + python -c "import sdk, data_pipeline; print('✅ Package imported successfully')" create-github-release: name: Create GitHub Release @@ -263,4 +263,4 @@ jobs: - name: Create incident run: | echo "📝 Creating incident report" - # gh issue create --title "Production deployment failed: ${{ needs.validate-version.outputs.version }}" --body "Automatic rollback initiated" \ No newline at end of file + # gh issue create --title "Production deployment failed: ${{ needs.validate-version.outputs.version }}" --body "Automatic rollback initiated" diff --git a/.gitignore b/.gitignore index e4796dd1..518d5f7f 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,9 @@ celerybeat.pid .env .env.development .env.production +.env.demo +.env.* +!*.example .venv env/ venv/ @@ -191,6 +194,7 @@ keys/ # Project Specific logs/ *.log +data/ data/cache/ data/backtest_results/ temp/ @@ -214,4 +218,4 @@ Thumbs.db # Test outputs test_results/ coverage/ -.pytest_cache/ \ No newline at end of file +.pytest_cache/ diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 2ea1bbdd..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,494 +0,0 @@ -# CLAUDE.md - -This file provides comprehensive guidance to Claude Code (claude.ai/code) when working with the Kalshi Trading Agent System. - -## Project Overview - -Autonomous multi-agent trading system for Kalshi sports event contracts using the Agentuity framework. The system uses real-time WebSocket streams, Redis pub/sub for data distribution, and sophisticated agents with LLM analysis to execute trades using the Kelly Criterion for optimal position sizing. - -## System Architecture - -### Data Flow Pipeline -``` -[External APIs] → [WebSocket Streams] → [Redis Pub/Sub] → [Agent Consumers] → [Trading Decisions] - ↓ ↓ ↓ - [Stream Manager] [Event Publisher] [Agentuity Agents] - ↓ ↓ ↓ - [Reliability Layer] [Redis Channels] [Kalshi API] -``` - -### Directory Structure (Production Names) -``` -. -├── agents/ # Agentuity platform agents (high-level logic) -├── agent_consumers/ # Redis consumers (data processing) -├── data_pipeline/ # Data ingestion and streaming -│ ├── orchestration/ # Stream coordination -│ ├── streaming/ # WebSocket clients -│ ├── data_sources/ # API integrations -│ └── reliability/ # Resilience components -├── trading_logic/ # Trading algorithms and tools -├── tests/ # Test suite -├── examples/ # Usage examples -└── docs/ # Documentation -``` - -## Key Design Decisions - -### 1. Redis Pub/Sub Architecture -**Decision:** Use Redis as central message broker -**Rationale:** -- Decouples data sources from consumers -- Enables horizontal scaling -- Provides buffering and persistence -- Supports multiple subscribers per channel - -### 2. Separation of Concerns -**Decision:** Split agents into consumers and platform agents -**Rationale:** -- `agent_consumers/`: Handle real-time data, no LLM calls -- `agents/`: Complex decisions with LLM analysis -- Clear performance boundaries -- Easier testing and debugging - -### 3. Kelly Criterion with Safety Factor -**Decision:** Use 25% of Kelly for position sizing -**Rationale:** -- Reduces risk of ruin -- Accounts for estimation errors -- Provides smoother equity curve -- Industry standard practice - -## Common Development Tasks - -### Adding a New Data Source - -1. **Create WebSocket Client** -```python -# data_pipeline/streaming/new_source.py -class NewSourceWebSocket: - async def connect(self): - # Implementation - - async def subscribe(self, channels): - # Implementation -``` - -2. **Add to Stream Manager** -```python -# data_pipeline/orchestration/unified_stream_manager.py -self.new_source_client = NewSourceWebSocket(config) -await self.new_source_client.connect() -``` - -3. **Define Redis Channel** -```python -# data_pipeline/orchestration/redis_event_publisher.py -await self.publish("newsource:events", event_data) -``` - -4. **Create Consumer** -```python -# agent_consumers/NewConsumer/consumer.py -class NewConsumer(BaseConsumer): - async def process_message(self, channel, data): - # Process events -``` - -### Implementing a Trading Strategy - -1. **Define Strategy in Agent** -```python -# agents/StrategyAnalyst/agent.py -@tool -def analyze_opportunity(market_data, sentiment): - # Strategy logic - return signal -``` - -2. **Process Signals in Consumer** -```python -# agent_consumers/DataCoordinator/data_coordinator.py -if signal.strength > THRESHOLD: - await self.publish_signal(signal) -``` - -3. **Execute Trade** -```python -# agents/TradeExecutor/agent.py -@tool -def execute_trade(signal): - position_size = calculate_kelly_position(signal) - order = place_order(market, side, position_size) - return order -``` - -### Testing Workflows - -1. **Test Redis Integration** -```bash -# Start Redis -redis-server - -# Run integration tests -pytest tests/test_redis_integration.py -``` - -2. **Test Individual Agent** -```bash -# Start in dev mode -agentuity dev - -# Test specific agent -agentuity agent test DataCoordinator -``` - -3. **Full System Test** -```bash -# Run all components -python examples/agent_redis_consumer.py all -``` - -## Code Style & Conventions - -### Python Standards -- Use Python 3.10+ features -- Type hints for all functions -- Async/await for I/O operations -- Dataclasses for data structures - -### Naming Conventions -```python -# Files and modules: snake_case -unified_stream_manager.py - -# Classes: PascalCase -class DataCoordinator: - pass - -# Functions and variables: snake_case -def calculate_position_size(): - pass - -# Constants: UPPER_SNAKE_CASE -MAX_POSITION_SIZE = 100 -``` - -### Import Organization -```python -# Standard library -import os -import asyncio -from datetime import datetime - -# Third-party -import redis -import websockets -from agentuity import Agent, tool - -# Local application -from data_pipeline.orchestration import StreamManager -from agent_consumers.base_consumer import BaseConsumer -from trading_logic.kelly_tools import calculate_kelly -``` - -### Error Handling Pattern -```python -async def robust_operation(): - """Standard error handling pattern.""" - max_retries = 3 - retry_delay = 1.0 - - for attempt in range(max_retries): - try: - result = await risky_operation() - return result - except TemporaryError as e: - logger.warning(f"Attempt {attempt + 1} failed: {e}") - if attempt < max_retries - 1: - await asyncio.sleep(retry_delay) - retry_delay *= 2 # Exponential backoff - else: - raise - except PermanentError as e: - logger.error(f"Permanent failure: {e}") - raise -``` - -## Trading Logic Implementation - -### Position Sizing -Always use Kelly Criterion with safety factors: -```python -# Never use full Kelly -position = kelly_fraction * 0.25 # 25% of Kelly - -# Apply hard limits -position = min(position, MAX_POSITION_SIZE) -position = max(position, MIN_POSITION_SIZE) - -# Check portfolio exposure -if total_exposure + position > MAX_PORTFOLIO_EXPOSURE: - position = MAX_PORTFOLIO_EXPOSURE - total_exposure -``` - -### Risk Management Rules -1. **Stop Loss**: Always set at order time -2. **Position Limits**: Max 5% per position -3. **Correlation**: Reduce size for correlated bets -4. **Drawdown**: Stop at 20% daily loss - -### Market Analysis -```python -# Check for arbitrage -if yes_price + no_price < 0.98: - # Arbitrage opportunity exists - -# Check liquidity -if volume < MIN_LIQUIDITY: - # Skip illiquid markets - -# Check spread -if abs(yes_price - no_price) > MAX_SPREAD: - # Market too wide -``` - -## Performance Considerations - -### WebSocket Management -- Maintain persistent connections -- Implement heartbeat/ping -- Buffer during disconnections -- Automatic reconnection - -### Redis Optimization -- Use connection pooling -- Batch publish when possible -- Set appropriate TTLs -- Monitor memory usage - -### Agent Performance -- Cache LLM responses -- Batch similar requests -- Use appropriate models -- Monitor token usage - -## Security Practices - -### API Keys -```bash -# Never hardcode keys -KALSHI_API_KEY_ID=xxx # In .env file - -# Use environment variables -key_id = os.getenv("KALSHI_API_KEY_ID") -``` - -### Data Privacy -- Don't log sensitive data -- Sanitize user inputs -- Encrypt stored credentials -- Audit access logs - -## Troubleshooting Guide - -### Common Issues - -#### WebSocket Disconnections -```python -# Check: Connection status -logger.info(f"WebSocket state: {ws.state}") - -# Solution: Automatic reconnection -await exponential_backoff_retry() -``` - -#### Redis Pub/Sub Issues -```bash -# Check: Redis connectivity -redis-cli ping - -# Check: Active subscriptions -redis-cli PUBSUB CHANNELS - -# Monitor: Message flow -redis-cli MONITOR -``` - -#### No Trading Signals -```python -# Check: Data pipeline -assert stream_manager.is_connected() - -# Check: Agent consumers -assert len(active_consumers) > 0 - -# Check: Signal thresholds -logger.info(f"Signal threshold: {SIGNAL_THRESHOLD}") -``` - -#### High Latency -```bash -# Profile: Message processing -python -m cProfile -s cumulative server.py - -# Check: Redis performance -redis-cli --latency - -# Monitor: System resources -htop -``` - -## Deployment Checklist - -### Pre-Deployment -- [ ] All tests passing -- [ ] Environment variables set -- [ ] Redis configured -- [ ] API credentials valid -- [ ] Risk limits configured - -### Deployment Steps -```bash -# 1. Set production environment -export ENVIRONMENT=production - -# 2. Verify configuration -agentuity config verify - -# 3. Deploy agents -agentuity deploy - -# 4. Monitor logs -agentuity logs --follow -``` - -### Post-Deployment -- [ ] Verify WebSocket connections -- [ ] Check Redis pub/sub -- [ ] Confirm agent health -- [ ] Monitor first trades -- [ ] Review error logs - -## Performance Benchmarks - -### Expected Metrics -- WebSocket latency: < 100ms -- Redis publish: < 10ms -- Agent processing: < 500ms -- End-to-end: < 1 second - -### Capacity Planning -- Redis: 10K messages/second -- WebSocket: 1K updates/second -- Agents: 100 decisions/minute -- Trades: 20 per day maximum - -## Emergency Procedures - -### Market Halt -```python -# Immediate stop -await risk_manager.emergency_stop() - -# Cancel all orders -await trade_executor.cancel_all_orders() - -# Notify team -await send_alert("Trading halted") -``` - -### Data Loss -```python -# Switch to cached data -await use_fallback_data() - -# Reduce position sizes -KELLY_FRACTION *= 0.5 - -# Log incident -logger.critical("Data loss detected") -``` - -## Testing Strategy - -### Unit Tests -Test individual components in isolation: -```bash -pytest tests/unit/ -v -``` - -### Integration Tests -Test component interactions: -```bash -pytest tests/integration/ -v -``` - -### End-to-End Tests -Test full system flow: -```bash -pytest tests/e2e/ -v -``` - -### Performance Tests -```bash -# Load testing -locust -f tests/load/locustfile.py - -# Stress testing -python tests/stress/stress_test.py -``` - -## Important Notes - -### Do's -- ✅ Always use try/except for external calls -- ✅ Log all trading decisions -- ✅ Validate data before processing -- ✅ Use type hints -- ✅ Write tests for new features - -### Don'ts -- ❌ Never commit secrets -- ❌ Don't bypass risk checks -- ❌ Avoid synchronous I/O -- ❌ Don't ignore error logs -- ❌ Never use full Kelly - -## Quick Commands Reference - -```bash -# Development -agentuity dev # Start dev server -uv run server.py # Run directly -uv run pytest tests/ # Run tests - -# Redis -redis-cli ping # Check Redis -redis-cli MONITOR # Monitor messages -redis-cli FLUSHALL # Clear all data - -# Deployment -agentuity deploy # Deploy to cloud -agentuity logs DataCoordinator # View agent logs -agentuity status # Check deployment - -# Monitoring -python examples/agent_redis_consumer.py all # Run all consumers -python examples/test_kalshi_websocket.py # Test WebSocket -``` - -## Contact & Support - -For questions about: -- **Architecture**: Review docs/ARCHITECTURE.md -- **Agents**: Check agents/README.md -- **Trading Logic**: See trading_logic/README.md -- **Data Pipeline**: Read data_pipeline/README.md - -When debugging: -1. Check logs first -2. Verify configuration -3. Test components individually -4. Review recent changes -5. Check system resources \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..50b37a1b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Optional: install uv for reproducible installs +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && ln -s /root/.local/bin/uv /usr/local/bin/uv + +WORKDIR /app + +# Install dependencies first (leverage caching) +COPY pyproject.toml uv.lock ./ +RUN uv sync --no-dev || pip install . + +# Copy source and configs +COPY neural_sdk ./neural_sdk +COPY config ./config + +# Default command does a smoke import +CMD ["python", "-c", "import neural_sdk; print('Neural SDK OK')"] + diff --git a/INVESTOR_DEMO.md b/INVESTOR_DEMO.md deleted file mode 100644 index c57cb5f4..00000000 --- a/INVESTOR_DEMO.md +++ /dev/null @@ -1,496 +0,0 @@ -# 🏆 Kalshi Trading Agent Platform -## AI-Powered Sports Prediction Market Trading System - ---- - -## Executive Summary - -### 📊 The Opportunity -- **$2B+ Daily Volume** in sports prediction markets -- **67% Average Spread** indicates market inefficiencies -- **<100ms Latency Advantage** over manual traders -- **24/7 Autonomous Operation** capturing opportunities humans miss - -### 🎯 Our Solution -An institutional-grade autonomous trading system that: -- Correlates real-time game data with market prices -- Uses AI to identify and exploit pricing inefficiencies -- Manages risk using proven quantitative methods -- Scales horizontally to trade hundreds of markets simultaneously - ---- - -## System Architecture Overview - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ KALSHI TRADING AGENT PLATFORM │ -├─────────────────────────────────────────────────────────────────┤ -│ │ -│ DATA INGESTION LAYER │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Kalshi │ │ ESPN │ │ Twitter │ │ -│ │ WebSocket │ │ Stream │ │ Sentiment │ │ -│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ -│ │ │ │ │ -│ ═══════╪═════════════════╪═════════════════╪═══════════ │ -│ ▼ ▼ ▼ │ -│ ┌────────────────────────────────────────────────┐ │ -│ │ UNIFIED STREAM MANAGER │ │ -│ │ • Event Correlation │ │ -│ │ • Time Synchronization │ │ -│ │ • Backpressure Control │ │ -│ └─────────────────────┬──────────────────────────┘ │ -│ │ │ -│ DISTRIBUTION LAYER ▼ │ -│ ┌────────────────────────────────────────────────┐ │ -│ │ REDIS PUB/SUB HUB │ │ -│ │ • 10K msg/sec throughput │ │ -│ │ • Channel-based routing │ │ -│ │ • Persistent message buffer │ │ -│ └──┬──────────┬──────────┬──────────┬──────────┘ │ -│ │ │ │ │ │ -│ INTELLIGENCE LAYER │ -│ ▼ ▼ ▼ ▼ │ -│ ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ │ -│ │Data │ │Market│ │Risk │ │Trade │ │ -│ │Coord.│ │Analy.│ │Mgr. │ │Exec. │ │ -│ └──────┘ └──────┘ └──────┘ └──────┘ │ -│ │ │ │ │ │ -│ EXECUTION LAYER │ │ │ -│ └──────────┴──────────┴──────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────┐ │ -│ │ KALSHI API │ │ -│ │ • Orders │ │ -│ │ • Positions │ │ -│ └──────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - ---- - -## 🚀 Live Data Flow Demonstration - -### Real-Time Market Monitoring -```python -# LIVE EXAMPLE: Super Bowl 2025 - Chiefs vs Bills -Market: SUPERBOWL-2025-WINNER -Current Prices: Chiefs YES: $0.65 | Bills YES: $0.35 - -📈 Data Sources Active: -- Kalshi WebSocket: ✅ Connected (12ms latency) -- ESPN GameCast: ✅ Streaming (45ms latency) -- Twitter Sentiment: ✅ Processing (2,341 tweets/min) - -🔄 Recent Events (Last 30 seconds): -[14:23:45] ESPN: Touchdown Chiefs! Score: 21-14 -[14:23:46] Twitter: Sentiment spike detected (+18% Chiefs) -[14:23:47] Kalshi: Price movement $0.62 → $0.65 (+4.8%) -[14:23:48] System: OPPORTUNITY - Market lagging game event -[14:23:49] Trade: BUY Chiefs YES @ $0.65 (100 contracts) -[14:23:51] Trade: FILLED @ $0.65 ✓ - -💰 P&L This Session: +$487.23 (12 trades, 83% win rate) -``` - ---- - -## 🧠 AI Agent Architecture - -### Always-On Agents (24/7 Monitoring) -``` -┌────────────────────────────────────────┐ -│ DATA COORDINATOR AGENT │ -│ • Correlates multi-source data │ -│ • Maintains market context │ -│ • Publishes unified events │ -└────────────────────────────────────────┘ - │ - ┌───────────────┼───────────────┐ - ▼ ▼ ▼ -┌─────────┐ ┌─────────┐ ┌─────────┐ -│Portfolio│ │ Market │ │ Risk │ -│ Monitor │ │ Scanner │ │ Monitor │ -└─────────┘ └─────────┘ └─────────┘ -``` - -### On-Demand Agents (Decision Making) -``` -┌────────────────────────────────────────┐ -│ STRATEGY ANALYST AGENT │ -│ • GPT-4 powered analysis │ -│ • Pattern recognition │ -│ • Probability calculation │ -└────────────────────────────────────────┘ - │ - ▼ -┌────────────────────────────────────────┐ -│ TRADE EXECUTOR AGENT │ -│ • Kelly Criterion sizing │ -│ • Order management │ -│ • Execution optimization │ -└────────────────────────────────────────┘ -``` - ---- - -## 📊 Backtesting & Performance - -### Historical Performance (2024 Season) -``` -Period: Jan 2024 - Dec 2024 -Markets Traded: 487 -Total Trades: 3,241 - -Performance Metrics: -┌─────────────────────────────────────┐ -│ Sharpe Ratio: 2.87 │ -│ Win Rate: 67.3% │ -│ Avg Win/Loss: 1.82 │ -│ Max Drawdown: -12.4% │ -│ Total Return: +187% │ -│ Profit Factor: 2.41 │ -└─────────────────────────────────────┘ - -Monthly Returns: -Jan: +14.2% Apr: +11.8% Jul: +18.3% Oct: +22.1% -Feb: +8.7% May: +15.4% Aug: +12.9% Nov: +19.7% -Mar: +9.3% Jun: -3.2% Sep: +16.5% Dec: +24.8% -``` - -### Strategy Optimization Results -```python -# Genetic Algorithm Optimization (10,000 iterations) -OPTIMAL PARAMETERS: -├── Kelly Multiplier: 0.25 (25% of full Kelly) -├── Stop Loss: 12% -├── Take Profit: 35% -├── Min Edge Required: 3.5% -├── Confidence Threshold: 0.72 -└── Max Position Size: 5% of capital - -# Walk-Forward Analysis (6 months out-of-sample) -In-Sample Sharpe: 2.87 -Out-of-Sample Sharpe: 2.64 ✓ (Robust) -``` - ---- - -## 🛡️ Risk Management - -### Multi-Layer Risk Control -``` -Level 1: Pre-Trade Checks -├── Market liquidity verification -├── Correlation analysis -├── Position sizing (Kelly Criterion) -└── Max exposure limits - -Level 2: Real-Time Monitoring -├── Stop-loss triggers -├── Drawdown circuit breakers -├── Volatility adjustment -└── Portfolio heat mapping - -Level 3: System Protection -├── API rate limiting -├── Connection redundancy -├── Data validation -└── Emergency shutdown -``` - -### Live Risk Dashboard -``` -Current Portfolio Status: -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Open Positions: 8 -Total Exposure: $24,531 (49% of capital) -Daily P&L: +$1,247 (+2.5%) -Risk Metrics: - • VaR (95%): $1,823 - • Correlation Risk: LOW - • System Health: 98/100 -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -``` - ---- - -## 💻 Technology Stack - -### Core Infrastructure -- **Language**: Python 3.11+ with async/await -- **Message Queue**: Redis Pub/Sub (10K msg/sec) -- **WebSockets**: Persistent bi-directional streams -- **AI/ML**: OpenAI GPT-4, Custom sentiment models -- **Monitoring**: Prometheus + Grafana dashboards - -### Performance Specifications -``` -Latency Benchmarks: -├── Market Data → Decision: <100ms -├── Decision → Execution: <50ms -├── End-to-End: <200ms -└── Failure Recovery: <2 seconds - -Capacity: -├── Concurrent Markets: 500+ -├── Events/Second: 10,000 -├── Decisions/Minute: 100 -└── Orders/Day: 1,000+ -``` - ---- - -## 📈 Competitive Advantages - -### 1. **Speed** - Microsecond Advantage -``` -Human Trader: 2-5 seconds to react -Our System: 0.1 seconds to execute -Advantage: 20-50x faster -``` - -### 2. **Scale** - Parallel Processing -``` -Human: Monitors 1-3 markets -System: Monitors 500+ markets -Advantage: 166x coverage -``` - -### 3. **Consistency** - No Emotions -``` -Human: 55% win rate (emotional decisions) -System: 67% win rate (data-driven) -Advantage: 22% improvement -``` - -### 4. **Intelligence** - AI Enhancement -```python -# Real correlation example -if espn_touchdown_event and not kalshi_price_moved: - confidence = calculate_edge(game_state, market_state) - if confidence > 0.72: - execute_trade(size=kelly_position(confidence)) -``` - ---- - -## 🎯 Market Opportunity - -### Total Addressable Market -- **Kalshi Daily Volume**: $2M+ and growing -- **Sports Betting Market**: $150B globally -- **Prediction Markets**: $10B+ by 2025 - -### Revenue Model -- **Performance Fee**: 20% of profits -- **Management Fee**: 2% AUM -- **License Revenue**: White-label to funds - -### Scalability Path -``` -Phase 1: Single Exchange (Kalshi) ✓ Complete -Phase 2: Multi-Exchange (Polymarket, Manifold) -Phase 3: Traditional Sports Books Integration -Phase 4: Custom Market Making -``` - ---- - -## 🚦 Live System Demo - -### Starting the Platform -```bash -# Initialize all components -$ python scripts/run_agents.py all - -[2024-08-30 14:30:00] Starting Kalshi Trading Platform... -[2024-08-30 14:30:01] ✓ Redis connected (localhost:6379) -[2024-08-30 14:30:02] ✓ Kalshi WebSocket connected -[2024-08-30 14:30:03] ✓ ESPN stream active (4 games) -[2024-08-30 14:30:04] ✓ Twitter sentiment analyzer online -[2024-08-30 14:30:05] ✓ 5 AI agents initialized -[2024-08-30 14:30:06] ✓ Risk manager active -[2024-08-30 14:30:07] System ready. Monitoring 12 markets... - -═══════════════════════════════════════════════ - KALSHI TRADING PLATFORM - LIVE -═══════════════════════════════════════════════ -Markets: 12 | Agents: 5 | Latency: 23ms -P&L Today: +$1,247.83 | Win Rate: 71% -═══════════════════════════════════════════════ -``` - -### Real-Time Market Action -```python -# Live opportunity detection -[14:31:23] 🎯 OPPORTUNITY DETECTED -Market: NFL-WEEK18-KC-BUF -Signal: BIG_PLAY_DIVERGENCE -├── ESPN: 75-yard TD pass (Chiefs) -├── Twitter: +2,341 mentions/min -├── Kalshi: No price movement yet -├── Edge: 8.3% (HIGH CONFIDENCE) -└── Action: BUY 250 contracts @ $0.64 - -[14:31:24] 📊 Executing Trade... -├── Kelly Position: $1,250 (2.5% of capital) -├── Order ID: ORD-2024-483921 -├── Status: PENDING → FILLED -└── Fill Price: $0.64 ✓ - -[14:31:28] 💰 Price Movement -├── Market moved: $0.64 → $0.69 -├── Unrealized P&L: +$125.00 -└── Signal accuracy: CONFIRMED ✓ -``` - ---- - -## 📊 Performance Analytics Dashboard - -``` -╔══════════════════════════════════════════════════════╗ -║ LIVE TRADING DASHBOARD ║ -╠══════════════════════════════════════════════════════╣ -║ ║ -║ Portfolio Performance (24H) ║ -║ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░ +4.7% ║ -║ ║ -║ Win Rate by Market Type ║ -║ NFL: ▓▓▓▓▓▓▓▓▓▓▓▓▓░ 72% ║ -║ NBA: ▓▓▓▓▓▓▓▓▓▓░░░░ 65% ║ -║ MLB: ▓▓▓▓▓▓▓▓▓▓▓░░░ 68% ║ -║ ║ -║ System Health ║ -║ CPU: ▓▓▓░░░░░░░░░░░ 23% ║ -║ MEM: ▓▓▓▓▓░░░░░░░░░ 41% ║ -║ NET: ▓▓░░░░░░░░░░░░ 15% ║ -║ ║ -║ Active Strategies ║ -║ • Momentum Following [ACTIVE] +$823 ║ -║ • Mean Reversion [ACTIVE] +$412 ║ -║ • Sentiment Arbitrage [ACTIVE] +$198 ║ -║ • Event Correlation [PAUSED] $0 ║ -║ ║ -╚══════════════════════════════════════════════════════╝ -``` - ---- - -## 🔮 Future Roadmap - -### Q1 2025: Enhanced Intelligence -- [ ] GPT-4 Vision for game footage analysis -- [ ] Custom transformer models for price prediction -- [ ] Reinforcement learning for strategy optimization - -### Q2 2025: Market Expansion -- [ ] Polymarket integration -- [ ] Augur protocol support -- [ ] Cross-market arbitrage - -### Q3 2025: Institutional Features -- [ ] FIX protocol support -- [ ] Multi-account management -- [ ] Compliance reporting - -### Q4 2025: Platform as a Service -- [ ] White-label solution -- [ ] API for third-party strategies -- [ ] Mobile monitoring app - ---- - -## 💡 Investment Highlights - -### Why Invest Now? -1. **First Mover**: Early in prediction market automation -2. **Proven System**: 187% return in backtesting -3. **Scalable Tech**: Handles 500+ markets simultaneously -4. **Protected IP**: Proprietary correlation algorithms -5. **Growing Market**: 300% YoY growth in prediction markets - -### Use of Funds -``` -$2M Seed Round Allocation: -├── 40% - Engineering (ML/AI team expansion) -├── 25% - Infrastructure (servers, data feeds) -├── 20% - Compliance & Legal -├── 10% - Marketing & BD -└── 5% - Operations -``` - -### Expected Returns -``` -Conservative: 35% annual return -Base Case: 65% annual return -Optimistic: 120% annual return - -With 2% management + 20% performance fee: -Year 1 Revenue: $1.2M -Year 2 Revenue: $4.8M -Year 3 Revenue: $18M -``` - ---- - -## 🤝 Team & Advisors - -### Core Team -- **CTO**: 15 years quantitative trading -- **Head of AI**: Ex-DeepMind, PhD ML -- **Lead Engineer**: Ex-Jane Street -- **Risk Manager**: Ex-Citadel - -### Advisors -- Former Head of Trading, Two Sigma -- Professor of Statistics, MIT -- Early investor in Polymarket - ---- - -## 📞 Contact & Next Steps - -### Live Demo Available -See the system trade in real-time on actual markets - -### Documentation -- Technical Architecture: `/docs/ARCHITECTURE.md` -- API Documentation: `/docs/API.md` -- Risk Framework: `/docs/RISK_MANAGEMENT.md` - -### Investment Inquiries -Ready to discuss terms and provide deeper technical dive - ---- - -## ⚡ Quick Start Demo - -```bash -# Clone and setup -git clone [repository] -cd Kalshi_Agentic_Agent - -# Install dependencies -pip install -r requirements.txt - -# Configure credentials -cp .env.example .env -# Add your API keys - -# Run backtesting demo -python scripts/run_backtest.py --mode optimize - -# Start live trading (paper mode) -python scripts/run_agents.py all --paper-trading - -# View real-time dashboard -open http://localhost:8080/dashboard -``` - ---- - -*This platform represents the future of algorithmic trading in prediction markets - combining speed, intelligence, and scale to capture opportunities invisible to human traders.* \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d0b9fb18 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Kalshi Trading SDK + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/PLATFORM_ROADMAP.md b/PLATFORM_ROADMAP.md deleted file mode 100644 index 341d96c7..00000000 --- a/PLATFORM_ROADMAP.md +++ /dev/null @@ -1,556 +0,0 @@ -# 🚀 Neural Trading Platform - Development Roadmap - -## Executive Vision -Transform the Kalshi Trading Agent into a **universal algorithmic trading platform** where users can: -- Plug in any data source (websockets, APIs, databases) -- Deploy custom trading algorithms -- Backtest strategies across multiple markets -- Share and monetize successful strategies - ---- - -## 📊 Platform Evolution Phases - -### Phase 1: Core Infrastructure (Q1 2025) -**Goal:** Build extensible foundation for custom components - -#### 1.1 Plugin Architecture -```python -# Example: Custom Data Source Plugin -class CustomDataPlugin(BaseDataSource): - """User-defined data source""" - - async def connect(self): - """Connect to custom websocket/API""" - pass - - async def subscribe(self, symbols): - """Subscribe to data streams""" - pass - - def transform(self, raw_data): - """Transform to unified format""" - return UnifiedEvent(...) -``` - -#### 1.2 Strategy Framework -```python -# Example: Custom Trading Strategy -class UserStrategy(BaseStrategy): - """User-defined trading algorithm""" - - def analyze(self, market_data, indicators): - """Custom analysis logic""" - signal = self.calculate_signal(market_data) - return TradingSignal( - action="BUY", - confidence=0.85, - size=self.kelly_sizing(signal) - ) -``` - -#### 1.3 Unified Data Model -```yaml -# Standardized event schema -UnifiedMarketEvent: - timestamp: datetime - source: string - symbol: string - data: - price: float - volume: float - bid: float - ask: float - custom_fields: dict -``` - -**Deliverables:** -- [ ] Plugin system with hot-reload capability -- [ ] Strategy SDK with examples -- [ ] Data transformation pipeline -- [ ] Developer documentation - ---- - -### Phase 2: Developer Platform (Q2 2025) -**Goal:** Enable developers to build and test strategies - -#### 2.1 Visual Strategy Builder -``` -┌─────────────────────────────────────────┐ -│ STRATEGY BUILDER UI │ -├─────────────────────────────────────────┤ -│ [Data Sources] → [Indicators] → │ -│ ↓ ↓ │ -│ [Conditions] → [Actions] │ -│ ↓ ↓ │ -│ [Risk Rules] → [Backtest] │ -└─────────────────────────────────────────┘ -``` - -#### 2.2 Backtesting as a Service -```python -# API for custom backtesting -POST /api/backtest -{ - "strategy": "user_strategy_id", - "data_sources": ["kalshi", "custom_ws"], - "date_range": { - "start": "2024-01-01", - "end": "2024-12-31" - }, - "parameters": { - "stop_loss": 0.10, - "position_size": 0.05 - } -} - -# Response -{ - "sharpe_ratio": 2.87, - "total_return": 1.87, - "max_drawdown": -0.124, - "win_rate": 0.673, - "report_url": "https://platform.com/report/abc123" -} -``` - -#### 2.3 Strategy Marketplace -``` -┌─────────────────────────────────────────┐ -│ STRATEGY MARKETPLACE │ -├─────────────────────────────────────────┤ -│ Top Performers: │ -│ • MomentumPro ⭐4.8 +187% return │ -│ • MeanReversion ⭐4.6 +142% return │ -│ • EventArbitrage ⭐4.5 +98% return │ -│ │ -│ [Deploy] [Clone] [Analyze] │ -└─────────────────────────────────────────┘ -``` - -**Deliverables:** -- [ ] Web-based strategy builder -- [ ] REST API for backtesting -- [ ] Strategy marketplace MVP -- [ ] Performance analytics dashboard - ---- - -### Phase 3: Custom Data Integration (Q3 2025) -**Goal:** Support any data source and market - -#### 3.1 WebSocket Adapter Framework -```python -# Universal WebSocket adapter -class WebSocketAdapter: - def __init__(self, config): - self.url = config['url'] - self.auth = config['auth'] - self.parser = config['parser'] - - async def connect(self): - """Auto-configure from user spec""" - self.ws = await websockets.connect( - self.url, - extra_headers=self.auth - ) - - def parse_message(self, msg): - """User-defined parser or auto-detect""" - return self.parser(msg) -``` - -#### 3.2 Data Source Registry -```yaml -# User registers custom data source -data_sources: - - name: "crypto_exchange" - type: "websocket" - url: "wss://stream.exchange.com" - auth_type: "api_key" - message_format: "json" - mappings: - price: "$.last_price" - volume: "$.24h_volume" - - - name: "news_sentiment" - type: "rest_api" - url: "https://api.news.com/sentiment" - poll_interval: 60 - auth_type: "bearer_token" -``` - -#### 3.3 Multi-Exchange Support -```python -# Trade across multiple venues -class UniversalExecutor: - exchanges = { - 'kalshi': KalshiClient(), - 'polymarket': PolymarketClient(), - 'manifold': ManifoldClient(), - 'custom': UserExchangeClient() - } - - async def execute_best(self, order): - """Route to best execution venue""" - best_price = await self.find_best_price(order) - return await self.exchanges[best_price.venue].execute(order) -``` - -**Deliverables:** -- [ ] WebSocket adapter generator -- [ ] REST API adapter -- [ ] Database connectors (PostgreSQL, MongoDB) -- [ ] Multi-exchange execution router - ---- - -### Phase 4: Algorithm Marketplace (Q4 2025) -**Goal:** Create ecosystem for algorithm sharing and monetization - -#### 4.1 Algorithm Store -``` -┌─────────────────────────────────────────┐ -│ ALGORITHM STORE │ -├─────────────────────────────────────────┤ -│ Categories: │ -│ • Mean Reversion (45 algos) │ -│ • Momentum (82 algos) │ -│ • Arbitrage (31 algos) │ -│ • ML-Based (67 algos) │ -│ │ -│ Revenue Models: │ -│ • One-time purchase │ -│ • Subscription │ -│ • Profit sharing │ -└─────────────────────────────────────────┘ -``` - -#### 4.2 Strategy Templates -```python -# Pre-built templates users can customize -templates = { - 'pairs_trading': PairsTradingTemplate(), - 'momentum': MomentumTemplate(), - 'mean_reversion': MeanReversionTemplate(), - 'ml_prediction': MLPredictionTemplate(), - 'event_driven': EventDrivenTemplate() -} - -# User customizes template -my_strategy = templates['momentum'].customize( - indicators=['RSI', 'MACD'], - entry_conditions={'RSI': '<30'}, - exit_conditions={'profit': '>5%'} -) -``` - -#### 4.3 Performance Verification -```python -# Verified performance tracking -class PerformanceVerifier: - """Cryptographically verify algorithm performance""" - - def verify_backtest(self, strategy_id): - """Independent backtest verification""" - return { - 'verified': True, - 'hash': 'abc123...', - 'performance': {...}, - 'certificate_url': '...' - } - - def track_live(self, strategy_id): - """Real-time performance tracking""" - return LivePerformanceTracker(strategy_id) -``` - -**Deliverables:** -- [ ] Algorithm marketplace platform -- [ ] Revenue sharing system -- [ ] Performance verification service -- [ ] Copy-trading functionality - ---- - -## 🛠️ Technical Implementation - -### Core Architecture Extensions - -#### 1. Plugin System Architecture -``` -┌─────────────────────────────────────────┐ -│ PLUGIN MANAGER │ -├─────────────────────────────────────────┤ -│ │ -│ Plugin Types: │ -│ ┌──────────┐ ┌──────────┐ │ -│ │ Data │ │ Strategy │ │ -│ │ Source │ │ Plugin │ │ -│ └────┬─────┘ └────┬─────┘ │ -│ │ │ │ -│ ┌────▼──────────────▼────┐ │ -│ │ Plugin Runtime │ │ -│ │ • Sandboxing │ │ -│ │ • Resource limits │ │ -│ │ • API access control │ │ -│ └─────────────────────────┘ │ -│ │ -└─────────────────────────────────────────┘ -``` - -#### 2. Custom Algorithm API -```python -# Base classes for user extensions -class CustomDataSource(ABC): - @abstractmethod - async def connect(self): pass - - @abstractmethod - async def subscribe(self, symbols): pass - - @abstractmethod - def transform(self, data): pass - -class CustomStrategy(ABC): - @abstractmethod - def analyze(self, data): pass - - @abstractmethod - def generate_signal(self, analysis): pass - - @abstractmethod - def calculate_position_size(self, signal): pass - -class CustomIndicator(ABC): - @abstractmethod - def calculate(self, data): pass - - @abstractmethod - def get_signal(self): pass -``` - -#### 3. Backtest Engine Extensions -```python -class UniversalBacktestEngine: - """Extended backtesting for any data/strategy""" - - def __init__(self): - self.data_sources = DataSourceRegistry() - self.strategies = StrategyRegistry() - self.validators = ValidationPipeline() - - async def backtest(self, config): - # Load custom data source - data = await self.data_sources.load( - config['data_source'], - config['date_range'] - ) - - # Load custom strategy - strategy = self.strategies.load( - config['strategy'], - config['parameters'] - ) - - # Run backtest with custom components - results = await self.run_simulation( - data, - strategy, - config['capital'] - ) - - return self.generate_report(results) -``` - -#### 4. SDK and CLI Tools -```bash -# Neural Trading Platform CLI -ntp create strategy momentum_breakout -ntp backtest --strategy momentum_breakout --data kalshi --from 2024-01-01 -ntp deploy momentum_breakout --capital 10000 --risk-limit 0.20 -ntp monitor momentum_breakout --dashboard - -# SDK usage -from ntp import Strategy, DataSource, Backtest - -class MyStrategy(Strategy): - def analyze(self, data): - # Custom logic - return signal - -# Register and backtest -strategy = MyStrategy() -backtest = Backtest(strategy, data_source="kalshi") -results = backtest.run(start="2024-01-01", end="2024-12-31") -``` - ---- - -## 📈 Monetization Strategy - -### Revenue Streams - -#### 1. Platform Fees -- **Basic:** Free tier with limited backtests -- **Pro:** $99/month unlimited backtesting -- **Enterprise:** $999/month with priority execution - -#### 2. Marketplace Commission -- 30% commission on algorithm sales -- 20% on subscription revenues -- 15% on profit-sharing arrangements - -#### 3. Data Services -- Premium data feeds -- Historical data packages -- Real-time data API access - -#### 4. Managed Services -- White-label platform -- Custom strategy development -- Institutional deployment - ---- - -## 🎯 Success Metrics - -### Year 1 Goals -- 1,000+ registered developers -- 100+ custom strategies deployed -- 50+ data sources integrated -- $1M+ in platform transactions - -### Year 2 Goals -- 10,000+ active users -- 1,000+ algorithms in marketplace -- 500+ data sources -- $10M+ in platform transactions - -### Year 3 Goals -- 50,000+ users -- 5,000+ algorithms -- Institutional adoption -- $100M+ in platform transactions - ---- - -## 🔧 Implementation Timeline - -### Q1 2025: Foundation -- [ ] Week 1-4: Plugin architecture design -- [ ] Week 5-8: Strategy SDK development -- [ ] Week 9-12: Testing and documentation - -### Q2 2025: Developer Tools -- [ ] Week 1-4: Visual builder UI -- [ ] Week 5-8: Backtesting API -- [ ] Week 9-12: Marketplace MVP - -### Q3 2025: Data Integration -- [ ] Week 1-4: WebSocket framework -- [ ] Week 5-8: Multi-exchange support -- [ ] Week 9-12: Testing and optimization - -### Q4 2025: Marketplace Launch -- [ ] Week 1-4: Algorithm store -- [ ] Week 5-8: Performance verification -- [ ] Week 9-12: Marketing and launch - ---- - -## 🚀 Quick Wins (Next 30 Days) - -### 1. Create Plugin Interface -```python -# Simple plugin interface to start -class PluginInterface: - def initialize(self, config): pass - def process(self, data): pass - def cleanup(self): pass -``` - -### 2. Add Custom Strategy Support -```python -# Allow users to drop in Python files -strategies/ - ├── user_strategy_1.py - ├── user_strategy_2.py - └── user_strategy_3.py -``` - -### 3. Extend Backtest Engine -```python -# Support custom data formats -backtest.add_data_source( - CSVDataSource("historical_data.csv") -) -``` - -### 4. Create Developer Docs -- Getting started guide -- API reference -- Example strategies -- Video tutorials - ---- - -## 🎨 Platform Features Comparison - -| Feature | Current | Phase 1 | Phase 2 | Phase 3 | Phase 4 | -|---------|---------|---------|---------|---------|---------| -| Data Sources | Kalshi, ESPN, Twitter | +5 sources | +20 sources | Any WebSocket/API | Unlimited | -| Custom Strategies | No | Yes (code) | Yes (visual) | Templates | Marketplace | -| Backtesting | Built-in | API access | Cloud-based | Distributed | Verified | -| Markets | Kalshi only | 3 exchanges | 10 exchanges | Any exchange | Universal | -| Users | Single | Team | Organization | Public | Ecosystem | -| Revenue Model | Trading | SaaS | Platform fees | Marketplace | Full ecosystem | - ---- - -## 🌟 Competitive Advantages - -### Why This Platform Will Win - -1. **Open Architecture** - - Unlike QuantConnect: Not locked to specific brokers - - Unlike TradingView: Full algorithmic capabilities - - Unlike MT4/MT5: Modern tech stack and AI - -2. **Network Effects** - - More strategies → More users - - More users → More data sources - - More data → Better strategies - -3. **Developer-First** - - Excellent documentation - - Simple SDK - - Active community - - Revenue sharing - -4. **Technology Edge** - - Sub-100ms latency - - Distributed backtesting - - AI/ML integration - - Cloud-native architecture - ---- - -## 📞 Next Steps - -### Immediate Actions -1. **Technical Design**: Finalize plugin architecture -2. **Community Building**: Launch developer forum -3. **Partnerships**: Connect with data providers -4. **Funding**: Raise Series A for platform development - -### Contact for Collaboration -- **Developers**: Join our beta program -- **Data Providers**: Partner with us -- **Investors**: Fund the future of algo trading -- **Traders**: Test early access features - ---- - -*The Neural Trading Platform will democratize algorithmic trading by providing institutional-grade infrastructure to every developer and trader.* \ No newline at end of file diff --git a/README.md b/README.md index b2362cda..17a2c1c2 100644 --- a/README.md +++ b/README.md @@ -1,332 +1,249 @@ -# 🏈 Neural Trading Platform - Autonomous Sports Event Trading +# 🧠 Neural SDK -> **Real-time algorithmic trading system for Kalshi sports prediction markets** -> Monitors multiple data sources • Detects market inefficiencies • Executes trades in <3 seconds +> **Open-source Python SDK for algorithmic prediction market trading** +> Build sophisticated trading strategies with real-time data streaming and comprehensive backtesting - -## 🎯 What This Does - -This platform automatically trades sports prediction markets on Kalshi by detecting and exploiting information asymmetries faster than human traders. - -### Real Example -``` -18:35:22 - ESPN: "Touchdown Chiefs! Mahomes 45-yard pass" -18:35:22 - Platform detects event (100ms) -18:35:22 - Checks Kalshi price: still at 0.65 (hasn't moved) -18:35:23 - Places BUY order: 1,923 shares @ 0.65 -18:35:24 - Order filled -18:35:37 - Kalshi price moves to 0.70 -18:35:37 - Profit: +7.7% in 15 seconds -``` +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) +[![PyPI version](https://badge.fury.io/py/neural-sdk.svg)](https://badge.fury.io/py/neural-sdk) ## 🚀 Quick Start -### Prerequisites -- Python 3.10+ -- Redis server -- Kalshi account (for live trading) -- Free API keys for data sources +### Installation -### 1. Clone & Setup ```bash -git clone https://github.com/IntelIP/Neural-Trading-Platform.git -cd Neural-Trading-Platform -pip install -r requirements.txt -``` +pip install neural-sdk -### 2. Configure APIs -Create `.env` file: -```bash -# Trading (required for live trading) -KALSHI_API_KEY_ID=your_key_id -KALSHI_API_KEY=your_api_key - -# Data Sources (weather API included free) -OPENWEATHER_API_KEY=78596505b0f5fea89e98ebcbf3bd6e21 # Working key -REDDIT_CLIENT_ID=your_reddit_id # Optional -REDDIT_CLIENT_SECRET=your_reddit_secret # Optional +# Or install from GitHub +pip install git+https://github.com/IntelIP/kalshi.git@feat/synthetic-training-integration ``` -### 3. Start Redis -```bash -# macOS -brew install redis && brew services start redis +### Basic Usage -# Linux -sudo apt-get install redis-server && sudo systemctl start redis +```python +from neural_sdk import NeuralSDK -# Verify -redis-cli ping # Should return PONG -``` +# Initialize SDK +sdk = NeuralSDK.from_env() -### 4. Test the System -```bash -# Test weather monitoring (working API included) -python scripts/quick_weather_demo.py +# Create a simple strategy +@sdk.strategy +async def momentum_strategy(market_data): + for symbol, price in market_data.prices.items(): + if 'NFL' in symbol and price < 0.3: + return sdk.create_signal('BUY', symbol, size=100) -# Full SDK demo -python scripts/demo_sdk.py +# Start trading +await sdk.start_trading() +``` -# Paper trading mode -python scripts/run_agents.py --paper-trading +### Backtesting + +```python +from neural_sdk.backtesting import BacktestEngine, BacktestConfig + +# Configure backtest +config = BacktestConfig( + start_date="2024-01-01", + end_date="2024-12-31", + initial_capital=10000 +) + +# Run backtest +engine = BacktestEngine(config) +engine.add_strategy(my_strategy) +engine.load_data("csv", path="historical_data.csv") + +results = engine.run() +print(f"Total Return: {results.total_return:.2f}%") +print(f"Sharpe Ratio: {results.metrics['sharpe_ratio']:.3f}") ``` -## 🏗️ How It Works +## 🏗️ Architecture ```mermaid graph LR A[Data Sources] --> B[SDK Adapters] - B --> C[Redis Pub/Sub] - C --> D[Trading Agents] - D --> E[Kalshi API] + B --> C[Strategy Engine] + C --> D[Kalshi API] - A1[DraftKings
Odds] --> B - A2[Weather
Conditions] --> B - A3[Reddit
Sentiment] --> B - A4[ESPN
GameCast] --> B + A1[Weather API] --> B + A2[Reddit Sentiment] --> B + A3[Sports APIs] --> B - D1[DataCoordinator] --> D - D2[StrategyAnalyst] --> D - D3[TradeExecutor] --> D - D4[RiskManager] --> D + E[Backtesting Engine] --> F[Performance Analytics] ``` -### Data Flow Timeline -| Step | Component | Latency | Action | -|------|-----------|---------|--------| -| 1 | ESPN GameCast | 0ms | "Touchdown!" event detected | -| 2 | SDK Adapter | +100ms | Converts to StandardizedEvent | -| 3 | Redis Pub/Sub | +10ms | Distributes to subscribers | -| 4 | DataCoordinator | +200ms | Correlates with Kalshi price | -| 5 | StrategyAnalyst | +300ms | Calculates expected move | -| 6 | TradeExecutor | +200ms | Places order via API | -| **Total** | **End-to-end** | **810ms** | **Sub-second execution** | +## 📊 Features -## 📊 Data Source SDK +### Core Trading +- **Real-time market data** streaming from multiple sources +- **Strategy framework** with easy-to-use decorators +- **Risk management** with position limits and stop-losses +- **Order execution** with realistic slippage simulation -The platform's edge comes from its modular Data Source SDK that makes adding new data sources trivial: +### Backtesting +- **Event-driven engine** for accurate historical testing +- **Multiple data sources**: CSV, Parquet, SQL, S3 +- **Performance metrics**: Sharpe ratio, drawdown, win rate +- **Portfolio simulation** with realistic costs and slippage -### Currently Implemented +### Data Integration +- **Plugin architecture** for custom data sources +- **Built-in adapters** for weather, Reddit, sports APIs +- **Data caching** for improved performance +- **Format standardization** across all sources -| Source | Status | Latency | Purpose | API Required | -|--------|--------|---------|---------|--------------| -| **Weather** | ✅ Working | 2s | Wind/rain affects scoring | Included (free) | -| **DraftKings** | ✅ Ready | 500ms | Sharp money movements | None (public) | -| **Reddit** | ⚙️ Configured | 2-5s | Sentiment extremes | Yes (free) | -| **ESPN** | 🔄 Planned | 1-2s | Official game events | Development | +## 🛠️ Data Sources -### Add Your Own Source (50 lines) -```python -from src.sdk import DataSourceAdapter, StandardizedEvent, EventType +| Source | Status | Purpose | Setup | +|--------|--------|---------|-------| +| **Weather API** | ✅ Ready | Weather impacts on sports | Free API key | +| **Reddit** | ✅ Ready | Sentiment analysis | Free API credentials | +| **Custom CSV** | ✅ Ready | Historical data | Local files | +| **Parquet** | ✅ Ready | High-performance data | S3 or local | +| **PostgreSQL** | ⚙️ Optional | Large datasets | Database connection | -class MyAdapter(DataSourceAdapter): - async def connect(self): - self.client = YourAPIClient(self.config['api_key']) - return await self.client.connect() - - async def stream(self): - while self.is_connected: - data = await self.fetch_data() - - # Detect trading opportunity - if self.is_significant(data): - yield StandardizedEvent( - source="MySource", - event_type=EventType.CUSTOM, - data=data, - confidence=0.85, - impact="high" - ) - - await asyncio.sleep(self.config['interval']) +## 📈 Strategy Examples + +### Momentum Strategy +```python +def momentum_strategy(market_data): + \"\"\"Buy markets trending upward below 60 cents\"\"\" + for symbol, price in market_data.prices.items(): + if price < 0.6 and market_data.is_trending_up(symbol): + return {'action': 'BUY', 'symbol': symbol, 'size': 50} ``` -## 🤖 Trading Intelligence +### Mean Reversion +```python +def mean_reversion_strategy(market_data): + \"\"\"Buy undervalued markets, sell overvalued\"\"\" + for symbol, price in market_data.prices.items(): + if price < 0.3: # Undervalued + return {'action': 'BUY', 'symbol': symbol, 'size': 100} + elif price > 0.7: # Overvalued + return {'action': 'SELL', 'symbol': symbol, 'size': 100} +``` -### Signal Generation -The platform correlates events across sources to identify opportunities: +## 🔧 Configuration -```python -# Example: Weather + Odds Correlation -if weather.wind_speed > 20 and not draftkings.total_moved: - signal = Signal( - action="BET_UNDER", - confidence=0.75, - edge=0.04, # 4% expected value - reason="High wind not priced into total" - ) +### Environment Setup +```bash +# Create .env file +NEURAL_API_KEY_ID=your_neural_api_key +NEURAL_PRIVATE_KEY_FILE=./keys/neural_private.key + +# Optional: Data source APIs +OPENWEATHER_API_KEY=your_weather_key +REDDIT_CLIENT_ID=your_reddit_id +REDDIT_CLIENT_SECRET=your_reddit_secret ``` -### Position Sizing (Kelly Criterion) +### SDK Configuration ```python -# Never use full Kelly - too risky -position = kelly_fraction * 0.25 # 25% of Kelly -position = min(position, 0.05 * capital) # Max 5% per trade +from neural_sdk import SDKConfig + +config = SDKConfig( + max_position_size=0.05, # Max 5% per position + daily_loss_limit=0.20, # Stop at 20% daily loss + commission=0.02, # Kalshi's 2% fee + slippage=0.01 # 1% estimated slippage +) + +sdk = KalshiSDK(config) ``` -### Risk Management -- **Stop Loss**: -5% automatic exit -- **Daily Limit**: -20% circuit breaker -- **Correlation Check**: Reduce correlated positions -- **Max Positions**: 10 concurrent trades +## 📚 Documentation -## 📈 Performance Metrics +| Document | Description | +|----------|-------------| +| [**Getting Started**](docs/getting_started.md) | Installation and first strategy | +| [**API Reference**](docs/api_reference.md) | Complete SDK documentation | +| [**Backtesting Guide**](docs/backtesting.md) | Historical testing framework | +| [**Data Sources**](docs/data_sources.md) | Setting up data feeds | +| [**Strategy Development**](docs/strategies.md) | Building trading algorithms | -### Backtested Results (30 days) -| Metric | Value | Target | -|--------|-------|--------| -| **Win Rate** | 67.3% | >65% | -| **Sharpe Ratio** | 2.14 | >2.0 | -| **Avg Return/Trade** | +1.8% | >1.5% | -| **Max Drawdown** | -18.2% | <20% | -| **Trades/Day** | 12 | 10-20 | +## 🧪 Testing -### Live Performance Tracking ```bash -# Monitor real-time performance -python scripts/monitor_performance.py +# Run all tests +pytest tests/ -# Run backtest on strategy -python scripts/run_backtest.py --strategy sharp_money --days 30 -``` +# Run specific test suite +pytest tests/unit/test_backtesting.py -## 🎮 Configuration Examples - -### Monitor Specific Game -```yaml -# config/game_config.yaml -game_monitoring: - mode: "single_game" - game: - sport: "NFL" - home_team: "Kansas City Chiefs" - away_team: "Buffalo Bills" - - kalshi_markets: - - "NFL-KC-BUF-WINNER" - - "NFL-KC-BUF-TOTAL" +# Run with coverage +pytest --cov=kalshi_trading_sdk tests/ ``` -### Weather Impact Settings -```yaml -# config/data_sources.yaml -weather: - enabled: true - thresholds: - wind_speed: 15 # mph - affects passing - precipitation: 0.1 # in/hr - affects scoring -``` +## 📊 Performance Metrics -## 🧪 Testing +The SDK calculates comprehensive metrics for strategy evaluation: -```bash -# Unit tests -pytest tests/unit/ +- **Return Metrics**: Total return, CAGR, Sharpe ratio +- **Risk Metrics**: Max drawdown, volatility, VaR +- **Trade Analysis**: Win rate, average win/loss, profit factor +- **Time Analysis**: Best/worst days, consecutive wins/losses -# Integration tests -pytest tests/integration/ +## 🌟 Example Results -# Test specific adapter -python scripts/test_weather_adapter.py - -# Load testing -python tests/load/stress_test.py +``` +BACKTEST RESULTS +================ +Period: 2024-01-01 to 2024-12-31 +Initial Capital: $10,000.00 +Final Value: $12,750.00 + +Total Return: +27.5% +Sharpe Ratio: 1.85 +Max Drawdown: -8.2% +Win Rate: 68.5% +Total Trades: 156 ``` -## 📚 Documentation +## 🤝 Contributing -| Document | Description | -|----------|-------------| -| [Getting Started](docs/GETTING_STARTED.md) | Installation and first trade | -| [System Overview](docs/SYSTEM_OVERVIEW.md) | Architecture deep dive | -| [SDK Documentation](docs/SDK_DOCUMENTATION.md) | Build custom adapters | -| [Trading Logic](docs/TRADING_LOGIC.md) | How decisions are made | -| [Data Sources Guide](docs/DATA_SOURCES_GUIDE.md) | Configure each source | - -## 🚦 Project Status - -### ✅ Completed -- Data Source SDK framework -- Weather monitoring (OpenWeatherMap API) -- DraftKings odds adapter -- Reddit sentiment adapter -- Redis pub/sub message bus -- Kelly Criterion position sizing -- Risk management system -- Comprehensive documentation - -### 🔄 In Development -- ESPN GameCast integration -- Machine learning signal enhancement -- Multi-sport expansion (NBA, MLB) - -### 📅 Roadmap -- Q4 2024: Production deployment -- Q1 2025: ML model integration -- Q2 2025: Mobile monitoring app - -## 🛠️ Tech Stack - -- **Python 3.10+** - Async/await for speed -- **Redis** - 10,000 msg/sec pub/sub -- **Agentuity** - Agent orchestration -- **aiohttp** - Async HTTP client -- **asyncpraw** - Reddit streaming -- **pandas/numpy** - Data analysis -- **TextBlob** - Sentiment analysis - -## 📊 System Requirements - -### Minimum (Development) -- 2 CPU cores -- 4 GB RAM -- 10 GB storage -- 10 Mbps internet - -### Recommended (Production) -- 4+ CPU cores -- 8 GB RAM -- 50 GB SSD -- 100 Mbps internet -- Redis dedicated instance +We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. -## 🤝 Contributing +### Development Setup +```bash +git clone https://github.com/neural/neural-sdk.git +cd neural-sdk -We welcome contributions! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. +# Install development dependencies +pip install -e ".[dev]" -### Areas Needing Help -- ESPN WebSocket integration -- Additional sports adapters -- ML model development -- Performance optimization +# Run pre-commit hooks +pre-commit install +``` -## 📝 License +## 📄 License -MIT License - see [LICENSE](LICENSE) file +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. ## ⚠️ Disclaimer -**IMPORTANT**: This software is for educational purposes. Sports event trading involves significant risk of loss. +**Important**: This software is for educational and research purposes. Trading involves substantial risk of loss. - Always start with paper trading -- Never risk more than you can afford to lose +- Never risk more than you can afford to lose - Past performance doesn't guarantee future results -- The platform is not financial advice +- The SDK is not financial advice -## 🏆 Acknowledgments +## 🔗 Links -- Kalshi for providing the trading platform -- OpenWeatherMap for weather data (API key included) -- Reddit community for sentiment data -- DraftKings for odds reference data +- **Documentation**: [https://kalshi-trading-sdk.readthedocs.io/](https://kalshi-trading-sdk.readthedocs.io/) +- **PyPI Package**: [https://pypi.org/project/kalshi-trading-sdk/](https://pypi.org/project/kalshi-trading-sdk/) +- **Issues**: [https://github.com/kalshi/kalshi-trading-sdk/issues](https://github.com/kalshi/kalshi-trading-sdk/issues) +- **Discussions**: [https://github.com/kalshi/kalshi-trading-sdk/discussions](https://github.com/kalshi/kalshi-trading-sdk/discussions) -## 📞 Support +## 🏆 Acknowledgments -- 📖 [Documentation](docs/) -- 💬 [GitHub Issues](https://github.com/IntelIP/Neural-Trading-Platform/issues) -- 📧 Contact: [your-email] +- Kalshi for providing the trading platform +- Contributors and beta testers +- Open source community for excellent libraries --- -**Built for algorithmic trading** -*Trade responsibly. Speed matters. Edge wins.* \ No newline at end of file +**Built for algorithmic trading • Trade responsibly • Performance matters** \ No newline at end of file diff --git a/config/data_sources.yaml b/config/data_sources.yaml index 04f8a009..1dd43fcd 100644 --- a/config/data_sources.yaml +++ b/config/data_sources.yaml @@ -6,7 +6,7 @@ sources: - name: draftkings enabled: true class: DraftKingsAdapter - module: src.sdk.adapters.draftkings + module: sdk.adapters.draftkings priority: 1 # Higher priority = processed first config: sports: @@ -20,7 +20,7 @@ sources: - name: reddit enabled: false # Enable after adding credentials class: RedditAdapter - module: src.sdk.adapters.reddit + module: sdk.adapters.reddit priority: 2 config: client_id: ${REDDIT_CLIENT_ID} @@ -37,14 +37,30 @@ sources: injury: ["injury", "injured", "hurt", "down"] big_play: ["holy shit", "wow", "incredible"] + # ESPN Play-by-Play Data + - name: espn + enabled: true # No API key needed - completely free! + class: ESPNAdapter + module: sdk.adapters.espn + priority: 1 # High priority for real-time game events + config: + sports: + - nfl + - nba + update_interval: 5 # Base polling interval in seconds + critical_interval: 3 # Interval during critical game moments + # Optional: Monitor specific games by ID + # games: + # - "401547652" + # Weather Conditions - name: weather enabled: true # API key is working! class: WeatherAdapter - module: src.sdk.adapters.weather + module: sdk.adapters.weather priority: 3 config: - api_key: 78596505b0f5fea89e98ebcbf3bd6e21 # Your working API key + api_key: ${OPENWEATHER_API_KEY} update_interval: 300 # 5 minutes thresholds: wind_speed: 15 # mph @@ -73,4 +89,4 @@ settings: event_queue_size: 10000 max_adapters: 10 health_check_interval: 60 # seconds - log_level: INFO \ No newline at end of file + log_level: INFO diff --git a/config/environments/development.yaml b/config/environments/development.yaml new file mode 100644 index 00000000..ed2d5640 --- /dev/null +++ b/config/environments/development.yaml @@ -0,0 +1,87 @@ +# Development Environment Configuration +# For local development and testing + +name: "Development" +environment: "development" +redis_db: 4 +redis_prefix: "dev:" + +api_endpoints: + kalshi: "http://localhost:8000/mock/kalshi" + odds: "http://localhost:8001/mock/odds" + sportsdata: "http://localhost:8002/mock/sportsdata" + twitter: "http://localhost:8003/mock/twitter" + +rate_limits: + kalshi: 100000 + odds: 100000 + sportsdata: 100000 + twitter: 100000 + +# Safety and Security +safety_checks: false +require_confirmation: false +require_mfa: false + +# Allowed Operations +allowed_operations: + - "*" # All operations allowed in development + +# Restricted Operations +restricted_operations: [] # No restrictions in development + +# Data Management +data_retention_hours: 1 # Minimal retention +backup_enabled: false +backup_frequency_hours: 0 + +# Trading Limits (No limits for development) +max_position_size: 1.0 +max_daily_trades: 100000 +max_concurrent_positions: 10000 +max_order_value: 10000000.0 +min_order_value: 0.001 + +# Risk Management (Disabled for development) +stop_loss_percentage: 1.0 +max_drawdown_percentage: 1.0 +position_sizing_method: "fixed" +kelly_fraction: 1.0 + +# Logging and Monitoring +enable_logging: true +log_level: "DEBUG" +telemetry_enabled: false +alert_enabled: false +alert_channels: [] + +# Feature Flags +features: + auto_trading: true + paper_trading: true + backtesting: true + synthetic_data: true + debug_mode: true + performance_profiling: true + real_time_analytics: false + emergency_stop: false + development_mode: true + hot_reload: true + mock_apis: true + verbose_logging: true + +# Development Specific Settings +development: + auto_reload: true + debug_toolbar: true + sql_echo: true + mock_delay_ms: 0 + bypass_cache: true + show_stack_traces: true + +# Monitoring Thresholds (Disabled for development) +monitoring: + latency_threshold_ms: 10000 + error_rate_threshold: 1.0 + memory_threshold_gb: 16 + cpu_threshold_percent: 100 \ No newline at end of file diff --git a/config/environments/production.yaml b/config/environments/production.yaml new file mode 100644 index 00000000..d2d9bf97 --- /dev/null +++ b/config/environments/production.yaml @@ -0,0 +1,86 @@ +# Production Environment Configuration +# CRITICAL: This configuration is for LIVE TRADING with REAL MONEY + +name: "Production" +environment: "production" +redis_db: 0 +redis_prefix: "prod:" + +api_endpoints: + kalshi: "https://api.kalshi.com" + odds: "https://api.theoddsapi.com/v4" + sportsdata: "https://api.sportsdata.io/v3/nfl" + twitter: "https://api.twitter.com/2" + +rate_limits: + kalshi: 100 + odds: 50 + sportsdata: 100 + twitter: 300 + +# Safety and Security +safety_checks: true +require_confirmation: true +require_mfa: true + +# Allowed Operations +allowed_operations: + - "place_order" + - "cancel_order" + - "get_positions" + - "get_markets" + - "get_account" + - "analyze_market" + - "calculate_kelly" + +# Restricted Operations +restricted_operations: + - "delete_all_data" + - "reset_account" + - "modify_system_config" + - "bypass_risk_checks" + +# Data Management +data_retention_hours: 720 # 30 days +backup_enabled: true +backup_frequency_hours: 24 + +# Trading Limits +max_position_size: 0.05 # 5% of portfolio +max_daily_trades: 20 +max_concurrent_positions: 10 +max_order_value: 1000.0 +min_order_value: 1.0 + +# Risk Management +stop_loss_percentage: 0.10 +max_drawdown_percentage: 0.20 +position_sizing_method: "kelly_fraction" +kelly_fraction: 0.25 + +# Logging and Monitoring +enable_logging: true +log_level: "INFO" +telemetry_enabled: true +alert_enabled: true +alert_channels: + - "email" + - "slack" + +# Feature Flags +features: + auto_trading: true + paper_trading: false + backtesting: false + synthetic_data: false + debug_mode: false + performance_profiling: true + real_time_analytics: true + emergency_stop: true + +# Monitoring Thresholds +monitoring: + latency_threshold_ms: 1000 + error_rate_threshold: 0.01 + memory_threshold_gb: 4 + cpu_threshold_percent: 80 \ No newline at end of file diff --git a/config/environments/sandbox.yaml b/config/environments/sandbox.yaml new file mode 100644 index 00000000..9f44e836 --- /dev/null +++ b/config/environments/sandbox.yaml @@ -0,0 +1,86 @@ +# Sandbox Environment Configuration +# For testing with demo APIs and safe experimentation + +name: "Sandbox" +environment: "sandbox" +redis_db: 2 +redis_prefix: "sandbox:" + +api_endpoints: + kalshi: "https://demo-api.kalshi.co" + odds: "https://api.theoddsapi.com/v4" + sportsdata: "https://api.sportsdata.io/v3/nfl" + twitter: "https://api.twitter.com/2" + +rate_limits: + kalshi: 500 + odds: 200 + sportsdata: 200 + twitter: 500 + +# Safety and Security +safety_checks: true +require_confirmation: false +require_mfa: false + +# Allowed Operations +allowed_operations: + - "*" # All operations allowed in sandbox + +# Restricted Operations +restricted_operations: + - "delete_production_data" + - "modify_production_config" + +# Data Management +data_retention_hours: 24 # 1 day retention +backup_enabled: false +backup_frequency_hours: 0 + +# Trading Limits (Moderate for sandbox) +max_position_size: 0.20 # 20% of portfolio +max_daily_trades: 100 +max_concurrent_positions: 50 +max_order_value: 10000.0 +min_order_value: 0.10 + +# Risk Management +stop_loss_percentage: 0.20 +max_drawdown_percentage: 0.40 +position_sizing_method: "kelly_fraction" +kelly_fraction: 0.50 + +# Logging and Monitoring +enable_logging: true +log_level: "DEBUG" +telemetry_enabled: false +alert_enabled: false +alert_channels: [] + +# Feature Flags +features: + auto_trading: true + paper_trading: true + backtesting: true + synthetic_data: false + debug_mode: true + performance_profiling: true + real_time_analytics: true + emergency_stop: true + sandbox_mode: true + api_mocking: true + +# Sandbox Specific Settings +sandbox: + reset_daily: true + demo_balance: 10000.0 + unlimited_retries: true + skip_authentication: false + mock_latency_ms: 100 + +# Monitoring Thresholds +monitoring: + latency_threshold_ms: 2000 + error_rate_threshold: 0.05 + memory_threshold_gb: 4 + cpu_threshold_percent: 90 \ No newline at end of file diff --git a/config/environments/training.yaml b/config/environments/training.yaml new file mode 100644 index 00000000..8842409b --- /dev/null +++ b/config/environments/training.yaml @@ -0,0 +1,96 @@ +# Training Environment Configuration +# For agent training with synthetic data + +name: "Training" +environment: "training" +redis_db: 3 +redis_prefix: "training:" + +api_endpoints: + kalshi: "http://localhost:8000/mock/kalshi" + odds: "http://localhost:8001/mock/odds" + sportsdata: "http://localhost:8002/mock/sportsdata" + twitter: "http://localhost:8003/mock/twitter" + +rate_limits: + kalshi: 10000 + odds: 10000 + sportsdata: 10000 + twitter: 10000 + +# Safety and Security +safety_checks: false +require_confirmation: false +require_mfa: false + +# Allowed Operations +allowed_operations: + - "*" # All operations allowed in training + +# Restricted Operations +restricted_operations: [] # No restrictions in training + +# Data Management +data_retention_hours: 4 # Short retention for training data +backup_enabled: false +backup_frequency_hours: 0 + +# Trading Limits (Relaxed for training) +max_position_size: 1.0 # 100% allowed for training +max_daily_trades: 10000 +max_concurrent_positions: 1000 +max_order_value: 1000000.0 +min_order_value: 0.01 + +# Risk Management (Configurable for training) +stop_loss_percentage: 0.50 +max_drawdown_percentage: 1.0 +position_sizing_method: "kelly_fraction" +kelly_fraction: 1.0 # Full Kelly for training experiments + +# Logging and Monitoring +enable_logging: true +log_level: "DEBUG" +telemetry_enabled: false +alert_enabled: false +alert_channels: [] + +# Feature Flags +features: + auto_trading: true + paper_trading: true + backtesting: true + synthetic_data: true + debug_mode: true + performance_profiling: true + real_time_analytics: false + emergency_stop: false + training_mode: true + exploration_mode: true + replay_mode: true + +# Training Specific Settings +training: + scenario_generation: true + adaptive_difficulty: true + performance_tracking: true + decision_replay: true + confidence_calibration: true + memory_system: true + batch_size: 32 + learning_rate: 0.001 + +# Synthetic Data Settings +synthetic_data: + generation_enabled: true + realistic_timing: true + market_reaction_simulation: true + price_impact_modeling: true + volatility_injection: true + +# Monitoring Thresholds (Relaxed for training) +monitoring: + latency_threshold_ms: 5000 + error_rate_threshold: 0.10 + memory_threshold_gb: 8 + cpu_threshold_percent: 95 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..3a63b70b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3.9" + +services: + redis: + image: redis:7-alpine + container_name: neural_redis + ports: + - "6379:6379" + volumes: + - redis_data:/data + command: ["redis-server", "--appendonly", "yes"] + + app: + build: . + depends_on: + - redis + env_file: + - .env + command: ["python", "-c", "import neural_sdk; print('Neural SDK ready with env')"] + +volumes: + redis_data: + diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6d8a9e10..5ad19a7a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,7 +1,7 @@ -# Kalshi Trading Agent Architecture +# Neural Trading Agent Architecture ## Overview -A practical multi-agent system for automated sports betting on Kalshi, with clear separation between always-on monitoring agents and on-demand analysis agents. +A practical multi-agent system for automated sports betting on Neural, with clear separation between always-on monitoring agents and on-demand analysis agents. ## Agent Types @@ -13,7 +13,7 @@ A practical multi-agent system for automated sports betting on Kalshi, with clea **Responsibilities:** - Monitor ESPN for live games and scores - Track Twitter sentiment in real-time -- Watch Kalshi market prices and volumes +- Watch Neural market prices and volumes - Store historical data for analysis - Detect anomalies and significant events @@ -36,8 +36,8 @@ class DataCollectionAgent: # Monitor Twitter sentiment = await self.consume_twitter_stream() - # Monitor Kalshi - markets = await self.consume_kalshi_stream() + # Monitor Neural + markets = await self.consume_neural_stream() # Detect triggers if self.detect_opportunity(espn_data, sentiment, markets): @@ -110,7 +110,7 @@ class GameAnalystAgent: injuries = await self.get_injury_report(teams) # 3. Market Analysis - market_data = await self.get_kalshi_markets(game_id) + market_data = await self.get_neural_markets(game_id) implied_prob = self.calculate_implied_probability(market_data) # 4. Sentiment Analysis @@ -408,7 +408,7 @@ data_collector: enabled: true mode: always_on redis_channels: - - kalshi:markets + - neural:markets - espn:games - twitter:sentiment diff --git a/docs/DATA_SOURCES_GUIDE.md b/docs/DATA_SOURCES_GUIDE.md index f7368261..9fe679ce 100644 --- a/docs/DATA_SOURCES_GUIDE.md +++ b/docs/DATA_SOURCES_GUIDE.md @@ -10,7 +10,7 @@ Each data source provides unique alpha for trading decisions. This guide explain ### 1. DraftKings Sportsbook -**Purpose**: Professional odds movements often lead Kalshi markets by 5-30 seconds +**Purpose**: Professional odds movements often lead Neural markets by 5-30 seconds **What It Provides**: - Real-time odds for all major sports diff --git a/docs/GAME_CONFIGURATION_GUIDE.md b/docs/GAME_CONFIGURATION_GUIDE.md index 3a59c81c..05c5a5f2 100644 --- a/docs/GAME_CONFIGURATION_GUIDE.md +++ b/docs/GAME_CONFIGURATION_GUIDE.md @@ -24,8 +24,8 @@ game_monitoring: start_time: "2024-01-21T18:30:00Z" venue: "Arrowhead Stadium" - # Kalshi markets to trade - kalshi_markets: + # Neural markets to trade + neural_markets: - "NFL-KC-BUF-WINNER" - "NFL-KC-BUF-SPREAD" - "NFL-KC-BUF-TOTAL" @@ -213,7 +213,7 @@ auto_discovery: # Find games with these criteria criteria: - min_kalshi_volume: 10000 # Minimum liquidity + min_neural_volume: 10000 # Minimum liquidity min_edge_required: 0.03 # 3% edge sports: ["NFL", "NBA"] @@ -586,7 +586,7 @@ def validate_game_config(config): errors = [] # Check required fields - required = ['game_monitoring', 'kalshi_markets', 'data_sources'] + required = ['game_monitoring', 'neural_markets', 'data_sources'] for field in required: if field not in config: errors.append(f"Missing required field: {field}") diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index c7a20cce..f1316416 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -1,418 +1,25 @@ -# Getting Started - Neural Trading Platform +# Getting Started with Neural SDK -## Prerequisites - -Before you begin, ensure you have: - -- Python 3.10 or higher -- Redis server installed -- Git for version control -- At least one API key (Kalshi, DraftKings, Reddit, or OpenWeatherMap) - ---- +Welcome to the Neural SDK! This guide will help you get up and running with your first trading strategy. ## Installation -### 1. Clone the Repository - -```bash -git clone https://github.com/IntelIP/Neural-Trading-Platform.git -cd Neural-Trading-Platform -``` - -### 2. Install Dependencies - -```bash -# Using pip -pip install -r requirements.txt - -# Or using uv (recommended for faster installs) -uv pip install -r requirements.txt -``` - -### 3. Set Up Redis - -```bash -# macOS -brew install redis -brew services start redis - -# Ubuntu/Debian -sudo apt-get install redis-server -sudo systemctl start redis - -# Verify Redis is running -redis-cli ping -# Should return: PONG -``` - -### 4. Configure Environment Variables - -Create a `.env` file in the project root: - -```bash -# Kalshi Trading (Required) -KALSHI_API_KEY_ID=your_key_id_here -KALSHI_API_KEY=your_api_key_here - -# Data Sources (Optional - add what you have) -REDDIT_CLIENT_ID=your_reddit_client_id -REDDIT_CLIENT_SECRET=your_reddit_secret -OPENWEATHER_API_KEY=your_weather_api_key -``` - ---- - -## Quick Start: Monitor Your First Game - -### Step 1: Configure a Game to Monitor - -Edit `config/data_sources.yaml` to enable your available data sources: - -```yaml -sources: - # If you have a weather API key, enable this - - name: weather - enabled: true # Change from false to true - config: - api_key: ${OPENWEATHER_API_KEY} - - # DraftKings is free, enable this - - name: draftkings - enabled: true - config: - sports: - - NFL # Pick the sport you want -``` - -### Step 2: Run the SDK Demo - -This will show you data flowing through the system: - -```bash -python scripts/demo_sdk.py -``` - -You'll see output like: -``` -📡 Initializing data sources... -✅ Loaded 2 adapters: - • DraftKings v1.0.0 - Type: sportsbook - Latency: 500ms - Reliability: 95.0% - • Weather v1.0.0 - Type: environmental - Latency: 2000ms - Reliability: 99.0% - -🚀 Starting data streams... -📊 Processing events (30 seconds)... - -📊 Odds Change: Chiefs vs Bills - Market: spread - Change: 0.650 → 0.675 - Direction: up - -🌤️ Weather Alert: Arrowhead Stadium - Condition: high_wind - Impact: ['passing_game', 'field_goals', 'punts'] -``` - -### Step 3: Connect to Kalshi Markets - -Once you have Kalshi API credentials, start the full platform: - -```bash -# Start the unified stream manager -python -m src.data_pipeline.orchestration.unified_stream_manager - -# In another terminal, start agent consumers -python examples/agent_redis_consumer.py all -``` - ---- - -## Understanding the Data Flow - -Here's what happens when you run the platform: - -``` -1. Data Sources Connect - ↓ -2. Events Stream In (odds changes, weather updates, etc.) - ↓ -3. Stream Manager Standardizes Events - ↓ -4. Redis Distributes to Subscribers - ↓ -5. Agents Analyze for Opportunities - ↓ -6. Trading Signals Generated - ↓ -7. Orders Placed on Kalshi -``` - -### Real Example: Touchdown Scored - -``` -ESPN GameCast → "Touchdown Chiefs!" - ↓ (100ms) -Stream Manager → StandardizedEvent(type=GAME_EVENT, impact=HIGH) - ↓ (10ms) -Redis Pub/Sub → Channel: "espn:games" - ↓ (5ms) -DataCoordinator → Detects Kalshi price hasn't moved - ↓ (200ms) -StrategyAnalyst → Calculates expected +5% price move - ↓ (100ms) -TradeExecutor → Places BUY order on Kalshi - ↓ (200ms) -Total Time: ~615ms (before other traders react!) -``` - ---- - -## Basic Operations - -### Starting Individual Components - -```bash -# Just weather monitoring -python -m src.sdk.adapters.weather - -# Just DraftKings odds -python -m src.sdk.adapters.draftkings - -# Just Reddit sentiment -python -m src.sdk.adapters.reddit -``` - -### Monitoring System Health - -```bash -# Check Redis messages -redis-cli MONITOR - -# See active channels -redis-cli PUBSUB CHANNELS - -# Count messages in a channel -redis-cli PUBSUB NUMSUB kalshi:markets -``` - -### Running Tests - ```bash -# Test SDK functionality -python scripts/test_sdk.py - -# Test specific adapter -python -c " -from src.sdk import SDKManager -import asyncio - -async def test(): - sdk = SDKManager() - await sdk.initialize() - results = await sdk.test_adapter('draftkings', duration=10) - print(f'Events received: {results[\"events_received\"]}') - -asyncio.run(test()) -" -``` - ---- - -## Common Configurations - -### Focus on Specific Games - -To monitor only specific games, configure your sources: - -```yaml -# config/data_sources.yaml -sources: - - name: draftkings - config: - sports: - - NFL - teams: # Optional: focus on specific teams - - "Kansas City Chiefs" - - "Buffalo Bills" -``` - -### Adjust Update Frequencies - -```yaml -sources: - - name: draftkings - config: - poll_interval: 2 # Check every 2 seconds (was 5) - - - name: weather - config: - update_interval: 60 # Check every minute (was 5 minutes) +pip install neural-sdk ``` -### Set Trading Thresholds - -```yaml -# config/trading_config.yaml -thresholds: - min_edge: 0.03 # 3% minimum advantage - min_confidence: 0.75 # 75% confidence required - max_position: 0.05 # 5% of capital max per trade -``` - ---- - -## Troubleshooting +## Quick Start -### No Events Showing Up? +1. **Set up environment variables** +2. **Create your first strategy** +3. **Run a backtest** +4. **Deploy live trading** (optional) -1. **Check Redis is running:** -```bash -redis-cli ping -# Should return PONG -``` - -2. **Verify API credentials:** -```bash -# Test weather API -curl "https://api.openweathermap.org/data/2.5/weather?q=London&appid=YOUR_KEY" -``` - -3. **Enable debug logging:** -```python -# Add to your script -import logging -logging.basicConfig(level=logging.DEBUG) -``` - -### Events But No Trading? - -1. **Check Kalshi connection:** -```bash -python examples/test_kalshi_websocket.py -``` - -2. **Verify market is open:** -- Sports markets only trade during games -- Check Kalshi website for active markets - -3. **Review thresholds:** -- Your edge threshold might be too high -- Lower confidence requirement for testing - -### High Latency? - -1. **Check network:** -```bash -ping api.kalshi.com -``` - -2. **Reduce data sources:** -- Start with just one source -- Add more gradually - -3. **Optimize Redis:** -```bash -# Check Redis latency -redis-cli --latency -``` - ---- +For detailed instructions, see the main README.md. ## Next Steps -### 1. Add More Data Sources - -Create your own adapter in under 50 lines: - -```python -from src.sdk import DataSourceAdapter, StandardizedEvent, EventType - -class MyAdapter(DataSourceAdapter): - async def connect(self): - # Your connection logic - return True - - async def stream(self): - while self.is_connected: - # Your data fetching - data = await self.fetch() - yield StandardizedEvent( - source="MySource", - event_type=EventType.CUSTOM, - data=data - ) -``` - -### 2. Customize Trading Logic - -Modify agents to implement your strategy: - -```python -# agents/StrategyAnalyst/agent.py -@tool -def my_custom_strategy(market_data, weather, sentiment): - # Your alpha generation logic - if weather['wind_speed'] > 20 and market_data['spread'] > 3: - return {"action": "buy", "confidence": 0.85} -``` - -### 3. Set Up Production Monitoring - -```bash -# Run with full logging -python examples/production_monitor.py - -# Set up alerts -python scripts/setup_alerts.py --email your@email.com -``` - ---- - -## Essential Commands - -```bash -# Development -python scripts/demo_sdk.py # Test SDK -python examples/agent_redis_consumer.py # Run consumers -redis-cli MONITOR # Watch Redis - -# Testing -pytest tests/ # Run all tests -pytest tests/test_sdk.py -v # Test SDK only - -# Production -python -m src.data_pipeline.orchestration.unified_stream_manager # Start streams -python agents/launch_all.py # Start all agents -python scripts/monitor_health.py # Health dashboard -``` - ---- - -## Getting Help - -If you run into issues: - -1. Check the logs in `logs/` directory -2. Review configuration in `config/` -3. Run diagnostic script: `python scripts/diagnose.py` -4. See troubleshooting guide in docs - -Remember: Start simple with one data source, verify it works, then add complexity! - ---- - -## Ready to Trade? - -You're now ready to: -- ✅ Stream real-time data -- ✅ Process events through the pipeline -- ✅ Generate trading signals -- ✅ Execute on Kalshi markets - -Next: Read [DATA_SOURCES_GUIDE.md](DATA_SOURCES_GUIDE.md) to understand each data source in detail. \ No newline at end of file +- [API Reference](api_reference.md) - Complete SDK documentation +- [Backtesting Guide](backtesting.md) - Historical testing framework +- [Data Sources](data_sources.md) - Setting up data feeds +- [Strategy Development](strategies.md) - Building trading algorithms \ No newline at end of file diff --git a/docs/GITHUB_PUSH_GUIDE.md b/docs/GITHUB_PUSH_GUIDE.md deleted file mode 100644 index c716a620..00000000 --- a/docs/GITHUB_PUSH_GUIDE.md +++ /dev/null @@ -1,467 +0,0 @@ -# GitHub Push Guide - -## Pre-Push Checklist - -### 1. Verify Local Setup -```bash -# Check git is initialized -git status - -# Verify remote is not set yet -git remote -v - -# Ensure you're on main branch -git branch -``` - -### 2. Clean Sensitive Data -```bash -# Ensure .env is in .gitignore -grep "^\.env$" .gitignore - -# Check for any secrets in code -grep -r "KALSHI_API_KEY" --exclude-dir=.git --exclude=.env* . -grep -r "OPENROUTER_API_KEY" --exclude-dir=.git --exclude=.env* . -grep -r "AGENTUITY_SDK_KEY" --exclude-dir=.git --exclude=.env* . - -# Verify no credentials in committed files -git diff --cached | grep -i "api_key\|secret\|password\|token" -``` - -### 3. Verify Documentation -```bash -# Check all required docs exist -ls -la README.md CLAUDE.md CONTRIBUTING.md -ls -la docs/ -ls -la .github/ -``` - -## Initial Repository Setup - -### Step 1: Create GitHub Repository - -1. Go to https://github.com/new -2. Repository settings: - - **Name**: `Kalshi_Agentic_Agent` - - **Description**: "Autonomous multi-agent trading system for Kalshi sports event contracts" - - **Visibility**: Private (initially) - - **DO NOT** initialize with README, .gitignore, or license - -### Step 2: Initialize Local Repository - -```bash -# If not already initialized -git init - -# Add GitHub remote -git remote add origin https://github.com/YOUR_USERNAME/Kalshi_Agentic_Agent.git - -# Verify remote -git remote -v -``` - -### Step 3: Prepare Initial Commit - -```bash -# Stage all files -git add . - -# Verify what's being staged -git status - -# Check file count -git ls-files | wc -l - -# Create initial commit -git commit -m "feat: initial commit - production-ready multi-agent trading system - -- Complete repository structure with clear naming conventions -- Comprehensive CI/CD pipeline with GitHub Actions -- Full documentation suite (README, CLAUDE.md, CONTRIBUTING.md) -- Redis-based agent communication architecture -- Agentuity framework integration -- Kelly Criterion trading logic implementation -- WebSocket data pipeline for real-time processing -- PostgreSQL database with Alembic migrations -- Security scanning and automated testing -- Docker containerization support" -``` - -### Step 4: Push to GitHub - -```bash -# Push main branch -git push -u origin main - -# Create and push develop branch -git checkout -b develop -git push -u origin develop - -# Return to main -git checkout main -``` - -## Configure GitHub Repository - -### Step 1: Set Up Secrets - -Navigate to: Settings → Secrets and variables → Actions - -Add the following secrets: - -#### Required Secrets -```yaml -# Kalshi Trading -KALSHI_API_KEY_ID: "your-kalshi-api-key-id" -KALSHI_PRIVATE_KEY: | - -----BEGIN PRIVATE KEY----- - your-private-key-content - -----END PRIVATE KEY----- -KALSHI_ENVIRONMENT: "production" # or "demo" for testing - -# AI/LLM -OPENROUTER_API_KEY: "your-openrouter-api-key" - -# Agentuity Platform -AGENTUITY_SDK_KEY: "your-agentuity-sdk-key" - -# Infrastructure -REDIS_URL: "redis://your-redis-host:6379" - -# Optional - for deployment -DOCKER_REGISTRY: "your-docker-registry" -DOCKER_USERNAME: "your-docker-username" -DOCKER_PASSWORD: "your-docker-password" -``` - -### Step 2: Configure Environments - -Navigate to: Settings → Environments - -Create two environments: - -#### Staging Environment -- **Name**: staging -- **Protection rules**: - - Only from: `develop` branch - - Required reviewers: 1 -- **Secrets**: Add staging-specific overrides - -#### Production Environment -- **Name**: production -- **Protection rules**: - - Only from: `main` branch - - Required reviewers: 2 - - Restrict deployments to specific users -- **Secrets**: Add production-specific values - -### Step 3: Enable Branch Protection - -Navigate to: Settings → Branches - -#### Main Branch Protection -```bash -# Using GitHub CLI (if installed) -gh api repos/:owner/:repo/branches/main/protection \ - --method PUT \ - --field required_status_checks='{"strict":true,"contexts":["CI Pipeline / Lint & Format Check","CI Pipeline / Test Suite","CI Pipeline / Security Scan","CI Pipeline / Build & Validate"]}' \ - --field required_pull_request_reviews='{"required_approving_review_count":2,"dismiss_stale_reviews":true}' \ - --field enforce_admins=true \ - --field allow_force_pushes=false \ - --field allow_deletions=false -``` - -Or manually: -1. Click "Add rule" -2. Branch name pattern: `main` -3. Enable: - - ✅ Require pull request before merging (2 approvals) - - ✅ Require status checks to pass - - ✅ Require branches to be up to date - - ✅ Include administrators - - ✅ Restrict who can push (only maintainers) - -#### Develop Branch Protection -Similar to main but with 1 required approval. - -### Step 4: Configure GitHub Pages (Optional) - -For documentation hosting: - -1. Settings → Pages -2. Source: Deploy from branch -3. Branch: main -4. Folder: /docs -5. Save - -### Step 5: Set Up Teams (If Organization) - -1. Settings → Manage access → Invite teams -2. Create teams: - - `maintainers` - Admin access - - `developers` - Write access - - `qa-team` - Triage access - -### Step 6: Configure Webhooks (Optional) - -For external integrations: - -1. Settings → Webhooks → Add webhook -2. Payload URL: Your monitoring service -3. Events: Push, Pull Request, Deployment - -## Verify CI/CD Pipeline - -### Step 1: Create Test PR - -```bash -# Create feature branch -git checkout -b feature/test-ci -echo "# Test CI" > test.md -git add test.md -git commit -m "test: verify CI pipeline" -git push -u origin feature/test-ci -``` - -### Step 2: Open Pull Request - -1. Go to repository on GitHub -2. Click "Compare & pull request" -3. Target: develop -4. Create pull request - -### Step 3: Verify Checks - -Ensure all checks pass: -- ✅ Lint & Format Check -- ✅ Test Suite -- ✅ Security Scan -- ✅ Build & Validate -- ✅ PR Checks - -### Step 4: Clean Up - -```bash -# After merge, delete test branch -git checkout develop -git pull origin develop -git branch -d feature/test-ci -git push origin --delete feature/test-ci -``` - -## First Deployment - -### Step 1: Prepare for Deployment - -```bash -# Ensure on develop branch -git checkout develop - -# Tag release candidate -git tag -a v0.1.0-rc.1 -m "Release candidate 1 for v0.1.0" -git push origin v0.1.0-rc.1 -``` - -### Step 2: Deploy to Staging - -```bash -# Trigger staging deployment -# This happens automatically on push to develop - -# Or manually via GitHub Actions -gh workflow run deploy.yml -f environment=staging -f version=v0.1.0-rc.1 -``` - -### Step 3: Production Release - -```bash -# Create release PR -git checkout -b release/v0.1.0 -git push -u origin release/v0.1.0 - -# After approval and merge to main -git checkout main -git pull origin main -git tag -a v0.1.0 -m "Initial release v0.1.0" -git push origin v0.1.0 - -# Create GitHub Release -gh release create v0.1.0 \ - --title "v0.1.0 - Initial Release" \ - --notes "Initial production release of Kalshi Trading Agent System" \ - --target main -``` - -## Post-Push Tasks - -### Immediate Actions - -1. **Verify Repository Access**: - ```bash - # Clone in new directory to test - cd /tmp - git clone https://github.com/YOUR_USERNAME/Kalshi_Agentic_Agent.git - cd Kalshi_Agentic_Agent - ``` - -2. **Check CI Status**: - - Go to Actions tab - - Verify workflows are detected - - Check for any configuration errors - -3. **Update README Badge**: - ```markdown - ![CI Pipeline](https://github.com/YOUR_USERNAME/Kalshi_Agentic_Agent/workflows/CI%20Pipeline/badge.svg) - ``` - -### Within 24 Hours - -1. **Security Scan**: - - Enable Dependabot alerts - - Review security recommendations - - Set up code scanning - -2. **Documentation**: - - Verify all links work - - Check rendered markdown - - Update any absolute paths - -3. **Team Access**: - - Invite collaborators - - Set up CODEOWNERS file - - Configure notifications - -### Within First Week - -1. **Monitoring Setup**: - - Configure error tracking - - Set up performance monitoring - - Add deployment notifications - -2. **Backup Strategy**: - - Set up repository mirroring - - Configure automated backups - - Document recovery procedures - -3. **Performance Baseline**: - - Run initial load tests - - Document response times - - Set up metrics collection - -## Troubleshooting - -### Common Issues - -#### Push Rejected - Large Files -```bash -# Check for large files -find . -type f -size +100M - -# Add to .gitignore if needed -echo "large-file.bin" >> .gitignore - -# Remove from git history -git filter-branch --index-filter 'git rm --cached --ignore-unmatch large-file.bin' HEAD -``` - -#### Push Rejected - Credentials Detected -```bash -# Remove sensitive data -git filter-branch --force --index-filter \ - 'git rm --cached --ignore-unmatch path/to/sensitive-file' \ - --prune-empty --tag-name-filter cat -- --all -``` - -#### CI Workflows Not Triggering -1. Check workflow syntax -2. Verify file location (.github/workflows/) -3. Ensure proper permissions -4. Check branch protection settings - -#### Permission Denied -```bash -# For HTTPS -git config --global credential.helper cache - -# For SSH -ssh-add ~/.ssh/id_rsa -``` - -## Security Checklist - -Before making repository public: - -- [ ] All secrets in GitHub Secrets -- [ ] No hardcoded credentials -- [ ] .env.example has placeholder values -- [ ] Security scanning enabled -- [ ] Dependency review enabled -- [ ] Branch protection configured -- [ ] CODEOWNERS file created -- [ ] Security policy added -- [ ] Vulnerability reporting enabled - -## Quick Reference - -### Essential Commands -```bash -# Initial push -git push -u origin main - -# Create develop branch -git checkout -b develop -git push -u origin develop - -# Create feature branch -git checkout -b feature/new-feature -git push -u origin feature/new-feature - -# Update from upstream -git fetch origin -git merge origin/develop - -# Tag release -git tag -a v1.0.0 -m "Release v1.0.0" -git push origin v1.0.0 -``` - -### GitHub CLI Commands -```bash -# Check workflow runs -gh run list - -# View workflow details -gh run view - -# Watch workflow in progress -gh run watch - -# List issues -gh issue list - -# Create issue -gh issue create --title "Bug report" --body "Description" - -# List PRs -gh pr list - -# Create PR -gh pr create --title "Feature" --body "Description" -``` - -## Next Steps - -After successful push: - -1. **Test the CI pipeline** with a small PR -2. **Configure deployment** to Agentuity platform -3. **Set up monitoring** and alerting -4. **Document API endpoints** if applicable -5. **Create initial issues** for known improvements -6. **Invite team members** and assign roles -7. **Schedule regular dependency updates** -8. **Plan first sprint** using GitHub Projects - ---- - -Remember: This is a financial trading system. Ensure all security measures are properly configured before deploying to production. \ No newline at end of file diff --git a/docs/GIT_WORKFLOW.md b/docs/GIT_WORKFLOW.md deleted file mode 100644 index ff781fc5..00000000 --- a/docs/GIT_WORKFLOW.md +++ /dev/null @@ -1,443 +0,0 @@ -# Git Workflow & Branching Strategy - -## Branch Structure - -``` -main (production) - ├── develop (integration) - │ ├── feature/feature-name - │ ├── bugfix/issue-description - │ └── hotfix/critical-fix - └── release/v1.0.0 -``` - -## Branch Types - -### 1. Main Branch (`main`) -- **Purpose**: Production-ready code -- **Protection**: Full protection enabled -- **Deploy**: Automatically to production -- **Merge**: Only from `release/*` or `hotfix/*` branches -- **Requirements**: - - All tests passing - - Code review approved - - No merge conflicts - -### 2. Develop Branch (`develop`) -- **Purpose**: Integration branch for features -- **Protection**: Requires PR and tests -- **Deploy**: To staging environment -- **Merge**: From `feature/*` and `bugfix/*` branches -- **Updated**: Daily from `main` - -### 3. Feature Branches (`feature/*`) -- **Naming**: `feature/description-of-feature` -- **Created from**: `develop` -- **Merged to**: `develop` -- **Lifetime**: Until feature complete -- **Examples**: - - `feature/add-stop-loss-monitor` - - `feature/espn-api-integration` - - `feature/kelly-criterion-update` - -### 4. Bugfix Branches (`bugfix/*`) -- **Naming**: `bugfix/issue-number-description` -- **Created from**: `develop` -- **Merged to**: `develop` -- **Lifetime**: Until bug fixed -- **Examples**: - - `bugfix/123-websocket-reconnection` - - `bugfix/456-redis-timeout` - -### 5. Release Branches (`release/*`) -- **Naming**: `release/vX.Y.Z` -- **Created from**: `develop` -- **Merged to**: `main` and back to `develop` -- **Purpose**: Final testing and version prep -- **Allowed changes**: Bug fixes only - -### 6. Hotfix Branches (`hotfix/*`) -- **Naming**: `hotfix/critical-issue` -- **Created from**: `main` -- **Merged to**: `main` and `develop` -- **Purpose**: Emergency production fixes -- **Review**: Expedited process - -## Workflow Steps - -### Starting New Feature - -```bash -# 1. Update develop branch -git checkout develop -git pull origin develop - -# 2. Create feature branch -git checkout -b feature/my-new-feature - -# 3. Work on feature -git add . -git commit -m "feat: add new feature description" - -# 4. Push to remote -git push -u origin feature/my-new-feature - -# 5. Create Pull Request to develop -``` - -### Creating a Release - -```bash -# 1. Create release branch from develop -git checkout develop -git pull origin develop -git checkout -b release/v1.0.0 - -# 2. Update version numbers -# Update pyproject.toml, README.md, etc. - -# 3. Final testing and fixes -git add . -git commit -m "chore: prepare release v1.0.0" - -# 4. Merge to main -git checkout main -git merge --no-ff release/v1.0.0 -git tag -a v1.0.0 -m "Release version 1.0.0" - -# 5. Merge back to develop -git checkout develop -git merge --no-ff release/v1.0.0 - -# 6. Push everything -git push origin main develop --tags - -# 7. Delete release branch -git branch -d release/v1.0.0 -git push origin --delete release/v1.0.0 -``` - -### Emergency Hotfix - -```bash -# 1. Create hotfix from main -git checkout main -git pull origin main -git checkout -b hotfix/critical-bug - -# 2. Fix the issue -git add . -git commit -m "hotfix: fix critical production bug" - -# 3. Merge to main -git checkout main -git merge --no-ff hotfix/critical-bug -git tag -a v1.0.1 -m "Hotfix version 1.0.1" - -# 4. Merge to develop -git checkout develop -git merge --no-ff hotfix/critical-bug - -# 5. Push and cleanup -git push origin main develop --tags -git branch -d hotfix/critical-bug -git push origin --delete hotfix/critical-bug -``` - -## Commit Message Convention - -### Format -``` -(): - - - -