data_processor.py

import pandas as pd
import numpy as np
import os
import requests
from io import StringIO
import time
from datetime import datetime

def load_data():
    """
    Load tariff and price data from reliable sources.
    Returns tariff_data, price_data, and available categories.
    """
    # Try to load real data from sources
    try:
        # Load tariff data from US International Trade Commission
        # This is a simplification - in a real app, you would use their API or download their datasets
        tariff_data = load_tariff_data()
        
        # Load price data from Bureau of Labor Statistics
        # This is a simplification - in a real app, you would use their API or download their datasets
        price_data = load_price_data()
        
        # Get unique categories from both datasets
        categories = sorted(set(tariff_data['Category'].unique()) & set(price_data['Category'].unique()))
        
        return tariff_data, price_data, categories
        
    except Exception as e:
        # Log the error
        print(f"Error loading data: {str(e)}")
        return None, None, []

def load_tariff_data():
    """
    Load tariff data from reliable sources.
    Returns a DataFrame with tariff data.
    """
    # Try to load from USITC HTS data
    # In a real implementation, this would connect to APIs or download datasets
    
    # For this demo, we'll create a dataset that represents plausible tariff data
    # based on real tariff categories and rates, but not actual data
    
    # Define years, categories, and countries
    years = range(2015, 2024)
    
    categories = [
        "Smartphones", "Laptops", "Desktop Computers", "Tablets",
        "Computer Monitors", "Television Displays", "Networking Equipment",
        "Computer Components", "Semiconductor Chips", "Storage Devices",
        "Audio Equipment", "Wearable Technology", "Smart Home Devices",
        "Gaming Consoles", "Camera Equipment"
    ]
    
    # Initialize empty DataFrame
    data = []
    
    # Starting tariff rates based loosely on real tariff schedules
    # These are the base rates in 2015
    base_rates = {
        "Smartphones": 0.0,            # Most had 0% under ITA agreement
        "Laptops": 0.0,                # Most had 0% under ITA agreement
        "Desktop Computers": 0.0,      # Most had 0% under ITA agreement
        "Tablets": 0.0,                # Most had 0% under ITA agreement
        "Computer Monitors": 3.7,      # Mixed rates based on specifications
        "Television Displays": 4.5,    # TV tariffs vary by size/technology
        "Networking Equipment": 1.2,   # Some networking equipment had low tariffs
        "Computer Components": 2.0,    # Mixed rates for components
        "Semiconductor Chips": 0.0,    # Most covered under ITA with 0%
        "Storage Devices": 1.5,        # Various rates based on technology
        "Audio Equipment": 5.0,        # Higher rates for consumer electronics
        "Wearable Technology": 2.5,    # New category with mixed classification
        "Smart Home Devices": 3.0,     # New category with mixed classification
        "Gaming Consoles": 1.5,        # Video game hardware
        "Camera Equipment": 2.8        # Various optical equipment rates
    }
    
    # Key events with tariff changes
    events = {
        2018: {
            # Section 301 Tariffs on Chinese goods
            "Computer Components": 10.0,
            "Networking Equipment": 10.0,
            "Television Displays": 10.0,
            "Smart Home Devices": 10.0,
            "Storage Devices": 10.0
        },
        2019: {
            # Expanded tariffs
            "Computer Components": 15.0,
            "Networking Equipment": 15.0,
            "Television Displays": 15.0,
            "Smart Home Devices": 15.0,
            "Storage Devices": 15.0,
            "Wearable Technology": 7.5,
            "Audio Equipment": 7.5
        },
        2020: {
            # Phase 1 trade deal reduction
            "Computer Components": 7.5,
            "Networking Equipment": 7.5,
            "Television Displays": 7.5,
            "Smart Home Devices": 7.5,
            "Storage Devices": 7.5,
            "Wearable Technology": 7.5,
            "Audio Equipment": 7.5
        },
        2022: {
            # Semiconductor and critical technology focus
            "Semiconductor Chips": 5.0
        }
    }
    
    # Generate data for each year and category
    for year in years:
        current_rates = base_rates.copy()
        
        # Apply any events that occurred up to and including this year
        for event_year, changes in events.items():
            if event_year <= year:
                for category, new_rate in changes.items():
                    current_rates[category] = new_rate
        
        # Add small random variations to represent detailed tariff changes
        for category in categories:
            # Generate quarterly data
            for quarter in range(1, 5):
                # Base rate for this category and year
                rate = current_rates[category]
                
                # Add small random variation (±0.5%)
                rate_with_noise = max(0, rate + np.random.uniform(-0.5, 0.5))
                
                # Add row to data
                data.append({
                    'Year': year,
                    'Quarter': quarter,
                    'Category': category,
                    'Tariff_Rate': rate_with_noise
                })
    
    # Convert to DataFrame
    df = pd.DataFrame(data)
    
    # Add a date column for time series analysis
    df['Date'] = pd.to_datetime(df['Year'].astype(str) + 'Q' + df['Quarter'].astype(str))
    
    return df

def load_price_data():
    """
    Load price data from reliable sources.
    Returns a DataFrame with price data.
    """
    # Try to load from BLS Consumer Price Index data
    # In a real implementation, this would connect to APIs or download datasets
    
    # For this demo, we'll create a dataset that represents plausible price data
    # based on real price trends, but not actual data
    
    # Define years, categories, and countries
    years = range(2015, 2024)
    
    categories = [
        "Smartphones", "Laptops", "Desktop Computers", "Tablets",
        "Computer Monitors", "Television Displays", "Networking Equipment",
        "Computer Components", "Semiconductor Chips", "Storage Devices",
        "Audio Equipment", "Wearable Technology", "Smart Home Devices",
        "Gaming Consoles", "Camera Equipment"
    ]
    
    # Initialize empty DataFrame
    data = []
    
    # Base prices (index values) for 2015
    base_indices = {cat: 100.0 for cat in categories}
    
    # Different price trends for different categories
    # Annual % change baseline (before tariff effects)
    price_trends = {
        "Smartphones": -2.0,          # Slight deflation as technology improves
        "Laptops": -3.0,              # Deflation as technology improves
        "Desktop Computers": -2.5,    # Deflation as technology improves
        "Tablets": -4.0,              # Significant deflation as market matures
        "Computer Monitors": -1.5,    # Moderate deflation
        "Television Displays": -5.0,  # Significant deflation
        "Networking Equipment": -1.0, # Slight deflation
        "Computer Components": -2.0,  # Moderate deflation
        "Semiconductor Chips": -1.0,  # Slight deflation
        "Storage Devices": -8.0,      # Strong deflation as capacity increases
        "Audio Equipment": 0.5,       # Slight inflation
        "Wearable Technology": -5.0,  # Strong deflation in new category
        "Smart Home Devices": -3.0,   # Deflation as market grows
        "Gaming Consoles": 1.0,       # Slight inflation
        "Camera Equipment": -1.0      # Slight deflation
    }
    
    # Impact of tariff events on prices (smaller than tariff as companies absorb some costs)
    # Price effects lag behind tariff implementation
    price_events = {
        2018: {
            # No immediate effect
        },
        2019: {
            # Section 301 Tariffs start showing price effects
            "Computer Components": 7.0,
            "Networking Equipment": 6.0,
            "Television Displays": 5.0,
            "Smart Home Devices": 6.0,
            "Storage Devices": 4.0
        },
        2020: {
            # Expanded tariffs price effects, partially mitigated by Phase 1 deal
            "Computer Components": 5.0,
            "Networking Equipment": 5.0,
            "Television Displays": 4.0,
            "Smart Home Devices": 4.0,
            "Storage Devices": 3.0,
            "Wearable Technology": 3.0,
            "Audio Equipment": 2.0
        },
        2021: {
            # Supply chain disruptions amplify price effects
            "Smartphones": 3.0,
            "Laptops": 5.0,
            "Desktop Computers": 6.0,
            "Computer Components": 8.0,
            "Semiconductor Chips": 10.0,
            "Gaming Consoles": 7.0
        },
        2022: {
            # Semiconductor chip issues
            "Semiconductor Chips": 6.0
        },
        2023: {
            # Gradual normalization
            "Semiconductor Chips": 2.0,
            "Computer Components": 2.0,
            "Storage Devices": 1.0
        }
    }
    
    # Current indices that we'll update year by year
    current_indices = base_indices.copy()
    
    # Generate data for each year and category
    for year in years:
        # Apply baseline price trends
        for category in categories:
            annual_change = price_trends[category]
            current_indices[category] *= (1 + annual_change / 100)
        
        # Apply any price events for this year
        if year in price_events:
            for category, price_effect in price_events[year].items():
                current_indices[category] *= (1 + price_effect / 100)
        
        # Generate quarterly data with some noise
        for quarter in range(1, 5):
            for category in categories:
                # Get current index
                index = current_indices[category]
                
                # Add small random variation (±1%)
                index_with_noise = index * (1 + np.random.uniform(-0.01, 0.01))
                
                # Add row to data
                data.append({
                    'Year': year,
                    'Quarter': quarter,
                    'Category': category,
                    'Price_Index': index_with_noise
                })
    
    # Convert to DataFrame
    df = pd.DataFrame(data)
    
    # Add a date column for time series analysis
    df['Date'] = pd.to_datetime(df['Year'].astype(str) + 'Q' + df['Quarter'].astype(str))
    
    return df

def process_historical_data(tariff_data, price_data):
    """
    Process tariff and price data to create a consolidated historical dataset.
    """
    # Aggregate tariff data by year and category
    tariff_by_year = tariff_data.groupby(['Year', 'Category'])['Tariff_Rate'].mean().reset_index()
    
    # Aggregate price data by year and category
    price_by_year = price_data.groupby(['Year', 'Category'])['Price_Index'].mean().reset_index()
    
    # Merge the datasets
    historical_data = pd.merge(
        tariff_by_year,
        price_by_year,
        on=['Year', 'Category'],
        how='inner'
    )
    
    return historical_data

def filter_data_by_category(data, categories):
    """
    Filter data to include only specified categories.
    """
    return data[data['Category'].isin(categories)]

def filter_data_by_year_range(data, start_year, end_year):
    """
    Filter data to include only years within the specified range.
    """
    return data[(data['Year'] >= start_year) & (data['Year'] <= end_year)]