analysis.py

# %% [markdown]
# # Trader Performance vs Market Sentiment
# ### Primetrade.ai Data Science Intern Assignment
# **Objective:** Analyze how market sentiment (Fear/Greed) relates to trader behavior and performance on Hyperliquid.

# %% [markdown]
# ## Part A: Data Preparation

# %%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import warnings
warnings.filterwarnings('ignore')

# 1. Load datasets
print("Loading data...")
df_sent = pd.read_csv('sentiment.csv')
df_trader = pd.read_csv('trader_data.csv')

# 2. Document rows/columns and missing values
print("--- Bitcoin Market Sentiment ---")
print(f"Rows: {df_sent.shape[0]}, Columns: {df_sent.shape[1]}")
print("Missing Values:\n", df_sent.isnull().sum())
print("Duplicates:", df_sent.duplicated().sum())

print("\n--- Historical Trader Data ---")
print(f"Rows: {df_trader.shape[0]}, Columns: {df_trader.shape[1]}")
print("Missing Values:\n", df_trader.isnull().sum())
print("Duplicates:", df_trader.duplicated().sum())

# 3. Convert timestamps and align dates
df_sent['date'] = pd.to_datetime(df_sent['date']).dt.date
df_trader['date'] = pd.to_datetime(df_trader['Timestamp IST'], format='%d-%m-%Y %H:%M').dt.date

# 4. Create Key Metrics
print("\nCalculating metrics per trader per day...")
# PnL
daily_pnl = df_trader.groupby(['Account', 'date'])['Closed PnL'].sum().reset_index()
# Trade Count
daily_trades = df_trader.groupby(['Account', 'date']).size().reset_index(name='Trade Count')
# Average Trade Size (USD)
daily_avg_size = df_trader.groupby(['Account', 'date'])['Size USD'].mean().reset_index(name='Avg Trade Size USD')
# Total Volume (USD) -> proxy for leverage distribution / trading size
daily_vol = df_trader.groupby(['Account', 'date'])['Size USD'].sum().reset_index(name='Total Volume USD')

# Win rate
def calc_win_rate(g):
    resolved = g[g['Closed PnL'] != 0]
    if len(resolved) == 0: return 0.0
    return len(resolved[resolved['Closed PnL'] > 0]) / len(resolved)
daily_win_rate = df_trader.groupby(['Account', 'date']).apply(calc_win_rate).reset_index(name='Win Rate')

# Long/Short Ratio
def calc_ls_ratio(g):
    longs = len(g[g['Direction'].str.lower() == 'buy'])
    shorts = len(g[g['Direction'].str.lower() == 'sell'])
    if shorts == 0: return float(longs)
    return longs / shorts
daily_ls_ratio = df_trader.groupby(['Account', 'date']).apply(calc_ls_ratio).reset_index(name='Long/Short Ratio')

# Merge trader metrics
daily_metrics = daily_pnl.merge(daily_trades, on=['Account','date']) \
    .merge(daily_avg_size, on=['Account','date']) \
    .merge(daily_vol, on=['Account','date']) \
    .merge(daily_win_rate, on=['Account','date']) \
    .merge(daily_ls_ratio, on=['Account','date'])

# Merge with sentiment
data = daily_metrics.merge(df_sent[['date', 'classification', 'value']], on='date', how='inner')
data['Sentiment_Broad'] = data['classification'].apply(lambda x: 'Fear' if 'fear' in str(x).lower() else ('Greed' if 'greed' in str(x).lower() else 'Neutral'))

# Save aggregated data for Streamlit
data.to_csv('aggregated_trader_metrics.csv', index=False)
print("Aggregated data saved to 'aggregated_trader_metrics.csv'")

# %% [markdown]
# ## Part B: Analysis

# %%
sns.set_theme(style="whitegrid")

# Question 1: Does performance differ between Fear vs Greed days?
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
sns.barplot(data=data[data['Sentiment_Broad'] != 'Neutral'], x='Sentiment_Broad', y='Closed PnL', ax=axes[0])
axes[0].set_title('Average Daily PnL: Fear vs Greed')

sns.boxplot(data=data[data['Sentiment_Broad'] != 'Neutral'], x='Sentiment_Broad', y='Win Rate', ax=axes[1])
axes[1].set_title('Win Rate Distribution: Fear vs Greed')
plt.savefig('performance_vs_sentiment.png')
plt.close()

# Question 2: Do traders change behavior based on sentiment?
fig, axes = plt.subplots(1, 3, figsize=(18, 5))
sns.barplot(data=data[data['Sentiment_Broad'] != 'Neutral'], x='Sentiment_Broad', y='Trade Count', ax=axes[0])
axes[0].set_title('Average Trade Frequency')

sns.barplot(data=data[data['Sentiment_Broad'] != 'Neutral'], x='Sentiment_Broad', y='Avg Trade Size USD', ax=axes[1])
axes[1].set_title('Average Trade Size (USD)')

sns.barplot(data=data[data['Sentiment_Broad'] != 'Neutral'], x='Sentiment_Broad', y='Long/Short Ratio', ax=axes[2])
axes[2].set_title('Long/Short Ratio')
plt.savefig('behavior_vs_sentiment.png')
plt.close()

# %% [markdown]
# ### Segmentation
# Identify segments:
# 1. Frequent vs Infrequent
# 2. Consistent Winners vs Losers
# 3. High Volume (Leverage proxy) vs Low Volume

# %%
trader_stats = data.groupby('Account').agg({
    'Trade Count': 'mean',
    'Win Rate': 'mean',
    'Total Volume USD': 'mean'
}).reset_index()

# Medians for segmentation
med_freq = trader_stats['Trade Count'].median()
med_win = trader_stats['Win Rate'].median()
med_vol = trader_stats['Total Volume USD'].median()

trader_stats['Frequency Segment'] = np.where(trader_stats['Trade Count'] > med_freq, 'Frequent', 'Infrequent')
trader_stats['Performance Segment'] = np.where(trader_stats['Win Rate'] > med_win, 'Winner', 'Loser')
trader_stats['Volume Segment'] = np.where(trader_stats['Total Volume USD'] > med_vol, 'High Volume', 'Low Volume')

data_seg = data.merge(trader_stats[['Account', 'Frequency Segment', 'Performance Segment', 'Volume Segment']], on='Account')

# Behavior of Frequent vs Infrequent traders on Fear vs Greed
plt.figure(figsize=(8, 5))
sns.barplot(data=data_seg[data_seg['Sentiment_Broad'] != 'Neutral'], x='Sentiment_Broad', y='Closed PnL', hue='Frequency Segment')
plt.title('PnL by Frequency Segment: Fear vs Greed')
plt.savefig('segment_pnl_vs_sentiment.png')
plt.close()

# %% [markdown]
# ## Part C & Bonus: Predictive Modeling / Clustering

# %%
# Clustering traders into behavioral archetypes
X = trader_stats[['Trade Count', 'Win Rate', 'Total Volume USD']].fillna(0)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

kmeans = KMeans(n_clusters=3, random_state=42)
trader_stats['Archetype'] = kmeans.fit_predict(X_scaled)

plt.figure(figsize=(8, 5))
sns.scatterplot(data=trader_stats, x='Trade Count', y='Win Rate', hue='Archetype', palette='viridis', size='Total Volume USD', sizes=(20, 200))
plt.title('Trader Behavioral Archetypes')
plt.savefig('trader_archetypes.png')
plt.close()

# Predictive Model: Predict next-day profitability bucket
# Shift PnL by -1 to get next day's PnL for the same trader
data_sorted = data_seg.sort_values(by=['Account', 'date'])
data_sorted['Next_Day_PnL'] = data_sorted.groupby('Account')['Closed PnL'].shift(-1)
data_sorted.dropna(subset=['Next_Day_PnL'], inplace=True)
data_sorted['Profitable_Next_Day'] = (data_sorted['Next_Day_PnL'] > 0).astype(int)

features = ['value', 'Trade Count', 'Avg Trade Size USD', 'Long/Short Ratio', 'Closed PnL']
X_model = data_sorted[features].fillna(0)
y_model = data_sorted['Profitable_Next_Day']

X_train, X_test, y_train, y_test = train_test_split(X_model, y_model, test_size=0.2, random_state=42)
clf = RandomForestClassifier(n_estimators=100, random_state=42)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print("\nPredictive Model Report (Predicting Next-Day Profitability):")
print(classification_report(y_test, y_pred))

# Save the model features importance
pd.Series(clf.feature_importances_, index=features).sort_values(ascending=False).plot(kind='bar', figsize=(6,4))
plt.title('Feature Importances for Predicting Next-Day Profitability')
plt.tight_layout()
plt.savefig('feature_importances.png')
plt.close()
print("Analysis complete. Generated output charts.")