app.py

# app.py
import streamlit as st
import pandas as pd
import numpy as np
import hashlib
import re
from datetime import datetime, timedelta
from collections import Counter, defaultdict
from typing import Dict, List, Tuple, Any, Optional
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots
import networkx as nx
from scipy import stats
from scipy.spatial.distance import jaccard
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
import io
import json
import chardet

# Configuration Streamlit
st.set_page_config(
    page_title="LeakDetector Pro - Analyse Forensique Avancée",
    page_icon="🕵️‍♂️",
    layout="wide",
    initial_sidebar_state="expanded"
)

# Style personnalisé
st.markdown("""
<style>
    .main-header {
        background: linear-gradient(90deg, #00d4ff 0%, #0066ff 100%);
        padding: 1rem;
        border-radius: 10px;
        margin-bottom: 2rem;
    }
    .score-card {
        background: linear-gradient(135deg, #1e1e2e 0%, #2a2a3e 100%);
        border-radius: 15px;
        padding: 1.5rem;
        text-align: center;
        box-shadow: 0 4px 15px rgba(0,0,0,0.3);
    }
    .metric-card {
        background: #1e1e2e;
        border-radius: 10px;
        padding: 1rem;
        margin: 0.5rem 0;
        border-left: 4px solid #00d4ff;
    }
    .warning-badge {
        background: #ff4444;
        color: white;
        padding: 0.2rem 0.5rem;
        border-radius: 5px;
        font-size: 0.8rem;
    }
    .success-badge {
        background: #00ff88;
        color: black;
        padding: 0.2rem 0.5rem;
        border-radius: 5px;
        font-size: 0.8rem;
    }
    .stButton > button {
        background: #00d4ff;
        color: black;
        font-weight: bold;
    }
</style>
""", unsafe_allow_html=True)

# Constantes et patterns
COMMON_HASHES = {
    'md5': {
        '5f4dcc3b5aa765d61d8327deb882cf99': 'password',
        'e10adc3949ba59abbe56e057f20f883e': '123456',
        'd8578edf8458ce06fbc5bb76a58c5ca4': 'qwerty',
        '25d55ad283aa400af464c76d713c07ad': '12345678',
        '7c6a180b36896a991383b64d67e48b34': 'admin',
        '098f6bcd4621d373cade4e832627b4f6': 'test'
    },
    'sha256': {
        '5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8': 'password',
        '8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92': '123456',
        'a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3': '123'
    }
}

FAKE_DOMAINS = {
    'jetables': ['test.com', 'example.com', 'mailinator.com', 'tempmail.com', 'fakeemail.com', 'yopmail.com', 'no-reply.com', 'guerrillamail.com', '10minutemail.com', 'throwaway.com'],
    'role_based': ['admin@', 'contact@', 'support@', 'info@', 'noreply@', 'no-reply@', 'webmaster@', 'postmaster@']
}

REGEX_PATTERNS = {
    'md5': re.compile(r'^[a-f0-9]{32}$', re.I),
    'sha256': re.compile(r'^[a-f0-9]{64}$', re.I),
    'email': re.compile(r'^[^\s@]+@[^\s@]+\.[^\s@]+$', re.I),
    'ipv4': re.compile(r'^(\d{1,3}\.){3}\d{1,3}$'),
    'ipv6': re.compile(r'^([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}$', re.I),
    'phone': re.compile(r'^[\+]?[(]?[0-9]{1,4}[)]?[-\s\.]?[(]?[0-9]{1,4}[)]?[-\s\.]?[0-9]{1,4}[-\s\.]?[0-9]{1,9}$'),
    'url': re.compile(r'^https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+'),
    'credit_card': re.compile(r'^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12})$'),
    'ssn': re.compile(r'^\d{3}-\d{2}-\d{4}$'),
    'date_iso': re.compile(r'^\d{4}-\d{2}-\d{2}$'),
    'timestamp': re.compile(r'^\d{10}$')
}

class LeakAnalyzer:
    """Analyseur scientifique avancé pour datasets de fuites de données"""
    
    def __init__(self, df: pd.DataFrame, filename: str):
        self.df = df
        self.filename = filename
        self.results = {}
        self.warnings = []
        self.scores = {}
        
    def calculate_entropy(self, series: pd.Series) -> float:
        """Calcule l'entropie de Shannon d'une série"""
        text = ' '.join(series.dropna().astype(str))
        if not text:
            return 0.0
        
        # Comptage des caractères
        char_count = Counter(text)
        text_length = len(text)
        
        # Calcul de l'entropie
        entropy = -sum((count / text_length) * np.log2(count / text_length) 
                      for count in char_count.values())
        return round(entropy, 3)
    
    def detect_column_type(self, series: pd.Series) -> Dict[str, Any]:
        """Détection avancée du type de données"""
        non_null = series.dropna()
        if len(non_null) == 0:
            return {'type': 'empty', 'confidence': 0}
        
        samples = non_null.head(100).astype(str)
        type_scores = defaultdict(int)
        
        for value in samples:
            for type_name, pattern in REGEX_PATTERNS.items():
                if pattern.match(value):
                    type_scores[type_name] += 1
        
        if type_scores:
            best_type = max(type_scores, key=type_scores.get)
            confidence = type_scores[best_type] / len(samples)
            return {'type': best_type, 'confidence': confidence}
        
        # Détection de types numériques
        if pd.api.types.is_numeric_dtype(series):
            return {'type': 'numeric', 'confidence': 0.9}
        
        # Détection de texte
        avg_len = samples.str.len().mean()
        if avg_len > 100:
            return {'type': 'long_text', 'confidence': 0.7}
        else:
            return {'type': 'short_text', 'confidence': 0.6}
    
    def analyze_temporal_dispersion(self, date_series: pd.Series) -> Dict[str, Any]:
        """Analyse avancée de la dispersion temporelle"""
        dates = pd.to_datetime(date_series, errors='coerce')
        dates = dates.dropna()
        
        if len(dates) < 2:
            return {'error': 'Pas assez de dates valides'}
        
        # Calcul des métriques
        min_date = dates.min()
        max_date = dates.max()
        date_range = max_date - min_date
        median_date = dates.median()
        
        # Détection de patterns
        date_diff = dates.diff().dropna()
        unique_gaps = date_diff.unique()
        
        # Test de distribution uniforme
        expected_uniform = pd.date_range(min_date, max_date, periods=len(dates))
        ks_stat = stats.ks_2samp(dates, expected_uniform) if len(dates) > 1 else (1, 0)
        
        # Détection de clusters temporels
        timestamps = dates.astype(np.int64) / 10**9
        clustering = DBSCAN(eps=86400*30, min_samples=min(10, len(dates)//10))  # 30 jours
        labels = clustering.fit_predict(timestamps.values.reshape(-1, 1))
        n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
        
        return {
            'min_date': min_date,
            'max_date': max_date,
            'range_days': date_range.days,
            'median_date': median_date,
            'unique_gaps': len(unique_gaps),
            'ks_statistic': ks_stat[0],
            'is_uniform': ks_stat[1] > 0.05,
            'n_clusters': n_clusters,
            'cluster_labels': labels.tolist(),
            'future_dates': (dates > datetime.now()).sum(),
            'invalid_dates': date_series.isna().sum(),
            'outliers': self.detect_temporal_outliers(dates)
        }
    
    def detect_temporal_outliers(self, dates: pd.Series) -> List[datetime]:
        """Détection des outliers temporels"""
        if len(dates) < 4:
            return []
        
        Q1 = dates.quantile(0.25)
        Q3 = dates.quantile(0.75)
        IQR = Q3 - Q1
        
        outliers = dates[(dates < Q1 - 1.5 * IQR) | (dates > Q3 + 1.5 * IQR)]
        return outliers.tolist()
    
    def analyze_hash_quality(self, hash_series: pd.Series, hash_type: str) -> Dict[str, Any]:
        """Analyse scientifique des qualités des hachages"""
        values = hash_series.dropna().astype(str)
        unique_hashes = values.nunique()
        total = len(values)
        
        # Diversité des hachages
        hash_diversity = unique_hashes / total if total > 0 else 0
        
        # Distribution des longueurs
        lengths = values.str.len()
        length_distribution = lengths.value_counts().to_dict()
        
        # Entropie moyenne
        avg_entropy = np.mean([self.calculate_entropy(pd.Series([v])) for v in values.head(1000)])
        
        # Détection de patterns suspects
        common_patterns = {
            'all_same': unique_hashes == 1,
            'too_many_identical': hash_diversity < 0.1,
            'abnormal_lengths': any(length != (32 if hash_type == 'md5' else 64) for length in length_distribution),
            'low_entropy': avg_entropy < 3.5,
            'sequential_patterns': self.detect_sequential_patterns(values)
        }
        
        # Vérification contre hashes communs
        common_hash_matches = {}
        for known_hash, known_password in COMMON_HASHES.get(hash_type, {}).items():
            count = (values == known_hash).sum()
            if count > 0:
                common_hash_matches[known_password] = count
        
        return {
            'total': total,
            'unique': unique_hashes,
            'diversity': hash_diversity,
            'avg_entropy': avg_entropy,
            'length_distribution': length_distribution,
            'suspicious_patterns': common_patterns,
            'common_hashes': common_hash_matches,
            'quality_score': min(100, (hash_diversity * 50) + (avg_entropy / 4.5 * 50))
        }
    
    def detect_sequential_patterns(self, series: pd.Series) -> bool:
        """Détecte si les données semblent générées séquentiellement"""
        sample = series.head(min(1000, len(series))).tolist()
        
        # Test sur les IDs numériques
        numeric_sample = []
        for val in sample:
            try:
                numeric_sample.append(int(str(val)))
            except:
                pass
        
        if len(numeric_sample) > 10:
            # Vérification de séquence parfaite
            is_sequential = all(numeric_sample[i] == numeric_sample[0] + i 
                              for i in range(min(100, len(numeric_sample))))
            return is_sequential
        
        return False
    
    def analyze_duplication_patterns(self) -> Dict[str, Any]:
        """Analyse approfondie des patterns de duplication"""
        # Duplication exacte
        exact_duplicates = self.df.duplicated().sum()
        duplicate_ratio = exact_duplicates / len(self.df)
        
        # Similarité de colonnes
        column_similarities = {}
        for i, col1 in enumerate(self.df.columns):
            for col2 in self.df.columns[i+1:]:
                if pd.api.types.is_string_dtype(self.df[col1]) and pd.api.types.is_string_dtype(self.df[col2]):
                    similarity = self.calculate_column_similarity(self.df[col1], self.df[col2])
                    if similarity > 0.8:
                        column_similarities[f"{col1} ↔ {col2}"] = similarity
        
        # Patterns de répétition par ligne
        row_patterns = {}
        for col in self.df.columns:
            value_counts = self.df[col].value_counts()
            most_common = value_counts.head(5).to_dict()
            row_patterns[col] = most_common
        
        return {
            'exact_duplicates': exact_duplicates,
            'duplicate_ratio': duplicate_ratio,
            'column_similarities': column_similarities,
            'row_patterns': row_patterns,
            'is_artificial': duplicate_ratio > 0.3 or len(column_similarities) > len(self.df.columns)
        }
    
    def calculate_column_similarity(self, col1: pd.Series, col2: pd.Series) -> float:
        """Calcule la similarité Jaccard entre deux colonnes"""
        non_null1 = set(col1.dropna().astype(str))
        non_null2 = set(col2.dropna().astype(str))
        
        if not non_null1 or not non_null2:
            return 0.0
        
        intersection = len(non_null1 & non_null2)
        union = len(non_null1 | non_null2)
        
        return intersection / union if union > 0 else 0.0
    
    def analyze_email_quality(self, email_series: pd.Series) -> Dict[str, Any]:
        """Analyse des emails pour détection d'anomalies"""
        emails = email_series.dropna().astype(str)
        
        # Extraction des domaines
        domains = emails.str.split('@').str[-1].value_counts()
        
        # Détection de domaines suspects
        suspicious_domains = {}
        for domain in domains.index:
            domain_lower = domain.lower()
            if any(fake in domain_lower for fake in FAKE_DOMAINS['jetables']):
                suspicious_domains[domain] = {
                    'count': domains[domain],
                    'reason': 'Domaine jetable'
                }
            elif any(role in domain_lower for role in FAKE_DOMAINS['role_based']):
                suspicious_domains[domain] = {
                    'count': domains[domain],
                    'reason': 'Email rôle'
                }
        
        # Analyse des patterns de nom d'utilisateur
        usernames = emails.str.split('@').str[0]
        username_patterns = {
            'only_numbers': (usernames.str.isdigit()).sum(),
            'too_short': (usernames.str.len() < 3).sum(),
            'too_long': (usernames.str.len() > 30).sum(),
            'special_chars': (usernames.str.contains(r'[^\w\.\-]')).sum()
        }
        
        # Calcul de diversité
        email_diversity = len(emails.unique()) / len(emails)
        
        return {
            'total_emails': len(emails),
            'unique_domains': len(domains),
            'top_domains': domains.head(10).to_dict(),
            'suspicious_domains': suspicious_domains,
            'username_patterns': username_patterns,
            'diversity_score': email_diversity * 100,
            'quality_score': min(100, (email_diversity * 50) + 
                               (1 - len(suspicious_domains) / max(1, len(domains))) * 50)
        }
    
    def analyze_data_freshness(self) -> Dict[str, Any]:
        """Analyse de la fraîcheur et de la cohérence des données"""
        freshness_indicators = {}
        
        # Recherche de timestamps de dernière mise à jour
        for col in self.df.columns:
            col_lower = col.lower()
            if any(keyword in col_lower for keyword in ['date', 'time', 'timestamp', 'updated', 'created', 'last_login']):
                dates = pd.to_datetime(self.df[col], errors='coerce')
                if len(dates.dropna()) > 0:
                    max_date = dates.max()
                    days_since = (datetime.now() - max_date).days if not pd.isna(max_date) else None
                    freshness_indicators[col] = {
                        'last_update': max_date,
                        'days_old': days_since,
                        'is_recent': days_since is not None and days_since < 30
                    }
        
        # Détection de dates dans le futur
        future_dates_count = 0
        for col in self.df.select_dtypes(include=['datetime64']).columns:
            future_dates_count += (self.df[col] > datetime.now()).sum()
        
        return {
            'freshness_indicators': freshness_indicators,
            'future_dates': future_dates_count,
            'has_recent_data': any(v.get('is_recent', False) for v in freshness_indicators.values())
        }
    
    def calculate_authenticity_score(self) -> Dict[str, Any]:
        """Calcule le score d'authenticité global avec pondération scientifique"""
        score_components = {}
        
        # 1. Analyse des hachages (poids 25%)
        hash_columns = []
        for col in self.df.columns:
            col_type = self.detect_column_type(self.df[col])
            if col_type['type'] in ['md5', 'sha256']:
                hash_analysis = self.analyze_hash_quality(self.df[col], col_type['type'])
                hash_columns.append(hash_analysis['quality_score'])
        
        score_components['hash_quality'] = np.mean(hash_columns) if hash_columns else 70
        
        # 2. Diversité des données (poids 20%)
        total_cells = len(self.df) * len(self.df.columns)
        unique_cells = sum(self.df[col].nunique() for col in self.df.columns)
        score_components['data_diversity'] = min(100, (unique_cells / total_cells) * 100)
        
        # 3. Absence de patterns artificiels (poids 20%)
        dup_analysis = self.analyze_duplication_patterns()
        artificial_penalty = dup_analysis['duplicate_ratio'] * 50 + len(dup_analysis['column_similarities']) * 5
        score_components['natural_patterns'] = max(0, 100 - artificial_penalty)
        
        # 4. Cohérence temporelle (poids 15%)
        date_columns = self.df.select_dtypes(include=['datetime64']).columns
        if len(date_columns) > 0:
            temp_analysis = self.analyze_temporal_dispersion(self.df[date_columns[0]])
            if 'is_uniform' in temp_analysis:
                time_score = 100 if temp_analysis['is_uniform'] else 50
                time_score -= min(30, temp_analysis.get('future_dates', 0) / len(self.df) * 100)
            else:
                time_score = 50
        else:
            time_score = 50
        score_components['temporal_coherence'] = time_score
        
        # 5. Qualité des emails (poids 10%)
        email_columns = [col for col in self.df.columns 
                        if self.detect_column_type(self.df[col])['type'] == 'email']
        if email_columns:
            email_analysis = self.analyze_email_quality(self.df[email_columns[0]])
            score_components['email_quality'] = email_analysis['quality_score']
        else:
            score_components['email_quality'] = 70
        
        # 6. Entropie globale (poids 10%)
        global_entropy = self.calculate_entropy(self.df.astype(str).sum(axis=1))
        score_components['global_entropy'] = min(100, (global_entropy / 8) * 100)
        
        # Calcul du score pondéré
        weights = {
            'hash_quality': 0.25,
            'data_diversity': 0.20,
            'natural_patterns': 0.20,
            'temporal_coherence': 0.15,
            'email_quality': 0.10,
            'global_entropy': 0.10
        }
        
        total_score = sum(score_components.get(k, 50) * weights[k] for k in weights)
        
        # Détermination du verdict
        if total_score >= 80:
            verdict = "✅ AUTHENTIQUE - Confiance élevée"
            color = "success"
        elif total_score >= 60:
            verdict = "⚠️ SUSPECT - Compilation possible"
            color = "warning"
        elif total_score >= 40:
            verdict = "🔴 FAIBLE - Artefacts de génération détectés"
            color = "danger"
        else:
            verdict = "🚨 FAUX-LEAK - Dataset synthétique ou fabrication"
            color = "critical"
        
        return {
            'total_score': round(total_score, 1),
            'components': score_components,
            'verdict': verdict,
            'color': color
        }
    
    def generate_forensic_report(self) -> Dict[str, Any]:
        """Génère un rapport forensique complet"""
        score_analysis = self.calculate_authenticity_score()
        
        report = {
            'file_info': {
                'filename': self.filename,
                'rows': len(self.df),
                'columns': len(self.df.columns),
                'memory_usage': self.df.memory_usage(deep=True).sum()
            },
            'columns_analysis': {},
            'authenticity': score_analysis,
            'warnings': [],
            'statistical_tests': {}
        }
        
        # Analyse détaillée par colonne
        for col in self.df.columns:
            col_type = self.detect_column_type(self.df[col])
            report['columns_analysis'][col] = {
                'type': col_type['type'],
                'null_ratio': self.df[col].isna().mean(),
                'unique_ratio': self.df[col].nunique() / len(self.df),
                'entropy': self.calculate_entropy(self.df[col])
            }
            
            # Tests spécifiques selon le type
            if col_type['type'] in ['md5', 'sha256']:
                hash_quality = self.analyze_hash_quality(self.df[col], col_type['type'])
                if hash_quality['suspicious_patterns']['low_entropy']:
                    report['warnings'].append(f"Colonne {col}: Entropie anormalement basse ({hash_quality['avg_entropy']})")
                if hash_quality['common_hashes']:
                    report['warnings'].append(f"Colonne {col}: {len(hash_quality['common_hashes'])} hash communs détectés")
            
            elif col_type['type'] == 'email':
                email_analysis = self.analyze_email_quality(self.df[col])
                if email_analysis['suspicious_domains']:
                    report['warnings'].append(f"Colonne {col}: {len(email_analysis['suspicious_domains'])} domaines suspects")
        
        # Tests statistiques globaux
        report['statistical_tests']['benford_law'] = self.test_benford_law()
        report['statistical_tests']['zipf_law'] = self.test_zipf_law()
        
        return report
    
    def test_benford_law(self) -> Dict[str, Any]:
        """Test de la loi de Benford sur les colonnes numériques"""
        numeric_cols = self.df.select_dtypes(include=[np.number]).columns
        results = {}
        
        for col in numeric_cols:
            # Extraction des premiers chiffres
            first_digits = self.df[col].dropna().astype(str).str[0]
            first_digits = first_digits[first_digits.str.isdigit()].astype(int)
            
            if len(first_digits) > 100:
                observed = first_digits.value_counts(normalize=True).sort_index()
                expected = np.log10(1 + 1/np.arange(1, 10))
                
                # Test du chi-deux
                chi2_stat = np.sum((observed.values - expected) ** 2 / expected)
                chi2_p = 1 - stats.chi2.cdf(chi2_stat, df=8)
                
                results[col] = {
                    'conforms': chi2_p > 0.05,
                    'chi2_pvalue': chi2_p,
                    'deviation': np.mean(np.abs(observed.values - expected))
                }
        
        return results
    
    def test_zipf_law(self) -> Dict[str, Any]:
        """Test de la loi de Zipf sur les fréquences"""
        results = {}
        
        for col in self.df.columns:
            frequencies = self.df[col].value_counts()
            if len(frequencies) > 100:
                ranks = np.arange(1, len(frequencies) + 1)
                log_ranks = np.log(ranks)
                log_freqs = np.log(frequencies.values)
                
                # Régression linéaire
                slope, intercept, r_value, p_value, std_err = stats.linregress(log_ranks, log_freqs)
                
                results[col] = {
                    'zipf_exponent': -slope,
                    'r_squared': r_value ** 2,
                    'follows_zipf': abs(slope + 1) < 0.2  # Exposant proche de -1
                }
        
        return results

def create_visualizations(analyzer: LeakAnalyzer):
    """Crée des visualisations avancées avec Plotly"""
    figs = {}
    
    # Distribution des types de données
    col_types = []
    for col in analyzer.df.columns:
        col_type = analyzer.detect_column_type(analyzer.df[col])
        col_types.append(col_type['type'])
    
    type_counts = Counter(col_types)
    figs['data_types'] = px.pie(
        values=list(type_counts.values()),
        names=list(type_counts.keys()),
        title="Distribution des types de données",
        color_discrete_sequence=px.colors.qualitative.Set3
    )
    
    # Heatmap de corrélation (colonnes numériques)
    numeric_cols = analyzer.df.select_dtypes(include=[np.number]).columns
    if len(numeric_cols) > 1:
        corr_matrix = analyzer.df[numeric_cols].corr()
        figs['correlation'] = px.imshow(
            corr_matrix,
            title="Matrice de corrélation",
            color_continuous_scale='RdBu',
            aspect='auto'
        )
    
    # Distribution des entropies par colonne
    entropies = {}
    for col in analyzer.df.columns:
        entropies[col] = analyzer.calculate_entropy(analyzer.df[col])
    
    entropy_df = pd.DataFrame([entropies]).melt(var_name='Colonne', value_name='Entropie')
    figs['entropy'] = px.bar(
        entropy_df,
        x='Colonne',
        y='Entropie',
        title="Entropie de Shannon par colonne",
        color='Entropie',
        color_continuous_scale='Viridis'
    )
    
    # Analyse temporelle si disponible
    date_cols = analyzer.df.select_dtypes(include=['datetime64']).columns
    if len(date_cols) > 0:
        temporal = analyzer.analyze_temporal_dispersion(analyzer.df[date_cols[0]])
        if 'min_date' in temporal:
            # Série temporelle
            dates = pd.to_datetime(analyzer.df[date_cols[0]], errors='coerce').dropna()
            if len(dates) > 0:
                fig_date = make_subplots(rows=2, cols=1, 
                                         subplot_titles=('Distribution temporelle', 'Fréquence mensuelle'))
                
                # Histogramme
                fig_date.add_trace(
                    go.Histogram(x=dates, nbinsx=50, name='Distribution'),
                    row=1, col=1
                )
                
                # Série de fréquences
                monthly_counts = dates.dt.to_period('M').value_counts().sort_index()
                fig_date.add_trace(
                    go.Scatter(x=monthly_counts.index.astype(str), y=monthly_counts.values,
                              mode='lines+markers', name='Fréquence mensuelle'),
                    row=2, col=1
                )
                
                fig_date.update_layout(height=600, title="Analyse temporelle des données")
                figs['temporal'] = fig_date
    
    return figs

def main():
    st.markdown("""
    <div class="main-header">
        <h1 style="color: white;">🕵️‍♂️ LeakDetector Pro - Édition Scientifique</h1>
        <p style="color: #cccccc;">Analyse forensique avancée des fuites de données | Détection de datasets artificiels</p>
    </div>
    """, unsafe_allow_html=True)
    
    # Sidebar
    with st.sidebar:
        st.markdown("## ⚙️ Configuration")
        analysis_depth = st.select_slider(
            "Profondeur d'analyse",
            options=['Rapide', 'Standard', 'Approfondie', 'Forensique'],
            value='Standard'
        )
        
        st.markdown("---")
        st.markdown("### 📊 Métriques activées")
        st.checkbox("Analyse des hachages", value=True, disabled=True)
        st.checkbox("Détection de patterns", value=True, disabled=True)
        st.checkbox("Tests statistiques", value=analysis_depth in ['Approfondie', 'Forensique'])
        st.checkbox("Visualisations avancées", value=True)
        st.checkbox("Rapport forensique", value=analysis_depth == 'Forensique')
        
        st.markdown("---")
        st.markdown("### ℹ️ À propos")
        st.info("""
        **LeakDetector Pro** utilise des méthodes scientifiques pour détecter:
        - Fabrication artificielle de données
        - Compilations suspectes
        - Patterns de génération scriptée
        - Anomalies statistiques
        """)
    
    # Zone principale
    uploaded_file = st.file_uploader(
        "📂 Charger un fichier de données (CSV, TSV, JSON, Excel)",
        type=['csv', 'tsv', 'json', 'xlsx', 'xls'],
        help="Formats supportés: CSV, TSV, JSON, Excel"
    )
    
    if uploaded_file is not None:
        # Lecture du fichier
        file_extension = uploaded_file.name.split('.')[-1].lower()
        
        try:
            if file_extension == 'csv':
                # Détection de l'encodage
                raw_data = uploaded_file.read()
                detected = chardet.detect(raw_data)
                uploaded_file.seek(0)
                df = pd.read_csv(uploaded_file, encoding=detected['encoding'])
            elif file_extension in ['xlsx', 'xls']:
                df = pd.read_excel(uploaded_file)
            elif file_extension == 'json':
                df = pd.read_json(uploaded_file)
            elif file_extension == 'tsv':
                df = pd.read_csv(uploaded_file, sep='\t')
            else:
                st.error(f"Format {file_extension} non supporté")
                return
            
            # Création de l'analyzer
            analyzer = LeakAnalyzer(df, uploaded_file.name)
            
            # Affichage des infos de base
            col1, col2, col3, col4 = st.columns(4)
            with col1:
                st.metric("📊 Lignes", f"{len(df):,}")
            with col2:
                st.metric("📋 Colonnes", len(df.columns))
            with col3:
                st.metric("💾 Taille", f"{uploaded_file.size / 1024:.1f} KB")
            with col4:
                missing = df.isna().sum().sum()
                st.metric("🔍 Données manquantes", f"{missing:,}")
            
            st.markdown("---")
            
            # Score d'authenticité
            score_analysis = analyzer.calculate_authenticity_score()
            
            col1, col2 = st.columns([1, 2])
            with col1:
                # Affichage du score avec gauge
                fig = go.Figure(go.Indicator(
                    mode = "gauge+number+delta",
                    value = score_analysis['total_score'],
                    domain = {'x': [0, 1], 'y': [0, 1]},
                    title = {'text': "Score d'authenticité"},
                    gauge = {
                        'axis': {'range': [0, 100]},
                        'bar': {'color': "darkblue"},
                        'steps': [
                            {'range': [0, 40], 'color': "red"},
                            {'range': [40, 60], 'color': "orange"},
                            {'range': [60, 80], 'color': "yellow"},
                            {'range': [80, 100], 'color': "green"}
                        ],
                        'threshold': {
                            'line': {'color': "red", 'width': 4},
                            'thickness': 0.75,
                            'value': 50
                        }
                    }
                ))
                fig.update_layout(height=300)
                st.plotly_chart(fig, use_container_width=True)
            
            with col2:
                st.markdown(f"""
                <div class="score-card">
                    <h3>Verdict Forensique</h3>
                    <h2 style="color: {'#00ff88' if 'AUTHENTIQUE' in score_analysis['verdict'] else '#ff4444'}">
                        {score_analysis['verdict']}
                    </h2>
                    <p>Niveau de confiance: {score_analysis['total_score']}%</p>
                </div>
                """, unsafe_allow_html=True)
            
            # Onglets pour analyses détaillées
            tab1, tab2, tab3, tab4, tab5 = st.tabs([
                "🔬 Analyse Détaillée", 
                "📈 Visualisations", 
                "🚨 Anomalies", 
                "🔐 Analyse des Hashes",
                "📋 Rapport"
            ])
            
            with tab1:
                st.subheader("Analyse par colonne")
                
                for col in df.columns:
                    col_type = analyzer.detect_column_type(df[col])
                    with st.expander(f"📊 {col} - Type: {col_type['type'].upper()}"):
                        metrics_col1, metrics_col2, metrics_col3 = st.columns(3)
                        with metrics_col1:
                            st.metric("Valeurs uniques", f"{df[col].nunique():,}")
                            st.metric("Taux d'unicité", f"{df[col].nunique()/len(df)*100:.1f}%")
                        with metrics_col2:
                            st.metric("Valeurs nulles", f"{df[col].isna().sum():,}")
                            st.metric("Taux de nullité", f"{df[col].isna().mean()*100:.1f}%")
                        with metrics_col3:
                            st.metric("Entropie", f"{analyzer.calculate_entropy(df[col]):.2f}")
                            if col_type['type'] in ['md5', 'sha256']:
                                hash_quality = analyzer.analyze_hash_quality(df[col], col_type['type'])
                                st.metric("Qualité hash", f"{hash_quality['quality_score']:.1f}%")
                        
                        # Aperçu des valeurs
                        st.markdown("**Top 10 des valeurs**")
                        st.dataframe(df[col].value_counts().head(10))
            
            with tab2:
                st.subheader("Visualisations Avancées")
                figs = create_visualizations(analyzer)
                
                for name, fig in figs.items():
                    st.plotly_chart(fig, use_container_width=True)
            
            with tab3:
                st.subheader("🚨 Détection d'Anomalies")
                
                # Collecte des anomalies
                anomalies = []
                
                # Vérification des doublons
                dup_analysis = analyzer.analyze_duplication_patterns()
                if dup_analysis['duplicate_ratio'] > 0.1:
                    anomalies.append({
                        'type': 'Duplication massive',
                        'severity': 'Élevée' if dup_analysis['duplicate_ratio'] > 0.3 else 'Moyenne',
                        'details': f"{dup_analysis['duplicate_ratio']*100:.1f}% de lignes dupliquées"
                    })
                
                # Vérification des colonnes
                for col in df.columns:
                    col_type = analyzer.detect_column_type(df[col])
                    
                    if col_type['type'] in ['md5', 'sha256']:
                        hash_quality = analyzer.analyze_hash_quality(df[col], col_type['type'])
                        if hash_quality['suspicious_patterns']['low_entropy']:
                            anomalies.append({
                                'type': f'Entropie basse - {col}',
                                'severity': 'Élevée',
                                'details': f"Entropie de {hash_quality['avg_entropy']} (normal: 3.8-4.5)"
                            })
                        
                        if hash_quality['common_hashes']:
                            anomalies.append({
                                'type': f'Hashs communs - {col}',
                                'severity': 'Moyenne',
                                'details': f"{len(hash_quality['common_hashes'])} hashs de mots de passe par défaut"
                            })
                    
                    elif col_type['type'] == 'email':
                        email_analysis = analyzer.analyze_email_quality(df[col])
                        if email_analysis['suspicious_domains']:
                            anomalies.append({
                                'type': f'Domaines suspects - {col}',
                                'severity': 'Faible',
                                'details': f"{len(email_analysis['suspicious_domains'])} domaines jetables ou rôle"
                            })
                
                # Affichage des anomalies
                if anomalies:
                    for anomaly in anomalies:
                        severity_color = {
                            'Élevée': '🔴',
                            'Moyenne': '🟡',
                            'Faible': '🟢'
                        }.get(anomaly['severity'], '⚪')
                        
                        st.markdown(f"""
                        <div class="metric-card">
                            <strong>{severity_color} {anomaly['type']}</strong><br/>
                            <span style="color: #cccccc;">Severité: {anomaly['severity']}</span><br/>
                            <small>{anomaly['details']}</small>
                        </div>
                        """, unsafe_allow_html=True)
                else:
                    st.success("✅ Aucune anomalie majeure détectée")
            
            with tab4:
                st.subheader("🔐 Analyse Forensique des Hachages")
                
                hash_columns = []
                for col in df.columns:
                    col_type = analyzer.detect_column_type(df[col])
                    if col_type['type'] in ['md5', 'sha256']:
                        hash_columns.append(col)
                
                if hash_columns:
                    for col in hash_columns:
                        st.markdown(f"### Colonne: {col}")
                        
                        hash_quality = analyzer.analyze_hash_quality(df[col], 
                            'md5' if REGEX_PATTERNS['md5'].match(str(df[col].iloc[0])) else 'sha256')
                        
                        met1, met2, met3, met4 = st.columns(4)
                        with met1:
                            st.metric("Total hachages", f"{hash_quality['total']:,}")
                        with met2:
                            st.metric("Hachages uniques", f"{hash_quality['unique']:,}")
                        with met3:
                            st.metric("Diversité", f"{hash_quality['diversity']*100:.1f}%")
                        with met4:
                            st.metric("Score qualité", f"{hash_quality['quality_score']:.1f}%")
                        
                        # Patterns suspects
                        suspicious = hash_quality['suspicious_patterns']
                        if any(suspicious.values()):
                            st.warning("⚠️ Patterns suspects détectés:")
                            for pattern, detected in suspicious.items():
                                if detected:
                                    pattern_name = pattern.replace('_', ' ').title()
                                    st.write(f"- {pattern_name}")
                        
                        # Hashs communs
                        if hash_quality['common_hashes']:
                            st.markdown("**🎯 Hashs de mots de passe par défaut détectés:**")
                            for pwd, count in hash_quality['common_hashes'].items():
                                st.write(f"- `{pwd}`: {count} occurrences")
                else:
                    st.info("Aucune colonne de hachage détectée dans ce dataset")
            
            with tab5:
                st.subheader("📋 Rapport Forensique Complet")
                
                if analysis_depth in ['Approfondie', 'Forensique']:
                    report = analyzer.generate_forensic_report()
                    
                    # Téléchargement du rapport
                    report_json = json.dumps(report, default=str, indent=2)
                    st.download_button(
                        label="📥 Télécharger le rapport (JSON)",
                        data=report_json,
                        file_name=f"forensic_report_{uploaded_file.name}.json",
                        mime="application/json"
                    )
                    
                    # Affichage des tests statistiques
                    st.markdown("## Tests Statistiques Avancés")
                    
                    # Loi de Benford
                    benford_results = report['statistical_tests']['benford_law']
                    if benford_results:
                        st.markdown("### Loi de Benford")
                        for col, result in benford_results.items():
                            st.markdown(f"""
                            **Colonne {col}**:
                            - Conformité: {'✅ Conforme' if result['conforms'] else '❌ Non conforme'}
                            - P-value Chi-2: {result['chi2_pvalue']:.4f}
                            - Déviation: {result['deviation']:.3f}
                            """)
                    
                    # Loi de Zipf
                    zipf_results = report['statistical_tests']['zipf_law']
                    if zipf_results:
                        st.markdown("### Loi de Zipf (Distribution des fréquences)")
                        for col, result in zipf_results.items():
                            st.markdown(f"""
                            **Colonne {col}**:
                            - Exposant: {result['zipf_exponent']:.2f}
                            - R²: {result['r_squared']:.3f}
                            - Suit la loi de Zipf: {'✅ Oui' if result['follows_zipf'] else '❌ Non'}
                            """)
                    
                    # Avertissements
                    if report['warnings']:
                        st.markdown("## ⚠️ Alertes Forensiques")
                        for warning in report['warnings']:
                            st.warning(warning)
                else:
                    st.info("Activez le mode 'Approfondie' ou 'Forensique' pour générer un rapport complet")
            
            # Prévisualisation des données
            with st.expander("📋 Prévisualisation des données (premières lignes)"):
                st.dataframe(df.head(100), use_container_width=True)
            
        except Exception as e:
            st.error(f"Erreur lors de l'analyse: {str(e)}")
            st.exception(e)
    
    else:
        # Message d'accueil
        st.markdown("""
        <div style="text-align: center; padding: 3rem;">
            <h2>🔬 Analyse Scientifique des Fuites de Données</h2>
            <p style="color: #cccccc; font-size: 1.1rem;">
                LeakDetector Pro utilise des méthodes forensiques avancées pour détecter<br>
                la fabrication artificielle, les compilations suspectes et les anomalies statistiques.
            </p>
            <br/>
            <h3>📊 Capacités d'analyse:</h3>
            <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 1rem; margin-top: 2rem;">
                <div>✅ Entropie de Shannon avancée</div>
                <div>✅ Tests statistiques (Benford, Zipf)</div>
                <div>✅ Détection de patterns générés</div>
                <div>✅ Analyse de similarité Jaccard</div>
                <div>✅ Clustering temporel avec DBSCAN</div>
                <div>✅ Visualisations interactives</div>
                <div>✅ Validation de hachages cryptographiques</div>
                <div>✅ Détection d'emails jetables</div>
                <div>✅ Matrice de corrélation</div>
            </div>
        </div>
        """, unsafe_allow_html=True)

if __name__ == "__main__":
    main()