From d39e3acc9ae5de26c6190f379c7d7f85f4faa844 Mon Sep 17 00:00:00 2001 From: Lorenzo Venerandi Date: Thu, 2 Apr 2026 17:16:44 +0200 Subject: [PATCH 1/4] fix: optimize IP fetching logic for improved performance and loading experience --- src/templates/static/js/map.js | 81 ++++++++++++++-------------------- 1 file changed, 32 insertions(+), 49 deletions(-) diff --git a/src/templates/static/js/map.js b/src/templates/static/js/map.js index 2b82d84..354d475 100644 --- a/src/templates/static/js/map.js +++ b/src/templates/static/js/map.js @@ -295,66 +295,49 @@ async function fetchAndBuildMap(limit) { }); attackerMap.addLayer(clusterGroup); - if (limit === 'all') { - // Stream pages of 1000, adding each batch with pop animation - let page = 1; - const pageSize = 1000; - while (true) { - const response = await fetch( - `${DASHBOARD_PATH}/api/all-ips?page=${page}&page_size=${pageSize}&sort_by=total_requests&sort_order=desc`, - { cache: 'no-store', headers } - ); - if (!response.ok) throw new Error('Failed to fetch IPs'); - const data = await response.json(); - const batch = data.ips || []; - allIps = allIps.concat(batch); - - // Build markers for this batch with pop animation - const batchMarkers = []; - batch.forEach(ip => { - if (!ip.country_code || !ip.category) return; - const marker = _createIpMarker(ip, true); - if (marker) { - mapMarkers.push(marker); - batchMarkers.push(marker); - } - }); - - // Add batch to cluster group (triggers pop-in) - if (batchMarkers.length > 0) { - const visible = batchMarkers.filter(m => !hiddenCategories.has(m.options.category)); - clusterGroup.addLayers(visible); - } - - // Update overlay counter - const overlay = document.getElementById('map-loading-overlay'); - if (overlay) overlay.textContent = `Loading IPs... ${allIps.length} loaded`; + const pageSize = 5000; + const maxIps = limit === 'all' ? Infinity : parseInt(limit, 10); + let page = 1; - if (page >= data.pagination.total_pages) break; - page++; + while (true) { + const fetchSize = Math.min(pageSize, maxIps - allIps.length); + if (fetchSize <= 0) break; - // Small delay to let the browser paint the batch - await new Promise(r => setTimeout(r, 80)); - } - } else { - // Single fetch, no animation needed - const pageSize = parseInt(limit, 10); const response = await fetch( - `${DASHBOARD_PATH}/api/all-ips?page=1&page_size=${pageSize}&sort_by=total_requests&sort_order=desc`, + `${DASHBOARD_PATH}/api/all-ips?page=${page}&page_size=${fetchSize}&sort_by=total_requests&sort_order=desc`, { cache: 'no-store', headers } ); if (!response.ok) throw new Error('Failed to fetch IPs'); const data = await response.json(); - allIps = data.ips || []; + const batch = data.ips || []; + allIps = allIps.concat(batch); - allIps.forEach(ip => { + // Build markers for this batch with pop animation + const batchMarkers = []; + batch.forEach(ip => { if (!ip.country_code || !ip.category) return; - const marker = _createIpMarker(ip, false); - if (marker) mapMarkers.push(marker); + const marker = _createIpMarker(ip, true); + if (marker) { + mapMarkers.push(marker); + batchMarkers.push(marker); + } }); - const visible = mapMarkers.filter(m => !hiddenCategories.has(m.options.category)); - clusterGroup.addLayers(visible); + // Add batch to cluster group (triggers pop-in) + if (batchMarkers.length > 0) { + const visible = batchMarkers.filter(m => !hiddenCategories.has(m.options.category)); + clusterGroup.addLayers(visible); + } + + // Update overlay counter + const overlay = document.getElementById('map-loading-overlay'); + if (overlay) overlay.textContent = `Loading IPs... ${allIps.length} loaded`; + + if (batch.length < fetchSize || page >= data.pagination.total_pages) break; + page++; + + // Small delay to let the browser paint the batch + await new Promise(r => setTimeout(r, 80)); } // Fit bounds to visible markers From fb13e287c3828ca1c67690fdd86d079dd1f615db Mon Sep 17 00:00:00 2001 From: Lorenzo Venerandi Date: Thu, 2 Apr 2026 17:18:00 +0200 Subject: [PATCH 2/4] fix: enhance attack trends panel with time span selection and improved navigation --- src/templates/jinja2/dashboard/index.html | 15 +++++++++++---- src/templates/static/js/charts.js | 11 ++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/templates/jinja2/dashboard/index.html b/src/templates/jinja2/dashboard/index.html index e3cee98..09f2bf9 100644 --- a/src/templates/jinja2/dashboard/index.html +++ b/src/templates/jinja2/dashboard/index.html @@ -141,10 +141,17 @@

Generated Deception Templates

Attack Trends (click legend to filter table)

-
- - Last 30 days - +
+ +
+ + Last 30 days + +
diff --git a/src/templates/static/js/charts.js b/src/templates/static/js/charts.js index 2de77de..c25d728 100644 --- a/src/templates/static/js/charts.js +++ b/src/templates/static/js/charts.js @@ -187,7 +187,7 @@ async function loadAttackTypesChart(canvasId, ipFilter, legendPosition) { */ let attackTrendsChart = null; let _trendsOffsetDays = 0; -const _trendsDays = 30; +let _trendsDays = 30; // Hash-based consistent colors (shared with doughnut chart) function _trendsHashCode(str) { @@ -382,6 +382,15 @@ function shiftTrendsPeriod(direction) { loadAttackTrendsChart(); } +/** Switch the trends time span (7, 30, 90 days) and reset to current period */ +function setTrendsSpan(days, btn) { + _trendsDays = days; + _trendsOffsetDays = 0; + document.querySelectorAll('#trends-span-selector .map-limit-btn').forEach(b => b.classList.remove('active')); + if (btn) btn.classList.add('active'); + loadAttackTrendsChart(); +} + /** Active attack type filter (null = show all) */ let _activeAttackTypeFilter = null; From a025b9a3ee41a2bb994ad25629925542e4d69455 Mon Sep 17 00:00:00 2001 From: Lorenzo Venerandi Date: Thu, 2 Apr 2026 17:31:46 +0200 Subject: [PATCH 3/4] fix: enhance attack trends and chart rendering with hourly granularity support --- src/database.py | 141 +++++++++++++++------- src/templates/jinja2/dashboard/index.html | 2 +- src/templates/static/js/charts.js | 8 ++ 3 files changed, 105 insertions(+), 46 deletions(-) diff --git a/src/database.py b/src/database.py index 368551d..22cec26 100644 --- a/src/database.py +++ b/src/database.py @@ -2414,7 +2414,8 @@ def get_attack_types_daily( self, limit: int = 10, days: int = 30, offset_days: int = 0 ) -> Dict[str, Any]: """ - Get daily attack type counts for a sliding window (for line chart). + Get attack type counts for a sliding window (for line chart). + Uses hourly granularity for spans <= 7 days, daily otherwise. Args: limit: Max attack types to return @@ -2422,7 +2423,7 @@ def get_attack_types_daily( offset_days: How many days back to shift the window end (0 = ending today, 30 = ending 30 days ago, etc.) - Returns top N attack types with their daily breakdown and totals. + Returns top N attack types with their breakdown and totals. """ session = self.session try: @@ -2430,6 +2431,7 @@ def get_attack_types_daily( end = datetime.now() - timedelta(days=offset_days) cutoff = end - timedelta(days=days) + use_hourly = days <= 7 # Time range filter used by both queries time_filter = [ @@ -2457,54 +2459,103 @@ def get_attack_types_daily( top_type_names = [row.attack_type for row in top_types_q] totals = {row.attack_type: row.total for row in top_types_q} - # Build list of dates in the period - dates = [] - for i in range(days, -1, -1): - d = (end - timedelta(days=i)).strftime("%Y-%m-%d") - dates.append(d) + if use_hourly: + # Hourly granularity: build list of hour slots + slots = [] + total_hours = days * 24 + for i in range(total_hours, -1, -1): + slot = (end - timedelta(hours=i)).strftime("%Y-%m-%d %H:00") + slots.append(slot) + + # Group by date + hour, portable across SQLite and PostgreSQL + # strftime works on SQLite, to_char on PostgreSQL + from sqlalchemy import literal_column + is_sqlite = "sqlite" in str(session.bind.url) + if is_sqlite: + hour_expr = func.strftime("%Y-%m-%d %H:00", AccessLog.timestamp) + else: + hour_expr = func.to_char(AccessLog.timestamp, "YYYY-MM-DD HH24:00") - # Get daily breakdown for those types using func.date() for portability - day_expr = func.date(AccessLog.timestamp) - daily_q = ( - session.query( - AttackDetection.attack_type, - day_expr.label("day"), - func.count(AttackDetection.id).label("count"), - ) - .join(AccessLog, AttackDetection.access_log_id == AccessLog.id) - .filter( - *time_filter, - AttackDetection.attack_type.in_(top_type_names), + hourly_q = ( + session.query( + AttackDetection.attack_type, + hour_expr.label("slot"), + func.count(AttackDetection.id).label("count"), + ) + .join(AccessLog, AttackDetection.access_log_id == AccessLog.id) + .filter( + *time_filter, + AttackDetection.attack_type.in_(top_type_names), + ) + .group_by(AttackDetection.attack_type, hour_expr) + .all() ) - .group_by(AttackDetection.attack_type, day_expr) - .all() - ) - # Build daily data per attack type - daily_data = {t: {d: 0 for d in dates} for t in top_type_names} - for row in daily_q: - # row.day may be a date object or string depending on the DB backend - day_str = ( - row.day.strftime("%Y-%m-%d") - if hasattr(row.day, "strftime") - else str(row.day) + slot_data = {t: {s: 0 for s in slots} for t in top_type_names} + for row in hourly_q: + slot_str = str(row.slot) + if row.attack_type in slot_data and slot_str in slot_data[row.attack_type]: + slot_data[row.attack_type][slot_str] = row.count + + return { + "attack_types": [ + { + "type": t, + "total": totals[t], + "daily": [slot_data[t][s] for s in slots], + } + for t in top_type_names + ], + "dates": slots, + } + else: + # Daily granularity + dates = [] + for i in range(days, -1, -1): + d = (end - timedelta(days=i)).strftime("%Y-%m-%d") + dates.append(d) + + # Get daily breakdown for those types using func.date() for portability + day_expr = func.date(AccessLog.timestamp) + daily_q = ( + session.query( + AttackDetection.attack_type, + day_expr.label("day"), + func.count(AttackDetection.id).label("count"), + ) + .join(AccessLog, AttackDetection.access_log_id == AccessLog.id) + .filter( + *time_filter, + AttackDetection.attack_type.in_(top_type_names), + ) + .group_by(AttackDetection.attack_type, day_expr) + .all() ) - if ( - row.attack_type in daily_data - and day_str in daily_data[row.attack_type] - ): - daily_data[row.attack_type][day_str] = row.count - return { - "attack_types": [ - { - "type": t, - "total": totals[t], - "daily": [daily_data[t][d] for d in dates], - } - for t in top_type_names - ], - "dates": dates, + # Build daily data per attack type + daily_data = {t: {d: 0 for d in dates} for t in top_type_names} + for row in daily_q: + day_str = ( + row.day.strftime("%Y-%m-%d") + if hasattr(row.day, "strftime") + else str(row.day) + ) + if ( + row.attack_type in daily_data + and day_str in daily_data[row.attack_type] + ): + daily_data[row.attack_type][day_str] = row.count + + return { + "attack_types": [ + { + "type": t, + "total": totals[t], + "daily": [daily_data[t][d] for d in dates], + } + for t in top_type_names + ], + "dates": dates, } finally: self.close_session() diff --git a/src/templates/jinja2/dashboard/index.html b/src/templates/jinja2/dashboard/index.html index 09f2bf9..2cd25b4 100644 --- a/src/templates/jinja2/dashboard/index.html +++ b/src/templates/jinja2/dashboard/index.html @@ -143,9 +143,9 @@

Generated Deception Templates

Attack Trends (click legend to filter table)

diff --git a/src/templates/static/js/charts.js b/src/templates/static/js/charts.js index c25d728..2eb269e 100644 --- a/src/templates/static/js/charts.js +++ b/src/templates/static/js/charts.js @@ -259,7 +259,15 @@ async function loadAttackTrendsChart(canvasId) { const oldMsg = canvas.parentElement.querySelector('.trends-empty-msg'); if (oldMsg) oldMsg.style.display = 'none'; + const isHourly = dates.length > 0 && dates[0].includes(':'); const shortLabels = dates.map(d => { + if (isHourly) { + // "2026-04-02 14:00" → "Apr 2 14:00" + const [datePart, timePart] = d.split(' '); + const dt = new Date(datePart + 'T00:00:00'); + const dayLabel = dt.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); + return `${dayLabel} ${timePart}`; + } const dt = new Date(d + 'T00:00:00'); return dt.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); }); From 6316611c0a0c48f5205cb3c00180c2c252a6da13 Mon Sep 17 00:00:00 2001 From: Lorenzo Venerandi Date: Thu, 2 Apr 2026 17:34:27 +0200 Subject: [PATCH 4/4] linted code --- src/database.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/database.py b/src/database.py index 22cec26..7a99fb7 100644 --- a/src/database.py +++ b/src/database.py @@ -2470,6 +2470,7 @@ def get_attack_types_daily( # Group by date + hour, portable across SQLite and PostgreSQL # strftime works on SQLite, to_char on PostgreSQL from sqlalchemy import literal_column + is_sqlite = "sqlite" in str(session.bind.url) if is_sqlite: hour_expr = func.strftime("%Y-%m-%d %H:00", AccessLog.timestamp) @@ -2494,7 +2495,10 @@ def get_attack_types_daily( slot_data = {t: {s: 0 for s in slots} for t in top_type_names} for row in hourly_q: slot_str = str(row.slot) - if row.attack_type in slot_data and slot_str in slot_data[row.attack_type]: + if ( + row.attack_type in slot_data + and slot_str in slot_data[row.attack_type] + ): slot_data[row.attack_type][slot_str] = row.count return { @@ -2556,7 +2560,7 @@ def get_attack_types_daily( for t in top_type_names ], "dates": dates, - } + } finally: self.close_session()