From 933ab3daa4eef17f9e2cf7e586baefd91aaa2505 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 14 Jun 2026 23:47:06 +0200 Subject: [PATCH 01/38] =?UTF-8?q?feat(coverage):=20frontend=20=E2=80=94=20?= =?UTF-8?q?Coverage=20dashboard=20+=20Reach=20overlay=20(flag-gated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- public/index.html | 4 + public/node-reach-coverage.css | 24 +++++ public/node-reach-coverage.js | 56 +++++++++++ public/node-reach.js | 45 ++++++++- public/roles.js | 5 + public/rx-coverage.js | 172 ++++++++++++++++++++++++++++++++ test-coverage-gate.js | 57 +++++++++++ test-node-reach-coverage-e2e.js | 56 +++++++++++ test-node-reach-coverage.js | 28 ++++++ 9 files changed, 446 insertions(+), 1 deletion(-) create mode 100644 public/node-reach-coverage.css create mode 100644 public/node-reach-coverage.js create mode 100644 public/rx-coverage.js create mode 100644 test-coverage-gate.js create mode 100644 test-node-reach-coverage-e2e.js create mode 100644 test-node-reach-coverage.js diff --git a/public/index.html b/public/index.html index 8f3a10a2..70fddcf6 100644 --- a/public/index.html +++ b/public/index.html @@ -40,6 +40,7 @@ + Tools Observers Analytics + Coverage Perf Lab @@ -218,7 +220,9 @@ + + diff --git a/public/node-reach-coverage.css b/public/node-reach-coverage.css new file mode 100644 index 00000000..4e4536a6 --- /dev/null +++ b/public/node-reach-coverage.css @@ -0,0 +1,24 @@ +/* Client-RX coverage styles (fork-only feature). Kept in a DEDICATED file — + separate from node-reach.css — so upstream's periodic node-reach.css rewrites + don't drop the coverage tier colours + leaderboard layout (as happened in the + v3.9.0 merge). Consumed by rx-coverage.js (standalone Coverage dashboard) and, + once re-grafted, the per-node Reach page coverage overlay. */ + +/* RX coverage hex layer (mobile client receptions) */ +:root { + --nq-cov-strong: #2ecc71; /* SF8: SNR ≥ −5 dB (good margin) */ + --nq-cov-mid: #e67e22; /* SF8: −9..−5 dB (near the limit) */ + --nq-cov-weak: #e74c3c; /* SF8: < −9 dB (poor, packet loss likely) */ + --nq-cov-grey: #95a5a6; /* heard, no SNR metric */ +} +.nq-cov-legend { display:flex; gap:12px; align-items:center; font-size:11px; color:var(--text-muted, #57606a); margin:4px 0 10px; } +.nq-cov-legend i { width:12px; height:12px; border-radius:2px; display:inline-block; margin-right:4px; vertical-align:middle; } + +/* Mobile RX coverage hub — leaderboard */ +.rxb { display:flex; flex-direction:column; gap:3px; } +.rxb-row { display:flex; align-items:center; gap:10px; padding:7px 10px; background:var(--section-bg, #f6f8fa); border:1px solid var(--border, #d0d7de); border-radius:6px; font-size:13px; cursor:pointer; } +.rxb-row.rxb-head { font-size:10px; text-transform:uppercase; color:var(--text-muted, #57606a); cursor:default; } +.rxb-row.sel { outline:2px solid var(--accent, #2ecc71); } +.rxb-rank { width:24px; text-align:right; color:var(--text-muted, #57606a); } +.rxb-name { flex:1; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } +.rxb-rec, .rxb-nodes { width:50px; text-align:right; font-variant-numeric:tabular-nums; } diff --git a/public/node-reach-coverage.js b/public/node-reach-coverage.js new file mode 100644 index 00000000..b9b3913a --- /dev/null +++ b/public/node-reach-coverage.js @@ -0,0 +1,56 @@ +/* === CoreScope — node-reach-coverage.js === + Draws per-node mobile RX coverage as an H3-style hex layer on the existing + Reach Leaflet map, from the /api/nodes/{pubkey}/rx-coverage GeoJSON. + No external deps; colours via CSS variables. */ +'use strict'; +(function () { + // coverageColorVar maps a GeoJSON feature's properties to a CSS variable name. + // Grey = received but no signal metric; otherwise SF8 SNR thresholds: ≥ −5 green + // (good margin), −9..−5 orange (near the limit), < −9 red (packet loss likely). + function coverageColorVar(props) { + if (!props || !props.has_sig || props.best_snr == null) return '--nq-cov-grey'; + var s = Number(props.best_snr); + if (s >= -5) return '--nq-cov-strong'; + if (s >= -9) return '--nq-cov-mid'; + return '--nq-cov-weak'; + } + + function cssColor(varName) { + try { + var v = getComputedStyle(document.documentElement).getPropertyValue(varName).trim(); + return v || '#888'; + } catch (e) { return '#888'; } + } + + // addLayer fetches coverage for the current map bbox/zoom and draws hex + // polygons. Returns a handle with off() so the caller can remove it. + function addLayer(map, pubkey) { + var group = L.layerGroup().addTo(map); + function refresh() { + var b = map.getBounds(); + var bbox = [b.getSouth(), b.getWest(), b.getNorth(), b.getEast()].join(','); + var url = '/api/nodes/' + encodeURIComponent(pubkey) + '/rx-coverage?bbox=' + bbox + '&z=' + map.getZoom(); + fetch(url).then(function (r) { return r.json(); }).then(function (fc) { + group.clearLayers(); + (fc.features || []).forEach(function (f) { + var ring = (f.geometry.coordinates[0] || []).map(function (c) { return [c[1], c[0]]; }); // [lon,lat]→[lat,lon] + var col = cssColor(coverageColorVar(f.properties)); + L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: 0.45 }) + .addTo(group) + .bindTooltip('n=' + f.properties.count + + (f.properties.best_snr != null ? ' · SNR ' + f.properties.best_snr : ' · no signal')); + }); + }).catch(function () { /* leave layer empty on error; never crash the reach page */ }); + } + map.on('moveend zoomend', refresh); + refresh(); + return { + off: function () { + map.off('moveend zoomend', refresh); + try { map.removeLayer(group); } catch (e) {} + } + }; + } + + window.NodeReachCoverage = { coverageColorVar: coverageColorVar, addLayer: addLayer }; +})(); diff --git a/public/node-reach.js b/public/node-reach.js index ecf1b03c..9ba09148 100644 --- a/public/node-reach.js +++ b/public/node-reach.js @@ -8,6 +8,16 @@ var current = null; var loadGen = 0; // bumped per load + on destroy; guards against in-flight races var DEFAULT_DAYS = 7; // single JS source for the default window (mirrors the server default) + var coverageOn = false; // mobile-client RX coverage hex layer (deep-linked ?coverage=1) + var covHandle = null; // handle from NodeReachCoverage.addLayer (off() to remove) + + // setCoverageHash reflects the coverage toggle in the URL hash without + // re-triggering the router (history.replaceState, not location.hash =). + function setCoverageHash(on) { + var h = location.hash.replace(/([?&])coverage=1/, '$1').replace(/[?&]$/, ''); + if (on) { h += (h.indexOf('?') >= 0 ? '&' : '?') + 'coverage=1'; } + try { history.replaceState(null, '', h || location.pathname + location.search); } catch (e) {} + } // Single source of the bottleneck tiers: colour + threshold + colour-blind // glyph + legend text. The map legend and the table both read from this. @@ -105,6 +115,8 @@ async function load(container, pubkey, days, isInitial) { var myGen = ++loadGen; current = { pubkey: pubkey, days: days }; + coverageOn = window.MC_CLIENT_RX_COVERAGE === true && (typeof getHashParams === 'function' && getHashParams().get('coverage') === '1'); + if (covHandle) { try { covHandle.off(); } catch (e) {} covHandle = null; } if (qmap) { qmap.destroy(); qmap = null; } if (isInitial) { container.innerHTML = '
Loading reach…
'; @@ -172,9 +184,16 @@ '' + '' + '' + + (window.MC_CLIENT_RX_COVERAGE ? '' : '') + '' + '' + '' + + (window.MC_CLIENT_RX_COVERAGE ? '
' + + 'strong' + + 'medium' + + 'weak' + + 'no signal' + + '
' : '') + '
' + '
' + '' + @@ -187,6 +206,7 @@ qmap = window.NodeReachMap.render('nqMap', n, TIERS); } + var lastList = []; // most recent filtered link list (so the coverage toggle can restore lines) // Two-way links are always shown; the two checkboxes add the asymmetric ones. function paint() { var inc = document.getElementById('nqIncoming').checked; @@ -205,13 +225,35 @@ var noGps = list.filter(function (l) { return l.lat == null || l.lon == null; }).length; document.getElementById('nqNoGps').textContent = noGps ? noGps + ' link' + (noGps === 1 ? '' : 's') + ' have no location and are not drawn on the map.' : ''; - if (qmap) qmap.setLinks(list); + lastList = list; + if (qmap) qmap.setLinks(coverageOn ? [] : list); } paint(); document.getElementById('nqIncoming').addEventListener('change', paint); document.getElementById('nqOutgoing').addEventListener('change', paint); document.getElementById('nqPrintBtn').addEventListener('click', printReport); + // Coverage overlay (fork): toggles the mobile-client RX hex layer in place and + // hides the link lines under it (declutter) — the table still lists every link. + var covLegend = document.getElementById('nqCovLegend'); + function applyCoverage() { + if (covLegend) covLegend.style.display = coverageOn ? 'flex' : 'none'; + if (coverageOn) { + if (qmap && window.NodeReachCoverage && !covHandle) covHandle = window.NodeReachCoverage.addLayer(qmap.map, pubkey); + } else if (covHandle) { + try { covHandle.off(); } catch (e) {} + covHandle = null; + } + if (qmap) qmap.setLinks(coverageOn ? [] : lastList); + } + var covCb = document.getElementById('nqCoverage'); + if (covCb) covCb.addEventListener('change', function (e) { + coverageOn = e.target.checked; + setCoverageHash(coverageOn); + applyCoverage(); + }); + if (coverageOn) applyCoverage(); // deep-linked ?coverage=1: add layer + hide lines now + wireTimeRange(container, pubkey); } @@ -225,6 +267,7 @@ function destroy() { loadGen++; // invalidate any in-flight load so it won't mutate a foreign container + if (covHandle) { try { covHandle.off(); } catch (e) {} covHandle = null; } if (qmap) { qmap.destroy(); qmap = null; } current = null; } diff --git a/public/roles.js b/public/roles.js index 57c7f303..6af73efc 100644 --- a/public/roles.js +++ b/public/roles.js @@ -533,6 +533,11 @@ // ─── Fetch server overrides ─── window.MeshConfigReady = fetch('/api/config/client').then(function (r) { return r.json(); }).then(function (cfg) { + window.MC_CLIENT_RX_COVERAGE = cfg.clientRxCoverage === true; + if (!window.MC_CLIENT_RX_COVERAGE) { + var covNav = document.querySelector('[data-route="rx-coverage"]'); + if (covNav) covNav.style.display = 'none'; + } if (cfg.roles) { if (cfg.roles.colors) { // #1407 — ROLE_COLORS is now a live getter; merge into the override map. diff --git a/public/rx-coverage.js b/public/rx-coverage.js new file mode 100644 index 00000000..de070890 --- /dev/null +++ b/public/rx-coverage.js @@ -0,0 +1,172 @@ +/* === CoreScope — rx-coverage.js === + Mobile RX coverage hub (route #/rx-coverage): + - global H3-style hex coverage map (all mobile observers), time-windowed + - leaderboard of top mobile observers (companion name + counts) + - click an observer to filter the map to just their coverage + Fork-only feature; isolated page (no changes to the core map). */ +'use strict'; +(function () { + var map = null, covLayer = null, days = 7, selectedRx = '', selectedName = '', boardCache = [], destroyed = false; + + function cssColor(varName) { + try { return getComputedStyle(document.documentElement).getPropertyValue(varName).trim() || '#888'; } + catch (e) { return '#888'; } + } + // SF8 SNR thresholds: ≥ −5 good margin, −9..−5 near the limit, < −9 packet loss + // likely. Grey = heard but no SNR metric. + function colorVar(p) { + if (!p || !p.has_sig || p.best_snr == null) return '--nq-cov-grey'; + var s = Number(p.best_snr); + if (s >= -5) return '--nq-cov-strong'; + if (s >= -9) return '--nq-cov-mid'; + return '--nq-cov-weak'; + } + + function dayBtn(d) { return ''; } + + function pageHtml() { + return '
' + + '

🗺️ Mobile RX coverage

' + + '
Where roaming CoreScope-RX clients heard nodes. Colour = best signal per cell.
' + + '
' + dayBtn(1) + dayBtn(7) + dayBtn(14) + dayBtn(30) + '
' + + '
strongmediumweakno signal
' + + '
' + + '
Top mobile observers
' + + '
' + + '
'; + } + + // coverageNodeRow renders one heard node: name (or heard_key prefix) + latest SNR + count. + function coverageNodeRow(n) { + var label = n.name ? escapeHtml(n.name) : '' + escapeHtml(n.prefix || '?') + ''; + var snr = (n.snr != null) ? Number(n.snr).toFixed(1) + ' dB' : 'no sig'; + return '
' + + '' + label + '' + + '' + snr + ' · ×' + n.count + '
'; + } + + // coverageNodesHtml lists the nodes directly heard in a cell (properties.nodes: + // {prefix, name, snr, count}, strongest latest-SNR first; prefix shown when the + // name is unresolved). Rendered in the hover tooltip; capped at 10 rows with a + // "(N more)" footer so dense cells don't produce an unwieldy tooltip. + var COVERAGE_NODE_CAP = 10; + function coverageNodesHtml(p) { + var nodes = (p && p.nodes) || []; + var head = '
' + + nodes.length + (nodes.length === 1 ? ' node heard here' : ' nodes heard here') + '
'; + if (!nodes.length) return head + '
n=' + (p ? p.count : 0) + '
'; + var rows = nodes.slice(0, COVERAGE_NODE_CAP).map(coverageNodeRow).join(''); + var more = (nodes.length > COVERAGE_NODE_CAP) + ? '
(' + (nodes.length - COVERAGE_NODE_CAP) + ' more)
' + : ''; + return head + '
' + rows + '
' + more; + } + + function drawCoverage() { + if (!map || destroyed) return; + var b = map.getBounds(); + var bbox = [b.getSouth(), b.getWest(), b.getNorth(), b.getEast()].join(','); + var url = '/api/rx-coverage?bbox=' + bbox + '&z=' + map.getZoom() + '&days=' + days + (selectedRx ? '&rx=' + encodeURIComponent(selectedRx) : ''); + fetch(url).then(function (r) { return r.json(); }).then(function (fc) { + if (destroyed || !covLayer) return; + covLayer.clearLayers(); + (fc.features || []).forEach(function (f) { + var ring = (f.geometry.coordinates[0] || []).map(function (c) { return [c[1], c[0]]; }); + var col = cssColor(colorVar(f.properties)); + L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: 0.45 }).addTo(covLayer) + .bindTooltip(coverageNodesHtml(f.properties)); + }); + }).catch(function () {}); + } + + function renderBoard() { + var el = document.getElementById('rxBoard'); + if (!el) return; + if (!boardCache.length) { el.innerHTML = '
No mobile observers in this window yet.
'; return; } + var rows = boardCache.map(function (o, i) { + var nm = o.name ? escapeHtml(o.name) : (o.pubkey.slice(0, 10) + '…'); + return '
' + + '' + (i + 1) + '' + nm + '' + + '' + o.receptions + '' + o.nodes + '
'; + }).join(''); + el.innerHTML = (selectedRx ? '' : '') + + '
#Observer (companion)pktsnodes
' + rows; + el.querySelectorAll('.rxb-row[data-rx]').forEach(function (r) { + r.addEventListener('click', function () { + selectedRx = r.dataset.rx; selectedName = r.dataset.name || ''; + renderBoard(); fitToObserver(); syncHash(); + }); + }); + var all = document.getElementById('rxAll'); + if (all) all.addEventListener('click', function () { selectedRx = ''; selectedName = ''; renderBoard(); drawCoverage(); syncHash(); }); + } + + // fitToObserver zooms the map to the selected observer's full coverage extent + // (fetched with a world bbox so it's independent of the current view), then the + // resulting moveend redraws the hexes at the fitted resolution. + function fitToObserver() { + if (!map || !selectedRx) { drawCoverage(); return; } + var url = '/api/rx-coverage?bbox=-90,-180,90,180&z=' + Math.max(8, map.getZoom()) + '&days=' + days + '&rx=' + encodeURIComponent(selectedRx); + fetch(url).then(function (r) { return r.json(); }).then(function (fc) { + if (destroyed || !map) return; + var minLat = 90, minLon = 180, maxLat = -90, maxLon = -180, any = false; + (fc.features || []).forEach(function (f) { + (f.geometry.coordinates[0] || []).forEach(function (c) { + any = true; + if (c[1] < minLat) minLat = c[1]; if (c[1] > maxLat) maxLat = c[1]; + if (c[0] < minLon) minLon = c[0]; if (c[0] > maxLon) maxLon = c[0]; + }); + }); + if (!any) { drawCoverage(); return; } // observer has no data in window → keep view + map.fitBounds([[minLat, minLon], [maxLat, maxLon]], { padding: [30, 30], maxZoom: 15 }); + drawCoverage(); // fitBounds may not fire moveend if the view is unchanged + }).catch(function () { drawCoverage(); }); + } + + function loadBoard() { + fetch('/api/rx-leaderboard?days=' + days + '&limit=25').then(function (r) { return r.json(); }) + .then(function (d) { if (destroyed) return; boardCache = d.observers || []; renderBoard(); }).catch(function () {}); + } + + function setDays(d) { + days = d; + var bar = document.getElementById('rxDays'); + if (bar) bar.querySelectorAll('button').forEach(function (b) { b.classList.toggle('active', +b.dataset.days === d); }); + loadBoard(); drawCoverage(); syncHash(); + } + + function syncHash() { + var q = 'days=' + days + (selectedRx ? '&rx=' + selectedRx : ''); + try { history.replaceState(null, '', '#/rx-coverage?' + q); } catch (e) {} + } + + function init(container) { + if (!window.MC_CLIENT_RX_COVERAGE) { + container.innerHTML = '
Coverage is not enabled on this deployment.
'; + return; + } + destroyed = false; selectedRx = ''; selectedName = ''; days = 7; boardCache = []; + try { + var p = (typeof getHashParams === 'function') ? getHashParams() : null; + if (p) { var dd = parseInt(p.get('days'), 10); if ([1, 7, 14, 30].indexOf(dd) >= 0) days = dd; selectedRx = (p.get('rx') || '').toLowerCase(); } + } catch (e) {} + container.innerHTML = pageHtml(); + map = L.map('rxMap', { zoomControl: true, attributionControl: false }).setView([51.0, 4.8], 8); + if (typeof window._applyTilesToNodeMap === 'function') window._applyTilesToNodeMap(map); + else L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { maxZoom: 19 }).addTo(map); + covLayer = L.layerGroup().addTo(map); + map.on('moveend zoomend', drawCoverage); + var bar = document.getElementById('rxDays'); + if (bar) bar.addEventListener('click', function (e) { var b = e.target.closest('button[data-days]'); if (b) setDays(+b.dataset.days); }); + setTimeout(function () { if (!destroyed && map) { map.invalidateSize(); if (selectedRx) fitToObserver(); else drawCoverage(); } }, 150); + loadBoard(); + } + + function destroy() { + destroyed = true; + if (map) { try { map.remove(); } catch (e) {} map = null; } + covLayer = null; + } + + registerPage('rx-coverage', { init: init, destroy: destroy }); +})(); diff --git a/test-coverage-gate.js b/test-coverage-gate.js new file mode 100644 index 00000000..7588d268 --- /dev/null +++ b/test-coverage-gate.js @@ -0,0 +1,57 @@ +'use strict'; +// Unit test for the client-RX coverage frontend gate (SP2). +// +// The coverage toggle (#nqCoverage) and its legend (#nqCovLegend) are built +// inline inside node-reach.js's load() — a large async function that depends on +// api()/Leaflet, so it can't be invoked headless. Instead we extract the exact +// actions-HTML concatenation block from the real source and evaluate it in a vm +// sandbox with both values of window.MC_CLIENT_RX_COVERAGE. This exercises the +// real source markup logic (no hand-copied duplicate) for the gate. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const src = fs.readFileSync(path.join(__dirname, 'public', 'node-reach.js'), 'utf8'); + +// Slice the actions-HTML expression: from the '
'"; +const startIdx = src.indexOf(startMarker); +assert.ok(startIdx >= 0, 'could not locate nq-actions block start in node-reach.js'); +const endMarker = "'
';"; +const endIdx = src.indexOf(endMarker, startIdx); +assert.ok(endIdx >= 0, 'could not locate nq-actions block end in node-reach.js'); +// Drop the trailing ";" so we can wrap the concatenation as a single expression. +let block = src.slice(startIdx, endIdx + endMarker.length).replace(/;\s*$/, ''); + +// Render the actions HTML for a given flag value via a controlled sandbox. +function renderActions(flag) { + const sandbox = { + window: { MC_CLIENT_RX_COVERAGE: flag }, + coverageOn: false, + statsHtml: '', + }; + vm.createContext(sandbox); + return vm.runInContext('(' + block + ')', sandbox); +} + +// Flag OFF ⇒ no coverage checkbox, no legend. +const off = renderActions(false); +assert.ok(!off.includes('id="nqCoverage"'), + 'flag false: actions HTML must NOT contain id="nqCoverage"'); +assert.ok(!off.includes('id="nqCovLegend"'), + 'flag false: actions HTML must NOT contain id="nqCovLegend"'); + +// Flag ON ⇒ coverage checkbox and legend present. +const on = renderActions(true); +assert.ok(on.includes('id="nqCoverage"'), + 'flag true: actions HTML MUST contain id="nqCoverage"'); +assert.ok(on.includes('id="nqCovLegend"'), + 'flag true: actions HTML MUST contain id="nqCovLegend"'); + +// Sanity: the non-gated controls render regardless of the flag. +assert.ok(off.includes('id="nqIncoming"') && on.includes('id="nqIncoming"'), + 'incoming filter must render irrespective of the coverage flag'); + +console.log('coverage gate (node-reach actions HTML) OK'); diff --git a/test-node-reach-coverage-e2e.js b/test-node-reach-coverage-e2e.js new file mode 100644 index 00000000..9e7f5dfb --- /dev/null +++ b/test-node-reach-coverage-e2e.js @@ -0,0 +1,56 @@ +// E2E for the RX coverage hex layer on the Reach page (#/nodes//reach?coverage=1). +// Defaults to localhost:3000 — NEVER point at prod (AGENTS.md). CI sets BASE_URL. +const { chromium } = require('playwright'); +const BASE = process.env.BASE_URL || 'http://localhost:3000'; + +(async () => { + const browser = await chromium.launch(); + const page = await browser.newPage(); + + const nodes = await (await page.request.get(BASE + '/api/nodes?role=repeater&limit=1')).json(); + if (!nodes.nodes || !nodes.nodes.length) { + console.log('node-reach-coverage E2E SKIP (no repeater in dataset)'); + await browser.close(); + return; + } + const pk = nodes.nodes[0].public_key; + + // 1. The coverage endpoint returns a GeoJSON FeatureCollection. + const cov = await (await page.request.get( + BASE + '/api/nodes/' + pk + '/rx-coverage?bbox=-90,-180,90,180&z=10')).json(); + if (cov.type !== 'FeatureCollection' || !Array.isArray(cov.features)) { + throw new Error('rx-coverage must return a FeatureCollection with a features array'); + } + + // 2. Bad bbox → 400. + const bad = await page.request.get(BASE + '/api/nodes/' + pk + '/rx-coverage'); + if (bad.status() !== 400) throw new Error('missing bbox should be 400, got ' + bad.status()); + + // 3. The Reach page exposes the coverage toggle. + await page.goto(BASE + '/#/nodes/' + pk + '/reach'); + await page.waitForSelector('.nq-head', { timeout: 20000 }); + const reach = await (await page.request.get(BASE + '/api/nodes/' + pk + '/reach?days=7')).json(); + if (reach.reliable_tokens && reach.reliable_tokens.length && (await page.locator('#nqRows').count())) { + await page.waitForSelector('#nqCoverage'); + + // 4. Enabling coverage issues a request to rx-coverage, shows the legend, and deep-links. + const waitCov = page.waitForRequest((r) => r.url().includes('/rx-coverage'), { timeout: 15000 }); + await page.check('#nqCoverage'); + await waitCov; + await page.waitForSelector('#nqCovLegend', { state: 'visible' }); + if (!/coverage=1/.test(await page.evaluate(() => location.hash))) { + throw new Error('coverage toggle did not deep-link ?coverage=1'); + } + + // 5. Toggling off hides the legend. + await page.uncheck('#nqCoverage'); + await page.waitForSelector('#nqCovLegend', { state: 'hidden' }); + } + + const errors = []; + page.on('console', (m) => { if (m.type() === 'error') errors.push(m.text()); }); + if (errors.length) throw new Error('console errors: ' + errors.join('; ')); + + console.log('node-reach-coverage E2E OK'); + await browser.close(); +})().catch((e) => { console.error('node-reach-coverage E2E FAIL:', e.message); process.exit(1); }); diff --git a/test-node-reach-coverage.js b/test-node-reach-coverage.js new file mode 100644 index 00000000..407b0ab8 --- /dev/null +++ b/test-node-reach-coverage.js @@ -0,0 +1,28 @@ +'use strict'; +// Unit test for node-reach-coverage.js color buckets. Loads the browser IIFE in +// a vm sandbox (pattern from test-frontend-helpers.js) and exercises the pure +// coverageColorVar mapping. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'node-reach-coverage.js'), 'utf8'); +const sandbox = { window: {}, document: {}, getComputedStyle: function () { return { getPropertyValue: function () { return ''; } }; } }; +vm.createContext(sandbox); +vm.runInContext(code, sandbox); + +const { coverageColorVar } = sandbox.window.NodeReachCoverage; + +// SF8 SNR thresholds: ≥ −5 strong, −9..−5 mid, < −9 weak. +assert.strictEqual(coverageColorVar({ has_sig: false }), '--nq-cov-grey', 'no-sig → grey'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: null }), '--nq-cov-grey', 'null snr → grey'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -3 }), '--nq-cov-strong', 'strong'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -5 }), '--nq-cov-strong', 'boundary strong (≥ −5)'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -6 }), '--nq-cov-mid', 'just below −5 → mid'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -9 }), '--nq-cov-mid', 'boundary mid (≥ −9)'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -10 }), '--nq-cov-weak', 'below −9 → weak'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -18 }), '--nq-cov-weak', 'weak'); +assert.strictEqual(coverageColorVar(null), '--nq-cov-grey', 'null props → grey'); + +console.log('node-reach-coverage color buckets OK'); From 1af0f16bfb9317563fca71a011e4be5d45a6145a Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 07:39:14 +0200 Subject: [PATCH 02/38] =?UTF-8?q?feat(coverage):=20ingestor=20=E2=80=94=20?= =?UTF-8?q?client=5Freceptions=20schema=20+=20ingest=20+=20opt-in=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- cmd/ingestor/client_reception.go | 206 ++++++++++++++++++++++++++ cmd/ingestor/client_reception_test.go | 140 +++++++++++++++++ cmd/ingestor/config.go | 42 ++++-- cmd/ingestor/coverage_gate_test.go | 65 ++++++++ cmd/ingestor/db.go | 31 +++- cmd/ingestor/main.go | 8 + 6 files changed, 476 insertions(+), 16 deletions(-) create mode 100644 cmd/ingestor/client_reception.go create mode 100644 cmd/ingestor/client_reception_test.go create mode 100644 cmd/ingestor/coverage_gate_test.go diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go new file mode 100644 index 00000000..2825c5be --- /dev/null +++ b/cmd/ingestor/client_reception.go @@ -0,0 +1,206 @@ +package main + +import ( + "log" + "strings" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +// handleClientPacket processes a packet from the mobile client RX topic +// (meshcore/client/{PUBLIC_KEY}/packets). Unlike observer packets, a roaming +// companion reports WHERE it directly heard a node, so we write a +// client_receptions row and never touch the observers/observations tables. +// rxPubkey is the companion pubkey from the topic (ACL-bound by the broker). +func handleClientPacket(store *Store, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string) { + rawHex, _ := msg["raw"].(string) + if rawHex == "" { + return + } + gps, ok := msg["gps"].(map[string]interface{}) + if !ok { + return // a client packet without a GPS fix is not coverage; drop + } + lat, latOK := toFloat64(gps["lat"]) + lon, lonOK := toFloat64(gps["lon"]) + if !latOK || !lonOK { + return + } + var accPtr *float64 + if acc, ok := toFloat64(gps["acc_m"]); ok { + accPtr = &acc + } + + decoded, err := DecodePacket(rawHex, channelKeys, false) + if err != nil { + log.Printf("MQTT [%s] client decode error: %v", tag, err) + return + } + + direction := "" + if v, ok := msg["direction"].(string); ok { + direction = v + } else if v, ok := msg["Direction"].(string); ok { + direction = v + } + + var snrPtr *float64 + if f, ok := toFloat64(firstPresent(msg, "SNR", "snr")); ok { + snrPtr = &f + } + var rssiPtr *int + if f, ok := toFloat64(firstPresent(msg, "RSSI", "rssi")); ok { + v := int(f) + rssiPtr = &v + } + + rxAt, _ := resolveRxTime(msg, tag) + isAdvert := decoded.Header.PayloadTypeName == "ADVERT" + + rec, ok := buildClientReception( + firstNonEmpty(rxPubkey, stringField(msg, "origin_id")), + direction, decoded.Header.RouteType, decoded.Path.Hops, decoded.Payload.PubKey, isAdvert, + snrPtr, rssiPtr, lat, lon, accPtr, rxAt, time.Now().UTC().Format(time.RFC3339), + ) + if !ok { + return + } + if _, err := store.InsertClientReception(rec); err != nil { + log.Printf("MQTT [%s] client_reception insert: %v", tag, err) + } + // Remember the companion's self-reported name (sent as "origin") so the + // leaderboard can show a name even if this companion never advertised. + if name := stringField(msg, "origin"); name != "" { + if err := store.UpsertClientObserver(rec.RxPubkey, name, time.Now().UTC().Format(time.RFC3339)); err != nil { + log.Printf("MQTT [%s] client_observer upsert: %v", tag, err) + } + } +} + +// UpsertClientObserver records/updates a mobile client's self-reported name. +// All writes live in the ingestor (read/write invariant #1283). +func (s *Store) UpsertClientObserver(pubkey, name, ts string) error { + if pubkey == "" || name == "" { + return nil + } + _, err := s.db.Exec(` + INSERT INTO client_observers (pubkey, name, last_seen) VALUES (?,?,?) + ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`, + strings.ToLower(pubkey), name, ts) + return err +} + +// firstPresent returns the first present value among the given keys. +func firstPresent(msg map[string]interface{}, keys ...string) interface{} { + for _, k := range keys { + if v, ok := msg[k]; ok { + return v + } + } + return nil +} + +// stringField returns msg[key] as a string, or "" if absent/not a string. +func stringField(msg map[string]interface{}, key string) string { + if v, ok := msg[key].(string); ok { + return v + } + return "" +} + +// ClientReception is one mobile RX coverage point: a companion (RxPubkey) +// directly heard a node (HeardKey) at a GPS position. Hex binning is done +// server-side from Lat/Lon at query time, so no cell id is stored here. +type ClientReception struct { + RxPubkey string + HeardKey string + HeardKeyLen int + RSSI *int + SNR *float64 + Lat float64 + Lon float64 + PosAccM *float64 + RxAt string + IngestedAt string + Src string +} + +// deriveHeardKey applies the RX capture HARD RULE: record only what the +// companion heard itself and directly. +// - direction must be "rx". +// - hops present AND a FLOOD route → the directly-heard node is the LAST hop +// (path[len-1] = the forwarder that just transmitted; each FLOOD forwarder +// appends its hash to the end). 1-byte (2 hex char) prefixes are rejected. +// - hops present on a DIRECT route → NOT attributable: direct forwarders +// consume the next hop from the FRONT (firmware Mesh.cpp removeSelfFromPath), +// so path[len-1] is the route's destination-side end, not who was heard. +// - hops empty + isAdvert → the 0-hop advertiser, by its full pubkey. +// - otherwise → not attributable (ok=false). +// +// Returns (heardKey lowercased, keylenBytes, src, ok). +func deriveHeardKey(direction string, routeType int, hops []string, advertPubkey string, isAdvert bool) (string, int, string, bool) { + if !strings.EqualFold(direction, "rx") { + return "", 0, "", false + } + if len(hops) > 0 { + // FLOOD routes (TRANSPORT_FLOOD 0, FLOOD 1) APPEND each forwarder's hash to + // the END of the path, so path[last] is the immediate RF transmitter. DIRECT + // routes (2, 3) consume the next hop from the FRONT, so path[last] is the + // route's destination-side end, NOT who was heard. + if routeType != packetpath.RouteTransportFlood && routeType != packetpath.RouteFlood { // direct route: path[last] is not the transmitter + return "", 0, "", false + } + last := strings.ToLower(strings.TrimSpace(hops[len(hops)-1])) + keylen := len(last) / 2 + if keylen < 2 { // exclude 1-byte (collision-prone), matching Reach + return "", 0, "", false + } + return last, keylen, "rxlog", true + } + if isAdvert && advertPubkey != "" { + pk := strings.ToLower(strings.TrimSpace(advertPubkey)) + return pk, len(pk) / 2, "advert", true + } + return "", 0, "", false +} + +// buildClientReception validates inputs and assembles a ClientReception, or +// returns ok=false when the packet is not attributable / out of range. +func buildClientReception( + rxPubkey, direction string, routeType int, hops []string, advertPubkey string, isAdvert bool, + snr *float64, rssi *int, lat, lon float64, posAccM *float64, rxAt, ingestedAt string, +) (*ClientReception, bool) { + if rxPubkey == "" || rxAt == "" { + return nil, false + } + if lat < -90 || lat > 90 || lon < -180 || lon > 180 { + return nil, false + } + heardKey, keylen, src, ok := deriveHeardKey(direction, routeType, hops, advertPubkey, isAdvert) + if !ok { + return nil, false + } + return &ClientReception{ + RxPubkey: strings.ToLower(rxPubkey), HeardKey: heardKey, HeardKeyLen: keylen, + RSSI: rssi, SNR: snr, Lat: lat, Lon: lon, PosAccM: posAccM, + RxAt: rxAt, IngestedAt: ingestedAt, Src: src, + }, true +} + +// InsertClientReception writes one coverage row. Idempotent via the +// UNIQUE(rx_pubkey, heard_key, rx_at) constraint; returns ins=false when the +// row already existed. All writes live in the ingestor (read/write invariant #1283). +func (s *Store) InsertClientReception(r *ClientReception) (bool, error) { + res, err := s.db.Exec(` + INSERT INTO client_receptions + (rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src) + VALUES (?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`, + r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil +} diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go new file mode 100644 index 00000000..abfe281a --- /dev/null +++ b/cmd/ingestor/client_reception_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "strings" + "testing" + + "github.com/meshcore-analyzer/packetpath" +) + +func TestClientReceptionsTableExists(t *testing.T) { + s := newTestStore(t) + cols := map[string]bool{} + rows, err := s.db.Query(`PRAGMA table_info(client_receptions)`) + if err != nil { + t.Fatalf("PRAGMA failed: %v", err) + } + defer rows.Close() + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dflt any + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + t.Fatal(err) + } + cols[name] = true + } + for _, want := range []string{"id", "rx_pubkey", "heard_key", "heard_keylen", "rssi", "snr", "lat", "lon", "pos_acc_m", "rx_at", "ingested_at", "src"} { + if !cols[want] { + t.Errorf("missing column %q in client_receptions", want) + } + } +} + +func crF(f float64) *float64 { return &f } +func crI(i int) *int { return &i } + +func TestDeriveHeardKey(t *testing.T) { + full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, nil, strings.ToUpper(full), true) + if !ok || l != 32 || src != "advert" || k != full { + t.Fatalf("0-hop advert: got k=%q l=%d src=%q ok=%v", k, l, src, ok) + } + k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, []string{"aa", "bbccdd"}, "", false) + if !ok || k != "bbccdd" || l != 3 || src != "rxlog" { + t.Fatalf("flood path: got k=%q l=%d src=%q ok=%v", k, l, src, ok) + } + // DIRECT route: path[last] is the route's far end, not the transmitter — must be rejected. + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteDirect, []string{"aa", "bbccdd"}, "", false); ok { + t.Fatalf("direct-route path must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteTransportDirect, []string{"aa", "bbccdd"}, "", false); ok { + t.Fatalf("transport-direct-route path must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, []string{"aa", "bb"}, "", false); ok { + t.Fatalf("1-byte last hop should be rejected") + } + if _, _, _, ok = deriveHeardKey("tx", packetpath.RouteFlood, []string{"aabbcc"}, "", false); ok { + t.Fatalf("tx must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, nil, "", false); ok { + t.Fatalf("no hops + non-advert must be rejected") + } +} + +func TestBuildClientReception(t *testing.T) { + acc := 8.0 + rec, ok := buildClientReception("companionpk", "rx", packetpath.RouteFlood, []string{"aa", "bbccdd"}, "", false, + crF(-7.5), crI(-92), 51.05, 3.72, &acc, "2026-06-09T12:00:00Z", "2026-06-09T12:00:01Z") + if !ok || rec.HeardKey != "bbccdd" || rec.HeardKeyLen != 3 || rec.Src != "rxlog" { + t.Fatalf("bad reception: %+v ok=%v", rec, ok) + } + if _, ok := buildClientReception("c", "rx", packetpath.RouteDirect, []string{"bbccdd"}, "", false, + crF(-7.5), crI(-92), 51.05, 3.72, nil, "t", "t"); ok { + t.Fatal("direct-route path must be rejected (not the transmitter)") + } + if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, []string{"bbccdd"}, "", false, nil, nil, 99.0, 3.72, nil, "t", "t"); ok { + t.Fatal("out-of-range lat must be rejected") + } +} + +func TestInsertClientReceptionRoundTripAndIdempotent(t *testing.T) { + s := newTestStore(t) + rec := &ClientReception{ + RxPubkey: "companionpk", HeardKey: "bbccdd", HeardKeyLen: 3, RSSI: crI(-92), + Lat: 51.05, Lon: 3.72, RxAt: "2026-06-09T12:00:00Z", IngestedAt: "2026-06-09T12:00:01Z", Src: "rxlog", + } + if ins, err := s.InsertClientReception(rec); err != nil || !ins { + t.Fatalf("first insert: ins=%v err=%v", ins, err) + } + if ins, err := s.InsertClientReception(rec); err != nil || ins { + t.Fatalf("second insert should be a no-op: ins=%v err=%v", ins, err) + } + var n int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n) + if n != 1 { + t.Fatalf("expected 1 row, got %d", n) + } +} + +func TestHandleClientPacketAdvertWritesReception(t *testing.T) { + s := newTestStore(t) + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + msg := map[string]interface{}{ + "raw": advertHex, + "direction": "rx", + "timestamp": "2026-06-09T12:00:00Z", + "origin": "MyMob", + "SNR": -7.0, + "RSSI": -92.0, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientPacket(s, "test", "companionpk", msg, nil) + + var obsName string + s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey='companionpk'`).Scan(&obsName) + if obsName != "MyMob" { + t.Fatalf("expected client_observers name 'MyMob', got %q", obsName) + } + + // This fixture is a relayed advert (non-empty path), so by the capture HARD + // RULE we record the directly-heard LAST hop (multibyte), not the originator. + // The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey. + var n, keylen int + var src string + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey='companionpk'`).Scan(&n, &keylen, &src); err != nil { + t.Fatal(err) + } + if n != 1 || keylen < 2 || src != "rxlog" { + t.Fatalf("expected 1 rxlog reception (multibyte last hop), got n=%d keylen=%d src=%q", n, keylen, src) + } + + // No GPS → no row. + handleClientPacket(s, "test", "companion2", map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil) + var n2 int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey='companion2'`).Scan(&n2) + if n2 != 0 { + t.Fatalf("packet without gps must be dropped, got %d rows", n2) + } +} diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 10e81195..b370f1fa 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -43,21 +43,22 @@ type MQTTLegacy struct { // Config holds the ingestor configuration, compatible with the Node.js config.json format. type Config struct { - DBPath string `json:"dbPath"` - MQTT *MQTTLegacy `json:"mqtt,omitempty"` - MQTTSources []MQTTSource `json:"mqttSources,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - ChannelKeysPath string `json:"channelKeysPath,omitempty"` - ChannelKeys map[string]string `json:"channelKeys,omitempty"` - HashChannels []string `json:"hashChannels,omitempty"` - HashRegions []string `json:"hashRegions,omitempty"` - Retention *RetentionConfig `json:"retention,omitempty"` - Metrics *MetricsConfig `json:"metrics,omitempty"` - Runtime *RuntimeConfig `json:"runtime,omitempty"` - GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` - ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"` - ValidateSignatures *bool `json:"validateSignatures,omitempty"` - DB *DBConfig `json:"db,omitempty"` + DBPath string `json:"dbPath"` + MQTT *MQTTLegacy `json:"mqtt,omitempty"` + MQTTSources []MQTTSource `json:"mqttSources,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + ChannelKeysPath string `json:"channelKeysPath,omitempty"` + ChannelKeys map[string]string `json:"channelKeys,omitempty"` + HashChannels []string `json:"hashChannels,omitempty"` + HashRegions []string `json:"hashRegions,omitempty"` + Retention *RetentionConfig `json:"retention,omitempty"` + Metrics *MetricsConfig `json:"metrics,omitempty"` + Runtime *RuntimeConfig `json:"runtime,omitempty"` + ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"` + GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` + ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"` + ValidateSignatures *bool `json:"validateSignatures,omitempty"` + DB *DBConfig `json:"db,omitempty"` // ObserverIATAWhitelist restricts which observer IATA regions are processed. // When non-empty, only observers whose IATA code (from the MQTT topic) matches @@ -128,6 +129,17 @@ func (f *ForeignAdvertConfig) IsDropMode() bool { return strings.EqualFold(strings.TrimSpace(f.Mode), "drop") } +// ClientRxCoverageConfig controls the opt-in mobile client-RX coverage feature. +type ClientRxCoverageConfig struct { + Enabled bool `json:"enabled"` +} + +// ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage +// feature is on. Absent/nil ⇒ off (the safe default). +func (c *Config) ClientRxCoverageEnabled() bool { + return c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled +} + // RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes. type RetentionConfig struct { NodeDays int `json:"nodeDays"` diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go new file mode 100644 index 00000000..de931d4e --- /dev/null +++ b/cmd/ingestor/coverage_gate_test.go @@ -0,0 +1,65 @@ +package main + +import "testing" + +// clientCoverageMsg builds a valid mobile client-RX coverage message on the +// dedicated topic meshcore/client//packets. The raw hex is a relayed +// advert with GPS, so handleClientPacket would write exactly one +// client_receptions row when the feature is enabled (see +// TestHandleClientPacketAdvertWritesReception). +func clientCoverageMsg() *mockMessage { + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + payload := []byte(`{"raw":"` + advertHex + `","direction":"rx","timestamp":"2026-06-09T12:00:00Z","origin":"MyMob","SNR":-7.0,"RSSI":-92.0,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`) + return &mockMessage{topic: "meshcore/client/companionpk/packets", payload: payload} +} + +func clientReceptionCount(t *testing.T, s *Store) int { + t.Helper() + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// TestClientRxCoverageEnabledDefault verifies the gate helper defaults OFF for +// nil/absent config and is only true when explicitly enabled. +func TestClientRxCoverageEnabledDefault(t *testing.T) { + if (&Config{}).ClientRxCoverageEnabled() { + t.Fatal("nil ClientRxCoverage must report disabled") + } + if (&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: false}}).ClientRxCoverageEnabled() { + t.Fatal("Enabled:false must report disabled") + } + if !(&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}).ClientRxCoverageEnabled() { + t.Fatal("Enabled:true must report enabled") + } +} + +// TestClientRxCoverageGateOff drives handleMessage with the feature OFF: the +// client-topic message must fall through and write no client_receptions rows. +func TestClientRxCoverageGateOff(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{} // ClientRxCoverage nil ⇒ disabled + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 0 { + t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n) + } +} + +// TestClientRxCoverageGateOn drives handleMessage with the feature ON: the +// client-topic message must be dispatched and write exactly one row. +func TestClientRxCoverageGateOn(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}} + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 1 { + t.Fatalf("feature ON: expected 1 client_receptions row, got %d", n) + } +} diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index a5d54fa0..6fa03ae6 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -271,6 +271,36 @@ func applySchema(db *sql.DB) error { -- the last_seen column exists (#1690) — keep it OUT of this base -- schema block so legacy DBs (table-exists, column-missing) don't -- trip on the CREATE INDEX before the ALTER runs. + + -- Mobile client RX coverage: a roaming companion = a mobile observer + -- with a moving GPS position, so it gets its own table rather than + -- observations (which assumes a fixed observer/location). + CREATE TABLE IF NOT EXISTS client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + heard_key TEXT NOT NULL, + heard_keylen INTEGER NOT NULL, + rssi INTEGER, + snr REAL, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + rx_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + src TEXT NOT NULL, + UNIQUE(rx_pubkey, heard_key, rx_at) + ); + CREATE INDEX IF NOT EXISTS idx_client_recept_heard ON client_receptions(heard_key); + CREATE INDEX IF NOT EXISTS idx_client_recept_rxpk ON client_receptions(rx_pubkey); + + -- Self-reported name of each mobile client (companion), from the SELF_INFO + -- name the app sends as "origin". Lets the leaderboard show a name even + -- when the companion never advertised (so it isn't in the nodes table). + CREATE TABLE IF NOT EXISTS client_observers ( + pubkey TEXT PRIMARY KEY, + name TEXT, + last_seen TEXT + ); ` if _, err := db.Exec(schema); err != nil { return fmt.Errorf("base schema: %w", err) @@ -1703,7 +1733,6 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, return pd } - // ─── Writer-lock instrumentation (issue #1340) ──────────────────────────── // // Make SQLite writer-lock starvation visible to operators. Per-component diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index a623da6f..5fa3216b 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -535,6 +535,14 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, return } + // Mobile client RX coverage: dedicated topic meshcore/client/{PUBLIC_KEY}/packets. + // A roaming companion reports where it directly heard a node; handled in isolation + // from the observer/observations path. EMQX ACL binds parts[2] to the client's own key. + if cfg.ClientRxCoverageEnabled() && len(parts) >= 4 && parts[1] == "client" && parts[3] == "packets" { + handleClientPacket(store, tag, parts[2], msg, channelKeys) + return + } + // Skip status/connection topics if topic == "meshcore/status" || topic == "meshcore/events/connection" { return From a5c6a9c4eed5040aa459245c1bc7a300946629aa Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 08:17:07 +0200 Subject: [PATCH 03/38] =?UTF-8?q?feat(coverage):=20server=20=E2=80=94=20Ge?= =?UTF-8?q?oJSON=20hex=20coverage=20endpoints=20+=20opt-in=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- cmd/server/config.go | 37 +++- cmd/server/coverage_gate_test.go | 67 +++++++ cmd/server/hexgrid.go | 106 ++++++++++ cmd/server/hexgrid_test.go | 29 +++ cmd/server/node_resolve.go | 61 ++++++ cmd/server/node_resolve_test.go | 51 +++++ cmd/server/openapi_known_gaps.json | 4 + cmd/server/rx_coverage.go | 255 ++++++++++++++++++++++++ cmd/server/rx_coverage_endpoint_test.go | 80 ++++++++ cmd/server/rx_coverage_test.go | 143 +++++++++++++ cmd/server/rx_dashboard.go | 217 ++++++++++++++++++++ cmd/server/rx_dashboard_test.go | 68 +++++++ cmd/server/types.go | 199 +++++++++--------- config.example.json | 2 + 14 files changed, 1210 insertions(+), 109 deletions(-) create mode 100644 cmd/server/coverage_gate_test.go create mode 100644 cmd/server/hexgrid.go create mode 100644 cmd/server/hexgrid_test.go create mode 100644 cmd/server/node_resolve.go create mode 100644 cmd/server/node_resolve_test.go create mode 100644 cmd/server/rx_coverage.go create mode 100644 cmd/server/rx_coverage_endpoint_test.go create mode 100644 cmd/server/rx_coverage_test.go create mode 100644 cmd/server/rx_dashboard.go create mode 100644 cmd/server/rx_dashboard_test.go diff --git a/cmd/server/config.go b/cmd/server/config.go index 2e1e4d48..3a6b414f 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -133,7 +133,7 @@ type Config struct { // Currently exposes runtime.maxMemoryMB which sets a soft memory limit // (GOMEMLIMIT) via runtime/debug.SetMemoryLimit at startup. The // GOMEMLIMIT environment variable, when set, takes precedence. - Runtime *RuntimeConfig `json:"runtime,omitempty"` + Runtime *RuntimeConfig `json:"runtime,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` Areas map[string]AreaEntry `json:"areas,omitempty"` @@ -160,7 +160,13 @@ type Config struct { obsBlacklistSetCached map[string]bool obsBlacklistOnce sync.Once - Compression *CompressionConfig `json:"compression,omitempty"` + Compression *CompressionConfig `json:"compression,omitempty"` + + // ClientRxCoverage gates the opt-in mobile client-RX coverage feature + // (corescope-rx companions publishing GPS-tagged receptions). Absent/nil + // ⇒ off; see ClientRxCoverageEnabled. + ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"` + ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"` NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"` @@ -250,6 +256,17 @@ func (c *Config) GZipEnabled() bool { return c.Compression != nil && c.Compression.GZip } +// ClientRxCoverageConfig gates the opt-in mobile client-RX coverage feature. +type ClientRxCoverageConfig struct { + Enabled bool `json:"enabled"` +} + +// ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage +// feature is on. Absent/nil ⇒ off (the safe default). +func (c *Config) ClientRxCoverageEnabled() bool { + return c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled +} + // WSCompressionEnabled returns true when WebSocket permessage-deflate is explicitly enabled. func (c *Config) WSCompressionEnabled() bool { return c.Compression != nil && c.Compression.Websocket @@ -303,10 +320,10 @@ type RuntimeConfig struct { } type RetentionConfig struct { - NodeDays int `json:"nodeDays"` - ObserverDays int `json:"observerDays"` - PacketDays int `json:"packetDays"` - MetricsDays int `json:"metricsDays"` + NodeDays int `json:"nodeDays"` + ObserverDays int `json:"observerDays"` + PacketDays int `json:"packetDays"` + MetricsDays int `json:"metricsDays"` } // DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). @@ -601,7 +618,6 @@ func (c *Config) ResolveDBPath(baseDir string) string { return filepath.Join(baseDir, "data", "meshcore.db") } - func (c *Config) NormalizeTimestampConfig() { defaults := defaultTimestampConfig() if c.Timestamps == nil { @@ -908,10 +924,11 @@ func (c *Config) IsObserverBlacklisted(id string) bool { // data slowly." Lower values give fresher data at higher CPU cost. // // RecomputeIntervalSeconds keys (all optional): -// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew +// +// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew type AnalyticsConfig struct { - DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"` - RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"` + DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"` + RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"` } // AnalyticsDefaultRecomputeInterval returns the configured default diff --git a/cmd/server/coverage_gate_test.go b/cmd/server/coverage_gate_test.go new file mode 100644 index 00000000..ed3192e8 --- /dev/null +++ b/cmd/server/coverage_gate_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" +) + +// gateTestServer builds a minimal *Server with the given client-RX coverage +// config and registers all routes, mirroring routes_test.go's setup but +// skipping the packet store (none of the gated routes need it for the 404 / +// registration assertions below). +func gateTestServer(t *testing.T, cov *ClientRxCoverageConfig) *mux.Router { + t.Helper() + db := setupTestDB(t) + cfg := &Config{Port: 3000, ClientRxCoverage: cov} + hub := NewHub() + srv := NewServer(db, cfg, hub) + router := mux.NewRouter() + srv.RegisterRoutes(router) + return router +} + +func TestCoverageRoutesGatedOff(t *testing.T) { + router := gateTestServer(t, nil) + + req := httptest.NewRequest("GET", "/api/rx-coverage", nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusNotFound { + t.Fatalf("expected 404 for /api/rx-coverage when disabled, got %d", rr.Code) + } + + creq := httptest.NewRequest("GET", "/api/config/client", nil) + crr := httptest.NewRecorder() + router.ServeHTTP(crr, creq) + if crr.Code != http.StatusOK { + t.Fatalf("config/client status %d body %s", crr.Code, crr.Body.String()) + } + if !strings.Contains(crr.Body.String(), `"clientRxCoverage":false`) { + t.Fatalf("expected clientRxCoverage:false in config body, got %s", crr.Body.String()) + } +} + +func TestCoverageRoutesGatedOn(t *testing.T) { + router := gateTestServer(t, &ClientRxCoverageConfig{Enabled: true}) + + req := httptest.NewRequest("GET", "/api/rx-coverage", nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code == http.StatusNotFound { + t.Fatalf("expected /api/rx-coverage to be registered when enabled, got 404") + } + + creq := httptest.NewRequest("GET", "/api/config/client", nil) + crr := httptest.NewRecorder() + router.ServeHTTP(crr, creq) + if crr.Code != http.StatusOK { + t.Fatalf("config/client status %d body %s", crr.Code, crr.Body.String()) + } + if !strings.Contains(crr.Body.String(), `"clientRxCoverage":true`) { + t.Fatalf("expected clientRxCoverage:true in config body, got %s", crr.Body.String()) + } +} diff --git a/cmd/server/hexgrid.go b/cmd/server/hexgrid.go new file mode 100644 index 00000000..41323781 --- /dev/null +++ b/cmd/server/hexgrid.go @@ -0,0 +1,106 @@ +package main + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// Pure-Go hexagonal binning for RX coverage display. We deliberately avoid the +// CGO-based uber/h3-go (this project builds with CGO_ENABLED=0). Points are +// projected to Web Mercator and snapped to a pointy-top hex grid whose size +// depends on the display resolution. Cell ids are "res:q:r" (axial coords). +// At city/region scale this looks like H3/mapme.sh coverage without any deps. + +const hexEarthRadius = 6378137.0 // Web Mercator sphere radius (m) + +// hexTargetPx is the desired on-screen hex size (point-to-point height) in CSS +// pixels. mercUPPZ0 is Web Mercator units per pixel at zoom 0 (world span / 256); +// Leaflet halves it each zoom level, independent of latitude. Sizing the hex in +// these units therefore renders it at a constant ~hexTargetPx at every zoom — the +// old fixed-meter buckets looked like specks when zoomed out (issue: hexes too small). +const hexTargetPx = 28.0 +const mercUPPZ0 = 156543.03392 + +func hexMercator(lat, lon float64) (float64, float64) { + x := hexEarthRadius * lon * math.Pi / 180 + y := hexEarthRadius * math.Log(math.Tan(math.Pi/4+lat*math.Pi/360)) + return x, y +} + +func hexInvMercator(x, y float64) (lat, lon float64) { + lon = x / hexEarthRadius * 180 / math.Pi + lat = (2*math.Atan(math.Exp(y/hexEarthRadius)) - math.Pi/2) * 180 / math.Pi + return lat, lon +} + +// hexSizeForRes is the hex circumradius (center→corner) in Web Mercator units for a +// display resolution. Resolution equals the Leaflet zoom level (see zoomToHexRes), so +// the size scales as 2^-zoom and the hex keeps a constant ~hexTargetPx on-screen size +// regardless of zoom. hexCellAt (binning) and hexBoundary (drawing) both read this, so +// they stay consistent for a given cell id. +func hexSizeForRes(res int) float64 { + return (hexTargetPx / 2) * mercUPPZ0 / math.Pow(2, float64(res)) +} + +// hexCellAt returns a stable cell id ("res:q:r") for the lat/lon at res. +func hexCellAt(lat, lon float64, res int) string { + size := hexSizeForRes(res) + x, y := hexMercator(lat, lon) + q := (math.Sqrt(3)/3*x - 1.0/3*y) / size + r := (2.0 / 3 * y) / size + qi, ri := hexRound(q, r) + return fmt.Sprintf("%d:%d:%d", res, qi, ri) +} + +// hexRound rounds fractional axial coords to the nearest hex via cube rounding. +func hexRound(q, r float64) (int, int) { + x, z := q, r + y := -x - z + rx, ry, rz := math.Round(x), math.Round(y), math.Round(z) + dx, dy, dz := math.Abs(rx-x), math.Abs(ry-y), math.Abs(rz-z) + switch { + case dx > dy && dx > dz: + rx = -ry - rz + case dy > dz: + ry = -rx - rz + default: + rz = -rx - ry + } + return int(rx), int(rz) +} + +// hexBoundary returns the cell's 6 corners as a closed [lon,lat] ring (GeoJSON +// order), or nil if the cell id is malformed. +func hexBoundary(cellID string) [][2]float64 { + res, q, r, ok := parseHexCell(cellID) + if !ok { + return nil + } + size := hexSizeForRes(res) + cx := size * (math.Sqrt(3)*float64(q) + math.Sqrt(3)/2*float64(r)) + cy := size * (1.5 * float64(r)) + ring := make([][2]float64, 0, 7) + for i := 0; i < 6; i++ { + ang := math.Pi / 180 * float64(60*i-30) + lat, lon := hexInvMercator(cx+size*math.Cos(ang), cy+size*math.Sin(ang)) + ring = append(ring, [2]float64{lon, lat}) + } + ring = append(ring, ring[0]) // close the ring + return ring +} + +func parseHexCell(id string) (res, q, r int, ok bool) { + p := strings.Split(id, ":") + if len(p) != 3 { + return 0, 0, 0, false + } + a, e1 := strconv.Atoi(p[0]) + b, e2 := strconv.Atoi(p[1]) + c, e3 := strconv.Atoi(p[2]) + if e1 != nil || e2 != nil || e3 != nil { + return 0, 0, 0, false + } + return a, b, c, true +} diff --git a/cmd/server/hexgrid_test.go b/cmd/server/hexgrid_test.go new file mode 100644 index 00000000..ce98bfcc --- /dev/null +++ b/cmd/server/hexgrid_test.go @@ -0,0 +1,29 @@ +package main + +import "testing" + +func TestHexCellAtStableAndDistinct(t *testing.T) { + a := hexCellAt(51.0500, 3.7200, 9) + b := hexCellAt(51.0500, 3.7200, 9) + if a == "" || a != b { + t.Fatalf("stable cell expected, got %q %q", a, b) + } + c := hexCellAt(51.2000, 3.7200, 9) // ~17 km away + if c == a { + t.Fatalf("distant point should differ, both %q", a) + } +} + +func TestHexBoundaryClosedRing(t *testing.T) { + cell := hexCellAt(51.05, 3.72, 9) + ring := hexBoundary(cell) + if len(ring) != 7 { + t.Fatalf("expected 7 points (closed hex), got %d", len(ring)) + } + if ring[0] != ring[6] { + t.Fatalf("ring not closed: %v vs %v", ring[0], ring[6]) + } + if hexBoundary("garbage") != nil { + t.Fatalf("malformed cell should return nil") + } +} diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go new file mode 100644 index 00000000..655740ba --- /dev/null +++ b/cmd/server/node_resolve.go @@ -0,0 +1,61 @@ +package main + +import ( + "encoding/json" + "net/http" + "regexp" + "strings" +) + +// ResolvePrefixResp is the tiny reply for /api/nodes/resolve — lets a client +// resolve a heard 2-3 byte path prefix (or full pubkey) to a node name without +// fetching the whole node list. Read-only. +type ResolvePrefixResp struct { + Prefix string `json:"prefix"` + Pubkey string `json:"pubkey,omitempty"` + Name string `json:"name,omitempty"` + Ambiguous bool `json:"ambiguous"` +} + +var hexPrefixRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) + +func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { + pfx := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("prefix"))) + if !hexPrefixRe.MatchString(pfx) { + http.Error(w, "prefix must be 2-64 hex chars", http.StatusBadRequest) + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + // LIMIT 2: we only need to know unique vs ambiguous. nodes.public_key is the + // PK and stored lowercase; pfx is validated hex so the LIKE pattern is safe. + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, pfx+"%") + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + defer rows.Close() + var pks, names []string + for rows.Next() { + var pk, nm string + if err := rows.Scan(&pk, &nm); err != nil { + http.Error(w, "scan failed", http.StatusInternalServerError) + return + } + pks = append(pks, pk) + names = append(names, nm) + } + + resp := ResolvePrefixResp{Prefix: pfx} + switch len(pks) { + case 1: + resp.Pubkey = pks[0] + resp.Name = names[0] + default: + resp.Ambiguous = len(pks) > 1 // 0 → not found (name empty), >1 → ambiguous + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} diff --git a/cmd/server/node_resolve_test.go b/cmd/server/node_resolve_test.go new file mode 100644 index 00000000..80b6daaf --- /dev/null +++ b/cmd/server/node_resolve_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func serveResolve(srv *Server, path string) *httptest.ResponseRecorder { + router := mux.NewRouter() + router.HandleFunc("/api/nodes/resolve", srv.handleResolvePrefix).Methods("GET") + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest("GET", path, nil)) + return rr +} + +func TestResolvePrefix(t *testing.T) { + db := setupTestDBv2(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('efef7943505052b47f1809488ea4b4d3942d4ed72d2b1953b90a9f5e62a65fb5','NodeUnique','repeater','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('aa11000000000000000000000000000000000000000000000000000000000000','NodeA','repeater','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('aa22000000000000000000000000000000000000000000000000000000000000','NodeB','repeater','t','t',1)`) + srv := &Server{db: db} + + // unique 3-byte prefix → name + var r1 ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=efef79").Body.Bytes(), &r1) + if r1.Name != "NodeUnique" || r1.Ambiguous { + t.Fatalf("unique: %+v", r1) + } + // colliding 1-byte prefix (aa…) → ambiguous, no name + var r2 ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=aa").Body.Bytes(), &r2) + if !r2.Ambiguous || r2.Name != "" { + t.Fatalf("ambiguous: %+v", r2) + } + // not found → empty name, not ambiguous + var r3 ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=dead").Body.Bytes(), &r3) + if r3.Name != "" || r3.Ambiguous { + t.Fatalf("notfound: %+v", r3) + } + // bad prefix → 400 + if serveResolve(srv, "/api/nodes/resolve?prefix=xyz").Code != 400 { + t.Fatal("non-hex prefix should be 400") + } +} diff --git a/cmd/server/openapi_known_gaps.json b/cmd/server/openapi_known_gaps.json index 174c39a9..7c87182a 100644 --- a/cmd/server/openapi_known_gaps.json +++ b/cmd/server/openapi_known_gaps.json @@ -13,14 +13,18 @@ "/api/healthz", "/api/known-channels", "/api/nodes/clock-skew", + "/api/nodes/resolve", "/api/nodes/{pubkey}/battery", "/api/nodes/{pubkey}/clock-skew", "/api/nodes/{pubkey}/reach", + "/api/nodes/{pubkey}/rx-coverage", "/api/observers/clock-skew", "/api/paths/inspect", "/api/perf/io", "/api/perf/sqlite", "/api/perf/write-sources", + "/api/rx-coverage", + "/api/rx-leaderboard", "/api/scope-stats", "/api/spec" ] diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go new file mode 100644 index 00000000..f79945ff --- /dev/null +++ b/cmd/server/rx_coverage.go @@ -0,0 +1,255 @@ +package main + +import ( + "encoding/json" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/gorilla/mux" +) + +// coverageRow is one raw reception read from client_receptions. +type coverageRow struct { + Lat, Lon float64 + SNR *float64 + RSSI *int + HeardKey string // directly-heard node key (2-3 byte prefix or full pubkey), lowercase + RxAt string // reception time (RFC3339); used to pick the latest SNR per node +} + +// GeoJSON output (named structs, no map[string]interface{} — AGENTS.md). +type CoverageFeatureCollection struct { + Type string `json:"type"` // "FeatureCollection" + Features []CoverageFeature `json:"features"` +} +type CoverageFeature struct { + Type string `json:"type"` // "Feature" + Geometry CoveragePolygon `json:"geometry"` + Properties CoverageProperties `json:"properties"` +} +type CoveragePolygon struct { + Type string `json:"type"` // "Polygon" + Coordinates [][][2]float64 `json:"coordinates"` // one ring: [ [ [lon,lat], ... ] ] +} +type CoverageProperties struct { + Cell string `json:"cell"` + Count int `json:"count"` + BestSNR *float64 `json:"best_snr"` + HasSig bool `json:"has_sig"` // false → render grey (no signal metric) + Nodes []CoverageNode `json:"nodes"` // per-node breakdown, strongest latest-SNR first +} + +// CoverageNode is one directly-heard node within a cell, with its latest SNR. +type CoverageNode struct { + Prefix string `json:"prefix"` // heard_key (resolved to Name when unique) + Name string `json:"name,omitempty"` // node name, empty if unknown/ambiguous prefix + SNR *float64 `json:"snr"` // latest SNR (by rx_at); nil → heard without signal + Count int `json:"count"` +} + +type covAgg struct { + count int + bestSNR *float64 + hasSig bool + nodes map[string]*covNodeAgg +} + +// covNodeAgg tracks, per directly-heard node within a cell, its reception count and +// the SNR of its most recent reception (by rx_at). name/prefix are the resolved node +// name (when known) and a display prefix fallback. +type covNodeAgg struct { + count int + latestAt string + latestSNR *float64 + name string + prefix string +} + +// nodeResolver maps a heard_key (2-3 byte prefix or full pubkey) to a canonical +// identity key and a display name. A unique match returns (pubkey, name) so the same +// node heard under different prefix lengths collapses into one bucket; unknown or +// ambiguous keys return (heardKey, "") and stay distinct. nil disables resolution. +type nodeResolver func(heardKey string) (key, name string) + +// aggregateCoverage bins raw rows into display-resolution hex cells, keeping the +// best (max) SNR per cell, and emits GeoJSON polygons. resolve (may be nil) collapses +// per-node receptions by resolved node identity. +func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) CoverageFeatureCollection { + byCell := map[string]*covAgg{} + for _, row := range rows { + cell := hexCellAt(row.Lat, row.Lon, res) + a := byCell[cell] + if a == nil { + a = &covAgg{} + byCell[cell] = a + } + a.count++ + if row.SNR != nil { + a.hasSig = true + if a.bestSNR == nil || *row.SNR > *a.bestSNR { + v := *row.SNR + a.bestSNR = &v + } + } + if row.HeardKey != "" { + if a.nodes == nil { + a.nodes = map[string]*covNodeAgg{} + } + key, name := row.HeardKey, "" + if resolve != nil { + if k, n := resolve(row.HeardKey); k != "" { + key, name = k, n + } + } + na := a.nodes[key] + if na == nil { + na = &covNodeAgg{prefix: row.HeardKey, name: name} + a.nodes[key] = na + } + if name != "" { + na.name = name + } + na.count++ + // rx_at is RFC3339, so lexical >= is chronological; keep the latest SNR. + if na.count == 1 || row.RxAt >= na.latestAt { + na.latestAt = row.RxAt + na.latestSNR = row.SNR + } + } + } + fc := CoverageFeatureCollection{Type: "FeatureCollection", Features: []CoverageFeature{}} + for cell, a := range byCell { + ring := hexBoundary(cell) + if ring == nil { + continue + } + fc.Features = append(fc.Features, CoverageFeature{ + Type: "Feature", + Geometry: CoveragePolygon{Type: "Polygon", Coordinates: [][][2]float64{ring}}, + Properties: CoverageProperties{ + Cell: cell, Count: a.count, BestSNR: a.bestSNR, HasSig: a.hasSig, + Nodes: sortedCoverageNodes(a.nodes), + }, + }) + } + return fc +} + +// sortedCoverageNodes flattens the per-node aggregates into a slice sorted by latest +// SNR descending (nodes heard without a signal sort last), tie-broken by count then +// prefix for a stable order. +func sortedCoverageNodes(m map[string]*covNodeAgg) []CoverageNode { + out := make([]CoverageNode, 0, len(m)) + for _, na := range m { + out = append(out, CoverageNode{Prefix: na.prefix, Name: na.name, SNR: na.latestSNR, Count: na.count}) + } + sort.Slice(out, func(i, j int) bool { + si, sj := out[i].SNR, out[j].SNR + if (si == nil) != (sj == nil) { + return si != nil // signal before no-signal + } + if si != nil && *si != *sj { + return *si > *sj + } + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Prefix < out[j].Prefix + }) + return out +} + +type bbox struct{ MinLat, MinLon, MaxLat, MaxLon float64 } + +// queryCoverageRows returns raw coverage rows where the directly-heard node +// matches the target pubkey by its 2-3 byte prefix (or full pubkey), within the +// bbox. Read-only (server RO connection). +func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) { + pk := strings.ToLower(pubkey) + rows, err := s.db.conn.Query(` + SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? + AND ( (heard_keylen = 32 AND heard_key = ?) + OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )`, + b.MinLat, b.MaxLat, b.MinLon, b.MaxLon, pk, pk) + if err != nil { + return nil, err + } + defer rows.Close() + return scanCoverageRows(rows) +} + +// mobileRxStats returns the total mobile-client receptions of a node (by its +// 2-3 byte prefix or full pubkey) and the number of distinct contributing clients. +func (s *Server) mobileRxStats(pubkey string) (count, clients int) { + if s.db == nil || s.db.conn == nil { + return 0, 0 + } + pk := strings.ToLower(pubkey) + s.db.conn.QueryRow(` + SELECT COUNT(*), COUNT(DISTINCT rx_pubkey) FROM client_receptions + WHERE (heard_keylen = 32 AND heard_key = ?) + OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key)`, + pk, pk).Scan(&count, &clients) + return count, clients +} + +// zoomToHexRes maps a Leaflet zoom level to the display resolution used for hex +// binning. Resolution == zoom (clamped to a sane range) so hex size tracks the map +// scale 1:1 and renders at a constant ~hexTargetPx (see hexSizeForRes). The clamp also +// guards the missing-param case (z parses to 0). +func zoomToHexRes(z int) int { + switch { + case z < 3: + return 3 + case z > 18: + return 18 + default: + return z + } +} + +func parseBBox(s string) (bbox, bool) { + p := strings.Split(s, ",") + if len(p) != 4 { + return bbox{}, false + } + v := make([]float64, 4) + for i := range p { + f, err := strconv.ParseFloat(strings.TrimSpace(p[i]), 64) + if err != nil { + return bbox{}, false + } + v[i] = f + } + return bbox{MinLat: v[0], MinLon: v[1], MaxLat: v[2], MaxLon: v[3]}, true +} + +// handleNodeRxCoverage serves per-node mobile RX coverage as a GeoJSON hex grid. +func (s *Server) handleNodeRxCoverage(w http.ResponseWriter, r *http.Request) { + if !s.requireClientRxCoverage(w, r) { + return + } + pubkey := strings.ToLower(mux.Vars(r)["pubkey"]) + b, ok := parseBBox(r.URL.Query().Get("bbox")) + if !ok { + http.Error(w, "bbox required as minLat,minLon,maxLat,maxLon", http.StatusBadRequest) + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + z, _ := strconv.Atoi(r.URL.Query().Get("z")) + rows, err := s.queryCoverageRows(pubkey, b) + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolver()) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(fc) +} diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go new file mode 100644 index 00000000..1825d551 --- /dev/null +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func seedCoverageDB(t *testing.T) *DB { + db := setupTestDBv2(t) + mustExecDB(t, db, `CREATE TABLE client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, + rssi INTEGER, snr REAL, lat REAL, lon REAL, pos_acc_m REAL, rx_at TEXT, ingested_at TEXT, src TEXT)`) + mustExecDB(t, db, `CREATE TABLE client_observers (pubkey TEXT PRIMARY KEY, name TEXT, last_seen TEXT)`) + return db +} + +func TestQueryCoverageRowsByPrefixAndBBox(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + + rows, err := srv.queryCoverageRows("aabbccddeeff00112233", bbox{MinLat: 50, MinLon: 3, MaxLat: 52, MaxLon: 4}) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 row by prefix, got %d", len(rows)) + } + rows, _ = srv.queryCoverageRows("aabbccddeeff00112233", bbox{MinLat: 0, MinLon: 0, MaxLat: 1, MaxLon: 1}) + if len(rows) != 0 { + t.Fatalf("bbox filter failed, got %d", len(rows)) + } +} + +func TestMobileRxStats(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('compA','aabbcc',3,-6,51.05,3.72,'t1','t','rxlog')`) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('compB','aabbcc',3,-8,51.06,3.73,'t2','t','rxlog')`) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('compA','ffeedd',3,-5,51.07,3.74,'t3','t','rxlog')`) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + c, cl := srv.mobileRxStats("aabbccddeeff00112233") + if c != 2 || cl != 2 { + t.Fatalf("got count=%d clients=%d, want 2/2", c, cl) + } +} + +func serveRxCoverage(srv *Server, path string) *httptest.ResponseRecorder { + router := mux.NewRouter() + router.HandleFunc("/api/nodes/{pubkey}/rx-coverage", srv.handleNodeRxCoverage).Methods("GET") + req := httptest.NewRequest("GET", path, nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr +} + +func TestRxCoverageEndpointGeoJSON(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + + rr := serveRxCoverage(srv, "/api/nodes/aabbccddeeff00112233/rx-coverage?bbox=50,3,52,4&z=12") + if rr.Code != 200 { + t.Fatalf("status %d body %s", rr.Code, rr.Body.String()) + } + var fc CoverageFeatureCollection + if err := json.Unmarshal(rr.Body.Bytes(), &fc); err != nil { + t.Fatalf("decode: %v", err) + } + if fc.Type != "FeatureCollection" || len(fc.Features) != 1 { + t.Fatalf("unexpected fc: %+v", fc) + } + if serveRxCoverage(srv, "/api/nodes/aabbcc/rx-coverage").Code != 400 { + t.Fatal("missing bbox should be 400") + } +} diff --git a/cmd/server/rx_coverage_test.go b/cmd/server/rx_coverage_test.go new file mode 100644 index 00000000..868753d5 --- /dev/null +++ b/cmd/server/rx_coverage_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "encoding/json" + "math" + "testing" +) + +func covF(f float64) *float64 { return &f } + +func TestAggregateCoverageBucketsBestSNR(t *testing.T) { + rows := []coverageRow{ + {Lat: 51.05000, Lon: 3.72000, SNR: covF(-12)}, + {Lat: 51.05001, Lon: 3.72001, SNR: covF(-6)}, // same cell, stronger + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + if p := fc.Features[0].Properties; p.BestSNR == nil || *p.BestSNR != -6 || p.Count != 2 || !p.HasSig { + t.Fatalf("bad props: %+v", fc.Features[0].Properties) + } + if g := fc.Features[0].Geometry; g.Type != "Polygon" || len(g.Coordinates) != 1 { + t.Fatalf("bad geometry: %+v", g) + } + if _, err := json.Marshal(fc); err != nil { + t.Fatalf("marshal: %v", err) + } +} + +func TestAggregateCoverageGreyWhenNoSignal(t *testing.T) { + fc := aggregateCoverage([]coverageRow{{Lat: 51.05, Lon: 3.72}}, 9, nil) + if len(fc.Features) != 1 || fc.Features[0].Properties.HasSig { + t.Fatalf("expected one grey (no-sig) cell, got %+v", fc.Features) + } +} + +// TestAggregateCoverageNodeBreakdown covers the per-cell node list: each heard node +// keeps its latest SNR (by rx_at) and reception count, sorted strongest-first with +// heard-without-signal nodes last. +func TestAggregateCoverageNodeBreakdown(t *testing.T) { + rows := []coverageRow{ + // node A: two receptions; the later one (t2) has the weaker SNR -10. + {Lat: 51.05, Lon: 3.72, SNR: covF(-4), HeardKey: "aabb", RxAt: "2026-06-01T10:00:00Z"}, + {Lat: 51.05001, Lon: 3.72001, SNR: covF(-10), HeardKey: "aabb", RxAt: "2026-06-02T10:00:00Z"}, + // node B: single reception, strongest latest SNR. + {Lat: 51.05, Lon: 3.72, SNR: covF(-6), HeardKey: "ccdd", RxAt: "2026-06-01T10:00:00Z"}, + // node C: heard without a signal metric. + {Lat: 51.05, Lon: 3.72, HeardKey: "eeff", RxAt: "2026-06-01T10:00:00Z"}, + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + nodes := fc.Features[0].Properties.Nodes + if len(nodes) != 3 { + t.Fatalf("expected 3 nodes, got %d (%+v)", len(nodes), nodes) + } + if nodes[0].Prefix != "ccdd" || nodes[0].SNR == nil || *nodes[0].SNR != -6 { + t.Errorf("node[0] want ccdd@-6 (strongest), got %+v", nodes[0]) + } + if nodes[1].Prefix != "aabb" || nodes[1].SNR == nil || *nodes[1].SNR != -10 || nodes[1].Count != 2 { + t.Errorf("node[1] want aabb latest -10 count 2, got %+v", nodes[1]) + } + if nodes[2].Prefix != "eeff" || nodes[2].SNR != nil { + t.Errorf("node[2] want eeff no-signal (last), got %+v", nodes[2]) + } +} + +// TestResolveHeardKey covers heard_key → (pubkey, name) resolution: a unique match +// returns the canonical pubkey + name; an ambiguous prefix (>1 node) and an +// unknown/empty key return the key itself with an empty name. +func TestResolveHeardKey(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbccdd11223344','Alice','repeater')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbcc99887766aa','Bob','repeater')`) + srv := &Server{db: db} + if k, n := srv.resolveHeardKey("aabbccdd"); k != "aabbccdd11223344" || n != "Alice" { + t.Errorf("unique prefix → (pubkey,Alice), got (%q,%q)", k, n) + } + if k, n := srv.resolveHeardKey("aabbcc"); k != "aabbcc" || n != "" { + t.Errorf("ambiguous prefix → (key,\"\"), got (%q,%q)", k, n) + } + if k, n := srv.resolveHeardKey("ffff"); k != "ffff" || n != "" { + t.Errorf("unknown prefix → (key,\"\"), got (%q,%q)", k, n) + } + if k, n := srv.resolveHeardKey(""); k != "" || n != "" { + t.Errorf("empty prefix → (\"\",\"\"), got (%q,%q)", k, n) + } +} + +// TestAggregateCoverageMergesResolvedNodes verifies that the same node heard under +// two different heard_keys (e.g. a 3-byte prefix and the full pubkey) collapses into a +// single entry — summed count, latest SNR — when the resolver maps both to one node. +func TestAggregateCoverageMergesResolvedNodes(t *testing.T) { + rows := []coverageRow{ + {Lat: 51.05, Lon: 3.72, SNR: covF(-4), HeardKey: "aabbcc", RxAt: "2026-06-01T10:00:00Z"}, + {Lat: 51.05, Lon: 3.72, SNR: covF(-9), HeardKey: "aabbccdd11223344", RxAt: "2026-06-03T10:00:00Z"}, + {Lat: 51.05, Lon: 3.72, SNR: covF(-7), HeardKey: "aabbcc", RxAt: "2026-06-02T10:00:00Z"}, + } + resolve := func(hk string) (string, string) { return "aabbccdd11223344", "Alice" } + fc := aggregateCoverage(rows, 9, resolve) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + nodes := fc.Features[0].Properties.Nodes + if len(nodes) != 1 { + t.Fatalf("expected 1 merged node, got %d (%+v)", len(nodes), nodes) + } + n := nodes[0] + if n.Name != "Alice" || n.Count != 3 || n.SNR == nil || *n.SNR != -9 { + t.Errorf("merged node want Alice count 3 latest -9, got %+v (snr=%v)", n, n.SNR) + } +} + +func TestZoomToHexRes(t *testing.T) { + // Resolution tracks zoom 1:1 within [3,18], clamped at the edges (z=0 is the + // missing-param case). + cases := map[int]int{0: 3, 3: 3, 8: 8, 16: 16, 18: 18, 25: 18} + for z, want := range cases { + if got := zoomToHexRes(z); got != want { + t.Fatalf("zoomToHexRes(%d)=%d, want %d", z, got, want) + } + } +} + +// TestHexSizeRendersConstantPx verifies the core fix: a hex sized for resolution +// res renders at a constant ~hexTargetPx on screen at the corresponding zoom level, +// instead of the old fixed-meter buckets that were ~2px when zoomed out. +func TestHexSizeRendersConstantPx(t *testing.T) { + for res := 4; res <= 16; res++ { + // On-screen point-to-point height = 2*circumradius / mercUnitsPerPixel(zoom), + // where mercUnitsPerPixel = mercUPPZ0 / 2^zoom and zoom == res. + px := 2 * hexSizeForRes(res) * math.Pow(2, float64(res)) / mercUPPZ0 + if math.Abs(px-hexTargetPx) > 0.001 { + t.Fatalf("res %d renders %.2fpx, want %.2fpx", res, px, hexTargetPx) + } + // Size must halve each zoom step (finer grid as you zoom in). + if ratio := hexSizeForRes(res) / hexSizeForRes(res+1); math.Abs(ratio-2) > 1e-9 { + t.Fatalf("res %d→%d size ratio %.4f, want 2", res, res+1, ratio) + } + } +} diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go new file mode 100644 index 00000000..58899c9f --- /dev/null +++ b/cmd/server/rx_dashboard.go @@ -0,0 +1,217 @@ +package main + +import ( + "database/sql" + "encoding/json" + "net/http" + "strconv" + "strings" + "time" +) + +// scanCoverageRows reads (lat,lon,snr,rssi,heard_key,rx_at) rows into coverageRow values. +func scanCoverageRows(rows *sql.Rows) ([]coverageRow, error) { + out := []coverageRow{} + for rows.Next() { + var lat, lon float64 + var snr sql.NullFloat64 + var rssi sql.NullInt64 + var heardKey, rxAt sql.NullString + if err := rows.Scan(&lat, &lon, &snr, &rssi, &heardKey, &rxAt); err != nil { + return nil, err + } + cr := coverageRow{Lat: lat, Lon: lon, HeardKey: strings.ToLower(heardKey.String), RxAt: rxAt.String} + if snr.Valid { + v := snr.Float64 + cr.SNR = &v + } + if rssi.Valid { + v := int(rssi.Int64) + cr.RSSI = &v + } + out = append(out, cr) + } + return out, rows.Err() +} + +// heardKeyResolver returns a request-scoped, memoized nodeResolver. It maps a heard_key +// to (pubkey, name) on a unique match — so the same node heard under different prefix +// lengths collapses into one entry — and to (heardKey, "") when unknown or ambiguous. +func (s *Server) heardKeyResolver() nodeResolver { + if s.db == nil || s.db.conn == nil { + return nil + } + type kv struct{ key, name string } + cache := map[string]kv{} + return func(heardKey string) (string, string) { + if v, ok := cache[heardKey]; ok { + return v.key, v.name + } + key, name := s.resolveHeardKey(heardKey) + cache[heardKey] = kv{key, name} + return key, name + } +} + +// resolveHeardKey resolves a heard_key (2-3 byte prefix or full pubkey) to a canonical +// (pubkey, name) on a unique match. Unknown or ambiguous (>1 match) keys return the +// heard_key itself with an empty name. LIMIT 2 is enough to tell unique from ambiguous. +func (s *Server) resolveHeardKey(heardKey string) (string, string) { + if heardKey == "" || !hexPrefixRe.MatchString(heardKey) { + return heardKey, "" + } + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, heardKey+"%") + if err != nil { + return heardKey, "" + } + defer rows.Close() + var pks, names []string + for rows.Next() { + var pk, n string + if err := rows.Scan(&pk, &n); err != nil { + return heardKey, "" + } + pks = append(pks, pk) + names = append(names, n) + } + if len(pks) == 1 { + return pks[0], names[0] + } + return heardKey, "" +} + +// queryCoverageFiltered returns coverage rows within a bbox, optionally filtered +// by heard node (prefix/pubkey), contributing client (rx_pubkey), and time window +// (days; 0 = all time). Powers the global and per-observer coverage maps. +func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]coverageRow, error) { + where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?"} + args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} + if node != "" { + pk := strings.ToLower(node) + where = append(where, "((heard_keylen = 32 AND heard_key = ?) OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key))") + args = append(args, pk, pk) + } + if rx != "" { + where = append(where, "rx_pubkey = ?") + args = append(args, strings.ToLower(rx)) + } + if days > 0 { + since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + where = append(where, "rx_at >= ?") + args = append(args, since) + } + rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanCoverageRows(rows) +} + +// handleRxCoverage serves global (or per-observer via ?rx=) coverage as GeoJSON +// hexbins, over a time window. ?node= also works (same as the per-node endpoint). +// requireClientRxCoverage writes a 404 and returns false when the opt-in +// client-RX coverage feature is disabled, so the coverage endpoints read as +// "not found" instead of serving data on deployments that haven't enabled it. +func (s *Server) requireClientRxCoverage(w http.ResponseWriter, r *http.Request) bool { + if !s.cfg.ClientRxCoverageEnabled() { + http.NotFound(w, r) + return false + } + return true +} + +func (s *Server) handleRxCoverage(w http.ResponseWriter, r *http.Request) { + if !s.requireClientRxCoverage(w, r) { + return + } + b, ok := parseBBox(r.URL.Query().Get("bbox")) + if !ok { + http.Error(w, "bbox required as minLat,minLon,maxLat,maxLon", http.StatusBadRequest) + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + days := clampDays(atoiDefault(r.URL.Query().Get("days"), 7)) + z, _ := strconv.Atoi(r.URL.Query().Get("z")) + rows, err := s.queryCoverageFiltered(r.URL.Query().Get("node"), r.URL.Query().Get("rx"), days, b) + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolver()) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(fc) +} + +// --- Leaderboard (top mobile observers) --- + +type LeaderObserver struct { + Pubkey string `json:"pubkey"` + Name string `json:"name"` + Receptions int `json:"receptions"` + Nodes int `json:"nodes"` +} +type RxLeaderboardResp struct { + Days int `json:"days"` + Observers []LeaderObserver `json:"observers"` +} + +func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { + since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + // Name preference: the node's advertised name, else the companion's + // self-reported name (client_observers), else empty (UI shows the prefix). + rows, err := s.db.conn.Query(` + SELECT cr.rx_pubkey, COALESCE(NULLIF(n.name,''), NULLIF(co.name,''), ''), COUNT(*), COUNT(DISTINCT cr.heard_key) + FROM client_receptions cr + LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey + LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey + WHERE cr.rx_at >= ? + GROUP BY cr.rx_pubkey + ORDER BY COUNT(*) DESC + LIMIT ?`, since, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []LeaderObserver{} + for rows.Next() { + var o LeaderObserver + if err := rows.Scan(&o.Pubkey, &o.Name, &o.Receptions, &o.Nodes); err != nil { + return nil, err + } + out = append(out, o) + } + return out, rows.Err() +} + +func (s *Server) handleRxLeaderboard(w http.ResponseWriter, r *http.Request) { + if !s.requireClientRxCoverage(w, r) { + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + days := clampDays(atoiDefault(r.URL.Query().Get("days"), 7)) + limit := atoiDefault(r.URL.Query().Get("limit"), 20) + if limit < 1 || limit > 100 { + limit = 20 + } + obs, err := s.rxLeaderboard(days, limit) + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(RxLeaderboardResp{Days: days, Observers: obs}) +} + +func atoiDefault(s string, d int) int { + if n, err := strconv.Atoi(strings.TrimSpace(s)); err == nil { + return n + } + return d +} diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go new file mode 100644 index 00000000..c2ce1d83 --- /dev/null +++ b/cmd/server/rx_dashboard_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "testing" + "time" +) + +func insRx(t *testing.T, db *DB, rx, hk, at string, lat, lon float64) { + mustExecDB(t, db, fmt.Sprintf( + `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('%s','%s',3,-6,%f,%f,'%s','x','rxlog')`, + rx, hk, lat, lon, at)) +} + +func TestQueryCoverageFiltered(t *testing.T) { + db := seedCoverageDB(t) + now := time.Now().UTC() + recent := now.Format(time.RFC3339) + old := now.AddDate(0, 0, -40).Format(time.RFC3339) + insRx(t, db, "compa", "aabbcc", recent, 51.05, 3.72) + insRx(t, db, "compb", "ffeedd", recent, 51.06, 3.73) + insRx(t, db, "compa", "aabbcc", old, 51.05, 3.72) + srv := &Server{db: db} + bb := bbox{MinLat: 50, MinLon: 3, MaxLat: 52, MaxLon: 4} + + if rows, _ := srv.queryCoverageFiltered("", "", 7, bb); len(rows) != 2 { + t.Fatalf("global 7d: want 2, got %d", len(rows)) + } + if rows, _ := srv.queryCoverageFiltered("", "compa", 7, bb); len(rows) != 1 { + t.Fatalf("observer compa 7d: want 1, got %d", len(rows)) + } + if rows, _ := srv.queryCoverageFiltered("", "", 0, bb); len(rows) != 3 { + t.Fatalf("global all-time: want 3, got %d", len(rows)) + } +} + +func TestRxLeaderboard(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('compa','MyCompanion','companion','t','t',1)`) + // compc is NOT in nodes, but reported its name via client_observers (fallback). + mustExecDB(t, db, `INSERT INTO client_observers (pubkey, name, last_seen) VALUES ('compc','MobOnly','t')`) + for i := 0; i < 3; i++ { + insRx(t, db, "compa", fmt.Sprintf("aabb%02d", i), recent, 51.05, 3.72) + } + insRx(t, db, "compc", "ddee00", recent, 51.05, 3.72) + insRx(t, db, "compc", "ddee01", recent, 51.05, 3.72) + insRx(t, db, "compb", "aabbcc", recent, 51.05, 3.72) // no name anywhere + srv := &Server{db: db} + + obs, err := srv.rxLeaderboard(7, 10) + if err != nil { + t.Fatal(err) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + if byPk["compa"].Name != "MyCompanion" || byPk["compa"].Receptions != 3 { + t.Fatalf("compa (nodes name): %+v", byPk["compa"]) + } + if byPk["compc"].Name != "MobOnly" || byPk["compc"].Receptions != 2 { + t.Fatalf("compc (client_observers fallback): %+v", byPk["compc"]) + } + if byPk["compb"].Name != "" { + t.Fatalf("compb should have no name: %+v", byPk["compb"]) + } +} diff --git a/cmd/server/types.go b/cmd/server/types.go index e3c71ab4..29b3b5dd 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -55,21 +55,21 @@ type TimeBucket struct { // ─── Stats ───────────────────────────────────────────────────────────────────── type StatsResponse struct { - TotalPackets int `json:"totalPackets"` - TotalTransmissions *int `json:"totalTransmissions"` - TotalObservations int `json:"totalObservations"` - TotalNodes int `json:"totalNodes"` - TotalNodesAllTime int `json:"totalNodesAllTime"` - TotalObservers int `json:"totalObservers"` - PacketsLastHour int `json:"packetsLastHour"` - PacketsLast24h int `json:"packetsLast24h"` - Engine string `json:"engine"` - Version string `json:"version"` - Commit string `json:"commit"` - BuildTime string `json:"buildTime"` - Counts RoleCounts `json:"counts"` - SignatureDrops int64 `json:"signatureDrops,omitempty"` - HashMigrationComplete bool `json:"hashMigrationComplete"` + TotalPackets int `json:"totalPackets"` + TotalTransmissions *int `json:"totalTransmissions"` + TotalObservations int `json:"totalObservations"` + TotalNodes int `json:"totalNodes"` + TotalNodesAllTime int `json:"totalNodesAllTime"` + TotalObservers int `json:"totalObservers"` + PacketsLastHour int `json:"packetsLastHour"` + PacketsLast24h int `json:"packetsLast24h"` + Engine string `json:"engine"` + Version string `json:"version"` + Commit string `json:"commit"` + BuildTime string `json:"buildTime"` + Counts RoleCounts `json:"counts"` + SignatureDrops int64 `json:"signatureDrops,omitempty"` + HashMigrationComplete bool `json:"hashMigrationComplete"` // Memory accounting (issue #832). All values in MB. // @@ -207,31 +207,31 @@ type EndpointStatsResp struct { } type PacketStoreIndexes struct { - ByHash int `json:"byHash"` - ByObserver int `json:"byObserver"` - ByNode int `json:"byNode"` + ByHash int `json:"byHash"` + ByObserver int `json:"byObserver"` + ByNode int `json:"byNode"` AdvertByObserver int `json:"advertByObserver"` } type PerfPacketStoreStats struct { - TotalLoaded int `json:"totalLoaded"` - TotalObservations int `json:"totalObservations"` - Evicted int `json:"evicted"` - Inserts int64 `json:"inserts"` - Queries int64 `json:"queries"` - InMemory int `json:"inMemory"` - SqliteOnly bool `json:"sqliteOnly"` - MaxPackets int `json:"maxPackets"` - EstimatedMB float64 `json:"estimatedMB"` - TrackedMB float64 `json:"trackedMB"` - AvgBytesPerPacket int64 `json:"avgBytesPerPacket"` - MaxMB int `json:"maxMB"` - Indexes PacketStoreIndexes `json:"indexes"` - HotStartupHours float64 `json:"hotStartupHours"` - BackgroundLoadComplete bool `json:"backgroundLoadComplete"` - BackgroundLoadFailed bool `json:"backgroundLoadFailed"` - BackgroundLoadProgress int64 `json:"backgroundLoadProgress"` - BackgroundLoadError string `json:"backgroundLoadError,omitempty"` + TotalLoaded int `json:"totalLoaded"` + TotalObservations int `json:"totalObservations"` + Evicted int `json:"evicted"` + Inserts int64 `json:"inserts"` + Queries int64 `json:"queries"` + InMemory int `json:"inMemory"` + SqliteOnly bool `json:"sqliteOnly"` + MaxPackets int `json:"maxPackets"` + EstimatedMB float64 `json:"estimatedMB"` + TrackedMB float64 `json:"trackedMB"` + AvgBytesPerPacket int64 `json:"avgBytesPerPacket"` + MaxMB int `json:"maxMB"` + Indexes PacketStoreIndexes `json:"indexes"` + HotStartupHours float64 `json:"hotStartupHours"` + BackgroundLoadComplete bool `json:"backgroundLoadComplete"` + BackgroundLoadFailed bool `json:"backgroundLoadFailed"` + BackgroundLoadProgress int64 `json:"backgroundLoadProgress"` + BackgroundLoadError string `json:"backgroundLoadError,omitempty"` // #1690: surface retention + coverage so operators can see how much // of the on-disk DB the in-memory store currently reflects. RetentionHours float64 `json:"retentionHours"` @@ -288,24 +288,24 @@ type GoRuntimeStats struct { // ─── Packets ─────────────────────────────────────────────────────────────────── type TransmissionResp struct { - ID int `json:"id"` - RawHex interface{} `json:"raw_hex"` - Hash string `json:"hash"` - FirstSeen string `json:"first_seen"` - Timestamp string `json:"timestamp"` - RouteType interface{} `json:"route_type"` - PayloadType interface{} `json:"payload_type"` - PayloadVersion interface{} `json:"payload_version,omitempty"` - DecodedJSON interface{} `json:"decoded_json"` - ObservationCount int `json:"observation_count"` - ObserverID interface{} `json:"observer_id"` - ObserverName interface{} `json:"observer_name"` - ObserverIATA interface{} `json:"observer_iata"` - SNR interface{} `json:"snr"` - RSSI interface{} `json:"rssi"` - PathJSON interface{} `json:"path_json"` - Direction interface{} `json:"direction"` - Score interface{} `json:"score,omitempty"` + ID int `json:"id"` + RawHex interface{} `json:"raw_hex"` + Hash string `json:"hash"` + FirstSeen string `json:"first_seen"` + Timestamp string `json:"timestamp"` + RouteType interface{} `json:"route_type"` + PayloadType interface{} `json:"payload_type"` + PayloadVersion interface{} `json:"payload_version,omitempty"` + DecodedJSON interface{} `json:"decoded_json"` + ObservationCount int `json:"observation_count"` + ObserverID interface{} `json:"observer_id"` + ObserverName interface{} `json:"observer_name"` + ObserverIATA interface{} `json:"observer_iata"` + SNR interface{} `json:"snr"` + RSSI interface{} `json:"rssi"` + PathJSON interface{} `json:"path_json"` + Direction interface{} `json:"direction"` + Score interface{} `json:"score,omitempty"` Observations []ObservationResp `json:"observations,omitempty"` } @@ -374,18 +374,18 @@ type DecodeResponse struct { // ─── Nodes ───────────────────────────────────────────────────────────────────── type NodeResp struct { - PublicKey string `json:"public_key"` - Name interface{} `json:"name"` - Role interface{} `json:"role"` - Lat interface{} `json:"lat"` - Lon interface{} `json:"lon"` - LastSeen interface{} `json:"last_seen"` - FirstSeen interface{} `json:"first_seen"` - AdvertCount int `json:"advert_count"` - HashSize interface{} `json:"hash_size,omitempty"` - HashSizeInconsistent bool `json:"hash_size_inconsistent,omitempty"` - HashSizesSeen []int `json:"hash_sizes_seen,omitempty"` - LastHeard interface{} `json:"last_heard,omitempty"` + PublicKey string `json:"public_key"` + Name interface{} `json:"name"` + Role interface{} `json:"role"` + Lat interface{} `json:"lat"` + Lon interface{} `json:"lon"` + LastSeen interface{} `json:"last_seen"` + FirstSeen interface{} `json:"first_seen"` + AdvertCount int `json:"advert_count"` + HashSize interface{} `json:"hash_size,omitempty"` + HashSizeInconsistent bool `json:"hash_size_inconsistent,omitempty"` + HashSizesSeen []int `json:"hash_sizes_seen,omitempty"` + LastHeard interface{} `json:"last_heard,omitempty"` } type NodeListResponse struct { @@ -669,7 +669,7 @@ type TopologyResponse struct { HopDistribution []TopologyHopDist `json:"hopDistribution"` TopRepeaters []TopRepeater `json:"topRepeaters"` TopPairs []TopPair `json:"topPairs"` - HopsVsSnr []HopsVsSnr `json:"hopsVsSnr"` + HopsVsSnr []HopsVsSnr `json:"hopsVsSnr"` Observers []ObserverRef `json:"observers"` PerObserverReach map[string]*ObserverReach `json:"perObserverReach"` MultiObsNodes []MultiObsNode `json:"multiObsNodes"` @@ -761,12 +761,12 @@ type DistOverTimeEntry struct { } type DistanceAnalyticsResponse struct { - Summary DistanceSummary `json:"summary"` - TopHops []DistanceHop `json:"topHops"` - TopPaths []DistancePath `json:"topPaths"` - CatStats map[string]*CategoryDistStats `json:"catStats"` - DistHistogram *Histogram `json:"distHistogram"` - DistOverTime []DistOverTimeEntry `json:"distOverTime"` + Summary DistanceSummary `json:"summary"` + TopHops []DistanceHop `json:"topHops"` + TopPaths []DistancePath `json:"topPaths"` + CatStats map[string]*CategoryDistStats `json:"catStats"` + DistHistogram *Histogram `json:"distHistogram"` + DistOverTime []DistOverTimeEntry `json:"distOverTime"` } // ─── Analytics — Hash Sizes ──────────────────────────────────────────────────── @@ -795,11 +795,11 @@ type MultiByteNode struct { } type HashSizeAnalyticsResponse struct { - Total int `json:"total"` - Distribution map[string]int `json:"distribution"` - Hourly []HashSizeHourly `json:"hourly"` - TopHops []HashSizeHop `json:"topHops"` - MultiByteNodes []MultiByteNode `json:"multiByteNodes"` + Total int `json:"total"` + Distribution map[string]int `json:"distribution"` + Hourly []HashSizeHourly `json:"hourly"` + TopHops []HashSizeHop `json:"topHops"` + MultiByteNodes []MultiByteNode `json:"multiByteNodes"` } // ─── Analytics — Subpaths ────────────────────────────────────────────────────── @@ -933,10 +933,10 @@ type SnrDistributionEntry struct { } type ObserverAnalyticsResponse struct { - Timeline []TimeBucket `json:"timeline"` - PacketTypes map[string]int `json:"packetTypes"` - NodesTimeline []TimeBucket `json:"nodesTimeline"` - SnrDistribution []SnrDistributionEntry `json:"snrDistribution"` + Timeline []TimeBucket `json:"timeline"` + PacketTypes map[string]int `json:"packetTypes"` + NodesTimeline []TimeBucket `json:"nodesTimeline"` + SnrDistribution []SnrDistributionEntry `json:"snrDistribution"` RecentPackets []map[string]interface{} `json:"recentPackets"` } @@ -999,24 +999,25 @@ type MapConfigResponse struct { } type ClientConfigResponse struct { - Roles interface{} `json:"roles"` - HealthThresholds interface{} `json:"healthThresholds"` - Map interface{} `json:"map"` - Tiles interface{} `json:"tiles,omitempty"` // deprecated - SnrThresholds interface{} `json:"snrThresholds"` - DistThresholds interface{} `json:"distThresholds"` - MaxHopDist interface{} `json:"maxHopDist"` - Limits interface{} `json:"limits"` - PerfSlowMs interface{} `json:"perfSlowMs"` - WsReconnectMs interface{} `json:"wsReconnectMs"` - CacheInvalidateMs interface{} `json:"cacheInvalidateMs"` - ExternalUrls interface{} `json:"externalUrls"` - PropagationBufferMs float64 `json:"propagationBufferMs"` - LiveMapMaxNodes int `json:"liveMapMaxNodes"` - Timestamps TimestampConfig `json:"timestamps"` - DebugAffinity bool `json:"debugAffinity,omitempty"` - MapDarkTileProvider string `json:"mapDarkTileProvider,omitempty"` // deprecated. TODO: remove after v3.5.0 + Roles interface{} `json:"roles"` + HealthThresholds interface{} `json:"healthThresholds"` + Map interface{} `json:"map"` + Tiles interface{} `json:"tiles,omitempty"` // deprecated + SnrThresholds interface{} `json:"snrThresholds"` + DistThresholds interface{} `json:"distThresholds"` + MaxHopDist interface{} `json:"maxHopDist"` + Limits interface{} `json:"limits"` + PerfSlowMs interface{} `json:"perfSlowMs"` + WsReconnectMs interface{} `json:"wsReconnectMs"` + CacheInvalidateMs interface{} `json:"cacheInvalidateMs"` + ExternalUrls interface{} `json:"externalUrls"` + PropagationBufferMs float64 `json:"propagationBufferMs"` + LiveMapMaxNodes int `json:"liveMapMaxNodes"` + Timestamps TimestampConfig `json:"timestamps"` + DebugAffinity bool `json:"debugAffinity,omitempty"` + MapDarkTileProvider string `json:"mapDarkTileProvider,omitempty"` // deprecated. TODO: remove after v3.5.0 Customizer CustomizerClientConfig `json:"customizer"` + ClientRxCoverage bool `json:"clientRxCoverage"` } // CustomizerClientConfig is the operator-side customizer-modal knobs that diff --git a/config.example.json b/config.example.json index ac2ce525..0fc986cc 100644 --- a/config.example.json +++ b/config.example.json @@ -357,6 +357,8 @@ ] }, "_comment_compression": "Opt-in HTTP gzip middleware + WebSocket permessage-deflate. Both default to false — enable ONLY when your upstream reverse proxy is NOT already compressing. gzip: enables the gzipMiddleware wrapper around the HTTP handler. websocket: sets gorilla websocket Upgrader.EnableCompression. level: gzip compression level 1..9 (1=BestSpeed, 9=BestCompression, default 6). minSizeBytes: advisory minimum response size below which compression would not pay off. contentTypes: MIME allow-list — only responses with these Content-Type values are compressed. Already-compressed types (image/*, video/*, audio/*, application/zip, application/x-gzip, application/pdf, application/octet-stream) are always skipped, as are responses whose handler already set Content-Encoding. Omit contentTypes to use the built-in default allow-list.", + "clientRxCoverage": { "enabled": false }, + "_comment_clientRxCoverage": "Opt-in mobile client-RX coverage (corescope-rx companions publishing GPS-tagged receptions to meshcore/client//packets). Default OFF: when disabled the ingestor writes no client_receptions, the /api/rx-coverage|rx-leaderboard|nodes/{pubkey}/rx-coverage endpoints 404, and the UI hides the Coverage dashboard + Reach overlay. Set enabled=true to turn it on.", "_comment_channelKeys": "Hex keys for decrypting channel messages. Key name = channel display name. public channel key is well-known.", "_comment_hashChannels": "Channel names whose keys are derived via SHA256. Key = SHA256(name)[:16]. Listed here so the ingestor can auto-derive keys.", "hashRegions": [ From b2de601af8af10adfc98f8b7b9e7c7ba74a3800d Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 18:30:56 +0200 Subject: [PATCH 04/38] feat(nodes): /api/nodes/resolve prefix lookup Co-Authored-By: Claude Opus 4.8 --- cmd/server/routes.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 7a43184a..95586c1b 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -267,10 +267,17 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/nodes/{pubkey}/clock-skew", s.handleNodeClockSkew).Methods("GET") r.HandleFunc("/api/observers/clock-skew", s.handleObserverClockSkew).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}/neighbors", s.handleNodeNeighbors).Methods("GET") - // Keep specific sub-routes (…/reach) registered BEFORE the catch-all - // /api/nodes/{pubkey} — mux matches in registration order, so reordering - // this below the catch-all would shadow it and break the route. + // Keep specific sub-routes (…/reach, …/rx-coverage) registered BEFORE the + // catch-all /api/nodes/{pubkey} — mux matches in registration order, so + // reordering these below the catch-all would shadow them and break the route. r.HandleFunc("/api/nodes/{pubkey}/reach", s.handleNodeReach).Methods("GET") + // Coverage routes are always registered; each handler 404s when the opt-in + // clientRxCoverage flag is off (a clean 404 rather than the SPA fallback that + // an unregistered /api route would hit). See requireClientRxCoverage. + r.HandleFunc("/api/nodes/{pubkey}/rx-coverage", s.handleNodeRxCoverage).Methods("GET") + r.HandleFunc("/api/nodes/resolve", s.handleResolvePrefix).Methods("GET") + r.HandleFunc("/api/rx-coverage", s.handleRxCoverage).Methods("GET") + r.HandleFunc("/api/rx-leaderboard", s.handleRxLeaderboard).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}", s.handleNodeDetail).Methods("GET") r.HandleFunc("/api/nodes", s.handleNodes).Methods("GET") @@ -445,6 +452,7 @@ func (s *Server) handleConfigClient(w http.ResponseWriter, r *http.Request) { MapDarkTileProvider: s.cfg.MapDarkTileProvider, Tiles: s.cfg.Tiles, Customizer: CustomizerClientConfig{DisabledTabs: disabledTabs}, + ClientRxCoverage: s.cfg.ClientRxCoverageEnabled(), }) } From c6e7f19252a64c5dc9b8c174536527f1cebfc001 Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 18:38:13 +0200 Subject: [PATCH 05/38] ci: run client-RX coverage tests (unit + e2e) Co-Authored-By: Claude Opus 4.8 --- .github/workflows/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 55a304ad..aeebe173 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -138,6 +138,8 @@ jobs: node test-issue-1509-nav-active-bg.js node test-issue-1509-detect-preset.js node test-live.js + node test-coverage-gate.js + node test-node-reach-coverage.js node test-issue-1107-live-layout.js node test-issue-1532-live-fullscreen.js node test-issue-1619-feed-detail-card-draggable.js @@ -463,6 +465,7 @@ jobs: CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt # #1616: slide-over focus-restore flake-gate. Runs the slide-over From 191c99803aa3bd7f16ab8144a55d1c562bf57ade Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 18:50:43 +0200 Subject: [PATCH 06/38] test(coverage): e2e skips when clientRxCoverage is disabled (CI-safe default-off) Co-Authored-By: Claude Opus 4.8 --- test-node-reach-coverage-e2e.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test-node-reach-coverage-e2e.js b/test-node-reach-coverage-e2e.js index 9e7f5dfb..af70be9f 100644 --- a/test-node-reach-coverage-e2e.js +++ b/test-node-reach-coverage-e2e.js @@ -7,6 +7,15 @@ const BASE = process.env.BASE_URL || 'http://localhost:3000'; const browser = await chromium.launch(); const page = await browser.newPage(); + // Coverage is opt-in (config flag, default off). Skip when the deployment under + // test hasn't enabled it — the endpoints 404 and the UI toggle is absent by design. + const clientCfg = await (await page.request.get(BASE + '/api/config/client')).json(); + if (clientCfg.clientRxCoverage !== true) { + console.log('node-reach-coverage E2E SKIP (clientRxCoverage disabled on this deployment)'); + await browser.close(); + return; + } + const nodes = await (await page.request.get(BASE + '/api/nodes?role=repeater&limit=1')).json(); if (!nodes.nodes || !nodes.nodes.length) { console.log('node-reach-coverage E2E SKIP (no repeater in dataset)'); From a3956bddabb45f200e6962ffd54f140bc6509141 Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 19:50:49 +0200 Subject: [PATCH 07/38] docs(coverage): add payload-contract doc + link the companion app (corescope-rx) The PR was missing docs/client-rx-coverage.md (the MQTT payload contract) and gave operators/users no pointer to the mobile capture app. Add the doc with a 'Companion app' section + operator enable steps, link corescope-rx from the config.example.json flag comment, and add a 'Get the companion app' link on the Coverage dashboard. Co-Authored-By: Claude Opus 4.8 --- config.example.json | 2 +- docs/client-rx-coverage.md | 148 +++++++++++++++++++++++++++++++++++++ public/rx-coverage.js | 2 +- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 docs/client-rx-coverage.md diff --git a/config.example.json b/config.example.json index 0fc986cc..58500a46 100644 --- a/config.example.json +++ b/config.example.json @@ -358,7 +358,7 @@ }, "_comment_compression": "Opt-in HTTP gzip middleware + WebSocket permessage-deflate. Both default to false — enable ONLY when your upstream reverse proxy is NOT already compressing. gzip: enables the gzipMiddleware wrapper around the HTTP handler. websocket: sets gorilla websocket Upgrader.EnableCompression. level: gzip compression level 1..9 (1=BestSpeed, 9=BestCompression, default 6). minSizeBytes: advisory minimum response size below which compression would not pay off. contentTypes: MIME allow-list — only responses with these Content-Type values are compressed. Already-compressed types (image/*, video/*, audio/*, application/zip, application/x-gzip, application/pdf, application/octet-stream) are always skipped, as are responses whose handler already set Content-Encoding. Omit contentTypes to use the built-in default allow-list.", "clientRxCoverage": { "enabled": false }, - "_comment_clientRxCoverage": "Opt-in mobile client-RX coverage (corescope-rx companions publishing GPS-tagged receptions to meshcore/client//packets). Default OFF: when disabled the ingestor writes no client_receptions, the /api/rx-coverage|rx-leaderboard|nodes/{pubkey}/rx-coverage endpoints 404, and the UI hides the Coverage dashboard + Reach overlay. Set enabled=true to turn it on.", + "_comment_clientRxCoverage": "Opt-in mobile client-RX coverage (corescope-rx companions publishing GPS-tagged receptions to meshcore/client//packets). Default OFF: when disabled the ingestor writes no client_receptions, the /api/rx-coverage|rx-leaderboard|nodes/{pubkey}/rx-coverage endpoints 404, and the UI hides the Coverage dashboard + Reach overlay. Set enabled=true to turn it on. Companion app (the mobile capture side users run) + setup: https://github.com/efiten/corescope-rx — see docs/client-rx-coverage.md.", "_comment_channelKeys": "Hex keys for decrypting channel messages. Key name = channel display name. public channel key is well-known.", "_comment_hashChannels": "Channel names whose keys are derived via SHA256. Key = SHA256(name)[:16]. Listed here so the ingestor can auto-derive keys.", "hashRegions": [ diff --git a/docs/client-rx-coverage.md b/docs/client-rx-coverage.md new file mode 100644 index 00000000..111dd37f --- /dev/null +++ b/docs/client-rx-coverage.md @@ -0,0 +1,148 @@ +# Client RX Coverage + +Crowdsourced RF coverage from mobile clients: a phone connects over BLE to a MeshCore +*companion* radio, captures which nodes the companion hears (with SNR/RSSI), tags each reception +with the phone's GPS position, and publishes it to MQTT. CoreScope ingests these into +`client_receptions` and renders per-node H3-style hex coverage on the Reach page. + +## Companion app — where to get it + +The mobile capture side is **[corescope-rx](https://github.com/efiten/corescope-rx)** — an +open-source (GPL-3.0) Android PWA. Operators who enable coverage point their users at it: it connects +over BLE to a MeshCore companion radio, captures directly-heard nodes + the phone's GPS, and publishes +the payload defined below. It's self-hostable and generic — a runtime `config.json` aims it at your +own MQTT broker + CoreScope instance (see its README). + +## Enabling coverage (operators) + +Coverage is **off by default**. To turn it on: + +1. In CoreScope's `config.json`, set `"clientRxCoverage": { "enabled": true }` and restart the server + and ingestor. +2. Make sure your broker lets a client publish to `meshcore/client/{PUBLIC_KEY}/packets` (the ingestor + already subscribes under `meshcore/#`). An EMQX ACL binding each client to its own `{PUBLIC_KEY}` + topic is recommended. +3. Point your users at [corescope-rx](https://github.com/efiten/corescope-rx) and they start + contributing. Results show on each node's Reach page (coverage toggle) and the `#/rx-coverage` + dashboard. + +The rest of this document is the MQTT payload contract the companion app implements. + +## Companion BLE source (verified against firmware) + +The mobile app's RX data comes from the companion's **`PUSH_CODE_LOG_RX_DATA` (0x88)** BLE frame: +`[0x88][snr×4 int8][rssi int8][raw packet bytes]`. This is emitted for **every** received +packet (promiscuous, incl. overheard flood traffic), not just messages addressed to the device: + +- `src/Dispatcher.cpp:198` calls `logRxRaw(getLastSNR(), getLastRSSI(), raw, len)` in `checkRecv()` + **unconditionally** — NOT behind `#if MESH_PACKET_LOGGING`. So it works on stock firmware. +- `examples/companion_radio/MyMesh.cpp:283` overrides it to write the 0x88 frame whenever the app + is connected over BLE (`_serial->isConnected()`). + +So per received packet the app gets SNR + RSSI + the raw bytes. It decodes the raw packet (standard +MeshCore format) to derive the directly-heard node (`path[last]` or 0-hop advert pubkey) and pairs it +with the phone's GPS. The bare advert push (`PUSH_CODE_ADVERT` 0x80) carries only a pubkey (no SNR/ +RSSI/path) and is NOT used — 0x88 already covers adverts (the raw advert is in its payload). + +Caveats: 0x88 is only sent while the app is BLE-connected; packets larger than `MAX_FRAME_SIZE` are +skipped; the firmware doc labels 0x88 "can be ignored" (messaging-app view) — for coverage it is the +primary frame. GPS is always the phone's, never the companion's. + +## MQTT topic & payload + +Topic: `meshcore/client/{PUBLIC_KEY}/packets` — `{PUBLIC_KEY}` is the companion's pubkey. The +broker (EMQX) should ACL-restrict each client to publish only under its own pubkey, which is how +"a connected companion may only inject under the keys that apply" is enforced. + +Payload — meshcoretomqtt-compatible packet, plus a `gps` object: + +```json +{ + "origin": "", + "origin_id": "", + "timestamp": "2026-06-09T12:00:00Z", + "type": "PACKET", + "direction": "rx", + "raw": "", + "SNR": -7, + "RSSI": -92, + "gps": { "lat": 51.05, "lon": 3.72, "acc_m": 8 } +} +``` + +- The discriminator is the `gps` object. A packet without `gps` is dropped (coverage needs a position). +- `raw` is decoded server-side to derive the directly-heard node and the path; `hash`/`path` fields + are not required. +- Subscription: the ingestor's default subscription (`meshcore/#`) already covers this topic. Sources + configured with an explicit topic list must add `meshcore/client/+/packets`. + +## Capture HARD RULE — only what was heard directly + +The app and ingestor record **only the node the companion physically received**, never upstream +relayers: + +- **FLOOD** packet **with a path** (≥1 hop) → record `path[len-1]` (the last forwarder = the + immediate RF transmitter). Confirmed against firmware `Mesh.cpp` (`routeRecvPacket` appends the + forwarder's hash to the END of the path) and CoreScope's `neighbor_builder.go:226-228`. +- **DIRECT** packet **with a path** → **NOT attributable, discarded.** Direct forwarders consume the + next hop from the FRONT (`Mesh.cpp removeSelfFromPath`), so `path[len-1]` is the route's + destination-side end, NOT the node we heard. Attributing it credits the SNR to the wrong (often + far-away) node. Only FLOOD routes (0,1) are recorded from a path. +- Packet **with no path** (0 hops) **and** an advert → record the advertiser's full pubkey. +- `direction` must be `rx`. 1-byte (2 hex char) prefixes are excluded (collision-prone, like Reach). +- The RSSI/SNR belong to the directly-received transmission, so they attach to the recorded node. +- The rest of the path is discarded for coverage. + +## Storage — `client_receptions` (ingestor-owned) + +A roaming companion is a mobile observer with a moving position, so it gets its own table (not +`observations`, which assumes a fixed observer location). Per the #1283 read/write invariant, the +table and all writes live in `cmd/ingestor/`. + +``` +client_receptions( + id, rx_pubkey, heard_key, heard_keylen, rssi, snr, + lat, lon, pos_acc_m, rx_at, ingested_at, src, + UNIQUE(rx_pubkey, heard_key, rx_at)) -- idempotent re-ingest +``` + +`heard_keylen` is 32 for a full pubkey (0-hop advert) or 2/3 for a multibyte prefix. `src` is +`advert` or `rxlog`. No hex cell is stored — binning is computed server-side from lat/lon. + +## Read API — coverage GeoJSON + +`GET /api/nodes/{pubkey}/rx-coverage?bbox={minLat,minLon,maxLat,maxLon}&z={zoom}` + +Returns a GeoJSON `FeatureCollection` of hexagons covering where clients heard the node, aggregated +server-side (read-only). Each feature: + +```json +{ "type": "Feature", + "geometry": { "type": "Polygon", "coordinates": [[[lon,lat], ...]] }, + "properties": { "cell": "9:123:-45", "count": 7, "best_snr": -6, "has_sig": true } } +``` + +- Hex binning is a pure-Go pointy-top grid over Web Mercator (`cmd/server/hexgrid.go`). We do **not** + use `uber/h3-go` because it is CGO and the project builds with `CGO_ENABLED=0`. +- `z` (Leaflet zoom) selects the hex resolution (zoom-adaptive). Raw points never leave the server + (privacy: contributors' tracks are not exposed). +- `best_snr` / `has_sig` drive the colour: green→orange by best SNR, grey when no signal metric. + +## Frontend + +Shown only in the Reach view (`#/nodes/{pubkey}/reach`), as a toggleable hex layer drawn on the +existing Leaflet map (`public/node-reach-coverage.js`), deep-linked via `?coverage=1`. No new +frontend dependencies. Colours come from CSS variables in `public/node-reach.css` +(`--nq-cov-strong|mid|weak|grey`). + +## Trust + +Identity = the companion pubkey (`rx_pubkey`). The broker ACL binds each client to its own +`{PUBLIC_KEY}` topic, so a client can only contribute under the key it physically holds. Optional +future hardening: have the companion sign a broker-issued token (the firmware exposes on-device +signing) — not required for the MVP. + +## Configurable values (future customizer) + +Hardcoded initially, tracked for the customizer per AGENTS.md rule 8: hex resolution per zoom +(`zoomToHexRes`), colour SNR thresholds (`coverageColorVar`), and any `rx_at` max-age validation. diff --git a/public/rx-coverage.js b/public/rx-coverage.js index de070890..923371e5 100644 --- a/public/rx-coverage.js +++ b/public/rx-coverage.js @@ -27,7 +27,7 @@ function pageHtml() { return '
' + '

🗺️ Mobile RX coverage

' + - '
Where roaming CoreScope-RX clients heard nodes. Colour = best signal per cell.
' + + '
Where roaming CoreScope-RX clients heard nodes. Colour = best signal per cell. Get the companion app →
' + '
' + dayBtn(1) + dayBtn(7) + dayBtn(14) + dayBtn(30) + '
' + '
strongmediumweakno signal
' + '
' + From 87bd7205d2b3e7324dca013688fedee6cbeb8b93 Mon Sep 17 00:00:00 2001 From: efiten Date: Mon, 15 Jun 2026 20:30:20 +0200 Subject: [PATCH 08/38] fix(coverage): inject Coverage nav link only when enabled (don't ship it in static nav) The static nav link broke upstream's nav-overflow e2e (test-nav-priority-1391): it counts all .nav-link elements regardless of display, so a hidden opt-in link still failed the expected-nav-set assertion. Remove the link from index.html and inject it from roles.js after Analytics only when clientRxCoverage is enabled, nudging applyNavPriority via a resize event. Default-off nav now exactly matches upstream (deterministic CI), and the link appears when the feature is on. Co-Authored-By: Claude Opus 4.8 --- public/index.html | 1 - public/roles.js | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index 70fddcf6..f272581c 100644 --- a/public/index.html +++ b/public/index.html @@ -127,7 +127,6 @@ Tools Observers Analytics - Coverage Perf Lab
diff --git a/public/roles.js b/public/roles.js index 6af73efc..90d63e2a 100644 --- a/public/roles.js +++ b/public/roles.js @@ -534,9 +534,20 @@ // ─── Fetch server overrides ─── window.MeshConfigReady = fetch('/api/config/client').then(function (r) { return r.json(); }).then(function (cfg) { window.MC_CLIENT_RX_COVERAGE = cfg.clientRxCoverage === true; - if (!window.MC_CLIENT_RX_COVERAGE) { - var covNav = document.querySelector('[data-route="rx-coverage"]'); - if (covNav) covNav.style.display = 'none'; + // Coverage is opt-in: the nav link is NOT in static HTML (so the default-off + // nav matches upstream and the nav-overflow tests). Inject it after Analytics + // only when enabled, then nudge applyNavPriority (it re-runs on 'resize'). + if (window.MC_CLIENT_RX_COVERAGE && !document.querySelector('.nav-links [data-route="rx-coverage"]')) { + var navAnchor = document.querySelector('.nav-links [data-route="analytics"]'); + if (navAnchor) { + var covLink = document.createElement('a'); + covLink.href = '#/rx-coverage'; + covLink.className = 'nav-link'; + covLink.setAttribute('data-route', 'rx-coverage'); + covLink.innerHTML = ' Coverage'; + navAnchor.insertAdjacentElement('afterend', covLink); + window.dispatchEvent(new Event('resize')); + } } if (cfg.roles) { if (cfg.roles.colors) { From 6643280c3eb9c28257bf9b801e1faeea0f81d7b8 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:24:26 +0200 Subject: [PATCH 09/38] fix(coverage): validate companion pubkey from client topic (#2, #10) The companion identity is the topic segment in meshcore/client//packets, which the broker is expected to ACL-bind to the publisher. On a broker without ACLs an attacker could publish under an arbitrary topic (e.g. !@#$) and pollute client_receptions / client_observers with junk pubkeys. Reject any topic pubkey that is not lowercase hex (^[0-9a-f]{2,64}$, mirroring the server-side hexPrefixRe) before any write. Because the topic value is now always validated, the firstNonEmpty(rxPubkey, origin_id) payload fallback is both unreachable (#10) and a trust hole, so it is removed; the companion identity comes only from the ACL-bound topic. Test: TestHandleClientPacketRejectsNonHexPubkey drives non-hex topic segments and asserts zero rows in both tables (fails without the guard). Existing fixtures updated to use a valid hex companion pubkey. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/client_reception.go | 19 ++++++++++++++- cmd/ingestor/client_reception_test.go | 34 +++++++++++++++++++++++---- cmd/ingestor/coverage_gate_test.go | 6 ++++- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index 2825c5be..ab670e65 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -2,18 +2,35 @@ package main import ( "log" + "regexp" "strings" "time" "github.com/meshcore-analyzer/packetpath" ) +// clientPubkeyRe validates the companion pubkey taken from the MQTT topic +// (meshcore/client//packets). A no-ACL broker would let a client +// publish under an arbitrary topic segment (e.g. "!@#$"), so we reject anything +// that is not lowercase hex before it reaches client_receptions/client_observers. +// Mirrors the server-side hexPrefixRe (cmd/server/node_resolve.go). +var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) + // handleClientPacket processes a packet from the mobile client RX topic // (meshcore/client/{PUBLIC_KEY}/packets). Unlike observer packets, a roaming // companion reports WHERE it directly heard a node, so we write a // client_receptions row and never touch the observers/observations tables. // rxPubkey is the companion pubkey from the topic (ACL-bound by the broker). func handleClientPacket(store *Store, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string) { + // The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex + // topic segments so a no-ACL broker can't pollute the coverage tables, and + // never fall back to a payload-supplied id (that would defeat the ACL trust + // model — see docs/client-rx-coverage.md). + rxPubkey = strings.ToLower(strings.TrimSpace(rxPubkey)) + if !clientPubkeyRe.MatchString(rxPubkey) { + log.Printf("MQTT [%s] client: invalid pubkey %.8q, dropping", tag, rxPubkey) + return + } rawHex, _ := msg["raw"].(string) if rawHex == "" { return @@ -59,7 +76,7 @@ func handleClientPacket(store *Store, tag, rxPubkey string, msg map[string]inter isAdvert := decoded.Header.PayloadTypeName == "ADVERT" rec, ok := buildClientReception( - firstNonEmpty(rxPubkey, stringField(msg, "origin_id")), + rxPubkey, direction, decoded.Header.RouteType, decoded.Path.Hops, decoded.Payload.PubKey, isAdvert, snrPtr, rssiPtr, lat, lon, accPtr, rxAt, time.Now().UTC().Format(time.RFC3339), ) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index abfe281a..2121811b 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -110,10 +110,10 @@ func TestHandleClientPacketAdvertWritesReception(t *testing.T) { "RSSI": -92.0, "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, } - handleClientPacket(s, "test", "companionpk", msg, nil) + handleClientPacket(s, "test", testCompanionPK, msg, nil) var obsName string - s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey='companionpk'`).Scan(&obsName) + s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=?`, testCompanionPK).Scan(&obsName) if obsName != "MyMob" { t.Fatalf("expected client_observers name 'MyMob', got %q", obsName) } @@ -123,7 +123,7 @@ func TestHandleClientPacketAdvertWritesReception(t *testing.T) { // The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey. var n, keylen int var src string - if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey='companionpk'`).Scan(&n, &keylen, &src); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { t.Fatal(err) } if n != 1 || keylen < 2 || src != "rxlog" { @@ -131,10 +131,34 @@ func TestHandleClientPacketAdvertWritesReception(t *testing.T) { } // No GPS → no row. - handleClientPacket(s, "test", "companion2", map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil) + const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" + handleClientPacket(s, "test", companion2, map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil) var n2 int - s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey='companion2'`).Scan(&n2) + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=?`, companion2).Scan(&n2) if n2 != 0 { t.Fatalf("packet without gps must be dropped, got %d rows", n2) } } + +// TestHandleClientPacketRejectsNonHexPubkey verifies the #2 fix: a companion +// pubkey from the topic that isn't lowercase hex (a no-ACL broker could publish +// meshcore/client/!@#$/packets) writes nothing to either coverage table. Without +// the clientPubkeyRe guard this fixture would insert a polluting row. +func TestHandleClientPacketRejectsNonHexPubkey(t *testing.T) { + s := newTestStore(t) + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + for _, bad := range []string{"!@#$", "companionpk", "", "g0g0", "xyz"} { + msg := map[string]interface{}{ + "raw": advertHex, "direction": "rx", "timestamp": "2026-06-09T12:00:00Z", + "origin": "Spoof", "SNR": -7.0, "RSSI": -92.0, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientPacket(s, "test", bad, msg, nil) + } + var nRecept, nObs int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&nRecept) + s.db.QueryRow(`SELECT COUNT(*) FROM client_observers`).Scan(&nObs) + if nRecept != 0 || nObs != 0 { + t.Fatalf("non-hex pubkey must write nothing, got %d receptions, %d observers", nRecept, nObs) + } +} diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go index de931d4e..d5761751 100644 --- a/cmd/ingestor/coverage_gate_test.go +++ b/cmd/ingestor/coverage_gate_test.go @@ -2,6 +2,10 @@ package main import "testing" +// testCompanionPK is a valid lowercase-hex companion pubkey for coverage tests. +// The topic segment must be hex (clientPubkeyRe) or handleClientPacket drops it. +const testCompanionPK = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2" + // clientCoverageMsg builds a valid mobile client-RX coverage message on the // dedicated topic meshcore/client//packets. The raw hex is a relayed // advert with GPS, so handleClientPacket would write exactly one @@ -10,7 +14,7 @@ import "testing" func clientCoverageMsg() *mockMessage { advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" payload := []byte(`{"raw":"` + advertHex + `","direction":"rx","timestamp":"2026-06-09T12:00:00Z","origin":"MyMob","SNR":-7.0,"RSSI":-92.0,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`) - return &mockMessage{topic: "meshcore/client/companionpk/packets", payload: payload} + return &mockMessage{topic: "meshcore/client/" + testCompanionPK + "/packets", payload: payload} } func clientReceptionCount(t *testing.T, s *Store) int { From 44fc6ac8ac4f76a8f4145d2159619d137c5f6693 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:25:39 +0200 Subject: [PATCH 10/38] fix(coverage): enforce observer blacklist on client topic (#1) handleMessage dispatched the meshcore/client//packets topic to handleClientPacket and returned before the IsObserverBlacklisted check that guards the observer path. A blacklisted operator could therefore keep feeding coverage data through the client topic. Check the blacklist for the companion pubkey at the top of the client dispatch branch and drop (with a log) before any write. Test: TestClientRxCoverageBlacklistedDropped drives handleMessage with the feature ON and the companion pubkey blacklisted, asserting zero rows; it fails without the gate because the old order inserted before the blacklist ran. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/coverage_gate_test.go | 19 +++++++++++++++++++ cmd/ingestor/main.go | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go index d5761751..a3f799df 100644 --- a/cmd/ingestor/coverage_gate_test.go +++ b/cmd/ingestor/coverage_gate_test.go @@ -67,3 +67,22 @@ func TestClientRxCoverageGateOn(t *testing.T) { t.Fatalf("feature ON: expected 1 client_receptions row, got %d", n) } } + +// TestClientRxCoverageBlacklistedDropped verifies the #1 fix: a blacklisted +// operator cannot skirt the observer blacklist via the client topic. With the +// feature ON but the companion pubkey blacklisted, no row is written. Without +// the gate the client dispatch runs before the blacklist check and inserts. +func TestClientRxCoverageBlacklistedDropped(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ + ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}, + ObserverBlacklist: []string{testCompanionPK}, + } + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 0 { + t.Fatalf("blacklisted companion: expected 0 client_receptions rows, got %d", n) + } +} diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 5fa3216b..896d80f4 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -539,6 +539,13 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, // A roaming companion reports where it directly heard a node; handled in isolation // from the observer/observations path. EMQX ACL binds parts[2] to the client's own key. if cfg.ClientRxCoverageEnabled() && len(parts) >= 4 && parts[1] == "client" && parts[3] == "packets" { + // The observer blacklist (checked below) only runs on the observer path, + // so a blacklisted operator could otherwise skirt it via the client topic + // (#1). Enforce it here before any coverage write. + if cfg.IsObserverBlacklisted(parts[2]) { + log.Printf("MQTT [%s] client %.8s blacklisted, dropping", tag, parts[2]) + return + } handleClientPacket(store, tag, parts[2], msg, channelKeys) return } From 197e5f2e7774e42969084a1b2536f8c4eda779e6 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:28:04 +0200 Subject: [PATCH 11/38] fix(coverage): nil-safe client-RX coverage gate (#4) requireClientRxCoverage dereferenced s.cfg directly while the coverage routes are registered unconditionally, so a nil server/cfg would panic instead of returning a clean 404. Guard s == nil / s.cfg == nil and make the (*Config).ClientRxCoverageEnabled() receiver nil-safe too. Test: TestRequireClientRxCoverageNilSafe drives handleRxCoverage with a nil cfg and asserts 404 (panics without the guard), plus the nil-receiver helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/config.go | 4 ++-- cmd/server/rx_dashboard.go | 5 ++++- cmd/server/rx_dashboard_test.go | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cmd/server/config.go b/cmd/server/config.go index 3a6b414f..da05686d 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -262,9 +262,9 @@ type ClientRxCoverageConfig struct { } // ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage -// feature is on. Absent/nil ⇒ off (the safe default). +// feature is on. Nil config or absent/nil section ⇒ off (the safe default). func (c *Config) ClientRxCoverageEnabled() bool { - return c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled + return c != nil && c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled } // WSCompressionEnabled returns true when WebSocket permessage-deflate is explicitly enabled. diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 58899c9f..4dc4c890 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -114,7 +114,10 @@ func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]cov // client-RX coverage feature is disabled, so the coverage endpoints read as // "not found" instead of serving data on deployments that haven't enabled it. func (s *Server) requireClientRxCoverage(w http.ResponseWriter, r *http.Request) bool { - if !s.cfg.ClientRxCoverageEnabled() { + // Routes are registered unconditionally, so guard against a nil server/cfg + // (e.g. handlers exercised in isolation) rather than panicking (#4). + // ClientRxCoverageEnabled is itself nil-receiver-safe. + if s == nil || s.cfg == nil || !s.cfg.ClientRxCoverageEnabled() { http.NotFound(w, r) return false } diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go index c2ce1d83..b6f38519 100644 --- a/cmd/server/rx_dashboard_test.go +++ b/cmd/server/rx_dashboard_test.go @@ -2,10 +2,33 @@ package main import ( "fmt" + "net/http" + "net/http/httptest" "testing" "time" ) +// TestRequireClientRxCoverageNilSafe verifies the #4 fix: coverage routes are +// registered unconditionally, so a nil server cfg (or nil *Config receiver) +// must 404 rather than panic. +func TestRequireClientRxCoverageNilSafe(t *testing.T) { + var nilCfg *Config + if nilCfg.ClientRxCoverageEnabled() { + t.Fatal("nil *Config must report disabled") + } + req := func(srv *Server) int { + rr := httptest.NewRecorder() + srv.handleRxCoverage(rr, httptest.NewRequest("GET", "/api/rx-coverage?bbox=50,3,52,4", nil)) + return rr.Code + } + if code := req(&Server{}); code != http.StatusNotFound { // cfg nil → would panic without the guard + t.Fatalf("nil cfg: want 404, got %d", code) + } + if code := req(&Server{cfg: &Config{}}); code != http.StatusNotFound { // feature disabled + t.Fatalf("disabled: want 404, got %d", code) + } +} + func insRx(t *testing.T, db *DB, rx, hk, at string, lat, lon float64) { mustExecDB(t, db, fmt.Sprintf( `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('%s','%s',3,-6,%f,%f,'%s','x','rxlog')`, From 4183b14a839e642ee65706e3cb4984a8fe55ae12 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:31:04 +0200 Subject: [PATCH 12/38] fix(coverage): harden /api/nodes/resolve enumeration + identity hiding (#15) The prefix resolver accepted 2-char hex prefixes, so the 256 one-byte prefixes enumerated every node name; and, unlike /api/nodes/search and /api/resolve-hops, it returned blacklisted / hidden-prefix (#1181) node identities the rest of the API hides. - Require >= 4 hex chars. 1-byte (2 hex) keys are never stored (the ingestor rejects heard keys shorter than 2 bytes), so the floor matches the data model while ruling out trivial full-table enumeration. - A unique match that is blacklisted or hidden now resolves as not-found. - Apply the same identity-hiding to resolveHeardKey so coverage tooltips don't leak hidden node names either. The endpoint is kept (single-prefix -> name lookup, distinct from the q= fuzzy search and the hop-context resolver) and stays in openapi_known_gaps.json. Per-IP rate limiting is left to follow-up #1. Tests: TestResolvePrefix gains <4-hex 400 cases and an aabb-collision ambiguous case; TestResolvePrefixHidesBlacklistedAndHidden asserts blacklisted/hidden matches resolve as not-found (both fail without the fix). Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/node_resolve.go | 21 ++++++++++++--- cmd/server/node_resolve_test.go | 45 +++++++++++++++++++++++++++++---- cmd/server/rx_dashboard.go | 5 ++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go index 655740ba..bf5bd5ff 100644 --- a/cmd/server/node_resolve.go +++ b/cmd/server/node_resolve.go @@ -19,10 +19,20 @@ type ResolvePrefixResp struct { var hexPrefixRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) +// minResolvePrefixHex is the shortest accepted prefix. 1-byte (2 hex) keys are +// never stored — the ingestor rejects heard keys shorter than 2 bytes — so the +// floor matches the data model and, by ruling out the 256 two-char prefixes, +// blunts trivial enumeration of every node name through this endpoint (#15). +const minResolvePrefixHex = 4 + func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { pfx := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("prefix"))) if !hexPrefixRe.MatchString(pfx) { - http.Error(w, "prefix must be 2-64 hex chars", http.StatusBadRequest) + http.Error(w, "prefix must be hex", http.StatusBadRequest) + return + } + if len(pfx) < minResolvePrefixHex { + http.Error(w, "prefix must be at least 4 hex chars", http.StatusBadRequest) return } if s.db == nil || s.db.conn == nil { @@ -51,8 +61,13 @@ func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { resp := ResolvePrefixResp{Prefix: pfx} switch len(pks) { case 1: - resp.Pubkey = pks[0] - resp.Name = names[0] + // Parity with /api/nodes/search and /api/resolve-hops: never reveal the + // identity of a blacklisted or hidden-prefix node (#1181). Report it as + // not-found rather than leaking the name the rest of the API hides. + if !s.cfg.IsBlacklisted(pks[0]) && !s.cfg.IsNameHidden(names[0]) { + resp.Pubkey = pks[0] + resp.Name = names[0] + } default: resp.Ambiguous = len(pks) > 1 // 0 → not found (name empty), >1 → ambiguous } diff --git a/cmd/server/node_resolve_test.go b/cmd/server/node_resolve_test.go index 80b6daaf..05f0bd3d 100644 --- a/cmd/server/node_resolve_test.go +++ b/cmd/server/node_resolve_test.go @@ -20,10 +20,11 @@ func TestResolvePrefix(t *testing.T) { db := setupTestDBv2(t) mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('efef7943505052b47f1809488ea4b4d3942d4ed72d2b1953b90a9f5e62a65fb5','NodeUnique','repeater','t','t',1)`) + // Two nodes sharing the 4-hex prefix aabb → ambiguous at the new minimum. mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES ('aa11000000000000000000000000000000000000000000000000000000000000','NodeA','repeater','t','t',1)`) + VALUES ('aabb110000000000000000000000000000000000000000000000000000000000','NodeA','repeater','t','t',1)`) mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES ('aa22000000000000000000000000000000000000000000000000000000000000','NodeB','repeater','t','t',1)`) + VALUES ('aabb220000000000000000000000000000000000000000000000000000000000','NodeB','repeater','t','t',1)`) srv := &Server{db: db} // unique 3-byte prefix → name @@ -32,9 +33,9 @@ func TestResolvePrefix(t *testing.T) { if r1.Name != "NodeUnique" || r1.Ambiguous { t.Fatalf("unique: %+v", r1) } - // colliding 1-byte prefix (aa…) → ambiguous, no name + // colliding 4-hex prefix (aabb…) → ambiguous, no name var r2 ResolvePrefixResp - json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=aa").Body.Bytes(), &r2) + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=aabb").Body.Bytes(), &r2) if !r2.Ambiguous || r2.Name != "" { t.Fatalf("ambiguous: %+v", r2) } @@ -44,8 +45,42 @@ func TestResolvePrefix(t *testing.T) { if r3.Name != "" || r3.Ambiguous { t.Fatalf("notfound: %+v", r3) } - // bad prefix → 400 + // non-hex prefix → 400 if serveResolve(srv, "/api/nodes/resolve?prefix=xyz").Code != 400 { t.Fatal("non-hex prefix should be 400") } + // #15: prefixes shorter than 4 hex are rejected (kills 256-prefix enumeration) + for _, short := range []string{"a", "aa", "abc"} { + if code := serveResolve(srv, "/api/nodes/resolve?prefix="+short).Code; code != 400 { + t.Fatalf("prefix %q (<4 hex) should be 400, got %d", short, code) + } + } +} + +// TestResolvePrefixHidesBlacklistedAndHidden verifies the #15 parity fix: a +// unique match that is blacklisted or whose name is hidden (#1181) resolves as +// not-found, never leaking an identity the rest of the API hides. +func TestResolvePrefixHidesBlacklistedAndHidden(t *testing.T) { + db := setupTestDBv2(t) + const blPK = "bbcc110000000000000000000000000000000000000000000000000000000000" + const hidPK = "ddee220000000000000000000000000000000000000000000000000000000000" + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('`+blPK+`','BlacklistedNode','repeater','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('`+hidPK+`','🚫HiddenNode','repeater','t','t',1)`) + srv := &Server{db: db, cfg: &Config{ + NodeBlacklist: []string{blPK}, + HiddenNamePrefixes: []string{"🚫"}, + }} + + var rb ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=bbcc11").Body.Bytes(), &rb) + if rb.Name != "" || rb.Pubkey != "" || rb.Ambiguous { + t.Fatalf("blacklisted node must resolve as not-found: %+v", rb) + } + var rh ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=ddee22").Body.Bytes(), &rh) + if rh.Name != "" || rh.Pubkey != "" || rh.Ambiguous { + t.Fatalf("hidden-prefix node must resolve as not-found: %+v", rh) + } } diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 4dc4c890..3eb4262c 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -75,6 +75,11 @@ func (s *Server) resolveHeardKey(heardKey string) (string, string) { names = append(names, n) } if len(pks) == 1 { + // Same identity-hiding parity as /api/nodes/resolve (#15, #1181): don't + // surface a blacklisted or hidden-prefix node's name in coverage tooltips. + if s.cfg.IsBlacklisted(pks[0]) || s.cfg.IsNameHidden(names[0]) { + return heardKey, "" + } return pks[0], names[0] } return heardKey, "" From 970b3b7050c9b36f64b1d8b68e8d42cc70a636a1 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:33:15 +0200 Subject: [PATCH 13/38] fix(coverage): escape companion pubkey in coverage leaderboard (#14) renderBoard built each leaderboard row by string concatenation and interpolated o.pubkey raw into data-rx="..." (and into the truncated fallback label when the observer has no name) while only o.name was escaped. A non-hex pubkey (possible on a no-ACL broker, or in rows ingested before the #2 validation) could break out of the attribute and inject markup. escapeHtml() both the data-rx value and the truncated-pubkey label. Test: test-rx-coverage-escape.js slices the real row-builder out of rx-coverage.js and renders it with a markup-bearing pubkey, asserting no raw tag survives (fails when the escaping is reverted). Co-Authored-By: Claude Opus 4.8 (1M context) --- public/rx-coverage.js | 4 +-- test-rx-coverage-escape.js | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 test-rx-coverage-escape.js diff --git a/public/rx-coverage.js b/public/rx-coverage.js index 923371e5..0a81234d 100644 --- a/public/rx-coverage.js +++ b/public/rx-coverage.js @@ -84,8 +84,8 @@ if (!el) return; if (!boardCache.length) { el.innerHTML = '
No mobile observers in this window yet.
'; return; } var rows = boardCache.map(function (o, i) { - var nm = o.name ? escapeHtml(o.name) : (o.pubkey.slice(0, 10) + '…'); - return '
' + + var nm = o.name ? escapeHtml(o.name) : (escapeHtml(o.pubkey.slice(0, 10)) + '…'); + return '
' + '' + (i + 1) + '' + nm + '' + '' + o.receptions + '' + o.nodes + '
'; }).join(''); diff --git a/test-rx-coverage-escape.js b/test-rx-coverage-escape.js new file mode 100644 index 00000000..16b1956b --- /dev/null +++ b/test-rx-coverage-escape.js @@ -0,0 +1,55 @@ +'use strict'; +// Unit test for #14: the mobile RX coverage leaderboard must HTML-escape the +// pubkey it interpolates into the row markup (data-rx="..." and the truncated +// fallback label), not only the name. A no-ACL broker / pre-validation rows +// could carry a non-hex pubkey, and the rest of the row is built by string +// concatenation, so an unescaped pubkey is an HTML-injection vector. +// +// Like test-coverage-gate.js we slice the real row-building expression out of +// public/rx-coverage.js and evaluate it in a vm sandbox — no hand-copied +// duplicate — so the test tracks the actual source. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const src = fs.readFileSync(path.join(__dirname, 'public', 'rx-coverage.js'), 'utf8'); + +// Slice from `var nm = o.name ...` through the end of the returned row string. +const startMarker = 'var nm = o.name ? escapeHtml(o.name)'; +const endMarker = "o.nodes + '
';"; +const startIdx = src.indexOf(startMarker); +assert.ok(startIdx >= 0, 'could not locate row-builder start in rx-coverage.js'); +const endIdx = src.indexOf(endMarker, startIdx); +assert.ok(endIdx >= 0, 'could not locate row-builder end in rx-coverage.js'); +const block = src.slice(startIdx, endIdx + endMarker.length); + +// Canonical escapeHtml (public/app.js). +function escapeHtml(s) { + if (s == null) return ''; + return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); +} + +function renderRow(o) { + const sandbox = { o: o, i: 0, selectedRx: '', escapeHtml: escapeHtml }; + vm.createContext(sandbox); + return vm.runInContext('(function () { ' + block + ' })()', sandbox); +} + +// Malicious pubkey that would break out of the data-rx attribute and inject a +// tag if interpolated raw. With escaping, no raw '<', '>' or attribute-closing +// '"' survives. +const evil = '">'; + +// Case 1: no name → pubkey used as the visible label fallback too. +const row1 = renderRow({ pubkey: evil, name: '', receptions: 1, nodes: 1 }); +assert.ok(row1.indexOf(' Date: Tue, 16 Jun 2026 09:36:35 +0200 Subject: [PATCH 14/38] fix(coverage): deterministic feature order + node name precedence (#8, #20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aggregateCoverage built its GeoJSON features by ranging a Go map, so feature order was randomized — defeating ETag/caching and making "first feature" e2e checks flaky (#8). And the per-node name was set with `if name != "" { name = ... }`, so when several heard_keys mapped to the same node the displayed name depended on row/map order (#20). - Sort fc.Features by cell before returning. - Lock the node identity (name and display-prefix fallback) to the most specific (longest) heard_key that resolved, tie-broken lexicographically, so a full-pubkey reception outranks a short prefix independent of order. Tests: TestAggregateCoverageDeterministicFeatureOrder asserts features come out sorted by cell; TestAggregateCoverageNamePrecedenceOrderIndependent feeds the same rows in both orders with a resolver that returns different names per heard_key and asserts the name is stable (fails under the old last-writer rule). Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/rx_coverage.go | 36 +++++++++++++++++++------ cmd/server/rx_coverage_test.go | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index f79945ff..5ff4f288 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -58,13 +58,16 @@ type covAgg struct { // covNodeAgg tracks, per directly-heard node within a cell, its reception count and // the SNR of its most recent reception (by rx_at). name/prefix are the resolved node -// name (when known) and a display prefix fallback. +// name (when known) and a display prefix fallback. nameKeyLen records the heard_key +// length that set the current name, so the chosen identity is the most specific one +// regardless of row order (#20). type covNodeAgg struct { - count int - latestAt string - latestSNR *float64 - name string - prefix string + count int + latestAt string + latestSNR *float64 + name string + nameKeyLen int + prefix string } // nodeResolver maps a heard_key (2-3 byte prefix or full pubkey) to a canonical @@ -105,11 +108,23 @@ func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) Covera } na := a.nodes[key] if na == nil { - na = &covNodeAgg{prefix: row.HeardKey, name: name} + na = &covNodeAgg{prefix: row.HeardKey} a.nodes[key] = na } - if name != "" { + // Lock the display identity to the MOST SPECIFIC (longest) heard_key + // that resolved to a non-empty name, tie-broken lexicographically, so + // the name no longer flaps with row/map order (#20). A full-pubkey + // reception thus outranks a short-prefix one for the same node. + if name != "" && (na.name == "" || len(row.HeardKey) > na.nameKeyLen || + (len(row.HeardKey) == na.nameKeyLen && name < na.name)) { na.name = name + na.nameKeyLen = len(row.HeardKey) + } + // Display-prefix fallback (shown when name is empty): same precedence so + // it is also order-independent. + if len(row.HeardKey) > len(na.prefix) || + (len(row.HeardKey) == len(na.prefix) && row.HeardKey < na.prefix) { + na.prefix = row.HeardKey } na.count++ // rx_at is RFC3339, so lexical >= is chronological; keep the latest SNR. @@ -134,6 +149,11 @@ func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) Covera }, }) } + // Map iteration is randomized, so sort features by cell for a deterministic + // payload — stable ETag/caching and a non-flaky "first feature" in e2e (#8). + sort.Slice(fc.Features, func(i, j int) bool { + return fc.Features[i].Properties.Cell < fc.Features[j].Properties.Cell + }) return fc } diff --git a/cmd/server/rx_coverage_test.go b/cmd/server/rx_coverage_test.go index 868753d5..e6ac92f9 100644 --- a/cmd/server/rx_coverage_test.go +++ b/cmd/server/rx_coverage_test.go @@ -113,6 +113,54 @@ func TestAggregateCoverageMergesResolvedNodes(t *testing.T) { } } +// TestAggregateCoverageDeterministicFeatureOrder verifies #8: features come out +// sorted by cell regardless of Go's randomized map iteration, so the GeoJSON is +// stable (cacheable / non-flaky e2e). +func TestAggregateCoverageDeterministicFeatureOrder(t *testing.T) { + rows := []coverageRow{ + {Lat: 51.0, Lon: 3.0, SNR: covF(-5)}, + {Lat: 48.0, Lon: 2.0, SNR: covF(-5)}, + {Lat: 52.0, Lon: 4.0, SNR: covF(-5)}, + {Lat: 40.0, Lon: -3.0, SNR: covF(-5)}, + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) < 2 { + t.Fatalf("expected multiple cells, got %d", len(fc.Features)) + } + for i := 1; i < len(fc.Features); i++ { + if fc.Features[i-1].Properties.Cell > fc.Features[i].Properties.Cell { + t.Fatalf("features not sorted by cell at %d: %q > %q", i, + fc.Features[i-1].Properties.Cell, fc.Features[i].Properties.Cell) + } + } +} + +// TestAggregateCoverageNamePrecedenceOrderIndependent verifies #20: when two +// heard_keys resolve to the same node but the resolver returns different display +// names, the most specific (longest) heard_key wins regardless of row order, so +// the name no longer depends on map/row iteration. +func TestAggregateCoverageNamePrecedenceOrderIndependent(t *testing.T) { + resolve := func(hk string) (string, string) { + if hk == "aabbccdd11223344" { + return "aabbccdd11223344", "Alice" + } + return "aabbccdd11223344", "AliceShortPrefix" + } + full := coverageRow{Lat: 51.05, Lon: 3.72, SNR: covF(-5), HeardKey: "aabbccdd11223344", RxAt: "2026-06-01T10:00:00Z"} + prefix := coverageRow{Lat: 51.05, Lon: 3.72, SNR: covF(-6), HeardKey: "aabbcc", RxAt: "2026-06-02T10:00:00Z"} + + for _, order := range [][]coverageRow{{full, prefix}, {prefix, full}} { + fc := aggregateCoverage(order, 9, resolve) + nodes := fc.Features[0].Properties.Nodes + if len(nodes) != 1 { + t.Fatalf("expected 1 merged node, got %d (%+v)", len(nodes), nodes) + } + if nodes[0].Name != "Alice" { + t.Fatalf("name precedence flapped with row order: got %q, want Alice", nodes[0].Name) + } + } +} + func TestZoomToHexRes(t *testing.T) { // Resolution tracks zoom 1:1 within [3,18], clamped at the edges (z=0 is the // missing-param case). From 22655fc15ec0cf6d46271476b65c1f7bab07cb88 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:39:15 +0200 Subject: [PATCH 15/38] fix(coverage): bound coverage response size (#11, #12) A wide bbox at high zoom over the 30-day window could return unbounded GeoJSON: every hex cell, each with every node heard there. - Cap the per-cell node breakdown at coverageCellNodeCap (25); set properties.nodes_truncated when more nodes were heard than returned (#11). The client only renders ~10, so this just bounds the wire payload. - Cap the feature collection at coverageFeatureCap (5000) cells, keeping the densest (count desc, cell asc tie-break for determinism) and setting the top-level truncated flag (#12). Both flags are omitempty so untruncated responses are unchanged. truncated/nodes_truncated are GeoJSON foreign members that Leaflet ignores. Tests: TestAggregateCoverageCapsNodesPerCell (30 nodes -> 25 + flag) and TestAggregateCoverageCapsFeatures (5625 cells -> 5000 + flag, still cell-sorted, small query untruncated). Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/rx_coverage.go | 55 +++++++++++++++++++++++++++------- cmd/server/rx_coverage_test.go | 50 +++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index 5ff4f288..69005e8b 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -19,10 +19,23 @@ type coverageRow struct { RxAt string // reception time (RFC3339); used to pick the latest SNR per node } +// coverageFeatureCap bounds the number of hex cells returned in one response. +// A wide bbox at high zoom over the 30-day window could otherwise emit multi-MB +// GeoJSON; when more cells exist the densest are kept and Truncated is set (#12). +const coverageFeatureCap = 5000 + +// coverageCellNodeCap bounds the per-cell node breakdown shipped on the wire +// (the client only renders the top ~10). NodesTruncated flags that more were +// heard than returned (#11). +const coverageCellNodeCap = 25 + // GeoJSON output (named structs, no map[string]interface{} — AGENTS.md). +// Truncated is a non-standard foreign member (ignored by GeoJSON consumers like +// Leaflet) that signals the cell list was capped at coverageFeatureCap. type CoverageFeatureCollection struct { - Type string `json:"type"` // "FeatureCollection" - Features []CoverageFeature `json:"features"` + Type string `json:"type"` // "FeatureCollection" + Features []CoverageFeature `json:"features"` + Truncated bool `json:"truncated,omitempty"` } type CoverageFeature struct { Type string `json:"type"` // "Feature" @@ -34,11 +47,12 @@ type CoveragePolygon struct { Coordinates [][][2]float64 `json:"coordinates"` // one ring: [ [ [lon,lat], ... ] ] } type CoverageProperties struct { - Cell string `json:"cell"` - Count int `json:"count"` - BestSNR *float64 `json:"best_snr"` - HasSig bool `json:"has_sig"` // false → render grey (no signal metric) - Nodes []CoverageNode `json:"nodes"` // per-node breakdown, strongest latest-SNR first + Cell string `json:"cell"` + Count int `json:"count"` + BestSNR *float64 `json:"best_snr"` + HasSig bool `json:"has_sig"` // false → render grey (no signal metric) + Nodes []CoverageNode `json:"nodes"` // per-node breakdown, strongest latest-SNR first + NodesTruncated bool `json:"nodes_truncated,omitempty"` // true → more nodes heard than returned (#11) } // CoverageNode is one directly-heard node within a cell, with its latest SNR. @@ -140,15 +154,30 @@ func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) Covera if ring == nil { continue } + nodes, nodesTrunc := sortedCoverageNodes(a.nodes) fc.Features = append(fc.Features, CoverageFeature{ Type: "Feature", Geometry: CoveragePolygon{Type: "Polygon", Coordinates: [][][2]float64{ring}}, Properties: CoverageProperties{ Cell: cell, Count: a.count, BestSNR: a.bestSNR, HasSig: a.hasSig, - Nodes: sortedCoverageNodes(a.nodes), + Nodes: nodes, NodesTruncated: nodesTrunc, }, }) } + // Bound the response: when more cells exist than coverageFeatureCap, keep the + // densest (highest count) and flag the truncation, so a wide/zoomed-out query + // can't emit unbounded multi-MB GeoJSON (#12). + if len(fc.Features) > coverageFeatureCap { + sort.Slice(fc.Features, func(i, j int) bool { + ci, cj := fc.Features[i].Properties.Count, fc.Features[j].Properties.Count + if ci != cj { + return ci > cj // densest first + } + return fc.Features[i].Properties.Cell < fc.Features[j].Properties.Cell // deterministic tie-break + }) + fc.Features = fc.Features[:coverageFeatureCap] + fc.Truncated = true + } // Map iteration is randomized, so sort features by cell for a deterministic // payload — stable ETag/caching and a non-flaky "first feature" in e2e (#8). sort.Slice(fc.Features, func(i, j int) bool { @@ -159,8 +188,9 @@ func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) Covera // sortedCoverageNodes flattens the per-node aggregates into a slice sorted by latest // SNR descending (nodes heard without a signal sort last), tie-broken by count then -// prefix for a stable order. -func sortedCoverageNodes(m map[string]*covNodeAgg) []CoverageNode { +// prefix for a stable order. The slice is capped at coverageCellNodeCap; truncated +// reports whether more nodes were heard in the cell than returned (#11). +func sortedCoverageNodes(m map[string]*covNodeAgg) (nodes []CoverageNode, truncated bool) { out := make([]CoverageNode, 0, len(m)) for _, na := range m { out = append(out, CoverageNode{Prefix: na.prefix, Name: na.name, SNR: na.latestSNR, Count: na.count}) @@ -178,7 +208,10 @@ func sortedCoverageNodes(m map[string]*covNodeAgg) []CoverageNode { } return out[i].Prefix < out[j].Prefix }) - return out + if len(out) > coverageCellNodeCap { + return out[:coverageCellNodeCap], true + } + return out, false } type bbox struct{ MinLat, MinLon, MaxLat, MaxLon float64 } diff --git a/cmd/server/rx_coverage_test.go b/cmd/server/rx_coverage_test.go index e6ac92f9..72c35ce1 100644 --- a/cmd/server/rx_coverage_test.go +++ b/cmd/server/rx_coverage_test.go @@ -2,10 +2,60 @@ package main import ( "encoding/json" + "fmt" "math" "testing" ) +// TestAggregateCoverageCapsNodesPerCell verifies #11: a cell that heard more than +// coverageCellNodeCap distinct nodes ships at most that many, with NodesTruncated set. +func TestAggregateCoverageCapsNodesPerCell(t *testing.T) { + rows := make([]coverageRow, 0, coverageCellNodeCap+5) + for i := 0; i < coverageCellNodeCap+5; i++ { + rows = append(rows, coverageRow{ + Lat: 51.05, Lon: 3.72, SNR: covF(float64(-i)), + HeardKey: fmt.Sprintf("aa%06x", i), RxAt: "2026-06-01T10:00:00Z", + }) + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + p := fc.Features[0].Properties + if len(p.Nodes) != coverageCellNodeCap || !p.NodesTruncated { + t.Fatalf("want %d nodes + truncated, got %d nodes truncated=%v", coverageCellNodeCap, len(p.Nodes), p.NodesTruncated) + } +} + +// TestAggregateCoverageCapsFeatures verifies #12: a query spanning more than +// coverageFeatureCap cells is bounded to that many features with Truncated set, +// and a smaller query is not truncated. +func TestAggregateCoverageCapsFeatures(t *testing.T) { + // 0.1° spacing >> a res-9 cell (~4 km), so each point lands in its own cell. + rows := make([]coverageRow, 0, coverageFeatureCap+200) + side := 75 // 75*75 = 5625 > 5000 + for i := 0; i < side*side; i++ { + lat := 10.0 + float64(i/side)*0.1 + lon := 10.0 + float64(i%side)*0.1 + rows = append(rows, coverageRow{Lat: lat, Lon: lon, SNR: covF(-5)}) + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != coverageFeatureCap || !fc.Truncated { + t.Fatalf("want %d features + truncated, got %d truncated=%v", coverageFeatureCap, len(fc.Features), fc.Truncated) + } + // Still sorted by cell after truncation. + for i := 1; i < len(fc.Features); i++ { + if fc.Features[i-1].Properties.Cell > fc.Features[i].Properties.Cell { + t.Fatalf("truncated features not sorted by cell at %d", i) + } + } + // A small query is not truncated. + small := aggregateCoverage(rows[:10], 9, nil) + if small.Truncated { + t.Fatalf("small query should not be truncated") + } +} + func covF(f float64) *float64 { return &f } func TestAggregateCoverageBucketsBestSNR(t *testing.T) { From 0b5bdfd8d91e364bf58528bc60206d6374631492 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:41:04 +0200 Subject: [PATCH 16/38] fix(coverage): wire mobileRxStats into per-node response (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mobileRxStats was defined and tested but never called by a handler — dead production code. Rather than drop it, surface the node-wide totals it computes: the per-node coverage endpoint now returns mobile_receptions and mobile_clients (distinct contributing companions) as foreign members on the FeatureCollection, so the UI can show "heard by N clients" independent of the current bbox/pan. Both are omitempty, so the global /api/rx-coverage payload is unchanged. Test: TestRxCoverageEndpointGeoJSON now asserts the wired-in totals (1/1 for the single seeded reception); fails if the stats aren't attached. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/rx_coverage.go | 9 +++++++++ cmd/server/rx_coverage_endpoint_test.go | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index 69005e8b..28689c26 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -36,6 +36,11 @@ type CoverageFeatureCollection struct { Type string `json:"type"` // "FeatureCollection" Features []CoverageFeature `json:"features"` Truncated bool `json:"truncated,omitempty"` + // Per-node summary (set only by the per-node endpoint): total mobile-client + // receptions of this node and how many distinct companions heard it. Foreign + // members, omitempty so the global endpoint's payload is unchanged (#3). + MobileReceptions int `json:"mobile_receptions,omitempty"` + MobileClients int `json:"mobile_clients,omitempty"` } type CoverageFeature struct { Type string `json:"type"` // "Feature" @@ -303,6 +308,10 @@ func (s *Server) handleNodeRxCoverage(w http.ResponseWriter, r *http.Request) { return } fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolver()) + // Attach the node-wide reception/contributor totals (#3): the bbox limits the + // hex features to the current view, but these summarise all of this node's + // mobile coverage so the UI can show "heard by N clients" regardless of pan. + fc.MobileReceptions, fc.MobileClients = s.mobileRxStats(pubkey) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(fc) } diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go index 1825d551..2d56ddaf 100644 --- a/cmd/server/rx_coverage_endpoint_test.go +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -74,6 +74,11 @@ func TestRxCoverageEndpointGeoJSON(t *testing.T) { if fc.Type != "FeatureCollection" || len(fc.Features) != 1 { t.Fatalf("unexpected fc: %+v", fc) } + // #3: the per-node response carries the node-wide mobile totals (wired in + // from mobileRxStats). One reception from one companion → 1/1. + if fc.MobileReceptions != 1 || fc.MobileClients != 1 { + t.Fatalf("want mobile_receptions=1 mobile_clients=1, got %d/%d", fc.MobileReceptions, fc.MobileClients) + } if serveRxCoverage(srv, "/api/nodes/aabbcc/rx-coverage").Code != 400 { t.Fatal("missing bbox should be 400") } From eac0e5114876e8f89441d551e8c24c923442ee02 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:42:04 +0200 Subject: [PATCH 17/38] fix(coverage): clamp latitude in hex grid to Mercator limit (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hexMercator diverges toward the poles (tan(π/4 + lat·π/360) → ∞), so a coverage submission past ~85.05° produced NaN hex rings via hexInvMercator. Clamp lat to ±hexMaxLat (85.05112878) in hexCellAt and document that coverage is defined only within that range. Test: TestHexCellAtClampsPolarLatitude drives ±89.9/±90° and asserts they bin to the clamped edge cell with a finite (non-NaN/Inf) boundary ring. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/hexgrid.go | 15 ++++++++++++++- cmd/server/hexgrid_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cmd/server/hexgrid.go b/cmd/server/hexgrid.go index 41323781..93c25f3b 100644 --- a/cmd/server/hexgrid.go +++ b/cmd/server/hexgrid.go @@ -44,8 +44,21 @@ func hexSizeForRes(res int) float64 { return (hexTargetPx / 2) * mercUPPZ0 / math.Pow(2, float64(res)) } -// hexCellAt returns a stable cell id ("res:q:r") for the lat/lon at res. +// hexMaxLat is the Web Mercator latitude limit. The projection (hexMercator) +// diverges toward ±90° — tan(π/4 + lat·π/360) → ∞ — so points beyond this would +// produce NaN cell rings via hexInvMercator. Coverage is therefore only defined +// within ±hexMaxLat; polar submissions are clamped to the edge (#17). +const hexMaxLat = 85.05112878 + +// hexCellAt returns a stable cell id ("res:q:r") for the lat/lon at res. Latitude +// is clamped to ±hexMaxLat so near-polar points bin to the edge instead of +// producing NaN geometry. func hexCellAt(lat, lon float64, res int) string { + if lat > hexMaxLat { + lat = hexMaxLat + } else if lat < -hexMaxLat { + lat = -hexMaxLat + } size := hexSizeForRes(res) x, y := hexMercator(lat, lon) q := (math.Sqrt(3)/3*x - 1.0/3*y) / size diff --git a/cmd/server/hexgrid_test.go b/cmd/server/hexgrid_test.go index ce98bfcc..f8938436 100644 --- a/cmd/server/hexgrid_test.go +++ b/cmd/server/hexgrid_test.go @@ -1,6 +1,31 @@ package main -import "testing" +import ( + "math" + "testing" +) + +// TestHexCellAtClampsPolarLatitude verifies #17: latitudes past the Web Mercator +// limit are clamped, so near-polar submissions bin to the edge cell and produce +// finite geometry instead of NaN rings. +func TestHexCellAtClampsPolarLatitude(t *testing.T) { + for _, lat := range []float64{89.9, 90.0, -89.9, -90.0} { + cell := hexCellAt(lat, 3.72, 9) + clamped := math.Copysign(hexMaxLat, lat) + if want := hexCellAt(clamped, 3.72, 9); cell != want { + t.Fatalf("lat %.1f should clamp to %q, got %q", lat, want, cell) + } + ring := hexBoundary(cell) + if ring == nil { + t.Fatalf("lat %.1f produced no ring", lat) + } + for _, pt := range ring { + if math.IsNaN(pt[0]) || math.IsNaN(pt[1]) || math.IsInf(pt[0], 0) || math.IsInf(pt[1], 0) { + t.Fatalf("lat %.1f produced non-finite ring point %v", lat, pt) + } + } + } +} func TestHexCellAtStableAndDistinct(t *testing.T) { a := hexCellAt(51.0500, 3.7200, 9) From 7cebe71a585cc4d8a61467fac680fc1b67020414 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 09:44:23 +0200 Subject: [PATCH 18/38] test(coverage): add true 0-hop advert reception test (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestHandleClientPacketAdvertWritesReception used a RELAYED advert (non-empty path), so it exercised the rxlog last-hop branch — not the 0-hop src='advert' path its name implied. Rename it to ...RelayedAdvert... and add TestHandleClientPacketZeroHopAdvertWritesReception, which rebuilds the same advert with zero hops (header + "00" + payload) and asserts the advertiser is stored by its full pubkey with src='advert', plus gps/snr capture. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/client_reception_test.go | 40 ++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 2121811b..2c1b0175 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -1,6 +1,7 @@ package main import ( + "database/sql" "strings" "testing" @@ -98,7 +99,7 @@ func TestInsertClientReceptionRoundTripAndIdempotent(t *testing.T) { } } -func TestHandleClientPacketAdvertWritesReception(t *testing.T) { +func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { s := newTestStore(t) advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" msg := map[string]interface{}{ @@ -140,6 +141,43 @@ func TestHandleClientPacketAdvertWritesReception(t *testing.T) { } } +// TestHandleClientPacketZeroHopAdvertWritesReception covers the #9 gap: the +// advert fixture used above is a RELAYED advert (non-empty path), so it exercises +// the rxlog last-hop branch, not the 0-hop src='advert' branch. Here we rebuild +// the same advert with zero hops — header (FLOOD ADVERT) + "00" (0 hops) + the +// same advert payload — so handleClientPacket stores the advertiser by its full +// pubkey with src='advert', and we assert gps/snr were captured too. +func TestHandleClientPacketZeroHopAdvertWritesReception(t *testing.T) { + s := newTestStore(t) + relayed := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + // relayed = header(2) + path-descriptor(2) + 5*2-byte hops(20) + payload. + payload := relayed[24:] + zeroHop := "1100" + payload + advertPubkey := strings.ToLower(payload[:64]) // advert payload starts with the 32-byte pubkey + + msg := map[string]interface{}{ + "raw": zeroHop, "direction": "rx", "timestamp": "2026-06-09T12:00:00Z", + "origin": "MyMob", "SNR": -7.0, "RSSI": -92.0, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientPacket(s, "test", testCompanionPK, msg, nil) + + var heardKey, src string + var keylen int + var snr sql.NullFloat64 + var lat, lon float64 + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK). + Scan(&heardKey, &keylen, &src, &snr, &lat, &lon); err != nil { + t.Fatalf("expected a 0-hop advert reception: %v", err) + } + if src != "advert" || keylen != 32 || heardKey != advertPubkey { + t.Fatalf("0-hop advert: want advert/32/%s, got %s/%d/%s", advertPubkey, src, keylen, heardKey) + } + if !snr.Valid || snr.Float64 != -7 || lat != 51.05 || lon != 3.72 { + t.Fatalf("gps/snr not captured: snr=%v lat=%f lon=%f", snr, lat, lon) + } +} + // TestHandleClientPacketRejectsNonHexPubkey verifies the #2 fix: a companion // pubkey from the topic that isn't lowercase hex (a no-ACL broker could publish // meshcore/client/!@#$/packets) writes nothing to either coverage table. Without From 15b20f8a05c720bc654d50ffda4fe881d4e55eb4 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:07:13 +0200 Subject: [PATCH 19/38] perf(coverage): index client_receptions for bbox+prefix query (#5, #18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-node coverage query filters by lat/lon bbox AND matches the heard node by full key or 2-3 byte prefix; the only indexes were single-column heard_key / rx_pubkey, so the bbox path (the sargable common filter) fell back to a full table scan. Add a composite (heard_key, heard_keylen, lat, lon) — which serves the heard_key-equality seek, carries lat/lon for the range, and supersedes the old single-column heard_key index — plus idx_client_recept_latlon so the planner can drive from a selective bbox (#18 covers heard_keylen). CREATE INDEX IF NOT EXISTS in the base schema covers fresh and existing DBs (the table is new in this PR). Test: TestClientReceptionsCoverageQueryUsesIndex asserts EXPLAIN QUERY PLAN uses a client_recept index and no longer SCANs the table. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/client_reception_test.go | 33 +++++++++++++++++++++++++++ cmd/ingestor/db.go | 9 +++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 2c1b0175..6be96720 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -36,6 +36,39 @@ func TestClientReceptionsTableExists(t *testing.T) { func crF(f float64) *float64 { return &f } func crI(i int) *int { return &i } +// TestClientReceptionsCoverageQueryUsesIndex verifies #5/#18: the dominant +// coverage query (bbox + full-key/prefix match) is served by an index rather +// than a full table scan. Without idx_client_recept_latlon / _heard_geo the plan +// is "SCAN client_receptions". +func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { + s := newTestStore(t) + q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? + AND ( (heard_keylen = 32 AND heard_key = ?) + OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` + rows, err := s.db.Query(q, 50.0, 52.0, 3.0, 4.0, "aabb", "aabb") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + if !strings.Contains(plan, "USING INDEX idx_client_recept") { + t.Fatalf("coverage query should use a client_recept index, plan was:\n%s", plan) + } + if strings.Contains(plan, "SCAN client_receptions") { + t.Fatalf("coverage query should not full-scan, plan was:\n%s", plan) + } +} + func TestDeriveHeardKey(t *testing.T) { full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, nil, strings.ToUpper(full), true) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 6fa03ae6..93f95670 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -290,7 +290,14 @@ func applySchema(db *sql.DB) error { src TEXT NOT NULL, UNIQUE(rx_pubkey, heard_key, rx_at) ); - CREATE INDEX IF NOT EXISTS idx_client_recept_heard ON client_receptions(heard_key); + -- Coverage queries filter by bbox AND match the heard node either by full + -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite + -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and + -- carries lat/lon so the bbox range is satisfied from the index; it also + -- supersedes the old single-column heard_key index. idx_client_recept_latlon + -- lets the planner instead drive from a selective bbox. (#5, #18) + CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); + CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); CREATE INDEX IF NOT EXISTS idx_client_recept_rxpk ON client_receptions(rx_pubkey); -- Self-reported name of each mobile client (companion), from the SELF_INFO From 102ec7e413229234429d09623e74115364248f49 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:10:03 +0200 Subject: [PATCH 20/38] fix(coverage): debounce redraws and surface fetch errors (#6, #7) Both coverage layers bound moveend/zoomend straight to an immediate fetch, so a single drag fired a storm of /api/rx-coverage requests (#6); and every coverage fetch swallowed errors with an empty .catch (#7), so failures were invisible. - Wrap the pan/zoom redraw in the shared 200ms debounce (keeping a stable reference so node-reach-coverage can still off() the handler). Direct redraws (day switch, fit-to-observer) stay immediate. - Replace the empty catches with console.warn; the leaderboard failure also shows a one-line in-page message. Test: test-node-reach-coverage-debounce.js loads addLayer with controllable timers + the real debounce, fires a 6-event burst and asserts exactly one coalesced fetch after the settle (7 fetches without the debounce). Co-Authored-By: Claude Opus 4.8 (1M context) --- public/node-reach-coverage.js | 12 +++-- public/rx-coverage.js | 15 ++++-- test-node-reach-coverage-debounce.js | 73 ++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 test-node-reach-coverage-debounce.js diff --git a/public/node-reach-coverage.js b/public/node-reach-coverage.js index b9b3913a..43e10dc2 100644 --- a/public/node-reach-coverage.js +++ b/public/node-reach-coverage.js @@ -40,13 +40,19 @@ .bindTooltip('n=' + f.properties.count + (f.properties.best_snr != null ? ' · SNR ' + f.properties.best_snr : ' · no signal')); }); - }).catch(function () { /* leave layer empty on error; never crash the reach page */ }); + }).catch(function (e) { + // Leave the layer empty on error; never crash the reach page. + console.warn('node-reach-coverage: coverage fetch failed', e); + }); } - map.on('moveend zoomend', refresh); + // Debounce pan/zoom redraws so dragging doesn't storm the coverage endpoint + // (#6). Keep the reference so off() can unbind the same handler. + var debouncedRefresh = (typeof debounce === 'function') ? debounce(refresh, 200) : refresh; + map.on('moveend zoomend', debouncedRefresh); refresh(); return { off: function () { - map.off('moveend zoomend', refresh); + map.off('moveend zoomend', debouncedRefresh); try { map.removeLayer(group); } catch (e) {} } }; diff --git a/public/rx-coverage.js b/public/rx-coverage.js index 0a81234d..65f97bc4 100644 --- a/public/rx-coverage.js +++ b/public/rx-coverage.js @@ -76,7 +76,7 @@ L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: 0.45 }).addTo(covLayer) .bindTooltip(coverageNodesHtml(f.properties)); }); - }).catch(function () {}); + }).catch(function (e) { console.warn('rx-coverage: coverage fetch failed', e); }); } function renderBoard() { @@ -120,12 +120,17 @@ if (!any) { drawCoverage(); return; } // observer has no data in window → keep view map.fitBounds([[minLat, minLon], [maxLat, maxLon]], { padding: [30, 30], maxZoom: 15 }); drawCoverage(); // fitBounds may not fire moveend if the view is unchanged - }).catch(function () { drawCoverage(); }); + }).catch(function (e) { console.warn('rx-coverage: observer extent fetch failed', e); drawCoverage(); }); } function loadBoard() { fetch('/api/rx-leaderboard?days=' + days + '&limit=25').then(function (r) { return r.json(); }) - .then(function (d) { if (destroyed) return; boardCache = d.observers || []; renderBoard(); }).catch(function () {}); + .then(function (d) { if (destroyed) return; boardCache = d.observers || []; renderBoard(); }) + .catch(function (e) { + console.warn('rx-coverage: leaderboard fetch failed', e); + var el = document.getElementById('rxBoard'); + if (el) el.innerHTML = '
Could not load mobile observers.
'; + }); } function setDays(d) { @@ -155,7 +160,9 @@ if (typeof window._applyTilesToNodeMap === 'function') window._applyTilesToNodeMap(map); else L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { maxZoom: 19 }).addTo(map); covLayer = L.layerGroup().addTo(map); - map.on('moveend zoomend', drawCoverage); + // Debounce pan/zoom redraws so dragging the map doesn't fire a storm of + // /api/rx-coverage requests (#6). Direct calls (setDays, fit) stay immediate. + map.on('moveend zoomend', debounce(drawCoverage, 200)); var bar = document.getElementById('rxDays'); if (bar) bar.addEventListener('click', function (e) { var b = e.target.closest('button[data-days]'); if (b) setDays(+b.dataset.days); }); setTimeout(function () { if (!destroyed && map) { map.invalidateSize(); if (selectedRx) fitToObserver(); else drawCoverage(); } }, 150); diff --git a/test-node-reach-coverage-debounce.js b/test-node-reach-coverage-debounce.js new file mode 100644 index 00000000..3299e904 --- /dev/null +++ b/test-node-reach-coverage-debounce.js @@ -0,0 +1,73 @@ +'use strict'; +// Unit test for #6: pan/zoom coverage redraws must be debounced so dragging the +// map fires at most one /api/...rx-coverage request per settle, not one per +// moveend. We load node-reach-coverage.js in a vm sandbox with controllable +// timers + the real debounce, bind the layer, fire a burst of map events, then +// advance the fake clock and assert exactly one extra fetch happened. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'node-reach-coverage.js'), 'utf8'); + +// Controllable timer queue. +let now = 0; +let nextId = 1; +let timers = []; +function setTimeoutStub(fn, ms) { const id = nextId++; timers.push({ id: id, fn: fn, at: now + (ms || 0) }); return id; } +function clearTimeoutStub(id) { timers = timers.filter(function (t) { return t.id !== id; }); } +function advance(ms) { + now += ms; + const due = timers.filter(function (t) { return t.at <= now; }); + timers = timers.filter(function (t) { return t.at > now; }); + due.forEach(function (t) { t.fn(); }); +} + +let fetchCount = 0; +function fakeFetch() { + fetchCount++; + // Chainable stub whose then/catch never invoke callbacks (we only count calls). + const chain = { then: function () { return chain; }, catch: function () { return chain; } }; + return chain; +} + +const fakeGroup = { addTo: function () { return fakeGroup; }, clearLayers: function () {}, }; +const map = { + handlers: {}, + on: function (ev, fn) { this.handlers[ev] = fn; }, + off: function () {}, + getBounds: function () { return { getSouth: function () { return 0; }, getWest: function () { return 0; }, getNorth: function () { return 1; }, getEast: function () { return 1; } }; }, + getZoom: function () { return 10; }, + removeLayer: function () {}, +}; + +const sandbox = { + window: {}, + console: { warn: function () {} }, + setTimeout: setTimeoutStub, + clearTimeout: clearTimeoutStub, + fetch: fakeFetch, + L: { layerGroup: function () { return fakeGroup; } }, + getComputedStyle: function () { return { getPropertyValue: function () { return ''; } }; }, + document: { documentElement: {} }, +}; +vm.createContext(sandbox); +// Define debounce IN the context so it uses the controllable timers above. +vm.runInContext('function debounce(fn, ms){var t; return function(){var a=arguments, c=this; clearTimeout(t); t=setTimeout(function(){fn.apply(c,a);}, ms);};}', sandbox); +vm.runInContext(code, sandbox); + +const handle = sandbox.window.NodeReachCoverage.addLayer(map, 'aabbccddeeff'); +assert.strictEqual(fetchCount, 1, 'addLayer should fetch once initially'); + +// Burst of pan/zoom events. +const fire = map.handlers['moveend zoomend']; +assert.strictEqual(typeof fire, 'function', 'moveend/zoomend handler must be bound'); +for (let i = 0; i < 6; i++) fire(); +assert.strictEqual(fetchCount, 1, 'burst must not fetch immediately (debounced)'); + +advance(200); +assert.strictEqual(fetchCount, 2, 'after settle, exactly one coalesced fetch (got ' + fetchCount + ')'); + +handle.off(); +console.log('node-reach-coverage debounce OK'); From dea1fc749d5b70d2fb07222a694e6dcabb6cc374 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:13:54 +0200 Subject: [PATCH 21/38] fix(coverage): toggle reach coverage legend via class not inline style (#19) nqCovLegend's visibility was driven by an inline style="display:flex|none" that applyCoverage rewrote, so CSS (print rules, themes) couldn't override it. Use an .is-hidden class instead (matching the .nav-more-wrap.is-hidden pattern) toggled via classList, and add .nq-cov-legend.is-hidden { display:none !important } to node-reach-coverage.css. Test: test-coverage-gate.js now asserts the rendered legend carries the is-hidden class and uses no inline display style (fails if the inline style returns). Co-Authored-By: Claude Opus 4.8 (1M context) --- public/node-reach-coverage.css | 3 +++ public/node-reach.js | 4 ++-- test-coverage-gate.js | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/public/node-reach-coverage.css b/public/node-reach-coverage.css index 4e4536a6..7070870d 100644 --- a/public/node-reach-coverage.css +++ b/public/node-reach-coverage.css @@ -12,6 +12,9 @@ --nq-cov-grey: #95a5a6; /* heard, no SNR metric */ } .nq-cov-legend { display:flex; gap:12px; align-items:center; font-size:11px; color:var(--text-muted, #57606a); margin:4px 0 10px; } +/* Toggled by node-reach.js applyCoverage via class, not inline style, so CSS + (print rules, themes) can still override visibility (#19). */ +.nq-cov-legend.is-hidden { display:none !important; } .nq-cov-legend i { width:12px; height:12px; border-radius:2px; display:inline-block; margin-right:4px; vertical-align:middle; } /* Mobile RX coverage hub — leaderboard */ diff --git a/public/node-reach.js b/public/node-reach.js index 9ba09148..f2e2408c 100644 --- a/public/node-reach.js +++ b/public/node-reach.js @@ -188,7 +188,7 @@ '' + '' + '' + - (window.MC_CLIENT_RX_COVERAGE ? '
' + + (window.MC_CLIENT_RX_COVERAGE ? '
' + 'strong' + 'medium' + 'weak' + @@ -237,7 +237,7 @@ // hides the link lines under it (declutter) — the table still lists every link. var covLegend = document.getElementById('nqCovLegend'); function applyCoverage() { - if (covLegend) covLegend.style.display = coverageOn ? 'flex' : 'none'; + if (covLegend) covLegend.classList.toggle('is-hidden', !coverageOn); if (coverageOn) { if (qmap && window.NodeReachCoverage && !covHandle) covHandle = window.NodeReachCoverage.addLayer(qmap.map, pubkey); } else if (covHandle) { diff --git a/test-coverage-gate.js b/test-coverage-gate.js index 7588d268..ec96315c 100644 --- a/test-coverage-gate.js +++ b/test-coverage-gate.js @@ -50,6 +50,14 @@ assert.ok(on.includes('id="nqCoverage"'), assert.ok(on.includes('id="nqCovLegend"'), 'flag true: actions HTML MUST contain id="nqCovLegend"'); +// #19: the legend visibility is class-driven (.is-hidden), not an inline +// style="display:..." that CSS can't override. coverageOn is false in this +// sandbox, so the legend must carry is-hidden and no inline display style. +assert.ok(/class="nq-cov-legend[^"]*\bis-hidden\b/.test(on), + 'flag true + coverage off: legend must use the is-hidden class'); +assert.ok(!on.includes('style="display:'), + 'legend must not use an inline display style (#19)'); + // Sanity: the non-gated controls render regardless of the flag. assert.ok(off.includes('id="nqIncoming"') && on.includes('id="nqIncoming"'), 'incoming filter must render irrespective of the coverage flag'); From f46f67df6b89f4d0ee4791acc45358df7effeaf3 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:16:42 +0200 Subject: [PATCH 22/38] test(coverage): benchmark coverage query at ~1M rows (#5/#18, carmack) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds BenchmarkCoverageQuery: seeds ~1M receptions across a metro-area bbox and times the per-node coverage query three ways. On a Ryzen 7 PRO 8845HS, modernc.org/sqlite, 1M rows (benchtime 5x): or_query_indexed 2.28 s/op (original OR/substr query, latlon index) or_query_table_scan 0.30 s/op (same query, indexes dropped) inlist_query_indexed 0.42 ms/op (sargable heard_key IN-list + composite) The OR/substr shape can't use the heard_key index, so the planner drives from the bbox and pays a random row fetch per candidate — slower than a plain scan. Rewriting the per-node match as a heard_key IN-list (next commit) lets the (heard_key, …) composite seek the few hundred matching rows: ~5400x faster than the indexed OR query and bounded by node, not table size. Benchmarks don't run in CI; this is on-demand evidence for the perf claim. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/coverage_query_bench_test.go | 104 ++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 cmd/ingestor/coverage_query_bench_test.go diff --git a/cmd/ingestor/coverage_query_bench_test.go b/cmd/ingestor/coverage_query_bench_test.go new file mode 100644 index 00000000..078ec864 --- /dev/null +++ b/cmd/ingestor/coverage_query_bench_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "fmt" + "math/rand" + "strings" + "testing" +) + +// coverageBenchSQL is the dominant per-node coverage query (mirrors +// cmd/server queryCoverageRows): a bbox range plus a full-key/2-3-byte-prefix +// match on the heard node. +const coverageBenchSQL = `SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? + AND ( (heard_keylen = 32 AND heard_key = ?) + OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` + +// BenchmarkCoverageQuery seeds ~1M receptions across a metro-area bbox and times +// the coverage query with the indexes (#5/#18) versus a forced full table scan. +// Run: go test -run x -bench BenchmarkCoverageQuery -benchtime 20x ./cmd/ingestor +func BenchmarkCoverageQuery(b *testing.B) { + const n = 1_000_000 + const prefixPool = 2000 // distinct 3-byte heard_key prefixes + + dir := b.TempDir() + s, err := OpenStore(dir + "/bench.db") + if err != nil { + b.Fatal(err) + } + defer s.Close() + + rng := rand.New(rand.NewSource(1)) + tx, err := s.db.Begin() + if err != nil { + b.Fatal(err) + } + stmt, err := tx.Prepare(`INSERT INTO client_receptions + (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES (?,?,?,?,?,?,?,?,?)`) + if err != nil { + b.Fatal(err) + } + for i := 0; i < n; i++ { + hk := fmt.Sprintf("%06x", rng.Intn(prefixPool)) + lat := 51.0 + rng.Float64()*0.4 // ~44 km metro span + lon := 3.5 + rng.Float64()*0.4 + rxpk := fmt.Sprintf("%064x", rng.Intn(500)) + // rx_at carries i so (rx_pubkey,heard_key,rx_at) stays unique. + if _, err := stmt.Exec(rxpk, hk, 3, -6.0, lat, lon, fmt.Sprintf("t%d", i), "x", "rxlog"); err != nil { + b.Fatal(err) + } + } + stmt.Close() + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + + // Target node whose 3-byte prefix (0003e8 = 1000) is in the pool, queried + // over a sub-bbox of the metro area. + target := "0003e8" + strings.Repeat("ab", 29) // 6 + 58 = 64 hex + + // OR/substr query (original shape): bbox range OR'd with a non-sargable + // substr prefix match. + runOR := func(b *testing.B) { + for i := 0; i < b.N; i++ { + rows, err := s.db.Query(coverageBenchSQL, 51.1, 51.3, 3.6, 3.8, target, target) + if err != nil { + b.Fatal(err) + } + for rows.Next() { + } + rows.Close() + } + } + + // IN-list query (sargable): the heard node's candidate keys are exactly the + // full pubkey and its 2/3-byte prefixes, so an IN-list seeks them via the + // heard_key-leading composite instead of scanning the bbox. + inListSQL := `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions + WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + runIN := func(b *testing.B) { + for i := 0; i < b.N; i++ { + rows, err := s.db.Query(inListSQL, target, target[:4], target[:6], 51.1, 51.3, 3.6, 3.8) + if err != nil { + b.Fatal(err) + } + for rows.Next() { + } + rows.Close() + } + } + + b.Run("or_query_indexed", runOR) + b.Run("inlist_query_indexed", runIN) + + // Drop the coverage indexes to measure the full-scan baseline. + for _, idx := range []string{"idx_client_recept_heard_geo", "idx_client_recept_latlon", "idx_client_recept_rxpk"} { + if _, err := s.db.Exec("DROP INDEX IF EXISTS " + idx); err != nil { + b.Fatal(err) + } + } + b.Run("or_query_table_scan", runOR) +} From 67358053a83eb0f8ea246f8c2664450beb978874 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:19:06 +0200 Subject: [PATCH 23/38] perf(coverage): make per-node coverage query sargable via heard_key IN-list (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark (previous commit) showed the OR/substr match — "heard_keylen=32 AND heard_key=? OR heard_keylen IN (2,3) AND substr(?,1,keylen*2)=heard_key" — can't use the heard_key index, so the planner drove from the bbox and paid a random row fetch per candidate: 2.28 s/op at 1M rows, slower even than a full scan (0.30 s). A node's heard_key is always exactly its full pubkey or its 2-/3-byte prefix, so replace the OR/substr with heard_key IN (pubkey, pubkey[:6], pubkey[:4]) — an equivalent but sargable predicate. The (heard_key, …) composite index then seeks the few hundred matching rows: 0.42 ms/op (~5400x faster), bounded by node not table size. Applied to queryCoverageRows, queryCoverageFiltered (node filter) and mobileRxStats via the shared coverageHeardKeyCandidates helper. Test: TestClientReceptionsCoverageQueryUsesIndex now EXPLAINs the IN-list shape and asserts an index seek, not a table scan. Existing coverage/endpoint tests (which assert the same result rows) still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/client_reception_test.go | 11 +++--- cmd/server/rx_coverage.go | 57 ++++++++++++++++++++++----- cmd/server/rx_dashboard.go | 10 +++-- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 6be96720..21762596 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -37,17 +37,16 @@ func crF(f float64) *float64 { return &f } func crI(i int) *int { return &i } // TestClientReceptionsCoverageQueryUsesIndex verifies #5/#18: the dominant -// coverage query (bbox + full-key/prefix match) is served by an index rather -// than a full table scan. Without idx_client_recept_latlon / _heard_geo the plan +// per-node coverage query (sargable heard_key IN-list + bbox, mirroring +// cmd/server coverageHeardKeyCandidates) seeks the heard_key composite index +// rather than scanning the table. Without idx_client_recept_heard_geo the plan // is "SCAN client_receptions". func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { s := newTestStore(t) q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? - AND ( (heard_keylen = 32 AND heard_key = ?) - OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` - rows, err := s.db.Query(q, 50.0, 52.0, 3.0, 4.0, "aabb", "aabb") + WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0) if err != nil { t.Fatal(err) } diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index 28689c26..0c96d6e8 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -221,18 +221,55 @@ func sortedCoverageNodes(m map[string]*covNodeAgg) (nodes []CoverageNode, trunca type bbox struct{ MinLat, MinLon, MaxLat, MaxLon float64 } +// coverageHeardKeyCandidates returns the exact heard_key values that identify a +// node: its full pubkey (stored with heard_keylen 32) and the 2-byte (4 hex) and +// 3-byte (6 hex) prefixes a relay logs. Matching heard_key IN (these) is +// equivalent to the old "heard_keylen=32 AND heard_key=? OR heard_keylen IN (2,3) +// AND substr(?,1,keylen*2)=heard_key", but sargable — so the (heard_key, …) +// composite index seeks the few matching rows instead of scanning the bbox (#5). +func coverageHeardKeyCandidates(pubkey string) []string { + pk := strings.ToLower(pubkey) + seen := map[string]bool{} + out := make([]string, 0, 3) + for _, c := range []string{pk, prefixOrEmpty(pk, 6), prefixOrEmpty(pk, 4)} { + if c != "" && !seen[c] { + seen[c] = true + out = append(out, c) + } + } + return out +} + +func prefixOrEmpty(s string, n int) string { + if len(s) >= n { + return s[:n] + } + return "" +} + +// sqlPlaceholders returns "?,?,…" with n placeholders (n >= 1). +func sqlPlaceholders(n int) string { + if n <= 1 { + return "?" + } + return strings.Repeat("?,", n-1) + "?" +} + // queryCoverageRows returns raw coverage rows where the directly-heard node // matches the target pubkey by its 2-3 byte prefix (or full pubkey), within the // bbox. Read-only (server RO connection). func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) { - pk := strings.ToLower(pubkey) + cands := coverageHeardKeyCandidates(pubkey) + args := make([]interface{}, 0, len(cands)+4) + for _, c := range cands { + args = append(args, c) + } + args = append(args, b.MinLat, b.MaxLat, b.MinLon, b.MaxLon) rows, err := s.db.conn.Query(` SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? - AND ( (heard_keylen = 32 AND heard_key = ?) - OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )`, - b.MinLat, b.MaxLat, b.MinLon, b.MaxLon, pk, pk) + WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`) + AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`, args...) if err != nil { return nil, err } @@ -246,12 +283,14 @@ func (s *Server) mobileRxStats(pubkey string) (count, clients int) { if s.db == nil || s.db.conn == nil { return 0, 0 } - pk := strings.ToLower(pubkey) + cands := coverageHeardKeyCandidates(pubkey) + args := make([]interface{}, len(cands)) + for i, c := range cands { + args[i] = c + } s.db.conn.QueryRow(` SELECT COUNT(*), COUNT(DISTINCT rx_pubkey) FROM client_receptions - WHERE (heard_keylen = 32 AND heard_key = ?) - OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key)`, - pk, pk).Scan(&count, &clients) + WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`)`, args...).Scan(&count, &clients) return count, clients } diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 3eb4262c..06dadf21 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -92,9 +92,13 @@ func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]cov where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?"} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if node != "" { - pk := strings.ToLower(node) - where = append(where, "((heard_keylen = 32 AND heard_key = ?) OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key))") - args = append(args, pk, pk) + // Sargable heard_key IN-list (see coverageHeardKeyCandidates) so the + // (heard_key, …) composite index is used instead of a substr() scan (#5). + cands := coverageHeardKeyCandidates(node) + where = append(where, "heard_key IN ("+sqlPlaceholders(len(cands))+")") + for _, c := range cands { + args = append(args, c) + } } if rx != "" { where = append(where, "rx_pubkey = ?") From 50da14166ccbc2d00f1540fb4aa4ee1cadd336ba Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:21:18 +0200 Subject: [PATCH 24/38] fix(coverage): await MeshConfigReady before reading coverage flag (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit window.MC_CLIENT_RX_COVERAGE is set inside the MeshConfigReady promise (roles.js). A direct land on #/rx-coverage or #/nodes//reach could run init/load before it resolved, reading the flag as false — so the dashboard showed "not enabled" and the Reach page omitted the coverage toggle, even with the feature on. - rx-coverage init() now defers to start() behind Promise.resolve(MeshConfigReady) (guarded by `destroyed` for navigation away mid-wait). - node-reach load() awaits MeshConfigReady before reading the flag / building the actions bar, with a loadGen staleness check so a superseding load wins. Test: test-rx-coverage-config-race.js drives init() with a pending MeshConfigReady and asserts it does not render synchronously, only deciding once config settles (fails against the old synchronous flag read). Co-Authored-By: Claude Opus 4.8 (1M context) --- public/node-reach.js | 5 ++++ public/rx-coverage.js | 12 ++++++++- test-rx-coverage-config-race.js | 43 +++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 test-rx-coverage-config-race.js diff --git a/public/node-reach.js b/public/node-reach.js index f2e2408c..2d4f2b1f 100644 --- a/public/node-reach.js +++ b/public/node-reach.js @@ -114,6 +114,11 @@ // pass false so the current report stays on screen until the swap (no flash). async function load(container, pubkey, days, isInitial) { var myGen = ++loadGen; + // Wait for server config so the coverage flag (read just below and in the + // actions bar markup) is populated even on a direct land on this route — a + // race that otherwise hides the coverage toggle (#13). + try { await window.MeshConfigReady; } catch (e) {} + if (myGen !== loadGen) return; // superseded by a newer load while waiting current = { pubkey: pubkey, days: days }; coverageOn = window.MC_CLIENT_RX_COVERAGE === true && (typeof getHashParams === 'function' && getHashParams().get('coverage') === '1'); if (covHandle) { try { covHandle.off(); } catch (e) {} covHandle = null; } diff --git a/public/rx-coverage.js b/public/rx-coverage.js index 65f97bc4..43008c06 100644 --- a/public/rx-coverage.js +++ b/public/rx-coverage.js @@ -146,11 +146,21 @@ } function init(container) { + destroyed = false; + // A direct land on #/rx-coverage can run before MeshConfigReady resolves, at + // which point MC_CLIENT_RX_COVERAGE is still undefined and the page would + // wrongly show "not enabled". Defer until server config is loaded (#13). + Promise.resolve(window.MeshConfigReady).then(function () { + if (!destroyed) start(container); + }); + } + + function start(container) { if (!window.MC_CLIENT_RX_COVERAGE) { container.innerHTML = '
Coverage is not enabled on this deployment.
'; return; } - destroyed = false; selectedRx = ''; selectedName = ''; days = 7; boardCache = []; + selectedRx = ''; selectedName = ''; days = 7; boardCache = []; try { var p = (typeof getHashParams === 'function') ? getHashParams() : null; if (p) { var dd = parseInt(p.get('days'), 10); if ([1, 7, 14, 30].indexOf(dd) >= 0) days = dd; selectedRx = (p.get('rx') || '').toLowerCase(); } diff --git a/test-rx-coverage-config-race.js b/test-rx-coverage-config-race.js new file mode 100644 index 00000000..02631088 --- /dev/null +++ b/test-rx-coverage-config-race.js @@ -0,0 +1,43 @@ +'use strict'; +// Unit test for #13: a direct land on #/rx-coverage must not read the coverage +// feature flag before MeshConfigReady resolves. init() should defer its +// enabled/disabled decision until the config promise settles, instead of +// synchronously rendering "not enabled" when the flag is still undefined. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'rx-coverage.js'), 'utf8'); + +let page = null; +let resolveConfig; +const sandbox = { + window: { MeshConfigReady: new Promise(function (r) { resolveConfig = r; }) }, // MC_CLIENT_RX_COVERAGE undefined for now + document: { getElementById: function () { return null; } }, + registerPage: function (name, obj) { page = obj; }, + console: { warn: function () {} }, + Promise: Promise, + setTimeout: function () {}, clearTimeout: function () {}, + fetch: function () { var c = { then: function () { return c; }, catch: function () { return c; } }; return c; }, + L: {}, getComputedStyle: function () { return { getPropertyValue: function () { return ''; } }; }, +}; +vm.createContext(sandbox); +vm.runInContext(code, sandbox); +assert.ok(page && typeof page.init === 'function', 'registerPage should expose init'); + +(async function () { + const container = { innerHTML: '' }; + page.init(container); + // Before MeshConfigReady resolves, init must NOT have decided yet. The old + // code read the (undefined) flag synchronously and rendered "not enabled". + assert.strictEqual(container.innerHTML, '', 'init must defer until MeshConfigReady resolves'); + + // Resolve config with the feature OFF → only now should it render not-enabled. + sandbox.window.MC_CLIENT_RX_COVERAGE = false; + resolveConfig(); + await new Promise(function (r) { setTimeout(r, 10); }); + assert.ok(/not enabled/i.test(container.innerHTML), 'after config (off) resolves, shows not-enabled'); + + console.log('rx-coverage config race OK'); +})().catch(function (e) { console.error(e); process.exit(1); }); From 6eba9371749af63a7a06f1857d9f7defb198bd6e Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:24:21 +0200 Subject: [PATCH 25/38] feat(coverage): bound client_receptions with a retention reaper (#1727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit client_receptions grows on every coverage submission and had no retention, so the opt-in feature's table grew unbounded — the maintainer's main scale worry. Add retention.clientRxDays (0 = disabled): PruneOldClientReceptions deletes client_receptions older than the window (by rx_at) and client_observers whose last_seen aged out, via the ingestor WriterTx (single-writer rule #1283). Wired into the maintenance loop at startup and on a daily ticker alongside packetDays, independent of the feature flag so data is reaped even after the feature is turned off. config.example.json documents the knob (default 30). Test: TestPruneOldClientReceptions seeds recent + 40-day-old rows and asserts the old ones (and stale companion names) are deleted, recent kept, and days=0 is a no-op. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/client_reception_test.go | 37 +++++++++++++++++++++++++++ cmd/ingestor/config.go | 13 ++++++++++ cmd/ingestor/main.go | 27 +++++++++++++++++++ cmd/ingestor/maintenance.go | 33 ++++++++++++++++++++++++ config.example.json | 3 ++- 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 21762596..cc8979a4 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -4,10 +4,47 @@ import ( "database/sql" "strings" "testing" + "time" "github.com/meshcore-analyzer/packetpath" ) +// TestPruneOldClientReceptions verifies the retention reaper bounds the coverage +// tables: rows older than the window (and stale companion names) are deleted, +// recent ones kept, and days=0 disables it. +func TestPruneOldClientReceptions(t *testing.T) { + s := newTestStore(t) + now := time.Now().UTC() + recent := now.AddDate(0, 0, -1).Format(time.RFC3339) + old := now.AddDate(0, 0, -40).Format(time.RFC3339) + const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" + + s.InsertClientReception(&ClientReception{RxPubkey: testCompanionPK, HeardKey: "aabbcc", HeardKeyLen: 3, Lat: 51, Lon: 3.7, RxAt: recent, IngestedAt: "x", Src: "rxlog"}) + s.InsertClientReception(&ClientReception{RxPubkey: testCompanionPK, HeardKey: "aabbcc", HeardKeyLen: 3, Lat: 51, Lon: 3.7, RxAt: old, IngestedAt: "x", Src: "rxlog"}) + s.UpsertClientObserver(testCompanionPK, "Fresh", recent) + s.UpsertClientObserver(companion2, "Stale", old) + + if n, _ := s.PruneOldClientReceptions(0); n != 0 { + t.Fatalf("days=0 must be a no-op, got %d", n) + } + n, err := s.PruneOldClientReceptions(7) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expected 1 old reception pruned, got %d", n) + } + var recN, obsN int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&recN) + s.db.QueryRow(`SELECT COUNT(*) FROM client_observers`).Scan(&obsN) + if recN != 1 { + t.Fatalf("expected 1 reception remaining (recent), got %d", recN) + } + if obsN != 1 { + t.Fatalf("expected 1 observer remaining (fresh), got %d", obsN) + } +} + func TestClientReceptionsTableExists(t *testing.T) { s := newTestStore(t) cols := map[string]bool{} diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index b370f1fa..be1d55f4 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -148,6 +148,10 @@ type RetentionConfig struct { // PacketDays is the retention window for transmissions (#1283). // Ownership moved from cmd/server to cmd/ingestor; 0 disables. PacketDays int `json:"packetDays"` + // ClientRxDays is the retention window (by rx_at) for mobile client-RX + // coverage rows in client_receptions / client_observers; 0 disables. Bounds + // the table the opt-in coverage feature would otherwise grow without limit. + ClientRxDays int `json:"clientRxDays"` } // PacketDaysOrZero returns the configured retention.packetDays or 0 @@ -159,6 +163,15 @@ func (c *Config) PacketDaysOrZero() int { return 0 } +// ClientRxDaysOrZero returns the configured retention.clientRxDays or 0 +// (disabled) if not set. +func (c *Config) ClientRxDaysOrZero() int { + if c.Retention != nil && c.Retention.ClientRxDays > 0 { + return c.Retention.ClientRxDays + } + return 0 +} + // MetricsConfig controls observer metrics collection. type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 896d80f4..6176e327 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -273,6 +273,18 @@ func main() { } } + // Client-RX coverage retention: bound the opt-in coverage tables (#1727). + // Independent of the feature flag, so data persists are reaped even after + // the feature is turned off. 0 = disabled. + clientRxDays := cfg.ClientRxDaysOrZero() + if clientRxDays > 0 { + if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil { + log.Printf("[prune] error: %v", err) + } else if n > 0 { + log.Printf("[prune] startup pruned %d client_receptions older than %d days", n, clientRxDays) + } + } + vacuumPages := cfg.IncrementalVacuumPages() store.RunIncrementalVacuum(vacuumPages) @@ -335,6 +347,21 @@ func main() { log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", packetDays) } + // Daily ticker for client-RX coverage retention (#1727). + if clientRxDays > 0 { + clientRxRetentionTicker := time.NewTicker(24 * time.Hour) + go func() { + for range clientRxRetentionTicker.C { + if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil { + log.Printf("[prune] error: %v", err) + } else if n > 0 { + store.RunIncrementalVacuum(vacuumPages) + } + } + }() + log.Printf("[prune] auto-prune enabled: client_receptions older than %d days will be removed daily", clientRxDays) + } + // Hourly WAL checkpoint to prevent unbounded WAL growth. // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; // if the server's read connection holds frames, remaining pages stay in the diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index 2b978bdc..fea189fe 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -48,6 +48,39 @@ func (s *Store) PruneOldPackets(days int) (int64, error) { return n, nil } +// PruneOldClientReceptions deletes mobile client-RX coverage rows older than +// `days` (by rx_at), and client_observers (companion names) whose last_seen has +// aged out. This bounds the otherwise-unbounded client_receptions table the +// opt-in coverage feature feeds. 0 disables. Owned by the ingestor writer +// (#1283). Returns the number of client_receptions rows deleted. +func (s *Store) PruneOldClientReceptions(days int) (int64, error) { + if days <= 0 { + return 0, nil + } + cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + + var n int64 + err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error { + res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < ?`, cutoff) + if err != nil { + return fmt.Errorf("prune client_receptions: %w", err) + } + n, _ = res.RowsAffected() + // Drop companion name rows not refreshed within the window. + if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < ?`, cutoff); err != nil { + return fmt.Errorf("prune client_observers: %w", err) + } + return nil + }) + if err != nil { + return 0, err + } + if n > 0 { + log.Printf("[prune] deleted %d client_receptions older than %d days", n, days) + } + return n, nil +} + // SoftDeleteBlacklistedObservers marks observers in the blacklist as // inactive=1 so they are hidden from API responses. Owned by ingestor // per #1287. Runs once at startup. diff --git a/config.example.json b/config.example.json index 58500a46..1770a5c9 100644 --- a/config.example.json +++ b/config.example.json @@ -11,7 +11,8 @@ "nodeDays": 7, "observerDays": 14, "packetDays": 30, - "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled). NOTE (#1283): all four retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." + "clientRxDays": 30, + "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled). clientRxDays: mobile client-RX coverage rows (client_receptions/client_observers) older than N days are deleted (0 = disabled) — bounds the opt-in coverage tables (#1727). NOTE (#1283): all retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." }, "db": { "vacuumOnStartup": false, From e1b7d089976c40ed22d44c9195fdd552e46ded0d Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 10:26:04 +0200 Subject: [PATCH 26/38] docs(coverage): document single flag, ACL trust requirement, retention (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - config.example.json: clarify clientRxCoverage.enabled is one flag read by BOTH the ingestor and the server (not per-process), and call out the ACL-broker trust requirement inline. - docs/client-rx-coverage.md: state plainly that the feature REQUIRES an ACL-capable broker — without it the topic (and thus GPS + attribution) is spoofable; enabling step made a hard requirement, not "recommended". List the server/ingestor defense-in-depth (#1/#2/#10/#14/#15) as blast-radius reduction, not a replacement for the ACL. Document retention.clientRxDays, the coverage indexes, the ±85.05° latitude clamp, and the response bounds / per-node totals so the read API matches the implementation. Co-Authored-By: Claude Opus 4.8 (1M context) --- config.example.json | 2 +- docs/client-rx-coverage.md | 60 +++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/config.example.json b/config.example.json index 1770a5c9..ec797097 100644 --- a/config.example.json +++ b/config.example.json @@ -359,7 +359,7 @@ }, "_comment_compression": "Opt-in HTTP gzip middleware + WebSocket permessage-deflate. Both default to false — enable ONLY when your upstream reverse proxy is NOT already compressing. gzip: enables the gzipMiddleware wrapper around the HTTP handler. websocket: sets gorilla websocket Upgrader.EnableCompression. level: gzip compression level 1..9 (1=BestSpeed, 9=BestCompression, default 6). minSizeBytes: advisory minimum response size below which compression would not pay off. contentTypes: MIME allow-list — only responses with these Content-Type values are compressed. Already-compressed types (image/*, video/*, audio/*, application/zip, application/x-gzip, application/pdf, application/octet-stream) are always skipped, as are responses whose handler already set Content-Encoding. Omit contentTypes to use the built-in default allow-list.", "clientRxCoverage": { "enabled": false }, - "_comment_clientRxCoverage": "Opt-in mobile client-RX coverage (corescope-rx companions publishing GPS-tagged receptions to meshcore/client//packets). Default OFF: when disabled the ingestor writes no client_receptions, the /api/rx-coverage|rx-leaderboard|nodes/{pubkey}/rx-coverage endpoints 404, and the UI hides the Coverage dashboard + Reach overlay. Set enabled=true to turn it on. Companion app (the mobile capture side users run) + setup: https://github.com/efiten/corescope-rx — see docs/client-rx-coverage.md.", + "_comment_clientRxCoverage": "Opt-in mobile client-RX coverage (corescope-rx companions publishing GPS-tagged receptions to meshcore/client//packets). Default OFF: when disabled the ingestor writes no client_receptions, the /api/rx-coverage|rx-leaderboard|nodes/{pubkey}/rx-coverage endpoints 404, and the UI hides the Coverage dashboard + Reach overlay. Set enabled=true to turn it on. SINGLE FLAG, BOTH PROCESSES: the ingestor and server each parse this same config.json, so this one clientRxCoverage.enabled entry gates both the ingest write path and the read endpoints — set it once, not per-process. TRUST: the feature requires an ACL-capable broker binding meshcore/client/{pubkey}/packets to that publisher; without ACLs the companion GPS is spoofable (see docs/client-rx-coverage.md). Retention: see retention.clientRxDays. Companion app + setup: https://github.com/efiten/corescope-rx.", "_comment_channelKeys": "Hex keys for decrypting channel messages. Key name = channel display name. public channel key is well-known.", "_comment_hashChannels": "Channel names whose keys are derived via SHA256. Key = SHA256(name)[:16]. Listed here so the ingestor can auto-derive keys.", "hashRegions": [ diff --git a/docs/client-rx-coverage.md b/docs/client-rx-coverage.md index 111dd37f..833958bb 100644 --- a/docs/client-rx-coverage.md +++ b/docs/client-rx-coverage.md @@ -18,11 +18,16 @@ own MQTT broker + CoreScope instance (see its README). Coverage is **off by default**. To turn it on: 1. In CoreScope's `config.json`, set `"clientRxCoverage": { "enabled": true }` and restart the server - and ingestor. -2. Make sure your broker lets a client publish to `meshcore/client/{PUBLIC_KEY}/packets` (the ingestor - already subscribes under `meshcore/#`). An EMQX ACL binding each client to its own `{PUBLIC_KEY}` - topic is recommended. -3. Point your users at [corescope-rx](https://github.com/efiten/corescope-rx) and they start + and ingestor. This is a **single flag read by both processes** — the ingestor and server each parse + the same `config.json`, so you set `clientRxCoverage.enabled` once and it gates both the ingest write + path and the read endpoints. There is no separate per-process flag. +2. **Required: an ACL-capable broker.** Bind `meshcore/client/{PUBLIC_KEY}/packets` so each client may + publish **only** under its own pubkey (e.g. an EMQX ACL keyed on the connected client's identity). + This is the trust boundary, not an optimization — see [Trust](#trust). The ingestor already + subscribes under `meshcore/#`. +3. Optionally set `retention.clientRxDays` to bound the coverage tables (see + [Storage](#storage--client_receptions-ingestor-owned)). +4. Point your users at [corescope-rx](https://github.com/efiten/corescope-rx) and they start contributing. Results show on each node's Reach page (coverage toggle) and the `#/rx-coverage` dashboard. @@ -109,6 +114,14 @@ client_receptions( `heard_keylen` is 32 for a full pubkey (0-hop advert) or 2/3 for a multibyte prefix. `src` is `advert` or `rxlog`. No hex cell is stored — binning is computed server-side from lat/lon. +Indexes: a composite `(heard_key, heard_keylen, lat, lon)` and a `(lat, lon)` index back the coverage +queries; the per-node query matches a sargable `heard_key IN (pubkey, prefix6, prefix4)` list so the +composite is used instead of a table scan (see the benchmark in `cmd/ingestor`). + +Retention: the table grows on every submission, so set `retention.clientRxDays` (ingestor) to delete +rows older than N days (and stale `client_observers`); `0` disables it. Without it the table is +unbounded. + ## Read API — coverage GeoJSON `GET /api/nodes/{pubkey}/rx-coverage?bbox={minLat,minLon,maxLat,maxLon}&z={zoom}` @@ -119,14 +132,22 @@ server-side (read-only). Each feature: ```json { "type": "Feature", "geometry": { "type": "Polygon", "coordinates": [[[lon,lat], ...]] }, - "properties": { "cell": "9:123:-45", "count": 7, "best_snr": -6, "has_sig": true } } + "properties": { "cell": "9:123:-45", "count": 7, "best_snr": -6, "has_sig": true, + "nodes": [{ "prefix": "aabbcc", "name": "Alice", "snr": -6, "count": 3 }], + "nodes_truncated": false } } ``` - Hex binning is a pure-Go pointy-top grid over Web Mercator (`cmd/server/hexgrid.go`). We do **not** - use `uber/h3-go` because it is CGO and the project builds with `CGO_ENABLED=0`. + use `uber/h3-go` because it is CGO and the project builds with `CGO_ENABLED=0`. Latitude is only + defined within ±85.05° (Web Mercator limit) and is clamped to that range. - `z` (Leaflet zoom) selects the hex resolution (zoom-adaptive). Raw points never leave the server (privacy: contributors' tracks are not exposed). - `best_snr` / `has_sig` drive the colour: green→orange by best SNR, grey when no signal metric. +- Features are sorted by `cell` for a deterministic (cacheable) payload. +- **Bounds:** the per-cell `nodes` list is capped (with `nodes_truncated`), and the collection is + capped at a fixed feature count — when exceeded, the densest cells are kept and the top-level + `truncated` flag is set. The per-node endpoint also returns `mobile_receptions` and `mobile_clients` + totals (node-wide, independent of the bbox). ## Frontend @@ -137,10 +158,27 @@ frontend dependencies. Colours come from CSS variables in `public/node-reach.css ## Trust -Identity = the companion pubkey (`rx_pubkey`). The broker ACL binds each client to its own -`{PUBLIC_KEY}` topic, so a client can only contribute under the key it physically holds. Optional -future hardening: have the companion sign a broker-issued token (the firmware exposes on-device -signing) — not required for the MVP. +Identity = the companion pubkey (`rx_pubkey`), taken from the `{PUBLIC_KEY}` topic segment. + +**The feature requires an ACL-capable broker.** The reported GPS position is the contributor's own +claim, so the only thing anchoring a reception to a real identity is the broker ACL binding +`meshcore/client/{PUBLIC_KEY}/packets` to the client that holds that key. **Without such an ACL, the +topic — and therefore the GPS and the heard-node attribution — is spoofable:** anyone who can publish +to the broker could inject coverage under any pubkey. Do not enable this feature on an open/no-ACL +broker if you trust the resulting map. + +Server/ingestor-side defense-in-depth (these reduce blast radius but do **not** replace the ACL): + +- The ingestor rejects any topic pubkey that is not lowercase hex before writing, and never falls back + to a payload-supplied id (`cmd/ingestor/client_reception.go`, #2/#10). +- A blacklisted operator cannot contribute via the client topic (the blacklist is enforced before the + coverage write, #1). +- The frontend HTML-escapes the pubkey it renders, so a junk pubkey can't inject markup (#14). +- `/api/nodes/resolve` and coverage tooltips never reveal blacklisted or hidden-prefix node identities + (#15). + +Optional future hardening: have the companion sign a broker-issued token (the firmware exposes +on-device signing) — not required for the MVP, tracked as a follow-up. ## Configurable values (future customizer) From 588b4afdf52341761b60061c3be32b1828a369a0 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 13:54:35 +0200 Subject: [PATCH 27/38] fix(coverage): hide blacklisted/hidden nodes on coverage + leaderboard (review r2) Round-1 hid the node *name* via the resolver, but two read endpoints still leaked the identities the rest of the API suppresses: - handleNodeRxCoverage returned GPS hex bins + mobile_receptions/mobile_clients for any {pubkey}, so a blacklisted or hidden-prefix node's coverage was mappable by anyone who knew the key. Now mirrors handleNodeReach: reject non-hex pubkeys (400) and 404 blacklisted / isPubkeyHidden nodes before any query. - rxLeaderboard had no blacklist/hidden filter, so a pre-PR or post-blacklist client_receptions row kept a banned operator on the board (with name). Now drops IsObserverBlacklisted contributors and blanks the name when IsBlacklisted(pubkey) || IsNameHidden(name). Result set is <=100, filtered in Go with the same helpers the resolver uses. Tests: TestNodeRxCoverageHidesBlacklistedAndHidden (404 for both) and TestRxLeaderboardHidesBlacklistedAndHidden (drop + blank); both fail without the gates. setupTestDBv2's nodes table gains foreign_advert so GetNodeByPubkey (used by isPubkeyHidden) works against the test schema. The per-node endpoint now requires a full 64-hex pubkey, matching handleNodeReach. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/coverage_test.go | 2 +- cmd/server/rx_coverage.go | 12 ++++++++ cmd/server/rx_coverage_endpoint_test.go | 35 +++++++++++++++++++-- cmd/server/rx_dashboard.go | 19 +++++++++++- cmd/server/rx_dashboard_test.go | 41 +++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 4 deletions(-) diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 1369553c..8dc83153 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -30,7 +30,7 @@ func setupTestDBv2(t *testing.T) *DB { CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL + battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0 ); CREATE TABLE observers ( id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index 0c96d6e8..cbc446c3 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -331,6 +331,18 @@ func (s *Server) handleNodeRxCoverage(w http.ResponseWriter, r *http.Request) { return } pubkey := strings.ToLower(mux.Vars(r)["pubkey"]) + // Mirror handleNodeReach's gate at this same {pubkey}: reject malformed keys, + // and 404 blacklisted / hidden-prefix nodes. Hiding only the node *name* (via + // heardKeyResolver) still leaked the GPS hex bins and mobile_receptions / + // mobile_clients counts for a node the rest of the API hides (#1727 r2). + if !isHexPubkey(pubkey) { + http.Error(w, "invalid pubkey: expected 64 hex chars", http.StatusBadRequest) + return + } + if (s.cfg != nil && s.cfg.IsBlacklisted(pubkey)) || s.isPubkeyHidden(pubkey) { + http.NotFound(w, r) + return + } b, ok := parseBBox(r.URL.Query().Get("bbox")) if !ok { http.Error(w, "bbox required as minLat,minLon,maxLat,maxLon", http.StatusBadRequest) diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go index 2d56ddaf..f5fa92a8 100644 --- a/cmd/server/rx_coverage_endpoint_test.go +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -57,13 +57,16 @@ func serveRxCoverage(srv *Server, path string) *httptest.ResponseRecorder { return rr } +// nodePK is a full 64-hex pubkey whose 3-byte prefix is the seeded heard_key. +const nodePK = "aabbcc0000000000000000000000000000000000000000000000000000000000" + func TestRxCoverageEndpointGeoJSON(t *testing.T) { db := seedCoverageDB(t) mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} - rr := serveRxCoverage(srv, "/api/nodes/aabbccddeeff00112233/rx-coverage?bbox=50,3,52,4&z=12") + rr := serveRxCoverage(srv, "/api/nodes/"+nodePK+"/rx-coverage?bbox=50,3,52,4&z=12") if rr.Code != 200 { t.Fatalf("status %d body %s", rr.Code, rr.Body.String()) } @@ -79,7 +82,35 @@ func TestRxCoverageEndpointGeoJSON(t *testing.T) { if fc.MobileReceptions != 1 || fc.MobileClients != 1 { t.Fatalf("want mobile_receptions=1 mobile_clients=1, got %d/%d", fc.MobileReceptions, fc.MobileClients) } - if serveRxCoverage(srv, "/api/nodes/aabbcc/rx-coverage").Code != 400 { + if serveRxCoverage(srv, "/api/nodes/"+nodePK+"/rx-coverage").Code != 400 { t.Fatal("missing bbox should be 400") } + // Non-hex pubkey is rejected up front (parity with handleNodeReach). + if serveRxCoverage(srv, "/api/nodes/nothex/rx-coverage?bbox=50,3,52,4").Code != 400 { + t.Fatal("non-hex pubkey should be 400") + } +} + +// TestNodeRxCoverageHidesBlacklistedAndHidden verifies #1727 r2 must-fix #1: the +// per-node coverage endpoint must 404 for blacklisted or hidden-prefix nodes, so +// their GPS hex bins / counts aren't retrievable at a pubkey the rest of the API +// hides — not just the node name. +func TestNodeRxCoverageHidesBlacklistedAndHidden(t *testing.T) { + const hidPK = "ddee110000000000000000000000000000000000000000000000000000000000" + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role,last_seen,first_seen,advert_count) VALUES ('`+hidPK+`','🚫Secret','repeater','t','t',1)`) + srv := &Server{db: db, cfg: &Config{ + ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}, + NodeBlacklist: []string{nodePK}, + HiddenNamePrefixes: []string{"🚫"}, + }} + + if code := serveRxCoverage(srv, "/api/nodes/"+nodePK+"/rx-coverage?bbox=50,3,52,4").Code; code != 404 { + t.Fatalf("blacklisted node coverage should be 404, got %d", code) + } + if code := serveRxCoverage(srv, "/api/nodes/"+hidPK+"/rx-coverage?bbox=50,3,52,4").Code; code != 404 { + t.Fatalf("hidden-prefix node coverage should be 404, got %d", code) + } } diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 06dadf21..6cc9692b 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -196,7 +196,24 @@ func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { } out = append(out, o) } - return out, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + // Identity hiding parity (#1727 r2): pre-PR rows / blacklist-after-ingest / + // config-reload lag could otherwise surface a hidden operator here. Drop + // observer-blacklisted contributors entirely, and blank the name of a + // node-blacklisted or hidden-prefix identity. nil cfg ⇒ all no-ops. + filtered := out[:0] + for _, o := range out { + if s.cfg.IsObserverBlacklisted(o.Pubkey) { + continue + } + if s.cfg.IsBlacklisted(o.Pubkey) || s.cfg.IsNameHidden(o.Name) { + o.Name = "" + } + filtered = append(filtered, o) + } + return filtered, nil } func (s *Server) handleRxLeaderboard(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go index b6f38519..aae1eeee 100644 --- a/cmd/server/rx_dashboard_test.go +++ b/cmd/server/rx_dashboard_test.go @@ -89,3 +89,44 @@ func TestRxLeaderboard(t *testing.T) { t.Fatalf("compb should have no name: %+v", byPk["compb"]) } } + +// TestRxLeaderboardHidesBlacklistedAndHidden verifies #1727 r2 must-fix #2: the +// leaderboard must drop observer-blacklisted contributors and blank the name of +// node-blacklisted or hidden-prefix identities (pre-PR / post-blacklist rows). +func TestRxLeaderboardHidesBlacklistedAndHidden(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aa01','GoodGuy','companion','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('cc03','BadNode','companion','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('dd04','🚫Hidden','companion','t','t',1)`) + insRx(t, db, "aa01", "aabb01", recent, 51.05, 3.72) // normal → kept with name + insRx(t, db, "bb02", "aabb02", recent, 51.05, 3.72) // observer-blacklisted → dropped + insRx(t, db, "cc03", "aabb03", recent, 51.05, 3.72) // node-blacklisted → name blanked + insRx(t, db, "dd04", "aabb04", recent, 51.05, 3.72) // hidden prefix → name blanked + srv := &Server{db: db, cfg: &Config{ + ObserverBlacklist: []string{"bb02"}, + NodeBlacklist: []string{"cc03"}, + HiddenNamePrefixes: []string{"🚫"}, + }} + + obs, err := srv.rxLeaderboard(7, 100) + if err != nil { + t.Fatal(err) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + if _, ok := byPk["bb02"]; ok { + t.Fatalf("observer-blacklisted contributor must be dropped, got %+v", byPk["bb02"]) + } + if byPk["aa01"].Name != "GoodGuy" { + t.Fatalf("normal contributor name should be kept: %+v", byPk["aa01"]) + } + if _, ok := byPk["cc03"]; !ok || byPk["cc03"].Name != "" { + t.Fatalf("node-blacklisted contributor should remain with a blanked name: %+v", byPk["cc03"]) + } + if _, ok := byPk["dd04"]; !ok || byPk["dd04"].Name != "" { + t.Fatalf("hidden-prefix contributor should remain with a blanked name: %+v", byPk["dd04"]) + } +} From 2b41dd25e0287bf334e4e5f21a3e68ee8c8f6869 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 15:50:17 +0200 Subject: [PATCH 28/38] perf(coverage): index rx_at for retention/leaderboard; drop redundant index (polish review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retention reaper's DELETE … WHERE rx_at < ? and the leaderboard's WHERE rx_at >= ? both filtered on an unindexed column → full scan under the writer lock (the reaper from 6eba9371 introduced the DELETE). Add idx_client_recept_rxat. Drop idx_client_recept_rxpk: it duplicates the rx_pubkey-leading index the UNIQUE(rx_pubkey, heard_key, rx_at) constraint already creates, which serves the ?rx= filter and leaderboard GROUP BY. Also drop the dead `na.count == 1` guard in aggregateCoverage (latestAt starts "", so the first row always satisfies rx_at >= latestAt) — no behavior change. Test: TestClientReceptionsRetentionUsesRxAtIndex asserts the DELETE plan seeks idx_client_recept_rxat instead of scanning. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/ingestor/client_reception_test.go | 24 ++++++++++++++++++++++++ cmd/ingestor/db.go | 8 +++++++- cmd/server/rx_coverage.go | 6 ++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index cc8979a4..cf3a0d25 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -105,6 +105,30 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { } } +// TestClientReceptionsRetentionUsesRxAtIndex verifies the retention reaper's +// DELETE ... WHERE rx_at < ? (and the leaderboard's rx_at window) seek the rx_at +// index rather than full-scanning under the writer lock (polish review). +func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { + s := newTestStore(t) + rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + if !strings.Contains(plan, "idx_client_recept_rxat") { + t.Fatalf("retention DELETE should use idx_client_recept_rxat, plan was:\n%s", plan) + } +} + func TestDeriveHeardKey(t *testing.T) { full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, nil, strings.ToUpper(full), true) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 93f95670..3b9849d2 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -298,7 +298,13 @@ func applySchema(db *sql.DB) error { -- lets the planner instead drive from a selective bbox. (#5, #18) CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); - CREATE INDEX IF NOT EXISTS idx_client_recept_rxpk ON client_receptions(rx_pubkey); + -- rx_at backs the retention reaper (DELETE WHERE rx_at < ?) and the + -- leaderboard window (WHERE rx_at >= ?); without it both full-scan under + -- the writer lock. A dedicated rx_pubkey index is redundant — the + -- UNIQUE(rx_pubkey, heard_key, rx_at) constraint already provides an + -- rx_pubkey-leading index for the ?rx= filter / leaderboard GROUP BY. + CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); + DROP INDEX IF EXISTS idx_client_recept_rxpk; -- Self-reported name of each mobile client (companion), from the SELF_INFO -- name the app sends as "origin". Lets the leaderboard show a name even diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index cbc446c3..f19fbd21 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -146,8 +146,10 @@ func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) Covera na.prefix = row.HeardKey } na.count++ - // rx_at is RFC3339, so lexical >= is chronological; keep the latest SNR. - if na.count == 1 || row.RxAt >= na.latestAt { + // rx_at is RFC3339, so lexical >= is chronological; keep the latest + // SNR. The first row always wins (latestAt starts "", and any value + // >= ""), so no separate count==1 guard is needed. + if row.RxAt >= na.latestAt { na.latestAt = row.RxAt na.latestSNR = row.SNR } From e147d8dfed182f8baece88f98f49f7548a3543fb Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 15:51:09 +0200 Subject: [PATCH 29/38] docs(coverage): warn that contributor location is public (privacy BLOCKER) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The polish review flagged that the per-observer view (/api/rx-coverage?rx=) is an unauthenticated, fine-grained movement trail of a single contributor, with no warning in onboarding. Per operator decision this stays an accepted tradeoff (opt-in, default OFF, fine resolution is what makes the aggregate map useful), but consent must be informed. Add a "Privacy — contributor location is public" section: it states plainly that contributions are world-readable, that the per-observer view reconstructs movements, and that a pseudonymous companion name does NOT mitigate it (locations are identifying; the pubkey links all points). Operators are told to warn users; contributors are told not to use a carried device if that matters. The enabling steps cross-link to it. Notes the further-hardening levers (lower clientRxDays, auth proxy, future coarsening / k-anonymity). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/client-rx-coverage.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/client-rx-coverage.md b/docs/client-rx-coverage.md index 833958bb..48468896 100644 --- a/docs/client-rx-coverage.md +++ b/docs/client-rx-coverage.md @@ -29,7 +29,8 @@ Coverage is **off by default**. To turn it on: [Storage](#storage--client_receptions-ingestor-owned)). 4. Point your users at [corescope-rx](https://github.com/efiten/corescope-rx) and they start contributing. Results show on each node's Reach page (coverage toggle) and the `#/rx-coverage` - dashboard. + dashboard. **Warn them first that their contribution is world-readable and a per-observer view can + reconstruct their movements — see [Privacy](#privacy--contributor-location-is-public).** The rest of this document is the MQTT payload contract the companion app implements. @@ -177,6 +178,29 @@ Server/ingestor-side defense-in-depth (these reduce blast radius but do **not** - `/api/nodes/resolve` and coverage tooltips never reveal blacklisted or hidden-prefix node identities (#15). +## Privacy — contributor location is public + +⚠️ **Enabling coverage publishes contributors' GPS-tagged receptions, and the per-observer view can +reconstruct a contributor's movements.** The hex map is read without authentication. The leaderboard +exposes each companion's pubkey, and clicking one filters the map to that single companion +(`/api/rx-coverage?rx=`); at high zoom over the retention window this is effectively a public +movement trail (home / work / commute) of whoever carries that companion. **A pseudonymous companion +name does not mitigate this** — the *locations themselves* are identifying (overnight clustering = home), +and all of one contributor's points are linked by the pubkey. + +This is an accepted tradeoff of the feature, not a bug: fine resolution is what makes the aggregate +coverage map useful, the feature is opt-in and OFF by default, and contributors choose to run the +companion. But the consent must be **informed**: + +- **Operators:** tell your users, before they contribute, that their coverage (including a per-observer + view of their own track) is world-readable for as long as `retention.clientRxDays` keeps it. +- **Contributors:** do not contribute from a device you carry on your person if a public record of where + you have been is a concern. Use a dedicated/stationary node, or accept that the trail is public. + +Operators who want to harden this further can lower `retention.clientRxDays`, run the dashboard behind +their own auth/proxy, or (future hardening) coarsen stored coordinates / apply a k-anonymity threshold +to the per-observer view. + Optional future hardening: have the companion sign a broker-issued token (the firmware exposes on-device signing) — not required for the MVP, tracked as a follow-up. From c11465a09738300c99edacd687138c136dde2500 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 16:00:29 +0200 Subject: [PATCH 30/38] perf(coverage): batch heard_key resolution to kill the N+1 (polish review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit heardKeyResolver issued one `LIKE ? LIMIT 2` per distinct heard_key per request on the writer-shared read connection. Replace it with heardKeyResolverFor(rows): collect the distinct heard_keys once and resolve them all in a single round-trip per 200-key chunk — a UNION ALL of per-prefix `LIMIT 2` subqueries (subquery form because a bare LIMIT on a UNION ALL term is a SQLite syntax error; prefixes are hexPrefixRe-validated so literal interpolation is injection-safe). Per-key work stays bounded at 2 rows, and the #15/#1181 blacklist/hidden hiding is preserved. resolveHeardKey is now a thin wrapper over the batch (single code path). Test: TestBatchResolveHeardKeys checks unique/ambiguous/unknown/hidden in one call; existing resolver + endpoint tests still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/rx_coverage.go | 2 +- cmd/server/rx_dashboard.go | 136 +++++++++++++++++++++++--------- cmd/server/rx_dashboard_test.go | 24 ++++++ 3 files changed, 123 insertions(+), 39 deletions(-) diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index f19fbd21..90283fcf 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -360,7 +360,7 @@ func (s *Server) handleNodeRxCoverage(w http.ResponseWriter, r *http.Request) { http.Error(w, "query failed", http.StatusInternalServerError) return } - fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolver()) + fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolverFor(rows)) // Attach the node-wide reception/contributor totals (#3): the bbox limits the // hex features to the current view, but these summarise all of this node's // mobile coverage so the UI can show "heard by N clients" regardless of pan. diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 6cc9692b..27ba1adc 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -34,55 +34,115 @@ func scanCoverageRows(rows *sql.Rows) ([]coverageRow, error) { return out, rows.Err() } -// heardKeyResolver returns a request-scoped, memoized nodeResolver. It maps a heard_key -// to (pubkey, name) on a unique match — so the same node heard under different prefix -// lengths collapses into one entry — and to (heardKey, "") when unknown or ambiguous. -func (s *Server) heardKeyResolver() nodeResolver { +// heardKeyResolverFor builds a nodeResolver for exactly the distinct heard_keys +// present in rows, resolving them all in one batched query instead of one query +// per key (the previous per-key resolver was N+1 on the read connection). Maps a +// heard_key to (pubkey, name) on a unique, non-hidden match; to (heardKey, "") +// otherwise. nil when there's no DB. +func (s *Server) heardKeyResolverFor(rows []coverageRow) nodeResolver { if s.db == nil || s.db.conn == nil { return nil } - type kv struct{ key, name string } - cache := map[string]kv{} + keys := make([]string, 0, len(rows)) + seen := map[string]bool{} + for _, r := range rows { + if r.HeardKey != "" && !seen[r.HeardKey] { + seen[r.HeardKey] = true + keys = append(keys, r.HeardKey) + } + } + resolved := s.batchResolveHeardKeys(keys) return func(heardKey string) (string, string) { - if v, ok := cache[heardKey]; ok { - return v.key, v.name + if v, ok := resolved[heardKey]; ok { + return v[0], v[1] } - key, name := s.resolveHeardKey(heardKey) - cache[heardKey] = kv{key, name} - return key, name + return heardKey, "" } } -// resolveHeardKey resolves a heard_key (2-3 byte prefix or full pubkey) to a canonical -// (pubkey, name) on a unique match. Unknown or ambiguous (>1 match) keys return the -// heard_key itself with an empty name. LIMIT 2 is enough to tell unique from ambiguous. -func (s *Server) resolveHeardKey(heardKey string) (string, string) { - if heardKey == "" || !hexPrefixRe.MatchString(heardKey) { - return heardKey, "" - } - rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, heardKey+"%") - if err != nil { - return heardKey, "" - } - defer rows.Close() - var pks, names []string - for rows.Next() { - var pk, n string - if err := rows.Scan(&pk, &n); err != nil { - return heardKey, "" +// batchResolveHeardKeys resolves many heard_keys (2-3 byte prefixes or full +// pubkeys) to their canonical (pubkey, name) in a single round-trip per chunk: a +// UNION ALL of one LIMIT-2 prefix lookup each, so per-key work stays bounded +// (2 rows) and the whole set costs one query, not N. A unique match returns +// [pubkey, name]; unknown / ambiguous / blacklisted / hidden-prefix keys (#15, +// #1181) return [heardKey, ""]. +func (s *Server) batchResolveHeardKeys(keys []string) map[string][2]string { + res := make(map[string][2]string, len(keys)) + valid := make([]string, 0, len(keys)) + seen := map[string]bool{} + for _, k := range keys { + if k == "" || seen[k] { + continue } - pks = append(pks, pk) - names = append(names, n) + seen[k] = true + if !hexPrefixRe.MatchString(k) { + res[k] = [2]string{k, ""} + continue + } + valid = append(valid, k) } - if len(pks) == 1 { - // Same identity-hiding parity as /api/nodes/resolve (#15, #1181): don't - // surface a blacklisted or hidden-prefix node's name in coverage tooltips. - if s.cfg.IsBlacklisted(pks[0]) || s.cfg.IsNameHidden(names[0]) { - return heardKey, "" + // SQLITE_MAX_COMPOUND_SELECT is 500 by default; chunk well under it. + const chunk = 200 + for i := 0; i < len(valid); i += chunk { + end := i + chunk + if end > len(valid) { + end = len(valid) + } + batch := valid[i:end] + parts := make([]string, len(batch)) + for j, k := range batch { + // k is hexPrefixRe-validated ([0-9a-f] only), so literal interpolation + // is injection-safe. The per-prefix LIMIT 2 lives in a subquery because + // a bare LIMIT on a UNION ALL term is a SQLite syntax error. + parts[j] = "SELECT * FROM (SELECT '" + k + "' AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE '" + k + "%' LIMIT 2)" + } + rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL ")) + if err != nil { + for _, k := range batch { + res[k] = [2]string{k, ""} + } + continue + } + type agg struct { + pk, name string + cnt int } - return pks[0], names[0] + acc := map[string]*agg{} + for rows.Next() { + var pfx, pk, nm string + if err := rows.Scan(&pfx, &pk, &nm); err != nil { + continue + } + a := acc[pfx] + if a == nil { + a = &agg{} + acc[pfx] = a + } + a.cnt++ + a.pk, a.name = pk, nm + } + rows.Close() + for _, k := range batch { + a := acc[k] + if a != nil && a.cnt == 1 && !s.cfg.IsBlacklisted(a.pk) && !s.cfg.IsNameHidden(a.name) { + res[k] = [2]string{a.pk, a.name} + } else { + res[k] = [2]string{k, ""} + } + } + } + return res +} + +// resolveHeardKey resolves a single heard_key (2-3 byte prefix or full pubkey) +// to its canonical (pubkey, name), or (heardKey, "") when unknown / ambiguous / +// hidden. Thin wrapper over batchResolveHeardKeys so there is one code path. +func (s *Server) resolveHeardKey(heardKey string) (string, string) { + v := s.batchResolveHeardKeys([]string{heardKey})[heardKey] + if v[0] == "" && v[1] == "" { + return heardKey, "" // empty/unresolved → echo the key } - return heardKey, "" + return v[0], v[1] } // queryCoverageFiltered returns coverage rows within a bbox, optionally filtered @@ -153,7 +213,7 @@ func (s *Server) handleRxCoverage(w http.ResponseWriter, r *http.Request) { http.Error(w, "query failed", http.StatusInternalServerError) return } - fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolver()) + fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolverFor(rows)) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(fc) } diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go index aae1eeee..611b57cf 100644 --- a/cmd/server/rx_dashboard_test.go +++ b/cmd/server/rx_dashboard_test.go @@ -90,6 +90,30 @@ func TestRxLeaderboard(t *testing.T) { } } +// TestBatchResolveHeardKeys verifies the N+1 fix: many heard_keys resolve in one +// batched call with the same unique/ambiguous/unknown/hidden semantics as the +// single-key path. +func TestBatchResolveHeardKeys(t *testing.T) { + db := setupTestDBv2(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbccdd11223344','Alice','repeater')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbcc99887766aa','Bob','repeater')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('ddee110000000000','🚫Hidden','repeater')`) + srv := &Server{db: db, cfg: &Config{HiddenNamePrefixes: []string{"🚫"}}} + + got := srv.batchResolveHeardKeys([]string{"aabbccdd", "aabbcc", "ffff", "ddee11", "aabbccdd"}) + cases := map[string][2]string{ + "aabbccdd": {"aabbccdd11223344", "Alice"}, // unique + "aabbcc": {"aabbcc", ""}, // ambiguous (Alice + Bob) + "ffff": {"ffff", ""}, // unknown + "ddee11": {"ddee11", ""}, // unique but hidden-prefix → not surfaced + } + for k, want := range cases { + if got[k] != want { + t.Errorf("batchResolveHeardKeys[%q] = %v, want %v", k, got[k], want) + } + } +} + // TestRxLeaderboardHidesBlacklistedAndHidden verifies #1727 r2 must-fix #2: the // leaderboard must drop observer-blacklisted contributors and blank the name of // node-blacklisted or hidden-prefix identities (pre-PR / post-blacklist rows). From d6ea742f2850484628f5f2192642d386f47c9b79 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Tue, 16 Jun 2026 16:04:03 +0200 Subject: [PATCH 31/38] fix(coverage): a11y + dark-theme for coverage layers (polish review, tufte) - Dark theme: the saturated --nq-cov-* palette only existed in :root and glared on dark basemaps. Add [data-theme="dark"] variants (matches the dashboard's theme-aware tokens). - Colour-blind: SNR tiers were hue-only. Add a redundant, monotonic fill-opacity ramp (strong>mid>weak>grey) in both coverage layers so orange vs red are distinguishable without relying on hue; the per-cell SNR stays in the tooltip. - Keyboard/SR: the leaderboard .rxb-row was a click-only
. Add role="button", tabindex=0, aria-pressed, aria-label, an Enter/Space keydown handler, and a :focus-visible outline. aria-pressed added to the day buttons. Tests: test-node-reach-coverage.js asserts the opacity ramp is strictly decreasing; test-rx-coverage-escape.js asserts the row carries role/tabindex/ aria-pressed. coverageFillOpacity is exported for the unit test. Co-Authored-By: Claude Opus 4.8 (1M context) --- public/node-reach-coverage.css | 10 ++++++++++ public/node-reach-coverage.js | 15 +++++++++++++-- public/rx-coverage.js | 26 ++++++++++++++++++++++---- test-node-reach-coverage.js | 12 +++++++++++- test-rx-coverage-escape.js | 8 +++++++- 5 files changed, 63 insertions(+), 8 deletions(-) diff --git a/public/node-reach-coverage.css b/public/node-reach-coverage.css index 7070870d..a8adcf5c 100644 --- a/public/node-reach-coverage.css +++ b/public/node-reach-coverage.css @@ -11,6 +11,14 @@ --nq-cov-weak: #e74c3c; /* SF8: < −9 dB (poor, packet loss likely) */ --nq-cov-grey: #95a5a6; /* heard, no SNR metric */ } +/* Dark-theme variants: the saturated mid-luminance defaults glare on dark + basemaps. Mirrors the rest of the dashboard's theme-aware tokens (#polish). */ +[data-theme="dark"] { + --nq-cov-strong: #3fb950; + --nq-cov-mid: #d29922; + --nq-cov-weak: #f85149; + --nq-cov-grey: #8b949e; +} .nq-cov-legend { display:flex; gap:12px; align-items:center; font-size:11px; color:var(--text-muted, #57606a); margin:4px 0 10px; } /* Toggled by node-reach.js applyCoverage via class, not inline style, so CSS (print rules, themes) can still override visibility (#19). */ @@ -22,6 +30,8 @@ .rxb-row { display:flex; align-items:center; gap:10px; padding:7px 10px; background:var(--section-bg, #f6f8fa); border:1px solid var(--border, #d0d7de); border-radius:6px; font-size:13px; cursor:pointer; } .rxb-row.rxb-head { font-size:10px; text-transform:uppercase; color:var(--text-muted, #57606a); cursor:default; } .rxb-row.sel { outline:2px solid var(--accent, #2ecc71); } +/* Visible keyboard focus for the now-focusable (#polish) leaderboard rows. */ +.rxb-row[data-rx]:focus-visible { outline:2px solid var(--link, #0969da); outline-offset:1px; } .rxb-rank { width:24px; text-align:right; color:var(--text-muted, #57606a); } .rxb-name { flex:1; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } .rxb-rec, .rxb-nodes { width:50px; text-align:right; font-variant-numeric:tabular-nums; } diff --git a/public/node-reach-coverage.js b/public/node-reach-coverage.js index 43e10dc2..a87f0c48 100644 --- a/public/node-reach-coverage.js +++ b/public/node-reach-coverage.js @@ -22,6 +22,17 @@ } catch (e) { return '#888'; } } + // coverageFillOpacity gives the SNR tier a redundant, non-hue cue (stronger = + // more opaque) so colour-blind users can distinguish tiers (#a11y). + function coverageFillOpacity(props) { + switch (coverageColorVar(props)) { + case '--nq-cov-strong': return 0.6; + case '--nq-cov-mid': return 0.48; + case '--nq-cov-weak': return 0.34; + default: return 0.22; + } + } + // addLayer fetches coverage for the current map bbox/zoom and draws hex // polygons. Returns a handle with off() so the caller can remove it. function addLayer(map, pubkey) { @@ -35,7 +46,7 @@ (fc.features || []).forEach(function (f) { var ring = (f.geometry.coordinates[0] || []).map(function (c) { return [c[1], c[0]]; }); // [lon,lat]→[lat,lon] var col = cssColor(coverageColorVar(f.properties)); - L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: 0.45 }) + L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: coverageFillOpacity(f.properties) }) .addTo(group) .bindTooltip('n=' + f.properties.count + (f.properties.best_snr != null ? ' · SNR ' + f.properties.best_snr : ' · no signal')); @@ -58,5 +69,5 @@ }; } - window.NodeReachCoverage = { coverageColorVar: coverageColorVar, addLayer: addLayer }; + window.NodeReachCoverage = { coverageColorVar: coverageColorVar, coverageFillOpacity: coverageFillOpacity, addLayer: addLayer }; })(); diff --git a/public/rx-coverage.js b/public/rx-coverage.js index 43008c06..bfd1ef7c 100644 --- a/public/rx-coverage.js +++ b/public/rx-coverage.js @@ -22,7 +22,7 @@ return '--nq-cov-weak'; } - function dayBtn(d) { return ''; } + function dayBtn(d) { return ''; } function pageHtml() { return '
' + @@ -62,6 +62,18 @@ return head + '
' + rows + '
' + more; } + // fillOpacityFor adds a redundant, non-hue cue to the SNR tier so the map is + // distinguishable for colour-blind users (orange vs red): stronger signal = + // more opaque. Pairs with the hue and the per-cell SNR in the tooltip (#a11y). + function fillOpacityFor(p) { + switch (colorVar(p)) { + case '--nq-cov-strong': return 0.6; + case '--nq-cov-mid': return 0.48; + case '--nq-cov-weak': return 0.34; + default: return 0.22; + } + } + function drawCoverage() { if (!map || destroyed) return; var b = map.getBounds(); @@ -73,7 +85,7 @@ (fc.features || []).forEach(function (f) { var ring = (f.geometry.coordinates[0] || []).map(function (c) { return [c[1], c[0]]; }); var col = cssColor(colorVar(f.properties)); - L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: 0.45 }).addTo(covLayer) + L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: fillOpacityFor(f.properties) }).addTo(covLayer) .bindTooltip(coverageNodesHtml(f.properties)); }); }).catch(function (e) { console.warn('rx-coverage: coverage fetch failed', e); }); @@ -85,16 +97,22 @@ if (!boardCache.length) { el.innerHTML = '
No mobile observers in this window yet.
'; return; } var rows = boardCache.map(function (o, i) { var nm = o.name ? escapeHtml(o.name) : (escapeHtml(o.pubkey.slice(0, 10)) + '…'); - return '
' + + return '
' + '' + (i + 1) + '' + nm + '' + '' + o.receptions + '' + o.nodes + '
'; }).join(''); el.innerHTML = (selectedRx ? '' : '') + '
#Observer (companion)pktsnodes
' + rows; el.querySelectorAll('.rxb-row[data-rx]').forEach(function (r) { - r.addEventListener('click', function () { + function activate() { selectedRx = r.dataset.rx; selectedName = r.dataset.name || ''; renderBoard(); fitToObserver(); syncHash(); + } + r.addEventListener('click', activate); + // Keyboard parity: Enter/Space activate the row like a button (#a11y). + r.addEventListener('keydown', function (e) { + if (e.key === 'Enter' || e.key === ' ' || e.key === 'Spacebar') { e.preventDefault(); activate(); } }); }); var all = document.getElementById('rxAll'); diff --git a/test-node-reach-coverage.js b/test-node-reach-coverage.js index 407b0ab8..ab8afcf6 100644 --- a/test-node-reach-coverage.js +++ b/test-node-reach-coverage.js @@ -25,4 +25,14 @@ assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -10 }), '--nq-cov assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -18 }), '--nq-cov-weak', 'weak'); assert.strictEqual(coverageColorVar(null), '--nq-cov-grey', 'null props → grey'); -console.log('node-reach-coverage color buckets OK'); +// #a11y: fill opacity is a redundant, monotonic non-hue cue for the SNR tier so +// colour-blind users can tell tiers apart. Must strictly decrease strong→grey. +const { coverageFillOpacity } = sandbox.window.NodeReachCoverage; +const oStrong = coverageFillOpacity({ has_sig: true, best_snr: -3 }); +const oMid = coverageFillOpacity({ has_sig: true, best_snr: -6 }); +const oWeak = coverageFillOpacity({ has_sig: true, best_snr: -10 }); +const oGrey = coverageFillOpacity({ has_sig: false }); +assert.ok(oStrong > oMid && oMid > oWeak && oWeak > oGrey, + 'opacity must ramp strong>mid>weak>grey, got ' + [oStrong, oMid, oWeak, oGrey].join(',')); + +console.log('node-reach-coverage color buckets + opacity ramp OK'); diff --git a/test-rx-coverage-escape.js b/test-rx-coverage-escape.js index 16b1956b..3556546c 100644 --- a/test-rx-coverage-escape.js +++ b/test-rx-coverage-escape.js @@ -52,4 +52,10 @@ assert.ok(row1.indexOf('<img') !== -1 || row1.indexOf('">') !== -1, ' const row2 = renderRow({ pubkey: evil, name: 'Mob', receptions: 2, nodes: 3 }); assert.ok(row2.indexOf('. +assert.ok(/role="button"/.test(row2), 'row must have role="button"'); +assert.ok(/tabindex="0"/.test(row2), 'row must be focusable (tabindex)'); +assert.ok(/aria-pressed="(true|false)"/.test(row2), 'row must expose aria-pressed'); + +console.log('rx-coverage pubkey escaping + row a11y OK'); From 5eac5cefe296d3a7844585038c757a9f4bd5f025 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 17 Jun 2026 07:50:52 +0200 Subject: [PATCH 32/38] fix(coverage): parameterize batch resolver, over-fetch leaderboard, log query errors (review r2) Round-2 must-fix items on the coverage dashboard: - #1 batchResolveHeardKeys: parameterize the per-prefix LIKE (bound args, no literal interpolation) so it stays injection-safe if hexPrefixRe ever widens. - #2 rxLeaderboard: the SQL LIMIT runs before the Go-side observer-blacklist drop, shrinking the public board below the requested limit. Over-fetch by the blacklist size and re-cap to limit. Test: blacklisted top contributors no longer shrink the result. - #4 batchResolveHeardKeys: log.Printf the query error instead of swallowing it (a silent fallback presents as 'every name ambiguous' with no signal). Co-Authored-By: Claude Opus 4.8 --- cmd/server/rx_dashboard.go | 31 ++++++++++++++++++++----- cmd/server/rx_dashboard_test.go | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 27ba1adc..bc423c33 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "encoding/json" + "log" "net/http" "strconv" "strings" @@ -90,14 +91,20 @@ func (s *Server) batchResolveHeardKeys(keys []string) map[string][2]string { } batch := valid[i:end] parts := make([]string, len(batch)) + args := make([]interface{}, 0, len(batch)*2) for j, k := range batch { - // k is hexPrefixRe-validated ([0-9a-f] only), so literal interpolation - // is injection-safe. The per-prefix LIMIT 2 lives in a subquery because - // a bare LIMIT on a UNION ALL term is a SQLite syntax error. - parts[j] = "SELECT * FROM (SELECT '" + k + "' AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE '" + k + "%' LIMIT 2)" + // Parameterized: the prefix flows in as bound args, never interpolated, + // so this stays injection-safe regardless of how hexPrefixRe later + // evolves. The per-prefix LIMIT 2 lives in a subquery because a bare + // LIMIT on a UNION ALL term is a SQLite syntax error. + parts[j] = "SELECT * FROM (SELECT ? AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE ? LIMIT 2)" + args = append(args, k, k+"%") } - rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL ")) + rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL "), args...) if err != nil { + // Don't fail the request, but don't fail silently either: a swallowed + // error here presents as "every name is ambiguous" with no signal. + log.Printf("WARN batchResolveHeardKeys: %v", err) for _, k := range batch { res[k] = [2]string{k, ""} } @@ -233,6 +240,15 @@ type RxLeaderboardResp struct { func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + // Over-fetch by the observer-blacklist size: the SQL LIMIT is applied before + // the Go-side blacklist drop below, so a blacklisted top contributor would + // otherwise shrink the public leaderboard under `limit`. At most + // len(observerBlacklist) rows are dropped, so limit+that guarantees >= limit + // survivors; we re-cap to limit after filtering. + sqlLimit := limit + if s.cfg != nil { + sqlLimit += len(s.cfg.ObserverBlacklist) + } // Name preference: the node's advertised name, else the companion's // self-reported name (client_observers), else empty (UI shows the prefix). rows, err := s.db.conn.Query(` @@ -243,7 +259,7 @@ func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { WHERE cr.rx_at >= ? GROUP BY cr.rx_pubkey ORDER BY COUNT(*) DESC - LIMIT ?`, since, limit) + LIMIT ?`, since, sqlLimit) if err != nil { return nil, err } @@ -272,6 +288,9 @@ func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { o.Name = "" } filtered = append(filtered, o) + if len(filtered) >= limit { + break + } } return filtered, nil } diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go index 611b57cf..cf5c5ceb 100644 --- a/cmd/server/rx_dashboard_test.go +++ b/cmd/server/rx_dashboard_test.go @@ -154,3 +154,44 @@ func TestRxLeaderboardHidesBlacklistedAndHidden(t *testing.T) { t.Fatalf("hidden-prefix contributor should remain with a blanked name: %+v", byPk["dd04"]) } } + +// TestRxLeaderboardLimitSurvivesBlacklistDrop verifies #1727 r2 must-fix #2: the +// SQL LIMIT runs before the Go-side observer-blacklist drop, so the leaderboard +// must over-fetch and still return `limit` non-blacklisted rows even when the +// top contributors are blacklisted (not limit-minus-dropped). +func TestRxLeaderboardLimitSurvivesBlacklistDrop(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + // Reception counts strictly descending so ORDER BY COUNT(*) DESC is deterministic: + // the two blacklisted observers are the top two, then five good ones. + counts := []struct { + pk string + n int + }{ + {"bk1", 10}, {"bk2", 9}, // observer-blacklisted (top of the board) + {"g1", 8}, {"g2", 7}, {"g3", 6}, {"g4", 5}, {"g5", 4}, + } + for _, c := range counts { + for i := 0; i < c.n; i++ { + insRx(t, db, c.pk, fmt.Sprintf("%s%04d", c.pk, i), recent, 51.05, 3.72) + } + } + srv := &Server{db: db, cfg: &Config{ObserverBlacklist: []string{"bk1", "bk2"}}} + + obs, err := srv.rxLeaderboard(7, 3) + if err != nil { + t.Fatal(err) + } + if len(obs) != 3 { + t.Fatalf("expected exactly 3 rows after dropping 2 blacklisted from the top, got %d: %+v", len(obs), obs) + } + want := []string{"g1", "g2", "g3"} + for i, o := range obs { + if o.Pubkey == "bk1" || o.Pubkey == "bk2" { + t.Fatalf("blacklisted observer %q leaked into the leaderboard", o.Pubkey) + } + if o.Pubkey != want[i] { + t.Fatalf("row %d = %q, want %q (top-3 non-blacklisted by count)", i, o.Pubkey, want[i]) + } + } +} From 220656404480b3e5d584b50b365ede14d250d908 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 17 Jun 2026 07:50:52 +0200 Subject: [PATCH 33/38] test(coverage): pin leaderboard query plan + correct rx_at index comment (review r2 #3) EXPLAIN shows the leaderboard SELECT is served by the UNIQUE(rx_pubkey,heard_key, rx_at) covering index, NOT idx_client_recept_rxat. Add an EXPLAIN test that pins it as index-backed (guards against a regression to a bare table scan under the writer lock; the table is retention-bounded so a covering scan is fine), and correct the db.go comment that wrongly claimed rx_at backs the leaderboard window. Co-Authored-By: Claude Opus 4.8 --- cmd/ingestor/client_reception_test.go | 39 +++++++++++++++++++++++++++ cmd/ingestor/db.go | 11 ++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index cf3a0d25..2bba8f18 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -129,6 +129,45 @@ func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { } } +// TestRxLeaderboardQueryIsIndexBacked pins the planner choice for the leaderboard +// SELECT (the rx_at-windowed, rx_pubkey-grouped query in cmd/server/rx_dashboard.go). +// SQLite serves it from the UNIQUE(rx_pubkey,heard_key,rx_at) constraint index as a +// COVERING scan (not idx_client_recept_rxat, and not a table-heap scan). The table +// is retention-bounded, so a covering scan is acceptable; this test guards against a +// silent regression to a bare table scan under the writer lock when the schema is +// next tweaked. Representative form (no JOINs — they don't change whether `cr` is +// index-backed). +func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { + s := newTestStore(t) + rows, err := s.db.Query(`EXPLAIN QUERY PLAN + SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) + FROM client_receptions cr + WHERE cr.rx_at >= ? + GROUP BY cr.rx_pubkey + ORDER BY COUNT(*) DESC + LIMIT ?`, "2026-01-01T00:00:00Z", 100) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + t.Logf("leaderboard plan:\n%s", plan) + // The concern is a bare table-heap scan, not which specific index wins. The + // plan must stay index-backed (covering or search) — a regression to a bare + // "SCAN cr" without an index fails here. + if !strings.Contains(plan, "INDEX") { + t.Fatalf("leaderboard SELECT must stay index-backed (no full table-heap scan), plan was:\n%s", plan) + } +} + func TestDeriveHeardKey(t *testing.T) { full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, nil, strings.ToUpper(full), true) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 3b9849d2..2b2e0c95 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -298,11 +298,12 @@ func applySchema(db *sql.DB) error { -- lets the planner instead drive from a selective bbox. (#5, #18) CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); - -- rx_at backs the retention reaper (DELETE WHERE rx_at < ?) and the - -- leaderboard window (WHERE rx_at >= ?); without it both full-scan under - -- the writer lock. A dedicated rx_pubkey index is redundant — the - -- UNIQUE(rx_pubkey, heard_key, rx_at) constraint already provides an - -- rx_pubkey-leading index for the ?rx= filter / leaderboard GROUP BY. + -- rx_at backs the retention reaper (DELETE WHERE rx_at < ?), which would + -- otherwise full-scan the table under the writer lock (verified by an + -- EXPLAIN test). The leaderboard (GROUP BY rx_pubkey, WHERE rx_at >= ?) is + -- served instead by the UNIQUE(rx_pubkey, heard_key, rx_at) constraint's + -- index as a COVERING scan (no table-heap access; the table is + -- retention-bounded), so a dedicated rx_pubkey index is redundant. CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); DROP INDEX IF EXISTS idx_client_recept_rxpk; From 997aaee042f8fff1f84d426ab70010a48f895909 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 17 Jun 2026 10:45:49 +0200 Subject: [PATCH 34/38] feat(coverage): rank mobile observers by frontier-weighted cell coverage Replace raw COUNT(*) leaderboard ranking with a per-observer score that sums 1/(observers covering each ~150m cell), plus cells/score fields. Spam-proof (parked node = 1 cell) and kills dense-area bias. Aggregated in Go over the window; ties broken by receptions then pubkey. Co-Authored-By: Claude Opus 4.8 --- cmd/server/rx_dashboard.go | 111 ++++++++++++++++++++++++-------- cmd/server/rx_dashboard_test.go | 49 ++++++++++++++ 2 files changed, 133 insertions(+), 27 deletions(-) diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index bc423c33..559da563 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -5,6 +5,7 @@ import ( "encoding/json" "log" "net/http" + "sort" "strconv" "strings" "time" @@ -228,58 +229,114 @@ func (s *Server) handleRxCoverage(w http.ResponseWriter, r *http.Request) { // --- Leaderboard (top mobile observers) --- type LeaderObserver struct { - Pubkey string `json:"pubkey"` - Name string `json:"name"` - Receptions int `json:"receptions"` - Nodes int `json:"nodes"` + Pubkey string `json:"pubkey"` + Name string `json:"name"` + Receptions int `json:"receptions"` + Nodes int `json:"nodes"` + Cells int `json:"cells"` // distinct fixed-res hex cells covered + Score float64 `json:"score"` // frontier-weighted coverage score } type RxLeaderboardResp struct { Days int `json:"days"` Observers []LeaderObserver `json:"observers"` } +// leaderboardHexRes is the fixed hex resolution used to bucket receptions into +// "cells visited" for the frontier-weighted score. ~150 m ground cells at our +// latitude: coarse enough that a parked node's GPS jitter stays in one cell, +// fine enough that real driving paints many. Independent of the coverage map's +// zoom-dependent render resolution so the ranking is stable across views. +const leaderboardHexRes = 13 + +// rxLeaderboard ranks mobile observers by frontier-weighted cell coverage over +// the time window. Each distinct cell an observer covers contributes +// 1/(observers covering that cell): a cell only they reached weighs 1.0, a cell +// shared by N observers weighs 1/N. This rewards expanding the map's edge and is +// spam-proof — a stationary node covers exactly one cell regardless of how many +// receptions it logs. Bucketing + the rarity weight can't be expressed in SQL, +// so we aggregate the window's rows in Go (a few thousand — cheap). func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - // Over-fetch by the observer-blacklist size: the SQL LIMIT is applied before - // the Go-side blacklist drop below, so a blacklisted top contributor would - // otherwise shrink the public leaderboard under `limit`. At most - // len(observerBlacklist) rows are dropped, so limit+that guarantees >= limit - // survivors; we re-cap to limit after filtering. - sqlLimit := limit - if s.cfg != nil { - sqlLimit += len(s.cfg.ObserverBlacklist) - } // Name preference: the node's advertised name, else the companion's // self-reported name (client_observers), else empty (UI shows the prefix). rows, err := s.db.conn.Query(` - SELECT cr.rx_pubkey, COALESCE(NULLIF(n.name,''), NULLIF(co.name,''), ''), COUNT(*), COUNT(DISTINCT cr.heard_key) + SELECT cr.rx_pubkey, COALESCE(NULLIF(n.name,''), NULLIF(co.name,''), ''), + cr.lat, cr.lon, cr.heard_key FROM client_receptions cr LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey - WHERE cr.rx_at >= ? - GROUP BY cr.rx_pubkey - ORDER BY COUNT(*) DESC - LIMIT ?`, since, sqlLimit) + WHERE cr.rx_at >= ?`, since) if err != nil { return nil, err } defer rows.Close() - out := []LeaderObserver{} + + type agg struct { + name string + receptions int + cells map[string]struct{} + nodes map[string]struct{} + } + obsAgg := map[string]*agg{} + cellObservers := map[string]map[string]struct{}{} // cell -> set of rx_pubkey + for rows.Next() { - var o LeaderObserver - if err := rows.Scan(&o.Pubkey, &o.Name, &o.Receptions, &o.Nodes); err != nil { + var pk, name, heardKey string + var lat, lon float64 + if err := rows.Scan(&pk, &name, &lat, &lon, &heardKey); err != nil { return nil, err } - out = append(out, o) + a := obsAgg[pk] + if a == nil { + a = &agg{name: name, cells: map[string]struct{}{}, nodes: map[string]struct{}{}} + obsAgg[pk] = a + } + a.receptions++ + a.nodes[heardKey] = struct{}{} + cell := hexCellAt(lat, lon, leaderboardHexRes) + a.cells[cell] = struct{}{} + set := cellObservers[cell] + if set == nil { + set = map[string]struct{}{} + cellObservers[cell] = set + } + set[pk] = struct{}{} } if err := rows.Err(); err != nil { return nil, err } - // Identity hiding parity (#1727 r2): pre-PR rows / blacklist-after-ingest / - // config-reload lag could otherwise surface a hidden operator here. Drop - // observer-blacklisted contributors entirely, and blank the name of a - // node-blacklisted or hidden-prefix identity. nil cfg ⇒ all no-ops. - filtered := out[:0] + + out := make([]LeaderObserver, 0, len(obsAgg)) + for pk, a := range obsAgg { + var score float64 + for cell := range a.cells { + score += 1.0 / float64(len(cellObservers[cell])) + } + out = append(out, LeaderObserver{ + Pubkey: pk, + Name: a.name, + Receptions: a.receptions, + Nodes: len(a.nodes), + Cells: len(a.cells), + Score: score, + }) + } + + // Rank by frontier score; ties broken by raw receptions then pubkey so the + // order is deterministic (keeps same-location fixtures stable). + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + if out[i].Receptions != out[j].Receptions { + return out[i].Receptions > out[j].Receptions + } + return out[i].Pubkey < out[j].Pubkey + }) + + // Identity hiding parity (#1727 r2): drop observer-blacklisted contributors, + // blank node-blacklisted / hidden-prefix names, cap at limit. nil cfg ⇒ no-ops. + filtered := make([]LeaderObserver, 0, limit) for _, o := range out { if s.cfg.IsObserverBlacklisted(o.Pubkey) { continue diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go index cf5c5ceb..afc99dd0 100644 --- a/cmd/server/rx_dashboard_test.go +++ b/cmd/server/rx_dashboard_test.go @@ -195,3 +195,52 @@ func TestRxLeaderboardLimitSurvivesBlacklistDrop(t *testing.T) { } } } + +// TestRxLeaderboardFrontierScore verifies the leaderboard ranks by frontier- +// weighted cell coverage, not raw reception count: a roaming observer with FEWER +// receptions but MORE distinct cells outranks a stationary spammer, a cell only +// one observer covers weighs 1.0, and a cell shared by N observers weighs 1/N. +func TestRxLeaderboardFrontierScore(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + // "park": 5 receptions all at ONE spot → 1 cell (stationary spammer). + for i := 0; i < 5; i++ { + insRx(t, db, "park", fmt.Sprintf("pk%04d", i), recent, 51.05, 3.72) + } + // "roam": 3 receptions at 3 far-apart spots → 3 distinct cells. The first + // coincides with park's cell, so that cell is shared by 2 observers. + insRx(t, db, "roam", "rm0001", recent, 51.05, 3.72) // shared with park + insRx(t, db, "roam", "rm0002", recent, 51.06, 3.72) // unique to roam + insRx(t, db, "roam", "rm0003", recent, 51.07, 3.72) // unique to roam + srv := &Server{db: db} + + obs, err := srv.rxLeaderboard(7, 10) + if err != nil { + t.Fatal(err) + } + if len(obs) != 2 { + t.Fatalf("want 2 observers, got %d: %+v", len(obs), obs) + } + // Roamer outranks the parked spammer despite fewer receptions. + if obs[0].Pubkey != "roam" || obs[1].Pubkey != "park" { + t.Fatalf("ranking: want [roam park], got [%s %s]", obs[0].Pubkey, obs[1].Pubkey) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + // park: 1 cell shared with roam → score 0.5; 5 receptions retained. + if byPk["park"].Cells != 1 || byPk["park"].Receptions != 5 { + t.Fatalf("park: %+v", byPk["park"]) + } + if d := byPk["park"].Score - 0.5; d > 1e-9 || d < -1e-9 { + t.Fatalf("park score: want 0.5, got %v", byPk["park"].Score) + } + // roam: shared cell (0.5) + 2 unique cells (1.0 each) = 2.5; 3 cells. + if byPk["roam"].Cells != 3 { + t.Fatalf("roam cells: want 3, got %d", byPk["roam"].Cells) + } + if d := byPk["roam"].Score - 2.5; d > 1e-9 || d < -1e-9 { + t.Fatalf("roam score: want 2.5, got %v", byPk["roam"].Score) + } +} From 1574e087c0e70d130d7a4023fd8475d6a4ac8a14 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 17 Jun 2026 10:50:20 +0200 Subject: [PATCH 35/38] docs(ingestor): refresh stale covering-scan comment after leaderboard rewrite The leaderboard no longer GROUPs BY rx_pubkey in SQL (it range-scans rx_at and aggregates in Go), so the old "served by the UNIQUE index as a COVERING scan" note was inaccurate. The conclusion stands: a dedicated rx_pubkey index is still redundant. Co-Authored-By: Claude Opus 4.8 --- cmd/ingestor/db.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 2b2e0c95..58cece26 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -298,12 +298,12 @@ func applySchema(db *sql.DB) error { -- lets the planner instead drive from a selective bbox. (#5, #18) CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); - -- rx_at backs the retention reaper (DELETE WHERE rx_at < ?), which would - -- otherwise full-scan the table under the writer lock (verified by an - -- EXPLAIN test). The leaderboard (GROUP BY rx_pubkey, WHERE rx_at >= ?) is - -- served instead by the UNIQUE(rx_pubkey, heard_key, rx_at) constraint's - -- index as a COVERING scan (no table-heap access; the table is - -- retention-bounded), so a dedicated rx_pubkey index is redundant. + -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the + -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per + -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without + -- this index either would full-scan the table under the writer lock + -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays + -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); DROP INDEX IF EXISTS idx_client_recept_rxpk; From 547e5624b995f82409053f45489aabd99cdc3993 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 17 Jun 2026 10:52:26 +0200 Subject: [PATCH 36/38] feat(coverage): sortable leaderboard table with cells + frontier score Render the Top mobile observers board as a table with score/cells columns, every column sortable (default score desc), and a hover tooltip on the score header explaining the frontier weighting. Row click-to-filter preserved. Co-Authored-By: Claude Opus 4.8 --- public/node-reach-coverage.css | 6 +++- public/rx-coverage.js | 62 ++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/public/node-reach-coverage.css b/public/node-reach-coverage.css index a8adcf5c..b58e88ff 100644 --- a/public/node-reach-coverage.css +++ b/public/node-reach-coverage.css @@ -34,4 +34,8 @@ .rxb-row[data-rx]:focus-visible { outline:2px solid var(--link, #0969da); outline-offset:1px; } .rxb-rank { width:24px; text-align:right; color:var(--text-muted, #57606a); } .rxb-name { flex:1; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } -.rxb-rec, .rxb-nodes { width:50px; text-align:right; font-variant-numeric:tabular-nums; } +.rxb-rec, .rxb-nodes, .rxb-cells, .rxb-score { width:56px; text-align:right; font-variant-numeric:tabular-nums; } +/* Sortable column headers in the leaderboard head row. */ +.rxb-head .rxb-sort { cursor:pointer; user-select:none; } +.rxb-head .rxb-sort:hover { color:var(--link, #0969da); } +.rxb-head .rxb-sort:focus-visible { outline:2px solid var(--link, #0969da); outline-offset:1px; } diff --git a/public/rx-coverage.js b/public/rx-coverage.js index bfd1ef7c..b2e3824e 100644 --- a/public/rx-coverage.js +++ b/public/rx-coverage.js @@ -91,26 +91,82 @@ }).catch(function (e) { console.warn('rx-coverage: coverage fetch failed', e); }); } + // Leaderboard sort state. Default = frontier score, descending. The rank (#) + // column is not sortable (it just reflects the current order). Numeric columns + // default to descending on first click; the name column to ascending. + var boardSort = { key: 'score', dir: 'desc' }; + var BOARD_COLS = [ + { key: 'name', label: 'Observer (companion)', cls: 'rxb-name' }, + { key: 'score', label: 'score', cls: 'rxb-score', + title: 'Score telt je gedekte cellen, waarbij elke cel zwaarder weegt naarmate minder andere waarnemers ze bereikt hebben — grensverleggende dekking weegt meer dan drukke zones opnieuw afrijden.' }, + { key: 'cells', label: 'cells', cls: 'rxb-cells', + title: 'Aantal unieke ~150 m-cellen waar deze waarnemer iets hoorde.' }, + { key: 'nodes', label: 'nodes', cls: 'rxb-nodes' }, + { key: 'receptions', label: 'pkts', cls: 'rxb-rec' } + ]; + + function sortBoard() { + var k = boardSort.key, dir = boardSort.dir === 'asc' ? 1 : -1; + boardCache.sort(function (a, b) { + if (k === 'name') { + var an = (a.name || a.pubkey).toLowerCase(), bn = (b.name || b.pubkey).toLowerCase(); + return an < bn ? -dir : an > bn ? dir : 0; + } + return (Number(a[k]) - Number(b[k])) * dir; + }); + } + + function boardHeadHtml() { + var cells = BOARD_COLS.map(function (c) { + var arrow = boardSort.key === c.key ? (boardSort.dir === 'asc' ? ' ▲' : ' ▼') : ''; + return '' + escapeHtml(c.label) + arrow + ''; + }).join(''); + return '
#' + cells + '
'; + } + function renderBoard() { var el = document.getElementById('rxBoard'); if (!el) return; if (!boardCache.length) { el.innerHTML = '
No mobile observers in this window yet.
'; return; } + sortBoard(); var rows = boardCache.map(function (o, i) { var nm = o.name ? escapeHtml(o.name) : (escapeHtml(o.pubkey.slice(0, 10)) + '…'); return '
' + '' + (i + 1) + '' + nm + '' + - '' + o.receptions + '' + o.nodes + '
'; + '' + Number(o.score).toFixed(1) + '' + + '' + o.cells + '' + + '' + o.nodes + '' + + '' + o.receptions + '
'; }).join(''); el.innerHTML = (selectedRx ? '' : '') + - '
#Observer (companion)pktsnodes
' + rows; + boardHeadHtml() + rows; + // Column sort handlers (click + keyboard). + el.querySelectorAll('.rxb-sort[data-sort]').forEach(function (h) { + function applySort() { + var k = h.dataset.sort; + if (boardSort.key === k) { + boardSort.dir = boardSort.dir === 'asc' ? 'desc' : 'asc'; + } else { + boardSort.key = k; + boardSort.dir = (k === 'name') ? 'asc' : 'desc'; + } + renderBoard(); + } + h.addEventListener('click', applySort); + h.addEventListener('keydown', function (e) { + if (e.key === 'Enter' || e.key === ' ' || e.key === 'Spacebar') { e.preventDefault(); applySort(); } + }); + }); + // Row click-to-filter (preserved from the original). el.querySelectorAll('.rxb-row[data-rx]').forEach(function (r) { function activate() { selectedRx = r.dataset.rx; selectedName = r.dataset.name || ''; renderBoard(); fitToObserver(); syncHash(); } r.addEventListener('click', activate); - // Keyboard parity: Enter/Space activate the row like a button (#a11y). r.addEventListener('keydown', function (e) { if (e.key === 'Enter' || e.key === ' ' || e.key === 'Spacebar') { e.preventDefault(); activate(); } }); From eabbae614bfea083c8899108ab647a2589769238 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 17 Jun 2026 11:51:05 +0200 Subject: [PATCH 37/38] chore(deploy): untrack environment-specific deploy scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy-live.sh and deploy-staging.sh contain host-specific deploy logic (on8ar.eu, mesh-internal, container names) — they belong only on the deploy host, not in version control. Remove them from the repo and gitignore them so future deploys (git reset --hard) no longer overwrite the host-local copies. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 4 ++++ deploy-live.sh | 27 --------------------------- deploy-staging.sh | 30 ------------------------------ 3 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 deploy-live.sh delete mode 100644 deploy-staging.sh diff --git a/.gitignore b/.gitignore index f44fb736..548c25d6 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ corescope-server cmd/server/server # Local-only planning and design files docs/superpowers/ + +# Environment-specific deploy scripts — live only on the deploy host, not tracked +deploy-live.sh +deploy-staging.sh diff --git a/deploy-live.sh b/deploy-live.sh deleted file mode 100644 index bc100d83..00000000 --- a/deploy-live.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)" -MATOMO_COMMIT="38c30f9" - -cd "$DEPLOY_DIR" - -echo "[deploy] Fetching latest from origin..." -git fetch origin - -echo "[deploy] Resetting to origin/master..." -git reset --hard origin/master - -echo "[deploy] Building Docker image..." -docker build -t meshcore-analyzer . - -echo "[deploy] Stopping old container (30s grace period)..." -docker stop -t 30 meshcore-analyzer && docker rm meshcore-analyzer -docker run -d --name meshcore-analyzer \ - --restart unless-stopped \ - -p 3000:3000 \ - -v "$(pwd)/config.json:/app/config.json:ro" \ - -v meshcore-data:/app/data \ - meshcore-analyzer - -echo "[deploy] Done. Live at https://analyzer.on8ar.eu" diff --git a/deploy-staging.sh b/deploy-staging.sh deleted file mode 100644 index 2c437e95..00000000 --- a/deploy-staging.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)" - -cd "$DEPLOY_DIR" - -echo "[staging] Fetching latest from origin..." -git fetch origin - -BRANCH="${1:-master}" -echo "[staging] Checking out $BRANCH..." -git reset --hard "origin/$BRANCH" - -echo "[staging] Building Docker image..." -docker build -t meshcore-analyzer-staging . - -echo "[staging] Stopping old container (30s grace period)..." -docker stop -t 30 meshcore-staging 2>/dev/null || true -docker rm meshcore-staging 2>/dev/null || true - -echo "[staging] Starting new container..." -docker run -d --name meshcore-staging \ - --restart unless-stopped \ - -p 3001:3000 \ - -v "$(pwd)/config.json:/app/config.json:ro" \ - -v meshcore-staging-data:/app/data \ - meshcore-analyzer-staging - -echo "[staging] Done. Live at https://staging.on8ar.eu" From f472a3f2f63a878784966e33933e77d1d089bf26 Mon Sep 17 00:00:00 2001 From: Erwin Fiten Date: Thu, 18 Jun 2026 09:18:32 +0200 Subject: [PATCH 38/38] fix(coverage): bound leaderboard scan + blacklist-proof scoring (review r2) Two production-data MAJORs in the new frontier-weighted leaderboard: - Unbounded scan: the Go-side rarity weighting dropped the SQL LIMIT, so an unauthenticated /api/rx-leaderboard streamed the whole window into maps. rxLeaderboard now takes a context.Context (QueryContext + batched ctx.Err() checks every 2048 rows), caps the scan at leaderboardScanCap (500k) ORDER BY rx_at DESC (keep most recent on truncation), and logs when the cap is hit. - Score dilution: cellObservers was populated before the blacklist filter, so a blacklisted-node operator parked in a cell silently lowered every legitimate observer's 1/N frontier weight. The per-cell denominator (cellCount) now excludes observer- and node-blacklisted pubkeys; name-hidden (non-blacklisted) contributors still count. Test: TestRxLeaderboardScoreNotDilutedByBlacklisted asserts a legit observer sharing a cell with a blacklisted one scores 1.0, not 0.5 (fails without the fix). Existing leaderboard/frontier tests updated for the new ctx parameter. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/server/rx_dashboard.go | 51 +++++++++++++++++++++++++++++---- cmd/server/rx_dashboard_test.go | 37 +++++++++++++++++++++--- 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 559da563..c4ae3b47 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -1,6 +1,7 @@ package main import ( + "context" "database/sql" "encoding/json" "log" @@ -248,24 +249,35 @@ type RxLeaderboardResp struct { // zoom-dependent render resolution so the ranking is stable across views. const leaderboardHexRes = 13 +// leaderboardScanCap bounds how many rows the leaderboard aggregates in memory. +// The endpoint is unauthenticated (only requireClientRxCoverage), and the Go-side +// rarity weighting can't push the GROUP BY into SQLite, so without a cap a wide +// window on a busy network would stream the whole table into maps. At the cap we +// log and return a partial (best-effort) ranking rather than OOM (#review r2). +const leaderboardScanCap = 500000 + // rxLeaderboard ranks mobile observers by frontier-weighted cell coverage over // the time window. Each distinct cell an observer covers contributes // 1/(observers covering that cell): a cell only they reached weighs 1.0, a cell // shared by N observers weighs 1/N. This rewards expanding the map's edge and is // spam-proof — a stationary node covers exactly one cell regardless of how many // receptions it logs. Bucketing + the rarity weight can't be expressed in SQL, -// so we aggregate the window's rows in Go (a few thousand — cheap). -func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { +// so we aggregate the window's rows in Go (bounded by leaderboardScanCap). +func (s *Server) rxLeaderboard(ctx context.Context, days, limit int) ([]LeaderObserver, error) { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) // Name preference: the node's advertised name, else the companion's // self-reported name (client_observers), else empty (UI shows the prefix). - rows, err := s.db.conn.Query(` + // Hard LIMIT bounds memory; ORDER BY rx_at DESC so a truncated window keeps + // the most recent receptions. + rows, err := s.db.conn.QueryContext(ctx, ` SELECT cr.rx_pubkey, COALESCE(NULLIF(n.name,''), NULLIF(co.name,''), ''), cr.lat, cr.lon, cr.heard_key FROM client_receptions cr LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey - WHERE cr.rx_at >= ?`, since) + WHERE cr.rx_at >= ? + ORDER BY cr.rx_at DESC + LIMIT ?`, since, leaderboardScanCap) if err != nil { return nil, err } @@ -280,12 +292,19 @@ func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { obsAgg := map[string]*agg{} cellObservers := map[string]map[string]struct{}{} // cell -> set of rx_pubkey + scanned := 0 for rows.Next() { + // Honour client cancellation/timeout on the long scan (checked in batches + // to avoid a per-row context mutex on up to 500k rows). + if scanned&2047 == 0 && ctx.Err() != nil { + return nil, ctx.Err() + } var pk, name, heardKey string var lat, lon float64 if err := rows.Scan(&pk, &name, &lat, &lon, &heardKey); err != nil { return nil, err } + scanned++ a := obsAgg[pk] if a == nil { a = &agg{name: name, cells: map[string]struct{}{}, nodes: map[string]struct{}{}} @@ -305,12 +324,32 @@ func (s *Server) rxLeaderboard(days, limit int) ([]LeaderObserver, error) { if err := rows.Err(); err != nil { return nil, err } + if scanned >= leaderboardScanCap { + log.Printf("[rx-leaderboard] scan hit cap %d over %dd window; ranking is partial (most-recent rows)", leaderboardScanCap, days) + } + + // Per-cell observer counts EXCLUDING blacklisted contributors, so an operator + // of a blacklisted node parked in a cell can't silently dilute everyone else's + // frontier weight (#review r2). Name-hidden (not blacklisted) observers are + // legitimate contributors and still count. + cellCount := make(map[string]int, len(cellObservers)) + for cell, set := range cellObservers { + n := 0 + for pk := range set { + if !s.cfg.IsObserverBlacklisted(pk) && !s.cfg.IsBlacklisted(pk) { + n++ + } + } + cellCount[cell] = n + } out := make([]LeaderObserver, 0, len(obsAgg)) for pk, a := range obsAgg { var score float64 for cell := range a.cells { - score += 1.0 / float64(len(cellObservers[cell])) + if c := cellCount[cell]; c > 0 { + score += 1.0 / float64(c) + } } out = append(out, LeaderObserver{ Pubkey: pk, @@ -365,7 +404,7 @@ func (s *Server) handleRxLeaderboard(w http.ResponseWriter, r *http.Request) { if limit < 1 || limit > 100 { limit = 20 } - obs, err := s.rxLeaderboard(days, limit) + obs, err := s.rxLeaderboard(r.Context(), days, limit) if err != nil { http.Error(w, "query failed", http.StatusInternalServerError) return diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go index afc99dd0..5d708bee 100644 --- a/cmd/server/rx_dashboard_test.go +++ b/cmd/server/rx_dashboard_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -71,7 +72,7 @@ func TestRxLeaderboard(t *testing.T) { insRx(t, db, "compb", "aabbcc", recent, 51.05, 3.72) // no name anywhere srv := &Server{db: db} - obs, err := srv.rxLeaderboard(7, 10) + obs, err := srv.rxLeaderboard(context.Background(), 7, 10) if err != nil { t.Fatal(err) } @@ -133,7 +134,7 @@ func TestRxLeaderboardHidesBlacklistedAndHidden(t *testing.T) { HiddenNamePrefixes: []string{"🚫"}, }} - obs, err := srv.rxLeaderboard(7, 100) + obs, err := srv.rxLeaderboard(context.Background(), 7, 100) if err != nil { t.Fatal(err) } @@ -178,7 +179,7 @@ func TestRxLeaderboardLimitSurvivesBlacklistDrop(t *testing.T) { } srv := &Server{db: db, cfg: &Config{ObserverBlacklist: []string{"bk1", "bk2"}}} - obs, err := srv.rxLeaderboard(7, 3) + obs, err := srv.rxLeaderboard(context.Background(), 7, 3) if err != nil { t.Fatal(err) } @@ -214,7 +215,7 @@ func TestRxLeaderboardFrontierScore(t *testing.T) { insRx(t, db, "roam", "rm0003", recent, 51.07, 3.72) // unique to roam srv := &Server{db: db} - obs, err := srv.rxLeaderboard(7, 10) + obs, err := srv.rxLeaderboard(context.Background(), 7, 10) if err != nil { t.Fatal(err) } @@ -244,3 +245,31 @@ func TestRxLeaderboardFrontierScore(t *testing.T) { t.Fatalf("roam score: want 2.5, got %v", byPk["roam"].Score) } } + +// TestRxLeaderboardScoreNotDilutedByBlacklisted verifies the #review-r2 fix: a +// blacklisted observer sharing a cell must NOT dilute a legitimate observer's +// frontier score. Without excluding blacklisted pubkeys from the per-cell count, +// the legit observer's only cell would weigh 1/2 = 0.5 instead of 1.0. +func TestRxLeaderboardScoreNotDilutedByBlacklisted(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + // Legit "good" and blacklisted "bad" both cover the same ~150 m cell. + insRx(t, db, "good", "aabb01", recent, 51.05, 3.72) + insRx(t, db, "bad", "aabb02", recent, 51.05, 3.72) + srv := &Server{db: db, cfg: &Config{ObserverBlacklist: []string{"bad"}}} + + obs, err := srv.rxLeaderboard(context.Background(), 7, 100) + if err != nil { + t.Fatal(err) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + if _, leaked := byPk["bad"]; leaked { + t.Fatalf("blacklisted observer must not appear: %+v", byPk["bad"]) + } + if d := byPk["good"].Score - 1.0; d > 1e-9 || d < -1e-9 { + t.Fatalf("blacklisted observer diluted the score: got %v, want 1.0", byPk["good"].Score) + } +}
#Neighbourwe hear