diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c4c468c43..64acc1e7d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -138,6 +138,8 @@ jobs: node test-issue-1509-nav-active-bg.js node test-issue-1509-detect-preset.js node test-live.js + node test-coverage-gate.js + node test-node-reach-coverage.js node test-issue-1107-live-layout.js node test-issue-1532-live-fullscreen.js node test-issue-1619-feed-detail-card-draggable.js @@ -464,6 +466,7 @@ jobs: CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt # #1616: slide-over focus-restore flake-gate. Runs the slide-over diff --git a/.gitignore b/.gitignore index f44fb7368..548c25d66 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ corescope-server cmd/server/server # Local-only planning and design files docs/superpowers/ + +# Environment-specific deploy scripts — live only on the deploy host, not tracked +deploy-live.sh +deploy-staging.sh diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go new file mode 100644 index 000000000..ab670e653 --- /dev/null +++ b/cmd/ingestor/client_reception.go @@ -0,0 +1,223 @@ +package main + +import ( + "log" + "regexp" + "strings" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +// clientPubkeyRe validates the companion pubkey taken from the MQTT topic +// (meshcore/client//packets). A no-ACL broker would let a client +// publish under an arbitrary topic segment (e.g. "!@#$"), so we reject anything +// that is not lowercase hex before it reaches client_receptions/client_observers. +// Mirrors the server-side hexPrefixRe (cmd/server/node_resolve.go). +var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) + +// handleClientPacket processes a packet from the mobile client RX topic +// (meshcore/client/{PUBLIC_KEY}/packets). Unlike observer packets, a roaming +// companion reports WHERE it directly heard a node, so we write a +// client_receptions row and never touch the observers/observations tables. +// rxPubkey is the companion pubkey from the topic (ACL-bound by the broker). +func handleClientPacket(store *Store, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string) { + // The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex + // topic segments so a no-ACL broker can't pollute the coverage tables, and + // never fall back to a payload-supplied id (that would defeat the ACL trust + // model — see docs/client-rx-coverage.md). + rxPubkey = strings.ToLower(strings.TrimSpace(rxPubkey)) + if !clientPubkeyRe.MatchString(rxPubkey) { + log.Printf("MQTT [%s] client: invalid pubkey %.8q, dropping", tag, rxPubkey) + return + } + rawHex, _ := msg["raw"].(string) + if rawHex == "" { + return + } + gps, ok := msg["gps"].(map[string]interface{}) + if !ok { + return // a client packet without a GPS fix is not coverage; drop + } + lat, latOK := toFloat64(gps["lat"]) + lon, lonOK := toFloat64(gps["lon"]) + if !latOK || !lonOK { + return + } + var accPtr *float64 + if acc, ok := toFloat64(gps["acc_m"]); ok { + accPtr = &acc + } + + decoded, err := DecodePacket(rawHex, channelKeys, false) + if err != nil { + log.Printf("MQTT [%s] client decode error: %v", tag, err) + return + } + + direction := "" + if v, ok := msg["direction"].(string); ok { + direction = v + } else if v, ok := msg["Direction"].(string); ok { + direction = v + } + + var snrPtr *float64 + if f, ok := toFloat64(firstPresent(msg, "SNR", "snr")); ok { + snrPtr = &f + } + var rssiPtr *int + if f, ok := toFloat64(firstPresent(msg, "RSSI", "rssi")); ok { + v := int(f) + rssiPtr = &v + } + + rxAt, _ := resolveRxTime(msg, tag) + isAdvert := decoded.Header.PayloadTypeName == "ADVERT" + + rec, ok := buildClientReception( + rxPubkey, + direction, decoded.Header.RouteType, decoded.Path.Hops, decoded.Payload.PubKey, isAdvert, + snrPtr, rssiPtr, lat, lon, accPtr, rxAt, time.Now().UTC().Format(time.RFC3339), + ) + if !ok { + return + } + if _, err := store.InsertClientReception(rec); err != nil { + log.Printf("MQTT [%s] client_reception insert: %v", tag, err) + } + // Remember the companion's self-reported name (sent as "origin") so the + // leaderboard can show a name even if this companion never advertised. + if name := stringField(msg, "origin"); name != "" { + if err := store.UpsertClientObserver(rec.RxPubkey, name, time.Now().UTC().Format(time.RFC3339)); err != nil { + log.Printf("MQTT [%s] client_observer upsert: %v", tag, err) + } + } +} + +// UpsertClientObserver records/updates a mobile client's self-reported name. +// All writes live in the ingestor (read/write invariant #1283). +func (s *Store) UpsertClientObserver(pubkey, name, ts string) error { + if pubkey == "" || name == "" { + return nil + } + _, err := s.db.Exec(` + INSERT INTO client_observers (pubkey, name, last_seen) VALUES (?,?,?) + ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`, + strings.ToLower(pubkey), name, ts) + return err +} + +// firstPresent returns the first present value among the given keys. +func firstPresent(msg map[string]interface{}, keys ...string) interface{} { + for _, k := range keys { + if v, ok := msg[k]; ok { + return v + } + } + return nil +} + +// stringField returns msg[key] as a string, or "" if absent/not a string. +func stringField(msg map[string]interface{}, key string) string { + if v, ok := msg[key].(string); ok { + return v + } + return "" +} + +// ClientReception is one mobile RX coverage point: a companion (RxPubkey) +// directly heard a node (HeardKey) at a GPS position. Hex binning is done +// server-side from Lat/Lon at query time, so no cell id is stored here. +type ClientReception struct { + RxPubkey string + HeardKey string + HeardKeyLen int + RSSI *int + SNR *float64 + Lat float64 + Lon float64 + PosAccM *float64 + RxAt string + IngestedAt string + Src string +} + +// deriveHeardKey applies the RX capture HARD RULE: record only what the +// companion heard itself and directly. +// - direction must be "rx". +// - hops present AND a FLOOD route → the directly-heard node is the LAST hop +// (path[len-1] = the forwarder that just transmitted; each FLOOD forwarder +// appends its hash to the end). 1-byte (2 hex char) prefixes are rejected. +// - hops present on a DIRECT route → NOT attributable: direct forwarders +// consume the next hop from the FRONT (firmware Mesh.cpp removeSelfFromPath), +// so path[len-1] is the route's destination-side end, not who was heard. +// - hops empty + isAdvert → the 0-hop advertiser, by its full pubkey. +// - otherwise → not attributable (ok=false). +// +// Returns (heardKey lowercased, keylenBytes, src, ok). +func deriveHeardKey(direction string, routeType int, hops []string, advertPubkey string, isAdvert bool) (string, int, string, bool) { + if !strings.EqualFold(direction, "rx") { + return "", 0, "", false + } + if len(hops) > 0 { + // FLOOD routes (TRANSPORT_FLOOD 0, FLOOD 1) APPEND each forwarder's hash to + // the END of the path, so path[last] is the immediate RF transmitter. DIRECT + // routes (2, 3) consume the next hop from the FRONT, so path[last] is the + // route's destination-side end, NOT who was heard. + if routeType != packetpath.RouteTransportFlood && routeType != packetpath.RouteFlood { // direct route: path[last] is not the transmitter + return "", 0, "", false + } + last := strings.ToLower(strings.TrimSpace(hops[len(hops)-1])) + keylen := len(last) / 2 + if keylen < 2 { // exclude 1-byte (collision-prone), matching Reach + return "", 0, "", false + } + return last, keylen, "rxlog", true + } + if isAdvert && advertPubkey != "" { + pk := strings.ToLower(strings.TrimSpace(advertPubkey)) + return pk, len(pk) / 2, "advert", true + } + return "", 0, "", false +} + +// buildClientReception validates inputs and assembles a ClientReception, or +// returns ok=false when the packet is not attributable / out of range. +func buildClientReception( + rxPubkey, direction string, routeType int, hops []string, advertPubkey string, isAdvert bool, + snr *float64, rssi *int, lat, lon float64, posAccM *float64, rxAt, ingestedAt string, +) (*ClientReception, bool) { + if rxPubkey == "" || rxAt == "" { + return nil, false + } + if lat < -90 || lat > 90 || lon < -180 || lon > 180 { + return nil, false + } + heardKey, keylen, src, ok := deriveHeardKey(direction, routeType, hops, advertPubkey, isAdvert) + if !ok { + return nil, false + } + return &ClientReception{ + RxPubkey: strings.ToLower(rxPubkey), HeardKey: heardKey, HeardKeyLen: keylen, + RSSI: rssi, SNR: snr, Lat: lat, Lon: lon, PosAccM: posAccM, + RxAt: rxAt, IngestedAt: ingestedAt, Src: src, + }, true +} + +// InsertClientReception writes one coverage row. Idempotent via the +// UNIQUE(rx_pubkey, heard_key, rx_at) constraint; returns ins=false when the +// row already existed. All writes live in the ingestor (read/write invariant #1283). +func (s *Store) InsertClientReception(r *ClientReception) (bool, error) { + res, err := s.db.Exec(` + INSERT INTO client_receptions + (rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src) + VALUES (?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`, + r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil +} diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go new file mode 100644 index 000000000..2bba8f184 --- /dev/null +++ b/cmd/ingestor/client_reception_test.go @@ -0,0 +1,334 @@ +package main + +import ( + "database/sql" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +// TestPruneOldClientReceptions verifies the retention reaper bounds the coverage +// tables: rows older than the window (and stale companion names) are deleted, +// recent ones kept, and days=0 disables it. +func TestPruneOldClientReceptions(t *testing.T) { + s := newTestStore(t) + now := time.Now().UTC() + recent := now.AddDate(0, 0, -1).Format(time.RFC3339) + old := now.AddDate(0, 0, -40).Format(time.RFC3339) + const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" + + s.InsertClientReception(&ClientReception{RxPubkey: testCompanionPK, HeardKey: "aabbcc", HeardKeyLen: 3, Lat: 51, Lon: 3.7, RxAt: recent, IngestedAt: "x", Src: "rxlog"}) + s.InsertClientReception(&ClientReception{RxPubkey: testCompanionPK, HeardKey: "aabbcc", HeardKeyLen: 3, Lat: 51, Lon: 3.7, RxAt: old, IngestedAt: "x", Src: "rxlog"}) + s.UpsertClientObserver(testCompanionPK, "Fresh", recent) + s.UpsertClientObserver(companion2, "Stale", old) + + if n, _ := s.PruneOldClientReceptions(0); n != 0 { + t.Fatalf("days=0 must be a no-op, got %d", n) + } + n, err := s.PruneOldClientReceptions(7) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expected 1 old reception pruned, got %d", n) + } + var recN, obsN int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&recN) + s.db.QueryRow(`SELECT COUNT(*) FROM client_observers`).Scan(&obsN) + if recN != 1 { + t.Fatalf("expected 1 reception remaining (recent), got %d", recN) + } + if obsN != 1 { + t.Fatalf("expected 1 observer remaining (fresh), got %d", obsN) + } +} + +func TestClientReceptionsTableExists(t *testing.T) { + s := newTestStore(t) + cols := map[string]bool{} + rows, err := s.db.Query(`PRAGMA table_info(client_receptions)`) + if err != nil { + t.Fatalf("PRAGMA failed: %v", err) + } + defer rows.Close() + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dflt any + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + t.Fatal(err) + } + cols[name] = true + } + for _, want := range []string{"id", "rx_pubkey", "heard_key", "heard_keylen", "rssi", "snr", "lat", "lon", "pos_acc_m", "rx_at", "ingested_at", "src"} { + if !cols[want] { + t.Errorf("missing column %q in client_receptions", want) + } + } +} + +func crF(f float64) *float64 { return &f } +func crI(i int) *int { return &i } + +// TestClientReceptionsCoverageQueryUsesIndex verifies #5/#18: the dominant +// per-node coverage query (sargable heard_key IN-list + bbox, mirroring +// cmd/server coverageHeardKeyCandidates) seeks the heard_key composite index +// rather than scanning the table. Without idx_client_recept_heard_geo the plan +// is "SCAN client_receptions". +func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { + s := newTestStore(t) + q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + if !strings.Contains(plan, "USING INDEX idx_client_recept") { + t.Fatalf("coverage query should use a client_recept index, plan was:\n%s", plan) + } + if strings.Contains(plan, "SCAN client_receptions") { + t.Fatalf("coverage query should not full-scan, plan was:\n%s", plan) + } +} + +// TestClientReceptionsRetentionUsesRxAtIndex verifies the retention reaper's +// DELETE ... WHERE rx_at < ? (and the leaderboard's rx_at window) seek the rx_at +// index rather than full-scanning under the writer lock (polish review). +func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { + s := newTestStore(t) + rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + if !strings.Contains(plan, "idx_client_recept_rxat") { + t.Fatalf("retention DELETE should use idx_client_recept_rxat, plan was:\n%s", plan) + } +} + +// TestRxLeaderboardQueryIsIndexBacked pins the planner choice for the leaderboard +// SELECT (the rx_at-windowed, rx_pubkey-grouped query in cmd/server/rx_dashboard.go). +// SQLite serves it from the UNIQUE(rx_pubkey,heard_key,rx_at) constraint index as a +// COVERING scan (not idx_client_recept_rxat, and not a table-heap scan). The table +// is retention-bounded, so a covering scan is acceptable; this test guards against a +// silent regression to a bare table scan under the writer lock when the schema is +// next tweaked. Representative form (no JOINs — they don't change whether `cr` is +// index-backed). +func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { + s := newTestStore(t) + rows, err := s.db.Query(`EXPLAIN QUERY PLAN + SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) + FROM client_receptions cr + WHERE cr.rx_at >= ? + GROUP BY cr.rx_pubkey + ORDER BY COUNT(*) DESC + LIMIT ?`, "2026-01-01T00:00:00Z", 100) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + t.Logf("leaderboard plan:\n%s", plan) + // The concern is a bare table-heap scan, not which specific index wins. The + // plan must stay index-backed (covering or search) — a regression to a bare + // "SCAN cr" without an index fails here. + if !strings.Contains(plan, "INDEX") { + t.Fatalf("leaderboard SELECT must stay index-backed (no full table-heap scan), plan was:\n%s", plan) + } +} + +func TestDeriveHeardKey(t *testing.T) { + full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, nil, strings.ToUpper(full), true) + if !ok || l != 32 || src != "advert" || k != full { + t.Fatalf("0-hop advert: got k=%q l=%d src=%q ok=%v", k, l, src, ok) + } + k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, []string{"aa", "bbccdd"}, "", false) + if !ok || k != "bbccdd" || l != 3 || src != "rxlog" { + t.Fatalf("flood path: got k=%q l=%d src=%q ok=%v", k, l, src, ok) + } + // DIRECT route: path[last] is the route's far end, not the transmitter — must be rejected. + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteDirect, []string{"aa", "bbccdd"}, "", false); ok { + t.Fatalf("direct-route path must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteTransportDirect, []string{"aa", "bbccdd"}, "", false); ok { + t.Fatalf("transport-direct-route path must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, []string{"aa", "bb"}, "", false); ok { + t.Fatalf("1-byte last hop should be rejected") + } + if _, _, _, ok = deriveHeardKey("tx", packetpath.RouteFlood, []string{"aabbcc"}, "", false); ok { + t.Fatalf("tx must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, nil, "", false); ok { + t.Fatalf("no hops + non-advert must be rejected") + } +} + +func TestBuildClientReception(t *testing.T) { + acc := 8.0 + rec, ok := buildClientReception("companionpk", "rx", packetpath.RouteFlood, []string{"aa", "bbccdd"}, "", false, + crF(-7.5), crI(-92), 51.05, 3.72, &acc, "2026-06-09T12:00:00Z", "2026-06-09T12:00:01Z") + if !ok || rec.HeardKey != "bbccdd" || rec.HeardKeyLen != 3 || rec.Src != "rxlog" { + t.Fatalf("bad reception: %+v ok=%v", rec, ok) + } + if _, ok := buildClientReception("c", "rx", packetpath.RouteDirect, []string{"bbccdd"}, "", false, + crF(-7.5), crI(-92), 51.05, 3.72, nil, "t", "t"); ok { + t.Fatal("direct-route path must be rejected (not the transmitter)") + } + if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, []string{"bbccdd"}, "", false, nil, nil, 99.0, 3.72, nil, "t", "t"); ok { + t.Fatal("out-of-range lat must be rejected") + } +} + +func TestInsertClientReceptionRoundTripAndIdempotent(t *testing.T) { + s := newTestStore(t) + rec := &ClientReception{ + RxPubkey: "companionpk", HeardKey: "bbccdd", HeardKeyLen: 3, RSSI: crI(-92), + Lat: 51.05, Lon: 3.72, RxAt: "2026-06-09T12:00:00Z", IngestedAt: "2026-06-09T12:00:01Z", Src: "rxlog", + } + if ins, err := s.InsertClientReception(rec); err != nil || !ins { + t.Fatalf("first insert: ins=%v err=%v", ins, err) + } + if ins, err := s.InsertClientReception(rec); err != nil || ins { + t.Fatalf("second insert should be a no-op: ins=%v err=%v", ins, err) + } + var n int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n) + if n != 1 { + t.Fatalf("expected 1 row, got %d", n) + } +} + +func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { + s := newTestStore(t) + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + msg := map[string]interface{}{ + "raw": advertHex, + "direction": "rx", + "timestamp": "2026-06-09T12:00:00Z", + "origin": "MyMob", + "SNR": -7.0, + "RSSI": -92.0, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientPacket(s, "test", testCompanionPK, msg, nil) + + var obsName string + s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=?`, testCompanionPK).Scan(&obsName) + if obsName != "MyMob" { + t.Fatalf("expected client_observers name 'MyMob', got %q", obsName) + } + + // This fixture is a relayed advert (non-empty path), so by the capture HARD + // RULE we record the directly-heard LAST hop (multibyte), not the originator. + // The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey. + var n, keylen int + var src string + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { + t.Fatal(err) + } + if n != 1 || keylen < 2 || src != "rxlog" { + t.Fatalf("expected 1 rxlog reception (multibyte last hop), got n=%d keylen=%d src=%q", n, keylen, src) + } + + // No GPS → no row. + const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" + handleClientPacket(s, "test", companion2, map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil) + var n2 int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=?`, companion2).Scan(&n2) + if n2 != 0 { + t.Fatalf("packet without gps must be dropped, got %d rows", n2) + } +} + +// TestHandleClientPacketZeroHopAdvertWritesReception covers the #9 gap: the +// advert fixture used above is a RELAYED advert (non-empty path), so it exercises +// the rxlog last-hop branch, not the 0-hop src='advert' branch. Here we rebuild +// the same advert with zero hops — header (FLOOD ADVERT) + "00" (0 hops) + the +// same advert payload — so handleClientPacket stores the advertiser by its full +// pubkey with src='advert', and we assert gps/snr were captured too. +func TestHandleClientPacketZeroHopAdvertWritesReception(t *testing.T) { + s := newTestStore(t) + relayed := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + // relayed = header(2) + path-descriptor(2) + 5*2-byte hops(20) + payload. + payload := relayed[24:] + zeroHop := "1100" + payload + advertPubkey := strings.ToLower(payload[:64]) // advert payload starts with the 32-byte pubkey + + msg := map[string]interface{}{ + "raw": zeroHop, "direction": "rx", "timestamp": "2026-06-09T12:00:00Z", + "origin": "MyMob", "SNR": -7.0, "RSSI": -92.0, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientPacket(s, "test", testCompanionPK, msg, nil) + + var heardKey, src string + var keylen int + var snr sql.NullFloat64 + var lat, lon float64 + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK). + Scan(&heardKey, &keylen, &src, &snr, &lat, &lon); err != nil { + t.Fatalf("expected a 0-hop advert reception: %v", err) + } + if src != "advert" || keylen != 32 || heardKey != advertPubkey { + t.Fatalf("0-hop advert: want advert/32/%s, got %s/%d/%s", advertPubkey, src, keylen, heardKey) + } + if !snr.Valid || snr.Float64 != -7 || lat != 51.05 || lon != 3.72 { + t.Fatalf("gps/snr not captured: snr=%v lat=%f lon=%f", snr, lat, lon) + } +} + +// TestHandleClientPacketRejectsNonHexPubkey verifies the #2 fix: a companion +// pubkey from the topic that isn't lowercase hex (a no-ACL broker could publish +// meshcore/client/!@#$/packets) writes nothing to either coverage table. Without +// the clientPubkeyRe guard this fixture would insert a polluting row. +func TestHandleClientPacketRejectsNonHexPubkey(t *testing.T) { + s := newTestStore(t) + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + for _, bad := range []string{"!@#$", "companionpk", "", "g0g0", "xyz"} { + msg := map[string]interface{}{ + "raw": advertHex, "direction": "rx", "timestamp": "2026-06-09T12:00:00Z", + "origin": "Spoof", "SNR": -7.0, "RSSI": -92.0, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientPacket(s, "test", bad, msg, nil) + } + var nRecept, nObs int + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&nRecept) + s.db.QueryRow(`SELECT COUNT(*) FROM client_observers`).Scan(&nObs) + if nRecept != 0 || nObs != 0 { + t.Fatalf("non-hex pubkey must write nothing, got %d receptions, %d observers", nRecept, nObs) + } +} diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 10e81195f..be1d55f4f 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -43,21 +43,22 @@ type MQTTLegacy struct { // Config holds the ingestor configuration, compatible with the Node.js config.json format. type Config struct { - DBPath string `json:"dbPath"` - MQTT *MQTTLegacy `json:"mqtt,omitempty"` - MQTTSources []MQTTSource `json:"mqttSources,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - ChannelKeysPath string `json:"channelKeysPath,omitempty"` - ChannelKeys map[string]string `json:"channelKeys,omitempty"` - HashChannels []string `json:"hashChannels,omitempty"` - HashRegions []string `json:"hashRegions,omitempty"` - Retention *RetentionConfig `json:"retention,omitempty"` - Metrics *MetricsConfig `json:"metrics,omitempty"` - Runtime *RuntimeConfig `json:"runtime,omitempty"` - GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` - ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"` - ValidateSignatures *bool `json:"validateSignatures,omitempty"` - DB *DBConfig `json:"db,omitempty"` + DBPath string `json:"dbPath"` + MQTT *MQTTLegacy `json:"mqtt,omitempty"` + MQTTSources []MQTTSource `json:"mqttSources,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + ChannelKeysPath string `json:"channelKeysPath,omitempty"` + ChannelKeys map[string]string `json:"channelKeys,omitempty"` + HashChannels []string `json:"hashChannels,omitempty"` + HashRegions []string `json:"hashRegions,omitempty"` + Retention *RetentionConfig `json:"retention,omitempty"` + Metrics *MetricsConfig `json:"metrics,omitempty"` + Runtime *RuntimeConfig `json:"runtime,omitempty"` + ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"` + GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` + ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"` + ValidateSignatures *bool `json:"validateSignatures,omitempty"` + DB *DBConfig `json:"db,omitempty"` // ObserverIATAWhitelist restricts which observer IATA regions are processed. // When non-empty, only observers whose IATA code (from the MQTT topic) matches @@ -128,6 +129,17 @@ func (f *ForeignAdvertConfig) IsDropMode() bool { return strings.EqualFold(strings.TrimSpace(f.Mode), "drop") } +// ClientRxCoverageConfig controls the opt-in mobile client-RX coverage feature. +type ClientRxCoverageConfig struct { + Enabled bool `json:"enabled"` +} + +// ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage +// feature is on. Absent/nil ⇒ off (the safe default). +func (c *Config) ClientRxCoverageEnabled() bool { + return c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled +} + // RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes. type RetentionConfig struct { NodeDays int `json:"nodeDays"` @@ -136,6 +148,10 @@ type RetentionConfig struct { // PacketDays is the retention window for transmissions (#1283). // Ownership moved from cmd/server to cmd/ingestor; 0 disables. PacketDays int `json:"packetDays"` + // ClientRxDays is the retention window (by rx_at) for mobile client-RX + // coverage rows in client_receptions / client_observers; 0 disables. Bounds + // the table the opt-in coverage feature would otherwise grow without limit. + ClientRxDays int `json:"clientRxDays"` } // PacketDaysOrZero returns the configured retention.packetDays or 0 @@ -147,6 +163,15 @@ func (c *Config) PacketDaysOrZero() int { return 0 } +// ClientRxDaysOrZero returns the configured retention.clientRxDays or 0 +// (disabled) if not set. +func (c *Config) ClientRxDaysOrZero() int { + if c.Retention != nil && c.Retention.ClientRxDays > 0 { + return c.Retention.ClientRxDays + } + return 0 +} + // MetricsConfig controls observer metrics collection. type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go new file mode 100644 index 000000000..a3f799df0 --- /dev/null +++ b/cmd/ingestor/coverage_gate_test.go @@ -0,0 +1,88 @@ +package main + +import "testing" + +// testCompanionPK is a valid lowercase-hex companion pubkey for coverage tests. +// The topic segment must be hex (clientPubkeyRe) or handleClientPacket drops it. +const testCompanionPK = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2" + +// clientCoverageMsg builds a valid mobile client-RX coverage message on the +// dedicated topic meshcore/client//packets. The raw hex is a relayed +// advert with GPS, so handleClientPacket would write exactly one +// client_receptions row when the feature is enabled (see +// TestHandleClientPacketAdvertWritesReception). +func clientCoverageMsg() *mockMessage { + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + payload := []byte(`{"raw":"` + advertHex + `","direction":"rx","timestamp":"2026-06-09T12:00:00Z","origin":"MyMob","SNR":-7.0,"RSSI":-92.0,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`) + return &mockMessage{topic: "meshcore/client/" + testCompanionPK + "/packets", payload: payload} +} + +func clientReceptionCount(t *testing.T, s *Store) int { + t.Helper() + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// TestClientRxCoverageEnabledDefault verifies the gate helper defaults OFF for +// nil/absent config and is only true when explicitly enabled. +func TestClientRxCoverageEnabledDefault(t *testing.T) { + if (&Config{}).ClientRxCoverageEnabled() { + t.Fatal("nil ClientRxCoverage must report disabled") + } + if (&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: false}}).ClientRxCoverageEnabled() { + t.Fatal("Enabled:false must report disabled") + } + if !(&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}).ClientRxCoverageEnabled() { + t.Fatal("Enabled:true must report enabled") + } +} + +// TestClientRxCoverageGateOff drives handleMessage with the feature OFF: the +// client-topic message must fall through and write no client_receptions rows. +func TestClientRxCoverageGateOff(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{} // ClientRxCoverage nil ⇒ disabled + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 0 { + t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n) + } +} + +// TestClientRxCoverageGateOn drives handleMessage with the feature ON: the +// client-topic message must be dispatched and write exactly one row. +func TestClientRxCoverageGateOn(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}} + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 1 { + t.Fatalf("feature ON: expected 1 client_receptions row, got %d", n) + } +} + +// TestClientRxCoverageBlacklistedDropped verifies the #1 fix: a blacklisted +// operator cannot skirt the observer blacklist via the client topic. With the +// feature ON but the companion pubkey blacklisted, no row is written. Without +// the gate the client dispatch runs before the blacklist check and inserts. +func TestClientRxCoverageBlacklistedDropped(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ + ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}, + ObserverBlacklist: []string{testCompanionPK}, + } + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 0 { + t.Fatalf("blacklisted companion: expected 0 client_receptions rows, got %d", n) + } +} diff --git a/cmd/ingestor/coverage_query_bench_test.go b/cmd/ingestor/coverage_query_bench_test.go new file mode 100644 index 000000000..078ec8643 --- /dev/null +++ b/cmd/ingestor/coverage_query_bench_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "fmt" + "math/rand" + "strings" + "testing" +) + +// coverageBenchSQL is the dominant per-node coverage query (mirrors +// cmd/server queryCoverageRows): a bbox range plus a full-key/2-3-byte-prefix +// match on the heard node. +const coverageBenchSQL = `SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? + AND ( (heard_keylen = 32 AND heard_key = ?) + OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` + +// BenchmarkCoverageQuery seeds ~1M receptions across a metro-area bbox and times +// the coverage query with the indexes (#5/#18) versus a forced full table scan. +// Run: go test -run x -bench BenchmarkCoverageQuery -benchtime 20x ./cmd/ingestor +func BenchmarkCoverageQuery(b *testing.B) { + const n = 1_000_000 + const prefixPool = 2000 // distinct 3-byte heard_key prefixes + + dir := b.TempDir() + s, err := OpenStore(dir + "/bench.db") + if err != nil { + b.Fatal(err) + } + defer s.Close() + + rng := rand.New(rand.NewSource(1)) + tx, err := s.db.Begin() + if err != nil { + b.Fatal(err) + } + stmt, err := tx.Prepare(`INSERT INTO client_receptions + (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES (?,?,?,?,?,?,?,?,?)`) + if err != nil { + b.Fatal(err) + } + for i := 0; i < n; i++ { + hk := fmt.Sprintf("%06x", rng.Intn(prefixPool)) + lat := 51.0 + rng.Float64()*0.4 // ~44 km metro span + lon := 3.5 + rng.Float64()*0.4 + rxpk := fmt.Sprintf("%064x", rng.Intn(500)) + // rx_at carries i so (rx_pubkey,heard_key,rx_at) stays unique. + if _, err := stmt.Exec(rxpk, hk, 3, -6.0, lat, lon, fmt.Sprintf("t%d", i), "x", "rxlog"); err != nil { + b.Fatal(err) + } + } + stmt.Close() + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + + // Target node whose 3-byte prefix (0003e8 = 1000) is in the pool, queried + // over a sub-bbox of the metro area. + target := "0003e8" + strings.Repeat("ab", 29) // 6 + 58 = 64 hex + + // OR/substr query (original shape): bbox range OR'd with a non-sargable + // substr prefix match. + runOR := func(b *testing.B) { + for i := 0; i < b.N; i++ { + rows, err := s.db.Query(coverageBenchSQL, 51.1, 51.3, 3.6, 3.8, target, target) + if err != nil { + b.Fatal(err) + } + for rows.Next() { + } + rows.Close() + } + } + + // IN-list query (sargable): the heard node's candidate keys are exactly the + // full pubkey and its 2/3-byte prefixes, so an IN-list seeks them via the + // heard_key-leading composite instead of scanning the bbox. + inListSQL := `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions + WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + runIN := func(b *testing.B) { + for i := 0; i < b.N; i++ { + rows, err := s.db.Query(inListSQL, target, target[:4], target[:6], 51.1, 51.3, 3.6, 3.8) + if err != nil { + b.Fatal(err) + } + for rows.Next() { + } + rows.Close() + } + } + + b.Run("or_query_indexed", runOR) + b.Run("inlist_query_indexed", runIN) + + // Drop the coverage indexes to measure the full-scan baseline. + for _, idx := range []string{"idx_client_recept_heard_geo", "idx_client_recept_latlon", "idx_client_recept_rxpk"} { + if _, err := s.db.Exec("DROP INDEX IF EXISTS " + idx); err != nil { + b.Fatal(err) + } + } + b.Run("or_query_table_scan", runOR) +} diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index a5d54fa03..58cece264 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -271,6 +271,50 @@ func applySchema(db *sql.DB) error { -- the last_seen column exists (#1690) — keep it OUT of this base -- schema block so legacy DBs (table-exists, column-missing) don't -- trip on the CREATE INDEX before the ALTER runs. + + -- Mobile client RX coverage: a roaming companion = a mobile observer + -- with a moving GPS position, so it gets its own table rather than + -- observations (which assumes a fixed observer/location). + CREATE TABLE IF NOT EXISTS client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + heard_key TEXT NOT NULL, + heard_keylen INTEGER NOT NULL, + rssi INTEGER, + snr REAL, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + rx_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + src TEXT NOT NULL, + UNIQUE(rx_pubkey, heard_key, rx_at) + ); + -- Coverage queries filter by bbox AND match the heard node either by full + -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite + -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and + -- carries lat/lon so the bbox range is satisfied from the index; it also + -- supersedes the old single-column heard_key index. idx_client_recept_latlon + -- lets the planner instead drive from a selective bbox. (#5, #18) + CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); + CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); + -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the + -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per + -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without + -- this index either would full-scan the table under the writer lock + -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays + -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. + CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); + DROP INDEX IF EXISTS idx_client_recept_rxpk; + + -- Self-reported name of each mobile client (companion), from the SELF_INFO + -- name the app sends as "origin". Lets the leaderboard show a name even + -- when the companion never advertised (so it isn't in the nodes table). + CREATE TABLE IF NOT EXISTS client_observers ( + pubkey TEXT PRIMARY KEY, + name TEXT, + last_seen TEXT + ); ` if _, err := db.Exec(schema); err != nil { return fmt.Errorf("base schema: %w", err) @@ -1703,7 +1747,6 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, return pd } - // ─── Writer-lock instrumentation (issue #1340) ──────────────────────────── // // Make SQLite writer-lock starvation visible to operators. Per-component diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index a623da6f6..6176e3273 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -273,6 +273,18 @@ func main() { } } + // Client-RX coverage retention: bound the opt-in coverage tables (#1727). + // Independent of the feature flag, so data persists are reaped even after + // the feature is turned off. 0 = disabled. + clientRxDays := cfg.ClientRxDaysOrZero() + if clientRxDays > 0 { + if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil { + log.Printf("[prune] error: %v", err) + } else if n > 0 { + log.Printf("[prune] startup pruned %d client_receptions older than %d days", n, clientRxDays) + } + } + vacuumPages := cfg.IncrementalVacuumPages() store.RunIncrementalVacuum(vacuumPages) @@ -335,6 +347,21 @@ func main() { log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", packetDays) } + // Daily ticker for client-RX coverage retention (#1727). + if clientRxDays > 0 { + clientRxRetentionTicker := time.NewTicker(24 * time.Hour) + go func() { + for range clientRxRetentionTicker.C { + if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil { + log.Printf("[prune] error: %v", err) + } else if n > 0 { + store.RunIncrementalVacuum(vacuumPages) + } + } + }() + log.Printf("[prune] auto-prune enabled: client_receptions older than %d days will be removed daily", clientRxDays) + } + // Hourly WAL checkpoint to prevent unbounded WAL growth. // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; // if the server's read connection holds frames, remaining pages stay in the @@ -535,6 +562,21 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, return } + // Mobile client RX coverage: dedicated topic meshcore/client/{PUBLIC_KEY}/packets. + // A roaming companion reports where it directly heard a node; handled in isolation + // from the observer/observations path. EMQX ACL binds parts[2] to the client's own key. + if cfg.ClientRxCoverageEnabled() && len(parts) >= 4 && parts[1] == "client" && parts[3] == "packets" { + // The observer blacklist (checked below) only runs on the observer path, + // so a blacklisted operator could otherwise skirt it via the client topic + // (#1). Enforce it here before any coverage write. + if cfg.IsObserverBlacklisted(parts[2]) { + log.Printf("MQTT [%s] client %.8s blacklisted, dropping", tag, parts[2]) + return + } + handleClientPacket(store, tag, parts[2], msg, channelKeys) + return + } + // Skip status/connection topics if topic == "meshcore/status" || topic == "meshcore/events/connection" { return diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index 2b978bdcd..fea189fe5 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -48,6 +48,39 @@ func (s *Store) PruneOldPackets(days int) (int64, error) { return n, nil } +// PruneOldClientReceptions deletes mobile client-RX coverage rows older than +// `days` (by rx_at), and client_observers (companion names) whose last_seen has +// aged out. This bounds the otherwise-unbounded client_receptions table the +// opt-in coverage feature feeds. 0 disables. Owned by the ingestor writer +// (#1283). Returns the number of client_receptions rows deleted. +func (s *Store) PruneOldClientReceptions(days int) (int64, error) { + if days <= 0 { + return 0, nil + } + cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + + var n int64 + err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error { + res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < ?`, cutoff) + if err != nil { + return fmt.Errorf("prune client_receptions: %w", err) + } + n, _ = res.RowsAffected() + // Drop companion name rows not refreshed within the window. + if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < ?`, cutoff); err != nil { + return fmt.Errorf("prune client_observers: %w", err) + } + return nil + }) + if err != nil { + return 0, err + } + if n > 0 { + log.Printf("[prune] deleted %d client_receptions older than %d days", n, days) + } + return n, nil +} + // SoftDeleteBlacklistedObservers marks observers in the blacklist as // inactive=1 so they are hidden from API responses. Owned by ingestor // per #1287. Runs once at startup. diff --git a/cmd/server/config.go b/cmd/server/config.go index 2e1e4d489..da05686dd 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -133,7 +133,7 @@ type Config struct { // Currently exposes runtime.maxMemoryMB which sets a soft memory limit // (GOMEMLIMIT) via runtime/debug.SetMemoryLimit at startup. The // GOMEMLIMIT environment variable, when set, takes precedence. - Runtime *RuntimeConfig `json:"runtime,omitempty"` + Runtime *RuntimeConfig `json:"runtime,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` Areas map[string]AreaEntry `json:"areas,omitempty"` @@ -160,7 +160,13 @@ type Config struct { obsBlacklistSetCached map[string]bool obsBlacklistOnce sync.Once - Compression *CompressionConfig `json:"compression,omitempty"` + Compression *CompressionConfig `json:"compression,omitempty"` + + // ClientRxCoverage gates the opt-in mobile client-RX coverage feature + // (corescope-rx companions publishing GPS-tagged receptions). Absent/nil + // ⇒ off; see ClientRxCoverageEnabled. + ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"` + ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"` NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"` @@ -250,6 +256,17 @@ func (c *Config) GZipEnabled() bool { return c.Compression != nil && c.Compression.GZip } +// ClientRxCoverageConfig gates the opt-in mobile client-RX coverage feature. +type ClientRxCoverageConfig struct { + Enabled bool `json:"enabled"` +} + +// ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage +// feature is on. Nil config or absent/nil section ⇒ off (the safe default). +func (c *Config) ClientRxCoverageEnabled() bool { + return c != nil && c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled +} + // WSCompressionEnabled returns true when WebSocket permessage-deflate is explicitly enabled. func (c *Config) WSCompressionEnabled() bool { return c.Compression != nil && c.Compression.Websocket @@ -303,10 +320,10 @@ type RuntimeConfig struct { } type RetentionConfig struct { - NodeDays int `json:"nodeDays"` - ObserverDays int `json:"observerDays"` - PacketDays int `json:"packetDays"` - MetricsDays int `json:"metricsDays"` + NodeDays int `json:"nodeDays"` + ObserverDays int `json:"observerDays"` + PacketDays int `json:"packetDays"` + MetricsDays int `json:"metricsDays"` } // DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). @@ -601,7 +618,6 @@ func (c *Config) ResolveDBPath(baseDir string) string { return filepath.Join(baseDir, "data", "meshcore.db") } - func (c *Config) NormalizeTimestampConfig() { defaults := defaultTimestampConfig() if c.Timestamps == nil { @@ -908,10 +924,11 @@ func (c *Config) IsObserverBlacklisted(id string) bool { // data slowly." Lower values give fresher data at higher CPU cost. // // RecomputeIntervalSeconds keys (all optional): -// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew +// +// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew type AnalyticsConfig struct { - DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"` - RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"` + DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"` + RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"` } // AnalyticsDefaultRecomputeInterval returns the configured default diff --git a/cmd/server/coverage_gate_test.go b/cmd/server/coverage_gate_test.go new file mode 100644 index 000000000..ed3192e8c --- /dev/null +++ b/cmd/server/coverage_gate_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" +) + +// gateTestServer builds a minimal *Server with the given client-RX coverage +// config and registers all routes, mirroring routes_test.go's setup but +// skipping the packet store (none of the gated routes need it for the 404 / +// registration assertions below). +func gateTestServer(t *testing.T, cov *ClientRxCoverageConfig) *mux.Router { + t.Helper() + db := setupTestDB(t) + cfg := &Config{Port: 3000, ClientRxCoverage: cov} + hub := NewHub() + srv := NewServer(db, cfg, hub) + router := mux.NewRouter() + srv.RegisterRoutes(router) + return router +} + +func TestCoverageRoutesGatedOff(t *testing.T) { + router := gateTestServer(t, nil) + + req := httptest.NewRequest("GET", "/api/rx-coverage", nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusNotFound { + t.Fatalf("expected 404 for /api/rx-coverage when disabled, got %d", rr.Code) + } + + creq := httptest.NewRequest("GET", "/api/config/client", nil) + crr := httptest.NewRecorder() + router.ServeHTTP(crr, creq) + if crr.Code != http.StatusOK { + t.Fatalf("config/client status %d body %s", crr.Code, crr.Body.String()) + } + if !strings.Contains(crr.Body.String(), `"clientRxCoverage":false`) { + t.Fatalf("expected clientRxCoverage:false in config body, got %s", crr.Body.String()) + } +} + +func TestCoverageRoutesGatedOn(t *testing.T) { + router := gateTestServer(t, &ClientRxCoverageConfig{Enabled: true}) + + req := httptest.NewRequest("GET", "/api/rx-coverage", nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code == http.StatusNotFound { + t.Fatalf("expected /api/rx-coverage to be registered when enabled, got 404") + } + + creq := httptest.NewRequest("GET", "/api/config/client", nil) + crr := httptest.NewRecorder() + router.ServeHTTP(crr, creq) + if crr.Code != http.StatusOK { + t.Fatalf("config/client status %d body %s", crr.Code, crr.Body.String()) + } + if !strings.Contains(crr.Body.String(), `"clientRxCoverage":true`) { + t.Fatalf("expected clientRxCoverage:true in config body, got %s", crr.Body.String()) + } +} diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 1369553cc..8dc83153f 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -30,7 +30,7 @@ func setupTestDBv2(t *testing.T) *DB { CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL + battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0 ); CREATE TABLE observers ( id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, diff --git a/cmd/server/hexgrid.go b/cmd/server/hexgrid.go new file mode 100644 index 000000000..93c25f3b7 --- /dev/null +++ b/cmd/server/hexgrid.go @@ -0,0 +1,119 @@ +package main + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// Pure-Go hexagonal binning for RX coverage display. We deliberately avoid the +// CGO-based uber/h3-go (this project builds with CGO_ENABLED=0). Points are +// projected to Web Mercator and snapped to a pointy-top hex grid whose size +// depends on the display resolution. Cell ids are "res:q:r" (axial coords). +// At city/region scale this looks like H3/mapme.sh coverage without any deps. + +const hexEarthRadius = 6378137.0 // Web Mercator sphere radius (m) + +// hexTargetPx is the desired on-screen hex size (point-to-point height) in CSS +// pixels. mercUPPZ0 is Web Mercator units per pixel at zoom 0 (world span / 256); +// Leaflet halves it each zoom level, independent of latitude. Sizing the hex in +// these units therefore renders it at a constant ~hexTargetPx at every zoom — the +// old fixed-meter buckets looked like specks when zoomed out (issue: hexes too small). +const hexTargetPx = 28.0 +const mercUPPZ0 = 156543.03392 + +func hexMercator(lat, lon float64) (float64, float64) { + x := hexEarthRadius * lon * math.Pi / 180 + y := hexEarthRadius * math.Log(math.Tan(math.Pi/4+lat*math.Pi/360)) + return x, y +} + +func hexInvMercator(x, y float64) (lat, lon float64) { + lon = x / hexEarthRadius * 180 / math.Pi + lat = (2*math.Atan(math.Exp(y/hexEarthRadius)) - math.Pi/2) * 180 / math.Pi + return lat, lon +} + +// hexSizeForRes is the hex circumradius (center→corner) in Web Mercator units for a +// display resolution. Resolution equals the Leaflet zoom level (see zoomToHexRes), so +// the size scales as 2^-zoom and the hex keeps a constant ~hexTargetPx on-screen size +// regardless of zoom. hexCellAt (binning) and hexBoundary (drawing) both read this, so +// they stay consistent for a given cell id. +func hexSizeForRes(res int) float64 { + return (hexTargetPx / 2) * mercUPPZ0 / math.Pow(2, float64(res)) +} + +// hexMaxLat is the Web Mercator latitude limit. The projection (hexMercator) +// diverges toward ±90° — tan(π/4 + lat·π/360) → ∞ — so points beyond this would +// produce NaN cell rings via hexInvMercator. Coverage is therefore only defined +// within ±hexMaxLat; polar submissions are clamped to the edge (#17). +const hexMaxLat = 85.05112878 + +// hexCellAt returns a stable cell id ("res:q:r") for the lat/lon at res. Latitude +// is clamped to ±hexMaxLat so near-polar points bin to the edge instead of +// producing NaN geometry. +func hexCellAt(lat, lon float64, res int) string { + if lat > hexMaxLat { + lat = hexMaxLat + } else if lat < -hexMaxLat { + lat = -hexMaxLat + } + size := hexSizeForRes(res) + x, y := hexMercator(lat, lon) + q := (math.Sqrt(3)/3*x - 1.0/3*y) / size + r := (2.0 / 3 * y) / size + qi, ri := hexRound(q, r) + return fmt.Sprintf("%d:%d:%d", res, qi, ri) +} + +// hexRound rounds fractional axial coords to the nearest hex via cube rounding. +func hexRound(q, r float64) (int, int) { + x, z := q, r + y := -x - z + rx, ry, rz := math.Round(x), math.Round(y), math.Round(z) + dx, dy, dz := math.Abs(rx-x), math.Abs(ry-y), math.Abs(rz-z) + switch { + case dx > dy && dx > dz: + rx = -ry - rz + case dy > dz: + ry = -rx - rz + default: + rz = -rx - ry + } + return int(rx), int(rz) +} + +// hexBoundary returns the cell's 6 corners as a closed [lon,lat] ring (GeoJSON +// order), or nil if the cell id is malformed. +func hexBoundary(cellID string) [][2]float64 { + res, q, r, ok := parseHexCell(cellID) + if !ok { + return nil + } + size := hexSizeForRes(res) + cx := size * (math.Sqrt(3)*float64(q) + math.Sqrt(3)/2*float64(r)) + cy := size * (1.5 * float64(r)) + ring := make([][2]float64, 0, 7) + for i := 0; i < 6; i++ { + ang := math.Pi / 180 * float64(60*i-30) + lat, lon := hexInvMercator(cx+size*math.Cos(ang), cy+size*math.Sin(ang)) + ring = append(ring, [2]float64{lon, lat}) + } + ring = append(ring, ring[0]) // close the ring + return ring +} + +func parseHexCell(id string) (res, q, r int, ok bool) { + p := strings.Split(id, ":") + if len(p) != 3 { + return 0, 0, 0, false + } + a, e1 := strconv.Atoi(p[0]) + b, e2 := strconv.Atoi(p[1]) + c, e3 := strconv.Atoi(p[2]) + if e1 != nil || e2 != nil || e3 != nil { + return 0, 0, 0, false + } + return a, b, c, true +} diff --git a/cmd/server/hexgrid_test.go b/cmd/server/hexgrid_test.go new file mode 100644 index 000000000..f8938436c --- /dev/null +++ b/cmd/server/hexgrid_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "math" + "testing" +) + +// TestHexCellAtClampsPolarLatitude verifies #17: latitudes past the Web Mercator +// limit are clamped, so near-polar submissions bin to the edge cell and produce +// finite geometry instead of NaN rings. +func TestHexCellAtClampsPolarLatitude(t *testing.T) { + for _, lat := range []float64{89.9, 90.0, -89.9, -90.0} { + cell := hexCellAt(lat, 3.72, 9) + clamped := math.Copysign(hexMaxLat, lat) + if want := hexCellAt(clamped, 3.72, 9); cell != want { + t.Fatalf("lat %.1f should clamp to %q, got %q", lat, want, cell) + } + ring := hexBoundary(cell) + if ring == nil { + t.Fatalf("lat %.1f produced no ring", lat) + } + for _, pt := range ring { + if math.IsNaN(pt[0]) || math.IsNaN(pt[1]) || math.IsInf(pt[0], 0) || math.IsInf(pt[1], 0) { + t.Fatalf("lat %.1f produced non-finite ring point %v", lat, pt) + } + } + } +} + +func TestHexCellAtStableAndDistinct(t *testing.T) { + a := hexCellAt(51.0500, 3.7200, 9) + b := hexCellAt(51.0500, 3.7200, 9) + if a == "" || a != b { + t.Fatalf("stable cell expected, got %q %q", a, b) + } + c := hexCellAt(51.2000, 3.7200, 9) // ~17 km away + if c == a { + t.Fatalf("distant point should differ, both %q", a) + } +} + +func TestHexBoundaryClosedRing(t *testing.T) { + cell := hexCellAt(51.05, 3.72, 9) + ring := hexBoundary(cell) + if len(ring) != 7 { + t.Fatalf("expected 7 points (closed hex), got %d", len(ring)) + } + if ring[0] != ring[6] { + t.Fatalf("ring not closed: %v vs %v", ring[0], ring[6]) + } + if hexBoundary("garbage") != nil { + t.Fatalf("malformed cell should return nil") + } +} diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go new file mode 100644 index 000000000..bf5bd5ff6 --- /dev/null +++ b/cmd/server/node_resolve.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/json" + "net/http" + "regexp" + "strings" +) + +// ResolvePrefixResp is the tiny reply for /api/nodes/resolve — lets a client +// resolve a heard 2-3 byte path prefix (or full pubkey) to a node name without +// fetching the whole node list. Read-only. +type ResolvePrefixResp struct { + Prefix string `json:"prefix"` + Pubkey string `json:"pubkey,omitempty"` + Name string `json:"name,omitempty"` + Ambiguous bool `json:"ambiguous"` +} + +var hexPrefixRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) + +// minResolvePrefixHex is the shortest accepted prefix. 1-byte (2 hex) keys are +// never stored — the ingestor rejects heard keys shorter than 2 bytes — so the +// floor matches the data model and, by ruling out the 256 two-char prefixes, +// blunts trivial enumeration of every node name through this endpoint (#15). +const minResolvePrefixHex = 4 + +func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { + pfx := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("prefix"))) + if !hexPrefixRe.MatchString(pfx) { + http.Error(w, "prefix must be hex", http.StatusBadRequest) + return + } + if len(pfx) < minResolvePrefixHex { + http.Error(w, "prefix must be at least 4 hex chars", http.StatusBadRequest) + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + // LIMIT 2: we only need to know unique vs ambiguous. nodes.public_key is the + // PK and stored lowercase; pfx is validated hex so the LIKE pattern is safe. + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, pfx+"%") + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + defer rows.Close() + var pks, names []string + for rows.Next() { + var pk, nm string + if err := rows.Scan(&pk, &nm); err != nil { + http.Error(w, "scan failed", http.StatusInternalServerError) + return + } + pks = append(pks, pk) + names = append(names, nm) + } + + resp := ResolvePrefixResp{Prefix: pfx} + switch len(pks) { + case 1: + // Parity with /api/nodes/search and /api/resolve-hops: never reveal the + // identity of a blacklisted or hidden-prefix node (#1181). Report it as + // not-found rather than leaking the name the rest of the API hides. + if !s.cfg.IsBlacklisted(pks[0]) && !s.cfg.IsNameHidden(names[0]) { + resp.Pubkey = pks[0] + resp.Name = names[0] + } + default: + resp.Ambiguous = len(pks) > 1 // 0 → not found (name empty), >1 → ambiguous + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} diff --git a/cmd/server/node_resolve_test.go b/cmd/server/node_resolve_test.go new file mode 100644 index 000000000..05f0bd3d5 --- /dev/null +++ b/cmd/server/node_resolve_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func serveResolve(srv *Server, path string) *httptest.ResponseRecorder { + router := mux.NewRouter() + router.HandleFunc("/api/nodes/resolve", srv.handleResolvePrefix).Methods("GET") + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest("GET", path, nil)) + return rr +} + +func TestResolvePrefix(t *testing.T) { + db := setupTestDBv2(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('efef7943505052b47f1809488ea4b4d3942d4ed72d2b1953b90a9f5e62a65fb5','NodeUnique','repeater','t','t',1)`) + // Two nodes sharing the 4-hex prefix aabb → ambiguous at the new minimum. + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('aabb110000000000000000000000000000000000000000000000000000000000','NodeA','repeater','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('aabb220000000000000000000000000000000000000000000000000000000000','NodeB','repeater','t','t',1)`) + srv := &Server{db: db} + + // unique 3-byte prefix → name + var r1 ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=efef79").Body.Bytes(), &r1) + if r1.Name != "NodeUnique" || r1.Ambiguous { + t.Fatalf("unique: %+v", r1) + } + // colliding 4-hex prefix (aabb…) → ambiguous, no name + var r2 ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=aabb").Body.Bytes(), &r2) + if !r2.Ambiguous || r2.Name != "" { + t.Fatalf("ambiguous: %+v", r2) + } + // not found → empty name, not ambiguous + var r3 ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=dead").Body.Bytes(), &r3) + if r3.Name != "" || r3.Ambiguous { + t.Fatalf("notfound: %+v", r3) + } + // non-hex prefix → 400 + if serveResolve(srv, "/api/nodes/resolve?prefix=xyz").Code != 400 { + t.Fatal("non-hex prefix should be 400") + } + // #15: prefixes shorter than 4 hex are rejected (kills 256-prefix enumeration) + for _, short := range []string{"a", "aa", "abc"} { + if code := serveResolve(srv, "/api/nodes/resolve?prefix="+short).Code; code != 400 { + t.Fatalf("prefix %q (<4 hex) should be 400, got %d", short, code) + } + } +} + +// TestResolvePrefixHidesBlacklistedAndHidden verifies the #15 parity fix: a +// unique match that is blacklisted or whose name is hidden (#1181) resolves as +// not-found, never leaking an identity the rest of the API hides. +func TestResolvePrefixHidesBlacklistedAndHidden(t *testing.T) { + db := setupTestDBv2(t) + const blPK = "bbcc110000000000000000000000000000000000000000000000000000000000" + const hidPK = "ddee220000000000000000000000000000000000000000000000000000000000" + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('`+blPK+`','BlacklistedNode','repeater','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('`+hidPK+`','🚫HiddenNode','repeater','t','t',1)`) + srv := &Server{db: db, cfg: &Config{ + NodeBlacklist: []string{blPK}, + HiddenNamePrefixes: []string{"🚫"}, + }} + + var rb ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=bbcc11").Body.Bytes(), &rb) + if rb.Name != "" || rb.Pubkey != "" || rb.Ambiguous { + t.Fatalf("blacklisted node must resolve as not-found: %+v", rb) + } + var rh ResolvePrefixResp + json.Unmarshal(serveResolve(srv, "/api/nodes/resolve?prefix=ddee22").Body.Bytes(), &rh) + if rh.Name != "" || rh.Pubkey != "" || rh.Ambiguous { + t.Fatalf("hidden-prefix node must resolve as not-found: %+v", rh) + } +} diff --git a/cmd/server/openapi_known_gaps.json b/cmd/server/openapi_known_gaps.json index 174c39a9d..7c87182a1 100644 --- a/cmd/server/openapi_known_gaps.json +++ b/cmd/server/openapi_known_gaps.json @@ -13,14 +13,18 @@ "/api/healthz", "/api/known-channels", "/api/nodes/clock-skew", + "/api/nodes/resolve", "/api/nodes/{pubkey}/battery", "/api/nodes/{pubkey}/clock-skew", "/api/nodes/{pubkey}/reach", + "/api/nodes/{pubkey}/rx-coverage", "/api/observers/clock-skew", "/api/paths/inspect", "/api/perf/io", "/api/perf/sqlite", "/api/perf/write-sources", + "/api/rx-coverage", + "/api/rx-leaderboard", "/api/scope-stats", "/api/spec" ] diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 7a43184a3..95586c1bb 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -267,10 +267,17 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/nodes/{pubkey}/clock-skew", s.handleNodeClockSkew).Methods("GET") r.HandleFunc("/api/observers/clock-skew", s.handleObserverClockSkew).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}/neighbors", s.handleNodeNeighbors).Methods("GET") - // Keep specific sub-routes (…/reach) registered BEFORE the catch-all - // /api/nodes/{pubkey} — mux matches in registration order, so reordering - // this below the catch-all would shadow it and break the route. + // Keep specific sub-routes (…/reach, …/rx-coverage) registered BEFORE the + // catch-all /api/nodes/{pubkey} — mux matches in registration order, so + // reordering these below the catch-all would shadow them and break the route. r.HandleFunc("/api/nodes/{pubkey}/reach", s.handleNodeReach).Methods("GET") + // Coverage routes are always registered; each handler 404s when the opt-in + // clientRxCoverage flag is off (a clean 404 rather than the SPA fallback that + // an unregistered /api route would hit). See requireClientRxCoverage. + r.HandleFunc("/api/nodes/{pubkey}/rx-coverage", s.handleNodeRxCoverage).Methods("GET") + r.HandleFunc("/api/nodes/resolve", s.handleResolvePrefix).Methods("GET") + r.HandleFunc("/api/rx-coverage", s.handleRxCoverage).Methods("GET") + r.HandleFunc("/api/rx-leaderboard", s.handleRxLeaderboard).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}", s.handleNodeDetail).Methods("GET") r.HandleFunc("/api/nodes", s.handleNodes).Methods("GET") @@ -445,6 +452,7 @@ func (s *Server) handleConfigClient(w http.ResponseWriter, r *http.Request) { MapDarkTileProvider: s.cfg.MapDarkTileProvider, Tiles: s.cfg.Tiles, Customizer: CustomizerClientConfig{DisabledTabs: disabledTabs}, + ClientRxCoverage: s.cfg.ClientRxCoverageEnabled(), }) } diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go new file mode 100644 index 000000000..90283fcfe --- /dev/null +++ b/cmd/server/rx_coverage.go @@ -0,0 +1,370 @@ +package main + +import ( + "encoding/json" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/gorilla/mux" +) + +// coverageRow is one raw reception read from client_receptions. +type coverageRow struct { + Lat, Lon float64 + SNR *float64 + RSSI *int + HeardKey string // directly-heard node key (2-3 byte prefix or full pubkey), lowercase + RxAt string // reception time (RFC3339); used to pick the latest SNR per node +} + +// coverageFeatureCap bounds the number of hex cells returned in one response. +// A wide bbox at high zoom over the 30-day window could otherwise emit multi-MB +// GeoJSON; when more cells exist the densest are kept and Truncated is set (#12). +const coverageFeatureCap = 5000 + +// coverageCellNodeCap bounds the per-cell node breakdown shipped on the wire +// (the client only renders the top ~10). NodesTruncated flags that more were +// heard than returned (#11). +const coverageCellNodeCap = 25 + +// GeoJSON output (named structs, no map[string]interface{} — AGENTS.md). +// Truncated is a non-standard foreign member (ignored by GeoJSON consumers like +// Leaflet) that signals the cell list was capped at coverageFeatureCap. +type CoverageFeatureCollection struct { + Type string `json:"type"` // "FeatureCollection" + Features []CoverageFeature `json:"features"` + Truncated bool `json:"truncated,omitempty"` + // Per-node summary (set only by the per-node endpoint): total mobile-client + // receptions of this node and how many distinct companions heard it. Foreign + // members, omitempty so the global endpoint's payload is unchanged (#3). + MobileReceptions int `json:"mobile_receptions,omitempty"` + MobileClients int `json:"mobile_clients,omitempty"` +} +type CoverageFeature struct { + Type string `json:"type"` // "Feature" + Geometry CoveragePolygon `json:"geometry"` + Properties CoverageProperties `json:"properties"` +} +type CoveragePolygon struct { + Type string `json:"type"` // "Polygon" + Coordinates [][][2]float64 `json:"coordinates"` // one ring: [ [ [lon,lat], ... ] ] +} +type CoverageProperties struct { + Cell string `json:"cell"` + Count int `json:"count"` + BestSNR *float64 `json:"best_snr"` + HasSig bool `json:"has_sig"` // false → render grey (no signal metric) + Nodes []CoverageNode `json:"nodes"` // per-node breakdown, strongest latest-SNR first + NodesTruncated bool `json:"nodes_truncated,omitempty"` // true → more nodes heard than returned (#11) +} + +// CoverageNode is one directly-heard node within a cell, with its latest SNR. +type CoverageNode struct { + Prefix string `json:"prefix"` // heard_key (resolved to Name when unique) + Name string `json:"name,omitempty"` // node name, empty if unknown/ambiguous prefix + SNR *float64 `json:"snr"` // latest SNR (by rx_at); nil → heard without signal + Count int `json:"count"` +} + +type covAgg struct { + count int + bestSNR *float64 + hasSig bool + nodes map[string]*covNodeAgg +} + +// covNodeAgg tracks, per directly-heard node within a cell, its reception count and +// the SNR of its most recent reception (by rx_at). name/prefix are the resolved node +// name (when known) and a display prefix fallback. nameKeyLen records the heard_key +// length that set the current name, so the chosen identity is the most specific one +// regardless of row order (#20). +type covNodeAgg struct { + count int + latestAt string + latestSNR *float64 + name string + nameKeyLen int + prefix string +} + +// nodeResolver maps a heard_key (2-3 byte prefix or full pubkey) to a canonical +// identity key and a display name. A unique match returns (pubkey, name) so the same +// node heard under different prefix lengths collapses into one bucket; unknown or +// ambiguous keys return (heardKey, "") and stay distinct. nil disables resolution. +type nodeResolver func(heardKey string) (key, name string) + +// aggregateCoverage bins raw rows into display-resolution hex cells, keeping the +// best (max) SNR per cell, and emits GeoJSON polygons. resolve (may be nil) collapses +// per-node receptions by resolved node identity. +func aggregateCoverage(rows []coverageRow, res int, resolve nodeResolver) CoverageFeatureCollection { + byCell := map[string]*covAgg{} + for _, row := range rows { + cell := hexCellAt(row.Lat, row.Lon, res) + a := byCell[cell] + if a == nil { + a = &covAgg{} + byCell[cell] = a + } + a.count++ + if row.SNR != nil { + a.hasSig = true + if a.bestSNR == nil || *row.SNR > *a.bestSNR { + v := *row.SNR + a.bestSNR = &v + } + } + if row.HeardKey != "" { + if a.nodes == nil { + a.nodes = map[string]*covNodeAgg{} + } + key, name := row.HeardKey, "" + if resolve != nil { + if k, n := resolve(row.HeardKey); k != "" { + key, name = k, n + } + } + na := a.nodes[key] + if na == nil { + na = &covNodeAgg{prefix: row.HeardKey} + a.nodes[key] = na + } + // Lock the display identity to the MOST SPECIFIC (longest) heard_key + // that resolved to a non-empty name, tie-broken lexicographically, so + // the name no longer flaps with row/map order (#20). A full-pubkey + // reception thus outranks a short-prefix one for the same node. + if name != "" && (na.name == "" || len(row.HeardKey) > na.nameKeyLen || + (len(row.HeardKey) == na.nameKeyLen && name < na.name)) { + na.name = name + na.nameKeyLen = len(row.HeardKey) + } + // Display-prefix fallback (shown when name is empty): same precedence so + // it is also order-independent. + if len(row.HeardKey) > len(na.prefix) || + (len(row.HeardKey) == len(na.prefix) && row.HeardKey < na.prefix) { + na.prefix = row.HeardKey + } + na.count++ + // rx_at is RFC3339, so lexical >= is chronological; keep the latest + // SNR. The first row always wins (latestAt starts "", and any value + // >= ""), so no separate count==1 guard is needed. + if row.RxAt >= na.latestAt { + na.latestAt = row.RxAt + na.latestSNR = row.SNR + } + } + } + fc := CoverageFeatureCollection{Type: "FeatureCollection", Features: []CoverageFeature{}} + for cell, a := range byCell { + ring := hexBoundary(cell) + if ring == nil { + continue + } + nodes, nodesTrunc := sortedCoverageNodes(a.nodes) + fc.Features = append(fc.Features, CoverageFeature{ + Type: "Feature", + Geometry: CoveragePolygon{Type: "Polygon", Coordinates: [][][2]float64{ring}}, + Properties: CoverageProperties{ + Cell: cell, Count: a.count, BestSNR: a.bestSNR, HasSig: a.hasSig, + Nodes: nodes, NodesTruncated: nodesTrunc, + }, + }) + } + // Bound the response: when more cells exist than coverageFeatureCap, keep the + // densest (highest count) and flag the truncation, so a wide/zoomed-out query + // can't emit unbounded multi-MB GeoJSON (#12). + if len(fc.Features) > coverageFeatureCap { + sort.Slice(fc.Features, func(i, j int) bool { + ci, cj := fc.Features[i].Properties.Count, fc.Features[j].Properties.Count + if ci != cj { + return ci > cj // densest first + } + return fc.Features[i].Properties.Cell < fc.Features[j].Properties.Cell // deterministic tie-break + }) + fc.Features = fc.Features[:coverageFeatureCap] + fc.Truncated = true + } + // Map iteration is randomized, so sort features by cell for a deterministic + // payload — stable ETag/caching and a non-flaky "first feature" in e2e (#8). + sort.Slice(fc.Features, func(i, j int) bool { + return fc.Features[i].Properties.Cell < fc.Features[j].Properties.Cell + }) + return fc +} + +// sortedCoverageNodes flattens the per-node aggregates into a slice sorted by latest +// SNR descending (nodes heard without a signal sort last), tie-broken by count then +// prefix for a stable order. The slice is capped at coverageCellNodeCap; truncated +// reports whether more nodes were heard in the cell than returned (#11). +func sortedCoverageNodes(m map[string]*covNodeAgg) (nodes []CoverageNode, truncated bool) { + out := make([]CoverageNode, 0, len(m)) + for _, na := range m { + out = append(out, CoverageNode{Prefix: na.prefix, Name: na.name, SNR: na.latestSNR, Count: na.count}) + } + sort.Slice(out, func(i, j int) bool { + si, sj := out[i].SNR, out[j].SNR + if (si == nil) != (sj == nil) { + return si != nil // signal before no-signal + } + if si != nil && *si != *sj { + return *si > *sj + } + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Prefix < out[j].Prefix + }) + if len(out) > coverageCellNodeCap { + return out[:coverageCellNodeCap], true + } + return out, false +} + +type bbox struct{ MinLat, MinLon, MaxLat, MaxLon float64 } + +// coverageHeardKeyCandidates returns the exact heard_key values that identify a +// node: its full pubkey (stored with heard_keylen 32) and the 2-byte (4 hex) and +// 3-byte (6 hex) prefixes a relay logs. Matching heard_key IN (these) is +// equivalent to the old "heard_keylen=32 AND heard_key=? OR heard_keylen IN (2,3) +// AND substr(?,1,keylen*2)=heard_key", but sargable — so the (heard_key, …) +// composite index seeks the few matching rows instead of scanning the bbox (#5). +func coverageHeardKeyCandidates(pubkey string) []string { + pk := strings.ToLower(pubkey) + seen := map[string]bool{} + out := make([]string, 0, 3) + for _, c := range []string{pk, prefixOrEmpty(pk, 6), prefixOrEmpty(pk, 4)} { + if c != "" && !seen[c] { + seen[c] = true + out = append(out, c) + } + } + return out +} + +func prefixOrEmpty(s string, n int) string { + if len(s) >= n { + return s[:n] + } + return "" +} + +// sqlPlaceholders returns "?,?,…" with n placeholders (n >= 1). +func sqlPlaceholders(n int) string { + if n <= 1 { + return "?" + } + return strings.Repeat("?,", n-1) + "?" +} + +// queryCoverageRows returns raw coverage rows where the directly-heard node +// matches the target pubkey by its 2-3 byte prefix (or full pubkey), within the +// bbox. Read-only (server RO connection). +func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) { + cands := coverageHeardKeyCandidates(pubkey) + args := make([]interface{}, 0, len(cands)+4) + for _, c := range cands { + args = append(args, c) + } + args = append(args, b.MinLat, b.MaxLat, b.MinLon, b.MaxLon) + rows, err := s.db.conn.Query(` + SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`) + AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanCoverageRows(rows) +} + +// mobileRxStats returns the total mobile-client receptions of a node (by its +// 2-3 byte prefix or full pubkey) and the number of distinct contributing clients. +func (s *Server) mobileRxStats(pubkey string) (count, clients int) { + if s.db == nil || s.db.conn == nil { + return 0, 0 + } + cands := coverageHeardKeyCandidates(pubkey) + args := make([]interface{}, len(cands)) + for i, c := range cands { + args[i] = c + } + s.db.conn.QueryRow(` + SELECT COUNT(*), COUNT(DISTINCT rx_pubkey) FROM client_receptions + WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`)`, args...).Scan(&count, &clients) + return count, clients +} + +// zoomToHexRes maps a Leaflet zoom level to the display resolution used for hex +// binning. Resolution == zoom (clamped to a sane range) so hex size tracks the map +// scale 1:1 and renders at a constant ~hexTargetPx (see hexSizeForRes). The clamp also +// guards the missing-param case (z parses to 0). +func zoomToHexRes(z int) int { + switch { + case z < 3: + return 3 + case z > 18: + return 18 + default: + return z + } +} + +func parseBBox(s string) (bbox, bool) { + p := strings.Split(s, ",") + if len(p) != 4 { + return bbox{}, false + } + v := make([]float64, 4) + for i := range p { + f, err := strconv.ParseFloat(strings.TrimSpace(p[i]), 64) + if err != nil { + return bbox{}, false + } + v[i] = f + } + return bbox{MinLat: v[0], MinLon: v[1], MaxLat: v[2], MaxLon: v[3]}, true +} + +// handleNodeRxCoverage serves per-node mobile RX coverage as a GeoJSON hex grid. +func (s *Server) handleNodeRxCoverage(w http.ResponseWriter, r *http.Request) { + if !s.requireClientRxCoverage(w, r) { + return + } + pubkey := strings.ToLower(mux.Vars(r)["pubkey"]) + // Mirror handleNodeReach's gate at this same {pubkey}: reject malformed keys, + // and 404 blacklisted / hidden-prefix nodes. Hiding only the node *name* (via + // heardKeyResolver) still leaked the GPS hex bins and mobile_receptions / + // mobile_clients counts for a node the rest of the API hides (#1727 r2). + if !isHexPubkey(pubkey) { + http.Error(w, "invalid pubkey: expected 64 hex chars", http.StatusBadRequest) + return + } + if (s.cfg != nil && s.cfg.IsBlacklisted(pubkey)) || s.isPubkeyHidden(pubkey) { + http.NotFound(w, r) + return + } + b, ok := parseBBox(r.URL.Query().Get("bbox")) + if !ok { + http.Error(w, "bbox required as minLat,minLon,maxLat,maxLon", http.StatusBadRequest) + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + z, _ := strconv.Atoi(r.URL.Query().Get("z")) + rows, err := s.queryCoverageRows(pubkey, b) + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolverFor(rows)) + // Attach the node-wide reception/contributor totals (#3): the bbox limits the + // hex features to the current view, but these summarise all of this node's + // mobile coverage so the UI can show "heard by N clients" regardless of pan. + fc.MobileReceptions, fc.MobileClients = s.mobileRxStats(pubkey) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(fc) +} diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go new file mode 100644 index 000000000..f5fa92a83 --- /dev/null +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func seedCoverageDB(t *testing.T) *DB { + db := setupTestDBv2(t) + mustExecDB(t, db, `CREATE TABLE client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, + rssi INTEGER, snr REAL, lat REAL, lon REAL, pos_acc_m REAL, rx_at TEXT, ingested_at TEXT, src TEXT)`) + mustExecDB(t, db, `CREATE TABLE client_observers (pubkey TEXT PRIMARY KEY, name TEXT, last_seen TEXT)`) + return db +} + +func TestQueryCoverageRowsByPrefixAndBBox(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + + rows, err := srv.queryCoverageRows("aabbccddeeff00112233", bbox{MinLat: 50, MinLon: 3, MaxLat: 52, MaxLon: 4}) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 row by prefix, got %d", len(rows)) + } + rows, _ = srv.queryCoverageRows("aabbccddeeff00112233", bbox{MinLat: 0, MinLon: 0, MaxLat: 1, MaxLon: 1}) + if len(rows) != 0 { + t.Fatalf("bbox filter failed, got %d", len(rows)) + } +} + +func TestMobileRxStats(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('compA','aabbcc',3,-6,51.05,3.72,'t1','t','rxlog')`) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('compB','aabbcc',3,-8,51.06,3.73,'t2','t','rxlog')`) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('compA','ffeedd',3,-5,51.07,3.74,'t3','t','rxlog')`) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + c, cl := srv.mobileRxStats("aabbccddeeff00112233") + if c != 2 || cl != 2 { + t.Fatalf("got count=%d clients=%d, want 2/2", c, cl) + } +} + +func serveRxCoverage(srv *Server, path string) *httptest.ResponseRecorder { + router := mux.NewRouter() + router.HandleFunc("/api/nodes/{pubkey}/rx-coverage", srv.handleNodeRxCoverage).Methods("GET") + req := httptest.NewRequest("GET", path, nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr +} + +// nodePK is a full 64-hex pubkey whose 3-byte prefix is the seeded heard_key. +const nodePK = "aabbcc0000000000000000000000000000000000000000000000000000000000" + +func TestRxCoverageEndpointGeoJSON(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + + rr := serveRxCoverage(srv, "/api/nodes/"+nodePK+"/rx-coverage?bbox=50,3,52,4&z=12") + if rr.Code != 200 { + t.Fatalf("status %d body %s", rr.Code, rr.Body.String()) + } + var fc CoverageFeatureCollection + if err := json.Unmarshal(rr.Body.Bytes(), &fc); err != nil { + t.Fatalf("decode: %v", err) + } + if fc.Type != "FeatureCollection" || len(fc.Features) != 1 { + t.Fatalf("unexpected fc: %+v", fc) + } + // #3: the per-node response carries the node-wide mobile totals (wired in + // from mobileRxStats). One reception from one companion → 1/1. + if fc.MobileReceptions != 1 || fc.MobileClients != 1 { + t.Fatalf("want mobile_receptions=1 mobile_clients=1, got %d/%d", fc.MobileReceptions, fc.MobileClients) + } + if serveRxCoverage(srv, "/api/nodes/"+nodePK+"/rx-coverage").Code != 400 { + t.Fatal("missing bbox should be 400") + } + // Non-hex pubkey is rejected up front (parity with handleNodeReach). + if serveRxCoverage(srv, "/api/nodes/nothex/rx-coverage?bbox=50,3,52,4").Code != 400 { + t.Fatal("non-hex pubkey should be 400") + } +} + +// TestNodeRxCoverageHidesBlacklistedAndHidden verifies #1727 r2 must-fix #1: the +// per-node coverage endpoint must 404 for blacklisted or hidden-prefix nodes, so +// their GPS hex bins / counts aren't retrievable at a pubkey the rest of the API +// hides — not just the node name. +func TestNodeRxCoverageHidesBlacklistedAndHidden(t *testing.T) { + const hidPK = "ddee110000000000000000000000000000000000000000000000000000000000" + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) + VALUES ('comp','aabbcc',3,-6,51.05,3.72,'t','t','rxlog')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role,last_seen,first_seen,advert_count) VALUES ('`+hidPK+`','🚫Secret','repeater','t','t',1)`) + srv := &Server{db: db, cfg: &Config{ + ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}, + NodeBlacklist: []string{nodePK}, + HiddenNamePrefixes: []string{"🚫"}, + }} + + if code := serveRxCoverage(srv, "/api/nodes/"+nodePK+"/rx-coverage?bbox=50,3,52,4").Code; code != 404 { + t.Fatalf("blacklisted node coverage should be 404, got %d", code) + } + if code := serveRxCoverage(srv, "/api/nodes/"+hidPK+"/rx-coverage?bbox=50,3,52,4").Code; code != 404 { + t.Fatalf("hidden-prefix node coverage should be 404, got %d", code) + } +} diff --git a/cmd/server/rx_coverage_test.go b/cmd/server/rx_coverage_test.go new file mode 100644 index 000000000..72c35ce16 --- /dev/null +++ b/cmd/server/rx_coverage_test.go @@ -0,0 +1,241 @@ +package main + +import ( + "encoding/json" + "fmt" + "math" + "testing" +) + +// TestAggregateCoverageCapsNodesPerCell verifies #11: a cell that heard more than +// coverageCellNodeCap distinct nodes ships at most that many, with NodesTruncated set. +func TestAggregateCoverageCapsNodesPerCell(t *testing.T) { + rows := make([]coverageRow, 0, coverageCellNodeCap+5) + for i := 0; i < coverageCellNodeCap+5; i++ { + rows = append(rows, coverageRow{ + Lat: 51.05, Lon: 3.72, SNR: covF(float64(-i)), + HeardKey: fmt.Sprintf("aa%06x", i), RxAt: "2026-06-01T10:00:00Z", + }) + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + p := fc.Features[0].Properties + if len(p.Nodes) != coverageCellNodeCap || !p.NodesTruncated { + t.Fatalf("want %d nodes + truncated, got %d nodes truncated=%v", coverageCellNodeCap, len(p.Nodes), p.NodesTruncated) + } +} + +// TestAggregateCoverageCapsFeatures verifies #12: a query spanning more than +// coverageFeatureCap cells is bounded to that many features with Truncated set, +// and a smaller query is not truncated. +func TestAggregateCoverageCapsFeatures(t *testing.T) { + // 0.1° spacing >> a res-9 cell (~4 km), so each point lands in its own cell. + rows := make([]coverageRow, 0, coverageFeatureCap+200) + side := 75 // 75*75 = 5625 > 5000 + for i := 0; i < side*side; i++ { + lat := 10.0 + float64(i/side)*0.1 + lon := 10.0 + float64(i%side)*0.1 + rows = append(rows, coverageRow{Lat: lat, Lon: lon, SNR: covF(-5)}) + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != coverageFeatureCap || !fc.Truncated { + t.Fatalf("want %d features + truncated, got %d truncated=%v", coverageFeatureCap, len(fc.Features), fc.Truncated) + } + // Still sorted by cell after truncation. + for i := 1; i < len(fc.Features); i++ { + if fc.Features[i-1].Properties.Cell > fc.Features[i].Properties.Cell { + t.Fatalf("truncated features not sorted by cell at %d", i) + } + } + // A small query is not truncated. + small := aggregateCoverage(rows[:10], 9, nil) + if small.Truncated { + t.Fatalf("small query should not be truncated") + } +} + +func covF(f float64) *float64 { return &f } + +func TestAggregateCoverageBucketsBestSNR(t *testing.T) { + rows := []coverageRow{ + {Lat: 51.05000, Lon: 3.72000, SNR: covF(-12)}, + {Lat: 51.05001, Lon: 3.72001, SNR: covF(-6)}, // same cell, stronger + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + if p := fc.Features[0].Properties; p.BestSNR == nil || *p.BestSNR != -6 || p.Count != 2 || !p.HasSig { + t.Fatalf("bad props: %+v", fc.Features[0].Properties) + } + if g := fc.Features[0].Geometry; g.Type != "Polygon" || len(g.Coordinates) != 1 { + t.Fatalf("bad geometry: %+v", g) + } + if _, err := json.Marshal(fc); err != nil { + t.Fatalf("marshal: %v", err) + } +} + +func TestAggregateCoverageGreyWhenNoSignal(t *testing.T) { + fc := aggregateCoverage([]coverageRow{{Lat: 51.05, Lon: 3.72}}, 9, nil) + if len(fc.Features) != 1 || fc.Features[0].Properties.HasSig { + t.Fatalf("expected one grey (no-sig) cell, got %+v", fc.Features) + } +} + +// TestAggregateCoverageNodeBreakdown covers the per-cell node list: each heard node +// keeps its latest SNR (by rx_at) and reception count, sorted strongest-first with +// heard-without-signal nodes last. +func TestAggregateCoverageNodeBreakdown(t *testing.T) { + rows := []coverageRow{ + // node A: two receptions; the later one (t2) has the weaker SNR -10. + {Lat: 51.05, Lon: 3.72, SNR: covF(-4), HeardKey: "aabb", RxAt: "2026-06-01T10:00:00Z"}, + {Lat: 51.05001, Lon: 3.72001, SNR: covF(-10), HeardKey: "aabb", RxAt: "2026-06-02T10:00:00Z"}, + // node B: single reception, strongest latest SNR. + {Lat: 51.05, Lon: 3.72, SNR: covF(-6), HeardKey: "ccdd", RxAt: "2026-06-01T10:00:00Z"}, + // node C: heard without a signal metric. + {Lat: 51.05, Lon: 3.72, HeardKey: "eeff", RxAt: "2026-06-01T10:00:00Z"}, + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + nodes := fc.Features[0].Properties.Nodes + if len(nodes) != 3 { + t.Fatalf("expected 3 nodes, got %d (%+v)", len(nodes), nodes) + } + if nodes[0].Prefix != "ccdd" || nodes[0].SNR == nil || *nodes[0].SNR != -6 { + t.Errorf("node[0] want ccdd@-6 (strongest), got %+v", nodes[0]) + } + if nodes[1].Prefix != "aabb" || nodes[1].SNR == nil || *nodes[1].SNR != -10 || nodes[1].Count != 2 { + t.Errorf("node[1] want aabb latest -10 count 2, got %+v", nodes[1]) + } + if nodes[2].Prefix != "eeff" || nodes[2].SNR != nil { + t.Errorf("node[2] want eeff no-signal (last), got %+v", nodes[2]) + } +} + +// TestResolveHeardKey covers heard_key → (pubkey, name) resolution: a unique match +// returns the canonical pubkey + name; an ambiguous prefix (>1 node) and an +// unknown/empty key return the key itself with an empty name. +func TestResolveHeardKey(t *testing.T) { + db := seedCoverageDB(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbccdd11223344','Alice','repeater')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbcc99887766aa','Bob','repeater')`) + srv := &Server{db: db} + if k, n := srv.resolveHeardKey("aabbccdd"); k != "aabbccdd11223344" || n != "Alice" { + t.Errorf("unique prefix → (pubkey,Alice), got (%q,%q)", k, n) + } + if k, n := srv.resolveHeardKey("aabbcc"); k != "aabbcc" || n != "" { + t.Errorf("ambiguous prefix → (key,\"\"), got (%q,%q)", k, n) + } + if k, n := srv.resolveHeardKey("ffff"); k != "ffff" || n != "" { + t.Errorf("unknown prefix → (key,\"\"), got (%q,%q)", k, n) + } + if k, n := srv.resolveHeardKey(""); k != "" || n != "" { + t.Errorf("empty prefix → (\"\",\"\"), got (%q,%q)", k, n) + } +} + +// TestAggregateCoverageMergesResolvedNodes verifies that the same node heard under +// two different heard_keys (e.g. a 3-byte prefix and the full pubkey) collapses into a +// single entry — summed count, latest SNR — when the resolver maps both to one node. +func TestAggregateCoverageMergesResolvedNodes(t *testing.T) { + rows := []coverageRow{ + {Lat: 51.05, Lon: 3.72, SNR: covF(-4), HeardKey: "aabbcc", RxAt: "2026-06-01T10:00:00Z"}, + {Lat: 51.05, Lon: 3.72, SNR: covF(-9), HeardKey: "aabbccdd11223344", RxAt: "2026-06-03T10:00:00Z"}, + {Lat: 51.05, Lon: 3.72, SNR: covF(-7), HeardKey: "aabbcc", RxAt: "2026-06-02T10:00:00Z"}, + } + resolve := func(hk string) (string, string) { return "aabbccdd11223344", "Alice" } + fc := aggregateCoverage(rows, 9, resolve) + if len(fc.Features) != 1 { + t.Fatalf("expected 1 cell, got %d", len(fc.Features)) + } + nodes := fc.Features[0].Properties.Nodes + if len(nodes) != 1 { + t.Fatalf("expected 1 merged node, got %d (%+v)", len(nodes), nodes) + } + n := nodes[0] + if n.Name != "Alice" || n.Count != 3 || n.SNR == nil || *n.SNR != -9 { + t.Errorf("merged node want Alice count 3 latest -9, got %+v (snr=%v)", n, n.SNR) + } +} + +// TestAggregateCoverageDeterministicFeatureOrder verifies #8: features come out +// sorted by cell regardless of Go's randomized map iteration, so the GeoJSON is +// stable (cacheable / non-flaky e2e). +func TestAggregateCoverageDeterministicFeatureOrder(t *testing.T) { + rows := []coverageRow{ + {Lat: 51.0, Lon: 3.0, SNR: covF(-5)}, + {Lat: 48.0, Lon: 2.0, SNR: covF(-5)}, + {Lat: 52.0, Lon: 4.0, SNR: covF(-5)}, + {Lat: 40.0, Lon: -3.0, SNR: covF(-5)}, + } + fc := aggregateCoverage(rows, 9, nil) + if len(fc.Features) < 2 { + t.Fatalf("expected multiple cells, got %d", len(fc.Features)) + } + for i := 1; i < len(fc.Features); i++ { + if fc.Features[i-1].Properties.Cell > fc.Features[i].Properties.Cell { + t.Fatalf("features not sorted by cell at %d: %q > %q", i, + fc.Features[i-1].Properties.Cell, fc.Features[i].Properties.Cell) + } + } +} + +// TestAggregateCoverageNamePrecedenceOrderIndependent verifies #20: when two +// heard_keys resolve to the same node but the resolver returns different display +// names, the most specific (longest) heard_key wins regardless of row order, so +// the name no longer depends on map/row iteration. +func TestAggregateCoverageNamePrecedenceOrderIndependent(t *testing.T) { + resolve := func(hk string) (string, string) { + if hk == "aabbccdd11223344" { + return "aabbccdd11223344", "Alice" + } + return "aabbccdd11223344", "AliceShortPrefix" + } + full := coverageRow{Lat: 51.05, Lon: 3.72, SNR: covF(-5), HeardKey: "aabbccdd11223344", RxAt: "2026-06-01T10:00:00Z"} + prefix := coverageRow{Lat: 51.05, Lon: 3.72, SNR: covF(-6), HeardKey: "aabbcc", RxAt: "2026-06-02T10:00:00Z"} + + for _, order := range [][]coverageRow{{full, prefix}, {prefix, full}} { + fc := aggregateCoverage(order, 9, resolve) + nodes := fc.Features[0].Properties.Nodes + if len(nodes) != 1 { + t.Fatalf("expected 1 merged node, got %d (%+v)", len(nodes), nodes) + } + if nodes[0].Name != "Alice" { + t.Fatalf("name precedence flapped with row order: got %q, want Alice", nodes[0].Name) + } + } +} + +func TestZoomToHexRes(t *testing.T) { + // Resolution tracks zoom 1:1 within [3,18], clamped at the edges (z=0 is the + // missing-param case). + cases := map[int]int{0: 3, 3: 3, 8: 8, 16: 16, 18: 18, 25: 18} + for z, want := range cases { + if got := zoomToHexRes(z); got != want { + t.Fatalf("zoomToHexRes(%d)=%d, want %d", z, got, want) + } + } +} + +// TestHexSizeRendersConstantPx verifies the core fix: a hex sized for resolution +// res renders at a constant ~hexTargetPx on screen at the corresponding zoom level, +// instead of the old fixed-meter buckets that were ~2px when zoomed out. +func TestHexSizeRendersConstantPx(t *testing.T) { + for res := 4; res <= 16; res++ { + // On-screen point-to-point height = 2*circumradius / mercUnitsPerPixel(zoom), + // where mercUnitsPerPixel = mercUPPZ0 / 2^zoom and zoom == res. + px := 2 * hexSizeForRes(res) * math.Pow(2, float64(res)) / mercUPPZ0 + if math.Abs(px-hexTargetPx) > 0.001 { + t.Fatalf("res %d renders %.2fpx, want %.2fpx", res, px, hexTargetPx) + } + // Size must halve each zoom step (finer grid as you zoom in). + if ratio := hexSizeForRes(res) / hexSizeForRes(res+1); math.Abs(ratio-2) > 1e-9 { + t.Fatalf("res %d→%d size ratio %.4f, want 2", res, res+1, ratio) + } + } +} diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go new file mode 100644 index 000000000..c4ae3b470 --- /dev/null +++ b/cmd/server/rx_dashboard.go @@ -0,0 +1,421 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "log" + "net/http" + "sort" + "strconv" + "strings" + "time" +) + +// scanCoverageRows reads (lat,lon,snr,rssi,heard_key,rx_at) rows into coverageRow values. +func scanCoverageRows(rows *sql.Rows) ([]coverageRow, error) { + out := []coverageRow{} + for rows.Next() { + var lat, lon float64 + var snr sql.NullFloat64 + var rssi sql.NullInt64 + var heardKey, rxAt sql.NullString + if err := rows.Scan(&lat, &lon, &snr, &rssi, &heardKey, &rxAt); err != nil { + return nil, err + } + cr := coverageRow{Lat: lat, Lon: lon, HeardKey: strings.ToLower(heardKey.String), RxAt: rxAt.String} + if snr.Valid { + v := snr.Float64 + cr.SNR = &v + } + if rssi.Valid { + v := int(rssi.Int64) + cr.RSSI = &v + } + out = append(out, cr) + } + return out, rows.Err() +} + +// heardKeyResolverFor builds a nodeResolver for exactly the distinct heard_keys +// present in rows, resolving them all in one batched query instead of one query +// per key (the previous per-key resolver was N+1 on the read connection). Maps a +// heard_key to (pubkey, name) on a unique, non-hidden match; to (heardKey, "") +// otherwise. nil when there's no DB. +func (s *Server) heardKeyResolverFor(rows []coverageRow) nodeResolver { + if s.db == nil || s.db.conn == nil { + return nil + } + keys := make([]string, 0, len(rows)) + seen := map[string]bool{} + for _, r := range rows { + if r.HeardKey != "" && !seen[r.HeardKey] { + seen[r.HeardKey] = true + keys = append(keys, r.HeardKey) + } + } + resolved := s.batchResolveHeardKeys(keys) + return func(heardKey string) (string, string) { + if v, ok := resolved[heardKey]; ok { + return v[0], v[1] + } + return heardKey, "" + } +} + +// batchResolveHeardKeys resolves many heard_keys (2-3 byte prefixes or full +// pubkeys) to their canonical (pubkey, name) in a single round-trip per chunk: a +// UNION ALL of one LIMIT-2 prefix lookup each, so per-key work stays bounded +// (2 rows) and the whole set costs one query, not N. A unique match returns +// [pubkey, name]; unknown / ambiguous / blacklisted / hidden-prefix keys (#15, +// #1181) return [heardKey, ""]. +func (s *Server) batchResolveHeardKeys(keys []string) map[string][2]string { + res := make(map[string][2]string, len(keys)) + valid := make([]string, 0, len(keys)) + seen := map[string]bool{} + for _, k := range keys { + if k == "" || seen[k] { + continue + } + seen[k] = true + if !hexPrefixRe.MatchString(k) { + res[k] = [2]string{k, ""} + continue + } + valid = append(valid, k) + } + // SQLITE_MAX_COMPOUND_SELECT is 500 by default; chunk well under it. + const chunk = 200 + for i := 0; i < len(valid); i += chunk { + end := i + chunk + if end > len(valid) { + end = len(valid) + } + batch := valid[i:end] + parts := make([]string, len(batch)) + args := make([]interface{}, 0, len(batch)*2) + for j, k := range batch { + // Parameterized: the prefix flows in as bound args, never interpolated, + // so this stays injection-safe regardless of how hexPrefixRe later + // evolves. The per-prefix LIMIT 2 lives in a subquery because a bare + // LIMIT on a UNION ALL term is a SQLite syntax error. + parts[j] = "SELECT * FROM (SELECT ? AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE ? LIMIT 2)" + args = append(args, k, k+"%") + } + rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL "), args...) + if err != nil { + // Don't fail the request, but don't fail silently either: a swallowed + // error here presents as "every name is ambiguous" with no signal. + log.Printf("WARN batchResolveHeardKeys: %v", err) + for _, k := range batch { + res[k] = [2]string{k, ""} + } + continue + } + type agg struct { + pk, name string + cnt int + } + acc := map[string]*agg{} + for rows.Next() { + var pfx, pk, nm string + if err := rows.Scan(&pfx, &pk, &nm); err != nil { + continue + } + a := acc[pfx] + if a == nil { + a = &agg{} + acc[pfx] = a + } + a.cnt++ + a.pk, a.name = pk, nm + } + rows.Close() + for _, k := range batch { + a := acc[k] + if a != nil && a.cnt == 1 && !s.cfg.IsBlacklisted(a.pk) && !s.cfg.IsNameHidden(a.name) { + res[k] = [2]string{a.pk, a.name} + } else { + res[k] = [2]string{k, ""} + } + } + } + return res +} + +// resolveHeardKey resolves a single heard_key (2-3 byte prefix or full pubkey) +// to its canonical (pubkey, name), or (heardKey, "") when unknown / ambiguous / +// hidden. Thin wrapper over batchResolveHeardKeys so there is one code path. +func (s *Server) resolveHeardKey(heardKey string) (string, string) { + v := s.batchResolveHeardKeys([]string{heardKey})[heardKey] + if v[0] == "" && v[1] == "" { + return heardKey, "" // empty/unresolved → echo the key + } + return v[0], v[1] +} + +// queryCoverageFiltered returns coverage rows within a bbox, optionally filtered +// by heard node (prefix/pubkey), contributing client (rx_pubkey), and time window +// (days; 0 = all time). Powers the global and per-observer coverage maps. +func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]coverageRow, error) { + where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?"} + args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} + if node != "" { + // Sargable heard_key IN-list (see coverageHeardKeyCandidates) so the + // (heard_key, …) composite index is used instead of a substr() scan (#5). + cands := coverageHeardKeyCandidates(node) + where = append(where, "heard_key IN ("+sqlPlaceholders(len(cands))+")") + for _, c := range cands { + args = append(args, c) + } + } + if rx != "" { + where = append(where, "rx_pubkey = ?") + args = append(args, strings.ToLower(rx)) + } + if days > 0 { + since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + where = append(where, "rx_at >= ?") + args = append(args, since) + } + rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanCoverageRows(rows) +} + +// handleRxCoverage serves global (or per-observer via ?rx=) coverage as GeoJSON +// hexbins, over a time window. ?node= also works (same as the per-node endpoint). +// requireClientRxCoverage writes a 404 and returns false when the opt-in +// client-RX coverage feature is disabled, so the coverage endpoints read as +// "not found" instead of serving data on deployments that haven't enabled it. +func (s *Server) requireClientRxCoverage(w http.ResponseWriter, r *http.Request) bool { + // Routes are registered unconditionally, so guard against a nil server/cfg + // (e.g. handlers exercised in isolation) rather than panicking (#4). + // ClientRxCoverageEnabled is itself nil-receiver-safe. + if s == nil || s.cfg == nil || !s.cfg.ClientRxCoverageEnabled() { + http.NotFound(w, r) + return false + } + return true +} + +func (s *Server) handleRxCoverage(w http.ResponseWriter, r *http.Request) { + if !s.requireClientRxCoverage(w, r) { + return + } + b, ok := parseBBox(r.URL.Query().Get("bbox")) + if !ok { + http.Error(w, "bbox required as minLat,minLon,maxLat,maxLon", http.StatusBadRequest) + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + days := clampDays(atoiDefault(r.URL.Query().Get("days"), 7)) + z, _ := strconv.Atoi(r.URL.Query().Get("z")) + rows, err := s.queryCoverageFiltered(r.URL.Query().Get("node"), r.URL.Query().Get("rx"), days, b) + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + fc := aggregateCoverage(rows, zoomToHexRes(z), s.heardKeyResolverFor(rows)) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(fc) +} + +// --- Leaderboard (top mobile observers) --- + +type LeaderObserver struct { + Pubkey string `json:"pubkey"` + Name string `json:"name"` + Receptions int `json:"receptions"` + Nodes int `json:"nodes"` + Cells int `json:"cells"` // distinct fixed-res hex cells covered + Score float64 `json:"score"` // frontier-weighted coverage score +} +type RxLeaderboardResp struct { + Days int `json:"days"` + Observers []LeaderObserver `json:"observers"` +} + +// leaderboardHexRes is the fixed hex resolution used to bucket receptions into +// "cells visited" for the frontier-weighted score. ~150 m ground cells at our +// latitude: coarse enough that a parked node's GPS jitter stays in one cell, +// fine enough that real driving paints many. Independent of the coverage map's +// zoom-dependent render resolution so the ranking is stable across views. +const leaderboardHexRes = 13 + +// leaderboardScanCap bounds how many rows the leaderboard aggregates in memory. +// The endpoint is unauthenticated (only requireClientRxCoverage), and the Go-side +// rarity weighting can't push the GROUP BY into SQLite, so without a cap a wide +// window on a busy network would stream the whole table into maps. At the cap we +// log and return a partial (best-effort) ranking rather than OOM (#review r2). +const leaderboardScanCap = 500000 + +// rxLeaderboard ranks mobile observers by frontier-weighted cell coverage over +// the time window. Each distinct cell an observer covers contributes +// 1/(observers covering that cell): a cell only they reached weighs 1.0, a cell +// shared by N observers weighs 1/N. This rewards expanding the map's edge and is +// spam-proof — a stationary node covers exactly one cell regardless of how many +// receptions it logs. Bucketing + the rarity weight can't be expressed in SQL, +// so we aggregate the window's rows in Go (bounded by leaderboardScanCap). +func (s *Server) rxLeaderboard(ctx context.Context, days, limit int) ([]LeaderObserver, error) { + since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + // Name preference: the node's advertised name, else the companion's + // self-reported name (client_observers), else empty (UI shows the prefix). + // Hard LIMIT bounds memory; ORDER BY rx_at DESC so a truncated window keeps + // the most recent receptions. + rows, err := s.db.conn.QueryContext(ctx, ` + SELECT cr.rx_pubkey, COALESCE(NULLIF(n.name,''), NULLIF(co.name,''), ''), + cr.lat, cr.lon, cr.heard_key + FROM client_receptions cr + LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey + LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey + WHERE cr.rx_at >= ? + ORDER BY cr.rx_at DESC + LIMIT ?`, since, leaderboardScanCap) + if err != nil { + return nil, err + } + defer rows.Close() + + type agg struct { + name string + receptions int + cells map[string]struct{} + nodes map[string]struct{} + } + obsAgg := map[string]*agg{} + cellObservers := map[string]map[string]struct{}{} // cell -> set of rx_pubkey + + scanned := 0 + for rows.Next() { + // Honour client cancellation/timeout on the long scan (checked in batches + // to avoid a per-row context mutex on up to 500k rows). + if scanned&2047 == 0 && ctx.Err() != nil { + return nil, ctx.Err() + } + var pk, name, heardKey string + var lat, lon float64 + if err := rows.Scan(&pk, &name, &lat, &lon, &heardKey); err != nil { + return nil, err + } + scanned++ + a := obsAgg[pk] + if a == nil { + a = &agg{name: name, cells: map[string]struct{}{}, nodes: map[string]struct{}{}} + obsAgg[pk] = a + } + a.receptions++ + a.nodes[heardKey] = struct{}{} + cell := hexCellAt(lat, lon, leaderboardHexRes) + a.cells[cell] = struct{}{} + set := cellObservers[cell] + if set == nil { + set = map[string]struct{}{} + cellObservers[cell] = set + } + set[pk] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, err + } + if scanned >= leaderboardScanCap { + log.Printf("[rx-leaderboard] scan hit cap %d over %dd window; ranking is partial (most-recent rows)", leaderboardScanCap, days) + } + + // Per-cell observer counts EXCLUDING blacklisted contributors, so an operator + // of a blacklisted node parked in a cell can't silently dilute everyone else's + // frontier weight (#review r2). Name-hidden (not blacklisted) observers are + // legitimate contributors and still count. + cellCount := make(map[string]int, len(cellObservers)) + for cell, set := range cellObservers { + n := 0 + for pk := range set { + if !s.cfg.IsObserverBlacklisted(pk) && !s.cfg.IsBlacklisted(pk) { + n++ + } + } + cellCount[cell] = n + } + + out := make([]LeaderObserver, 0, len(obsAgg)) + for pk, a := range obsAgg { + var score float64 + for cell := range a.cells { + if c := cellCount[cell]; c > 0 { + score += 1.0 / float64(c) + } + } + out = append(out, LeaderObserver{ + Pubkey: pk, + Name: a.name, + Receptions: a.receptions, + Nodes: len(a.nodes), + Cells: len(a.cells), + Score: score, + }) + } + + // Rank by frontier score; ties broken by raw receptions then pubkey so the + // order is deterministic (keeps same-location fixtures stable). + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + if out[i].Receptions != out[j].Receptions { + return out[i].Receptions > out[j].Receptions + } + return out[i].Pubkey < out[j].Pubkey + }) + + // Identity hiding parity (#1727 r2): drop observer-blacklisted contributors, + // blank node-blacklisted / hidden-prefix names, cap at limit. nil cfg ⇒ no-ops. + filtered := make([]LeaderObserver, 0, limit) + for _, o := range out { + if s.cfg.IsObserverBlacklisted(o.Pubkey) { + continue + } + if s.cfg.IsBlacklisted(o.Pubkey) || s.cfg.IsNameHidden(o.Name) { + o.Name = "" + } + filtered = append(filtered, o) + if len(filtered) >= limit { + break + } + } + return filtered, nil +} + +func (s *Server) handleRxLeaderboard(w http.ResponseWriter, r *http.Request) { + if !s.requireClientRxCoverage(w, r) { + return + } + if s.db == nil || s.db.conn == nil { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + days := clampDays(atoiDefault(r.URL.Query().Get("days"), 7)) + limit := atoiDefault(r.URL.Query().Get("limit"), 20) + if limit < 1 || limit > 100 { + limit = 20 + } + obs, err := s.rxLeaderboard(r.Context(), days, limit) + if err != nil { + http.Error(w, "query failed", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(RxLeaderboardResp{Days: days, Observers: obs}) +} + +func atoiDefault(s string, d int) int { + if n, err := strconv.Atoi(strings.TrimSpace(s)); err == nil { + return n + } + return d +} diff --git a/cmd/server/rx_dashboard_test.go b/cmd/server/rx_dashboard_test.go new file mode 100644 index 000000000..5d708bee9 --- /dev/null +++ b/cmd/server/rx_dashboard_test.go @@ -0,0 +1,275 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// TestRequireClientRxCoverageNilSafe verifies the #4 fix: coverage routes are +// registered unconditionally, so a nil server cfg (or nil *Config receiver) +// must 404 rather than panic. +func TestRequireClientRxCoverageNilSafe(t *testing.T) { + var nilCfg *Config + if nilCfg.ClientRxCoverageEnabled() { + t.Fatal("nil *Config must report disabled") + } + req := func(srv *Server) int { + rr := httptest.NewRecorder() + srv.handleRxCoverage(rr, httptest.NewRequest("GET", "/api/rx-coverage?bbox=50,3,52,4", nil)) + return rr.Code + } + if code := req(&Server{}); code != http.StatusNotFound { // cfg nil → would panic without the guard + t.Fatalf("nil cfg: want 404, got %d", code) + } + if code := req(&Server{cfg: &Config{}}); code != http.StatusNotFound { // feature disabled + t.Fatalf("disabled: want 404, got %d", code) + } +} + +func insRx(t *testing.T, db *DB, rx, hk, at string, lat, lon float64) { + mustExecDB(t, db, fmt.Sprintf( + `INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) VALUES ('%s','%s',3,-6,%f,%f,'%s','x','rxlog')`, + rx, hk, lat, lon, at)) +} + +func TestQueryCoverageFiltered(t *testing.T) { + db := seedCoverageDB(t) + now := time.Now().UTC() + recent := now.Format(time.RFC3339) + old := now.AddDate(0, 0, -40).Format(time.RFC3339) + insRx(t, db, "compa", "aabbcc", recent, 51.05, 3.72) + insRx(t, db, "compb", "ffeedd", recent, 51.06, 3.73) + insRx(t, db, "compa", "aabbcc", old, 51.05, 3.72) + srv := &Server{db: db} + bb := bbox{MinLat: 50, MinLon: 3, MaxLat: 52, MaxLon: 4} + + if rows, _ := srv.queryCoverageFiltered("", "", 7, bb); len(rows) != 2 { + t.Fatalf("global 7d: want 2, got %d", len(rows)) + } + if rows, _ := srv.queryCoverageFiltered("", "compa", 7, bb); len(rows) != 1 { + t.Fatalf("observer compa 7d: want 1, got %d", len(rows)) + } + if rows, _ := srv.queryCoverageFiltered("", "", 0, bb); len(rows) != 3 { + t.Fatalf("global all-time: want 3, got %d", len(rows)) + } +} + +func TestRxLeaderboard(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('compa','MyCompanion','companion','t','t',1)`) + // compc is NOT in nodes, but reported its name via client_observers (fallback). + mustExecDB(t, db, `INSERT INTO client_observers (pubkey, name, last_seen) VALUES ('compc','MobOnly','t')`) + for i := 0; i < 3; i++ { + insRx(t, db, "compa", fmt.Sprintf("aabb%02d", i), recent, 51.05, 3.72) + } + insRx(t, db, "compc", "ddee00", recent, 51.05, 3.72) + insRx(t, db, "compc", "ddee01", recent, 51.05, 3.72) + insRx(t, db, "compb", "aabbcc", recent, 51.05, 3.72) // no name anywhere + srv := &Server{db: db} + + obs, err := srv.rxLeaderboard(context.Background(), 7, 10) + if err != nil { + t.Fatal(err) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + if byPk["compa"].Name != "MyCompanion" || byPk["compa"].Receptions != 3 { + t.Fatalf("compa (nodes name): %+v", byPk["compa"]) + } + if byPk["compc"].Name != "MobOnly" || byPk["compc"].Receptions != 2 { + t.Fatalf("compc (client_observers fallback): %+v", byPk["compc"]) + } + if byPk["compb"].Name != "" { + t.Fatalf("compb should have no name: %+v", byPk["compb"]) + } +} + +// TestBatchResolveHeardKeys verifies the N+1 fix: many heard_keys resolve in one +// batched call with the same unique/ambiguous/unknown/hidden semantics as the +// single-key path. +func TestBatchResolveHeardKeys(t *testing.T) { + db := setupTestDBv2(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbccdd11223344','Alice','repeater')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('aabbcc99887766aa','Bob','repeater')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key,name,role) VALUES ('ddee110000000000','🚫Hidden','repeater')`) + srv := &Server{db: db, cfg: &Config{HiddenNamePrefixes: []string{"🚫"}}} + + got := srv.batchResolveHeardKeys([]string{"aabbccdd", "aabbcc", "ffff", "ddee11", "aabbccdd"}) + cases := map[string][2]string{ + "aabbccdd": {"aabbccdd11223344", "Alice"}, // unique + "aabbcc": {"aabbcc", ""}, // ambiguous (Alice + Bob) + "ffff": {"ffff", ""}, // unknown + "ddee11": {"ddee11", ""}, // unique but hidden-prefix → not surfaced + } + for k, want := range cases { + if got[k] != want { + t.Errorf("batchResolveHeardKeys[%q] = %v, want %v", k, got[k], want) + } + } +} + +// TestRxLeaderboardHidesBlacklistedAndHidden verifies #1727 r2 must-fix #2: the +// leaderboard must drop observer-blacklisted contributors and blank the name of +// node-blacklisted or hidden-prefix identities (pre-PR / post-blacklist rows). +func TestRxLeaderboardHidesBlacklistedAndHidden(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aa01','GoodGuy','companion','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('cc03','BadNode','companion','t','t',1)`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('dd04','🚫Hidden','companion','t','t',1)`) + insRx(t, db, "aa01", "aabb01", recent, 51.05, 3.72) // normal → kept with name + insRx(t, db, "bb02", "aabb02", recent, 51.05, 3.72) // observer-blacklisted → dropped + insRx(t, db, "cc03", "aabb03", recent, 51.05, 3.72) // node-blacklisted → name blanked + insRx(t, db, "dd04", "aabb04", recent, 51.05, 3.72) // hidden prefix → name blanked + srv := &Server{db: db, cfg: &Config{ + ObserverBlacklist: []string{"bb02"}, + NodeBlacklist: []string{"cc03"}, + HiddenNamePrefixes: []string{"🚫"}, + }} + + obs, err := srv.rxLeaderboard(context.Background(), 7, 100) + if err != nil { + t.Fatal(err) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + if _, ok := byPk["bb02"]; ok { + t.Fatalf("observer-blacklisted contributor must be dropped, got %+v", byPk["bb02"]) + } + if byPk["aa01"].Name != "GoodGuy" { + t.Fatalf("normal contributor name should be kept: %+v", byPk["aa01"]) + } + if _, ok := byPk["cc03"]; !ok || byPk["cc03"].Name != "" { + t.Fatalf("node-blacklisted contributor should remain with a blanked name: %+v", byPk["cc03"]) + } + if _, ok := byPk["dd04"]; !ok || byPk["dd04"].Name != "" { + t.Fatalf("hidden-prefix contributor should remain with a blanked name: %+v", byPk["dd04"]) + } +} + +// TestRxLeaderboardLimitSurvivesBlacklistDrop verifies #1727 r2 must-fix #2: the +// SQL LIMIT runs before the Go-side observer-blacklist drop, so the leaderboard +// must over-fetch and still return `limit` non-blacklisted rows even when the +// top contributors are blacklisted (not limit-minus-dropped). +func TestRxLeaderboardLimitSurvivesBlacklistDrop(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + // Reception counts strictly descending so ORDER BY COUNT(*) DESC is deterministic: + // the two blacklisted observers are the top two, then five good ones. + counts := []struct { + pk string + n int + }{ + {"bk1", 10}, {"bk2", 9}, // observer-blacklisted (top of the board) + {"g1", 8}, {"g2", 7}, {"g3", 6}, {"g4", 5}, {"g5", 4}, + } + for _, c := range counts { + for i := 0; i < c.n; i++ { + insRx(t, db, c.pk, fmt.Sprintf("%s%04d", c.pk, i), recent, 51.05, 3.72) + } + } + srv := &Server{db: db, cfg: &Config{ObserverBlacklist: []string{"bk1", "bk2"}}} + + obs, err := srv.rxLeaderboard(context.Background(), 7, 3) + if err != nil { + t.Fatal(err) + } + if len(obs) != 3 { + t.Fatalf("expected exactly 3 rows after dropping 2 blacklisted from the top, got %d: %+v", len(obs), obs) + } + want := []string{"g1", "g2", "g3"} + for i, o := range obs { + if o.Pubkey == "bk1" || o.Pubkey == "bk2" { + t.Fatalf("blacklisted observer %q leaked into the leaderboard", o.Pubkey) + } + if o.Pubkey != want[i] { + t.Fatalf("row %d = %q, want %q (top-3 non-blacklisted by count)", i, o.Pubkey, want[i]) + } + } +} + +// TestRxLeaderboardFrontierScore verifies the leaderboard ranks by frontier- +// weighted cell coverage, not raw reception count: a roaming observer with FEWER +// receptions but MORE distinct cells outranks a stationary spammer, a cell only +// one observer covers weighs 1.0, and a cell shared by N observers weighs 1/N. +func TestRxLeaderboardFrontierScore(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + // "park": 5 receptions all at ONE spot → 1 cell (stationary spammer). + for i := 0; i < 5; i++ { + insRx(t, db, "park", fmt.Sprintf("pk%04d", i), recent, 51.05, 3.72) + } + // "roam": 3 receptions at 3 far-apart spots → 3 distinct cells. The first + // coincides with park's cell, so that cell is shared by 2 observers. + insRx(t, db, "roam", "rm0001", recent, 51.05, 3.72) // shared with park + insRx(t, db, "roam", "rm0002", recent, 51.06, 3.72) // unique to roam + insRx(t, db, "roam", "rm0003", recent, 51.07, 3.72) // unique to roam + srv := &Server{db: db} + + obs, err := srv.rxLeaderboard(context.Background(), 7, 10) + if err != nil { + t.Fatal(err) + } + if len(obs) != 2 { + t.Fatalf("want 2 observers, got %d: %+v", len(obs), obs) + } + // Roamer outranks the parked spammer despite fewer receptions. + if obs[0].Pubkey != "roam" || obs[1].Pubkey != "park" { + t.Fatalf("ranking: want [roam park], got [%s %s]", obs[0].Pubkey, obs[1].Pubkey) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + // park: 1 cell shared with roam → score 0.5; 5 receptions retained. + if byPk["park"].Cells != 1 || byPk["park"].Receptions != 5 { + t.Fatalf("park: %+v", byPk["park"]) + } + if d := byPk["park"].Score - 0.5; d > 1e-9 || d < -1e-9 { + t.Fatalf("park score: want 0.5, got %v", byPk["park"].Score) + } + // roam: shared cell (0.5) + 2 unique cells (1.0 each) = 2.5; 3 cells. + if byPk["roam"].Cells != 3 { + t.Fatalf("roam cells: want 3, got %d", byPk["roam"].Cells) + } + if d := byPk["roam"].Score - 2.5; d > 1e-9 || d < -1e-9 { + t.Fatalf("roam score: want 2.5, got %v", byPk["roam"].Score) + } +} + +// TestRxLeaderboardScoreNotDilutedByBlacklisted verifies the #review-r2 fix: a +// blacklisted observer sharing a cell must NOT dilute a legitimate observer's +// frontier score. Without excluding blacklisted pubkeys from the per-cell count, +// the legit observer's only cell would weigh 1/2 = 0.5 instead of 1.0. +func TestRxLeaderboardScoreNotDilutedByBlacklisted(t *testing.T) { + db := seedCoverageDB(t) + recent := time.Now().UTC().Format(time.RFC3339) + // Legit "good" and blacklisted "bad" both cover the same ~150 m cell. + insRx(t, db, "good", "aabb01", recent, 51.05, 3.72) + insRx(t, db, "bad", "aabb02", recent, 51.05, 3.72) + srv := &Server{db: db, cfg: &Config{ObserverBlacklist: []string{"bad"}}} + + obs, err := srv.rxLeaderboard(context.Background(), 7, 100) + if err != nil { + t.Fatal(err) + } + byPk := map[string]LeaderObserver{} + for _, o := range obs { + byPk[o.Pubkey] = o + } + if _, leaked := byPk["bad"]; leaked { + t.Fatalf("blacklisted observer must not appear: %+v", byPk["bad"]) + } + if d := byPk["good"].Score - 1.0; d > 1e-9 || d < -1e-9 { + t.Fatalf("blacklisted observer diluted the score: got %v, want 1.0", byPk["good"].Score) + } +} diff --git a/cmd/server/types.go b/cmd/server/types.go index e3c71ab4f..29b3b5dda 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -55,21 +55,21 @@ type TimeBucket struct { // ─── Stats ───────────────────────────────────────────────────────────────────── type StatsResponse struct { - TotalPackets int `json:"totalPackets"` - TotalTransmissions *int `json:"totalTransmissions"` - TotalObservations int `json:"totalObservations"` - TotalNodes int `json:"totalNodes"` - TotalNodesAllTime int `json:"totalNodesAllTime"` - TotalObservers int `json:"totalObservers"` - PacketsLastHour int `json:"packetsLastHour"` - PacketsLast24h int `json:"packetsLast24h"` - Engine string `json:"engine"` - Version string `json:"version"` - Commit string `json:"commit"` - BuildTime string `json:"buildTime"` - Counts RoleCounts `json:"counts"` - SignatureDrops int64 `json:"signatureDrops,omitempty"` - HashMigrationComplete bool `json:"hashMigrationComplete"` + TotalPackets int `json:"totalPackets"` + TotalTransmissions *int `json:"totalTransmissions"` + TotalObservations int `json:"totalObservations"` + TotalNodes int `json:"totalNodes"` + TotalNodesAllTime int `json:"totalNodesAllTime"` + TotalObservers int `json:"totalObservers"` + PacketsLastHour int `json:"packetsLastHour"` + PacketsLast24h int `json:"packetsLast24h"` + Engine string `json:"engine"` + Version string `json:"version"` + Commit string `json:"commit"` + BuildTime string `json:"buildTime"` + Counts RoleCounts `json:"counts"` + SignatureDrops int64 `json:"signatureDrops,omitempty"` + HashMigrationComplete bool `json:"hashMigrationComplete"` // Memory accounting (issue #832). All values in MB. // @@ -207,31 +207,31 @@ type EndpointStatsResp struct { } type PacketStoreIndexes struct { - ByHash int `json:"byHash"` - ByObserver int `json:"byObserver"` - ByNode int `json:"byNode"` + ByHash int `json:"byHash"` + ByObserver int `json:"byObserver"` + ByNode int `json:"byNode"` AdvertByObserver int `json:"advertByObserver"` } type PerfPacketStoreStats struct { - TotalLoaded int `json:"totalLoaded"` - TotalObservations int `json:"totalObservations"` - Evicted int `json:"evicted"` - Inserts int64 `json:"inserts"` - Queries int64 `json:"queries"` - InMemory int `json:"inMemory"` - SqliteOnly bool `json:"sqliteOnly"` - MaxPackets int `json:"maxPackets"` - EstimatedMB float64 `json:"estimatedMB"` - TrackedMB float64 `json:"trackedMB"` - AvgBytesPerPacket int64 `json:"avgBytesPerPacket"` - MaxMB int `json:"maxMB"` - Indexes PacketStoreIndexes `json:"indexes"` - HotStartupHours float64 `json:"hotStartupHours"` - BackgroundLoadComplete bool `json:"backgroundLoadComplete"` - BackgroundLoadFailed bool `json:"backgroundLoadFailed"` - BackgroundLoadProgress int64 `json:"backgroundLoadProgress"` - BackgroundLoadError string `json:"backgroundLoadError,omitempty"` + TotalLoaded int `json:"totalLoaded"` + TotalObservations int `json:"totalObservations"` + Evicted int `json:"evicted"` + Inserts int64 `json:"inserts"` + Queries int64 `json:"queries"` + InMemory int `json:"inMemory"` + SqliteOnly bool `json:"sqliteOnly"` + MaxPackets int `json:"maxPackets"` + EstimatedMB float64 `json:"estimatedMB"` + TrackedMB float64 `json:"trackedMB"` + AvgBytesPerPacket int64 `json:"avgBytesPerPacket"` + MaxMB int `json:"maxMB"` + Indexes PacketStoreIndexes `json:"indexes"` + HotStartupHours float64 `json:"hotStartupHours"` + BackgroundLoadComplete bool `json:"backgroundLoadComplete"` + BackgroundLoadFailed bool `json:"backgroundLoadFailed"` + BackgroundLoadProgress int64 `json:"backgroundLoadProgress"` + BackgroundLoadError string `json:"backgroundLoadError,omitempty"` // #1690: surface retention + coverage so operators can see how much // of the on-disk DB the in-memory store currently reflects. RetentionHours float64 `json:"retentionHours"` @@ -288,24 +288,24 @@ type GoRuntimeStats struct { // ─── Packets ─────────────────────────────────────────────────────────────────── type TransmissionResp struct { - ID int `json:"id"` - RawHex interface{} `json:"raw_hex"` - Hash string `json:"hash"` - FirstSeen string `json:"first_seen"` - Timestamp string `json:"timestamp"` - RouteType interface{} `json:"route_type"` - PayloadType interface{} `json:"payload_type"` - PayloadVersion interface{} `json:"payload_version,omitempty"` - DecodedJSON interface{} `json:"decoded_json"` - ObservationCount int `json:"observation_count"` - ObserverID interface{} `json:"observer_id"` - ObserverName interface{} `json:"observer_name"` - ObserverIATA interface{} `json:"observer_iata"` - SNR interface{} `json:"snr"` - RSSI interface{} `json:"rssi"` - PathJSON interface{} `json:"path_json"` - Direction interface{} `json:"direction"` - Score interface{} `json:"score,omitempty"` + ID int `json:"id"` + RawHex interface{} `json:"raw_hex"` + Hash string `json:"hash"` + FirstSeen string `json:"first_seen"` + Timestamp string `json:"timestamp"` + RouteType interface{} `json:"route_type"` + PayloadType interface{} `json:"payload_type"` + PayloadVersion interface{} `json:"payload_version,omitempty"` + DecodedJSON interface{} `json:"decoded_json"` + ObservationCount int `json:"observation_count"` + ObserverID interface{} `json:"observer_id"` + ObserverName interface{} `json:"observer_name"` + ObserverIATA interface{} `json:"observer_iata"` + SNR interface{} `json:"snr"` + RSSI interface{} `json:"rssi"` + PathJSON interface{} `json:"path_json"` + Direction interface{} `json:"direction"` + Score interface{} `json:"score,omitempty"` Observations []ObservationResp `json:"observations,omitempty"` } @@ -374,18 +374,18 @@ type DecodeResponse struct { // ─── Nodes ───────────────────────────────────────────────────────────────────── type NodeResp struct { - PublicKey string `json:"public_key"` - Name interface{} `json:"name"` - Role interface{} `json:"role"` - Lat interface{} `json:"lat"` - Lon interface{} `json:"lon"` - LastSeen interface{} `json:"last_seen"` - FirstSeen interface{} `json:"first_seen"` - AdvertCount int `json:"advert_count"` - HashSize interface{} `json:"hash_size,omitempty"` - HashSizeInconsistent bool `json:"hash_size_inconsistent,omitempty"` - HashSizesSeen []int `json:"hash_sizes_seen,omitempty"` - LastHeard interface{} `json:"last_heard,omitempty"` + PublicKey string `json:"public_key"` + Name interface{} `json:"name"` + Role interface{} `json:"role"` + Lat interface{} `json:"lat"` + Lon interface{} `json:"lon"` + LastSeen interface{} `json:"last_seen"` + FirstSeen interface{} `json:"first_seen"` + AdvertCount int `json:"advert_count"` + HashSize interface{} `json:"hash_size,omitempty"` + HashSizeInconsistent bool `json:"hash_size_inconsistent,omitempty"` + HashSizesSeen []int `json:"hash_sizes_seen,omitempty"` + LastHeard interface{} `json:"last_heard,omitempty"` } type NodeListResponse struct { @@ -669,7 +669,7 @@ type TopologyResponse struct { HopDistribution []TopologyHopDist `json:"hopDistribution"` TopRepeaters []TopRepeater `json:"topRepeaters"` TopPairs []TopPair `json:"topPairs"` - HopsVsSnr []HopsVsSnr `json:"hopsVsSnr"` + HopsVsSnr []HopsVsSnr `json:"hopsVsSnr"` Observers []ObserverRef `json:"observers"` PerObserverReach map[string]*ObserverReach `json:"perObserverReach"` MultiObsNodes []MultiObsNode `json:"multiObsNodes"` @@ -761,12 +761,12 @@ type DistOverTimeEntry struct { } type DistanceAnalyticsResponse struct { - Summary DistanceSummary `json:"summary"` - TopHops []DistanceHop `json:"topHops"` - TopPaths []DistancePath `json:"topPaths"` - CatStats map[string]*CategoryDistStats `json:"catStats"` - DistHistogram *Histogram `json:"distHistogram"` - DistOverTime []DistOverTimeEntry `json:"distOverTime"` + Summary DistanceSummary `json:"summary"` + TopHops []DistanceHop `json:"topHops"` + TopPaths []DistancePath `json:"topPaths"` + CatStats map[string]*CategoryDistStats `json:"catStats"` + DistHistogram *Histogram `json:"distHistogram"` + DistOverTime []DistOverTimeEntry `json:"distOverTime"` } // ─── Analytics — Hash Sizes ──────────────────────────────────────────────────── @@ -795,11 +795,11 @@ type MultiByteNode struct { } type HashSizeAnalyticsResponse struct { - Total int `json:"total"` - Distribution map[string]int `json:"distribution"` - Hourly []HashSizeHourly `json:"hourly"` - TopHops []HashSizeHop `json:"topHops"` - MultiByteNodes []MultiByteNode `json:"multiByteNodes"` + Total int `json:"total"` + Distribution map[string]int `json:"distribution"` + Hourly []HashSizeHourly `json:"hourly"` + TopHops []HashSizeHop `json:"topHops"` + MultiByteNodes []MultiByteNode `json:"multiByteNodes"` } // ─── Analytics — Subpaths ────────────────────────────────────────────────────── @@ -933,10 +933,10 @@ type SnrDistributionEntry struct { } type ObserverAnalyticsResponse struct { - Timeline []TimeBucket `json:"timeline"` - PacketTypes map[string]int `json:"packetTypes"` - NodesTimeline []TimeBucket `json:"nodesTimeline"` - SnrDistribution []SnrDistributionEntry `json:"snrDistribution"` + Timeline []TimeBucket `json:"timeline"` + PacketTypes map[string]int `json:"packetTypes"` + NodesTimeline []TimeBucket `json:"nodesTimeline"` + SnrDistribution []SnrDistributionEntry `json:"snrDistribution"` RecentPackets []map[string]interface{} `json:"recentPackets"` } @@ -999,24 +999,25 @@ type MapConfigResponse struct { } type ClientConfigResponse struct { - Roles interface{} `json:"roles"` - HealthThresholds interface{} `json:"healthThresholds"` - Map interface{} `json:"map"` - Tiles interface{} `json:"tiles,omitempty"` // deprecated - SnrThresholds interface{} `json:"snrThresholds"` - DistThresholds interface{} `json:"distThresholds"` - MaxHopDist interface{} `json:"maxHopDist"` - Limits interface{} `json:"limits"` - PerfSlowMs interface{} `json:"perfSlowMs"` - WsReconnectMs interface{} `json:"wsReconnectMs"` - CacheInvalidateMs interface{} `json:"cacheInvalidateMs"` - ExternalUrls interface{} `json:"externalUrls"` - PropagationBufferMs float64 `json:"propagationBufferMs"` - LiveMapMaxNodes int `json:"liveMapMaxNodes"` - Timestamps TimestampConfig `json:"timestamps"` - DebugAffinity bool `json:"debugAffinity,omitempty"` - MapDarkTileProvider string `json:"mapDarkTileProvider,omitempty"` // deprecated. TODO: remove after v3.5.0 + Roles interface{} `json:"roles"` + HealthThresholds interface{} `json:"healthThresholds"` + Map interface{} `json:"map"` + Tiles interface{} `json:"tiles,omitempty"` // deprecated + SnrThresholds interface{} `json:"snrThresholds"` + DistThresholds interface{} `json:"distThresholds"` + MaxHopDist interface{} `json:"maxHopDist"` + Limits interface{} `json:"limits"` + PerfSlowMs interface{} `json:"perfSlowMs"` + WsReconnectMs interface{} `json:"wsReconnectMs"` + CacheInvalidateMs interface{} `json:"cacheInvalidateMs"` + ExternalUrls interface{} `json:"externalUrls"` + PropagationBufferMs float64 `json:"propagationBufferMs"` + LiveMapMaxNodes int `json:"liveMapMaxNodes"` + Timestamps TimestampConfig `json:"timestamps"` + DebugAffinity bool `json:"debugAffinity,omitempty"` + MapDarkTileProvider string `json:"mapDarkTileProvider,omitempty"` // deprecated. TODO: remove after v3.5.0 Customizer CustomizerClientConfig `json:"customizer"` + ClientRxCoverage bool `json:"clientRxCoverage"` } // CustomizerClientConfig is the operator-side customizer-modal knobs that diff --git a/config.example.json b/config.example.json index ac2ce525a..ec797097a 100644 --- a/config.example.json +++ b/config.example.json @@ -11,7 +11,8 @@ "nodeDays": 7, "observerDays": 14, "packetDays": 30, - "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled). NOTE (#1283): all four retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." + "clientRxDays": 30, + "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled). clientRxDays: mobile client-RX coverage rows (client_receptions/client_observers) older than N days are deleted (0 = disabled) — bounds the opt-in coverage tables (#1727). NOTE (#1283): all retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." }, "db": { "vacuumOnStartup": false, @@ -357,6 +358,8 @@ ] }, "_comment_compression": "Opt-in HTTP gzip middleware + WebSocket permessage-deflate. Both default to false — enable ONLY when your upstream reverse proxy is NOT already compressing. gzip: enables the gzipMiddleware wrapper around the HTTP handler. websocket: sets gorilla websocket Upgrader.EnableCompression. level: gzip compression level 1..9 (1=BestSpeed, 9=BestCompression, default 6). minSizeBytes: advisory minimum response size below which compression would not pay off. contentTypes: MIME allow-list — only responses with these Content-Type values are compressed. Already-compressed types (image/*, video/*, audio/*, application/zip, application/x-gzip, application/pdf, application/octet-stream) are always skipped, as are responses whose handler already set Content-Encoding. Omit contentTypes to use the built-in default allow-list.", + "clientRxCoverage": { "enabled": false }, + "_comment_clientRxCoverage": "Opt-in mobile client-RX coverage (corescope-rx companions publishing GPS-tagged receptions to meshcore/client//packets). Default OFF: when disabled the ingestor writes no client_receptions, the /api/rx-coverage|rx-leaderboard|nodes/{pubkey}/rx-coverage endpoints 404, and the UI hides the Coverage dashboard + Reach overlay. Set enabled=true to turn it on. SINGLE FLAG, BOTH PROCESSES: the ingestor and server each parse this same config.json, so this one clientRxCoverage.enabled entry gates both the ingest write path and the read endpoints — set it once, not per-process. TRUST: the feature requires an ACL-capable broker binding meshcore/client/{pubkey}/packets to that publisher; without ACLs the companion GPS is spoofable (see docs/client-rx-coverage.md). Retention: see retention.clientRxDays. Companion app + setup: https://github.com/efiten/corescope-rx.", "_comment_channelKeys": "Hex keys for decrypting channel messages. Key name = channel display name. public channel key is well-known.", "_comment_hashChannels": "Channel names whose keys are derived via SHA256. Key = SHA256(name)[:16]. Listed here so the ingestor can auto-derive keys.", "hashRegions": [ diff --git a/deploy-live.sh b/deploy-live.sh deleted file mode 100644 index bc100d835..000000000 --- a/deploy-live.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)" -MATOMO_COMMIT="38c30f9" - -cd "$DEPLOY_DIR" - -echo "[deploy] Fetching latest from origin..." -git fetch origin - -echo "[deploy] Resetting to origin/master..." -git reset --hard origin/master - -echo "[deploy] Building Docker image..." -docker build -t meshcore-analyzer . - -echo "[deploy] Stopping old container (30s grace period)..." -docker stop -t 30 meshcore-analyzer && docker rm meshcore-analyzer -docker run -d --name meshcore-analyzer \ - --restart unless-stopped \ - -p 3000:3000 \ - -v "$(pwd)/config.json:/app/config.json:ro" \ - -v meshcore-data:/app/data \ - meshcore-analyzer - -echo "[deploy] Done. Live at https://analyzer.on8ar.eu" diff --git a/deploy-staging.sh b/deploy-staging.sh deleted file mode 100644 index 2c437e95f..000000000 --- a/deploy-staging.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)" - -cd "$DEPLOY_DIR" - -echo "[staging] Fetching latest from origin..." -git fetch origin - -BRANCH="${1:-master}" -echo "[staging] Checking out $BRANCH..." -git reset --hard "origin/$BRANCH" - -echo "[staging] Building Docker image..." -docker build -t meshcore-analyzer-staging . - -echo "[staging] Stopping old container (30s grace period)..." -docker stop -t 30 meshcore-staging 2>/dev/null || true -docker rm meshcore-staging 2>/dev/null || true - -echo "[staging] Starting new container..." -docker run -d --name meshcore-staging \ - --restart unless-stopped \ - -p 3001:3000 \ - -v "$(pwd)/config.json:/app/config.json:ro" \ - -v meshcore-staging-data:/app/data \ - meshcore-analyzer-staging - -echo "[staging] Done. Live at https://staging.on8ar.eu" diff --git a/docs/client-rx-coverage.md b/docs/client-rx-coverage.md new file mode 100644 index 000000000..484688963 --- /dev/null +++ b/docs/client-rx-coverage.md @@ -0,0 +1,210 @@ +# Client RX Coverage + +Crowdsourced RF coverage from mobile clients: a phone connects over BLE to a MeshCore +*companion* radio, captures which nodes the companion hears (with SNR/RSSI), tags each reception +with the phone's GPS position, and publishes it to MQTT. CoreScope ingests these into +`client_receptions` and renders per-node H3-style hex coverage on the Reach page. + +## Companion app — where to get it + +The mobile capture side is **[corescope-rx](https://github.com/efiten/corescope-rx)** — an +open-source (GPL-3.0) Android PWA. Operators who enable coverage point their users at it: it connects +over BLE to a MeshCore companion radio, captures directly-heard nodes + the phone's GPS, and publishes +the payload defined below. It's self-hostable and generic — a runtime `config.json` aims it at your +own MQTT broker + CoreScope instance (see its README). + +## Enabling coverage (operators) + +Coverage is **off by default**. To turn it on: + +1. In CoreScope's `config.json`, set `"clientRxCoverage": { "enabled": true }` and restart the server + and ingestor. This is a **single flag read by both processes** — the ingestor and server each parse + the same `config.json`, so you set `clientRxCoverage.enabled` once and it gates both the ingest write + path and the read endpoints. There is no separate per-process flag. +2. **Required: an ACL-capable broker.** Bind `meshcore/client/{PUBLIC_KEY}/packets` so each client may + publish **only** under its own pubkey (e.g. an EMQX ACL keyed on the connected client's identity). + This is the trust boundary, not an optimization — see [Trust](#trust). The ingestor already + subscribes under `meshcore/#`. +3. Optionally set `retention.clientRxDays` to bound the coverage tables (see + [Storage](#storage--client_receptions-ingestor-owned)). +4. Point your users at [corescope-rx](https://github.com/efiten/corescope-rx) and they start + contributing. Results show on each node's Reach page (coverage toggle) and the `#/rx-coverage` + dashboard. **Warn them first that their contribution is world-readable and a per-observer view can + reconstruct their movements — see [Privacy](#privacy--contributor-location-is-public).** + +The rest of this document is the MQTT payload contract the companion app implements. + +## Companion BLE source (verified against firmware) + +The mobile app's RX data comes from the companion's **`PUSH_CODE_LOG_RX_DATA` (0x88)** BLE frame: +`[0x88][snr×4 int8][rssi int8][raw packet bytes]`. This is emitted for **every** received +packet (promiscuous, incl. overheard flood traffic), not just messages addressed to the device: + +- `src/Dispatcher.cpp:198` calls `logRxRaw(getLastSNR(), getLastRSSI(), raw, len)` in `checkRecv()` + **unconditionally** — NOT behind `#if MESH_PACKET_LOGGING`. So it works on stock firmware. +- `examples/companion_radio/MyMesh.cpp:283` overrides it to write the 0x88 frame whenever the app + is connected over BLE (`_serial->isConnected()`). + +So per received packet the app gets SNR + RSSI + the raw bytes. It decodes the raw packet (standard +MeshCore format) to derive the directly-heard node (`path[last]` or 0-hop advert pubkey) and pairs it +with the phone's GPS. The bare advert push (`PUSH_CODE_ADVERT` 0x80) carries only a pubkey (no SNR/ +RSSI/path) and is NOT used — 0x88 already covers adverts (the raw advert is in its payload). + +Caveats: 0x88 is only sent while the app is BLE-connected; packets larger than `MAX_FRAME_SIZE` are +skipped; the firmware doc labels 0x88 "can be ignored" (messaging-app view) — for coverage it is the +primary frame. GPS is always the phone's, never the companion's. + +## MQTT topic & payload + +Topic: `meshcore/client/{PUBLIC_KEY}/packets` — `{PUBLIC_KEY}` is the companion's pubkey. The +broker (EMQX) should ACL-restrict each client to publish only under its own pubkey, which is how +"a connected companion may only inject under the keys that apply" is enforced. + +Payload — meshcoretomqtt-compatible packet, plus a `gps` object: + +```json +{ + "origin": "", + "origin_id": "", + "timestamp": "2026-06-09T12:00:00Z", + "type": "PACKET", + "direction": "rx", + "raw": "", + "SNR": -7, + "RSSI": -92, + "gps": { "lat": 51.05, "lon": 3.72, "acc_m": 8 } +} +``` + +- The discriminator is the `gps` object. A packet without `gps` is dropped (coverage needs a position). +- `raw` is decoded server-side to derive the directly-heard node and the path; `hash`/`path` fields + are not required. +- Subscription: the ingestor's default subscription (`meshcore/#`) already covers this topic. Sources + configured with an explicit topic list must add `meshcore/client/+/packets`. + +## Capture HARD RULE — only what was heard directly + +The app and ingestor record **only the node the companion physically received**, never upstream +relayers: + +- **FLOOD** packet **with a path** (≥1 hop) → record `path[len-1]` (the last forwarder = the + immediate RF transmitter). Confirmed against firmware `Mesh.cpp` (`routeRecvPacket` appends the + forwarder's hash to the END of the path) and CoreScope's `neighbor_builder.go:226-228`. +- **DIRECT** packet **with a path** → **NOT attributable, discarded.** Direct forwarders consume the + next hop from the FRONT (`Mesh.cpp removeSelfFromPath`), so `path[len-1]` is the route's + destination-side end, NOT the node we heard. Attributing it credits the SNR to the wrong (often + far-away) node. Only FLOOD routes (0,1) are recorded from a path. +- Packet **with no path** (0 hops) **and** an advert → record the advertiser's full pubkey. +- `direction` must be `rx`. 1-byte (2 hex char) prefixes are excluded (collision-prone, like Reach). +- The RSSI/SNR belong to the directly-received transmission, so they attach to the recorded node. +- The rest of the path is discarded for coverage. + +## Storage — `client_receptions` (ingestor-owned) + +A roaming companion is a mobile observer with a moving position, so it gets its own table (not +`observations`, which assumes a fixed observer location). Per the #1283 read/write invariant, the +table and all writes live in `cmd/ingestor/`. + +``` +client_receptions( + id, rx_pubkey, heard_key, heard_keylen, rssi, snr, + lat, lon, pos_acc_m, rx_at, ingested_at, src, + UNIQUE(rx_pubkey, heard_key, rx_at)) -- idempotent re-ingest +``` + +`heard_keylen` is 32 for a full pubkey (0-hop advert) or 2/3 for a multibyte prefix. `src` is +`advert` or `rxlog`. No hex cell is stored — binning is computed server-side from lat/lon. + +Indexes: a composite `(heard_key, heard_keylen, lat, lon)` and a `(lat, lon)` index back the coverage +queries; the per-node query matches a sargable `heard_key IN (pubkey, prefix6, prefix4)` list so the +composite is used instead of a table scan (see the benchmark in `cmd/ingestor`). + +Retention: the table grows on every submission, so set `retention.clientRxDays` (ingestor) to delete +rows older than N days (and stale `client_observers`); `0` disables it. Without it the table is +unbounded. + +## Read API — coverage GeoJSON + +`GET /api/nodes/{pubkey}/rx-coverage?bbox={minLat,minLon,maxLat,maxLon}&z={zoom}` + +Returns a GeoJSON `FeatureCollection` of hexagons covering where clients heard the node, aggregated +server-side (read-only). Each feature: + +```json +{ "type": "Feature", + "geometry": { "type": "Polygon", "coordinates": [[[lon,lat], ...]] }, + "properties": { "cell": "9:123:-45", "count": 7, "best_snr": -6, "has_sig": true, + "nodes": [{ "prefix": "aabbcc", "name": "Alice", "snr": -6, "count": 3 }], + "nodes_truncated": false } } +``` + +- Hex binning is a pure-Go pointy-top grid over Web Mercator (`cmd/server/hexgrid.go`). We do **not** + use `uber/h3-go` because it is CGO and the project builds with `CGO_ENABLED=0`. Latitude is only + defined within ±85.05° (Web Mercator limit) and is clamped to that range. +- `z` (Leaflet zoom) selects the hex resolution (zoom-adaptive). Raw points never leave the server + (privacy: contributors' tracks are not exposed). +- `best_snr` / `has_sig` drive the colour: green→orange by best SNR, grey when no signal metric. +- Features are sorted by `cell` for a deterministic (cacheable) payload. +- **Bounds:** the per-cell `nodes` list is capped (with `nodes_truncated`), and the collection is + capped at a fixed feature count — when exceeded, the densest cells are kept and the top-level + `truncated` flag is set. The per-node endpoint also returns `mobile_receptions` and `mobile_clients` + totals (node-wide, independent of the bbox). + +## Frontend + +Shown only in the Reach view (`#/nodes/{pubkey}/reach`), as a toggleable hex layer drawn on the +existing Leaflet map (`public/node-reach-coverage.js`), deep-linked via `?coverage=1`. No new +frontend dependencies. Colours come from CSS variables in `public/node-reach.css` +(`--nq-cov-strong|mid|weak|grey`). + +## Trust + +Identity = the companion pubkey (`rx_pubkey`), taken from the `{PUBLIC_KEY}` topic segment. + +**The feature requires an ACL-capable broker.** The reported GPS position is the contributor's own +claim, so the only thing anchoring a reception to a real identity is the broker ACL binding +`meshcore/client/{PUBLIC_KEY}/packets` to the client that holds that key. **Without such an ACL, the +topic — and therefore the GPS and the heard-node attribution — is spoofable:** anyone who can publish +to the broker could inject coverage under any pubkey. Do not enable this feature on an open/no-ACL +broker if you trust the resulting map. + +Server/ingestor-side defense-in-depth (these reduce blast radius but do **not** replace the ACL): + +- The ingestor rejects any topic pubkey that is not lowercase hex before writing, and never falls back + to a payload-supplied id (`cmd/ingestor/client_reception.go`, #2/#10). +- A blacklisted operator cannot contribute via the client topic (the blacklist is enforced before the + coverage write, #1). +- The frontend HTML-escapes the pubkey it renders, so a junk pubkey can't inject markup (#14). +- `/api/nodes/resolve` and coverage tooltips never reveal blacklisted or hidden-prefix node identities + (#15). + +## Privacy — contributor location is public + +⚠️ **Enabling coverage publishes contributors' GPS-tagged receptions, and the per-observer view can +reconstruct a contributor's movements.** The hex map is read without authentication. The leaderboard +exposes each companion's pubkey, and clicking one filters the map to that single companion +(`/api/rx-coverage?rx=`); at high zoom over the retention window this is effectively a public +movement trail (home / work / commute) of whoever carries that companion. **A pseudonymous companion +name does not mitigate this** — the *locations themselves* are identifying (overnight clustering = home), +and all of one contributor's points are linked by the pubkey. + +This is an accepted tradeoff of the feature, not a bug: fine resolution is what makes the aggregate +coverage map useful, the feature is opt-in and OFF by default, and contributors choose to run the +companion. But the consent must be **informed**: + +- **Operators:** tell your users, before they contribute, that their coverage (including a per-observer + view of their own track) is world-readable for as long as `retention.clientRxDays` keeps it. +- **Contributors:** do not contribute from a device you carry on your person if a public record of where + you have been is a concern. Use a dedicated/stationary node, or accept that the trail is public. + +Operators who want to harden this further can lower `retention.clientRxDays`, run the dashboard behind +their own auth/proxy, or (future hardening) coarsen stored coordinates / apply a k-anonymity threshold +to the per-observer view. + +Optional future hardening: have the companion sign a broker-issued token (the firmware exposes +on-device signing) — not required for the MVP, tracked as a follow-up. + +## Configurable values (future customizer) + +Hardcoded initially, tracked for the customizer per AGENTS.md rule 8: hex resolution per zoom +(`zoomToHexRes`), colour SNR thresholds (`coverageColorVar`), and any `rx_at` max-age validation. diff --git a/public/index.html b/public/index.html index 8f3a10a21..f272581c4 100644 --- a/public/index.html +++ b/public/index.html @@ -40,6 +40,7 @@ + + + diff --git a/public/node-reach-coverage.css b/public/node-reach-coverage.css new file mode 100644 index 000000000..b58e88ff5 --- /dev/null +++ b/public/node-reach-coverage.css @@ -0,0 +1,41 @@ +/* Client-RX coverage styles (fork-only feature). Kept in a DEDICATED file — + separate from node-reach.css — so upstream's periodic node-reach.css rewrites + don't drop the coverage tier colours + leaderboard layout (as happened in the + v3.9.0 merge). Consumed by rx-coverage.js (standalone Coverage dashboard) and, + once re-grafted, the per-node Reach page coverage overlay. */ + +/* RX coverage hex layer (mobile client receptions) */ +:root { + --nq-cov-strong: #2ecc71; /* SF8: SNR ≥ −5 dB (good margin) */ + --nq-cov-mid: #e67e22; /* SF8: −9..−5 dB (near the limit) */ + --nq-cov-weak: #e74c3c; /* SF8: < −9 dB (poor, packet loss likely) */ + --nq-cov-grey: #95a5a6; /* heard, no SNR metric */ +} +/* Dark-theme variants: the saturated mid-luminance defaults glare on dark + basemaps. Mirrors the rest of the dashboard's theme-aware tokens (#polish). */ +[data-theme="dark"] { + --nq-cov-strong: #3fb950; + --nq-cov-mid: #d29922; + --nq-cov-weak: #f85149; + --nq-cov-grey: #8b949e; +} +.nq-cov-legend { display:flex; gap:12px; align-items:center; font-size:11px; color:var(--text-muted, #57606a); margin:4px 0 10px; } +/* Toggled by node-reach.js applyCoverage via class, not inline style, so CSS + (print rules, themes) can still override visibility (#19). */ +.nq-cov-legend.is-hidden { display:none !important; } +.nq-cov-legend i { width:12px; height:12px; border-radius:2px; display:inline-block; margin-right:4px; vertical-align:middle; } + +/* Mobile RX coverage hub — leaderboard */ +.rxb { display:flex; flex-direction:column; gap:3px; } +.rxb-row { display:flex; align-items:center; gap:10px; padding:7px 10px; background:var(--section-bg, #f6f8fa); border:1px solid var(--border, #d0d7de); border-radius:6px; font-size:13px; cursor:pointer; } +.rxb-row.rxb-head { font-size:10px; text-transform:uppercase; color:var(--text-muted, #57606a); cursor:default; } +.rxb-row.sel { outline:2px solid var(--accent, #2ecc71); } +/* Visible keyboard focus for the now-focusable (#polish) leaderboard rows. */ +.rxb-row[data-rx]:focus-visible { outline:2px solid var(--link, #0969da); outline-offset:1px; } +.rxb-rank { width:24px; text-align:right; color:var(--text-muted, #57606a); } +.rxb-name { flex:1; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } +.rxb-rec, .rxb-nodes, .rxb-cells, .rxb-score { width:56px; text-align:right; font-variant-numeric:tabular-nums; } +/* Sortable column headers in the leaderboard head row. */ +.rxb-head .rxb-sort { cursor:pointer; user-select:none; } +.rxb-head .rxb-sort:hover { color:var(--link, #0969da); } +.rxb-head .rxb-sort:focus-visible { outline:2px solid var(--link, #0969da); outline-offset:1px; } diff --git a/public/node-reach-coverage.js b/public/node-reach-coverage.js new file mode 100644 index 000000000..a87f0c484 --- /dev/null +++ b/public/node-reach-coverage.js @@ -0,0 +1,73 @@ +/* === CoreScope — node-reach-coverage.js === + Draws per-node mobile RX coverage as an H3-style hex layer on the existing + Reach Leaflet map, from the /api/nodes/{pubkey}/rx-coverage GeoJSON. + No external deps; colours via CSS variables. */ +'use strict'; +(function () { + // coverageColorVar maps a GeoJSON feature's properties to a CSS variable name. + // Grey = received but no signal metric; otherwise SF8 SNR thresholds: ≥ −5 green + // (good margin), −9..−5 orange (near the limit), < −9 red (packet loss likely). + function coverageColorVar(props) { + if (!props || !props.has_sig || props.best_snr == null) return '--nq-cov-grey'; + var s = Number(props.best_snr); + if (s >= -5) return '--nq-cov-strong'; + if (s >= -9) return '--nq-cov-mid'; + return '--nq-cov-weak'; + } + + function cssColor(varName) { + try { + var v = getComputedStyle(document.documentElement).getPropertyValue(varName).trim(); + return v || '#888'; + } catch (e) { return '#888'; } + } + + // coverageFillOpacity gives the SNR tier a redundant, non-hue cue (stronger = + // more opaque) so colour-blind users can distinguish tiers (#a11y). + function coverageFillOpacity(props) { + switch (coverageColorVar(props)) { + case '--nq-cov-strong': return 0.6; + case '--nq-cov-mid': return 0.48; + case '--nq-cov-weak': return 0.34; + default: return 0.22; + } + } + + // addLayer fetches coverage for the current map bbox/zoom and draws hex + // polygons. Returns a handle with off() so the caller can remove it. + function addLayer(map, pubkey) { + var group = L.layerGroup().addTo(map); + function refresh() { + var b = map.getBounds(); + var bbox = [b.getSouth(), b.getWest(), b.getNorth(), b.getEast()].join(','); + var url = '/api/nodes/' + encodeURIComponent(pubkey) + '/rx-coverage?bbox=' + bbox + '&z=' + map.getZoom(); + fetch(url).then(function (r) { return r.json(); }).then(function (fc) { + group.clearLayers(); + (fc.features || []).forEach(function (f) { + var ring = (f.geometry.coordinates[0] || []).map(function (c) { return [c[1], c[0]]; }); // [lon,lat]→[lat,lon] + var col = cssColor(coverageColorVar(f.properties)); + L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: coverageFillOpacity(f.properties) }) + .addTo(group) + .bindTooltip('n=' + f.properties.count + + (f.properties.best_snr != null ? ' · SNR ' + f.properties.best_snr : ' · no signal')); + }); + }).catch(function (e) { + // Leave the layer empty on error; never crash the reach page. + console.warn('node-reach-coverage: coverage fetch failed', e); + }); + } + // Debounce pan/zoom redraws so dragging doesn't storm the coverage endpoint + // (#6). Keep the reference so off() can unbind the same handler. + var debouncedRefresh = (typeof debounce === 'function') ? debounce(refresh, 200) : refresh; + map.on('moveend zoomend', debouncedRefresh); + refresh(); + return { + off: function () { + map.off('moveend zoomend', debouncedRefresh); + try { map.removeLayer(group); } catch (e) {} + } + }; + } + + window.NodeReachCoverage = { coverageColorVar: coverageColorVar, coverageFillOpacity: coverageFillOpacity, addLayer: addLayer }; +})(); diff --git a/public/node-reach.js b/public/node-reach.js index ecf1b03cc..2d4f2b1f0 100644 --- a/public/node-reach.js +++ b/public/node-reach.js @@ -8,6 +8,16 @@ var current = null; var loadGen = 0; // bumped per load + on destroy; guards against in-flight races var DEFAULT_DAYS = 7; // single JS source for the default window (mirrors the server default) + var coverageOn = false; // mobile-client RX coverage hex layer (deep-linked ?coverage=1) + var covHandle = null; // handle from NodeReachCoverage.addLayer (off() to remove) + + // setCoverageHash reflects the coverage toggle in the URL hash without + // re-triggering the router (history.replaceState, not location.hash =). + function setCoverageHash(on) { + var h = location.hash.replace(/([?&])coverage=1/, '$1').replace(/[?&]$/, ''); + if (on) { h += (h.indexOf('?') >= 0 ? '&' : '?') + 'coverage=1'; } + try { history.replaceState(null, '', h || location.pathname + location.search); } catch (e) {} + } // Single source of the bottleneck tiers: colour + threshold + colour-blind // glyph + legend text. The map legend and the table both read from this. @@ -104,7 +114,14 @@ // pass false so the current report stays on screen until the swap (no flash). async function load(container, pubkey, days, isInitial) { var myGen = ++loadGen; + // Wait for server config so the coverage flag (read just below and in the + // actions bar markup) is populated even on a direct land on this route — a + // race that otherwise hides the coverage toggle (#13). + try { await window.MeshConfigReady; } catch (e) {} + if (myGen !== loadGen) return; // superseded by a newer load while waiting current = { pubkey: pubkey, days: days }; + coverageOn = window.MC_CLIENT_RX_COVERAGE === true && (typeof getHashParams === 'function' && getHashParams().get('coverage') === '1'); + if (covHandle) { try { covHandle.off(); } catch (e) {} covHandle = null; } if (qmap) { qmap.destroy(); qmap = null; } if (isInitial) { container.innerHTML = '
Loading reach…
'; @@ -172,9 +189,16 @@ '' + '' + '' + + (window.MC_CLIENT_RX_COVERAGE ? '' : '') + '' + '' + '' + + (window.MC_CLIENT_RX_COVERAGE ? '
' + + 'strong' + + 'medium' + + 'weak' + + 'no signal' + + '
' : '') + '
' + '
' + '' + @@ -187,6 +211,7 @@ qmap = window.NodeReachMap.render('nqMap', n, TIERS); } + var lastList = []; // most recent filtered link list (so the coverage toggle can restore lines) // Two-way links are always shown; the two checkboxes add the asymmetric ones. function paint() { var inc = document.getElementById('nqIncoming').checked; @@ -205,13 +230,35 @@ var noGps = list.filter(function (l) { return l.lat == null || l.lon == null; }).length; document.getElementById('nqNoGps').textContent = noGps ? noGps + ' link' + (noGps === 1 ? '' : 's') + ' have no location and are not drawn on the map.' : ''; - if (qmap) qmap.setLinks(list); + lastList = list; + if (qmap) qmap.setLinks(coverageOn ? [] : list); } paint(); document.getElementById('nqIncoming').addEventListener('change', paint); document.getElementById('nqOutgoing').addEventListener('change', paint); document.getElementById('nqPrintBtn').addEventListener('click', printReport); + // Coverage overlay (fork): toggles the mobile-client RX hex layer in place and + // hides the link lines under it (declutter) — the table still lists every link. + var covLegend = document.getElementById('nqCovLegend'); + function applyCoverage() { + if (covLegend) covLegend.classList.toggle('is-hidden', !coverageOn); + if (coverageOn) { + if (qmap && window.NodeReachCoverage && !covHandle) covHandle = window.NodeReachCoverage.addLayer(qmap.map, pubkey); + } else if (covHandle) { + try { covHandle.off(); } catch (e) {} + covHandle = null; + } + if (qmap) qmap.setLinks(coverageOn ? [] : lastList); + } + var covCb = document.getElementById('nqCoverage'); + if (covCb) covCb.addEventListener('change', function (e) { + coverageOn = e.target.checked; + setCoverageHash(coverageOn); + applyCoverage(); + }); + if (coverageOn) applyCoverage(); // deep-linked ?coverage=1: add layer + hide lines now + wireTimeRange(container, pubkey); } @@ -225,6 +272,7 @@ function destroy() { loadGen++; // invalidate any in-flight load so it won't mutate a foreign container + if (covHandle) { try { covHandle.off(); } catch (e) {} covHandle = null; } if (qmap) { qmap.destroy(); qmap = null; } current = null; } diff --git a/public/roles.js b/public/roles.js index 57c7f303b..90d63e2ad 100644 --- a/public/roles.js +++ b/public/roles.js @@ -533,6 +533,22 @@ // ─── Fetch server overrides ─── window.MeshConfigReady = fetch('/api/config/client').then(function (r) { return r.json(); }).then(function (cfg) { + window.MC_CLIENT_RX_COVERAGE = cfg.clientRxCoverage === true; + // Coverage is opt-in: the nav link is NOT in static HTML (so the default-off + // nav matches upstream and the nav-overflow tests). Inject it after Analytics + // only when enabled, then nudge applyNavPriority (it re-runs on 'resize'). + if (window.MC_CLIENT_RX_COVERAGE && !document.querySelector('.nav-links [data-route="rx-coverage"]')) { + var navAnchor = document.querySelector('.nav-links [data-route="analytics"]'); + if (navAnchor) { + var covLink = document.createElement('a'); + covLink.href = '#/rx-coverage'; + covLink.className = 'nav-link'; + covLink.setAttribute('data-route', 'rx-coverage'); + covLink.innerHTML = ' Coverage'; + navAnchor.insertAdjacentElement('afterend', covLink); + window.dispatchEvent(new Event('resize')); + } + } if (cfg.roles) { if (cfg.roles.colors) { // #1407 — ROLE_COLORS is now a live getter; merge into the override map. diff --git a/public/rx-coverage.js b/public/rx-coverage.js new file mode 100644 index 000000000..b2e3824eb --- /dev/null +++ b/public/rx-coverage.js @@ -0,0 +1,263 @@ +/* === CoreScope — rx-coverage.js === + Mobile RX coverage hub (route #/rx-coverage): + - global H3-style hex coverage map (all mobile observers), time-windowed + - leaderboard of top mobile observers (companion name + counts) + - click an observer to filter the map to just their coverage + Fork-only feature; isolated page (no changes to the core map). */ +'use strict'; +(function () { + var map = null, covLayer = null, days = 7, selectedRx = '', selectedName = '', boardCache = [], destroyed = false; + + function cssColor(varName) { + try { return getComputedStyle(document.documentElement).getPropertyValue(varName).trim() || '#888'; } + catch (e) { return '#888'; } + } + // SF8 SNR thresholds: ≥ −5 good margin, −9..−5 near the limit, < −9 packet loss + // likely. Grey = heard but no SNR metric. + function colorVar(p) { + if (!p || !p.has_sig || p.best_snr == null) return '--nq-cov-grey'; + var s = Number(p.best_snr); + if (s >= -5) return '--nq-cov-strong'; + if (s >= -9) return '--nq-cov-mid'; + return '--nq-cov-weak'; + } + + function dayBtn(d) { return ''; } + + function pageHtml() { + return '
' + + '

🗺️ Mobile RX coverage

' + + '
Where roaming CoreScope-RX clients heard nodes. Colour = best signal per cell. Get the companion app →
' + + '
' + dayBtn(1) + dayBtn(7) + dayBtn(14) + dayBtn(30) + '
' + + '
strongmediumweakno signal
' + + '
' + + '
Top mobile observers
' + + '
' + + '
'; + } + + // coverageNodeRow renders one heard node: name (or heard_key prefix) + latest SNR + count. + function coverageNodeRow(n) { + var label = n.name ? escapeHtml(n.name) : '' + escapeHtml(n.prefix || '?') + ''; + var snr = (n.snr != null) ? Number(n.snr).toFixed(1) + ' dB' : 'no sig'; + return '
' + + '' + label + '' + + '' + snr + ' · ×' + n.count + '
'; + } + + // coverageNodesHtml lists the nodes directly heard in a cell (properties.nodes: + // {prefix, name, snr, count}, strongest latest-SNR first; prefix shown when the + // name is unresolved). Rendered in the hover tooltip; capped at 10 rows with a + // "(N more)" footer so dense cells don't produce an unwieldy tooltip. + var COVERAGE_NODE_CAP = 10; + function coverageNodesHtml(p) { + var nodes = (p && p.nodes) || []; + var head = '
' + + nodes.length + (nodes.length === 1 ? ' node heard here' : ' nodes heard here') + '
'; + if (!nodes.length) return head + '
n=' + (p ? p.count : 0) + '
'; + var rows = nodes.slice(0, COVERAGE_NODE_CAP).map(coverageNodeRow).join(''); + var more = (nodes.length > COVERAGE_NODE_CAP) + ? '
(' + (nodes.length - COVERAGE_NODE_CAP) + ' more)
' + : ''; + return head + '
' + rows + '
' + more; + } + + // fillOpacityFor adds a redundant, non-hue cue to the SNR tier so the map is + // distinguishable for colour-blind users (orange vs red): stronger signal = + // more opaque. Pairs with the hue and the per-cell SNR in the tooltip (#a11y). + function fillOpacityFor(p) { + switch (colorVar(p)) { + case '--nq-cov-strong': return 0.6; + case '--nq-cov-mid': return 0.48; + case '--nq-cov-weak': return 0.34; + default: return 0.22; + } + } + + function drawCoverage() { + if (!map || destroyed) return; + var b = map.getBounds(); + var bbox = [b.getSouth(), b.getWest(), b.getNorth(), b.getEast()].join(','); + var url = '/api/rx-coverage?bbox=' + bbox + '&z=' + map.getZoom() + '&days=' + days + (selectedRx ? '&rx=' + encodeURIComponent(selectedRx) : ''); + fetch(url).then(function (r) { return r.json(); }).then(function (fc) { + if (destroyed || !covLayer) return; + covLayer.clearLayers(); + (fc.features || []).forEach(function (f) { + var ring = (f.geometry.coordinates[0] || []).map(function (c) { return [c[1], c[0]]; }); + var col = cssColor(colorVar(f.properties)); + L.polygon(ring, { color: col, weight: 1, fillColor: col, fillOpacity: fillOpacityFor(f.properties) }).addTo(covLayer) + .bindTooltip(coverageNodesHtml(f.properties)); + }); + }).catch(function (e) { console.warn('rx-coverage: coverage fetch failed', e); }); + } + + // Leaderboard sort state. Default = frontier score, descending. The rank (#) + // column is not sortable (it just reflects the current order). Numeric columns + // default to descending on first click; the name column to ascending. + var boardSort = { key: 'score', dir: 'desc' }; + var BOARD_COLS = [ + { key: 'name', label: 'Observer (companion)', cls: 'rxb-name' }, + { key: 'score', label: 'score', cls: 'rxb-score', + title: 'Score telt je gedekte cellen, waarbij elke cel zwaarder weegt naarmate minder andere waarnemers ze bereikt hebben — grensverleggende dekking weegt meer dan drukke zones opnieuw afrijden.' }, + { key: 'cells', label: 'cells', cls: 'rxb-cells', + title: 'Aantal unieke ~150 m-cellen waar deze waarnemer iets hoorde.' }, + { key: 'nodes', label: 'nodes', cls: 'rxb-nodes' }, + { key: 'receptions', label: 'pkts', cls: 'rxb-rec' } + ]; + + function sortBoard() { + var k = boardSort.key, dir = boardSort.dir === 'asc' ? 1 : -1; + boardCache.sort(function (a, b) { + if (k === 'name') { + var an = (a.name || a.pubkey).toLowerCase(), bn = (b.name || b.pubkey).toLowerCase(); + return an < bn ? -dir : an > bn ? dir : 0; + } + return (Number(a[k]) - Number(b[k])) * dir; + }); + } + + function boardHeadHtml() { + var cells = BOARD_COLS.map(function (c) { + var arrow = boardSort.key === c.key ? (boardSort.dir === 'asc' ? ' ▲' : ' ▼') : ''; + return '' + escapeHtml(c.label) + arrow + ''; + }).join(''); + return '
#' + cells + '
'; + } + + function renderBoard() { + var el = document.getElementById('rxBoard'); + if (!el) return; + if (!boardCache.length) { el.innerHTML = '
No mobile observers in this window yet.
'; return; } + sortBoard(); + var rows = boardCache.map(function (o, i) { + var nm = o.name ? escapeHtml(o.name) : (escapeHtml(o.pubkey.slice(0, 10)) + '…'); + return '
' + + '' + (i + 1) + '' + nm + '' + + '' + Number(o.score).toFixed(1) + '' + + '' + o.cells + '' + + '' + o.nodes + '' + + '' + o.receptions + '
'; + }).join(''); + el.innerHTML = (selectedRx ? '' : '') + + boardHeadHtml() + rows; + // Column sort handlers (click + keyboard). + el.querySelectorAll('.rxb-sort[data-sort]').forEach(function (h) { + function applySort() { + var k = h.dataset.sort; + if (boardSort.key === k) { + boardSort.dir = boardSort.dir === 'asc' ? 'desc' : 'asc'; + } else { + boardSort.key = k; + boardSort.dir = (k === 'name') ? 'asc' : 'desc'; + } + renderBoard(); + } + h.addEventListener('click', applySort); + h.addEventListener('keydown', function (e) { + if (e.key === 'Enter' || e.key === ' ' || e.key === 'Spacebar') { e.preventDefault(); applySort(); } + }); + }); + // Row click-to-filter (preserved from the original). + el.querySelectorAll('.rxb-row[data-rx]').forEach(function (r) { + function activate() { + selectedRx = r.dataset.rx; selectedName = r.dataset.name || ''; + renderBoard(); fitToObserver(); syncHash(); + } + r.addEventListener('click', activate); + r.addEventListener('keydown', function (e) { + if (e.key === 'Enter' || e.key === ' ' || e.key === 'Spacebar') { e.preventDefault(); activate(); } + }); + }); + var all = document.getElementById('rxAll'); + if (all) all.addEventListener('click', function () { selectedRx = ''; selectedName = ''; renderBoard(); drawCoverage(); syncHash(); }); + } + + // fitToObserver zooms the map to the selected observer's full coverage extent + // (fetched with a world bbox so it's independent of the current view), then the + // resulting moveend redraws the hexes at the fitted resolution. + function fitToObserver() { + if (!map || !selectedRx) { drawCoverage(); return; } + var url = '/api/rx-coverage?bbox=-90,-180,90,180&z=' + Math.max(8, map.getZoom()) + '&days=' + days + '&rx=' + encodeURIComponent(selectedRx); + fetch(url).then(function (r) { return r.json(); }).then(function (fc) { + if (destroyed || !map) return; + var minLat = 90, minLon = 180, maxLat = -90, maxLon = -180, any = false; + (fc.features || []).forEach(function (f) { + (f.geometry.coordinates[0] || []).forEach(function (c) { + any = true; + if (c[1] < minLat) minLat = c[1]; if (c[1] > maxLat) maxLat = c[1]; + if (c[0] < minLon) minLon = c[0]; if (c[0] > maxLon) maxLon = c[0]; + }); + }); + if (!any) { drawCoverage(); return; } // observer has no data in window → keep view + map.fitBounds([[minLat, minLon], [maxLat, maxLon]], { padding: [30, 30], maxZoom: 15 }); + drawCoverage(); // fitBounds may not fire moveend if the view is unchanged + }).catch(function (e) { console.warn('rx-coverage: observer extent fetch failed', e); drawCoverage(); }); + } + + function loadBoard() { + fetch('/api/rx-leaderboard?days=' + days + '&limit=25').then(function (r) { return r.json(); }) + .then(function (d) { if (destroyed) return; boardCache = d.observers || []; renderBoard(); }) + .catch(function (e) { + console.warn('rx-coverage: leaderboard fetch failed', e); + var el = document.getElementById('rxBoard'); + if (el) el.innerHTML = '
Could not load mobile observers.
'; + }); + } + + function setDays(d) { + days = d; + var bar = document.getElementById('rxDays'); + if (bar) bar.querySelectorAll('button').forEach(function (b) { b.classList.toggle('active', +b.dataset.days === d); }); + loadBoard(); drawCoverage(); syncHash(); + } + + function syncHash() { + var q = 'days=' + days + (selectedRx ? '&rx=' + selectedRx : ''); + try { history.replaceState(null, '', '#/rx-coverage?' + q); } catch (e) {} + } + + function init(container) { + destroyed = false; + // A direct land on #/rx-coverage can run before MeshConfigReady resolves, at + // which point MC_CLIENT_RX_COVERAGE is still undefined and the page would + // wrongly show "not enabled". Defer until server config is loaded (#13). + Promise.resolve(window.MeshConfigReady).then(function () { + if (!destroyed) start(container); + }); + } + + function start(container) { + if (!window.MC_CLIENT_RX_COVERAGE) { + container.innerHTML = '
Coverage is not enabled on this deployment.
'; + return; + } + selectedRx = ''; selectedName = ''; days = 7; boardCache = []; + try { + var p = (typeof getHashParams === 'function') ? getHashParams() : null; + if (p) { var dd = parseInt(p.get('days'), 10); if ([1, 7, 14, 30].indexOf(dd) >= 0) days = dd; selectedRx = (p.get('rx') || '').toLowerCase(); } + } catch (e) {} + container.innerHTML = pageHtml(); + map = L.map('rxMap', { zoomControl: true, attributionControl: false }).setView([51.0, 4.8], 8); + if (typeof window._applyTilesToNodeMap === 'function') window._applyTilesToNodeMap(map); + else L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { maxZoom: 19 }).addTo(map); + covLayer = L.layerGroup().addTo(map); + // Debounce pan/zoom redraws so dragging the map doesn't fire a storm of + // /api/rx-coverage requests (#6). Direct calls (setDays, fit) stay immediate. + map.on('moveend zoomend', debounce(drawCoverage, 200)); + var bar = document.getElementById('rxDays'); + if (bar) bar.addEventListener('click', function (e) { var b = e.target.closest('button[data-days]'); if (b) setDays(+b.dataset.days); }); + setTimeout(function () { if (!destroyed && map) { map.invalidateSize(); if (selectedRx) fitToObserver(); else drawCoverage(); } }, 150); + loadBoard(); + } + + function destroy() { + destroyed = true; + if (map) { try { map.remove(); } catch (e) {} map = null; } + covLayer = null; + } + + registerPage('rx-coverage', { init: init, destroy: destroy }); +})(); diff --git a/test-coverage-gate.js b/test-coverage-gate.js new file mode 100644 index 000000000..ec96315c2 --- /dev/null +++ b/test-coverage-gate.js @@ -0,0 +1,65 @@ +'use strict'; +// Unit test for the client-RX coverage frontend gate (SP2). +// +// The coverage toggle (#nqCoverage) and its legend (#nqCovLegend) are built +// inline inside node-reach.js's load() — a large async function that depends on +// api()/Leaflet, so it can't be invoked headless. Instead we extract the exact +// actions-HTML concatenation block from the real source and evaluate it in a vm +// sandbox with both values of window.MC_CLIENT_RX_COVERAGE. This exercises the +// real source markup logic (no hand-copied duplicate) for the gate. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const src = fs.readFileSync(path.join(__dirname, 'public', 'node-reach.js'), 'utf8'); + +// Slice the actions-HTML expression: from the '
'"; +const startIdx = src.indexOf(startMarker); +assert.ok(startIdx >= 0, 'could not locate nq-actions block start in node-reach.js'); +const endMarker = "'
';"; +const endIdx = src.indexOf(endMarker, startIdx); +assert.ok(endIdx >= 0, 'could not locate nq-actions block end in node-reach.js'); +// Drop the trailing ";" so we can wrap the concatenation as a single expression. +let block = src.slice(startIdx, endIdx + endMarker.length).replace(/;\s*$/, ''); + +// Render the actions HTML for a given flag value via a controlled sandbox. +function renderActions(flag) { + const sandbox = { + window: { MC_CLIENT_RX_COVERAGE: flag }, + coverageOn: false, + statsHtml: '', + }; + vm.createContext(sandbox); + return vm.runInContext('(' + block + ')', sandbox); +} + +// Flag OFF ⇒ no coverage checkbox, no legend. +const off = renderActions(false); +assert.ok(!off.includes('id="nqCoverage"'), + 'flag false: actions HTML must NOT contain id="nqCoverage"'); +assert.ok(!off.includes('id="nqCovLegend"'), + 'flag false: actions HTML must NOT contain id="nqCovLegend"'); + +// Flag ON ⇒ coverage checkbox and legend present. +const on = renderActions(true); +assert.ok(on.includes('id="nqCoverage"'), + 'flag true: actions HTML MUST contain id="nqCoverage"'); +assert.ok(on.includes('id="nqCovLegend"'), + 'flag true: actions HTML MUST contain id="nqCovLegend"'); + +// #19: the legend visibility is class-driven (.is-hidden), not an inline +// style="display:..." that CSS can't override. coverageOn is false in this +// sandbox, so the legend must carry is-hidden and no inline display style. +assert.ok(/class="nq-cov-legend[^"]*\bis-hidden\b/.test(on), + 'flag true + coverage off: legend must use the is-hidden class'); +assert.ok(!on.includes('style="display:'), + 'legend must not use an inline display style (#19)'); + +// Sanity: the non-gated controls render regardless of the flag. +assert.ok(off.includes('id="nqIncoming"') && on.includes('id="nqIncoming"'), + 'incoming filter must render irrespective of the coverage flag'); + +console.log('coverage gate (node-reach actions HTML) OK'); diff --git a/test-node-reach-coverage-debounce.js b/test-node-reach-coverage-debounce.js new file mode 100644 index 000000000..3299e9048 --- /dev/null +++ b/test-node-reach-coverage-debounce.js @@ -0,0 +1,73 @@ +'use strict'; +// Unit test for #6: pan/zoom coverage redraws must be debounced so dragging the +// map fires at most one /api/...rx-coverage request per settle, not one per +// moveend. We load node-reach-coverage.js in a vm sandbox with controllable +// timers + the real debounce, bind the layer, fire a burst of map events, then +// advance the fake clock and assert exactly one extra fetch happened. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'node-reach-coverage.js'), 'utf8'); + +// Controllable timer queue. +let now = 0; +let nextId = 1; +let timers = []; +function setTimeoutStub(fn, ms) { const id = nextId++; timers.push({ id: id, fn: fn, at: now + (ms || 0) }); return id; } +function clearTimeoutStub(id) { timers = timers.filter(function (t) { return t.id !== id; }); } +function advance(ms) { + now += ms; + const due = timers.filter(function (t) { return t.at <= now; }); + timers = timers.filter(function (t) { return t.at > now; }); + due.forEach(function (t) { t.fn(); }); +} + +let fetchCount = 0; +function fakeFetch() { + fetchCount++; + // Chainable stub whose then/catch never invoke callbacks (we only count calls). + const chain = { then: function () { return chain; }, catch: function () { return chain; } }; + return chain; +} + +const fakeGroup = { addTo: function () { return fakeGroup; }, clearLayers: function () {}, }; +const map = { + handlers: {}, + on: function (ev, fn) { this.handlers[ev] = fn; }, + off: function () {}, + getBounds: function () { return { getSouth: function () { return 0; }, getWest: function () { return 0; }, getNorth: function () { return 1; }, getEast: function () { return 1; } }; }, + getZoom: function () { return 10; }, + removeLayer: function () {}, +}; + +const sandbox = { + window: {}, + console: { warn: function () {} }, + setTimeout: setTimeoutStub, + clearTimeout: clearTimeoutStub, + fetch: fakeFetch, + L: { layerGroup: function () { return fakeGroup; } }, + getComputedStyle: function () { return { getPropertyValue: function () { return ''; } }; }, + document: { documentElement: {} }, +}; +vm.createContext(sandbox); +// Define debounce IN the context so it uses the controllable timers above. +vm.runInContext('function debounce(fn, ms){var t; return function(){var a=arguments, c=this; clearTimeout(t); t=setTimeout(function(){fn.apply(c,a);}, ms);};}', sandbox); +vm.runInContext(code, sandbox); + +const handle = sandbox.window.NodeReachCoverage.addLayer(map, 'aabbccddeeff'); +assert.strictEqual(fetchCount, 1, 'addLayer should fetch once initially'); + +// Burst of pan/zoom events. +const fire = map.handlers['moveend zoomend']; +assert.strictEqual(typeof fire, 'function', 'moveend/zoomend handler must be bound'); +for (let i = 0; i < 6; i++) fire(); +assert.strictEqual(fetchCount, 1, 'burst must not fetch immediately (debounced)'); + +advance(200); +assert.strictEqual(fetchCount, 2, 'after settle, exactly one coalesced fetch (got ' + fetchCount + ')'); + +handle.off(); +console.log('node-reach-coverage debounce OK'); diff --git a/test-node-reach-coverage-e2e.js b/test-node-reach-coverage-e2e.js new file mode 100644 index 000000000..af70be9f5 --- /dev/null +++ b/test-node-reach-coverage-e2e.js @@ -0,0 +1,65 @@ +// E2E for the RX coverage hex layer on the Reach page (#/nodes//reach?coverage=1). +// Defaults to localhost:3000 — NEVER point at prod (AGENTS.md). CI sets BASE_URL. +const { chromium } = require('playwright'); +const BASE = process.env.BASE_URL || 'http://localhost:3000'; + +(async () => { + const browser = await chromium.launch(); + const page = await browser.newPage(); + + // Coverage is opt-in (config flag, default off). Skip when the deployment under + // test hasn't enabled it — the endpoints 404 and the UI toggle is absent by design. + const clientCfg = await (await page.request.get(BASE + '/api/config/client')).json(); + if (clientCfg.clientRxCoverage !== true) { + console.log('node-reach-coverage E2E SKIP (clientRxCoverage disabled on this deployment)'); + await browser.close(); + return; + } + + const nodes = await (await page.request.get(BASE + '/api/nodes?role=repeater&limit=1')).json(); + if (!nodes.nodes || !nodes.nodes.length) { + console.log('node-reach-coverage E2E SKIP (no repeater in dataset)'); + await browser.close(); + return; + } + const pk = nodes.nodes[0].public_key; + + // 1. The coverage endpoint returns a GeoJSON FeatureCollection. + const cov = await (await page.request.get( + BASE + '/api/nodes/' + pk + '/rx-coverage?bbox=-90,-180,90,180&z=10')).json(); + if (cov.type !== 'FeatureCollection' || !Array.isArray(cov.features)) { + throw new Error('rx-coverage must return a FeatureCollection with a features array'); + } + + // 2. Bad bbox → 400. + const bad = await page.request.get(BASE + '/api/nodes/' + pk + '/rx-coverage'); + if (bad.status() !== 400) throw new Error('missing bbox should be 400, got ' + bad.status()); + + // 3. The Reach page exposes the coverage toggle. + await page.goto(BASE + '/#/nodes/' + pk + '/reach'); + await page.waitForSelector('.nq-head', { timeout: 20000 }); + const reach = await (await page.request.get(BASE + '/api/nodes/' + pk + '/reach?days=7')).json(); + if (reach.reliable_tokens && reach.reliable_tokens.length && (await page.locator('#nqRows').count())) { + await page.waitForSelector('#nqCoverage'); + + // 4. Enabling coverage issues a request to rx-coverage, shows the legend, and deep-links. + const waitCov = page.waitForRequest((r) => r.url().includes('/rx-coverage'), { timeout: 15000 }); + await page.check('#nqCoverage'); + await waitCov; + await page.waitForSelector('#nqCovLegend', { state: 'visible' }); + if (!/coverage=1/.test(await page.evaluate(() => location.hash))) { + throw new Error('coverage toggle did not deep-link ?coverage=1'); + } + + // 5. Toggling off hides the legend. + await page.uncheck('#nqCoverage'); + await page.waitForSelector('#nqCovLegend', { state: 'hidden' }); + } + + const errors = []; + page.on('console', (m) => { if (m.type() === 'error') errors.push(m.text()); }); + if (errors.length) throw new Error('console errors: ' + errors.join('; ')); + + console.log('node-reach-coverage E2E OK'); + await browser.close(); +})().catch((e) => { console.error('node-reach-coverage E2E FAIL:', e.message); process.exit(1); }); diff --git a/test-node-reach-coverage.js b/test-node-reach-coverage.js new file mode 100644 index 000000000..ab8afcf66 --- /dev/null +++ b/test-node-reach-coverage.js @@ -0,0 +1,38 @@ +'use strict'; +// Unit test for node-reach-coverage.js color buckets. Loads the browser IIFE in +// a vm sandbox (pattern from test-frontend-helpers.js) and exercises the pure +// coverageColorVar mapping. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'node-reach-coverage.js'), 'utf8'); +const sandbox = { window: {}, document: {}, getComputedStyle: function () { return { getPropertyValue: function () { return ''; } }; } }; +vm.createContext(sandbox); +vm.runInContext(code, sandbox); + +const { coverageColorVar } = sandbox.window.NodeReachCoverage; + +// SF8 SNR thresholds: ≥ −5 strong, −9..−5 mid, < −9 weak. +assert.strictEqual(coverageColorVar({ has_sig: false }), '--nq-cov-grey', 'no-sig → grey'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: null }), '--nq-cov-grey', 'null snr → grey'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -3 }), '--nq-cov-strong', 'strong'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -5 }), '--nq-cov-strong', 'boundary strong (≥ −5)'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -6 }), '--nq-cov-mid', 'just below −5 → mid'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -9 }), '--nq-cov-mid', 'boundary mid (≥ −9)'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -10 }), '--nq-cov-weak', 'below −9 → weak'); +assert.strictEqual(coverageColorVar({ has_sig: true, best_snr: -18 }), '--nq-cov-weak', 'weak'); +assert.strictEqual(coverageColorVar(null), '--nq-cov-grey', 'null props → grey'); + +// #a11y: fill opacity is a redundant, monotonic non-hue cue for the SNR tier so +// colour-blind users can tell tiers apart. Must strictly decrease strong→grey. +const { coverageFillOpacity } = sandbox.window.NodeReachCoverage; +const oStrong = coverageFillOpacity({ has_sig: true, best_snr: -3 }); +const oMid = coverageFillOpacity({ has_sig: true, best_snr: -6 }); +const oWeak = coverageFillOpacity({ has_sig: true, best_snr: -10 }); +const oGrey = coverageFillOpacity({ has_sig: false }); +assert.ok(oStrong > oMid && oMid > oWeak && oWeak > oGrey, + 'opacity must ramp strong>mid>weak>grey, got ' + [oStrong, oMid, oWeak, oGrey].join(',')); + +console.log('node-reach-coverage color buckets + opacity ramp OK'); diff --git a/test-rx-coverage-config-race.js b/test-rx-coverage-config-race.js new file mode 100644 index 000000000..02631088f --- /dev/null +++ b/test-rx-coverage-config-race.js @@ -0,0 +1,43 @@ +'use strict'; +// Unit test for #13: a direct land on #/rx-coverage must not read the coverage +// feature flag before MeshConfigReady resolves. init() should defer its +// enabled/disabled decision until the config promise settles, instead of +// synchronously rendering "not enabled" when the flag is still undefined. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'rx-coverage.js'), 'utf8'); + +let page = null; +let resolveConfig; +const sandbox = { + window: { MeshConfigReady: new Promise(function (r) { resolveConfig = r; }) }, // MC_CLIENT_RX_COVERAGE undefined for now + document: { getElementById: function () { return null; } }, + registerPage: function (name, obj) { page = obj; }, + console: { warn: function () {} }, + Promise: Promise, + setTimeout: function () {}, clearTimeout: function () {}, + fetch: function () { var c = { then: function () { return c; }, catch: function () { return c; } }; return c; }, + L: {}, getComputedStyle: function () { return { getPropertyValue: function () { return ''; } }; }, +}; +vm.createContext(sandbox); +vm.runInContext(code, sandbox); +assert.ok(page && typeof page.init === 'function', 'registerPage should expose init'); + +(async function () { + const container = { innerHTML: '' }; + page.init(container); + // Before MeshConfigReady resolves, init must NOT have decided yet. The old + // code read the (undefined) flag synchronously and rendered "not enabled". + assert.strictEqual(container.innerHTML, '', 'init must defer until MeshConfigReady resolves'); + + // Resolve config with the feature OFF → only now should it render not-enabled. + sandbox.window.MC_CLIENT_RX_COVERAGE = false; + resolveConfig(); + await new Promise(function (r) { setTimeout(r, 10); }); + assert.ok(/not enabled/i.test(container.innerHTML), 'after config (off) resolves, shows not-enabled'); + + console.log('rx-coverage config race OK'); +})().catch(function (e) { console.error(e); process.exit(1); }); diff --git a/test-rx-coverage-escape.js b/test-rx-coverage-escape.js new file mode 100644 index 000000000..3556546c3 --- /dev/null +++ b/test-rx-coverage-escape.js @@ -0,0 +1,61 @@ +'use strict'; +// Unit test for #14: the mobile RX coverage leaderboard must HTML-escape the +// pubkey it interpolates into the row markup (data-rx="..." and the truncated +// fallback label), not only the name. A no-ACL broker / pre-validation rows +// could carry a non-hex pubkey, and the rest of the row is built by string +// concatenation, so an unescaped pubkey is an HTML-injection vector. +// +// Like test-coverage-gate.js we slice the real row-building expression out of +// public/rx-coverage.js and evaluate it in a vm sandbox — no hand-copied +// duplicate — so the test tracks the actual source. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const src = fs.readFileSync(path.join(__dirname, 'public', 'rx-coverage.js'), 'utf8'); + +// Slice from `var nm = o.name ...` through the end of the returned row string. +const startMarker = 'var nm = o.name ? escapeHtml(o.name)'; +const endMarker = "o.nodes + '';"; +const startIdx = src.indexOf(startMarker); +assert.ok(startIdx >= 0, 'could not locate row-builder start in rx-coverage.js'); +const endIdx = src.indexOf(endMarker, startIdx); +assert.ok(endIdx >= 0, 'could not locate row-builder end in rx-coverage.js'); +const block = src.slice(startIdx, endIdx + endMarker.length); + +// Canonical escapeHtml (public/app.js). +function escapeHtml(s) { + if (s == null) return ''; + return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); +} + +function renderRow(o) { + const sandbox = { o: o, i: 0, selectedRx: '', escapeHtml: escapeHtml }; + vm.createContext(sandbox); + return vm.runInContext('(function () { ' + block + ' })()', sandbox); +} + +// Malicious pubkey that would break out of the data-rx attribute and inject a +// tag if interpolated raw. With escaping, no raw '<', '>' or attribute-closing +// '"' survives. +const evil = '">'; + +// Case 1: no name → pubkey used as the visible label fallback too. +const row1 = renderRow({ pubkey: evil, name: '', receptions: 1, nodes: 1 }); +assert.ok(row1.indexOf('. +assert.ok(/role="button"/.test(row2), 'row must have role="button"'); +assert.ok(/tabindex="0"/.test(row2), 'row must be focusable (tabindex)'); +assert.ok(/aria-pressed="(true|false)"/.test(row2), 'row must expose aria-pressed'); + +console.log('rx-coverage pubkey escaping + row a11y OK');
#Neighbourwe hear