Persist node stats, fix export completeness, dead code cleanup

src/meshchat/database/schema.py

@@ -64,7 +64,21 @@
     first_seen      TEXT,
     last_heard      TEXT,
     packet_count    INTEGER DEFAULT 0,
-    text_count      INTEGER DEFAULT 0
+    text_count      INTEGER DEFAULT 0,
+    -- Signal/hop/transport state of the most recently heard packet, plus
+    -- session-lifetime RF/MQTT/position/telemetry counters — previously
+    -- tracked only on the in-memory NodeSnapshot, so a node the app had
+    -- tracked for weeks started every restart with these reset to
+    -- zero/unknown until it was heard again live. See MonitorStore._write_node.
+    last_snr        REAL,
+    last_rssi       INTEGER,
+    last_hops_used  INTEGER,
+    last_hop_start  INTEGER,
+    last_via_mqtt   INTEGER,
+    rf_count        INTEGER DEFAULT 0,
+    via_mqtt_count  INTEGER DEFAULT 0,
+    position_count  INTEGER DEFAULT 0,
+    telemetry_count INTEGER DEFAULT 0
 );
 
 CREATE TABLE IF NOT EXISTS positions (
@@ -155,3 +169,18 @@ def _migrate(conn) -> None:
         conn.execute("ALTER TABLE messages ADD COLUMN text TEXT")
     if "destination_num" not in cols:
         conn.execute("ALTER TABLE messages ADD COLUMN destination_num INTEGER")
+
+    node_cols = {row[1] for row in conn.execute("PRAGMA table_info(nodes)").fetchall()}
+    for col, decl in (
+        ("last_snr", "REAL"),
+        ("last_rssi", "INTEGER"),
+        ("last_hops_used", "INTEGER"),
+        ("last_hop_start", "INTEGER"),
+        ("last_via_mqtt", "INTEGER"),
+        ("rf_count", "INTEGER DEFAULT 0"),
+        ("via_mqtt_count", "INTEGER DEFAULT 0"),
+        ("position_count", "INTEGER DEFAULT 0"),
+        ("telemetry_count", "INTEGER DEFAULT 0"),
+    ):
+        if col not in node_cols:
+            conn.execute(f"ALTER TABLE nodes ADD COLUMN {col} {decl}")

src/meshchat/services/export_service.py

@@ -63,7 +63,13 @@ def export_packets_csv(
                     "hop_start":          pkt.hop_start if pkt.hop_start is not None else "",
                     "hop_limit":          pkt.hop_limit if pkt.hop_limit is not None else "",
                     "hops_used":          pkt.hops_used if pkt.hops_used is not None else "",
-                    "via_mqtt":           int(bool(pkt.via_mqtt)),
+                    # is-not-None here too: via_mqtt is bool | None on older
+                    # firmware that never reports the field at all, and
+                    # `int(bool(None))` == 0 was indistinguishable from a
+                    # packet confirmed to be direct RF — silently collapsing
+                    # "unknown" into "confirmed not MQTT" for anyone doing
+                    # offline RF-vs-MQTT analysis of the export.
+                    "via_mqtt":           int(pkt.via_mqtt) if pkt.via_mqtt is not None else "",
                     "transport_mechanism": pkt.transport_mechanism or "",
                 }
                 if include_text:

src/meshchat/services/monitor_store.py

@@ -38,6 +38,15 @@ def _dt_str(dt: datetime | None) -> str | None:
     return dt.isoformat()
 
 
+def _parse_dt(iso_str: str | None) -> datetime | None:
+    if not iso_str:
+        return None
+    try:
+        return datetime.fromisoformat(iso_str)
+    except ValueError:
+        return None
+
+
 class MonitorStore:
     """
     SQLite persistence for the Network Monitor.
@@ -128,6 +137,50 @@ def read_packets(
             log.error("MonitorStore.read_packets failed: %s", exc)
             return []
 
+    def read_packets_as_objects(self, session_id: str, limit: int = 200_000) -> list[NetworkPacket]:
+        """Like read_packets(), reconstructed as NetworkPacket objects.
+
+        Used for exporting a session's full packet history — beyond what
+        PacketIngestor's bounded in-memory ring buffer retains. `text` and
+        `raw_metadata_json` are always None here: message text isn't stored
+        in the packets table at all (it lives in `messages`, keyed by
+        chat-relevant fields the packets table doesn't carry), so callers
+        that need it must merge it back in themselves from whatever
+        still-in-memory packets happen to overlap.
+        """
+        rows = self.read_packets(session_id, limit=limit)
+        packets = []
+        for r in rows:
+            via_mqtt = r.get("via_mqtt")
+            pki_encrypted = r.get("pki_encrypted")
+            want_ack = r.get("want_ack")
+            packets.append(NetworkPacket(
+                session_id=r["session_id"],
+                observed_at=_parse_dt(r.get("observed_at")) or datetime.now(timezone.utc),
+                rx_time=_parse_dt(r.get("rx_time")),
+                sender_num=r.get("sender_num"),
+                sender_id=r.get("sender_id"),
+                destination_num=r.get("destination_num"),
+                packet_id=r.get("packet_id"),
+                channel_index=r.get("channel_index"),
+                portnum=r.get("portnum"),
+                portnum_name=r.get("portnum_name") or "",
+                text=None,
+                payload_size=r.get("payload_size"),
+                rx_snr=r.get("rx_snr"),
+                rx_rssi=r.get("rx_rssi"),
+                hop_start=r.get("hop_start"),
+                hop_limit=r.get("hop_limit"),
+                hops_used=r.get("hops_used"),
+                via_mqtt=bool(via_mqtt) if via_mqtt is not None else None,
+                transport_mechanism=r.get("transport_mechanism"),
+                pki_encrypted=bool(pki_encrypted) if pki_encrypted is not None else None,
+                want_ack=bool(want_ack) if want_ack is not None else None,
+                priority=r.get("priority"),
+                raw_metadata_json=None,
+            ))
+        return packets
+
     def packet_count(self, session_id: str) -> int:
         try:
             conn = self._read_conn()
@@ -414,8 +467,10 @@ def _write_node(self, conn: sqlite3.Connection, node: NodeSnapshot) -> None:
         conn.execute(
             """INSERT INTO nodes
             (node_num, node_id, long_name, short_name, role, hw_model,
-             first_seen, last_heard, packet_count, text_count)
-            VALUES (?,?,?,?,?,?,?,?,?,?)
+             first_seen, last_heard, packet_count, text_count,
+             last_snr, last_rssi, last_hops_used, last_hop_start, last_via_mqtt,
+             rf_count, via_mqtt_count, position_count, telemetry_count)
+            VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
             ON CONFLICT(node_num) DO UPDATE SET
                 node_id=excluded.node_id,
                 long_name=COALESCE(excluded.long_name, long_name),
@@ -436,12 +491,41 @@ def _write_node(self, conn: sqlite3.Connection, node: NodeSnapshot) -> None:
                 -- fields default to 0, never None), so a bare MAX is fine.
                 last_heard=COALESCE(MAX(excluded.last_heard, last_heard), excluded.last_heard, last_heard),
                 packet_count=MAX(excluded.packet_count, packet_count),
-                text_count=MAX(excluded.text_count, text_count)""",
+                text_count=MAX(excluded.text_count, text_count),
+                -- The last_* fields describe the single most recent packet,
+                -- not a running total — MAX() would be wrong here (e.g. a
+                -- fresher 0-hop packet must not lose to a staler 3-hop one
+                -- just because 3 > 0). Take them together only when this
+                -- write's last_heard is at least as new as what's stored,
+                -- so they always describe the same observation last_heard
+                -- itself was just updated to.
+                last_snr = CASE WHEN excluded.last_heard IS NOT NULL
+                    AND (last_heard IS NULL OR excluded.last_heard >= last_heard)
+                    THEN excluded.last_snr ELSE last_snr END,
+                last_rssi = CASE WHEN excluded.last_heard IS NOT NULL
+                    AND (last_heard IS NULL OR excluded.last_heard >= last_heard)
+                    THEN excluded.last_rssi ELSE last_rssi END,
+                last_hops_used = CASE WHEN excluded.last_heard IS NOT NULL
+                    AND (last_heard IS NULL OR excluded.last_heard >= last_heard)
+                    THEN excluded.last_hops_used ELSE last_hops_used END,
+                last_hop_start = CASE WHEN excluded.last_heard IS NOT NULL
+                    AND (last_heard IS NULL OR excluded.last_heard >= last_heard)
+                    THEN excluded.last_hop_start ELSE last_hop_start END,
+                last_via_mqtt = CASE WHEN excluded.last_heard IS NOT NULL
+                    AND (last_heard IS NULL OR excluded.last_heard >= last_heard)
+                    THEN excluded.last_via_mqtt ELSE last_via_mqtt END,
+                rf_count=MAX(excluded.rf_count, rf_count),
+                via_mqtt_count=MAX(excluded.via_mqtt_count, via_mqtt_count),
+                position_count=MAX(excluded.position_count, position_count),
+                telemetry_count=MAX(excluded.telemetry_count, telemetry_count)""",
             (
                 node.node_num, node.node_id, node.long_name, node.short_name,
                 node.role, node.hw_model,
                 _dt_str(node.first_seen), _dt_str(node.last_heard),
                 node.packet_count, node.text_count,
+                node.last_snr, node.last_rssi, node.last_hops_used, node.last_hop_start,
+                int(node.last_via_mqtt) if node.last_via_mqtt is not None else None,
+                node.rf_count, node.via_mqtt_count, node.position_count, node.telemetry_count,
             ),
         )
 

src/meshchat/services/packet_ingestor.py

@@ -349,9 +349,15 @@ def _update_node(self, pkt: NetworkPacket, raw: dict) -> None:
             node.last_hop_start = pkt.hop_start
             node.last_via_mqtt = pkt.via_mqtt
 
-            if pkt.is_via_mqtt:
+            # Tri-state, not pkt.is_via_mqtt (bool(via_mqtt), which coerces
+            # None to False): older firmware that never reports viaMqtt at
+            # all must not be counted as confirmed RF — these counters now
+            # persist across restarts (see MonitorStore._write_node /
+            # PacketIngestor.seed_from_store), so misclassifying "unknown"
+            # as "confirmed RF" here would keep compounding every session.
+            if pkt.via_mqtt is True:
                 node.via_mqtt_count += 1
-            else:
+            elif pkt.via_mqtt is False:
                 node.rf_count += 1
 
             if pkt.portnum == PORTNUM_TEXT:
@@ -635,6 +641,28 @@ def seed_from_store(self, node_rows: list[dict], positions_by_num: dict[int, dic
                 node.last_heard = node.last_heard or _parse_dt(row.get("last_heard"))
                 node.packet_count = max(node.packet_count, row.get("packet_count") or 0)
                 node.text_count = max(node.text_count, row.get("text_count") or 0)
+                node.rf_count = max(node.rf_count, row.get("rf_count") or 0)
+                node.via_mqtt_count = max(node.via_mqtt_count, row.get("via_mqtt_count") or 0)
+                node.position_count = max(node.position_count, row.get("position_count") or 0)
+                node.telemetry_count = max(node.telemetry_count, row.get("telemetry_count") or 0)
+                # Only restore the "last observation" fields if this node
+                # hasn't already been heard live this session — a fresh live
+                # packet is always newer than whatever was persisted before
+                # this app run started, and seed_from_store only fills in
+                # nodes not yet touched this session anyway (see the
+                # `node is None` branch above), but a defensive guard here
+                # keeps that true even if seeding order ever changes.
+                if node.last_snr is None:
+                    node.last_snr = row.get("last_snr")
+                if node.last_rssi is None:
+                    node.last_rssi = row.get("last_rssi")
+                if node.last_hops_used is None:
+                    node.last_hops_used = row.get("last_hops_used")
+                if node.last_hop_start is None:
+                    node.last_hop_start = row.get("last_hop_start")
+                if node.last_via_mqtt is None:
+                    raw_via_mqtt = row.get("last_via_mqtt")
+                    node.last_via_mqtt = bool(raw_via_mqtt) if raw_via_mqtt is not None else None
                 snap = _copy_node_snapshot(node)
 
             self.node_updated.emit(snap)