Fix dashboard refresh and use Ecowitt 24-hour rainfall

docs/operations.md

@@ -95,6 +95,21 @@ Useful web routes:
   `mqtt_ingest_health` reports callback counts, latency, and contention. Slow
   callback warnings identify only the MQTT topic and never include payloads.
 
+Dashboard values normally refresh every 15 seconds. A failed response or a
+request exceeding 12 seconds (including the JSON body) shows **Live updates
+delayed** above the dashboard. Existing values remain visible while automatic
+retries continue; the notice clears after a successful update. Expanding
+metrics or changing their display uses the latest received values.
+
+For comparable Web UI profiles, close other dashboard tabs on the tested hub
+and run `testApparatus/profile_webui.py` with the same sample count and timeout
+for each host. The profiler uses headless Chromium, reports progress, and
+checks completion and success of the dashboard's live-update cycle. Its
+navigation timeout includes waiting for server headers; failed navigation is
+recorded and the corresponding modal scenarios are skipped. Review both load
+and refresh errors alongside timing summaries. The profiler timeout does not
+change the dashboard's own 12-second request deadline.
+
 For Ecowitt troubleshooting, confirm the Sensorius host can reach the saved GW
 URL over plain HTTP, the gateway remains on the same LAN address, and **Find
 Sensors** succeeds in **General Settings > Add Device > Ecowitt Gateway**. A

docs/sensors.md

@@ -30,6 +30,9 @@ WeeWX station ingest:
 - Copies the WeeWX station model, station type, and driver into the station
   sensor config when a readable WeeWX config exists on the host.
 - Adds station metrics to the same dashboard and DB paths.
+- Continues deriving `Rain Last 24h` from interval `Rain`. The window query uses
+  existing sensor/metric/epoch indexes and retains compatibility with older
+  readings that have no epoch value.
 
 Ecowitt gateway ingest:
 
@@ -59,9 +62,16 @@ See the Ecowitt section in the [user guide](user_guide.md#ecowitt-gateway).
   reading set is written to SQLite.
 - Uses distinct channel metric names such as `WH31 CH1 Temperature_F`,
   `Soil Moisture CH3`, `PM2.5 CH2`, and `Leaf Wetness CH1`.
-- Stores Ecowitt rain day/week/month/year values as cumulative metrics. Only a
-  restart-safe day-total delta is written as interval `Rain`, allowing the
-  logger to derive `Rain Last 24h` correctly. The configured gateway reset
+- Stores Ecowitt rain day/week/month/year values as cumulative metrics and uses
+  the selected traditional or piezo source's `0x7C` value directly as
+  `Rain Last 24h`, converting its declared units to inches. This rolling total
+  differs from `Rain Day`, which resets at the configured hour. Ecowitt totals
+  are neither derived during ingestion nor recalculated for dashboard reads.
+  A packet without a usable `0x7C` clears the current total to unavailable;
+  older gateway firmware does not receive a derived substitute. Existing
+  historical records remain intact, including totals derived by older releases.
+- A restart-safe day-total delta is still written as interval `Rain` for graphs
+  and existing consumers. The configured gateway reset
   hour is checked before subtracting counters, including when the new day's
   counter has already reached or exceeded the prior day's total.
 - Makes observed Ecowitt metrics available behind the dashboard sensor-row

sensorius/__init__.py

@@ -4,4 +4,4 @@
 release notes, and supporting tooling can report a consistent build identity.
 """
 
-__version__ = "v0.26.253.8"
+__version__ = "v0.26.257.1"

sensorius/saiDataLogger.py

@@ -712,37 +712,41 @@ def _sum_metric_window_on_conn(
         dedupe_epoch_minute: bool = False,
     ) -> float | None:
         start_epoch = float(end_epoch) - float(window_sec)
+        # Keep indexed epoch ranges separate from the legacy NULL-epoch path.
+        # LOWER()/COALESCE() around indexed columns forced whole-table scans.
+        window_query = """
+            SELECT ts_epoch AS epoch, value FROM readings
+            WHERE sensor_id = ? COLLATE NOCASE AND metric = ? COLLATE NOCASE
+              AND ts_epoch >= ? AND ts_epoch <= ? AND value IS NOT NULL
+            UNION ALL
+            SELECT CAST(strftime('%s', timestamp) AS REAL) AS epoch, value FROM readings
+            WHERE sensor_id = ? COLLATE NOCASE AND metric = ? COLLATE NOCASE
+              AND ts_epoch IS NULL AND value IS NOT NULL
+              AND CAST(strftime('%s', timestamp) AS REAL) >= ?
+              AND CAST(strftime('%s', timestamp) AS REAL) <= ?
+        """
+        params = (sensor_id, metric, start_epoch, float(end_epoch)) * 2
         if dedupe_epoch_minute:
             row = conn.execute(
-                """
+                f"""
                 SELECT SUM(bucket_value)
                 FROM (
                     SELECT
-                        CAST(COALESCE(ts_epoch, CAST(strftime('%s', timestamp) AS REAL)) / 60 AS INTEGER) AS epoch_minute,
+                        CAST(epoch / 60 AS INTEGER) AS epoch_minute,
                         MAX(COALESCE(value, 0)) AS bucket_value
-                    FROM readings
-                    WHERE LOWER(sensor_id) = LOWER(?)
-                      AND LOWER(metric) = LOWER(?)
-                      AND value IS NOT NULL
-                      AND COALESCE(ts_epoch, CAST(strftime('%s', timestamp) AS REAL)) >= ?
-                      AND COALESCE(ts_epoch, CAST(strftime('%s', timestamp) AS REAL)) <= ?
+                    FROM ({window_query})
                     GROUP BY epoch_minute
                 )
                 """,
-                (sensor_id, metric, start_epoch, float(end_epoch)),
+                params,
             ).fetchone()
         else:
             row = conn.execute(
-                """
+                f"""
                 SELECT SUM(COALESCE(value, 0))
-                FROM readings
-                WHERE LOWER(sensor_id) = LOWER(?)
-                  AND LOWER(metric) = LOWER(?)
-                  AND value IS NOT NULL
-                  AND COALESCE(ts_epoch, CAST(strftime('%s', timestamp) AS REAL)) >= ?
-                  AND COALESCE(ts_epoch, CAST(strftime('%s', timestamp) AS REAL)) <= ?
+                FROM ({window_query})
                 """,
-                (sensor_id, metric, start_epoch, float(end_epoch)),
+                params,
             ).fetchone()
         if not row or row[0] is None:
             return None
@@ -785,6 +789,8 @@ def _derive_rain_window_metrics_on_conn(
         *,
         end_epoch: float,
     ) -> dict:
+        if str(sensor_id).lower().startswith("ecowitt-"):
+            return {}
         if self._metric_key(values, RAIN_INTERVAL_METRIC) is None:
             return {}
         total = self._sum_metric_window_on_conn(
@@ -801,6 +807,8 @@ def _derive_rain_window_metrics_on_conn(
 
     def _with_fallback_derived_metrics(self, sensor_id: str, values: dict) -> dict:
         out = dict(values or {})
+        if str(sensor_id).lower().startswith("ecowitt-"):
+            return out
         if self._metric_key(out, RAIN_INTERVAL_METRIC) is not None:
             total = self.get_metric_sum_for_window(
                 sensor_id,
@@ -1433,11 +1441,17 @@ def get_time_series(self, sensor_id: str, metric: str,
     # ------------------------------- SENSOR API -------------------
 
     def log_readings(self, timestamp, sensor_id, values: dict):
-        """Fast writer using a dedicated WAL connection + in-RAM snapshot."""
+        """Persist readings and preserve Ecowitt's gateway-supplied rain total."""
         timestamp, ts_epoch = _normalize_timestamp_input(
             timestamp, getattr(self, "local_tz", LOCAL_TIMEZONE)
         )
-        raw_values = self._strip_derived_input_metrics(values or {})
+        if str(sensor_id).lower().startswith("ecowitt-"):
+            raw_values = dict(values or {})
+            # Missing gateway totals must clear old snapshots, including totals
+            # derived by earlier releases. Historical records remain untouched.
+            raw_values.setdefault(WEEWX_RAIN_24H_METRIC, None)
+        else:
+            raw_values = self._strip_derived_input_metrics(values or {})
         if any(
             not isinstance(value, (type(None), int, float, str, bytes, bytearray, memoryview))
             or (isinstance(value, int) and not -(2 ** 63) <= value < 2 ** 63)

sensorius/saiHtml.py

@@ -1716,6 +1716,7 @@ def _status_color_hex(status: str) -> str:
     yield "</h2>"
 
     yield "<p id='update_time'>--</p>"
+    yield "<p id='dashboard_refresh_status' role='status' hidden>Live updates delayed. Displayed readings may be out of date. Retrying automatically.</p>"
 
     yield "<style>"
     yield ".dash-top-row{display:grid;grid-template-columns:repeat(auto-fit,minmax(206px,230px));justify-content:center;align-items:stretch;column-gap:.75rem;row-gap:.75rem;margin:1rem auto 0;width:min(100%,1198px);position:relative;}"
@@ -2347,11 +2348,13 @@ def _format_stat_value(stat_value):
     yield "const dashboardExtrasWarmRetryMs = 5000;"
     yield "const micrographFetchTimeoutMs = 20000;"
     yield "const micrographInflightStaleMs = 30000;"
-    yield "function fetchWithTimeout(url, options = {}, timeoutMs = dashboardFetchTimeoutMs) {"
+    yield "function fetchWithTimeout(url, options = {}, timeoutMs = dashboardFetchTimeoutMs, consumeResponse = null) {"
     yield "  const controller = new AbortController();"
-    yield "  const timer = window.setTimeout(() => controller.abort(), timeoutMs);"
+    yield "  let timer;"
+    yield "  const timeout = new Promise((resolve, reject) => { timer = window.setTimeout(() => { controller.abort(); reject(new Error('Request timed out')); }, timeoutMs); });"
     yield "  const opts = Object.assign({}, options, { signal: controller.signal });"
-    yield "  return fetch(url, opts).finally(() => window.clearTimeout(timer));"
+    yield "  const response = fetch(url, opts).then(res => consumeResponse ? consumeResponse(res) : res);"
+    yield "  return Promise.race([response, timeout]).finally(() => window.clearTimeout(timer));"
     yield "}"
     yield f"const displayStyle = {json.dumps(display_style_js)};"
     yield "window.displayStyle = displayStyle;"
@@ -5219,6 +5222,8 @@ def _format_stat_value(stat_value):
     yield "  window.__updateGaugesLastDurationMs = Math.max(0, finishedAt - startedAt);"
     yield "  window.__updateGaugesLastOk = !!(result && result.ok);"
     yield "  window.__updateGaugesLastError = String((result && result.error) || '');"
+    yield "  const refreshStatus = document.getElementById('dashboard_refresh_status');"
+    yield "  if (refreshStatus) refreshStatus.hidden = window.__updateGaugesLastOk;"
     yield "  window.__updateGaugesInFlight = false;"
     yield "  window.__updateGaugesStartedAt = 0;"
     yield "  if (window.__updateGaugesQueued) {"
@@ -5235,29 +5240,27 @@ def _format_stat_value(stat_value):
     yield "    return;"
     yield "  }"
     yield "  const nowForInflight = Date.now();"
-    yield "  if (window.__updateGaugesInFlight && (nowForInflight - window.__updateGaugesStartedAt) < (dashboardFetchTimeoutMs + 3000)) {"
+    yield "  if (window.__updateGaugesInFlight) {"
     yield "    window.__updateGaugesQueued = true;"
     yield "    return;"
     yield "  }"
     yield "  window.__updateGaugesInFlight = true;"
     yield "  window.__updateGaugesStartedAt = nowForInflight;"
     yield "  window.__updateGaugesRunSeq = (window.__updateGaugesRunSeq || 0) + 1;"
-    yield "  window.__updateGaugesLastOk = false;"
-    yield "  window.__updateGaugesLastError = '';"
     yield "  const finishOptions = { ignoreVisibility, ignoreModal };"
+    yield "  try {"
     yield "  const sensorIdEl = document.getElementById('sensor_id');"
     yield "  const sensorId = sensorIdEl ? sensorIdEl.value : 'All';"
     yield "  let d = null;"
     #yield "  console.warn('updateGauges: step 1 - get sensor data');"
 
     yield "  try {"
-    yield "    const res = await fetchWithTimeout(window.location.pathname + '?json_only=true&sensor_id=' + encodeURIComponent(sensorId), { cache: 'no-store' }, dashboardFetchTimeoutMs);"
-    yield "    if (!res.ok) {"
-    yield "      console.warn('updateGauges: non-OK response', res.status);"
-    yield "      finishUpdateGaugesRun(finishOptions, { ok: false, error: `http_${res.status}` });"
-    yield "      return;"
-    yield "    }"
-    yield "    d = await res.json();"
+    yield "    d = await fetchWithTimeout(window.location.pathname + '?json_only=true&sensor_id=' + encodeURIComponent(sensorId), { cache: 'no-store' }, dashboardFetchTimeoutMs, async res => {"
+    yield "      if (!res.ok) throw new Error(`http_${res.status}`);"
+    yield "      return await res.json();"
+    yield "    });"
+    yield "    if (!d || typeof d !== 'object' || !d.values || typeof d.values !== 'object' || Array.isArray(d.values)) throw new Error('Invalid dashboard response');"
+    yield "    for (const [sid, latest] of Object.entries(d.values || {})) currentValues[sid] = Object.assign({}, latest);"
     yield "    __lastJsonOnly = d;"
     yield "    __lastJsonOnlyAtMs = Date.now();"
     yield "    try {"
@@ -5451,6 +5454,10 @@ def _format_stat_value(stat_value):
     yield "    window.notifyFullscreenGraphDataChanged('sensor');"
     yield "  }"
     yield "  finishUpdateGaugesRun(finishOptions, { ok: true });"
+    yield "  } catch (e) {"
+    yield "    console.warn('updateGauges: update failed', e);"
+    yield "    finishUpdateGaugesRun(finishOptions, { ok: false, error: String(e) });"
+    yield "  }"
     yield ""
     yield "}"
     yield "window.updateGauges = updateGauges;"

sensorius/sensor_modules/station_ecowitt.py

@@ -261,6 +261,7 @@ def _parse_rain(items: Any, out: dict[str, float]) -> None:
     names = {
         "0x0d": "Rain Event",
         "0x0e": "Rain Rate",
+        "0x7c": "Rain Last 24h",
         "0x10": "Rain Day",
         "0x11": "Rain Week",
         "0x12": "Rain Month",