diff --git a/docs/sensorius_contract.md b/docs/sensorius_contract.md index 7bafc83..3a29ee0 100644 --- a/docs/sensorius_contract.md +++ b/docs/sensorius_contract.md @@ -15,6 +15,8 @@ define the contract. When other docs drift, this document wins. - Nodus publishes retained compact `meta` on connect/reconnect. - Nodus publishes retained `meta/switch` with detailed switch channel topics in the startup identity publish batch when switch channels are present. +- Nodus II publishes retained `meta/config` with saved display and configuration + settings alongside compact startup metadata. Sensorius consumes both snapshots. - After accepted runtime changes, Nodus publishes only `meta/patch`. - Sensorius paces ordinary runtime config writes one key at a time per physical Nodus host and waits for `ack` plus successful `result`. @@ -302,8 +304,10 @@ The payload must include: - `location_group.location`, `location_group.members` - `sensor.sensor_id`, `sensor.location`, `sensor.hardware`, `sensor.data_topic`, - `sensor.event_topic`, `sensor.availability_topic`, - `sensor.display_metrics`, `sensor.display_styles` + `sensor.event_topic`, `sensor.availability_topic` +- `sensor.display_metrics`, `sensor.display_styles` in full metadata (including + Nodus III); compact Nodus II metadata advertises `config_topic` and supplies + these fields in retained `meta/config` instead - optional `sensor.calibration.Device.ALTITUDE_METERS`, reflecting the device TOML calibration altitude in metres (number or numeric string); Sensorius mirrors explicit values, including zero, and preserves existing shadow @@ -759,6 +763,23 @@ Implemented behavior: target version when the target differs from the prior version. A final confirmation timeout is a failed update, not a successful update. +## Saved configuration companion (`meta/config`) + +Nodus II publishes retained `nodus//meta/config`, advertised by +`config_topic` in compact `meta`, with schema `nodus-meta-config/v1`. Its +`sensor` block identifies `sensor_id` and `config_file` and includes ordered +`display_metrics`, matching `display_styles`, and saved calibration. The +companion also carries `time` and `homeassistant` settings. Saved device TOML +values are authoritative, including after device-side edits and a restart. + +Sensorius subscribes to both snapshots (including base-prefixed topics), +accepts either arrival order, and merges configuration into its discovery +cache and `/home//Sensorius/sensor_settings//sensor.toml`. +Missing display fields must preserve existing shadow settings; factory defaults +are only used to seed missing settings. Full metadata remains supported. +Metric slots must keep their positions, including blank slots and temporary +duplicates during a reorder. Styles belong to the corresponding slot. + ## `meta/patch` Canonical non-retained patch payload: diff --git a/sensorius/__init__.py b/sensorius/__init__.py index 247bbb9..92c6f98 100644 --- a/sensorius/__init__.py +++ b/sensorius/__init__.py @@ -4,4 +4,4 @@ release notes, and supporting tooling can report a consistent build identity. """ -__version__ = "v0.26.269.1" +__version__ = "v0.26.270.1" diff --git a/sensorius/saiMQTTIngest.py b/sensorius/saiMQTTIngest.py index a5f2fc4..e2ad5be 100644 --- a/sensorius/saiMQTTIngest.py +++ b/sensorius/saiMQTTIngest.py @@ -477,6 +477,7 @@ def __init__(self, broker="localhost", client_id="rpi_ingest", mqtt_clients=None "nodus/+/status/heartbeat", "nodus/+/meta", "nodus/+/meta/switch", + "nodus/+/meta/config", "nodus/+/fwupdate/result", "nodus/+/meta/patch", "nodus/+/calibration/ack", @@ -537,6 +538,7 @@ def __init__(self, broker="localhost", client_id="rpi_ingest", mqtt_clients=None f"{self.base_topic}/nodus/+/event/calibration_result", f"{self.base_topic}/nodus/+/meta", f"{self.base_topic}/nodus/+/meta/switch", + f"{self.base_topic}/nodus/+/meta/config", f"{self.base_topic}/nodus/+/fwupdate/result", f"{self.base_topic}/nodus/+/meta/patch", f"{self.base_topic}/nodus/+/onboard/hello", @@ -793,12 +795,13 @@ def refresh_nodus_retained_metadata(self, device_id: str = "") -> dict: except Exception: pass - candidates = ["nodus/+/meta", "nodus/+/meta/switch"] + candidates = ["nodus/+/meta", "nodus/+/meta/switch", "nodus/+/meta/config"] if self.base_topic: candidates.extend( [ f"{self.base_topic}/nodus/+/meta", f"{self.base_topic}/nodus/+/meta/switch", + f"{self.base_topic}/nodus/+/meta/config", ] ) topics = [topic for topic in candidates if topic in self.registered_topics] @@ -2335,6 +2338,15 @@ def _on_message_unlocked(self, client, userdata, msg): ) if ok_patch: return + if ( + (not self.nodus_debug_data_only) + and len(parts) == id_index + 3 + and parts[id_index + 1:id_index + 3] == ["meta", "config"] + ): + self._parse_nodus_config_meta( + data, topic_device_id=parts[id_index], retain=retain, + ) + return if ( (not self.nodus_debug_data_only) and len(parts) == id_index + 3 @@ -2840,9 +2852,9 @@ def _parse_nodus_values_payload(self, payload_text: str, data: dict | None) -> d out[key] = val return out or None - def _normalize_display_metrics(self, raw_metrics) -> list[str]: + def _normalize_display_metrics(self, raw_metrics, *, preserve_slots=False) -> list[str]: """ - Normalize display metric hints into an ordered de-duplicated list. + Normalize metric hints; preserve positional slots when writing settings. """ if isinstance(raw_metrics, dict): values = [raw_metrics.get(f"METRIC_{idx}", "") for idx in range(1, 7)] @@ -2851,6 +2863,9 @@ def _normalize_display_metrics(self, raw_metrics) -> list[str]: else: return [] + if preserve_slots: + return [str(value or "").strip() for value in values[:6]] + ordered: list[str] = [] seen: set[str] = set() for raw in values: @@ -3180,10 +3195,24 @@ def _sensor_targets() -> list[dict]: targets.append(primary) return targets + def _display_slots(sensor: dict, field: str) -> OrderedDict[str, str]: + raw = sensor.get(field) + if raw is None: + from .saiSensorSettingsManager import SensorSettingsManager + + sensor_id = str(sensor.get("sensor_id") or "").strip() + if sensor_id: + try: + display = SensorSettingsManager().load(sensor_id).get("Display", {}) + except FileNotFoundError: + display = {} + raw = display.get("Style", {}) if field == "display_styles" else display + return self._meta_metric_slot_map(raw) + if section_key == "display": targets = _sensor_targets() for sensor in targets: - display_metrics = self._meta_metric_slot_map(sensor.get("display_metrics")) + display_metrics = _display_slots(sensor, "display_metrics") display_metrics[key_upper] = "" if value is None else str(value) sensor["display_metrics"] = dict(display_metrics) return bool(targets) @@ -3191,7 +3220,7 @@ def _sensor_targets() -> list[dict]: if section_key == "display.style": targets = _sensor_targets() for sensor in targets: - display_styles = self._meta_metric_slot_map(sensor.get("display_styles")) + display_styles = _display_slots(sensor, "display_styles") display_styles[key_upper] = "" if value is None else str(value) sensor["display_styles"] = dict(display_styles) return bool(targets) @@ -3940,6 +3969,17 @@ def _coerce_switch_state(raw_state) -> bool | None: firmware_version = str(meta.get("version") or "").strip() board_type = _extract_nodus_board_type(meta) + # Config companions and compact identity may arrive in either order. + # Preserve only omitted configuration fields; explicit new values win. + previous = self.discovery_cache.get(base) or {} + previous_sensor = previous.get("sensor") or {} + incoming_sensor = meta.get("sensor") + if isinstance(incoming_sensor, dict) and isinstance(previous_sensor, dict): + if incoming_sensor.get("sensor_id") == previous_sensor.get("sensor_id"): + for field in ("display_metrics", "display_styles", "calibration"): + if field not in incoming_sensor and field in previous_sensor: + incoming_sensor[field] = copy.deepcopy(previous_sensor[field]) + primary_sensor_blob = dict(meta.get("sensor")) if isinstance(meta.get("sensor"), dict) else {} if not primary_sensor_blob: top_sensor_id = str(meta.get("sensor_id") or meta.get("SENSOR_ID") or "").strip() @@ -4071,15 +4111,14 @@ def _coerce_switch_state(raw_state) -> bool | None: ) sensor_serial = self._extract_sensor_serial(sensor_blob, meta) display_metrics = self._normalize_display_metrics( - sensor_blob.get("display_metrics") - or sensor_blob.get("metrics") - or sensor_blob.get("Display") + sensor_blob.get("display_metrics", sensor_blob.get("metrics", sensor_blob.get("Display"))), + preserve_slots=True, ) display_styles = self._normalize_display_styles( sensor_blob.get("display_styles") or sensor_blob.get("styles") ) if display_metrics: - self.expected_gauge_map[sensor_id] = display_metrics + self.expected_gauge_map[sensor_id] = self._normalize_display_metrics(display_metrics) register_sensor = getattr(self.data_logger, "register_sensor", None) if callable(register_sensor): @@ -4122,7 +4161,7 @@ def _coerce_switch_state(raw_state) -> bool | None: "hardware": sensor_hardware, "physical_device_id": base, "config_file": config_file, - "display_metrics": display_metrics, + "display_metrics": display_metrics or None, "display_styles": display_styles, "calibration": sensor_blob.get("calibration"), }) @@ -4342,6 +4381,38 @@ def _coerce_switch_state(raw_state) -> bool | None: return True, subscribed + def _parse_nodus_config_meta(self, payload, *, topic_device_id: str, retain=False): + """Merge a retained saved-settings companion into device discovery.""" + if not isinstance(payload, dict) or payload.get("schema") != "nodus-meta-config/v1": + return False, False + device_id = str(payload.get("device_id") or topic_device_id).strip() + if device_id != topic_device_id: + return False, False + base = self._normalize_host_key(device_id) or device_id + meta = copy.deepcopy(self.discovery_cache.get(base) or {}) + meta.update(schema="nodus-meta/v1", device_id=device_id) + sensor = payload.get("sensor") + if isinstance(sensor, dict): + existing = meta.get("sensor") or {} + if existing.get("sensor_id") != sensor.get("sensor_id"): + existing = {} + meta["sensor"] = {**existing, **copy.deepcopy(sensor)} + for child in meta.get("sensors", []): + if isinstance(child, dict) and child.get("sensor_id") == sensor.get("sensor_id"): + child.update(copy.deepcopy(sensor)) + for field in ("time", "homeassistant"): + if isinstance(payload.get(field), dict): + meta[field] = copy.deepcopy(payload[field]) + result = self._parse_and_subscribe_from_nodus_meta( + meta, topic_device_id=device_id, retain=retain, + ) + info = {"HOSTNAME": base} + for field, section in (("time", "Time"), ("homeassistant", "HomeAssistant")): + if isinstance(payload.get(field), dict): + info[section] = {key.upper(): value for key, value in payload[field].items()} + self._ensure_settings_from_itaot(info, base, [], []) + return result + def _parse_and_subscribe_from_nodus_switch_meta( self, payload: dict, @@ -5982,7 +6053,8 @@ def _canonical_location(value: str | None) -> str: or "" ).strip() remote_display_metrics = self._normalize_display_metrics( - s.get("display_metrics") or s.get("metrics") + s.get("display_metrics", s.get("metrics")), + preserve_slots=True, ) remote_display_styles = self._normalize_display_styles( s.get("display_styles") or s.get("styles") @@ -6044,9 +6116,11 @@ def _canonical_location(value: str | None) -> str: if "Display" not in data or not isinstance(data["Display"], dict): data["Display"] = OrderedDict() display = data["Display"] - chosen_metrics = remote_display_metrics or _display_defaults_for_device( - device_name or device_type, - sensor_hardware, + # Missing compact metadata is not a reset to factory defaults. + chosen_metrics = remote_display_metrics or ( + [str(display.get(f"METRIC_{idx}", "") or "") for idx in range(1, 7)] + if any(f"METRIC_{idx}" in display for idx in range(1, 7)) + else _display_defaults_for_device(device_name or device_type, sensor_hardware) ) for idx in range(6): metric_key = f"METRIC_{idx + 1}" diff --git a/testApparatus/test_mqtt_ingest_liveness.py b/testApparatus/test_mqtt_ingest_liveness.py index 1eece2e..fe0d534 100644 --- a/testApparatus/test_mqtt_ingest_liveness.py +++ b/testApparatus/test_mqtt_ingest_liveness.py @@ -534,14 +534,18 @@ def test_reonboarding_refresh_renews_retained_metadata_subscriptions(monkeypatch assert set(result["topics"]) == { "nodus/+/meta", "nodus/+/meta/switch", + "nodus/+/meta/config", "sensorius/nodus/+/meta", "sensorius/nodus/+/meta/switch", + "sensorius/nodus/+/meta/config", } assert set(ingest.client.subs) == { ("nodus/+/meta", 0), ("nodus/+/meta/switch", 0), + ("nodus/+/meta/config", 0), ("sensorius/nodus/+/meta", 0), ("sensorius/nodus/+/meta/switch", 0), + ("sensorius/nodus/+/meta/config", 0), } @@ -3395,7 +3399,7 @@ def test_ensure_settings_from_itaot_overwrites_shadow_locations_when_payload_is_ assert switch_saved["Switch"]["SWITCH_LOCATION"] == "Unknown" -def test_ensure_settings_from_itaot_resets_sensor_display_metrics_to_meta_defaults(tmp_path, monkeypatch): +def test_ensure_settings_from_itaot_preserves_display_when_metadata_omits_it(tmp_path, monkeypatch): ingest = _build_ingest(monkeypatch) sensor_root = tmp_path / "sensor_settings" @@ -3459,12 +3463,8 @@ def test_ensure_settings_from_itaot_resets_sensor_display_metrics_to_meta_defaul ) sensor_saved = sensor_mgr.load("apvpd-test123") - assert sensor_saved["Display"]["METRIC_1"] == "Air Quality" - assert sensor_saved["Display"]["METRIC_2"] == "Temperature" - assert sensor_saved["Display"]["METRIC_3"] == "Rel-Humidity" - assert sensor_saved["Display"]["METRIC_4"] == "Ambient VPD" - assert sensor_saved["Display"]["METRIC_5"] == "Dewpoint Deficit" - assert sensor_saved["Display"]["METRIC_6"] == "dewVPD Risk" + for index in range(1, 7): + assert sensor_saved["Display"][f"METRIC_{index}"] == f"Old {index}" def test_nodus_meta_clears_switch_shadow_wiring_when_meta_fields_are_blank(tmp_path, monkeypatch): @@ -3981,3 +3981,85 @@ def receive(payload): receive(cached) assert mgr.load(sid)['Calibration']['Device']['ALTITUDE_METERS'] == 1719 assert mgr.load('aqi-1jm5s1')['Calibration']['Device']['ALTITUDE_METERS'] == 1200 + + +@pytest.mark.parametrize("config_first", [False, True]) +@pytest.mark.parametrize("prefix", ["", "sensorius/"]) +def test_cpynodus_ii_saved_display_companion_and_incremental_slots( + tmp_path, monkeypatch, config_first, prefix, +): + """Replay II startup/restart and paced display edits through MQTT ingestion.""" + ingest = _build_ingest(monkeypatch) + real_sensor_mgr = saiSensorSettingsManager.SensorSettingsManager + real_switch_mgr = saiSwitchSettingsManager.SwitchSettingsManager + monkeypatch.setattr(saiSensorSettingsManager, "SensorSettingsManager", + lambda *_a, **_k: real_sensor_mgr(str(tmp_path / "sensors"))) + monkeypatch.setattr(saiSwitchSettingsManager, "SwitchSettingsManager", + lambda *_a, **_k: real_switch_mgr(str(tmp_path / "switches"))) + monkeypatch.setattr(saiSettings.saiSettings, "DEFAULT_BASE_DIR", str(tmp_path / "systems")) + mgr = real_sensor_mgr(str(tmp_path / "sensors")) + sid = "co2-pmoopn" + old_metrics = ["CO2", "Temperature", "Rel-Humidity", "Ambient VPD", "Dewpoint Deficit", "dewVPD Risk"] + metrics = ["Temperature", "Rel-Humidity", "Ambient VPD", "Dew Point Deficit", "DewVPD Risk", "CO2"] + styles = ["Gauge", "Graph6hr", "Graph24hr", "Gauge", "Graph6hr", "Graph24hr"] + mgr.save(sid, {"Sensor": {"SENSOR_ID": sid, "DEVICE": "co2", "TYPE": "nodus"}, + "Display": {**dict(zip([f"METRIC_{i}" for i in range(1, 7)], old_metrics)), + "METRIC_DISPLAY_MODE": "Pick 6"}}) + identity = {"sensor_id": sid, "device": "co2", "config_file": "sensor_i2c.toml"} + compact = {"schema": "nodus-meta/v1", "device_id": sid, "mcu": "pico2w", + "config_topic": f"nodus/{sid}/meta/config", + "sensor": {**identity, "data_topic": f"nodus/{sid}/data"}} + companion = {"schema": "nodus-meta-config/v1", "device_id": sid, + "sensor": {**identity, "display_metrics": metrics, "display_styles": styles}, + "time": {"TZ": "UTC"}} + + def receive(suffix, payload, retain=True): + ingest._on_message(ingest.client, None, + _Msg(f"{prefix}nodus/{sid}/{suffix}", json.dumps(payload), retain=retain)) + + assert f"{prefix}nodus/+/meta/config" in ingest.registered_topics + if config_first: + receive("meta/config", companion) + receive("meta", compact) + else: + receive("meta", compact) + assert mgr.load(sid)["Display"]["METRIC_1"] == "CO2" + # A patch can arrive before the saved-settings companion. Preserve + # other slots using the existing shadow until the snapshot arrives. + receive("meta/patch", {"schema": "nodus-meta-patch/v1", "device_id": sid, + "source": "config_set", "message_id": "cfg-early", + "updates": [{"sensor_id": sid, "name": "sensor_i2c.toml", + "section": "Display", "key": "METRIC_2", "value": "CO2"}]}, False) + early = mgr.load(sid)["Display"] + assert early["METRIC_1"] == early["METRIC_2"] == "CO2" + assert early["METRIC_3"] == "Rel-Humidity" + assert early["METRIC_6"] == "dewVPD Risk" + receive("meta/config", companion) + saved = mgr.load(sid) + assert [saved["Display"][f"METRIC_{i}"] for i in range(1, 7)] == metrics + assert [saved["Display"]["Style"][f"METRIC_{i}"] for i in range(1, 7)] == styles + assert saved["Nodus"]["CONFIG_FILE"] == "sensor_i2c.toml" + assert saved["Display"]["METRIC_DISPLAY_MODE"] == "Pick 6" + + # Reordering one key at a time temporarily duplicates a metric. Do not + # shift any other slot, including the styles paired with those slots. + for slot, value in [(1, "CO2"), (6, "Temperature"), (3, "")]: + receive("meta/patch", {"schema": "nodus-meta-patch/v1", "device_id": sid, + "source": "config_set", "message_id": f"cfg-{slot}", + "updates": [{"sensor_id": sid, "name": "sensor_i2c.toml", + "section": "Display", "key": f"METRIC_{slot}", "value": value}]}, False) + metrics[slot - 1] = value + saved = mgr.load(sid) + assert [saved["Display"][f"METRIC_{i}"] for i in range(1, 7)] == metrics + assert [saved["Display"]["Style"][f"METRIC_{i}"] for i in range(1, 7)] == styles + receive("meta/patch", {"schema": "nodus-meta-patch/v1", "device_id": sid, + "source": "config_set", "message_id": "cfg-style", + "updates": [{"sensor_id": sid, "name": "sensor_i2c.toml", + "section": "Display.Style", "key": "METRIC_6", "value": "Gauge"}]}, False) + assert mgr.load(sid)["Display"]["Style"]["METRIC_6"] == "Gauge" + + # A device-side edit followed by restart must replace the hub shadow. + companion["sensor"]["display_metrics"] = list(reversed(old_metrics)) + receive("meta", compact) + receive("meta/config", companion) + assert mgr.load(sid)["Display"]["METRIC_1"] == "dewVPD Risk"