From 2291a002f0ed29a743ca333f20ec0687d3ec4e47 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 21:53:49 +0200 Subject: [PATCH 01/24] A voice command is refused before it is sent if it would pass the daily budget The voice agent only checked whether the budget was already spent, so the last command of the day went through in full and could end it over budget. It now estimates the command from its request size, as contexts and AI Task do, holds the estimate while the call runs, and reports the size to the tracker so the estimate learns from voice requests too. --- custom_components/jev/conversation.py | 27 ++++++++++++++------- tests/test_conversation.py | 35 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 359094b..07cc3a2 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -53,6 +53,7 @@ from .coordinator import JevRuntimeData from .entity import build_device_info from .interpret import build_questions, interpret +from .payload import payload_bytes from .snapshot import async_snapshot _LOGGER = logging.getLogger(__name__) @@ -154,14 +155,7 @@ async def _async_handle_message( ) -> conversation.ConversationResult: runtime: JevRuntimeData = self._entry.runtime_data - # The budget covers voice as well as sensors, because a satellite that - # mishears a wake word all night is exactly the runaway it exists to stop. runtime.usage.roll_over(dt_util.now().date()) - if runtime.usage.would_exceed(): - return await self._fall_back( - user_input, "the daily token budget is spent", "budget_spent" - ) - snapshot = async_snapshot(self.hass, MAX_CONVERSATION_ENTITIES) if not snapshot.entities: return await self._fall_back(user_input, "no entities are exposed to Assist") @@ -169,8 +163,23 @@ async def _async_handle_message( questions = build_questions(user_input.text, snapshot, MAX_CONVERSATION_ENTITIES) state = snapshot.as_state() | {"command": user_input.text} + # The budget covers voice as well as sensors, because a satellite that + # mishears a wake word all night is exactly the runaway it exists to stop. + # The check is on this command's estimate, the same as a context's, so the + # last command of the day cannot take the total past the budget. + request_bytes = payload_bytes(state, questions, runtime.model) + estimate = runtime.usage.estimate_tokens(request_bytes) + if runtime.usage.would_exceed_with(estimate): + return await self._fall_back( + user_input, + f"the daily token budget has {runtime.usage.remaining()} tokens left " + f"and this command needs about {estimate}", + "budget_spent", + ) + try: - response = await runtime.client.ask(state, questions) + with runtime.usage.reservation(estimate): + response = await runtime.client.ask(state, questions) except JevAuthError as err: _LOGGER.error("TypeSafe rejected the API key: %s", err) self._entry.async_start_reauth(self.hass) @@ -183,7 +192,7 @@ async def _async_handle_message( user_input, f"TypeSafe did not answer: {err}", "unavailable" ) - runtime.usage.record(response.usage.input_tokens) + runtime.usage.record(response.usage.input_tokens, request_bytes) runtime.model_version = response.model or runtime.model_version runtime.usage.notify() diff --git a/tests/test_conversation.py b/tests/test_conversation.py index a684f01..16a3e64 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -26,6 +26,7 @@ ) from custom_components.jev.conversation import _render_state_answer from custom_components.jev.interpret import find_brightness +from custom_components.jev.payload import payload_bytes from .conftest import PROBE_TOKENS, build_response @@ -344,6 +345,40 @@ async def test_a_spent_budget_stops_voice_too(hass, house, mock_client): assert "budget is spent" in result.response.speech["plain"]["speech"] +async def test_a_command_that_would_pass_the_budget_is_not_sent(hass, house, mock_client): + # One token short of the budget. would_exceed() says there is room, and on + # 1.15 the command went through and ended the day about 1,000 tokens over. + hass.config_entries.async_update_entry(house, options={"daily_token_budget": 1000}) + await hass.async_block_till_done() + house.runtime_data.usage.input_tokens = 999 + mock_client.ask.reset_mock() + + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + result = await converse(hass, "kitchen light on") + await hass.async_block_till_done() + + assert mock_client.ask.await_count == 0 + assert calls == [] + assert "budget is spent" in result.response.speech["plain"]["speech"] + + +async def test_a_voice_command_teaches_the_estimate(hass, house, mock_client): + # The estimate reads bytes per token from the last call it could measure. A + # voice command that did not report its size left the estimate on whatever + # the last context taught it, which is a different shape of request. + mock_client.ask.return_value = build_response(**answer_set()) + usage = house.runtime_data.usage + usage.bytes_per_token = 1.0 + + await converse(hass, "kitchen light on") + await hass.async_block_till_done() + + sent_state, sent_questions = mock_client.ask.await_args.args + sent = payload_bytes(sent_state, sent_questions, house.runtime_data.model) + assert usage.bytes_per_token == sent / 321 + + async def test_a_rejected_key_is_said_out_loud(hass, house, mock_client): from jevclient import JevAuthError From d2c5853e5b738a1b8f98b8d1f389d6247da9561a Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 21:55:39 +0200 Subject: [PATCH 02/24] The Assist debug view shows what Jev answered for each command Each command adds one agent detail to the conversation trace: the decision, the latency and tokens, and every answer with its probabilities. The debug view then shows why a command picked a device or fell back, without a diagnostics download. --- custom_components/jev/conversation.py | 27 +++++++++++++++++++++------ tests/test_conversation.py | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 07cc3a2..f56f8a4 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -197,13 +197,28 @@ async def _async_handle_message( runtime.usage.notify() decision = interpret(response, user_input.text, snapshot, self._min_confidence) - runtime.conversation_traces.appendleft( + trace = { + "text": user_input.text, + "latency_ms": response.latency_ms, + "input_tokens": response.usage.input_tokens, + "exposed_entities": len(snapshot.entities), + **asdict(decision), + } + runtime.conversation_traces.appendleft(trace) + # The Assist debug view shows this beside the pipeline's own steps. It + # carries each answer as well, with its distribution, because "why did it + # pick the office light" is answered by the entity question's + # probabilities and by nothing in the decision alone. Diagnostics keep the + # shorter record, since they hold the last few commands in memory. + chat_log.async_trace( { - "text": user_input.text, - "latency_ms": response.latency_ms, - "input_tokens": response.usage.input_tokens, - "exposed_entities": len(snapshot.entities), - **asdict(decision), + "jev": trace + | { + "model": response.model, + "answers": { + key: asdict(answer) for key, answer in response.answers.items() + }, + } } ) diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 16a3e64..f001889 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -10,6 +10,7 @@ import pytest from homeassistant.components import conversation +from homeassistant.components.conversation.trace import async_get_traces from homeassistant.components.homeassistant.exposed_entities import async_expose_entity from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.core import Context, ServiceCall @@ -515,6 +516,32 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH +async def test_the_assist_debug_view_shows_what_jev_answered(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer( + choice="light.kitchen", + probabilities={"light.kitchen": 0.8, "light.office": 0.2}, + confidence=0.8, + ) + ) + ) + await converse(hass, "kitchen light on") + await hass.async_block_till_done() + + events = async_get_traces()[-1].as_dict()["events"] + details = [e["data"]["jev"] for e in events if e["event_type"] == "agent_detail"] + assert len(details) == 1 + jev = details[0] + assert jev["text"] == "kitchen light on" + assert jev["intent_type"] == "HassTurnOn" + assert jev["input_tokens"] == 321 + assert jev["answers"]["entity"]["probabilities"] == { + "light.kitchen": 0.8, + "light.office": 0.2, + } + + @pytest.mark.parametrize( ("text", "expected"), [ From 8b241a30d7a34598a50e9bd423a8fe18f878742f Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 22:02:30 +0200 Subject: [PATCH 03/24] Voice asks which device you mean when two fit the name When the action is clear and the entity answer is split between two exposed devices, the agent asks "Do you mean A or B?" and keeps the microphone open. The reply gets one small question about those two devices, and the first command then runs on the one it picked. A reply that picks neither is handled as a new command. The question lasts as long as the chat session. --- custom_components/jev/conversation.py | 191 +++++++++++++++- custom_components/jev/interpret.py | 63 +++++- custom_components/jev/strings.json | 1 + custom_components/jev/translations/cs.json | 1 + custom_components/jev/translations/da.json | 1 + custom_components/jev/translations/de.json | 1 + custom_components/jev/translations/en.json | 1 + custom_components/jev/translations/es.json | 1 + custom_components/jev/translations/fr.json | 1 + custom_components/jev/translations/it.json | 1 + custom_components/jev/translations/nl.json | 1 + custom_components/jev/translations/pl.json | 1 + custom_components/jev/translations/pt-BR.json | 1 + custom_components/jev/translations/ru.json | 1 + custom_components/jev/translations/sv.json | 1 + .../jev/translations/zh-Hans.json | 1 + tests/test_conversation.py | 207 +++++++++++++++++- 17 files changed, 463 insertions(+), 12 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index f56f8a4..7bdc7c0 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -25,8 +25,9 @@ import logging import re from collections.abc import Mapping -from dataclasses import asdict, dataclass -from typing import Literal +from dataclasses import asdict, dataclass, field, replace +from datetime import datetime +from typing import Any, Literal from homeassistant.components import conversation from homeassistant.components.conversation.models import AbstractConversationAgent @@ -37,10 +38,18 @@ from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.helpers import template, translation +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback from homeassistant.util import dt as dt_util from homeassistant.util import language as language_util -from jevclient import JevAuthError, JevError +from jevclient import ( + Choice, + ChoiceAnswer, + JevAuthError, + JevError, + JevResponse, + Question, +) from .const import ( CONF_ALLOW_WHOLE_HOME, @@ -52,9 +61,9 @@ ) from .coordinator import JevRuntimeData from .entity import build_device_info -from .interpret import build_questions, interpret +from .interpret import NONE, Interpretation, build_questions, interpret, spoken_name from .payload import payload_bytes -from .snapshot import async_snapshot +from .snapshot import HomeSnapshot, async_snapshot _LOGGER = logging.getLogger(__name__) @@ -76,11 +85,25 @@ "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", } PARALLEL_UPDATES = 0 +@dataclass(slots=True) +class _Pending: + """A command held while the agent asks which device it meant.""" + + text: str + response: JevResponse + snapshot: HomeSnapshot + candidates: tuple[str, str] + expires: datetime = field( + default_factory=lambda: dt_util.utcnow() + CONVERSATION_TIMEOUT + ) + + async def async_setup_entry( hass: HomeAssistant, entry: ConfigEntry, @@ -101,6 +124,8 @@ def __init__(self, entry: ConfigEntry) -> None: self._attr_unique_id = f"{entry.entry_id}_conversation" runtime: JevRuntimeData = entry.runtime_data self._attr_device_info = build_device_info(entry.entry_id, runtime) + # Conversation id to the command waiting on its reply. + self._pending: dict[str, _Pending] = {} @property def supported_languages(self) -> list[str] | Literal["*"]: @@ -156,12 +181,48 @@ async def _async_handle_message( runtime: JevRuntimeData = self._entry.runtime_data runtime.usage.roll_over(dt_util.now().date()) + if (pending := self._take_pending(chat_log.conversation_id)) is not None: + resolved = await self._resolve(user_input, chat_log, pending) + if resolved is not None: + return resolved + snapshot = async_snapshot(self.hass, MAX_CONVERSATION_ENTITIES) if not snapshot.entities: return await self._fall_back(user_input, "no entities are exposed to Assist") questions = build_questions(user_input.text, snapshot, MAX_CONVERSATION_ENTITIES) state = snapshot.as_state() | {"command": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + decision = interpret(response, user_input.text, snapshot, self._min_confidence) + self._trace( + chat_log, + response, + { + "text": user_input.text, + "exposed_entities": len(snapshot.entities), + **asdict(decision), + }, + ) + + if decision.candidates is not None: + return await self._ask_which( + user_input, + chat_log, + _Pending(user_input.text, response, snapshot, decision.candidates), + ) + return await self._act(user_input, decision, user_input.text) + + async def _ask( + self, + user_input: conversation.ConversationInput, + state: dict[str, Any], + questions: dict[str, Question], + ) -> JevResponse | conversation.ConversationResult: + """One call to Jev, inside the budget. A result means it did not answer.""" + runtime: JevRuntimeData = self._entry.runtime_data # The budget covers voice as well as sensors, because a satellite that # mishears a wake word all night is exactly the runaway it exists to stop. @@ -195,14 +256,20 @@ async def _async_handle_message( runtime.usage.record(response.usage.input_tokens, request_bytes) runtime.model_version = response.model or runtime.model_version runtime.usage.notify() + return response - decision = interpret(response, user_input.text, snapshot, self._min_confidence) + def _trace( + self, + chat_log: conversation.ChatLog, + response: JevResponse, + record: dict[str, Any], + ) -> None: + """Keep what one call decided, for diagnostics and the Assist debug view.""" + runtime: JevRuntimeData = self._entry.runtime_data trace = { - "text": user_input.text, "latency_ms": response.latency_ms, "input_tokens": response.usage.input_tokens, - "exposed_entities": len(snapshot.entities), - **asdict(decision), + **record, } runtime.conversation_traces.appendleft(trace) # The Assist debug view shows this beside the pipeline's own steps. It @@ -222,6 +289,110 @@ async def _async_handle_message( } ) + # --- asking which device --- + + def _take_pending(self, conversation_id: str) -> _Pending | None: + """The command waiting on this conversation's reply, if it is still live. + + A pending command lasts as long as Home Assistant keeps the chat session, + so a reply that arrives in a new session is never read as an answer. + """ + now = dt_util.utcnow() + for key in [k for k, v in self._pending.items() if v.expires < now]: + del self._pending[key] + return self._pending.pop(conversation_id, None) + + async def _ask_which( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult: + """Ask which of two devices the command meant, and keep the command.""" + first, second = (pending.snapshot.by_id(e) for e in pending.candidates) + assert first is not None and second is not None + names = spoken_name(first, second) + assert names is not None + language = user_input.language or self.hass.config.language + text = (await self._lines(language))["which_device"].format( + first=names[0], second=names[1] + ) + self._pending[chat_log.conversation_id] = pending + # In the chat log, the next turn in this conversation carries the question + # it answers, and an LLM fallback agent reads the same history. + chat_log.async_add_assistant_content_without_tools( + conversation.AssistantContent(agent_id=self.entity_id, content=text) + ) + response = ha_intent.IntentResponse(language=user_input.language) + response.async_set_speech(text) + # A satellite opens the microphone again for the answer. + return conversation.ConversationResult( + response=response, + conversation_id=chat_log.conversation_id, + continue_conversation=True, + ) + + async def _resolve( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult | None: + """Run the kept command on the device the reply picked. + + None when the reply picked neither, so the reply is handled as a new + command: "no, the kitchen light" and "never mind, lock up" both are one. + """ + snapshot = pending.snapshot + options: dict[str, Any] = { + entity_id: described.as_option() + for entity_id in pending.candidates + if (described := snapshot.by_id(entity_id)) is not None + } + options[NONE] = "Neither of these, or a different request" + questions: dict[str, Question] = { + "which": Choice("Which device does the reply pick?", options) + } + state = {"command": pending.text, "reply": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + answer = response.answers.get("which") + picked = ( + answer.choice + if isinstance(answer, ChoiceAnswer) + and answer.choice in pending.candidates + and answer.confidence >= self._min_confidence + else None + ) + self._trace( + chat_log, + response, + {"text": user_input.text, "answers_command": pending.text, "picked": picked}, + ) + if picked is None: + return None + + # The first call's answers stand, with the entity question settled. The + # command is read from its own sentence again, so a brightness it named + # still comes from the text. + settled = ChoiceAnswer(choice=picked, probabilities={picked: 1.0}, confidence=1.0) + first = replace( + pending.response, answers=pending.response.answers | {"entity": settled} + ) + decision = interpret(first, pending.text, snapshot, self._min_confidence) + return await self._act(user_input, decision, pending.text) + + # --- acting --- + + async def _act( + self, + user_input: conversation.ConversationInput, + decision: Interpretation, + text: str, + ) -> conversation.ConversationResult: + """Carry out one decision, or say why not.""" if decision.already_satisfied is not None: name, settled = decision.already_satisfied return await self._speak(user_input, f"already_{settled}", name=name) @@ -248,7 +419,7 @@ async def _async_handle_message( DOMAIN, decision.intent_type, decision.slots, - user_input.text, + text, user_input.context, language=user_input.language, assistant=conversation.DOMAIN, diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 4e83f75..feccaab 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -20,7 +20,7 @@ from homeassistant.helpers import intent as ha_intent from jevclient import Choice, ChoiceAnswer, JevResponse, Noul, NoulAnswer, Question -from .snapshot import HomeSnapshot +from .snapshot import ExposedEntity, HomeSnapshot NONE = "none_of_these" @@ -129,6 +129,8 @@ class Interpretation: action_probabilities: dict[str, float] = field(default_factory=dict) # (entity name, the state it is already in) when there is nothing left to do. already_satisfied: tuple[str, str] | None = None + # Two entity ids the command could mean, when the agent should ask which. + candidates: tuple[str, str] | None = None @property def should_fall_back(self) -> bool: @@ -326,6 +328,20 @@ def out(reason: str) -> Interpretation: # unbounded off is not something to infer from one ambiguous sentence. slots["name"] = {"value": "all"} targets_everything = True + elif pair := _two_that_fit(entity, snapshot, min_confidence): + # The action is sure and the device is one of two. Asking costs one short + # question, and handing the sentence to the fallback agent gets the same + # guess made again by something that does not know it was a guess. + return Interpretation( + None, + {}, + action.choice, + action.confidence, + "two devices fit the name", + fallback=False, + action_probabilities=dict(action.probabilities or {}), + candidates=pair, + ) else: return out("no target named with enough confidence") @@ -363,6 +379,51 @@ def out(reason: str) -> Interpretation: ) +# The least share of the entity answer a device needs to be offered as one of two. +# With this, the two named devices hold at least 40% between them and the rest is +# spread across the others. Not measured on a real instance yet. +ASK_BACK_FLOOR = 0.2 + + +def _two_that_fit( + entity: ChoiceAnswer | None, snapshot: HomeSnapshot, min_confidence: float +) -> tuple[str, str] | None: + """The two devices a command could mean, when it is one of them and not a third. + + Both must be exposed, each must hold ASK_BACK_FLOOR of the answer, and the two + together must reach the confidence the agent acts on. A third device above the + floor means the question would not settle it, so the agent does not ask. + """ + if entity is None or not entity.probabilities: + return None + ranked = sorted( + ( + (probability, entity_id) + for entity_id, probability in entity.probabilities.items() + if entity_id != NONE and probability >= ASK_BACK_FLOOR + ), + reverse=True, + ) + if len(ranked) != 2 or ranked[0][0] + ranked[1][0] < min_confidence: + return None + first, second = (snapshot.by_id(entity_id) for _, entity_id in ranked) + if first is None or second is None or spoken_name(first, second) is None: + return None + return first.entity_id, second.entity_id + + +def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | None: + """How to say the two apart: by name, or by room when the names are the same. + + None when neither tells them apart, since "the fan or the fan" asks nothing. + """ + if one.name.casefold() != other.name.casefold(): + return one.name, other.name + if one.area and other.area and one.area.casefold() != other.area.casefold(): + return f"{one.name} ({one.area})", f"{other.name} ({other.area})" + return None + + # What "already done" looks like for each action the check covers. _SETTLED = {"turn_on": "on", "turn_off": "off"} diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..62b062a 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -367,6 +367,7 @@ "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..525226a 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -367,6 +367,7 @@ "budget_spent": "Denní rozpočet tokenů je vyčerpán, takže to dnes udělat nemohu.", "auth_failed": "TypeSafe odmítl klíč API. Zkontrolujte ho v nastavení Jev.", "unavailable": "TypeSafe neodpověděl. Zkuste to za chvíli znovu.", + "which_device": "Myslíte {first}, nebo {second}?", "preview_over_budget": "Denní rozpočet tokenů je vyčerpán, takže zkušební odpověď není. Otázka se přesto uloží.", "preview_nothing_yet": "Žádná zkušební odpověď: zatím není na co se ptát." }, diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..b589629 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -367,6 +367,7 @@ "budget_spent": "Det daglige tokenbudget er brugt, så det kan jeg ikke gøre i dag.", "auth_failed": "TypeSafe afviste API-nøglen. Tjek den i indstillingerne for Jev.", "unavailable": "TypeSafe svarede ikke. Prøv igen om et øjeblik.", + "which_device": "Mener du {first} eller {second}?", "preview_over_budget": "Det daglige token-budget er brugt op, så der er intet prøvesvar. Spørgsmålet bliver gemt alligevel.", "preview_nothing_yet": "Intet prøvesvar: der er endnu intet at spørge om." }, diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..42f16eb 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -367,6 +367,7 @@ "budget_spent": "Das tägliche Token-Budget ist aufgebraucht, deshalb kann ich das heute nicht tun.", "auth_failed": "TypeSafe hat den API-Schlüssel abgelehnt. Prüfe ihn in den Jev-Einstellungen.", "unavailable": "TypeSafe hat nicht geantwortet. Versuche es gleich noch einmal.", + "which_device": "Meinst du {first} oder {second}?", "preview_over_budget": "Das Tagesbudget für Tokens ist aufgebraucht, es gibt also keine Probeantwort. Die Frage wird trotzdem gespeichert.", "preview_nothing_yet": "Keine Probeantwort: Es gibt noch nichts, wonach sich fragen ließe." }, diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..62b062a 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -367,6 +367,7 @@ "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..be3ef4f 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -367,6 +367,7 @@ "budget_spent": "El presupuesto diario de tokens se ha agotado, así que hoy no puedo hacerlo.", "auth_failed": "TypeSafe rechazó la clave de API. Revísala en los ajustes de Jev.", "unavailable": "TypeSafe no respondió. Inténtalo de nuevo en un momento.", + "which_device": "¿Te refieres a {first} o a {second}?", "preview_over_budget": "El presupuesto diario de tokens está agotado, así que no hay respuesta de prueba. La pregunta sí se guarda.", "preview_nothing_yet": "Sin respuesta de prueba: todavía no hay nada sobre lo que preguntar." }, diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..922dae4 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -367,6 +367,7 @@ "budget_spent": "Le budget quotidien de jetons est épuisé, je ne peux donc pas le faire aujourd'hui.", "auth_failed": "TypeSafe a refusé la clé API. Vérifiez-la dans les paramètres de Jev.", "unavailable": "TypeSafe n'a pas répondu. Réessayez dans un instant.", + "which_device": "Voulez-vous dire {first} ou {second} ?", "preview_over_budget": "Le budget quotidien de jetons est épuisé, il n'y a donc pas de réponse d'essai. La question est quand même enregistrée.", "preview_nothing_yet": "Pas de réponse d'essai : il n'y a encore rien sur quoi poser la question." }, diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..bfcd33c 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -367,6 +367,7 @@ "budget_spent": "Il budget giornaliero di token è esaurito, quindi oggi non posso farlo.", "auth_failed": "TypeSafe ha rifiutato la chiave API. Controllala nelle impostazioni di Jev.", "unavailable": "TypeSafe non ha risposto. Riprova tra un momento.", + "which_device": "Intendi {first} o {second}?", "preview_over_budget": "Il budget giornaliero di token è esaurito, quindi non c'è una risposta di prova. La domanda viene comunque salvata.", "preview_nothing_yet": "Nessuna risposta di prova: non c'è ancora nulla su cui fare la domanda." }, diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..5f378b4 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -367,6 +367,7 @@ "budget_spent": "Het dagelijkse tokenbudget is op, dus dat kan ik vandaag niet doen.", "auth_failed": "TypeSafe heeft de API-sleutel geweigerd. Controleer hem in de instellingen van Jev.", "unavailable": "TypeSafe gaf geen antwoord. Probeer het zo nog eens.", + "which_device": "Bedoel je {first} of {second}?", "preview_over_budget": "Het daglimiet voor tokens is op, dus er is geen proefantwoord. De vraag wordt wel opgeslagen.", "preview_nothing_yet": "Geen proefantwoord: er is nog niets om over te vragen." }, diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..1d21f47 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -367,6 +367,7 @@ "budget_spent": "Dzienny budżet tokenów się wyczerpał, więc dziś nie mogę tego zrobić.", "auth_failed": "TypeSafe odrzucił klucz API. Sprawdź go w ustawieniach Jev.", "unavailable": "TypeSafe nie odpowiedział. Spróbuj ponownie za chwilę.", + "which_device": "Chodzi o {first} czy {second}?", "preview_over_budget": "Dzienny limit tokenów jest wyczerpany, więc nie ma próbnej odpowiedzi. Pytanie i tak zostanie zapisane.", "preview_nothing_yet": "Brak próbnej odpowiedzi: nie ma jeszcze o co pytać." }, diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..ce9669b 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -367,6 +367,7 @@ "budget_spent": "O orçamento diário de tokens acabou, então não posso fazer isso hoje.", "auth_failed": "A TypeSafe rejeitou a chave de API. Verifique-a nas configurações do Jev.", "unavailable": "A TypeSafe não respondeu. Tente novamente em instantes.", + "which_device": "Você quer dizer {first} ou {second}?", "preview_over_budget": "O orçamento diário de tokens acabou, então não há resposta de teste. A pergunta ainda é salva.", "preview_nothing_yet": "Sem resposta de teste: ainda não há nada sobre o que perguntar." }, diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..3235ef6 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -367,6 +367,7 @@ "budget_spent": "Дневной бюджет токенов исчерпан, поэтому сегодня я не могу это сделать.", "auth_failed": "TypeSafe отклонил ключ API. Проверьте его в настройках Jev.", "unavailable": "TypeSafe не ответил. Попробуйте ещё раз чуть позже.", + "which_device": "Вы имеете в виду {first} или {second}?", "preview_over_budget": "Дневной бюджет токенов исчерпан, поэтому пробного ответа нет. Вопрос всё равно сохраняется.", "preview_nothing_yet": "Пробного ответа нет: пока не о чем спрашивать." }, diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..6f9f9c0 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -367,6 +367,7 @@ "budget_spent": "Den dagliga tokenbudgeten är slut, så det kan jag inte göra i dag.", "auth_failed": "TypeSafe avvisade API-nyckeln. Kontrollera den i inställningarna för Jev.", "unavailable": "TypeSafe svarade inte. Försök igen om en stund.", + "which_device": "Menar du {first} eller {second}?", "preview_over_budget": "Den dagliga budgeten för token är slut, så det finns inget provsvar. Frågan sparas ändå.", "preview_nothing_yet": "Inget provsvar: det finns inget att fråga om än." }, diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..562c1c1 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -367,6 +367,7 @@ "budget_spent": "今天的令牌预算已用完，所以今天无法执行。", "auth_failed": "TypeSafe 拒绝了 API 密钥。请在 Jev 设置中检查。", "unavailable": "TypeSafe 没有响应。请稍后再试。", + "which_device": "你是指{first}还是{second}？", "preview_over_budget": "每日令牌预算已用完，因此没有试答结果。问题仍会保存。", "preview_nothing_yet": "没有试答结果：目前还没有可提问的内容。" }, diff --git a/tests/test_conversation.py b/tests/test_conversation.py index f001889..4b8d171 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -6,6 +6,7 @@ """ from dataclasses import replace +from datetime import timedelta from unittest.mock import patch import pytest @@ -16,7 +17,9 @@ from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar from homeassistant.helpers import entity_registry as er +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.setup import async_setup_component +from homeassistant.util import dt as dt_util from jevclient import ChoiceAnswer, NoulAnswer from custom_components.jev.const import ( @@ -26,7 +29,7 @@ CONVERSATION_TRACE_LENGTH, ) from custom_components.jev.conversation import _render_state_answer -from custom_components.jev.interpret import find_brightness +from custom_components.jev.interpret import NONE, find_brightness from custom_components.jev.payload import payload_bytes from .conftest import PROBE_TOKENS, build_response @@ -1318,3 +1321,205 @@ async def test_a_room_past_the_entity_cap_is_not_offered(hass, config_entry): assert [e.entity_id for e in snapshot.entities] == ["light.a", "light.b"] assert snapshot.areas == ["Attic"] + + +# --- asking which device --- + + +def unsure_between(first, second, first_share=0.5, second_share=0.45, **rest): + """An answer set whose entity answer is split between two devices.""" + shares = {first: first_share, second: second_share, NONE: 0.05, **rest} + return answer_set( + entity=ChoiceAnswer( + choice=first, probabilities=shares, confidence=max(shares.values()) + ), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + + +def reply(choice, confidence=0.95): + return {"which": ChoiceAnswer(choice=choice, probabilities={}, confidence=confidence)} + + +async def converse_in(hass, text, conversation_id): + return await conversation.async_converse( + hass, text, conversation_id, Context(), language="en", agent_id=AGENT + ) + + +async def test_two_devices_that_fit_the_name_get_a_question(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "light on") + await hass.async_block_till_done() + + assert calls == [] + assert result.continue_conversation is True + assert result.conversation_id + assert ( + result.response.speech["plain"]["speech"] + == "Do you mean Kitchen light or Office light?" + ) + + +async def test_the_reply_runs_the_first_command_on_the_device_it_picks( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + # The reply was one small question about the two devices, not a new command. + state, questions = mock_client.ask.await_args.args + assert list(questions) == ["which"] + assert state == {"command": "light on", "reply": "the office one"} + assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} + + +async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply(NONE) + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + result = await converse_in(hass, "never mind", asked.conversation_id) + await hass.async_block_till_done() + + # The reply question, then the whole reply as a command of its own, which + # this mock answers with the same split and so asks again. + assert mock_client.ask.await_count == 2 + assert "action" in mock_client.ask.await_args.args[1] + assert result.continue_conversation is True + + +async def test_an_unsure_reply_acts_on_nothing(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), + **reply("light.office", confidence=0.4), + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "hmm", asked.conversation_id) + await hass.async_block_till_done() + + assert calls == [] + + +async def test_a_reply_in_another_conversation_is_not_an_answer(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + await converse(hass, "light on") + mock_client.ask.reset_mock() + + await converse(hass, "the office one") + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_question_left_unanswered_expires_with_the_session( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + later = dt_util.utcnow() + CONVERSATION_TIMEOUT + timedelta(seconds=1) + with patch("custom_components.jev.conversation.dt_util.utcnow", return_value=later): + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_third_device_in_the_running_means_no_question(hass, house, mock_client): + hass.states.async_set("light.hall", "off", {"friendly_name": "Hall light"}) + async_expose_entity(hass, conversation.DOMAIN, "light.hall", True) + mock_client.ask.return_value = build_response( + **unsure_between( + "light.kitchen", "light.office", 0.4, 0.3, **{"light.hall": 0.25} + ) + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_two_devices_with_nothing_to_tell_them_apart_get_no_question( + hass, house, mock_client +): + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + er.async_get(hass).async_update_entity("light.office", area_id=kitchen.id) + hass.states.async_set("light.office", "off", {"friendly_name": "Kitchen light"}) + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_the_same_name_in_two_rooms_is_asked_by_room(hass, house, mock_client): + hass.states.async_set("light.office", "off", {"friendly_name": "Kitchen light"}) + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Kitchen light (Kitchen) or Kitchen light (Office)?" + ) + + +async def test_the_question_is_asked_in_the_pipeline_language(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "lamp aan", language="nl") + + assert ( + result.response.speech["plain"]["speech"] + == "Bedoel je Kitchen light of Office light?" + ) + + +async def test_a_reply_past_the_budget_is_refused_like_a_command( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + # Set on the account, not in the options: an options change reloads the entry, + # and a reloaded agent holds no question to answer. + house.runtime_data.usage.budget = 10 + mock_client.ask.reset_mock() + + result = await converse_in(hass, "the office one", asked.conversation_id) + + assert mock_client.ask.await_count == 0 + assert "budget is spent" in result.response.speech["plain"]["speech"] From e97c4699a365c1f0dbb8dbea5ee1bc48b6735944 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 22:05:25 +0200 Subject: [PATCH 04/24] Actions are refused before they are sent if they would pass the daily budget jev.noul, jev.choice, jev.score and jev.ask never checked the budget. A script that looped on an action spent without a limit. They now make the same check as contexts, AI Task and voice: estimate the request, refuse it with a message that names the estimate, what is left and the budget, and hold the estimate while the call runs. --- custom_components/jev/services.py | 20 +++++++++-- custom_components/jev/strings.json | 3 ++ custom_components/jev/translations/cs.json | 3 ++ custom_components/jev/translations/da.json | 3 ++ custom_components/jev/translations/de.json | 3 ++ custom_components/jev/translations/en.json | 3 ++ custom_components/jev/translations/es.json | 3 ++ custom_components/jev/translations/fr.json | 3 ++ custom_components/jev/translations/it.json | 3 ++ custom_components/jev/translations/nl.json | 3 ++ custom_components/jev/translations/pl.json | 3 ++ custom_components/jev/translations/pt-BR.json | 3 ++ custom_components/jev/translations/ru.json | 3 ++ custom_components/jev/translations/sv.json | 3 ++ .../jev/translations/zh-Hans.json | 3 ++ tests/test_services.py | 36 +++++++++++++++++++ 16 files changed, 95 insertions(+), 3 deletions(-) diff --git a/custom_components/jev/services.py b/custom_components/jev/services.py index fafb146..ee87e3c 100644 --- a/custom_components/jev/services.py +++ b/custom_components/jev/services.py @@ -217,9 +217,25 @@ async def _ask( type(state).__name__, state, ) + usage = entry.runtime_data.usage + usage.roll_over(dt_util.now().date()) request_bytes = payload_bytes(state, questions, entry.runtime_data.model) + # The same check a context and the voice agent make: a script that loops on an + # action is the runaway the budget exists for. + estimate = usage.estimate_tokens(request_bytes) + if usage.would_exceed_with(estimate): + raise HomeAssistantError( + translation_domain=DOMAIN, + translation_key="action_over_budget", + translation_placeholders={ + "estimate": str(estimate), + "remaining": str(usage.remaining()), + "budget": str(usage.budget), + }, + ) try: - response = await entry.runtime_data.client.ask(state, questions) + with usage.reservation(estimate): + response = await entry.runtime_data.client.ask(state, questions) except JevAuthError as err: entry.async_start_reauth(hass) raise HomeAssistantError( @@ -233,8 +249,6 @@ async def _ask( translation_key="ask_failed", translation_placeholders={"reason": str(err)}, ) from err - usage = entry.runtime_data.usage - usage.roll_over(dt_util.now().date()) usage.record(response.usage.input_tokens, request_bytes) entry.runtime_data.model_version = response.model or entry.runtime_data.model_version usage.notify() diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..18e8bd2 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Task {task} costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." }, + "action_over_budget": { + "message": "This action costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." + }, "reserved_field": { "message": "The field {field} cannot be used, because the result carries the confidence behind every answer under that name. Rename the field." }, diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..2ab23e5 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Úloha {task} stojí asi {estimate} vstupních tokenů a z denního rozpočtu {budget} zbývá {remaining}. Pro pokračování zvyšte nebo zrušte rozpočet v možnostech integrace." }, + "action_over_budget": { + "message": "Tato akce stojí asi {estimate} vstupních tokenů a z denního rozpočtu {budget} zbývá {remaining}. Pokud chcete pokračovat, zvyšte nebo zrušte rozpočet v možnostech integrace." + }, "reserved_field": { "message": "Pole {field} nelze použít, protože výsledek pod tímto názvem nese jistotu každé odpovědi. Pole přejmenujte." }, diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..969b972 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Opgaven {task} koster omkring {estimate} inputtokens, og der er {remaining} tilbage af dagsbudgettet på {budget}. Hæv eller ryd budgettet i integrationens indstillinger for at fortsætte." }, + "action_over_budget": { + "message": "Denne handling koster omkring {estimate} input-tokens, og der er {remaining} tilbage af det daglige budget på {budget}. Hæv eller fjern budgettet i integrationens indstillinger for at fortsætte." + }, "reserved_field": { "message": "Feltet {field} kan ikke bruges, fordi resultatet bærer sikkerheden bag hvert svar under det navn. Omdøb feltet." }, diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..c5e0dfb 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Die Aufgabe {task} kostet etwa {estimate} Eingabetokens, und vom Tagesbudget von {budget} sind noch {remaining} übrig. Erhöhe oder entferne das Budget in den Optionen der Integration, um fortzufahren." }, + "action_over_budget": { + "message": "Diese Aktion kostet etwa {estimate} Eingabe-Tokens, und vom Tagesbudget von {budget} sind noch {remaining} übrig. Erhöhe oder entferne das Budget in den Optionen der Integration, um fortzufahren." + }, "reserved_field": { "message": "Das Feld {field} kann nicht verwendet werden, weil das Ergebnis unter diesem Namen die Sicherheit hinter jeder Antwort mitliefert. Benenne das Feld um." }, diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..18e8bd2 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Task {task} costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." }, + "action_over_budget": { + "message": "This action costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." + }, "reserved_field": { "message": "The field {field} cannot be used, because the result carries the confidence behind every answer under that name. Rename the field." }, diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..74444d2 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "La tarea {task} cuesta unos {estimate} tokens de entrada y quedan {remaining} del presupuesto diario de {budget}. Aumenta o borra el presupuesto en las opciones de la integración para continuar." }, + "action_over_budget": { + "message": "Esta acción cuesta unos {estimate} tokens de entrada y quedan {remaining} del presupuesto diario de {budget}. Aumenta o elimina el presupuesto en las opciones de la integración para continuar." + }, "reserved_field": { "message": "El campo {field} no se puede usar, porque el resultado lleva bajo ese nombre la confianza detrás de cada respuesta. Cambia el nombre del campo." }, diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..57c8422 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "La tâche {task} coûte environ {estimate} jetons d'entrée et il reste {remaining} sur le budget quotidien de {budget}. Augmentez ou supprimez le budget dans les options de l'intégration pour continuer." }, + "action_over_budget": { + "message": "Cette action coûte environ {estimate} jetons d'entrée et il reste {remaining} sur le budget quotidien de {budget}. Augmentez ou supprimez le budget dans les options de l'intégration pour continuer." + }, "reserved_field": { "message": "Le champ {field} ne peut pas être utilisé, car le résultat transporte sous ce nom la confiance derrière chaque réponse. Renommez le champ." }, diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..17eaf2e 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "L'attività {task} costa circa {estimate} token di input e restano {remaining} del budget giornaliero di {budget}. Aumenta o cancella il budget nelle opzioni dell'integrazione per continuare." }, + "action_over_budget": { + "message": "Questa azione costa circa {estimate} token di input e restano {remaining} del budget giornaliero di {budget}. Aumenta o rimuovi il budget nelle opzioni dell'integrazione per continuare." + }, "reserved_field": { "message": "Il campo {field} non può essere usato, perché il risultato porta con quel nome la confidenza dietro ogni risposta. Rinomina il campo." }, diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..303002f 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Taak {task} kost ongeveer {estimate} invoertokens en van het dagbudget van {budget} zijn er nog {remaining} over. Verhoog of wis het budget in de opties van de integratie om door te gaan." }, + "action_over_budget": { + "message": "Deze actie kost ongeveer {estimate} invoertokens en er is nog {remaining} over van het dagbudget van {budget}. Verhoog of wis het budget in de opties van de integratie om door te gaan." + }, "reserved_field": { "message": "Het veld {field} kan niet worden gebruikt, omdat het resultaat onder die naam de zekerheid achter elk antwoord meegeeft. Geef het veld een andere naam." }, diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..d498bd1 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Zadanie {task} kosztuje około {estimate} tokenów wejściowych, a z dziennego budżetu {budget} zostało {remaining}. Zwiększ lub wyczyść budżet w opcjach integracji, aby kontynuować." }, + "action_over_budget": { + "message": "Ta akcja kosztuje około {estimate} tokenów wejściowych, a z dziennego budżetu {budget} zostało {remaining}. Aby kontynuować, zwiększ lub usuń budżet w opcjach integracji." + }, "reserved_field": { "message": "Pola {field} nie można użyć, ponieważ wynik przenosi pod tą nazwą pewność każdej odpowiedzi. Zmień nazwę pola." }, diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..3f185a8 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "A tarefa {task} custa cerca de {estimate} tokens de entrada e restam {remaining} do orçamento diário de {budget}. Aumente ou limpe o orçamento nas opções da integração para continuar." }, + "action_over_budget": { + "message": "Esta ação custa cerca de {estimate} tokens de entrada e restam {remaining} do orçamento diário de {budget}. Aumente ou remova o orçamento nas opções da integração para continuar." + }, "reserved_field": { "message": "O campo {field} não pode ser usado, porque o resultado leva sob esse nome a confiança por trás de cada resposta. Renomeie o campo." }, diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..2d1f5de 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Задача {task} стоит около {estimate} входных токенов, а из дневного бюджета {budget} осталось {remaining}. Чтобы продолжить, увеличьте или очистите бюджет в параметрах интеграции." }, + "action_over_budget": { + "message": "Это действие стоит около {estimate} входных токенов, а от дневного бюджета {budget} осталось {remaining}. Чтобы продолжить, увеличьте или снимите бюджет в параметрах интеграции." + }, "reserved_field": { "message": "Поле {field} использовать нельзя, потому что под этим именем результат передаёт уверенность каждого ответа. Переименуйте поле." }, diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..3a6f50b 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "Uppgiften {task} kostar ungefär {estimate} indatatokens och {remaining} av dagsbudgeten på {budget} återstår. Höj eller rensa budgeten i integrationens alternativ för att fortsätta." }, + "action_over_budget": { + "message": "Den här åtgärden kostar ungefär {estimate} indatatokens och det finns {remaining} kvar av den dagliga budgeten på {budget}. Höj eller ta bort budgeten i integrationens alternativ för att fortsätta." + }, "reserved_field": { "message": "Fältet {field} kan inte användas, eftersom resultatet bär säkerheten bakom varje svar under det namnet. Byt namn på fältet." }, diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..bce6a03 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -306,6 +306,9 @@ "task_over_budget": { "message": "任务 {task} 大约需要 {estimate} 个输入令牌，而每日预算 {budget} 只剩 {remaining}。请在集成选项中提高或清除预算后再继续。" }, + "action_over_budget": { + "message": "此操作约需 {estimate} 个输入令牌，每日预算 {budget} 还剩 {remaining}。要继续，请在集成选项中提高或清除预算。" + }, "reserved_field": { "message": "字段 {field} 不能使用，因为结果在这个名称下携带每个答案背后的置信度。请重命名该字段。" }, diff --git a/tests/test_services.py b/tests/test_services.py index df47b4c..a412b09 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -437,3 +437,39 @@ async def test_an_action_with_two_entries_and_none_named_is_refused( {"state": "x", "instructions": "y", "config_entry": second.entry_id}, ) assert "noul" in named + + +async def test_an_action_that_would_pass_the_budget_is_not_sent( + hass, loaded_entry, mock_client +): + usage = loaded_entry.runtime_data.usage + usage.budget = 1000 + usage.input_tokens = 999 + mock_client.ask.reset_mock() + + with pytest.raises(HomeAssistantError) as err: + await call(hass, "noul", {"state": "x", "instructions": "Is it done?"}) + + assert mock_client.ask.await_count == 0 + assert err.value.translation_key == "action_over_budget" + placeholders = err.value.translation_placeholders + assert placeholders["remaining"] == "1" + assert placeholders["budget"] == "1000" + assert int(placeholders["estimate"]) > 1 + + +async def test_an_action_holds_its_estimate_while_it_runs( + hass, loaded_entry, mock_client +): + usage = loaded_entry.runtime_data.usage + held = [] + + async def watch(state, questions): + held.append(usage.reserved) + return build_response(answer=NoulAnswer(noul=0.5)) + + mock_client.ask.side_effect = watch + await call(hass, "noul", {"state": "x", "instructions": "Is it done?"}) + + assert held[0] > 0 + assert usage.reserved == 0 From 7dbbaac084a5f66eb51c9995e0dc21b22cd359c6 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 22:11:35 +0200 Subject: [PATCH 05/24] Other LLM agents can ask Jev yes/no and pick-one questions, when the options allow it --- custom_components/jev/config_flow.py | 5 + custom_components/jev/const.py | 4 + custom_components/jev/llm.py | 187 ++++++++++++++++++ custom_components/jev/strings.json | 6 +- custom_components/jev/translations/cs.json | 6 +- custom_components/jev/translations/da.json | 6 +- custom_components/jev/translations/de.json | 6 +- custom_components/jev/translations/en.json | 6 +- custom_components/jev/translations/es.json | 6 +- custom_components/jev/translations/fr.json | 6 +- custom_components/jev/translations/it.json | 6 +- custom_components/jev/translations/nl.json | 6 +- custom_components/jev/translations/pl.json | 6 +- custom_components/jev/translations/pt-BR.json | 6 +- custom_components/jev/translations/ru.json | 6 +- custom_components/jev/translations/sv.json | 6 +- .../jev/translations/zh-Hans.json | 6 +- tests/test_llm.py | 158 +++++++++++++++ 18 files changed, 410 insertions(+), 28 deletions(-) create mode 100644 custom_components/jev/llm.py create mode 100644 tests/test_llm.py diff --git a/custom_components/jev/config_flow.py b/custom_components/jev/config_flow.py index 9f98fce..17df10a 100644 --- a/custom_components/jev/config_flow.py +++ b/custom_components/jev/config_flow.py @@ -42,6 +42,7 @@ CONF_ALLOW_WHOLE_HOME, CONF_DAILY_TOKEN_BUDGET, CONF_FALLBACK_AGENT, + CONF_LLM_TOOLS, CONF_MIN_CONFIDENCE, CONF_MODEL, CONF_PRICE_PER_MILLION, @@ -399,6 +400,10 @@ async def async_step_init( CONF_ALLOW_WHOLE_HOME, default=options.get(CONF_ALLOW_WHOLE_HOME, False), ): bool, + vol.Optional( + CONF_LLM_TOOLS, + default=options.get(CONF_LLM_TOOLS, False), + ): bool, } ) return self.async_show_form(step_id="init", data_schema=schema) diff --git a/custom_components/jev/const.py b/custom_components/jev/const.py index 126ac69..98525ef 100644 --- a/custom_components/jev/const.py +++ b/custom_components/jev/const.py @@ -101,6 +101,10 @@ CONF_MIN_CONFIDENCE: Final = "min_confidence" CONF_ALLOW_WHOLE_HOME: Final = "allow_whole_home" +# --- Tools for other LLM agents --- + +CONF_LLM_TOOLS: Final = "llm_tools" + # Below this, the router hands the sentence to the fallback agent rather than # guessing. 0.6 is a starting point and not a calibrated figure: TypeSafe publishes # no calibration evidence for confidence, so treat it as an ordering and measure it diff --git a/custom_components/jev/llm.py b/custom_components/jev/llm.py new file mode 100644 index 0000000..fd85a6e --- /dev/null +++ b/custom_components/jev/llm.py @@ -0,0 +1,187 @@ +"""Jev's two judgements as tools for an LLM conversation agent. + +An LLM agent reads the house through the Assist API and answers in prose. What it +cannot give is a probability it has been calibrated to: asked "is the washing +machine done", it says yes or no. These tools hand that one judgement to Jev and +return the number, so the agent can say how sure it is, or not act at all. + +The tools are off unless the entry's options turn them on. Every tool's schema +goes into the prompt of every Assist LLM agent on every turn, which the user pays +for at that agent's provider whether a tool is called or not. + +Each call goes through the matching action, so the daily budget, the error +messages and the usage sensors are the actions' own. +""" + +from __future__ import annotations + +from typing import Any, override + +import voluptuous as vol +from homeassistant.components.homeassistant.exposed_entities import ( + async_should_expose, +) +from homeassistant.components.llm import LLMTools +from homeassistant.const import ATTR_ENTITY_ID +from homeassistant.core import HomeAssistant, callback +from homeassistant.helpers.llm import LLM_API_ASSIST, LLMContext, Tool, ToolInput +from homeassistant.util.json import JsonObjectType + +from .const import ( + ATTR_CONFIG_ENTRY, + CONF_INSTRUCTIONS, + CONF_LLM_TOOLS, + CONF_OPTIONS, + CONF_STATE_TEMPLATE, + DOMAIN, + MAX_CONVERSATION_ENTITIES, + SERVICE_CHOICE, + SERVICE_NOUL, +) + +_QUESTION = "question" +_FACTS = "facts" +_ACCOUNT = "account" + +_QUESTION_DESCRIPTION = "The question, in plain words." +_FACTS_DESCRIPTION = ( + "Anything the question depends on that is not an entity state, such as what " + "the user said. Optional." +) + + +class _JevTool(Tool): + """One Jev action, judged against what Assist may see.""" + + action: str + + def __init__(self, entries: dict[str, str], extra: dict[Any, Any]) -> None: + # entries maps an entry title to its id. With two entries the model picks + # one, because each has its own key and its own budget. + self._entries = entries + fields: dict[Any, Any] = { + vol.Required(_QUESTION, description=_QUESTION_DESCRIPTION): str, + **extra, + vol.Optional(_FACTS, description=_FACTS_DESCRIPTION): str, + } + if len(entries) > 1: + fields[ + vol.Required(_ACCOUNT, description="Which Jev account pays for this.") + ] = vol.In(sorted(entries)) + self.parameters = vol.Schema(fields) + + def _data( + self, hass: HomeAssistant, args: dict[str, Any], llm_context: LLMContext + ) -> dict[str, Any]: + account = args.get(_ACCOUNT) or next(iter(self._entries)) + data: dict[str, Any] = { + CONF_INSTRUCTIONS: args[_QUESTION], + ATTR_CONFIG_ENTRY: self._entries[account], + ATTR_ENTITY_ID: _exposed(hass, llm_context), + } + if facts := args.get(_FACTS): + # A mapping, because the action renders a string state as a template, + # and a template can read entities that are not exposed to Assist. + data[CONF_STATE_TEMPLATE] = {_FACTS: facts} + return data + + async def _call( + self, hass: HomeAssistant, data: dict[str, Any], llm_context: LLMContext + ) -> dict[str, Any]: + response = await hass.services.async_call( + DOMAIN, + self.action, + data, + blocking=True, + context=llm_context.context, + return_response=True, + ) + assert response is not None + return dict(response) + + +class NoulTool(_JevTool): + name = f"{DOMAIN}__noul" + description = ( + "Ask Jev how likely a yes/no statement about the home is, judged from the " + "states of the entities exposed to Assist. Returns the probability that the " + "answer is yes, from 0 to 1. Use it for a judgement, such as whether a " + "machine is done or whether anyone is likely home, not for a state you can " + "read directly." + ) + action = SERVICE_NOUL + + def __init__(self, entries: dict[str, str]) -> None: + super().__init__(entries, {}) + + @override + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext + ) -> JsonObjectType: + args = self.parameters(tool_input.tool_args) + result = await self._call(hass, self._data(hass, args, llm_context), llm_context) + return {"probability_yes": result["noul"]} + + +class ChoiceTool(_JevTool): + name = f"{DOMAIN}__choice" + description = ( + "Ask Jev which of a set of options best describes the home, judged from the " + "states of the entities exposed to Assist. Returns the chosen option, the " + "probability of every option, and a confidence from 0 to 1." + ) + action = SERVICE_CHOICE + + def __init__(self, entries: dict[str, str]) -> None: + super().__init__( + entries, + { + vol.Required(CONF_OPTIONS, description="Two or more options."): vol.All( + [str], vol.Length(min=2) + ) + }, + ) + + @override + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext + ) -> JsonObjectType: + args = self.parameters(tool_input.tool_args) + data = self._data(hass, args, llm_context) | {CONF_OPTIONS: args[CONF_OPTIONS]} + result = await self._call(hass, data, llm_context) + return { + "choice": result["choice"], + "probabilities": result["probabilities"], + "confidence": result["confidence"], + } + + +def _exposed(hass: HomeAssistant, llm_context: LLMContext) -> list[str]: + """The entities this assistant may see, capped as the voice agent caps them. + + Jev judges from what the user exposed to Assist and nothing wider, the same + list the LLM agent itself reads. + """ + exposed = sorted( + state.entity_id + for state in hass.states.async_all() + if async_should_expose(hass, llm_context.assistant, state.entity_id) + ) + return exposed[:MAX_CONVERSATION_ENTITIES] + + +@callback +def async_get_tools( + hass: HomeAssistant, llm_context: LLMContext, api_id: str +) -> LLMTools | None: + """The tools, for each loaded entry whose options turn them on.""" + if api_id != LLM_API_ASSIST: + return None + entries = { + entry.title: entry.entry_id + for entry in hass.config_entries.async_loaded_entries(DOMAIN) + if entry.options.get(CONF_LLM_TOOLS, False) + } + if not entries: + return None + return LLMTools(tools=[NoulTool(entries), ChoiceTool(entries)]) diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 18e8bd2..3262c2c 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -82,14 +82,16 @@ "price_per_million": "Price per million input tokens, in USD", "fallback_agent": "Fall back to this agent", "min_confidence": "Act only above this confidence (0 to 1)", - "allow_whole_home": "Allow commands that name no room or device" + "allow_whole_home": "Allow commands that name no room or device", + "llm_tools": "Offer Jev as a tool to other LLM agents" }, "data_description": { "daily_token_budget": "Counted from midnight in the Home Assistant time zone. 0 means no limit.", "price_per_million": "Sets the estimated cost sensor and the cost in the question preview. Nothing that is sent depends on it.", "fallback_agent": "Receives the whole sentence when Jev is not sure what to do.", "min_confidence": "Below this, Jev does nothing and passes the command to the fallback agent.", - "allow_whole_home": "Turning everything off is always allowed, with or without this." + "allow_whole_home": "Turning everything off is always allowed, with or without this.", + "llm_tools": "Adds two tools to the Assist API, one for yes/no questions and one for picking an option. Each LLM agent that uses Assist gets their descriptions in every prompt, which costs tokens at that agent's provider. Each call also counts against the budget above." } } } diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index 2ab23e5..1c3c123 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -82,14 +82,16 @@ "price_per_million": "Cena za milion vstupních tokenů v USD", "fallback_agent": "Záložní agent", "min_confidence": "Jednat jen nad touto jistotou (0 až 1)", - "allow_whole_home": "Povolit příkazy, které neuvádějí místnost ani zařízení" + "allow_whole_home": "Povolit příkazy, které neuvádějí místnost ani zařízení", + "llm_tools": "Nabídnout Jev jako nástroj ostatním LLM agentům" }, "data_description": { "daily_token_budget": "Počítá se od půlnoci v časovém pásmu Home Assistantu. 0 znamená bez limitu.", "price_per_million": "Určuje senzor odhadované ceny a cenu v náhledu otázky. Nic z toho, co se odesílá, na tom nezávisí.", "fallback_agent": "Dostane celou větu, když si Jev není jistý, co udělat.", "min_confidence": "Pod touto hodnotou Jev nic neudělá a předá příkaz záložnímu agentovi.", - "allow_whole_home": "Vypnout všechno je dovoleno vždy, s touto volbou i bez ní." + "allow_whole_home": "Vypnout všechno je dovoleno vždy, s touto volbou i bez ní.", + "llm_tools": "Přidá do API Assist dva nástroje, jeden pro otázky ano/ne a jeden pro výběr možnosti. Každý LLM agent, který používá Assist, dostane jejich popisy v každém promptu, což stojí tokeny u poskytovatele tohoto agenta. Každé volání se také počítá do rozpočtu výše." } } } diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index 969b972..6a86556 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -82,14 +82,16 @@ "price_per_million": "Pris pr. million input-tokens, i USD", "fallback_agent": "Fald tilbage til denne agent", "min_confidence": "Handl kun over denne konfidens (0 til 1)", - "allow_whole_home": "Tillad kommandoer, der ikke nævner et rum eller en enhed" + "allow_whole_home": "Tillad kommandoer, der ikke nævner et rum eller en enhed", + "llm_tools": "Tilbyd Jev som værktøj til andre LLM-agenter" }, "data_description": { "daily_token_budget": "Tælles fra midnat i Home Assistants tidszone. 0 betyder ingen grænse.", "price_per_million": "Bestemmer sensoren for anslået pris og prisen i forhåndsvisningen af spørgsmålet. Intet af det, der sendes, afhænger af den.", "fallback_agent": "Modtager hele sætningen, når Jev ikke er sikker på, hvad der skal gøres.", "min_confidence": "Under denne værdi gør Jev intet og sender kommandoen videre til reserveagenten.", - "allow_whole_home": "At slukke alt er altid tilladt, med eller uden denne indstilling." + "allow_whole_home": "At slukke alt er altid tilladt, med eller uden denne indstilling.", + "llm_tools": "Tilføjer to værktøjer til Assist-API'et, et til ja/nej-spørgsmål og et til at vælge en mulighed. Hver LLM-agent, der bruger Assist, får deres beskrivelser i hver prompt, hvilket koster tokens hos den agents udbyder. Hvert kald tæller også med i budgettet ovenfor." } } } diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index c5e0dfb..6ec3aa5 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -82,14 +82,16 @@ "price_per_million": "Preis pro Million Eingabe-Tokens, in USD", "fallback_agent": "Auf diesen Agenten zurückfallen", "min_confidence": "Nur oberhalb dieser Konfidenz handeln (0 bis 1)", - "allow_whole_home": "Befehle erlauben, die keinen Raum und kein Gerät nennen" + "allow_whole_home": "Befehle erlauben, die keinen Raum und kein Gerät nennen", + "llm_tools": "Jev anderen LLM-Agenten als Werkzeug anbieten" }, "data_description": { "daily_token_budget": "Gezählt ab Mitternacht in der Zeitzone von Home Assistant. 0 bedeutet kein Limit.", "price_per_million": "Bestimmt den Sensor für die geschätzten Kosten und die Kosten in der Vorschau einer Frage. Nichts, was gesendet wird, hängt davon ab.", "fallback_agent": "Bekommt den ganzen Satz, wenn Jev nicht sicher ist, was zu tun ist.", "min_confidence": "Darunter tut Jev nichts und gibt den Befehl an den Ersatz-Agenten weiter.", - "allow_whole_home": "Alles ausschalten ist immer erlaubt, mit oder ohne diese Option." + "allow_whole_home": "Alles ausschalten ist immer erlaubt, mit oder ohne diese Option.", + "llm_tools": "Fügt der Assist-API zwei Werkzeuge hinzu, eines für Ja/Nein-Fragen und eines für die Wahl einer Option. Jeder LLM-Agent, der Assist nutzt, erhält ihre Beschreibungen in jedem Prompt, was beim Anbieter dieses Agenten Tokens kostet. Jeder Aufruf zählt außerdem zum Budget oben." } } } diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 18e8bd2..3262c2c 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -82,14 +82,16 @@ "price_per_million": "Price per million input tokens, in USD", "fallback_agent": "Fall back to this agent", "min_confidence": "Act only above this confidence (0 to 1)", - "allow_whole_home": "Allow commands that name no room or device" + "allow_whole_home": "Allow commands that name no room or device", + "llm_tools": "Offer Jev as a tool to other LLM agents" }, "data_description": { "daily_token_budget": "Counted from midnight in the Home Assistant time zone. 0 means no limit.", "price_per_million": "Sets the estimated cost sensor and the cost in the question preview. Nothing that is sent depends on it.", "fallback_agent": "Receives the whole sentence when Jev is not sure what to do.", "min_confidence": "Below this, Jev does nothing and passes the command to the fallback agent.", - "allow_whole_home": "Turning everything off is always allowed, with or without this." + "allow_whole_home": "Turning everything off is always allowed, with or without this.", + "llm_tools": "Adds two tools to the Assist API, one for yes/no questions and one for picking an option. Each LLM agent that uses Assist gets their descriptions in every prompt, which costs tokens at that agent's provider. Each call also counts against the budget above." } } } diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index 74444d2..51f08a2 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -82,14 +82,16 @@ "price_per_million": "Precio por millón de tokens de entrada, en USD", "fallback_agent": "Recurrir a este agente", "min_confidence": "Actuar solo por encima de esta confianza (de 0 a 1)", - "allow_whole_home": "Permitir órdenes que no nombren ninguna habitación ni dispositivo" + "allow_whole_home": "Permitir órdenes que no nombren ninguna habitación ni dispositivo", + "llm_tools": "Ofrecer Jev como herramienta a otros agentes LLM" }, "data_description": { "daily_token_budget": "Se cuenta desde la medianoche en la zona horaria de Home Assistant. 0 significa sin límite.", "price_per_million": "Fija el sensor de coste estimado y el coste en la vista previa de la pregunta. Nada de lo que se envía depende de ello.", "fallback_agent": "Recibe la frase entera cuando Jev no está seguro de qué hacer.", "min_confidence": "Por debajo de este valor, Jev no hace nada y pasa el comando al agente de respaldo.", - "allow_whole_home": "Apagarlo todo siempre está permitido, con o sin esta opción." + "allow_whole_home": "Apagarlo todo siempre está permitido, con o sin esta opción.", + "llm_tools": "Añade dos herramientas a la API de Assist, una para preguntas de sí o no y otra para elegir una opción. Cada agente LLM que usa Assist recibe sus descripciones en cada prompt, lo que cuesta tokens en el proveedor de ese agente. Cada llamada también cuenta para el presupuesto de arriba." } } } diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index 57c8422..2ac97b6 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -82,14 +82,16 @@ "price_per_million": "Prix par million de jetons d'entrée, en USD", "fallback_agent": "Se replier sur cet agent", "min_confidence": "N'agir qu'au-dessus de cette confiance (0 à 1)", - "allow_whole_home": "Autoriser les commandes qui ne nomment ni pièce ni appareil" + "allow_whole_home": "Autoriser les commandes qui ne nomment ni pièce ni appareil", + "llm_tools": "Proposer Jev comme outil aux autres agents LLM" }, "data_description": { "daily_token_budget": "Compté à partir de minuit dans le fuseau horaire de Home Assistant. 0 signifie aucune limite.", "price_per_million": "Détermine le capteur de coût estimé et le coût dans l'aperçu de la question. Rien de ce qui est envoyé n'en dépend.", "fallback_agent": "Reçoit la phrase entière quand Jev ne sait pas quoi faire.", "min_confidence": "En dessous, Jev ne fait rien et transmet la commande à l'agent de secours.", - "allow_whole_home": "Tout éteindre est toujours autorisé, avec ou sans cette option." + "allow_whole_home": "Tout éteindre est toujours autorisé, avec ou sans cette option.", + "llm_tools": "Ajoute deux outils à l'API Assist, l'un pour les questions oui/non et l'autre pour choisir une option. Chaque agent LLM qui utilise Assist reçoit leurs descriptions dans chaque prompt, ce qui coûte des tokens chez le fournisseur de cet agent. Chaque appel compte aussi dans le budget ci-dessus." } } } diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 17eaf2e..2877df8 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -82,14 +82,16 @@ "price_per_million": "Prezzo per milione di token di input, in USD", "fallback_agent": "Ripiega su questo agente", "min_confidence": "Agisci solo sopra questa confidenza (da 0 a 1)", - "allow_whole_home": "Consenti comandi che non nominano una stanza o un dispositivo" + "allow_whole_home": "Consenti comandi che non nominano una stanza o un dispositivo", + "llm_tools": "Offri Jev come strumento ad altri agenti LLM" }, "data_description": { "daily_token_budget": "Conteggiato dalla mezzanotte nel fuso orario di Home Assistant. 0 significa nessun limite.", "price_per_million": "Determina il sensore del costo stimato e il costo nell'anteprima della domanda. Niente di ciò che viene inviato dipende da questo.", "fallback_agent": "Riceve l'intera frase quando Jev non è sicuro di cosa fare.", "min_confidence": "Sotto questo valore Jev non fa nulla e passa il comando all'agente di riserva.", - "allow_whole_home": "Spegnere tutto è sempre consentito, con o senza questa opzione." + "allow_whole_home": "Spegnere tutto è sempre consentito, con o senza questa opzione.", + "llm_tools": "Aggiunge due strumenti all'API Assist, uno per le domande sì/no e uno per scegliere un'opzione. Ogni agente LLM che usa Assist riceve le loro descrizioni in ogni prompt, il che costa token presso il fornitore di quell'agente. Ogni chiamata conta anche nel budget qui sopra." } } } diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 303002f..c87dc1e 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -82,14 +82,16 @@ "price_per_million": "Prijs per miljoen invoertokens, in USD", "fallback_agent": "Val terug op deze agent", "min_confidence": "Alleen handelen boven dit vertrouwen (0 tot 1)", - "allow_whole_home": "Sta opdrachten toe die geen kamer of apparaat noemen" + "allow_whole_home": "Sta opdrachten toe die geen kamer of apparaat noemen", + "llm_tools": "Jev als tool aanbieden aan andere LLM-agents" }, "data_description": { "daily_token_budget": "Geteld vanaf middernacht in de tijdzone van Home Assistant. 0 betekent geen limiet.", "price_per_million": "Bepaalt de sensor voor de geschatte kosten en de kosten in het voorbeeld van een vraag. Niets van wat wordt verstuurd hangt ervan af.", "fallback_agent": "Krijgt de hele zin wanneer Jev niet zeker weet wat te doen.", "min_confidence": "Daaronder doet Jev niets en geeft het de opdracht door aan de terugvalagent.", - "allow_whole_home": "Alles uitzetten mag altijd, met of zonder deze optie." + "allow_whole_home": "Alles uitzetten mag altijd, met of zonder deze optie.", + "llm_tools": "Voegt twee tools toe aan de Assist-API, een voor ja/nee-vragen en een voor het kiezen van een optie. Elke LLM-agent die Assist gebruikt, krijgt hun beschrijvingen in elke prompt, en dat kost tokens bij de provider van die agent. Elke aanroep telt ook mee voor het budget hierboven." } } } diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index d498bd1..5c09f77 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -82,14 +82,16 @@ "price_per_million": "Cena za milion tokenów wejściowych, w USD", "fallback_agent": "Agent zapasowy", "min_confidence": "Działaj tylko powyżej tej pewności (0 do 1)", - "allow_whole_home": "Zezwalaj na polecenia, które nie wskazują pokoju ani urządzenia" + "allow_whole_home": "Zezwalaj na polecenia, które nie wskazują pokoju ani urządzenia", + "llm_tools": "Udostępnij Jev jako narzędzie innym agentom LLM" }, "data_description": { "daily_token_budget": "Liczone od północy w strefie czasowej Home Assistanta. 0 oznacza brak limitu.", "price_per_million": "Ustala czujnik szacowanego kosztu i koszt w podglądzie pytania. Nic z tego, co jest wysyłane, od tego nie zależy.", "fallback_agent": "Dostaje całe zdanie, gdy Jev nie jest pewny, co zrobić.", "min_confidence": "Poniżej tej wartości Jev nic nie robi i przekazuje polecenie agentowi zapasowemu.", - "allow_whole_home": "Wyłączenie wszystkiego jest zawsze dozwolone, z tą opcją lub bez niej." + "allow_whole_home": "Wyłączenie wszystkiego jest zawsze dozwolone, z tą opcją lub bez niej.", + "llm_tools": "Dodaje do API Assist dwa narzędzia, jedno do pytań tak/nie i jedno do wyboru opcji. Każdy agent LLM korzystający z Assist dostaje ich opisy w każdym prompcie, co kosztuje tokeny u dostawcy tego agenta. Każde wywołanie liczy się też do budżetu powyżej." } } } diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 3f185a8..3f696cd 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -82,14 +82,16 @@ "price_per_million": "Preço por milhão de tokens de entrada, em USD", "fallback_agent": "Recorrer a este agente", "min_confidence": "Agir apenas acima desta confiança (0 a 1)", - "allow_whole_home": "Permitir comandos que não citam cômodo nem dispositivo" + "allow_whole_home": "Permitir comandos que não citam cômodo nem dispositivo", + "llm_tools": "Oferecer o Jev como ferramenta para outros agentes LLM" }, "data_description": { "daily_token_budget": "Contado a partir da meia-noite no fuso horário do Home Assistant. 0 significa sem limite.", "price_per_million": "Define o sensor de custo estimado e o custo na prévia da pergunta. Nada do que é enviado depende disso.", "fallback_agent": "Recebe a frase inteira quando o Jev não tem certeza do que fazer.", "min_confidence": "Abaixo disso, o Jev não faz nada e passa o comando para o agente reserva.", - "allow_whole_home": "Desligar tudo é sempre permitido, com ou sem esta opção." + "allow_whole_home": "Desligar tudo é sempre permitido, com ou sem esta opção.", + "llm_tools": "Adiciona duas ferramentas à API do Assist, uma para perguntas de sim ou não e outra para escolher uma opção. Cada agente LLM que usa o Assist recebe as descrições delas em todo prompt, o que custa tokens no provedor desse agente. Cada chamada também conta no orçamento acima." } } } diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 2d1f5de..71209c6 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -82,14 +82,16 @@ "price_per_million": "Цена за миллион входных токенов, в USD", "fallback_agent": "Резервный агент", "min_confidence": "Действовать только выше этой уверенности (от 0 до 1)", - "allow_whole_home": "Разрешить команды, в которых не названы комната или устройство" + "allow_whole_home": "Разрешить команды, в которых не названы комната или устройство", + "llm_tools": "Предложить Jev как инструмент другим LLM-агентам" }, "data_description": { "daily_token_budget": "Считается с полуночи в часовом поясе Home Assistant. 0 означает без ограничения.", "price_per_million": "Задаёт датчик оценочной стоимости и стоимость в предпросмотре вопроса. От него не зависит ничего из того, что отправляется.", "fallback_agent": "Получает всю фразу, когда Jev не уверен, что делать.", "min_confidence": "Ниже этого значения Jev ничего не делает и передаёт команду резервному агенту.", - "allow_whole_home": "Выключить всё можно всегда, с этой настройкой или без неё." + "allow_whole_home": "Выключить всё можно всегда, с этой настройкой или без неё.", + "llm_tools": "Добавляет в API Assist два инструмента: один для вопросов да/нет и один для выбора варианта. Каждый LLM-агент, использующий Assist, получает их описания в каждом запросе, и это стоит токенов у провайдера этого агента. Каждый вызов также учитывается в бюджете выше." } } } diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index 3a6f50b..dcfb7aa 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -82,14 +82,16 @@ "price_per_million": "Pris per miljon indatatoken, i USD", "fallback_agent": "Fall tillbaka på den här agenten", "min_confidence": "Agera bara över den här säkerheten (0 till 1)", - "allow_whole_home": "Tillåt kommandon som inte nämner rum eller enhet" + "allow_whole_home": "Tillåt kommandon som inte nämner rum eller enhet", + "llm_tools": "Erbjud Jev som verktyg till andra LLM-agenter" }, "data_description": { "daily_token_budget": "Räknas från midnatt i Home Assistants tidszon. 0 betyder ingen gräns.", "price_per_million": "Styr sensorn för uppskattad kostnad och kostnaden i förhandsvisningen av frågan. Inget av det som skickas beror på den.", "fallback_agent": "Får hela meningen när Jev inte är säker på vad som ska göras.", "min_confidence": "Under detta värde gör Jev ingenting och skickar kommandot vidare till reservagenten.", - "allow_whole_home": "Att stänga av allt är alltid tillåtet, med eller utan detta val." + "allow_whole_home": "Att stänga av allt är alltid tillåtet, med eller utan detta val.", + "llm_tools": "Lägger till två verktyg i Assist-API:et, ett för ja/nej-frågor och ett för att välja ett alternativ. Varje LLM-agent som använder Assist får deras beskrivningar i varje prompt, vilket kostar tokens hos den agentens leverantör. Varje anrop räknas också mot budgeten ovan." } } } diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index bce6a03..b1d3a8b 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -82,14 +82,16 @@ "price_per_million": "每百万输入令牌的价格，单位 USD", "fallback_agent": "回退到此代理", "min_confidence": "仅在置信度高于此值时执行（0 到 1）", - "allow_whole_home": "允许未指明房间或设备的命令" + "allow_whole_home": "允许未指明房间或设备的命令", + "llm_tools": "将 Jev 作为工具提供给其他 LLM 代理" }, "data_description": { "daily_token_budget": "从 Home Assistant 时区的午夜开始计算。0 表示不限制。", "price_per_million": "决定预估费用传感器和问题预览中的费用。发送的内容都不受它影响。", "fallback_agent": "当 Jev 不确定该做什么时，接收完整的句子。", "min_confidence": "低于此值时，Jev 不执行任何操作，并把命令交给备用代理。", - "allow_whole_home": "无论是否开启此项，关闭所有设备始终允许。" + "allow_whole_home": "无论是否开启此项，关闭所有设备始终允许。", + "llm_tools": "向 Assist API 添加两个工具，一个用于是/否问题，一个用于选择选项。每个使用 Assist 的 LLM 代理都会在每条提示中收到它们的描述，这会在该代理的提供商处消耗 token。每次调用也计入上面的预算。" } } } diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..2db7366 --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,158 @@ +"""Jev's judgements as tools for another LLM agent, through the Assist API.""" + +import pytest +import voluptuous as vol +from homeassistant.components import conversation +from homeassistant.components.homeassistant.exposed_entities import async_expose_entity +from homeassistant.const import CONF_API_KEY +from homeassistant.core import Context +from homeassistant.helpers import llm +from homeassistant.setup import async_setup_component +from jevclient import ChoiceAnswer, NoulAnswer +from pytest_homeassistant_custom_component.common import MockConfigEntry + +from custom_components.jev.const import CONF_LLM_TOOLS, DOMAIN + +from .conftest import PROBE_TOKENS, build_response + + +def assist_context() -> llm.LLMContext: + return llm.LLMContext( + platform="test", + context=Context(), + language="en", + assistant=conversation.DOMAIN, + device_id=None, + ) + + +async def tool_names(hass) -> list[str]: + api = await llm.async_get_api(hass, llm.LLM_API_ASSIST, assist_context()) + return [tool.name for tool in api.tools] + + +async def call_tool(hass, name, args): + api = await llm.async_get_api(hass, llm.LLM_API_ASSIST, assist_context()) + return await api.async_call_tool(llm.ToolInput(tool_name=name, tool_args=args)) + + +@pytest.fixture +async def house(hass, mock_client, config_entry): + """A loaded entry with the tools on, one exposed sensor and one private one.""" + assert await async_setup_component(hass, "conversation", {}) + assert await async_setup_component(hass, "llm", {}) + hass.states.async_set("sensor.washer_power", "1.2", {"unit_of_measurement": "W"}) + hass.states.async_set("lock.front_door", "unlocked") + async_expose_entity(hass, conversation.DOMAIN, "sensor.washer_power", True) + async_expose_entity(hass, conversation.DOMAIN, "lock.front_door", False) + + config_entry.add_to_hass(hass) + hass.config_entries.async_update_entry(config_entry, options={CONF_LLM_TOOLS: True}) + assert await hass.config_entries.async_setup(config_entry.entry_id) + await hass.async_block_till_done() + mock_client.ask.reset_mock() + return config_entry + + +async def test_the_tools_are_off_until_the_options_turn_them_on( + hass, mock_client, config_entry +): + """Every tool schema is prompt text an agent's provider bills on every turn.""" + assert await async_setup_component(hass, "conversation", {}) + assert await async_setup_component(hass, "llm", {}) + config_entry.add_to_hass(hass) + assert await hass.config_entries.async_setup(config_entry.entry_id) + await hass.async_block_till_done() + assert not [name for name in await tool_names(hass) if name.startswith("jev__")] + + hass.config_entries.async_update_entry(config_entry, options={CONF_LLM_TOOLS: True}) + await hass.async_block_till_done() + assert {"jev__noul", "jev__choice"} <= set(await tool_names(hass)) + + +async def test_another_api_gets_no_tools(hass, house): + from custom_components.jev.llm import async_get_tools + + assert async_get_tools(hass, assist_context(), "some_other_api") is None + + +async def test_noul_judges_only_what_assist_may_see(hass, house, mock_client): + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.83)) + result = await call_tool( + hass, "jev__noul", {"question": "Is the washing machine done?"} + ) + assert result == {"probability_yes": 0.83} + + state, questions = mock_client.ask.await_args.args + assert [e["entity_id"] for e in state["entities"]] == ["sensor.washer_power"] + assert questions["answer"].instructions == "Is the washing machine done?" + assert "note" not in state + + +async def test_facts_are_sent_as_they_are_and_never_rendered(hass, house, mock_client): + """A rendered template could read the lock, which is not exposed to Assist.""" + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.5)) + facts = "The user says: {{ states('lock.front_door') }}" + await call_tool(hass, "jev__noul", {"question": "Is it done?", "facts": facts}) + + state, _ = mock_client.ask.await_args.args + assert state["note"] == {"facts": facts} + assert "unlocked" not in str(state) + + +async def test_choice_returns_the_distribution(hass, house, mock_client): + mock_client.ask.return_value = build_response( + answer=ChoiceAnswer( + choice="washing", + probabilities={"washing": 0.9, "done": 0.1}, + confidence=0.8, + ) + ) + result = await call_tool( + hass, + "jev__choice", + {"question": "What is the washer doing?", "options": ["washing", "done"]}, + ) + assert result == { + "choice": "washing", + "probabilities": {"washing": 0.9, "done": 0.1}, + "confidence": 0.8, + } + _, questions = mock_client.ask.await_args.args + assert set(questions["answer"].criteria) == {"washing", "done"} + + +async def test_a_choice_needs_two_options(hass, house, mock_client): + with pytest.raises(vol.Invalid): + await call_tool( + hass, "jev__choice", {"question": "Which?", "options": ["only one"]} + ) + assert mock_client.ask.await_count == 0 + + +async def test_with_two_accounts_the_agent_names_the_one_that_pays( + hass, house, mock_client +): + """Each entry has its own key and budget, so neither is picked for the agent.""" + guest = MockConfigEntry( + domain=DOMAIN, + title="Jev guest", + data={CONF_API_KEY: "another-key-not-a-real-one"}, + options={CONF_LLM_TOOLS: True}, + unique_id="fedcba9876543210", + ) + guest.add_to_hass(hass) + assert await hass.config_entries.async_setup(guest.entry_id) + await hass.async_block_till_done() + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.4)) + + with pytest.raises(vol.Invalid): + await call_tool(hass, "jev__noul", {"question": "Is it done?"}) + + house_calls = house.runtime_data.usage.calls + result = await call_tool( + hass, "jev__noul", {"question": "Is it done?", "account": "Jev guest"} + ) + assert result == {"probability_yes": 0.4} + assert house.runtime_data.usage.calls == house_calls + assert guest.runtime_data.usage.input_tokens == PROBE_TOKENS + 321 From 46fa039cacf467d3a41070b2c00ebe787bde0316 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 22:18:50 +0200 Subject: [PATCH 06/24] A new action finds the threshold that fits a yes/no sensor, from its recorded history --- custom_components/jev/calibrate.py | 256 ++++++++++++++++++ custom_components/jev/const.py | 1 + custom_components/jev/icons.json | 3 +- custom_components/jev/manifest.json | 2 +- custom_components/jev/services.py | 8 +- custom_components/jev/services.yaml | 25 ++ custom_components/jev/strings.json | 34 +++ custom_components/jev/translations/cs.json | 34 +++ custom_components/jev/translations/da.json | 34 +++ custom_components/jev/translations/de.json | 34 +++ custom_components/jev/translations/en.json | 34 +++ custom_components/jev/translations/es.json | 34 +++ custom_components/jev/translations/fr.json | 34 +++ custom_components/jev/translations/it.json | 34 +++ custom_components/jev/translations/nl.json | 34 +++ custom_components/jev/translations/pl.json | 34 +++ custom_components/jev/translations/pt-BR.json | 34 +++ custom_components/jev/translations/ru.json | 34 +++ custom_components/jev/translations/sv.json | 34 +++ .../jev/translations/zh-Hans.json | 34 +++ tests/test_calibrate.py | 175 ++++++++++++ tests/test_services.py | 10 + 22 files changed, 953 insertions(+), 3 deletions(-) create mode 100644 custom_components/jev/calibrate.py create mode 100644 tests/test_calibrate.py diff --git a/custom_components/jev/calibrate.py b/custom_components/jev/calibrate.py new file mode 100644 index 0000000..a6ee4f0 --- /dev/null +++ b/custom_components/jev/calibrate.py @@ -0,0 +1,256 @@ +"""Pick a threshold from what actually happened, instead of guessing one. + +A noul is a probability, and the threshold that turns it into yes or no is the +user's call. 0.5 is where a noul says "cannot tell", not where a given house's +washing machine is done. This action reads the recorder: the noul's history next to +the history of an entity that says what was really true, such as a door contact or +a smart plug's own "running" state. For every threshold it measures how much of the +time a yes was right (precision) and how much of the true time it said yes +(recall), and returns the threshold that balances the two best. + +Nothing is sent to TypeSafe, so it costs no tokens. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from statistics import median +from typing import Any, Final + +import voluptuous as vol +from homeassistant.components.recorder import history +from homeassistant.const import ATTR_ENTITY_ID, STATE_ON +from homeassistant.core import HomeAssistant, ServiceCall, ServiceResponse, State +from homeassistant.exceptions import ServiceValidationError +from homeassistant.helpers import config_validation as cv +from homeassistant.helpers.recorder import get_instance +from homeassistant.util import dt as dt_util + +from .const import DOMAIN + +CONF_TRUTH_ENTITY: Final = "truth_entity_id" +CONF_TRUTH_STATE: Final = "truth_state" +CONF_DAYS: Final = "days" + +# The recorder keeps 10 days by default (`purge_keep_days`), so a week is inside +# what a default install still has. A longer window only reads further back. +DEFAULT_DAYS: Final = 7 + +# Every hundredth. The noul sensor itself rounds to three places, but nobody sets a +# threshold that fine, and two neighbouring hundredths rarely differ in outcome. +CANDIDATES: Final = tuple(round(i / 100, 2) for i in range(1, 100)) + +# The coarse table returned next to the best threshold, so a caller can see how +# steep the trade-off is around it. +TABLE: Final = tuple(round(i / 10, 1) for i in range(1, 10)) + +CALIBRATE_SCHEMA: Final = vol.Schema( + { + vol.Required(ATTR_ENTITY_ID): cv.entity_id, + vol.Required(CONF_TRUTH_ENTITY): cv.entity_id, + vol.Optional(CONF_TRUTH_STATE, default=STATE_ON): cv.string, + vol.Optional(CONF_DAYS, default=DEFAULT_DAYS): vol.All( + vol.Coerce(int), vol.Range(min=1) + ), + } +) + + +@dataclass(frozen=True, slots=True) +class Span: + """A stretch of time where neither the probability nor the truth changed.""" + + probability: float + truth: bool + seconds: float + + +@dataclass(frozen=True, slots=True) +class Outcome: + """What one threshold would have said over the spans, in seconds.""" + + threshold: float + true_yes: float + false_yes: float + missed: float + + @property + def precision(self) -> float | None: + said_yes = self.true_yes + self.false_yes + return self.true_yes / said_yes if said_yes else None + + @property + def recall(self) -> float | None: + was_true = self.true_yes + self.missed + return self.true_yes / was_true if was_true else None + + @property + def f1(self) -> float: + precision, recall = self.precision, self.recall + if not precision or not recall: + return 0.0 + return 2 * precision * recall / (precision + recall) + + def as_dict(self) -> dict[str, Any]: + return { + "threshold": self.threshold, + "precision": _round(self.precision), + "recall": _round(self.recall), + "f1": round(self.f1, 3), + } + + +def _round(value: float | None) -> float | None: + return None if value is None else round(value, 3) + + +def outcome(spans: list[Span], threshold: float) -> Outcome: + true_yes = false_yes = missed = 0.0 + for span in spans: + said_yes = span.probability >= threshold + if said_yes and span.truth: + true_yes += span.seconds + elif said_yes: + false_yes += span.seconds + elif span.truth: + missed += span.seconds + return Outcome(threshold, true_yes, false_yes, missed) + + +def best(spans: list[Span]) -> Outcome: + """The threshold with the highest F1. + + Several neighbouring thresholds often tie, because no probability fell between + them. The middle of the tied ones is returned rather than an edge, so a noul + that lands a little off its usual values still falls on the same side. + """ + outcomes = [outcome(spans, threshold) for threshold in CANDIDATES] + top = max(o.f1 for o in outcomes) + tied = [o.threshold for o in outcomes if o.f1 == top] + return outcome(spans, round(median(tied), 2)) + + +def build_spans( + probabilities: list[State], + truths: list[State], + truth_state: str, + start: datetime, + end: datetime, +) -> list[Span]: + """Cut the window at every change of either entity. + + The state an entity already had when the window opened counts from the start of + the window. A span where the probability is not a number, such as unavailable + while the API was down, is left out rather than guessed. + """ + changes = sorted( + [(max(s.last_changed, start), "p", s.state) for s in probabilities] + + [(max(s.last_changed, start), "t", s.state) for s in truths], + key=lambda change: change[0], + ) + spans: list[Span] = [] + probability: float | None = None + truth: bool | None = None + for index, (when, kind, value) in enumerate(changes): + if kind == "p": + probability = _number(value) + else: + truth = value == truth_state + until = changes[index + 1][0] if index + 1 < len(changes) else end + seconds = (until - when).total_seconds() + if probability is not None and truth is not None and seconds > 0: + spans.append(Span(probability, truth, seconds)) + return spans + + +def _number(value: str) -> float | None: + try: + return float(value) + except ValueError: + return None + + +async def async_calibrate(hass: HomeAssistant, call: ServiceCall) -> ServiceResponse: + source = call.data[ATTR_ENTITY_ID] + truth_entity = call.data[CONF_TRUTH_ENTITY] + truth_state = call.data[CONF_TRUTH_STATE] + days = call.data[CONF_DAYS] + if "recorder" not in hass.config.components: + raise ServiceValidationError( + translation_domain=DOMAIN, translation_key="calibrate_needs_recorder" + ) + end = dt_util.utcnow() + start = end - timedelta(days=days) + found = await get_instance(hass).async_add_executor_job( + _history, hass, start, end, [source, truth_entity] + ) + spans = build_spans( + found.get(source, []), found.get(truth_entity, []), truth_state, start, end + ) + if not spans: + raise ServiceValidationError( + translation_domain=DOMAIN, + translation_key="calibrate_no_history", + translation_placeholders={ + "entity": source, + "truth": truth_entity, + "days": str(days), + }, + ) + true_seconds = sum(span.seconds for span in spans if span.truth) + # With only one side there is nothing to separate, and every threshold scores + # the same. + if true_seconds in (0, sum(span.seconds for span in spans)): + raise ServiceValidationError( + translation_domain=DOMAIN, + translation_key=( + "calibrate_never_true" if true_seconds == 0 else "calibrate_always_true" + ), + translation_placeholders={ + "truth": truth_entity, + "state": truth_state, + "days": str(days), + }, + ) + return { + **best(spans).as_dict(), + "hours": round(sum(span.seconds for span in spans) / 3600, 1), + "hours_true": round(true_seconds / 3600, 1), + # F1 over two occasions is noise, whatever its value. These are the counts + # to judge it by. + "times_true": _times_true(spans), + "probability_changes": sum( + 1 for s in found.get(source, []) if _number(s.state) is not None + ), + "table": [outcome(spans, threshold).as_dict() for threshold in TABLE], + } + + +def _times_true(spans: list[Span]) -> int: + """How many separate stretches the truth was on for.""" + count = 0 + previous = False + for span in spans: + if span.truth and not previous: + count += 1 + previous = span.truth + return count + + +def _history( + hass: HomeAssistant, start: datetime, end: datetime, entity_ids: list[str] +) -> dict[str, list[State]]: + found = history.get_significant_states( + hass, + start, + end, + entity_ids, + include_start_time_state=True, + significant_changes_only=False, + no_attributes=True, + ) + return { + entity_id: [s for s in states if isinstance(s, State)] + for entity_id, states in found.items() + } diff --git a/custom_components/jev/const.py b/custom_components/jev/const.py index 126ac69..dde5175 100644 --- a/custom_components/jev/const.py +++ b/custom_components/jev/const.py @@ -75,6 +75,7 @@ SERVICE_NOUL: Final = "noul" SERVICE_CHOICE: Final = "choice" SERVICE_SCORE: Final = "score" +SERVICE_CALIBRATE: Final = "calibrate" CONF_TRUE_MEANS: Final = "true_means" CONF_FALSE_MEANS: Final = "false_means" diff --git a/custom_components/jev/icons.json b/custom_components/jev/icons.json index 02c1706..9bdb5ed 100644 --- a/custom_components/jev/icons.json +++ b/custom_components/jev/icons.json @@ -30,6 +30,7 @@ "noul": "mdi:comment-question-outline", "choice": "mdi:format-list-checks", "score": "mdi:ruler", - "ask": "mdi:comment-multiple-outline" + "ask": "mdi:comment-multiple-outline", + "calibrate": "mdi:tune-vertical" } } diff --git a/custom_components/jev/manifest.json b/custom_components/jev/manifest.json index 439ca71..1ed6a55 100644 --- a/custom_components/jev/manifest.json +++ b/custom_components/jev/manifest.json @@ -1,7 +1,7 @@ { "domain": "jev", "name": "Jev (TypeSafe)", - "after_dependencies": ["conversation"], + "after_dependencies": ["conversation", "recorder"], "codeowners": ["@abovecolin"], "config_flow": true, "dependencies": [], diff --git a/custom_components/jev/services.py b/custom_components/jev/services.py index fafb146..0c5604e 100644 --- a/custom_components/jev/services.py +++ b/custom_components/jev/services.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: from . import JevConfigEntry +from .calibrate import CALIBRATE_SCHEMA, async_calibrate from .const import ( ATTR_ANSWERS, ATTR_CONFIG_ENTRY, @@ -64,6 +65,7 @@ CONF_TRUE_MEANS, DOMAIN, SERVICE_ASK, + SERVICE_CALIBRATE, SERVICE_CHOICE, SERVICE_NOUL, SERVICE_SCORE, @@ -283,7 +285,7 @@ def answer_as_dict(answer: Any) -> dict[str, Any]: def async_register_services(hass: HomeAssistant) -> None: - """Register all four actions once, the first time the component loads.""" + """Register all five actions once, the first time the component loads.""" if hass.services.has_service(DOMAIN, SERVICE_NOUL): return @@ -382,11 +384,15 @@ async def _ask_many(call: ServiceCall) -> ServiceResponse: **_envelope(response), } + async def _calibrate(call: ServiceCall) -> ServiceResponse: + return await async_calibrate(hass, call) + for name, handler, schema in ( (SERVICE_NOUL, _noul, NOUL_SCHEMA), (SERVICE_CHOICE, _choice, CHOICE_SCHEMA), (SERVICE_SCORE, _score, SCORE_SCHEMA), (SERVICE_ASK, _ask_many, ASK_SCHEMA), + (SERVICE_CALIBRATE, _calibrate, CALIBRATE_SCHEMA), ): hass.services.async_register( DOMAIN, name, handler, schema=schema, supports_response=SupportsResponse.ONLY diff --git a/custom_components/jev/services.yaml b/custom_components/jev/services.yaml index bab8b08..e35dd40 100644 --- a/custom_components/jev/services.yaml +++ b/custom_components/jev/services.yaml @@ -196,3 +196,28 @@ ask: selector: config_entry: integration: jev + +calibrate: + fields: + entity_id: + required: true + selector: + entity: + domain: sensor + truth_entity_id: + required: true + selector: + entity: {} + truth_state: + required: false + default: "on" + selector: + text: + days: + required: false + default: 7 + selector: + number: + min: 1 + mode: box + unit_of_measurement: days diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..68f7d28 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -228,6 +228,28 @@ "description": "Off by default. Sends every attribute of the picked entities, not just the value, unit, device class and area. A weather forecast or a media player artwork list runs to thousands of tokens, and you pay for them on every evaluation." } } + }, + "calibrate": { + "name": "Find the best threshold", + "description": "Compare the recorded history of a yes/no probability with an entity that shows what was really true, and get the threshold that would have been right most often. Reads only the recorder, so it costs no tokens.", + "fields": { + "entity_id": { + "name": "Probability", + "description": "A Jev yes/no sensor, or any sensor whose state is a probability from 0 to 1." + }, + "truth_entity_id": { + "name": "What was really true", + "description": "An entity whose state shows the real answer, such as a door contact or a plug that reports when a machine runs." + }, + "truth_state": { + "name": "State that means yes", + "description": "The state of that entity that means yes. on if you leave it empty." + }, + "days": { + "name": "Days of history", + "description": "How far back to read. The recorder keeps 10 days unless you changed it." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe rate limited the request: {reason}" + }, + "calibrate_needs_recorder": { + "message": "This action reads the recorder, and the recorder is not running." + }, + "calibrate_no_history": { + "message": "No time in the last {days} days has both a number from {entity} and a state from {truth}. Check that the recorder records both, and that {entity} reports a number." + }, + "calibrate_never_true": { + "message": "{truth} was never {state} in the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." + }, + "calibrate_always_true": { + "message": "{truth} was {state} for all of the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." } }, "common": { diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..8925543 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -228,6 +228,28 @@ "description": "Ve výchozím stavu vypnuto. Posílá každý atribut vybraných entit, nejen hodnotu, jednotku, třídu zařízení a oblast. Předpověď počasí nebo seznam obalů alb v přehrávači médií vydá na tisíce tokenů a ty zaplatíte při každém vyhodnocení." } } + }, + "calibrate": { + "name": "Najít nejlepší práh", + "description": "Porovná zaznamenanou historii pravděpodobnosti ano/ne s entitou, která ukazuje, co skutečně platilo, a vrátí práh, který by nejčastěji měl pravdu. Čte jen recorder, takže nestojí žádné tokeny.", + "fields": { + "entity_id": { + "name": "Pravděpodobnost", + "description": "Senzor ano/ne od Jev, nebo jakýkoli senzor, jehož stav je pravděpodobnost od 0 do 1." + }, + "truth_entity_id": { + "name": "Co skutečně platilo", + "description": "Entita, jejíž stav ukazuje skutečnou odpověď, například kontakt dveří nebo zásuvka, která hlásí, kdy spotřebič běží." + }, + "truth_state": { + "name": "Stav znamenající ano", + "description": "Stav té entity, který znamená ano. on, pokud pole necháte prázdné." + }, + "days": { + "name": "Dny historie", + "description": "Jak daleko zpět číst. Recorder uchovává 10 dní, pokud jste to nezměnili." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe omezil požadavek: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Tato akce čte recorder a recorder neběží." + }, + "calibrate_no_history": { + "message": "V posledních {days} dnech není žádný okamžik s číslem z {entity} i stavem z {truth}. Zkontrolujte, že recorder zaznamenává obojí a že {entity} hlásí číslo." + }, + "calibrate_never_true": { + "message": "{truth} nebyl v posledních {days} dnech nikdy {state}, takže není s čím porovnat. Čtěte delší období, nebo zkontrolujte, který stav znamená ano." + }, + "calibrate_always_true": { + "message": "{truth} byl celých posledních {days} dní {state}, takže není s čím porovnat. Čtěte delší období, nebo zkontrolujte, který stav znamená ano." } }, "common": { diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..bacdca1 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -228,6 +228,28 @@ "description": "Slået fra som standard. Sender alle attributter for de valgte entiteter, ikke kun værdien, måleenheden, enhedsklassen og området. En vejrudsigt eller en medieafspillers liste med albumbilleder løber op i tusindvis af tokens, og dem betaler du for ved hver evaluering." } } + }, + "calibrate": { + "name": "Find den bedste tærskel", + "description": "Sammenligner den gemte historik for en ja/nej-sandsynlighed med en enhed, der viser, hvad der faktisk var sandt, og giver den tærskel, der oftest ville have haft ret. Læser kun recorderen og koster derfor ingen tokens.", + "fields": { + "entity_id": { + "name": "Sandsynlighed", + "description": "En Jev ja/nej-sensor eller en anden sensor, hvis tilstand er en sandsynlighed fra 0 til 1." + }, + "truth_entity_id": { + "name": "Hvad der faktisk var sandt", + "description": "En enhed, hvis tilstand viser det rigtige svar, f.eks. en dørkontakt eller et stik, der melder, når et apparat kører." + }, + "truth_state": { + "name": "Tilstand der betyder ja", + "description": "Den tilstand for enheden, der betyder ja. on hvis du lader feltet være tomt." + }, + "days": { + "name": "Dages historik", + "description": "Hvor langt tilbage der læses. Recorderen gemmer 10 dage, medmindre du har ændret det." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe begrænsede forespørgslen: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Denne handling læser recorderen, og recorderen kører ikke." + }, + "calibrate_no_history": { + "message": "Intet tidspunkt i de seneste {days} dage har både et tal fra {entity} og en tilstand fra {truth}. Kontroller, at recorderen gemmer begge, og at {entity} melder et tal." + }, + "calibrate_never_true": { + "message": "{truth} var aldrig {state} i de seneste {days} dage, så der er intet at sammenligne med. Læs en længere periode, eller kontroller, hvilken tilstand der betyder ja." + }, + "calibrate_always_true": { + "message": "{truth} var {state} i hele de seneste {days} dage, så der er intet at sammenligne med. Læs en længere periode, eller kontroller, hvilken tilstand der betyder ja." } }, "common": { diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..39e9f8f 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -228,6 +228,28 @@ "description": "Standardmäßig aus. Sendet jedes Attribut der gewählten Entitäten mit, nicht nur Wert, Einheit, Geräteklasse und Bereich. Eine Wettervorhersage oder die Coverliste eines Media Players geht in die Tausende von Tokens, und die bezahlst du bei jeder Auswertung." } } + }, + "calibrate": { + "name": "Besten Schwellenwert finden", + "description": "Vergleicht den aufgezeichneten Verlauf einer Ja/Nein-Wahrscheinlichkeit mit einer Entität, die zeigt, was wirklich zutraf, und liefert den Schwellenwert, der am häufigsten richtig gewesen wäre. Liest nur den Recorder und kostet daher keine Tokens.", + "fields": { + "entity_id": { + "name": "Wahrscheinlichkeit", + "description": "Ein Jev-Ja/Nein-Sensor oder ein anderer Sensor, dessen Zustand eine Wahrscheinlichkeit von 0 bis 1 ist." + }, + "truth_entity_id": { + "name": "Was wirklich zutraf", + "description": "Eine Entität, deren Zustand die echte Antwort zeigt, etwa ein Türkontakt oder eine Steckdose, die meldet, wann ein Gerät läuft." + }, + "truth_state": { + "name": "Zustand für Ja", + "description": "Der Zustand dieser Entität, der Ja bedeutet. on, wenn du das Feld leer lässt." + }, + "days": { + "name": "Tage Verlauf", + "description": "Wie weit zurück gelesen wird. Der Recorder behält 10 Tage, sofern du das nicht geändert hast." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe hat die Anfrage gedrosselt: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Diese Aktion liest den Recorder, und der Recorder läuft nicht." + }, + "calibrate_no_history": { + "message": "In den letzten {days} Tagen gibt es keinen Zeitraum mit einer Zahl von {entity} und einem Zustand von {truth}. Prüfe, ob der Recorder beide aufzeichnet und ob {entity} eine Zahl meldet." + }, + "calibrate_never_true": { + "message": "{truth} war in den letzten {days} Tagen nie {state}, also gibt es nichts zum Vergleichen. Lies einen längeren Zeitraum oder prüfe, welcher Zustand Ja bedeutet." + }, + "calibrate_always_true": { + "message": "{truth} war die gesamten letzten {days} Tage {state}, also gibt es nichts zum Vergleichen. Lies einen längeren Zeitraum oder prüfe, welcher Zustand Ja bedeutet." } }, "common": { diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..68f7d28 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -228,6 +228,28 @@ "description": "Off by default. Sends every attribute of the picked entities, not just the value, unit, device class and area. A weather forecast or a media player artwork list runs to thousands of tokens, and you pay for them on every evaluation." } } + }, + "calibrate": { + "name": "Find the best threshold", + "description": "Compare the recorded history of a yes/no probability with an entity that shows what was really true, and get the threshold that would have been right most often. Reads only the recorder, so it costs no tokens.", + "fields": { + "entity_id": { + "name": "Probability", + "description": "A Jev yes/no sensor, or any sensor whose state is a probability from 0 to 1." + }, + "truth_entity_id": { + "name": "What was really true", + "description": "An entity whose state shows the real answer, such as a door contact or a plug that reports when a machine runs." + }, + "truth_state": { + "name": "State that means yes", + "description": "The state of that entity that means yes. on if you leave it empty." + }, + "days": { + "name": "Days of history", + "description": "How far back to read. The recorder keeps 10 days unless you changed it." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe rate limited the request: {reason}" + }, + "calibrate_needs_recorder": { + "message": "This action reads the recorder, and the recorder is not running." + }, + "calibrate_no_history": { + "message": "No time in the last {days} days has both a number from {entity} and a state from {truth}. Check that the recorder records both, and that {entity} reports a number." + }, + "calibrate_never_true": { + "message": "{truth} was never {state} in the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." + }, + "calibrate_always_true": { + "message": "{truth} was {state} for all of the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." } }, "common": { diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..5d8edd9 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -228,6 +228,28 @@ "description": "Desactivado por defecto. Envía todos los atributos de las entidades elegidas, no solo el valor, la unidad, la clase de dispositivo y el área. Una previsión meteorológica o la lista de carátulas de un reproductor multimedia llega a miles de tokens, y los pagas en cada evaluación." } } + }, + "calibrate": { + "name": "Encontrar el mejor umbral", + "description": "Compara el historial grabado de una probabilidad de sí o no con una entidad que muestra lo que era cierto de verdad, y devuelve el umbral que habría acertado más veces. Solo lee el recorder, así que no cuesta tokens.", + "fields": { + "entity_id": { + "name": "Probabilidad", + "description": "Un sensor de sí o no de Jev, o cualquier sensor cuyo estado sea una probabilidad de 0 a 1." + }, + "truth_entity_id": { + "name": "Lo que era cierto", + "description": "Una entidad cuyo estado da la respuesta real, como un contacto de puerta o un enchufe que indica cuándo funciona un aparato." + }, + "truth_state": { + "name": "Estado que significa sí", + "description": "El estado de esa entidad que significa sí. on si lo dejas vacío." + }, + "days": { + "name": "Días de historial", + "description": "Hasta dónde leer hacia atrás. El recorder guarda 10 días, salvo que lo hayas cambiado." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe limitó la solicitud: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Esta acción lee el recorder, y el recorder no está en marcha." + }, + "calibrate_no_history": { + "message": "Ningún momento de los últimos {days} días tiene a la vez un número de {entity} y un estado de {truth}. Comprueba que el recorder guarda los dos, y que {entity} da un número." + }, + "calibrate_never_true": { + "message": "{truth} nunca estuvo en {state} en los últimos {days} días, así que no hay nada con qué comparar. Lee un periodo más largo, o comprueba qué estado significa sí." + }, + "calibrate_always_true": { + "message": "{truth} estuvo en {state} durante todos los últimos {days} días, así que no hay nada con qué comparar. Lee un periodo más largo, o comprueba qué estado significa sí." } }, "common": { diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..338c90a 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -228,6 +228,28 @@ "description": "Désactivé par défaut. Envoie tous les attributs des entités choisies, pas seulement la valeur, l'unité, la classe d'appareil et la zone. Une prévision météo ou une liste de pochettes d'un lecteur multimédia représente des milliers de jetons, et vous les payez à chaque évaluation." } } + }, + "calibrate": { + "name": "Trouver le meilleur seuil", + "description": "Compare l'historique enregistré d'une probabilité oui/non avec une entité qui montre ce qui était vraiment vrai, et donne le seuil qui aurait eu raison le plus souvent. Lit seulement le recorder, donc ne coûte aucun token.", + "fields": { + "entity_id": { + "name": "Probabilité", + "description": "Un capteur oui/non Jev, ou tout capteur dont l'état est une probabilité de 0 à 1." + }, + "truth_entity_id": { + "name": "Ce qui était vraiment vrai", + "description": "Une entité dont l'état donne la vraie réponse, comme un contact de porte ou une prise qui signale quand un appareil tourne." + }, + "truth_state": { + "name": "État qui veut dire oui", + "description": "L'état de cette entité qui veut dire oui. on si vous laissez vide." + }, + "days": { + "name": "Jours d'historique", + "description": "Jusqu'où remonter. Le recorder garde 10 jours, sauf si vous l'avez modifié." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe a limité la requête : {reason}" + }, + "calibrate_needs_recorder": { + "message": "Cette action lit le recorder, et le recorder ne tourne pas." + }, + "calibrate_no_history": { + "message": "Aucun moment des {days} derniers jours n'a à la fois un nombre de {entity} et un état de {truth}. Vérifiez que le recorder enregistre les deux, et que {entity} donne un nombre." + }, + "calibrate_never_true": { + "message": "{truth} n'a jamais été {state} pendant les {days} derniers jours, il n'y a donc rien à comparer. Lisez une période plus longue, ou vérifiez quel état veut dire oui." + }, + "calibrate_always_true": { + "message": "{truth} a été {state} pendant tous les {days} derniers jours, il n'y a donc rien à comparer. Lisez une période plus longue, ou vérifiez quel état veut dire oui." } }, "common": { diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..0abc0d6 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -228,6 +228,28 @@ "description": "Disattivo per impostazione predefinita. Invia ogni attributo delle entità scelte, non solo il valore, l'unità, la classe del dispositivo e l'area. Una previsione meteo o l'elenco delle copertine di un lettore multimediale arriva a migliaia di token, e li paghi a ogni valutazione." } } + }, + "calibrate": { + "name": "Trova la soglia migliore", + "description": "Confronta la cronologia registrata di una probabilità sì/no con un'entità che mostra cosa era davvero vero, e restituisce la soglia che avrebbe indovinato più spesso. Legge solo il recorder, quindi non costa token.", + "fields": { + "entity_id": { + "name": "Probabilità", + "description": "Un sensore sì/no di Jev, o qualsiasi sensore il cui stato sia una probabilità da 0 a 1." + }, + "truth_entity_id": { + "name": "Cosa era davvero vero", + "description": "Un'entità il cui stato dà la risposta reale, come un contatto porta o una presa che segnala quando un apparecchio è in funzione." + }, + "truth_state": { + "name": "Stato che significa sì", + "description": "Lo stato di quell'entità che significa sì. on se lo lasci vuoto." + }, + "days": { + "name": "Giorni di cronologia", + "description": "Quanto indietro leggere. Il recorder conserva 10 giorni, salvo modifiche." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe ha limitato la richiesta: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Questa azione legge il recorder, e il recorder non è in esecuzione." + }, + "calibrate_no_history": { + "message": "Nessun momento degli ultimi {days} giorni ha sia un numero da {entity} sia uno stato da {truth}. Controlla che il recorder registri entrambi e che {entity} dia un numero." + }, + "calibrate_never_true": { + "message": "{truth} non è mai stato {state} negli ultimi {days} giorni, quindi non c'è nulla da confrontare. Leggi un periodo più lungo, o controlla quale stato significa sì." + }, + "calibrate_always_true": { + "message": "{truth} è stato {state} per tutti gli ultimi {days} giorni, quindi non c'è nulla da confrontare. Leggi un periodo più lungo, o controlla quale stato significa sì." } }, "common": { diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..d46aca9 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -228,6 +228,28 @@ "description": "Staat uit. Stuurt alle attributen van de gekozen entiteiten mee, niet alleen de waarde, eenheid, device class en ruimte. Een weersverwachting of een lijst met albumhoezen loopt in de duizenden tokens, en die betaal je bij elke evaluatie." } } + }, + "calibrate": { + "name": "Beste drempel bepalen", + "description": "Vergelijkt de opgeslagen geschiedenis van een ja/nee-kans met een entiteit die laat zien wat echt waar was, en geeft de drempel die het vaakst goed zou zijn geweest. Leest alleen de recorder en kost dus geen tokens.", + "fields": { + "entity_id": { + "name": "Kans", + "description": "Een Jev ja/nee-sensor, of een andere sensor waarvan de status een kans van 0 tot 1 is." + }, + "truth_entity_id": { + "name": "Wat echt waar was", + "description": "Een entiteit waarvan de status het echte antwoord geeft, zoals een deurcontact of een stekker die meldt wanneer een apparaat draait." + }, + "truth_state": { + "name": "Status die ja betekent", + "description": "De status van die entiteit die ja betekent. on als je dit leeg laat." + }, + "days": { + "name": "Dagen geschiedenis", + "description": "Hoe ver terug er gelezen wordt. De recorder bewaart 10 dagen, tenzij je dat hebt aangepast." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe heeft het verzoek begrensd: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Deze actie leest de recorder, en de recorder draait niet." + }, + "calibrate_no_history": { + "message": "Er is in de laatste {days} dagen geen moment met zowel een getal van {entity} als een status van {truth}. Controleer of de recorder beide opslaat, en of {entity} een getal geeft." + }, + "calibrate_never_true": { + "message": "{truth} was in de laatste {days} dagen nooit {state}, dus er is niets om mee te vergelijken. Lees een langere periode, of controleer welke status ja betekent." + }, + "calibrate_always_true": { + "message": "{truth} was de hele laatste {days} dagen {state}, dus er is niets om mee te vergelijken. Lees een langere periode, of controleer welke status ja betekent." } }, "common": { diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..f68e736 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -228,6 +228,28 @@ "description": "Domyślnie wyłączone. Wysyła każdy atrybut wybranych encji, nie tylko wartość, jednostkę, klasę urządzenia i obszar. Prognoza pogody albo lista okładek odtwarzacza multimediów to tysiące tokenów, za które płacisz przy każdej ocenie." } } + }, + "calibrate": { + "name": "Znajdź najlepszy próg", + "description": "Porównuje zapisaną historię prawdopodobieństwa tak/nie z encją, która pokazuje, co było naprawdę prawdą, i zwraca próg, który najczęściej miałby rację. Czyta tylko recorder, więc nie kosztuje tokenów.", + "fields": { + "entity_id": { + "name": "Prawdopodobieństwo", + "description": "Czujnik tak/nie z Jev albo dowolny czujnik, którego stan jest prawdopodobieństwem od 0 do 1." + }, + "truth_entity_id": { + "name": "Co było prawdą", + "description": "Encja, której stan pokazuje prawdziwą odpowiedź, na przykład kontaktron drzwi albo gniazdko, które zgłasza, kiedy urządzenie pracuje." + }, + "truth_state": { + "name": "Stan oznaczający tak", + "description": "Stan tej encji, który oznacza tak. on, jeśli zostawisz pole puste." + }, + "days": { + "name": "Dni historii", + "description": "Jak daleko wstecz czytać. Recorder przechowuje 10 dni, chyba że to zmieniono." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe ograniczył żądanie: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Ta akcja czyta recorder, a recorder nie działa." + }, + "calibrate_no_history": { + "message": "W ostatnich {days} dniach nie ma chwili, w której jest zarówno liczba z {entity}, jak i stan z {truth}. Sprawdź, czy recorder zapisuje obie encje i czy {entity} podaje liczbę." + }, + "calibrate_never_true": { + "message": "{truth} ani razu nie miał stanu {state} w ostatnich {days} dniach, więc nie ma z czym porównać. Czytaj dłuższy okres albo sprawdź, który stan oznacza tak." + }, + "calibrate_always_true": { + "message": "{truth} miał stan {state} przez całe ostatnie {days} dni, więc nie ma z czym porównać. Czytaj dłuższy okres albo sprawdź, który stan oznacza tak." } }, "common": { diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..0f63aca 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -228,6 +228,28 @@ "description": "Desligado por padrão. Envia todos os atributos das entidades escolhidas, não apenas o valor, a unidade, a classe de dispositivo e a área. Uma previsão do tempo ou a lista de capas de um reprodutor de mídia chega a milhares de tokens, e você paga por eles em cada avaliação." } } + }, + "calibrate": { + "name": "Encontrar o melhor limite", + "description": "Compara o histórico gravado de uma probabilidade de sim ou não com uma entidade que mostra o que era de fato verdade, e retorna o limite que teria acertado mais vezes. Lê apenas o recorder, então não custa tokens.", + "fields": { + "entity_id": { + "name": "Probabilidade", + "description": "Um sensor de sim ou não do Jev, ou qualquer sensor cujo estado seja uma probabilidade de 0 a 1." + }, + "truth_entity_id": { + "name": "O que era verdade", + "description": "Uma entidade cujo estado mostra a resposta real, como um contato de porta ou uma tomada que informa quando um aparelho está ligado." + }, + "truth_state": { + "name": "Estado que significa sim", + "description": "O estado dessa entidade que significa sim. on se você deixar vazio." + }, + "days": { + "name": "Dias de histórico", + "description": "Até onde ler para trás. O recorder guarda 10 dias, a menos que você tenha mudado isso." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "O TypeSafe limitou a solicitação: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Esta ação lê o recorder, e o recorder não está em execução." + }, + "calibrate_no_history": { + "message": "Nenhum momento dos últimos {days} dias tem ao mesmo tempo um número de {entity} e um estado de {truth}. Verifique se o recorder grava os dois e se {entity} informa um número." + }, + "calibrate_never_true": { + "message": "{truth} nunca esteve em {state} nos últimos {days} dias, então não há com o que comparar. Leia um período maior, ou verifique qual estado significa sim." + }, + "calibrate_always_true": { + "message": "{truth} esteve em {state} durante todos os últimos {days} dias, então não há com o que comparar. Leia um período maior, ou verifique qual estado significa sim." } }, "common": { diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..ce2a4ba 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -228,6 +228,28 @@ "description": "По умолчанию выключено. Передаёт каждый атрибут выбранных объектов, а не только значение, единицу измерения, класс устройства и помещение. Прогноз погоды или список обложек в медиаплеере занимает тысячи токенов, и вы платите за них при каждой оценке." } } + }, + "calibrate": { + "name": "Найти лучший порог", + "description": "Сравнивает записанную историю вероятности да/нет с объектом, который показывает, что было на самом деле, и возвращает порог, который чаще всего оказался бы прав. Читает только recorder, поэтому не тратит токены.", + "fields": { + "entity_id": { + "name": "Вероятность", + "description": "Датчик да/нет от Jev или любой датчик, состояние которого является вероятностью от 0 до 1." + }, + "truth_entity_id": { + "name": "Что было на самом деле", + "description": "Объект, состояние которого показывает настоящий ответ, например датчик двери или розетка, которая сообщает, когда работает прибор." + }, + "truth_state": { + "name": "Состояние, означающее да", + "description": "Состояние этого объекта, которое означает да. on, если оставить поле пустым." + }, + "days": { + "name": "Дней истории", + "description": "Насколько далеко назад читать. Recorder хранит 10 дней, если вы это не меняли." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe ограничил запрос: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Это действие читает recorder, а recorder не запущен." + }, + "calibrate_no_history": { + "message": "За последние {days} дней нет ни одного момента, где есть и число от {entity}, и состояние от {truth}. Проверьте, что recorder записывает оба объекта и что {entity} сообщает число." + }, + "calibrate_never_true": { + "message": "{truth} ни разу не был в состоянии {state} за последние {days} дней, поэтому сравнивать не с чем. Прочитайте более длинный период или проверьте, какое состояние означает да." + }, + "calibrate_always_true": { + "message": "{truth} был в состоянии {state} все последние {days} дней, поэтому сравнивать не с чем. Прочитайте более длинный период или проверьте, какое состояние означает да." } }, "common": { diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..488ca5c 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -228,6 +228,28 @@ "description": "Av som standard. Skickar alla attribut för de valda entiteterna, inte bara värde, måttenhet, enhetsklass och område. En väderprognos eller en mediaspelares lista med omslagsbilder blir tusentals token, och du betalar för dem vid varje utvärdering." } } + }, + "calibrate": { + "name": "Hitta den bästa tröskeln", + "description": "Jämför den inspelade historiken för en ja/nej-sannolikhet med en entitet som visar vad som verkligen var sant, och ger den tröskel som oftast skulle ha haft rätt. Läser bara recordern och kostar därför inga tokens.", + "fields": { + "entity_id": { + "name": "Sannolikhet", + "description": "En Jev ja/nej-sensor eller en annan sensor vars tillstånd är en sannolikhet från 0 till 1." + }, + "truth_entity_id": { + "name": "Vad som verkligen var sant", + "description": "En entitet vars tillstånd visar det riktiga svaret, till exempel en dörrkontakt eller ett uttag som rapporterar när en apparat är igång." + }, + "truth_state": { + "name": "Tillstånd som betyder ja", + "description": "Tillståndet för entiteten som betyder ja. on om du lämnar fältet tomt." + }, + "days": { + "name": "Dagar historik", + "description": "Hur långt tillbaka som läses. Recordern sparar 10 dagar om du inte har ändrat det." + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe begränsade begäran: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Den här åtgärden läser recordern, och recordern körs inte." + }, + "calibrate_no_history": { + "message": "Ingen tidpunkt de senaste {days} dagarna har både ett tal från {entity} och ett tillstånd från {truth}. Kontrollera att recordern sparar båda och att {entity} rapporterar ett tal." + }, + "calibrate_never_true": { + "message": "{truth} var aldrig {state} de senaste {days} dagarna, så det finns inget att jämföra med. Läs en längre period, eller kontrollera vilket tillstånd som betyder ja." + }, + "calibrate_always_true": { + "message": "{truth} var {state} under hela de senaste {days} dagarna, så det finns inget att jämföra med. Läs en längre period, eller kontrollera vilket tillstånd som betyder ja." } }, "common": { diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..0bf139d 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -228,6 +228,28 @@ "description": "默认关闭。会发送所选实体的每一个属性，而不只是值、单位、设备类别和区域。一份天气预报或一个媒体播放器的封面列表可达数千个令牌，而且每次评估都要为此付费。" } } + }, + "calibrate": { + "name": "找出最佳阈值", + "description": "将是/否概率的记录历史与显示真实情况的实体进行比较，并返回最常判断正确的阈值。只读取 recorder，因此不消耗 token。", + "fields": { + "entity_id": { + "name": "概率", + "description": "Jev 的是/否传感器，或任何状态为 0 到 1 概率的传感器。" + }, + "truth_entity_id": { + "name": "真实情况", + "description": "状态显示真实答案的实体，例如门磁或报告设备何时运行的插座。" + }, + "truth_state": { + "name": "表示是的状态", + "description": "该实体表示“是”的状态。留空时为 on。" + }, + "days": { + "name": "历史天数", + "description": "向前读取多少天。除非你修改过，recorder 保留 10 天。" + } + } } }, "entity": { @@ -341,6 +363,18 @@ }, "rate_limited": { "message": "TypeSafe 限制了该请求的速率：{reason}" + }, + "calibrate_needs_recorder": { + "message": "此操作需要读取 recorder，但 recorder 未运行。" + }, + "calibrate_no_history": { + "message": "过去 {days} 天中，没有任何时刻同时有 {entity} 的数值和 {truth} 的状态。请检查 recorder 是否记录了两者，以及 {entity} 是否报告数值。" + }, + "calibrate_never_true": { + "message": "过去 {days} 天中 {truth} 从未处于 {state}，因此没有可比较的内容。请读取更长的时间段，或检查哪个状态表示“是”。" + }, + "calibrate_always_true": { + "message": "过去 {days} 天中 {truth} 一直处于 {state}，因此没有可比较的内容。请读取更长的时间段，或检查哪个状态表示“是”。" } }, "common": { diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py new file mode 100644 index 0000000..874e7ad --- /dev/null +++ b/tests/test_calibrate.py @@ -0,0 +1,175 @@ +"""jev.calibrate: a threshold measured against what was really true.""" + +from datetime import timedelta + +import pytest +from homeassistant.exceptions import ServiceValidationError +from homeassistant.util import dt as dt_util +from pytest_homeassistant_custom_component.components.recorder.common import ( + async_wait_recording_done, +) + +from custom_components.jev.calibrate import Span, best, build_spans, outcome +from custom_components.jev.const import DOMAIN + +PROBABILITY = "sensor.laundry_done" +TRUTH = "binary_sensor.laundry_door" + + +@pytest.fixture +def mock_recorder_before_hass(async_test_recorder): + """The recorder's database has to exist before hass starts.""" + + +@pytest.fixture(autouse=True) +def recorder(recorder_mock): + return recorder_mock + + +def at_hours(*states): + """States one hour apart, as the recorder would return them.""" + from homeassistant.core import State + + start = dt_util.parse_datetime("2026-09-22 08:00:00+00:00") + return [ + State(entity_id, value, last_changed=start + timedelta(hours=hour)) + for entity_id, value, hour in states + ], start + + +def test_every_change_of_either_entity_cuts_a_span(): + states, start = at_hours( + ("sensor.p", "0.2", 0), + ("binary_sensor.t", "off", 0), + ("sensor.p", "0.7", 1), + ("binary_sensor.t", "on", 1), + ) + probabilities = [s for s in states if s.entity_id == "sensor.p"] + truths = [s for s in states if s.entity_id == "binary_sensor.t"] + spans = build_spans(probabilities, truths, "on", start, start + timedelta(hours=3)) + assert [(s.probability, s.truth, s.seconds) for s in spans] == [ + (0.2, False, 3600.0), + (0.7, True, 7200.0), + ] + + +def test_a_state_from_before_the_window_counts_from_its_start(): + states, start = at_hours(("sensor.p", "0.9", -5), ("binary_sensor.t", "on", -2)) + spans = build_spans(states[:1], states[1:], "on", start, start + timedelta(hours=1)) + assert spans == [Span(0.9, True, 3600.0)] + + +def test_time_without_a_number_is_left_out(): + """Unavailable while the API was down is not a probability of anything.""" + states, start = at_hours( + ("sensor.p", "0.9", 0), + ("binary_sensor.t", "on", 0), + ("sensor.p", "unavailable", 1), + ("sensor.p", "0.8", 2), + ) + probabilities = [s for s in states if s.entity_id == "sensor.p"] + truths = [s for s in states if s.entity_id == "binary_sensor.t"] + spans = build_spans(probabilities, truths, "on", start, start + timedelta(hours=3)) + assert sum(s.seconds for s in spans) == 7200.0 + assert {s.probability for s in spans} == {0.9, 0.8} + + +def test_precision_and_recall_are_weighted_by_time(): + spans = [Span(0.8, True, 30), Span(0.8, False, 10), Span(0.2, True, 60)] + result = outcome(spans, 0.5) + assert result.precision == 0.75 + assert result.recall == 30 / 90 + + +def test_nothing_said_yes_has_no_precision_and_no_f1(): + result = outcome([Span(0.2, True, 60), Span(0.1, False, 60)], 0.5) + assert result.precision is None + assert result.f1 == 0.0 + + +def test_the_middle_of_the_tied_thresholds_is_returned(): + """Every threshold from 0.21 to 0.41 separates these perfectly. 0.31 is central.""" + spans = [ + Span(0.2, False, 3600), + Span(0.7, True, 3600), + Span(0.41, True, 3600), + Span(0.1, False, 3600), + ] + result = best(spans) + assert result.threshold == 0.31 + assert (result.precision, result.recall) == (1.0, 1.0) + + +async def record_a_day(hass, freezer): + """Four hours: the door opens with the noul at 0.7, and closes at 0.1.""" + start = dt_util.utcnow() - timedelta(hours=4) + for hour, probability, door in ( + (0, "0.2", "off"), + (1, "0.7", "on"), + (2, "0.41", "on"), + (3, "0.1", "off"), + ): + freezer.move_to(start + timedelta(hours=hour)) + hass.states.async_set(PROBABILITY, probability) + hass.states.async_set(TRUTH, door) + await hass.async_block_till_done() + freezer.move_to(start + timedelta(hours=4)) + await async_wait_recording_done(hass) + + +async def calibrate(hass, **data): + return await hass.services.async_call( + DOMAIN, + "calibrate", + {"entity_id": PROBABILITY, "truth_entity_id": TRUTH, **data}, + blocking=True, + return_response=True, + ) + + +async def test_calibrate_reads_the_recorder_and_returns_the_best_threshold( + hass, loaded_entry, mock_client, freezer +): + mock_client.ask.reset_mock() + await record_a_day(hass, freezer) + result = await calibrate(hass) + + assert result["threshold"] == 0.31 + assert (result["precision"], result["recall"], result["f1"]) == (1.0, 1.0, 1.0) + assert result["hours"] == 4.0 + assert result["hours_true"] == 2.0 + assert result["times_true"] == 1 + assert result["probability_changes"] == 4 + assert result["table"][0] == { + "threshold": 0.1, + "precision": 0.5, + "recall": 1.0, + "f1": 0.667, + } + # It reads history and nothing else, so it costs no tokens. + assert mock_client.ask.await_count == 0 + + +async def test_truth_that_never_happened_is_refused(hass, loaded_entry, freezer): + await record_a_day(hass, freezer) + with pytest.raises(ServiceValidationError) as err: + await calibrate(hass, truth_state="open") + assert err.value.translation_key == "calibrate_never_true" + + +async def test_truth_that_never_changed_is_refused(hass, loaded_entry, freezer): + hass.states.async_set("binary_sensor.always", "on") + await hass.async_block_till_done() + freezer.tick(timedelta(hours=4)) + await record_a_day(hass, freezer) + with pytest.raises(ServiceValidationError) as err: + await calibrate(hass, truth_entity_id="binary_sensor.always") + assert err.value.translation_key == "calibrate_always_true" + + +async def test_an_entity_with_no_history_is_refused(hass, loaded_entry): + await async_wait_recording_done(hass) + with pytest.raises(ServiceValidationError) as err: + await calibrate(hass) + assert err.value.translation_key == "calibrate_no_history" + assert err.value.translation_placeholders["entity"] == PROBABILITY diff --git a/tests/test_services.py b/tests/test_services.py index df47b4c..3ed47ba 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -437,3 +437,13 @@ async def test_an_action_with_two_entries_and_none_named_is_refused( {"state": "x", "instructions": "y", "config_entry": second.entry_id}, ) assert "noul" in named + + +async def test_without_the_recorder_it_says_so(hass, loaded_entry): + with pytest.raises(ServiceValidationError) as err: + await call( + hass, + "calibrate", + {"entity_id": "sensor.x", "truth_entity_id": "binary_sensor.y"}, + ) + assert err.value.translation_key == "calibrate_needs_recorder" From 5054338b2e2a08cd66c0ff9a273492217f3f674f Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 22:23:08 +0200 Subject: [PATCH 07/24] The docs cover calibration, the LLM tools, the ask-back, a month of cost and the Bayesian sensor --- mkdocs.yml | 10 ++++- site-docs/actions.md | 6 +++ site-docs/bayesian.md | 73 +++++++++++++++++++++++++++++++++++ site-docs/calibrate.md | 80 +++++++++++++++++++++++++++++++++++++++ site-docs/conversation.md | 23 ++++++++++- site-docs/cost.md | 31 +++++++++++++-- site-docs/limitations.md | 21 ++++++++++ site-docs/llm-tools.md | 47 +++++++++++++++++++++++ 8 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 site-docs/bayesian.md create mode 100644 site-docs/calibrate.md create mode 100644 site-docs/llm-tools.md diff --git a/mkdocs.yml b/mkdocs.yml index 80c1b7b..b40339a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,7 +62,7 @@ plugins: A Home Assistant custom integration, installed through HACS. Questions become sensors: a probability, one of a set of options, or a number on a scale. Four actions (jev.noul, jev.choice, jev.score, jev.ask) answer inside an - automation, an AI Task entity answers ai_task.generate_data, and a + automation, jev.calibrate fits a threshold from history, an AI Task entity answers ai_task.generate_data, and a conversation agent handles Assist. Every call is billed in input tokens against an optional daily budget. sections: @@ -79,10 +79,13 @@ plugins: - actions.md: The four actions, their fields and their response variables - ai-task.md: The ai_task.generate_data entity and the structures it accepts - examples.md: Worked automations + - calibrate.md: jev.calibrate, a threshold measured from recorder history Voice: - conversation.md: The Assist conversation agent and what it will not control + - llm-tools.md: The jev__noul and jev__choice tools for other LLM agents Reference: - - cost.md: Token billing, the usage sensors and the daily budget + - bayesian.md: When to use Jev and when the Bayesian binary sensor + - cost.md: Token billing, the usage sensors, a month worked out and the daily budget - measurements.md: Latency and answer quality measured against the live API - troubleshooting.md: Debug logging and the common failures - limitations.md: What Jev does not do @@ -119,9 +122,12 @@ nav: - Actions: actions.md - AI Task: ai-task.md - Examples: examples.md + - Tuning a threshold: calibrate.md - Voice: - Conversation agent: conversation.md + - Tools for other LLM agents: llm-tools.md - Reference: + - Jev and the Bayesian sensor: bayesian.md - What it costs: cost.md - Measurements: measurements.md - Troubleshooting: troubleshooting.md diff --git a/site-docs/actions.md b/site-docs/actions.md index 5e45f0d..b22e7e1 100644 --- a/site-docs/actions.md +++ b/site-docs/actions.md @@ -109,6 +109,12 @@ the five you might need and discarding four is cheaper than two round trips. An object arrives as an object, not as a stringified dict. The model reads the field names as labels, so name them for what they hold. +## Tune a threshold + +`jev.calibrate` is a fifth action that asks nothing. It compares the recorded history +of a noul sensor with an entity that shows what was really true, and returns the +threshold that fits best. It costs no tokens. See [tuning a threshold](calibrate.md). + ## Errors name the limit Every budget failure names the budget, the limit and your ask, because an agent diff --git a/site-docs/bayesian.md b/site-docs/bayesian.md new file mode 100644 index 0000000..22c6430 --- /dev/null +++ b/site-docs/bayesian.md @@ -0,0 +1,73 @@ +# Jev and the Bayesian sensor + +Home Assistant already has a sensor that turns several readings into one probability, +the [Bayesian binary sensor](https://www.home-assistant.io/integrations/bayesian/). +Both give you a number from 0 to 1 and a threshold that turns it into on or off. They +get the number in different ways, and that decides which one fits. + +## How each gets its number + +The Bayesian sensor multiplies probabilities you give it. You set a prior, and for +each observation you say how often it is seen when the answer is yes +(`prob_given_true`) and when it is no (`prob_given_false`). An observation is a state, +a numeric range or a template. The sensor combines them with Bayes' rule and treats +them as independent of each other. + +Jev reads the states and your question in words, and the model gives the probability. +You write no probabilities. You write what the question means, and optionally what +counts as yes and as no. + +## Side by side + +| | Bayesian sensor | Jev noul | +|---|---|---| +| You supply | a prior and two probabilities per observation | a question in words | +| Where it runs | in Home Assistant | a call to the TypeSafe API | +| Cost | nothing | input tokens, see [what it costs](cost.md) | +| Time to update | when a state changes | 250 to 580 ms warm, measured from the Netherlands | +| Same input, same output | yes | no. One sentence gave 0.25, 0.28 and 0.31 in three runs | +| Reads free text | only through a template you write | yes, a note or a transcript is ordinary input | +| Observations that depend on each other | counted as independent, so they count twice | read together | +| Why it said what it said | each observation's share is visible | a number with no reasoning | + +## Which to use + +Use the Bayesian sensor when you know, or can measure, how often each reading goes with +the answer, and the readings are few and mostly independent. "Somebody is home" from a +phone, a door and motion in the hall is the textbook case. It costs nothing, it works +without the internet, and you can see why it changed. + +Use Jev when the probabilities are the part you cannot write down. A washing machine +whose power draw rises and falls through a programme, a doorbell transcript, or three +readings that only mean something together are hard to express as +`prob_given_true`. Jev takes them as they are. + +If you already have a working Bayesian sensor, keep it. Jev does not make one wrong. + +## Using both + +A Jev noul sensor is a number, so a Bayesian sensor can use it as a `numeric_state` +observation, next to readings that it handles well on its own: + +```yaml +binary_sensor: + - platform: bayesian + name: Laundry waiting + prior: 0.2 + probability_threshold: 0.8 + observations: + - platform: numeric_state + entity_id: sensor.jev_laundry_done + above: 0.6 + prob_given_true: 0.9 + prob_given_false: 0.1 + - platform: state + entity_id: binary_sensor.laundry_door + to_state: "off" + prob_given_true: 0.95 + prob_given_false: 0.5 +``` + +The probabilities in that example are placeholders. Measure your own, for example with +[jev.calibrate](calibrate.md), which reports precision and recall for a Jev sensor +against an entity that shows what was really true. diff --git a/site-docs/calibrate.md b/site-docs/calibrate.md new file mode 100644 index 0000000..dbabd07 --- /dev/null +++ b/site-docs/calibrate.md @@ -0,0 +1,80 @@ +# Tuning a threshold + +A noul is a probability, and the threshold that turns it into yes or no is your +choice. 0.5 is where the model says it cannot tell. It is not where your washing +machine is done. `jev.calibrate` finds a threshold from what really happened in your +house. + +## What it needs + +- a probability sensor, such as a Jev noul sensor +- an entity that shows what was really true, such as a door contact, or a smart plug + that reports when a machine runs +- the recorder, which keeps the history of both + +It reads the recorder and nothing else. It does not call TypeSafe, so it costs no +tokens. + +```yaml +- action: jev.calibrate + response_variable: fit + data: + entity_id: sensor.jev_laundry_done + truth_entity_id: binary_sensor.laundry_door + truth_state: "off" + days: 7 +``` + +| Field | Default | Meaning | +|---|---|---| +| `entity_id` | required | The probability sensor | +| `truth_entity_id` | required | The entity that shows what was true | +| `truth_state` | `on` | The state of that entity that means yes | +| `days` | 7 | How far back to read. The recorder keeps 10 days unless you changed `purge_keep_days` | + +## What it returns + +```yaml +threshold: 0.31 +precision: 1.0 +recall: 1.0 +f1: 1.0 +hours: 4.0 +hours_true: 2.0 +times_true: 1 +probability_changes: 4 +table: + - {threshold: 0.1, precision: 0.5, recall: 1.0, f1: 0.667} + # ... one row for each tenth up to 0.9 +``` + +The action cuts the window at every change of either entity, and counts time, not +state changes: + +- **precision** is the part of the time the threshold said yes that was really true. + A low precision means false alarms. +- **recall** is the part of the true time where the threshold said yes. A low recall + means missed cases. +- **f1** balances the two. The action tries every hundredth from 0.01 to 0.99 and + returns the one with the highest f1. When several tie, it returns the middle one, so + a probability a little off its usual values still lands on the same side. + +Use `table` when you care more about one side. For a notification you would rather +miss than repeat, pick a row with higher precision. + +## How much to trust it + +Read `times_true` before you read `f1`. One wash is one occasion, and an f1 of 1.0 over +one occasion tells you almost nothing. A week with a machine that runs every other day +gives three or four. Collect more occasions before you move a threshold far. + +Time when the probability sensor has no number, for example while the API was down, +is left out. + +## When it refuses + +| Error | Why | +|---|---| +| The recorder is not running | The action has nothing to read | +| No history | No time in the window has both a number and a truth state | +| Never true, or always true | With only one side, every threshold gets the same score | diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 0279310..1ca6361 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -52,6 +52,7 @@ hallway" would reach a lock exposed there and unlock it. | An entity you did not expose to Assist | Never described to the model at all | | No room and no device named | Refused, unless you allow it. `turn_off` is exempt. Below the confidence floor, fallback | | Two kinds of device, no kind named, whole house | Asks which kind. With one kind exposed, it acts on that kind | +| Two devices fit the name about equally | Asks which one, see below | | A spent budget, a rejected key or no answer | Fallback. With no fallback agent it says which of the three it was | !!! info "It only sees what Assist sees" @@ -60,7 +61,23 @@ hallway" would reach a lock exposed there and unlock it. widen that. This is the voice path only. The [four actions](actions.md) send whatever an automation targets, exposed or not. -## What it costs +## When two devices fit the name + +"Turn on the lamp" with a desk lamp and a floor lamp exposed can split the model's +probability between the two, with neither above the confidence floor. When exactly two +exposed devices each have at least 0.2, and the two together reach the floor, the agent +asks **"Do you mean Desk lamp or Floor lamp?"** and keeps the conversation open. When +the two have the same name, it adds the room: "Lamp (Office) or Lamp (Bedroom)". + +Your reply, such as "the desk one", is one more request, and it counts against the +budget. It asks which of the two the reply picks. A confident pick carries out the +first command on that device. A reply that picks neither is handled as a new command. +The question expires after five minutes, the same time Home Assistant keeps a +conversation open. + +The 0.2 is not measured. It keeps a third device with a small share from turning the +question into a guess between two. + Every command counts against the same daily token budget as your sensors. A satellite that mishears a wake word all night trips that tripwire instead of running @@ -118,6 +135,10 @@ only sees what Jev could not route, which is the cheap arrangement. ## Diagnostics +The Assist debug view (**Settings**, **Voice assistants**, the pipeline's menu, +**Debug**) shows what Jev answered for each command: the model, every answer with its +probabilities, and the reason for the decision. + The last 20 decisions the agent made are in the integration's diagnostics, with the reason for every decision and the action distribution behind it. The sentence itself is redacted, because the file is meant to be pasted into an issue. diff --git a/site-docs/cost.md b/site-docs/cost.md index a8c7d0c..d192230 100644 --- a/site-docs/cost.md +++ b/site-docs/cost.md @@ -43,6 +43,29 @@ roughly 110 tokens per entity per call. thousands of tokens on every evaluation, which is why it is off by default. Access tokens, entity pictures and coordinates are never sent, even with it on. +## A month, worked out + +The formula is calls per month, times input tokens per call, times the price. The +examples use the published $0.042 per million input tokens and a 30 day month. + +| Setup | Calls a month | Tokens a call | A month | +|---|---|---|---| +| One context on 2 entities, every 5 minutes | 8,640 | about 405 | $0.15 | +| 20 spoken commands a day, 5 entities exposed | 600 | 1,371 at most | $0.035 | +| One context on 250 entities, every 5 minutes | 8,640 | about 16,500 | $5.99 | + +Where the token counts come from: + +- A request with one entity measured 339 input tokens, and each entity record adds + 65.8, so two entities are about 405. See [what an entity costs](measurements.md#what-an-entity-costs). +- A spoken command with five entities exposed measured 1,329 to 1,371 input tokens. +- 250 entities is the cap on a target, at 65.8 tokens each. + +Every 5 minutes is the default scan interval, and it is the most a context on a +schedule asks. A context that wakes on entity changes asks at most once every 30 +seconds, so a busy one can ask up to 10 times as often. Your own numbers are in +`sensor.jev_input_tokens_today` and `sensor.jev_estimated_cost_today`. + ## The budget is a tripwire Set **Daily input token budget** in the integration options. The check runs before @@ -89,10 +112,10 @@ that either of those cleared would not be a daily budget. - YAML contexts name no entry, so they belong to the first enabled Jev entry. A second entry does not ask them again and bill them twice. -The two one-off paths, a conversation command and the preview in the question editor, -still check what has already been spent rather than estimating the call ahead. Both -are started by a person and both have somewhere to fall back to, so the worst case is -one request over the line rather than a runaway. +A conversation command, an action and an AI Task estimate their call ahead, like a +context does. The preview in the question editor checks only what has already been +spent. A person starts it and it has nothing to fall back to, so the worst case is one +request over the line rather than a runaway. ### How the estimate is worked out diff --git a/site-docs/limitations.md b/site-docs/limitations.md index 2c1e0d4..d8e29f3 100644 --- a/site-docs/limitations.md +++ b/site-docs/limitations.md @@ -61,6 +61,27 @@ Locks are refused on purpose: Home Assistant reads turn_on on a lock as `lock.lo which is the opposite way round from the spoken command. Garage, gate and door covers are refused for the same reason. +## No local model + +Jev needs the TypeSafe API. There is no local mode. + +A llama.cpp fork adds a `POST /v1/decision` endpoint that answers a schema of enum, +boolean and number fields in one batched pass, with a probability for each answer. I +read its interface to see whether Jev could use it as a local backend. It cannot, as +the fork is now: + +- It returns only the probability of the value it chose, not the distribution over + all values. +- A noul maps exactly, because the probability of yes follows from the probability + of the chosen value. +- A choice needs the probability of every option, and a score's level is a weighted + average over all levels, so neither can be built from one probability. +- The request shape is different, so Jev would need a second client. + +If that endpoint returns the full distribution for each field, a local backend is +worth doing. Until then, only the yes/no questions could run locally, with the other +two types still going to TypeSafe. + ## It is not a core integration `quality_scale.yaml` tracks this against Home Assistant's quality scale at 47 done diff --git a/site-docs/llm-tools.md b/site-docs/llm-tools.md new file mode 100644 index 0000000..6abb88b --- /dev/null +++ b/site-docs/llm-tools.md @@ -0,0 +1,47 @@ +# Tools for other LLM agents + +An LLM conversation agent can read your house through the Assist API. What it cannot +give is a probability. Ask it whether the washing machine is done, and it says yes or +no. With this option on, the agent can hand that one question to Jev and get the +number back. + +## Turn it on + +**Settings**, **Devices & services**, **Jev**, **Configure**, then **Offer Jev as a +tool to other LLM agents**. It is off by default. + +It is off because every LLM agent that uses Assist gets the tool descriptions in every +prompt. That costs tokens at that agent's provider on every turn, whether the agent +calls a tool or not. + +## The two tools + +| Tool | The agent gives | The agent gets | +|---|---|---| +| `jev__noul` | `question`, optional `facts` | `probability_yes`, 0 to 1 | +| `jev__choice` | `question`, `options` (2 or more), optional `facts` | `choice`, `probabilities`, `confidence` | + +Each tool call is a call to [jev.noul or jev.choice](actions.md). It counts against +the same daily budget, and it shows in the same usage sensors. + +## What Jev sees + +Jev judges the entities exposed to that assistant, the same list the agent itself +reads, up to 150. `facts` carries anything else the question depends on, such as what +the user said. + +Jev sends `facts` as it is. The actions render a string `state` as a template, and the +tools do not, because a template could read an entity that you did not expose to +Assist. + +## More than one entry + +When two or more Jev entries have the option on, each tool has an `account` +parameter, and the agent must name the entry that pays. Each entry has its own key and +its own budget, so Jev does not choose one for the agent. + +## The Jev conversation agent + +The Jev conversation agent does not use these tools. It asks Jev directly, and it +falls back to another agent for what it cannot route. See +[conversation agent](conversation.md). From 5e6bf6fac29eaeeb3db199887fc7997df84d081e Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:05:51 +0200 Subject: [PATCH 08/24] The budget estimate counts the part of the bill every request pays A 138 byte action was estimated at 70 tokens and billed 278. The estimate was bytes over a ratio, with no fixed part, and the ratio came from the last call alone. After that small call, a 6 KB request was estimated at 14,327 tokens, billed 3,277 one call earlier, and refused on every try. The estimate is now 250 tokens plus the body over the ratio, and only a call whose body was billed at least 250 tokens updates the ratio. --- custom_components/jev/const.py | 37 ++++++++++++++------ custom_components/jev/coordinator.py | 17 +++++---- site-docs/cost.md | 18 +++++----- site-docs/measurements.md | 52 ++++++++++++++++++++++------ tests/test_init.py | 2 +- tests/test_payload.py | 28 +++++++++++++++ tests/test_services.py | 38 +++++++++++++++++--- 7 files changed, 152 insertions(+), 40 deletions(-) diff --git a/custom_components/jev/const.py b/custom_components/jev/const.py index 126ac69..e0adf7a 100644 --- a/custom_components/jev/const.py +++ b/custom_components/jev/const.py @@ -44,17 +44,34 @@ DEFAULT_SCAN_INTERVAL_SECONDS: Final = 300 TRIGGER_DEBOUNCE_SECONDS: Final = 5.0 -# What the pre-flight budget check divides payload bytes by before any call of its -# own has measured the real ratio. This one is derived rather than measured end to -# end: the five-entity conversation payload in site-docs/measurements.md is 3,142 -# bytes and measured 1,329 to 1,371 input tokens live, which is 2.29 to 2.36 bytes -# per token. The low end is the one that over-estimates the cost. +# What every request is billed before its body counts, whatever its size. Measured +# live on 2026-09-24 (site-docs/measurements.md): jev.noul with a 138 byte body was +# billed 278 input tokens and one with 6,136 bytes 3,277, and jev.ask with 1 and 8 +# questions, 137 and 613 bytes, was billed 279 and 377. A straight line through +# each pair crosses zero bytes at 209 and 251 tokens. 250 is near the higher one. # -# Every answered call replaces it with the ratio that endpoint actually reported, -# so an endpoint with another tokeniser calibrates this in one request. A hardcoded -# divisor would be a landmine the day someone points the entry at OpenRouter or at -# a gateway of their own. -COLD_START_BYTES_PER_TOKEN: Final = 2.29 +# Without it, the estimate was bytes over a ratio and nothing else. A 138 byte +# action was estimated at 70 tokens and billed 278. +REQUEST_OVERHEAD_TOKENS: Final = 250 + +# What the pre-flight budget check divides the body bytes by before any call of its +# own has measured the real ratio. The five-entity conversation payload in +# site-docs/measurements.md is 3,142 bytes and measured 1,329 to 1,371 input tokens +# live. Less the fixed part, that is 2.80 to 2.91 bytes per token. The low end is the +# one that over-estimates the cost. +# +# An answered call with a body worth measuring replaces it with the ratio that +# endpoint actually reported, so an endpoint with another tokeniser calibrates this +# in one request. A hardcoded divisor would be a landmine the day someone points the +# entry at OpenRouter or at a gateway of their own. +COLD_START_BYTES_PER_TOKEN: Final = 2.8 + +# A call is worth measuring when its body was billed at least as much as the fixed +# part. Below that, a few tokens of rounding in the fixed part swing the ratio. +# Measured: with the ratio taken from the 278 token action, the next 6,136 byte +# request was estimated at 14,327 tokens and billed 3,277, and it was refused again +# on every try, because a refused call measures nothing. +MIN_MEASURED_BODY_TOKENS: Final = REQUEST_OVERHEAD_TOKENS # The estimate is a tripwire, not an accounting figure. Sixteen live commands on one # payload shape varied by 3% (site-docs/measurements.md), so 20% sits well past any diff --git a/custom_components/jev/coordinator.py b/custom_components/jev/coordinator.py index 3fb2d3d..08dc498 100644 --- a/custom_components/jev/coordinator.py +++ b/custom_components/jev/coordinator.py @@ -44,7 +44,9 @@ DOMAIN, ISSUE_BUDGET_EXCEEDED, ISSUE_BUDGET_SPENT, + MIN_MEASURED_BODY_TOKENS, MIN_UPDATE_INTERVAL_SECONDS, + REQUEST_OVERHEAD_TOKENS, STORE_SAVE_DELAY_SECONDS, TRIGGER_DEBOUNCE_SECONDS, ) @@ -73,9 +75,10 @@ class UsageAccount: budget: int = 0 price_per_million: float = USD_PER_MILLION_INPUT_TOKENS budget_exceeded: bool = False - # Measured from the last answered call rather than assumed, and deliberately - # not persisted: it describes the endpoint, not the day, and the first call - # after a restart measures it again. + # The bytes of a request body per billed token, not counting the fixed part. + # Measured from the last answered call with a body worth measuring rather than + # assumed, and deliberately not persisted: it describes the endpoint, not the + # day, and the first large call after a restart measures it again. bytes_per_token: float = COLD_START_BYTES_PER_TOKEN # Estimates of requests that are in flight. Two contexts refreshing together # each saw the same total before either answer came back, so both fitted and @@ -170,8 +173,9 @@ def roll_over(self, today: date) -> None: def record(self, input_tokens: int, payload_bytes: int | None = None) -> None: self.calls += 1 self.input_tokens += input_tokens - if payload_bytes and input_tokens > 0: - self.bytes_per_token = payload_bytes / input_tokens + body_tokens = input_tokens - REQUEST_OVERHEAD_TOKENS + if payload_bytes and body_tokens >= MIN_MEASURED_BODY_TOKENS: + self.bytes_per_token = payload_bytes / body_tokens self._save() async def async_flush(self) -> None: @@ -199,7 +203,8 @@ def would_exceed(self) -> bool: def estimate_tokens(self, request_bytes: int) -> int: """What a request of this size will be billed, over-estimated on purpose.""" - return ceil(request_bytes / self.bytes_per_token * BUDGET_ESTIMATE_MARGIN) + body_tokens = request_bytes / self.bytes_per_token + return ceil((REQUEST_OVERHEAD_TOKENS + body_tokens) * BUDGET_ESTIMATE_MARGIN) def would_exceed_with(self, estimate: int) -> bool: """Whether a request costing `estimate` would end the day over budget. diff --git a/site-docs/cost.md b/site-docs/cost.md index a8c7d0c..a1babbf 100644 --- a/site-docs/cost.md +++ b/site-docs/cost.md @@ -51,8 +51,9 @@ into a token estimate, and a call that would not fit in what is left of the budg never sent. A budget is a limit on what gets spent, and one that only notices after the spending is a report. -The estimate divides the request size by the bytes per token of the last answered -call. A question, an action and an AI Task all update that ratio. +The estimate is a fixed 250 tokens that every request pays, plus the body size +divided by a bytes-per-token ratio. A question, an action and an AI Task all update +that ratio. When it trips: @@ -96,12 +97,13 @@ one request over the line rather than a runaway. ### How the estimate is worked out -The estimate is `bytes / bytes-per-token`, with a 1.2 margin. The ratio is not -hardcoded: the first call after a restart uses 2.29 bytes per token, measured against -the live API, and every answered call after that replaces it with the payload size -divided by the input tokens the endpoint actually reported. An endpoint that counts -tokens differently, OpenRouter or a gateway of your own, is measured rather than -assumed within one call. See the +The estimate is `(250 + bytes / bytes-per-token) * 1.2`. The 250 is what every +request is billed before its body counts. The ratio is not hardcoded: the first call +after a restart uses 2.80 bytes per token, measured against the live API. After that, +every answered call with a body of at least 250 tokens replaces it with what the +endpoint actually billed. A small call leaves it alone, because its bill is nearly all +fixed part. An endpoint that counts tokens differently, OpenRouter or a gateway of +your own, is measured rather than assumed. See the [receipt](measurements.md#bytes-per-input-token). !!! tip "Size it past anything real" diff --git a/site-docs/measurements.md b/site-docs/measurements.md index 09a053d..539e936 100644 --- a/site-docs/measurements.md +++ b/site-docs/measurements.md @@ -154,27 +154,59 @@ the entities. The daily budget has to refuse a call before it is sent, and the only token count there is comes back with the reply. So the size of the request is measured locally and -divided by a bytes-per-token ratio. +turned into tokens: a fixed part that every request pays, plus the body bytes divided +by a bytes-per-token ratio. + +### The fixed part + +Four requests against the live API on 2026-09-24, body bytes measured locally for the +same request: + +| Request | Body bytes | Input tokens billed | +|---|---|---| +| `jev.noul`, one short state line | 138 | 278 | +| `jev.noul`, 6 KB of state | 6,136 | 3,277 | +| `jev.ask`, 1 question | 137 | 279 | +| `jev.ask`, 8 questions | 613 | 377 | + +A straight line through each pair crosses zero bytes at 209 tokens (the two `noul` +rows) and at 251 tokens (the two `ask` rows). The fixed part is 250. It is paid per +request, not per question: seven more questions added 98 tokens, not seven times 250. + +Before this, the estimate was the bytes over a ratio and nothing else. The 138 byte +action was estimated at 70 tokens and billed 278. + +### The ratio The cold-start ratio comes from the two readings above. The conversation payload with 5 exposed entities and 7 questions is 3,142 bytes, and the same shape against the live -API reported 1,329 to 1,371 input tokens per command: +API reported 1,329 to 1,371 input tokens per command. Less the fixed part: | | | |---|---| -| 3,142 / 1,371 | 2.29 bytes per token | -| 3,142 / 1,329 | 2.36 bytes per token | +| 3,142 / (1,371 - 250) | 2.80 bytes per token | +| 3,142 / (1,329 - 250) | 2.91 bytes per token | -2.29 is the seed, because the lower ratio is the larger token estimate and an estimate +2.80 is the seed, because the lower ratio is the larger token estimate and an estimate that refuses slightly early beats one that lets a call through. This is a derivation across two runs rather than one payload counted both ways: the byte figures were measured locally with no API call, and the token figures came from a -different set of sixteen live commands on the same fixtures. That is exactly why the -ratio is a seed and not a constant. Every answered call replaces it with its own -payload bytes divided by the input tokens the endpoint reported, so a different -tokenizer, a gateway, or OpenRouter is measured rather than assumed, and the assumption -lasts one call. +different set of sixteen live commands on the same fixtures. That is why the ratio is +a seed and not a constant. An answered call replaces it with its own body bytes +divided by the tokens it was billed past the fixed part, so a different tokenizer, a +gateway, or OpenRouter is measured rather than assumed. + +Only a call whose body was billed at least 250 tokens replaces it. The 278 token +action above has 28 tokens of body, and a ratio taken from all of it said 0.5 bytes +per token. With that ratio, the 6 KB request was estimated at 14,327 tokens after being billed +3,277 one call earlier, and it was refused on every try, because a refused call +measures nothing. + +The ratio depends on what the bytes are. The 6 KB state was a repeated two-byte word +and came to 2.0 bytes per token. The questions of the `ask` rows came to 4.9. The first +6 KB request after a restart was estimated below what it was billed. The next one was +not. The estimate carries a 1.2 margin. Input tokens across those sixteen commands varied by 3.2%, 1,329 to 1,371 for the same seven questions, so 20% sits well past anything diff --git a/tests/test_init.py b/tests/test_init.py index d426d6a..be5a39c 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -147,7 +147,7 @@ async def test_the_budget_stops_the_next_call_and_keeps_what_it_has(hass, mock_c mock_client.ask.return_value = build_response( laundry_laundry_forgotten=NoulAnswer(noul=0.81) ) - await setup_with_context(hass, _budget_entry(400)) + await setup_with_context(hass, _budget_entry(700)) assert hass.states.get("sensor.jev_input_tokens_today").state == PROBED_AND_ASKED assert hass.states.get("sensor.jev_laundry_forgotten").state == "0.81" diff --git a/tests/test_payload.py b/tests/test_payload.py index 442828e..1e434cf 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -7,12 +7,15 @@ """ import json +from datetime import date from typing import Any, ClassVar import pytest from aiohttp.payload import JsonPayload from jevclient import Choice, JevClient, Noul, Score +from custom_components.jev.const import BUDGET_ESTIMATE_MARGIN +from custom_components.jev.coordinator import UsageAccount from custom_components.jev.payload import payload_bytes REPLY = { @@ -92,3 +95,28 @@ async def test_the_model_is_part_of_what_is_measured(): short = payload_bytes("x", QUESTIONS, "a") long = payload_bytes("x", QUESTIONS, "a" * 40) assert long - short == 39 + + +# Measured live on 2026-09-24: body bytes, then the input tokens the API billed. +_BILLED = [ + (138, 278), # jev.noul, one short state line + (613, 377), # jev.ask, eight questions + (3142, 1371), # the five-entity conversation payload, the dearest of sixteen +] + + +@pytest.mark.parametrize(("request_bytes", "billed"), _BILLED) +def test_the_cold_start_estimate_is_above_what_was_billed(request_bytes, billed): + """Before any call has measured the ratio, the estimate must not be low. + + Without the fixed part, the 138 byte action was estimated at 70 tokens. + """ + usage = UsageAccount(day=date(2026, 9, 24)) + assert billed <= usage.estimate_tokens(request_bytes) <= billed * 1.5 + + +def test_one_measured_call_sets_the_estimate_for_the_next(): + """6,136 bytes of dense state was billed 3,277 tokens. The same again fits.""" + usage = UsageAccount(day=date(2026, 9, 24)) + usage.record(3277, 6136) + assert 3277 <= usage.estimate_tokens(6136) <= 3277 * BUDGET_ESTIMATE_MARGIN + 1 diff --git a/tests/test_services.py b/tests/test_services.py index df47b4c..8d463fa 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1,14 +1,16 @@ """The four actions, including what they refuse.""" +from dataclasses import replace + import pytest import voluptuous as vol from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.const import CONF_API_KEY from homeassistant.exceptions import HomeAssistantError, ServiceValidationError -from jevclient import ChoiceAnswer, NoulAnswer, ScoreAnswer +from jevclient import ChoiceAnswer, NoulAnswer, ScoreAnswer, Usage from pytest_homeassistant_custom_component.common import MockConfigEntry -from custom_components.jev.const import DOMAIN +from custom_components.jev.const import DOMAIN, REQUEST_OVERHEAD_TOKENS from custom_components.jev.payload import payload_bytes from .conftest import build_response @@ -43,8 +45,16 @@ async def test_noul_returns_the_probability_and_the_threshold( async def test_an_action_teaches_the_budget_how_big_a_token_is( hass, loaded_entry, mock_client ): - """The budget estimate divides request bytes by this ratio, so every call counts.""" - mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.81)) + """The budget estimate divides body bytes by this ratio, so every call counts. + + The fixed part of the bill is taken off first, because it is paid whatever the + body holds. + """ + billed = REQUEST_OVERHEAD_TOKENS + 900 + mock_client.ask.return_value = replace( + build_response(answer=NoulAnswer(noul=0.81)), + usage=Usage(input_tokens=billed, output_tokens=20), + ) await call( hass, "noul", @@ -53,7 +63,25 @@ async def test_an_action_teaches_the_budget_how_big_a_token_is( state, questions = mock_client.ask.call_args.args runtime = loaded_entry.runtime_data sent = payload_bytes(state, questions, runtime.model) - assert runtime.usage.bytes_per_token == sent / 321 + assert runtime.usage.bytes_per_token == sent / 900 + + +async def test_a_small_call_leaves_the_ratio_alone(hass, loaded_entry, mock_client): + """278 tokens for 138 bytes is nearly all fixed part, and says little about bytes. + + Measured live: with the ratio taken from that call, the next 6,136 byte request + was estimated at 14,327 tokens and billed 3,277. It was refused on every try, + because a refused call measures nothing. + """ + runtime = loaded_entry.runtime_data + before = runtime.usage.bytes_per_token + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.81)) + await call( + hass, + "noul", + {"state": "The machine has drawn 1.2 W.", "instructions": "Is it done?"}, + ) + assert runtime.usage.bytes_per_token == before async def test_the_threshold_is_the_callers_and_nothing_else( From ac3804e4152525067953e1719b008ec99a3959f5 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:08:46 +0200 Subject: [PATCH 09/24] A voice command that acts says what it did Jev ran the action through the intent layer, which says nothing, and the Assist dialog showed no reply at all. Home Assistant's own agent picks a sentence from home-assistant-intents by what the command targeted. Jev now reads the same key off the slots it sent and renders that sentence, and says "Done." where the package has none, as for a toggle. --- custom_components/jev/conversation.py | 134 +++++++++++++++++- custom_components/jev/strings.json | 1 + custom_components/jev/translations/cs.json | 1 + custom_components/jev/translations/da.json | 1 + custom_components/jev/translations/de.json | 1 + custom_components/jev/translations/en.json | 1 + custom_components/jev/translations/es.json | 1 + custom_components/jev/translations/fr.json | 1 + custom_components/jev/translations/it.json | 1 + custom_components/jev/translations/nl.json | 1 + custom_components/jev/translations/pl.json | 1 + custom_components/jev/translations/pt-BR.json | 1 + custom_components/jev/translations/ru.json | 1 + custom_components/jev/translations/sv.json | 1 + .../jev/translations/zh-Hans.json | 1 + site-docs/conversation.md | 5 + tests/test_conversation.py | 78 ++++++++++ 17 files changed, 227 insertions(+), 4 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 359094b..bfbf49a 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -26,7 +26,7 @@ import re from collections.abc import Mapping from dataclasses import asdict, dataclass -from typing import Literal +from typing import Any, Literal from homeassistant.components import conversation from homeassistant.components.conversation.models import AbstractConversationAgent @@ -52,7 +52,7 @@ ) from .coordinator import JevRuntimeData from .entity import build_device_info -from .interpret import build_questions, interpret +from .interpret import ACTIONS, build_questions, interpret from .snapshot import async_snapshot _LOGGER = logging.getLogger(__name__) @@ -71,6 +71,7 @@ "intent_failed": "Sorry, that did not work.", "already_on": "{name} is already on.", "already_off": "{name} is already off.", + "done": "Done.", "query_not_found": "I could not find that.", "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", @@ -241,12 +242,23 @@ async def _async_handle_message( _LOGGER.error("intent %s failed: %s", decision.intent_type, err) return await self._speak(user_input, "intent_failed") - # Only a state question needs our lines, and loading them reads translations. + # Loading our lines reads translations, so only a reply without a sentence of + # its own loads them. + language = user_input.language or self.hass.config.language if intent_response.response_type is ha_intent.IntentResponseType.QUERY_ANSWER: - language = user_input.language or self.hass.config.language await _speak_the_answer( self.hass, intent_response, language, await self._lines(language) ) + elif ( + intent_response.response_type is ha_intent.IntentResponseType.ACTION_DONE + and not intent_response.speech + ): + spoken = await _render_action_answer( + self.hass, intent_response, decision.intent_type, decision.slots, language + ) + intent_response.async_set_speech( + spoken or (await self._lines(language))["done"] + ) return conversation.ConversationResult( response=intent_response, conversation_id=user_input.conversation_id ) @@ -347,6 +359,9 @@ class _Shipped: state_answer: str | None writes_the_state_word: bool errors: Mapping[str, str] + # The sentences for each action intent, keyed by the response name the default + # agent's own sentence data picks. + action_answers: Mapping[str, Mapping[str, str]] # Read once per language, keyed by the language that was asked for rather than the @@ -390,6 +405,16 @@ def _load_shipped(language: str) -> _Shipped | None: for key, text in responses.get("errors", {}).items() if isinstance(text, str) and text.strip() }, + action_answers={ + intent_type: { + key: text + for key, text in ( + responses.get("intents", {}).get(intent_type) or {} + ).items() + if isinstance(text, str) and text.strip() + } + for intent_type in ACTIONS.values() + }, ) @@ -517,6 +542,107 @@ def spoken(state: State) -> _SpokenState: return ", ".join(parts) if parts else None +# One kind of device across a room or the whole house. The names differ between +# languages: English writes light_all and Polish lights_all. +_AREA_RESPONSES = {"light": ("lights_area",), "fan": ("fans_area",)} +_ALL_RESPONSES = {"light": ("light_all", "lights_all"), "fan": ("fan_all",)} + +_SLOT_REFERENCE = re.compile(r"slots\.(\w+)") + + +def _response_keys( + intent_type: str, slots: Mapping[str, Any], domain: str | None +) -> tuple[str, ...]: + """The response names to try, closest first, for what this command did. + + The default agent reads the name from the sentence it matched. This agent has + no sentence, so it reads the same thing off the slots it sent. + """ + if intent_type == "HassLightSet": + return ("brightness",) + kinds = slots.get("domain", {}).get("value") or [] + kind = kinds[0] if len(kinds) == 1 else None + closest: tuple[str, ...] + if "area" in slots: + closest = _AREA_RESPONSES.get(kind or "", ()) + elif slots.get("name", {}).get("value") == "all": + closest = _ALL_RESPONSES.get(kind or "", ()) + elif domain is not None: + # A scene is activated rather than turned on, and German says "Licht + # eingeschaltet" for one light, where English has no sentence of its own. + closest = (domain,) + else: + closest = () + return (*closest, "default") + + +async def _render_action_answer( + hass: HomeAssistant, + response: ha_intent.IntentResponse, + intent_type: str | None, + slots: Mapping[str, Any], + language: str, +) -> str | None: + """The sentence the default agent says after the same action, or None. + + `async_handle` does the action and says nothing, and the Assist dialog shows no + reply at all for an empty one. The default agent's words come from the same + package as the state answers, written by the people who translate Assist. + None when the package has nothing that fits, and the agent says "Done." instead. + """ + shipped = await _shipped(hass, language) + if shipped is None or intent_type is None: + return None + answers = shipped.action_answers.get(intent_type, {}) + states = [*response.matched_states, *response.unmatched_states] + first = states[0] if states else None + # The name slot is the device's own name, as the model picked it. "all" is not a + # name to say back. + speech_slots = { + key: value["value"] + for key, value in slots.items() + if key in ("name", "area") + and isinstance(value.get("value"), str) + and value["value"] != "all" + } | response.speech_slots + for key in _response_keys( + intent_type, slots, first.domain if first is not None else None + ): + text = answers.get(key) + if text is None: + continue + # German says "{{ slots.name }} eingeschaltet". Rendered for a room, with no + # name to put there, that is a sentence that starts with a blank. + if any(name not in speech_slots for name in _SLOT_REFERENCE.findall(text)): + continue + try: + rendered = template.Template(text, hass).async_render( + { + "slots": speech_slots, + "state": template.TemplateState(hass, first) if first else None, + "query": { + "matched": [ + template.TemplateState(hass, state) + for state in response.matched_states + ], + "unmatched": [ + template.TemplateState(hass, state) + for state in response.unmatched_states + ], + }, + }, + parse_result=False, + ) + except TemplateError as err: + _LOGGER.debug( + "the %s answer for %s did not render: %s", key, intent_type, err + ) + continue + if sentence := " ".join(str(rendered).split()): + return sentence + return None + + async def _speak_the_answer( hass: HomeAssistant, response: ha_intent.IntentResponse, diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..add11f9 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -350,6 +350,7 @@ "intent_failed": "Sorry, that did not work.", "already_on": "{name} is already on.", "already_off": "{name} is already off.", + "done": "Done.", "preview_alone": "Sent as its own request.", "preview_grouped": "Sent in one request together with: {others}.", "preview_cost": "Asked once for this preview: {tokens} input tokens, about ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..70c7489 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -350,6 +350,7 @@ "intent_failed": "Promiňte, to se nepovedlo.", "already_on": "{name} už je zapnuto.", "already_off": "{name} už je vypnuto.", + "done": "Hotovo.", "preview_alone": "Odesláno jako vlastní požadavek.", "preview_grouped": "Odesláno v jednom požadavku spolu s: {others}.", "preview_cost": "Jednou dotázáno pro tento náhled: {tokens} vstupních tokenů, asi ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..0ed2896 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -350,6 +350,7 @@ "intent_failed": "Beklager, det virkede ikke.", "already_on": "{name} er allerede tændt.", "already_off": "{name} er allerede slukket.", + "done": "Færdig.", "preview_alone": "Sendes som sin egen anmodning.", "preview_grouped": "Sendes i én anmodning sammen med: {others}.", "preview_cost": "Spurgt én gang til denne forhåndsvisning: {tokens} input-tokens, cirka ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..55c44c6 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -350,6 +350,7 @@ "intent_failed": "Entschuldigung, das hat nicht geklappt.", "already_on": "{name} ist schon an.", "already_off": "{name} ist schon aus.", + "done": "Erledigt.", "preview_alone": "Geht als eigene Anfrage raus.", "preview_grouped": "Geht in einer Anfrage zusammen mit: {others}.", "preview_cost": "Einmal für diese Vorschau gefragt: {tokens} Eingabe-Tokens, etwa ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..add11f9 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -350,6 +350,7 @@ "intent_failed": "Sorry, that did not work.", "already_on": "{name} is already on.", "already_off": "{name} is already off.", + "done": "Done.", "preview_alone": "Sent as its own request.", "preview_grouped": "Sent in one request together with: {others}.", "preview_cost": "Asked once for this preview: {tokens} input tokens, about ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..81c5f1d 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -350,6 +350,7 @@ "intent_failed": "Lo siento, eso no ha funcionado.", "already_on": "{name} ya está encendido.", "already_off": "{name} ya está apagado.", + "done": "Hecho.", "preview_alone": "Se envía en una solicitud propia.", "preview_grouped": "Se envía en una sola solicitud junto con: {others}.", "preview_cost": "Preguntado una vez para esta vista previa: {tokens} tokens de entrada, unos ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..82ffa86 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -350,6 +350,7 @@ "intent_failed": "Désolé, cela n'a pas fonctionné.", "already_on": "{name} est déjà allumé.", "already_off": "{name} est déjà éteint.", + "done": "C'est fait.", "preview_alone": "Envoyée dans sa propre requête.", "preview_grouped": "Envoyée dans une seule requête avec : {others}.", "preview_cost": "Posée une fois pour cet aperçu : {tokens} jetons d'entrée, environ ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..3bec9ee 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -350,6 +350,7 @@ "intent_failed": "Scusa, non ha funzionato.", "already_on": "{name} è già acceso.", "already_off": "{name} è già spento.", + "done": "Fatto.", "preview_alone": "Inviata come richiesta a sé stante.", "preview_grouped": "Inviata in un'unica richiesta insieme a: {others}.", "preview_cost": "Chiesto una volta per questa anteprima: {tokens} token di input, circa ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..7fe8e58 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -350,6 +350,7 @@ "intent_failed": "Sorry, dat is niet gelukt.", "already_on": "{name} staat al aan.", "already_off": "{name} staat al uit.", + "done": "Gedaan.", "preview_alone": "Gaat als eigen verzoek.", "preview_grouped": "Gaat in een verzoek samen met: {others}.", "preview_cost": "Een keer gevraagd voor deze controle: {tokens} invoertokens, ongeveer ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..0a8868d 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -350,6 +350,7 @@ "intent_failed": "Przepraszam, to się nie udało.", "already_on": "{name} jest już włączone.", "already_off": "{name} jest już wyłączone.", + "done": "Gotowe.", "preview_alone": "Wysyłane jako osobne żądanie.", "preview_grouped": "Wysyłane w jednym żądaniu razem z: {others}.", "preview_cost": "Zapytano raz na potrzeby tego podglądu: {tokens} tokenów wejściowych, około ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..6558e22 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -350,6 +350,7 @@ "intent_failed": "Desculpe, isso não funcionou.", "already_on": "{name} já está ligado.", "already_off": "{name} já está desligado.", + "done": "Pronto.", "preview_alone": "Enviada como requisição própria.", "preview_grouped": "Enviada em uma requisição junto com: {others}.", "preview_cost": "Perguntado uma vez para esta prévia: {tokens} tokens de entrada, cerca de ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..300d6bc 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -350,6 +350,7 @@ "intent_failed": "Извините, не получилось.", "already_on": "{name} уже включён.", "already_off": "{name} уже выключен.", + "done": "Готово.", "preview_alone": "Отправляется отдельным запросом.", "preview_grouped": "Отправляется одним запросом вместе с: {others}.", "preview_cost": "Один вопрос для этого предпросмотра: {tokens} входных токенов, примерно ${cost}, {ms} мс.", diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..83d0b03 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -350,6 +350,7 @@ "intent_failed": "Tyvärr, det gick inte.", "already_on": "{name} är redan på.", "already_off": "{name} är redan av.", + "done": "Klart.", "preview_alone": "Skickas som en egen förfrågan.", "preview_grouped": "Skickas i en förfrågan tillsammans med: {others}.", "preview_cost": "Frågade en gång för den här förhandsgranskningen: {tokens} indatatoken, ungefär ${cost}, {ms} ms.", diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..bc7c05f 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -350,6 +350,7 @@ "intent_failed": "抱歉，没有成功。", "already_on": "{name} 已经开着了。", "already_off": "{name} 已经关着了。", + "done": "好的，完成了。", "preview_alone": "作为单独的请求发送。", "preview_grouped": "与以下问题合并在一个请求中发送：{others}。", "preview_cost": "为本次预览提问一次：{tokens} 个输入令牌，约 ${cost}，{ms} 毫秒。", diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 0279310..5f62a44 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -41,6 +41,11 @@ A room command always carries the kinds of device the model was shown. Home Assistant otherwise acts on every exposed entity in the room, so "turn off the hallway" would reach a lock exposed there and unlock it. +After an action it says the sentence Home Assistant's own agent says for the same +command, in the pipeline's language, such as "Turned on the light". Those sentences +come from Home Assistant's translations. Where they have none, as for a toggle, it +says "Done." + ## What it refuses | Case | What happens | diff --git a/tests/test_conversation.py b/tests/test_conversation.py index a684f01..5c660fa 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -1256,3 +1256,81 @@ async def test_a_room_past_the_entity_cap_is_not_offered(hass, config_entry): assert [e.entity_id for e in snapshot.entities] == ["light.a", "light.b"] assert snapshot.areas == ["Attic"] + + +_ENTITY = {} +_AREA = { + "target_type": ChoiceAnswer(choice="area", probabilities={}, confidence=0.94), + "entity": ChoiceAnswer(choice="none_of_these", probabilities={}, confidence=0.9), + "area": ChoiceAnswer(choice="Office", probabilities={}, confidence=0.93), +} +_ALL = { + "target_type": ChoiceAnswer(choice="everything", probabilities={}, confidence=0.95), + "entity": ChoiceAnswer(choice="none_of_these", probabilities={}, confidence=0.9), +} + + +@pytest.mark.parametrize( + ("language", "service", "text", "answers"), + [ + ("en", "turn_on", "turn on the kitchen light", _ENTITY), + ("en", "turn_off", "turn off the lights in the office", _AREA), + ("en", "turn_off", "turn off all the lights", _ALL), + ("nl", "turn_on", "zet de kitchen light aan", _ENTITY), + ("nl", "turn_off", "zet de lampen in de office uit", _AREA), + ("de", "turn_on", "schalte kitchen light ein", _ENTITY), + ("pl", "turn_off", "wyłącz światła w office", _AREA), + ], +) +async def test_an_action_says_what_the_default_agent_says( + hass, house, mock_client, language, service, text, answers +): + """An action used to reply with no sentence, and the Assist dialog showed nothing. + + The reference is the default agent itself, on a sentence it matches, for the + same command. + """ + action = ChoiceAnswer(choice=service, probabilities={}, confidence=0.98) + mock_client.ask.return_value = build_response(**answer_set(action=action, **answers)) + hass.services.async_register("light", service, lambda call: None) + + ours = await converse(hass, text, language=language) + theirs = await converse( + hass, text, agent_id="conversation.home_assistant", language=language + ) + + spoken = ours.response.speech["plain"]["speech"] + assert spoken + assert spoken == theirs.response.speech["plain"]["speech"] + + +@pytest.mark.parametrize(("language", "expected"), [("en", "Done."), ("nl", "Gedaan.")]) +async def test_an_action_with_no_sentence_of_its_own_says_done( + hass, house, mock_client, language, expected +): + """home-assistant-intents writes nothing for a toggle.""" + mock_client.ask.return_value = build_response( + **answer_set( + action=ChoiceAnswer(choice="toggle", probabilities={}, confidence=0.92) + ) + ) + hass.services.async_register("light", "toggle", lambda call: None) + + result = await converse(hass, "flip the kitchen light", language=language) + + assert result.response.speech["plain"]["speech"] == expected + + +async def test_a_brightness_command_says_it_was_set(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set( + action=ChoiceAnswer( + choice="set_brightness", probabilities={}, confidence=0.95 + ) + ) + ) + hass.services.async_register("light", "turn_on", lambda call: None) + + result = await converse(hass, "set the kitchen light to 40%") + + assert result.response.speech["plain"]["speech"] == "Brightness set" From 214e5b62bb177794e445ccf2e99d5be9a9381d2e Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:12:05 +0200 Subject: [PATCH 10/24] "Turn off all the lights" counts as every device of one kind The everything option read "The whole house, with no room or device named". For "turn off all the lights" the model put 0.44 to 0.52 on it, below the 0.6 floor, and the command fell back. With the option naming every device of one kind, "all the lights", "every light" and "the lights" acted 6 of 6 times live. Commands naming an exposed device or a room still named it, 8 of 8, and "turn on all the lights" was still refused by the whole-house gate. --- custom_components/jev/interpret.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 4e83f75..cdb27d5 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -189,7 +189,8 @@ def build_questions( { "entity": "One particular device is named", "area": "A room or area is named, covering what is in it", - "everything": "The whole house, with no room or device named", + "everything": "Every device, or every device of one kind such as " + "all the lights, with no room or device named", NONE: "No target is named at all", }, ), From 5214f97eba23662a9a4940805715d38ad3c49a9e Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:19:02 +0200 Subject: [PATCH 11/24] The voice refusal for the budget is true when tokens remain, and is an error The check refuses a command whose estimate does not fit, which happens with tokens left. The line said the budget "is spent". It now says not enough is left, in all 13 languages. With no fallback agent, the reply was an action_done response. The Assist dialog and a satellite read that as a command that went through. It is now an error, no_intent_match for a sentence not understood and failed_to_handle for the budget, a rejected key and no answer, as the default agent answers. --- custom_components/jev/conversation.py | 21 +++++++++++++++---- custom_components/jev/strings.json | 2 +- custom_components/jev/translations/cs.json | 2 +- custom_components/jev/translations/da.json | 2 +- custom_components/jev/translations/de.json | 2 +- custom_components/jev/translations/en.json | 2 +- custom_components/jev/translations/es.json | 2 +- custom_components/jev/translations/fr.json | 2 +- custom_components/jev/translations/it.json | 2 +- custom_components/jev/translations/nl.json | 2 +- custom_components/jev/translations/pl.json | 2 +- custom_components/jev/translations/pt-BR.json | 2 +- custom_components/jev/translations/ru.json | 2 +- custom_components/jev/translations/sv.json | 2 +- .../jev/translations/zh-Hans.json | 2 +- tests/test_conversation.py | 8 +++++-- 16 files changed, 37 insertions(+), 20 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 07cc3a2..e37818d 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -73,7 +73,9 @@ "already_on": "{name} is already on.", "already_off": "{name} is already off.", "query_not_found": "I could not find that.", - "budget_spent": "The daily token budget is spent, so I cannot do that today.", + "budget_spent": ( + "Not enough of the daily token budget is left for that, so I cannot do it today." + ), "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", } @@ -276,7 +278,14 @@ async def _fall_back( agent = self._fallback_agent _LOGGER.debug("falling back to %s because %s", agent or "nobody", why) if agent is None: - return await self._speak(user_input, line) + # An error, as the default agent answers one. A satellite and the Assist + # dialog treat an action_done reply as a command that went through. + code = ( + ha_intent.IntentResponseErrorCode.NO_INTENT_MATCH + if line == "not_understood" + else ha_intent.IntentResponseErrorCode.FAILED_TO_HANDLE + ) + return await self._speak(user_input, line, error=code) result = await conversation.async_converse( self.hass, user_input.text, @@ -325,13 +334,17 @@ async def _speak( self, user_input: conversation.ConversationInput, key: str, + error: ha_intent.IntentResponseErrorCode | None = None, **placeholders: str, ) -> conversation.ConversationResult: """Say one of our own lines, with its placeholders filled in.""" language = user_input.language or self.hass.config.language - text = (await self._lines(language))[key] + text = (await self._lines(language))[key].format(**placeholders) response = ha_intent.IntentResponse(language=user_input.language) - response.async_set_speech(text.format(**placeholders)) + if error is None: + response.async_set_speech(text) + else: + response.async_set_error(error, text) return conversation.ConversationResult( response=response, conversation_id=user_input.conversation_id ) diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..1502dc0 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -364,7 +364,7 @@ "preview_mismatched": "The API answered, but not to the question that was asked.", "preview_template_error": "The template does not render: {reason}", "query_not_found": "I could not find that.", - "budget_spent": "The daily token budget is spent, so I cannot do that today.", + "budget_spent": "Not enough of the daily token budget is left for that, so I cannot do it today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..b9261e9 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -364,7 +364,7 @@ "preview_mismatched": "API odpovědělo, ale ne na otázku, která byla položena.", "preview_template_error": "Šablona se nevykreslí: {reason}", "query_not_found": "To jsem nenašel.", - "budget_spent": "Denní rozpočet tokenů je vyčerpán, takže to dnes udělat nemohu.", + "budget_spent": "Z denního rozpočtu tokenů na to nezbývá dost, takže to dnes udělat nemohu.", "auth_failed": "TypeSafe odmítl klíč API. Zkontrolujte ho v nastavení Jev.", "unavailable": "TypeSafe neodpověděl. Zkuste to za chvíli znovu.", "preview_over_budget": "Denní rozpočet tokenů je vyčerpán, takže zkušební odpověď není. Otázka se přesto uloží.", diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..162a8fb 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -364,7 +364,7 @@ "preview_mismatched": "API'en svarede, men ikke på det spørgsmål, der blev stillet.", "preview_template_error": "Skabelonen kan ikke gengives: {reason}", "query_not_found": "Det kunne jeg ikke finde.", - "budget_spent": "Det daglige tokenbudget er brugt, så det kan jeg ikke gøre i dag.", + "budget_spent": "Der er ikke nok tilbage af det daglige tokenbudget til det, så det kan jeg ikke gøre i dag.", "auth_failed": "TypeSafe afviste API-nøglen. Tjek den i indstillingerne for Jev.", "unavailable": "TypeSafe svarede ikke. Prøv igen om et øjeblik.", "preview_over_budget": "Det daglige token-budget er brugt op, så der er intet prøvesvar. Spørgsmålet bliver gemt alligevel.", diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..69fbe6c 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -364,7 +364,7 @@ "preview_mismatched": "Die API hat geantwortet, aber nicht auf die gestellte Frage.", "preview_template_error": "Die Vorlage lässt sich nicht rendern: {reason}", "query_not_found": "Das konnte ich nicht finden.", - "budget_spent": "Das tägliche Token-Budget ist aufgebraucht, deshalb kann ich das heute nicht tun.", + "budget_spent": "Vom täglichen Token-Budget ist dafür nicht mehr genug übrig, deshalb kann ich das heute nicht tun.", "auth_failed": "TypeSafe hat den API-Schlüssel abgelehnt. Prüfe ihn in den Jev-Einstellungen.", "unavailable": "TypeSafe hat nicht geantwortet. Versuche es gleich noch einmal.", "preview_over_budget": "Das Tagesbudget für Tokens ist aufgebraucht, es gibt also keine Probeantwort. Die Frage wird trotzdem gespeichert.", diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..1502dc0 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -364,7 +364,7 @@ "preview_mismatched": "The API answered, but not to the question that was asked.", "preview_template_error": "The template does not render: {reason}", "query_not_found": "I could not find that.", - "budget_spent": "The daily token budget is spent, so I cannot do that today.", + "budget_spent": "Not enough of the daily token budget is left for that, so I cannot do it today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..731deeb 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -364,7 +364,7 @@ "preview_mismatched": "La API ha respondido, pero no a la pregunta que se le hizo.", "preview_template_error": "La plantilla no se procesa: {reason}", "query_not_found": "No he podido encontrarlo.", - "budget_spent": "El presupuesto diario de tokens se ha agotado, así que hoy no puedo hacerlo.", + "budget_spent": "No queda suficiente presupuesto diario de tokens para eso, así que hoy no puedo hacerlo.", "auth_failed": "TypeSafe rechazó la clave de API. Revísala en los ajustes de Jev.", "unavailable": "TypeSafe no respondió. Inténtalo de nuevo en un momento.", "preview_over_budget": "El presupuesto diario de tokens está agotado, así que no hay respuesta de prueba. La pregunta sí se guarda.", diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..96b7c78 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -364,7 +364,7 @@ "preview_mismatched": "L'API a répondu, mais pas à la question qui a été posée.", "preview_template_error": "Le rendu du modèle échoue : {reason}", "query_not_found": "Je n'ai pas trouvé cela.", - "budget_spent": "Le budget quotidien de jetons est épuisé, je ne peux donc pas le faire aujourd'hui.", + "budget_spent": "Il ne reste pas assez de budget quotidien de jetons pour cela, je ne peux donc pas le faire aujourd'hui.", "auth_failed": "TypeSafe a refusé la clé API. Vérifiez-la dans les paramètres de Jev.", "unavailable": "TypeSafe n'a pas répondu. Réessayez dans un instant.", "preview_over_budget": "Le budget quotidien de jetons est épuisé, il n'y a donc pas de réponse d'essai. La question est quand même enregistrée.", diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..dbbc17d 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -364,7 +364,7 @@ "preview_mismatched": "L'API ha risposto, ma non alla domanda che è stata posta.", "preview_template_error": "Il template non si elabora: {reason}", "query_not_found": "Non sono riuscito a trovarlo.", - "budget_spent": "Il budget giornaliero di token è esaurito, quindi oggi non posso farlo.", + "budget_spent": "Non resta abbastanza budget giornaliero di token per questo, quindi oggi non posso farlo.", "auth_failed": "TypeSafe ha rifiutato la chiave API. Controllala nelle impostazioni di Jev.", "unavailable": "TypeSafe non ha risposto. Riprova tra un momento.", "preview_over_budget": "Il budget giornaliero di token è esaurito, quindi non c'è una risposta di prova. La domanda viene comunque salvata.", diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..3a937ab 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -364,7 +364,7 @@ "preview_mismatched": "De API antwoordde, maar niet op de vraag die gesteld is.", "preview_template_error": "De template rendert niet: {reason}", "query_not_found": "Dat kon ik niet vinden.", - "budget_spent": "Het dagelijkse tokenbudget is op, dus dat kan ik vandaag niet doen.", + "budget_spent": "Er is niet genoeg over van het dagelijkse tokenbudget, dus dat kan ik vandaag niet doen.", "auth_failed": "TypeSafe heeft de API-sleutel geweigerd. Controleer hem in de instellingen van Jev.", "unavailable": "TypeSafe gaf geen antwoord. Probeer het zo nog eens.", "preview_over_budget": "Het daglimiet voor tokens is op, dus er is geen proefantwoord. De vraag wordt wel opgeslagen.", diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..2ad1ed1 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -364,7 +364,7 @@ "preview_mismatched": "API odpowiedziało, ale nie na zadane pytanie.", "preview_template_error": "Szablon się nie renderuje: {reason}", "query_not_found": "Nie udało mi się tego znaleźć.", - "budget_spent": "Dzienny budżet tokenów się wyczerpał, więc dziś nie mogę tego zrobić.", + "budget_spent": "W dziennym budżecie tokenów nie zostało na to dość, więc dziś nie mogę tego zrobić.", "auth_failed": "TypeSafe odrzucił klucz API. Sprawdź go w ustawieniach Jev.", "unavailable": "TypeSafe nie odpowiedział. Spróbuj ponownie za chwilę.", "preview_over_budget": "Dzienny limit tokenów jest wyczerpany, więc nie ma próbnej odpowiedzi. Pytanie i tak zostanie zapisane.", diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..dab4a50 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -364,7 +364,7 @@ "preview_mismatched": "A API respondeu, mas não à pergunta que foi feita.", "preview_template_error": "O template não é renderizado: {reason}", "query_not_found": "Não consegui encontrar isso.", - "budget_spent": "O orçamento diário de tokens acabou, então não posso fazer isso hoje.", + "budget_spent": "Não resta orçamento diário de tokens suficiente para isso, então não posso fazer isso hoje.", "auth_failed": "A TypeSafe rejeitou a chave de API. Verifique-a nas configurações do Jev.", "unavailable": "A TypeSafe não respondeu. Tente novamente em instantes.", "preview_over_budget": "O orçamento diário de tokens acabou, então não há resposta de teste. A pergunta ainda é salva.", diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..9742937 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -364,7 +364,7 @@ "preview_mismatched": "API ответил, но не на тот вопрос, который был задан.", "preview_template_error": "Шаблон не обрабатывается: {reason}", "query_not_found": "Мне не удалось это найти.", - "budget_spent": "Дневной бюджет токенов исчерпан, поэтому сегодня я не могу это сделать.", + "budget_spent": "В дневном бюджете токенов на это не хватает, поэтому сегодня я не могу это сделать.", "auth_failed": "TypeSafe отклонил ключ API. Проверьте его в настройках Jev.", "unavailable": "TypeSafe не ответил. Попробуйте ещё раз чуть позже.", "preview_over_budget": "Дневной бюджет токенов исчерпан, поэтому пробного ответа нет. Вопрос всё равно сохраняется.", diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..5f41075 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -364,7 +364,7 @@ "preview_mismatched": "API:et svarade, men inte på frågan som ställdes.", "preview_template_error": "Mallen renderas inte: {reason}", "query_not_found": "Det kunde jag inte hitta.", - "budget_spent": "Den dagliga tokenbudgeten är slut, så det kan jag inte göra i dag.", + "budget_spent": "Det finns inte tillräckligt kvar av den dagliga tokenbudgeten för det, så det kan jag inte göra i dag.", "auth_failed": "TypeSafe avvisade API-nyckeln. Kontrollera den i inställningarna för Jev.", "unavailable": "TypeSafe svarade inte. Försök igen om en stund.", "preview_over_budget": "Den dagliga budgeten för token är slut, så det finns inget provsvar. Frågan sparas ändå.", diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..1490080 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -364,7 +364,7 @@ "preview_mismatched": "API 作出了回答，但回答的不是所提的那个问题。", "preview_template_error": "模板无法渲染：{reason}", "query_not_found": "我找不到那个。", - "budget_spent": "今天的令牌预算已用完，所以今天无法执行。", + "budget_spent": "今天剩余的令牌预算不足，所以无法执行。", "auth_failed": "TypeSafe 拒绝了 API 密钥。请在 Jev 设置中检查。", "unavailable": "TypeSafe 没有响应。请稍后再试。", "preview_over_budget": "每日令牌预算已用完，因此没有试答结果。问题仍会保存。", diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 16a3e64..18304fd 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -15,6 +15,7 @@ from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar from homeassistant.helpers import entity_registry as er +from homeassistant.helpers import intent as ha_intent from homeassistant.setup import async_setup_component from jevclient import ChoiceAnswer, NoulAnswer @@ -198,6 +199,7 @@ async def test_a_compound_command_acts_on_nothing(hass, house, mock_client): assert calls == [] assert "did not understand" in result.response.speech["plain"]["speech"] + assert result.response.error_code is ha_intent.IntentResponseErrorCode.NO_INTENT_MATCH async def test_low_confidence_acts_on_nothing(hass, house, mock_client): @@ -342,7 +344,8 @@ async def test_a_spent_budget_stops_voice_too(hass, house, mock_client): assert mock_client.ask.await_count == 0 assert calls == [] - assert "budget is spent" in result.response.speech["plain"]["speech"] + assert "budget is left" in result.response.speech["plain"]["speech"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR async def test_a_command_that_would_pass_the_budget_is_not_sent(hass, house, mock_client): @@ -360,7 +363,8 @@ async def test_a_command_that_would_pass_the_budget_is_not_sent(hass, house, moc assert mock_client.ask.await_count == 0 assert calls == [] - assert "budget is spent" in result.response.speech["plain"]["speech"] + assert "budget is left" in result.response.speech["plain"]["speech"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR async def test_a_voice_command_teaches_the_estimate(hass, house, mock_client): From 72d9c946f359b4597d7266a7df208d7675f55b81 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:26:50 +0200 Subject: [PATCH 12/24] Show the decision in the Assist dialog and the pipeline debug events chat_log.async_trace() writes to the conversation trace, and nothing in Home Assistant 2026.9.3 reads that: pipeline_debug/get showed only run-start, intent-start, intent-end and run-end. The agent now sends the record to the chat log's delta listener as thinking_content. The pipeline stores it as an intent-progress event, and the Assist dialog appends thinking_content under the reply. On a development instance pipeline_debug/get held the event for an action and for a refusal, with the decision, the slots, the top three options of each answer, the model, the input tokens and the latency. The delta goes to the listener only and is not added to the chat log, so a fallback agent reading the same conversation does not see it. --- custom_components/jev/conversation.py | 58 +++++++++++++++------- tests/test_conversation.py | 71 +++++++++++++++++++++------ 2 files changed, 96 insertions(+), 33 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 481dfee..ea35674 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -22,11 +22,12 @@ from __future__ import annotations +import json import logging import re from collections.abc import Mapping from dataclasses import asdict, dataclass -from typing import Literal +from typing import Any, Literal from homeassistant.components import conversation from homeassistant.components.conversation.models import AbstractConversationAgent @@ -40,7 +41,7 @@ from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback from homeassistant.util import dt as dt_util from homeassistant.util import language as language_util -from jevclient import JevAuthError, JevError +from jevclient import ChoiceAnswer, JevAuthError, JevError, JevResponse, NoulAnswer from .const import ( CONF_ALLOW_WHOLE_HOME, @@ -207,22 +208,18 @@ async def _async_handle_message( **asdict(decision), } runtime.conversation_traces.appendleft(trace) - # The Assist debug view shows this beside the pipeline's own steps. It - # carries each answer as well, with its distribution, because "why did it - # pick the office light" is answered by the entity question's - # probabilities and by nothing in the decision alone. Diagnostics keep the - # shorter record, since they hold the last few commands in memory. - chat_log.async_trace( - { - "jev": trace - | { - "model": response.model, - "answers": { - key: asdict(answer) for key, answer in response.answers.items() - }, - } - } - ) + # The pipeline records a chat log delta as an intent-progress event: the + # Assist dialog shows its thinking_content under the reply, and the run's + # debug events keep it. The delta goes to the listener only, not into the + # log, so a fallback agent reading this conversation never takes it for + # something said. It carries each answer's distribution, because "why did + # it pick the office light" is answered by the entity question and by + # nothing in the decision alone. + if chat_log.delta_listener is not None: + chat_log.delta_listener( + chat_log, + {"role": "assistant", "thinking_content": _reasoning(trace, response)}, + ) if decision.already_satisfied is not None: name, settled = decision.already_satisfied @@ -365,6 +362,31 @@ async def _speak( ) +def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: + """The trace as lines a person reads in the Assist dialog.""" + lines = [ + f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " + f"confidence {trace['confidence']:.2f}", + f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", + ] + for key, answer in response.answers.items(): + if isinstance(answer, ChoiceAnswer): + ranked = sorted((answer.probabilities or {}).items(), key=lambda kv: -kv[1])[ + :3 + ] + spread = ", ".join(f"{k} {p:.2f}" for k, p in ranked) + lines.append(f"{key}: {answer.choice} {answer.confidence:.2f} ({spread})") + elif isinstance(answer, NoulAnswer): + lines.append(f"{key}: {answer.noul:.2f}") + else: + lines.append(f"{key}: {json.dumps(asdict(answer), ensure_ascii=False)}") + lines.append( + f"{response.model}, {trace['input_tokens']} input tokens, " + f"{trace['latency_ms']:.0f} ms, {trace['exposed_entities']} entities" + ) + return "\n".join(lines) + + # A template that compares the state against an English word writes the state word # itself, in its own language. Hand it a translated one and every branch falls # through. Measured against home-assistant-intents 2026.8.28: 3 of the 47 templates diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 7ae102f..dfde94a 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -10,11 +10,11 @@ import pytest from homeassistant.components import conversation -from homeassistant.components.conversation.trace import async_get_traces from homeassistant.components.homeassistant.exposed_entities import async_expose_entity from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar +from homeassistant.helpers import chat_session from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.setup import async_setup_component @@ -520,7 +520,34 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH -async def test_the_assist_debug_view_shows_what_jev_answered(hass, house, mock_client): +async def converse_in_a_pipeline(hass, text): + """Converse the way assist_pipeline does, with a listener on the chat log.""" + deltas = [] + with ( + chat_session.async_get_chat_session(hass, None) as session, + conversation.async_get_chat_log( + hass, + session, + conversation.ConversationInput( + text=text, + context=Context(), + conversation_id=session.conversation_id, + device_id=None, + satellite_id=None, + language="en", + agent_id=AGENT, + ), + chat_log_delta_listener=lambda _log, delta: deltas.append(delta), + ) as chat_log, + ): + result = await conversation.async_converse( + hass, text, session.conversation_id, Context(), "en", agent_id=AGENT + ) + content = list(chat_log.content) + return result, deltas, content + + +async def test_the_assist_dialog_shows_what_jev_answered(hass, house, mock_client): mock_client.ask.return_value = build_response( **answer_set( entity=ChoiceAnswer( @@ -530,20 +557,34 @@ async def test_the_assist_debug_view_shows_what_jev_answered(hass, house, mock_c ) ) ) - await converse(hass, "kitchen light on") - await hass.async_block_till_done() + _, deltas, content = await converse_in_a_pipeline(hass, "kitchen light on") - events = async_get_traces()[-1].as_dict()["events"] - details = [e["data"]["jev"] for e in events if e["event_type"] == "agent_detail"] - assert len(details) == 1 - jev = details[0] - assert jev["text"] == "kitchen light on" - assert jev["intent_type"] == "HassTurnOn" - assert jev["input_tokens"] == 321 - assert jev["answers"]["entity"]["probabilities"] == { - "light.kitchen": 0.8, - "light.office": 0.2, - } + assert len(deltas) == 1 + assert deltas[0]["role"] == "assistant" + shown = deltas[0]["thinking_content"] + assert "Jev: turn_on, ok, confidence 0.97" in shown + assert "entity: light.kitchen 0.80 (light.kitchen 0.80, light.office 0.20)" in shown + assert "321 input tokens" in shown + # Shown to the pipeline, never written into the conversation a fallback reads. + assert [c.role for c in content] == ["system", "user"] + + +async def test_the_assist_dialog_shows_why_jev_refused(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set(compound=NoulAnswer(noul=0.95)) + ) + result, deltas, _ = await converse_in_a_pipeline(hass, "two things at once") + + assert "several commands in one sentence" in deltas[0]["thinking_content"] + assert "compound: 0.95" in deltas[0]["thinking_content"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR + + +async def test_a_command_outside_a_pipeline_still_answers(hass, house, mock_client): + """No listener, as from conversation.process: nothing to show it to.""" + mock_client.ask.return_value = build_response(**answer_set()) + result = await converse(hass, "kitchen light on") + assert result.response.response_type is not ha_intent.IntentResponseType.ERROR @pytest.mark.parametrize( From 39740f2a17e429222296be8d553b6ad77ced8798 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:31:06 +0200 Subject: [PATCH 13/24] Ask which device from the names, not from the probabilities On a development instance with two lights both called "Lamp", "turn on the lamp" came back as one of them at 1.00, every time. The entity answer never split, so the ask-back never asked, and the agent acted on a guess that looked sure. The candidates now come from the exposed names of the chosen device's kind. The whole name said beats a name that only shares a word with the command, so "the lamp" is Lamp beside a Desk lamp, and "the desk lamp" is the Desk lamp. Two names that fit equally well get the question. More than two fall back. A room the model is sure of narrows the list first, so "the lamp in the office" acts. When no name fits the words at all, the model's answer stands as before. A reply that picks a device reads the command again without the check, so it does not ask the same question twice. --- custom_components/jev/conversation.py | 4 +- custom_components/jev/interpret.py | 127 +++++++++++++++++--------- tests/test_conversation.py | 86 +++++++++++++++-- 3 files changed, 166 insertions(+), 51 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 962232d..530bec2 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -381,7 +381,9 @@ async def _resolve( first = replace( pending.response, answers=pending.response.answers | {"entity": settled} ) - decision = interpret(first, pending.text, snapshot, self._min_confidence) + decision = interpret( + first, pending.text, snapshot, self._min_confidence, ask_back=False + ) return await self._act(user_input, decision, pending.text) # --- acting --- diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index feccaab..6abaa85 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -227,8 +227,13 @@ def interpret( text: str, snapshot: HomeSnapshot, min_confidence: float, + *, + ask_back: bool = True, ) -> Interpretation: - """Read the answers that matter and ignore the rest.""" + """Read the answers that matter and ignore the rest. + + ask_back=False reads a command whose device a reply has already picked. + """ def choice(key: str) -> ChoiceAnswer | None: answer = response.answers.get(key) @@ -293,6 +298,28 @@ def out(reason: str) -> Interpretation: area = choice("area") slots: dict[str, Any] = {} targets_everything = False + named_area = ( + area.choice + if area is not None and area.choice != NONE and area.confidence >= min_confidence + else None + ) + + def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: + # The action is sure and the device is one of two. Asking costs one short + # question, and handing the sentence to the fallback agent gets the same + # guess made again by something that does not know it was a guess. + if spoken_name(first, second) is None: + return out("two devices fit the name and nothing tells them apart") + return Interpretation( + None, + {}, + action.choice, + action.confidence, + "two devices fit the name", + fallback=False, + action_probabilities=dict(action.probabilities or {}), + candidates=(first.entity_id, second.entity_id), + ) # Trust the confident answer rather than the ordering. Measured: a scope answer # of one_room at 0.41 alongside a device answer at 1.00, where branching on @@ -305,6 +332,12 @@ def out(reason: str) -> Interpretation: described = snapshot.by_id(entity.choice) if described is None: return out("named a device that is not exposed") + if ask_back: + tied = _fit_as_well(text, described, snapshot, named_area) + if len(tied) == 2: + return ask(*tied) + if len(tied) > 2: + return out(f"{len(tied)} devices fit the name") slots["name"] = {"value": described.name} # The domain keeps a same-named entity the model was never shown, a lock # called "Front door" beside a cover called "Front door", out of the match. @@ -328,20 +361,13 @@ def out(reason: str) -> Interpretation: # unbounded off is not something to infer from one ambiguous sentence. slots["name"] = {"value": "all"} targets_everything = True - elif pair := _two_that_fit(entity, snapshot, min_confidence): - # The action is sure and the device is one of two. Asking costs one short - # question, and handing the sentence to the fallback agent gets the same - # guess made again by something that does not know it was a guess. - return Interpretation( - None, - {}, - action.choice, - action.confidence, - "two devices fit the name", - fallback=False, - action_probabilities=dict(action.probabilities or {}), - candidates=pair, - ) + elif ( + ask_back + and entity is not None + and (unsure := snapshot.by_id(entity.choice)) is not None + and len(tied := _fit_as_well(text, unsure, snapshot, named_area)) == 2 + ): + return ask(*tied) else: return out("no target named with enough confidence") @@ -379,37 +405,39 @@ def out(reason: str) -> Interpretation: ) -# The least share of the entity answer a device needs to be offered as one of two. -# With this, the two named devices hold at least 40% between them and the rest is -# spread across the others. Not measured on a real instance yet. -ASK_BACK_FLOOR = 0.2 - +def _fit_as_well( + text: str, chosen: ExposedEntity, snapshot: HomeSnapshot, area: str | None +) -> list[ExposedEntity]: + """The devices of the chosen kind whose names fit the words as well as its own. -def _two_that_fit( - entity: ChoiceAnswer | None, snapshot: HomeSnapshot, min_confidence: float -) -> tuple[str, str] | None: - """The two devices a command could mean, when it is one of them and not a third. + The model gives one device all of its answer even when two fit: measured on a + development instance with two lights both called "Lamp", "turn on the lamp" + came back as one of them at 1.00, every time. The probabilities cannot say the + name was shared, the names can. A room the command names narrows the list. - Both must be exposed, each must hold ASK_BACK_FLOOR of the answer, and the two - together must reach the confidence the agent acts on. A third device above the - floor means the question would not settle it, so the agent does not ask. + Only the chosen device when it fits best alone, or when no name fits the words + at all, which is where the model's reading is all there is. """ - if entity is None or not entity.probabilities: - return None - ranked = sorted( - ( - (probability, entity_id) - for entity_id, probability in entity.probabilities.items() - if entity_id != NONE and probability >= ASK_BACK_FLOOR - ), - reverse=True, - ) - if len(ranked) != 2 or ranked[0][0] + ranked[1][0] < min_confidence: - return None - first, second = (snapshot.by_id(entity_id) for _, entity_id in ranked) - if first is None or second is None or spoken_name(first, second) is None: - return None - return first.entity_id, second.entity_id + kind = [ + e + for e in snapshot.entities + if e.domain == chosen.domain and (area is None or e.area == area) + ] + if chosen not in kind: + return [chosen] + fit = {e.entity_id: _name_fit(text, e.name) for e in kind} + best = max(fit.values()) + if best == 0 or fit[chosen.entity_id] < best: + return [chosen] + return [e for e in kind if fit[e.entity_id] == best] + + +def _name_fit(text: str, name: str) -> int: + """How well the words fit a name. The whole name said beats any part of it.""" + if said := _words_said(text, name): + return 100 + said + words = set(re.findall(r"\w+", text.casefold())) + return len(set(name.casefold().split()) & words) def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | None: @@ -424,6 +452,19 @@ def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | N return None +def _words_said(text: str, name: str) -> int: + """How many words of the name the text says, as one phrase, or 0. + + Whole words only, so a hidden "Lamp" is not found inside "lamps". In a language + written without spaces a name rarely stands apart, so the check seldom fires. + """ + words = name.casefold().split() + if not words: + return 0 + phrase = r"\s+".join(re.escape(w) for w in words) + return len(words) if re.search(rf"(? Date: Thu, 24 Sep 2026 08:32:00 +0200 Subject: [PATCH 14/24] Calibrate reports hours to two places A test window with the door open for two minutes returned hours_true 0.0 beside times_true 1, which read as the never-true refusal failing to fire. Both hour counts now keep two places, so the same window reads 0.03. --- custom_components/jev/calibrate.py | 6 ++++-- tests/test_calibrate.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/custom_components/jev/calibrate.py b/custom_components/jev/calibrate.py index a6ee4f0..07d2bc4 100644 --- a/custom_components/jev/calibrate.py +++ b/custom_components/jev/calibrate.py @@ -215,8 +215,10 @@ async def async_calibrate(hass: HomeAssistant, call: ServiceCall) -> ServiceResp ) return { **best(spans).as_dict(), - "hours": round(sum(span.seconds for span in spans) / 3600, 1), - "hours_true": round(true_seconds / 3600, 1), + # Two places, so a door open for two minutes reads 0.03 and not 0.0, which + # looked like the never-true refusal had failed to fire. + "hours": round(sum(span.seconds for span in spans) / 3600, 2), + "hours_true": round(true_seconds / 3600, 2), # F1 over two occasions is noise, whatever its value. These are the counts # to judge it by. "times_true": _times_true(spans), diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py index 874e7ad..07eaa3b 100644 --- a/tests/test_calibrate.py +++ b/tests/test_calibrate.py @@ -117,6 +117,22 @@ async def record_a_day(hass, freezer): await async_wait_recording_done(hass) +async def record_two_minutes_open(hass, freezer): + """An hour with the door open for two minutes of it.""" + start = dt_util.utcnow() - timedelta(hours=1) + for minute, probability, door in ( + (0, "0.2", "off"), + (30, "0.8", "on"), + (32, "0.1", "off"), + ): + freezer.move_to(start + timedelta(minutes=minute)) + hass.states.async_set(PROBABILITY, probability) + hass.states.async_set(TRUTH, door) + await hass.async_block_till_done() + freezer.move_to(start + timedelta(hours=1)) + await async_wait_recording_done(hass) + + async def calibrate(hass, **data): return await hass.services.async_call( DOMAIN, @@ -173,3 +189,13 @@ async def test_an_entity_with_no_history_is_refused(hass, loaded_entry): await calibrate(hass) assert err.value.translation_key == "calibrate_no_history" assert err.value.translation_placeholders["entity"] == PROBABILITY + + +async def test_a_short_time_true_is_not_rounded_to_nothing( + hass, loaded_entry, mock_client, freezer +): + await record_two_minutes_open(hass, freezer) + result = await calibrate(hass) + + assert result["hours_true"] == 0.03 + assert result["times_true"] == 1 From ffa47bdcadd5e66a8306b4042c49eea65782deea Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:33:17 +0200 Subject: [PATCH 15/24] docs: ask-back from names, the Assist dialog note, the budget refusal The ask-back section described the probability split. It now describes the name comparison. It also restores the "What it costs" heading that the section had replaced. The diagnostics section named the debug view as the place for the agent detail. The note now shows in the Assist dialog and as an intent-progress event in the pipeline debug view. The refusal row and the budget sensor now say what the budget actually refuses: a command whose estimate does not fit, with tokens left. --- site-docs/conversation.md | 34 +++++++++++++++++++++------------- site-docs/cost.md | 4 +++- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 1ca6361..3d322a2 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -52,8 +52,8 @@ hallway" would reach a lock exposed there and unlock it. | An entity you did not expose to Assist | Never described to the model at all | | No room and no device named | Refused, unless you allow it. `turn_off` is exempt. Below the confidence floor, fallback | | Two kinds of device, no kind named, whole house | Asks which kind. With one kind exposed, it acts on that kind | -| Two devices fit the name about equally | Asks which one, see below | -| A spent budget, a rejected key or no answer | Fallback. With no fallback agent it says which of the three it was | +| Two devices whose names fit the command equally well | Asks which one, see below | +| Too little budget left for the command, a rejected key or no answer | Fallback. With no fallback agent it says which of the three it was, as an error reply | !!! info "It only sees what Assist sees" The agent describes only entities you exposed to Assist. You already decided @@ -63,11 +63,18 @@ hallway" would reach a lock exposed there and unlock it. ## When two devices fit the name -"Turn on the lamp" with a desk lamp and a floor lamp exposed can split the model's -probability between the two, with neither above the confidence floor. When exactly two -exposed devices each have at least 0.2, and the two together reach the floor, the agent -asks **"Do you mean Desk lamp or Floor lamp?"** and keeps the conversation open. When -the two have the same name, it adds the room: "Lamp (Office) or Lamp (Bedroom)". +The agent compares what you said with the names of the exposed devices of the kind +the model picked. The whole name said wins: with a "Lamp" and a "Desk lamp", "turn on +the lamp" is the Lamp and "turn on the desk lamp" is the Desk lamp. When two names fit +equally well, as "the lamp" does for a Desk lamp and a Floor lamp, the agent asks +**"Do you mean Desk lamp or Floor lamp?"** and keeps the conversation open. When the +two have the same name, it adds the room: "Lamp (Office) or Lamp (Bedroom)". A room +you name settles it first, so "the lamp in the office" acts. + +The names decide this, not the model's confidence. On a test instance with two lights +both called "Lamp", the model put 1.00 on one of them, every time, so a confidence +split never showed that there were two. With three or more that fit equally well, the +command goes to the fallback agent. Your reply, such as "the desk one", is one more request, and it counts against the budget. It asks which of the two the reply picks. A confident pick carries out the @@ -75,9 +82,7 @@ first command on that device. A reply that picks neither is handled as a new com The question expires after five minutes, the same time Home Assistant keeps a conversation open. -The 0.2 is not measured. It keeps a third device with a small share from turning the -question into a guess between two. - +## What it costs Every command counts against the same daily token budget as your sensors. A satellite that mishears a wake word all night trips that tripwire instead of running @@ -135,9 +140,12 @@ only sees what Jev could not route, which is the cheap arrangement. ## Diagnostics -The Assist debug view (**Settings**, **Voice assistants**, the pipeline's menu, -**Debug**) shows what Jev answered for each command: the model, every answer with its -probabilities, and the reason for the decision. +In the Assist dialog, each reply from Jev has a note under it with what Jev answered: +the decision and its reason, the slots, each answer with its top three options, the +model, the input tokens and the time the call took. The pipeline's debug view +(**Settings**, **Voice assistants**, the pipeline's menu, **Debug**) keeps the same +note as an `intent-progress` event of the run. The note goes to the pipeline only. It +is not added to the conversation, so a fallback agent does not read it. The last 20 decisions the agent made are in the integration's diagnostics, with the reason for every decision and the action distribution behind it. The sentence itself diff --git a/site-docs/cost.md b/site-docs/cost.md index d192230..1ea4df1 100644 --- a/site-docs/cost.md +++ b/site-docs/cost.md @@ -83,7 +83,9 @@ When it trips: - the answer sensors of that context go unavailable, because Jev was not asked and there is no answer for right now. The last answers are not thrown away, and the next call that fits replaces them -- `binary_sensor.jev_daily_budget_exceeded` turns on +- `binary_sensor.jev_daily_budget_exceeded` turns on. It shows that contexts have + stopped, so a spoken command, an action or an AI Task that the budget refuses + does not turn it on. Each of those says so to whoever started it - a repair issue explains it, naming the budget, what has been used, and the context that was refused with the tokens it needed From 43aa5e1a3cfd2387c0e4c001b8cb472ef425d98f Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:37:19 +0200 Subject: [PATCH 16/24] Check that a voice command records its size, not the ratio it gives --- tests/test_conversation.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 18304fd..99302b2 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -17,7 +17,7 @@ from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.setup import async_setup_component -from jevclient import ChoiceAnswer, NoulAnswer +from jevclient import ChoiceAnswer, NoulAnswer, Usage from custom_components.jev.const import ( CONF_ALLOW_WHOLE_HOME, @@ -371,16 +371,18 @@ async def test_a_voice_command_teaches_the_estimate(hass, house, mock_client): # The estimate reads bytes per token from the last call it could measure. A # voice command that did not report its size left the estimate on whatever # the last context taught it, which is a different shape of request. - mock_client.ask.return_value = build_response(**answer_set()) + mock_client.ask.return_value = replace( + build_response(**answer_set()), usage=Usage(input_tokens=1371, output_tokens=42) + ) usage = house.runtime_data.usage - usage.bytes_per_token = 1.0 - await converse(hass, "kitchen light on") - await hass.async_block_till_done() + with patch.object(usage, "record", wraps=usage.record) as record: + await converse(hass, "kitchen light on") + await hass.async_block_till_done() sent_state, sent_questions = mock_client.ask.await_args.args sent = payload_bytes(sent_state, sent_questions, house.runtime_data.model) - assert usage.bytes_per_token == sent / 321 + record.assert_called_once_with(1371, sent) async def test_a_rejected_key_is_said_out_loud(hass, house, mock_client): From ab601d59abd31f5713f3810d6d1cc54e19999013 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:41:53 +0200 Subject: [PATCH 17/24] Ask about a shared name when the model put most of its answer on none --- custom_components/jev/interpret.py | 15 ++++++++++++++- tests/test_conversation.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 6abaa85..6d74d9d 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -364,7 +364,7 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: elif ( ask_back and entity is not None - and (unsure := snapshot.by_id(entity.choice)) is not None + and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None and len(tied := _fit_as_well(text, unsure, snapshot, named_area)) == 2 ): return ask(*tied) @@ -405,6 +405,19 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: ) +def _likeliest_device(entity: ChoiceAnswer) -> str: + """The device the model gave most of its answer to, even if "none" got more. + + Measured on a development instance with two lights both called "Lamp": "turn on + the lamp" came back as none_of_these 0.55, one Lamp 0.44 and the other 0.01. The + model split its answer because the name was shared, so the name decides. + """ + devices = {k: v for k, v in (entity.probabilities or {}).items() if k != NONE} + if entity.choice != NONE or not devices: + return entity.choice + return max(devices, key=lambda k: devices[k]) + + def _fit_as_well( text: str, chosen: ExposedEntity, snapshot: HomeSnapshot, area: str | None ) -> list[ExposedEntity]: diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 73fddd4..2aba929 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -1439,6 +1439,31 @@ async def test_a_sure_answer_is_still_asked_about_when_the_name_is_shared( ) +async def test_a_shared_name_is_asked_about_when_none_got_most_of_the_answer( + hass, house, mock_client +): + """Measured: none_of_these 0.55, one Lamp 0.44, the other 0.01.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + shares = {NONE: 0.55, "light.kitchen": 0.44, "light.office": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp") + await hass.async_block_till_done() + + assert calls == [] + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_client): rename(hass, "light.kitchen", "Lamp") rename(hass, "light.office", "Lamp") From cf7280ac260d086b87996e4708c5a4ba227e786b Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:52:49 +0200 Subject: [PATCH 18/24] Show a reply's pick in the Assist dialog without failing the reply --- custom_components/jev/conversation.py | 25 ++++++++++++++++--------- tests/test_conversation.py | 22 ++++++++++++++++++++-- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 530bec2..a81282f 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -538,11 +538,17 @@ async def _speak( def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: """The trace as lines a person reads in the Assist dialog.""" - lines = [ - f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " - f"confidence {trace['confidence']:.2f}", - f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", - ] + if "answers_command" in trace: + lines = [ + f'Jev: a reply to "{trace["answers_command"]}", ' + f"picked {trace['picked'] or 'neither'}" + ] + else: + lines = [ + f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " + f"confidence {trace['confidence']:.2f}", + f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", + ] for key, answer in response.answers.items(): if isinstance(answer, ChoiceAnswer): ranked = sorted((answer.probabilities or {}).items(), key=lambda kv: -kv[1])[ @@ -554,10 +560,11 @@ def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: lines.append(f"{key}: {answer.noul:.2f}") else: lines.append(f"{key}: {json.dumps(asdict(answer), ensure_ascii=False)}") - lines.append( - f"{response.model}, {trace['input_tokens']} input tokens, " - f"{trace['latency_ms']:.0f} ms, {trace['exposed_entities']} entities" - ) + footer = f"{response.model}, {trace['input_tokens']} input tokens, " + footer += f"{trace['latency_ms']:.0f} ms" + if "exposed_entities" in trace: + footer += f", {trace['exposed_entities']} entities" + lines.append(footer) return "\n".join(lines) diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 2aba929..78372bd 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -525,11 +525,11 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH -async def converse_in_a_pipeline(hass, text): +async def converse_in_a_pipeline(hass, text, conversation_id=None): """Converse the way assist_pipeline does, with a listener on the chat log.""" deltas = [] with ( - chat_session.async_get_chat_session(hass, None) as session, + chat_session.async_get_chat_session(hass, conversation_id) as session, conversation.async_get_chat_log( hass, session, @@ -1529,6 +1529,24 @@ async def test_the_reply_runs_the_first_command_on_the_device_it_picks( assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} +async def test_the_assist_dialog_shows_what_the_reply_picked(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + hass.services.async_register("light", "turn_on", lambda call: None) + + asked = await converse(hass, "light on") + result, deltas, _ = await converse_in_a_pipeline( + hass, "the office one", asked.conversation_id + ) + + assert result.response.response_type is ha_intent.IntentResponseType.ACTION_DONE + [delta] = deltas + assert delta["thinking_content"].startswith( + 'Jev: a reply to "light on", picked light.office\n' + ) + + async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_client): mock_client.ask.return_value = build_response( **unsure_between("light.kitchen", "light.office"), **reply(NONE) From b3db1af4afd1d2d80487718c610a68db77dc7b8a Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:57:56 +0200 Subject: [PATCH 19/24] A reply that asks for something else is a new command, not a pick --- custom_components/jev/conversation.py | 16 ++++++++++++++- tests/test_conversation.py | 28 ++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index a81282f..43a43f1 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -49,6 +49,7 @@ JevAuthError, JevError, JevResponse, + Noul, NoulAnswer, Question, ) @@ -351,7 +352,18 @@ async def _resolve( } options[NONE] = "Neither of these, or a different request" questions: dict[str, Question] = { - "which": Choice("Which device does the reply pick?", options) + "which": Choice("Which device does the reply pick?", options), + # "Never mind, turn off the lamp in the bedroom" names one of the two, so + # the choice alone picks it and the kept command, turn on, runs on it. + # Measured on a development instance, twelve replies to "turn on the + # lamp": the six that only pick a device scored 0.08 to 0.26 here, and + # the six that ask for something else, that one included, 0.91 to 0.97. + "new_request": Noul( + "Does the reply ask for something of its own, rather than only " + "saying which device the command meant?", + true="The reply is a new instruction, or changes what should happen", + false="The reply only picks a device, however it is phrased", + ), } state = {"command": pending.text, "reply": user_input.text} response = await self._ask(user_input, state, questions) @@ -359,11 +371,13 @@ async def _resolve( return response answer = response.answers.get("which") + new_request = response.answers.get("new_request") picked = ( answer.choice if isinstance(answer, ChoiceAnswer) and answer.choice in pending.candidates and answer.confidence >= self._min_confidence + and not (isinstance(new_request, NoulAnswer) and new_request.noul >= 0.5) else None ) self._trace( diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 78372bd..75439a3 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -1524,7 +1524,7 @@ async def test_the_reply_runs_the_first_command_on_the_device_it_picks( assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] # The reply was one small question about the two devices, not a new command. state, questions = mock_client.ask.await_args.args - assert list(questions) == ["which"] + assert list(questions) == ["which", "new_request"] assert state == {"command": "light on", "reply": "the office one"} assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} @@ -1564,6 +1564,32 @@ async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_cli assert result.continue_conversation is False +async def test_a_reply_that_names_a_device_in_a_new_command_runs_that_command( + hass, house, mock_client +): + """Measured: "never mind, turn off the lamp in the bedroom" picked that lamp.""" + new_request = {"new_request": NoulAnswer(noul=0.96)} + turn_off = answer_set( + action=ChoiceAnswer(choice="turn_off", probabilities={}, confidence=0.98), + entity=ChoiceAnswer(choice="light.office", probabilities={}, confidence=0.97), + ) + mock_client.ask.side_effect = [ + build_response(**unsure_between("light.kitchen", "light.office")), + build_response(**reply("light.office"), **new_request), + build_response(**turn_off), + ] + turned_on, turned_off = [], [] + hass.services.async_register("light", "turn_on", turned_on.append) + hass.services.async_register("light", "turn_off", turned_off.append) + + asked = await converse(hass, "light on") + await converse_in(hass, "no, turn off the office light", asked.conversation_id) + await hass.async_block_till_done() + + assert turned_on == [] + assert [e for c in turned_off for e in c.data["entity_id"]] == ["light.office"] + + async def test_an_unsure_reply_acts_on_nothing(hass, house, mock_client): mock_client.ask.return_value = build_response( **unsure_between("light.kitchen", "light.office"), From 6efa72a4caf2f57c33444da829e9d7ffc27c2e4e Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 09:03:12 +0200 Subject: [PATCH 20/24] Say what a reply that asks for something else does --- site-docs/conversation.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 3d322a2..68c6827 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -72,13 +72,17 @@ two have the same name, it adds the room: "Lamp (Office) or Lamp (Bedroom)". A r you name settles it first, so "the lamp in the office" acts. The names decide this, not the model's confidence. On a test instance with two lights -both called "Lamp", the model put 1.00 on one of them, every time, so a confidence -split never showed that there were two. With three or more that fit equally well, the -command goes to the fallback agent. +both called "Lamp", the model put 1.00 on one of them in one session. In another it put +0.55 on "none of these", 0.44 on one Lamp and 0.01 on the other. Neither split showed +that there were two. With three or more that fit equally well, the command goes to the +fallback agent. Your reply, such as "the desk one", is one more request, and it counts against the -budget. It asks which of the two the reply picks. A confident pick carries out the -first command on that device. A reply that picks neither is handled as a new command. +budget. It asks which of the two the reply picks, and if the reply asks for something +of its own. A confident pick carries out the first command on that device. A reply +that picks neither, or that is a new instruction, is handled as a new command. So +"never mind, turn off the lamp in the bedroom" turns that lamp off, and does not run +the first command on it. The question expires after five minutes, the same time Home Assistant keeps a conversation open. From 604f44047c5d00663870be62ba547395b22f8fd2 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 11:37:50 +0200 Subject: [PATCH 21/24] Settle a shared name by the room of the satellite that heard it Home Assistant's own agent prefers the satellite's area when a name fits devices in several rooms. Jev asked instead. It now reads the satellite entity's area, then its device's area, the same lookup, and asks only when that room does not hold exactly one of the devices. --- custom_components/jev/conversation.py | 12 +++- custom_components/jev/interpret.py | 49 ++++++++++---- custom_components/jev/snapshot.py | 18 +++++ tests/test_conversation.py | 94 +++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 15 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 43a43f1..66f4f95 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -66,7 +66,7 @@ from .entity import build_device_info from .interpret import NONE, Interpretation, build_questions, interpret, spoken_name from .payload import payload_bytes -from .snapshot import HomeSnapshot, async_snapshot +from .snapshot import HomeSnapshot, async_heard_in, async_snapshot _LOGGER = logging.getLogger(__name__) @@ -201,7 +201,15 @@ async def _async_handle_message( if isinstance(response, conversation.ConversationResult): return response - decision = interpret(response, user_input.text, snapshot, self._min_confidence) + decision = interpret( + response, + user_input.text, + snapshot, + self._min_confidence, + heard_in=async_heard_in( + self.hass, user_input.satellite_id, user_input.device_id + ), + ) self._trace( chat_log, response, diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 6d74d9d..9027d30 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -229,10 +229,12 @@ def interpret( min_confidence: float, *, ask_back: bool = True, + heard_in: str | None = None, ) -> Interpretation: """Read the answers that matter and ignore the rest. ask_back=False reads a command whose device a reply has already picked. + heard_in is the area id of the satellite or device that heard the command. """ def choice(key: str) -> ChoiceAnswer | None: @@ -321,9 +323,27 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: candidates=(first.entity_id, second.entity_id), ) + def pick(chosen: ExposedEntity, *, sure: bool) -> ExposedEntity | Interpretation: + # sure is False when the model put most of its answer on none. Then only a + # name that two devices share is a reason to go on. + tied = _fit_as_well(text, chosen, snapshot, named_area) + if not sure and len(tied) < 2: + return out("no target named with enough confidence") + # A name that fits two devices is settled by the room it was said in, as + # Home Assistant's own agent settles it. A room the command names came first. + here = [e for e in tied if heard_in is not None and e.area_id == heard_in] + if len(tied) > 1 and len(here) == 1: + return here[0] + if len(tied) == 1: + return tied[0] + if len(tied) == 2: + return ask(*tied) + return out(f"{len(tied)} devices fit the name") + # Trust the confident answer rather than the ordering. Measured: a scope answer # of one_room at 0.41 alongside a device answer at 1.00, where branching on # scope first threw away the certain answer and acted on the whole house. + described: ExposedEntity | None = None if ( entity is not None and entity.choice != NONE @@ -333,17 +353,10 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: if described is None: return out("named a device that is not exposed") if ask_back: - tied = _fit_as_well(text, described, snapshot, named_area) - if len(tied) == 2: - return ask(*tied) - if len(tied) > 2: - return out(f"{len(tied)} devices fit the name") - slots["name"] = {"value": described.name} - # The domain keeps a same-named entity the model was never shown, a lock - # called "Front door" beside a cover called "Front door", out of the match. - slots["domain"] = {"value": [described.domain]} - if described.area_id: - slots["preferred_area_id"] = {"value": described.area_id} + picked = pick(described, sure=True) + if isinstance(picked, Interpretation): + return picked + described = picked elif area is not None and area.choice != NONE and area.confidence >= min_confidence: slots["area"] = {"value": area.choice} elif ( @@ -365,12 +378,22 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: ask_back and entity is not None and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None - and len(tied := _fit_as_well(text, unsure, snapshot, named_area)) == 2 ): - return ask(*tied) + picked = pick(unsure, sure=False) + if isinstance(picked, Interpretation): + return picked + described = picked else: return out("no target named with enough confidence") + if described is not None: + slots["name"] = {"value": described.name} + # The domain keeps a same-named entity the model was never shown, a lock + # called "Front door" beside a cover called "Front door", out of the match. + slots["domain"] = {"value": [described.domain]} + if described.area_id: + slots["preferred_area_id"] = {"value": described.area_id} + # An area always carries a domain. With none, Home Assistant acts on every # exposed entity in the room whatever its domain, so turn_off on a hallway with a # light and a lock unlocked the lock. Without a confident answer, the domains the diff --git a/custom_components/jev/snapshot.py b/custom_components/jev/snapshot.py index e111258..6efdf62 100644 --- a/custom_components/jev/snapshot.py +++ b/custom_components/jev/snapshot.py @@ -102,6 +102,24 @@ def as_state(self) -> dict[str, object]: } +@callback +def async_heard_in( + hass: HomeAssistant, satellite_id: str | None, device_id: str | None +) -> str | None: + """The area id of the satellite or device that heard a command, if it has one. + + The same lookup as Home Assistant's own agent: the satellite entity's area, then + its device's area. + """ + if satellite_id and (entry := er.async_get(hass).async_get(satellite_id)): + if entry.area_id is not None: + return entry.area_id + device_id = entry.device_id + if device_id and (device := dr.async_get(hass).async_get(device_id)): + return device.area_id + return None + + @callback def async_snapshot(hass: HomeAssistant, limit: int) -> HomeSnapshot: """Collect the exposed, controllable entities, newest registry state.""" diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 75439a3..2c5ba3d 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -16,6 +16,7 @@ from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar from homeassistant.helpers import chat_session +from homeassistant.helpers import device_registry as dr from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT @@ -1483,6 +1484,99 @@ async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_clie assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] +def a_satellite_in(hass, house, area_name): + """A voice device placed in the named area, as a satellite is.""" + area = ar.async_get(hass).async_get_area_by_name(area_name) + assert area is not None + devices = dr.async_get(hass) + device = devices.async_get_or_create( + config_entry_id=house.entry_id, identifiers={("test", area_name)} + ) + devices.async_update_device(device.id, area_id=area.id) + return device.id + + +async def test_the_room_a_satellite_is_in_settles_a_shared_name(hass, house, mock_client): + """Home Assistant's own agent prefers the satellite's area, and so does Jev.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Office"), + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + + +async def test_a_satellite_entity_area_settles_a_shared_name_the_model_was_unsure_of( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + entities = er.async_get(hass) + satellite = entities.async_get_or_create("assist_satellite", "test", "kitchen") + entities.async_update_entity(satellite.entity_id, area_id=kitchen.id) + shares = {NONE: 0.55, "light.office": 0.44, "light.kitchen": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + satellite_id=satellite.entity_id, + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.kitchen"] + + +async def test_a_satellite_in_another_room_still_gets_the_question( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + ar.async_get(hass).async_create("Hall") + mock_client.ask.return_value = build_response(**answer_set()) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Hall"), + ) + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + @pytest.mark.parametrize( ("text", "chosen"), [ From 11846bad66116acb3b3ae4f2f509fca8ec182f3b Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:17:32 +0200 Subject: [PATCH 22/24] A hidden device named in full is not swapped for an exposed one With an unexposed "Desk lamp" and an exposed "Lamp", "turn on the desk lamp" came back from the model as Lamp and turned it on, 3 of 3 times on a development instance. The model can only pick from what it was shown, and Lamp was the closest option. The snapshot now keeps the names of entities the model is not shown: not exposed, a lock or entrance cover, or past the cap. They stay in Home Assistant. When the command says one of them in more words than it says the chosen device, the agent falls back and records "named a device that is not exposed". --- custom_components/jev/interpret.py | 18 ++++++++++ custom_components/jev/snapshot.py | 20 ++++++++--- site-docs/conversation.md | 1 + tests/test_conversation.py | 55 ++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 9027d30..3d41bdb 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -352,6 +352,12 @@ def pick(chosen: ExposedEntity, *, sure: bool) -> ExposedEntity | Interpretation described = snapshot.by_id(entity.choice) if described is None: return out("named a device that is not exposed") + # The model picks the closest option it was shown, and a hidden device is + # not one of them. Measured on a development instance with an unexposed + # "Desk lamp" and an exposed "Lamp": "turn on the desk lamp" came back as + # Lamp and switched it on, three times of three. + if _a_hidden_name_fits_better(text, described, snapshot): + return out("named a device that is not exposed") if ask_back: picked = pick(described, sure=True) if isinstance(picked, Interpretation): @@ -379,6 +385,10 @@ def pick(chosen: ExposedEntity, *, sure: bool) -> ExposedEntity | Interpretation and entity is not None and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None ): + # "Turn on the desk lamp" with a hidden desk lamp would otherwise ask + # which of two exposed lamps was meant. + if _a_hidden_name_fits_better(text, unsure, snapshot): + return out("named a device that is not exposed") picked = pick(unsure, sure=False) if isinstance(picked, Interpretation): return picked @@ -488,6 +498,14 @@ def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | N return None +def _a_hidden_name_fits_better( + text: str, chosen: ExposedEntity, snapshot: HomeSnapshot +) -> bool: + """A hidden name that the command says in more words than the chosen one.""" + said = _words_said(text, chosen.name) + return any(_words_said(text, name) > said for name in snapshot.hidden_names) + + def _words_said(text: str, name: str) -> int: """How many words of the name the text says, as one phrase, or 0. diff --git a/custom_components/jev/snapshot.py b/custom_components/jev/snapshot.py index 6efdf62..4f28280 100644 --- a/custom_components/jev/snapshot.py +++ b/custom_components/jev/snapshot.py @@ -69,6 +69,10 @@ class HomeSnapshot: entities: list[ExposedEntity] = field(default_factory=list) areas: list[str] = field(default_factory=list) floors: list[str] = field(default_factory=list) + # Names of the entities the model is not shown: not exposed, left out above, or + # past the cap. They never leave Home Assistant. interpret() reads them so that + # "the desk lamp" cannot land on an exposed "Lamp" when the desk lamp is hidden. + hidden_names: list[str] = field(default_factory=list) @property def domains(self) -> list[str]: @@ -140,13 +144,17 @@ def area_id_of(entity_id: str) -> str | None: return None found: list[ExposedEntity] = [] - for state in hass.states.async_all(CONTROLLABLE): - if not async_should_expose(hass, CONVERSATION_DOMAIN, state.entity_id): - continue + hidden: list[str] = [] + for state in hass.states.async_all(): if ( - state.domain == "cover" - and state.attributes.get("device_class") in ENTRANCE_COVERS + state.domain not in CONTROLLABLE + or not async_should_expose(hass, CONVERSATION_DOMAIN, state.entity_id) + or ( + state.domain == "cover" + and state.attributes.get("device_class") in ENTRANCE_COVERS + ) ): + hidden.append(state.name) continue area_id = area_id_of(state.entity_id) area = areas.async_get_area(area_id) if area_id else None @@ -163,6 +171,7 @@ def area_id_of(entity_id: str) -> str | None: # Sorted so the option list is stable between requests, which makes a trace # readable when the same command is tried twice. found.sort(key=lambda e: e.entity_id) + hidden.extend(e.name for e in found[limit:]) found = found[:limit] # Counted after the cap, so a room that only had entities past the limit is not # offered as somewhere the command could go. @@ -186,4 +195,5 @@ def area_id_of(entity_id: str) -> str | None: floors=sorted( f.name for f in floors.async_list_floors() if f.floor_id in used_floor_ids ), + hidden_names=hidden, ) diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 0279310..b14103f 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -50,6 +50,7 @@ hallway" would reach a lock exposed there and unlock it. | Needs words written or looked up | Fallback | | A lock or a garage, gate or door cover | Fallback. The agent never describes one | | An entity you did not expose to Assist | Never described to the model at all | +| A hidden device named in full, next to an exposed one with a shorter name | Fallback. "Turn on the desk lamp" does not turn on "Lamp" | | No room and no device named | Refused, unless you allow it. `turn_off` is exempt. Below the confidence floor, fallback | | Two kinds of device, no kind named, whole house | Asks which kind. With one kind exposed, it acts on that kind | | A spent budget, a rejected key or no answer | Fallback. With no fallback agent it says which of the three it was | diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 2c5ba3d..8a1f83f 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -453,6 +453,39 @@ async def test_a_device_that_is_not_exposed_is_never_acted_on(hass, house, mock_ assert calls == [] +@pytest.mark.parametrize( + ("text", "acts"), + [ + # The hidden name is said, in more words than the one the model picked. + ("turn on the kitchen light strip", False), + ("turn on the private light", False), + # Only the exposed name is said, or the hidden one is not said whole. + ("turn on the kitchen light", True), + ("turn on the kitchen lights", True), + ], +) +async def test_a_hidden_device_named_in_full_is_not_swapped_for_an_exposed_one( + hass, house, mock_client, text, acts +): + """Measured: an unexposed "Desk lamp" was said, and the exposed "Lamp" went on.""" + hass.states.async_set("light.strip", "off", {"friendly_name": "Kitchen light strip"}) + async_expose_entity(hass, conversation.DOMAIN, "light.strip", False) + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + await converse(hass, text) + await hass.async_block_till_done() + + assert bool(calls) is acts + sent = mock_client.ask.call_args.args[0] + assert "light.strip" not in str(sent) + assert "Kitchen light strip" not in str(sent["entities"]) + if not acts: + trace = house.runtime_data.conversation_traces[0] + assert trace["reason"] == "named a device that is not exposed" + + async def test_brightness_is_read_from_the_text_not_the_model(hass, house, mock_client): """Jev judges and does not calculate, so the number comes out of a regex.""" mock_client.ask.return_value = build_response( @@ -1370,6 +1403,8 @@ async def test_a_room_past_the_entity_cap_is_not_offered(hass, config_entry): assert [e.entity_id for e in snapshot.entities] == ["light.a", "light.b"] assert snapshot.areas == ["Attic"] + # Past the cap is as good as hidden: the model never saw it to pick it. + assert snapshot.hidden_names == ["c"] # --- asking which device --- @@ -1465,6 +1500,26 @@ async def test_a_shared_name_is_asked_about_when_none_got_most_of_the_answer( ) +async def test_a_hidden_name_is_not_asked_about_as_a_shared_one(hass, house, mock_client): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + hass.states.async_set("light.desk", "off", {"friendly_name": "Desk lamp"}) + async_expose_entity(hass, conversation.DOMAIN, "light.desk", False) + shares = {NONE: 0.55, "light.kitchen": 0.44, "light.office": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + + result = await converse(hass, "turn on the desk lamp") + + assert result.continue_conversation is False + trace = house.runtime_data.conversation_traces[0] + assert trace["reason"] == "named a device that is not exposed" + + async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_client): rename(hass, "light.kitchen", "Lamp") rename(hass, "light.office", "Lamp") From 6b7168fb294735e92c598de23d51ef9c3d55688b Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 11:38:48 +0200 Subject: [PATCH 23/24] Say that the satellite's room settles a shared name --- site-docs/conversation.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 68c6827..a5ff0b8 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -69,7 +69,9 @@ the lamp" is the Lamp and "turn on the desk lamp" is the Desk lamp. When two nam equally well, as "the lamp" does for a Desk lamp and a Floor lamp, the agent asks **"Do you mean Desk lamp or Floor lamp?"** and keeps the conversation open. When the two have the same name, it adds the room: "Lamp (Office) or Lamp (Bedroom)". A room -you name settles it first, so "the lamp in the office" acts. +you name settles it first, so "the lamp in the office" acts. Next comes the room of +the satellite that heard you, as for Home Assistant's own agent: "turn on the lamp" +said to the office satellite turns on the office Lamp. The names decide this, not the model's confidence. On a test instance with two lights both called "Lamp", the model put 1.00 on one of them in one session. In another it put From e47c2155321e6bdbe1afae115a4a9ce445fe31fc Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 11:45:05 +0200 Subject: [PATCH 24/24] Count the 1.16 tree in the quality scale, and 1.16.0 --- custom_components/jev/manifest.json | 2 +- quality_scale.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/custom_components/jev/manifest.json b/custom_components/jev/manifest.json index 439ca71..1a3e2ea 100644 --- a/custom_components/jev/manifest.json +++ b/custom_components/jev/manifest.json @@ -10,5 +10,5 @@ "iot_class": "cloud_polling", "issue_tracker": "https://github.com/AboveColin/HA-Jev/issues", "requirements": ["jevclient==1.2.0"], - "version": "1.15.1" + "version": "1.16.0" } diff --git a/quality_scale.yaml b/quality_scale.yaml index fc7bd97..448028b 100644 --- a/quality_scale.yaml +++ b/quality_scale.yaml @@ -87,7 +87,7 @@ rules: agent all start it when TypeSafe rejects the key. test-coverage: status: done - comment: 97% measured across 407 tests, enforced in CI. + comment: 97% measured across 474 tests, enforced in CI. # Gold devices: done @@ -133,12 +133,12 @@ rules: placeholder. A question sensor is named by the user's own question text, which is theirs to write and cannot be translated by anyone else. The conversation agent and the AI Task entity take the device name, which is what every core - provider of either does, and the ten sentences the agent speaks itself go + provider of either does, and the twelve sentences the agent speaks itself go through the common translation category, so a Dutch pipeline answers in Dutch. exception-translations: status: done comment: >- - Twenty-eight keys, including the library's own limit errors, which are + Thirty-four keys, including the library's own limit errors, which are re-raised with the real numbers as placeholders rather than passed through in English. The coordinator's UpdateFailed and setup's ConfigEntryNotReady and ConfigEntryAuthFailed are translated too, since both reach the integrations @@ -148,7 +148,7 @@ rules: icon-translations: status: done comment: >- - icons.json covers every translated entity and all four actions, and a test + icons.json covers every translated entity and all five actions, and a test asserts that. The conversation entity takes the conversation domain's own icon, as core's agents do. reconfiguration-flow: @@ -176,4 +176,4 @@ rules: Assistant ships aiohttp and httpx, not httpx2. strict-typing: status: done - comment: mypy --strict passes on all 20 modules, enforced in CI. + comment: mypy --strict passes on all 22 modules, enforced in CI.