From 8b241a30d7a34598a50e9bd423a8fe18f878742f Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 22:02:30 +0200 Subject: [PATCH 1/6] Voice asks which device you mean when two fit the name When the action is clear and the entity answer is split between two exposed devices, the agent asks "Do you mean A or B?" and keeps the microphone open. The reply gets one small question about those two devices, and the first command then runs on the one it picked. A reply that picks neither is handled as a new command. The question lasts as long as the chat session. --- custom_components/jev/conversation.py | 191 +++++++++++++++- custom_components/jev/interpret.py | 63 +++++- custom_components/jev/strings.json | 1 + custom_components/jev/translations/cs.json | 1 + custom_components/jev/translations/da.json | 1 + custom_components/jev/translations/de.json | 1 + custom_components/jev/translations/en.json | 1 + custom_components/jev/translations/es.json | 1 + custom_components/jev/translations/fr.json | 1 + custom_components/jev/translations/it.json | 1 + custom_components/jev/translations/nl.json | 1 + custom_components/jev/translations/pl.json | 1 + custom_components/jev/translations/pt-BR.json | 1 + custom_components/jev/translations/ru.json | 1 + custom_components/jev/translations/sv.json | 1 + .../jev/translations/zh-Hans.json | 1 + tests/test_conversation.py | 207 +++++++++++++++++- 17 files changed, 463 insertions(+), 12 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index f56f8a4..7bdc7c0 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -25,8 +25,9 @@ import logging import re from collections.abc import Mapping -from dataclasses import asdict, dataclass -from typing import Literal +from dataclasses import asdict, dataclass, field, replace +from datetime import datetime +from typing import Any, Literal from homeassistant.components import conversation from homeassistant.components.conversation.models import AbstractConversationAgent @@ -37,10 +38,18 @@ from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.helpers import template, translation +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback from homeassistant.util import dt as dt_util from homeassistant.util import language as language_util -from jevclient import JevAuthError, JevError +from jevclient import ( + Choice, + ChoiceAnswer, + JevAuthError, + JevError, + JevResponse, + Question, +) from .const import ( CONF_ALLOW_WHOLE_HOME, @@ -52,9 +61,9 @@ ) from .coordinator import JevRuntimeData from .entity import build_device_info -from .interpret import build_questions, interpret +from .interpret import NONE, Interpretation, build_questions, interpret, spoken_name from .payload import payload_bytes -from .snapshot import async_snapshot +from .snapshot import HomeSnapshot, async_snapshot _LOGGER = logging.getLogger(__name__) @@ -76,11 +85,25 @@ "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", } PARALLEL_UPDATES = 0 +@dataclass(slots=True) +class _Pending: + """A command held while the agent asks which device it meant.""" + + text: str + response: JevResponse + snapshot: HomeSnapshot + candidates: tuple[str, str] + expires: datetime = field( + default_factory=lambda: dt_util.utcnow() + CONVERSATION_TIMEOUT + ) + + async def async_setup_entry( hass: HomeAssistant, entry: ConfigEntry, @@ -101,6 +124,8 @@ def __init__(self, entry: ConfigEntry) -> None: self._attr_unique_id = f"{entry.entry_id}_conversation" runtime: JevRuntimeData = entry.runtime_data self._attr_device_info = build_device_info(entry.entry_id, runtime) + # Conversation id to the command waiting on its reply. + self._pending: dict[str, _Pending] = {} @property def supported_languages(self) -> list[str] | Literal["*"]: @@ -156,12 +181,48 @@ async def _async_handle_message( runtime: JevRuntimeData = self._entry.runtime_data runtime.usage.roll_over(dt_util.now().date()) + if (pending := self._take_pending(chat_log.conversation_id)) is not None: + resolved = await self._resolve(user_input, chat_log, pending) + if resolved is not None: + return resolved + snapshot = async_snapshot(self.hass, MAX_CONVERSATION_ENTITIES) if not snapshot.entities: return await self._fall_back(user_input, "no entities are exposed to Assist") questions = build_questions(user_input.text, snapshot, MAX_CONVERSATION_ENTITIES) state = snapshot.as_state() | {"command": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + decision = interpret(response, user_input.text, snapshot, self._min_confidence) + self._trace( + chat_log, + response, + { + "text": user_input.text, + "exposed_entities": len(snapshot.entities), + **asdict(decision), + }, + ) + + if decision.candidates is not None: + return await self._ask_which( + user_input, + chat_log, + _Pending(user_input.text, response, snapshot, decision.candidates), + ) + return await self._act(user_input, decision, user_input.text) + + async def _ask( + self, + user_input: conversation.ConversationInput, + state: dict[str, Any], + questions: dict[str, Question], + ) -> JevResponse | conversation.ConversationResult: + """One call to Jev, inside the budget. A result means it did not answer.""" + runtime: JevRuntimeData = self._entry.runtime_data # The budget covers voice as well as sensors, because a satellite that # mishears a wake word all night is exactly the runaway it exists to stop. @@ -195,14 +256,20 @@ async def _async_handle_message( runtime.usage.record(response.usage.input_tokens, request_bytes) runtime.model_version = response.model or runtime.model_version runtime.usage.notify() + return response - decision = interpret(response, user_input.text, snapshot, self._min_confidence) + def _trace( + self, + chat_log: conversation.ChatLog, + response: JevResponse, + record: dict[str, Any], + ) -> None: + """Keep what one call decided, for diagnostics and the Assist debug view.""" + runtime: JevRuntimeData = self._entry.runtime_data trace = { - "text": user_input.text, "latency_ms": response.latency_ms, "input_tokens": response.usage.input_tokens, - "exposed_entities": len(snapshot.entities), - **asdict(decision), + **record, } runtime.conversation_traces.appendleft(trace) # The Assist debug view shows this beside the pipeline's own steps. It @@ -222,6 +289,110 @@ async def _async_handle_message( } ) + # --- asking which device --- + + def _take_pending(self, conversation_id: str) -> _Pending | None: + """The command waiting on this conversation's reply, if it is still live. + + A pending command lasts as long as Home Assistant keeps the chat session, + so a reply that arrives in a new session is never read as an answer. + """ + now = dt_util.utcnow() + for key in [k for k, v in self._pending.items() if v.expires < now]: + del self._pending[key] + return self._pending.pop(conversation_id, None) + + async def _ask_which( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult: + """Ask which of two devices the command meant, and keep the command.""" + first, second = (pending.snapshot.by_id(e) for e in pending.candidates) + assert first is not None and second is not None + names = spoken_name(first, second) + assert names is not None + language = user_input.language or self.hass.config.language + text = (await self._lines(language))["which_device"].format( + first=names[0], second=names[1] + ) + self._pending[chat_log.conversation_id] = pending + # In the chat log, the next turn in this conversation carries the question + # it answers, and an LLM fallback agent reads the same history. + chat_log.async_add_assistant_content_without_tools( + conversation.AssistantContent(agent_id=self.entity_id, content=text) + ) + response = ha_intent.IntentResponse(language=user_input.language) + response.async_set_speech(text) + # A satellite opens the microphone again for the answer. + return conversation.ConversationResult( + response=response, + conversation_id=chat_log.conversation_id, + continue_conversation=True, + ) + + async def _resolve( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult | None: + """Run the kept command on the device the reply picked. + + None when the reply picked neither, so the reply is handled as a new + command: "no, the kitchen light" and "never mind, lock up" both are one. + """ + snapshot = pending.snapshot + options: dict[str, Any] = { + entity_id: described.as_option() + for entity_id in pending.candidates + if (described := snapshot.by_id(entity_id)) is not None + } + options[NONE] = "Neither of these, or a different request" + questions: dict[str, Question] = { + "which": Choice("Which device does the reply pick?", options) + } + state = {"command": pending.text, "reply": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + answer = response.answers.get("which") + picked = ( + answer.choice + if isinstance(answer, ChoiceAnswer) + and answer.choice in pending.candidates + and answer.confidence >= self._min_confidence + else None + ) + self._trace( + chat_log, + response, + {"text": user_input.text, "answers_command": pending.text, "picked": picked}, + ) + if picked is None: + return None + + # The first call's answers stand, with the entity question settled. The + # command is read from its own sentence again, so a brightness it named + # still comes from the text. + settled = ChoiceAnswer(choice=picked, probabilities={picked: 1.0}, confidence=1.0) + first = replace( + pending.response, answers=pending.response.answers | {"entity": settled} + ) + decision = interpret(first, pending.text, snapshot, self._min_confidence) + return await self._act(user_input, decision, pending.text) + + # --- acting --- + + async def _act( + self, + user_input: conversation.ConversationInput, + decision: Interpretation, + text: str, + ) -> conversation.ConversationResult: + """Carry out one decision, or say why not.""" if decision.already_satisfied is not None: name, settled = decision.already_satisfied return await self._speak(user_input, f"already_{settled}", name=name) @@ -248,7 +419,7 @@ async def _async_handle_message( DOMAIN, decision.intent_type, decision.slots, - user_input.text, + text, user_input.context, language=user_input.language, assistant=conversation.DOMAIN, diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 4e83f75..feccaab 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -20,7 +20,7 @@ from homeassistant.helpers import intent as ha_intent from jevclient import Choice, ChoiceAnswer, JevResponse, Noul, NoulAnswer, Question -from .snapshot import HomeSnapshot +from .snapshot import ExposedEntity, HomeSnapshot NONE = "none_of_these" @@ -129,6 +129,8 @@ class Interpretation: action_probabilities: dict[str, float] = field(default_factory=dict) # (entity name, the state it is already in) when there is nothing left to do. already_satisfied: tuple[str, str] | None = None + # Two entity ids the command could mean, when the agent should ask which. + candidates: tuple[str, str] | None = None @property def should_fall_back(self) -> bool: @@ -326,6 +328,20 @@ def out(reason: str) -> Interpretation: # unbounded off is not something to infer from one ambiguous sentence. slots["name"] = {"value": "all"} targets_everything = True + elif pair := _two_that_fit(entity, snapshot, min_confidence): + # The action is sure and the device is one of two. Asking costs one short + # question, and handing the sentence to the fallback agent gets the same + # guess made again by something that does not know it was a guess. + return Interpretation( + None, + {}, + action.choice, + action.confidence, + "two devices fit the name", + fallback=False, + action_probabilities=dict(action.probabilities or {}), + candidates=pair, + ) else: return out("no target named with enough confidence") @@ -363,6 +379,51 @@ def out(reason: str) -> Interpretation: ) +# The least share of the entity answer a device needs to be offered as one of two. +# With this, the two named devices hold at least 40% between them and the rest is +# spread across the others. Not measured on a real instance yet. +ASK_BACK_FLOOR = 0.2 + + +def _two_that_fit( + entity: ChoiceAnswer | None, snapshot: HomeSnapshot, min_confidence: float +) -> tuple[str, str] | None: + """The two devices a command could mean, when it is one of them and not a third. + + Both must be exposed, each must hold ASK_BACK_FLOOR of the answer, and the two + together must reach the confidence the agent acts on. A third device above the + floor means the question would not settle it, so the agent does not ask. + """ + if entity is None or not entity.probabilities: + return None + ranked = sorted( + ( + (probability, entity_id) + for entity_id, probability in entity.probabilities.items() + if entity_id != NONE and probability >= ASK_BACK_FLOOR + ), + reverse=True, + ) + if len(ranked) != 2 or ranked[0][0] + ranked[1][0] < min_confidence: + return None + first, second = (snapshot.by_id(entity_id) for _, entity_id in ranked) + if first is None or second is None or spoken_name(first, second) is None: + return None + return first.entity_id, second.entity_id + + +def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | None: + """How to say the two apart: by name, or by room when the names are the same. + + None when neither tells them apart, since "the fan or the fan" asks nothing. + """ + if one.name.casefold() != other.name.casefold(): + return one.name, other.name + if one.area and other.area and one.area.casefold() != other.area.casefold(): + return f"{one.name} ({one.area})", f"{other.name} ({other.area})" + return None + + # What "already done" looks like for each action the check covers. _SETTLED = {"turn_on": "on", "turn_off": "off"} diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..62b062a 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -367,6 +367,7 @@ "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..525226a 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -367,6 +367,7 @@ "budget_spent": "Denní rozpočet tokenů je vyčerpán, takže to dnes udělat nemohu.", "auth_failed": "TypeSafe odmítl klíč API. Zkontrolujte ho v nastavení Jev.", "unavailable": "TypeSafe neodpověděl. Zkuste to za chvíli znovu.", + "which_device": "Myslíte {first}, nebo {second}?", "preview_over_budget": "Denní rozpočet tokenů je vyčerpán, takže zkušební odpověď není. Otázka se přesto uloží.", "preview_nothing_yet": "Žádná zkušební odpověď: zatím není na co se ptát." }, diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..b589629 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -367,6 +367,7 @@ "budget_spent": "Det daglige tokenbudget er brugt, så det kan jeg ikke gøre i dag.", "auth_failed": "TypeSafe afviste API-nøglen. Tjek den i indstillingerne for Jev.", "unavailable": "TypeSafe svarede ikke. Prøv igen om et øjeblik.", + "which_device": "Mener du {first} eller {second}?", "preview_over_budget": "Det daglige token-budget er brugt op, så der er intet prøvesvar. Spørgsmålet bliver gemt alligevel.", "preview_nothing_yet": "Intet prøvesvar: der er endnu intet at spørge om." }, diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..42f16eb 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -367,6 +367,7 @@ "budget_spent": "Das tägliche Token-Budget ist aufgebraucht, deshalb kann ich das heute nicht tun.", "auth_failed": "TypeSafe hat den API-Schlüssel abgelehnt. Prüfe ihn in den Jev-Einstellungen.", "unavailable": "TypeSafe hat nicht geantwortet. Versuche es gleich noch einmal.", + "which_device": "Meinst du {first} oder {second}?", "preview_over_budget": "Das Tagesbudget für Tokens ist aufgebraucht, es gibt also keine Probeantwort. Die Frage wird trotzdem gespeichert.", "preview_nothing_yet": "Keine Probeantwort: Es gibt noch nichts, wonach sich fragen ließe." }, diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..62b062a 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -367,6 +367,7 @@ "budget_spent": "The daily token budget is spent, so I cannot do that today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..be3ef4f 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -367,6 +367,7 @@ "budget_spent": "El presupuesto diario de tokens se ha agotado, así que hoy no puedo hacerlo.", "auth_failed": "TypeSafe rechazó la clave de API. Revísala en los ajustes de Jev.", "unavailable": "TypeSafe no respondió. Inténtalo de nuevo en un momento.", + "which_device": "¿Te refieres a {first} o a {second}?", "preview_over_budget": "El presupuesto diario de tokens está agotado, así que no hay respuesta de prueba. La pregunta sí se guarda.", "preview_nothing_yet": "Sin respuesta de prueba: todavía no hay nada sobre lo que preguntar." }, diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..922dae4 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -367,6 +367,7 @@ "budget_spent": "Le budget quotidien de jetons est épuisé, je ne peux donc pas le faire aujourd'hui.", "auth_failed": "TypeSafe a refusé la clé API. Vérifiez-la dans les paramètres de Jev.", "unavailable": "TypeSafe n'a pas répondu. Réessayez dans un instant.", + "which_device": "Voulez-vous dire {first} ou {second} ?", "preview_over_budget": "Le budget quotidien de jetons est épuisé, il n'y a donc pas de réponse d'essai. La question est quand même enregistrée.", "preview_nothing_yet": "Pas de réponse d'essai : il n'y a encore rien sur quoi poser la question." }, diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..bfcd33c 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -367,6 +367,7 @@ "budget_spent": "Il budget giornaliero di token è esaurito, quindi oggi non posso farlo.", "auth_failed": "TypeSafe ha rifiutato la chiave API. Controllala nelle impostazioni di Jev.", "unavailable": "TypeSafe non ha risposto. Riprova tra un momento.", + "which_device": "Intendi {first} o {second}?", "preview_over_budget": "Il budget giornaliero di token è esaurito, quindi non c'è una risposta di prova. La domanda viene comunque salvata.", "preview_nothing_yet": "Nessuna risposta di prova: non c'è ancora nulla su cui fare la domanda." }, diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..5f378b4 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -367,6 +367,7 @@ "budget_spent": "Het dagelijkse tokenbudget is op, dus dat kan ik vandaag niet doen.", "auth_failed": "TypeSafe heeft de API-sleutel geweigerd. Controleer hem in de instellingen van Jev.", "unavailable": "TypeSafe gaf geen antwoord. Probeer het zo nog eens.", + "which_device": "Bedoel je {first} of {second}?", "preview_over_budget": "Het daglimiet voor tokens is op, dus er is geen proefantwoord. De vraag wordt wel opgeslagen.", "preview_nothing_yet": "Geen proefantwoord: er is nog niets om over te vragen." }, diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..1d21f47 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -367,6 +367,7 @@ "budget_spent": "Dzienny budżet tokenów się wyczerpał, więc dziś nie mogę tego zrobić.", "auth_failed": "TypeSafe odrzucił klucz API. Sprawdź go w ustawieniach Jev.", "unavailable": "TypeSafe nie odpowiedział. Spróbuj ponownie za chwilę.", + "which_device": "Chodzi o {first} czy {second}?", "preview_over_budget": "Dzienny limit tokenów jest wyczerpany, więc nie ma próbnej odpowiedzi. Pytanie i tak zostanie zapisane.", "preview_nothing_yet": "Brak próbnej odpowiedzi: nie ma jeszcze o co pytać." }, diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..ce9669b 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -367,6 +367,7 @@ "budget_spent": "O orçamento diário de tokens acabou, então não posso fazer isso hoje.", "auth_failed": "A TypeSafe rejeitou a chave de API. Verifique-a nas configurações do Jev.", "unavailable": "A TypeSafe não respondeu. Tente novamente em instantes.", + "which_device": "Você quer dizer {first} ou {second}?", "preview_over_budget": "O orçamento diário de tokens acabou, então não há resposta de teste. A pergunta ainda é salva.", "preview_nothing_yet": "Sem resposta de teste: ainda não há nada sobre o que perguntar." }, diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..3235ef6 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -367,6 +367,7 @@ "budget_spent": "Дневной бюджет токенов исчерпан, поэтому сегодня я не могу это сделать.", "auth_failed": "TypeSafe отклонил ключ API. Проверьте его в настройках Jev.", "unavailable": "TypeSafe не ответил. Попробуйте ещё раз чуть позже.", + "which_device": "Вы имеете в виду {first} или {second}?", "preview_over_budget": "Дневной бюджет токенов исчерпан, поэтому пробного ответа нет. Вопрос всё равно сохраняется.", "preview_nothing_yet": "Пробного ответа нет: пока не о чем спрашивать." }, diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..6f9f9c0 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -367,6 +367,7 @@ "budget_spent": "Den dagliga tokenbudgeten är slut, så det kan jag inte göra i dag.", "auth_failed": "TypeSafe avvisade API-nyckeln. Kontrollera den i inställningarna för Jev.", "unavailable": "TypeSafe svarade inte. Försök igen om en stund.", + "which_device": "Menar du {first} eller {second}?", "preview_over_budget": "Den dagliga budgeten för token är slut, så det finns inget provsvar. Frågan sparas ändå.", "preview_nothing_yet": "Inget provsvar: det finns inget att fråga om än." }, diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..562c1c1 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -367,6 +367,7 @@ "budget_spent": "今天的令牌预算已用完，所以今天无法执行。", "auth_failed": "TypeSafe 拒绝了 API 密钥。请在 Jev 设置中检查。", "unavailable": "TypeSafe 没有响应。请稍后再试。", + "which_device": "你是指{first}还是{second}？", "preview_over_budget": "每日令牌预算已用完，因此没有试答结果。问题仍会保存。", "preview_nothing_yet": "没有试答结果：目前还没有可提问的内容。" }, diff --git a/tests/test_conversation.py b/tests/test_conversation.py index f001889..4b8d171 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -6,6 +6,7 @@ """ from dataclasses import replace +from datetime import timedelta from unittest.mock import patch import pytest @@ -16,7 +17,9 @@ from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar from homeassistant.helpers import entity_registry as er +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.setup import async_setup_component +from homeassistant.util import dt as dt_util from jevclient import ChoiceAnswer, NoulAnswer from custom_components.jev.const import ( @@ -26,7 +29,7 @@ CONVERSATION_TRACE_LENGTH, ) from custom_components.jev.conversation import _render_state_answer -from custom_components.jev.interpret import find_brightness +from custom_components.jev.interpret import NONE, find_brightness from custom_components.jev.payload import payload_bytes from .conftest import PROBE_TOKENS, build_response @@ -1318,3 +1321,205 @@ async def test_a_room_past_the_entity_cap_is_not_offered(hass, config_entry): assert [e.entity_id for e in snapshot.entities] == ["light.a", "light.b"] assert snapshot.areas == ["Attic"] + + +# --- asking which device --- + + +def unsure_between(first, second, first_share=0.5, second_share=0.45, **rest): + """An answer set whose entity answer is split between two devices.""" + shares = {first: first_share, second: second_share, NONE: 0.05, **rest} + return answer_set( + entity=ChoiceAnswer( + choice=first, probabilities=shares, confidence=max(shares.values()) + ), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + + +def reply(choice, confidence=0.95): + return {"which": ChoiceAnswer(choice=choice, probabilities={}, confidence=confidence)} + + +async def converse_in(hass, text, conversation_id): + return await conversation.async_converse( + hass, text, conversation_id, Context(), language="en", agent_id=AGENT + ) + + +async def test_two_devices_that_fit_the_name_get_a_question(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "light on") + await hass.async_block_till_done() + + assert calls == [] + assert result.continue_conversation is True + assert result.conversation_id + assert ( + result.response.speech["plain"]["speech"] + == "Do you mean Kitchen light or Office light?" + ) + + +async def test_the_reply_runs_the_first_command_on_the_device_it_picks( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + # The reply was one small question about the two devices, not a new command. + state, questions = mock_client.ask.await_args.args + assert list(questions) == ["which"] + assert state == {"command": "light on", "reply": "the office one"} + assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} + + +async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply(NONE) + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + result = await converse_in(hass, "never mind", asked.conversation_id) + await hass.async_block_till_done() + + # The reply question, then the whole reply as a command of its own, which + # this mock answers with the same split and so asks again. + assert mock_client.ask.await_count == 2 + assert "action" in mock_client.ask.await_args.args[1] + assert result.continue_conversation is True + + +async def test_an_unsure_reply_acts_on_nothing(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), + **reply("light.office", confidence=0.4), + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "hmm", asked.conversation_id) + await hass.async_block_till_done() + + assert calls == [] + + +async def test_a_reply_in_another_conversation_is_not_an_answer(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + await converse(hass, "light on") + mock_client.ask.reset_mock() + + await converse(hass, "the office one") + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_question_left_unanswered_expires_with_the_session( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + later = dt_util.utcnow() + CONVERSATION_TIMEOUT + timedelta(seconds=1) + with patch("custom_components.jev.conversation.dt_util.utcnow", return_value=later): + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_third_device_in_the_running_means_no_question(hass, house, mock_client): + hass.states.async_set("light.hall", "off", {"friendly_name": "Hall light"}) + async_expose_entity(hass, conversation.DOMAIN, "light.hall", True) + mock_client.ask.return_value = build_response( + **unsure_between( + "light.kitchen", "light.office", 0.4, 0.3, **{"light.hall": 0.25} + ) + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_two_devices_with_nothing_to_tell_them_apart_get_no_question( + hass, house, mock_client +): + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + er.async_get(hass).async_update_entity("light.office", area_id=kitchen.id) + hass.states.async_set("light.office", "off", {"friendly_name": "Kitchen light"}) + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_the_same_name_in_two_rooms_is_asked_by_room(hass, house, mock_client): + hass.states.async_set("light.office", "off", {"friendly_name": "Kitchen light"}) + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Kitchen light (Kitchen) or Kitchen light (Office)?" + ) + + +async def test_the_question_is_asked_in_the_pipeline_language(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "lamp aan", language="nl") + + assert ( + result.response.speech["plain"]["speech"] + == "Bedoel je Kitchen light of Office light?" + ) + + +async def test_a_reply_past_the_budget_is_refused_like_a_command( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + # Set on the account, not in the options: an options change reloads the entry, + # and a reloaded agent holds no question to answer. + house.runtime_data.usage.budget = 10 + mock_client.ask.reset_mock() + + result = await converse_in(hass, "the office one", asked.conversation_id) + + assert mock_client.ask.await_count == 0 + assert "budget is spent" in result.response.speech["plain"]["speech"] From 39740f2a17e429222296be8d553b6ad77ced8798 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:31:06 +0200 Subject: [PATCH 2/6] Ask which device from the names, not from the probabilities On a development instance with two lights both called "Lamp", "turn on the lamp" came back as one of them at 1.00, every time. The entity answer never split, so the ask-back never asked, and the agent acted on a guess that looked sure. The candidates now come from the exposed names of the chosen device's kind. The whole name said beats a name that only shares a word with the command, so "the lamp" is Lamp beside a Desk lamp, and "the desk lamp" is the Desk lamp. Two names that fit equally well get the question. More than two fall back. A room the model is sure of narrows the list first, so "the lamp in the office" acts. When no name fits the words at all, the model's answer stands as before. A reply that picks a device reads the command again without the check, so it does not ask the same question twice. --- custom_components/jev/conversation.py | 4 +- custom_components/jev/interpret.py | 127 +++++++++++++++++--------- tests/test_conversation.py | 86 +++++++++++++++-- 3 files changed, 166 insertions(+), 51 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 962232d..530bec2 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -381,7 +381,9 @@ async def _resolve( first = replace( pending.response, answers=pending.response.answers | {"entity": settled} ) - decision = interpret(first, pending.text, snapshot, self._min_confidence) + decision = interpret( + first, pending.text, snapshot, self._min_confidence, ask_back=False + ) return await self._act(user_input, decision, pending.text) # --- acting --- diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index feccaab..6abaa85 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -227,8 +227,13 @@ def interpret( text: str, snapshot: HomeSnapshot, min_confidence: float, + *, + ask_back: bool = True, ) -> Interpretation: - """Read the answers that matter and ignore the rest.""" + """Read the answers that matter and ignore the rest. + + ask_back=False reads a command whose device a reply has already picked. + """ def choice(key: str) -> ChoiceAnswer | None: answer = response.answers.get(key) @@ -293,6 +298,28 @@ def out(reason: str) -> Interpretation: area = choice("area") slots: dict[str, Any] = {} targets_everything = False + named_area = ( + area.choice + if area is not None and area.choice != NONE and area.confidence >= min_confidence + else None + ) + + def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: + # The action is sure and the device is one of two. Asking costs one short + # question, and handing the sentence to the fallback agent gets the same + # guess made again by something that does not know it was a guess. + if spoken_name(first, second) is None: + return out("two devices fit the name and nothing tells them apart") + return Interpretation( + None, + {}, + action.choice, + action.confidence, + "two devices fit the name", + fallback=False, + action_probabilities=dict(action.probabilities or {}), + candidates=(first.entity_id, second.entity_id), + ) # Trust the confident answer rather than the ordering. Measured: a scope answer # of one_room at 0.41 alongside a device answer at 1.00, where branching on @@ -305,6 +332,12 @@ def out(reason: str) -> Interpretation: described = snapshot.by_id(entity.choice) if described is None: return out("named a device that is not exposed") + if ask_back: + tied = _fit_as_well(text, described, snapshot, named_area) + if len(tied) == 2: + return ask(*tied) + if len(tied) > 2: + return out(f"{len(tied)} devices fit the name") slots["name"] = {"value": described.name} # The domain keeps a same-named entity the model was never shown, a lock # called "Front door" beside a cover called "Front door", out of the match. @@ -328,20 +361,13 @@ def out(reason: str) -> Interpretation: # unbounded off is not something to infer from one ambiguous sentence. slots["name"] = {"value": "all"} targets_everything = True - elif pair := _two_that_fit(entity, snapshot, min_confidence): - # The action is sure and the device is one of two. Asking costs one short - # question, and handing the sentence to the fallback agent gets the same - # guess made again by something that does not know it was a guess. - return Interpretation( - None, - {}, - action.choice, - action.confidence, - "two devices fit the name", - fallback=False, - action_probabilities=dict(action.probabilities or {}), - candidates=pair, - ) + elif ( + ask_back + and entity is not None + and (unsure := snapshot.by_id(entity.choice)) is not None + and len(tied := _fit_as_well(text, unsure, snapshot, named_area)) == 2 + ): + return ask(*tied) else: return out("no target named with enough confidence") @@ -379,37 +405,39 @@ def out(reason: str) -> Interpretation: ) -# The least share of the entity answer a device needs to be offered as one of two. -# With this, the two named devices hold at least 40% between them and the rest is -# spread across the others. Not measured on a real instance yet. -ASK_BACK_FLOOR = 0.2 - +def _fit_as_well( + text: str, chosen: ExposedEntity, snapshot: HomeSnapshot, area: str | None +) -> list[ExposedEntity]: + """The devices of the chosen kind whose names fit the words as well as its own. -def _two_that_fit( - entity: ChoiceAnswer | None, snapshot: HomeSnapshot, min_confidence: float -) -> tuple[str, str] | None: - """The two devices a command could mean, when it is one of them and not a third. + The model gives one device all of its answer even when two fit: measured on a + development instance with two lights both called "Lamp", "turn on the lamp" + came back as one of them at 1.00, every time. The probabilities cannot say the + name was shared, the names can. A room the command names narrows the list. - Both must be exposed, each must hold ASK_BACK_FLOOR of the answer, and the two - together must reach the confidence the agent acts on. A third device above the - floor means the question would not settle it, so the agent does not ask. + Only the chosen device when it fits best alone, or when no name fits the words + at all, which is where the model's reading is all there is. """ - if entity is None or not entity.probabilities: - return None - ranked = sorted( - ( - (probability, entity_id) - for entity_id, probability in entity.probabilities.items() - if entity_id != NONE and probability >= ASK_BACK_FLOOR - ), - reverse=True, - ) - if len(ranked) != 2 or ranked[0][0] + ranked[1][0] < min_confidence: - return None - first, second = (snapshot.by_id(entity_id) for _, entity_id in ranked) - if first is None or second is None or spoken_name(first, second) is None: - return None - return first.entity_id, second.entity_id + kind = [ + e + for e in snapshot.entities + if e.domain == chosen.domain and (area is None or e.area == area) + ] + if chosen not in kind: + return [chosen] + fit = {e.entity_id: _name_fit(text, e.name) for e in kind} + best = max(fit.values()) + if best == 0 or fit[chosen.entity_id] < best: + return [chosen] + return [e for e in kind if fit[e.entity_id] == best] + + +def _name_fit(text: str, name: str) -> int: + """How well the words fit a name. The whole name said beats any part of it.""" + if said := _words_said(text, name): + return 100 + said + words = set(re.findall(r"\w+", text.casefold())) + return len(set(name.casefold().split()) & words) def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | None: @@ -424,6 +452,19 @@ def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | N return None +def _words_said(text: str, name: str) -> int: + """How many words of the name the text says, as one phrase, or 0. + + Whole words only, so a hidden "Lamp" is not found inside "lamps". In a language + written without spaces a name rarely stands apart, so the check seldom fires. + """ + words = name.casefold().split() + if not words: + return 0 + phrase = r"\s+".join(re.escape(w) for w in words) + return len(words) if re.search(rf"(? Date: Thu, 24 Sep 2026 08:41:53 +0200 Subject: [PATCH 3/6] Ask about a shared name when the model put most of its answer on none --- custom_components/jev/interpret.py | 15 ++++++++++++++- tests/test_conversation.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 6abaa85..6d74d9d 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -364,7 +364,7 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: elif ( ask_back and entity is not None - and (unsure := snapshot.by_id(entity.choice)) is not None + and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None and len(tied := _fit_as_well(text, unsure, snapshot, named_area)) == 2 ): return ask(*tied) @@ -405,6 +405,19 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: ) +def _likeliest_device(entity: ChoiceAnswer) -> str: + """The device the model gave most of its answer to, even if "none" got more. + + Measured on a development instance with two lights both called "Lamp": "turn on + the lamp" came back as none_of_these 0.55, one Lamp 0.44 and the other 0.01. The + model split its answer because the name was shared, so the name decides. + """ + devices = {k: v for k, v in (entity.probabilities or {}).items() if k != NONE} + if entity.choice != NONE or not devices: + return entity.choice + return max(devices, key=lambda k: devices[k]) + + def _fit_as_well( text: str, chosen: ExposedEntity, snapshot: HomeSnapshot, area: str | None ) -> list[ExposedEntity]: diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 73fddd4..2aba929 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -1439,6 +1439,31 @@ async def test_a_sure_answer_is_still_asked_about_when_the_name_is_shared( ) +async def test_a_shared_name_is_asked_about_when_none_got_most_of_the_answer( + hass, house, mock_client +): + """Measured: none_of_these 0.55, one Lamp 0.44, the other 0.01.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + shares = {NONE: 0.55, "light.kitchen": 0.44, "light.office": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp") + await hass.async_block_till_done() + + assert calls == [] + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_client): rename(hass, "light.kitchen", "Lamp") rename(hass, "light.office", "Lamp") From cf7280ac260d086b87996e4708c5a4ba227e786b Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:52:49 +0200 Subject: [PATCH 4/6] Show a reply's pick in the Assist dialog without failing the reply --- custom_components/jev/conversation.py | 25 ++++++++++++++++--------- tests/test_conversation.py | 22 ++++++++++++++++++++-- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 530bec2..a81282f 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -538,11 +538,17 @@ async def _speak( def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: """The trace as lines a person reads in the Assist dialog.""" - lines = [ - f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " - f"confidence {trace['confidence']:.2f}", - f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", - ] + if "answers_command" in trace: + lines = [ + f'Jev: a reply to "{trace["answers_command"]}", ' + f"picked {trace['picked'] or 'neither'}" + ] + else: + lines = [ + f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " + f"confidence {trace['confidence']:.2f}", + f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", + ] for key, answer in response.answers.items(): if isinstance(answer, ChoiceAnswer): ranked = sorted((answer.probabilities or {}).items(), key=lambda kv: -kv[1])[ @@ -554,10 +560,11 @@ def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: lines.append(f"{key}: {answer.noul:.2f}") else: lines.append(f"{key}: {json.dumps(asdict(answer), ensure_ascii=False)}") - lines.append( - f"{response.model}, {trace['input_tokens']} input tokens, " - f"{trace['latency_ms']:.0f} ms, {trace['exposed_entities']} entities" - ) + footer = f"{response.model}, {trace['input_tokens']} input tokens, " + footer += f"{trace['latency_ms']:.0f} ms" + if "exposed_entities" in trace: + footer += f", {trace['exposed_entities']} entities" + lines.append(footer) return "\n".join(lines) diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 2aba929..78372bd 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -525,11 +525,11 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH -async def converse_in_a_pipeline(hass, text): +async def converse_in_a_pipeline(hass, text, conversation_id=None): """Converse the way assist_pipeline does, with a listener on the chat log.""" deltas = [] with ( - chat_session.async_get_chat_session(hass, None) as session, + chat_session.async_get_chat_session(hass, conversation_id) as session, conversation.async_get_chat_log( hass, session, @@ -1529,6 +1529,24 @@ async def test_the_reply_runs_the_first_command_on_the_device_it_picks( assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} +async def test_the_assist_dialog_shows_what_the_reply_picked(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + hass.services.async_register("light", "turn_on", lambda call: None) + + asked = await converse(hass, "light on") + result, deltas, _ = await converse_in_a_pipeline( + hass, "the office one", asked.conversation_id + ) + + assert result.response.response_type is ha_intent.IntentResponseType.ACTION_DONE + [delta] = deltas + assert delta["thinking_content"].startswith( + 'Jev: a reply to "light on", picked light.office\n' + ) + + async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_client): mock_client.ask.return_value = build_response( **unsure_between("light.kitchen", "light.office"), **reply(NONE) From b3db1af4afd1d2d80487718c610a68db77dc7b8a Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:57:56 +0200 Subject: [PATCH 5/6] A reply that asks for something else is a new command, not a pick --- custom_components/jev/conversation.py | 16 ++++++++++++++- tests/test_conversation.py | 28 ++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index a81282f..43a43f1 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -49,6 +49,7 @@ JevAuthError, JevError, JevResponse, + Noul, NoulAnswer, Question, ) @@ -351,7 +352,18 @@ async def _resolve( } options[NONE] = "Neither of these, or a different request" questions: dict[str, Question] = { - "which": Choice("Which device does the reply pick?", options) + "which": Choice("Which device does the reply pick?", options), + # "Never mind, turn off the lamp in the bedroom" names one of the two, so + # the choice alone picks it and the kept command, turn on, runs on it. + # Measured on a development instance, twelve replies to "turn on the + # lamp": the six that only pick a device scored 0.08 to 0.26 here, and + # the six that ask for something else, that one included, 0.91 to 0.97. + "new_request": Noul( + "Does the reply ask for something of its own, rather than only " + "saying which device the command meant?", + true="The reply is a new instruction, or changes what should happen", + false="The reply only picks a device, however it is phrased", + ), } state = {"command": pending.text, "reply": user_input.text} response = await self._ask(user_input, state, questions) @@ -359,11 +371,13 @@ async def _resolve( return response answer = response.answers.get("which") + new_request = response.answers.get("new_request") picked = ( answer.choice if isinstance(answer, ChoiceAnswer) and answer.choice in pending.candidates and answer.confidence >= self._min_confidence + and not (isinstance(new_request, NoulAnswer) and new_request.noul >= 0.5) else None ) self._trace( diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 78372bd..75439a3 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -1524,7 +1524,7 @@ async def test_the_reply_runs_the_first_command_on_the_device_it_picks( assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] # The reply was one small question about the two devices, not a new command. state, questions = mock_client.ask.await_args.args - assert list(questions) == ["which"] + assert list(questions) == ["which", "new_request"] assert state == {"command": "light on", "reply": "the office one"} assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} @@ -1564,6 +1564,32 @@ async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_cli assert result.continue_conversation is False +async def test_a_reply_that_names_a_device_in_a_new_command_runs_that_command( + hass, house, mock_client +): + """Measured: "never mind, turn off the lamp in the bedroom" picked that lamp.""" + new_request = {"new_request": NoulAnswer(noul=0.96)} + turn_off = answer_set( + action=ChoiceAnswer(choice="turn_off", probabilities={}, confidence=0.98), + entity=ChoiceAnswer(choice="light.office", probabilities={}, confidence=0.97), + ) + mock_client.ask.side_effect = [ + build_response(**unsure_between("light.kitchen", "light.office")), + build_response(**reply("light.office"), **new_request), + build_response(**turn_off), + ] + turned_on, turned_off = [], [] + hass.services.async_register("light", "turn_on", turned_on.append) + hass.services.async_register("light", "turn_off", turned_off.append) + + asked = await converse(hass, "light on") + await converse_in(hass, "no, turn off the office light", asked.conversation_id) + await hass.async_block_till_done() + + assert turned_on == [] + assert [e for c in turned_off for e in c.data["entity_id"]] == ["light.office"] + + async def test_an_unsure_reply_acts_on_nothing(hass, house, mock_client): mock_client.ask.return_value = build_response( **unsure_between("light.kitchen", "light.office"), From 604f44047c5d00663870be62ba547395b22f8fd2 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 11:37:50 +0200 Subject: [PATCH 6/6] Settle a shared name by the room of the satellite that heard it Home Assistant's own agent prefers the satellite's area when a name fits devices in several rooms. Jev asked instead. It now reads the satellite entity's area, then its device's area, the same lookup, and asks only when that room does not hold exactly one of the devices. --- custom_components/jev/conversation.py | 12 +++- custom_components/jev/interpret.py | 49 ++++++++++---- custom_components/jev/snapshot.py | 18 +++++ tests/test_conversation.py | 94 +++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 15 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 43a43f1..66f4f95 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -66,7 +66,7 @@ from .entity import build_device_info from .interpret import NONE, Interpretation, build_questions, interpret, spoken_name from .payload import payload_bytes -from .snapshot import HomeSnapshot, async_snapshot +from .snapshot import HomeSnapshot, async_heard_in, async_snapshot _LOGGER = logging.getLogger(__name__) @@ -201,7 +201,15 @@ async def _async_handle_message( if isinstance(response, conversation.ConversationResult): return response - decision = interpret(response, user_input.text, snapshot, self._min_confidence) + decision = interpret( + response, + user_input.text, + snapshot, + self._min_confidence, + heard_in=async_heard_in( + self.hass, user_input.satellite_id, user_input.device_id + ), + ) self._trace( chat_log, response, diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 6d74d9d..9027d30 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -229,10 +229,12 @@ def interpret( min_confidence: float, *, ask_back: bool = True, + heard_in: str | None = None, ) -> Interpretation: """Read the answers that matter and ignore the rest. ask_back=False reads a command whose device a reply has already picked. + heard_in is the area id of the satellite or device that heard the command. """ def choice(key: str) -> ChoiceAnswer | None: @@ -321,9 +323,27 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: candidates=(first.entity_id, second.entity_id), ) + def pick(chosen: ExposedEntity, *, sure: bool) -> ExposedEntity | Interpretation: + # sure is False when the model put most of its answer on none. Then only a + # name that two devices share is a reason to go on. + tied = _fit_as_well(text, chosen, snapshot, named_area) + if not sure and len(tied) < 2: + return out("no target named with enough confidence") + # A name that fits two devices is settled by the room it was said in, as + # Home Assistant's own agent settles it. A room the command names came first. + here = [e for e in tied if heard_in is not None and e.area_id == heard_in] + if len(tied) > 1 and len(here) == 1: + return here[0] + if len(tied) == 1: + return tied[0] + if len(tied) == 2: + return ask(*tied) + return out(f"{len(tied)} devices fit the name") + # Trust the confident answer rather than the ordering. Measured: a scope answer # of one_room at 0.41 alongside a device answer at 1.00, where branching on # scope first threw away the certain answer and acted on the whole house. + described: ExposedEntity | None = None if ( entity is not None and entity.choice != NONE @@ -333,17 +353,10 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: if described is None: return out("named a device that is not exposed") if ask_back: - tied = _fit_as_well(text, described, snapshot, named_area) - if len(tied) == 2: - return ask(*tied) - if len(tied) > 2: - return out(f"{len(tied)} devices fit the name") - slots["name"] = {"value": described.name} - # The domain keeps a same-named entity the model was never shown, a lock - # called "Front door" beside a cover called "Front door", out of the match. - slots["domain"] = {"value": [described.domain]} - if described.area_id: - slots["preferred_area_id"] = {"value": described.area_id} + picked = pick(described, sure=True) + if isinstance(picked, Interpretation): + return picked + described = picked elif area is not None and area.choice != NONE and area.confidence >= min_confidence: slots["area"] = {"value": area.choice} elif ( @@ -365,12 +378,22 @@ def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: ask_back and entity is not None and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None - and len(tied := _fit_as_well(text, unsure, snapshot, named_area)) == 2 ): - return ask(*tied) + picked = pick(unsure, sure=False) + if isinstance(picked, Interpretation): + return picked + described = picked else: return out("no target named with enough confidence") + if described is not None: + slots["name"] = {"value": described.name} + # The domain keeps a same-named entity the model was never shown, a lock + # called "Front door" beside a cover called "Front door", out of the match. + slots["domain"] = {"value": [described.domain]} + if described.area_id: + slots["preferred_area_id"] = {"value": described.area_id} + # An area always carries a domain. With none, Home Assistant acts on every # exposed entity in the room whatever its domain, so turn_off on a hallway with a # light and a lock unlocked the lock. Without a confident answer, the domains the diff --git a/custom_components/jev/snapshot.py b/custom_components/jev/snapshot.py index e111258..6efdf62 100644 --- a/custom_components/jev/snapshot.py +++ b/custom_components/jev/snapshot.py @@ -102,6 +102,24 @@ def as_state(self) -> dict[str, object]: } +@callback +def async_heard_in( + hass: HomeAssistant, satellite_id: str | None, device_id: str | None +) -> str | None: + """The area id of the satellite or device that heard a command, if it has one. + + The same lookup as Home Assistant's own agent: the satellite entity's area, then + its device's area. + """ + if satellite_id and (entry := er.async_get(hass).async_get(satellite_id)): + if entry.area_id is not None: + return entry.area_id + device_id = entry.device_id + if device_id and (device := dr.async_get(hass).async_get(device_id)): + return device.area_id + return None + + @callback def async_snapshot(hass: HomeAssistant, limit: int) -> HomeSnapshot: """Collect the exposed, controllable entities, newest registry state.""" diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 75439a3..2c5ba3d 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -16,6 +16,7 @@ from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar from homeassistant.helpers import chat_session +from homeassistant.helpers import device_registry as dr from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT @@ -1483,6 +1484,99 @@ async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_clie assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] +def a_satellite_in(hass, house, area_name): + """A voice device placed in the named area, as a satellite is.""" + area = ar.async_get(hass).async_get_area_by_name(area_name) + assert area is not None + devices = dr.async_get(hass) + device = devices.async_get_or_create( + config_entry_id=house.entry_id, identifiers={("test", area_name)} + ) + devices.async_update_device(device.id, area_id=area.id) + return device.id + + +async def test_the_room_a_satellite_is_in_settles_a_shared_name(hass, house, mock_client): + """Home Assistant's own agent prefers the satellite's area, and so does Jev.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Office"), + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + + +async def test_a_satellite_entity_area_settles_a_shared_name_the_model_was_unsure_of( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + entities = er.async_get(hass) + satellite = entities.async_get_or_create("assist_satellite", "test", "kitchen") + entities.async_update_entity(satellite.entity_id, area_id=kitchen.id) + shares = {NONE: 0.55, "light.office": 0.44, "light.kitchen": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + satellite_id=satellite.entity_id, + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.kitchen"] + + +async def test_a_satellite_in_another_room_still_gets_the_question( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + ar.async_get(hass).async_create("Hall") + mock_client.ask.return_value = build_response(**answer_set()) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Hall"), + ) + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + @pytest.mark.parametrize( ("text", "chosen"), [