diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index ea35674..66f4f95 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -26,7 +26,8 @@ import logging import re from collections.abc import Mapping -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field, replace +from datetime import datetime from typing import Any, Literal from homeassistant.components import conversation @@ -38,10 +39,20 @@ from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.helpers import template, translation +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback from homeassistant.util import dt as dt_util from homeassistant.util import language as language_util -from jevclient import ChoiceAnswer, JevAuthError, JevError, JevResponse, NoulAnswer +from jevclient import ( + Choice, + ChoiceAnswer, + JevAuthError, + JevError, + JevResponse, + Noul, + NoulAnswer, + Question, +) from .const import ( CONF_ALLOW_WHOLE_HOME, @@ -53,9 +64,9 @@ ) from .coordinator import JevRuntimeData from .entity import build_device_info -from .interpret import build_questions, interpret +from .interpret import NONE, Interpretation, build_questions, interpret, spoken_name from .payload import payload_bytes -from .snapshot import async_snapshot +from .snapshot import HomeSnapshot, async_heard_in, async_snapshot _LOGGER = logging.getLogger(__name__) @@ -79,11 +90,25 @@ ), "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", } PARALLEL_UPDATES = 0 +@dataclass(slots=True) +class _Pending: + """A command held while the agent asks which device it meant.""" + + text: str + response: JevResponse + snapshot: HomeSnapshot + candidates: tuple[str, str] + expires: datetime = field( + default_factory=lambda: dt_util.utcnow() + CONVERSATION_TIMEOUT + ) + + async def async_setup_entry( hass: HomeAssistant, entry: ConfigEntry, @@ -104,6 +129,8 @@ def __init__(self, entry: ConfigEntry) -> None: self._attr_unique_id = f"{entry.entry_id}_conversation" runtime: JevRuntimeData = entry.runtime_data self._attr_device_info = build_device_info(entry.entry_id, runtime) + # Conversation id to the command waiting on its reply. + self._pending: dict[str, _Pending] = {} @property def supported_languages(self) -> list[str] | Literal["*"]: @@ -159,12 +186,56 @@ async def _async_handle_message( runtime: JevRuntimeData = self._entry.runtime_data runtime.usage.roll_over(dt_util.now().date()) + if (pending := self._take_pending(chat_log.conversation_id)) is not None: + resolved = await self._resolve(user_input, chat_log, pending) + if resolved is not None: + return resolved + snapshot = async_snapshot(self.hass, MAX_CONVERSATION_ENTITIES) if not snapshot.entities: return await self._fall_back(user_input, "no entities are exposed to Assist") questions = build_questions(user_input.text, snapshot, MAX_CONVERSATION_ENTITIES) state = snapshot.as_state() | {"command": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + decision = interpret( + response, + user_input.text, + snapshot, + self._min_confidence, + heard_in=async_heard_in( + self.hass, user_input.satellite_id, user_input.device_id + ), + ) + self._trace( + chat_log, + response, + { + "text": user_input.text, + "exposed_entities": len(snapshot.entities), + **asdict(decision), + }, + ) + + if decision.candidates is not None: + return await self._ask_which( + user_input, + chat_log, + _Pending(user_input.text, response, snapshot, decision.candidates), + ) + return await self._act(user_input, decision, user_input.text) + + async def _ask( + self, + user_input: conversation.ConversationInput, + state: dict[str, Any], + questions: dict[str, Question], + ) -> JevResponse | conversation.ConversationResult: + """One call to Jev, inside the budget. A result means it did not answer.""" + runtime: JevRuntimeData = self._entry.runtime_data # The budget covers voice as well as sensors, because a satellite that # mishears a wake word all night is exactly the runaway it exists to stop. @@ -198,14 +269,20 @@ async def _async_handle_message( runtime.usage.record(response.usage.input_tokens, request_bytes) runtime.model_version = response.model or runtime.model_version runtime.usage.notify() + return response - decision = interpret(response, user_input.text, snapshot, self._min_confidence) + def _trace( + self, + chat_log: conversation.ChatLog, + response: JevResponse, + record: dict[str, Any], + ) -> None: + """Keep what one call decided, for diagnostics and the Assist dialog.""" + runtime: JevRuntimeData = self._entry.runtime_data trace = { - "text": user_input.text, "latency_ms": response.latency_ms, "input_tokens": response.usage.input_tokens, - "exposed_entities": len(snapshot.entities), - **asdict(decision), + **record, } runtime.conversation_traces.appendleft(trace) # The pipeline records a chat log delta as an intent-progress event: the @@ -221,6 +298,125 @@ async def _async_handle_message( {"role": "assistant", "thinking_content": _reasoning(trace, response)}, ) + # --- asking which device --- + + def _take_pending(self, conversation_id: str) -> _Pending | None: + """The command waiting on this conversation's reply, if it is still live. + + A pending command lasts as long as Home Assistant keeps the chat session, + so a reply that arrives in a new session is never read as an answer. + """ + now = dt_util.utcnow() + for key in [k for k, v in self._pending.items() if v.expires < now]: + del self._pending[key] + return self._pending.pop(conversation_id, None) + + async def _ask_which( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult: + """Ask which of two devices the command meant, and keep the command.""" + first, second = (pending.snapshot.by_id(e) for e in pending.candidates) + assert first is not None and second is not None + names = spoken_name(first, second) + assert names is not None + language = user_input.language or self.hass.config.language + text = (await self._lines(language))["which_device"].format( + first=names[0], second=names[1] + ) + self._pending[chat_log.conversation_id] = pending + # In the chat log, the next turn in this conversation carries the question + # it answers, and an LLM fallback agent reads the same history. + chat_log.async_add_assistant_content_without_tools( + conversation.AssistantContent(agent_id=self.entity_id, content=text) + ) + response = ha_intent.IntentResponse(language=user_input.language) + response.async_set_speech(text) + # A satellite opens the microphone again for the answer. + return conversation.ConversationResult( + response=response, + conversation_id=chat_log.conversation_id, + continue_conversation=True, + ) + + async def _resolve( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult | None: + """Run the kept command on the device the reply picked. + + None when the reply picked neither, so the reply is handled as a new + command: "no, the kitchen light" and "never mind, lock up" both are one. + """ + snapshot = pending.snapshot + options: dict[str, Any] = { + entity_id: described.as_option() + for entity_id in pending.candidates + if (described := snapshot.by_id(entity_id)) is not None + } + options[NONE] = "Neither of these, or a different request" + questions: dict[str, Question] = { + "which": Choice("Which device does the reply pick?", options), + # "Never mind, turn off the lamp in the bedroom" names one of the two, so + # the choice alone picks it and the kept command, turn on, runs on it. + # Measured on a development instance, twelve replies to "turn on the + # lamp": the six that only pick a device scored 0.08 to 0.26 here, and + # the six that ask for something else, that one included, 0.91 to 0.97. + "new_request": Noul( + "Does the reply ask for something of its own, rather than only " + "saying which device the command meant?", + true="The reply is a new instruction, or changes what should happen", + false="The reply only picks a device, however it is phrased", + ), + } + state = {"command": pending.text, "reply": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + answer = response.answers.get("which") + new_request = response.answers.get("new_request") + picked = ( + answer.choice + if isinstance(answer, ChoiceAnswer) + and answer.choice in pending.candidates + and answer.confidence >= self._min_confidence + and not (isinstance(new_request, NoulAnswer) and new_request.noul >= 0.5) + else None + ) + self._trace( + chat_log, + response, + {"text": user_input.text, "answers_command": pending.text, "picked": picked}, + ) + if picked is None: + return None + + # The first call's answers stand, with the entity question settled. The + # command is read from its own sentence again, so a brightness it named + # still comes from the text. + settled = ChoiceAnswer(choice=picked, probabilities={picked: 1.0}, confidence=1.0) + first = replace( + pending.response, answers=pending.response.answers | {"entity": settled} + ) + decision = interpret( + first, pending.text, snapshot, self._min_confidence, ask_back=False + ) + return await self._act(user_input, decision, pending.text) + + # --- acting --- + + async def _act( + self, + user_input: conversation.ConversationInput, + decision: Interpretation, + text: str, + ) -> conversation.ConversationResult: + """Carry out one decision, or say why not.""" if decision.already_satisfied is not None: name, settled = decision.already_satisfied return await self._speak(user_input, f"already_{settled}", name=name) @@ -247,7 +443,7 @@ async def _async_handle_message( DOMAIN, decision.intent_type, decision.slots, - user_input.text, + text, user_input.context, language=user_input.language, assistant=conversation.DOMAIN, @@ -364,11 +560,17 @@ async def _speak( def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: """The trace as lines a person reads in the Assist dialog.""" - lines = [ - f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " - f"confidence {trace['confidence']:.2f}", - f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", - ] + if "answers_command" in trace: + lines = [ + f'Jev: a reply to "{trace["answers_command"]}", ' + f"picked {trace['picked'] or 'neither'}" + ] + else: + lines = [ + f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " + f"confidence {trace['confidence']:.2f}", + f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", + ] for key, answer in response.answers.items(): if isinstance(answer, ChoiceAnswer): ranked = sorted((answer.probabilities or {}).items(), key=lambda kv: -kv[1])[ @@ -380,10 +582,11 @@ def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: lines.append(f"{key}: {answer.noul:.2f}") else: lines.append(f"{key}: {json.dumps(asdict(answer), ensure_ascii=False)}") - lines.append( - f"{response.model}, {trace['input_tokens']} input tokens, " - f"{trace['latency_ms']:.0f} ms, {trace['exposed_entities']} entities" - ) + footer = f"{response.model}, {trace['input_tokens']} input tokens, " + footer += f"{trace['latency_ms']:.0f} ms" + if "exposed_entities" in trace: + footer += f", {trace['exposed_entities']} entities" + lines.append(footer) return "\n".join(lines) diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 4e83f75..9027d30 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -20,7 +20,7 @@ from homeassistant.helpers import intent as ha_intent from jevclient import Choice, ChoiceAnswer, JevResponse, Noul, NoulAnswer, Question -from .snapshot import HomeSnapshot +from .snapshot import ExposedEntity, HomeSnapshot NONE = "none_of_these" @@ -129,6 +129,8 @@ class Interpretation: action_probabilities: dict[str, float] = field(default_factory=dict) # (entity name, the state it is already in) when there is nothing left to do. already_satisfied: tuple[str, str] | None = None + # Two entity ids the command could mean, when the agent should ask which. + candidates: tuple[str, str] | None = None @property def should_fall_back(self) -> bool: @@ -225,8 +227,15 @@ def interpret( text: str, snapshot: HomeSnapshot, min_confidence: float, + *, + ask_back: bool = True, + heard_in: str | None = None, ) -> Interpretation: - """Read the answers that matter and ignore the rest.""" + """Read the answers that matter and ignore the rest. + + ask_back=False reads a command whose device a reply has already picked. + heard_in is the area id of the satellite or device that heard the command. + """ def choice(key: str) -> ChoiceAnswer | None: answer = response.answers.get(key) @@ -291,10 +300,50 @@ def out(reason: str) -> Interpretation: area = choice("area") slots: dict[str, Any] = {} targets_everything = False + named_area = ( + area.choice + if area is not None and area.choice != NONE and area.confidence >= min_confidence + else None + ) + + def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: + # The action is sure and the device is one of two. Asking costs one short + # question, and handing the sentence to the fallback agent gets the same + # guess made again by something that does not know it was a guess. + if spoken_name(first, second) is None: + return out("two devices fit the name and nothing tells them apart") + return Interpretation( + None, + {}, + action.choice, + action.confidence, + "two devices fit the name", + fallback=False, + action_probabilities=dict(action.probabilities or {}), + candidates=(first.entity_id, second.entity_id), + ) + + def pick(chosen: ExposedEntity, *, sure: bool) -> ExposedEntity | Interpretation: + # sure is False when the model put most of its answer on none. Then only a + # name that two devices share is a reason to go on. + tied = _fit_as_well(text, chosen, snapshot, named_area) + if not sure and len(tied) < 2: + return out("no target named with enough confidence") + # A name that fits two devices is settled by the room it was said in, as + # Home Assistant's own agent settles it. A room the command names came first. + here = [e for e in tied if heard_in is not None and e.area_id == heard_in] + if len(tied) > 1 and len(here) == 1: + return here[0] + if len(tied) == 1: + return tied[0] + if len(tied) == 2: + return ask(*tied) + return out(f"{len(tied)} devices fit the name") # Trust the confident answer rather than the ordering. Measured: a scope answer # of one_room at 0.41 alongside a device answer at 1.00, where branching on # scope first threw away the certain answer and acted on the whole house. + described: ExposedEntity | None = None if ( entity is not None and entity.choice != NONE @@ -303,12 +352,11 @@ def out(reason: str) -> Interpretation: described = snapshot.by_id(entity.choice) if described is None: return out("named a device that is not exposed") - slots["name"] = {"value": described.name} - # The domain keeps a same-named entity the model was never shown, a lock - # called "Front door" beside a cover called "Front door", out of the match. - slots["domain"] = {"value": [described.domain]} - if described.area_id: - slots["preferred_area_id"] = {"value": described.area_id} + if ask_back: + picked = pick(described, sure=True) + if isinstance(picked, Interpretation): + return picked + described = picked elif area is not None and area.choice != NONE and area.confidence >= min_confidence: slots["area"] = {"value": area.choice} elif ( @@ -326,9 +374,26 @@ def out(reason: str) -> Interpretation: # unbounded off is not something to infer from one ambiguous sentence. slots["name"] = {"value": "all"} targets_everything = True + elif ( + ask_back + and entity is not None + and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None + ): + picked = pick(unsure, sure=False) + if isinstance(picked, Interpretation): + return picked + described = picked else: return out("no target named with enough confidence") + if described is not None: + slots["name"] = {"value": described.name} + # The domain keeps a same-named entity the model was never shown, a lock + # called "Front door" beside a cover called "Front door", out of the match. + slots["domain"] = {"value": [described.domain]} + if described.area_id: + slots["preferred_area_id"] = {"value": described.area_id} + # An area always carries a domain. With none, Home Assistant acts on every # exposed entity in the room whatever its domain, so turn_off on a hallway with a # light and a lock unlocked the lock. Without a confident answer, the domains the @@ -363,6 +428,79 @@ def out(reason: str) -> Interpretation: ) +def _likeliest_device(entity: ChoiceAnswer) -> str: + """The device the model gave most of its answer to, even if "none" got more. + + Measured on a development instance with two lights both called "Lamp": "turn on + the lamp" came back as none_of_these 0.55, one Lamp 0.44 and the other 0.01. The + model split its answer because the name was shared, so the name decides. + """ + devices = {k: v for k, v in (entity.probabilities or {}).items() if k != NONE} + if entity.choice != NONE or not devices: + return entity.choice + return max(devices, key=lambda k: devices[k]) + + +def _fit_as_well( + text: str, chosen: ExposedEntity, snapshot: HomeSnapshot, area: str | None +) -> list[ExposedEntity]: + """The devices of the chosen kind whose names fit the words as well as its own. + + The model gives one device all of its answer even when two fit: measured on a + development instance with two lights both called "Lamp", "turn on the lamp" + came back as one of them at 1.00, every time. The probabilities cannot say the + name was shared, the names can. A room the command names narrows the list. + + Only the chosen device when it fits best alone, or when no name fits the words + at all, which is where the model's reading is all there is. + """ + kind = [ + e + for e in snapshot.entities + if e.domain == chosen.domain and (area is None or e.area == area) + ] + if chosen not in kind: + return [chosen] + fit = {e.entity_id: _name_fit(text, e.name) for e in kind} + best = max(fit.values()) + if best == 0 or fit[chosen.entity_id] < best: + return [chosen] + return [e for e in kind if fit[e.entity_id] == best] + + +def _name_fit(text: str, name: str) -> int: + """How well the words fit a name. The whole name said beats any part of it.""" + if said := _words_said(text, name): + return 100 + said + words = set(re.findall(r"\w+", text.casefold())) + return len(set(name.casefold().split()) & words) + + +def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | None: + """How to say the two apart: by name, or by room when the names are the same. + + None when neither tells them apart, since "the fan or the fan" asks nothing. + """ + if one.name.casefold() != other.name.casefold(): + return one.name, other.name + if one.area and other.area and one.area.casefold() != other.area.casefold(): + return f"{one.name} ({one.area})", f"{other.name} ({other.area})" + return None + + +def _words_said(text: str, name: str) -> int: + """How many words of the name the text says, as one phrase, or 0. + + Whole words only, so a hidden "Lamp" is not found inside "lamps". In a language + written without spaces a name rarely stands apart, so the check seldom fires. + """ + words = name.casefold().split() + if not words: + return 0 + phrase = r"\s+".join(re.escape(w) for w in words) + return len(words) if re.search(rf"(? dict[str, object]: } +@callback +def async_heard_in( + hass: HomeAssistant, satellite_id: str | None, device_id: str | None +) -> str | None: + """The area id of the satellite or device that heard a command, if it has one. + + The same lookup as Home Assistant's own agent: the satellite entity's area, then + its device's area. + """ + if satellite_id and (entry := er.async_get(hass).async_get(satellite_id)): + if entry.area_id is not None: + return entry.area_id + device_id = entry.device_id + if device_id and (device := dr.async_get(hass).async_get(device_id)): + return device.area_id + return None + + @callback def async_snapshot(hass: HomeAssistant, limit: int) -> HomeSnapshot: """Collect the exposed, controllable entities, newest registry state.""" diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 1502dc0..f8a8f60 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -367,6 +367,7 @@ "budget_spent": "Not enough of the daily token budget is left for that, so I cannot do it today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index b9261e9..c533e97 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -367,6 +367,7 @@ "budget_spent": "Z denního rozpočtu tokenů na to nezbývá dost, takže to dnes udělat nemohu.", "auth_failed": "TypeSafe odmítl klíč API. Zkontrolujte ho v nastavení Jev.", "unavailable": "TypeSafe neodpověděl. Zkuste to za chvíli znovu.", + "which_device": "Myslíte {first}, nebo {second}?", "preview_over_budget": "Denní rozpočet tokenů je vyčerpán, takže zkušební odpověď není. Otázka se přesto uloží.", "preview_nothing_yet": "Žádná zkušební odpověď: zatím není na co se ptát." }, diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index 162a8fb..a819dda 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -367,6 +367,7 @@ "budget_spent": "Der er ikke nok tilbage af det daglige tokenbudget til det, så det kan jeg ikke gøre i dag.", "auth_failed": "TypeSafe afviste API-nøglen. Tjek den i indstillingerne for Jev.", "unavailable": "TypeSafe svarede ikke. Prøv igen om et øjeblik.", + "which_device": "Mener du {first} eller {second}?", "preview_over_budget": "Det daglige token-budget er brugt op, så der er intet prøvesvar. Spørgsmålet bliver gemt alligevel.", "preview_nothing_yet": "Intet prøvesvar: der er endnu intet at spørge om." }, diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 69fbe6c..42b4c84 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -367,6 +367,7 @@ "budget_spent": "Vom täglichen Token-Budget ist dafür nicht mehr genug übrig, deshalb kann ich das heute nicht tun.", "auth_failed": "TypeSafe hat den API-Schlüssel abgelehnt. Prüfe ihn in den Jev-Einstellungen.", "unavailable": "TypeSafe hat nicht geantwortet. Versuche es gleich noch einmal.", + "which_device": "Meinst du {first} oder {second}?", "preview_over_budget": "Das Tagesbudget für Tokens ist aufgebraucht, es gibt also keine Probeantwort. Die Frage wird trotzdem gespeichert.", "preview_nothing_yet": "Keine Probeantwort: Es gibt noch nichts, wonach sich fragen ließe." }, diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 1502dc0..f8a8f60 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -367,6 +367,7 @@ "budget_spent": "Not enough of the daily token budget is left for that, so I cannot do it today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index 731deeb..c989e5e 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -367,6 +367,7 @@ "budget_spent": "No queda suficiente presupuesto diario de tokens para eso, así que hoy no puedo hacerlo.", "auth_failed": "TypeSafe rechazó la clave de API. Revísala en los ajustes de Jev.", "unavailable": "TypeSafe no respondió. Inténtalo de nuevo en un momento.", + "which_device": "¿Te refieres a {first} o a {second}?", "preview_over_budget": "El presupuesto diario de tokens está agotado, así que no hay respuesta de prueba. La pregunta sí se guarda.", "preview_nothing_yet": "Sin respuesta de prueba: todavía no hay nada sobre lo que preguntar." }, diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index 96b7c78..b961887 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -367,6 +367,7 @@ "budget_spent": "Il ne reste pas assez de budget quotidien de jetons pour cela, je ne peux donc pas le faire aujourd'hui.", "auth_failed": "TypeSafe a refusé la clé API. Vérifiez-la dans les paramètres de Jev.", "unavailable": "TypeSafe n'a pas répondu. Réessayez dans un instant.", + "which_device": "Voulez-vous dire {first} ou {second} ?", "preview_over_budget": "Le budget quotidien de jetons est épuisé, il n'y a donc pas de réponse d'essai. La question est quand même enregistrée.", "preview_nothing_yet": "Pas de réponse d'essai : il n'y a encore rien sur quoi poser la question." }, diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index dbbc17d..3beed89 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -367,6 +367,7 @@ "budget_spent": "Non resta abbastanza budget giornaliero di token per questo, quindi oggi non posso farlo.", "auth_failed": "TypeSafe ha rifiutato la chiave API. Controllala nelle impostazioni di Jev.", "unavailable": "TypeSafe non ha risposto. Riprova tra un momento.", + "which_device": "Intendi {first} o {second}?", "preview_over_budget": "Il budget giornaliero di token è esaurito, quindi non c'è una risposta di prova. La domanda viene comunque salvata.", "preview_nothing_yet": "Nessuna risposta di prova: non c'è ancora nulla su cui fare la domanda." }, diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 3a937ab..5bb0a66 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -367,6 +367,7 @@ "budget_spent": "Er is niet genoeg over van het dagelijkse tokenbudget, dus dat kan ik vandaag niet doen.", "auth_failed": "TypeSafe heeft de API-sleutel geweigerd. Controleer hem in de instellingen van Jev.", "unavailable": "TypeSafe gaf geen antwoord. Probeer het zo nog eens.", + "which_device": "Bedoel je {first} of {second}?", "preview_over_budget": "Het daglimiet voor tokens is op, dus er is geen proefantwoord. De vraag wordt wel opgeslagen.", "preview_nothing_yet": "Geen proefantwoord: er is nog niets om over te vragen." }, diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index 2ad1ed1..9275006 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -367,6 +367,7 @@ "budget_spent": "W dziennym budżecie tokenów nie zostało na to dość, więc dziś nie mogę tego zrobić.", "auth_failed": "TypeSafe odrzucił klucz API. Sprawdź go w ustawieniach Jev.", "unavailable": "TypeSafe nie odpowiedział. Spróbuj ponownie za chwilę.", + "which_device": "Chodzi o {first} czy {second}?", "preview_over_budget": "Dzienny limit tokenów jest wyczerpany, więc nie ma próbnej odpowiedzi. Pytanie i tak zostanie zapisane.", "preview_nothing_yet": "Brak próbnej odpowiedzi: nie ma jeszcze o co pytać." }, diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index dab4a50..925dff2 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -367,6 +367,7 @@ "budget_spent": "Não resta orçamento diário de tokens suficiente para isso, então não posso fazer isso hoje.", "auth_failed": "A TypeSafe rejeitou a chave de API. Verifique-a nas configurações do Jev.", "unavailable": "A TypeSafe não respondeu. Tente novamente em instantes.", + "which_device": "Você quer dizer {first} ou {second}?", "preview_over_budget": "O orçamento diário de tokens acabou, então não há resposta de teste. A pergunta ainda é salva.", "preview_nothing_yet": "Sem resposta de teste: ainda não há nada sobre o que perguntar." }, diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 9742937..da6a902 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -367,6 +367,7 @@ "budget_spent": "В дневном бюджете токенов на это не хватает, поэтому сегодня я не могу это сделать.", "auth_failed": "TypeSafe отклонил ключ API. Проверьте его в настройках Jev.", "unavailable": "TypeSafe не ответил. Попробуйте ещё раз чуть позже.", + "which_device": "Вы имеете в виду {first} или {second}?", "preview_over_budget": "Дневной бюджет токенов исчерпан, поэтому пробного ответа нет. Вопрос всё равно сохраняется.", "preview_nothing_yet": "Пробного ответа нет: пока не о чем спрашивать." }, diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index 5f41075..e405633 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -367,6 +367,7 @@ "budget_spent": "Det finns inte tillräckligt kvar av den dagliga tokenbudgeten för det, så det kan jag inte göra i dag.", "auth_failed": "TypeSafe avvisade API-nyckeln. Kontrollera den i inställningarna för Jev.", "unavailable": "TypeSafe svarade inte. Försök igen om en stund.", + "which_device": "Menar du {first} eller {second}?", "preview_over_budget": "Den dagliga budgeten för token är slut, så det finns inget provsvar. Frågan sparas ändå.", "preview_nothing_yet": "Inget provsvar: det finns inget att fråga om än." }, diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 1490080..3e5eb3d 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -367,6 +367,7 @@ "budget_spent": "今天剩余的令牌预算不足,所以无法执行。", "auth_failed": "TypeSafe 拒绝了 API 密钥。请在 Jev 设置中检查。", "unavailable": "TypeSafe 没有响应。请稍后再试。", + "which_device": "你是指{first}还是{second}?", "preview_over_budget": "每日令牌预算已用完,因此没有试答结果。问题仍会保存。", "preview_nothing_yet": "没有试答结果:目前还没有可提问的内容。" }, diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 89ac0dc..2c5ba3d 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -6,6 +6,7 @@ """ from dataclasses import replace +from datetime import timedelta from unittest.mock import patch import pytest @@ -15,9 +16,12 @@ from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar from homeassistant.helpers import chat_session +from homeassistant.helpers import device_registry as dr from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.setup import async_setup_component +from homeassistant.util import dt as dt_util from jevclient import ChoiceAnswer, NoulAnswer, Usage from custom_components.jev.const import ( @@ -27,7 +31,7 @@ CONVERSATION_TRACE_LENGTH, ) from custom_components.jev.conversation import _render_state_answer -from custom_components.jev.interpret import find_brightness +from custom_components.jev.interpret import NONE, find_brightness from custom_components.jev.payload import payload_bytes from .conftest import PROBE_TOKENS, build_response @@ -522,11 +526,11 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH -async def converse_in_a_pipeline(hass, text): +async def converse_in_a_pipeline(hass, text, conversation_id=None): """Converse the way assist_pipeline does, with a listener on the chat log.""" deltas = [] with ( - chat_session.async_get_chat_session(hass, None) as session, + chat_session.async_get_chat_session(hass, conversation_id) as session, conversation.async_get_chat_log( hass, session, @@ -1310,7 +1314,8 @@ async def test_two_devices_with_one_name_resolve_by_room(hass, house, mock_clien **answer_set( entity=ChoiceAnswer( choice="light.office_ceiling", probabilities={}, confidence=1.0 - ) + ), + area=ChoiceAnswer(choice="Office", probabilities={}, confidence=0.95), ) ) calls = [] @@ -1365,3 +1370,438 @@ async def test_a_room_past_the_entity_cap_is_not_offered(hass, config_entry): assert [e.entity_id for e in snapshot.entities] == ["light.a", "light.b"] assert snapshot.areas == ["Attic"] + + +# --- asking which device --- + + +def unsure_between(first, second, first_share=0.5, second_share=0.45, **rest): + """An answer set whose entity answer is split between two devices.""" + shares = {first: first_share, second: second_share, NONE: 0.05, **rest} + return answer_set( + entity=ChoiceAnswer( + choice=first, probabilities=shares, confidence=max(shares.values()) + ), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + + +def rename(hass, entity_id, name): + """Rename in the registry too, which is where the intent layer matches names.""" + er.async_get(hass).async_update_entity(entity_id, name=name) + hass.states.async_set(entity_id, "off", {"friendly_name": name}) + + +def reply(choice, confidence=0.95): + return {"which": ChoiceAnswer(choice=choice, probabilities={}, confidence=confidence)} + + +async def converse_in(hass, text, conversation_id): + return await conversation.async_converse( + hass, text, conversation_id, Context(), language="en", agent_id=AGENT + ) + + +async def test_two_devices_that_fit_the_name_get_a_question(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "light on") + await hass.async_block_till_done() + + assert calls == [] + assert result.continue_conversation is True + assert result.conversation_id + assert ( + result.response.speech["plain"]["speech"] + == "Do you mean Kitchen light or Office light?" + ) + + +async def test_a_sure_answer_is_still_asked_about_when_the_name_is_shared( + hass, house, mock_client +): + """Measured: two lights called "Lamp", and the model gave one of them 1.00.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp") + await hass.async_block_till_done() + + assert calls == [] + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + +async def test_a_shared_name_is_asked_about_when_none_got_most_of_the_answer( + hass, house, mock_client +): + """Measured: none_of_these 0.55, one Lamp 0.44, the other 0.01.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + shares = {NONE: 0.55, "light.kitchen": 0.44, "light.office": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp") + await hass.async_block_till_done() + + assert calls == [] + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + +async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_client): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice="light.office", probabilities={}, confidence=1.0), + area=ChoiceAnswer(choice="Office", probabilities={}, confidence=0.99), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp in the office") + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + + +def a_satellite_in(hass, house, area_name): + """A voice device placed in the named area, as a satellite is.""" + area = ar.async_get(hass).async_get_area_by_name(area_name) + assert area is not None + devices = dr.async_get(hass) + device = devices.async_get_or_create( + config_entry_id=house.entry_id, identifiers={("test", area_name)} + ) + devices.async_update_device(device.id, area_id=area.id) + return device.id + + +async def test_the_room_a_satellite_is_in_settles_a_shared_name(hass, house, mock_client): + """Home Assistant's own agent prefers the satellite's area, and so does Jev.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Office"), + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + + +async def test_a_satellite_entity_area_settles_a_shared_name_the_model_was_unsure_of( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + entities = er.async_get(hass) + satellite = entities.async_get_or_create("assist_satellite", "test", "kitchen") + entities.async_update_entity(satellite.entity_id, area_id=kitchen.id) + shares = {NONE: 0.55, "light.office": 0.44, "light.kitchen": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + satellite_id=satellite.entity_id, + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.kitchen"] + + +async def test_a_satellite_in_another_room_still_gets_the_question( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + ar.async_get(hass).async_create("Hall") + mock_client.ask.return_value = build_response(**answer_set()) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Hall"), + ) + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + +@pytest.mark.parametrize( + ("text", "chosen"), + [ + # The whole name said beats a name that only shares a word with it. + ("turn on the lamp", "light.kitchen"), + ("turn on the desk lamp", "light.office"), + ], +) +async def test_the_whole_name_said_is_the_device_meant( + hass, house, mock_client, text, chosen +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Desk lamp") + mock_client.ask.return_value = build_response( + **answer_set(entity=ChoiceAnswer(choice=chosen, probabilities={}, confidence=1.0)) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + await converse(hass, text) + await hass.async_block_till_done() + + assert [e for c in calls for e in c.data["entity_id"]] == [chosen] + + +async def test_the_reply_runs_the_first_command_on_the_device_it_picks( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + # The reply was one small question about the two devices, not a new command. + state, questions = mock_client.ask.await_args.args + assert list(questions) == ["which", "new_request"] + assert state == {"command": "light on", "reply": "the office one"} + assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} + + +async def test_the_assist_dialog_shows_what_the_reply_picked(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + hass.services.async_register("light", "turn_on", lambda call: None) + + asked = await converse(hass, "light on") + result, deltas, _ = await converse_in_a_pipeline( + hass, "the office one", asked.conversation_id + ) + + assert result.response.response_type is ha_intent.IntentResponseType.ACTION_DONE + [delta] = deltas + assert delta["thinking_content"].startswith( + 'Jev: a reply to "light on", picked light.office\n' + ) + + +async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply(NONE) + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + result = await converse_in(hass, "never mind", asked.conversation_id) + await hass.async_block_till_done() + + # The reply question, then the whole reply as a command of its own. It fits + # no device's name, so it is not asked about again. + assert mock_client.ask.await_count == 2 + assert "action" in mock_client.ask.await_args.args[1] + assert result.continue_conversation is False + + +async def test_a_reply_that_names_a_device_in_a_new_command_runs_that_command( + hass, house, mock_client +): + """Measured: "never mind, turn off the lamp in the bedroom" picked that lamp.""" + new_request = {"new_request": NoulAnswer(noul=0.96)} + turn_off = answer_set( + action=ChoiceAnswer(choice="turn_off", probabilities={}, confidence=0.98), + entity=ChoiceAnswer(choice="light.office", probabilities={}, confidence=0.97), + ) + mock_client.ask.side_effect = [ + build_response(**unsure_between("light.kitchen", "light.office")), + build_response(**reply("light.office"), **new_request), + build_response(**turn_off), + ] + turned_on, turned_off = [], [] + hass.services.async_register("light", "turn_on", turned_on.append) + hass.services.async_register("light", "turn_off", turned_off.append) + + asked = await converse(hass, "light on") + await converse_in(hass, "no, turn off the office light", asked.conversation_id) + await hass.async_block_till_done() + + assert turned_on == [] + assert [e for c in turned_off for e in c.data["entity_id"]] == ["light.office"] + + +async def test_an_unsure_reply_acts_on_nothing(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), + **reply("light.office", confidence=0.4), + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "hmm", asked.conversation_id) + await hass.async_block_till_done() + + assert calls == [] + + +async def test_a_reply_in_another_conversation_is_not_an_answer(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + await converse(hass, "light on") + mock_client.ask.reset_mock() + + await converse(hass, "the office one") + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_question_left_unanswered_expires_with_the_session( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + later = dt_util.utcnow() + CONVERSATION_TIMEOUT + timedelta(seconds=1) + with patch("custom_components.jev.conversation.dt_util.utcnow", return_value=later): + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_third_device_in_the_running_means_no_question(hass, house, mock_client): + hass.states.async_set("light.hall", "off", {"friendly_name": "Hall light"}) + async_expose_entity(hass, conversation.DOMAIN, "light.hall", True) + mock_client.ask.return_value = build_response( + **unsure_between( + "light.kitchen", "light.office", 0.4, 0.3, **{"light.hall": 0.25} + ) + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_two_devices_with_nothing_to_tell_them_apart_get_no_question( + hass, house, mock_client +): + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + er.async_get(hass).async_update_entity("light.office", area_id=kitchen.id) + rename(hass, "light.office", "Kitchen light") + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_the_same_name_in_two_rooms_is_asked_by_room(hass, house, mock_client): + rename(hass, "light.office", "Kitchen light") + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Kitchen light (Kitchen) or Kitchen light (Office)?" + ) + + +async def test_the_question_is_asked_in_the_pipeline_language(hass, house, mock_client): + rename(hass, "light.kitchen", "Lamp keuken") + rename(hass, "light.office", "Lamp kantoor") + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "lamp aan", language="nl") + + assert ( + result.response.speech["plain"]["speech"] + == "Bedoel je Lamp keuken of Lamp kantoor?" + ) + + +async def test_a_reply_past_the_budget_is_refused_like_a_command( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + # Set on the account, not in the options: an options change reloads the entry, + # and a reloaded agent holds no question to answer. + house.runtime_data.usage.budget = 10 + mock_client.ask.reset_mock() + + result = await converse_in(hass, "the office one", asked.conversation_id) + + assert mock_client.ask.await_count == 0 + assert "budget is left" in result.response.speech["plain"]["speech"]