diff --git a/custom_components/jev/calibrate.py b/custom_components/jev/calibrate.py new file mode 100644 index 0000000..07d2bc4 --- /dev/null +++ b/custom_components/jev/calibrate.py @@ -0,0 +1,258 @@ +"""Pick a threshold from what actually happened, instead of guessing one. + +A noul is a probability, and the threshold that turns it into yes or no is the +user's call. 0.5 is where a noul says "cannot tell", not where a given house's +washing machine is done. This action reads the recorder: the noul's history next to +the history of an entity that says what was really true, such as a door contact or +a smart plug's own "running" state. For every threshold it measures how much of the +time a yes was right (precision) and how much of the true time it said yes +(recall), and returns the threshold that balances the two best. + +Nothing is sent to TypeSafe, so it costs no tokens. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from statistics import median +from typing import Any, Final + +import voluptuous as vol +from homeassistant.components.recorder import history +from homeassistant.const import ATTR_ENTITY_ID, STATE_ON +from homeassistant.core import HomeAssistant, ServiceCall, ServiceResponse, State +from homeassistant.exceptions import ServiceValidationError +from homeassistant.helpers import config_validation as cv +from homeassistant.helpers.recorder import get_instance +from homeassistant.util import dt as dt_util + +from .const import DOMAIN + +CONF_TRUTH_ENTITY: Final = "truth_entity_id" +CONF_TRUTH_STATE: Final = "truth_state" +CONF_DAYS: Final = "days" + +# The recorder keeps 10 days by default (`purge_keep_days`), so a week is inside +# what a default install still has. A longer window only reads further back. +DEFAULT_DAYS: Final = 7 + +# Every hundredth. The noul sensor itself rounds to three places, but nobody sets a +# threshold that fine, and two neighbouring hundredths rarely differ in outcome. +CANDIDATES: Final = tuple(round(i / 100, 2) for i in range(1, 100)) + +# The coarse table returned next to the best threshold, so a caller can see how +# steep the trade-off is around it. +TABLE: Final = tuple(round(i / 10, 1) for i in range(1, 10)) + +CALIBRATE_SCHEMA: Final = vol.Schema( + { + vol.Required(ATTR_ENTITY_ID): cv.entity_id, + vol.Required(CONF_TRUTH_ENTITY): cv.entity_id, + vol.Optional(CONF_TRUTH_STATE, default=STATE_ON): cv.string, + vol.Optional(CONF_DAYS, default=DEFAULT_DAYS): vol.All( + vol.Coerce(int), vol.Range(min=1) + ), + } +) + + +@dataclass(frozen=True, slots=True) +class Span: + """A stretch of time where neither the probability nor the truth changed.""" + + probability: float + truth: bool + seconds: float + + +@dataclass(frozen=True, slots=True) +class Outcome: + """What one threshold would have said over the spans, in seconds.""" + + threshold: float + true_yes: float + false_yes: float + missed: float + + @property + def precision(self) -> float | None: + said_yes = self.true_yes + self.false_yes + return self.true_yes / said_yes if said_yes else None + + @property + def recall(self) -> float | None: + was_true = self.true_yes + self.missed + return self.true_yes / was_true if was_true else None + + @property + def f1(self) -> float: + precision, recall = self.precision, self.recall + if not precision or not recall: + return 0.0 + return 2 * precision * recall / (precision + recall) + + def as_dict(self) -> dict[str, Any]: + return { + "threshold": self.threshold, + "precision": _round(self.precision), + "recall": _round(self.recall), + "f1": round(self.f1, 3), + } + + +def _round(value: float | None) -> float | None: + return None if value is None else round(value, 3) + + +def outcome(spans: list[Span], threshold: float) -> Outcome: + true_yes = false_yes = missed = 0.0 + for span in spans: + said_yes = span.probability >= threshold + if said_yes and span.truth: + true_yes += span.seconds + elif said_yes: + false_yes += span.seconds + elif span.truth: + missed += span.seconds + return Outcome(threshold, true_yes, false_yes, missed) + + +def best(spans: list[Span]) -> Outcome: + """The threshold with the highest F1. + + Several neighbouring thresholds often tie, because no probability fell between + them. The middle of the tied ones is returned rather than an edge, so a noul + that lands a little off its usual values still falls on the same side. + """ + outcomes = [outcome(spans, threshold) for threshold in CANDIDATES] + top = max(o.f1 for o in outcomes) + tied = [o.threshold for o in outcomes if o.f1 == top] + return outcome(spans, round(median(tied), 2)) + + +def build_spans( + probabilities: list[State], + truths: list[State], + truth_state: str, + start: datetime, + end: datetime, +) -> list[Span]: + """Cut the window at every change of either entity. + + The state an entity already had when the window opened counts from the start of + the window. A span where the probability is not a number, such as unavailable + while the API was down, is left out rather than guessed. + """ + changes = sorted( + [(max(s.last_changed, start), "p", s.state) for s in probabilities] + + [(max(s.last_changed, start), "t", s.state) for s in truths], + key=lambda change: change[0], + ) + spans: list[Span] = [] + probability: float | None = None + truth: bool | None = None + for index, (when, kind, value) in enumerate(changes): + if kind == "p": + probability = _number(value) + else: + truth = value == truth_state + until = changes[index + 1][0] if index + 1 < len(changes) else end + seconds = (until - when).total_seconds() + if probability is not None and truth is not None and seconds > 0: + spans.append(Span(probability, truth, seconds)) + return spans + + +def _number(value: str) -> float | None: + try: + return float(value) + except ValueError: + return None + + +async def async_calibrate(hass: HomeAssistant, call: ServiceCall) -> ServiceResponse: + source = call.data[ATTR_ENTITY_ID] + truth_entity = call.data[CONF_TRUTH_ENTITY] + truth_state = call.data[CONF_TRUTH_STATE] + days = call.data[CONF_DAYS] + if "recorder" not in hass.config.components: + raise ServiceValidationError( + translation_domain=DOMAIN, translation_key="calibrate_needs_recorder" + ) + end = dt_util.utcnow() + start = end - timedelta(days=days) + found = await get_instance(hass).async_add_executor_job( + _history, hass, start, end, [source, truth_entity] + ) + spans = build_spans( + found.get(source, []), found.get(truth_entity, []), truth_state, start, end + ) + if not spans: + raise ServiceValidationError( + translation_domain=DOMAIN, + translation_key="calibrate_no_history", + translation_placeholders={ + "entity": source, + "truth": truth_entity, + "days": str(days), + }, + ) + true_seconds = sum(span.seconds for span in spans if span.truth) + # With only one side there is nothing to separate, and every threshold scores + # the same. + if true_seconds in (0, sum(span.seconds for span in spans)): + raise ServiceValidationError( + translation_domain=DOMAIN, + translation_key=( + "calibrate_never_true" if true_seconds == 0 else "calibrate_always_true" + ), + translation_placeholders={ + "truth": truth_entity, + "state": truth_state, + "days": str(days), + }, + ) + return { + **best(spans).as_dict(), + # Two places, so a door open for two minutes reads 0.03 and not 0.0, which + # looked like the never-true refusal had failed to fire. + "hours": round(sum(span.seconds for span in spans) / 3600, 2), + "hours_true": round(true_seconds / 3600, 2), + # F1 over two occasions is noise, whatever its value. These are the counts + # to judge it by. + "times_true": _times_true(spans), + "probability_changes": sum( + 1 for s in found.get(source, []) if _number(s.state) is not None + ), + "table": [outcome(spans, threshold).as_dict() for threshold in TABLE], + } + + +def _times_true(spans: list[Span]) -> int: + """How many separate stretches the truth was on for.""" + count = 0 + previous = False + for span in spans: + if span.truth and not previous: + count += 1 + previous = span.truth + return count + + +def _history( + hass: HomeAssistant, start: datetime, end: datetime, entity_ids: list[str] +) -> dict[str, list[State]]: + found = history.get_significant_states( + hass, + start, + end, + entity_ids, + include_start_time_state=True, + significant_changes_only=False, + no_attributes=True, + ) + return { + entity_id: [s for s in states if isinstance(s, State)] + for entity_id, states in found.items() + } diff --git a/custom_components/jev/config_flow.py b/custom_components/jev/config_flow.py index 9f98fce..17df10a 100644 --- a/custom_components/jev/config_flow.py +++ b/custom_components/jev/config_flow.py @@ -42,6 +42,7 @@ CONF_ALLOW_WHOLE_HOME, CONF_DAILY_TOKEN_BUDGET, CONF_FALLBACK_AGENT, + CONF_LLM_TOOLS, CONF_MIN_CONFIDENCE, CONF_MODEL, CONF_PRICE_PER_MILLION, @@ -399,6 +400,10 @@ async def async_step_init( CONF_ALLOW_WHOLE_HOME, default=options.get(CONF_ALLOW_WHOLE_HOME, False), ): bool, + vol.Optional( + CONF_LLM_TOOLS, + default=options.get(CONF_LLM_TOOLS, False), + ): bool, } ) return self.async_show_form(step_id="init", data_schema=schema) diff --git a/custom_components/jev/const.py b/custom_components/jev/const.py index 126ac69..2f4c4b5 100644 --- a/custom_components/jev/const.py +++ b/custom_components/jev/const.py @@ -44,17 +44,34 @@ DEFAULT_SCAN_INTERVAL_SECONDS: Final = 300 TRIGGER_DEBOUNCE_SECONDS: Final = 5.0 -# What the pre-flight budget check divides payload bytes by before any call of its -# own has measured the real ratio. This one is derived rather than measured end to -# end: the five-entity conversation payload in site-docs/measurements.md is 3,142 -# bytes and measured 1,329 to 1,371 input tokens live, which is 2.29 to 2.36 bytes -# per token. The low end is the one that over-estimates the cost. +# What every request is billed before its body counts, whatever its size. Measured +# live on 2026-09-24 (site-docs/measurements.md): jev.noul with a 138 byte body was +# billed 278 input tokens and one with 6,136 bytes 3,277, and jev.ask with 1 and 8 +# questions, 137 and 613 bytes, was billed 279 and 377. A straight line through +# each pair crosses zero bytes at 209 and 251 tokens. 250 is near the higher one. # -# Every answered call replaces it with the ratio that endpoint actually reported, -# so an endpoint with another tokeniser calibrates this in one request. A hardcoded -# divisor would be a landmine the day someone points the entry at OpenRouter or at -# a gateway of their own. -COLD_START_BYTES_PER_TOKEN: Final = 2.29 +# Without it, the estimate was bytes over a ratio and nothing else. A 138 byte +# action was estimated at 70 tokens and billed 278. +REQUEST_OVERHEAD_TOKENS: Final = 250 + +# What the pre-flight budget check divides the body bytes by before any call of its +# own has measured the real ratio. The five-entity conversation payload in +# site-docs/measurements.md is 3,142 bytes and measured 1,329 to 1,371 input tokens +# live. Less the fixed part, that is 2.80 to 2.91 bytes per token. The low end is the +# one that over-estimates the cost. +# +# An answered call with a body worth measuring replaces it with the ratio that +# endpoint actually reported, so an endpoint with another tokeniser calibrates this +# in one request. A hardcoded divisor would be a landmine the day someone points the +# entry at OpenRouter or at a gateway of their own. +COLD_START_BYTES_PER_TOKEN: Final = 2.8 + +# A call is worth measuring when its body was billed at least as much as the fixed +# part. Below that, a few tokens of rounding in the fixed part swing the ratio. +# Measured: with the ratio taken from the 278 token action, the next 6,136 byte +# request was estimated at 14,327 tokens and billed 3,277, and it was refused again +# on every try, because a refused call measures nothing. +MIN_MEASURED_BODY_TOKENS: Final = REQUEST_OVERHEAD_TOKENS # The estimate is a tripwire, not an accounting figure. Sixteen live commands on one # payload shape varied by 3% (site-docs/measurements.md), so 20% sits well past any @@ -75,6 +92,7 @@ SERVICE_NOUL: Final = "noul" SERVICE_CHOICE: Final = "choice" SERVICE_SCORE: Final = "score" +SERVICE_CALIBRATE: Final = "calibrate" CONF_TRUE_MEANS: Final = "true_means" CONF_FALSE_MEANS: Final = "false_means" @@ -101,6 +119,10 @@ CONF_MIN_CONFIDENCE: Final = "min_confidence" CONF_ALLOW_WHOLE_HOME: Final = "allow_whole_home" +# --- Tools for other LLM agents --- + +CONF_LLM_TOOLS: Final = "llm_tools" + # Below this, the router hands the sentence to the fallback agent rather than # guessing. 0.6 is a starting point and not a calibrated figure: TypeSafe publishes # no calibration evidence for confidence, so treat it as an ordering and measure it diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 359094b..3863b87 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -22,11 +22,13 @@ from __future__ import annotations +import json import logging import re from collections.abc import Mapping -from dataclasses import asdict, dataclass -from typing import Literal +from dataclasses import asdict, dataclass, field, replace +from datetime import datetime +from typing import Any, Literal from homeassistant.components import conversation from homeassistant.components.conversation.models import AbstractConversationAgent @@ -37,10 +39,20 @@ from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.helpers import template, translation +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback from homeassistant.util import dt as dt_util from homeassistant.util import language as language_util -from jevclient import JevAuthError, JevError +from jevclient import ( + Choice, + ChoiceAnswer, + JevAuthError, + JevError, + JevResponse, + Noul, + NoulAnswer, + Question, +) from .const import ( CONF_ALLOW_WHOLE_HOME, @@ -52,8 +64,16 @@ ) from .coordinator import JevRuntimeData from .entity import build_device_info -from .interpret import build_questions, interpret -from .snapshot import async_snapshot +from .interpret import ( + ACTIONS, + NONE, + Interpretation, + build_questions, + interpret, + spoken_name, +) +from .payload import payload_bytes +from .snapshot import HomeSnapshot, async_heard_in, async_snapshot _LOGGER = logging.getLogger(__name__) @@ -71,15 +91,32 @@ "intent_failed": "Sorry, that did not work.", "already_on": "{name} is already on.", "already_off": "{name} is already off.", + "done": "Done.", "query_not_found": "I could not find that.", - "budget_spent": "The daily token budget is spent, so I cannot do that today.", + "budget_spent": ( + "Not enough of the daily token budget is left for that, so I cannot do it today." + ), "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", } PARALLEL_UPDATES = 0 +@dataclass(slots=True) +class _Pending: + """A command held while the agent asks which device it meant.""" + + text: str + response: JevResponse + snapshot: HomeSnapshot + candidates: tuple[str, str] + expires: datetime = field( + default_factory=lambda: dt_util.utcnow() + CONVERSATION_TIMEOUT + ) + + async def async_setup_entry( hass: HomeAssistant, entry: ConfigEntry, @@ -100,6 +137,8 @@ def __init__(self, entry: ConfigEntry) -> None: self._attr_unique_id = f"{entry.entry_id}_conversation" runtime: JevRuntimeData = entry.runtime_data self._attr_device_info = build_device_info(entry.entry_id, runtime) + # Conversation id to the command waiting on its reply. + self._pending: dict[str, _Pending] = {} @property def supported_languages(self) -> list[str] | Literal["*"]: @@ -154,13 +193,11 @@ async def _async_handle_message( ) -> conversation.ConversationResult: runtime: JevRuntimeData = self._entry.runtime_data - # The budget covers voice as well as sensors, because a satellite that - # mishears a wake word all night is exactly the runaway it exists to stop. runtime.usage.roll_over(dt_util.now().date()) - if runtime.usage.would_exceed(): - return await self._fall_back( - user_input, "the daily token budget is spent", "budget_spent" - ) + if (pending := self._take_pending(chat_log.conversation_id)) is not None: + resolved = await self._resolve(user_input, chat_log, pending) + if resolved is not None: + return resolved snapshot = async_snapshot(self.hass, MAX_CONVERSATION_ENTITIES) if not snapshot.entities: @@ -168,9 +205,63 @@ async def _async_handle_message( questions = build_questions(user_input.text, snapshot, MAX_CONVERSATION_ENTITIES) state = snapshot.as_state() | {"command": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + decision = interpret( + response, + user_input.text, + snapshot, + self._min_confidence, + heard_in=async_heard_in( + self.hass, user_input.satellite_id, user_input.device_id + ), + ) + self._trace( + chat_log, + response, + { + "text": user_input.text, + "exposed_entities": len(snapshot.entities), + **asdict(decision), + }, + ) + + if decision.candidates is not None: + return await self._ask_which( + user_input, + chat_log, + _Pending(user_input.text, response, snapshot, decision.candidates), + ) + return await self._act(user_input, decision, user_input.text) + + async def _ask( + self, + user_input: conversation.ConversationInput, + state: dict[str, Any], + questions: dict[str, Question], + ) -> JevResponse | conversation.ConversationResult: + """One call to Jev, inside the budget. A result means it did not answer.""" + runtime: JevRuntimeData = self._entry.runtime_data + + # The budget covers voice as well as sensors, because a satellite that + # mishears a wake word all night is exactly the runaway it exists to stop. + # The check is on this command's estimate, the same as a context's, so the + # last command of the day cannot take the total past the budget. + request_bytes = payload_bytes(state, questions, runtime.model) + estimate = runtime.usage.estimate_tokens(request_bytes) + if runtime.usage.would_exceed_with(estimate): + return await self._fall_back( + user_input, + f"the daily token budget has {runtime.usage.remaining()} tokens left " + f"and this command needs about {estimate}", + "budget_spent", + ) try: - response = await runtime.client.ask(state, questions) + with runtime.usage.reservation(estimate): + response = await runtime.client.ask(state, questions) except JevAuthError as err: _LOGGER.error("TypeSafe rejected the API key: %s", err) self._entry.async_start_reauth(self.hass) @@ -183,21 +274,157 @@ async def _async_handle_message( user_input, f"TypeSafe did not answer: {err}", "unavailable" ) - runtime.usage.record(response.usage.input_tokens) + runtime.usage.record(response.usage.input_tokens, request_bytes) runtime.model_version = response.model or runtime.model_version runtime.usage.notify() + return response - decision = interpret(response, user_input.text, snapshot, self._min_confidence) - runtime.conversation_traces.appendleft( - { - "text": user_input.text, - "latency_ms": response.latency_ms, - "input_tokens": response.usage.input_tokens, - "exposed_entities": len(snapshot.entities), - **asdict(decision), - } + def _trace( + self, + chat_log: conversation.ChatLog, + response: JevResponse, + record: dict[str, Any], + ) -> None: + """Keep what one call decided, for diagnostics and the Assist dialog.""" + runtime: JevRuntimeData = self._entry.runtime_data + trace = { + "latency_ms": response.latency_ms, + "input_tokens": response.usage.input_tokens, + **record, + } + runtime.conversation_traces.appendleft(trace) + # The pipeline records a chat log delta as an intent-progress event: the + # Assist dialog shows its thinking_content under the reply, and the run's + # debug events keep it. The delta goes to the listener only, not into the + # log, so a fallback agent reading this conversation never takes it for + # something said. It carries each answer's distribution, because "why did + # it pick the office light" is answered by the entity question and by + # nothing in the decision alone. + if chat_log.delta_listener is not None: + chat_log.delta_listener( + chat_log, + {"role": "assistant", "thinking_content": _reasoning(trace, response)}, + ) + + # --- asking which device --- + + def _take_pending(self, conversation_id: str) -> _Pending | None: + """The command waiting on this conversation's reply, if it is still live. + + A pending command lasts as long as Home Assistant keeps the chat session, + so a reply that arrives in a new session is never read as an answer. + """ + now = dt_util.utcnow() + for key in [k for k, v in self._pending.items() if v.expires < now]: + del self._pending[key] + return self._pending.pop(conversation_id, None) + + async def _ask_which( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult: + """Ask which of two devices the command meant, and keep the command.""" + first, second = (pending.snapshot.by_id(e) for e in pending.candidates) + assert first is not None and second is not None + names = spoken_name(first, second) + assert names is not None + language = user_input.language or self.hass.config.language + text = (await self._lines(language))["which_device"].format( + first=names[0], second=names[1] ) + self._pending[chat_log.conversation_id] = pending + # In the chat log, the next turn in this conversation carries the question + # it answers, and an LLM fallback agent reads the same history. + chat_log.async_add_assistant_content_without_tools( + conversation.AssistantContent(agent_id=self.entity_id, content=text) + ) + response = ha_intent.IntentResponse(language=user_input.language) + response.async_set_speech(text) + # A satellite opens the microphone again for the answer. + return conversation.ConversationResult( + response=response, + conversation_id=chat_log.conversation_id, + continue_conversation=True, + ) + + async def _resolve( + self, + user_input: conversation.ConversationInput, + chat_log: conversation.ChatLog, + pending: _Pending, + ) -> conversation.ConversationResult | None: + """Run the kept command on the device the reply picked. + + None when the reply picked neither, so the reply is handled as a new + command: "no, the kitchen light" and "never mind, lock up" both are one. + """ + snapshot = pending.snapshot + options: dict[str, Any] = { + entity_id: described.as_option() + for entity_id in pending.candidates + if (described := snapshot.by_id(entity_id)) is not None + } + options[NONE] = "Neither of these, or a different request" + questions: dict[str, Question] = { + "which": Choice("Which device does the reply pick?", options), + # "Never mind, turn off the lamp in the bedroom" names one of the two, so + # the choice alone picks it and the kept command, turn on, runs on it. + # Measured on a development instance, twelve replies to "turn on the + # lamp": the six that only pick a device scored 0.08 to 0.26 here, and + # the six that ask for something else, that one included, 0.91 to 0.97. + "new_request": Noul( + "Does the reply ask for something of its own, rather than only " + "saying which device the command meant?", + true="The reply is a new instruction, or changes what should happen", + false="The reply only picks a device, however it is phrased", + ), + } + state = {"command": pending.text, "reply": user_input.text} + response = await self._ask(user_input, state, questions) + if isinstance(response, conversation.ConversationResult): + return response + + answer = response.answers.get("which") + new_request = response.answers.get("new_request") + picked = ( + answer.choice + if isinstance(answer, ChoiceAnswer) + and answer.choice in pending.candidates + and answer.confidence >= self._min_confidence + and not (isinstance(new_request, NoulAnswer) and new_request.noul >= 0.5) + else None + ) + self._trace( + chat_log, + response, + {"text": user_input.text, "answers_command": pending.text, "picked": picked}, + ) + if picked is None: + return None + + # The first call's answers stand, with the entity question settled. The + # command is read from its own sentence again, so a brightness it named + # still comes from the text. + settled = ChoiceAnswer(choice=picked, probabilities={picked: 1.0}, confidence=1.0) + first = replace( + pending.response, answers=pending.response.answers | {"entity": settled} + ) + decision = interpret( + first, pending.text, snapshot, self._min_confidence, ask_back=False + ) + return await self._act(user_input, decision, pending.text) + # --- acting --- + + async def _act( + self, + user_input: conversation.ConversationInput, + decision: Interpretation, + text: str, + ) -> conversation.ConversationResult: + """Carry out one decision, or say why not.""" if decision.already_satisfied is not None: name, settled = decision.already_satisfied return await self._speak(user_input, f"already_{settled}", name=name) @@ -224,7 +451,7 @@ async def _async_handle_message( DOMAIN, decision.intent_type, decision.slots, - user_input.text, + text, user_input.context, language=user_input.language, assistant=conversation.DOMAIN, @@ -241,12 +468,23 @@ async def _async_handle_message( _LOGGER.error("intent %s failed: %s", decision.intent_type, err) return await self._speak(user_input, "intent_failed") - # Only a state question needs our lines, and loading them reads translations. + # Loading our lines reads translations, so only a reply without a sentence of + # its own loads them. + language = user_input.language or self.hass.config.language if intent_response.response_type is ha_intent.IntentResponseType.QUERY_ANSWER: - language = user_input.language or self.hass.config.language await _speak_the_answer( self.hass, intent_response, language, await self._lines(language) ) + elif ( + intent_response.response_type is ha_intent.IntentResponseType.ACTION_DONE + and not intent_response.speech + ): + spoken = await _render_action_answer( + self.hass, intent_response, decision.intent_type, decision.slots, language + ) + intent_response.async_set_speech( + spoken or (await self._lines(language))["done"] + ) return conversation.ConversationResult( response=intent_response, conversation_id=user_input.conversation_id ) @@ -267,7 +505,14 @@ async def _fall_back( agent = self._fallback_agent _LOGGER.debug("falling back to %s because %s", agent or "nobody", why) if agent is None: - return await self._speak(user_input, line) + # An error, as the default agent answers one. A satellite and the Assist + # dialog treat an action_done reply as a command that went through. + code = ( + ha_intent.IntentResponseErrorCode.NO_INTENT_MATCH + if line == "not_understood" + else ha_intent.IntentResponseErrorCode.FAILED_TO_HANDLE + ) + return await self._speak(user_input, line, error=code) result = await conversation.async_converse( self.hass, user_input.text, @@ -316,18 +561,54 @@ async def _speak( self, user_input: conversation.ConversationInput, key: str, + error: ha_intent.IntentResponseErrorCode | None = None, **placeholders: str, ) -> conversation.ConversationResult: """Say one of our own lines, with its placeholders filled in.""" language = user_input.language or self.hass.config.language - text = (await self._lines(language))[key] + text = (await self._lines(language))[key].format(**placeholders) response = ha_intent.IntentResponse(language=user_input.language) - response.async_set_speech(text.format(**placeholders)) + if error is None: + response.async_set_speech(text) + else: + response.async_set_error(error, text) return conversation.ConversationResult( response=response, conversation_id=user_input.conversation_id ) +def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: + """The trace as lines a person reads in the Assist dialog.""" + if "answers_command" in trace: + lines = [ + f'Jev: a reply to "{trace["answers_command"]}", ' + f"picked {trace['picked'] or 'neither'}" + ] + else: + lines = [ + f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " + f"confidence {trace['confidence']:.2f}", + f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", + ] + for key, answer in response.answers.items(): + if isinstance(answer, ChoiceAnswer): + ranked = sorted((answer.probabilities or {}).items(), key=lambda kv: -kv[1])[ + :3 + ] + spread = ", ".join(f"{k} {p:.2f}" for k, p in ranked) + lines.append(f"{key}: {answer.choice} {answer.confidence:.2f} ({spread})") + elif isinstance(answer, NoulAnswer): + lines.append(f"{key}: {answer.noul:.2f}") + else: + lines.append(f"{key}: {json.dumps(asdict(answer), ensure_ascii=False)}") + footer = f"{response.model}, {trace['input_tokens']} input tokens, " + footer += f"{trace['latency_ms']:.0f} ms" + if "exposed_entities" in trace: + footer += f", {trace['exposed_entities']} entities" + lines.append(footer) + return "\n".join(lines) + + # A template that compares the state against an English word writes the state word # itself, in its own language. Hand it a translated one and every branch falls # through. Measured against home-assistant-intents 2026.8.28: 3 of the 47 templates @@ -347,6 +628,9 @@ class _Shipped: state_answer: str | None writes_the_state_word: bool errors: Mapping[str, str] + # The sentences for each action intent, keyed by the response name the default + # agent's own sentence data picks. + action_answers: Mapping[str, Mapping[str, str]] # Read once per language, keyed by the language that was asked for rather than the @@ -390,6 +674,16 @@ def _load_shipped(language: str) -> _Shipped | None: for key, text in responses.get("errors", {}).items() if isinstance(text, str) and text.strip() }, + action_answers={ + intent_type: { + key: text + for key, text in ( + responses.get("intents", {}).get(intent_type) or {} + ).items() + if isinstance(text, str) and text.strip() + } + for intent_type in ACTIONS.values() + }, ) @@ -517,6 +811,107 @@ def spoken(state: State) -> _SpokenState: return ", ".join(parts) if parts else None +# One kind of device across a room or the whole house. The names differ between +# languages: English writes light_all and Polish lights_all. +_AREA_RESPONSES = {"light": ("lights_area",), "fan": ("fans_area",)} +_ALL_RESPONSES = {"light": ("light_all", "lights_all"), "fan": ("fan_all",)} + +_SLOT_REFERENCE = re.compile(r"slots\.(\w+)") + + +def _response_keys( + intent_type: str, slots: Mapping[str, Any], domain: str | None +) -> tuple[str, ...]: + """The response names to try, closest first, for what this command did. + + The default agent reads the name from the sentence it matched. This agent has + no sentence, so it reads the same thing off the slots it sent. + """ + if intent_type == "HassLightSet": + return ("brightness",) + kinds = slots.get("domain", {}).get("value") or [] + kind = kinds[0] if len(kinds) == 1 else None + closest: tuple[str, ...] + if "area" in slots: + closest = _AREA_RESPONSES.get(kind or "", ()) + elif slots.get("name", {}).get("value") == "all": + closest = _ALL_RESPONSES.get(kind or "", ()) + elif domain is not None: + # A scene is activated rather than turned on, and German says "Licht + # eingeschaltet" for one light, where English has no sentence of its own. + closest = (domain,) + else: + closest = () + return (*closest, "default") + + +async def _render_action_answer( + hass: HomeAssistant, + response: ha_intent.IntentResponse, + intent_type: str | None, + slots: Mapping[str, Any], + language: str, +) -> str | None: + """The sentence the default agent says after the same action, or None. + + `async_handle` does the action and says nothing, and the Assist dialog shows no + reply at all for an empty one. The default agent's words come from the same + package as the state answers, written by the people who translate Assist. + None when the package has nothing that fits, and the agent says "Done." instead. + """ + shipped = await _shipped(hass, language) + if shipped is None or intent_type is None: + return None + answers = shipped.action_answers.get(intent_type, {}) + states = [*response.matched_states, *response.unmatched_states] + first = states[0] if states else None + # The name slot is the device's own name, as the model picked it. "all" is not a + # name to say back. + speech_slots = { + key: value["value"] + for key, value in slots.items() + if key in ("name", "area") + and isinstance(value.get("value"), str) + and value["value"] != "all" + } | response.speech_slots + for key in _response_keys( + intent_type, slots, first.domain if first is not None else None + ): + text = answers.get(key) + if text is None: + continue + # German says "{{ slots.name }} eingeschaltet". Rendered for a room, with no + # name to put there, that is a sentence that starts with a blank. + if any(name not in speech_slots for name in _SLOT_REFERENCE.findall(text)): + continue + try: + rendered = template.Template(text, hass).async_render( + { + "slots": speech_slots, + "state": template.TemplateState(hass, first) if first else None, + "query": { + "matched": [ + template.TemplateState(hass, state) + for state in response.matched_states + ], + "unmatched": [ + template.TemplateState(hass, state) + for state in response.unmatched_states + ], + }, + }, + parse_result=False, + ) + except TemplateError as err: + _LOGGER.debug( + "the %s answer for %s did not render: %s", key, intent_type, err + ) + continue + if sentence := " ".join(str(rendered).split()): + return sentence + return None + + async def _speak_the_answer( hass: HomeAssistant, response: ha_intent.IntentResponse, diff --git a/custom_components/jev/coordinator.py b/custom_components/jev/coordinator.py index 3fb2d3d..08dc498 100644 --- a/custom_components/jev/coordinator.py +++ b/custom_components/jev/coordinator.py @@ -44,7 +44,9 @@ DOMAIN, ISSUE_BUDGET_EXCEEDED, ISSUE_BUDGET_SPENT, + MIN_MEASURED_BODY_TOKENS, MIN_UPDATE_INTERVAL_SECONDS, + REQUEST_OVERHEAD_TOKENS, STORE_SAVE_DELAY_SECONDS, TRIGGER_DEBOUNCE_SECONDS, ) @@ -73,9 +75,10 @@ class UsageAccount: budget: int = 0 price_per_million: float = USD_PER_MILLION_INPUT_TOKENS budget_exceeded: bool = False - # Measured from the last answered call rather than assumed, and deliberately - # not persisted: it describes the endpoint, not the day, and the first call - # after a restart measures it again. + # The bytes of a request body per billed token, not counting the fixed part. + # Measured from the last answered call with a body worth measuring rather than + # assumed, and deliberately not persisted: it describes the endpoint, not the + # day, and the first large call after a restart measures it again. bytes_per_token: float = COLD_START_BYTES_PER_TOKEN # Estimates of requests that are in flight. Two contexts refreshing together # each saw the same total before either answer came back, so both fitted and @@ -170,8 +173,9 @@ def roll_over(self, today: date) -> None: def record(self, input_tokens: int, payload_bytes: int | None = None) -> None: self.calls += 1 self.input_tokens += input_tokens - if payload_bytes and input_tokens > 0: - self.bytes_per_token = payload_bytes / input_tokens + body_tokens = input_tokens - REQUEST_OVERHEAD_TOKENS + if payload_bytes and body_tokens >= MIN_MEASURED_BODY_TOKENS: + self.bytes_per_token = payload_bytes / body_tokens self._save() async def async_flush(self) -> None: @@ -199,7 +203,8 @@ def would_exceed(self) -> bool: def estimate_tokens(self, request_bytes: int) -> int: """What a request of this size will be billed, over-estimated on purpose.""" - return ceil(request_bytes / self.bytes_per_token * BUDGET_ESTIMATE_MARGIN) + body_tokens = request_bytes / self.bytes_per_token + return ceil((REQUEST_OVERHEAD_TOKENS + body_tokens) * BUDGET_ESTIMATE_MARGIN) def would_exceed_with(self, estimate: int) -> bool: """Whether a request costing `estimate` would end the day over budget. diff --git a/custom_components/jev/icons.json b/custom_components/jev/icons.json index 02c1706..9bdb5ed 100644 --- a/custom_components/jev/icons.json +++ b/custom_components/jev/icons.json @@ -30,6 +30,7 @@ "noul": "mdi:comment-question-outline", "choice": "mdi:format-list-checks", "score": "mdi:ruler", - "ask": "mdi:comment-multiple-outline" + "ask": "mdi:comment-multiple-outline", + "calibrate": "mdi:tune-vertical" } } diff --git a/custom_components/jev/interpret.py b/custom_components/jev/interpret.py index 4e83f75..9bfec7f 100644 --- a/custom_components/jev/interpret.py +++ b/custom_components/jev/interpret.py @@ -20,7 +20,7 @@ from homeassistant.helpers import intent as ha_intent from jevclient import Choice, ChoiceAnswer, JevResponse, Noul, NoulAnswer, Question -from .snapshot import HomeSnapshot +from .snapshot import ExposedEntity, HomeSnapshot NONE = "none_of_these" @@ -129,6 +129,8 @@ class Interpretation: action_probabilities: dict[str, float] = field(default_factory=dict) # (entity name, the state it is already in) when there is nothing left to do. already_satisfied: tuple[str, str] | None = None + # Two entity ids the command could mean, when the agent should ask which. + candidates: tuple[str, str] | None = None @property def should_fall_back(self) -> bool: @@ -189,7 +191,8 @@ def build_questions( { "entity": "One particular device is named", "area": "A room or area is named, covering what is in it", - "everything": "The whole house, with no room or device named", + "everything": "Every device, or every device of one kind such as " + "all the lights, with no room or device named", NONE: "No target is named at all", }, ), @@ -225,8 +228,15 @@ def interpret( text: str, snapshot: HomeSnapshot, min_confidence: float, + *, + ask_back: bool = True, + heard_in: str | None = None, ) -> Interpretation: - """Read the answers that matter and ignore the rest.""" + """Read the answers that matter and ignore the rest. + + ask_back=False reads a command whose device a reply has already picked. + heard_in is the area id of the satellite or device that heard the command. + """ def choice(key: str) -> ChoiceAnswer | None: answer = response.answers.get(key) @@ -291,10 +301,50 @@ def out(reason: str) -> Interpretation: area = choice("area") slots: dict[str, Any] = {} targets_everything = False + named_area = ( + area.choice + if area is not None and area.choice != NONE and area.confidence >= min_confidence + else None + ) + + def ask(first: ExposedEntity, second: ExposedEntity) -> Interpretation: + # The action is sure and the device is one of two. Asking costs one short + # question, and handing the sentence to the fallback agent gets the same + # guess made again by something that does not know it was a guess. + if spoken_name(first, second) is None: + return out("two devices fit the name and nothing tells them apart") + return Interpretation( + None, + {}, + action.choice, + action.confidence, + "two devices fit the name", + fallback=False, + action_probabilities=dict(action.probabilities or {}), + candidates=(first.entity_id, second.entity_id), + ) + + def pick(chosen: ExposedEntity, *, sure: bool) -> ExposedEntity | Interpretation: + # sure is False when the model put most of its answer on none. Then only a + # name that two devices share is a reason to go on. + tied = _fit_as_well(text, chosen, snapshot, named_area) + if not sure and len(tied) < 2: + return out("no target named with enough confidence") + # A name that fits two devices is settled by the room it was said in, as + # Home Assistant's own agent settles it. A room the command names came first. + here = [e for e in tied if heard_in is not None and e.area_id == heard_in] + if len(tied) > 1 and len(here) == 1: + return here[0] + if len(tied) == 1: + return tied[0] + if len(tied) == 2: + return ask(*tied) + return out(f"{len(tied)} devices fit the name") # Trust the confident answer rather than the ordering. Measured: a scope answer # of one_room at 0.41 alongside a device answer at 1.00, where branching on # scope first threw away the certain answer and acted on the whole house. + described: ExposedEntity | None = None if ( entity is not None and entity.choice != NONE @@ -303,12 +353,17 @@ def out(reason: str) -> Interpretation: described = snapshot.by_id(entity.choice) if described is None: return out("named a device that is not exposed") - slots["name"] = {"value": described.name} - # The domain keeps a same-named entity the model was never shown, a lock - # called "Front door" beside a cover called "Front door", out of the match. - slots["domain"] = {"value": [described.domain]} - if described.area_id: - slots["preferred_area_id"] = {"value": described.area_id} + # The model picks the closest option it was shown, and a hidden device is + # not one of them. Measured on a development instance with an unexposed + # "Desk lamp" and an exposed "Lamp": "turn on the desk lamp" came back as + # Lamp and switched it on, three times of three. + if _a_hidden_name_fits_better(text, described, snapshot): + return out("named a device that is not exposed") + if ask_back: + picked = pick(described, sure=True) + if isinstance(picked, Interpretation): + return picked + described = picked elif area is not None and area.choice != NONE and area.confidence >= min_confidence: slots["area"] = {"value": area.choice} elif ( @@ -326,9 +381,30 @@ def out(reason: str) -> Interpretation: # unbounded off is not something to infer from one ambiguous sentence. slots["name"] = {"value": "all"} targets_everything = True + elif ( + ask_back + and entity is not None + and (unsure := snapshot.by_id(_likeliest_device(entity))) is not None + ): + # "Turn on the desk lamp" with a hidden desk lamp would otherwise ask + # which of two exposed lamps was meant. + if _a_hidden_name_fits_better(text, unsure, snapshot): + return out("named a device that is not exposed") + picked = pick(unsure, sure=False) + if isinstance(picked, Interpretation): + return picked + described = picked else: return out("no target named with enough confidence") + if described is not None: + slots["name"] = {"value": described.name} + # The domain keeps a same-named entity the model was never shown, a lock + # called "Front door" beside a cover called "Front door", out of the match. + slots["domain"] = {"value": [described.domain]} + if described.area_id: + slots["preferred_area_id"] = {"value": described.area_id} + # An area always carries a domain. With none, Home Assistant acts on every # exposed entity in the room whatever its domain, so turn_off on a hallway with a # light and a lock unlocked the lock. Without a confident answer, the domains the @@ -363,6 +439,87 @@ def out(reason: str) -> Interpretation: ) +def _likeliest_device(entity: ChoiceAnswer) -> str: + """The device the model gave most of its answer to, even if "none" got more. + + Measured on a development instance with two lights both called "Lamp": "turn on + the lamp" came back as none_of_these 0.55, one Lamp 0.44 and the other 0.01. The + model split its answer because the name was shared, so the name decides. + """ + devices = {k: v for k, v in (entity.probabilities or {}).items() if k != NONE} + if entity.choice != NONE or not devices: + return entity.choice + return max(devices, key=lambda k: devices[k]) + + +def _fit_as_well( + text: str, chosen: ExposedEntity, snapshot: HomeSnapshot, area: str | None +) -> list[ExposedEntity]: + """The devices of the chosen kind whose names fit the words as well as its own. + + The model gives one device all of its answer even when two fit: measured on a + development instance with two lights both called "Lamp", "turn on the lamp" + came back as one of them at 1.00, every time. The probabilities cannot say the + name was shared, the names can. A room the command names narrows the list. + + Only the chosen device when it fits best alone, or when no name fits the words + at all, which is where the model's reading is all there is. + """ + kind = [ + e + for e in snapshot.entities + if e.domain == chosen.domain and (area is None or e.area == area) + ] + if chosen not in kind: + return [chosen] + fit = {e.entity_id: _name_fit(text, e.name) for e in kind} + best = max(fit.values()) + if best == 0 or fit[chosen.entity_id] < best: + return [chosen] + return [e for e in kind if fit[e.entity_id] == best] + + +def _name_fit(text: str, name: str) -> int: + """How well the words fit a name. The whole name said beats any part of it.""" + if said := _words_said(text, name): + return 100 + said + words = set(re.findall(r"\w+", text.casefold())) + return len(set(name.casefold().split()) & words) + + +def spoken_name(one: ExposedEntity, other: ExposedEntity) -> tuple[str, str] | None: + """How to say the two apart: by name, or by room when the names are the same. + + None when neither tells them apart, since "the fan or the fan" asks nothing. + """ + if one.name.casefold() != other.name.casefold(): + return one.name, other.name + if one.area and other.area and one.area.casefold() != other.area.casefold(): + return f"{one.name} ({one.area})", f"{other.name} ({other.area})" + return None + + +def _a_hidden_name_fits_better( + text: str, chosen: ExposedEntity, snapshot: HomeSnapshot +) -> bool: + """A hidden name that the command says in more words than the chosen one.""" + said = _words_said(text, chosen.name) + return any(_words_said(text, name) > said for name in snapshot.hidden_names) + + +def _words_said(text: str, name: str) -> int: + """How many words of the name the text says, as one phrase, or 0. + + Whole words only, so a hidden "Lamp" is not found inside "lamps". In a language + written without spaces a name rarely stands apart, so the check seldom fires. + """ + words = name.casefold().split() + if not words: + return 0 + phrase = r"\s+".join(re.escape(w) for w in words) + return len(words) if re.search(rf"(? None: + # entries maps an entry title to its id. With two entries the model picks + # one, because each has its own key and its own budget. + self._entries = entries + fields: dict[Any, Any] = { + vol.Required(_QUESTION, description=_QUESTION_DESCRIPTION): str, + **extra, + vol.Optional(_FACTS, description=_FACTS_DESCRIPTION): str, + } + if len(entries) > 1: + fields[ + vol.Required(_ACCOUNT, description="Which Jev account pays for this.") + ] = vol.In(sorted(entries)) + self.parameters = vol.Schema(fields) + + def _data( + self, hass: HomeAssistant, args: dict[str, Any], llm_context: LLMContext + ) -> dict[str, Any]: + account = args.get(_ACCOUNT) or next(iter(self._entries)) + data: dict[str, Any] = { + CONF_INSTRUCTIONS: args[_QUESTION], + ATTR_CONFIG_ENTRY: self._entries[account], + ATTR_ENTITY_ID: _exposed(hass, llm_context), + } + if facts := args.get(_FACTS): + # A mapping, because the action renders a string state as a template, + # and a template can read entities that are not exposed to Assist. + data[CONF_STATE_TEMPLATE] = {_FACTS: facts} + return data + + async def _call( + self, hass: HomeAssistant, data: dict[str, Any], llm_context: LLMContext + ) -> dict[str, Any]: + response = await hass.services.async_call( + DOMAIN, + self.action, + data, + blocking=True, + context=llm_context.context, + return_response=True, + ) + assert response is not None + return dict(response) + + +class NoulTool(_JevTool): + name = f"{DOMAIN}__noul" + description = ( + "Ask Jev how likely a yes/no statement about the home is, judged from the " + "states of the entities exposed to Assist. Returns the probability that the " + "answer is yes, from 0 to 1. Use it for a judgement, such as whether a " + "machine is done or whether anyone is likely home, not for a state you can " + "read directly." + ) + action = SERVICE_NOUL + + def __init__(self, entries: dict[str, str]) -> None: + super().__init__(entries, {}) + + @override + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext + ) -> JsonObjectType: + args = self.parameters(tool_input.tool_args) + result = await self._call(hass, self._data(hass, args, llm_context), llm_context) + return {"probability_yes": result["noul"]} + + +class ChoiceTool(_JevTool): + name = f"{DOMAIN}__choice" + description = ( + "Ask Jev which of a set of options best describes the home, judged from the " + "states of the entities exposed to Assist. Returns the chosen option, the " + "probability of every option, and a confidence from 0 to 1." + ) + action = SERVICE_CHOICE + + def __init__(self, entries: dict[str, str]) -> None: + super().__init__( + entries, + { + vol.Required(CONF_OPTIONS, description="Two or more options."): vol.All( + [str], vol.Length(min=2) + ) + }, + ) + + @override + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext + ) -> JsonObjectType: + args = self.parameters(tool_input.tool_args) + data = self._data(hass, args, llm_context) | {CONF_OPTIONS: args[CONF_OPTIONS]} + result = await self._call(hass, data, llm_context) + return { + "choice": result["choice"], + "probabilities": result["probabilities"], + "confidence": result["confidence"], + } + + +def _exposed(hass: HomeAssistant, llm_context: LLMContext) -> list[str]: + """The entities this assistant may see, capped as the voice agent caps them. + + Jev judges from what the user exposed to Assist and nothing wider, the same + list the LLM agent itself reads. + """ + exposed = sorted( + state.entity_id + for state in hass.states.async_all() + if async_should_expose(hass, llm_context.assistant, state.entity_id) + ) + return exposed[:MAX_CONVERSATION_ENTITIES] + + +@callback +def async_get_tools( + hass: HomeAssistant, llm_context: LLMContext, api_id: str +) -> LLMTools | None: + """The tools, for each loaded entry whose options turn them on.""" + if api_id != LLM_API_ASSIST: + return None + entries = { + entry.title: entry.entry_id + for entry in hass.config_entries.async_loaded_entries(DOMAIN) + if entry.options.get(CONF_LLM_TOOLS, False) + } + if not entries: + return None + return LLMTools(tools=[NoulTool(entries), ChoiceTool(entries)]) diff --git a/custom_components/jev/manifest.json b/custom_components/jev/manifest.json index 439ca71..c8e9c3e 100644 --- a/custom_components/jev/manifest.json +++ b/custom_components/jev/manifest.json @@ -1,7 +1,7 @@ { "domain": "jev", "name": "Jev (TypeSafe)", - "after_dependencies": ["conversation"], + "after_dependencies": ["conversation", "recorder"], "codeowners": ["@abovecolin"], "config_flow": true, "dependencies": [], @@ -10,5 +10,5 @@ "iot_class": "cloud_polling", "issue_tracker": "https://github.com/AboveColin/HA-Jev/issues", "requirements": ["jevclient==1.2.0"], - "version": "1.15.1" + "version": "1.16.0" } diff --git a/custom_components/jev/services.py b/custom_components/jev/services.py index fafb146..807022e 100644 --- a/custom_components/jev/services.py +++ b/custom_components/jev/services.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: from . import JevConfigEntry +from .calibrate import CALIBRATE_SCHEMA, async_calibrate from .const import ( ATTR_ANSWERS, ATTR_CONFIG_ENTRY, @@ -64,6 +65,7 @@ CONF_TRUE_MEANS, DOMAIN, SERVICE_ASK, + SERVICE_CALIBRATE, SERVICE_CHOICE, SERVICE_NOUL, SERVICE_SCORE, @@ -217,9 +219,25 @@ async def _ask( type(state).__name__, state, ) + usage = entry.runtime_data.usage + usage.roll_over(dt_util.now().date()) request_bytes = payload_bytes(state, questions, entry.runtime_data.model) + # The same check a context and the voice agent make: a script that loops on an + # action is the runaway the budget exists for. + estimate = usage.estimate_tokens(request_bytes) + if usage.would_exceed_with(estimate): + raise HomeAssistantError( + translation_domain=DOMAIN, + translation_key="action_over_budget", + translation_placeholders={ + "estimate": str(estimate), + "remaining": str(usage.remaining()), + "budget": str(usage.budget), + }, + ) try: - response = await entry.runtime_data.client.ask(state, questions) + with usage.reservation(estimate): + response = await entry.runtime_data.client.ask(state, questions) except JevAuthError as err: entry.async_start_reauth(hass) raise HomeAssistantError( @@ -233,8 +251,6 @@ async def _ask( translation_key="ask_failed", translation_placeholders={"reason": str(err)}, ) from err - usage = entry.runtime_data.usage - usage.roll_over(dt_util.now().date()) usage.record(response.usage.input_tokens, request_bytes) entry.runtime_data.model_version = response.model or entry.runtime_data.model_version usage.notify() @@ -283,7 +299,7 @@ def answer_as_dict(answer: Any) -> dict[str, Any]: def async_register_services(hass: HomeAssistant) -> None: - """Register all four actions once, the first time the component loads.""" + """Register all five actions once, the first time the component loads.""" if hass.services.has_service(DOMAIN, SERVICE_NOUL): return @@ -382,11 +398,15 @@ async def _ask_many(call: ServiceCall) -> ServiceResponse: **_envelope(response), } + async def _calibrate(call: ServiceCall) -> ServiceResponse: + return await async_calibrate(hass, call) + for name, handler, schema in ( (SERVICE_NOUL, _noul, NOUL_SCHEMA), (SERVICE_CHOICE, _choice, CHOICE_SCHEMA), (SERVICE_SCORE, _score, SCORE_SCHEMA), (SERVICE_ASK, _ask_many, ASK_SCHEMA), + (SERVICE_CALIBRATE, _calibrate, CALIBRATE_SCHEMA), ): hass.services.async_register( DOMAIN, name, handler, schema=schema, supports_response=SupportsResponse.ONLY diff --git a/custom_components/jev/services.yaml b/custom_components/jev/services.yaml index bab8b08..e35dd40 100644 --- a/custom_components/jev/services.yaml +++ b/custom_components/jev/services.yaml @@ -196,3 +196,28 @@ ask: selector: config_entry: integration: jev + +calibrate: + fields: + entity_id: + required: true + selector: + entity: + domain: sensor + truth_entity_id: + required: true + selector: + entity: {} + truth_state: + required: false + default: "on" + selector: + text: + days: + required: false + default: 7 + selector: + number: + min: 1 + mode: box + unit_of_measurement: days diff --git a/custom_components/jev/snapshot.py b/custom_components/jev/snapshot.py index e111258..4f28280 100644 --- a/custom_components/jev/snapshot.py +++ b/custom_components/jev/snapshot.py @@ -69,6 +69,10 @@ class HomeSnapshot: entities: list[ExposedEntity] = field(default_factory=list) areas: list[str] = field(default_factory=list) floors: list[str] = field(default_factory=list) + # Names of the entities the model is not shown: not exposed, left out above, or + # past the cap. They never leave Home Assistant. interpret() reads them so that + # "the desk lamp" cannot land on an exposed "Lamp" when the desk lamp is hidden. + hidden_names: list[str] = field(default_factory=list) @property def domains(self) -> list[str]: @@ -102,6 +106,24 @@ def as_state(self) -> dict[str, object]: } +@callback +def async_heard_in( + hass: HomeAssistant, satellite_id: str | None, device_id: str | None +) -> str | None: + """The area id of the satellite or device that heard a command, if it has one. + + The same lookup as Home Assistant's own agent: the satellite entity's area, then + its device's area. + """ + if satellite_id and (entry := er.async_get(hass).async_get(satellite_id)): + if entry.area_id is not None: + return entry.area_id + device_id = entry.device_id + if device_id and (device := dr.async_get(hass).async_get(device_id)): + return device.area_id + return None + + @callback def async_snapshot(hass: HomeAssistant, limit: int) -> HomeSnapshot: """Collect the exposed, controllable entities, newest registry state.""" @@ -122,13 +144,17 @@ def area_id_of(entity_id: str) -> str | None: return None found: list[ExposedEntity] = [] - for state in hass.states.async_all(CONTROLLABLE): - if not async_should_expose(hass, CONVERSATION_DOMAIN, state.entity_id): - continue + hidden: list[str] = [] + for state in hass.states.async_all(): if ( - state.domain == "cover" - and state.attributes.get("device_class") in ENTRANCE_COVERS + state.domain not in CONTROLLABLE + or not async_should_expose(hass, CONVERSATION_DOMAIN, state.entity_id) + or ( + state.domain == "cover" + and state.attributes.get("device_class") in ENTRANCE_COVERS + ) ): + hidden.append(state.name) continue area_id = area_id_of(state.entity_id) area = areas.async_get_area(area_id) if area_id else None @@ -145,6 +171,7 @@ def area_id_of(entity_id: str) -> str | None: # Sorted so the option list is stable between requests, which makes a trace # readable when the same command is tried twice. found.sort(key=lambda e: e.entity_id) + hidden.extend(e.name for e in found[limit:]) found = found[:limit] # Counted after the cap, so a room that only had entities past the limit is not # offered as somewhere the command could go. @@ -168,4 +195,5 @@ def area_id_of(entity_id: str) -> str | None: floors=sorted( f.name for f in floors.async_list_floors() if f.floor_id in used_floor_ids ), + hidden_names=hidden, ) diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 54cf0a1..ff188f6 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -82,14 +82,16 @@ "price_per_million": "Price per million input tokens, in USD", "fallback_agent": "Fall back to this agent", "min_confidence": "Act only above this confidence (0 to 1)", - "allow_whole_home": "Allow commands that name no room or device" + "allow_whole_home": "Allow commands that name no room or device", + "llm_tools": "Offer Jev as a tool to other LLM agents" }, "data_description": { "daily_token_budget": "Counted from midnight in the Home Assistant time zone. 0 means no limit.", "price_per_million": "Sets the estimated cost sensor and the cost in the question preview. Nothing that is sent depends on it.", "fallback_agent": "Receives the whole sentence when Jev is not sure what to do.", "min_confidence": "Below this, Jev does nothing and passes the command to the fallback agent.", - "allow_whole_home": "Turning everything off is always allowed, with or without this." + "allow_whole_home": "Turning everything off is always allowed, with or without this.", + "llm_tools": "Adds two tools to the Assist API, one for yes/no questions and one for picking an option. Each LLM agent that uses Assist gets their descriptions in every prompt, which costs tokens at that agent's provider. Each call also counts against the budget above." } } } @@ -228,6 +230,28 @@ "description": "Off by default. Sends every attribute of the picked entities, not just the value, unit, device class and area. A weather forecast or a media player artwork list runs to thousands of tokens, and you pay for them on every evaluation." } } + }, + "calibrate": { + "name": "Find the best threshold", + "description": "Compare the recorded history of a yes/no probability with an entity that shows what was really true, and get the threshold that would have been right most often. Reads only the recorder, so it costs no tokens.", + "fields": { + "entity_id": { + "name": "Probability", + "description": "A Jev yes/no sensor, or any sensor whose state is a probability from 0 to 1." + }, + "truth_entity_id": { + "name": "What was really true", + "description": "An entity whose state shows the real answer, such as a door contact or a plug that reports when a machine runs." + }, + "truth_state": { + "name": "State that means yes", + "description": "The state of that entity that means yes. on if you leave it empty." + }, + "days": { + "name": "Days of history", + "description": "How far back to read. The recorder keeps 10 days unless you changed it." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Task {task} costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." }, + "action_over_budget": { + "message": "This action costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." + }, "reserved_field": { "message": "The field {field} cannot be used, because the result carries the confidence behind every answer under that name. Rename the field." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe rate limited the request: {reason}" + }, + "calibrate_needs_recorder": { + "message": "This action reads the recorder, and the recorder is not running." + }, + "calibrate_no_history": { + "message": "No time in the last {days} days has both a number from {entity} and a state from {truth}. Check that the recorder records both, and that {entity} reports a number." + }, + "calibrate_never_true": { + "message": "{truth} was never {state} in the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." + }, + "calibrate_always_true": { + "message": "{truth} was {state} for all of the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Sorry, that did not work.", "already_on": "{name} is already on.", "already_off": "{name} is already off.", + "done": "Done.", "preview_alone": "Sent as its own request.", "preview_grouped": "Sent in one request together with: {others}.", "preview_cost": "Asked once for this preview: {tokens} input tokens, about ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "The API answered, but not to the question that was asked.", "preview_template_error": "The template does not render: {reason}", "query_not_found": "I could not find that.", - "budget_spent": "The daily token budget is spent, so I cannot do that today.", + "budget_spent": "Not enough of the daily token budget is left for that, so I cannot do it today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index dd03fe5..d102fa1 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -82,14 +82,16 @@ "price_per_million": "Cena za milion vstupních tokenů v USD", "fallback_agent": "Záložní agent", "min_confidence": "Jednat jen nad touto jistotou (0 až 1)", - "allow_whole_home": "Povolit příkazy, které neuvádějí místnost ani zařízení" + "allow_whole_home": "Povolit příkazy, které neuvádějí místnost ani zařízení", + "llm_tools": "Nabídnout Jev jako nástroj ostatním LLM agentům" }, "data_description": { "daily_token_budget": "Počítá se od půlnoci v časovém pásmu Home Assistantu. 0 znamená bez limitu.", "price_per_million": "Určuje senzor odhadované ceny a cenu v náhledu otázky. Nic z toho, co se odesílá, na tom nezávisí.", "fallback_agent": "Dostane celou větu, když si Jev není jistý, co udělat.", "min_confidence": "Pod touto hodnotou Jev nic neudělá a předá příkaz záložnímu agentovi.", - "allow_whole_home": "Vypnout všechno je dovoleno vždy, s touto volbou i bez ní." + "allow_whole_home": "Vypnout všechno je dovoleno vždy, s touto volbou i bez ní.", + "llm_tools": "Přidá do API Assist dva nástroje, jeden pro otázky ano/ne a jeden pro výběr možnosti. Každý LLM agent, který používá Assist, dostane jejich popisy v každém promptu, což stojí tokeny u poskytovatele tohoto agenta. Každé volání se také počítá do rozpočtu výše." } } } @@ -228,6 +230,28 @@ "description": "Ve výchozím stavu vypnuto. Posílá každý atribut vybraných entit, nejen hodnotu, jednotku, třídu zařízení a oblast. Předpověď počasí nebo seznam obalů alb v přehrávači médií vydá na tisíce tokenů a ty zaplatíte při každém vyhodnocení." } } + }, + "calibrate": { + "name": "Najít nejlepší práh", + "description": "Porovná zaznamenanou historii pravděpodobnosti ano/ne s entitou, která ukazuje, co skutečně platilo, a vrátí práh, který by nejčastěji měl pravdu. Čte jen recorder, takže nestojí žádné tokeny.", + "fields": { + "entity_id": { + "name": "Pravděpodobnost", + "description": "Senzor ano/ne od Jev, nebo jakýkoli senzor, jehož stav je pravděpodobnost od 0 do 1." + }, + "truth_entity_id": { + "name": "Co skutečně platilo", + "description": "Entita, jejíž stav ukazuje skutečnou odpověď, například kontakt dveří nebo zásuvka, která hlásí, kdy spotřebič běží." + }, + "truth_state": { + "name": "Stav znamenající ano", + "description": "Stav té entity, který znamená ano. on, pokud pole necháte prázdné." + }, + "days": { + "name": "Dny historie", + "description": "Jak daleko zpět číst. Recorder uchovává 10 dní, pokud jste to nezměnili." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Úloha {task} stojí asi {estimate} vstupních tokenů a z denního rozpočtu {budget} zbývá {remaining}. Pro pokračování zvyšte nebo zrušte rozpočet v možnostech integrace." }, + "action_over_budget": { + "message": "Tato akce stojí asi {estimate} vstupních tokenů a z denního rozpočtu {budget} zbývá {remaining}. Pokud chcete pokračovat, zvyšte nebo zrušte rozpočet v možnostech integrace." + }, "reserved_field": { "message": "Pole {field} nelze použít, protože výsledek pod tímto názvem nese jistotu každé odpovědi. Pole přejmenujte." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe omezil požadavek: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Tato akce čte recorder a recorder neběží." + }, + "calibrate_no_history": { + "message": "V posledních {days} dnech není žádný okamžik s číslem z {entity} i stavem z {truth}. Zkontrolujte, že recorder zaznamenává obojí a že {entity} hlásí číslo." + }, + "calibrate_never_true": { + "message": "{truth} nebyl v posledních {days} dnech nikdy {state}, takže není s čím porovnat. Čtěte delší období, nebo zkontrolujte, který stav znamená ano." + }, + "calibrate_always_true": { + "message": "{truth} byl celých posledních {days} dní {state}, takže není s čím porovnat. Čtěte delší období, nebo zkontrolujte, který stav znamená ano." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Promiňte, to se nepovedlo.", "already_on": "{name} už je zapnuto.", "already_off": "{name} už je vypnuto.", + "done": "Hotovo.", "preview_alone": "Odesláno jako vlastní požadavek.", "preview_grouped": "Odesláno v jednom požadavku spolu s: {others}.", "preview_cost": "Jednou dotázáno pro tento náhled: {tokens} vstupních tokenů, asi ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "API odpovědělo, ale ne na otázku, která byla položena.", "preview_template_error": "Šablona se nevykreslí: {reason}", "query_not_found": "To jsem nenašel.", - "budget_spent": "Denní rozpočet tokenů je vyčerpán, takže to dnes udělat nemohu.", + "budget_spent": "Z denního rozpočtu tokenů na to nezbývá dost, takže to dnes udělat nemohu.", "auth_failed": "TypeSafe odmítl klíč API. Zkontrolujte ho v nastavení Jev.", "unavailable": "TypeSafe neodpověděl. Zkuste to za chvíli znovu.", + "which_device": "Myslíte {first}, nebo {second}?", "preview_over_budget": "Denní rozpočet tokenů je vyčerpán, takže zkušební odpověď není. Otázka se přesto uloží.", "preview_nothing_yet": "Žádná zkušební odpověď: zatím není na co se ptát." }, diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index bbdea05..0730c57 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -82,14 +82,16 @@ "price_per_million": "Pris pr. million input-tokens, i USD", "fallback_agent": "Fald tilbage til denne agent", "min_confidence": "Handl kun over denne konfidens (0 til 1)", - "allow_whole_home": "Tillad kommandoer, der ikke nævner et rum eller en enhed" + "allow_whole_home": "Tillad kommandoer, der ikke nævner et rum eller en enhed", + "llm_tools": "Tilbyd Jev som værktøj til andre LLM-agenter" }, "data_description": { "daily_token_budget": "Tælles fra midnat i Home Assistants tidszone. 0 betyder ingen grænse.", "price_per_million": "Bestemmer sensoren for anslået pris og prisen i forhåndsvisningen af spørgsmålet. Intet af det, der sendes, afhænger af den.", "fallback_agent": "Modtager hele sætningen, når Jev ikke er sikker på, hvad der skal gøres.", "min_confidence": "Under denne værdi gør Jev intet og sender kommandoen videre til reserveagenten.", - "allow_whole_home": "At slukke alt er altid tilladt, med eller uden denne indstilling." + "allow_whole_home": "At slukke alt er altid tilladt, med eller uden denne indstilling.", + "llm_tools": "Tilføjer to værktøjer til Assist-API'et, et til ja/nej-spørgsmål og et til at vælge en mulighed. Hver LLM-agent, der bruger Assist, får deres beskrivelser i hver prompt, hvilket koster tokens hos den agents udbyder. Hvert kald tæller også med i budgettet ovenfor." } } } @@ -228,6 +230,28 @@ "description": "Slået fra som standard. Sender alle attributter for de valgte entiteter, ikke kun værdien, måleenheden, enhedsklassen og området. En vejrudsigt eller en medieafspillers liste med albumbilleder løber op i tusindvis af tokens, og dem betaler du for ved hver evaluering." } } + }, + "calibrate": { + "name": "Find den bedste tærskel", + "description": "Sammenligner den gemte historik for en ja/nej-sandsynlighed med en enhed, der viser, hvad der faktisk var sandt, og giver den tærskel, der oftest ville have haft ret. Læser kun recorderen og koster derfor ingen tokens.", + "fields": { + "entity_id": { + "name": "Sandsynlighed", + "description": "En Jev ja/nej-sensor eller en anden sensor, hvis tilstand er en sandsynlighed fra 0 til 1." + }, + "truth_entity_id": { + "name": "Hvad der faktisk var sandt", + "description": "En enhed, hvis tilstand viser det rigtige svar, f.eks. en dørkontakt eller et stik, der melder, når et apparat kører." + }, + "truth_state": { + "name": "Tilstand der betyder ja", + "description": "Den tilstand for enheden, der betyder ja. on hvis du lader feltet være tomt." + }, + "days": { + "name": "Dages historik", + "description": "Hvor langt tilbage der læses. Recorderen gemmer 10 dage, medmindre du har ændret det." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Opgaven {task} koster omkring {estimate} inputtokens, og der er {remaining} tilbage af dagsbudgettet på {budget}. Hæv eller ryd budgettet i integrationens indstillinger for at fortsætte." }, + "action_over_budget": { + "message": "Denne handling koster omkring {estimate} input-tokens, og der er {remaining} tilbage af det daglige budget på {budget}. Hæv eller fjern budgettet i integrationens indstillinger for at fortsætte." + }, "reserved_field": { "message": "Feltet {field} kan ikke bruges, fordi resultatet bærer sikkerheden bag hvert svar under det navn. Omdøb feltet." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe begrænsede forespørgslen: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Denne handling læser recorderen, og recorderen kører ikke." + }, + "calibrate_no_history": { + "message": "Intet tidspunkt i de seneste {days} dage har både et tal fra {entity} og en tilstand fra {truth}. Kontroller, at recorderen gemmer begge, og at {entity} melder et tal." + }, + "calibrate_never_true": { + "message": "{truth} var aldrig {state} i de seneste {days} dage, så der er intet at sammenligne med. Læs en længere periode, eller kontroller, hvilken tilstand der betyder ja." + }, + "calibrate_always_true": { + "message": "{truth} var {state} i hele de seneste {days} dage, så der er intet at sammenligne med. Læs en længere periode, eller kontroller, hvilken tilstand der betyder ja." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Beklager, det virkede ikke.", "already_on": "{name} er allerede tændt.", "already_off": "{name} er allerede slukket.", + "done": "Færdig.", "preview_alone": "Sendes som sin egen anmodning.", "preview_grouped": "Sendes i én anmodning sammen med: {others}.", "preview_cost": "Spurgt én gang til denne forhåndsvisning: {tokens} input-tokens, cirka ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "API'en svarede, men ikke på det spørgsmål, der blev stillet.", "preview_template_error": "Skabelonen kan ikke gengives: {reason}", "query_not_found": "Det kunne jeg ikke finde.", - "budget_spent": "Det daglige tokenbudget er brugt, så det kan jeg ikke gøre i dag.", + "budget_spent": "Der er ikke nok tilbage af det daglige tokenbudget til det, så det kan jeg ikke gøre i dag.", "auth_failed": "TypeSafe afviste API-nøglen. Tjek den i indstillingerne for Jev.", "unavailable": "TypeSafe svarede ikke. Prøv igen om et øjeblik.", + "which_device": "Mener du {first} eller {second}?", "preview_over_budget": "Det daglige token-budget er brugt op, så der er intet prøvesvar. Spørgsmålet bliver gemt alligevel.", "preview_nothing_yet": "Intet prøvesvar: der er endnu intet at spørge om." }, diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index 89c7dbc..6b4e1ed 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -82,14 +82,16 @@ "price_per_million": "Preis pro Million Eingabe-Tokens, in USD", "fallback_agent": "Auf diesen Agenten zurückfallen", "min_confidence": "Nur oberhalb dieser Konfidenz handeln (0 bis 1)", - "allow_whole_home": "Befehle erlauben, die keinen Raum und kein Gerät nennen" + "allow_whole_home": "Befehle erlauben, die keinen Raum und kein Gerät nennen", + "llm_tools": "Jev anderen LLM-Agenten als Werkzeug anbieten" }, "data_description": { "daily_token_budget": "Gezählt ab Mitternacht in der Zeitzone von Home Assistant. 0 bedeutet kein Limit.", "price_per_million": "Bestimmt den Sensor für die geschätzten Kosten und die Kosten in der Vorschau einer Frage. Nichts, was gesendet wird, hängt davon ab.", "fallback_agent": "Bekommt den ganzen Satz, wenn Jev nicht sicher ist, was zu tun ist.", "min_confidence": "Darunter tut Jev nichts und gibt den Befehl an den Ersatz-Agenten weiter.", - "allow_whole_home": "Alles ausschalten ist immer erlaubt, mit oder ohne diese Option." + "allow_whole_home": "Alles ausschalten ist immer erlaubt, mit oder ohne diese Option.", + "llm_tools": "Fügt der Assist-API zwei Werkzeuge hinzu, eines für Ja/Nein-Fragen und eines für die Wahl einer Option. Jeder LLM-Agent, der Assist nutzt, erhält ihre Beschreibungen in jedem Prompt, was beim Anbieter dieses Agenten Tokens kostet. Jeder Aufruf zählt außerdem zum Budget oben." } } } @@ -228,6 +230,28 @@ "description": "Standardmäßig aus. Sendet jedes Attribut der gewählten Entitäten mit, nicht nur Wert, Einheit, Geräteklasse und Bereich. Eine Wettervorhersage oder die Coverliste eines Media Players geht in die Tausende von Tokens, und die bezahlst du bei jeder Auswertung." } } + }, + "calibrate": { + "name": "Besten Schwellenwert finden", + "description": "Vergleicht den aufgezeichneten Verlauf einer Ja/Nein-Wahrscheinlichkeit mit einer Entität, die zeigt, was wirklich zutraf, und liefert den Schwellenwert, der am häufigsten richtig gewesen wäre. Liest nur den Recorder und kostet daher keine Tokens.", + "fields": { + "entity_id": { + "name": "Wahrscheinlichkeit", + "description": "Ein Jev-Ja/Nein-Sensor oder ein anderer Sensor, dessen Zustand eine Wahrscheinlichkeit von 0 bis 1 ist." + }, + "truth_entity_id": { + "name": "Was wirklich zutraf", + "description": "Eine Entität, deren Zustand die echte Antwort zeigt, etwa ein Türkontakt oder eine Steckdose, die meldet, wann ein Gerät läuft." + }, + "truth_state": { + "name": "Zustand für Ja", + "description": "Der Zustand dieser Entität, der Ja bedeutet. on, wenn du das Feld leer lässt." + }, + "days": { + "name": "Tage Verlauf", + "description": "Wie weit zurück gelesen wird. Der Recorder behält 10 Tage, sofern du das nicht geändert hast." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Die Aufgabe {task} kostet etwa {estimate} Eingabetokens, und vom Tagesbudget von {budget} sind noch {remaining} übrig. Erhöhe oder entferne das Budget in den Optionen der Integration, um fortzufahren." }, + "action_over_budget": { + "message": "Diese Aktion kostet etwa {estimate} Eingabe-Tokens, und vom Tagesbudget von {budget} sind noch {remaining} übrig. Erhöhe oder entferne das Budget in den Optionen der Integration, um fortzufahren." + }, "reserved_field": { "message": "Das Feld {field} kann nicht verwendet werden, weil das Ergebnis unter diesem Namen die Sicherheit hinter jeder Antwort mitliefert. Benenne das Feld um." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe hat die Anfrage gedrosselt: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Diese Aktion liest den Recorder, und der Recorder läuft nicht." + }, + "calibrate_no_history": { + "message": "In den letzten {days} Tagen gibt es keinen Zeitraum mit einer Zahl von {entity} und einem Zustand von {truth}. Prüfe, ob der Recorder beide aufzeichnet und ob {entity} eine Zahl meldet." + }, + "calibrate_never_true": { + "message": "{truth} war in den letzten {days} Tagen nie {state}, also gibt es nichts zum Vergleichen. Lies einen längeren Zeitraum oder prüfe, welcher Zustand Ja bedeutet." + }, + "calibrate_always_true": { + "message": "{truth} war die gesamten letzten {days} Tage {state}, also gibt es nichts zum Vergleichen. Lies einen längeren Zeitraum oder prüfe, welcher Zustand Ja bedeutet." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Entschuldigung, das hat nicht geklappt.", "already_on": "{name} ist schon an.", "already_off": "{name} ist schon aus.", + "done": "Erledigt.", "preview_alone": "Geht als eigene Anfrage raus.", "preview_grouped": "Geht in einer Anfrage zusammen mit: {others}.", "preview_cost": "Einmal für diese Vorschau gefragt: {tokens} Eingabe-Tokens, etwa ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "Die API hat geantwortet, aber nicht auf die gestellte Frage.", "preview_template_error": "Die Vorlage lässt sich nicht rendern: {reason}", "query_not_found": "Das konnte ich nicht finden.", - "budget_spent": "Das tägliche Token-Budget ist aufgebraucht, deshalb kann ich das heute nicht tun.", + "budget_spent": "Vom täglichen Token-Budget ist dafür nicht mehr genug übrig, deshalb kann ich das heute nicht tun.", "auth_failed": "TypeSafe hat den API-Schlüssel abgelehnt. Prüfe ihn in den Jev-Einstellungen.", "unavailable": "TypeSafe hat nicht geantwortet. Versuche es gleich noch einmal.", + "which_device": "Meinst du {first} oder {second}?", "preview_over_budget": "Das Tagesbudget für Tokens ist aufgebraucht, es gibt also keine Probeantwort. Die Frage wird trotzdem gespeichert.", "preview_nothing_yet": "Keine Probeantwort: Es gibt noch nichts, wonach sich fragen ließe." }, diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 54cf0a1..ff188f6 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -82,14 +82,16 @@ "price_per_million": "Price per million input tokens, in USD", "fallback_agent": "Fall back to this agent", "min_confidence": "Act only above this confidence (0 to 1)", - "allow_whole_home": "Allow commands that name no room or device" + "allow_whole_home": "Allow commands that name no room or device", + "llm_tools": "Offer Jev as a tool to other LLM agents" }, "data_description": { "daily_token_budget": "Counted from midnight in the Home Assistant time zone. 0 means no limit.", "price_per_million": "Sets the estimated cost sensor and the cost in the question preview. Nothing that is sent depends on it.", "fallback_agent": "Receives the whole sentence when Jev is not sure what to do.", "min_confidence": "Below this, Jev does nothing and passes the command to the fallback agent.", - "allow_whole_home": "Turning everything off is always allowed, with or without this." + "allow_whole_home": "Turning everything off is always allowed, with or without this.", + "llm_tools": "Adds two tools to the Assist API, one for yes/no questions and one for picking an option. Each LLM agent that uses Assist gets their descriptions in every prompt, which costs tokens at that agent's provider. Each call also counts against the budget above." } } } @@ -228,6 +230,28 @@ "description": "Off by default. Sends every attribute of the picked entities, not just the value, unit, device class and area. A weather forecast or a media player artwork list runs to thousands of tokens, and you pay for them on every evaluation." } } + }, + "calibrate": { + "name": "Find the best threshold", + "description": "Compare the recorded history of a yes/no probability with an entity that shows what was really true, and get the threshold that would have been right most often. Reads only the recorder, so it costs no tokens.", + "fields": { + "entity_id": { + "name": "Probability", + "description": "A Jev yes/no sensor, or any sensor whose state is a probability from 0 to 1." + }, + "truth_entity_id": { + "name": "What was really true", + "description": "An entity whose state shows the real answer, such as a door contact or a plug that reports when a machine runs." + }, + "truth_state": { + "name": "State that means yes", + "description": "The state of that entity that means yes. on if you leave it empty." + }, + "days": { + "name": "Days of history", + "description": "How far back to read. The recorder keeps 10 days unless you changed it." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Task {task} costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." }, + "action_over_budget": { + "message": "This action costs about {estimate} input tokens and {remaining} of the {budget} daily budget are left. Raise or clear the budget in the integration options to continue." + }, "reserved_field": { "message": "The field {field} cannot be used, because the result carries the confidence behind every answer under that name. Rename the field." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe rate limited the request: {reason}" + }, + "calibrate_needs_recorder": { + "message": "This action reads the recorder, and the recorder is not running." + }, + "calibrate_no_history": { + "message": "No time in the last {days} days has both a number from {entity} and a state from {truth}. Check that the recorder records both, and that {entity} reports a number." + }, + "calibrate_never_true": { + "message": "{truth} was never {state} in the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." + }, + "calibrate_always_true": { + "message": "{truth} was {state} for all of the last {days} days, so there is nothing to compare against. Read a longer period, or check which state means yes." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Sorry, that did not work.", "already_on": "{name} is already on.", "already_off": "{name} is already off.", + "done": "Done.", "preview_alone": "Sent as its own request.", "preview_grouped": "Sent in one request together with: {others}.", "preview_cost": "Asked once for this preview: {tokens} input tokens, about ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "The API answered, but not to the question that was asked.", "preview_template_error": "The template does not render: {reason}", "query_not_found": "I could not find that.", - "budget_spent": "The daily token budget is spent, so I cannot do that today.", + "budget_spent": "Not enough of the daily token budget is left for that, so I cannot do it today.", "auth_failed": "TypeSafe rejected the API key. Check it in the Jev settings.", "unavailable": "TypeSafe did not answer. Try again in a moment.", + "which_device": "Do you mean {first} or {second}?", "preview_over_budget": "The daily token budget is spent, so there is no trial answer. The question still saves.", "preview_nothing_yet": "No trial answer: there is nothing to ask about yet." }, diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index a3e9da7..0a56ffc 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -82,14 +82,16 @@ "price_per_million": "Precio por millón de tokens de entrada, en USD", "fallback_agent": "Recurrir a este agente", "min_confidence": "Actuar solo por encima de esta confianza (de 0 a 1)", - "allow_whole_home": "Permitir órdenes que no nombren ninguna habitación ni dispositivo" + "allow_whole_home": "Permitir órdenes que no nombren ninguna habitación ni dispositivo", + "llm_tools": "Ofrecer Jev como herramienta a otros agentes LLM" }, "data_description": { "daily_token_budget": "Se cuenta desde la medianoche en la zona horaria de Home Assistant. 0 significa sin límite.", "price_per_million": "Fija el sensor de coste estimado y el coste en la vista previa de la pregunta. Nada de lo que se envía depende de ello.", "fallback_agent": "Recibe la frase entera cuando Jev no está seguro de qué hacer.", "min_confidence": "Por debajo de este valor, Jev no hace nada y pasa el comando al agente de respaldo.", - "allow_whole_home": "Apagarlo todo siempre está permitido, con o sin esta opción." + "allow_whole_home": "Apagarlo todo siempre está permitido, con o sin esta opción.", + "llm_tools": "Añade dos herramientas a la API de Assist, una para preguntas de sí o no y otra para elegir una opción. Cada agente LLM que usa Assist recibe sus descripciones en cada prompt, lo que cuesta tokens en el proveedor de ese agente. Cada llamada también cuenta para el presupuesto de arriba." } } } @@ -228,6 +230,28 @@ "description": "Desactivado por defecto. Envía todos los atributos de las entidades elegidas, no solo el valor, la unidad, la clase de dispositivo y el área. Una previsión meteorológica o la lista de carátulas de un reproductor multimedia llega a miles de tokens, y los pagas en cada evaluación." } } + }, + "calibrate": { + "name": "Encontrar el mejor umbral", + "description": "Compara el historial grabado de una probabilidad de sí o no con una entidad que muestra lo que era cierto de verdad, y devuelve el umbral que habría acertado más veces. Solo lee el recorder, así que no cuesta tokens.", + "fields": { + "entity_id": { + "name": "Probabilidad", + "description": "Un sensor de sí o no de Jev, o cualquier sensor cuyo estado sea una probabilidad de 0 a 1." + }, + "truth_entity_id": { + "name": "Lo que era cierto", + "description": "Una entidad cuyo estado da la respuesta real, como un contacto de puerta o un enchufe que indica cuándo funciona un aparato." + }, + "truth_state": { + "name": "Estado que significa sí", + "description": "El estado de esa entidad que significa sí. on si lo dejas vacío." + }, + "days": { + "name": "Días de historial", + "description": "Hasta dónde leer hacia atrás. El recorder guarda 10 días, salvo que lo hayas cambiado." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "La tarea {task} cuesta unos {estimate} tokens de entrada y quedan {remaining} del presupuesto diario de {budget}. Aumenta o borra el presupuesto en las opciones de la integración para continuar." }, + "action_over_budget": { + "message": "Esta acción cuesta unos {estimate} tokens de entrada y quedan {remaining} del presupuesto diario de {budget}. Aumenta o elimina el presupuesto en las opciones de la integración para continuar." + }, "reserved_field": { "message": "El campo {field} no se puede usar, porque el resultado lleva bajo ese nombre la confianza detrás de cada respuesta. Cambia el nombre del campo." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe limitó la solicitud: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Esta acción lee el recorder, y el recorder no está en marcha." + }, + "calibrate_no_history": { + "message": "Ningún momento de los últimos {days} días tiene a la vez un número de {entity} y un estado de {truth}. Comprueba que el recorder guarda los dos, y que {entity} da un número." + }, + "calibrate_never_true": { + "message": "{truth} nunca estuvo en {state} en los últimos {days} días, así que no hay nada con qué comparar. Lee un periodo más largo, o comprueba qué estado significa sí." + }, + "calibrate_always_true": { + "message": "{truth} estuvo en {state} durante todos los últimos {days} días, así que no hay nada con qué comparar. Lee un periodo más largo, o comprueba qué estado significa sí." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Lo siento, eso no ha funcionado.", "already_on": "{name} ya está encendido.", "already_off": "{name} ya está apagado.", + "done": "Hecho.", "preview_alone": "Se envía en una solicitud propia.", "preview_grouped": "Se envía en una sola solicitud junto con: {others}.", "preview_cost": "Preguntado una vez para esta vista previa: {tokens} tokens de entrada, unos ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "La API ha respondido, pero no a la pregunta que se le hizo.", "preview_template_error": "La plantilla no se procesa: {reason}", "query_not_found": "No he podido encontrarlo.", - "budget_spent": "El presupuesto diario de tokens se ha agotado, así que hoy no puedo hacerlo.", + "budget_spent": "No queda suficiente presupuesto diario de tokens para eso, así que hoy no puedo hacerlo.", "auth_failed": "TypeSafe rechazó la clave de API. Revísala en los ajustes de Jev.", "unavailable": "TypeSafe no respondió. Inténtalo de nuevo en un momento.", + "which_device": "¿Te refieres a {first} o a {second}?", "preview_over_budget": "El presupuesto diario de tokens está agotado, así que no hay respuesta de prueba. La pregunta sí se guarda.", "preview_nothing_yet": "Sin respuesta de prueba: todavía no hay nada sobre lo que preguntar." }, diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index d05d12e..9bba0a2 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -82,14 +82,16 @@ "price_per_million": "Prix par million de jetons d'entrée, en USD", "fallback_agent": "Se replier sur cet agent", "min_confidence": "N'agir qu'au-dessus de cette confiance (0 à 1)", - "allow_whole_home": "Autoriser les commandes qui ne nomment ni pièce ni appareil" + "allow_whole_home": "Autoriser les commandes qui ne nomment ni pièce ni appareil", + "llm_tools": "Proposer Jev comme outil aux autres agents LLM" }, "data_description": { "daily_token_budget": "Compté à partir de minuit dans le fuseau horaire de Home Assistant. 0 signifie aucune limite.", "price_per_million": "Détermine le capteur de coût estimé et le coût dans l'aperçu de la question. Rien de ce qui est envoyé n'en dépend.", "fallback_agent": "Reçoit la phrase entière quand Jev ne sait pas quoi faire.", "min_confidence": "En dessous, Jev ne fait rien et transmet la commande à l'agent de secours.", - "allow_whole_home": "Tout éteindre est toujours autorisé, avec ou sans cette option." + "allow_whole_home": "Tout éteindre est toujours autorisé, avec ou sans cette option.", + "llm_tools": "Ajoute deux outils à l'API Assist, l'un pour les questions oui/non et l'autre pour choisir une option. Chaque agent LLM qui utilise Assist reçoit leurs descriptions dans chaque prompt, ce qui coûte des tokens chez le fournisseur de cet agent. Chaque appel compte aussi dans le budget ci-dessus." } } } @@ -228,6 +230,28 @@ "description": "Désactivé par défaut. Envoie tous les attributs des entités choisies, pas seulement la valeur, l'unité, la classe d'appareil et la zone. Une prévision météo ou une liste de pochettes d'un lecteur multimédia représente des milliers de jetons, et vous les payez à chaque évaluation." } } + }, + "calibrate": { + "name": "Trouver le meilleur seuil", + "description": "Compare l'historique enregistré d'une probabilité oui/non avec une entité qui montre ce qui était vraiment vrai, et donne le seuil qui aurait eu raison le plus souvent. Lit seulement le recorder, donc ne coûte aucun token.", + "fields": { + "entity_id": { + "name": "Probabilité", + "description": "Un capteur oui/non Jev, ou tout capteur dont l'état est une probabilité de 0 à 1." + }, + "truth_entity_id": { + "name": "Ce qui était vraiment vrai", + "description": "Une entité dont l'état donne la vraie réponse, comme un contact de porte ou une prise qui signale quand un appareil tourne." + }, + "truth_state": { + "name": "État qui veut dire oui", + "description": "L'état de cette entité qui veut dire oui. on si vous laissez vide." + }, + "days": { + "name": "Jours d'historique", + "description": "Jusqu'où remonter. Le recorder garde 10 jours, sauf si vous l'avez modifié." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "La tâche {task} coûte environ {estimate} jetons d'entrée et il reste {remaining} sur le budget quotidien de {budget}. Augmentez ou supprimez le budget dans les options de l'intégration pour continuer." }, + "action_over_budget": { + "message": "Cette action coûte environ {estimate} jetons d'entrée et il reste {remaining} sur le budget quotidien de {budget}. Augmentez ou supprimez le budget dans les options de l'intégration pour continuer." + }, "reserved_field": { "message": "Le champ {field} ne peut pas être utilisé, car le résultat transporte sous ce nom la confiance derrière chaque réponse. Renommez le champ." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe a limité la requête : {reason}" + }, + "calibrate_needs_recorder": { + "message": "Cette action lit le recorder, et le recorder ne tourne pas." + }, + "calibrate_no_history": { + "message": "Aucun moment des {days} derniers jours n'a à la fois un nombre de {entity} et un état de {truth}. Vérifiez que le recorder enregistre les deux, et que {entity} donne un nombre." + }, + "calibrate_never_true": { + "message": "{truth} n'a jamais été {state} pendant les {days} derniers jours, il n'y a donc rien à comparer. Lisez une période plus longue, ou vérifiez quel état veut dire oui." + }, + "calibrate_always_true": { + "message": "{truth} a été {state} pendant tous les {days} derniers jours, il n'y a donc rien à comparer. Lisez une période plus longue, ou vérifiez quel état veut dire oui." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Désolé, cela n'a pas fonctionné.", "already_on": "{name} est déjà allumé.", "already_off": "{name} est déjà éteint.", + "done": "C'est fait.", "preview_alone": "Envoyée dans sa propre requête.", "preview_grouped": "Envoyée dans une seule requête avec : {others}.", "preview_cost": "Posée une fois pour cet aperçu : {tokens} jetons d'entrée, environ ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "L'API a répondu, mais pas à la question qui a été posée.", "preview_template_error": "Le rendu du modèle échoue : {reason}", "query_not_found": "Je n'ai pas trouvé cela.", - "budget_spent": "Le budget quotidien de jetons est épuisé, je ne peux donc pas le faire aujourd'hui.", + "budget_spent": "Il ne reste pas assez de budget quotidien de jetons pour cela, je ne peux donc pas le faire aujourd'hui.", "auth_failed": "TypeSafe a refusé la clé API. Vérifiez-la dans les paramètres de Jev.", "unavailable": "TypeSafe n'a pas répondu. Réessayez dans un instant.", + "which_device": "Voulez-vous dire {first} ou {second} ?", "preview_over_budget": "Le budget quotidien de jetons est épuisé, il n'y a donc pas de réponse d'essai. La question est quand même enregistrée.", "preview_nothing_yet": "Pas de réponse d'essai : il n'y a encore rien sur quoi poser la question." }, diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 9d88f65..c9a4031 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -82,14 +82,16 @@ "price_per_million": "Prezzo per milione di token di input, in USD", "fallback_agent": "Ripiega su questo agente", "min_confidence": "Agisci solo sopra questa confidenza (da 0 a 1)", - "allow_whole_home": "Consenti comandi che non nominano una stanza o un dispositivo" + "allow_whole_home": "Consenti comandi che non nominano una stanza o un dispositivo", + "llm_tools": "Offri Jev come strumento ad altri agenti LLM" }, "data_description": { "daily_token_budget": "Conteggiato dalla mezzanotte nel fuso orario di Home Assistant. 0 significa nessun limite.", "price_per_million": "Determina il sensore del costo stimato e il costo nell'anteprima della domanda. Niente di ciò che viene inviato dipende da questo.", "fallback_agent": "Riceve l'intera frase quando Jev non è sicuro di cosa fare.", "min_confidence": "Sotto questo valore Jev non fa nulla e passa il comando all'agente di riserva.", - "allow_whole_home": "Spegnere tutto è sempre consentito, con o senza questa opzione." + "allow_whole_home": "Spegnere tutto è sempre consentito, con o senza questa opzione.", + "llm_tools": "Aggiunge due strumenti all'API Assist, uno per le domande sì/no e uno per scegliere un'opzione. Ogni agente LLM che usa Assist riceve le loro descrizioni in ogni prompt, il che costa token presso il fornitore di quell'agente. Ogni chiamata conta anche nel budget qui sopra." } } } @@ -228,6 +230,28 @@ "description": "Disattivo per impostazione predefinita. Invia ogni attributo delle entità scelte, non solo il valore, l'unità, la classe del dispositivo e l'area. Una previsione meteo o l'elenco delle copertine di un lettore multimediale arriva a migliaia di token, e li paghi a ogni valutazione." } } + }, + "calibrate": { + "name": "Trova la soglia migliore", + "description": "Confronta la cronologia registrata di una probabilità sì/no con un'entità che mostra cosa era davvero vero, e restituisce la soglia che avrebbe indovinato più spesso. Legge solo il recorder, quindi non costa token.", + "fields": { + "entity_id": { + "name": "Probabilità", + "description": "Un sensore sì/no di Jev, o qualsiasi sensore il cui stato sia una probabilità da 0 a 1." + }, + "truth_entity_id": { + "name": "Cosa era davvero vero", + "description": "Un'entità il cui stato dà la risposta reale, come un contatto porta o una presa che segnala quando un apparecchio è in funzione." + }, + "truth_state": { + "name": "Stato che significa sì", + "description": "Lo stato di quell'entità che significa sì. on se lo lasci vuoto." + }, + "days": { + "name": "Giorni di cronologia", + "description": "Quanto indietro leggere. Il recorder conserva 10 giorni, salvo modifiche." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "L'attività {task} costa circa {estimate} token di input e restano {remaining} del budget giornaliero di {budget}. Aumenta o cancella il budget nelle opzioni dell'integrazione per continuare." }, + "action_over_budget": { + "message": "Questa azione costa circa {estimate} token di input e restano {remaining} del budget giornaliero di {budget}. Aumenta o rimuovi il budget nelle opzioni dell'integrazione per continuare." + }, "reserved_field": { "message": "Il campo {field} non può essere usato, perché il risultato porta con quel nome la confidenza dietro ogni risposta. Rinomina il campo." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe ha limitato la richiesta: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Questa azione legge il recorder, e il recorder non è in esecuzione." + }, + "calibrate_no_history": { + "message": "Nessun momento degli ultimi {days} giorni ha sia un numero da {entity} sia uno stato da {truth}. Controlla che il recorder registri entrambi e che {entity} dia un numero." + }, + "calibrate_never_true": { + "message": "{truth} non è mai stato {state} negli ultimi {days} giorni, quindi non c'è nulla da confrontare. Leggi un periodo più lungo, o controlla quale stato significa sì." + }, + "calibrate_always_true": { + "message": "{truth} è stato {state} per tutti gli ultimi {days} giorni, quindi non c'è nulla da confrontare. Leggi un periodo più lungo, o controlla quale stato significa sì." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Scusa, non ha funzionato.", "already_on": "{name} è già acceso.", "already_off": "{name} è già spento.", + "done": "Fatto.", "preview_alone": "Inviata come richiesta a sé stante.", "preview_grouped": "Inviata in un'unica richiesta insieme a: {others}.", "preview_cost": "Chiesto una volta per questa anteprima: {tokens} token di input, circa ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "L'API ha risposto, ma non alla domanda che è stata posta.", "preview_template_error": "Il template non si elabora: {reason}", "query_not_found": "Non sono riuscito a trovarlo.", - "budget_spent": "Il budget giornaliero di token è esaurito, quindi oggi non posso farlo.", + "budget_spent": "Non resta abbastanza budget giornaliero di token per questo, quindi oggi non posso farlo.", "auth_failed": "TypeSafe ha rifiutato la chiave API. Controllala nelle impostazioni di Jev.", "unavailable": "TypeSafe non ha risposto. Riprova tra un momento.", + "which_device": "Intendi {first} o {second}?", "preview_over_budget": "Il budget giornaliero di token è esaurito, quindi non c'è una risposta di prova. La domanda viene comunque salvata.", "preview_nothing_yet": "Nessuna risposta di prova: non c'è ancora nulla su cui fare la domanda." }, diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 48220fd..39e803e 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -82,14 +82,16 @@ "price_per_million": "Prijs per miljoen invoertokens, in USD", "fallback_agent": "Val terug op deze agent", "min_confidence": "Alleen handelen boven dit vertrouwen (0 tot 1)", - "allow_whole_home": "Sta opdrachten toe die geen kamer of apparaat noemen" + "allow_whole_home": "Sta opdrachten toe die geen kamer of apparaat noemen", + "llm_tools": "Jev als tool aanbieden aan andere LLM-agents" }, "data_description": { "daily_token_budget": "Geteld vanaf middernacht in de tijdzone van Home Assistant. 0 betekent geen limiet.", "price_per_million": "Bepaalt de sensor voor de geschatte kosten en de kosten in het voorbeeld van een vraag. Niets van wat wordt verstuurd hangt ervan af.", "fallback_agent": "Krijgt de hele zin wanneer Jev niet zeker weet wat te doen.", "min_confidence": "Daaronder doet Jev niets en geeft het de opdracht door aan de terugvalagent.", - "allow_whole_home": "Alles uitzetten mag altijd, met of zonder deze optie." + "allow_whole_home": "Alles uitzetten mag altijd, met of zonder deze optie.", + "llm_tools": "Voegt twee tools toe aan de Assist-API, een voor ja/nee-vragen en een voor het kiezen van een optie. Elke LLM-agent die Assist gebruikt, krijgt hun beschrijvingen in elke prompt, en dat kost tokens bij de provider van die agent. Elke aanroep telt ook mee voor het budget hierboven." } } } @@ -228,6 +230,28 @@ "description": "Staat uit. Stuurt alle attributen van de gekozen entiteiten mee, niet alleen de waarde, eenheid, device class en ruimte. Een weersverwachting of een lijst met albumhoezen loopt in de duizenden tokens, en die betaal je bij elke evaluatie." } } + }, + "calibrate": { + "name": "Beste drempel bepalen", + "description": "Vergelijkt de opgeslagen geschiedenis van een ja/nee-kans met een entiteit die laat zien wat echt waar was, en geeft de drempel die het vaakst goed zou zijn geweest. Leest alleen de recorder en kost dus geen tokens.", + "fields": { + "entity_id": { + "name": "Kans", + "description": "Een Jev ja/nee-sensor, of een andere sensor waarvan de status een kans van 0 tot 1 is." + }, + "truth_entity_id": { + "name": "Wat echt waar was", + "description": "Een entiteit waarvan de status het echte antwoord geeft, zoals een deurcontact of een stekker die meldt wanneer een apparaat draait." + }, + "truth_state": { + "name": "Status die ja betekent", + "description": "De status van die entiteit die ja betekent. on als je dit leeg laat." + }, + "days": { + "name": "Dagen geschiedenis", + "description": "Hoe ver terug er gelezen wordt. De recorder bewaart 10 dagen, tenzij je dat hebt aangepast." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Taak {task} kost ongeveer {estimate} invoertokens en van het dagbudget van {budget} zijn er nog {remaining} over. Verhoog of wis het budget in de opties van de integratie om door te gaan." }, + "action_over_budget": { + "message": "Deze actie kost ongeveer {estimate} invoertokens en er is nog {remaining} over van het dagbudget van {budget}. Verhoog of wis het budget in de opties van de integratie om door te gaan." + }, "reserved_field": { "message": "Het veld {field} kan niet worden gebruikt, omdat het resultaat onder die naam de zekerheid achter elk antwoord meegeeft. Geef het veld een andere naam." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe heeft het verzoek begrensd: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Deze actie leest de recorder, en de recorder draait niet." + }, + "calibrate_no_history": { + "message": "Er is in de laatste {days} dagen geen moment met zowel een getal van {entity} als een status van {truth}. Controleer of de recorder beide opslaat, en of {entity} een getal geeft." + }, + "calibrate_never_true": { + "message": "{truth} was in de laatste {days} dagen nooit {state}, dus er is niets om mee te vergelijken. Lees een langere periode, of controleer welke status ja betekent." + }, + "calibrate_always_true": { + "message": "{truth} was de hele laatste {days} dagen {state}, dus er is niets om mee te vergelijken. Lees een langere periode, of controleer welke status ja betekent." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Sorry, dat is niet gelukt.", "already_on": "{name} staat al aan.", "already_off": "{name} staat al uit.", + "done": "Gedaan.", "preview_alone": "Gaat als eigen verzoek.", "preview_grouped": "Gaat in een verzoek samen met: {others}.", "preview_cost": "Een keer gevraagd voor deze controle: {tokens} invoertokens, ongeveer ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "De API antwoordde, maar niet op de vraag die gesteld is.", "preview_template_error": "De template rendert niet: {reason}", "query_not_found": "Dat kon ik niet vinden.", - "budget_spent": "Het dagelijkse tokenbudget is op, dus dat kan ik vandaag niet doen.", + "budget_spent": "Er is niet genoeg over van het dagelijkse tokenbudget, dus dat kan ik vandaag niet doen.", "auth_failed": "TypeSafe heeft de API-sleutel geweigerd. Controleer hem in de instellingen van Jev.", "unavailable": "TypeSafe gaf geen antwoord. Probeer het zo nog eens.", + "which_device": "Bedoel je {first} of {second}?", "preview_over_budget": "Het daglimiet voor tokens is op, dus er is geen proefantwoord. De vraag wordt wel opgeslagen.", "preview_nothing_yet": "Geen proefantwoord: er is nog niets om over te vragen." }, diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index a5765eb..8c2b8c5 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -82,14 +82,16 @@ "price_per_million": "Cena za milion tokenów wejściowych, w USD", "fallback_agent": "Agent zapasowy", "min_confidence": "Działaj tylko powyżej tej pewności (0 do 1)", - "allow_whole_home": "Zezwalaj na polecenia, które nie wskazują pokoju ani urządzenia" + "allow_whole_home": "Zezwalaj na polecenia, które nie wskazują pokoju ani urządzenia", + "llm_tools": "Udostępnij Jev jako narzędzie innym agentom LLM" }, "data_description": { "daily_token_budget": "Liczone od północy w strefie czasowej Home Assistanta. 0 oznacza brak limitu.", "price_per_million": "Ustala czujnik szacowanego kosztu i koszt w podglądzie pytania. Nic z tego, co jest wysyłane, od tego nie zależy.", "fallback_agent": "Dostaje całe zdanie, gdy Jev nie jest pewny, co zrobić.", "min_confidence": "Poniżej tej wartości Jev nic nie robi i przekazuje polecenie agentowi zapasowemu.", - "allow_whole_home": "Wyłączenie wszystkiego jest zawsze dozwolone, z tą opcją lub bez niej." + "allow_whole_home": "Wyłączenie wszystkiego jest zawsze dozwolone, z tą opcją lub bez niej.", + "llm_tools": "Dodaje do API Assist dwa narzędzia, jedno do pytań tak/nie i jedno do wyboru opcji. Każdy agent LLM korzystający z Assist dostaje ich opisy w każdym prompcie, co kosztuje tokeny u dostawcy tego agenta. Każde wywołanie liczy się też do budżetu powyżej." } } } @@ -228,6 +230,28 @@ "description": "Domyślnie wyłączone. Wysyła każdy atrybut wybranych encji, nie tylko wartość, jednostkę, klasę urządzenia i obszar. Prognoza pogody albo lista okładek odtwarzacza multimediów to tysiące tokenów, za które płacisz przy każdej ocenie." } } + }, + "calibrate": { + "name": "Znajdź najlepszy próg", + "description": "Porównuje zapisaną historię prawdopodobieństwa tak/nie z encją, która pokazuje, co było naprawdę prawdą, i zwraca próg, który najczęściej miałby rację. Czyta tylko recorder, więc nie kosztuje tokenów.", + "fields": { + "entity_id": { + "name": "Prawdopodobieństwo", + "description": "Czujnik tak/nie z Jev albo dowolny czujnik, którego stan jest prawdopodobieństwem od 0 do 1." + }, + "truth_entity_id": { + "name": "Co było prawdą", + "description": "Encja, której stan pokazuje prawdziwą odpowiedź, na przykład kontaktron drzwi albo gniazdko, które zgłasza, kiedy urządzenie pracuje." + }, + "truth_state": { + "name": "Stan oznaczający tak", + "description": "Stan tej encji, który oznacza tak. on, jeśli zostawisz pole puste." + }, + "days": { + "name": "Dni historii", + "description": "Jak daleko wstecz czytać. Recorder przechowuje 10 dni, chyba że to zmieniono." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Zadanie {task} kosztuje około {estimate} tokenów wejściowych, a z dziennego budżetu {budget} zostało {remaining}. Zwiększ lub wyczyść budżet w opcjach integracji, aby kontynuować." }, + "action_over_budget": { + "message": "Ta akcja kosztuje około {estimate} tokenów wejściowych, a z dziennego budżetu {budget} zostało {remaining}. Aby kontynuować, zwiększ lub usuń budżet w opcjach integracji." + }, "reserved_field": { "message": "Pola {field} nie można użyć, ponieważ wynik przenosi pod tą nazwą pewność każdej odpowiedzi. Zmień nazwę pola." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe ograniczył żądanie: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Ta akcja czyta recorder, a recorder nie działa." + }, + "calibrate_no_history": { + "message": "W ostatnich {days} dniach nie ma chwili, w której jest zarówno liczba z {entity}, jak i stan z {truth}. Sprawdź, czy recorder zapisuje obie encje i czy {entity} podaje liczbę." + }, + "calibrate_never_true": { + "message": "{truth} ani razu nie miał stanu {state} w ostatnich {days} dniach, więc nie ma z czym porównać. Czytaj dłuższy okres albo sprawdź, który stan oznacza tak." + }, + "calibrate_always_true": { + "message": "{truth} miał stan {state} przez całe ostatnie {days} dni, więc nie ma z czym porównać. Czytaj dłuższy okres albo sprawdź, który stan oznacza tak." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Przepraszam, to się nie udało.", "already_on": "{name} jest już włączone.", "already_off": "{name} jest już wyłączone.", + "done": "Gotowe.", "preview_alone": "Wysyłane jako osobne żądanie.", "preview_grouped": "Wysyłane w jednym żądaniu razem z: {others}.", "preview_cost": "Zapytano raz na potrzeby tego podglądu: {tokens} tokenów wejściowych, około ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "API odpowiedziało, ale nie na zadane pytanie.", "preview_template_error": "Szablon się nie renderuje: {reason}", "query_not_found": "Nie udało mi się tego znaleźć.", - "budget_spent": "Dzienny budżet tokenów się wyczerpał, więc dziś nie mogę tego zrobić.", + "budget_spent": "W dziennym budżecie tokenów nie zostało na to dość, więc dziś nie mogę tego zrobić.", "auth_failed": "TypeSafe odrzucił klucz API. Sprawdź go w ustawieniach Jev.", "unavailable": "TypeSafe nie odpowiedział. Spróbuj ponownie za chwilę.", + "which_device": "Chodzi o {first} czy {second}?", "preview_over_budget": "Dzienny limit tokenów jest wyczerpany, więc nie ma próbnej odpowiedzi. Pytanie i tak zostanie zapisane.", "preview_nothing_yet": "Brak próbnej odpowiedzi: nie ma jeszcze o co pytać." }, diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 22b1c26..da23996 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -82,14 +82,16 @@ "price_per_million": "Preço por milhão de tokens de entrada, em USD", "fallback_agent": "Recorrer a este agente", "min_confidence": "Agir apenas acima desta confiança (0 a 1)", - "allow_whole_home": "Permitir comandos que não citam cômodo nem dispositivo" + "allow_whole_home": "Permitir comandos que não citam cômodo nem dispositivo", + "llm_tools": "Oferecer o Jev como ferramenta para outros agentes LLM" }, "data_description": { "daily_token_budget": "Contado a partir da meia-noite no fuso horário do Home Assistant. 0 significa sem limite.", "price_per_million": "Define o sensor de custo estimado e o custo na prévia da pergunta. Nada do que é enviado depende disso.", "fallback_agent": "Recebe a frase inteira quando o Jev não tem certeza do que fazer.", "min_confidence": "Abaixo disso, o Jev não faz nada e passa o comando para o agente reserva.", - "allow_whole_home": "Desligar tudo é sempre permitido, com ou sem esta opção." + "allow_whole_home": "Desligar tudo é sempre permitido, com ou sem esta opção.", + "llm_tools": "Adiciona duas ferramentas à API do Assist, uma para perguntas de sim ou não e outra para escolher uma opção. Cada agente LLM que usa o Assist recebe as descrições delas em todo prompt, o que custa tokens no provedor desse agente. Cada chamada também conta no orçamento acima." } } } @@ -228,6 +230,28 @@ "description": "Desligado por padrão. Envia todos os atributos das entidades escolhidas, não apenas o valor, a unidade, a classe de dispositivo e a área. Uma previsão do tempo ou a lista de capas de um reprodutor de mídia chega a milhares de tokens, e você paga por eles em cada avaliação." } } + }, + "calibrate": { + "name": "Encontrar o melhor limite", + "description": "Compara o histórico gravado de uma probabilidade de sim ou não com uma entidade que mostra o que era de fato verdade, e retorna o limite que teria acertado mais vezes. Lê apenas o recorder, então não custa tokens.", + "fields": { + "entity_id": { + "name": "Probabilidade", + "description": "Um sensor de sim ou não do Jev, ou qualquer sensor cujo estado seja uma probabilidade de 0 a 1." + }, + "truth_entity_id": { + "name": "O que era verdade", + "description": "Uma entidade cujo estado mostra a resposta real, como um contato de porta ou uma tomada que informa quando um aparelho está ligado." + }, + "truth_state": { + "name": "Estado que significa sim", + "description": "O estado dessa entidade que significa sim. on se você deixar vazio." + }, + "days": { + "name": "Dias de histórico", + "description": "Até onde ler para trás. O recorder guarda 10 dias, a menos que você tenha mudado isso." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "A tarefa {task} custa cerca de {estimate} tokens de entrada e restam {remaining} do orçamento diário de {budget}. Aumente ou limpe o orçamento nas opções da integração para continuar." }, + "action_over_budget": { + "message": "Esta ação custa cerca de {estimate} tokens de entrada e restam {remaining} do orçamento diário de {budget}. Aumente ou remova o orçamento nas opções da integração para continuar." + }, "reserved_field": { "message": "O campo {field} não pode ser usado, porque o resultado leva sob esse nome a confiança por trás de cada resposta. Renomeie o campo." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "O TypeSafe limitou a solicitação: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Esta ação lê o recorder, e o recorder não está em execução." + }, + "calibrate_no_history": { + "message": "Nenhum momento dos últimos {days} dias tem ao mesmo tempo um número de {entity} e um estado de {truth}. Verifique se o recorder grava os dois e se {entity} informa um número." + }, + "calibrate_never_true": { + "message": "{truth} nunca esteve em {state} nos últimos {days} dias, então não há com o que comparar. Leia um período maior, ou verifique qual estado significa sim." + }, + "calibrate_always_true": { + "message": "{truth} esteve em {state} durante todos os últimos {days} dias, então não há com o que comparar. Leia um período maior, ou verifique qual estado significa sim." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Desculpe, isso não funcionou.", "already_on": "{name} já está ligado.", "already_off": "{name} já está desligado.", + "done": "Pronto.", "preview_alone": "Enviada como requisição própria.", "preview_grouped": "Enviada em uma requisição junto com: {others}.", "preview_cost": "Perguntado uma vez para esta prévia: {tokens} tokens de entrada, cerca de ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "A API respondeu, mas não à pergunta que foi feita.", "preview_template_error": "O template não é renderizado: {reason}", "query_not_found": "Não consegui encontrar isso.", - "budget_spent": "O orçamento diário de tokens acabou, então não posso fazer isso hoje.", + "budget_spent": "Não resta orçamento diário de tokens suficiente para isso, então não posso fazer isso hoje.", "auth_failed": "A TypeSafe rejeitou a chave de API. Verifique-a nas configurações do Jev.", "unavailable": "A TypeSafe não respondeu. Tente novamente em instantes.", + "which_device": "Você quer dizer {first} ou {second}?", "preview_over_budget": "O orçamento diário de tokens acabou, então não há resposta de teste. A pergunta ainda é salva.", "preview_nothing_yet": "Sem resposta de teste: ainda não há nada sobre o que perguntar." }, diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 1f50471..df4df42 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -82,14 +82,16 @@ "price_per_million": "Цена за миллион входных токенов, в USD", "fallback_agent": "Резервный агент", "min_confidence": "Действовать только выше этой уверенности (от 0 до 1)", - "allow_whole_home": "Разрешить команды, в которых не названы комната или устройство" + "allow_whole_home": "Разрешить команды, в которых не названы комната или устройство", + "llm_tools": "Предложить Jev как инструмент другим LLM-агентам" }, "data_description": { "daily_token_budget": "Считается с полуночи в часовом поясе Home Assistant. 0 означает без ограничения.", "price_per_million": "Задаёт датчик оценочной стоимости и стоимость в предпросмотре вопроса. От него не зависит ничего из того, что отправляется.", "fallback_agent": "Получает всю фразу, когда Jev не уверен, что делать.", "min_confidence": "Ниже этого значения Jev ничего не делает и передаёт команду резервному агенту.", - "allow_whole_home": "Выключить всё можно всегда, с этой настройкой или без неё." + "allow_whole_home": "Выключить всё можно всегда, с этой настройкой или без неё.", + "llm_tools": "Добавляет в API Assist два инструмента: один для вопросов да/нет и один для выбора варианта. Каждый LLM-агент, использующий Assist, получает их описания в каждом запросе, и это стоит токенов у провайдера этого агента. Каждый вызов также учитывается в бюджете выше." } } } @@ -228,6 +230,28 @@ "description": "По умолчанию выключено. Передаёт каждый атрибут выбранных объектов, а не только значение, единицу измерения, класс устройства и помещение. Прогноз погоды или список обложек в медиаплеере занимает тысячи токенов, и вы платите за них при каждой оценке." } } + }, + "calibrate": { + "name": "Найти лучший порог", + "description": "Сравнивает записанную историю вероятности да/нет с объектом, который показывает, что было на самом деле, и возвращает порог, который чаще всего оказался бы прав. Читает только recorder, поэтому не тратит токены.", + "fields": { + "entity_id": { + "name": "Вероятность", + "description": "Датчик да/нет от Jev или любой датчик, состояние которого является вероятностью от 0 до 1." + }, + "truth_entity_id": { + "name": "Что было на самом деле", + "description": "Объект, состояние которого показывает настоящий ответ, например датчик двери или розетка, которая сообщает, когда работает прибор." + }, + "truth_state": { + "name": "Состояние, означающее да", + "description": "Состояние этого объекта, которое означает да. on, если оставить поле пустым." + }, + "days": { + "name": "Дней истории", + "description": "Насколько далеко назад читать. Recorder хранит 10 дней, если вы это не меняли." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Задача {task} стоит около {estimate} входных токенов, а из дневного бюджета {budget} осталось {remaining}. Чтобы продолжить, увеличьте или очистите бюджет в параметрах интеграции." }, + "action_over_budget": { + "message": "Это действие стоит около {estimate} входных токенов, а от дневного бюджета {budget} осталось {remaining}. Чтобы продолжить, увеличьте или снимите бюджет в параметрах интеграции." + }, "reserved_field": { "message": "Поле {field} использовать нельзя, потому что под этим именем результат передаёт уверенность каждого ответа. Переименуйте поле." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe ограничил запрос: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Это действие читает recorder, а recorder не запущен." + }, + "calibrate_no_history": { + "message": "За последние {days} дней нет ни одного момента, где есть и число от {entity}, и состояние от {truth}. Проверьте, что recorder записывает оба объекта и что {entity} сообщает число." + }, + "calibrate_never_true": { + "message": "{truth} ни разу не был в состоянии {state} за последние {days} дней, поэтому сравнивать не с чем. Прочитайте более длинный период или проверьте, какое состояние означает да." + }, + "calibrate_always_true": { + "message": "{truth} был в состоянии {state} все последние {days} дней, поэтому сравнивать не с чем. Прочитайте более длинный период или проверьте, какое состояние означает да." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Извините, не получилось.", "already_on": "{name} уже включён.", "already_off": "{name} уже выключен.", + "done": "Готово.", "preview_alone": "Отправляется отдельным запросом.", "preview_grouped": "Отправляется одним запросом вместе с: {others}.", "preview_cost": "Один вопрос для этого предпросмотра: {tokens} входных токенов, примерно ${cost}, {ms} мс.", @@ -364,9 +404,10 @@ "preview_mismatched": "API ответил, но не на тот вопрос, который был задан.", "preview_template_error": "Шаблон не обрабатывается: {reason}", "query_not_found": "Мне не удалось это найти.", - "budget_spent": "Дневной бюджет токенов исчерпан, поэтому сегодня я не могу это сделать.", + "budget_spent": "В дневном бюджете токенов на это не хватает, поэтому сегодня я не могу это сделать.", "auth_failed": "TypeSafe отклонил ключ API. Проверьте его в настройках Jev.", "unavailable": "TypeSafe не ответил. Попробуйте ещё раз чуть позже.", + "which_device": "Вы имеете в виду {first} или {second}?", "preview_over_budget": "Дневной бюджет токенов исчерпан, поэтому пробного ответа нет. Вопрос всё равно сохраняется.", "preview_nothing_yet": "Пробного ответа нет: пока не о чем спрашивать." }, diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index d49d5b2..82f016e 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -82,14 +82,16 @@ "price_per_million": "Pris per miljon indatatoken, i USD", "fallback_agent": "Fall tillbaka på den här agenten", "min_confidence": "Agera bara över den här säkerheten (0 till 1)", - "allow_whole_home": "Tillåt kommandon som inte nämner rum eller enhet" + "allow_whole_home": "Tillåt kommandon som inte nämner rum eller enhet", + "llm_tools": "Erbjud Jev som verktyg till andra LLM-agenter" }, "data_description": { "daily_token_budget": "Räknas från midnatt i Home Assistants tidszon. 0 betyder ingen gräns.", "price_per_million": "Styr sensorn för uppskattad kostnad och kostnaden i förhandsvisningen av frågan. Inget av det som skickas beror på den.", "fallback_agent": "Får hela meningen när Jev inte är säker på vad som ska göras.", "min_confidence": "Under detta värde gör Jev ingenting och skickar kommandot vidare till reservagenten.", - "allow_whole_home": "Att stänga av allt är alltid tillåtet, med eller utan detta val." + "allow_whole_home": "Att stänga av allt är alltid tillåtet, med eller utan detta val.", + "llm_tools": "Lägger till två verktyg i Assist-API:et, ett för ja/nej-frågor och ett för att välja ett alternativ. Varje LLM-agent som använder Assist får deras beskrivningar i varje prompt, vilket kostar tokens hos den agentens leverantör. Varje anrop räknas också mot budgeten ovan." } } } @@ -228,6 +230,28 @@ "description": "Av som standard. Skickar alla attribut för de valda entiteterna, inte bara värde, måttenhet, enhetsklass och område. En väderprognos eller en mediaspelares lista med omslagsbilder blir tusentals token, och du betalar för dem vid varje utvärdering." } } + }, + "calibrate": { + "name": "Hitta den bästa tröskeln", + "description": "Jämför den inspelade historiken för en ja/nej-sannolikhet med en entitet som visar vad som verkligen var sant, och ger den tröskel som oftast skulle ha haft rätt. Läser bara recordern och kostar därför inga tokens.", + "fields": { + "entity_id": { + "name": "Sannolikhet", + "description": "En Jev ja/nej-sensor eller en annan sensor vars tillstånd är en sannolikhet från 0 till 1." + }, + "truth_entity_id": { + "name": "Vad som verkligen var sant", + "description": "En entitet vars tillstånd visar det riktiga svaret, till exempel en dörrkontakt eller ett uttag som rapporterar när en apparat är igång." + }, + "truth_state": { + "name": "Tillstånd som betyder ja", + "description": "Tillståndet för entiteten som betyder ja. on om du lämnar fältet tomt." + }, + "days": { + "name": "Dagar historik", + "description": "Hur långt tillbaka som läses. Recordern sparar 10 dagar om du inte har ändrat det." + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "Uppgiften {task} kostar ungefär {estimate} indatatokens och {remaining} av dagsbudgeten på {budget} återstår. Höj eller rensa budgeten i integrationens alternativ för att fortsätta." }, + "action_over_budget": { + "message": "Den här åtgärden kostar ungefär {estimate} indatatokens och det finns {remaining} kvar av den dagliga budgeten på {budget}. Höj eller ta bort budgeten i integrationens alternativ för att fortsätta." + }, "reserved_field": { "message": "Fältet {field} kan inte användas, eftersom resultatet bär säkerheten bakom varje svar under det namnet. Byt namn på fältet." }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe begränsade begäran: {reason}" + }, + "calibrate_needs_recorder": { + "message": "Den här åtgärden läser recordern, och recordern körs inte." + }, + "calibrate_no_history": { + "message": "Ingen tidpunkt de senaste {days} dagarna har både ett tal från {entity} och ett tillstånd från {truth}. Kontrollera att recordern sparar båda och att {entity} rapporterar ett tal." + }, + "calibrate_never_true": { + "message": "{truth} var aldrig {state} de senaste {days} dagarna, så det finns inget att jämföra med. Läs en längre period, eller kontrollera vilket tillstånd som betyder ja." + }, + "calibrate_always_true": { + "message": "{truth} var {state} under hela de senaste {days} dagarna, så det finns inget att jämföra med. Läs en längre period, eller kontrollera vilket tillstånd som betyder ja." } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "Tyvärr, det gick inte.", "already_on": "{name} är redan på.", "already_off": "{name} är redan av.", + "done": "Klart.", "preview_alone": "Skickas som en egen förfrågan.", "preview_grouped": "Skickas i en förfrågan tillsammans med: {others}.", "preview_cost": "Frågade en gång för den här förhandsgranskningen: {tokens} indatatoken, ungefär ${cost}, {ms} ms.", @@ -364,9 +404,10 @@ "preview_mismatched": "API:et svarade, men inte på frågan som ställdes.", "preview_template_error": "Mallen renderas inte: {reason}", "query_not_found": "Det kunde jag inte hitta.", - "budget_spent": "Den dagliga tokenbudgeten är slut, så det kan jag inte göra i dag.", + "budget_spent": "Det finns inte tillräckligt kvar av den dagliga tokenbudgeten för det, så det kan jag inte göra i dag.", "auth_failed": "TypeSafe avvisade API-nyckeln. Kontrollera den i inställningarna för Jev.", "unavailable": "TypeSafe svarade inte. Försök igen om en stund.", + "which_device": "Menar du {first} eller {second}?", "preview_over_budget": "Den dagliga budgeten för token är slut, så det finns inget provsvar. Frågan sparas ändå.", "preview_nothing_yet": "Inget provsvar: det finns inget att fråga om än." }, diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index 85f813c..3041a51 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -82,14 +82,16 @@ "price_per_million": "每百万输入令牌的价格,单位 USD", "fallback_agent": "回退到此代理", "min_confidence": "仅在置信度高于此值时执行(0 到 1)", - "allow_whole_home": "允许未指明房间或设备的命令" + "allow_whole_home": "允许未指明房间或设备的命令", + "llm_tools": "将 Jev 作为工具提供给其他 LLM 代理" }, "data_description": { "daily_token_budget": "从 Home Assistant 时区的午夜开始计算。0 表示不限制。", "price_per_million": "决定预估费用传感器和问题预览中的费用。发送的内容都不受它影响。", "fallback_agent": "当 Jev 不确定该做什么时,接收完整的句子。", "min_confidence": "低于此值时,Jev 不执行任何操作,并把命令交给备用代理。", - "allow_whole_home": "无论是否开启此项,关闭所有设备始终允许。" + "allow_whole_home": "无论是否开启此项,关闭所有设备始终允许。", + "llm_tools": "向 Assist API 添加两个工具,一个用于是/否问题,一个用于选择选项。每个使用 Assist 的 LLM 代理都会在每条提示中收到它们的描述,这会在该代理的提供商处消耗 token。每次调用也计入上面的预算。" } } } @@ -228,6 +230,28 @@ "description": "默认关闭。会发送所选实体的每一个属性,而不只是值、单位、设备类别和区域。一份天气预报或一个媒体播放器的封面列表可达数千个令牌,而且每次评估都要为此付费。" } } + }, + "calibrate": { + "name": "找出最佳阈值", + "description": "将是/否概率的记录历史与显示真实情况的实体进行比较,并返回最常判断正确的阈值。只读取 recorder,因此不消耗 token。", + "fields": { + "entity_id": { + "name": "概率", + "description": "Jev 的是/否传感器,或任何状态为 0 到 1 概率的传感器。" + }, + "truth_entity_id": { + "name": "真实情况", + "description": "状态显示真实答案的实体,例如门磁或报告设备何时运行的插座。" + }, + "truth_state": { + "name": "表示是的状态", + "description": "该实体表示“是”的状态。留空时为 on。" + }, + "days": { + "name": "历史天数", + "description": "向前读取多少天。除非你修改过,recorder 保留 10 天。" + } + } } }, "entity": { @@ -306,6 +330,9 @@ "task_over_budget": { "message": "任务 {task} 大约需要 {estimate} 个输入令牌,而每日预算 {budget} 只剩 {remaining}。请在集成选项中提高或清除预算后再继续。" }, + "action_over_budget": { + "message": "此操作约需 {estimate} 个输入令牌,每日预算 {budget} 还剩 {remaining}。要继续,请在集成选项中提高或清除预算。" + }, "reserved_field": { "message": "字段 {field} 不能使用,因为结果在这个名称下携带每个答案背后的置信度。请重命名该字段。" }, @@ -341,6 +368,18 @@ }, "rate_limited": { "message": "TypeSafe 限制了该请求的速率:{reason}" + }, + "calibrate_needs_recorder": { + "message": "此操作需要读取 recorder,但 recorder 未运行。" + }, + "calibrate_no_history": { + "message": "过去 {days} 天中,没有任何时刻同时有 {entity} 的数值和 {truth} 的状态。请检查 recorder 是否记录了两者,以及 {entity} 是否报告数值。" + }, + "calibrate_never_true": { + "message": "过去 {days} 天中 {truth} 从未处于 {state},因此没有可比较的内容。请读取更长的时间段,或检查哪个状态表示“是”。" + }, + "calibrate_always_true": { + "message": "过去 {days} 天中 {truth} 一直处于 {state},因此没有可比较的内容。请读取更长的时间段,或检查哪个状态表示“是”。" } }, "common": { @@ -350,6 +389,7 @@ "intent_failed": "抱歉,没有成功。", "already_on": "{name} 已经开着了。", "already_off": "{name} 已经关着了。", + "done": "好的,完成了。", "preview_alone": "作为单独的请求发送。", "preview_grouped": "与以下问题合并在一个请求中发送:{others}。", "preview_cost": "为本次预览提问一次:{tokens} 个输入令牌,约 ${cost},{ms} 毫秒。", @@ -364,9 +404,10 @@ "preview_mismatched": "API 作出了回答,但回答的不是所提的那个问题。", "preview_template_error": "模板无法渲染:{reason}", "query_not_found": "我找不到那个。", - "budget_spent": "今天的令牌预算已用完,所以今天无法执行。", + "budget_spent": "今天剩余的令牌预算不足,所以无法执行。", "auth_failed": "TypeSafe 拒绝了 API 密钥。请在 Jev 设置中检查。", "unavailable": "TypeSafe 没有响应。请稍后再试。", + "which_device": "你是指{first}还是{second}?", "preview_over_budget": "每日令牌预算已用完,因此没有试答结果。问题仍会保存。", "preview_nothing_yet": "没有试答结果:目前还没有可提问的内容。" }, diff --git a/mkdocs.yml b/mkdocs.yml index 80c1b7b..b40339a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,7 +62,7 @@ plugins: A Home Assistant custom integration, installed through HACS. Questions become sensors: a probability, one of a set of options, or a number on a scale. Four actions (jev.noul, jev.choice, jev.score, jev.ask) answer inside an - automation, an AI Task entity answers ai_task.generate_data, and a + automation, jev.calibrate fits a threshold from history, an AI Task entity answers ai_task.generate_data, and a conversation agent handles Assist. Every call is billed in input tokens against an optional daily budget. sections: @@ -79,10 +79,13 @@ plugins: - actions.md: The four actions, their fields and their response variables - ai-task.md: The ai_task.generate_data entity and the structures it accepts - examples.md: Worked automations + - calibrate.md: jev.calibrate, a threshold measured from recorder history Voice: - conversation.md: The Assist conversation agent and what it will not control + - llm-tools.md: The jev__noul and jev__choice tools for other LLM agents Reference: - - cost.md: Token billing, the usage sensors and the daily budget + - bayesian.md: When to use Jev and when the Bayesian binary sensor + - cost.md: Token billing, the usage sensors, a month worked out and the daily budget - measurements.md: Latency and answer quality measured against the live API - troubleshooting.md: Debug logging and the common failures - limitations.md: What Jev does not do @@ -119,9 +122,12 @@ nav: - Actions: actions.md - AI Task: ai-task.md - Examples: examples.md + - Tuning a threshold: calibrate.md - Voice: - Conversation agent: conversation.md + - Tools for other LLM agents: llm-tools.md - Reference: + - Jev and the Bayesian sensor: bayesian.md - What it costs: cost.md - Measurements: measurements.md - Troubleshooting: troubleshooting.md diff --git a/quality_scale.yaml b/quality_scale.yaml index fc7bd97..448028b 100644 --- a/quality_scale.yaml +++ b/quality_scale.yaml @@ -87,7 +87,7 @@ rules: agent all start it when TypeSafe rejects the key. test-coverage: status: done - comment: 97% measured across 407 tests, enforced in CI. + comment: 97% measured across 474 tests, enforced in CI. # Gold devices: done @@ -133,12 +133,12 @@ rules: placeholder. A question sensor is named by the user's own question text, which is theirs to write and cannot be translated by anyone else. The conversation agent and the AI Task entity take the device name, which is what every core - provider of either does, and the ten sentences the agent speaks itself go + provider of either does, and the twelve sentences the agent speaks itself go through the common translation category, so a Dutch pipeline answers in Dutch. exception-translations: status: done comment: >- - Twenty-eight keys, including the library's own limit errors, which are + Thirty-four keys, including the library's own limit errors, which are re-raised with the real numbers as placeholders rather than passed through in English. The coordinator's UpdateFailed and setup's ConfigEntryNotReady and ConfigEntryAuthFailed are translated too, since both reach the integrations @@ -148,7 +148,7 @@ rules: icon-translations: status: done comment: >- - icons.json covers every translated entity and all four actions, and a test + icons.json covers every translated entity and all five actions, and a test asserts that. The conversation entity takes the conversation domain's own icon, as core's agents do. reconfiguration-flow: @@ -176,4 +176,4 @@ rules: Assistant ships aiohttp and httpx, not httpx2. strict-typing: status: done - comment: mypy --strict passes on all 20 modules, enforced in CI. + comment: mypy --strict passes on all 22 modules, enforced in CI. diff --git a/site-docs/actions.md b/site-docs/actions.md index 5e45f0d..b22e7e1 100644 --- a/site-docs/actions.md +++ b/site-docs/actions.md @@ -109,6 +109,12 @@ the five you might need and discarding four is cheaper than two round trips. An object arrives as an object, not as a stringified dict. The model reads the field names as labels, so name them for what they hold. +## Tune a threshold + +`jev.calibrate` is a fifth action that asks nothing. It compares the recorded history +of a noul sensor with an entity that shows what was really true, and returns the +threshold that fits best. It costs no tokens. See [tuning a threshold](calibrate.md). + ## Errors name the limit Every budget failure names the budget, the limit and your ask, because an agent diff --git a/site-docs/bayesian.md b/site-docs/bayesian.md new file mode 100644 index 0000000..22c6430 --- /dev/null +++ b/site-docs/bayesian.md @@ -0,0 +1,73 @@ +# Jev and the Bayesian sensor + +Home Assistant already has a sensor that turns several readings into one probability, +the [Bayesian binary sensor](https://www.home-assistant.io/integrations/bayesian/). +Both give you a number from 0 to 1 and a threshold that turns it into on or off. They +get the number in different ways, and that decides which one fits. + +## How each gets its number + +The Bayesian sensor multiplies probabilities you give it. You set a prior, and for +each observation you say how often it is seen when the answer is yes +(`prob_given_true`) and when it is no (`prob_given_false`). An observation is a state, +a numeric range or a template. The sensor combines them with Bayes' rule and treats +them as independent of each other. + +Jev reads the states and your question in words, and the model gives the probability. +You write no probabilities. You write what the question means, and optionally what +counts as yes and as no. + +## Side by side + +| | Bayesian sensor | Jev noul | +|---|---|---| +| You supply | a prior and two probabilities per observation | a question in words | +| Where it runs | in Home Assistant | a call to the TypeSafe API | +| Cost | nothing | input tokens, see [what it costs](cost.md) | +| Time to update | when a state changes | 250 to 580 ms warm, measured from the Netherlands | +| Same input, same output | yes | no. One sentence gave 0.25, 0.28 and 0.31 in three runs | +| Reads free text | only through a template you write | yes, a note or a transcript is ordinary input | +| Observations that depend on each other | counted as independent, so they count twice | read together | +| Why it said what it said | each observation's share is visible | a number with no reasoning | + +## Which to use + +Use the Bayesian sensor when you know, or can measure, how often each reading goes with +the answer, and the readings are few and mostly independent. "Somebody is home" from a +phone, a door and motion in the hall is the textbook case. It costs nothing, it works +without the internet, and you can see why it changed. + +Use Jev when the probabilities are the part you cannot write down. A washing machine +whose power draw rises and falls through a programme, a doorbell transcript, or three +readings that only mean something together are hard to express as +`prob_given_true`. Jev takes them as they are. + +If you already have a working Bayesian sensor, keep it. Jev does not make one wrong. + +## Using both + +A Jev noul sensor is a number, so a Bayesian sensor can use it as a `numeric_state` +observation, next to readings that it handles well on its own: + +```yaml +binary_sensor: + - platform: bayesian + name: Laundry waiting + prior: 0.2 + probability_threshold: 0.8 + observations: + - platform: numeric_state + entity_id: sensor.jev_laundry_done + above: 0.6 + prob_given_true: 0.9 + prob_given_false: 0.1 + - platform: state + entity_id: binary_sensor.laundry_door + to_state: "off" + prob_given_true: 0.95 + prob_given_false: 0.5 +``` + +The probabilities in that example are placeholders. Measure your own, for example with +[jev.calibrate](calibrate.md), which reports precision and recall for a Jev sensor +against an entity that shows what was really true. diff --git a/site-docs/calibrate.md b/site-docs/calibrate.md new file mode 100644 index 0000000..dbabd07 --- /dev/null +++ b/site-docs/calibrate.md @@ -0,0 +1,80 @@ +# Tuning a threshold + +A noul is a probability, and the threshold that turns it into yes or no is your +choice. 0.5 is where the model says it cannot tell. It is not where your washing +machine is done. `jev.calibrate` finds a threshold from what really happened in your +house. + +## What it needs + +- a probability sensor, such as a Jev noul sensor +- an entity that shows what was really true, such as a door contact, or a smart plug + that reports when a machine runs +- the recorder, which keeps the history of both + +It reads the recorder and nothing else. It does not call TypeSafe, so it costs no +tokens. + +```yaml +- action: jev.calibrate + response_variable: fit + data: + entity_id: sensor.jev_laundry_done + truth_entity_id: binary_sensor.laundry_door + truth_state: "off" + days: 7 +``` + +| Field | Default | Meaning | +|---|---|---| +| `entity_id` | required | The probability sensor | +| `truth_entity_id` | required | The entity that shows what was true | +| `truth_state` | `on` | The state of that entity that means yes | +| `days` | 7 | How far back to read. The recorder keeps 10 days unless you changed `purge_keep_days` | + +## What it returns + +```yaml +threshold: 0.31 +precision: 1.0 +recall: 1.0 +f1: 1.0 +hours: 4.0 +hours_true: 2.0 +times_true: 1 +probability_changes: 4 +table: + - {threshold: 0.1, precision: 0.5, recall: 1.0, f1: 0.667} + # ... one row for each tenth up to 0.9 +``` + +The action cuts the window at every change of either entity, and counts time, not +state changes: + +- **precision** is the part of the time the threshold said yes that was really true. + A low precision means false alarms. +- **recall** is the part of the true time where the threshold said yes. A low recall + means missed cases. +- **f1** balances the two. The action tries every hundredth from 0.01 to 0.99 and + returns the one with the highest f1. When several tie, it returns the middle one, so + a probability a little off its usual values still lands on the same side. + +Use `table` when you care more about one side. For a notification you would rather +miss than repeat, pick a row with higher precision. + +## How much to trust it + +Read `times_true` before you read `f1`. One wash is one occasion, and an f1 of 1.0 over +one occasion tells you almost nothing. A week with a machine that runs every other day +gives three or four. Collect more occasions before you move a threshold far. + +Time when the probability sensor has no number, for example while the API was down, +is left out. + +## When it refuses + +| Error | Why | +|---|---| +| The recorder is not running | The action has nothing to read | +| No history | No time in the window has both a number and a truth state | +| Never true, or always true | With only one side, every threshold gets the same score | diff --git a/site-docs/conversation.md b/site-docs/conversation.md index 0279310..4fa246e 100644 --- a/site-docs/conversation.md +++ b/site-docs/conversation.md @@ -41,6 +41,11 @@ A room command always carries the kinds of device the model was shown. Home Assistant otherwise acts on every exposed entity in the room, so "turn off the hallway" would reach a lock exposed there and unlock it. +After an action it says the sentence Home Assistant's own agent says for the same +command, in the pipeline's language, such as "Turned on the light". Those sentences +come from Home Assistant's translations. Where they have none, as for a toggle, it +says "Done." + ## What it refuses | Case | What happens | @@ -50,9 +55,11 @@ hallway" would reach a lock exposed there and unlock it. | Needs words written or looked up | Fallback | | A lock or a garage, gate or door cover | Fallback. The agent never describes one | | An entity you did not expose to Assist | Never described to the model at all | +| A hidden device named in full, next to an exposed one with a shorter name | Fallback. "Turn on the desk lamp" does not turn on "Lamp" | | No room and no device named | Refused, unless you allow it. `turn_off` is exempt. Below the confidence floor, fallback | | Two kinds of device, no kind named, whole house | Asks which kind. With one kind exposed, it acts on that kind | -| A spent budget, a rejected key or no answer | Fallback. With no fallback agent it says which of the three it was | +| Two devices whose names fit the command equally well | Asks which one, see below | +| Too little budget left for the command, a rejected key or no answer | Fallback. With no fallback agent it says which of the three it was, as an error reply | !!! info "It only sees what Assist sees" The agent describes only entities you exposed to Assist. You already decided @@ -60,6 +67,33 @@ hallway" would reach a lock exposed there and unlock it. widen that. This is the voice path only. The [four actions](actions.md) send whatever an automation targets, exposed or not. +## When two devices fit the name + +The agent compares what you said with the names of the exposed devices of the kind +the model picked. The whole name said wins: with a "Lamp" and a "Desk lamp", "turn on +the lamp" is the Lamp and "turn on the desk lamp" is the Desk lamp. When two names fit +equally well, as "the lamp" does for a Desk lamp and a Floor lamp, the agent asks +**"Do you mean Desk lamp or Floor lamp?"** and keeps the conversation open. When the +two have the same name, it adds the room: "Lamp (Office) or Lamp (Bedroom)". A room +you name settles it first, so "the lamp in the office" acts. Next comes the room of +the satellite that heard you, as for Home Assistant's own agent: "turn on the lamp" +said to the office satellite turns on the office Lamp. + +The names decide this, not the model's confidence. On a test instance with two lights +both called "Lamp", the model put 1.00 on one of them in one session. In another it put +0.55 on "none of these", 0.44 on one Lamp and 0.01 on the other. Neither split showed +that there were two. With three or more that fit equally well, the command goes to the +fallback agent. + +Your reply, such as "the desk one", is one more request, and it counts against the +budget. It asks which of the two the reply picks, and if the reply asks for something +of its own. A confident pick carries out the first command on that device. A reply +that picks neither, or that is a new instruction, is handled as a new command. So +"never mind, turn off the lamp in the bedroom" turns that lamp off, and does not run +the first command on it. +The question expires after five minutes, the same time Home Assistant keeps a +conversation open. + ## What it costs Every command counts against the same daily token budget as your sensors. A @@ -118,6 +152,13 @@ only sees what Jev could not route, which is the cheap arrangement. ## Diagnostics +In the Assist dialog, each reply from Jev has a note under it with what Jev answered: +the decision and its reason, the slots, each answer with its top three options, the +model, the input tokens and the time the call took. The pipeline's debug view +(**Settings**, **Voice assistants**, the pipeline's menu, **Debug**) keeps the same +note as an `intent-progress` event of the run. The note goes to the pipeline only. It +is not added to the conversation, so a fallback agent does not read it. + The last 20 decisions the agent made are in the integration's diagnostics, with the reason for every decision and the action distribution behind it. The sentence itself is redacted, because the file is meant to be pasted into an issue. diff --git a/site-docs/cost.md b/site-docs/cost.md index a8c7d0c..3952e57 100644 --- a/site-docs/cost.md +++ b/site-docs/cost.md @@ -43,6 +43,29 @@ roughly 110 tokens per entity per call. thousands of tokens on every evaluation, which is why it is off by default. Access tokens, entity pictures and coordinates are never sent, even with it on. +## A month, worked out + +The formula is calls per month, times input tokens per call, times the price. The +examples use the published $0.042 per million input tokens and a 30 day month. + +| Setup | Calls a month | Tokens a call | A month | +|---|---|---|---| +| One context on 2 entities, every 5 minutes | 8,640 | about 405 | $0.15 | +| 20 spoken commands a day, 5 entities exposed | 600 | 1,371 at most | $0.035 | +| One context on 250 entities, every 5 minutes | 8,640 | about 16,500 | $5.99 | + +Where the token counts come from: + +- A request with one entity measured 339 input tokens, and each entity record adds + 65.8, so two entities are about 405. See [what an entity costs](measurements.md#what-an-entity-costs). +- A spoken command with five entities exposed measured 1,329 to 1,371 input tokens. +- 250 entities is the cap on a target, at 65.8 tokens each. + +Every 5 minutes is the default scan interval, and it is the most a context on a +schedule asks. A context that wakes on entity changes asks at most once every 30 +seconds, so a busy one can ask up to 10 times as often. Your own numbers are in +`sensor.jev_input_tokens_today` and `sensor.jev_estimated_cost_today`. + ## The budget is a tripwire Set **Daily input token budget** in the integration options. The check runs before @@ -51,8 +74,9 @@ into a token estimate, and a call that would not fit in what is left of the budg never sent. A budget is a limit on what gets spent, and one that only notices after the spending is a report. -The estimate divides the request size by the bytes per token of the last answered -call. A question, an action and an AI Task all update that ratio. +The estimate is a fixed 250 tokens that every request pays, plus the body size +divided by a bytes-per-token ratio. A question, an action and an AI Task all update +that ratio. When it trips: @@ -60,7 +84,9 @@ When it trips: - the answer sensors of that context go unavailable, because Jev was not asked and there is no answer for right now. The last answers are not thrown away, and the next call that fits replaces them -- `binary_sensor.jev_daily_budget_exceeded` turns on +- `binary_sensor.jev_daily_budget_exceeded` turns on. It shows that contexts have + stopped, so a spoken command, an action or an AI Task that the budget refuses + does not turn it on. Each of those says so to whoever started it - a repair issue explains it, naming the budget, what has been used, and the context that was refused with the tokens it needed @@ -89,19 +115,20 @@ that either of those cleared would not be a daily budget. - YAML contexts name no entry, so they belong to the first enabled Jev entry. A second entry does not ask them again and bill them twice. -The two one-off paths, a conversation command and the preview in the question editor, -still check what has already been spent rather than estimating the call ahead. Both -are started by a person and both have somewhere to fall back to, so the worst case is -one request over the line rather than a runaway. +A conversation command, an action and an AI Task estimate their call ahead, like a +context does. The preview in the question editor checks only what has already been +spent. A person starts it and it has nothing to fall back to, so the worst case is one +request over the line rather than a runaway. ### How the estimate is worked out -The estimate is `bytes / bytes-per-token`, with a 1.2 margin. The ratio is not -hardcoded: the first call after a restart uses 2.29 bytes per token, measured against -the live API, and every answered call after that replaces it with the payload size -divided by the input tokens the endpoint actually reported. An endpoint that counts -tokens differently, OpenRouter or a gateway of your own, is measured rather than -assumed within one call. See the +The estimate is `(250 + bytes / bytes-per-token) * 1.2`. The 250 is what every +request is billed before its body counts. The ratio is not hardcoded: the first call +after a restart uses 2.80 bytes per token, measured against the live API. After that, +every answered call with a body of at least 250 tokens replaces it with what the +endpoint actually billed. A small call leaves it alone, because its bill is nearly all +fixed part. An endpoint that counts tokens differently, OpenRouter or a gateway of +your own, is measured rather than assumed. See the [receipt](measurements.md#bytes-per-input-token). !!! tip "Size it past anything real" diff --git a/site-docs/limitations.md b/site-docs/limitations.md index 2c1e0d4..d8e29f3 100644 --- a/site-docs/limitations.md +++ b/site-docs/limitations.md @@ -61,6 +61,27 @@ Locks are refused on purpose: Home Assistant reads turn_on on a lock as `lock.lo which is the opposite way round from the spoken command. Garage, gate and door covers are refused for the same reason. +## No local model + +Jev needs the TypeSafe API. There is no local mode. + +A llama.cpp fork adds a `POST /v1/decision` endpoint that answers a schema of enum, +boolean and number fields in one batched pass, with a probability for each answer. I +read its interface to see whether Jev could use it as a local backend. It cannot, as +the fork is now: + +- It returns only the probability of the value it chose, not the distribution over + all values. +- A noul maps exactly, because the probability of yes follows from the probability + of the chosen value. +- A choice needs the probability of every option, and a score's level is a weighted + average over all levels, so neither can be built from one probability. +- The request shape is different, so Jev would need a second client. + +If that endpoint returns the full distribution for each field, a local backend is +worth doing. Until then, only the yes/no questions could run locally, with the other +two types still going to TypeSafe. + ## It is not a core integration `quality_scale.yaml` tracks this against Home Assistant's quality scale at 47 done diff --git a/site-docs/llm-tools.md b/site-docs/llm-tools.md new file mode 100644 index 0000000..6abb88b --- /dev/null +++ b/site-docs/llm-tools.md @@ -0,0 +1,47 @@ +# Tools for other LLM agents + +An LLM conversation agent can read your house through the Assist API. What it cannot +give is a probability. Ask it whether the washing machine is done, and it says yes or +no. With this option on, the agent can hand that one question to Jev and get the +number back. + +## Turn it on + +**Settings**, **Devices & services**, **Jev**, **Configure**, then **Offer Jev as a +tool to other LLM agents**. It is off by default. + +It is off because every LLM agent that uses Assist gets the tool descriptions in every +prompt. That costs tokens at that agent's provider on every turn, whether the agent +calls a tool or not. + +## The two tools + +| Tool | The agent gives | The agent gets | +|---|---|---| +| `jev__noul` | `question`, optional `facts` | `probability_yes`, 0 to 1 | +| `jev__choice` | `question`, `options` (2 or more), optional `facts` | `choice`, `probabilities`, `confidence` | + +Each tool call is a call to [jev.noul or jev.choice](actions.md). It counts against +the same daily budget, and it shows in the same usage sensors. + +## What Jev sees + +Jev judges the entities exposed to that assistant, the same list the agent itself +reads, up to 150. `facts` carries anything else the question depends on, such as what +the user said. + +Jev sends `facts` as it is. The actions render a string `state` as a template, and the +tools do not, because a template could read an entity that you did not expose to +Assist. + +## More than one entry + +When two or more Jev entries have the option on, each tool has an `account` +parameter, and the agent must name the entry that pays. Each entry has its own key and +its own budget, so Jev does not choose one for the agent. + +## The Jev conversation agent + +The Jev conversation agent does not use these tools. It asks Jev directly, and it +falls back to another agent for what it cannot route. See +[conversation agent](conversation.md). diff --git a/site-docs/measurements.md b/site-docs/measurements.md index 09a053d..539e936 100644 --- a/site-docs/measurements.md +++ b/site-docs/measurements.md @@ -154,27 +154,59 @@ the entities. The daily budget has to refuse a call before it is sent, and the only token count there is comes back with the reply. So the size of the request is measured locally and -divided by a bytes-per-token ratio. +turned into tokens: a fixed part that every request pays, plus the body bytes divided +by a bytes-per-token ratio. + +### The fixed part + +Four requests against the live API on 2026-09-24, body bytes measured locally for the +same request: + +| Request | Body bytes | Input tokens billed | +|---|---|---| +| `jev.noul`, one short state line | 138 | 278 | +| `jev.noul`, 6 KB of state | 6,136 | 3,277 | +| `jev.ask`, 1 question | 137 | 279 | +| `jev.ask`, 8 questions | 613 | 377 | + +A straight line through each pair crosses zero bytes at 209 tokens (the two `noul` +rows) and at 251 tokens (the two `ask` rows). The fixed part is 250. It is paid per +request, not per question: seven more questions added 98 tokens, not seven times 250. + +Before this, the estimate was the bytes over a ratio and nothing else. The 138 byte +action was estimated at 70 tokens and billed 278. + +### The ratio The cold-start ratio comes from the two readings above. The conversation payload with 5 exposed entities and 7 questions is 3,142 bytes, and the same shape against the live -API reported 1,329 to 1,371 input tokens per command: +API reported 1,329 to 1,371 input tokens per command. Less the fixed part: | | | |---|---| -| 3,142 / 1,371 | 2.29 bytes per token | -| 3,142 / 1,329 | 2.36 bytes per token | +| 3,142 / (1,371 - 250) | 2.80 bytes per token | +| 3,142 / (1,329 - 250) | 2.91 bytes per token | -2.29 is the seed, because the lower ratio is the larger token estimate and an estimate +2.80 is the seed, because the lower ratio is the larger token estimate and an estimate that refuses slightly early beats one that lets a call through. This is a derivation across two runs rather than one payload counted both ways: the byte figures were measured locally with no API call, and the token figures came from a -different set of sixteen live commands on the same fixtures. That is exactly why the -ratio is a seed and not a constant. Every answered call replaces it with its own -payload bytes divided by the input tokens the endpoint reported, so a different -tokenizer, a gateway, or OpenRouter is measured rather than assumed, and the assumption -lasts one call. +different set of sixteen live commands on the same fixtures. That is why the ratio is +a seed and not a constant. An answered call replaces it with its own body bytes +divided by the tokens it was billed past the fixed part, so a different tokenizer, a +gateway, or OpenRouter is measured rather than assumed. + +Only a call whose body was billed at least 250 tokens replaces it. The 278 token +action above has 28 tokens of body, and a ratio taken from all of it said 0.5 bytes +per token. With that ratio, the 6 KB request was estimated at 14,327 tokens after being billed +3,277 one call earlier, and it was refused on every try, because a refused call +measures nothing. + +The ratio depends on what the bytes are. The 6 KB state was a repeated two-byte word +and came to 2.0 bytes per token. The questions of the `ask` rows came to 4.9. The first +6 KB request after a restart was estimated below what it was billed. The next one was +not. The estimate carries a 1.2 margin. Input tokens across those sixteen commands varied by 3.2%, 1,329 to 1,371 for the same seven questions, so 20% sits well past anything diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py new file mode 100644 index 0000000..07eaa3b --- /dev/null +++ b/tests/test_calibrate.py @@ -0,0 +1,201 @@ +"""jev.calibrate: a threshold measured against what was really true.""" + +from datetime import timedelta + +import pytest +from homeassistant.exceptions import ServiceValidationError +from homeassistant.util import dt as dt_util +from pytest_homeassistant_custom_component.components.recorder.common import ( + async_wait_recording_done, +) + +from custom_components.jev.calibrate import Span, best, build_spans, outcome +from custom_components.jev.const import DOMAIN + +PROBABILITY = "sensor.laundry_done" +TRUTH = "binary_sensor.laundry_door" + + +@pytest.fixture +def mock_recorder_before_hass(async_test_recorder): + """The recorder's database has to exist before hass starts.""" + + +@pytest.fixture(autouse=True) +def recorder(recorder_mock): + return recorder_mock + + +def at_hours(*states): + """States one hour apart, as the recorder would return them.""" + from homeassistant.core import State + + start = dt_util.parse_datetime("2026-09-22 08:00:00+00:00") + return [ + State(entity_id, value, last_changed=start + timedelta(hours=hour)) + for entity_id, value, hour in states + ], start + + +def test_every_change_of_either_entity_cuts_a_span(): + states, start = at_hours( + ("sensor.p", "0.2", 0), + ("binary_sensor.t", "off", 0), + ("sensor.p", "0.7", 1), + ("binary_sensor.t", "on", 1), + ) + probabilities = [s for s in states if s.entity_id == "sensor.p"] + truths = [s for s in states if s.entity_id == "binary_sensor.t"] + spans = build_spans(probabilities, truths, "on", start, start + timedelta(hours=3)) + assert [(s.probability, s.truth, s.seconds) for s in spans] == [ + (0.2, False, 3600.0), + (0.7, True, 7200.0), + ] + + +def test_a_state_from_before_the_window_counts_from_its_start(): + states, start = at_hours(("sensor.p", "0.9", -5), ("binary_sensor.t", "on", -2)) + spans = build_spans(states[:1], states[1:], "on", start, start + timedelta(hours=1)) + assert spans == [Span(0.9, True, 3600.0)] + + +def test_time_without_a_number_is_left_out(): + """Unavailable while the API was down is not a probability of anything.""" + states, start = at_hours( + ("sensor.p", "0.9", 0), + ("binary_sensor.t", "on", 0), + ("sensor.p", "unavailable", 1), + ("sensor.p", "0.8", 2), + ) + probabilities = [s for s in states if s.entity_id == "sensor.p"] + truths = [s for s in states if s.entity_id == "binary_sensor.t"] + spans = build_spans(probabilities, truths, "on", start, start + timedelta(hours=3)) + assert sum(s.seconds for s in spans) == 7200.0 + assert {s.probability for s in spans} == {0.9, 0.8} + + +def test_precision_and_recall_are_weighted_by_time(): + spans = [Span(0.8, True, 30), Span(0.8, False, 10), Span(0.2, True, 60)] + result = outcome(spans, 0.5) + assert result.precision == 0.75 + assert result.recall == 30 / 90 + + +def test_nothing_said_yes_has_no_precision_and_no_f1(): + result = outcome([Span(0.2, True, 60), Span(0.1, False, 60)], 0.5) + assert result.precision is None + assert result.f1 == 0.0 + + +def test_the_middle_of_the_tied_thresholds_is_returned(): + """Every threshold from 0.21 to 0.41 separates these perfectly. 0.31 is central.""" + spans = [ + Span(0.2, False, 3600), + Span(0.7, True, 3600), + Span(0.41, True, 3600), + Span(0.1, False, 3600), + ] + result = best(spans) + assert result.threshold == 0.31 + assert (result.precision, result.recall) == (1.0, 1.0) + + +async def record_a_day(hass, freezer): + """Four hours: the door opens with the noul at 0.7, and closes at 0.1.""" + start = dt_util.utcnow() - timedelta(hours=4) + for hour, probability, door in ( + (0, "0.2", "off"), + (1, "0.7", "on"), + (2, "0.41", "on"), + (3, "0.1", "off"), + ): + freezer.move_to(start + timedelta(hours=hour)) + hass.states.async_set(PROBABILITY, probability) + hass.states.async_set(TRUTH, door) + await hass.async_block_till_done() + freezer.move_to(start + timedelta(hours=4)) + await async_wait_recording_done(hass) + + +async def record_two_minutes_open(hass, freezer): + """An hour with the door open for two minutes of it.""" + start = dt_util.utcnow() - timedelta(hours=1) + for minute, probability, door in ( + (0, "0.2", "off"), + (30, "0.8", "on"), + (32, "0.1", "off"), + ): + freezer.move_to(start + timedelta(minutes=minute)) + hass.states.async_set(PROBABILITY, probability) + hass.states.async_set(TRUTH, door) + await hass.async_block_till_done() + freezer.move_to(start + timedelta(hours=1)) + await async_wait_recording_done(hass) + + +async def calibrate(hass, **data): + return await hass.services.async_call( + DOMAIN, + "calibrate", + {"entity_id": PROBABILITY, "truth_entity_id": TRUTH, **data}, + blocking=True, + return_response=True, + ) + + +async def test_calibrate_reads_the_recorder_and_returns_the_best_threshold( + hass, loaded_entry, mock_client, freezer +): + mock_client.ask.reset_mock() + await record_a_day(hass, freezer) + result = await calibrate(hass) + + assert result["threshold"] == 0.31 + assert (result["precision"], result["recall"], result["f1"]) == (1.0, 1.0, 1.0) + assert result["hours"] == 4.0 + assert result["hours_true"] == 2.0 + assert result["times_true"] == 1 + assert result["probability_changes"] == 4 + assert result["table"][0] == { + "threshold": 0.1, + "precision": 0.5, + "recall": 1.0, + "f1": 0.667, + } + # It reads history and nothing else, so it costs no tokens. + assert mock_client.ask.await_count == 0 + + +async def test_truth_that_never_happened_is_refused(hass, loaded_entry, freezer): + await record_a_day(hass, freezer) + with pytest.raises(ServiceValidationError) as err: + await calibrate(hass, truth_state="open") + assert err.value.translation_key == "calibrate_never_true" + + +async def test_truth_that_never_changed_is_refused(hass, loaded_entry, freezer): + hass.states.async_set("binary_sensor.always", "on") + await hass.async_block_till_done() + freezer.tick(timedelta(hours=4)) + await record_a_day(hass, freezer) + with pytest.raises(ServiceValidationError) as err: + await calibrate(hass, truth_entity_id="binary_sensor.always") + assert err.value.translation_key == "calibrate_always_true" + + +async def test_an_entity_with_no_history_is_refused(hass, loaded_entry): + await async_wait_recording_done(hass) + with pytest.raises(ServiceValidationError) as err: + await calibrate(hass) + assert err.value.translation_key == "calibrate_no_history" + assert err.value.translation_placeholders["entity"] == PROBABILITY + + +async def test_a_short_time_true_is_not_rounded_to_nothing( + hass, loaded_entry, mock_client, freezer +): + await record_two_minutes_open(hass, freezer) + result = await calibrate(hass) + + assert result["hours_true"] == 0.03 + assert result["times_true"] == 1 diff --git a/tests/test_conversation.py b/tests/test_conversation.py index a684f01..a1230ec 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -6,6 +6,7 @@ """ from dataclasses import replace +from datetime import timedelta from unittest.mock import patch import pytest @@ -14,9 +15,14 @@ from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar +from homeassistant.helpers import chat_session +from homeassistant.helpers import device_registry as dr from homeassistant.helpers import entity_registry as er +from homeassistant.helpers import intent as ha_intent +from homeassistant.helpers.chat_session import CONVERSATION_TIMEOUT from homeassistant.setup import async_setup_component -from jevclient import ChoiceAnswer, NoulAnswer +from homeassistant.util import dt as dt_util +from jevclient import ChoiceAnswer, NoulAnswer, Usage from custom_components.jev.const import ( CONF_ALLOW_WHOLE_HOME, @@ -25,7 +31,8 @@ CONVERSATION_TRACE_LENGTH, ) from custom_components.jev.conversation import _render_state_answer -from custom_components.jev.interpret import find_brightness +from custom_components.jev.interpret import NONE, find_brightness +from custom_components.jev.payload import payload_bytes from .conftest import PROBE_TOKENS, build_response @@ -197,6 +204,7 @@ async def test_a_compound_command_acts_on_nothing(hass, house, mock_client): assert calls == [] assert "did not understand" in result.response.speech["plain"]["speech"] + assert result.response.error_code is ha_intent.IntentResponseErrorCode.NO_INTENT_MATCH async def test_low_confidence_acts_on_nothing(hass, house, mock_client): @@ -341,7 +349,45 @@ async def test_a_spent_budget_stops_voice_too(hass, house, mock_client): assert mock_client.ask.await_count == 0 assert calls == [] - assert "budget is spent" in result.response.speech["plain"]["speech"] + assert "budget is left" in result.response.speech["plain"]["speech"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR + + +async def test_a_command_that_would_pass_the_budget_is_not_sent(hass, house, mock_client): + # One token short of the budget. would_exceed() says there is room, and on + # 1.15 the command went through and ended the day about 1,000 tokens over. + hass.config_entries.async_update_entry(house, options={"daily_token_budget": 1000}) + await hass.async_block_till_done() + house.runtime_data.usage.input_tokens = 999 + mock_client.ask.reset_mock() + + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + result = await converse(hass, "kitchen light on") + await hass.async_block_till_done() + + assert mock_client.ask.await_count == 0 + assert calls == [] + assert "budget is left" in result.response.speech["plain"]["speech"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR + + +async def test_a_voice_command_teaches_the_estimate(hass, house, mock_client): + # The estimate reads bytes per token from the last call it could measure. A + # voice command that did not report its size left the estimate on whatever + # the last context taught it, which is a different shape of request. + mock_client.ask.return_value = replace( + build_response(**answer_set()), usage=Usage(input_tokens=1371, output_tokens=42) + ) + usage = house.runtime_data.usage + + with patch.object(usage, "record", wraps=usage.record) as record: + await converse(hass, "kitchen light on") + await hass.async_block_till_done() + + sent_state, sent_questions = mock_client.ask.await_args.args + sent = payload_bytes(sent_state, sent_questions, house.runtime_data.model) + record.assert_called_once_with(1371, sent) async def test_a_rejected_key_is_said_out_loud(hass, house, mock_client): @@ -407,6 +453,39 @@ async def test_a_device_that_is_not_exposed_is_never_acted_on(hass, house, mock_ assert calls == [] +@pytest.mark.parametrize( + ("text", "acts"), + [ + # The hidden name is said, in more words than the one the model picked. + ("turn on the kitchen light strip", False), + ("turn on the private light", False), + # Only the exposed name is said, or the hidden one is not said whole. + ("turn on the kitchen light", True), + ("turn on the kitchen lights", True), + ], +) +async def test_a_hidden_device_named_in_full_is_not_swapped_for_an_exposed_one( + hass, house, mock_client, text, acts +): + """Measured: an unexposed "Desk lamp" was said, and the exposed "Lamp" went on.""" + hass.states.async_set("light.strip", "off", {"friendly_name": "Kitchen light strip"}) + async_expose_entity(hass, conversation.DOMAIN, "light.strip", False) + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + await converse(hass, text) + await hass.async_block_till_done() + + assert bool(calls) is acts + sent = mock_client.ask.call_args.args[0] + assert "light.strip" not in str(sent) + assert "Kitchen light strip" not in str(sent["entities"]) + if not acts: + trace = house.runtime_data.conversation_traces[0] + assert trace["reason"] == "named a device that is not exposed" + + async def test_brightness_is_read_from_the_text_not_the_model(hass, house, mock_client): """Jev judges and does not calculate, so the number comes out of a regex.""" mock_client.ask.return_value = build_response( @@ -480,6 +559,73 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH +async def converse_in_a_pipeline(hass, text, conversation_id=None): + """Converse the way assist_pipeline does, with a listener on the chat log.""" + deltas = [] + with ( + chat_session.async_get_chat_session(hass, conversation_id) as session, + conversation.async_get_chat_log( + hass, + session, + conversation.ConversationInput( + text=text, + context=Context(), + conversation_id=session.conversation_id, + device_id=None, + satellite_id=None, + language="en", + agent_id=AGENT, + ), + chat_log_delta_listener=lambda _log, delta: deltas.append(delta), + ) as chat_log, + ): + result = await conversation.async_converse( + hass, text, session.conversation_id, Context(), "en", agent_id=AGENT + ) + content = list(chat_log.content) + return result, deltas, content + + +async def test_the_assist_dialog_shows_what_jev_answered(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer( + choice="light.kitchen", + probabilities={"light.kitchen": 0.8, "light.office": 0.2}, + confidence=0.8, + ) + ) + ) + _, deltas, content = await converse_in_a_pipeline(hass, "kitchen light on") + + assert len(deltas) == 1 + assert deltas[0]["role"] == "assistant" + shown = deltas[0]["thinking_content"] + assert "Jev: turn_on, ok, confidence 0.97" in shown + assert "entity: light.kitchen 0.80 (light.kitchen 0.80, light.office 0.20)" in shown + assert "321 input tokens" in shown + # Shown to the pipeline, never written into the conversation a fallback reads. + assert [c.role for c in content] == ["system", "user"] + + +async def test_the_assist_dialog_shows_why_jev_refused(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set(compound=NoulAnswer(noul=0.95)) + ) + result, deltas, _ = await converse_in_a_pipeline(hass, "two things at once") + + assert "several commands in one sentence" in deltas[0]["thinking_content"] + assert "compound: 0.95" in deltas[0]["thinking_content"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR + + +async def test_a_command_outside_a_pipeline_still_answers(hass, house, mock_client): + """No listener, as from conversation.process: nothing to show it to.""" + mock_client.ask.return_value = build_response(**answer_set()) + result = await converse(hass, "kitchen light on") + assert result.response.response_type is not ha_intent.IntentResponseType.ERROR + + @pytest.mark.parametrize( ("text", "expected"), [ @@ -1201,7 +1347,8 @@ async def test_two_devices_with_one_name_resolve_by_room(hass, house, mock_clien **answer_set( entity=ChoiceAnswer( choice="light.office_ceiling", probabilities={}, confidence=1.0 - ) + ), + area=ChoiceAnswer(choice="Office", probabilities={}, confidence=0.95), ) ) calls = [] @@ -1256,3 +1403,538 @@ async def test_a_room_past_the_entity_cap_is_not_offered(hass, config_entry): assert [e.entity_id for e in snapshot.entities] == ["light.a", "light.b"] assert snapshot.areas == ["Attic"] + # Past the cap is as good as hidden: the model never saw it to pick it. + assert snapshot.hidden_names == ["c"] + + +_ENTITY = {} +_AREA = { + "target_type": ChoiceAnswer(choice="area", probabilities={}, confidence=0.94), + "entity": ChoiceAnswer(choice="none_of_these", probabilities={}, confidence=0.9), + "area": ChoiceAnswer(choice="Office", probabilities={}, confidence=0.93), +} +_ALL = { + "target_type": ChoiceAnswer(choice="everything", probabilities={}, confidence=0.95), + "entity": ChoiceAnswer(choice="none_of_these", probabilities={}, confidence=0.9), +} + + +@pytest.mark.parametrize( + ("language", "service", "text", "answers"), + [ + ("en", "turn_on", "turn on the kitchen light", _ENTITY), + ("en", "turn_off", "turn off the lights in the office", _AREA), + ("en", "turn_off", "turn off all the lights", _ALL), + ("nl", "turn_on", "zet de kitchen light aan", _ENTITY), + ("nl", "turn_off", "zet de lampen in de office uit", _AREA), + ("de", "turn_on", "schalte kitchen light ein", _ENTITY), + ("pl", "turn_off", "wyłącz światła w office", _AREA), + ], +) +async def test_an_action_says_what_the_default_agent_says( + hass, house, mock_client, language, service, text, answers +): + """An action used to reply with no sentence, and the Assist dialog showed nothing. + + The reference is the default agent itself, on a sentence it matches, for the + same command. + """ + action = ChoiceAnswer(choice=service, probabilities={}, confidence=0.98) + mock_client.ask.return_value = build_response(**answer_set(action=action, **answers)) + hass.services.async_register("light", service, lambda call: None) + + ours = await converse(hass, text, language=language) + theirs = await converse( + hass, text, agent_id="conversation.home_assistant", language=language + ) + + spoken = ours.response.speech["plain"]["speech"] + assert spoken + assert spoken == theirs.response.speech["plain"]["speech"] + + +@pytest.mark.parametrize(("language", "expected"), [("en", "Done."), ("nl", "Gedaan.")]) +async def test_an_action_with_no_sentence_of_its_own_says_done( + hass, house, mock_client, language, expected +): + """home-assistant-intents writes nothing for a toggle.""" + mock_client.ask.return_value = build_response( + **answer_set( + action=ChoiceAnswer(choice="toggle", probabilities={}, confidence=0.92) + ) + ) + hass.services.async_register("light", "toggle", lambda call: None) + + result = await converse(hass, "flip the kitchen light", language=language) + + assert result.response.speech["plain"]["speech"] == expected + + +async def test_a_brightness_command_says_it_was_set(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set( + action=ChoiceAnswer( + choice="set_brightness", probabilities={}, confidence=0.95 + ) + ) + ) + hass.services.async_register("light", "turn_on", lambda call: None) + + result = await converse(hass, "set the kitchen light to 40%") + + assert result.response.speech["plain"]["speech"] == "Brightness set" + + +# --- asking which device --- + + +def unsure_between(first, second, first_share=0.5, second_share=0.45, **rest): + """An answer set whose entity answer is split between two devices.""" + shares = {first: first_share, second: second_share, NONE: 0.05, **rest} + return answer_set( + entity=ChoiceAnswer( + choice=first, probabilities=shares, confidence=max(shares.values()) + ), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + + +def rename(hass, entity_id, name): + """Rename in the registry too, which is where the intent layer matches names.""" + er.async_get(hass).async_update_entity(entity_id, name=name) + hass.states.async_set(entity_id, "off", {"friendly_name": name}) + + +def reply(choice, confidence=0.95): + return {"which": ChoiceAnswer(choice=choice, probabilities={}, confidence=confidence)} + + +async def converse_in(hass, text, conversation_id): + return await conversation.async_converse( + hass, text, conversation_id, Context(), language="en", agent_id=AGENT + ) + + +async def test_two_devices_that_fit_the_name_get_a_question(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "light on") + await hass.async_block_till_done() + + assert calls == [] + assert result.continue_conversation is True + assert result.conversation_id + assert ( + result.response.speech["plain"]["speech"] + == "Do you mean Kitchen light or Office light?" + ) + + +async def test_a_sure_answer_is_still_asked_about_when_the_name_is_shared( + hass, house, mock_client +): + """Measured: two lights called "Lamp", and the model gave one of them 1.00.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp") + await hass.async_block_till_done() + + assert calls == [] + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + +async def test_a_shared_name_is_asked_about_when_none_got_most_of_the_answer( + hass, house, mock_client +): + """Measured: none_of_these 0.55, one Lamp 0.44, the other 0.01.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + shares = {NONE: 0.55, "light.kitchen": 0.44, "light.office": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp") + await hass.async_block_till_done() + + assert calls == [] + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + +async def test_a_hidden_name_is_not_asked_about_as_a_shared_one(hass, house, mock_client): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + hass.states.async_set("light.desk", "off", {"friendly_name": "Desk lamp"}) + async_expose_entity(hass, conversation.DOMAIN, "light.desk", False) + shares = {NONE: 0.55, "light.kitchen": 0.44, "light.office": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + + result = await converse(hass, "turn on the desk lamp") + + assert result.continue_conversation is False + trace = house.runtime_data.conversation_traces[0] + assert trace["reason"] == "named a device that is not exposed" + + +async def test_a_room_that_is_named_settles_a_shared_name(hass, house, mock_client): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice="light.office", probabilities={}, confidence=1.0), + area=ChoiceAnswer(choice="Office", probabilities={}, confidence=0.99), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await converse(hass, "turn on the lamp in the office") + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + + +def a_satellite_in(hass, house, area_name): + """A voice device placed in the named area, as a satellite is.""" + area = ar.async_get(hass).async_get_area_by_name(area_name) + assert area is not None + devices = dr.async_get(hass) + device = devices.async_get_or_create( + config_entry_id=house.entry_id, identifiers={("test", area_name)} + ) + devices.async_update_device(device.id, area_id=area.id) + return device.id + + +async def test_the_room_a_satellite_is_in_settles_a_shared_name(hass, house, mock_client): + """Home Assistant's own agent prefers the satellite's area, and so does Jev.""" + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + mock_client.ask.return_value = build_response(**answer_set()) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Office"), + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + + +async def test_a_satellite_entity_area_settles_a_shared_name_the_model_was_unsure_of( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + entities = er.async_get(hass) + satellite = entities.async_get_or_create("assist_satellite", "test", "kitchen") + entities.async_update_entity(satellite.entity_id, area_id=kitchen.id) + shares = {NONE: 0.55, "light.office": 0.44, "light.kitchen": 0.01} + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer(choice=NONE, probabilities=shares, confidence=0.55), + area=ChoiceAnswer(choice=NONE, probabilities={}, confidence=0.9), + ) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + satellite_id=satellite.entity_id, + ) + await hass.async_block_till_done() + + assert result.continue_conversation is False + assert [e for c in calls for e in c.data["entity_id"]] == ["light.kitchen"] + + +async def test_a_satellite_in_another_room_still_gets_the_question( + hass, house, mock_client +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Lamp") + ar.async_get(hass).async_create("Hall") + mock_client.ask.return_value = build_response(**answer_set()) + + result = await conversation.async_converse( + hass, + "turn on the lamp", + None, + Context(), + language="en", + agent_id=AGENT, + device_id=a_satellite_in(hass, house, "Hall"), + ) + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Lamp (Kitchen) or Lamp (Office)?" + ) + + +@pytest.mark.parametrize( + ("text", "chosen"), + [ + # The whole name said beats a name that only shares a word with it. + ("turn on the lamp", "light.kitchen"), + ("turn on the desk lamp", "light.office"), + ], +) +async def test_the_whole_name_said_is_the_device_meant( + hass, house, mock_client, text, chosen +): + rename(hass, "light.kitchen", "Lamp") + rename(hass, "light.office", "Desk lamp") + mock_client.ask.return_value = build_response( + **answer_set(entity=ChoiceAnswer(choice=chosen, probabilities={}, confidence=1.0)) + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + await converse(hass, text) + await hass.async_block_till_done() + + assert [e for c in calls for e in c.data["entity_id"]] == [chosen] + + +async def test_the_reply_runs_the_first_command_on_the_device_it_picks( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert [e for c in calls for e in c.data["entity_id"]] == ["light.office"] + # The reply was one small question about the two devices, not a new command. + state, questions = mock_client.ask.await_args.args + assert list(questions) == ["which", "new_request"] + assert state == {"command": "light on", "reply": "the office one"} + assert set(questions["which"].criteria) == {"light.kitchen", "light.office", NONE} + + +async def test_the_assist_dialog_shows_what_the_reply_picked(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + hass.services.async_register("light", "turn_on", lambda call: None) + + asked = await converse(hass, "light on") + result, deltas, _ = await converse_in_a_pipeline( + hass, "the office one", asked.conversation_id + ) + + assert result.response.response_type is ha_intent.IntentResponseType.ACTION_DONE + [delta] = deltas + assert delta["thinking_content"].startswith( + 'Jev: a reply to "light on", picked light.office\n' + ) + + +async def test_a_reply_that_picks_neither_is_a_new_command(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply(NONE) + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + result = await converse_in(hass, "never mind", asked.conversation_id) + await hass.async_block_till_done() + + # The reply question, then the whole reply as a command of its own. It fits + # no device's name, so it is not asked about again. + assert mock_client.ask.await_count == 2 + assert "action" in mock_client.ask.await_args.args[1] + assert result.continue_conversation is False + + +async def test_a_reply_that_names_a_device_in_a_new_command_runs_that_command( + hass, house, mock_client +): + """Measured: "never mind, turn off the lamp in the bedroom" picked that lamp.""" + new_request = {"new_request": NoulAnswer(noul=0.96)} + turn_off = answer_set( + action=ChoiceAnswer(choice="turn_off", probabilities={}, confidence=0.98), + entity=ChoiceAnswer(choice="light.office", probabilities={}, confidence=0.97), + ) + mock_client.ask.side_effect = [ + build_response(**unsure_between("light.kitchen", "light.office")), + build_response(**reply("light.office"), **new_request), + build_response(**turn_off), + ] + turned_on, turned_off = [], [] + hass.services.async_register("light", "turn_on", turned_on.append) + hass.services.async_register("light", "turn_off", turned_off.append) + + asked = await converse(hass, "light on") + await converse_in(hass, "no, turn off the office light", asked.conversation_id) + await hass.async_block_till_done() + + assert turned_on == [] + assert [e for c in turned_off for e in c.data["entity_id"]] == ["light.office"] + + +async def test_an_unsure_reply_acts_on_nothing(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), + **reply("light.office", confidence=0.4), + ) + calls = [] + hass.services.async_register("light", "turn_on", lambda call: calls.append(call)) + + asked = await converse(hass, "light on") + await converse_in(hass, "hmm", asked.conversation_id) + await hass.async_block_till_done() + + assert calls == [] + + +async def test_a_reply_in_another_conversation_is_not_an_answer(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + await converse(hass, "light on") + mock_client.ask.reset_mock() + + await converse(hass, "the office one") + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_question_left_unanswered_expires_with_the_session( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + mock_client.ask.reset_mock() + + later = dt_util.utcnow() + CONVERSATION_TIMEOUT + timedelta(seconds=1) + with patch("custom_components.jev.conversation.dt_util.utcnow", return_value=later): + await converse_in(hass, "the office one", asked.conversation_id) + await hass.async_block_till_done() + + assert "which" not in mock_client.ask.await_args_list[0].args[1] + + +async def test_a_third_device_in_the_running_means_no_question(hass, house, mock_client): + hass.states.async_set("light.hall", "off", {"friendly_name": "Hall light"}) + async_expose_entity(hass, conversation.DOMAIN, "light.hall", True) + mock_client.ask.return_value = build_response( + **unsure_between( + "light.kitchen", "light.office", 0.4, 0.3, **{"light.hall": 0.25} + ) + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_two_devices_with_nothing_to_tell_them_apart_get_no_question( + hass, house, mock_client +): + kitchen = ar.async_get(hass).async_get_area_by_name("Kitchen") + assert kitchen is not None + er.async_get(hass).async_update_entity("light.office", area_id=kitchen.id) + rename(hass, "light.office", "Kitchen light") + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.continue_conversation is False + assert "did not understand" in result.response.speech["plain"]["speech"] + + +async def test_the_same_name_in_two_rooms_is_asked_by_room(hass, house, mock_client): + rename(hass, "light.office", "Kitchen light") + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "light on") + + assert result.response.speech["plain"]["speech"] == ( + "Do you mean Kitchen light (Kitchen) or Kitchen light (Office)?" + ) + + +async def test_the_question_is_asked_in_the_pipeline_language(hass, house, mock_client): + rename(hass, "light.kitchen", "Lamp keuken") + rename(hass, "light.office", "Lamp kantoor") + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office") + ) + + result = await converse(hass, "lamp aan", language="nl") + + assert ( + result.response.speech["plain"]["speech"] + == "Bedoel je Lamp keuken of Lamp kantoor?" + ) + + +async def test_a_reply_past_the_budget_is_refused_like_a_command( + hass, house, mock_client +): + mock_client.ask.return_value = build_response( + **unsure_between("light.kitchen", "light.office"), **reply("light.office") + ) + asked = await converse(hass, "light on") + # Set on the account, not in the options: an options change reloads the entry, + # and a reloaded agent holds no question to answer. + house.runtime_data.usage.budget = 10 + mock_client.ask.reset_mock() + + result = await converse_in(hass, "the office one", asked.conversation_id) + + assert mock_client.ask.await_count == 0 + assert "budget is left" in result.response.speech["plain"]["speech"] diff --git a/tests/test_init.py b/tests/test_init.py index d426d6a..be5a39c 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -147,7 +147,7 @@ async def test_the_budget_stops_the_next_call_and_keeps_what_it_has(hass, mock_c mock_client.ask.return_value = build_response( laundry_laundry_forgotten=NoulAnswer(noul=0.81) ) - await setup_with_context(hass, _budget_entry(400)) + await setup_with_context(hass, _budget_entry(700)) assert hass.states.get("sensor.jev_input_tokens_today").state == PROBED_AND_ASKED assert hass.states.get("sensor.jev_laundry_forgotten").state == "0.81" diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..2db7366 --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,158 @@ +"""Jev's judgements as tools for another LLM agent, through the Assist API.""" + +import pytest +import voluptuous as vol +from homeassistant.components import conversation +from homeassistant.components.homeassistant.exposed_entities import async_expose_entity +from homeassistant.const import CONF_API_KEY +from homeassistant.core import Context +from homeassistant.helpers import llm +from homeassistant.setup import async_setup_component +from jevclient import ChoiceAnswer, NoulAnswer +from pytest_homeassistant_custom_component.common import MockConfigEntry + +from custom_components.jev.const import CONF_LLM_TOOLS, DOMAIN + +from .conftest import PROBE_TOKENS, build_response + + +def assist_context() -> llm.LLMContext: + return llm.LLMContext( + platform="test", + context=Context(), + language="en", + assistant=conversation.DOMAIN, + device_id=None, + ) + + +async def tool_names(hass) -> list[str]: + api = await llm.async_get_api(hass, llm.LLM_API_ASSIST, assist_context()) + return [tool.name for tool in api.tools] + + +async def call_tool(hass, name, args): + api = await llm.async_get_api(hass, llm.LLM_API_ASSIST, assist_context()) + return await api.async_call_tool(llm.ToolInput(tool_name=name, tool_args=args)) + + +@pytest.fixture +async def house(hass, mock_client, config_entry): + """A loaded entry with the tools on, one exposed sensor and one private one.""" + assert await async_setup_component(hass, "conversation", {}) + assert await async_setup_component(hass, "llm", {}) + hass.states.async_set("sensor.washer_power", "1.2", {"unit_of_measurement": "W"}) + hass.states.async_set("lock.front_door", "unlocked") + async_expose_entity(hass, conversation.DOMAIN, "sensor.washer_power", True) + async_expose_entity(hass, conversation.DOMAIN, "lock.front_door", False) + + config_entry.add_to_hass(hass) + hass.config_entries.async_update_entry(config_entry, options={CONF_LLM_TOOLS: True}) + assert await hass.config_entries.async_setup(config_entry.entry_id) + await hass.async_block_till_done() + mock_client.ask.reset_mock() + return config_entry + + +async def test_the_tools_are_off_until_the_options_turn_them_on( + hass, mock_client, config_entry +): + """Every tool schema is prompt text an agent's provider bills on every turn.""" + assert await async_setup_component(hass, "conversation", {}) + assert await async_setup_component(hass, "llm", {}) + config_entry.add_to_hass(hass) + assert await hass.config_entries.async_setup(config_entry.entry_id) + await hass.async_block_till_done() + assert not [name for name in await tool_names(hass) if name.startswith("jev__")] + + hass.config_entries.async_update_entry(config_entry, options={CONF_LLM_TOOLS: True}) + await hass.async_block_till_done() + assert {"jev__noul", "jev__choice"} <= set(await tool_names(hass)) + + +async def test_another_api_gets_no_tools(hass, house): + from custom_components.jev.llm import async_get_tools + + assert async_get_tools(hass, assist_context(), "some_other_api") is None + + +async def test_noul_judges_only_what_assist_may_see(hass, house, mock_client): + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.83)) + result = await call_tool( + hass, "jev__noul", {"question": "Is the washing machine done?"} + ) + assert result == {"probability_yes": 0.83} + + state, questions = mock_client.ask.await_args.args + assert [e["entity_id"] for e in state["entities"]] == ["sensor.washer_power"] + assert questions["answer"].instructions == "Is the washing machine done?" + assert "note" not in state + + +async def test_facts_are_sent_as_they_are_and_never_rendered(hass, house, mock_client): + """A rendered template could read the lock, which is not exposed to Assist.""" + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.5)) + facts = "The user says: {{ states('lock.front_door') }}" + await call_tool(hass, "jev__noul", {"question": "Is it done?", "facts": facts}) + + state, _ = mock_client.ask.await_args.args + assert state["note"] == {"facts": facts} + assert "unlocked" not in str(state) + + +async def test_choice_returns_the_distribution(hass, house, mock_client): + mock_client.ask.return_value = build_response( + answer=ChoiceAnswer( + choice="washing", + probabilities={"washing": 0.9, "done": 0.1}, + confidence=0.8, + ) + ) + result = await call_tool( + hass, + "jev__choice", + {"question": "What is the washer doing?", "options": ["washing", "done"]}, + ) + assert result == { + "choice": "washing", + "probabilities": {"washing": 0.9, "done": 0.1}, + "confidence": 0.8, + } + _, questions = mock_client.ask.await_args.args + assert set(questions["answer"].criteria) == {"washing", "done"} + + +async def test_a_choice_needs_two_options(hass, house, mock_client): + with pytest.raises(vol.Invalid): + await call_tool( + hass, "jev__choice", {"question": "Which?", "options": ["only one"]} + ) + assert mock_client.ask.await_count == 0 + + +async def test_with_two_accounts_the_agent_names_the_one_that_pays( + hass, house, mock_client +): + """Each entry has its own key and budget, so neither is picked for the agent.""" + guest = MockConfigEntry( + domain=DOMAIN, + title="Jev guest", + data={CONF_API_KEY: "another-key-not-a-real-one"}, + options={CONF_LLM_TOOLS: True}, + unique_id="fedcba9876543210", + ) + guest.add_to_hass(hass) + assert await hass.config_entries.async_setup(guest.entry_id) + await hass.async_block_till_done() + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.4)) + + with pytest.raises(vol.Invalid): + await call_tool(hass, "jev__noul", {"question": "Is it done?"}) + + house_calls = house.runtime_data.usage.calls + result = await call_tool( + hass, "jev__noul", {"question": "Is it done?", "account": "Jev guest"} + ) + assert result == {"probability_yes": 0.4} + assert house.runtime_data.usage.calls == house_calls + assert guest.runtime_data.usage.input_tokens == PROBE_TOKENS + 321 diff --git a/tests/test_payload.py b/tests/test_payload.py index 442828e..1e434cf 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -7,12 +7,15 @@ """ import json +from datetime import date from typing import Any, ClassVar import pytest from aiohttp.payload import JsonPayload from jevclient import Choice, JevClient, Noul, Score +from custom_components.jev.const import BUDGET_ESTIMATE_MARGIN +from custom_components.jev.coordinator import UsageAccount from custom_components.jev.payload import payload_bytes REPLY = { @@ -92,3 +95,28 @@ async def test_the_model_is_part_of_what_is_measured(): short = payload_bytes("x", QUESTIONS, "a") long = payload_bytes("x", QUESTIONS, "a" * 40) assert long - short == 39 + + +# Measured live on 2026-09-24: body bytes, then the input tokens the API billed. +_BILLED = [ + (138, 278), # jev.noul, one short state line + (613, 377), # jev.ask, eight questions + (3142, 1371), # the five-entity conversation payload, the dearest of sixteen +] + + +@pytest.mark.parametrize(("request_bytes", "billed"), _BILLED) +def test_the_cold_start_estimate_is_above_what_was_billed(request_bytes, billed): + """Before any call has measured the ratio, the estimate must not be low. + + Without the fixed part, the 138 byte action was estimated at 70 tokens. + """ + usage = UsageAccount(day=date(2026, 9, 24)) + assert billed <= usage.estimate_tokens(request_bytes) <= billed * 1.5 + + +def test_one_measured_call_sets_the_estimate_for_the_next(): + """6,136 bytes of dense state was billed 3,277 tokens. The same again fits.""" + usage = UsageAccount(day=date(2026, 9, 24)) + usage.record(3277, 6136) + assert 3277 <= usage.estimate_tokens(6136) <= 3277 * BUDGET_ESTIMATE_MARGIN + 1 diff --git a/tests/test_services.py b/tests/test_services.py index df47b4c..26ccc6b 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1,14 +1,16 @@ """The four actions, including what they refuse.""" +from dataclasses import replace + import pytest import voluptuous as vol from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.const import CONF_API_KEY from homeassistant.exceptions import HomeAssistantError, ServiceValidationError -from jevclient import ChoiceAnswer, NoulAnswer, ScoreAnswer +from jevclient import ChoiceAnswer, NoulAnswer, ScoreAnswer, Usage from pytest_homeassistant_custom_component.common import MockConfigEntry -from custom_components.jev.const import DOMAIN +from custom_components.jev.const import DOMAIN, REQUEST_OVERHEAD_TOKENS from custom_components.jev.payload import payload_bytes from .conftest import build_response @@ -43,8 +45,16 @@ async def test_noul_returns_the_probability_and_the_threshold( async def test_an_action_teaches_the_budget_how_big_a_token_is( hass, loaded_entry, mock_client ): - """The budget estimate divides request bytes by this ratio, so every call counts.""" - mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.81)) + """The budget estimate divides body bytes by this ratio, so every call counts. + + The fixed part of the bill is taken off first, because it is paid whatever the + body holds. + """ + billed = REQUEST_OVERHEAD_TOKENS + 900 + mock_client.ask.return_value = replace( + build_response(answer=NoulAnswer(noul=0.81)), + usage=Usage(input_tokens=billed, output_tokens=20), + ) await call( hass, "noul", @@ -53,7 +63,25 @@ async def test_an_action_teaches_the_budget_how_big_a_token_is( state, questions = mock_client.ask.call_args.args runtime = loaded_entry.runtime_data sent = payload_bytes(state, questions, runtime.model) - assert runtime.usage.bytes_per_token == sent / 321 + assert runtime.usage.bytes_per_token == sent / 900 + + +async def test_a_small_call_leaves_the_ratio_alone(hass, loaded_entry, mock_client): + """278 tokens for 138 bytes is nearly all fixed part, and says little about bytes. + + Measured live: with the ratio taken from that call, the next 6,136 byte request + was estimated at 14,327 tokens and billed 3,277. It was refused on every try, + because a refused call measures nothing. + """ + runtime = loaded_entry.runtime_data + before = runtime.usage.bytes_per_token + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.81)) + await call( + hass, + "noul", + {"state": "The machine has drawn 1.2 W.", "instructions": "Is it done?"}, + ) + assert runtime.usage.bytes_per_token == before async def test_the_threshold_is_the_callers_and_nothing_else( @@ -437,3 +465,49 @@ async def test_an_action_with_two_entries_and_none_named_is_refused( {"state": "x", "instructions": "y", "config_entry": second.entry_id}, ) assert "noul" in named + + +async def test_without_the_recorder_it_says_so(hass, loaded_entry): + with pytest.raises(ServiceValidationError) as err: + await call( + hass, + "calibrate", + {"entity_id": "sensor.x", "truth_entity_id": "binary_sensor.y"}, + ) + assert err.value.translation_key == "calibrate_needs_recorder" + + +async def test_an_action_that_would_pass_the_budget_is_not_sent( + hass, loaded_entry, mock_client +): + usage = loaded_entry.runtime_data.usage + usage.budget = 1000 + usage.input_tokens = 999 + mock_client.ask.reset_mock() + + with pytest.raises(HomeAssistantError) as err: + await call(hass, "noul", {"state": "x", "instructions": "Is it done?"}) + + assert mock_client.ask.await_count == 0 + assert err.value.translation_key == "action_over_budget" + placeholders = err.value.translation_placeholders + assert placeholders["remaining"] == "1" + assert placeholders["budget"] == "1000" + assert int(placeholders["estimate"]) > 1 + + +async def test_an_action_holds_its_estimate_while_it_runs( + hass, loaded_entry, mock_client +): + usage = loaded_entry.runtime_data.usage + held = [] + + async def watch(state, questions): + held.append(usage.reserved) + return build_response(answer=NoulAnswer(noul=0.5)) + + mock_client.ask.side_effect = watch + await call(hass, "noul", {"state": "x", "instructions": "Is it done?"}) + + assert held[0] > 0 + assert usage.reserved == 0