diff --git a/custom_components/jev/config_flow.py b/custom_components/jev/config_flow.py index 9f98fce..17df10a 100644 --- a/custom_components/jev/config_flow.py +++ b/custom_components/jev/config_flow.py @@ -42,6 +42,7 @@ CONF_ALLOW_WHOLE_HOME, CONF_DAILY_TOKEN_BUDGET, CONF_FALLBACK_AGENT, + CONF_LLM_TOOLS, CONF_MIN_CONFIDENCE, CONF_MODEL, CONF_PRICE_PER_MILLION, @@ -399,6 +400,10 @@ async def async_step_init( CONF_ALLOW_WHOLE_HOME, default=options.get(CONF_ALLOW_WHOLE_HOME, False), ): bool, + vol.Optional( + CONF_LLM_TOOLS, + default=options.get(CONF_LLM_TOOLS, False), + ): bool, } ) return self.async_show_form(step_id="init", data_schema=schema) diff --git a/custom_components/jev/const.py b/custom_components/jev/const.py index 126ac69..98525ef 100644 --- a/custom_components/jev/const.py +++ b/custom_components/jev/const.py @@ -101,6 +101,10 @@ CONF_MIN_CONFIDENCE: Final = "min_confidence" CONF_ALLOW_WHOLE_HOME: Final = "allow_whole_home" +# --- Tools for other LLM agents --- + +CONF_LLM_TOOLS: Final = "llm_tools" + # Below this, the router hands the sentence to the fallback agent rather than # guessing. 0.6 is a starting point and not a calibrated figure: TypeSafe publishes # no calibration evidence for confidence, so treat it as an ordering and measure it diff --git a/custom_components/jev/llm.py b/custom_components/jev/llm.py new file mode 100644 index 0000000..fd85a6e --- /dev/null +++ b/custom_components/jev/llm.py @@ -0,0 +1,187 @@ +"""Jev's two judgements as tools for an LLM conversation agent. + +An LLM agent reads the house through the Assist API and answers in prose. What it +cannot give is a probability it has been calibrated to: asked "is the washing +machine done", it says yes or no. These tools hand that one judgement to Jev and +return the number, so the agent can say how sure it is, or not act at all. + +The tools are off unless the entry's options turn them on. Every tool's schema +goes into the prompt of every Assist LLM agent on every turn, which the user pays +for at that agent's provider whether a tool is called or not. + +Each call goes through the matching action, so the daily budget, the error +messages and the usage sensors are the actions' own. +""" + +from __future__ import annotations + +from typing import Any, override + +import voluptuous as vol +from homeassistant.components.homeassistant.exposed_entities import ( + async_should_expose, +) +from homeassistant.components.llm import LLMTools +from homeassistant.const import ATTR_ENTITY_ID +from homeassistant.core import HomeAssistant, callback +from homeassistant.helpers.llm import LLM_API_ASSIST, LLMContext, Tool, ToolInput +from homeassistant.util.json import JsonObjectType + +from .const import ( + ATTR_CONFIG_ENTRY, + CONF_INSTRUCTIONS, + CONF_LLM_TOOLS, + CONF_OPTIONS, + CONF_STATE_TEMPLATE, + DOMAIN, + MAX_CONVERSATION_ENTITIES, + SERVICE_CHOICE, + SERVICE_NOUL, +) + +_QUESTION = "question" +_FACTS = "facts" +_ACCOUNT = "account" + +_QUESTION_DESCRIPTION = "The question, in plain words." +_FACTS_DESCRIPTION = ( + "Anything the question depends on that is not an entity state, such as what " + "the user said. Optional." +) + + +class _JevTool(Tool): + """One Jev action, judged against what Assist may see.""" + + action: str + + def __init__(self, entries: dict[str, str], extra: dict[Any, Any]) -> None: + # entries maps an entry title to its id. With two entries the model picks + # one, because each has its own key and its own budget. + self._entries = entries + fields: dict[Any, Any] = { + vol.Required(_QUESTION, description=_QUESTION_DESCRIPTION): str, + **extra, + vol.Optional(_FACTS, description=_FACTS_DESCRIPTION): str, + } + if len(entries) > 1: + fields[ + vol.Required(_ACCOUNT, description="Which Jev account pays for this.") + ] = vol.In(sorted(entries)) + self.parameters = vol.Schema(fields) + + def _data( + self, hass: HomeAssistant, args: dict[str, Any], llm_context: LLMContext + ) -> dict[str, Any]: + account = args.get(_ACCOUNT) or next(iter(self._entries)) + data: dict[str, Any] = { + CONF_INSTRUCTIONS: args[_QUESTION], + ATTR_CONFIG_ENTRY: self._entries[account], + ATTR_ENTITY_ID: _exposed(hass, llm_context), + } + if facts := args.get(_FACTS): + # A mapping, because the action renders a string state as a template, + # and a template can read entities that are not exposed to Assist. + data[CONF_STATE_TEMPLATE] = {_FACTS: facts} + return data + + async def _call( + self, hass: HomeAssistant, data: dict[str, Any], llm_context: LLMContext + ) -> dict[str, Any]: + response = await hass.services.async_call( + DOMAIN, + self.action, + data, + blocking=True, + context=llm_context.context, + return_response=True, + ) + assert response is not None + return dict(response) + + +class NoulTool(_JevTool): + name = f"{DOMAIN}__noul" + description = ( + "Ask Jev how likely a yes/no statement about the home is, judged from the " + "states of the entities exposed to Assist. Returns the probability that the " + "answer is yes, from 0 to 1. Use it for a judgement, such as whether a " + "machine is done or whether anyone is likely home, not for a state you can " + "read directly." + ) + action = SERVICE_NOUL + + def __init__(self, entries: dict[str, str]) -> None: + super().__init__(entries, {}) + + @override + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext + ) -> JsonObjectType: + args = self.parameters(tool_input.tool_args) + result = await self._call(hass, self._data(hass, args, llm_context), llm_context) + return {"probability_yes": result["noul"]} + + +class ChoiceTool(_JevTool): + name = f"{DOMAIN}__choice" + description = ( + "Ask Jev which of a set of options best describes the home, judged from the " + "states of the entities exposed to Assist. Returns the chosen option, the " + "probability of every option, and a confidence from 0 to 1." + ) + action = SERVICE_CHOICE + + def __init__(self, entries: dict[str, str]) -> None: + super().__init__( + entries, + { + vol.Required(CONF_OPTIONS, description="Two or more options."): vol.All( + [str], vol.Length(min=2) + ) + }, + ) + + @override + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext + ) -> JsonObjectType: + args = self.parameters(tool_input.tool_args) + data = self._data(hass, args, llm_context) | {CONF_OPTIONS: args[CONF_OPTIONS]} + result = await self._call(hass, data, llm_context) + return { + "choice": result["choice"], + "probabilities": result["probabilities"], + "confidence": result["confidence"], + } + + +def _exposed(hass: HomeAssistant, llm_context: LLMContext) -> list[str]: + """The entities this assistant may see, capped as the voice agent caps them. + + Jev judges from what the user exposed to Assist and nothing wider, the same + list the LLM agent itself reads. + """ + exposed = sorted( + state.entity_id + for state in hass.states.async_all() + if async_should_expose(hass, llm_context.assistant, state.entity_id) + ) + return exposed[:MAX_CONVERSATION_ENTITIES] + + +@callback +def async_get_tools( + hass: HomeAssistant, llm_context: LLMContext, api_id: str +) -> LLMTools | None: + """The tools, for each loaded entry whose options turn them on.""" + if api_id != LLM_API_ASSIST: + return None + entries = { + entry.title: entry.entry_id + for entry in hass.config_entries.async_loaded_entries(DOMAIN) + if entry.options.get(CONF_LLM_TOOLS, False) + } + if not entries: + return None + return LLMTools(tools=[NoulTool(entries), ChoiceTool(entries)]) diff --git a/custom_components/jev/strings.json b/custom_components/jev/strings.json index 18e8bd2..3262c2c 100644 --- a/custom_components/jev/strings.json +++ b/custom_components/jev/strings.json @@ -82,14 +82,16 @@ "price_per_million": "Price per million input tokens, in USD", "fallback_agent": "Fall back to this agent", "min_confidence": "Act only above this confidence (0 to 1)", - "allow_whole_home": "Allow commands that name no room or device" + "allow_whole_home": "Allow commands that name no room or device", + "llm_tools": "Offer Jev as a tool to other LLM agents" }, "data_description": { "daily_token_budget": "Counted from midnight in the Home Assistant time zone. 0 means no limit.", "price_per_million": "Sets the estimated cost sensor and the cost in the question preview. Nothing that is sent depends on it.", "fallback_agent": "Receives the whole sentence when Jev is not sure what to do.", "min_confidence": "Below this, Jev does nothing and passes the command to the fallback agent.", - "allow_whole_home": "Turning everything off is always allowed, with or without this." + "allow_whole_home": "Turning everything off is always allowed, with or without this.", + "llm_tools": "Adds two tools to the Assist API, one for yes/no questions and one for picking an option. Each LLM agent that uses Assist gets their descriptions in every prompt, which costs tokens at that agent's provider. Each call also counts against the budget above." } } } diff --git a/custom_components/jev/translations/cs.json b/custom_components/jev/translations/cs.json index 2ab23e5..1c3c123 100644 --- a/custom_components/jev/translations/cs.json +++ b/custom_components/jev/translations/cs.json @@ -82,14 +82,16 @@ "price_per_million": "Cena za milion vstupních tokenů v USD", "fallback_agent": "Záložní agent", "min_confidence": "Jednat jen nad touto jistotou (0 až 1)", - "allow_whole_home": "Povolit příkazy, které neuvádějí místnost ani zařízení" + "allow_whole_home": "Povolit příkazy, které neuvádějí místnost ani zařízení", + "llm_tools": "Nabídnout Jev jako nástroj ostatním LLM agentům" }, "data_description": { "daily_token_budget": "Počítá se od půlnoci v časovém pásmu Home Assistantu. 0 znamená bez limitu.", "price_per_million": "Určuje senzor odhadované ceny a cenu v náhledu otázky. Nic z toho, co se odesílá, na tom nezávisí.", "fallback_agent": "Dostane celou větu, když si Jev není jistý, co udělat.", "min_confidence": "Pod touto hodnotou Jev nic neudělá a předá příkaz záložnímu agentovi.", - "allow_whole_home": "Vypnout všechno je dovoleno vždy, s touto volbou i bez ní." + "allow_whole_home": "Vypnout všechno je dovoleno vždy, s touto volbou i bez ní.", + "llm_tools": "Přidá do API Assist dva nástroje, jeden pro otázky ano/ne a jeden pro výběr možnosti. Každý LLM agent, který používá Assist, dostane jejich popisy v každém promptu, což stojí tokeny u poskytovatele tohoto agenta. Každé volání se také počítá do rozpočtu výše." } } } diff --git a/custom_components/jev/translations/da.json b/custom_components/jev/translations/da.json index 969b972..6a86556 100644 --- a/custom_components/jev/translations/da.json +++ b/custom_components/jev/translations/da.json @@ -82,14 +82,16 @@ "price_per_million": "Pris pr. million input-tokens, i USD", "fallback_agent": "Fald tilbage til denne agent", "min_confidence": "Handl kun over denne konfidens (0 til 1)", - "allow_whole_home": "Tillad kommandoer, der ikke nævner et rum eller en enhed" + "allow_whole_home": "Tillad kommandoer, der ikke nævner et rum eller en enhed", + "llm_tools": "Tilbyd Jev som værktøj til andre LLM-agenter" }, "data_description": { "daily_token_budget": "Tælles fra midnat i Home Assistants tidszone. 0 betyder ingen grænse.", "price_per_million": "Bestemmer sensoren for anslået pris og prisen i forhåndsvisningen af spørgsmålet. Intet af det, der sendes, afhænger af den.", "fallback_agent": "Modtager hele sætningen, når Jev ikke er sikker på, hvad der skal gøres.", "min_confidence": "Under denne værdi gør Jev intet og sender kommandoen videre til reserveagenten.", - "allow_whole_home": "At slukke alt er altid tilladt, med eller uden denne indstilling." + "allow_whole_home": "At slukke alt er altid tilladt, med eller uden denne indstilling.", + "llm_tools": "Tilføjer to værktøjer til Assist-API'et, et til ja/nej-spørgsmål og et til at vælge en mulighed. Hver LLM-agent, der bruger Assist, får deres beskrivelser i hver prompt, hvilket koster tokens hos den agents udbyder. Hvert kald tæller også med i budgettet ovenfor." } } } diff --git a/custom_components/jev/translations/de.json b/custom_components/jev/translations/de.json index c5e0dfb..6ec3aa5 100644 --- a/custom_components/jev/translations/de.json +++ b/custom_components/jev/translations/de.json @@ -82,14 +82,16 @@ "price_per_million": "Preis pro Million Eingabe-Tokens, in USD", "fallback_agent": "Auf diesen Agenten zurückfallen", "min_confidence": "Nur oberhalb dieser Konfidenz handeln (0 bis 1)", - "allow_whole_home": "Befehle erlauben, die keinen Raum und kein Gerät nennen" + "allow_whole_home": "Befehle erlauben, die keinen Raum und kein Gerät nennen", + "llm_tools": "Jev anderen LLM-Agenten als Werkzeug anbieten" }, "data_description": { "daily_token_budget": "Gezählt ab Mitternacht in der Zeitzone von Home Assistant. 0 bedeutet kein Limit.", "price_per_million": "Bestimmt den Sensor für die geschätzten Kosten und die Kosten in der Vorschau einer Frage. Nichts, was gesendet wird, hängt davon ab.", "fallback_agent": "Bekommt den ganzen Satz, wenn Jev nicht sicher ist, was zu tun ist.", "min_confidence": "Darunter tut Jev nichts und gibt den Befehl an den Ersatz-Agenten weiter.", - "allow_whole_home": "Alles ausschalten ist immer erlaubt, mit oder ohne diese Option." + "allow_whole_home": "Alles ausschalten ist immer erlaubt, mit oder ohne diese Option.", + "llm_tools": "Fügt der Assist-API zwei Werkzeuge hinzu, eines für Ja/Nein-Fragen und eines für die Wahl einer Option. Jeder LLM-Agent, der Assist nutzt, erhält ihre Beschreibungen in jedem Prompt, was beim Anbieter dieses Agenten Tokens kostet. Jeder Aufruf zählt außerdem zum Budget oben." } } } diff --git a/custom_components/jev/translations/en.json b/custom_components/jev/translations/en.json index 18e8bd2..3262c2c 100644 --- a/custom_components/jev/translations/en.json +++ b/custom_components/jev/translations/en.json @@ -82,14 +82,16 @@ "price_per_million": "Price per million input tokens, in USD", "fallback_agent": "Fall back to this agent", "min_confidence": "Act only above this confidence (0 to 1)", - "allow_whole_home": "Allow commands that name no room or device" + "allow_whole_home": "Allow commands that name no room or device", + "llm_tools": "Offer Jev as a tool to other LLM agents" }, "data_description": { "daily_token_budget": "Counted from midnight in the Home Assistant time zone. 0 means no limit.", "price_per_million": "Sets the estimated cost sensor and the cost in the question preview. Nothing that is sent depends on it.", "fallback_agent": "Receives the whole sentence when Jev is not sure what to do.", "min_confidence": "Below this, Jev does nothing and passes the command to the fallback agent.", - "allow_whole_home": "Turning everything off is always allowed, with or without this." + "allow_whole_home": "Turning everything off is always allowed, with or without this.", + "llm_tools": "Adds two tools to the Assist API, one for yes/no questions and one for picking an option. Each LLM agent that uses Assist gets their descriptions in every prompt, which costs tokens at that agent's provider. Each call also counts against the budget above." } } } diff --git a/custom_components/jev/translations/es.json b/custom_components/jev/translations/es.json index 74444d2..51f08a2 100644 --- a/custom_components/jev/translations/es.json +++ b/custom_components/jev/translations/es.json @@ -82,14 +82,16 @@ "price_per_million": "Precio por millón de tokens de entrada, en USD", "fallback_agent": "Recurrir a este agente", "min_confidence": "Actuar solo por encima de esta confianza (de 0 a 1)", - "allow_whole_home": "Permitir órdenes que no nombren ninguna habitación ni dispositivo" + "allow_whole_home": "Permitir órdenes que no nombren ninguna habitación ni dispositivo", + "llm_tools": "Ofrecer Jev como herramienta a otros agentes LLM" }, "data_description": { "daily_token_budget": "Se cuenta desde la medianoche en la zona horaria de Home Assistant. 0 significa sin límite.", "price_per_million": "Fija el sensor de coste estimado y el coste en la vista previa de la pregunta. Nada de lo que se envía depende de ello.", "fallback_agent": "Recibe la frase entera cuando Jev no está seguro de qué hacer.", "min_confidence": "Por debajo de este valor, Jev no hace nada y pasa el comando al agente de respaldo.", - "allow_whole_home": "Apagarlo todo siempre está permitido, con o sin esta opción." + "allow_whole_home": "Apagarlo todo siempre está permitido, con o sin esta opción.", + "llm_tools": "Añade dos herramientas a la API de Assist, una para preguntas de sí o no y otra para elegir una opción. Cada agente LLM que usa Assist recibe sus descripciones en cada prompt, lo que cuesta tokens en el proveedor de ese agente. Cada llamada también cuenta para el presupuesto de arriba." } } } diff --git a/custom_components/jev/translations/fr.json b/custom_components/jev/translations/fr.json index 57c8422..2ac97b6 100644 --- a/custom_components/jev/translations/fr.json +++ b/custom_components/jev/translations/fr.json @@ -82,14 +82,16 @@ "price_per_million": "Prix par million de jetons d'entrée, en USD", "fallback_agent": "Se replier sur cet agent", "min_confidence": "N'agir qu'au-dessus de cette confiance (0 à 1)", - "allow_whole_home": "Autoriser les commandes qui ne nomment ni pièce ni appareil" + "allow_whole_home": "Autoriser les commandes qui ne nomment ni pièce ni appareil", + "llm_tools": "Proposer Jev comme outil aux autres agents LLM" }, "data_description": { "daily_token_budget": "Compté à partir de minuit dans le fuseau horaire de Home Assistant. 0 signifie aucune limite.", "price_per_million": "Détermine le capteur de coût estimé et le coût dans l'aperçu de la question. Rien de ce qui est envoyé n'en dépend.", "fallback_agent": "Reçoit la phrase entière quand Jev ne sait pas quoi faire.", "min_confidence": "En dessous, Jev ne fait rien et transmet la commande à l'agent de secours.", - "allow_whole_home": "Tout éteindre est toujours autorisé, avec ou sans cette option." + "allow_whole_home": "Tout éteindre est toujours autorisé, avec ou sans cette option.", + "llm_tools": "Ajoute deux outils à l'API Assist, l'un pour les questions oui/non et l'autre pour choisir une option. Chaque agent LLM qui utilise Assist reçoit leurs descriptions dans chaque prompt, ce qui coûte des tokens chez le fournisseur de cet agent. Chaque appel compte aussi dans le budget ci-dessus." } } } diff --git a/custom_components/jev/translations/it.json b/custom_components/jev/translations/it.json index 17eaf2e..2877df8 100644 --- a/custom_components/jev/translations/it.json +++ b/custom_components/jev/translations/it.json @@ -82,14 +82,16 @@ "price_per_million": "Prezzo per milione di token di input, in USD", "fallback_agent": "Ripiega su questo agente", "min_confidence": "Agisci solo sopra questa confidenza (da 0 a 1)", - "allow_whole_home": "Consenti comandi che non nominano una stanza o un dispositivo" + "allow_whole_home": "Consenti comandi che non nominano una stanza o un dispositivo", + "llm_tools": "Offri Jev come strumento ad altri agenti LLM" }, "data_description": { "daily_token_budget": "Conteggiato dalla mezzanotte nel fuso orario di Home Assistant. 0 significa nessun limite.", "price_per_million": "Determina il sensore del costo stimato e il costo nell'anteprima della domanda. Niente di ciò che viene inviato dipende da questo.", "fallback_agent": "Riceve l'intera frase quando Jev non è sicuro di cosa fare.", "min_confidence": "Sotto questo valore Jev non fa nulla e passa il comando all'agente di riserva.", - "allow_whole_home": "Spegnere tutto è sempre consentito, con o senza questa opzione." + "allow_whole_home": "Spegnere tutto è sempre consentito, con o senza questa opzione.", + "llm_tools": "Aggiunge due strumenti all'API Assist, uno per le domande sì/no e uno per scegliere un'opzione. Ogni agente LLM che usa Assist riceve le loro descrizioni in ogni prompt, il che costa token presso il fornitore di quell'agente. Ogni chiamata conta anche nel budget qui sopra." } } } diff --git a/custom_components/jev/translations/nl.json b/custom_components/jev/translations/nl.json index 303002f..c87dc1e 100644 --- a/custom_components/jev/translations/nl.json +++ b/custom_components/jev/translations/nl.json @@ -82,14 +82,16 @@ "price_per_million": "Prijs per miljoen invoertokens, in USD", "fallback_agent": "Val terug op deze agent", "min_confidence": "Alleen handelen boven dit vertrouwen (0 tot 1)", - "allow_whole_home": "Sta opdrachten toe die geen kamer of apparaat noemen" + "allow_whole_home": "Sta opdrachten toe die geen kamer of apparaat noemen", + "llm_tools": "Jev als tool aanbieden aan andere LLM-agents" }, "data_description": { "daily_token_budget": "Geteld vanaf middernacht in de tijdzone van Home Assistant. 0 betekent geen limiet.", "price_per_million": "Bepaalt de sensor voor de geschatte kosten en de kosten in het voorbeeld van een vraag. Niets van wat wordt verstuurd hangt ervan af.", "fallback_agent": "Krijgt de hele zin wanneer Jev niet zeker weet wat te doen.", "min_confidence": "Daaronder doet Jev niets en geeft het de opdracht door aan de terugvalagent.", - "allow_whole_home": "Alles uitzetten mag altijd, met of zonder deze optie." + "allow_whole_home": "Alles uitzetten mag altijd, met of zonder deze optie.", + "llm_tools": "Voegt twee tools toe aan de Assist-API, een voor ja/nee-vragen en een voor het kiezen van een optie. Elke LLM-agent die Assist gebruikt, krijgt hun beschrijvingen in elke prompt, en dat kost tokens bij de provider van die agent. Elke aanroep telt ook mee voor het budget hierboven." } } } diff --git a/custom_components/jev/translations/pl.json b/custom_components/jev/translations/pl.json index d498bd1..5c09f77 100644 --- a/custom_components/jev/translations/pl.json +++ b/custom_components/jev/translations/pl.json @@ -82,14 +82,16 @@ "price_per_million": "Cena za milion tokenów wejściowych, w USD", "fallback_agent": "Agent zapasowy", "min_confidence": "Działaj tylko powyżej tej pewności (0 do 1)", - "allow_whole_home": "Zezwalaj na polecenia, które nie wskazują pokoju ani urządzenia" + "allow_whole_home": "Zezwalaj na polecenia, które nie wskazują pokoju ani urządzenia", + "llm_tools": "Udostępnij Jev jako narzędzie innym agentom LLM" }, "data_description": { "daily_token_budget": "Liczone od północy w strefie czasowej Home Assistanta. 0 oznacza brak limitu.", "price_per_million": "Ustala czujnik szacowanego kosztu i koszt w podglądzie pytania. Nic z tego, co jest wysyłane, od tego nie zależy.", "fallback_agent": "Dostaje całe zdanie, gdy Jev nie jest pewny, co zrobić.", "min_confidence": "Poniżej tej wartości Jev nic nie robi i przekazuje polecenie agentowi zapasowemu.", - "allow_whole_home": "Wyłączenie wszystkiego jest zawsze dozwolone, z tą opcją lub bez niej." + "allow_whole_home": "Wyłączenie wszystkiego jest zawsze dozwolone, z tą opcją lub bez niej.", + "llm_tools": "Dodaje do API Assist dwa narzędzia, jedno do pytań tak/nie i jedno do wyboru opcji. Każdy agent LLM korzystający z Assist dostaje ich opisy w każdym prompcie, co kosztuje tokeny u dostawcy tego agenta. Każde wywołanie liczy się też do budżetu powyżej." } } } diff --git a/custom_components/jev/translations/pt-BR.json b/custom_components/jev/translations/pt-BR.json index 3f185a8..3f696cd 100644 --- a/custom_components/jev/translations/pt-BR.json +++ b/custom_components/jev/translations/pt-BR.json @@ -82,14 +82,16 @@ "price_per_million": "Preço por milhão de tokens de entrada, em USD", "fallback_agent": "Recorrer a este agente", "min_confidence": "Agir apenas acima desta confiança (0 a 1)", - "allow_whole_home": "Permitir comandos que não citam cômodo nem dispositivo" + "allow_whole_home": "Permitir comandos que não citam cômodo nem dispositivo", + "llm_tools": "Oferecer o Jev como ferramenta para outros agentes LLM" }, "data_description": { "daily_token_budget": "Contado a partir da meia-noite no fuso horário do Home Assistant. 0 significa sem limite.", "price_per_million": "Define o sensor de custo estimado e o custo na prévia da pergunta. Nada do que é enviado depende disso.", "fallback_agent": "Recebe a frase inteira quando o Jev não tem certeza do que fazer.", "min_confidence": "Abaixo disso, o Jev não faz nada e passa o comando para o agente reserva.", - "allow_whole_home": "Desligar tudo é sempre permitido, com ou sem esta opção." + "allow_whole_home": "Desligar tudo é sempre permitido, com ou sem esta opção.", + "llm_tools": "Adiciona duas ferramentas à API do Assist, uma para perguntas de sim ou não e outra para escolher uma opção. Cada agente LLM que usa o Assist recebe as descrições delas em todo prompt, o que custa tokens no provedor desse agente. Cada chamada também conta no orçamento acima." } } } diff --git a/custom_components/jev/translations/ru.json b/custom_components/jev/translations/ru.json index 2d1f5de..71209c6 100644 --- a/custom_components/jev/translations/ru.json +++ b/custom_components/jev/translations/ru.json @@ -82,14 +82,16 @@ "price_per_million": "Цена за миллион входных токенов, в USD", "fallback_agent": "Резервный агент", "min_confidence": "Действовать только выше этой уверенности (от 0 до 1)", - "allow_whole_home": "Разрешить команды, в которых не названы комната или устройство" + "allow_whole_home": "Разрешить команды, в которых не названы комната или устройство", + "llm_tools": "Предложить Jev как инструмент другим LLM-агентам" }, "data_description": { "daily_token_budget": "Считается с полуночи в часовом поясе Home Assistant. 0 означает без ограничения.", "price_per_million": "Задаёт датчик оценочной стоимости и стоимость в предпросмотре вопроса. От него не зависит ничего из того, что отправляется.", "fallback_agent": "Получает всю фразу, когда Jev не уверен, что делать.", "min_confidence": "Ниже этого значения Jev ничего не делает и передаёт команду резервному агенту.", - "allow_whole_home": "Выключить всё можно всегда, с этой настройкой или без неё." + "allow_whole_home": "Выключить всё можно всегда, с этой настройкой или без неё.", + "llm_tools": "Добавляет в API Assist два инструмента: один для вопросов да/нет и один для выбора варианта. Каждый LLM-агент, использующий Assist, получает их описания в каждом запросе, и это стоит токенов у провайдера этого агента. Каждый вызов также учитывается в бюджете выше." } } } diff --git a/custom_components/jev/translations/sv.json b/custom_components/jev/translations/sv.json index 3a6f50b..dcfb7aa 100644 --- a/custom_components/jev/translations/sv.json +++ b/custom_components/jev/translations/sv.json @@ -82,14 +82,16 @@ "price_per_million": "Pris per miljon indatatoken, i USD", "fallback_agent": "Fall tillbaka på den här agenten", "min_confidence": "Agera bara över den här säkerheten (0 till 1)", - "allow_whole_home": "Tillåt kommandon som inte nämner rum eller enhet" + "allow_whole_home": "Tillåt kommandon som inte nämner rum eller enhet", + "llm_tools": "Erbjud Jev som verktyg till andra LLM-agenter" }, "data_description": { "daily_token_budget": "Räknas från midnatt i Home Assistants tidszon. 0 betyder ingen gräns.", "price_per_million": "Styr sensorn för uppskattad kostnad och kostnaden i förhandsvisningen av frågan. Inget av det som skickas beror på den.", "fallback_agent": "Får hela meningen när Jev inte är säker på vad som ska göras.", "min_confidence": "Under detta värde gör Jev ingenting och skickar kommandot vidare till reservagenten.", - "allow_whole_home": "Att stänga av allt är alltid tillåtet, med eller utan detta val." + "allow_whole_home": "Att stänga av allt är alltid tillåtet, med eller utan detta val.", + "llm_tools": "Lägger till två verktyg i Assist-API:et, ett för ja/nej-frågor och ett för att välja ett alternativ. Varje LLM-agent som använder Assist får deras beskrivningar i varje prompt, vilket kostar tokens hos den agentens leverantör. Varje anrop räknas också mot budgeten ovan." } } } diff --git a/custom_components/jev/translations/zh-Hans.json b/custom_components/jev/translations/zh-Hans.json index bce6a03..b1d3a8b 100644 --- a/custom_components/jev/translations/zh-Hans.json +++ b/custom_components/jev/translations/zh-Hans.json @@ -82,14 +82,16 @@ "price_per_million": "每百万输入令牌的价格,单位 USD", "fallback_agent": "回退到此代理", "min_confidence": "仅在置信度高于此值时执行(0 到 1)", - "allow_whole_home": "允许未指明房间或设备的命令" + "allow_whole_home": "允许未指明房间或设备的命令", + "llm_tools": "将 Jev 作为工具提供给其他 LLM 代理" }, "data_description": { "daily_token_budget": "从 Home Assistant 时区的午夜开始计算。0 表示不限制。", "price_per_million": "决定预估费用传感器和问题预览中的费用。发送的内容都不受它影响。", "fallback_agent": "当 Jev 不确定该做什么时,接收完整的句子。", "min_confidence": "低于此值时,Jev 不执行任何操作,并把命令交给备用代理。", - "allow_whole_home": "无论是否开启此项,关闭所有设备始终允许。" + "allow_whole_home": "无论是否开启此项,关闭所有设备始终允许。", + "llm_tools": "向 Assist API 添加两个工具,一个用于是/否问题,一个用于选择选项。每个使用 Assist 的 LLM 代理都会在每条提示中收到它们的描述,这会在该代理的提供商处消耗 token。每次调用也计入上面的预算。" } } } diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..2db7366 --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,158 @@ +"""Jev's judgements as tools for another LLM agent, through the Assist API.""" + +import pytest +import voluptuous as vol +from homeassistant.components import conversation +from homeassistant.components.homeassistant.exposed_entities import async_expose_entity +from homeassistant.const import CONF_API_KEY +from homeassistant.core import Context +from homeassistant.helpers import llm +from homeassistant.setup import async_setup_component +from jevclient import ChoiceAnswer, NoulAnswer +from pytest_homeassistant_custom_component.common import MockConfigEntry + +from custom_components.jev.const import CONF_LLM_TOOLS, DOMAIN + +from .conftest import PROBE_TOKENS, build_response + + +def assist_context() -> llm.LLMContext: + return llm.LLMContext( + platform="test", + context=Context(), + language="en", + assistant=conversation.DOMAIN, + device_id=None, + ) + + +async def tool_names(hass) -> list[str]: + api = await llm.async_get_api(hass, llm.LLM_API_ASSIST, assist_context()) + return [tool.name for tool in api.tools] + + +async def call_tool(hass, name, args): + api = await llm.async_get_api(hass, llm.LLM_API_ASSIST, assist_context()) + return await api.async_call_tool(llm.ToolInput(tool_name=name, tool_args=args)) + + +@pytest.fixture +async def house(hass, mock_client, config_entry): + """A loaded entry with the tools on, one exposed sensor and one private one.""" + assert await async_setup_component(hass, "conversation", {}) + assert await async_setup_component(hass, "llm", {}) + hass.states.async_set("sensor.washer_power", "1.2", {"unit_of_measurement": "W"}) + hass.states.async_set("lock.front_door", "unlocked") + async_expose_entity(hass, conversation.DOMAIN, "sensor.washer_power", True) + async_expose_entity(hass, conversation.DOMAIN, "lock.front_door", False) + + config_entry.add_to_hass(hass) + hass.config_entries.async_update_entry(config_entry, options={CONF_LLM_TOOLS: True}) + assert await hass.config_entries.async_setup(config_entry.entry_id) + await hass.async_block_till_done() + mock_client.ask.reset_mock() + return config_entry + + +async def test_the_tools_are_off_until_the_options_turn_them_on( + hass, mock_client, config_entry +): + """Every tool schema is prompt text an agent's provider bills on every turn.""" + assert await async_setup_component(hass, "conversation", {}) + assert await async_setup_component(hass, "llm", {}) + config_entry.add_to_hass(hass) + assert await hass.config_entries.async_setup(config_entry.entry_id) + await hass.async_block_till_done() + assert not [name for name in await tool_names(hass) if name.startswith("jev__")] + + hass.config_entries.async_update_entry(config_entry, options={CONF_LLM_TOOLS: True}) + await hass.async_block_till_done() + assert {"jev__noul", "jev__choice"} <= set(await tool_names(hass)) + + +async def test_another_api_gets_no_tools(hass, house): + from custom_components.jev.llm import async_get_tools + + assert async_get_tools(hass, assist_context(), "some_other_api") is None + + +async def test_noul_judges_only_what_assist_may_see(hass, house, mock_client): + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.83)) + result = await call_tool( + hass, "jev__noul", {"question": "Is the washing machine done?"} + ) + assert result == {"probability_yes": 0.83} + + state, questions = mock_client.ask.await_args.args + assert [e["entity_id"] for e in state["entities"]] == ["sensor.washer_power"] + assert questions["answer"].instructions == "Is the washing machine done?" + assert "note" not in state + + +async def test_facts_are_sent_as_they_are_and_never_rendered(hass, house, mock_client): + """A rendered template could read the lock, which is not exposed to Assist.""" + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.5)) + facts = "The user says: {{ states('lock.front_door') }}" + await call_tool(hass, "jev__noul", {"question": "Is it done?", "facts": facts}) + + state, _ = mock_client.ask.await_args.args + assert state["note"] == {"facts": facts} + assert "unlocked" not in str(state) + + +async def test_choice_returns_the_distribution(hass, house, mock_client): + mock_client.ask.return_value = build_response( + answer=ChoiceAnswer( + choice="washing", + probabilities={"washing": 0.9, "done": 0.1}, + confidence=0.8, + ) + ) + result = await call_tool( + hass, + "jev__choice", + {"question": "What is the washer doing?", "options": ["washing", "done"]}, + ) + assert result == { + "choice": "washing", + "probabilities": {"washing": 0.9, "done": 0.1}, + "confidence": 0.8, + } + _, questions = mock_client.ask.await_args.args + assert set(questions["answer"].criteria) == {"washing", "done"} + + +async def test_a_choice_needs_two_options(hass, house, mock_client): + with pytest.raises(vol.Invalid): + await call_tool( + hass, "jev__choice", {"question": "Which?", "options": ["only one"]} + ) + assert mock_client.ask.await_count == 0 + + +async def test_with_two_accounts_the_agent_names_the_one_that_pays( + hass, house, mock_client +): + """Each entry has its own key and budget, so neither is picked for the agent.""" + guest = MockConfigEntry( + domain=DOMAIN, + title="Jev guest", + data={CONF_API_KEY: "another-key-not-a-real-one"}, + options={CONF_LLM_TOOLS: True}, + unique_id="fedcba9876543210", + ) + guest.add_to_hass(hass) + assert await hass.config_entries.async_setup(guest.entry_id) + await hass.async_block_till_done() + mock_client.ask.return_value = build_response(answer=NoulAnswer(noul=0.4)) + + with pytest.raises(vol.Invalid): + await call_tool(hass, "jev__noul", {"question": "Is it done?"}) + + house_calls = house.runtime_data.usage.calls + result = await call_tool( + hass, "jev__noul", {"question": "Is it done?", "account": "Jev guest"} + ) + assert result == {"probability_yes": 0.4} + assert house.runtime_data.usage.calls == house_calls + assert guest.runtime_data.usage.input_tokens == PROBE_TOKENS + 321