From d2c5853e5b738a1b8f98b8d1f389d6247da9561a Mon Sep 17 00:00:00 2001 From: AboveColin Date: Wed, 23 Sep 2026 21:55:39 +0200 Subject: [PATCH 1/2] The Assist debug view shows what Jev answered for each command Each command adds one agent detail to the conversation trace: the decision, the latency and tokens, and every answer with its probabilities. The debug view then shows why a command picked a device or fell back, without a diagnostics download. --- custom_components/jev/conversation.py | 27 +++++++++++++++++++++------ tests/test_conversation.py | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 07cc3a2..f56f8a4 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -197,13 +197,28 @@ async def _async_handle_message( runtime.usage.notify() decision = interpret(response, user_input.text, snapshot, self._min_confidence) - runtime.conversation_traces.appendleft( + trace = { + "text": user_input.text, + "latency_ms": response.latency_ms, + "input_tokens": response.usage.input_tokens, + "exposed_entities": len(snapshot.entities), + **asdict(decision), + } + runtime.conversation_traces.appendleft(trace) + # The Assist debug view shows this beside the pipeline's own steps. It + # carries each answer as well, with its distribution, because "why did it + # pick the office light" is answered by the entity question's + # probabilities and by nothing in the decision alone. Diagnostics keep the + # shorter record, since they hold the last few commands in memory. + chat_log.async_trace( { - "text": user_input.text, - "latency_ms": response.latency_ms, - "input_tokens": response.usage.input_tokens, - "exposed_entities": len(snapshot.entities), - **asdict(decision), + "jev": trace + | { + "model": response.model, + "answers": { + key: asdict(answer) for key, answer in response.answers.items() + }, + } } ) diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 16a3e64..f001889 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -10,6 +10,7 @@ import pytest from homeassistant.components import conversation +from homeassistant.components.conversation.trace import async_get_traces from homeassistant.components.homeassistant.exposed_entities import async_expose_entity from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.core import Context, ServiceCall @@ -515,6 +516,32 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH +async def test_the_assist_debug_view_shows_what_jev_answered(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set( + entity=ChoiceAnswer( + choice="light.kitchen", + probabilities={"light.kitchen": 0.8, "light.office": 0.2}, + confidence=0.8, + ) + ) + ) + await converse(hass, "kitchen light on") + await hass.async_block_till_done() + + events = async_get_traces()[-1].as_dict()["events"] + details = [e["data"]["jev"] for e in events if e["event_type"] == "agent_detail"] + assert len(details) == 1 + jev = details[0] + assert jev["text"] == "kitchen light on" + assert jev["intent_type"] == "HassTurnOn" + assert jev["input_tokens"] == 321 + assert jev["answers"]["entity"]["probabilities"] == { + "light.kitchen": 0.8, + "light.office": 0.2, + } + + @pytest.mark.parametrize( ("text", "expected"), [ From 72d9c946f359b4597d7266a7df208d7675f55b81 Mon Sep 17 00:00:00 2001 From: AboveColin Date: Thu, 24 Sep 2026 08:26:50 +0200 Subject: [PATCH 2/2] Show the decision in the Assist dialog and the pipeline debug events chat_log.async_trace() writes to the conversation trace, and nothing in Home Assistant 2026.9.3 reads that: pipeline_debug/get showed only run-start, intent-start, intent-end and run-end. The agent now sends the record to the chat log's delta listener as thinking_content. The pipeline stores it as an intent-progress event, and the Assist dialog appends thinking_content under the reply. On a development instance pipeline_debug/get held the event for an action and for a refusal, with the decision, the slots, the top three options of each answer, the model, the input tokens and the latency. The delta goes to the listener only and is not added to the chat log, so a fallback agent reading the same conversation does not see it. --- custom_components/jev/conversation.py | 58 +++++++++++++++------- tests/test_conversation.py | 71 +++++++++++++++++++++------ 2 files changed, 96 insertions(+), 33 deletions(-) diff --git a/custom_components/jev/conversation.py b/custom_components/jev/conversation.py index 481dfee..ea35674 100644 --- a/custom_components/jev/conversation.py +++ b/custom_components/jev/conversation.py @@ -22,11 +22,12 @@ from __future__ import annotations +import json import logging import re from collections.abc import Mapping from dataclasses import asdict, dataclass -from typing import Literal +from typing import Any, Literal from homeassistant.components import conversation from homeassistant.components.conversation.models import AbstractConversationAgent @@ -40,7 +41,7 @@ from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback from homeassistant.util import dt as dt_util from homeassistant.util import language as language_util -from jevclient import JevAuthError, JevError +from jevclient import ChoiceAnswer, JevAuthError, JevError, JevResponse, NoulAnswer from .const import ( CONF_ALLOW_WHOLE_HOME, @@ -207,22 +208,18 @@ async def _async_handle_message( **asdict(decision), } runtime.conversation_traces.appendleft(trace) - # The Assist debug view shows this beside the pipeline's own steps. It - # carries each answer as well, with its distribution, because "why did it - # pick the office light" is answered by the entity question's - # probabilities and by nothing in the decision alone. Diagnostics keep the - # shorter record, since they hold the last few commands in memory. - chat_log.async_trace( - { - "jev": trace - | { - "model": response.model, - "answers": { - key: asdict(answer) for key, answer in response.answers.items() - }, - } - } - ) + # The pipeline records a chat log delta as an intent-progress event: the + # Assist dialog shows its thinking_content under the reply, and the run's + # debug events keep it. The delta goes to the listener only, not into the + # log, so a fallback agent reading this conversation never takes it for + # something said. It carries each answer's distribution, because "why did + # it pick the office light" is answered by the entity question and by + # nothing in the decision alone. + if chat_log.delta_listener is not None: + chat_log.delta_listener( + chat_log, + {"role": "assistant", "thinking_content": _reasoning(trace, response)}, + ) if decision.already_satisfied is not None: name, settled = decision.already_satisfied @@ -365,6 +362,31 @@ async def _speak( ) +def _reasoning(trace: Mapping[str, Any], response: JevResponse) -> str: + """The trace as lines a person reads in the Assist dialog.""" + lines = [ + f"Jev: {trace['action'] or 'no action'}, {trace['reason']}, " + f"confidence {trace['confidence']:.2f}", + f"Slots: {json.dumps(trace['slots'], ensure_ascii=False)}", + ] + for key, answer in response.answers.items(): + if isinstance(answer, ChoiceAnswer): + ranked = sorted((answer.probabilities or {}).items(), key=lambda kv: -kv[1])[ + :3 + ] + spread = ", ".join(f"{k} {p:.2f}" for k, p in ranked) + lines.append(f"{key}: {answer.choice} {answer.confidence:.2f} ({spread})") + elif isinstance(answer, NoulAnswer): + lines.append(f"{key}: {answer.noul:.2f}") + else: + lines.append(f"{key}: {json.dumps(asdict(answer), ensure_ascii=False)}") + lines.append( + f"{response.model}, {trace['input_tokens']} input tokens, " + f"{trace['latency_ms']:.0f} ms, {trace['exposed_entities']} entities" + ) + return "\n".join(lines) + + # A template that compares the state against an English word writes the state word # itself, in its own language. Hand it a translated one and every branch falls # through. Measured against home-assistant-intents 2026.8.28: 3 of the 47 templates diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 7ae102f..dfde94a 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -10,11 +10,11 @@ import pytest from homeassistant.components import conversation -from homeassistant.components.conversation.trace import async_get_traces from homeassistant.components.homeassistant.exposed_entities import async_expose_entity from homeassistant.config_entries import SOURCE_REAUTH from homeassistant.core import Context, ServiceCall from homeassistant.helpers import area_registry as ar +from homeassistant.helpers import chat_session from homeassistant.helpers import entity_registry as er from homeassistant.helpers import intent as ha_intent from homeassistant.setup import async_setup_component @@ -520,7 +520,34 @@ async def test_traces_are_bounded(hass, house, mock_client): assert len(house.runtime_data.conversation_traces) == CONVERSATION_TRACE_LENGTH -async def test_the_assist_debug_view_shows_what_jev_answered(hass, house, mock_client): +async def converse_in_a_pipeline(hass, text): + """Converse the way assist_pipeline does, with a listener on the chat log.""" + deltas = [] + with ( + chat_session.async_get_chat_session(hass, None) as session, + conversation.async_get_chat_log( + hass, + session, + conversation.ConversationInput( + text=text, + context=Context(), + conversation_id=session.conversation_id, + device_id=None, + satellite_id=None, + language="en", + agent_id=AGENT, + ), + chat_log_delta_listener=lambda _log, delta: deltas.append(delta), + ) as chat_log, + ): + result = await conversation.async_converse( + hass, text, session.conversation_id, Context(), "en", agent_id=AGENT + ) + content = list(chat_log.content) + return result, deltas, content + + +async def test_the_assist_dialog_shows_what_jev_answered(hass, house, mock_client): mock_client.ask.return_value = build_response( **answer_set( entity=ChoiceAnswer( @@ -530,20 +557,34 @@ async def test_the_assist_debug_view_shows_what_jev_answered(hass, house, mock_c ) ) ) - await converse(hass, "kitchen light on") - await hass.async_block_till_done() + _, deltas, content = await converse_in_a_pipeline(hass, "kitchen light on") - events = async_get_traces()[-1].as_dict()["events"] - details = [e["data"]["jev"] for e in events if e["event_type"] == "agent_detail"] - assert len(details) == 1 - jev = details[0] - assert jev["text"] == "kitchen light on" - assert jev["intent_type"] == "HassTurnOn" - assert jev["input_tokens"] == 321 - assert jev["answers"]["entity"]["probabilities"] == { - "light.kitchen": 0.8, - "light.office": 0.2, - } + assert len(deltas) == 1 + assert deltas[0]["role"] == "assistant" + shown = deltas[0]["thinking_content"] + assert "Jev: turn_on, ok, confidence 0.97" in shown + assert "entity: light.kitchen 0.80 (light.kitchen 0.80, light.office 0.20)" in shown + assert "321 input tokens" in shown + # Shown to the pipeline, never written into the conversation a fallback reads. + assert [c.role for c in content] == ["system", "user"] + + +async def test_the_assist_dialog_shows_why_jev_refused(hass, house, mock_client): + mock_client.ask.return_value = build_response( + **answer_set(compound=NoulAnswer(noul=0.95)) + ) + result, deltas, _ = await converse_in_a_pipeline(hass, "two things at once") + + assert "several commands in one sentence" in deltas[0]["thinking_content"] + assert "compound: 0.95" in deltas[0]["thinking_content"] + assert result.response.response_type is ha_intent.IntentResponseType.ERROR + + +async def test_a_command_outside_a_pipeline_still_answers(hass, house, mock_client): + """No listener, as from conversation.process: nothing to show it to.""" + mock_client.ask.return_value = build_response(**answer_set()) + result = await converse(hass, "kitchen light on") + assert result.response.response_type is not ha_intent.IntentResponseType.ERROR @pytest.mark.parametrize(