From d7de500b75faf77e5da8998ec179b603792d37eb Mon Sep 17 00:00:00 2001 From: audit Date: Fri, 14 Aug 2026 02:10:24 +0700 Subject: [PATCH 1/6] feat(realtime): persist chat telemetry and replay over SSE --- backend/agent_runtime/__init__.py | 17 +- backend/agent_runtime/llm_loop.py | 51 +- backend/agent_runtime/notifier.py | 11 +- backend/agent_runtime/runtime.py | 506 +++++--- backend/channels/discord.py | 15 +- backend/channels/telegram.py | 15 +- backend/channels/whatsapp.py | 18 +- backend/event_stream.py | 58 +- backend/realtime_store.py | 609 ++++++++++ backend/scheduler.py | 15 +- backend/slash_commands.py | 5 +- backend/tools/agent_messaging.py | 12 +- backend/update_manager.py | 7 + models/mixins/chat_delegation.py | 4 + routes/agents.py | 124 +- routes/realtime.py | 1315 ++++----------------- routes/sessions.py | 29 +- routes/update.py | 3 +- routes/workplaces.py | 3 +- static/js/chat-ui.js | 85 +- static/js/chat-ui/transport.js | 85 +- static/js/chat-ui/turn.js | 2 +- static/js/realtime.js | 96 +- templates/agent_detail.html | 565 +++------ templates/sessions.html | 391 ++---- unit_tests/test_chat_buffer_replay.py | 292 +++-- unit_tests/test_chat_throttle_seq.py | 63 - unit_tests/test_frontend_sse_lifecycle.py | 94 +- unit_tests/test_state_changed_sse.py | 120 +- 29 files changed, 2015 insertions(+), 2595 deletions(-) create mode 100644 backend/realtime_store.py delete mode 100644 unit_tests/test_chat_throttle_seq.py diff --git a/backend/agent_runtime/__init__.py b/backend/agent_runtime/__init__.py index 51c154e2..e4bbc07c 100644 --- a/backend/agent_runtime/__init__.py +++ b/backend/agent_runtime/__init__.py @@ -167,9 +167,18 @@ def _send_free_notification(agent_id: str): from models.db import db notify_msg = "Hey! I'm done and ready to help again. Is there anything I can do?" + message_id = None try: - db.add_chat_message(session_id, 'assistant', notify_msg, - agent_id=agent_id, metadata={"free_notification": True}) + message_id = db.add_chat_message( + session_id, 'assistant', notify_msg, + agent_id=agent_id, metadata={"free_notification": True}, + ) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'final', 'session_id': session_id, 'content': notify_msg, + 'metadata': {'free_notification': True}, 'message_id': message_id, + }) except Exception as e: log.error("[AgentFreeNotify] Failed to save notification message: %s", e) @@ -181,6 +190,10 @@ def _send_free_notification(agent_id: str): 'session_id': session_id, 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': notify_msg, + 'message_id': message_id, + 'metadata': {"free_notification": True}, + 'role': 'assistant', }) except Exception as e: log.error("[AgentFreeNotify] Failed to emit message_received event: %s", e) diff --git a/backend/agent_runtime/llm_loop.py b/backend/agent_runtime/llm_loop.py index 012fe6fe..1b204f1d 100644 --- a/backend/agent_runtime/llm_loop.py +++ b/backend/agent_runtime/llm_loop.py @@ -419,6 +419,9 @@ def run_tool_loop(agent: Dict[str, Any], from backend.event_stream import event_stream from models.chatlog import chatlog_manager + def _message_id(value): + return value if type(value) in (int, str) else None + agent_id = agent['id'] db_agent_id = session_db_agent_id or agent_id # which per-agent DB owns this session external_user_id = agent_context.get('user_id') @@ -440,7 +443,6 @@ def run_tool_loop(agent: Dict[str, Any], _parent_agent_id = None _loop_ts = int(time.time() * 1000) chatlog.append({'type': 'turn_begin', 'session_id': session_id, 'ts': _loop_ts}) - event_stream.emit('turn_begin', {'session_id': session_id, 'ts': _loop_ts}) tool_trace = [] timeline = [] @@ -510,10 +512,13 @@ def _emit_task_lifecycle_event(event_name, task_ids): def _finalize_gate_response(response: str, source: str): duration = round(time.time() - _loop_start_time, 1) metadata = {'plugin_gate': source, 'thinking_duration': duration} - db.add_chat_message(session_id, 'assistant', response, - agent_id=db_agent_id, metadata=metadata) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', response, + agent_id=db_agent_id, metadata=metadata, + )) chatlog.append({'type': 'final', 'session_id': session_id, - 'content': response, 'metadata': metadata}) + 'content': response, 'metadata': metadata, + 'message_id': message_id}) chatlog.append({'type': 'turn_end', 'session_id': session_id, 'thinking_duration': duration}) event_stream.emit('final_answer', { @@ -949,11 +954,14 @@ def _get_agent_config_ig(agt_id: str) -> dict: _logger.info("Stop signal received during ATG execution for session %s", session_id) stop_msg = "Agent stopped by user request." _atg_stop_dur = round(time.time() - _loop_start_time, 1) - db.add_chat_message(session_id, 'assistant', stop_msg, agent_id=db_agent_id, - metadata={"timeline": timeline, "stopped": True, - "thinking_duration": _atg_stop_dur}) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', stop_msg, agent_id=db_agent_id, + metadata={"timeline": timeline, "stopped": True, + "thinking_duration": _atg_stop_dur}, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': stop_msg, - 'metadata': {'stopped': True, 'thinking_duration': _atg_stop_dur}}) + 'metadata': {'stopped': True, 'thinking_duration': _atg_stop_dur}, + 'message_id': message_id}) chatlog.append({'type': 'turn_end', 'session_id': session_id, 'thinking_duration': _atg_stop_dur}) event_stream.emit('final_answer', { @@ -1183,10 +1191,14 @@ def _get_agent_config_ig(agt_id: str) -> dict: _logger.info("Stop signal received for session %s — aborting loop", session_id) stop_msg = "Agent stopped by user request." _stop_dur = round(time.time() - _loop_start_time, 1) - db.add_chat_message(session_id, 'assistant', stop_msg, agent_id=db_agent_id, - metadata={"timeline": timeline, "stopped": True, "thinking_duration": _stop_dur}) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', stop_msg, agent_id=db_agent_id, + metadata={"timeline": timeline, "stopped": True, + "thinking_duration": _stop_dur}, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': stop_msg, - 'metadata': {'stopped': True, 'thinking_duration': _stop_dur}}) + 'metadata': {'stopped': True, 'thinking_duration': _stop_dur}, + 'message_id': message_id}) _stop_inj = ("[SYSTEM] Your previous reasoning and response were forcefully " "interrupted by the user via /stop before completion. " "Await the user's next instruction.") @@ -1983,9 +1995,12 @@ def _get_agent_config_ig(agt_id: str) -> dict: 'content': _display_content, 'is_final': True, 'send_as_message': True, }) - db.add_chat_message(session_id, 'assistant', _display_content, agent_id=db_agent_id, metadata=meta) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', _display_content, + agent_id=db_agent_id, metadata=meta, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': _display_content, - 'metadata': _cl_meta}) + 'metadata': _cl_meta, 'message_id': message_id}) chatlog.append({'type': 'turn_end', 'session_id': session_id, 'thinking_duration': _final_dur}) # Archive sub-agent session at turn-end — single-turn only. Explorer & # kb-organizer are single-shot, so they archive on completion (no need to @@ -2812,10 +2827,14 @@ def _normalize(s): _logger.info("Stop signal received for session %s — aborting after tools", session_id) stop_msg = "Agent stopped by user request." _stopb_dur = round(time.time() - _loop_start_time, 1) - db.add_chat_message(session_id, 'assistant', stop_msg, agent_id=db_agent_id, - metadata={"timeline": timeline, "stopped": True, "thinking_duration": _stopb_dur}) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', stop_msg, agent_id=db_agent_id, + metadata={"timeline": timeline, "stopped": True, + "thinking_duration": _stopb_dur}, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': stop_msg, - 'metadata': {'stopped': True, 'thinking_duration': _stopb_dur}}) + 'metadata': {'stopped': True, 'thinking_duration': _stopb_dur}, + 'message_id': message_id}) _stopb_inj = ("[SYSTEM] Your previous reasoning and response were forcefully " "interrupted by the user via /stop before completion. " "Await the user's next instruction.") diff --git a/backend/agent_runtime/notifier.py b/backend/agent_runtime/notifier.py index 73397cb6..32df1f6e 100644 --- a/backend/agent_runtime/notifier.py +++ b/backend/agent_runtime/notifier.py @@ -190,10 +190,17 @@ def notify_agent(agent_id: str, tag: str, message: str, ) else: meta = dict(metadata) if metadata else {} - db.add_chat_message( + message_id = db.add_chat_message( target_session_id, role='user', content=full_message, agent_id=_db_agent_id, metadata=meta if meta else None, ) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(_db_agent_id, target_session_id).append({ + 'type': 'user', 'session_id': target_session_id, + 'content': full_message, 'metadata': meta, + 'message_id': message_id, + }) from backend.event_stream import event_stream event_stream.emit('message_received', { 'agent_id': agent_id, @@ -202,6 +209,8 @@ def notify_agent(agent_id: str, tag: str, message: str, 'channel_id': channel_id, 'message': full_message, 'metadata': meta, + 'message_id': message_id, + 'role': 'user', }) if deliver_external and channel_id: from backend.channels.registry import channel_manager diff --git a/backend/agent_runtime/runtime.py b/backend/agent_runtime/runtime.py index cff9c16f..c48018bd 100644 --- a/backend/agent_runtime/runtime.py +++ b/backend/agent_runtime/runtime.py @@ -16,6 +16,7 @@ import queue import threading import traceback +import uuid from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from contextlib import contextmanager @@ -101,7 +102,6 @@ def _append_attachment_context(content: str, attachment_infos, attachment_info, WORKER_JOIN_TIMEOUT_SECONDS = 5.0 # Max time to wait for worker threads to finish on shutdown WORKER_JOIN_MIN_TIMEOUT = 0.1 # Minimum timeout per worker join iteration (seconds) DEFAULT_BUFFER_SECONDS = 2 # Default message buffering delay when agent has no config (seconds) -SESSION_BUFFER_CLEANUP_DELAY = 30.0 # Delay before cleaning up SSE session buffers (seconds) def _llm_log_path(agent_id: str) -> str: @@ -309,6 +309,7 @@ class SessionContext: external_user_id: str channel_id: Optional[str] = None session_db_agent_id: Optional[str] = None + turn_id: str = field(default_factory=lambda: uuid.uuid4().hex) class _QueueTask: @@ -340,12 +341,11 @@ def __init__(self, agent: dict, ctx: SessionContext, # (b) _session_store._stop_flags_guard # (c) _session_store._inject_queues_guard # (d) _session_store._busy_guard -# (e) _agent_tracker._guard -# (f) _cleanup_tracker._guard -# (g) _llm_serializer._summarize_guard -# (h) _llm_serializer._llm_lock -# (i) _shutdown_mgr._lock -# (j) instance._buffer_lock +# (e) _cleanup_tracker._guard +# (f) _llm_serializer._summarize_guard +# (g) _llm_serializer._llm_lock +# (h) _shutdown_mgr._lock +# (i) instance._buffer_lock # # 2. GUARD-LOCK PATTERN: Each mutable dict has a dedicated "guard" lock. # The guard protects structuring operations (get-or-create, pop, clear). @@ -362,8 +362,6 @@ def __init__(self, agent: dict, ctx: SessionContext, # 5. INVARIANTS: # • Every session_id present in _cleanup_tracker._ttl MUST also have # entries in _session_store (or be in the process of being cleaned up). -# • _agent_tracker._busy[agent_id] exists only while an agent is -# actively processing a turn; cleared on completion or TTL expiry. # • _shutdown_mgr._event, once set, is never cleared (shutdown is final). # # ───────────────────────────────────────────────────────────────────────────── @@ -449,41 +447,6 @@ def __init__(self) -> None: self._busy_guard = threading.Lock() -class _AgentTracker: - """Track which agents are currently busy processing a session. - - Thread-safety: - The _guard lock protects all reads and writes to the _busy dict, - which is mutated by worker threads when agents start or finish - processing turns. - - Acquired by: _set_agent_busy(), _clear_agent_busy(), - is_agent_busy(), get_busy_agents(). - Released: immediately after the dict operation (short critical - section — < 1ms). - - Deadlock risk: NONE — _guard is never nested with any other lock. - It is acquired independently each time. - - TTL staleness: entries older than the TTL (default 600s) are - treated as stale and auto-expired. This protects against hung - threads that never clear their busy flag. - - Invariants: - • An agent_id appears in _busy only while it is actively - processing an LLM turn. - • Each entry has {session_id: str, started_at: float}. - • At most one entry per agent_id (set overwrites). - """ - - def __init__(self) -> None: - # agent_id -> {session_id: str, started_at: float} - # Guarded by _guard — prevents races between agent_busy set/clear - # calls coming from different worker threads. - self._busy: Dict[str, dict] = {} - self._guard = threading.Lock() - - class _CleanupTracker: """TTL-based idle-session cleanup: session timestamps + periodic timer. @@ -639,9 +602,8 @@ def __init__(self) -> None: class AgentRuntime: - # State containers — reduce class-level attributes from 23 to 6 + # State containers — reduce class-level attributes from 23 to 5 _session_store = _SessionStore() - _agent_tracker = _AgentTracker() _cleanup_tracker = _CleanupTracker() _llm_serializer = _LLMSerializer() _shutdown_mgr = _ShutdownManager() @@ -765,6 +727,13 @@ def __init__(self): self._buffer_timers: Dict[str, threading.Timer] = {} self._buffer_lock = threading.Lock() self._workers: list[threading.Thread] = [] + try: + from backend.realtime_store import realtime_store + interrupted = realtime_store.interrupt_stale_turns() + if interrupted: + _logger.warning("Marked %d turn(s) interrupted after restart", len(interrupted)) + except Exception as exc: + _logger.error("Failed to recover durable active turns: %s", exc) # Read worker count from DB (user-configurable), fall back to config default try: from models.db import db as _db @@ -874,6 +843,7 @@ def _worker(self) -> None: not bool(_resp), _resp == "(No response)", task.ctx.channel_id) except Exception as e: _logger.error("Worker error for session %s: %s", task.ctx.session_id, e, exc_info=True) + self._fail_queued_task(task, str(e)) task.result = { "response": "An unexpected error occurred. Please try again.", "error": True, @@ -926,50 +896,60 @@ def _is_busy(self, session_id: str) -> bool: with self._session_store._busy_guard: return self._session_store._busy.get(session_id, False) - def _set_agent_busy(self, agent_id: str, session_id: str) -> None: - with self._agent_tracker._guard: - self._agent_tracker._busy[agent_id] = {'session_id': session_id, 'started_at': time.time()} + def _mark_task_queued(self, task: '_QueueTask') -> None: + from backend.realtime_store import realtime_store + turn_id, _created = realtime_store.queue_turn( + task.agent['id'], task.ctx.session_id, task.ctx.turn_id, + ) + task.ctx.turn_id = turn_id - def _clear_agent_busy(self, agent_id: str) -> None: - with self._agent_tracker._guard: - self._agent_tracker._busy.pop(agent_id, None) + def _put_task(self, task: '_QueueTask', *, already_queued: bool = False) -> None: + if not already_queued: + self._mark_task_queued(task) + try: + self._message_queue.put(task) + except Exception: + self._fail_queued_task(task, 'queue_failed') + raise - def is_agent_busy(self, agent_id: str, ttl: int = 600) -> bool: - """Return True if agent is currently processing an LLM turn. + def _fail_queued_task(self, task: '_QueueTask', reason: str) -> None: + from backend.realtime_store import realtime_store + if not any(t['turn_id'] == task.ctx.turn_id for t in realtime_store.active_turns()): + return + event_stream.emit('turn_complete', { + 'agent_id': task.agent.get('id', ''), + 'agent_name': task.agent.get('name', ''), + 'session_id': task.ctx.session_id, + 'external_user_id': task.ctx.external_user_id, + 'channel_id': task.ctx.channel_id, + 'turn_id': task.ctx.turn_id, + 'response': '', + 'tool_trace': [], + 'is_error': True, + 'interrupted': True, + 'reason': reason, + }) + realtime_store.finish_turn(task.ctx.turn_id) + remaining = realtime_store.busy_agents().get(task.agent.get('id', '')) + event_stream.emit('agent_busy_changed', { + 'agent_id': task.agent.get('id', ''), + 'busy': bool(remaining), + 'session_id': remaining.get('session_id', task.ctx.session_id) if remaining else task.ctx.session_id, + 'session_ids': remaining.get('session_ids', []) if remaining else [], + 'active_count': remaining.get('active_count', 0) if remaining else 0, + 'state': remaining.get('state', 'idle') if remaining else 'idle', + 'turn_id': None, + }) - A TTL guard treats entries older than `ttl` seconds as stale (e.g. a - thread that hung and never cleared its flag). Default is 10 minutes. - """ - with self._agent_tracker._guard: - entry = self._agent_tracker._busy.get(agent_id) - if not entry: - return False - if time.time() - entry['started_at'] > ttl: - # Auto-expire stale entry - self._clear_agent_busy(agent_id) - return False - return True + def is_agent_busy(self, agent_id: str, ttl: int = 600) -> bool: + """Return durable queued/running state. ``ttl`` is kept for callers.""" + from backend.realtime_store import realtime_store + return bool(realtime_store.active_turns(agent_id=agent_id)) def get_busy_agents(self, ttl: int = 600) -> dict: - """Return a snapshot of all currently busy agents (respects TTL).""" - now = time.time() - with self._agent_tracker._guard: - snapshot = dict(self._agent_tracker._busy) - result = {} - stale = [] - for agent_id, entry in snapshot.items(): - elapsed = now - entry['started_at'] - if elapsed > ttl: - stale.append(agent_id) - else: - result[agent_id] = { - 'session_id': entry['session_id'], - 'started_at': entry['started_at'], - 'elapsed': round(elapsed, 1), - } - for agent_id in stale: - self._clear_agent_busy(agent_id) - return result + """Return all durable queued/running turns grouped by agent.""" + from backend.realtime_store import realtime_store + return realtime_store.busy_agents() @contextmanager def _buffer_timer(self, session_id: str, buffer_seconds: float, @@ -1043,7 +1023,8 @@ def summarize_session(self, agent: dict, session_id: str) -> bool: def _run_bash_exec(self, agent: Dict[str, Any], session_id: str, db_agent_id: str, external_user_id: str, - message: str) -> str: + message: str, client_message_id: str | None = None + ) -> tuple[str, int | str | None, int | str | None]: """Run a web user's "!" directly and persist it for UI display only. The command and its output are saved with a `bash_exec` metadata flag so @@ -1054,7 +1035,7 @@ def _run_bash_exec(self, agent: Dict[str, Any], session_id: str, """ cmd = message.lstrip()[1:].strip() if not cmd: - return "Usage: `!` — run a shell command directly (web only)." + return "Usage: `!` — run a shell command directly (web only).", None, None from backend.tools import bash exec_agent = {**agent, 'session_id': session_id, '_skip_safety': True} @@ -1082,19 +1063,39 @@ def _run_bash_exec(self, agent: Dict[str, Any], session_id: str, response += f"\n_(exit code {exit_code})_" # Persist for UI display only — hidden from LLM via the `bash_exec` flag. - _db_retry(db.add_chat_message, session_id, 'user', message, - agent_id=db_agent_id, metadata={'bash_exec': True}, - label="save bash command") - _db_retry(db.add_chat_message, session_id, 'assistant', response, - agent_id=db_agent_id, metadata={'bash_exec': True}, - label="save bash output") + command_meta = {'bash_exec': True} + if client_message_id: + command_meta['client_message_id'] = client_message_id + message_id = _db_retry( + db.add_chat_message, session_id, 'user', message, + agent_id=db_agent_id, metadata=command_meta, + label="save bash command", + ) + response_id = _db_retry( + db.add_chat_message, session_id, 'assistant', response, + agent_id=db_agent_id, metadata={'bash_exec': True}, + label="save bash output", + ) + message_id = message_id if type(message_id) in (int, str) else None + response_id = response_id if type(response_id) in (int, str) else None _cl = chatlog_manager.get(db_agent_id, session_id) _cl.append({'type': 'user', 'session_id': session_id, 'content': message, - 'sender_id': external_user_id, 'metadata': {'bash_exec': True}}) + 'sender_id': external_user_id, 'metadata': command_meta, + 'message_id': message_id}) _cl.append({'type': 'system', 'session_id': session_id, 'content': response, - 'metadata': {'bash_exec': True}}) + 'metadata': {'bash_exec': True}, 'message_id': response_id}) + for role, content, saved_id, meta in ( + ('user', message, message_id, command_meta), + ('assistant', response, response_id, {'bash_exec': True})): + event_stream.emit('message_received', { + 'agent_id': agent['id'], 'session_id': session_id, + 'external_user_id': external_user_id, 'message': content, + 'message_id': saved_id, + 'client_message_id': meta.get('client_message_id'), + 'metadata': meta, 'role': role, + }) self._prefetcher.invalidate(session_id) - return response + return response, message_id, response_id def handle_message(self, agent_id: str, external_user_id: str, message: str, channel_id: Optional[str] = None, @@ -1187,11 +1188,15 @@ def handle_message(self, agent_id: str, external_user_id: str, # On a channel, a "!"-prefixed message falls through as ordinary user text. if message.lstrip().startswith('!') and channel_id is None \ and agent.get('bash_exec_enabled'): - response = self._run_bash_exec( + response, message_id, response_id = self._run_bash_exec( agent, session_id, db_agent_id, external_user_id, message, + (metadata or {}).get('client_message_id'), ) return {"response": response, "tool_trace": [], "timeline": [], - "slash_command": True, "bash_exec": True} + "slash_command": True, "bash_exec": True, + "message_id": message_id, + "response_message_id": response_id, + "client_message_id": (metadata or {}).get('client_message_id')} # Slash command interception — execute before saving message or sending to LLM parsed = parse_command(message) @@ -1203,23 +1208,49 @@ def handle_message(self, agent_id: str, external_user_id: str, ) if response is not None: # Command was recognized — save command echo and response, then return - _db_retry(db.add_chat_message, session_id, 'user', message, - agent_id=db_agent_id, metadata={"slash_command": True}, - label="save command message") - _db_retry(db.add_chat_message, session_id, 'assistant', response, - agent_id=db_agent_id, metadata={"slash_command": True}, - label="save command response") + command_meta = {"slash_command": True} + if metadata and metadata.get('client_message_id'): + command_meta['client_message_id'] = metadata['client_message_id'] + message_id = _db_retry( + db.add_chat_message, session_id, 'user', message, + agent_id=db_agent_id, metadata=command_meta, + label="save command message", + ) + response_id = _db_retry( + db.add_chat_message, session_id, 'assistant', response, + agent_id=db_agent_id, metadata={"slash_command": True}, + label="save command response", + ) _cl = chatlog_manager.get(db_agent_id, session_id) _cl.append({'type': 'user', 'session_id': session_id, 'content': message, 'sender_id': external_user_id, - 'metadata': {'slash_command': True}}) + 'metadata': command_meta, 'message_id': message_id}) _cl.append({'type': 'system', 'session_id': session_id, 'content': response, - 'metadata': {'slash_command': True}}) + 'metadata': {'slash_command': True}, 'message_id': response_id}) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': message, 'message_id': message_id, + 'client_message_id': command_meta.get('client_message_id'), + 'metadata': command_meta, 'role': 'user', + }) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': response, 'message_id': response_id, + 'metadata': {'slash_command': True}, 'role': 'assistant', + }) # Signal the client to clear the chat UI when the clear command was used extra = {"clear_ui": True} if cmd_name == "clear" else {} extra["slash_command"] = True # flag so frontend skips thinking bubble self._prefetcher.invalidate(session_id) - return {"response": response, "tool_trace": [], "timeline": [], **extra} + return { + "response": response, "tool_trace": [], "timeline": [], + "message_id": message_id, + "response_message_id": response_id, + "client_message_id": command_meta.get('client_message_id'), + **extra, + } # Unknown command — fall through to normal LLM processing # Save user message (store image reference and any extra metadata) @@ -1274,11 +1305,15 @@ def handle_message(self, agent_id: str, external_user_id: str, meta['agent_message'] = True meta['from_agent_id'] = sender_id meta['from_agent_name'] = sender_agent.get('name', sender_id) if sender_agent else sender_id - _db_retry(db.add_chat_message, session_id, 'user', message or "[Image]", - agent_id=db_agent_id, metadata=meta if meta else None, label="save user message") + message_id = _db_retry( + db.add_chat_message, session_id, 'user', message or "[Image]", + agent_id=db_agent_id, metadata=meta if meta else None, + label="save user message", + ) _cl_user = chatlog_manager.get(db_agent_id, session_id) _cl_user_entry = {'type': 'user', 'session_id': session_id, - 'content': message or '[Image]', 'sender_id': external_user_id} + 'content': message or '[Image]', 'sender_id': external_user_id, + 'message_id': message_id} if meta: _cl_user_entry['metadata'] = meta _cl_user.append(_cl_user_entry) @@ -1299,6 +1334,9 @@ def handle_message(self, agent_id: str, external_user_id: str, 'audio_url': audio_url, 'video_url': video_url, 'metadata': meta, + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'role': 'user', }) # A plain human reply in the exact originating session resumes the @@ -1322,6 +1360,8 @@ def handle_message(self, agent_id: str, external_user_id: str, "tool_trace": [], "timeline": [], "escalation_routed": routed, + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), } # Busy-ack: if the agent-level concurrency gate is saturated, send an @@ -1345,14 +1385,24 @@ def handle_message(self, agent_id: str, external_user_id: str, ) _ack_meta = {"busy_ack": True, "concurrency_limited": True, "concurrency_active": _cap["active"], "concurrency_max": _cap["max"]} - _db_retry(db.add_chat_message, session_id, 'assistant', _ack_text, - agent_id=db_agent_id, metadata=_ack_meta, - label="save busy ack") + _ack_id = _db_retry( + db.add_chat_message, session_id, 'assistant', _ack_text, + agent_id=db_agent_id, metadata=_ack_meta, + label="save busy ack", + ) + _ack_id = _ack_id if type(_ack_id) in (int, str) else None chatlog_manager.get(db_agent_id, session_id).append({ 'type': 'final', 'session_id': session_id, 'content': _ack_text, 'metadata': _ack_meta, + 'message_id': _ack_id, + }) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': _ack_text, 'message_id': _ack_id, + 'metadata': _ack_meta, 'role': 'assistant', }) event_stream.emit('concurrency_limited', { 'agent_id': agent_id, @@ -1387,9 +1437,11 @@ def handle_message(self, agent_id: str, external_user_id: str, # in a DIFFERENT session, reject this message with a contextual explanation. # Check focus first (requires DB read) only when agent-level busy is confirmed. if agent.get('enable_agent_state') and self.is_agent_busy(agent_id): - with self._agent_tracker._guard: - busy_entry = self._agent_tracker._busy.get(agent_id) - if busy_entry and busy_entry['session_id'] != session_id: + from backend.realtime_store import realtime_store + busy_sessions = { + turn['session_id'] for turn in realtime_store.active_turns(agent_id=agent_id) + } + if busy_sessions and session_id not in busy_sessions: ms = self._restore_agent_state(agent_id) if ms and ms.focus: busy_msg = self._handle_busy_rejection( @@ -1416,7 +1468,11 @@ def handle_message(self, agent_id: str, external_user_id: str, 'channel_id': channel_id, 'message': message, }) - return {"response": None, "injected": True, "tool_trace": [], "timeline": []} + return { + "response": None, "injected": True, "tool_trace": [], "timeline": [], + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), + } # Message buffering: debounce rapid messages, then queue # Skip when skip_buffer=True (e.g. API routes need synchronous response) @@ -1429,6 +1485,7 @@ def handle_message(self, agent_id: str, external_user_id: str, task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id, session_db_agent_id=db_agent_id if is_subagent else None), send_via_channel=True) + self._mark_task_queued(task) timer = threading.Timer(buffer_seconds, self._enqueue_buffered, args=(task,)) timer.daemon = True with self._buffer_lock: @@ -1441,8 +1498,14 @@ def handle_message(self, agent_id: str, external_user_id: str, # If start() fails, cancel the timer and remove it from the dict with self._buffer_lock: self._buffer_timers.pop(session_id, None) + self._fail_queued_task(task, 'buffer_timer_failed') raise - return {"response": None, "buffered": True, "tool_trace": [], "timeline": []} + return { + "response": None, "buffered": True, "tool_trace": [], "timeline": [], + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), + "turn_id": task.ctx.turn_id, + } # Inter-agent messages: fire-and-forget (don't block the sender's worker thread). # The sub-agent/target processes asynchronously and results are delivered via @@ -1455,16 +1518,27 @@ def handle_message(self, agent_id: str, external_user_id: str, task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id, session_db_agent_id=db_agent_id if is_subagent else None), send_via_channel=False) - self._message_queue.put(task) - return {"response": None, "async": True, "tool_trace": [], "timeline": []} + self._put_task(task) + return { + "response": None, "async": True, "tool_trace": [], "timeline": [], + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), + "turn_id": task.ctx.turn_id, + } # No buffering — queue immediately and wait for result task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id, session_db_agent_id=db_agent_id if is_subagent else None), send_via_channel=bool(channel_id)) - self._message_queue.put(task) + self._put_task(task) task.event.wait() - return task.result + result = task.result or {} + result.update({ + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'turn_id': task.ctx.turn_id, + }) + return result def _enqueue_buffered(self, task: '_QueueTask') -> None: """Queue a buffered task, cleaning up its timer even on failure.""" @@ -1474,7 +1548,7 @@ def _enqueue_buffered(self, task: '_QueueTask') -> None: except Exception: pass # Timer may already be gone; the context manager handles cleanup try: - self._message_queue.put(task) + self._put_task(task, already_queued=True) except Exception: _logger.error("Failed to enqueue buffered task for session %s: %s", task.ctx.session_id, traceback.format_exc()) @@ -1503,12 +1577,28 @@ def _process_and_respond(self, agent: dict, ctx: SessionContext) -> dict: def _do_process(self, agent: dict, ctx: SessionContext) -> dict: """Internal: build messages and call LLM (must hold session lock).""" agent_id = agent['id'] + from backend.realtime_store import realtime_store + if not realtime_store.start_turn(ctx.turn_id): + ctx.turn_id, _ = realtime_store.queue_turn( + agent_id, ctx.session_id, uuid.uuid4().hex, + ) + realtime_store.start_turn(ctx.turn_id) self._set_busy(ctx.session_id, True) - self._set_agent_busy(agent_id, ctx.session_id) + active = realtime_store.busy_agents().get(agent_id, {}) event_stream.emit('agent_busy_changed', { 'agent_id': agent_id, 'busy': True, 'session_id': ctx.session_id, + 'session_ids': active.get('session_ids', [ctx.session_id]), + 'active_count': active.get('active_count', 1), + 'state': 'running', + 'turn_id': ctx.turn_id, + }) + event_stream.emit('turn_begin', { + 'agent_id': agent_id, + 'session_id': ctx.session_id, + 'turn_id': ctx.turn_id, + 'ts': int(time.time() * 1000), }) _turn_start = time.time() _turn_complete_emitted = False @@ -1542,10 +1632,8 @@ def _do_process(self, agent: dict, ctx: SessionContext) -> dict: 'tool_trace': [], 'is_error': True, 'thinking_duration': _err_dur, + 'turn_id': ctx.turn_id, }) - self._bg_executor.submit( - lambda sid=ctx.session_id: (time.sleep(SESSION_BUFFER_CLEANUP_DELAY), event_stream.cleanup_session_buffer(sid)), - ) result = { "response": "An unexpected error occurred. Please try again.", "error": True, @@ -1564,11 +1652,16 @@ def _do_process(self, agent: dict, ctx: SessionContext) -> dict: return result finally: self._set_busy(ctx.session_id, False) - self._clear_agent_busy(agent_id) + realtime_store.finish_turn(ctx.turn_id) + remaining = realtime_store.busy_agents().get(agent_id) event_stream.emit('agent_busy_changed', { 'agent_id': agent_id, - 'busy': False, - 'session_id': ctx.session_id, + 'busy': bool(remaining), + 'session_id': remaining.get('session_id', ctx.session_id) if remaining else ctx.session_id, + 'session_ids': remaining.get('session_ids', []) if remaining else [], + 'active_count': remaining.get('active_count', 0) if remaining else 0, + 'state': remaining.get('state', 'idle') if remaining else 'idle', + 'turn_id': None, }) # Drain any messages that arrived in the injection queue just as the loop # was finishing (race between _is_busy check and loop exit). They are @@ -1583,8 +1676,12 @@ def _do_process(self, agent: dict, ctx: SessionContext) -> dict: if orphaned: _logger.warning("%d orphaned injected message(s) for %s — re-processing as new turn", len(orphaned), ctx.session_id) - task = _QueueTask(agent, ctx, send_via_channel=bool(ctx.channel_id)) - self._message_queue.put(task) + next_ctx = SessionContext( + ctx.session_id, ctx.external_user_id, ctx.channel_id, + ctx.session_db_agent_id, + ) + task = _QueueTask(agent, next_ctx, send_via_channel=bool(ctx.channel_id)) + self._put_task(task) def _check_evonet_offline(self, agent: dict, ctx: SessionContext): """Return a completed turn result dict if the agent's Tunnel Workplace is offline, @@ -1613,13 +1710,17 @@ def _check_evonet_offline(self, agent: dict, ctx: SessionContext): ) db_agent_id = ctx.session_db_agent_id or agent['id'] - _db_retry(db.add_chat_message, ctx.session_id, 'assistant', reply, - agent_id=db_agent_id, metadata={'evonet_offline': True}, - label="save evonet offline reply") + message_id = _db_retry( + db.add_chat_message, ctx.session_id, 'assistant', reply, + agent_id=db_agent_id, metadata={'evonet_offline': True}, + label="save evonet offline reply", + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog_manager.get(db_agent_id, ctx.session_id).append({ 'type': 'final', 'session_id': ctx.session_id, 'content': reply, 'metadata': {'evonet_offline': True}, + 'message_id': message_id, }) if ctx.channel_id: try: @@ -1646,6 +1747,7 @@ def _check_evonet_offline(self, agent: dict, ctx: SessionContext): 'tool_trace': [], 'is_error': True, 'thinking_duration': 0, + 'message_id': message_id, }) return {'response': reply, 'tool_trace': [], 'error': True} @@ -2440,6 +2542,14 @@ def _heartbeat(): if is_error: result["error"] = True + last_assistant = db.get_last_assistant_message( + ctx.session_id, agent_id=db_agent_id, + ) + response_message_id = ( + last_assistant.get('id') + if last_assistant and last_assistant.get('content') == response_text + else None + ) # Emit turn_complete event event_stream.emit('turn_complete', { 'agent_id': agent_id, @@ -2451,12 +2561,9 @@ def _heartbeat(): 'tool_trace': tool_trace, 'is_error': is_error, 'thinking_duration': round(time.time() - _inner_turn_start, 1), + 'turn_id': ctx.turn_id, + 'message_id': response_message_id, }) - # Clean up per-session buffer after a delay to allow gap-fill requests to complete. - # Use executor to avoid timer leak (old timer never cancelled). - self._bg_executor.submit( - lambda sid=ctx.session_id: (time.sleep(SESSION_BUFFER_CLEANUP_DELAY), event_stream.cleanup_session_buffer(sid)), - ) return result @@ -2507,7 +2614,7 @@ def process_in_session(self, processing_agent_id: str, session_id: str, agent=agent, ctx=SessionContext(session_id, external_user_id, channel_id, session_db_agent_id), ) - self._message_queue.put(task) + self._put_task(task) def get_compiled_context(self, agent_id: str, user_id: str = None) -> dict: """Return the compiled system prompt and tool definitions for an agent.""" @@ -2628,12 +2735,22 @@ def _handle_busy_rejection(self, agent_id: str, agent_state: Any, reply = (f"Sorry, I'm busy with {reason}. " f"Want me to let you know when I'm done?") - _db_retry(db.add_chat_message, session_id, 'assistant', reply, - agent_id=agent_id, metadata={"busy_rejection": True}, - label="save busy rejection") + message_id = _db_retry( + db.add_chat_message, session_id, 'assistant', reply, + agent_id=agent_id, metadata={"busy_rejection": True}, + label="save busy rejection", + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog_manager.get(agent_id, session_id).append({'type': 'final', 'session_id': session_id, 'content': reply, - 'metadata': {'busy_rejection': True}}) + 'metadata': {'busy_rejection': True}, + 'message_id': message_id}) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': reply, 'message_id': message_id, + 'metadata': {'busy_rejection': True}, 'role': 'assistant', + }) # Record the deferral so the pending user message is auto-resumed once the # agent is free and unfocused (drained in _send_free_notification). Queued # on the opt-in branch too — the real answer supersedes the notification. @@ -2703,6 +2820,13 @@ def clear_session(self, agent_id: str, external_user_id: str, channel_id: Option """Clear chat history for a user's session.""" session_id = db.get_or_create_session(agent_id, external_user_id, channel_id) db.clear_session(session_id, agent_id=agent_id) + from backend.realtime_store import realtime_store + realtime_store.purge_session(session_id) + event_stream.emit('session_clear', { + 'agent_id': agent_id, + 'session_id': session_id, + 'turn_id': None, + }) self._session_skill_mds.pop(session_id, None) self._session_skill_tools.pop(session_id, None) @@ -2723,15 +2847,35 @@ def get_session_skills(self, session_id: str) -> list[dict]: return [{"skill_id": sk_id, "tool_count": len(tool_defs)} for sk_id, tool_defs in skills_data.items()] - def send_as_bot(self, session_id: str, text: str) -> bool: + def send_as_bot(self, session_id: str, text: str, + metadata: dict | None = None) -> bool: """Admin takeover: save message as assistant and send via channel.""" session = db.get_session_with_details(session_id) if not session: return False - db.add_chat_message(session_id, 'assistant', text, agent_id=session['agent_id']) + meta = {'admin_takeover': True} + if metadata: + meta.update(metadata) + message_id = db.add_chat_message( + session_id, 'assistant', text, + agent_id=session['agent_id'], metadata=meta, + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog_manager.get(session['agent_id'], session_id).append( {'type': 'final', 'session_id': session_id, 'content': text, - 'metadata': {'admin_takeover': True}}) + 'metadata': meta, 'message_id': message_id}) + event_stream.emit('message_received', { + 'agent_id': session['agent_id'], + 'session_id': session_id, + 'external_user_id': session.get('external_user_id', ''), + 'channel_id': session.get('channel_id'), + 'message': text, + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'metadata': meta, + 'role': 'assistant', + 'sender': 'admin', + }) # Send via channel if available if session.get('channel_id'): instance = channel_manager._active.get(session['channel_id']) @@ -2808,18 +2952,27 @@ def send_file_as_bot(self, session_id: str, file_path: str, else: content = f"[File: {filename}]" + # Also persist in main chat messages table + message_id = db.add_chat_message( + session_id, 'assistant', content, agent_id=agent_id, + metadata={'attachment_info': attachment_info}, + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog = chatlog_manager.get(agent_id, session_id) chatlog.append({ 'type': 'final', 'session_id': session_id, 'content': content, 'metadata': {'attachment_info': attachment_info, 'channel': channel_type}, + 'message_id': message_id, + }) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': content, 'message_id': message_id, + 'metadata': {'attachment_info': attachment_info}, + 'role': 'assistant', 'sender': agent_id, }) - - # Also persist in main chat messages table - db.add_chat_message(session_id, 'assistant', content, - agent_id=agent_id, - metadata={'attachment_info': attachment_info}) return True @@ -2907,16 +3060,30 @@ def send_as_user(self, session_id: str, text: str, ) if response is not None: # Command was recognized — save command echo and response, then return - db.add_chat_message(session_id, 'user', text, - agent_id=agent_id, metadata={'slash_command': True}) - db.add_chat_message(session_id, 'assistant', response, - agent_id=agent_id, metadata={'slash_command': True}) + command_meta = {'slash_command': True} + if metadata and metadata.get('client_message_id'): + command_meta['client_message_id'] = metadata['client_message_id'] + message_id = db.add_chat_message( + session_id, 'user', text, + agent_id=agent_id, metadata=command_meta, + ) + response_id = db.add_chat_message( + session_id, 'assistant', response, + agent_id=agent_id, metadata={'slash_command': True}, + ) _cl = chatlog_manager.get(agent_id, session_id) _cl.append({'type': 'user', 'session_id': session_id, 'content': text, 'sender_id': external_user_id, - 'metadata': {'slash_command': True}}) + 'metadata': command_meta, 'message_id': message_id}) _cl.append({'type': 'system', 'session_id': session_id, 'content': response, - 'metadata': {'slash_command': True}}) + 'metadata': {'slash_command': True}, 'message_id': response_id}) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': text, 'message_id': message_id, + 'client_message_id': command_meta.get('client_message_id'), + 'metadata': command_meta, 'role': 'user', + }) agent = db.get_agent(agent_id) # Check for any attachments created by the handler (e.g. /dump) attachment_info = None @@ -2949,13 +3116,8 @@ def send_as_user(self, session_id: str, text: str, 'thinking_duration': 0.0, 'slash_command': True, 'attachment_info': attachment_info, + 'message_id': response_id, }) - # Signal the client to clear the chat UI when the clear command was used - if cmd_name == 'clear': - event_stream.emit('session_clear', { - 'session_id': session_id, - 'agent_id': agent_id, - }) self._prefetcher.invalidate(session_id) return response # return response text so caller can include it in API response # Unknown command — fall through to normal LLM processing @@ -2969,10 +3131,12 @@ def send_as_user(self, session_id: str, text: str, meta['video_url'] = video_url if metadata: meta.update(metadata) - db.add_chat_message(session_id, 'user', text, agent_id=agent_id, metadata=meta) + message_id = db.add_chat_message( + session_id, 'user', text, agent_id=agent_id, metadata=meta, + ) chatlog_manager.get(agent_id, session_id).append( {'type': 'user', 'session_id': session_id, 'content': text, - 'metadata': meta}) + 'metadata': meta, 'message_id': message_id}) # Invalidate prefetched context — a new message arrived self._prefetcher.invalidate(session_id) @@ -2989,6 +3153,10 @@ def send_as_user(self, session_id: str, text: str, 'image_url': image_url, 'audio_url': audio_url, 'video_url': video_url, + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'metadata': meta, + 'role': 'user', }) # Mid-loop injection: if session is currently processing, inject message @@ -3012,7 +3180,7 @@ def send_as_user(self, session_id: str, text: str, if agent and agent.get('enabled', True): task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id), send_via_channel=False) - self._message_queue.put(task) + self._put_task(task) return True @@ -3030,4 +3198,4 @@ def resume_session(self, agent: dict, session_id: str, return task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id), send_via_channel=send_via_channel) - self._message_queue.put(task) + self._put_task(task) diff --git a/backend/channels/discord.py b/backend/channels/discord.py index cc18772b..e78f29ca 100644 --- a/backend/channels/discord.py +++ b/backend/channels/discord.py @@ -392,7 +392,20 @@ async def _handle_message(self, message, is_dm: bool): # Respect the per-session bot toggle. if not db.is_session_bot_enabled(session_id, agent_id=agent_id): - db.add_chat_message(session_id, 'user', text or '[Image]', agent_id=agent_id) + stored = text or '[Image]' + message_id = db.add_chat_message(session_id, 'user', stored, agent_id=agent_id) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'user', 'session_id': session_id, 'content': stored, + 'sender_id': user_id, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': user_id, 'channel_id': channel_id, + 'message': stored, 'message_id': message_id, 'role': 'user', + }) return # Include the replied-to bot message as context, when present. diff --git a/backend/channels/telegram.py b/backend/channels/telegram.py index ae363e9d..f1be2528 100644 --- a/backend/channels/telegram.py +++ b/backend/channels/telegram.py @@ -600,7 +600,20 @@ async def handle_message(update: Update, context: ContextTypes.DEFAULT_TYPE): # Check if bot is enabled for this session if not db.is_session_bot_enabled(session_id, agent_id=agent_id): - db.add_chat_message(session_id, 'user', text or '[Image]', agent_id=agent_id) + stored = text or '[Image]' + message_id = db.add_chat_message(session_id, 'user', stored, agent_id=agent_id) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'user', 'session_id': session_id, 'content': stored, + 'sender_id': user_id, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': user_id, 'channel_id': channel_id, + 'message': stored, 'message_id': message_id, 'role': 'user', + }) return # Detect reply/quote: include replied message content as context diff --git a/backend/channels/whatsapp.py b/backend/channels/whatsapp.py index 84999a52..c7ec5488 100644 --- a/backend/channels/whatsapp.py +++ b/backend/channels/whatsapp.py @@ -1011,8 +1011,22 @@ def handle_callback(self, payload: dict): if not db.is_session_bot_enabled(session_id, agent_id=agent_id): _logger.info("WhatsApp message stored only — bot disabled for session %s (sender=%s)", session_id, sender) - db.add_chat_message(session_id, 'user', final_text or text or '[Attachment]', - agent_id=agent_id) + stored = final_text or text or '[Attachment]' + message_id = db.add_chat_message( + session_id, 'user', stored, agent_id=agent_id, + ) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'user', 'session_id': session_id, 'content': stored, + 'sender_id': session_user_id, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': session_user_id, 'channel_id': self.channel_id, + 'message': stored, 'message_id': message_id, 'role': 'user', + }) return _logger.info("WhatsApp message received from %s (channel %s)", sender, self.channel_id) diff --git a/backend/event_stream.py b/backend/event_stream.py index 80c33da0..9253f7b3 100644 --- a/backend/event_stream.py +++ b/backend/event_stream.py @@ -17,7 +17,6 @@ Events are logged to logs/events.log (configurable via EVENT_LOG_FILE in .env). """ -import collections import itertools import logging import os @@ -28,11 +27,9 @@ _logger = logging.getLogger(__name__) -# Event types that the per-session chat SSE stream forwards to the browser. -# A per-session "chat seq" is assigned ONLY to these (see EventStream.emit), so the -# sequence the browser sees is contiguous and gap-detection never misfires on -# unrelated/global events. Keep in sync with the live stream + gap-fill transforms -# in routes/agents.py. +# Event types forwarded by the deprecated per-session stream. The durable +# gateway uses realtime_events.id instead; this counter remains only so old +# clients keep working during the compatibility window. CHAT_FORWARDED_EVENTS = frozenset({ 'turn_begin', 'llm_thinking', @@ -62,19 +59,15 @@ def __init__(self): self._listeners: Dict[str, List[Callable]] = {} self._lock = threading.Lock() self._log_lock = threading.Lock() - self._log_buffer: collections.deque = collections.deque(maxlen=1000) + self._log_buffer: List[str] = [] self._log_timer: Optional[threading.Timer] = None self._LOG_FLUSH_INTERVAL = 2.0 - self._LOG_BUFFER_LIMIT = 50 # soft flush trigger; deque maxlen=1000 is hard cap + self._LOG_BUFFER_LIMIT = 50 self._executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix='event') self._log_file: str = None # resolved lazily to avoid import-time circular deps - # Sequence numbering and ring buffers for gap-fill recovery + # Raw in-process sequence is retained for plugin compatibility only. self._seq_counter = itertools.count(1) self._buffer_lock = threading.Lock() - self._global_buffer: collections.deque = collections.deque(maxlen=1000) - self._session_buffers: Dict[str, collections.deque] = {} - # Per-session monotonic counter over CHAT_FORWARDED_EVENTS only, so the - # browser's chat stream sees a contiguous (gap-free) sequence. self._session_chat_seq: Dict[str, int] = {} self._web_listeners: Dict[str, int] = {} @@ -147,22 +140,20 @@ def emit(self, event_name: str, data: dict): seq = next(self._seq_counter) data['_seq'] = seq data['_event'] = event_name - # Store in ring buffers for gap-fill queries session_id = data.get('session_id') - chat_seq = None - entry = {'seq': seq, 'event': event_name, 'data': data} with self._buffer_lock: - # Assign a contiguous per-session chat seq for forwarded events only. if session_id and event_name in CHAT_FORWARDED_EVENTS: chat_seq = self._session_chat_seq.get(session_id, 0) + 1 self._session_chat_seq[session_id] = chat_seq data['_chat_seq'] = chat_seq - entry['chat_seq'] = chat_seq - self._global_buffer.append(entry) - if session_id: - if session_id not in self._session_buffers: - self._session_buffers[session_id] = collections.deque(maxlen=500) - self._session_buffers[session_id].append(entry) + # Journal synchronously before asynchronous plugin listeners. This is + # what gives browser replay a stable total order even though raw plugin + # callbacks still run concurrently. + try: + from backend.realtime_store import record_internal_event + record_internal_event(event_name, data) + except Exception as exc: + _logger.error("Failed to journal realtime event '%s': %s", event_name, exc) preview = ', '.join(f'{k}={str(v)[:120]}' for k, v in data.items() if not k.startswith('_')) self._write_log(f"[seq={seq}] {event_name} | {preview}") with self._lock: @@ -170,27 +161,8 @@ def emit(self, event_name: str, data: dict): for cb in listeners: self._executor.submit(self._safe_call, event_name, cb, data) - def get_events_in_range(self, session_id: str, after_seq: int, up_to_seq: int) -> list: - """Return chat-forwarded events for session_id where - after_seq < chat_seq <= up_to_seq (chat_seq is the per-session chat seq).""" - with self._buffer_lock: - buf = self._session_buffers.get(session_id, collections.deque()) - if not buf: - return [] - return [e for e in buf - if 'chat_seq' in e and after_seq < e['chat_seq'] <= up_to_seq] - - def get_session_events(self, session_id: str, after_seq: int = 0) -> list: - """Return chat-forwarded events for session_id with chat_seq > after_seq.""" - with self._buffer_lock: - buf = self._session_buffers.get(session_id, collections.deque()) - return [e for e in buf if 'chat_seq' in e and e['chat_seq'] > after_seq] - def cleanup_session_buffer(self, session_id: str): - """Remove per-session buffer (called after turn completes).""" - with self._buffer_lock: - self._session_buffers.pop(session_id, None) - self._session_chat_seq.pop(session_id, None) + """Deprecated no-op; durable event retention replaces delayed cleanup.""" def register_web_listener(self, session_id: str): with self._lock: diff --git a/backend/realtime_store.py b/backend/realtime_store.py new file mode 100644 index 00000000..00b89ac9 --- /dev/null +++ b/backend/realtime_store.py @@ -0,0 +1,609 @@ +"""Durable realtime journal and active-turn projection. + +The journal is the source of truth for browser replay. The existing +``EventStream`` remains the in-process plugin bus; it records normalized public +events here before dispatching asynchronous listeners. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import sqlite3 +import threading +import time +import uuid +from contextlib import contextmanager +from datetime import datetime +import config + + +log = logging.getLogger(__name__) + +RETENTION_MS = 24 * 60 * 60 * 1000 +_CLEANUP_INTERVAL_MS = 60 * 60 * 1000 +_ATTACHMENT_KEYS = frozenset({ + 'attachment_id', 'filename', 'mime_type', 'size_bytes', 'is_image', +}) + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def _json_default(value): + if isinstance(value, datetime): + return value.isoformat() + return str(value) + + +def _public_attachment(value): + if not isinstance(value, dict): + return None + return {key: value.get(key) for key in _ATTACHMENT_KEYS if key in value} + + +def _public_metadata(value) -> dict: + """Remove browser-local URLs and backend paths from SSE payloads.""" + if not isinstance(value, dict): + return {} + result = {} + for key, item in value.items(): + if key in {'image_url', 'audio_url', 'video_url', 'file_path', 'path'}: + continue + if key == 'attachment_info': + result[key] = _public_attachment(item) + elif key == 'attachment_infos' and isinstance(item, list): + result[key] = [clean for clean in map(_public_attachment, item) if clean] + else: + result[key] = item + return result + + +def _public_message(value: str) -> str: + # Attachment markers are part of the model prompt, but local paths do not + # belong in browser broadcasts. + return re.sub( + r'(\[Attached:[^\]]*?)\s+path=[^\]]+(\])', r'\1\2', value or '', + ) + + +class RealtimeStore: + def __init__(self, db_path: str | None = None): + self.db_path = db_path + self._tls = threading.local() + self._schema_lock = threading.Lock() + self._schema_paths: set[str] = set() + self._condition = threading.Condition() + self._cleanup_lock = threading.Lock() + self._last_cleanup_ms = 0 + + @contextmanager + def _connect(self): + db_path = self._resolve_db_path() + self._ensure_schema(db_path) + conn = getattr(self._tls, 'conn', None) + if conn is not None and ( + getattr(self._tls, 'db_path', None) != db_path + or getattr(self._tls, 'pid', None) != os.getpid()): + conn.close() + conn = None + if conn is None: + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect( + f'file:{db_path}?mode=rwc&busy_timeout=10000', + uri=True, timeout=10, + ) + conn.row_factory = sqlite3.Row + conn.execute('PRAGMA busy_timeout=10000') + conn.execute('PRAGMA journal_mode=WAL') + conn.execute('PRAGMA synchronous=NORMAL') + self._tls.conn = conn + self._tls.db_path = db_path + self._tls.pid = os.getpid() + with conn: + yield conn + + def _resolve_db_path(self) -> str: + if self.db_path: + return os.path.abspath(self.db_path) + try: + from models.db import db + return os.path.abspath(db.db_path) + except Exception: + return os.path.abspath(config.DB_PATH) + + def _ensure_schema(self, db_path: str): + if db_path in self._schema_paths: + return + with self._schema_lock: + if db_path in self._schema_paths: + return + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect(db_path, timeout=10) + try: + conn.execute('PRAGMA journal_mode=WAL') + conn.executescript(""" + CREATE TABLE IF NOT EXISTS realtime_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + occurred_at_ms INTEGER NOT NULL, + expires_at_ms INTEGER, + channel TEXT NOT NULL, + event_type TEXT NOT NULL, + agent_id TEXT, + session_id TEXT, + workplace_id TEXT, + turn_id TEXT, + payload_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_realtime_session_id + ON realtime_events(session_id, id); + CREATE INDEX IF NOT EXISTS idx_realtime_channel_id + ON realtime_events(channel, id); + CREATE INDEX IF NOT EXISTS idx_realtime_workplace_id + ON realtime_events(workplace_id, id); + CREATE INDEX IF NOT EXISTS idx_realtime_turn_id + ON realtime_events(turn_id, id); + CREATE INDEX IF NOT EXISTS idx_realtime_expiry + ON realtime_events(expires_at_ms); + + CREATE TABLE IF NOT EXISTS active_turns ( + turn_id TEXT PRIMARY KEY, + agent_id TEXT NOT NULL, + session_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued', 'running')), + queued_at_ms INTEGER NOT NULL, + started_at_ms INTEGER, + updated_at_ms INTEGER NOT NULL, + owner_pid INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_active_turns_agent + ON active_turns(agent_id, updated_at_ms); + CREATE INDEX IF NOT EXISTS idx_active_turns_session + ON active_turns(session_id, updated_at_ms); + """) + conn.commit() + finally: + conn.close() + self._schema_paths.add(db_path) + + def close(self): + conn = getattr(self._tls, 'conn', None) + if conn is not None: + conn.close() + self._tls.conn = None + self._tls.db_path = None + self._tls.pid = None + + def high_water(self) -> int: + with self._connect() as conn: + row = conn.execute('SELECT COALESCE(MAX(id), 0) AS id FROM realtime_events').fetchone() + return int(row['id']) + + def publish(self, channel: str, event_type: str, payload: dict, *, + agent_id: str | None = None, session_id: str | None = None, + workplace_id: str | None = None, turn_id: str | None = None, + occurred_at_ms: int | None = None) -> int: + now = occurred_at_ms or _now_ms() + expires_at = None if turn_id else now + RETENTION_MS + body = json.dumps(payload, separators=(',', ':'), default=_json_default) + with self._connect() as conn: + cursor = conn.execute(""" + INSERT INTO realtime_events ( + occurred_at_ms, expires_at_ms, channel, event_type, + agent_id, session_id, workplace_id, turn_id, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (now, expires_at, channel, event_type, agent_id, session_id, + workplace_id, turn_id, body)) + event_id = int(cursor.lastrowid) + with self._condition: + self._condition.notify_all() + self._maybe_cleanup(now) + return event_id + + def events_after(self, after_id: int, channels: set[str], *, + session_id: str | None = None, + agent_id: str | None = None, + workplace_id: str | None = None, + up_to_id: int | None = None, + active_only: bool = False, + limit: int = 500) -> list[dict]: + clauses = ['e.id > ?', '(e.expires_at_ms IS NULL OR e.expires_at_ms > ?)'] + params: list = [after_id, _now_ms()] + if up_to_id is not None: + clauses.append('e.id <= ?') + params.append(up_to_id) + if active_only: + clauses.append('EXISTS (SELECT 1 FROM active_turns a WHERE a.turn_id = e.turn_id)') + + channel_clauses = [] + global_channels = channels - {'chat', 'workplace'} + if global_channels: + placeholders = ','.join('?' for _ in global_channels) + channel_clauses.append(f'e.channel IN ({placeholders})') + params.extend(sorted(global_channels)) + if 'chat' in channels and session_id: + # Session-scoped approval/status events share the chat stream so a + # chat-only consumer still sees queued/busy/approval state. + channel_clauses.append( + "(e.session_id = ? AND e.channel IN ('chat','status','approvals'))" + ) + params.append(session_id) + if 'workplace' in channels and workplace_id: + channel_clauses.append("(e.channel = 'workplace' AND e.workplace_id = ?)") + params.append(workplace_id) + if not channel_clauses: + return [] + clauses.append('(' + ' OR '.join(channel_clauses) + ')') + + if session_id and 'chat' in channels: + clauses.append("(e.channel != 'chat' OR e.session_id = ?)") + params.append(session_id) + if agent_id and 'chat' in channels: + clauses.append("(e.channel != 'chat' OR e.agent_id IS NULL OR e.agent_id = ?)") + params.append(agent_id) + + params.append(limit) + sql = f""" + SELECT e.* FROM realtime_events e + WHERE {' AND '.join(clauses)} + ORDER BY e.id ASC LIMIT ? + """ + with self._connect() as conn: + rows = conn.execute(sql, params).fetchall() + return [self._decode_event(row) for row in rows] + + @staticmethod + def _decode_event(row: sqlite3.Row) -> dict: + payload = json.loads(row['payload_json']) + payload.update({ + 'event_id': row['id'], + 'seq': row['id'], + 'timestamp': row['occurred_at_ms'], + 'channel': row['channel'], + }) + if row['agent_id']: + payload.setdefault('agent_id', row['agent_id']) + if row['session_id']: + payload.setdefault('session_id', row['session_id']) + if row['workplace_id']: + payload.setdefault('workplace_id', row['workplace_id']) + if row['turn_id']: + payload['turn_id'] = row['turn_id'] + return { + 'id': row['id'], + 'timestamp': row['occurred_at_ms'], + 'channel': row['channel'], + 'event': row['event_type'], + 'agent_id': row['agent_id'], + 'session_id': row['session_id'], + 'workplace_id': row['workplace_id'], + 'turn_id': row['turn_id'], + 'data': payload, + } + + def wait_for_events(self, after_id: int, timeout: float = 15) -> None: + if self.high_water() > after_id: + return + with self._condition: + self._condition.wait(timeout) + + def queue_turn(self, agent_id: str, session_id: str, + turn_id: str | None = None) -> tuple[str, bool]: + """Create queued state, reusing an existing debounced queue entry.""" + now = _now_ms() + with self._connect() as conn: + conn.execute('BEGIN IMMEDIATE') + existing = conn.execute(""" + SELECT turn_id FROM active_turns + WHERE session_id = ? AND state = 'queued' + ORDER BY queued_at_ms ASC LIMIT 1 + """, (session_id,)).fetchone() + if existing: + return existing['turn_id'], False + turn_id = turn_id or uuid.uuid4().hex + conn.execute(""" + INSERT INTO active_turns ( + turn_id, agent_id, session_id, state, queued_at_ms, + started_at_ms, updated_at_ms, owner_pid + ) VALUES (?, ?, ?, 'queued', ?, NULL, ?, ?) + """, (turn_id, agent_id, session_id, now, now, os.getpid())) + active = self.busy_agents().get(agent_id, {}) + self.publish('status', 'turn_queued', { + 'agent_id': agent_id, + 'session_id': session_id, + 'busy': True, + 'state': 'queued', + }, agent_id=agent_id, session_id=session_id, turn_id=turn_id, + occurred_at_ms=now) + self.publish('status', 'agent_busy_changed', { + 'agent_id': agent_id, + 'session_id': active.get('session_id', session_id), + 'session_ids': active.get('session_ids', [session_id]), + 'active_count': active.get('active_count', 1), + 'busy': True, + 'state': 'queued', + }, agent_id=agent_id, session_id=session_id, turn_id=turn_id, + occurred_at_ms=now) + return turn_id, True + + def start_turn(self, turn_id: str) -> dict | None: + now = _now_ms() + with self._connect() as conn: + conn.execute(""" + UPDATE active_turns SET state = 'running', started_at_ms = ?, + updated_at_ms = ?, owner_pid = ? WHERE turn_id = ? + """, (now, now, os.getpid(), turn_id)) + row = conn.execute( + 'SELECT * FROM active_turns WHERE turn_id = ?', (turn_id,), + ).fetchone() + with self._condition: + self._condition.notify_all() + return dict(row) if row else None + + def finish_turn(self, turn_id: str) -> None: + now = _now_ms() + with self._connect() as conn: + conn.execute( + 'UPDATE realtime_events SET expires_at_ms = ? WHERE turn_id = ?', + (now + RETENTION_MS, turn_id), + ) + conn.execute('DELETE FROM active_turns WHERE turn_id = ?', (turn_id,)) + with self._condition: + self._condition.notify_all() + + def current_turn_id(self, session_id: str) -> str | None: + with self._connect() as conn: + row = conn.execute(""" + SELECT turn_id FROM active_turns WHERE session_id = ? + ORDER BY CASE state WHEN 'running' THEN 0 ELSE 1 END, + updated_at_ms DESC LIMIT 1 + """, (session_id,)).fetchone() + return row['turn_id'] if row else None + + def active_turns(self, *, agent_id: str | None = None, + session_id: str | None = None) -> list[dict]: + clauses = [] + params = [] + if agent_id: + clauses.append('agent_id = ?') + params.append(agent_id) + if session_id: + clauses.append('session_id = ?') + params.append(session_id) + where = (' WHERE ' + ' AND '.join(clauses)) if clauses else '' + with self._connect() as conn: + rows = conn.execute( + 'SELECT * FROM active_turns' + where + ' ORDER BY queued_at_ms', + params, + ).fetchall() + return [dict(row) for row in rows] + + def busy_agents(self) -> dict: + now = _now_ms() + result = {} + for turn in self.active_turns(): + entry = result.setdefault(turn['agent_id'], { + 'session_id': turn['session_id'], + 'session_ids': [], + 'active_count': 0, + 'started_at': (turn['started_at_ms'] or turn['queued_at_ms']) / 1000, + 'state': turn['state'], + }) + if turn['session_id'] not in entry['session_ids']: + entry['session_ids'].append(turn['session_id']) + entry['active_count'] += 1 + if turn['state'] == 'running': + entry['state'] = 'running' + entry['session_id'] = turn['session_id'] + entry['started_at'] = (turn['started_at_ms'] or turn['queued_at_ms']) / 1000 + for entry in result.values(): + entry['elapsed'] = round(now / 1000 - entry['started_at'], 1) + return result + + def purge_session(self, session_id: str) -> None: + with self._connect() as conn: + conn.execute('DELETE FROM active_turns WHERE session_id = ?', (session_id,)) + conn.execute('DELETE FROM realtime_events WHERE session_id = ?', (session_id,)) + with self._condition: + self._condition.notify_all() + + def purge_all(self) -> None: + with self._connect() as conn: + conn.execute('DELETE FROM active_turns') + conn.execute('DELETE FROM realtime_events') + with self._condition: + self._condition.notify_all() + + def interrupt_stale_turns(self) -> list[dict]: + self._maybe_cleanup(_now_ms()) + stale = [] + for turn in self.active_turns(): + if turn.get('owner_pid') == os.getpid(): + continue + stale.append(turn) + for turn in stale: + payload = { + 'agent_id': turn['agent_id'], + 'session_id': turn['session_id'], + 'response': '', + 'interrupted': True, + 'is_error': True, + 'reason': 'server_restart', + } + self.publish('chat', 'done', payload, + agent_id=turn['agent_id'], session_id=turn['session_id'], + turn_id=turn['turn_id']) + self.publish('status', 'agent_busy_changed', { + 'agent_id': turn['agent_id'], 'session_id': turn['session_id'], + 'busy': False, 'interrupted': True, + }, agent_id=turn['agent_id'], session_id=turn['session_id'], + turn_id=turn['turn_id']) + self.finish_turn(turn['turn_id']) + return stale + + def _maybe_cleanup(self, now_ms: int) -> None: + if now_ms - self._last_cleanup_ms < _CLEANUP_INTERVAL_MS: + return + if not self._cleanup_lock.acquire(blocking=False): + return + try: + if now_ms - self._last_cleanup_ms < _CLEANUP_INTERVAL_MS: + return + with self._connect() as conn: + conn.execute( + 'DELETE FROM realtime_events WHERE expires_at_ms IS NOT NULL AND expires_at_ms <= ?', + (now_ms,), + ) + self._last_cleanup_ms = now_ms + finally: + self._cleanup_lock.release() + +realtime_store = RealtimeStore() + + +def record_internal_event(event_name: str, data: dict) -> list[int]: + """Normalize one raw runtime event into durable public SSE events.""" + session_id = data.get('session_id') or None + agent_id = data.get('agent_id') or None + workplace_id = data.get('workplace_id') or None + if event_name == 'session_clear' and session_id: + realtime_store.purge_session(session_id) + turn_id = ( + data.get('turn_id') if 'turn_id' in data + else (realtime_store.current_turn_id(session_id) if session_id else None) + ) + metadata = _public_metadata(data.get('metadata')) + specs: list[tuple[str, str, dict]] = [] + + if event_name == 'turn_begin': + specs.append(('chat', 'turn_begin', {'ts': data.get('ts', _now_ms())})) + elif event_name == 'llm_thinking': + specs.append(('chat', 'thinking', {'content': data.get('thinking', '')})) + elif event_name == 'tool_call_started': + specs.append(('chat', 'tool_call_started', { + 'tool': data.get('tool_name', ''), 'args': data.get('tool_args', {}), + 'param_types': data.get('param_types', {}), + })) + elif event_name == 'tool_executed': + specs.append(('chat', 'tool_executed', { + 'tool': data.get('tool_name', ''), 'args': data.get('tool_args', {}), + 'result': data.get('tool_result', {}), 'error': data.get('has_error', False), + })) + elif event_name in {'state:changed', 'tasks:auto_transition', 'tasks:stale'}: + keys = ('mode', 'plan_file', 'tasks', 'loaded_skills', 'task_ids') + specs.append(('chat', event_name, {key: data[key] for key in keys if key in data})) + elif event_name == 'llm_response_chunk': + specs.append(('chat', 'response_chunk', { + 'content': data.get('content', ''), 'is_final': data.get('is_final', False), + 'send_as_message': data.get('send_as_message', False), + })) + elif event_name == 'turn_complete': + done = { + 'thinking_duration': data.get('thinking_duration'), + 'response': data.get('response', ''), + 'slash_command': data.get('slash_command', False), + 'attachment_info': _public_attachment(data.get('attachment_info')), + 'message_id': data.get('message_id'), + 'is_error': data.get('is_error', False), + 'interrupted': data.get('interrupted', False), + } + specs.append(('chat', 'done', done)) + if data.get('response') and not data.get('is_error'): + specs.append(('status', 'agent_turn_complete', { + 'agent_id': agent_id or '', + 'agent_name': data.get('agent_name', ''), + 'response': data.get('response', ''), + 'session_id': session_id or '', + 'external_user_id': data.get('external_user_id', ''), + })) + elif event_name == 'agent_busy_changed': + specs.append(('status', 'agent_busy_changed', { + 'agent_id': agent_id or '', 'busy': data.get('busy', False), + 'session_id': session_id or '', + 'session_ids': data.get('session_ids', [session_id] if session_id else []), + 'active_count': data.get('active_count', 1 if data.get('busy') else 0), + 'state': data.get('state', 'running' if data.get('busy') else 'idle'), + })) + elif event_name == 'message_received': + specs.append(('chat', 'message_received', { + 'message': _public_message(data.get('message', '')), + 'content': _public_message(data.get('message', '')), + 'role': data.get('role', 'user'), + 'message_id': data.get('message_id'), + 'client_message_id': data.get('client_message_id') or metadata.get('client_message_id'), + 'metadata': metadata, + 'external_user_id': data.get('external_user_id', ''), + 'sender': data.get('sender') or data.get('external_user_id', ''), + })) + elif event_name in {'message_injected', 'message_injection_applied'}: + specs.append(('chat', event_name, { + 'message': data.get('message', ''), 'content': data.get('content', ''), + 'count': data.get('count', 1), + })) + elif event_name == 'llm_retry': + specs.append(('chat', 'retry', { + 'retry_count': data.get('retry_count', 0), + 'max_retries': data.get('max_retries', 0), + 'error_type': data.get('error_type', ''), + 'message': data.get('user_message', ''), + })) + elif event_name in {'approval_required', 'approval_resolved'}: + if event_name == 'approval_required': + payload = { + 'approval_id': data.get('approval_id', ''), + 'agent_id': agent_id or '', + 'source_agent_id': data.get('source_agent_id', ''), + 'source_agent_name': data.get('source_agent_name', ''), + 'tool': data.get('tool_name', ''), 'args': data.get('tool_args', {}), + 'approval_info': data.get('approval_info', {}), + 'reasons': data.get('reasons', []), 'score': data.get('score'), + } + else: + payload = { + 'approval_id': data.get('approval_id', ''), + 'decision': data.get('decision', ''), + 'timed_out': data.get('timed_out', False), + } + specs.append(('approvals', event_name, payload)) + elif event_name in {'whatsapp_bridge_status', 'panel_updated'}: + specs.append(('status', event_name, { + key: data.get(key, '') for key in ('agent_id', 'channel_id', 'status') + if key in data + })) + elif event_name in { + 'connector_connected', 'connector_disconnected', 'connector_paired', + 'workplace_status_changed', + }: + specs.append(('workplace', event_name, { + key: value for key, value in data.items() if not key.startswith('_') + })) + elif event_name in {'update_status', 'update_done'}: + specs.append(('update', event_name, { + key: value for key, value in data.items() if not key.startswith('_') + })) + elif event_name == 'whatsapp_restriction_warning': + specs.append(('chat', event_name, { + 'content': data.get('content', ''), 'metadata': metadata, + })) + elif event_name == 'session_clear': + specs.append(('chat', event_name, { + 'session_id': session_id or '', 'agent_id': agent_id or '', + })) + elif event_name == 'turn_split': + specs.append(('chat', event_name, {})) + elif event_name == 'evonic:agent-state-changed': + specs.append(('chat', 'state_changed', { + 'agent_id': agent_id or '', 'session_id': session_id or '', + })) + + ids = [] + for channel, public_name, payload in specs: + ids.append(realtime_store.publish( + channel, public_name, payload, + agent_id=agent_id, session_id=session_id, + workplace_id=workplace_id, turn_id=turn_id, + )) + return ids diff --git a/backend/scheduler.py b/backend/scheduler.py index 64c3be22..f227c7f7 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -759,8 +759,21 @@ def _action_static_message(self, config: dict): if external_user_id != '__scheduler__' and channel_id: session_id = main_db.get_or_create_session( agent_id, external_user_id, channel_id) - main_db.add_chat_message( + message_id = main_db.add_chat_message( session_id, 'assistant', message, agent_id=agent_id) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'final', 'session_id': session_id, + 'content': message, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': message, 'message_id': message_id, + 'role': 'assistant', 'sender': 'scheduler', + }) # Push via channel (Telegram, etc.) so the user sees it immediately. # Only return on successful delivery — if the channel is down or diff --git a/backend/slash_commands.py b/backend/slash_commands.py index b84432e4..0131dd6f 100644 --- a/backend/slash_commands.py +++ b/backend/slash_commands.py @@ -235,7 +235,6 @@ def clear_handler( no_archive = not archive_requested db.clear_session(session_id, agent_id, no_archive=no_archive) - # Clear in-memory loaded skill state so skill badges disappear from session state UI from backend.agent_runtime import agent_runtime agent_runtime._session_skill_mds.pop(session_id, None) @@ -277,7 +276,9 @@ def clear_handler( # Emit session_clear event try: from backend.event_stream import event_stream - event_stream.emit('session_clear', {'session_id': session_id, 'agent_id': agent_id}) + event_stream.emit('session_clear', { + 'session_id': session_id, 'agent_id': agent_id, 'turn_id': None, + }) except Exception: pass diff --git a/backend/tools/agent_messaging.py b/backend/tools/agent_messaging.py index afcab6e9..e40f2803 100644 --- a/backend/tools/agent_messaging.py +++ b/backend/tools/agent_messaging.py @@ -1116,17 +1116,27 @@ def _exec_send_channel_message(args: dict, agent_context: dict) -> dict: # ---- Record in chat log ---- try: - db.add_chat_message( + message_id = db.add_chat_message( session_id, 'assistant', message, agent_id=sender_id, metadata={'channel_send': True}, ) + message_id = message_id if type(message_id) in (int, str) else None from models.chatlog import chatlog_manager chatlog_manager.get(sender_id, session_id).append({ 'type': 'final', 'session_id': session_id, 'content': message, 'metadata': {'channel_send': True}, + 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': sender_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': message, 'message_id': message_id, + 'metadata': {'channel_send': True}, + 'role': 'assistant', 'sender': sender_id, }) except Exception as e: _logger.warning("send_channel_message: chat log error: %s", e) diff --git a/backend/update_manager.py b/backend/update_manager.py index 5de87940..d00b8a48 100644 --- a/backend/update_manager.py +++ b/backend/update_manager.py @@ -268,6 +268,13 @@ def _append_log(level: str, message: str): def _notify_listeners(): snapshot = get_status() + try: + from backend.event_stream import event_stream + event_stream.emit('update_status', snapshot) + if snapshot.get('status') in ('success', 'failed'): + event_stream.emit('update_done', {'status': snapshot['status']}) + except Exception as exc: + log.warning('Failed to publish update status: %s', exc) dead = [] for q in _listeners: try: diff --git a/models/mixins/chat_delegation.py b/models/mixins/chat_delegation.py index cfce6703..327d41ba 100644 --- a/models/mixins/chat_delegation.py +++ b/models/mixins/chat_delegation.py @@ -132,6 +132,8 @@ def clear_session(self, session_id: str, agent_id: str = None, no_archive: bool llm_trace_manager.get(agent_id, session_id).clear() llm_trace_manager.evict(agent_id, session_id) self._remove_session_index(session_id) + from backend.realtime_store import realtime_store + realtime_store.purge_session(session_id) def get_last_message_timestamp(self, session_id: str, agent_id: str = None) -> Optional[float]: """Return the unix timestamp of the most recent message in a session. @@ -201,6 +203,8 @@ def delete_session(self, session_id: str, agent_id: str = None) -> bool: pass self._refresh_session_count(agent_id) self._remove_session_index(session_id) + from backend.realtime_store import realtime_store + realtime_store.purge_session(session_id) # Wipe attachments tied to this session (rows + on-disk files) so # they don't linger unreachable after the conversation is gone. try: diff --git a/routes/agents.py b/routes/agents.py index 184b258f..081c78db 100644 --- a/routes/agents.py +++ b/routes/agents.py @@ -9,8 +9,9 @@ import uuid import queue import logging +from urllib.parse import urlencode from typing import Dict, Any, List, Optional -from flask import Blueprint, render_template, jsonify, request, Response, session, stream_with_context, g +from flask import Blueprint, render_template, jsonify, request, Response, session, stream_with_context, g, redirect from models.db import db from models.chatlog import chatlog_manager, _DISPLAY_TYPES from backend.agent_portability import AgentPortabilityError, export_agent, import_agent, preflight_import @@ -1646,6 +1647,7 @@ def api_chat(agent_id): if request.content_type and request.content_type.startswith('multipart/form-data'): message = (request.form.get('message') or '').strip() user_id = (request.form.get('user_id') or 'anonymous').strip() + client_message_id = (request.form.get('client_message_id') or '').strip() files = [f for f in request.files.getlist('files') if f and f.filename] if not files: legacy_file = request.files.get('file') @@ -1654,8 +1656,12 @@ def api_chat(agent_id): data = request.get_json() or {} message = data.get('message', '').strip() user_id = data.get('user_id', 'anonymous') + client_message_id = (data.get('client_message_id') or '').strip() files = [] + if client_message_id and not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', client_message_id): + return jsonify({'error': 'Invalid client_message_id'}), 400 + if not message and not files: return jsonify({'error': 'Message is required'}), 400 @@ -1708,9 +1714,9 @@ def api_chat(agent_id): image_url = image_urls[0] if image_urls else None attachment_info = attachment_infos[0] if attachment_infos else None - upload_meta = None + upload_meta = {'client_message_id': client_message_id} if client_message_id else None if attachment_infos: - upload_meta = {'attachment_infos': attachment_infos} + upload_meta = dict(upload_meta or {}, attachment_infos=attachment_infos) if len(attachment_infos) == 1: upload_meta['attachment_info'] = attachment_info @@ -1721,14 +1727,24 @@ def api_chat(agent_id): metadata=upload_meta, ) if result.get('buffered'): - resp = {'success': True, 'buffered': True} + resp = { + 'success': True, 'buffered': True, + 'message_id': result.get('message_id'), + 'client_message_id': result.get('client_message_id'), + 'turn_id': result.get('turn_id'), + } if attachment_infos: resp['attachment_infos'] = attachment_infos if len(attachment_infos) == 1: resp['attachment_info'] = attachment_info return jsonify(resp) if result.get('injected'): - resp = {'success': True, 'injected': True} + resp = { + 'success': True, 'injected': True, + 'message_id': result.get('message_id'), + 'client_message_id': result.get('client_message_id'), + 'turn_id': result.get('turn_id'), + } if attachment_infos: resp['attachment_infos'] = attachment_infos if len(attachment_infos) == 1: @@ -1742,6 +1758,10 @@ def api_chat(agent_id): 'slash_command': result.get('slash_command', False), 'bash_exec': result.get('bash_exec', False), 'clear_ui': result.get('clear_ui', False), + 'message_id': result.get('message_id'), + 'response_message_id': result.get('response_message_id'), + 'client_message_id': result.get('client_message_id'), + 'turn_id': result.get('turn_id'), } if attachment_infos: resp['attachment_infos'] = attachment_infos @@ -1768,6 +1788,8 @@ def api_chat_jsonl(agent_id): Response: {"entries": [...], "has_more": bool} has_more is true when exactly `limit` entries were returned. """ + from backend.realtime_store import realtime_store + realtime_cursor = realtime_store.high_water() user_id = request.args.get('user_id', 'anonymous') session_id = request.args.get('session_id') to_ts = request.args.get('to_ts', type=int) @@ -1794,15 +1816,21 @@ def api_chat_jsonl(agent_id): # fall through to tail_by_messages instead. all_entries = chatlog.get_entries_after_ts(after_ts, types=_DISPLAY_TYPES) entries = all_entries[:limit] - return jsonify({'entries': entries, 'has_more': len(all_entries) > limit}) + response = jsonify({'entries': entries, 'has_more': len(all_entries) > limit}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response # Backward (tail) scan: entries older than to_ts, counted by logical messages entries, has_more = chatlog.tail_by_messages(limit=limit, to_ts=to_ts) - return jsonify({'entries': entries, 'has_more': has_more}) + response = jsonify({'entries': entries, 'has_more': has_more}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response @agents_bp.route('/api/agents//chat/history', methods=['GET']) def api_chat_history(agent_id): + from backend.realtime_store import realtime_store + realtime_cursor = realtime_store.high_water() user_id = request.args.get('user_id', 'anonymous') session_id = db.get_session_id(agent_id, user_id) or db.get_or_create_session(agent_id, user_id) messages = db.get_session_messages(session_id, limit=50, agent_id=agent_id) @@ -1828,7 +1856,9 @@ def api_chat_history(agent_id): if m.get('metadata'): entry['metadata'] = m['metadata'] filtered.append(entry) - return jsonify({'messages': filtered}) + response = jsonify({'messages': filtered}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response @agents_bp.route('/api/agents//chat/poll', methods=['GET']) @@ -2367,6 +2397,14 @@ def api_chat_stream(agent_id): session_id = request.args.get('session_id') if not session_id: return jsonify({'error': 'session_id required'}), 400 + after = request.args.get('after', '0') + return redirect( + '/api/realtime/stream?' + urlencode({ + 'chat': 1, 'agent_id': agent_id, + 'session_id': session_id, 'after': after, + }), + code=307, + ) from backend.event_stream import event_stream @@ -2601,6 +2639,7 @@ def api_approvals_stream(): """Global SSE endpoint — pushes ALL approval events (any agent, any session) to every connected client. DEPRECATED: Use unified GET /api/realtime/stream?channels=approvals instead.""" + return redirect('/api/realtime/stream?channels=approvals', code=307) import logging as _log_depr _log_depr.getLogger(__name__).warning( "DEPRECATED endpoint /api/approvals/stream used — " @@ -2694,7 +2733,7 @@ def generate(): @agents_bp.route('/api/agents//chat/events', methods=['GET']) def api_chat_events(agent_id): - """Fetch missed SSE events by sequence range for gap-detection recovery.""" + """Compatibility reader backed by the durable realtime journal.""" session_id = request.args.get('session_id') after_seq = request.args.get('after', type=int) up_to_seq = request.args.get('up_to', type=int) @@ -2703,61 +2742,17 @@ def api_chat_events(agent_id): if up_to_seq is not None and up_to_seq - after_seq > 200: return jsonify({'error': 'range too large (max 200)'}), 400 - from backend.event_stream import event_stream - - _TRANSFORM_MAP = { - 'turn_begin': ('turn_begin', lambda d: {'ts': d.get('ts', 0)}), - 'llm_thinking': ('thinking', lambda d: {'content': d.get('thinking', '')}), - 'tool_call_started': ('tool_call_started', lambda d: {'tool': d.get('tool_name', ''), 'args': d.get('tool_args', {}), 'param_types': d.get('param_types', {})}), - 'tool_executed': ('tool_executed', lambda d: {'tool': d.get('tool_name', ''), 'args': d.get('tool_args', {}), 'result': d.get('tool_result', {}), 'error': d.get('has_error', False)}), - 'state:changed': ('state:changed', lambda d: {key: d[key] for key in ('mode', 'plan_file', 'tasks', 'loaded_skills') if key in d}), - 'tasks:auto_transition': ('tasks:auto_transition', lambda d: {key: d[key] for key in ('task_ids', 'tasks') if key in d}), - 'tasks:stale': ('tasks:stale', lambda d: {key: d[key] for key in ('task_ids', 'tasks') if key in d}), - 'llm_response_chunk':('response_chunk', lambda d: {'content': d.get('content', ''), 'is_final': d.get('is_final', False), 'send_as_message': d.get('send_as_message', False)}), - 'turn_complete': ('done', lambda d: { - 'thinking_duration': d.get('thinking_duration'), - 'response': d.get('response', ''), - 'slash_command': d.get('slash_command', False), - }), - 'approval_required': ('approval_required', lambda d: {'approval_id': d.get('approval_id', ''), 'agent_id': d.get('agent_id', ''), 'source_agent_id': d.get('source_agent_id', ''), 'source_agent_name': d.get('source_agent_name', ''), 'tool': d.get('tool_name', ''), 'args': d.get('tool_args', {}), 'approval_info': d.get('approval_info', {}), 'reasons': d.get('reasons', []), 'score': d.get('score')}), - 'approval_resolved': ('approval_resolved', lambda d: {'approval_id': d.get('approval_id', ''), 'decision': d.get('decision', ''), 'timed_out': d.get('timed_out', False)}), - 'llm_retry': ('retry', lambda d: {'retry_count': d.get('retry_count', 0), 'max_retries': d.get('max_retries', 0), 'error_type': d.get('error_type', ''), 'message': d.get('user_message', '')}), - 'message_injected': ('message_injected', lambda d: {'message': d.get('message', '')}), - 'message_injection_applied': ('message_injection_applied', lambda d: {'content': d.get('content', ''), 'count': d.get('count', 1)}), - 'message_received': ('message_received', lambda d: {'message': d.get('message', ''), 'metadata': d.get('metadata', {})}), - 'session_clear': ('session_clear', lambda d: {'session_id': d.get('session_id', ''), 'agent_id': d.get('agent_id', '')}), - 'turn_split': ('turn_split', lambda d: {}), - 'evonic:agent-state-changed': ('state_changed', lambda d: {'agent_id': d.get('agent_id', ''), 'session_id': d.get('session_id', '')}), - } - - if up_to_seq is None: - raw = event_stream.get_session_events(session_id, after_seq) - else: - raw = event_stream.get_events_in_range(session_id, after_seq, up_to_seq) - - # Strip boundary events (turn_complete, session_clear) on fresh requests so - # restoreActiveReasoning() never replays completed turns or past session_clear - # events that would create a stale thinking bubble. Mirror the SSE stream logic - # at lines 1668-1674. Only strip when after_seq==0; on gap-fill reconnects - # (after_seq>0), the client hasn't seen these events and needs them. - if after_seq == 0: - last_boundary = -1 - for i, e in enumerate(raw): - if e['event'] in ('turn_complete', 'session_clear'): - last_boundary = i - if last_boundary >= 0: - raw = raw[last_boundary + 1:] - - events = [] - for entry in raw: - event_name = entry['event'] - if event_name in _TRANSFORM_MAP: - sse_name, transform = _TRANSFORM_MAP[event_name] - payload = transform(entry['data']) - payload['seq'] = entry['chat_seq'] - events.append({'event': sse_name, 'seq': entry['chat_seq'], 'data': payload}) - - return jsonify({'events': events}) + from backend.realtime_store import realtime_store + active_only = after_seq == 0 and up_to_seq is None + rows = realtime_store.events_after( + after_seq, {'chat'}, session_id=session_id, agent_id=agent_id, + up_to_id=up_to_seq, active_only=active_only, + limit=200 if up_to_seq is not None else 5000, + ) + return jsonify({'events': [ + {'event': row['event'], 'seq': row['id'], 'data': row['data']} + for row in rows + ]}) @agents_bp.route('/api/agents//chat/approve', methods=['POST']) @@ -2831,6 +2826,7 @@ def api_agents_status_stream(): data: {"agent_id": "...", "agent_name": "...", "response": "...", "external_user_id": "...", "session_id": "..."} """ + return redirect('/api/realtime/stream?channels=status', code=307) import logging as _log_depr _log_depr.getLogger(__name__).warning( "DEPRECATED endpoint /api/agents/status/stream used — " diff --git a/routes/realtime.py b/routes/realtime.py index c027777f..659bb57a 100644 --- a/routes/realtime.py +++ b/routes/realtime.py @@ -1,1201 +1,308 @@ -""" -Unified Real-Time SSE Endpoint — consolidates 5 separate SSE connections -into 1 multiplexed connection with per-channel priority queuing. +"""Durable, multiplexed Server-Sent Events gateway.""" -Endpoint: GET /api/realtime/stream +from __future__ import annotations -Query parameters (opt-in channels): - channels — comma-separated: status,approvals,update - chat — 1 to include per-session chat events - session_id — chat session ID (required when chat=1) - agent_id — agent ID (required when chat=1) - after — chat event resume seq - workplace — workplace ID for connector events - chat_throttle — throttle interval ms for chat events (default 100) -""" - -import collections import json import logging -import math -import os -import queue -import random -import signal -import socket import threading import time -from datetime import datetime, timedelta +import uuid +from datetime import datetime from flask import Blueprint, Response, request, stream_with_context -log = logging.getLogger(__name__) - -realtime_bp = Blueprint('realtime', __name__) - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -RING_SIZES = { - 'chat': 256, - 'approvals': 8, - 'status': 32, - 'update': 16, - 'workplace': 16, -} - -RING_STRATEGIES = { - 'chat': 'drop_oldest', - 'approvals': 'drop_oldest', # on overflow keep the newest approval, never drop it - 'status': 'drop_oldest', - 'update': 'drop_oldest', - 'workplace': 'drop_oldest', -} - -CHANNEL_PRIORITY = { - 'update': 0, # highest — small, rare, must be fast - 'status': 0, - 'approvals': 1, # user-facing modal - 'chat': 2, # high throughput, tolerable delay - 'workplace': 2, # high throughput, tolerable delay -} - -# Weighted round-robin: 1 L0/L1 event per 5 L2 events -L2_WEIGHT = 5 - -HEARTBEAT_INTERVAL = 15 # seconds -HEARTBEAT_MAX_FAILURES = 3 -TCP_KEEPIDLE = 60 -TCP_KEEPINTVL = 10 -TCP_KEEPCNT = 3 - -CIRCUIT_BREAKER_WINDOW = 10 # seconds -CIRCUIT_BREAKER_THRESHOLD = 3 -CIRCUIT_BREAKER_COOLDOWN = 60 - -# Per-channel buffer for pause/resume -PAUSE_BUFFER = { - 'chat': 64, - 'workplace': 16, -} - -# --------------------------------------------------------------------------- -# Bounded Ring Buffer -# --------------------------------------------------------------------------- - -class BoundedRing: - """Thread-safe bounded queue with configurable overflow strategy.""" - - def __init__(self, channel: str, maxlen: int, strategy: str): - self.channel = channel - self.maxlen = maxlen - self.strategy = strategy - self._lock = threading.Lock() - self._q = collections.deque(maxlen=maxlen) - self._dropped_count = 0 - self._seq = 0 - - def put(self, item): - """Put an item. Returns (inserted, dropped_count_for_this_put).""" - dropped = 0 - with self._lock: - if len(self._q) >= self.maxlen: - if self.strategy == 'drop_oldest': - self._q.popleft() - dropped = 1 - elif self.strategy == 'drop_newest': - dropped = 1 - # don't actually enqueue — drop the new item - self._dropped_count += 1 - return (False, 1) - self._seq += 1 - self._q.append((self._seq, item)) - if dropped: - self._dropped_count += 1 - return (True, dropped) - - def get(self): - """Get the oldest item, or None if empty.""" - with self._lock: - if self._q: - return self._q.popleft() - return None - - def get_many(self, max_count: int): - """Get up to max_count items.""" - items = [] - with self._lock: - while self._q and len(items) < max_count: - items.append(self._q.popleft()) - return items - - def get_all(self): - """Get all items.""" - with self._lock: - items = list(self._q) - self._q.clear() - return items - - def drain_dropped(self) -> int: - """Atomically read and reset dropped count.""" - with self._lock: - c = self._dropped_count - self._dropped_count = 0 - return c - - def peek_all(self): - """Return all items without dequeuing (for snapshot).""" - with self._lock: - return list(self._q) - - def size(self): - with self._lock: - return len(self._q) - - -# --------------------------------------------------------------------------- -# Circuit Breaker -# --------------------------------------------------------------------------- +from backend.realtime_store import realtime_store -class CircuitBreaker: - """Per-channel circuit breaker with sliding window crash tracking.""" - def __init__(self, channel: str): - self.channel = channel - self._lock = threading.Lock() - self._crashes = [] # list of crash timestamps - self._open_since = None - self._disabled = False - - def record_crash(self) -> bool: - """Record a crash. Returns True if circuit should open (stop restarting).""" - now = time.time() - with self._lock: - # Clean old entries outside window - cutoff = now - CIRCUIT_BREAKER_WINDOW - self._crashes = [t for t in self._crashes if t > cutoff] - self._crashes.append(now) - if len(self._crashes) >= CIRCUIT_BREAKER_THRESHOLD: - self._open_since = now - self._disabled = True - return True - return False - - def is_disabled(self) -> bool: - with self._lock: - if not self._disabled: - return False - # Check cooldown - if self._open_since and (time.time() - self._open_since) > CIRCUIT_BREAKER_COOLDOWN: - self._disabled = False - self._crashes = [] - self._open_since = None - return False - return True - - def reset(self): - with self._lock: - self._crashes = [] - self._open_since = None - self._disabled = False - - -# --------------------------------------------------------------------------- -# Connection State -# --------------------------------------------------------------------------- +log = logging.getLogger(__name__) +realtime_bp = Blueprint('realtime', __name__) -# Global registry of active connections (for pause/resume by session) -_connections: dict = {} # key: connection_id -> RealtimeConnection +HEARTBEAT_INTERVAL = 15 +WEB_SSE_HEARTBEAT_MAX_AGE = 45 +_ALLOWED_CHANNELS = {'chat', 'status', 'approvals', 'update', 'workplace'} +_connections: dict[str, dict] = {} _conn_lock = threading.Lock() -# Heartbeat-aware web SSE delivery check: maximum age (seconds) of the -# last heartbeat before a connection is considered disconnected. -WEB_SSE_HEARTBEAT_MAX_AGE = 45 # 3 × HEARTBEAT_INTERVAL - def has_active_web_sse(session_id: str) -> bool: - """Return True if any SSE connection for *session_id* has had a - heartbeat within WEB_SSE_HEARTBEAT_MAX_AGE seconds. - - This is more reliable than has_web_listener() which only checks - listener registration, not actual delivery. - """ now = time.monotonic() with _conn_lock: - for conn in _connections.values(): - if conn.chat_session_id == session_id: - if (conn.last_heartbeat_time > 0 and - (now - conn.last_heartbeat_time) < WEB_SSE_HEARTBEAT_MAX_AGE): - return True - return False - - -class RealtimeConnection: - """Per-connection state for the unified SSE stream.""" - - def __init__(self, conn_id: str, channels: set, chat_session_id: str = None, - agent_id: str = None, after_seq: int = 0, - workplace_id: str = None, chat_throttle_ms: int = 100, - expires_at: float = None): - self.conn_id = conn_id - self.channels = channels - self.chat_session_id = chat_session_id - self.agent_id = agent_id - self.after_seq = after_seq - self.last_global_seq = after_seq - self.last_chat_seq = after_seq - self.workplace_id = workplace_id - self.chat_throttle_ms = chat_throttle_ms - self.expires_at = expires_at - self.paused = False - self._stop_event = threading.Event() - # Per-channel pause buffers - self._pause_buffers: dict[str, BoundedRing] = {} - self.last_write_ok = True - # Reference to per-channel rings (set by api_realtime_stream) - # so api_realtime_resume can flush pause buffers back into them. - self.rings: dict[str, BoundedRing] | None = None - # Last heartbeat timestamp (monotonic). Updated by the generator. - # Used by has_active_web_listener() to verify delivery. - self.last_heartbeat_time: float = 0.0 - - def stop(self): - self._stop_event.set() - - def is_stopped(self) -> bool: - return self._stop_event.is_set() - - def pause(self): - self.paused = True - - def resume(self): - self.paused = False - - def check_expired(self) -> bool: - if self.expires_at and time.time() > self.expires_at: - return True - return False - + return any( + item.get('session_id') == session_id + and now - item.get('last_heartbeat', 0) < WEB_SSE_HEARTBEAT_MAX_AGE + for item in _connections.values() + ) -# --------------------------------------------------------------------------- -# SSE formatting helpers -# --------------------------------------------------------------------------- -def _json_default(obj): - """Custom JSON default handler: convert non-serializable types to strings.""" - if isinstance(obj, datetime): - return obj.isoformat() - log.warning("_format_sse_event: non-serializable type %s in SSE data", type(obj).__name__) - return str(obj) +def _json_default(value): + if isinstance(value, datetime): + return value.isoformat() + return str(value) -def _format_sse_event(event_name: str, data: dict, seq_id: str = None, - global_seq: int = None) -> str: - """Format a single SSE event with optional id and event fields.""" +def _format_sse_event(event_name: str, data: dict, + event_id: int | str | None = None) -> str: lines = [] - if seq_id: - lines.append(f"id: {seq_id}") - elif global_seq is not None: - lines.append(f"id: {global_seq}") + if event_id is not None: + lines.append(f'id: {event_id}') if event_name: - lines.append(f"event: {event_name}") - lines.append(f"data: {json.dumps(data, separators=(',', ':'), default=_json_default)}") - return "\n".join(lines) + "\n\n" - - -def _format_sse_comment(comment: str) -> str: - """Format an SSE comment line.""" - return f": {comment}\n\n" + lines.append(f'event: {event_name}') + lines.append('data: ' + json.dumps( + data, separators=(',', ':'), default=_json_default, + )) + return '\n'.join(lines) + '\n\n' -# --------------------------------------------------------------------------- -# State snapshot (atomic subscribe) -# --------------------------------------------------------------------------- - -# Agent busy-status snapshot, cached briefly so the multiple SSE connections -# a page opens at load share one db.get_agents() read. Staleness is bounded: -# changes after the snapshot arrive via agent_busy_changed events anyway. -_status_snapshot_cache = {'ts': 0.0, 'events': None} -_status_snapshot_lock = threading.Lock() -_STATUS_SNAPSHOT_TTL = 2.0 +def _parse_cursor(value) -> int: + text = str(value or '').strip() + if ':' in text: # Accept IDs produced by the former channel:seq gateway. + text = text.rsplit(':', 1)[-1] + try: + return max(0, int(text)) + except (TypeError, ValueError): + return 0 -def _get_status_snapshot_events() -> list: - from models.db import db - now = time.monotonic() - with _status_snapshot_lock: - cached = _status_snapshot_cache['events'] - if cached is not None and now - _status_snapshot_cache['ts'] < _STATUS_SNAPSHOT_TTL: - return cached - events = [] - for agent in db.get_agents(): - events.append(('agent_busy_changed', { - 'agent_id': agent['id'], - 'busy': agent.get('busy', False), - 'session_id': agent.get('current_session_id', ''), - })) - _status_snapshot_cache['ts'] = now - _status_snapshot_cache['events'] = events - return events +def _snapshot_payload(channel: str, payload: dict) -> dict: + result = dict(payload) + result.setdefault('timestamp', int(time.time() * 1000)) + result.setdefault('channel', channel) + result['snapshot'] = True + return result -def _build_snapshot(channels: set, agent_id: str = None, - session_id: str = None, - workplace_id: str = None) -> list: - """Capture current state snapshot for requested channels.""" - events = [] +def _build_snapshot(channels: set[str], session_id: str | None = None, + agent_id: str | None = None, + workplace_id: str | None = None) -> list[tuple[str, str, dict]]: + events: list[tuple[str, str, dict]] = [] - if 'status' in channels: + if 'status' in channels or session_id: try: - events.extend(_get_status_snapshot_events()) - except Exception as e: - log.warning("realtime snapshot: failed to get agent statuses: %s", e) + from models.db import db + busy = realtime_store.busy_agents() + for agent in db.get_agents(): + if session_id and 'status' not in channels and agent['id'] != agent_id: + continue + entry = busy.get(agent['id']) + events.append(('status', 'agent_busy_changed', { + 'agent_id': agent['id'], + 'busy': bool(entry), + 'session_id': entry.get('session_id', '') if entry else '', + 'session_ids': entry.get('session_ids', []) if entry else [], + 'active_count': entry.get('active_count', 0) if entry else 0, + 'state': entry.get('state', 'idle') if entry else 'idle', + })) + except Exception as exc: + log.warning('realtime status snapshot failed: %s', exc) if 'approvals' in channels or session_id: - from models.db import db try: - pending = db.get_pending_tool_approvals() - for app in (pending or []): - events.append(('approval_required', { - 'approval_id': app.get('id', ''), - 'agent_id': app.get('agent_id', ''), - 'source_agent_id': app.get('source_agent_id', ''), - 'source_agent_name': app.get('source_agent_name', ''), - 'tool': app.get('tool_name', ''), - 'args': app.get('tool_args', {}), - 'approval_info': app.get('approval_info', {}), - 'reasons': app.get('reasons', []), - 'score': app.get('score'), + from models.db import db + for approval in db.get_pending_tool_approvals() or []: + approval_session = approval.get('session_id') + if session_id and 'approvals' not in channels \ + and approval_session != session_id: + continue + events.append(('approvals', 'approval_required', { + 'approval_id': approval.get('id', ''), + 'agent_id': approval.get('agent_id', ''), + 'source_agent_id': approval.get('source_agent_id', ''), + 'source_agent_name': approval.get('source_agent_name', ''), + 'tool': approval.get('tool_name', ''), + 'args': approval.get('tool_args', {}), + 'approval_info': approval.get('approval_info', {}), + 'reasons': approval.get('reasons', []), + 'score': approval.get('score'), })) - except Exception as e: - log.warning("realtime snapshot: failed to get pending approvals: %s", e) + except Exception as exc: + log.warning('realtime approval snapshot failed: %s', exc) if 'update' in channels: try: - from routes.update import update_manager - status = update_manager.get_status() - events.append(('update_status', status)) - except Exception as e: - log.warning("realtime snapshot: failed to get update status: %s", e) - - return events - - -# --------------------------------------------------------------------------- -# Producer factories (per-channel) — Task isolation -# --------------------------------------------------------------------------- - -def _producer_status(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event): - """Producer: listen to agent_busy_changed, turn_complete, whatsapp_bridge_status and panel_updated events.""" - from backend.event_stream import event_stream - - def busy_handler(data): - ring.put(('agent_busy_changed', { - 'agent_id': data.get('agent_id', ''), - 'busy': data.get('busy', False), - 'session_id': data.get('session_id', ''), - })) - - def turn_handler(data): - response = data.get('response', '') - if not response or data.get('is_error'): - return - ring.put(('agent_turn_complete', { - 'agent_id': data.get('agent_id', ''), - 'agent_name': data.get('agent_name', ''), - 'response': response, - 'session_id': data.get('session_id', ''), - 'external_user_id': data.get('external_user_id', ''), - })) - - def wa_bridge_handler(data): - ring.put(('whatsapp_bridge_status', { - 'agent_id': data.get('agent_id', ''), - 'channel_id': data.get('channel_id', ''), - 'status': data.get('status', ''), - })) - - def panel_handler(data): - ring.put(('panel_updated', { - 'agent_id': data.get('agent_id', ''), - })) - - event_stream.on('agent_busy_changed', busy_handler) - event_stream.on('turn_complete', turn_handler) - event_stream.on('whatsapp_bridge_status', wa_bridge_handler) - event_stream.on('panel_updated', panel_handler) - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - event_stream.off('agent_busy_changed', busy_handler) - event_stream.off('turn_complete', turn_handler) - event_stream.off('whatsapp_bridge_status', wa_bridge_handler) - event_stream.off('panel_updated', panel_handler) - - -def _producer_approval(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event): - """Producer: listen to approval_required and approval_resolved events.""" - from backend.event_stream import event_stream - - def approval_handler(data): - ring.put(('approval_required', { - 'approval_id': data.get('approval_id', ''), - 'agent_id': data.get('agent_id', ''), - 'source_agent_id': data.get('source_agent_id', ''), - 'source_agent_name': data.get('source_agent_name', ''), - 'tool': data.get('tool_name', ''), - 'args': data.get('tool_args', {}), - 'approval_info': data.get('approval_info', {}), - 'reasons': data.get('reasons', []), - 'score': data.get('score'), - })) - - def resolved_handler(data): - ring.put(('approval_resolved', { - 'approval_id': data.get('approval_id', ''), - 'decision': data.get('decision', ''), - 'timed_out': data.get('timed_out', False), - })) - - event_stream.on('approval_required', approval_handler) - event_stream.on('approval_resolved', resolved_handler) - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - event_stream.off('approval_required', approval_handler) - event_stream.off('approval_resolved', resolved_handler) - - -def _producer_chat(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event, session_id: str, - after_seq: int = 0): - """Producer: listen to per-session chat events.""" - from backend.event_stream import event_stream - - _TRANSFORMS = { - 'turn_begin': ('turn_begin', lambda d: {'ts': d.get('ts', 0)}), - 'llm_thinking': ('thinking', lambda d: {'content': d.get('thinking', '')}), - 'tool_call_started': ('tool_call_started', lambda d: { - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'param_types': d.get('param_types', {}), - }), - 'tool_executed': ('tool_executed', lambda d: { - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'result': d.get('tool_result', {}), - 'error': d.get('has_error', False), - }), - 'state:changed': ('state:changed', lambda d: { - key: d[key] for key in ('mode', 'plan_file', 'tasks', 'loaded_skills') - if key in d - }), - 'tasks:auto_transition': ('tasks:auto_transition', lambda d: { - key: d[key] for key in ('task_ids', 'tasks') if key in d - }), - 'tasks:stale': ('tasks:stale', lambda d: { - key: d[key] for key in ('task_ids', 'tasks') if key in d - }), - 'llm_response_chunk': ('response_chunk', lambda d: { - 'content': d.get('content', ''), - 'is_final': d.get('is_final', False), - 'send_as_message': d.get('send_as_message', False), - }), - 'turn_complete': ('done', lambda d: { - 'thinking_duration': d.get('thinking_duration'), - 'response': d.get('response', ''), - 'slash_command': d.get('slash_command', False), - 'attachment_info': d.get('attachment_info'), - }), - 'approval_required': ('approval_required', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'agent_id': d.get('agent_id', ''), - 'source_agent_id': d.get('source_agent_id', ''), - 'source_agent_name': d.get('source_agent_name', ''), - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'approval_info': d.get('approval_info', {}), - 'reasons': d.get('reasons', []), - 'score': d.get('score'), - }), - 'approval_resolved': ('approval_resolved', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'decision': d.get('decision', ''), - 'timed_out': d.get('timed_out', False), - }), - 'llm_retry': ('retry', lambda d: { - 'retry_count': d.get('retry_count', 0), - 'max_retries': d.get('max_retries', 0), - 'error_type': d.get('error_type', ''), - 'message': d.get('user_message', ''), - }), - 'message_injected': ('message_injected', lambda d: { - 'message': d.get('message', ''), - }), - 'message_injection_applied': ('message_injection_applied', lambda d: { - 'content': d.get('content', ''), - 'count': d.get('count', 1), - }), - 'message_received': ('message_received', lambda d: { - 'message': d.get('message', ''), - 'metadata': d.get('metadata', {}), - }), - 'whatsapp_restriction_warning': ('whatsapp_restriction_warning', lambda d: { - 'content': d.get('content', ''), - 'metadata': d.get('metadata', {}), - }), - 'session_clear': ('session_clear', lambda d: { - 'session_id': d.get('session_id', ''), - 'agent_id': d.get('agent_id', ''), - }), - 'turn_split': ('turn_split', lambda d: {}), - 'evonic:agent-state-changed': ('state_changed', lambda d: { - 'agent_id': d.get('agent_id', ''), - 'session_id': d.get('session_id', ''), - }), - } - - def make_handler(evt_name, sse_name, transform): - def handler(data): - if data.get('session_id') != session_id: - return - try: - payload = transform(data) if transform else data - if payload is not None: - # Use the contiguous per-session chat seq (not the global _seq) - # so the browser's gap detector sees a gap-free sequence and - # doesn't fire a phantom gap-fill on every event. Matches the - # legacy /chat/stream + /chat/events gap-fill endpoint. - payload['seq'] = data.get('_chat_seq') - ring.put((sse_name, payload)) - except Exception: - pass - return handler - - handlers = {} - for evt_name, (sse_name, transform) in _TRANSFORMS.items(): - h = make_handler(evt_name, sse_name, transform) - handlers[evt_name] = h - event_stream.on(evt_name, h) - - event_stream.register_web_listener(session_id) - - # Replay the in-progress session buffer so a client connecting at/after the - # POST that starts a turn still sees the turn's opening events (turn_begin, - # early thinking, first tool call). The legacy /chat/stream did this; without - # it the unified path loses those events and the UI shows only a spinner - # until a manual refresh. Subscribe-then-replay ordering (after event_stream.on - # above) means live events arriving during replay are also queued; the client - # dedups the overlap by the contiguous _chat_seq. - try: - buffered = event_stream.get_session_events(session_id, after_seq) - # On a fresh connect, drop everything up to and including the last - # completed turn / session_clear so we never replay a finished turn. - if after_seq == 0: - last_boundary = -1 - for i, e in enumerate(buffered): - if e['event'] in ('turn_complete', 'session_clear'): - last_boundary = i - if last_boundary >= 0: - buffered = buffered[last_boundary + 1:] - for entry in buffered: - st = _TRANSFORMS.get(entry['event']) - if not st: - continue - sse_name, transform = st - payload = transform(entry['data']) - payload['seq'] = entry.get('chat_seq') - ring.put((sse_name, payload)) - except Exception: - pass - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - event_stream.unregister_web_listener(session_id) - for evt_name, h in handlers.items(): - event_stream.off(evt_name, h) - - -def _producer_update(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event): - """Producer: listen to update manager status changes.""" - from routes.update import update_manager - - listener_q = update_manager.register_listener() - - try: - while not stop_event.is_set(): - try: - snapshot = listener_q.get(timeout=1) - ring.put(('update_status', snapshot)) - if snapshot.get('status') in ('success', 'failed'): - ring.put(('update_done', {'status': snapshot['status']})) - except queue.Empty: - pass - finally: - update_manager.unregister_listener(listener_q) - - -def _producer_workplace(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event, workplace_id: str): - """Producer: listen to workplace connector events for a specific workplace.""" - from backend.event_stream import event_stream - - _WATCHED = ('connector_connected', 'connector_disconnected', - 'connector_paired', 'workplace_status_changed') - - def handler(data): - if data.get('workplace_id') == workplace_id: - ring.put((data['_event'], dict(data))) - - for ev in _WATCHED: - event_stream.on(ev, handler) - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - for ev in _WATCHED: - event_stream.off(ev, handler) - + from backend import update_manager + events.append(('update', 'update_status', update_manager.get_status())) + except Exception as exc: + log.warning('realtime update snapshot failed: %s', exc) -# --------------------------------------------------------------------------- -# Priority-aware event scheduler -# --------------------------------------------------------------------------- - -def _priority_round_robin(rings: dict, conn: RealtimeConnection) -> list: - """Extract events from per-channel rings using weighted round-robin. - - Returns list of (channel, seq, sse_name, payload) tuples. - """ - result = [] - l2_count = 0 - - # First pass: L0 channels (update, status) — 1 event each - for ch in ('update', 'status'): - if ch not in rings: - continue - item = rings[ch].get() - if item: - seq, (sse_name, payload) = item - result.append((ch, seq, sse_name, payload)) - - # L1 channels (approvals) — 1 event - if 'approvals' in rings: - item = rings['approvals'].get() - if item: - seq, (sse_name, payload) = item - result.append(('approvals', seq, sse_name, payload)) - - # L2 channels (chat, workplace) — up to L2_WEIGHT events each - for ch in ('chat', 'workplace'): - if ch not in rings: - continue - for _ in range(L2_WEIGHT): - item = rings[ch].get() - if item: - seq, (sse_name, payload) = item - result.append((ch, seq, sse_name, payload)) - l2_count += 1 - else: - break - - return result - - -# --------------------------------------------------------------------------- -# Differential push for chat -# --------------------------------------------------------------------------- - -class ChatThrottle: - """Batches thinking chunks: push first chunk immediately, then batch - every throttle_ms, then push final event.""" - - def __init__(self, throttle_ms: int = 100): - self.throttle_ms = throttle_ms - self._batch = [] - self._first_sent = False - self._last_flush = 0 - # Highest chat seq among batched chunks — stamped on the merged event so - # the client's _lastSeq advances over the chunks folded into the batch, - # avoiding a spurious gap-fill (and duplicate CoT) per batch boundary. - self._batch_seq = None - - def _merged_thinking(self): - """Build the merged 'thinking' event from the current batch and reset it.""" - batched_content = ''.join(self._batch) - self._batch = [] - seq = self._batch_seq - self._batch_seq = None - if not batched_content: - return None - ev = {'content': batched_content} - if seq is not None: - ev['seq'] = seq - return ('thinking', ev) - - def feed(self, sse_name: str, payload: dict): - """Feed a chat event. Returns list of events to emit now (may be empty).""" - now_ms = time.monotonic() * 1000 - - # 'thinking' chunks get batched - if sse_name == 'thinking': - if not self._first_sent: - self._first_sent = True - self._last_flush = now_ms - return [('thinking', payload)] - - self._batch.append(payload.get('content', '')) - if payload.get('seq') is not None: - self._batch_seq = payload.get('seq') - - if (now_ms - self._last_flush) >= self.throttle_ms: - self._last_flush = now_ms - merged = self._merged_thinking() - return [merged] if merged else [] - return [] - - # Non-thinking event: flush any pending batch first - result = [] - merged = self._merged_thinking() - if merged: - result.append(merged) - - result.append((sse_name, payload)) - self._first_sent = False - return result - - def flush(self): - """Flush any remaining batched content. Returns list of events.""" - merged = self._merged_thinking() - return [merged] if merged else [] + if 'workplace' in channels and workplace_id: + try: + from backend.workplaces.manager import workplace_manager + events.append(( + 'workplace', 'workplace_status_changed', + workplace_manager.get_status(workplace_id), + )) + except Exception as exc: + log.warning('realtime workplace snapshot failed: %s', exc) + return events -# --------------------------------------------------------------------------- -# Main SSE endpoint -# --------------------------------------------------------------------------- @realtime_bp.route('/api/realtime/stream', methods=['GET']) def api_realtime_stream(): - """Unified multiplexed SSE endpoint. + channels = { + value.strip() for value in request.args.get('channels', '').split(',') + if value.strip() + } + unknown = channels - _ALLOWED_CHANNELS + if unknown: + return Response( + json.dumps({'error': 'unknown channels', 'channels': sorted(unknown)}), + status=400, mimetype='application/json', + ) - Consolidates 5 separate EventSource connections into 1: - - /api/agents/status/stream -> channels=status - - /api/approvals/stream -> channels=approvals - - /api/system/update/stream -> channels=update - - /api/agents//chat/stream -> chat=1&session_id=...&agent_id=... - - /api/workplaces//events -> workplace= - """ - # Parse query parameters - channels_str = request.args.get('channels', '') - channels = set(filter(None, [ch.strip() for ch in channels_str.split(',')])) chat_enabled = request.args.get('chat') == '1' session_id = request.args.get('session_id', '').strip() or None agent_id = request.args.get('agent_id', '').strip() or None - after_seq = request.args.get('after', 0, type=int) workplace_id = request.args.get('workplace', '').strip() or None - chat_throttle_ms = request.args.get('chat_throttle', 100, type=int) - - # Validate chat parameters - if chat_enabled and (not session_id or not agent_id): - return Response( - json.dumps({'error': 'session_id and agent_id required when chat=1'}), - status=400, - mimetype='application/json' - ) - - # Release thread-local DB connection (SSE thread is long-lived) - from models.db import db - db.close() - - # Build channel set - all_channels = set(channels) if chat_enabled: - all_channels.add('chat') + if not session_id or not agent_id: + return Response( + json.dumps({'error': 'session_id and agent_id required when chat=1'}), + status=400, mimetype='application/json', + ) + channels.add('chat') if workplace_id: - all_channels.add('workplace') - - if not all_channels: + channels.add('workplace') + if not channels: return Response( json.dumps({'error': 'At least one channel must be requested'}), - status=400, - mimetype='application/json' + status=400, mimetype='application/json', ) - # SSE connection limiting — max 5 concurrent per user/IP (FINDING-004) - from flask import session as _flask_session + query_cursor = _parse_cursor(request.args.get('after')) + header_cursor = _parse_cursor(request.headers.get('Last-Event-ID')) + cursor = max(query_cursor, header_cursor) + fresh_connection = header_cursor == 0 + + from flask import session as flask_session from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_id = ( - f"user:{_flask_session.get('_user_id', 'admin')}" - if _flask_session.get('authenticated') + sse_identity = ( + f"user:{flask_session.get('_user_id', 'admin')}" + if flask_session.get('authenticated') else f"ip:{request.remote_addr or '0.0.0.0'}" ) - _sse_allowed, _sse_count = sse_register(_sse_id) - if not _sse_allowed: + allowed, _count = sse_register(sse_identity) + if not allowed: return Response( json.dumps({ 'error': 'too_many_sse_connections', 'message': f'Maximum {SSE_MAX_CONCURRENT} concurrent SSE connections allowed.', 'retry_after': 30, }), - status=429, - headers={'Retry-After': '30'}, - mimetype='application/json' - ) - - # Thundering herd mitigation — check approximate connection count - with _conn_lock: - conn_count = len(_connections) - max_conn = int(os.environ.get('WORKER_CONNECTIONS', 512)) - if max_conn > 0 and conn_count >= max_conn * 0.8: - sse_unregister(_sse_id) - return Response( - json.dumps({'error': 'Server busy, please retry later'}), - status=503, - headers={'Retry-After': '10'}, - mimetype='application/json' + status=429, headers={'Retry-After': '30'}, + mimetype='application/json', ) - # Generate a connection ID - conn_id = f"{id(request)}:{time.monotonic()}" - - # Token expiry: 24h from now - expires_at = time.time() + 86400 - - conn = RealtimeConnection( - conn_id=conn_id, - channels=all_channels, - chat_session_id=session_id, - agent_id=agent_id, - after_seq=after_seq, - workplace_id=workplace_id, - chat_throttle_ms=chat_throttle_ms, - expires_at=expires_at, - ) - - # Register connection + from models.db import db + db.close() + connection_id = uuid.uuid4().hex + connected_at = time.time() with _conn_lock: - _connections[conn_id] = conn - - # Build per-channel bounded rings - rings: dict[str, BoundedRing] = {} - for ch in all_channels: - strategy = RING_STRATEGIES.get(ch, 'drop_oldest') - size = RING_SIZES.get(ch, 32) - rings[ch] = BoundedRing(ch, size, strategy) - - # Build circuit breakers - breakers: dict[str, CircuitBreaker] = {} - for ch in all_channels: - breakers[ch] = CircuitBreaker(ch) - - # Store rings ref on the connection so api_realtime_resume can flush - # pause buffers back into the rings after resume. - conn.rings = rings - - # Start producer threads (task isolation) - producers = {} - stop_event = conn._stop_event # shared stop signal - - _PRODUCERS = { - 'status': (_producer_status, {}), - 'approvals': (_producer_approval, {}), - 'update': (_producer_update, {}), - } - - for ch in all_channels: - if ch in _PRODUCERS: - fn, kwargs = _PRODUCERS[ch] - t = threading.Thread( - target=_start_producer, - args=(ch, fn, rings[ch], breakers[ch], stop_event, kwargs), - daemon=True, - name=f"realtime-producer-{ch}-{conn_id[:12]}" - ) - producers[ch] = t - t.start() - elif ch == 'chat' and session_id: - t = threading.Thread( - target=_start_producer, - args=(ch, _producer_chat, rings[ch], breakers[ch], - stop_event, {'session_id': session_id, 'after_seq': after_seq}), - daemon=True, - name=f"realtime-producer-{ch}-{conn_id[:12]}" - ) - producers[ch] = t - t.start() - elif ch == 'workplace' and workplace_id: - t = threading.Thread( - target=_start_producer, - args=(ch, _producer_workplace, rings[ch], breakers[ch], - stop_event, {'workplace_id': workplace_id}), - daemon=True, - name=f"realtime-producer-{ch}-{conn_id[:12]}" - ) - producers[ch] = t - t.start() - - # Chat throttler for differential push - chat_throttle = ChatThrottle(chat_throttle_ms) if 'chat' in all_channels else None - - # Global sequence counter - global_seq = after_seq - chat_seq = after_seq - - # Build state snapshot - snapshot_events = _build_snapshot(all_channels, agent_id, session_id, workplace_id) - - # Setup TCP_NODELAY for time-sensitive channels (approval, status, update) - # This is done during the first write in the generator + _connections[connection_id] = { + 'session_id': session_id, + 'last_heartbeat': time.monotonic(), + } - # SSE generator with priority scheduler @stream_with_context def generate(): - nonlocal global_seq, chat_seq - - # Set TCP keepalive and SIGPIPE handling on the socket + nonlocal cursor try: - # Ignore SIGPIPE at process level if not already done - try: - signal.signal(signal.SIGPIPE, signal.SIG_IGN) - except (ValueError, OSError): - pass # can only be set in main thread - except Exception: - pass - - try: - # --- Phase 1: Emit retry with jitter --- - retry_ms = random.randint(3000, 8000) - yield f"retry: {retry_ms}\n" - - # --- Phase 2: Push state snapshot --- - for event_name, data in snapshot_events: - global_seq += 1 - yield _format_sse_event(event_name, data, - global_seq=global_seq) - - # --- Phase 3: Forward live events with priority scheduler --- - last_heartbeat = time.monotonic() - heartbeat_failures = 0 - - while not conn.is_stopped(): - # Check token expiry - if conn.check_expired(): - global_seq += 1 - yield _format_sse_event('auth_expired', - {'message': 'Token expired, please reconnect'}, - global_seq=global_seq) - conn.stop() + yield 'retry: 3000\n\n' + + # Snapshots repair current state on a fresh page load. Reconnects + # use only durable replay from Last-Event-ID. + if fresh_connection: + for channel, event_name, payload in _build_snapshot( + channels, session_id, agent_id, workplace_id): + yield _format_sse_event( + event_name, _snapshot_payload(channel, payload), + ) + + if fresh_connection and session_id: + high_water = cursor or realtime_store.high_water() + snapshot_cursor = 0 + while snapshot_cursor < high_water: + active_events = realtime_store.events_after( + snapshot_cursor, {'chat'}, session_id=session_id, + agent_id=agent_id, up_to_id=high_water, + active_only=True, limit=500, + ) + if not active_events: + break + for event in active_events: + snapshot_cursor = event['id'] + data = dict(event['data']) + data['snapshot'] = True + data['source_event_id'] = event['id'] + data['event_id'] = 0 + data['seq'] = 0 + yield _format_sse_event(event['event'], data) + if cursor == 0: + cursor = high_water + + # A brand-new global stream starts from current state. Historical + # rows are replayed only when the browser supplies a cursor. + if fresh_connection and cursor == 0: + cursor = realtime_store.high_water() + + yield _format_sse_event('ready', { + 'event_id': cursor, 'seq': cursor, + 'timestamp': int(time.time() * 1000), + 'channel': 'system', + }, cursor) + + while True: + if time.time() - connected_at >= 24 * 60 * 60: + yield _format_sse_event('auth_expired', { + 'message': 'Connection expired, please reconnect', + }, cursor) break - # Heartbeat - now = time.monotonic() - if now - last_heartbeat >= HEARTBEAT_INTERVAL: - try: - yield "event: heartbeat\ndata: {}\n\n" - heartbeat_failures = 0 - conn.last_heartbeat_time = now - except (BrokenPipeError, OSError): - heartbeat_failures += 1 - if heartbeat_failures >= HEARTBEAT_MAX_FAILURES: - conn.stop() - break - time.sleep(1) - continue - last_heartbeat = now - - # Priority-aware extraction from rings - events = _priority_round_robin(rings, conn) - - if not events: - # No events — short sleep to avoid busy-wait - time.sleep(0.05) + observed_high_water = realtime_store.high_water() + events = realtime_store.events_after( + cursor, channels, session_id=session_id, agent_id=agent_id, + workplace_id=workplace_id, up_to_id=observed_high_water, + ) + if events: + for event in events: + cursor = event['id'] + yield _format_sse_event( + event['event'], event['data'], event['id'], + ) continue - for channel, seq, sse_name, payload in events: - if conn.is_stopped(): - break - - global_seq += 1 - - # Build composite id for per-channel resume - if channel == 'chat': - chat_seq += 1 - seq_id = f"chat:{chat_seq}" - else: - seq_id = f"{channel}:{global_seq}" - - # Differential push for chat - if channel == 'chat' and chat_throttle: - throttled = chat_throttle.feed(sse_name, payload) - if not throttled: - continue - for t_name, t_payload in throttled: - global_seq += 1 - chat_seq += 1 - yield _format_sse_event(t_name, t_payload, - seq_id=f"chat:{chat_seq}", - global_seq=global_seq) - else: - # Check if paused — buffer chat/workplace events - if conn.paused and channel in ('chat', 'workplace'): - buf = conn._pause_buffers.get(channel) - if buf is None: - max_buf = PAUSE_BUFFER.get(channel, 32) - buf = BoundedRing(channel, max_buf, 'drop_oldest') - conn._pause_buffers[channel] = buf - buf.put((sse_name, payload)) - continue - - try: - yield _format_sse_event(sse_name, payload, - seq_id=seq_id, - global_seq=global_seq) - conn.last_write_ok = True - except (BrokenPipeError, OSError) as e: - log.warning("realtime %s: write failed: %s", conn_id, e) - conn.stop() - break - - # Check for dropped events per channel - for ch_name, ring in rings.items(): - dropped = ring.drain_dropped() - if dropped > 0: - try: - yield _format_sse_comment(f"x-sse-dropped {ch_name}:{dropped}") - except (BrokenPipeError, OSError): - conn.stop() - break - - # Flush chat throttler on disconnect - if chat_throttle: - for t_name, t_payload in chat_throttle.flush(): - try: - global_seq += 1 - chat_seq += 1 - yield _format_sse_event(t_name, t_payload, - seq_id=f"chat:{chat_seq}", - global_seq=global_seq) - except (BrokenPipeError, OSError): - break - + realtime_store.wait_for_events(observed_high_water, HEARTBEAT_INTERVAL) + if realtime_store.high_water() <= observed_high_water: + with _conn_lock: + item = _connections.get(connection_id) + if item is not None: + item['last_heartbeat'] = time.monotonic() + yield 'event: heartbeat\ndata: {}\n\n' except GeneratorExit: pass finally: - # Cleanup: stop all producers - conn.stop() - for ch, t in producers.items(): - t.join(timeout=2) - - # Remove connection from registry + realtime_store.close() with _conn_lock: - _connections.pop(conn_id, None) - - # Unregister SSE connection (FINDING-004) - sse_unregister(_sse_id) - - # Check for circuit-breaker channel_disabled events - for ch_name in all_channels: - if breakers.get(ch_name) and breakers[ch_name].is_disabled(): - pass # Already disabled - - log.debug("realtime %s: connection closed", conn_id[:20]) + _connections.pop(connection_id, None) + sse_unregister(sse_identity) return Response( - generate(), - mimetype='text/event-stream', + generate(), mimetype='text/event-stream', headers={ 'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no', 'Connection': 'keep-alive', - } + }, ) -def _start_producer(channel: str, producer_fn, ring: BoundedRing, - breaker: CircuitBreaker, stop_event: threading.Event, - kwargs: dict): - """Run a producer with task isolation and circuit breaker logic.""" - while not stop_event.is_set(): - if breaker.is_disabled(): - log.warning("realtime: channel %s disabled by circuit breaker", channel) - return - try: - producer_fn(ring, breaker, stop_event, **kwargs) - break # producer returned normally - except Exception as e: - log.error("realtime: producer %s crashed: %s", channel, e, exc_info=True) - should_stop = breaker.record_crash() - if should_stop: - log.error("realtime: channel %s circuit breaker open — stopping", channel) - return - # Wait before retry - stop_event.wait(1) - - -# --------------------------------------------------------------------------- -# Pause/Resume endpoint (internal — called by client) -# --------------------------------------------------------------------------- - @realtime_bp.route('/api/realtime/pause', methods=['POST']) def api_realtime_pause(): - """Pause chat+workplace event delivery for this session.""" - data = request.get_json() or {} - session_id = data.get('session_id', '').strip() - if not session_id: - return Response(json.dumps({'error': 'session_id required'}), status=400, - mimetype='application/json') - - with _conn_lock: - for conn in list(_connections.values()): - if conn.chat_session_id == session_id: - conn.pause() + # Delivery is durable now. Browser visibility may pause rendering locally; + # the server no longer needs a second per-connection buffer. return Response(json.dumps({'ok': True}), mimetype='application/json') @realtime_bp.route('/api/realtime/resume', methods=['POST']) def api_realtime_resume(): - """Resume chat+workplace event delivery and flush paused buffer.""" - data = request.get_json() or {} - session_id = data.get('session_id', '').strip() - if not session_id: - return Response(json.dumps({'error': 'session_id required'}), status=400, - mimetype='application/json') - - with _conn_lock: - for conn in list(_connections.values()): - if conn.chat_session_id == session_id: - conn.resume() - # Flush pause buffers — drain events that accumulated - # while paused and re-insert them into the per-channel - # rings so the generator yields them on the next pass. - if conn.rings: - for ch in ('chat', 'workplace'): - buf = conn._pause_buffers.get(ch) - if buf: - ring = conn.rings.get(ch) - if ring: - for _seq, item in buf.get_all(): - ring.put(item) return Response(json.dumps({'ok': True}), mimetype='application/json') - - -# --------------------------------------------------------------------------- -# Deprecated old-SSE endpoint wrappers (keep functional, log deprecation) -# --------------------------------------------------------------------------- - -_deprecated_logged: set = set() - -def _warn_deprecated_once(msg: str): - if msg not in _deprecated_logged: - log.warning("DEPRECATED: %s", msg) - _deprecated_logged.add(msg) diff --git a/routes/sessions.py b/routes/sessions.py index 6a7fb3ab..0d2e5fff 100644 --- a/routes/sessions.py +++ b/routes/sessions.py @@ -182,21 +182,25 @@ def api_list_sessions(): sessions, total = db.get_all_sessions(search=search, limit=limit, offset=offset, exclude_test=exclude_test) # Tag sessions that are currently being processed by an agent - from backend.agent_runtime import agent_runtime + from backend.realtime_store import realtime_store for s in sessions: - s['is_active'] = agent_runtime._is_busy(s['id']) + s['is_active'] = bool(realtime_store.active_turns(session_id=s['id'])) return jsonify({'sessions': sessions, 'total': total}) @sessions_bp.route('/api/sessions/') def api_get_session(session_id): + from backend.realtime_store import realtime_store + realtime_cursor = realtime_store.high_water() session = db.get_session_with_details(session_id) if not session: return jsonify({'error': 'Session not found'}), 404 limit = request.args.get('limit', 200, type=int) before_id = request.args.get('before_id', None, type=int) messages, has_more = db.get_session_messages_full(session_id, limit=limit, before_id=before_id) - return jsonify({'session': session, 'messages': messages, 'has_more': has_more}) + response = jsonify({'session': session, 'messages': messages, 'has_more': has_more}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response @sessions_bp.route('/api/sessions//poll') @@ -213,13 +217,18 @@ def api_session_reply(session_id): if request.content_type and request.content_type.startswith('multipart/form-data'): text = (request.form.get('text') or '').strip() perspective = (request.form.get('perspective') or 'A').strip() + client_message_id = (request.form.get('client_message_id') or '').strip() file = request.files.get('file') else: - data = request.get_json() + data = request.get_json() or {} text = (data.get('text') or '').strip() perspective = (data.get('perspective') or 'A').strip() + client_message_id = (data.get('client_message_id') or '').strip() file = None + if client_message_id and not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', client_message_id): + return jsonify({'error': 'Invalid client_message_id'}), 400 + if not text and not file: return jsonify({'error': 'Text or file is required'}), 400 @@ -285,16 +294,22 @@ def api_session_reply(session_id): text = f"{info_line}\n\n{text}" if text else info_line if perspective == 'A': + upload_meta = dict(upload_meta or {}) + if client_message_id: + upload_meta['client_message_id'] = client_message_id result = agent_runtime.send_as_user(session_id, text, image_url=image_url, metadata=upload_meta) else: - result = agent_runtime.send_as_bot(session_id, text) + result = agent_runtime.send_as_bot( + session_id, text, + metadata={'client_message_id': client_message_id} if client_message_id else None, + ) if not result: return jsonify({'error': 'Session not found'}), 404 # Build response — for slash commands, include the response text directly - resp = {'success': True} + resp = {'success': True, 'client_message_id': client_message_id or None} if isinstance(result, str): # send_as_user returned the slash command response text resp['slash_command'] = True @@ -366,6 +381,8 @@ def api_clear_all_sessions(): """Delete all chat sessions, messages, summaries, and attachments across all agents.""" db.clear_all_sessions() + from backend.realtime_store import realtime_store + realtime_store.purge_all() audit.log_session(user_id='admin', session_id='*', action='clear_all', ip=request.remote_addr or '') return jsonify({'success': True}) diff --git a/routes/update.py b/routes/update.py index d3b6f981..a01780a8 100644 --- a/routes/update.py +++ b/routes/update.py @@ -3,7 +3,7 @@ import json import queue -from flask import Blueprint, Response, jsonify, render_template, request, stream_with_context +from flask import Blueprint, Response, jsonify, render_template, request, stream_with_context, redirect from backend import update_manager @@ -67,6 +67,7 @@ def api_update_restart(): def api_update_stream(): """SSE endpoint for real-time update progress. DEPRECATED: Use unified GET /api/realtime/stream?channels=update instead.""" + return redirect('/api/realtime/stream?channels=update', code=307) import logging as _log_depr _log_depr.getLogger(__name__).warning( "DEPRECATED endpoint /api/system/update/stream used — " diff --git a/routes/workplaces.py b/routes/workplaces.py index c639bbb3..ad7ec2e6 100644 --- a/routes/workplaces.py +++ b/routes/workplaces.py @@ -8,7 +8,7 @@ import queue import threading -from flask import Blueprint, Response, jsonify, render_template, request, stream_with_context +from flask import Blueprint, Response, jsonify, render_template, request, stream_with_context, redirect from models.db import db @@ -154,6 +154,7 @@ def api_workplace_status(workplace_id): def api_workplace_events(workplace_id): """SSE stream for real-time workplace status changes (connector connect/disconnect, status). DEPRECATED: Use unified GET /api/realtime/stream?workplace= instead.""" + return redirect(f'/api/realtime/stream?workplace={workplace_id}', code=307) import logging as _log_depr _log_depr.getLogger(__name__).warning( "DEPRECATED endpoint /api/workplaces//events used — " diff --git a/static/js/chat-ui.js b/static/js/chat-ui.js index 7337d743..ac4b9677 100644 --- a/static/js/chat-ui.js +++ b/static/js/chat-ui.js @@ -2301,8 +2301,7 @@ const SSE_EVENTS = [ 'turn_begin', 'turn_split', 'thinking', 'tool_call_started', 'tool_executed', 'state:changed', 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', 'approval_required', 'approval_resolved', 'retry', 'message_injected', 'message_injection_applied', 'message_received', 'whatsapp_restriction_warning', 'session_clear', - 'state_changed', - 'heartbeat', + 'state_changed', 'turn_queued', 'ready', 'heartbeat', 'auth_expired', ]; // If no event (including heartbeats) arrives within this window, the connection @@ -2318,8 +2317,6 @@ class SSEAdapter { this._lastSeq = opts.afterSeq || 0; this._handler = null; this._es = null; - this._fillingGap = false; - this._pendingQueue = []; this._log = log('sse'); this._lastEventAt = 0; this._livenessInterval = null; @@ -2407,10 +2404,10 @@ class SSEAdapter { es.onerror = () => { this._log.warn('SSE error/closed', url); - console.warn('[sse] error/closed _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length); + console.warn('[sse] error/closed _lastSeq=', this._lastSeq); es.close(); if (this._es === es) this._es = null; - // Only reconnect if this was NOT an intentional stop (e.g. after 'done') + // Only reconnect if this adapter was not explicitly stopped. if (this._intentionallyStopped) { this._log.info('intentionally stopped — no reconnect'); return; @@ -2451,7 +2448,7 @@ class SSEAdapter { resumeUrl = u.pathname + u.search; } this._log.info('reconnecting from seq', this._lastSeq, resumeUrl); - console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length, 'url=', resumeUrl); + console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, 'url=', resumeUrl); this._connect(resumeUrl); }, delay); }; @@ -2459,80 +2456,21 @@ class SSEAdapter { _handleRaw(evtName, data) { const seq = data.seq || 0; - - if (this._fillingGap) { - this._log.debug('queued while filling gap', evtName, 'seq', seq, 'queueLen', this._pendingQueue.length); - if (this._pendingQueue.length >= 1 && this._pendingQueue.length % 10 === 0) { - console.warn('[sse] pendingQueue grew to', this._pendingQueue.length, 'while filling gap — possible reconnect storm'); - } - this._pendingQueue.push({ evtName, data }); - return; - } - if (seq && seq <= this._lastSeq) { this._log.debug('dedup skip', evtName, 'seq', seq, '≤ lastSeq', this._lastSeq); - console.log('[sse] dedup skip', evtName, 'seq=', seq, '_lastSeq=', this._lastSeq); return; } - - if (seq && this._lastSeq > 0 && seq > this._lastSeq + 1) { - this._log.warn('seq gap detected', this._lastSeq, '→', seq, '— filling'); - this._fillingGap = true; - this._pendingQueue.push({ evtName, data }); - this._fillGap(this._lastSeq, seq).then(() => { - this._fillingGap = false; - console.warn('[sse] draining pendingQueue len=', this._pendingQueue.length, '_lastSeq=', this._lastSeq); - this._drainQueue(); - }); - return; - } - if (seq) this._lastSeq = seq; + if (evtName === 'ready') return; this._dispatch(evtName, data); } - async _fillGap(afterSeq, upToSeq) { - try { - const agentId = this._agentId || this._url.match(/\/agents\/([^/?]+)\//)?.[1] || ''; - const res = await $.getJSON( - `/api/agents/${encodeURIComponent(agentId)}/chat/events?session_id=${encodeURIComponent(this._sessionId)}&after=${afterSeq}&up_to=${upToSeq}` - ); - const evts = res.events || []; - this._log.warn('gap-fill response: afterSeq=' + afterSeq + ' upToSeq=' + upToSeq + ' returned=' + evts.length + ' seqs=' + evts.map(e=>e.seq).join(',')); - console.warn('[gap-fill] returned', evts.length, 'events for after=', afterSeq, 'up_to=', upToSeq, 'seqs:', evts.map(e=>e.seq).join(',')); - for (const ev of evts) { - if (ev.seq <= this._lastSeq) continue; - this._lastSeq = ev.seq; - this._dispatch(ev.event, ev.data); - } - } catch (err) { - this._log.warn('gap-fill failed', err, '— skipping gap'); - this._lastSeq = upToSeq - 1; - } - } - - // Drain _pendingQueue asynchronously — one event per animation frame so we - // never block the main thread with a large synchronous burst. - _drainQueue() { - if (this._pendingQueue.length === 0) { - console.warn('[sse] queue drain done _lastSeq=', this._lastSeq); - return; - } - const item = this._pendingQueue.shift(); - const itemSeq = item.data.seq || 0; - if (itemSeq && itemSeq <= this._lastSeq) { - console.log('[sse] queue dedup skip', item.evtName, 'seq=', itemSeq); - // Skip but continue draining without waiting — dedup is cheap - this._drainQueue(); + _dispatch(evtName, data) { + if (evtName === 'auth_expired') { + this.stop(); + window.location.href = '/login'; return; } - if (itemSeq) this._lastSeq = itemSeq; - this._dispatch(item.evtName, item.data); - // Yield to the browser between each real event - requestAnimationFrame(() => this._drainQueue()); - } - - _dispatch(evtName, data) { if (evtName === 'state_changed') { // Not turn-scoped — bridge straight to the document-level event that // agent_detail.html / sessions.html already listen for (debounced refresh). @@ -2547,11 +2485,8 @@ class SSEAdapter { this._handler({ event: 'session_clear', data, seq: data.seq || 0 }); return; } - // done: stop reconnecting after this if (evtName === 'done') { - console.warn('[sse] _dispatch done _lastSeq=', this._lastSeq, 'data.seq=', data.seq); this._handler({ event: 'done', data, seq: data.seq || 0 }); - this.stop(); return; } this._handler({ event: evtName, data, seq: data.seq || 0 }); @@ -2990,7 +2925,7 @@ class Turn { console.warn('[turn] done event turn=%s _finalized=%s _finalContent=%s', this.id, this._finalized, !!this._finalContent); this._finalizeBubble(data.thinking_duration); // Fire final:response so page-level code can render the response bubble - // synchronously — no dependency on pollForResponse JSONL poll. + // synchronously from the durable stream. if (this._finalContent) { console.warn('[turn] firing final:response turn=%s contentLen=%d', this.id, this._finalContent.length); this._onTrigger('final:response', { diff --git a/static/js/chat-ui/transport.js b/static/js/chat-ui/transport.js index 9b2de2dd..b8b0f077 100644 --- a/static/js/chat-ui/transport.js +++ b/static/js/chat-ui/transport.js @@ -14,8 +14,7 @@ const SSE_EVENTS = [ 'turn_begin', 'turn_split', 'thinking', 'tool_call_started', 'tool_executed', 'state:changed', 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', 'approval_required', 'approval_resolved', 'retry', 'message_injected', 'message_injection_applied', 'message_received', 'whatsapp_restriction_warning', 'session_clear', - 'state_changed', - 'heartbeat', + 'state_changed', 'turn_queued', 'ready', 'heartbeat', 'auth_expired', ]; // If no event (including heartbeats) arrives within this window, the connection @@ -28,7 +27,7 @@ export class SSEAdapter { * @param {object} [opts] * @param {string} [opts.agentId] * @param {string} [opts.sessionId] - * @param {number} [opts.afterSeq=0] - resume from this seq (gap-fill will request from here) + * @param {number} [opts.afterSeq=0] - durable journal resume cursor */ constructor(url, opts = {}) { this._url = url; @@ -37,8 +36,6 @@ export class SSEAdapter { this._lastSeq = opts.afterSeq || 0; this._handler = null; this._es = null; - this._fillingGap = false; - this._pendingQueue = []; this._log = log('sse'); this._lastEventAt = 0; this._livenessInterval = null; @@ -126,10 +123,10 @@ export class SSEAdapter { es.onerror = () => { this._log.warn('SSE error/closed', url); - console.warn('[sse] error/closed _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length); + console.warn('[sse] error/closed _lastSeq=', this._lastSeq); es.close(); if (this._es === es) this._es = null; - // Only reconnect if this was NOT an intentional stop (e.g. after 'done') + // Only reconnect if this adapter was not explicitly stopped. if (this._intentionallyStopped) { this._log.info('intentionally stopped — no reconnect'); return; @@ -170,7 +167,7 @@ export class SSEAdapter { resumeUrl = u.pathname + u.search; } this._log.info('reconnecting from seq', this._lastSeq, resumeUrl); - console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length, 'url=', resumeUrl); + console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, 'url=', resumeUrl); this._connect(resumeUrl); }, delay); }; @@ -178,80 +175,21 @@ export class SSEAdapter { _handleRaw(evtName, data) { const seq = data.seq || 0; - - if (this._fillingGap) { - this._log.debug('queued while filling gap', evtName, 'seq', seq, 'queueLen', this._pendingQueue.length); - if (this._pendingQueue.length >= 1 && this._pendingQueue.length % 10 === 0) { - console.warn('[sse] pendingQueue grew to', this._pendingQueue.length, 'while filling gap — possible reconnect storm'); - } - this._pendingQueue.push({ evtName, data }); - return; - } - if (seq && seq <= this._lastSeq) { this._log.debug('dedup skip', evtName, 'seq', seq, '≤ lastSeq', this._lastSeq); - console.log('[sse] dedup skip', evtName, 'seq=', seq, '_lastSeq=', this._lastSeq); - return; - } - - if (seq && this._lastSeq > 0 && seq > this._lastSeq + 1) { - this._log.warn('seq gap detected', this._lastSeq, '→', seq, '— filling'); - this._fillingGap = true; - this._pendingQueue.push({ evtName, data }); - this._fillGap(this._lastSeq, seq).then(() => { - this._fillingGap = false; - console.warn('[sse] draining pendingQueue len=', this._pendingQueue.length, '_lastSeq=', this._lastSeq); - this._drainQueue(); - }); return; } - if (seq) this._lastSeq = seq; + if (evtName === 'ready') return; this._dispatch(evtName, data); } - async _fillGap(afterSeq, upToSeq) { - try { - const agentId = this._agentId || this._url.match(/\/agents\/([^/?]+)\//)?.[1] || ''; - const res = await $.getJSON( - `/api/agents/${encodeURIComponent(agentId)}/chat/events?session_id=${encodeURIComponent(this._sessionId)}&after=${afterSeq}&up_to=${upToSeq}` - ); - const evts = res.events || []; - this._log.warn('gap-fill response: afterSeq=' + afterSeq + ' upToSeq=' + upToSeq + ' returned=' + evts.length + ' seqs=' + evts.map(e=>e.seq).join(',')); - console.warn('[gap-fill] returned', evts.length, 'events for after=', afterSeq, 'up_to=', upToSeq, 'seqs:', evts.map(e=>e.seq).join(',')); - for (const ev of evts) { - if (ev.seq <= this._lastSeq) continue; - this._lastSeq = ev.seq; - this._dispatch(ev.event, ev.data); - } - } catch (err) { - this._log.warn('gap-fill failed', err, '— skipping gap'); - this._lastSeq = upToSeq - 1; - } - } - - // Drain _pendingQueue asynchronously — one event per animation frame so we - // never block the main thread with a large synchronous burst. - _drainQueue() { - if (this._pendingQueue.length === 0) { - console.warn('[sse] queue drain done _lastSeq=', this._lastSeq); - return; - } - const item = this._pendingQueue.shift(); - const itemSeq = item.data.seq || 0; - if (itemSeq && itemSeq <= this._lastSeq) { - console.log('[sse] queue dedup skip', item.evtName, 'seq=', itemSeq); - // Skip but continue draining without waiting — dedup is cheap - this._drainQueue(); + _dispatch(evtName, data) { + if (evtName === 'auth_expired') { + this.stop(); + window.location.href = '/login'; return; } - if (itemSeq) this._lastSeq = itemSeq; - this._dispatch(item.evtName, item.data); - // Yield to the browser between each real event - requestAnimationFrame(() => this._drainQueue()); - } - - _dispatch(evtName, data) { if (evtName === 'state_changed') { // Not turn-scoped — bridge straight to the document-level event that // agent_detail.html / sessions.html already listen for (debounced refresh). @@ -266,11 +204,8 @@ export class SSEAdapter { this._handler({ event: 'session_clear', data, seq: data.seq || 0 }); return; } - // done: stop reconnecting after this if (evtName === 'done') { - console.warn('[sse] _dispatch done _lastSeq=', this._lastSeq, 'data.seq=', data.seq); this._handler({ event: 'done', data, seq: data.seq || 0 }); - this.stop(); return; } this._handler({ event: evtName, data, seq: data.seq || 0 }); diff --git a/static/js/chat-ui/turn.js b/static/js/chat-ui/turn.js index ba5ab90e..6798f52f 100644 --- a/static/js/chat-ui/turn.js +++ b/static/js/chat-ui/turn.js @@ -357,7 +357,7 @@ export class Turn { console.warn('[turn] done event turn=%s _finalized=%s _finalContent=%s', this.id, this._finalized, !!this._finalContent); this._finalizeBubble(data.thinking_duration); // Fire final:response so page-level code can render the response bubble - // synchronously — no dependency on pollForResponse JSONL poll. + // synchronously from the durable stream. if (this._finalContent) { console.warn('[turn] firing final:response turn=%s contentLen=%d', this.id, this._finalContent.length); this._onTrigger('final:response', { diff --git a/static/js/realtime.js b/static/js/realtime.js index 16f1a7ec..1135f838 100644 --- a/static/js/realtime.js +++ b/static/js/realtime.js @@ -11,7 +11,6 @@ * sessionId: 'abc123', * agentId: 'my-agent', * workplace: 'wp-1', - * chatThrottle: 100, * }); * * rt.on('status', 'agent_busy_changed', (data) => { ... }); @@ -22,20 +21,6 @@ var RealtimeClient = (function () { 'use strict'; - // ---- Channel definitions ---- - var CHANNEL_PRIORITY = { - status: 0, update: 0, // Level 0: system/update - approvals: 1, // Level 1: user-facing - chat: 2, workplace: 2, // Level 2: high throughput - heartbeat: 0, - auth_expired: 0, - channel_disabled: 0, - }; - - // Per-channel resume sequence trackers - var _channelSeqs = {}; - var _channelIds = {}; // channel -> last SSE id - function RealtimeClient(opts) { opts = opts || {}; this._channels = (opts.channels || 'status,approvals,update').split(',').map(function (s) { return s.trim(); }); @@ -44,13 +29,11 @@ var RealtimeClient = (function () { this._agentId = opts.agentId || ''; this._after = opts.after || 0; this._workplace = opts.workplace || ''; - this._chatThrottle = opts.chatThrottle || 100; this._es = null; this._handlers = {}; // channel -> [handler] this._started = false; this._intentionallyStopped = false; this._paused = false; - this._pauseBuffer = {}; // channel -> [events] buffered during pause this._onAuthExpired = opts.onAuthExpired || function () { window.location.href = '/login'; }; this._visibilityBound = false; this._unloadHandlers = []; // cleanup hooks registered by consumers @@ -92,27 +75,13 @@ var RealtimeClient = (function () { RealtimeClient.prototype.pause = function () { if (this._paused) return; this._paused = true; - if (this._es && this._es.readyState === EventSource.OPEN) { - // Send pause signal via a separate fetch - this._sendCommand('pause'); - } + this._disconnect(); }; RealtimeClient.prototype.resume = function () { if (!this._paused) return; this._paused = false; - if (this._es && this._es.readyState === EventSource.OPEN) { - this._sendCommand('resume'); - } - // Replay buffered events - var self = this; - Object.keys(this._pauseBuffer).forEach(function (ch) { - var buf = self._pauseBuffer[ch]; - while (buf && buf.length) { - var item = buf.shift(); - self._dispatch(ch, item.evtName, item.data); - } - }); + if (this._started) this._connect(); }; // ---- Internal: Connection lifecycle ---- @@ -124,15 +93,14 @@ var RealtimeClient = (function () { params.push('chat=1'); if (this._sessionId) params.push('session_id=' + encodeURIComponent(this._sessionId)); if (this._agentId) params.push('agent_id=' + encodeURIComponent(this._agentId)); - if (this._after) params.push('after=' + this._after); } + if (this._after) params.push('after=' + this._after); if (this._workplace) params.push('workplace=' + encodeURIComponent(this._workplace)); - if (this._chatThrottle) params.push('chat_throttle=' + this._chatThrottle); return '/api/realtime/stream?' + params.join('&'); }; RealtimeClient.prototype._connect = function () { - if (this._intentionallyStopped) return; + if (this._intentionallyStopped || this._paused || this._es) return; var self = this; var url = this._buildUrl(); @@ -167,11 +135,13 @@ var RealtimeClient = (function () { 'approval_required', 'approval_resolved', 'update_status', 'update_done', 'turn_begin', 'thinking', 'tool_call_started', 'tool_executed', + 'state:changed', 'state_changed', 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', 'retry', 'message_injected', - 'message_injection_applied', 'whatsapp_restriction_warning', 'session_clear', 'turn_split', + 'message_injection_applied', 'message_received', 'turn_queued', + 'whatsapp_restriction_warning', 'session_clear', 'turn_split', 'connector_connected', 'connector_disconnected', 'connector_paired', 'workplace_status_changed', - 'heartbeat', 'auth_expired', 'channel_disabled', + 'ready', 'heartbeat', 'auth_expired', 'channel_disabled', ]; ALL_EVENTS.forEach(function (evtName) { @@ -189,7 +159,7 @@ var RealtimeClient = (function () { // Auto-reconnect with jitter var delay = 2000 + Math.floor(Math.random() * 5000); setTimeout(function () { - if (self._intentionallyStopped) return; + if (self._intentionallyStopped || self._paused) return; self._connect(); }, delay); }; @@ -202,24 +172,12 @@ var RealtimeClient = (function () { } }; - RealtimeClient.prototype._sendCommand = function (cmd) { - try { - var xhr = new XMLHttpRequest(); - xhr.open('POST', '/api/realtime/' + cmd, true); - xhr.send(); - } catch (_) {} - }; - // ---- Internal: Event routing ---- RealtimeClient.prototype._routeEvent = function (evtName, data, lastEventId) { - // Track per-channel seq from composite SSE id (e.g. "chat:892") + // Durable journal IDs are global and may legitimately skip after scope filtering. if (lastEventId) { - var parts = lastEventId.split(':'); - if (parts.length === 2) { - _channelIds[parts[0]] = lastEventId; - _channelSeqs[parts[0]] = parseInt(parts[1], 10) || 0; - } + this._after = Math.max(this._after, parseInt(lastEventId, 10) || 0); } // Map event name to channel @@ -241,15 +199,6 @@ var RealtimeClient = (function () { if (evtName === 'heartbeat') return; // no-op - // Pause buffering for chat/workplace events - if (this._paused && (channel === 'chat' || channel === 'workplace')) { - if (!this._pauseBuffer[channel]) this._pauseBuffer[channel] = []; - if (this._pauseBuffer[channel].length < 100) { - this._pauseBuffer[channel].push({ evtName: evtName, data: data }); - } - return; - } - this._dispatch(channel, evtName, data); }; @@ -279,6 +228,7 @@ var RealtimeClient = (function () { RealtimeClient.prototype._eventToChannel = function (evtName) { // Status channel events if (evtName === 'agent_busy_changed' || evtName === 'agent_turn_complete' || + evtName === 'turn_queued' || evtName === 'whatsapp_bridge_status') { return 'status'; } @@ -293,9 +243,11 @@ var RealtimeClient = (function () { // Chat channel events if (evtName === 'turn_begin' || evtName === 'thinking' || evtName === 'tool_call_started' || evtName === 'tool_executed' || + evtName === 'state:changed' || evtName === 'state_changed' || + evtName === 'tasks:auto_transition' || evtName === 'tasks:stale' || evtName === 'response_chunk' || evtName === 'done' || evtName === 'retry' || evtName === 'message_injected' || - evtName === 'message_injection_applied' || + evtName === 'message_injection_applied' || evtName === 'message_received' || evtName === 'whatsapp_restriction_warning' || evtName === 'session_clear' || evtName === 'turn_split') { return 'chat'; @@ -330,24 +282,6 @@ var RealtimeClient = (function () { this._unloadHandlers.push(fn); }; - // ---- SSE comment handler (invoked by caller when EventSource - // comment events are intercepted — see x-sse-dropped below) ---- - - RealtimeClient.prototype._handleComment = function (comment) { - // :x-sse-dropped N — server lost N events on a channel - var match = comment.match(/^x-sse-dropped\s+(\d+)/); - if (match) { - var dropped = parseInt(match[1], 10); - console.warn('[realtime] stream thinned:', dropped, 'events dropped'); - this._dispatch('stream_thinned', { dropped: dropped }); - } - // :error channel= — producer error - var errMatch = comment.match(/^error\s+channel=(\S+)/); - if (errMatch) { - console.warn('[realtime] producer error on channel:', errMatch[1]); - } - }; - return RealtimeClient; })(); diff --git a/templates/agent_detail.html b/templates/agent_detail.html index e54b963f..39c7a348 100644 --- a/templates/agent_detail.html +++ b/templates/agent_detail.html @@ -4573,8 +4573,8 @@

Allowed window._evRealtime.on('status', 'panel_updated', function (payload) { if (payload && payload.agent_id === AGENT_ID) refreshPluginTab('panel'); }); - // Busy state pushed by the unified status stream — replaces the old - // 5s /busy poll. _chatBusy (this session's own turn) takes precedence. + // Busy state is pushed by the unified stream; this session's own + // active turn takes precedence over the agent-wide snapshot. window._evRealtime.on('status', 'agent_busy_changed', function (payload) { if (payload && payload.agent_id === AGENT_ID && !_chatBusy) { updateBusyBadge(payload.busy); @@ -4834,8 +4834,6 @@

Allowed } // ==================== Chat ==================== -let lastChatTs = 0; // epoch-ms cursor for JSONL-based polling -let lastTurnEndTs = 0; // ts of last turn_end rendered from chatlog history let _chatSessionId = null; // cached session_id for this user async function _getChatSessionId() { @@ -4850,17 +4848,18 @@

Allowed return _chatSessionId; } -// Pass { persistent: true } when called from incremental idle polling so the thinking -// bubble persists across multiple poll calls within a single turn. -function renderEntries(entries, { persistent = false } = {}) { +function renderEntries(entries) { // Render JSONL entries into the chat UI, grouping thinking/tool events into bubbles. - let thinkingId = persistent ? _pollThinkingId : null; + let thinkingId = null; for (const entry of entries) { - if (entry.ts > lastChatTs) lastChatTs = entry.ts; const type = entry.type; if (type === 'user') { if (thinkingId) { chatUI.finalizeThinkingBubble(thinkingId); thinkingId = null; } - chatUI.appendMessage('user', entry.content, {metadata: entry.metadata}); + const $el = chatUI.appendMessage('user', entry.content, {metadata: entry.metadata}); + if (entry.message_id) { + $el.attr('data-message-id', entry.message_id); + _seenRealtimeMessages.add(`${_chatSessionId}:${entry.message_id}`); + } } else if (type === 'thinking') { if (!thinkingId) thinkingId = chatUI.showThinkingIndicator(); chatUI.appendTimelineEntry(thinkingId, {type: 'thinking', content: entry.content}); @@ -4886,7 +4885,11 @@

Allowed } else if (meta.error) { chatUI.appendMessage('error', entry.content, {metadata: meta}); } else { - chatUI.appendMessage('assistant', entry.content, {metadata: meta}); + const $el = chatUI.appendMessage('assistant', entry.content, {metadata: meta}); + if (entry.message_id) { + $el.attr('data-message-id', entry.message_id); + _seenRealtimeMessages.add(`${_chatSessionId}:${entry.message_id}`); + } } } else if (type === 'error') { const meta = entry.metadata || {}; @@ -4896,15 +4899,11 @@

Allowed chatUI.appendMessage('system', entry.content, {metadata: entry.metadata}); } else if (type === 'turn_end') { if (thinkingId) { chatUI.finalizeThinkingBubble(thinkingId, entry.thinking_duration); thinkingId = null; } - if (entry.ts && entry.ts > lastTurnEndTs) lastTurnEndTs = entry.ts; } // turn_begin, pending: no rendering needed for history } - if (persistent) { - _pollThinkingId = thinkingId; // save for next incremental poll call - } else if (thinkingId) { - // In-progress turn detected: remove the incomplete bubble so restoreActiveReasoning - // can create a fresh one from the event buffer without duplicating entries. + if (thinkingId) { + // Active telemetry is rebuilt from the durable stream after history loads. chatUI.removeThinkingIndicator(thinkingId); } } @@ -4932,141 +4931,173 @@

Allowed async function loadChatHistory() { const epoch = window._agentEpoch; - _pollThinkingId = null; - lastChatTs = 0; _showChatLoadingSkeleton(); try { // Fetch session_id and history in parallel — don't block history on session lookup const sidPromise = _getChatSessionId(); const res = await fetch(`/api/agents/${AGENT_ID}/chat?user_id=web_test&limit=50`); await sidPromise; // ensure cache is primed before restoreActiveReasoning runs + const realtimeCursor = Number(res.headers.get('X-Evonic-Realtime-Cursor') || 0); const data = await res.json(); if (epoch !== window._agentEpoch) return; // agent switched mid-fetch — discard chatUI.clearContainer(); chatUI.batchRender(() => renderEntries(data.entries || [])); + window._chatHistoryCursor = realtimeCursor; } catch(e) { /* ignore load errors */ } if (epoch !== window._agentEpoch) return; if (window.finishNavProgress) window.finishNavProgress(); // end soft-switch loading bar (no-op otherwise) chatLoaded = true; setTimeout(() => chatUI.scrollToBottom(), 50); - if (!_chatBusy) startIdlePoll(); restoreActiveReasoning(); // fire-and-forget — history already rendered } +const _seenRealtimeMessages = new Set(); +const _optimisticMessages = new Map(); + +function newClientMessageId() { + return (window.crypto && crypto.randomUUID) + ? crypto.randomUUID() + : `${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + document.addEventListener('evonic:message-received', function (event) { - const meta = (event.detail && event.detail.metadata) || {}; - if (meta.escalated_from_agent_session) loadChatHistory(); + const data = event.detail || {}; + if (!data.message_id || (data.session_id && data.session_id !== _chatSessionId)) return; + const key = `${_chatSessionId}:${data.message_id}`; + if (_seenRealtimeMessages.has(key)) return; + _seenRealtimeMessages.add(key); + const optimistic = data.client_message_id && _optimisticMessages.get(data.client_message_id); + if (optimistic) { + optimistic.attr('data-message-id', data.message_id); + _optimisticMessages.delete(data.client_message_id); + return; + } + chatUI.appendMessage(data.role || 'user', data.content || data.message || '', { + metadata: data.metadata || {}, + timestamp: data.timestamp ? new Date(data.timestamp).toISOString() : null, + }).attr('data-message-id', data.message_id); + chatUI.scrollToBottom(); }); -let _restoringActiveReasoning = false; +let _agentChatEs = null; +let _agentChatSessionId = null; +let _agentChatCursor = 0; +const _renderedTurnIds = new Set(); -async function restoreActiveReasoning() { - if (_restoringActiveReasoning) return; - _restoringActiveReasoning = true; - const epoch = window._agentEpoch; - try { - // Use cached session_id — already fetched in loadChatHistory() - const sessionId = await _getChatSessionId(); - if (!sessionId || _currentTurn) return; // an in-page turn already owns this session - - // Busy ownership is authoritative after refresh. The event buffer may be - // empty for a long synchronous tool call, so do not require a recent - // thinking/tool event before restoring the indicator. - const [eRes, busyRes] = await Promise.all([ - fetch(`/api/agents/${encodeURIComponent(AGENT_ID)}/chat/events?session_id=${encodeURIComponent(sessionId)}&after=0`), - fetch(`/api/agents/${encodeURIComponent(AGENT_ID)}/busy`), - ]); - if (_currentTurn) return; // user sent a message while fetching - const eData = eRes.ok ? await eRes.json() : {events: []}; - const busyState = busyRes.ok ? await busyRes.json() : {}; - if (epoch !== window._agentEpoch || sessionId !== _chatSessionId) return; // agent/session switched mid-fetch - if (_currentTurn) return; // bail before touching shared state - if (!busyState.busy || busyState.session_id !== sessionId) return; - const events = eData.events || []; - - // Only replay events from the current in-progress turn. - // Discard everything up to and including the last 'done' OR 'turn_split' so - // that pre-split events (already rendered by renderEntries from JSONL) are not - // replayed again — which would create a duplicate thinking bubble for turn 1. - const lastDoneIdx = events.reduce((idx, ev, i) => ev.event === 'done' ? i : idx, -1); - const lastSplitIdx = events.reduce((idx, ev, i) => ev.event === 'turn_split' ? i : idx, -1); - const lastClearIdx = events.reduce((idx, ev, i) => ev.event === 'session_clear' ? i : idx, -1); - const cutIdx = Math.max(lastDoneIdx, lastSplitIdx, lastClearIdx); - const replayEvents = cutIdx >= 0 ? events.slice(cutIdx + 1) : events; - - // The event buffer can retain an incomplete tail after a restart or other - // interrupted turn. /busy above remains authoritative; replay only enriches - // the indicator and never creates activity by itself. - // Guard: if the in-progress turn started before the last turn_end rendered - // from chatlog history, that turn was already fully finalized — skip to avoid - // creating a duplicate thinking bubble above completed messages. - const firstReplayEvt = replayEvents.find(ev => ev.event === 'turn_begin'); - if (firstReplayEvt && firstReplayEvt.data && firstReplayEvt.data.ts && - firstReplayEvt.data.ts < lastTurnEndTs) { - return; - } +function _ensureRealtimeTurn(startTs = null) { + if (_currentTurn && _currentTurn.thinkingId && + !_currentTurn.thinkingId._finalized) return _currentTurn.thinkingId; + const container = document.getElementById('chat-messages'); + const users = container ? container.querySelectorAll('[data-msg-role="user"]') : []; + const anchor = users.length ? users[users.length - 1] : null; + const turn = chatUI.showThinkingIndicator(startTs, anchor); + _currentTurn = { + abortController: null, thinkingId: turn, + }; + _chatBusy = true; + updateBusyBadge(true); + return turn; +} + +function _renderTurnResponse(data) { + if (!data.response) return; + if (data.message_id && _seenRealtimeMessages.has(`${_chatSessionId}:${data.message_id}`)) return; + const key = data.turn_id || `response:${data.event_id}`; + if (_renderedTurnIds.has(key)) return; + _renderedTurnIds.add(key); + const opts = data.attachment_info + ? {metadata: {attachment_info: data.attachment_info}} + : {metadata: {}}; + const $el = chatUI.appendMessage(data.is_error ? 'error' : 'assistant', data.response, opts); + if (data.message_id) { + $el.attr('data-message-id', data.message_id); + _seenRealtimeMessages.add(`${_chatSessionId}:${data.message_id}`); + } + chatUI.scrollToBottom(); +} - // Stop the idle poll before creating the live bubble. The idle poll may - // have fired during our async awaits (before _currentTurn was set) and - // created a stale _pollThinkingId that would duplicate the live bubble. - // pollForResponse → setChatBusy(false) will restart the idle poll when done. - stopIdlePoll(); - - // Replay trace into a new thinking bubble. - // Find the last user message to anchor the bubble below it. - const container = document.getElementById('chat-messages'); - const userWrappers = container ? container.querySelectorAll('[data-msg-role="user"]') : []; - const userMsgEl = userWrappers.length ? userWrappers[userWrappers.length - 1] : null; - let resumeSeq = 0; - const turnBeginEvt = replayEvents.find(ev => ev.event === 'turn_begin'); - let thinkingId = chatUI.showThinkingIndicator(turnBeginEvt ? turnBeginEvt.data.ts : null, userMsgEl); - for (const ev of replayEvents) { - if (ev.seq > resumeSeq) resumeSeq = ev.seq; - if (ev.event === 'thinking') { - chatUI.appendTimelineEntry(thinkingId, {type: 'thinking', content: ev.data.content}); - } else if (ev.event === 'tool_call_started') { - chatUI.appendTimelineEntry(thinkingId, {type: 'tool_call', tool: ev.data.tool, args: ev.data.args, param_types: ev.data.param_types || {}}); - } else if (ev.event === 'tool_executed') { - chatUI.appendTimelineEntry(thinkingId, {type: 'tool_result', tool: ev.data.tool, result: ev.data.result, error: ev.data.error}); - } else if (ev.event === 'response_chunk' && ev.data.is_final && ev.data.content) { - chatUI.appendTimelineEntry(thinkingId, {type: 'response', content: ev.data.content}); - } else if (ev.event === 'turn_split') { - chatUI.finalizeThinkingBubble(thinkingId); - thinkingId = chatUI.showThinkingIndicator(null, userMsgEl); - } - } +function _handleAgentRealtime(evtName, data, sessionId) { + if (_chatSessionId !== sessionId) return; + const seq = Number(data.seq || data.event_id || 0); + if (seq && seq <= _agentChatCursor) return; + if (seq) _agentChatCursor = seq; - _currentTurn = { abortController: null, thinkingId, stream: null, pollTimer: null, _finalRendered: false }; - - chatUI.scrollToBottom(); - - // Connect SSE from resumeSeq so the backend only sends the gap - _currentTurn.stream = chatUI.connectThinkingStream( - `/api/agents/${encodeURIComponent(AGENT_ID)}/chat/stream?session_id=${encodeURIComponent(sessionId)}&after=${resumeSeq}`, - thinkingId, - { userMsgEl, - onSplit: (newId) => { _currentTurn.thinkingId = newId; }, - onDone: () => { if (_currentTurn) _currentTurn.thinkingId = null; }, - onFinalResponse: (content) => { - if (_currentTurn) _currentTurn._finalRendered = true; - chatUI.appendMessage('assistant', content, { metadata: {} }); - chatUI.scrollToBottom(); - } - } - ); + if (evtName === 'message_received') { + document.dispatchEvent(new CustomEvent('evonic:message-received', {detail: data})); + return; + } + if (evtName === 'state_changed') { + document.dispatchEvent(new CustomEvent('evonic:agent-state-changed', {detail: data})); + return; + } + if (evtName === 'agent_busy_changed') { + if (data.agent_id && data.agent_id !== AGENT_ID) return; + const ownsSession = data.session_id === sessionId || + (data.session_ids || []).includes(sessionId); + updateBusyBadge(Boolean(data.busy)); + if (data.busy && ownsSession) _ensureRealtimeTurn(); + return; + } + if (evtName === 'turn_queued') { + _ensureRealtimeTurn(data.timestamp || null); + return; + } + if (evtName === 'session_clear') { + chatUI.clear(); + _currentTurn = null; + return; + } + + let turn = _ensureRealtimeTurn(evtName === 'turn_begin' ? data.ts : null); + if (evtName === 'turn_split') { + turn.ingest({event: evtName, data, seq}); + turn = chatUI.showThinkingIndicator(); + _currentTurn.thinkingId = turn; + return; + } + turn.ingest({event: evtName, data, seq}); + if (evtName === 'done') { + _renderTurnResponse(data); + _currentTurn = null; + _chatBusy = false; + updateBusyBadge(false); + } +} - // Poll until the final response lands - pollForResponse(thinkingId); - } catch(e) { /* silently ignore */ } - finally { _restoringActiveReasoning = false; } +async function restoreActiveReasoning() { + const sessionId = await _getChatSessionId(); + if (!sessionId) return; + if (_agentChatEs && _agentChatSessionId === sessionId) return; + if (_agentChatEs) _agentChatEs.close(); + _agentChatSessionId = sessionId; + _agentChatCursor = Number(window._chatHistoryCursor || 0); + const url = `/api/realtime/stream?chat=1&agent_id=${encodeURIComponent(AGENT_ID)}&session_id=${encodeURIComponent(sessionId)}&after=${_agentChatCursor}`; + const es = new EventSource(url); + _agentChatEs = es; + const events = [ + 'agent_busy_changed', 'turn_queued', 'turn_begin', 'thinking', + 'tool_call_started', 'tool_executed', 'state:changed', 'state_changed', + 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', + 'approval_required', 'approval_resolved', 'retry', 'turn_split', + 'message_injected', 'message_injection_applied', 'message_received', + 'whatsapp_restriction_warning', 'session_clear', + ]; + events.forEach(evtName => es.addEventListener(evtName, event => { + let data = {}; + try { data = JSON.parse(event.data); } catch (_) {} + _handleAgentRealtime(evtName, data, sessionId); + })); + es.addEventListener('auth_expired', () => { + es.close(); + window.location.href = '/login'; + }); + es.onerror = () => { + // Native EventSource reconnects with Last-Event-ID; keep the cursor intact. + }; } -let chatPollTimer = null; -let chatIdleTimer = null; -let chatPolling = false; let _chatBusy = false; -let _currentTurn = null; // { abortController, thinkingId, stream, pollTimer } -let _pollThinkingId = null; // active thinking bubble from idle poll (persists across calls) +let _currentTurn = null; // { abortController, thinkingId } function updateBusyBadge(busy) { const dot = document.getElementById('state-busy-dot'); @@ -5085,29 +5116,13 @@

Allowed function _destroyCurrentTurn() { if (!_currentTurn) return; - console.warn('[_destroyCurrentTurn] hasStream=%s hasPollTimer=%s hasThinkingId=%s', !!_currentTurn.stream, !!_currentTurn.pollTimer, !!_currentTurn.thinkingId); if (_currentTurn.abortController) _currentTurn.abortController.abort(); - // Explicitly stop the SSEAdapter — chatUI.closeStream() is a no-op shim. - // Without this, old adapters keep reconnecting from a stale _lastSeq, - // fetching all historical events on every reconnect → OOM renderer crash. - if (_currentTurn.stream && _currentTurn.stream.stop) { - console.warn('[_destroyCurrentTurn] stopping stream _lastSeq=', _currentTurn.stream._lastSeq); - _currentTurn.stream.stop(); - _currentTurn.stream = null; - } if (_currentTurn.thinkingId) { chatUI.removeThinkingIndicator(_currentTurn.thinkingId); } - if (_currentTurn.pollTimer) { - clearInterval(_currentTurn.pollTimer); - } _currentTurn = null; - // Safety net: if the SSE gap-fill prevented 'done' delivery, the Turn object - // may still be in chatUI._turns with _finalized=false. hasActiveStream() would - // then return true permanently, blocking the idle poll from showing the response. - // Force-clear any stranded non-finalized turns so the idle poll can resume. if (chatUI.hasActiveStream()) { - console.warn('[_destroyCurrentTurn] stranded active turn detected — force-clearing so idle poll can resume'); + console.warn('[_destroyCurrentTurn] clearing stranded active turn'); chatUI.clearActiveSpinner(); } } @@ -5120,74 +5135,14 @@

Allowed if (!busy) { input.focus(); if (!chatUI.hasActiveStream()) _destroyCurrentTurn(); - startIdlePoll(); - } else { - stopIdlePoll(); } } -// One-shot busy-state resync (external sessions: notifier, other channels). -// Live updates arrive via the shared status stream (agent_busy_changed); this -// only fetches the current snapshot on load / after a soft agent switch. -let _busyPolling = false; -async function resyncBusyBadge() { - if (_chatBusy || _busyPolling) return; // skip if own turn is busy or a fetch is in-flight - _busyPolling = true; - try { - const res = await fetch(`/api/agents/${AGENT_ID}/busy`); - const data = await res.json(); - if (!_chatBusy) updateBusyBadge(data.busy); - } catch (e) { /* ignore */ } finally { _busyPolling = false; } -} -resyncBusyBadge(); // initial state; the stream covers everything after this - -// Stop all polling on page unload to free HTTP connections during navigation +// Stop the durable stream on navigation; EventSource handles reconnects itself. window.addEventListener('beforeunload', () => { - stopIdlePoll(); + if (_agentChatEs) _agentChatEs.close(); }); -function startIdlePoll() { - if (chatIdleTimer) return; // already running - chatIdleTimer = setInterval(async () => { - // Skip during user-initiated busy state, or when there's an SSE stream - // that's NOT from the idle poll (tracked via _pollThinkingId). - if (_chatBusy) return; - if (chatUI.hasActiveStream() && !_pollThinkingId) return; - try { - const sid = await _getChatSessionId(); - const qs = sid - ? `session_id=${encodeURIComponent(sid)}&after_ts=${lastChatTs}&limit=20` - : `user_id=web_test&after_ts=${lastChatTs}&limit=20`; - const res = await fetch(`/api/agents/${AGENT_ID}/chat?${qs}`); - const data = await res.json(); - // Re-check: user may have sent a message while fetching. - // Don't block if the only active stream is our own idle-poll bubble. - if (_chatBusy) return; - if (chatUI.hasActiveStream() && !_pollThinkingId) return; - const entries = data.entries || []; - if (!entries.length) return; - const wasAtBottom = chatUI.isNearBottom(80); - // Advance cursor over ALL entries (including user echoes) so we don't re-fetch them - for (const e of entries) { if (e.ts > lastChatTs) lastChatTs = e.ts; } - // Skip user echoes (already shown locally); render assistant/system events - const newEntries = entries.filter(e => e.type !== 'user'); - renderEntries(newEntries, { persistent: true }); - if (newEntries.length && wasAtBottom) chatUI.scrollToBottom(); - } catch(e) { /* ignore network errors during idle poll */ } - }, 3000); -} - -function stopIdlePoll() { - if (chatIdleTimer) { - clearInterval(chatIdleTimer); - chatIdleTimer = null; - } - if (_pollThinkingId) { - chatUI.removeThinkingIndicator(_pollThinkingId); - _pollThinkingId = null; - } -} - // Chat-input keydown handler: Enter submits on desktop, inserts newline on mobile. // Mobile = viewport < 1024px (matching enterChatFullscreen breakpoint). // Desktop = viewport >= 1024px: Enter without Shift submits, Shift+Enter inserts newline. @@ -5716,12 +5671,7 @@

Allowed function stopChat() { _destroyCurrentTurn(); - if (chatPollTimer) { - clearInterval(chatPollTimer); - chatPollTimer = null; - } - chatPolling = false; - setChatBusy(false); // will call startIdlePoll() + setChatBusy(false); } async function sendChat() { @@ -5758,6 +5708,9 @@

Allowed info.is_image ? `[Image #${++imageSlot}]` : `[File: ${info.filename}]`).join(' '); const $msgEl = chatUI.appendMessage('user', displayText, {metadata: optimisticMeta}); + const clientMessageId = newClientMessageId(); + $msgEl.attr('data-client-message-id', clientMessageId); + _optimisticMessages.set(clientMessageId, $msgEl); if (files.length) chatUI.markMessageUploading($msgEl); // Capture the user message wrapper for anchoring the thinking bubble const container = document.getElementById('chat-messages'); @@ -5772,6 +5725,7 @@

Allowed const formData = new FormData(); formData.append('message', msg); formData.append('user_id', 'web_test'); + formData.append('client_message_id', clientMessageId); files.forEach(file => formData.append('files', file)); return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); @@ -5792,7 +5746,7 @@

Allowed return fetch(url, { method: 'POST', headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({message: msg, user_id: 'web_test'}), + body: JSON.stringify({message: msg, user_id: 'web_test', client_message_id: clientMessageId}), signal, }).then(r => r.json()); } @@ -5807,17 +5761,23 @@

Allowed return; } if (!data.success) { + if ($msgEl.attr('data-message-id')) return; $msgEl.remove(); chatUI.appendMessage('error', 'Error: ' + (data.error || 'Unknown error')); } else { - const $resp = chatUI.appendMessage('assistant', data.response); + let $resp = $(); + if (!data.turn_id || !_renderedTurnIds.has(data.turn_id)) { + if (data.turn_id) _renderedTurnIds.add(data.turn_id); + $resp = chatUI.appendMessage('assistant', data.response); + } // Reposition the slash command response immediately after the user // message so it doesn't appear below an active thinking bubble that // turn_split may have inserted in the meantime. if (userMsgEl && $resp && $resp.length) $(userMsgEl).after($resp); - await syncLastChatTs(); } } catch(e) { + _optimisticMessages.delete(clientMessageId); + if ($msgEl.attr('data-message-id')) return; $msgEl.remove(); chatUI.appendMessage('error', 'Network error: ' + e.message); } @@ -5830,13 +5790,6 @@

Allowed // Destroy any previous turn before starting a new one _destroyCurrentTurn(); chatUI.clearActiveSpinner(); - // Kill any stale poll (e.g. from restoreActiveReasoning running concurrently) - if (chatPollTimer) { - clearInterval(chatPollTimer); - chatPollTimer = null; - chatPolling = false; - } - // Create thinking bubble eagerly, anchored right below the user message. // This guarantees correct DOM ordering — the bubble always sits between // the user message and whatever follows, never above a prior finalized bubble. @@ -5845,42 +5798,18 @@

Allowed const abortController = new AbortController(); const signal = abortController.signal; - // Fetch session_id so we can open SSE stream before the LLM starts - let sessionId = null; - try { - const sRes = await fetch(`/api/agents/${AGENT_ID}/chat/session?user_id=web_test`, {signal}); - const sData = await sRes.json(); - sessionId = sData.session_id || null; - } catch(e) { /* SSE won't work but chat still functions */ } - - _currentTurn = { abortController, thinkingId, stream: null, pollTimer: null, _finalRendered: false }; - - if (sessionId) { - _currentTurn.stream = chatUI.connectThinkingStream( - `/api/agents/${AGENT_ID}/chat/stream?session_id=${encodeURIComponent(sessionId)}`, - thinkingId, - { - userMsgEl: userMsgEl, - onSplit: (newId) => { thinkingId = newId; _currentTurn.thinkingId = newId; }, - onDone: () => { thinkingId = null; if (_currentTurn) _currentTurn.thinkingId = null; }, - onFinalResponse: (content) => { - if (_currentTurn) _currentTurn._finalRendered = true; - chatUI.appendMessage('assistant', content, { metadata: {} }); - chatUI.scrollToBottom(); - }, - } - ); - } + _currentTurn = { abortController, thinkingId }; + restoreActiveReasoning(); try { const data = await postChat(`/api/agents/${AGENT_ID}/chat`, signal); if (data.buffered) { - pollForResponse(thinkingId); return; } else if (data.injected) { chatUI.removeThinkingIndicator(thinkingId); chatUI.markLastUserBubbleQueued(); } else if (!data.success) { + if ($msgEl.attr('data-message-id')) return; chatUI.removeThinkingIndicator(thinkingId); $msgEl.remove(); chatUI.appendMessage('error', 'Error: ' + (data.error || 'Unknown error')); @@ -5891,16 +5820,22 @@

Allowed chatUI.appendMessage('system', data.response, { metadata: { slash_command: true } }); thinkingId = null; if (_currentTurn) _currentTurn.thinkingId = null; chatUI.closeStream(); - await syncLastChatTs(); } else if (data.slash_command) { // Slash command response — no LLM involved, remove the eager thinking bubble chatUI.removeThinkingIndicator(thinkingId); thinkingId = null; if (_currentTurn) _currentTurn.thinkingId = null; chatUI.closeStream(); const cmdMeta = data.bash_exec ? { bash_exec: true } : { slash_command: true }; - chatUI.appendMessage('assistant', data.response, { metadata: cmdMeta }); + const responseKey = data.response_message_id + ? `${_chatSessionId}:${data.response_message_id}` : null; + if (!responseKey || !_seenRealtimeMessages.has(responseKey)) { + const $response = chatUI.appendMessage('assistant', data.response, { metadata: cmdMeta }); + if (data.response_message_id) { + $response.attr('data-message-id', data.response_message_id); + _seenRealtimeMessages.add(responseKey); + } + } chatUI.scrollToBottom(); - await syncLastChatTs(); } else { // Non-buffered success: agent responded immediately. // Thinking bubble was already created eagerly; populate it and close SSE. @@ -5926,11 +5861,15 @@

Allowed chatUI.finalizeThinkingBubble(thinkingId, (data.metadata || {}).thinking_duration); thinkingId = null; if (_currentTurn) _currentTurn.thinkingId = null; chatUI.closeStream(); - chatUI.appendMessage(data.error ? 'error' : 'assistant', data.response); - await syncLastChatTs(); + if (data.turn_id && !_renderedTurnIds.has(data.turn_id)) { + _renderedTurnIds.add(data.turn_id); + chatUI.appendMessage(data.error ? 'error' : 'assistant', data.response); + } } } catch(e) { if (e.name === 'AbortError') return; // stopChat() already cleaned up + _optimisticMessages.delete(clientMessageId); + if ($msgEl.attr('data-message-id')) return; chatUI.closeStream(); chatUI.removeThinkingIndicator(thinkingId); $msgEl.remove(); @@ -5942,121 +5881,6 @@

Allowed setTimeout(refreshAgentState, 1000); } -async function syncLastChatTs() { - try { - const sid = await _getChatSessionId(); - const qs = sid ? `session_id=${encodeURIComponent(sid)}&limit=1` : `user_id=web_test&limit=1`; - const res = await fetch(`/api/agents/${AGENT_ID}/chat?${qs}`); - const data = await res.json(); - for (const e of (data.entries || [])) { - if (e.ts > lastChatTs) lastChatTs = e.ts; - } - } catch(e) { /* ignore */ } -} - -function pollForResponse(existingThinkingId) { - // Poll every 1s until a 'final' or 'error' entry arrives in JSONL. - // existingThinkingId: thinking bubble already shown (created eagerly in sendChat/restoreActiveReasoning). - if (!_currentTurn) return; // no active turn to poll for - if (_currentTurn.pollTimer) clearInterval(_currentTurn.pollTimer); - let thinkingId = existingThinkingId || null; - // Collect intermediate entries across poll ticks for SSE fallback rendering. - // If SSE fails to deliver activity entries (race: agent completed before SSE connected), - // these are replayed into the thinking bubble when 'final' arrives. - const collectedIntermediates = []; - const pollStartTs = lastChatTs; // snapshot cursor at poll start - _currentTurn.pollTimer = setInterval(async () => { - if (chatPolling) return; - chatPolling = true; - try { - // Sync thinkingId from _currentTurn in case turn_split created a new bubble - if (_currentTurn && _currentTurn.thinkingId && _currentTurn.thinkingId !== thinkingId) { - thinkingId = _currentTurn.thinkingId; - } - const sid = await _getChatSessionId(); - const qs = sid - ? `session_id=${encodeURIComponent(sid)}&after_ts=${lastChatTs}&limit=50` - : `user_id=web_test&after_ts=${lastChatTs}&limit=50`; - const res = await fetch(`/api/agents/${AGENT_ID}/chat?${qs}`); - const data = await res.json(); - const entries = data.entries || []; - if (!entries.length) return; // bubble already exists with spinner, just wait - console.warn('[pollForResponse] got %d entries afterTs=%s hasStream=%s _currentTurn=%s _finalRendered=%s', - entries.length, lastChatTs, chatUI.hasActiveStream(), !!_currentTurn, _currentTurn ? _currentTurn._finalRendered : 'n/a'); - const wasAtBottom = chatUI.isNearBottom(80); - let gotFinal = false; - for (const entry of entries) { - if (entry.ts <= lastChatTs) continue; - lastChatTs = entry.ts; - if (entry.type === 'final' || entry.type === 'error') { - console.warn('[pollForResponse] found %s entry ts=%s contentLen=%d', entry.type, entry.ts, (entry.content||'').length); - chatUI.closeStream(); - // SSE fallback: if SSE race-lost (agent finished before stream connected), - // the thinking bubble will be empty. Replay intermediates from JSONL history. - if (collectedIntermediates.length > 0) { - if (chatUI.getTimelineEntryCount(thinkingId) === 0) { - for (const ie of collectedIntermediates) { - if (ie.type === 'thinking') { - chatUI.appendTimelineEntry(thinkingId, {type: 'thinking', content: ie.content}); - } else if (ie.type === 'tool_call') { - chatUI.appendTimelineEntry(thinkingId, {type: 'tool_call', tool: ie.function, args: ie.params || {}, param_types: {}}); - } else if (ie.type === 'tool_output') { - let result; - try { result = JSON.parse(ie.content); } catch(e) { result = {data: ie.content}; } - chatUI.appendTimelineEntry(thinkingId, {type: 'tool_result', tool: ie.function, result, error: !!ie.error}); - } else if (ie.type === 'intermediate') { - chatUI.appendTimelineEntry(thinkingId, {type: 'response', content: ie.content}); - } - } - } - } - chatUI.finalizeThinkingBubble(thinkingId, (entry.metadata || {}).thinking_duration); - thinkingId = null; if (_currentTurn) _currentTurn.thinkingId = null; - const fMeta = entry.metadata || {}; - // Skip appendMessage if SSE onFinalResponse already rendered the bubble - if (_currentTurn && _currentTurn._finalRendered) { - _currentTurn._finalRendered = false; - } else if (fMeta.concurrency_limited || fMeta.busy_ack) { - chatUI.appendMessage('system', '[SYSTEM/Concurrency]\n' + entry.content, {metadata: fMeta}); - } else { - chatUI.appendMessage(entry.type === 'error' ? 'error' : 'assistant', - entry.content, {metadata: fMeta}); - } - gotFinal = true; - } else if (['thinking', 'tool_call', 'tool_output', 'intermediate'].includes(entry.type)) { - collectedIntermediates.push(entry); - // Render progressively if SSE isn't connected (SSE handles live rendering otherwise) - if (!chatUI.hasActiveStream()) { - if (entry.type === 'thinking') { - chatUI.appendTimelineEntry(thinkingId, {type: 'thinking', content: entry.content}); - } else if (entry.type === 'tool_call') { - chatUI.appendTimelineEntry(thinkingId, {type: 'tool_call', tool: entry.function, args: entry.params || {}, param_types: {}}); - } else if (entry.type === 'tool_output') { - let result; - try { result = JSON.parse(entry.content); } catch(e) { result = {data: entry.content}; } - chatUI.appendTimelineEntry(thinkingId, {type: 'tool_result', tool: entry.function, result, error: !!entry.error}); - } else if (entry.type === 'intermediate') { - chatUI.appendTimelineEntry(thinkingId, {type: 'response', content: entry.content}); - } - if (wasAtBottom) chatUI.scrollToBottom(); - } - } - // Other types are handled by SSE live stream; skip to avoid duplication - } - if (wasAtBottom) chatUI.scrollToBottom(); - if (gotFinal) { - console.warn('[pollForResponse] gotFinal=true _currentTurn=%s clearing pollTimer', !!_currentTurn); - clearInterval(_currentTurn.pollTimer); - _currentTurn.pollTimer = null; - setChatBusy(false); - setTimeout(refreshSessionState, 0); - setTimeout(refreshAgentState, 1000); - } - } catch(e) { console.error('[pollForResponse]', e); } - finally { chatPolling = false; } - }, 1000); -} - function escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; @@ -6183,7 +6007,6 @@

Allowed async function clearChat() { document.getElementById('chat-messages').innerHTML = ''; chatLoaded = false; - lastChatTs = 0; await fetch(`/api/agents/${AGENT_ID}/chat/clear`, {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({user_id: 'web_test'})}); // Clear summary panels (desktop + mobile) const emptySummary = '

No summary yet. Start chatting to generate one.

'; @@ -7564,7 +7387,10 @@

Clone AgentClone Agent lastChatTs) lastChatTs = entry.ts; const type = entry.type; entryCount++; if (type === 'user') { @@ -662,7 +657,11 @@

${esc(msg)}`; return; } + let realtimeCursor = Number(res.headers.get('X-Evonic-Realtime-Cursor') || 0); const data = await res.json(); if (data.error) return; @@ -945,6 +938,10 @@
ev.event === 'turn_begin'); - _beginSessionTurn(begin?.data?.ts || null, userMsgEl); - for (const ev of replayEvents) { - if (ev.seq > resumeSeq) resumeSeq = ev.seq; - _sessionTurn.lastSeq = Math.max(_sessionTurn.lastSeq, ev.seq || 0); - dispatchSessionEvent(ev.event, ev.data, sessionId); - } - restoredReasoning = true; - saveReasoningState(sessionId, resumeSeq); - chatUI.scrollToBottom(); - } - } - - // An immediate refresh can beat the first buffered SSE event. In that - // empty-buffer window, restore a placeholder only when this exact - // session is still the agent's active turn. - if (!restoredReasoning && ownsActiveTurn) { - const userMessages = container.querySelectorAll('[data-msg-role="user"]'); - const userMsgEl = userMessages.length ? userMessages[userMessages.length - 1] : null; - _beginSessionTurn(null, userMsgEl); - saveReasoningState(sessionId, resumeSeq); - restoredReasoning = true; - chatUI.scrollToBottom(); - } - } catch (_) { - // Treat failed recovery checks as stale state; live SSE can still - // create a fresh bubble when a subsequent event arrives. - } - if (!restoredReasoning) clearReasoningState(); - } - - // Connect SSE for real-time thinking events, then start polling. - connectSessionStream(sessionId, resumeSeq); - startPolling(); + // The durable stream restores queued/running state and any active telemetry. + connectSessionStream(sessionId, realtimeCursor); } function _renderSqliteMessages(messages, container) { @@ -1071,76 +1002,21 @@
0 && seq > _lastSeq + 1) { - console.warn(`[sse] gap detected ${evtName} seq=${seq} _lastSeq=${_lastSeq} gap=${seq-_lastSeq-1}`); - _fillingGap = true; - _gapQueue.push({evtName, data}); - fillGap(_lastSeq, seq).then(() => { - _fillingGap = false; - console.log(`[gap-fill] draining queue len=${_gapQueue.length}`); - while (_gapQueue.length > 0) { - const item = _gapQueue.shift(); - const itemSeq = item.data.seq || 0; - if (itemSeq && itemSeq <= _lastSeq) continue; - if (itemSeq) _lastSeq = itemSeq; - dispatchSessionEvent(item.evtName, item.data, sessionId); - } - }); return; } if (seq) _lastSeq = seq; if (seq) _sessionTurn.lastSeq = Math.max(_sessionTurn.lastSeq, seq); - saveReasoningState(sessionId, _lastSeq); - // LOG-B: count and log every event actually dispatched - _dispatchCount++; - console.log(`[sse] dispatch #${_dispatchCount} ${evtName} seq=${seq} _lastSeq=${_lastSeq} sseId=${!!_sseThinkingId}`); dispatch(data); } - for (const evtName of ['turn_begin', 'thinking', 'tool_call_started', 'tool_executed', 'state:changed', 'response_chunk', 'turn_split', 'whatsapp_restriction_warning', 'session_clear', 'approval_required', 'approval_resolved']) { + for (const evtName of ['agent_busy_changed', 'turn_queued', 'turn_begin', 'thinking', 'tool_call_started', 'tool_executed', 'state:changed', 'response_chunk', 'turn_split', 'whatsapp_restriction_warning', 'session_clear', 'approval_required', 'approval_resolved']) { es.addEventListener(evtName, e => { if (currentSessionId !== sessionId) return; const data = JSON.parse(e.data); - // Normalize raw SSE field names to match gap-fill/replay format so - // dispatchSessionEvent receives consistent field names regardless of path. + // Normalize legacy field aliases before dispatching. if (evtName === 'tool_call_started') { data.tool = data.tool_name || data.tool; data.args = data.tool_args || data.args; @@ -1237,6 +1064,17 @@
chatUI.scrollToBottom()); _pendingSlashResponse = payload.response; } - _sseCompleted = true; - // Reconnect so we're ready for the next incoming message - es.close(); - if (_sessionEs === es) { - _sessionEs = null; - const doneGen = _selectGeneration; - setTimeout(() => { - if (currentSessionId === sessionId && _selectGeneration === doneGen) connectSessionStream(sessionId, _sessionTurn.lastSeq); - }, 500); - } }); es.onerror = () => { - // SSE dropped — reconnect after a short delay if session is still active. - // Do NOT clear reasoning state here: the turn may still be running, - // and the saved state lets a page refresh recover properly. - // Do NOT finalize the thinking bubble — keep _sseThinkingId alive so the - // reconnected stream continues appending to the same bubble. - _pendingIntermediateResponse = null; - es.close(); - if (_sessionEs === es) { - _sessionEs = null; - setTimeout(() => { - if (currentSessionId === sessionId) connectSessionStream(sessionId, _sessionTurn.lastSeq); - }, 2000); - } + // Native EventSource reconnects with Last-Event-ID. The durable journal + // fills the disconnect window without replacing the active bubble. }; } @@ -1368,9 +1181,9 @@
({ error: err.responseJSON?.error || 'Request failed' })); } $btn.prop('disabled', false); if (res.error) { + _optimisticMessages.delete(clientMessageId); + if ($msgEl.attr('data-message-id')) return; _pendingReplyText = null; _pendingFileUpload = false; if (optimisticThinkingId && _sseThinkingId === optimisticThinkingId) { @@ -1733,7 +1552,7 @@