diff --git a/app.py b/app.py index 7ae5fa3b..22610307 100644 --- a/app.py +++ b/app.py @@ -330,7 +330,8 @@ def redirect_settings_to_system(subpath=None): # Smoke-test import (used by the updater to validate a new tree before # restarting): import all modules but skip channel/scheduler startup. _smoke_test = _os.environ.get('EVONIC_SMOKE_TEST') == '1' -if (not _reloader_active or _is_reloader_child) and not _smoke_test: +_testing = _os.environ.get('EVONIC_TESTING') == '1' +if (not _reloader_active or _is_reloader_child) and not _smoke_test and not _testing: # Run SYSTEM.md migration eagerly (not lazily on first GET /api/agents). # Agents that predate the on-disk SYSTEM.md feature need their file written # before they start processing messages — otherwise read_file("/_self/SYSTEM.md") diff --git a/backend/agent_runtime/__init__.py b/backend/agent_runtime/__init__.py index 51c154e2..e4bbc07c 100644 --- a/backend/agent_runtime/__init__.py +++ b/backend/agent_runtime/__init__.py @@ -167,9 +167,18 @@ def _send_free_notification(agent_id: str): from models.db import db notify_msg = "Hey! I'm done and ready to help again. Is there anything I can do?" + message_id = None try: - db.add_chat_message(session_id, 'assistant', notify_msg, - agent_id=agent_id, metadata={"free_notification": True}) + message_id = db.add_chat_message( + session_id, 'assistant', notify_msg, + agent_id=agent_id, metadata={"free_notification": True}, + ) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'final', 'session_id': session_id, 'content': notify_msg, + 'metadata': {'free_notification': True}, 'message_id': message_id, + }) except Exception as e: log.error("[AgentFreeNotify] Failed to save notification message: %s", e) @@ -181,6 +190,10 @@ def _send_free_notification(agent_id: str): 'session_id': session_id, 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': notify_msg, + 'message_id': message_id, + 'metadata': {"free_notification": True}, + 'role': 'assistant', }) except Exception as e: log.error("[AgentFreeNotify] Failed to emit message_received event: %s", e) diff --git a/backend/agent_runtime/llm_loop.py b/backend/agent_runtime/llm_loop.py index 012fe6fe..1e896a1a 100644 --- a/backend/agent_runtime/llm_loop.py +++ b/backend/agent_runtime/llm_loop.py @@ -419,6 +419,9 @@ def run_tool_loop(agent: Dict[str, Any], from backend.event_stream import event_stream from models.chatlog import chatlog_manager + def _message_id(value): + return value if type(value) in (int, str) else None + agent_id = agent['id'] db_agent_id = session_db_agent_id or agent_id # which per-agent DB owns this session external_user_id = agent_context.get('user_id') @@ -440,7 +443,6 @@ def run_tool_loop(agent: Dict[str, Any], _parent_agent_id = None _loop_ts = int(time.time() * 1000) chatlog.append({'type': 'turn_begin', 'session_id': session_id, 'ts': _loop_ts}) - event_stream.emit('turn_begin', {'session_id': session_id, 'ts': _loop_ts}) tool_trace = [] timeline = [] @@ -510,10 +512,13 @@ def _emit_task_lifecycle_event(event_name, task_ids): def _finalize_gate_response(response: str, source: str): duration = round(time.time() - _loop_start_time, 1) metadata = {'plugin_gate': source, 'thinking_duration': duration} - db.add_chat_message(session_id, 'assistant', response, - agent_id=db_agent_id, metadata=metadata) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', response, + agent_id=db_agent_id, metadata=metadata, + )) chatlog.append({'type': 'final', 'session_id': session_id, - 'content': response, 'metadata': metadata}) + 'content': response, 'metadata': metadata, + 'message_id': message_id}) chatlog.append({'type': 'turn_end', 'session_id': session_id, 'thinking_duration': duration}) event_stream.emit('final_answer', { @@ -949,11 +954,14 @@ def _get_agent_config_ig(agt_id: str) -> dict: _logger.info("Stop signal received during ATG execution for session %s", session_id) stop_msg = "Agent stopped by user request." _atg_stop_dur = round(time.time() - _loop_start_time, 1) - db.add_chat_message(session_id, 'assistant', stop_msg, agent_id=db_agent_id, - metadata={"timeline": timeline, "stopped": True, - "thinking_duration": _atg_stop_dur}) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', stop_msg, agent_id=db_agent_id, + metadata={"timeline": timeline, "stopped": True, + "thinking_duration": _atg_stop_dur}, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': stop_msg, - 'metadata': {'stopped': True, 'thinking_duration': _atg_stop_dur}}) + 'metadata': {'stopped': True, 'thinking_duration': _atg_stop_dur}, + 'message_id': message_id}) chatlog.append({'type': 'turn_end', 'session_id': session_id, 'thinking_duration': _atg_stop_dur}) event_stream.emit('final_answer', { @@ -1183,10 +1191,14 @@ def _get_agent_config_ig(agt_id: str) -> dict: _logger.info("Stop signal received for session %s — aborting loop", session_id) stop_msg = "Agent stopped by user request." _stop_dur = round(time.time() - _loop_start_time, 1) - db.add_chat_message(session_id, 'assistant', stop_msg, agent_id=db_agent_id, - metadata={"timeline": timeline, "stopped": True, "thinking_duration": _stop_dur}) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', stop_msg, agent_id=db_agent_id, + metadata={"timeline": timeline, "stopped": True, + "thinking_duration": _stop_dur}, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': stop_msg, - 'metadata': {'stopped': True, 'thinking_duration': _stop_dur}}) + 'metadata': {'stopped': True, 'thinking_duration': _stop_dur}, + 'message_id': message_id}) _stop_inj = ("[SYSTEM] Your previous reasoning and response were forcefully " "interrupted by the user via /stop before completion. " "Await the user's next instruction.") @@ -1983,9 +1995,12 @@ def _get_agent_config_ig(agt_id: str) -> dict: 'content': _display_content, 'is_final': True, 'send_as_message': True, }) - db.add_chat_message(session_id, 'assistant', _display_content, agent_id=db_agent_id, metadata=meta) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', _display_content, + agent_id=db_agent_id, metadata=meta, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': _display_content, - 'metadata': _cl_meta}) + 'metadata': _cl_meta, 'message_id': message_id}) chatlog.append({'type': 'turn_end', 'session_id': session_id, 'thinking_duration': _final_dur}) # Archive sub-agent session at turn-end — single-turn only. Explorer & # kb-organizer are single-shot, so they archive on completion (no need to @@ -2344,13 +2359,8 @@ def _normalize(s): }) # Escalation: ensure a human can see the approval. - # We always fan-out to BOTH web SSE AND messaging channels, - # because has_web_listener() is unreliable — it only checks - # listener registration, not actual SSE delivery. If SSE - # disconnects and reconnects, the approval event may already - # be gone from the ring buffer. Web SSE delivers the approval - # modal in the browser; messaging channels deliver a fallback - # notification via Telegram/WhatsApp. + # Always fan out to BOTH durable web SSE and messaging channels. + # Messaging remains the out-of-browser fallback. # List of (session_id, external_user_id, channel_id) that received # approval_required — used to fan-out approval_resolved to all of them. _escalation_targets: list = [] @@ -2385,31 +2395,8 @@ def _normalize(s): }) _escalation_targets.append((_human_session_id, _web_uid, _web_cid)) - # Channel (Telegram/WhatsApp): always notify via the super - # agent's messaging channel as a fallback. We no longer gate - # this behind has_web_listener() because the registration - # may exist while the SSE connection is not delivering. - - # Verify web SSE delivery with heartbeat-aware check. - # has_web_listener() only confirms a callback is registered; - # this confirms the SSE connection is actually sending heartbeats. - _web_sse_active = False - try: - from routes.realtime import has_active_web_sse - _web_sse_active = ( - has_active_web_sse(session_id) or - (_human_session_id and has_active_web_sse(_human_session_id)) - ) - except Exception: - pass # routes.realtime may not be importable in all contexts - - if not _web_sse_active: - _logger.info( - "approval %s: web SSE appears inactive for session %s " - "(heartbeat not received within window) — relying on " - "messaging channel fallback", - pending.approval_id, session_id, - ) + # Channel (Telegram/WhatsApp) remains an unconditional + # fallback; browser connection liveness is not authoritative. _super = db.get_super_agent() if _super and _super['id'] != agent_id: _su_uid, _su_cid = _resolve_agent_target(_super['id']) @@ -2812,10 +2799,14 @@ def _normalize(s): _logger.info("Stop signal received for session %s — aborting after tools", session_id) stop_msg = "Agent stopped by user request." _stopb_dur = round(time.time() - _loop_start_time, 1) - db.add_chat_message(session_id, 'assistant', stop_msg, agent_id=db_agent_id, - metadata={"timeline": timeline, "stopped": True, "thinking_duration": _stopb_dur}) + message_id = _message_id(db.add_chat_message( + session_id, 'assistant', stop_msg, agent_id=db_agent_id, + metadata={"timeline": timeline, "stopped": True, + "thinking_duration": _stopb_dur}, + )) chatlog.append({'type': 'final', 'session_id': session_id, 'content': stop_msg, - 'metadata': {'stopped': True, 'thinking_duration': _stopb_dur}}) + 'metadata': {'stopped': True, 'thinking_duration': _stopb_dur}, + 'message_id': message_id}) _stopb_inj = ("[SYSTEM] Your previous reasoning and response were forcefully " "interrupted by the user via /stop before completion. " "Await the user's next instruction.") diff --git a/backend/agent_runtime/notifier.py b/backend/agent_runtime/notifier.py index 73397cb6..32df1f6e 100644 --- a/backend/agent_runtime/notifier.py +++ b/backend/agent_runtime/notifier.py @@ -190,10 +190,17 @@ def notify_agent(agent_id: str, tag: str, message: str, ) else: meta = dict(metadata) if metadata else {} - db.add_chat_message( + message_id = db.add_chat_message( target_session_id, role='user', content=full_message, agent_id=_db_agent_id, metadata=meta if meta else None, ) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(_db_agent_id, target_session_id).append({ + 'type': 'user', 'session_id': target_session_id, + 'content': full_message, 'metadata': meta, + 'message_id': message_id, + }) from backend.event_stream import event_stream event_stream.emit('message_received', { 'agent_id': agent_id, @@ -202,6 +209,8 @@ def notify_agent(agent_id: str, tag: str, message: str, 'channel_id': channel_id, 'message': full_message, 'metadata': meta, + 'message_id': message_id, + 'role': 'user', }) if deliver_external and channel_id: from backend.channels.registry import channel_manager diff --git a/backend/agent_runtime/runtime.py b/backend/agent_runtime/runtime.py index cff9c16f..487a7b9e 100644 --- a/backend/agent_runtime/runtime.py +++ b/backend/agent_runtime/runtime.py @@ -16,6 +16,7 @@ import queue import threading import traceback +import uuid from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from contextlib import contextmanager @@ -101,7 +102,6 @@ def _append_attachment_context(content: str, attachment_infos, attachment_info, WORKER_JOIN_TIMEOUT_SECONDS = 5.0 # Max time to wait for worker threads to finish on shutdown WORKER_JOIN_MIN_TIMEOUT = 0.1 # Minimum timeout per worker join iteration (seconds) DEFAULT_BUFFER_SECONDS = 2 # Default message buffering delay when agent has no config (seconds) -SESSION_BUFFER_CLEANUP_DELAY = 30.0 # Delay before cleaning up SSE session buffers (seconds) def _llm_log_path(agent_id: str) -> str: @@ -309,6 +309,7 @@ class SessionContext: external_user_id: str channel_id: Optional[str] = None session_db_agent_id: Optional[str] = None + turn_id: str = field(default_factory=lambda: uuid.uuid4().hex) class _QueueTask: @@ -340,12 +341,11 @@ def __init__(self, agent: dict, ctx: SessionContext, # (b) _session_store._stop_flags_guard # (c) _session_store._inject_queues_guard # (d) _session_store._busy_guard -# (e) _agent_tracker._guard -# (f) _cleanup_tracker._guard -# (g) _llm_serializer._summarize_guard -# (h) _llm_serializer._llm_lock -# (i) _shutdown_mgr._lock -# (j) instance._buffer_lock +# (e) _cleanup_tracker._guard +# (f) _llm_serializer._summarize_guard +# (g) _llm_serializer._llm_lock +# (h) _shutdown_mgr._lock +# (i) instance._buffer_lock # # 2. GUARD-LOCK PATTERN: Each mutable dict has a dedicated "guard" lock. # The guard protects structuring operations (get-or-create, pop, clear). @@ -362,8 +362,6 @@ def __init__(self, agent: dict, ctx: SessionContext, # 5. INVARIANTS: # • Every session_id present in _cleanup_tracker._ttl MUST also have # entries in _session_store (or be in the process of being cleaned up). -# • _agent_tracker._busy[agent_id] exists only while an agent is -# actively processing a turn; cleared on completion or TTL expiry. # • _shutdown_mgr._event, once set, is never cleared (shutdown is final). # # ───────────────────────────────────────────────────────────────────────────── @@ -449,41 +447,6 @@ def __init__(self) -> None: self._busy_guard = threading.Lock() -class _AgentTracker: - """Track which agents are currently busy processing a session. - - Thread-safety: - The _guard lock protects all reads and writes to the _busy dict, - which is mutated by worker threads when agents start or finish - processing turns. - - Acquired by: _set_agent_busy(), _clear_agent_busy(), - is_agent_busy(), get_busy_agents(). - Released: immediately after the dict operation (short critical - section — < 1ms). - - Deadlock risk: NONE — _guard is never nested with any other lock. - It is acquired independently each time. - - TTL staleness: entries older than the TTL (default 600s) are - treated as stale and auto-expired. This protects against hung - threads that never clear their busy flag. - - Invariants: - • An agent_id appears in _busy only while it is actively - processing an LLM turn. - • Each entry has {session_id: str, started_at: float}. - • At most one entry per agent_id (set overwrites). - """ - - def __init__(self) -> None: - # agent_id -> {session_id: str, started_at: float} - # Guarded by _guard — prevents races between agent_busy set/clear - # calls coming from different worker threads. - self._busy: Dict[str, dict] = {} - self._guard = threading.Lock() - - class _CleanupTracker: """TTL-based idle-session cleanup: session timestamps + periodic timer. @@ -639,9 +602,8 @@ def __init__(self) -> None: class AgentRuntime: - # State containers — reduce class-level attributes from 23 to 6 + # State containers — reduce class-level attributes from 23 to 5 _session_store = _SessionStore() - _agent_tracker = _AgentTracker() _cleanup_tracker = _CleanupTracker() _llm_serializer = _LLMSerializer() _shutdown_mgr = _ShutdownManager() @@ -753,6 +715,9 @@ def _touch_session(cls, session_id: str) -> None: """Mark session as active (called on every turn).""" with cls._cleanup_tracker._guard: cls._cleanup_tracker._ttl[session_id] = time.time() + if (os.environ.get('EVONIC_TESTING') == '1' + or os.environ.get('EVONIC_SMOKE_TEST') == '1'): + return if cls._cleanup_tracker._timer is None: cls._cleanup_tracker._timer = threading.Timer(cls._cleanup_tracker._interval, cls._cleanup_idle_sessions) cls._cleanup_tracker._timer.daemon = True @@ -765,28 +730,41 @@ def __init__(self): self._buffer_timers: Dict[str, threading.Timer] = {} self._buffer_lock = threading.Lock() self._workers: list[threading.Thread] = [] - # Read worker count from DB (user-configurable), fall back to config default - try: - from models.db import db as _db - _db_workers = _db.get_setting('agent_queue_workers') - initial_workers = max(1, min(32, int(_db_workers))) if _db_workers else AGENT_QUEUE_WORKERS - except Exception: - initial_workers = AGENT_QUEUE_WORKERS - for i in range(initial_workers): - t = threading.Thread(target=self._worker, name=f'agent-worker-{i}', daemon=True) - t.start() - self._workers.append(t) - _logger.info("Started %d queue worker(s)", initial_workers) + background_enabled = not ( + os.environ.get('EVONIC_TESTING') == '1' + or os.environ.get('EVONIC_SMOKE_TEST') == '1' + ) + if background_enabled: + try: + from backend.realtime_store import realtime_store + interrupted = realtime_store.interrupt_stale_turns() + if interrupted: + _logger.warning("Marked %d turn(s) interrupted after restart", len(interrupted)) + except Exception as exc: + _logger.error("Failed to recover durable active turns: %s", exc) + # Read worker count from DB (user-configurable), fall back to config default + try: + from models.db import db as _db + _db_workers = _db.get_setting('agent_queue_workers') + initial_workers = max(1, min(32, int(_db_workers))) if _db_workers else AGENT_QUEUE_WORKERS + except Exception: + initial_workers = AGENT_QUEUE_WORKERS + for i in range(initial_workers): + t = threading.Thread(target=self._worker, name=f'agent-worker-{i}', daemon=True) + t.start() + self._workers.append(t) + _logger.info("Started %d queue worker(s)", initial_workers) AgentRuntime._llm_serializer._concurrency_mgr = ConcurrencyManager() self._session_skill_mds: Dict[str, Dict[str, str]] = {} # session_id -> {skill_id: system_md} self._session_skill_tools: Dict[str, Dict[str, list]] = {} # session_id -> {skill_id: [tool_defs]} self._prefetcher = TurnPrefetcher() # pre-loads messages for next turn - # Register signal handlers + atexit for graceful shutdown (once only) - AgentRuntime._register_signal_handlers() - atexit.register(self._atexit_shutdown) # Schedule periodic cleanup of stale buffer timers self._buffer_timer_stats = {"created": 0, "cancelled": 0, "leaked": 0} - self._stale_timer_cleanup() + if background_enabled: + # Register signal handlers + atexit for graceful shutdown (once only) + AgentRuntime._register_signal_handlers() + atexit.register(self._atexit_shutdown) + self._stale_timer_cleanup() def _atexit_shutdown(self) -> None: AgentRuntime.graceful_shutdown() @@ -863,7 +841,7 @@ def _worker(self) -> None: ) except Exception as e: _logger.error("Channel send error for session %s: %s", task.ctx.session_id, e) - elif task.send_via_channel: + elif task.send_via_channel and not result.get('stopped'): # DIAGNOSTIC (shared-channel reply loss): the reply was generated # and saved (so it shows in the web session) but the channel send # was skipped. Log exactly which precondition failed. @@ -874,6 +852,7 @@ def _worker(self) -> None: not bool(_resp), _resp == "(No response)", task.ctx.channel_id) except Exception as e: _logger.error("Worker error for session %s: %s", task.ctx.session_id, e, exc_info=True) + self._fail_queued_task(task, str(e)) task.result = { "response": "An unexpected error occurred. Please try again.", "error": True, @@ -926,50 +905,79 @@ def _is_busy(self, session_id: str) -> bool: with self._session_store._busy_guard: return self._session_store._busy.get(session_id, False) - def _set_agent_busy(self, agent_id: str, session_id: str) -> None: - with self._agent_tracker._guard: - self._agent_tracker._busy[agent_id] = {'session_id': session_id, 'started_at': time.time()} + def _mark_task_queued(self, task: '_QueueTask') -> None: + from backend.realtime_store import realtime_store + # ponytail: one runtime lock preserves queue/stop event order; use + # per-session locks only if message-ingress contention is measured. + with self._buffer_lock: + turn_id, _created = realtime_store.queue_turn( + task.agent['id'], task.ctx.session_id, task.ctx.turn_id, + ) + task.ctx.turn_id = turn_id + # Queue insertion is the acceptance point: a stop racing before it + # applies to the old turn; a stop racing after it can cancel this row. + self._get_stop_event(task.ctx.session_id).clear() + from backend.tools.lib.process_tracker import process_tracker + process_tracker.clear_stop(task.ctx.session_id) + + def _put_task(self, task: '_QueueTask', *, already_queued: bool = False) -> None: + if not already_queued: + self._mark_task_queued(task) + try: + self._message_queue.put(task) + except Exception: + self._fail_queued_task(task, 'queue_failed') + raise - def _clear_agent_busy(self, agent_id: str) -> None: - with self._agent_tracker._guard: - self._agent_tracker._busy.pop(agent_id, None) + def _fail_queued_task(self, task: '_QueueTask', reason: str) -> None: + from backend.realtime_store import realtime_store + with self._buffer_lock: + cancelled = realtime_store.cancel_queued_turns( + task.ctx.session_id, task.ctx.turn_id, + ) + if not cancelled: + return + self._emit_cancelled_turn(cancelled[0], reason, task) + + def _emit_cancelled_turn(self, turn: dict, reason: str, + task: '_QueueTask | None' = None) -> None: + from backend.realtime_store import realtime_store + agent_id = turn['agent_id'] + session_id = turn['session_id'] + event_stream.emit('turn_complete', { + 'agent_id': agent_id, + 'agent_name': task.agent.get('name', '') if task else '', + 'session_id': session_id, + 'external_user_id': task.ctx.external_user_id if task else '', + 'channel_id': task.ctx.channel_id if task else None, + 'turn_id': turn['turn_id'], + 'response': '', + 'tool_trace': [], + 'is_error': True, + 'interrupted': True, + 'reason': reason, + }) + realtime_store.finish_turn(turn['turn_id']) + remaining = realtime_store.busy_agents().get(agent_id) + event_stream.emit('agent_busy_changed', { + 'agent_id': agent_id, + 'busy': bool(remaining), + 'session_id': remaining.get('session_id', session_id) if remaining else session_id, + 'session_ids': remaining.get('session_ids', []) if remaining else [], + 'active_count': remaining.get('active_count', 0) if remaining else 0, + 'state': remaining.get('state', 'idle') if remaining else 'idle', + 'turn_id': None, + }) def is_agent_busy(self, agent_id: str, ttl: int = 600) -> bool: - """Return True if agent is currently processing an LLM turn. - - A TTL guard treats entries older than `ttl` seconds as stale (e.g. a - thread that hung and never cleared its flag). Default is 10 minutes. - """ - with self._agent_tracker._guard: - entry = self._agent_tracker._busy.get(agent_id) - if not entry: - return False - if time.time() - entry['started_at'] > ttl: - # Auto-expire stale entry - self._clear_agent_busy(agent_id) - return False - return True + """Return durable queued/running state. ``ttl`` is kept for callers.""" + from backend.realtime_store import realtime_store + return bool(realtime_store.active_turns(agent_id=agent_id)) def get_busy_agents(self, ttl: int = 600) -> dict: - """Return a snapshot of all currently busy agents (respects TTL).""" - now = time.time() - with self._agent_tracker._guard: - snapshot = dict(self._agent_tracker._busy) - result = {} - stale = [] - for agent_id, entry in snapshot.items(): - elapsed = now - entry['started_at'] - if elapsed > ttl: - stale.append(agent_id) - else: - result[agent_id] = { - 'session_id': entry['session_id'], - 'started_at': entry['started_at'], - 'elapsed': round(elapsed, 1), - } - for agent_id in stale: - self._clear_agent_busy(agent_id) - return result + """Return all durable queued/running turns grouped by agent.""" + from backend.realtime_store import realtime_store + return realtime_store.busy_agents() @contextmanager def _buffer_timer(self, session_id: str, buffer_seconds: float, @@ -1013,11 +1021,14 @@ def request_stop(self, session_id: str) -> None: """Signal the agent loop for this session to stop after the current LLM call. Also cancels any pending buffer timer so no new task is enqueued. Kills any running tool subprocess immediately via process_tracker.""" + from backend.realtime_store import realtime_store with self._buffer_lock: + self._get_stop_event(session_id).set() timer = self._buffer_timers.pop(session_id, None) - if timer is not None: - timer.cancel() - self._get_stop_event(session_id).set() + if timer is not None: + timer.cancel() + for turn in realtime_store.cancel_queued_turns(session_id): + self._emit_cancelled_turn(turn, 'stopped') # Kill any running tool subprocess for this session from backend.tools.lib.process_tracker import process_tracker process_tracker.kill(session_id) @@ -1043,7 +1054,8 @@ def summarize_session(self, agent: dict, session_id: str) -> bool: def _run_bash_exec(self, agent: Dict[str, Any], session_id: str, db_agent_id: str, external_user_id: str, - message: str) -> str: + message: str, client_message_id: str | None = None + ) -> tuple[str, int | str | None, int | str | None]: """Run a web user's "!" directly and persist it for UI display only. The command and its output are saved with a `bash_exec` metadata flag so @@ -1054,7 +1066,7 @@ def _run_bash_exec(self, agent: Dict[str, Any], session_id: str, """ cmd = message.lstrip()[1:].strip() if not cmd: - return "Usage: `!` — run a shell command directly (web only)." + return "Usage: `!` — run a shell command directly (web only).", None, None from backend.tools import bash exec_agent = {**agent, 'session_id': session_id, '_skip_safety': True} @@ -1082,19 +1094,39 @@ def _run_bash_exec(self, agent: Dict[str, Any], session_id: str, response += f"\n_(exit code {exit_code})_" # Persist for UI display only — hidden from LLM via the `bash_exec` flag. - _db_retry(db.add_chat_message, session_id, 'user', message, - agent_id=db_agent_id, metadata={'bash_exec': True}, - label="save bash command") - _db_retry(db.add_chat_message, session_id, 'assistant', response, - agent_id=db_agent_id, metadata={'bash_exec': True}, - label="save bash output") + command_meta = {'bash_exec': True} + if client_message_id: + command_meta['client_message_id'] = client_message_id + message_id = _db_retry( + db.add_chat_message, session_id, 'user', message, + agent_id=db_agent_id, metadata=command_meta, + label="save bash command", + ) + response_id = _db_retry( + db.add_chat_message, session_id, 'assistant', response, + agent_id=db_agent_id, metadata={'bash_exec': True}, + label="save bash output", + ) + message_id = message_id if type(message_id) in (int, str) else None + response_id = response_id if type(response_id) in (int, str) else None _cl = chatlog_manager.get(db_agent_id, session_id) _cl.append({'type': 'user', 'session_id': session_id, 'content': message, - 'sender_id': external_user_id, 'metadata': {'bash_exec': True}}) + 'sender_id': external_user_id, 'metadata': command_meta, + 'message_id': message_id}) _cl.append({'type': 'system', 'session_id': session_id, 'content': response, - 'metadata': {'bash_exec': True}}) + 'metadata': {'bash_exec': True}, 'message_id': response_id}) + for role, content, saved_id, meta in ( + ('user', message, message_id, command_meta), + ('assistant', response, response_id, {'bash_exec': True})): + event_stream.emit('message_received', { + 'agent_id': agent['id'], 'session_id': session_id, + 'external_user_id': external_user_id, 'message': content, + 'message_id': saved_id, + 'client_message_id': meta.get('client_message_id'), + 'metadata': meta, 'role': role, + }) self._prefetcher.invalidate(session_id) - return response + return response, message_id, response_id def handle_message(self, agent_id: str, external_user_id: str, message: str, channel_id: Optional[str] = None, @@ -1187,11 +1219,15 @@ def handle_message(self, agent_id: str, external_user_id: str, # On a channel, a "!"-prefixed message falls through as ordinary user text. if message.lstrip().startswith('!') and channel_id is None \ and agent.get('bash_exec_enabled'): - response = self._run_bash_exec( + response, message_id, response_id = self._run_bash_exec( agent, session_id, db_agent_id, external_user_id, message, + (metadata or {}).get('client_message_id'), ) return {"response": response, "tool_trace": [], "timeline": [], - "slash_command": True, "bash_exec": True} + "slash_command": True, "bash_exec": True, + "message_id": message_id, + "response_message_id": response_id, + "client_message_id": (metadata or {}).get('client_message_id')} # Slash command interception — execute before saving message or sending to LLM parsed = parse_command(message) @@ -1203,23 +1239,49 @@ def handle_message(self, agent_id: str, external_user_id: str, ) if response is not None: # Command was recognized — save command echo and response, then return - _db_retry(db.add_chat_message, session_id, 'user', message, - agent_id=db_agent_id, metadata={"slash_command": True}, - label="save command message") - _db_retry(db.add_chat_message, session_id, 'assistant', response, - agent_id=db_agent_id, metadata={"slash_command": True}, - label="save command response") + command_meta = {"slash_command": True} + if metadata and metadata.get('client_message_id'): + command_meta['client_message_id'] = metadata['client_message_id'] + message_id = _db_retry( + db.add_chat_message, session_id, 'user', message, + agent_id=db_agent_id, metadata=command_meta, + label="save command message", + ) + response_id = _db_retry( + db.add_chat_message, session_id, 'assistant', response, + agent_id=db_agent_id, metadata={"slash_command": True}, + label="save command response", + ) _cl = chatlog_manager.get(db_agent_id, session_id) _cl.append({'type': 'user', 'session_id': session_id, 'content': message, 'sender_id': external_user_id, - 'metadata': {'slash_command': True}}) + 'metadata': command_meta, 'message_id': message_id}) _cl.append({'type': 'system', 'session_id': session_id, 'content': response, - 'metadata': {'slash_command': True}}) + 'metadata': {'slash_command': True}, 'message_id': response_id}) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': message, 'message_id': message_id, + 'client_message_id': command_meta.get('client_message_id'), + 'metadata': command_meta, 'role': 'user', + }) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': response, 'message_id': response_id, + 'metadata': {'slash_command': True}, 'role': 'assistant', + }) # Signal the client to clear the chat UI when the clear command was used extra = {"clear_ui": True} if cmd_name == "clear" else {} extra["slash_command"] = True # flag so frontend skips thinking bubble self._prefetcher.invalidate(session_id) - return {"response": response, "tool_trace": [], "timeline": [], **extra} + return { + "response": response, "tool_trace": [], "timeline": [], + "message_id": message_id, + "response_message_id": response_id, + "client_message_id": command_meta.get('client_message_id'), + **extra, + } # Unknown command — fall through to normal LLM processing # Save user message (store image reference and any extra metadata) @@ -1274,11 +1336,15 @@ def handle_message(self, agent_id: str, external_user_id: str, meta['agent_message'] = True meta['from_agent_id'] = sender_id meta['from_agent_name'] = sender_agent.get('name', sender_id) if sender_agent else sender_id - _db_retry(db.add_chat_message, session_id, 'user', message or "[Image]", - agent_id=db_agent_id, metadata=meta if meta else None, label="save user message") + message_id = _db_retry( + db.add_chat_message, session_id, 'user', message or "[Image]", + agent_id=db_agent_id, metadata=meta if meta else None, + label="save user message", + ) _cl_user = chatlog_manager.get(db_agent_id, session_id) _cl_user_entry = {'type': 'user', 'session_id': session_id, - 'content': message or '[Image]', 'sender_id': external_user_id} + 'content': message or '[Image]', 'sender_id': external_user_id, + 'message_id': message_id} if meta: _cl_user_entry['metadata'] = meta _cl_user.append(_cl_user_entry) @@ -1299,6 +1365,9 @@ def handle_message(self, agent_id: str, external_user_id: str, 'audio_url': audio_url, 'video_url': video_url, 'metadata': meta, + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'role': 'user', }) # A plain human reply in the exact originating session resumes the @@ -1322,6 +1391,8 @@ def handle_message(self, agent_id: str, external_user_id: str, "tool_trace": [], "timeline": [], "escalation_routed": routed, + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), } # Busy-ack: if the agent-level concurrency gate is saturated, send an @@ -1345,14 +1416,24 @@ def handle_message(self, agent_id: str, external_user_id: str, ) _ack_meta = {"busy_ack": True, "concurrency_limited": True, "concurrency_active": _cap["active"], "concurrency_max": _cap["max"]} - _db_retry(db.add_chat_message, session_id, 'assistant', _ack_text, - agent_id=db_agent_id, metadata=_ack_meta, - label="save busy ack") + _ack_id = _db_retry( + db.add_chat_message, session_id, 'assistant', _ack_text, + agent_id=db_agent_id, metadata=_ack_meta, + label="save busy ack", + ) + _ack_id = _ack_id if type(_ack_id) in (int, str) else None chatlog_manager.get(db_agent_id, session_id).append({ 'type': 'final', 'session_id': session_id, 'content': _ack_text, 'metadata': _ack_meta, + 'message_id': _ack_id, + }) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': _ack_text, 'message_id': _ack_id, + 'metadata': _ack_meta, 'role': 'assistant', }) event_stream.emit('concurrency_limited', { 'agent_id': agent_id, @@ -1387,9 +1468,11 @@ def handle_message(self, agent_id: str, external_user_id: str, # in a DIFFERENT session, reject this message with a contextual explanation. # Check focus first (requires DB read) only when agent-level busy is confirmed. if agent.get('enable_agent_state') and self.is_agent_busy(agent_id): - with self._agent_tracker._guard: - busy_entry = self._agent_tracker._busy.get(agent_id) - if busy_entry and busy_entry['session_id'] != session_id: + from backend.realtime_store import realtime_store + busy_sessions = { + turn['session_id'] for turn in realtime_store.active_turns(agent_id=agent_id) + } + if busy_sessions and session_id not in busy_sessions: ms = self._restore_agent_state(agent_id) if ms and ms.focus: busy_msg = self._handle_busy_rejection( @@ -1416,7 +1499,11 @@ def handle_message(self, agent_id: str, external_user_id: str, 'channel_id': channel_id, 'message': message, }) - return {"response": None, "injected": True, "tool_trace": [], "timeline": []} + return { + "response": None, "injected": True, "tool_trace": [], "timeline": [], + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), + } # Message buffering: debounce rapid messages, then queue # Skip when skip_buffer=True (e.g. API routes need synchronous response) @@ -1429,6 +1516,7 @@ def handle_message(self, agent_id: str, external_user_id: str, task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id, session_db_agent_id=db_agent_id if is_subagent else None), send_via_channel=True) + self._mark_task_queued(task) timer = threading.Timer(buffer_seconds, self._enqueue_buffered, args=(task,)) timer.daemon = True with self._buffer_lock: @@ -1441,8 +1529,14 @@ def handle_message(self, agent_id: str, external_user_id: str, # If start() fails, cancel the timer and remove it from the dict with self._buffer_lock: self._buffer_timers.pop(session_id, None) + self._fail_queued_task(task, 'buffer_timer_failed') raise - return {"response": None, "buffered": True, "tool_trace": [], "timeline": []} + return { + "response": None, "buffered": True, "tool_trace": [], "timeline": [], + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), + "turn_id": task.ctx.turn_id, + } # Inter-agent messages: fire-and-forget (don't block the sender's worker thread). # The sub-agent/target processes asynchronously and results are delivered via @@ -1455,16 +1549,27 @@ def handle_message(self, agent_id: str, external_user_id: str, task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id, session_db_agent_id=db_agent_id if is_subagent else None), send_via_channel=False) - self._message_queue.put(task) - return {"response": None, "async": True, "tool_trace": [], "timeline": []} + self._put_task(task) + return { + "response": None, "async": True, "tool_trace": [], "timeline": [], + "message_id": message_id, + "client_message_id": meta.get('client_message_id'), + "turn_id": task.ctx.turn_id, + } # No buffering — queue immediately and wait for result task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id, session_db_agent_id=db_agent_id if is_subagent else None), send_via_channel=bool(channel_id)) - self._message_queue.put(task) + self._put_task(task) task.event.wait() - return task.result + result = task.result or {} + result.update({ + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'turn_id': task.ctx.turn_id, + }) + return result def _enqueue_buffered(self, task: '_QueueTask') -> None: """Queue a buffered task, cleaning up its timer even on failure.""" @@ -1474,7 +1579,7 @@ def _enqueue_buffered(self, task: '_QueueTask') -> None: except Exception: pass # Timer may already be gone; the context manager handles cleanup try: - self._message_queue.put(task) + self._put_task(task, already_queued=True) except Exception: _logger.error("Failed to enqueue buffered task for session %s: %s", task.ctx.session_id, traceback.format_exc()) @@ -1503,12 +1608,28 @@ def _process_and_respond(self, agent: dict, ctx: SessionContext) -> dict: def _do_process(self, agent: dict, ctx: SessionContext) -> dict: """Internal: build messages and call LLM (must hold session lock).""" agent_id = agent['id'] + from backend.realtime_store import realtime_store + if not realtime_store.start_turn(ctx.turn_id): + return { + 'response': None, 'stopped': True, + 'tool_trace': [], 'timeline': [], + } self._set_busy(ctx.session_id, True) - self._set_agent_busy(agent_id, ctx.session_id) + active = realtime_store.busy_agents().get(agent_id, {}) event_stream.emit('agent_busy_changed', { 'agent_id': agent_id, 'busy': True, 'session_id': ctx.session_id, + 'session_ids': active.get('session_ids', [ctx.session_id]), + 'active_count': active.get('active_count', 1), + 'state': 'running', + 'turn_id': ctx.turn_id, + }) + event_stream.emit('turn_begin', { + 'agent_id': agent_id, + 'session_id': ctx.session_id, + 'turn_id': ctx.turn_id, + 'ts': int(time.time() * 1000), }) _turn_start = time.time() _turn_complete_emitted = False @@ -1542,10 +1663,8 @@ def _do_process(self, agent: dict, ctx: SessionContext) -> dict: 'tool_trace': [], 'is_error': True, 'thinking_duration': _err_dur, + 'turn_id': ctx.turn_id, }) - self._bg_executor.submit( - lambda sid=ctx.session_id: (time.sleep(SESSION_BUFFER_CLEANUP_DELAY), event_stream.cleanup_session_buffer(sid)), - ) result = { "response": "An unexpected error occurred. Please try again.", "error": True, @@ -1564,11 +1683,16 @@ def _do_process(self, agent: dict, ctx: SessionContext) -> dict: return result finally: self._set_busy(ctx.session_id, False) - self._clear_agent_busy(agent_id) + realtime_store.finish_turn(ctx.turn_id) + remaining = realtime_store.busy_agents().get(agent_id) event_stream.emit('agent_busy_changed', { 'agent_id': agent_id, - 'busy': False, - 'session_id': ctx.session_id, + 'busy': bool(remaining), + 'session_id': remaining.get('session_id', ctx.session_id) if remaining else ctx.session_id, + 'session_ids': remaining.get('session_ids', []) if remaining else [], + 'active_count': remaining.get('active_count', 0) if remaining else 0, + 'state': remaining.get('state', 'idle') if remaining else 'idle', + 'turn_id': None, }) # Drain any messages that arrived in the injection queue just as the loop # was finishing (race between _is_busy check and loop exit). They are @@ -1583,8 +1707,12 @@ def _do_process(self, agent: dict, ctx: SessionContext) -> dict: if orphaned: _logger.warning("%d orphaned injected message(s) for %s — re-processing as new turn", len(orphaned), ctx.session_id) - task = _QueueTask(agent, ctx, send_via_channel=bool(ctx.channel_id)) - self._message_queue.put(task) + next_ctx = SessionContext( + ctx.session_id, ctx.external_user_id, ctx.channel_id, + ctx.session_db_agent_id, + ) + task = _QueueTask(agent, next_ctx, send_via_channel=bool(ctx.channel_id)) + self._put_task(task) def _check_evonet_offline(self, agent: dict, ctx: SessionContext): """Return a completed turn result dict if the agent's Tunnel Workplace is offline, @@ -1613,13 +1741,17 @@ def _check_evonet_offline(self, agent: dict, ctx: SessionContext): ) db_agent_id = ctx.session_db_agent_id or agent['id'] - _db_retry(db.add_chat_message, ctx.session_id, 'assistant', reply, - agent_id=db_agent_id, metadata={'evonet_offline': True}, - label="save evonet offline reply") + message_id = _db_retry( + db.add_chat_message, ctx.session_id, 'assistant', reply, + agent_id=db_agent_id, metadata={'evonet_offline': True}, + label="save evonet offline reply", + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog_manager.get(db_agent_id, ctx.session_id).append({ 'type': 'final', 'session_id': ctx.session_id, 'content': reply, 'metadata': {'evonet_offline': True}, + 'message_id': message_id, }) if ctx.channel_id: try: @@ -1646,6 +1778,7 @@ def _check_evonet_offline(self, agent: dict, ctx: SessionContext): 'tool_trace': [], 'is_error': True, 'thinking_duration': 0, + 'message_id': message_id, }) return {'response': reply, 'tool_trace': [], 'error': True} @@ -1660,11 +1793,6 @@ def _do_process_inner(self, agent: dict, ctx: SessionContext) -> dict: agent_id = agent['id'] db_agent_id = ctx.session_db_agent_id or agent_id - # Clear any stale stop flag so a previous /stop doesn't kill this new request - self._get_stop_event(ctx.session_id).clear() - from backend.tools.lib.process_tracker import process_tracker - process_tracker.clear_stop(ctx.session_id) - # Send typing indicator now that processing is actually starting if ctx.channel_id: instance = channel_manager._active.get(ctx.channel_id) @@ -2440,6 +2568,15 @@ def _heartbeat(): if is_error: result["error"] = True + last_assistant = db.get_last_assistant_message( + ctx.session_id, agent_id=db_agent_id, + ) + response_message_id = ( + last_assistant.get('id') + if last_assistant and last_assistant.get('content') == response_text + else None + ) + result['response_message_id'] = response_message_id # Emit turn_complete event event_stream.emit('turn_complete', { 'agent_id': agent_id, @@ -2451,12 +2588,9 @@ def _heartbeat(): 'tool_trace': tool_trace, 'is_error': is_error, 'thinking_duration': round(time.time() - _inner_turn_start, 1), + 'turn_id': ctx.turn_id, + 'message_id': response_message_id, }) - # Clean up per-session buffer after a delay to allow gap-fill requests to complete. - # Use executor to avoid timer leak (old timer never cancelled). - self._bg_executor.submit( - lambda sid=ctx.session_id: (time.sleep(SESSION_BUFFER_CLEANUP_DELAY), event_stream.cleanup_session_buffer(sid)), - ) return result @@ -2507,7 +2641,7 @@ def process_in_session(self, processing_agent_id: str, session_id: str, agent=agent, ctx=SessionContext(session_id, external_user_id, channel_id, session_db_agent_id), ) - self._message_queue.put(task) + self._put_task(task) def get_compiled_context(self, agent_id: str, user_id: str = None) -> dict: """Return the compiled system prompt and tool definitions for an agent.""" @@ -2628,12 +2762,22 @@ def _handle_busy_rejection(self, agent_id: str, agent_state: Any, reply = (f"Sorry, I'm busy with {reason}. " f"Want me to let you know when I'm done?") - _db_retry(db.add_chat_message, session_id, 'assistant', reply, - agent_id=agent_id, metadata={"busy_rejection": True}, - label="save busy rejection") + message_id = _db_retry( + db.add_chat_message, session_id, 'assistant', reply, + agent_id=agent_id, metadata={"busy_rejection": True}, + label="save busy rejection", + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog_manager.get(agent_id, session_id).append({'type': 'final', 'session_id': session_id, 'content': reply, - 'metadata': {'busy_rejection': True}}) + 'metadata': {'busy_rejection': True}, + 'message_id': message_id}) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': reply, 'message_id': message_id, + 'metadata': {'busy_rejection': True}, 'role': 'assistant', + }) # Record the deferral so the pending user message is auto-resumed once the # agent is free and unfocused (drained in _send_free_notification). Queued # on the opt-in branch too — the real answer supersedes the notification. @@ -2703,6 +2847,11 @@ def clear_session(self, agent_id: str, external_user_id: str, channel_id: Option """Clear chat history for a user's session.""" session_id = db.get_or_create_session(agent_id, external_user_id, channel_id) db.clear_session(session_id, agent_id=agent_id) + event_stream.emit('session_clear', { + 'agent_id': agent_id, + 'session_id': session_id, + 'turn_id': None, + }) self._session_skill_mds.pop(session_id, None) self._session_skill_tools.pop(session_id, None) @@ -2723,15 +2872,35 @@ def get_session_skills(self, session_id: str) -> list[dict]: return [{"skill_id": sk_id, "tool_count": len(tool_defs)} for sk_id, tool_defs in skills_data.items()] - def send_as_bot(self, session_id: str, text: str) -> bool: + def send_as_bot(self, session_id: str, text: str, + metadata: dict | None = None) -> bool: """Admin takeover: save message as assistant and send via channel.""" session = db.get_session_with_details(session_id) if not session: return False - db.add_chat_message(session_id, 'assistant', text, agent_id=session['agent_id']) + meta = {'admin_takeover': True} + if metadata: + meta.update(metadata) + message_id = db.add_chat_message( + session_id, 'assistant', text, + agent_id=session['agent_id'], metadata=meta, + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog_manager.get(session['agent_id'], session_id).append( {'type': 'final', 'session_id': session_id, 'content': text, - 'metadata': {'admin_takeover': True}}) + 'metadata': meta, 'message_id': message_id}) + event_stream.emit('message_received', { + 'agent_id': session['agent_id'], + 'session_id': session_id, + 'external_user_id': session.get('external_user_id', ''), + 'channel_id': session.get('channel_id'), + 'message': text, + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'metadata': meta, + 'role': 'assistant', + 'sender': 'admin', + }) # Send via channel if available if session.get('channel_id'): instance = channel_manager._active.get(session['channel_id']) @@ -2808,18 +2977,27 @@ def send_file_as_bot(self, session_id: str, file_path: str, else: content = f"[File: {filename}]" + # Also persist in main chat messages table + message_id = db.add_chat_message( + session_id, 'assistant', content, agent_id=agent_id, + metadata={'attachment_info': attachment_info}, + ) + message_id = message_id if type(message_id) in (int, str) else None chatlog = chatlog_manager.get(agent_id, session_id) chatlog.append({ 'type': 'final', 'session_id': session_id, 'content': content, 'metadata': {'attachment_info': attachment_info, 'channel': channel_type}, + 'message_id': message_id, + }) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': content, 'message_id': message_id, + 'metadata': {'attachment_info': attachment_info}, + 'role': 'assistant', 'sender': agent_id, }) - - # Also persist in main chat messages table - db.add_chat_message(session_id, 'assistant', content, - agent_id=agent_id, - metadata={'attachment_info': attachment_info}) return True @@ -2907,16 +3085,30 @@ def send_as_user(self, session_id: str, text: str, ) if response is not None: # Command was recognized — save command echo and response, then return - db.add_chat_message(session_id, 'user', text, - agent_id=agent_id, metadata={'slash_command': True}) - db.add_chat_message(session_id, 'assistant', response, - agent_id=agent_id, metadata={'slash_command': True}) + command_meta = {'slash_command': True} + if metadata and metadata.get('client_message_id'): + command_meta['client_message_id'] = metadata['client_message_id'] + message_id = db.add_chat_message( + session_id, 'user', text, + agent_id=agent_id, metadata=command_meta, + ) + response_id = db.add_chat_message( + session_id, 'assistant', response, + agent_id=agent_id, metadata={'slash_command': True}, + ) _cl = chatlog_manager.get(agent_id, session_id) _cl.append({'type': 'user', 'session_id': session_id, 'content': text, 'sender_id': external_user_id, - 'metadata': {'slash_command': True}}) + 'metadata': command_meta, 'message_id': message_id}) _cl.append({'type': 'system', 'session_id': session_id, 'content': response, - 'metadata': {'slash_command': True}}) + 'metadata': {'slash_command': True}, 'message_id': response_id}) + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': text, 'message_id': message_id, + 'client_message_id': command_meta.get('client_message_id'), + 'metadata': command_meta, 'role': 'user', + }) agent = db.get_agent(agent_id) # Check for any attachments created by the handler (e.g. /dump) attachment_info = None @@ -2949,13 +3141,8 @@ def send_as_user(self, session_id: str, text: str, 'thinking_duration': 0.0, 'slash_command': True, 'attachment_info': attachment_info, + 'message_id': response_id, }) - # Signal the client to clear the chat UI when the clear command was used - if cmd_name == 'clear': - event_stream.emit('session_clear', { - 'session_id': session_id, - 'agent_id': agent_id, - }) self._prefetcher.invalidate(session_id) return response # return response text so caller can include it in API response # Unknown command — fall through to normal LLM processing @@ -2969,10 +3156,12 @@ def send_as_user(self, session_id: str, text: str, meta['video_url'] = video_url if metadata: meta.update(metadata) - db.add_chat_message(session_id, 'user', text, agent_id=agent_id, metadata=meta) + message_id = db.add_chat_message( + session_id, 'user', text, agent_id=agent_id, metadata=meta, + ) chatlog_manager.get(agent_id, session_id).append( {'type': 'user', 'session_id': session_id, 'content': text, - 'metadata': meta}) + 'metadata': meta, 'message_id': message_id}) # Invalidate prefetched context — a new message arrived self._prefetcher.invalidate(session_id) @@ -2989,6 +3178,10 @@ def send_as_user(self, session_id: str, text: str, 'image_url': image_url, 'audio_url': audio_url, 'video_url': video_url, + 'message_id': message_id, + 'client_message_id': meta.get('client_message_id'), + 'metadata': meta, + 'role': 'user', }) # Mid-loop injection: if session is currently processing, inject message @@ -3012,7 +3205,7 @@ def send_as_user(self, session_id: str, text: str, if agent and agent.get('enabled', True): task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id), send_via_channel=False) - self._message_queue.put(task) + self._put_task(task) return True @@ -3030,4 +3223,4 @@ def resume_session(self, agent: dict, session_id: str, return task = _QueueTask(agent, SessionContext(session_id, external_user_id, channel_id), send_via_channel=send_via_channel) - self._message_queue.put(task) + self._put_task(task) diff --git a/backend/channels/discord.py b/backend/channels/discord.py index cc18772b..e78f29ca 100644 --- a/backend/channels/discord.py +++ b/backend/channels/discord.py @@ -392,7 +392,20 @@ async def _handle_message(self, message, is_dm: bool): # Respect the per-session bot toggle. if not db.is_session_bot_enabled(session_id, agent_id=agent_id): - db.add_chat_message(session_id, 'user', text or '[Image]', agent_id=agent_id) + stored = text or '[Image]' + message_id = db.add_chat_message(session_id, 'user', stored, agent_id=agent_id) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'user', 'session_id': session_id, 'content': stored, + 'sender_id': user_id, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': user_id, 'channel_id': channel_id, + 'message': stored, 'message_id': message_id, 'role': 'user', + }) return # Include the replied-to bot message as context, when present. diff --git a/backend/channels/telegram.py b/backend/channels/telegram.py index ae363e9d..f1be2528 100644 --- a/backend/channels/telegram.py +++ b/backend/channels/telegram.py @@ -600,7 +600,20 @@ async def handle_message(update: Update, context: ContextTypes.DEFAULT_TYPE): # Check if bot is enabled for this session if not db.is_session_bot_enabled(session_id, agent_id=agent_id): - db.add_chat_message(session_id, 'user', text or '[Image]', agent_id=agent_id) + stored = text or '[Image]' + message_id = db.add_chat_message(session_id, 'user', stored, agent_id=agent_id) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'user', 'session_id': session_id, 'content': stored, + 'sender_id': user_id, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': user_id, 'channel_id': channel_id, + 'message': stored, 'message_id': message_id, 'role': 'user', + }) return # Detect reply/quote: include replied message content as context diff --git a/backend/channels/whatsapp.py b/backend/channels/whatsapp.py index 84999a52..c7ec5488 100644 --- a/backend/channels/whatsapp.py +++ b/backend/channels/whatsapp.py @@ -1011,8 +1011,22 @@ def handle_callback(self, payload: dict): if not db.is_session_bot_enabled(session_id, agent_id=agent_id): _logger.info("WhatsApp message stored only — bot disabled for session %s (sender=%s)", session_id, sender) - db.add_chat_message(session_id, 'user', final_text or text or '[Attachment]', - agent_id=agent_id) + stored = final_text or text or '[Attachment]' + message_id = db.add_chat_message( + session_id, 'user', stored, agent_id=agent_id, + ) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'user', 'session_id': session_id, 'content': stored, + 'sender_id': session_user_id, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': session_user_id, 'channel_id': self.channel_id, + 'message': stored, 'message_id': message_id, 'role': 'user', + }) return _logger.info("WhatsApp message received from %s (channel %s)", sender, self.channel_id) diff --git a/backend/event_stream.py b/backend/event_stream.py index 80c33da0..33ce1bd9 100644 --- a/backend/event_stream.py +++ b/backend/event_stream.py @@ -17,7 +17,6 @@ Events are logged to logs/events.log (configurable via EVENT_LOG_FILE in .env). """ -import collections import itertools import logging import os @@ -28,55 +27,19 @@ _logger = logging.getLogger(__name__) -# Event types that the per-session chat SSE stream forwards to the browser. -# A per-session "chat seq" is assigned ONLY to these (see EventStream.emit), so the -# sequence the browser sees is contiguous and gap-detection never misfires on -# unrelated/global events. Keep in sync with the live stream + gap-fill transforms -# in routes/agents.py. -CHAT_FORWARDED_EVENTS = frozenset({ - 'turn_begin', - 'llm_thinking', - 'tool_call_started', - 'tool_executed', - 'state:changed', - 'tasks:auto_transition', - 'tasks:stale', - 'llm_response_chunk', - 'turn_complete', - 'required_tool_enforced', - 'approval_required', - 'approval_resolved', - 'llm_retry', - 'message_injected', - 'message_injection_applied', - 'message_received', - 'whatsapp_restriction_warning', - 'session_clear', - 'turn_split', - 'evonic:agent-state-changed', -}) - - class EventStream: def __init__(self): self._listeners: Dict[str, List[Callable]] = {} self._lock = threading.Lock() self._log_lock = threading.Lock() - self._log_buffer: collections.deque = collections.deque(maxlen=1000) + self._log_buffer: List[str] = [] self._log_timer: Optional[threading.Timer] = None self._LOG_FLUSH_INTERVAL = 2.0 - self._LOG_BUFFER_LIMIT = 50 # soft flush trigger; deque maxlen=1000 is hard cap + self._LOG_BUFFER_LIMIT = 50 self._executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix='event') self._log_file: str = None # resolved lazily to avoid import-time circular deps - # Sequence numbering and ring buffers for gap-fill recovery + # Raw in-process sequence is retained for plugin compatibility only. self._seq_counter = itertools.count(1) - self._buffer_lock = threading.Lock() - self._global_buffer: collections.deque = collections.deque(maxlen=1000) - self._session_buffers: Dict[str, collections.deque] = {} - # Per-session monotonic counter over CHAT_FORWARDED_EVENTS only, so the - # browser's chat stream sees a contiguous (gap-free) sequence. - self._session_chat_seq: Dict[str, int] = {} - self._web_listeners: Dict[str, int] = {} def _get_log_file(self) -> str: if self._log_file is None: @@ -147,22 +110,14 @@ def emit(self, event_name: str, data: dict): seq = next(self._seq_counter) data['_seq'] = seq data['_event'] = event_name - # Store in ring buffers for gap-fill queries - session_id = data.get('session_id') - chat_seq = None - entry = {'seq': seq, 'event': event_name, 'data': data} - with self._buffer_lock: - # Assign a contiguous per-session chat seq for forwarded events only. - if session_id and event_name in CHAT_FORWARDED_EVENTS: - chat_seq = self._session_chat_seq.get(session_id, 0) + 1 - self._session_chat_seq[session_id] = chat_seq - data['_chat_seq'] = chat_seq - entry['chat_seq'] = chat_seq - self._global_buffer.append(entry) - if session_id: - if session_id not in self._session_buffers: - self._session_buffers[session_id] = collections.deque(maxlen=500) - self._session_buffers[session_id].append(entry) + # Journal synchronously before asynchronous plugin listeners. This is + # what gives browser replay a stable total order even though raw plugin + # callbacks still run concurrently. + try: + from backend.realtime_store import record_internal_event + record_internal_event(event_name, data) + except Exception as exc: + _logger.error("Failed to journal realtime event '%s': %s", event_name, exc) preview = ', '.join(f'{k}={str(v)[:120]}' for k, v in data.items() if not k.startswith('_')) self._write_log(f"[seq={seq}] {event_name} | {preview}") with self._lock: @@ -170,44 +125,6 @@ def emit(self, event_name: str, data: dict): for cb in listeners: self._executor.submit(self._safe_call, event_name, cb, data) - def get_events_in_range(self, session_id: str, after_seq: int, up_to_seq: int) -> list: - """Return chat-forwarded events for session_id where - after_seq < chat_seq <= up_to_seq (chat_seq is the per-session chat seq).""" - with self._buffer_lock: - buf = self._session_buffers.get(session_id, collections.deque()) - if not buf: - return [] - return [e for e in buf - if 'chat_seq' in e and after_seq < e['chat_seq'] <= up_to_seq] - - def get_session_events(self, session_id: str, after_seq: int = 0) -> list: - """Return chat-forwarded events for session_id with chat_seq > after_seq.""" - with self._buffer_lock: - buf = self._session_buffers.get(session_id, collections.deque()) - return [e for e in buf if 'chat_seq' in e and e['chat_seq'] > after_seq] - - def cleanup_session_buffer(self, session_id: str): - """Remove per-session buffer (called after turn completes).""" - with self._buffer_lock: - self._session_buffers.pop(session_id, None) - self._session_chat_seq.pop(session_id, None) - - def register_web_listener(self, session_id: str): - with self._lock: - self._web_listeners[session_id] = self._web_listeners.get(session_id, 0) + 1 - - def unregister_web_listener(self, session_id: str): - with self._lock: - count = self._web_listeners.get(session_id, 0) - 1 - if count <= 0: - self._web_listeners.pop(session_id, None) - else: - self._web_listeners[session_id] = count - - def has_web_listener(self, session_id: str) -> bool: - with self._lock: - return self._web_listeners.get(session_id, 0) > 0 - def _safe_call(self, event_name: str, cb: Callable, data: dict): try: cb(data) diff --git a/backend/realtime_store.py b/backend/realtime_store.py new file mode 100644 index 00000000..25ec6897 --- /dev/null +++ b/backend/realtime_store.py @@ -0,0 +1,843 @@ +"""Durable realtime journal and active-turn projection. + +The journal is the source of truth for browser replay. The existing +``EventStream`` remains the in-process plugin bus; it records normalized public +events here before dispatching asynchronous listeners. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import sqlite3 +import threading +import time +import uuid +from contextlib import contextmanager +from datetime import datetime +import config + + +log = logging.getLogger(__name__) + +RETENTION_MS = 60 * 60 * 1000 +_CLEANUP_INTERVAL_MS = 60 * 60 * 1000 +MAX_EVENT_PAYLOAD_BYTES = 256 * 1024 +_PAYLOAD_PREVIEW_BYTES = 32 * 1024 +_ATTACHMENT_KEYS = frozenset({ + 'attachment_id', 'filename', 'mime_type', 'size_bytes', 'is_image', +}) + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def _json_default(value): + if isinstance(value, datetime): + return value.isoformat() + return str(value) + + +def _json_dumps(value) -> str: + return json.dumps(value, separators=(',', ':'), default=_json_default) + + +def _utf8_prefix(value: str, max_bytes: int) -> str: + return value.encode('utf-8')[:max_bytes].decode('utf-8', errors='ignore') + + +def _bounded_payload_json(payload: dict) -> str: + """Serialize one browser payload without letting telemetry grow unbounded.""" + body = _json_dumps(payload) + original_body = body + original_bytes = len(body.encode('utf-8')) + if original_bytes <= MAX_EVENT_PAYLOAD_BYTES: + return body + + bounded = dict(payload) + field_sizes = sorted( + ( + (len(_json_dumps(value).encode('utf-8')), key) + for key, value in payload.items() + ), + reverse=True, + ) + truncated_fields = {} + for field_bytes, key in field_sizes: + value = payload[key] + if isinstance(value, str): + bounded[key] = _utf8_prefix(value, _PAYLOAD_PREVIEW_BYTES) + else: + raw_value = _json_dumps(value) + bounded[key] = { + 'truncated': True, + 'original_bytes': len(raw_value.encode('utf-8')), + 'preview': _utf8_prefix(raw_value, _PAYLOAD_PREVIEW_BYTES), + } + truncated_fields[key] = field_bytes + bounded['truncated'] = True + bounded['original_bytes'] = original_bytes + bounded['truncated_fields'] = truncated_fields + body = _json_dumps(bounded) + if len(body.encode('utf-8')) <= MAX_EVENT_PAYLOAD_BYTES: + return body + + # Pathological objects with many small keys still get one valid bounded row. + fallback = { + 'truncated': True, + 'original_bytes': original_bytes, + } + for key in ( + 'tool', 'error', 'message_id', 'client_message_id', 'approval_id', + 'decision', 'is_final', 'is_error', 'interrupted', 'reason'): + if key not in payload: + continue + value = payload.get(key) + if isinstance(value, (bool, int, float)) or value is None: + fallback[key] = value + elif isinstance(value, str): + fallback[key] = _utf8_prefix(value, 2048) + + preview_budget = MAX_EVENT_PAYLOAD_BYTES - len(_json_dumps(fallback).encode('utf-8')) - 64 + fallback['preview'] = _utf8_prefix(original_body, max(0, preview_budget)) + body = _json_dumps(fallback) + while len(body.encode('utf-8')) > MAX_EVENT_PAYLOAD_BYTES and fallback['preview']: + fallback['preview'] = _utf8_prefix( + fallback['preview'], max(0, len(fallback['preview'].encode('utf-8')) - 1024), + ) + body = _json_dumps(fallback) + return body + + +def _public_attachment(value): + if not isinstance(value, dict): + return None + return {key: value.get(key) for key in _ATTACHMENT_KEYS if key in value} + + +def _public_metadata(value) -> dict: + """Remove browser-local URLs and backend paths from SSE payloads.""" + if not isinstance(value, dict): + return {} + result = {} + for key, item in value.items(): + if key in {'image_url', 'audio_url', 'video_url', 'file_path', 'path'}: + continue + if key == 'attachment_info': + result[key] = _public_attachment(item) + elif key == 'attachment_infos' and isinstance(item, list): + result[key] = [clean for clean in map(_public_attachment, item) if clean] + else: + result[key] = item + return result + + +def _public_message(value: str) -> str: + # Attachment markers are part of the model prompt, but local paths do not + # belong in browser broadcasts. + return re.sub( + r'(\[Attached:[^\]]*?)\s+path=[^\]]+(\])', r'\1\2', value or '', + ) + + +class RealtimeStore: + def __init__(self, db_path: str | None = None): + self.db_path = db_path + self._conn = None + self._conn_path = None + self._conn_pid = None + # ponytail: one lock keeps the journal's connection count bounded; + # add a small pool only if realtime DB contention is measured. + self._conn_lock = threading.Lock() + self._schema_lock = threading.Lock() + self._schema_paths: set[str] = set() + self._condition = threading.Condition() + self._cleanup_lock = threading.Lock() + self._last_cleanup_ms = 0 + self._recovery_lock = threading.Lock() + self._recovered: set[tuple[int, str]] = set() + + @contextmanager + def _connect(self): + db_path = self._resolve_db_path() + self._ensure_schema(db_path) + with self._conn_lock: + conn = self._conn + if conn is not None and ( + self._conn_path != db_path or self._conn_pid != os.getpid()): + conn.close() + conn = None + self._conn = None + if conn is not None: + try: + conn.execute('SELECT 1') + except sqlite3.Error: + conn.close() + conn = None + self._conn = None + if conn is None: + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect( + f'file:{db_path}?mode=rwc&busy_timeout=10000', + uri=True, timeout=10, check_same_thread=False, + ) + conn.row_factory = sqlite3.Row + conn.execute('PRAGMA busy_timeout=10000') + conn.execute('PRAGMA journal_mode=WAL') + conn.execute('PRAGMA synchronous=NORMAL') + self._conn = conn + self._conn_path = db_path + self._conn_pid = os.getpid() + with conn: + yield conn + + def _resolve_db_path(self) -> str: + if self.db_path: + return os.path.abspath(self.db_path) + app_db_path = config.DB_PATH + try: + from models.db import db + app_db_path = db.db_path + except Exception: + pass + return os.path.abspath( + os.path.join(os.path.dirname(app_db_path), 'realtime.db') + ) + + def _ensure_schema(self, db_path: str): + if db_path in self._schema_paths: + return + with self._schema_lock: + if db_path in self._schema_paths: + return + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect(db_path, timeout=10) + try: + conn.execute('PRAGMA journal_mode=WAL') + conn.executescript(""" + CREATE TABLE IF NOT EXISTS realtime_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + occurred_at_ms INTEGER NOT NULL, + expires_at_ms INTEGER, + channel TEXT NOT NULL, + event_type TEXT NOT NULL, + agent_id TEXT, + session_id TEXT, + workplace_id TEXT, + turn_id TEXT, + payload_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_realtime_session_id + ON realtime_events(session_id, id); + CREATE INDEX IF NOT EXISTS idx_realtime_channel_id + ON realtime_events(channel, id); + CREATE INDEX IF NOT EXISTS idx_realtime_workplace_id + ON realtime_events(workplace_id, id); + CREATE INDEX IF NOT EXISTS idx_realtime_turn_id + ON realtime_events(turn_id, id); + CREATE INDEX IF NOT EXISTS idx_realtime_expiry + ON realtime_events(expires_at_ms); + + CREATE TABLE IF NOT EXISTS active_turns ( + turn_id TEXT PRIMARY KEY, + agent_id TEXT NOT NULL, + session_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued', 'running')), + queued_at_ms INTEGER NOT NULL, + started_at_ms INTEGER, + updated_at_ms INTEGER NOT NULL, + owner_pid INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_active_turns_agent + ON active_turns(agent_id, updated_at_ms); + CREATE INDEX IF NOT EXISTS idx_active_turns_session + ON active_turns(session_id, updated_at_ms); + + CREATE TABLE IF NOT EXISTS realtime_replay_floors ( + session_id TEXT PRIMARY KEY, + event_id INTEGER NOT NULL + ); + """) + conn.commit() + finally: + conn.close() + self._schema_paths.add(db_path) + + def close(self): + with self._conn_lock: + if self._conn is not None: + self._conn.close() + self._conn = None + self._conn_path = None + self._conn_pid = None + + def high_water(self) -> int: + with self._connect() as conn: + row = conn.execute(""" + SELECT COALESCE( + (SELECT seq FROM sqlite_sequence WHERE name = 'realtime_events'), + 0 + ) AS id + """).fetchone() + return int(row['id']) + + def _advance_replay_floor(self, conn: sqlite3.Connection, + session_id: str, event_id: int | None) -> None: + if not session_id or not event_id: + return + conn.execute(""" + INSERT INTO realtime_replay_floors(session_id, event_id) VALUES (?, ?) + ON CONFLICT(session_id) DO UPDATE + SET event_id = MAX(event_id, excluded.event_id) + """, (session_id, int(event_id))) + + def replay_floor(self, session_id: str) -> int: + """Highest session event ID that may no longer be replayable.""" + if not session_id: + return 0 + with self._connect() as conn: + row = conn.execute(""" + SELECT MAX( + COALESCE(( + SELECT event_id FROM realtime_replay_floors + WHERE session_id = ? + ), 0), + COALESCE(( + SELECT MAX(id) FROM realtime_events + WHERE session_id = ? AND expires_at_ms IS NOT NULL + AND expires_at_ms <= ? + ), 0) + ) AS id + """, (session_id, session_id, _now_ms())).fetchone() + return int(row['id']) + + def last_session_clear_id(self, session_id: str, up_to_id: int) -> int: + with self._connect() as conn: + row = conn.execute(""" + SELECT COALESCE(MAX(id), 0) AS id FROM realtime_events + WHERE session_id = ? AND event_type = 'session_clear' + AND id <= ? + AND (expires_at_ms IS NULL OR expires_at_ms > ?) + """, (session_id, up_to_id, _now_ms())).fetchone() + return int(row['id']) + + def publish(self, channel: str, event_type: str, payload: dict, *, + agent_id: str | None = None, session_id: str | None = None, + workplace_id: str | None = None, turn_id: str | None = None, + occurred_at_ms: int | None = None) -> int: + now = occurred_at_ms or _now_ms() + expires_at = None if turn_id else now + RETENTION_MS + body = _bounded_payload_json(payload) + with self._connect() as conn: + cursor = conn.execute(""" + INSERT INTO realtime_events ( + occurred_at_ms, expires_at_ms, channel, event_type, + agent_id, session_id, workplace_id, turn_id, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (now, expires_at, channel, event_type, agent_id, session_id, + workplace_id, turn_id, body)) + event_id = int(cursor.lastrowid) + with self._condition: + self._condition.notify_all() + self._maybe_cleanup(now) + return event_id + + def events_after(self, after_id: int, channels: set[str], *, + session_id: str | None = None, + agent_id: str | None = None, + workplace_id: str | None = None, + up_to_id: int | None = None, + active_only: bool = False, + limit: int = 500) -> list[dict]: + clauses = ['e.id > ?', '(e.expires_at_ms IS NULL OR e.expires_at_ms > ?)'] + params: list = [after_id, _now_ms()] + if up_to_id is not None: + clauses.append('e.id <= ?') + params.append(up_to_id) + if active_only: + clauses.append('EXISTS (SELECT 1 FROM active_turns a WHERE a.turn_id = e.turn_id)') + + channel_clauses = [] + global_channels = channels - {'chat', 'workplace'} + if global_channels: + placeholders = ','.join('?' for _ in global_channels) + channel_clauses.append(f'e.channel IN ({placeholders})') + params.extend(sorted(global_channels)) + if 'chat' in channels and session_id: + # Session-scoped approval/status events share the chat stream so a + # chat-only consumer still sees queued/busy/approval state. + channel_clauses.append( + "(e.session_id = ? AND e.channel IN ('chat','status','approvals'))" + ) + params.append(session_id) + if 'workplace' in channels and workplace_id: + channel_clauses.append("(e.channel = 'workplace' AND e.workplace_id = ?)") + params.append(workplace_id) + if not channel_clauses: + return [] + clauses.append('(' + ' OR '.join(channel_clauses) + ')') + + if session_id and 'chat' in channels: + clauses.append("(e.channel != 'chat' OR e.session_id = ?)") + params.append(session_id) + if agent_id and 'chat' in channels: + clauses.append("(e.channel != 'chat' OR e.agent_id IS NULL OR e.agent_id = ?)") + params.append(agent_id) + + params.append(limit) + sql = f""" + SELECT e.* FROM realtime_events e + WHERE {' AND '.join(clauses)} + ORDER BY e.id ASC LIMIT ? + """ + with self._connect() as conn: + rows = conn.execute(sql, params).fetchall() + return [self._decode_event(row) for row in rows] + + @staticmethod + def _decode_event(row: sqlite3.Row) -> dict: + payload = json.loads(row['payload_json']) + payload.update({ + 'event_id': row['id'], + 'seq': row['id'], + 'timestamp': row['occurred_at_ms'], + 'channel': row['channel'], + }) + if row['agent_id']: + payload.setdefault('agent_id', row['agent_id']) + if row['session_id']: + payload.setdefault('session_id', row['session_id']) + if row['workplace_id']: + payload.setdefault('workplace_id', row['workplace_id']) + if row['turn_id']: + payload['turn_id'] = row['turn_id'] + return { + 'id': row['id'], + 'timestamp': row['occurred_at_ms'], + 'channel': row['channel'], + 'event': row['event_type'], + 'agent_id': row['agent_id'], + 'session_id': row['session_id'], + 'workplace_id': row['workplace_id'], + 'turn_id': row['turn_id'], + 'data': payload, + } + + def wait_for_events(self, after_id: int, timeout: float = 15) -> None: + with self._condition: + self._condition.wait_for( + lambda: self.high_water() > after_id, + timeout, + ) + + def queue_turn(self, agent_id: str, session_id: str, + turn_id: str | None = None) -> tuple[str, bool]: + """Create queued state, reusing an existing debounced queue entry.""" + now = _now_ms() + with self._connect() as conn: + conn.execute('BEGIN IMMEDIATE') + existing = conn.execute(""" + SELECT turn_id FROM active_turns + WHERE session_id = ? AND state = 'queued' + ORDER BY queued_at_ms ASC LIMIT 1 + """, (session_id,)).fetchone() + if existing: + return existing['turn_id'], False + turn_id = turn_id or uuid.uuid4().hex + conn.execute(""" + INSERT INTO active_turns ( + turn_id, agent_id, session_id, state, queued_at_ms, + started_at_ms, updated_at_ms, owner_pid + ) VALUES (?, ?, ?, 'queued', ?, NULL, ?, ?) + """, (turn_id, agent_id, session_id, now, now, os.getpid())) + active = self.busy_agents().get(agent_id, {}) + self.publish('status', 'turn_queued', { + 'agent_id': agent_id, + 'session_id': session_id, + 'busy': True, + 'state': 'queued', + }, agent_id=agent_id, session_id=session_id, turn_id=turn_id, + occurred_at_ms=now) + self.publish('status', 'agent_busy_changed', { + 'agent_id': agent_id, + 'session_id': active.get('session_id', session_id), + 'session_ids': active.get('session_ids', [session_id]), + 'active_count': active.get('active_count', 1), + 'busy': True, + 'state': 'queued', + }, agent_id=agent_id, session_id=session_id, turn_id=turn_id, + occurred_at_ms=now) + return turn_id, True + + def start_turn(self, turn_id: str) -> dict | None: + now = _now_ms() + with self._connect() as conn: + cursor = conn.execute(""" + UPDATE active_turns SET state = 'running', started_at_ms = ?, + updated_at_ms = ?, owner_pid = ? + WHERE turn_id = ? AND state = 'queued' + """, (now, now, os.getpid(), turn_id)) + if cursor.rowcount != 1: + return None + row = conn.execute( + 'SELECT * FROM active_turns WHERE turn_id = ?', (turn_id,), + ).fetchone() + with self._condition: + self._condition.notify_all() + return dict(row) if row else None + + def cancel_queued_turns(self, session_id: str, + turn_id: str | None = None) -> list[dict]: + """Atomically remove queued work so a racing worker cannot start it.""" + now = _now_ms() + with self._connect() as conn: + conn.execute('BEGIN IMMEDIATE') + clauses = ["session_id = ?", "state = 'queued'"] + params = [session_id] + if turn_id: + clauses.append('turn_id = ?') + params.append(turn_id) + rows = conn.execute( + f"SELECT * FROM active_turns WHERE {' AND '.join(clauses)}", + params, + ).fetchall() + turn_ids = [row['turn_id'] for row in rows] + if turn_ids: + placeholders = ','.join('?' for _ in turn_ids) + conn.execute( + f'DELETE FROM active_turns WHERE turn_id IN ({placeholders}) ' + "AND state = 'queued'", + turn_ids, + ) + conn.execute( + f'UPDATE realtime_events SET expires_at_ms = ? ' + f'WHERE turn_id IN ({placeholders})', + [now + RETENTION_MS, *turn_ids], + ) + if rows: + with self._condition: + self._condition.notify_all() + return [dict(row) for row in rows] + + def finish_turn(self, turn_id: str) -> None: + now = _now_ms() + with self._connect() as conn: + conn.execute( + 'UPDATE realtime_events SET expires_at_ms = ? WHERE turn_id = ?', + (now + RETENTION_MS, turn_id), + ) + conn.execute('DELETE FROM active_turns WHERE turn_id = ?', (turn_id,)) + with self._condition: + self._condition.notify_all() + + def current_turn_id(self, session_id: str) -> str | None: + with self._connect() as conn: + row = conn.execute(""" + SELECT turn_id FROM active_turns WHERE session_id = ? + ORDER BY CASE state WHEN 'running' THEN 0 ELSE 1 END, + updated_at_ms DESC LIMIT 1 + """, (session_id,)).fetchone() + return row['turn_id'] if row else None + + def active_turns(self, *, agent_id: str | None = None, + session_id: str | None = None) -> list[dict]: + clauses = [] + params = [] + if agent_id: + clauses.append('agent_id = ?') + params.append(agent_id) + if session_id: + clauses.append('session_id = ?') + params.append(session_id) + where = (' WHERE ' + ' AND '.join(clauses)) if clauses else '' + with self._connect() as conn: + rows = conn.execute( + 'SELECT * FROM active_turns' + where + ' ORDER BY queued_at_ms', + params, + ).fetchall() + return [dict(row) for row in rows] + + def busy_agents(self) -> dict: + now = _now_ms() + result = {} + for turn in self.active_turns(): + entry = result.setdefault(turn['agent_id'], { + 'session_id': turn['session_id'], + 'session_ids': [], + 'active_count': 0, + 'started_at': (turn['started_at_ms'] or turn['queued_at_ms']) / 1000, + 'state': turn['state'], + }) + if turn['session_id'] not in entry['session_ids']: + entry['session_ids'].append(turn['session_id']) + entry['active_count'] += 1 + if turn['state'] == 'running': + entry['state'] = 'running' + entry['session_id'] = turn['session_id'] + entry['started_at'] = (turn['started_at_ms'] or turn['queued_at_ms']) / 1000 + for entry in result.values(): + entry['elapsed'] = round(now / 1000 - entry['started_at'], 1) + return result + + def purge_session(self, session_id: str) -> None: + with self._connect() as conn: + row = conn.execute(""" + SELECT MAX(id) AS id FROM realtime_events + WHERE session_id = ? AND ( + turn_id IS NULL OR NOT EXISTS ( + SELECT 1 FROM active_turns a + WHERE a.turn_id = realtime_events.turn_id + ) + ) + """, (session_id,)).fetchone() + self._advance_replay_floor(conn, session_id, row['id']) + conn.execute(""" + DELETE FROM realtime_events + WHERE session_id = ? AND ( + turn_id IS NULL OR NOT EXISTS ( + SELECT 1 FROM active_turns a + WHERE a.turn_id = realtime_events.turn_id + ) + ) + """, (session_id,)) + with self._condition: + self._condition.notify_all() + + def purge_all(self) -> None: + with self._connect() as conn: + rows = conn.execute(""" + SELECT session_id, MAX(id) AS id FROM realtime_events + WHERE session_id IS NOT NULL AND ( + turn_id IS NULL OR NOT EXISTS ( + SELECT 1 FROM active_turns a + WHERE a.turn_id = realtime_events.turn_id + ) + ) + GROUP BY session_id + """).fetchall() + for row in rows: + self._advance_replay_floor(conn, row['session_id'], row['id']) + conn.execute(""" + DELETE FROM realtime_events + WHERE turn_id IS NULL OR NOT EXISTS ( + SELECT 1 FROM active_turns a + WHERE a.turn_id = realtime_events.turn_id + ) + """) + with self._condition: + self._condition.notify_all() + + def interrupt_stale_turns(self) -> list[dict]: + key = (os.getpid(), self._resolve_db_path()) + with self._recovery_lock: + if key in self._recovered: + return [] + self._maybe_cleanup(_now_ms()) + stale = self.active_turns() + for turn in stale: + with self._connect() as conn: + terminal = conn.execute(""" + SELECT 1 FROM realtime_events + WHERE turn_id = ? AND event_type = 'done' LIMIT 1 + """, (turn['turn_id'],)).fetchone() + if not terminal: + payload = { + 'agent_id': turn['agent_id'], + 'session_id': turn['session_id'], + 'response': '', + 'interrupted': True, + 'is_error': True, + 'reason': 'server_restart', + } + self.publish('chat', 'done', payload, + agent_id=turn['agent_id'], session_id=turn['session_id'], + turn_id=turn['turn_id']) + self.finish_turn(turn['turn_id']) + remaining = self.busy_agents().get(turn['agent_id']) + self.publish('status', 'agent_busy_changed', { + 'agent_id': turn['agent_id'], + 'session_id': remaining.get('session_id', turn['session_id']) if remaining else turn['session_id'], + 'session_ids': remaining.get('session_ids', []) if remaining else [], + 'active_count': remaining.get('active_count', 0) if remaining else 0, + 'state': remaining.get('state', 'idle') if remaining else 'idle', + 'busy': bool(remaining), + 'interrupted': True, + }, agent_id=turn['agent_id'], session_id=turn['session_id']) + self._recovered.add(key) + return stale + + def _maybe_cleanup(self, now_ms: int) -> None: + if now_ms - self._last_cleanup_ms < _CLEANUP_INTERVAL_MS: + return + if not self._cleanup_lock.acquire(blocking=False): + return + try: + if now_ms - self._last_cleanup_ms < _CLEANUP_INTERVAL_MS: + return + with self._connect() as conn: + rows = conn.execute(""" + SELECT session_id, MAX(id) AS id FROM realtime_events + WHERE session_id IS NOT NULL AND expires_at_ms IS NOT NULL + AND expires_at_ms <= ? + GROUP BY session_id + """, (now_ms,)).fetchall() + for row in rows: + self._advance_replay_floor(conn, row['session_id'], row['id']) + conn.execute( + 'DELETE FROM realtime_events WHERE expires_at_ms IS NOT NULL AND expires_at_ms <= ?', + (now_ms,), + ) + self._last_cleanup_ms = now_ms + finally: + self._cleanup_lock.release() + +realtime_store = RealtimeStore() + + +def record_internal_event(event_name: str, data: dict) -> list[int]: + """Normalize one raw runtime event into durable public SSE events.""" + session_id = data.get('session_id') or None + agent_id = data.get('agent_id') or None + workplace_id = data.get('workplace_id') or None + metadata = _public_metadata(data.get('metadata')) + specs: list[tuple[str, str, dict]] = [] + + if event_name == 'turn_begin': + specs.append(('chat', 'turn_begin', {'ts': data.get('ts', _now_ms())})) + elif event_name == 'llm_thinking': + specs.append(('chat', 'thinking', {'content': data.get('thinking', '')})) + elif event_name == 'tool_call_started': + specs.append(('chat', 'tool_call_started', { + 'tool': data.get('tool_name', ''), 'args': data.get('tool_args', {}), + 'param_types': data.get('param_types', {}), + })) + elif event_name == 'tool_executed': + specs.append(('chat', 'tool_executed', { + 'tool': data.get('tool_name', ''), 'args': data.get('tool_args', {}), + 'result': data.get('tool_result', {}), 'error': data.get('has_error', False), + })) + elif event_name in {'state:changed', 'tasks:auto_transition', 'tasks:stale'}: + keys = ('mode', 'plan_file', 'tasks', 'loaded_skills', 'task_ids') + specs.append(('chat', event_name, {key: data[key] for key in keys if key in data})) + elif event_name == 'llm_response_chunk': + specs.append(('chat', 'response_chunk', { + 'content': data.get('content', ''), 'is_final': data.get('is_final', False), + 'send_as_message': data.get('send_as_message', False), + })) + elif event_name == 'turn_complete': + done = { + 'thinking_duration': data.get('thinking_duration'), + 'response': data.get('response', ''), + 'slash_command': data.get('slash_command', False), + 'attachment_info': _public_attachment(data.get('attachment_info')), + 'message_id': data.get('message_id'), + 'is_error': data.get('is_error', False), + 'interrupted': data.get('interrupted', False), + 'reason': data.get('reason'), + } + specs.append(('chat', 'done', done)) + if data.get('response') and not data.get('is_error'): + specs.append(('status', 'agent_turn_complete', { + 'agent_id': agent_id or '', + 'agent_name': data.get('agent_name', ''), + 'response': data.get('response', ''), + 'session_id': session_id or '', + 'external_user_id': data.get('external_user_id', ''), + })) + elif event_name == 'agent_busy_changed': + specs.append(('status', 'agent_busy_changed', { + 'agent_id': agent_id or '', 'busy': data.get('busy', False), + 'session_id': session_id or '', + 'session_ids': data.get('session_ids', [session_id] if session_id else []), + 'active_count': data.get('active_count', 1 if data.get('busy') else 0), + 'state': data.get('state', 'running' if data.get('busy') else 'idle'), + })) + elif event_name == 'message_received': + specs.append(('chat', 'message_received', { + 'message': _public_message(data.get('message', '')), + 'content': _public_message(data.get('message', '')), + 'role': data.get('role', 'user'), + 'message_id': data.get('message_id'), + 'client_message_id': data.get('client_message_id') or metadata.get('client_message_id'), + 'metadata': metadata, + 'external_user_id': data.get('external_user_id', ''), + 'sender': data.get('sender') or data.get('external_user_id', ''), + })) + elif event_name in {'message_injected', 'message_injection_applied'}: + specs.append(('chat', event_name, { + 'message': data.get('message', ''), 'content': data.get('content', ''), + 'count': data.get('count', 1), + })) + elif event_name == 'llm_retry': + specs.append(('chat', 'retry', { + 'retry_count': data.get('retry_count', 0), + 'max_retries': data.get('max_retries', 0), + 'error_type': data.get('error_type', ''), + 'message': data.get('user_message', ''), + })) + elif event_name in {'approval_required', 'approval_resolved'}: + if event_name == 'approval_required': + payload = { + 'approval_id': data.get('approval_id', ''), + 'agent_id': agent_id or '', + 'source_agent_id': data.get('source_agent_id', ''), + 'source_agent_name': data.get('source_agent_name', ''), + 'tool': data.get('tool_name', ''), 'args': data.get('tool_args', {}), + 'approval_info': data.get('approval_info', {}), + 'reasons': data.get('reasons', []), 'score': data.get('score'), + } + else: + payload = { + 'approval_id': data.get('approval_id', ''), + 'decision': data.get('decision', ''), + 'timed_out': data.get('timed_out', False), + } + specs.append(('approvals', event_name, payload)) + elif event_name in {'whatsapp_bridge_status', 'panel_updated'}: + specs.append(('status', event_name, { + key: data.get(key, '') for key in ('agent_id', 'channel_id', 'status') + if key in data + })) + elif event_name in { + 'connector_connected', 'connector_disconnected', 'connector_paired', + 'workplace_status_changed', + }: + specs.append(('workplace', event_name, { + key: value for key, value in data.items() if not key.startswith('_') + })) + elif event_name in {'update_status', 'update_done'}: + specs.append(('update', event_name, { + key: value for key, value in data.items() if not key.startswith('_') + })) + elif event_name == 'whatsapp_restriction_warning': + specs.append(('chat', event_name, { + 'content': data.get('content', ''), 'metadata': metadata, + })) + elif event_name == 'session_clear': + specs.append(('chat', event_name, { + 'session_id': session_id or '', 'agent_id': agent_id or '', + })) + elif event_name == 'turn_split': + specs.append(('chat', event_name, {})) + elif event_name == 'evonic:agent-state-changed': + specs.append(('chat', 'state_changed', { + 'agent_id': agent_id or '', 'session_id': session_id or '', + })) + + if not specs: + return [] + turn_id = ( + data.get('turn_id') if 'turn_id' in data + else (realtime_store.current_turn_id(session_id) if session_id else None) + ) + ids = [] + for channel, public_name, payload in specs: + ids.append(realtime_store.publish( + channel, public_name, payload, + agent_id=agent_id, session_id=session_id, + workplace_id=workplace_id, turn_id=turn_id, + )) + return ids diff --git a/backend/scheduler.py b/backend/scheduler.py index 64c3be22..f227c7f7 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -759,8 +759,21 @@ def _action_static_message(self, config: dict): if external_user_id != '__scheduler__' and channel_id: session_id = main_db.get_or_create_session( agent_id, external_user_id, channel_id) - main_db.add_chat_message( + message_id = main_db.add_chat_message( session_id, 'assistant', message, agent_id=agent_id) + message_id = message_id if type(message_id) in (int, str) else None + from models.chatlog import chatlog_manager + chatlog_manager.get(agent_id, session_id).append({ + 'type': 'final', 'session_id': session_id, + 'content': message, 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': agent_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': message, 'message_id': message_id, + 'role': 'assistant', 'sender': 'scheduler', + }) # Push via channel (Telegram, etc.) so the user sees it immediately. # Only return on successful delivery — if the channel is down or diff --git a/backend/slash_commands.py b/backend/slash_commands.py index b84432e4..0131dd6f 100644 --- a/backend/slash_commands.py +++ b/backend/slash_commands.py @@ -235,7 +235,6 @@ def clear_handler( no_archive = not archive_requested db.clear_session(session_id, agent_id, no_archive=no_archive) - # Clear in-memory loaded skill state so skill badges disappear from session state UI from backend.agent_runtime import agent_runtime agent_runtime._session_skill_mds.pop(session_id, None) @@ -277,7 +276,9 @@ def clear_handler( # Emit session_clear event try: from backend.event_stream import event_stream - event_stream.emit('session_clear', {'session_id': session_id, 'agent_id': agent_id}) + event_stream.emit('session_clear', { + 'session_id': session_id, 'agent_id': agent_id, 'turn_id': None, + }) except Exception: pass diff --git a/backend/tools/agent_messaging.py b/backend/tools/agent_messaging.py index afcab6e9..dc21029f 100644 --- a/backend/tools/agent_messaging.py +++ b/backend/tools/agent_messaging.py @@ -121,6 +121,8 @@ def _get_message_depth(agent_context: dict) -> int: "Send a message to another agent on this platform. " "The message is delivered asynchronously — the target agent will process it " "and their reply will be automatically forwarded back to you (fire-and-forget). " + "Call this tool directly; do not invoke Evonic's internal messaging executor " + "through bash, Python, or another tool. " "Use this for delegation, collaboration, or requesting specialist help. " "By default, the message is delivered to the agent's inter-agent session " "(__agent__<sender-id>). Pass 'session' to target a specific session." @@ -501,19 +503,38 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: if session_id: metadata['session_id'] = session_id + # Register synchronous internal waits before dispatch so an immediate reply + # cannot beat the waiter into the registry. This option is intentionally not + # exposed in the public tool schema. + wait_for_reply = bool(args.get('wait_for_reply', False)) + wait_timeout = _WAIT_TIMEOUT_DEFAULT + reply_queue = None + if wait_for_reply: + wait_timeout = int(args.get('wait_timeout', _WAIT_TIMEOUT_DEFAULT)) + wait_timeout = max(_WAIT_TIMEOUT_MIN, min(wait_timeout, _WAIT_TIMEOUT_MAX)) + reply_queue = Queue() + with _WAIT_REGISTRY_LOCK: + _WAIT_REGISTRY[reply_to_id] = reply_queue + # Deliver via notify_agent (handles routing, dedup, and LLM triggering) from backend.agent_runtime.notifier import notify_agent target_session = args.get('session', '').strip() if args.get('session') else None - result = notify_agent( - agent_id=target_id, - tag=f"AGENT/{sender_name}", - message=message, - external_user_id=(f"{_AGENT_MSG_PREFIX}{sender_id}" if not target_session else None), - channel_id=None, - session_id=target_session, - dedup=False, - metadata=metadata, - ) + try: + result = notify_agent( + agent_id=target_id, + tag=f"AGENT/{sender_name}", + message=message, + external_user_id=(f"{_AGENT_MSG_PREFIX}{sender_id}" if not target_session else None), + channel_id=None, + session_id=target_session, + dedup=False, + metadata=metadata, + ) + except Exception: + if wait_for_reply: + with _WAIT_REGISTRY_LOCK: + _WAIT_REGISTRY.pop(reply_to_id, None) + raise _logger.info( "Agent message sent: '%s' → '%s' (depth=%d, reply_to=%s, report_to=%s, " @@ -523,6 +544,9 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: ) if not result.get('success'): + if wait_for_reply: + with _WAIT_REGISTRY_LOCK: + _WAIT_REGISTRY.pop(reply_to_id, None) reason = result.get('reason', 'unknown') _logger.error( "Agent message FAILED: '%s' → '%s', notify_agent reason=%s, result=%s", @@ -535,15 +559,7 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: } # ---- wait_for_reply (internal-only, NOT in tool definition) ---- - wait_for_reply = args.get('wait_for_reply', False) if wait_for_reply: - wait_timeout = int(args.get('wait_timeout', _WAIT_TIMEOUT_DEFAULT)) - wait_timeout = max(_WAIT_TIMEOUT_MIN, min(wait_timeout, _WAIT_TIMEOUT_MAX)) - reply_queue: Queue = Queue() - - with _WAIT_REGISTRY_LOCK: - _WAIT_REGISTRY[reply_to_id] = reply_queue - _logger.info( "Agent '%s' waiting for reply from '%s' (reply_to=%s, timeout=%ds).", sender_id, target_id, reply_to_id, wait_timeout, @@ -551,7 +567,9 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: wait_start = time.time() try: - answer = reply_queue.get(timeout=wait_timeout) + from backend.agent_runtime.concurrency import paused_model_gate + with paused_model_gate(): + answer = reply_queue.get(timeout=wait_timeout) elapsed = time.time() - wait_start _logger.info( "Agent '%s' received reply from '%s' after %.1fs (reply_to=%s).", @@ -574,8 +592,6 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: "Agent '%s' wait_for_reply timed out after %ds for '%s' (reply_to=%s).", sender_id, wait_timeout, target_id, reply_to_id, ) - with _WAIT_REGISTRY_LOCK: - _WAIT_REGISTRY.pop(reply_to_id, None) return { 'success': True, 'wait_for_reply': True, @@ -592,8 +608,6 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: "Agent '%s' wait_for_reply error for '%s': %s", sender_id, target_id, e, ) - with _WAIT_REGISTRY_LOCK: - _WAIT_REGISTRY.pop(reply_to_id, None) return { 'success': True, 'wait_for_reply': True, @@ -603,6 +617,9 @@ def _exec_send_agent_message(args: dict, agent_context: dict) -> dict: "The reply will arrive in your next turn." ), } + finally: + with _WAIT_REGISTRY_LOCK: + _WAIT_REGISTRY.pop(reply_to_id, None) return { 'success': True, @@ -813,19 +830,36 @@ def _on_final_answer(data: dict) -> None: ) return - if not meta or meta.get('from_agent_id') != sender_id or not meta.get('report_to_id'): + if not meta or meta.get('from_agent_id') != sender_id: + _logger.warning( + "Auto-forward skip: no matching request metadata found for sender '%s' in session '%s'.", + sender_id, session_id, + ) + return + + reply_to_id = meta.get('reply_to_id') + if reply_to_id: + with _WAIT_REGISTRY_LOCK: + reply_queue = _WAIT_REGISTRY.pop(reply_to_id, None) + if reply_queue is not None: + reply_queue.put_nowait(answer) + _logger.info( + "Wait registry: woke up waiter for reply_to=%s (sender=%s, agent_b=%s).", + reply_to_id, sender_id, agent_b_id, + ) + + report_to_id = meta.get('report_to_id') + if not report_to_id: _logger.warning( - "Auto-forward skip: no routable request metadata found for sender '%s' in session '%s'.", + "Auto-forward skip: request from '%s' in session '%s' has no human route.", sender_id, session_id, ) return - report_to_id = meta['report_to_id'] report_to_channel_id = meta.get('report_to_channel_id') or None session_id_from_meta = meta.get('session_id') original_depth = meta.get('agent_message_depth', 0) subagent_user_direct = meta.get('subagent_user_direct', False) - reply_to_id = meta.get('reply_to_id') skip_auto_forward = meta.get('skip_auto_forward', False) if skip_auto_forward: @@ -915,23 +949,6 @@ def _on_final_answer(data: dict) -> None: "Auto-forward failed for '%s' → '%s': %s", agent_b_id, sender_id, e, ) - # ---- wake-up: signal any waiting send_agent_message(wait_for_reply=true) ---- - if reply_to_id: - with _WAIT_REGISTRY_LOCK: - q = _WAIT_REGISTRY.pop(reply_to_id, None) - if q is not None: - try: - q.put(answer, timeout=1) - _logger.info( - "Wait registry: woke up waiter for reply_to=%s (sender=%s, agent_b=%s).", - reply_to_id, sender_id, agent_b_id, - ) - except Exception: - _logger.warning( - "Wait registry: failed to wake waiter for reply_to=%s (full queue?).", - reply_to_id, - ) - # NOTE: _on_final_answer listener is registered in # backend/agent_runtime/__init__.py at startup, not here, # so it fires regardless of whether agent_messaging tools are loaded. @@ -1116,17 +1133,27 @@ def _exec_send_channel_message(args: dict, agent_context: dict) -> dict: # ---- Record in chat log ---- try: - db.add_chat_message( + message_id = db.add_chat_message( session_id, 'assistant', message, agent_id=sender_id, metadata={'channel_send': True}, ) + message_id = message_id if type(message_id) in (int, str) else None from models.chatlog import chatlog_manager chatlog_manager.get(sender_id, session_id).append({ 'type': 'final', 'session_id': session_id, 'content': message, 'metadata': {'channel_send': True}, + 'message_id': message_id, + }) + from backend.event_stream import event_stream + event_stream.emit('message_received', { + 'agent_id': sender_id, 'session_id': session_id, + 'external_user_id': external_user_id, 'channel_id': channel_id, + 'message': message, 'message_id': message_id, + 'metadata': {'channel_send': True}, + 'role': 'assistant', 'sender': sender_id, }) except Exception as e: _logger.warning("send_channel_message: chat log error: %s", e) diff --git a/backend/update_manager.py b/backend/update_manager.py index 5de87940..a7dc5f5f 100644 --- a/backend/update_manager.py +++ b/backend/update_manager.py @@ -2,7 +2,7 @@ Server-side update state manager. Provides daily-cached update checks, background update execution with log -capture, and SSE listener management for real-time web UI notifications. +capture, and durable realtime notifications for the web UI. Progress state is persisted to disk to survive crashes and restarts. @@ -13,7 +13,6 @@ import json import logging import os -import queue import re import subprocess import sys @@ -195,7 +194,6 @@ def _persist_state(state: dict) -> None: # --------------------------------------------------------------------------- _lock = threading.Lock() -_listeners: list = [] # list of queue.Queue, one per SSE client # Total pipeline steps: fetch + reset + reinstall deps + doctor --fix + smoke test TOTAL_STEPS = 5 @@ -250,10 +248,10 @@ def _persist_state(state: dict) -> None: # --------------------------------------------------------------------------- -# SSE listener helpers +# Realtime publishing # --------------------------------------------------------------------------- -def _append_log(level: str, message: str): +def _append_log(level: str, message: str, *, terminal: bool = False): entry = { 'ts': datetime.now().strftime('%H:%M:%S'), 'level': level, @@ -263,56 +261,18 @@ def _append_log(level: str, message: str): _state['logs'].append(entry) # Persist state after log update _persist_state(_state) - _notify_listeners() + _publish_status(terminal=terminal) -def _notify_listeners(): +def _publish_status(*, terminal: bool = False): snapshot = get_status() - dead = [] - for q in _listeners: - try: - q.put_nowait(snapshot) - except queue.Full: - dead.append(q) - for q in dead: - try: - _listeners.remove(q) - except ValueError: - pass - - -def register_listener() -> queue.Queue: - q = queue.Queue(maxsize=200) - _listeners.append(q) - return q - - -def unregister_listener(q: queue.Queue): try: - _listeners.remove(q) - except ValueError: - pass - -_cleanup_started = False - - -def _start_listener_cleanup(interval: int = 600): - """Periodically prune dead listener queues to prevent unbounded list growth. - - SSE clients that disconnect without calling unregister_listener() leave - stale queue objects behind. This daemon thread calls _notify_listeners() - every ``interval`` seconds — the existing dead-queue detection in - _notify_listeners() handles removal. - """ - global _cleanup_started - if _cleanup_started: - return - _cleanup_started = True - def _cleanup_loop(): - while True: - time.sleep(interval) - _notify_listeners() - threading.Thread(target=_cleanup_loop, daemon=True, name='listener-cleanup').start() + from backend.event_stream import event_stream + event_stream.emit('update_status', snapshot) + if terminal and snapshot.get('status') in ('success', 'failed'): + event_stream.emit('update_done', {'status': snapshot['status']}) + except Exception as exc: + log.warning('Failed to publish update status: %s', exc) # --------------------------------------------------------------------------- @@ -339,13 +299,13 @@ def send_failure(self, step, total, error): with _lock: _state['status'] = 'failed' _state['error'] = str(error) - _append_log('error', f'FAILED at step {step}/{total}: {error}') + _append_log('error', f'FAILED at step {step}/{total}: {error}', terminal=True) def send_success(self, tag): with _lock: _state['status'] = 'success' _state['progress'] = 100 - _append_log('info', f'Update to {tag} successful') + _append_log('info', f'Update to {tag} successful', terminal=True) # --------------------------------------------------------------------------- @@ -786,7 +746,6 @@ def start_update(tag=None) -> dict: _persist_state(_state) _append_log('info', f'Starting update to {target}...') - _notify_listeners() t = threading.Thread(target=_run_update_thread, args=(target,), daemon=True) t.start() @@ -798,7 +757,7 @@ def _run_update_thread(target): Applies the update via the shared apply_update pipeline (git → reinstall → doctor --fix → smoke test → auto-rollback on failure). Progress is broadcast - to SSE listeners via WebNotifier at each pipeline phase so operators can see + through durable realtime events at each pipeline phase so operators can see which step is running during a long update. On success the user restarts via the existing "Restart" action (unchanged two-step flow). """ @@ -818,9 +777,7 @@ def _run_update_thread(target): with _lock: _state['status'] = 'failed' _state['error'] = str(e) - _append_log('error', f'Unexpected error: {e}') - finally: - _notify_listeners() + _append_log('error', f'Unexpected error: {e}', terminal=True) def trigger_rollback() -> dict: @@ -833,7 +790,6 @@ def trigger_rollback() -> dict: _persist_state(_state) _append_log('info', 'Starting rollback...') - _notify_listeners() def _do_rollback(): try: @@ -842,19 +798,18 @@ def _do_rollback(): with _lock: _state['status'] = 'failed' _state['error'] = result['error'] - _append_log('error', f"Rollback failed: {result['error']}") + _append_log('error', f"Rollback failed: {result['error']}", terminal=True) else: with _lock: _state['status'] = 'success' _state['step_label'] = 'Rollback complete' _state['current_version'] = _get_current_version() - _append_log('info', f"Rollback successful to {result['target'][:8]}") + _append_log('info', f"Rollback successful to {result['target'][:8]}", terminal=True) except Exception as e: with _lock: _state['status'] = 'failed' _state['error'] = str(e) - _append_log('error', f'Rollback error: {e}') - _notify_listeners() + _append_log('error', f'Rollback error: {e}', terminal=True) threading.Thread(target=_do_rollback, daemon=True).start() return {'success': True} @@ -868,7 +823,6 @@ def trigger_restart() -> dict: restarts, which was causing the 'Update complete!' banner to reappear. """ _append_log('info', 'Restart scheduled...') - _notify_listeners() # Reset state to idle BEFORE restart so the persisted state does not # carry over 'success' status after the server restarts. Must happen @@ -887,7 +841,3 @@ def trigger_restart() -> dict: from backend.restart import restart_service restart_service() return {'success': True, 'restarting': True} - - -# Start periodic listener cleanup on module import -_start_listener_cleanup() diff --git a/models/chat.py b/models/chat.py index a0c9d8c7..2bb0c294 100644 --- a/models/chat.py +++ b/models/chat.py @@ -364,11 +364,12 @@ def get_session_messages(self, session_id: str, limit: int = 50) -> List[Dict[st return rows def get_latest_agent_request_metadata(self, session_id: str, sender_agent_id: str = None) -> Optional[dict]: - """Return the newest routable agent-request metadata in a session. + """Return the newest reply-capable agent request in a session. Only user-role rows are candidates. Invalid JSON, non-agent messages, - sender mismatches, and requests without ``report_to_id`` are skipped so - later background notifications cannot shadow a routable delegation. + sender mismatches, and requests without either a human route or a + synchronous reply ID are skipped so background notifications cannot + shadow an active delegation. """ with self._connect() as conn: conn.row_factory = sqlite3.Row @@ -386,7 +387,7 @@ def get_latest_agent_request_metadata(self, session_id: str, sender_agent_id: st continue if sender_agent_id and metadata.get('from_agent_id') != sender_agent_id: continue - if metadata.get('report_to_id'): + if metadata.get('report_to_id') or metadata.get('reply_to_id'): return metadata return None diff --git a/models/mixins/chat_delegation.py b/models/mixins/chat_delegation.py index cfce6703..d1c502ba 100644 --- a/models/mixins/chat_delegation.py +++ b/models/mixins/chat_delegation.py @@ -132,6 +132,15 @@ def clear_session(self, session_id: str, agent_id: str = None, no_archive: bool llm_trace_manager.get(agent_id, session_id).clear() llm_trace_manager.evict(agent_id, session_id) self._remove_session_index(session_id) + try: + from backend.realtime_store import realtime_store + realtime_store.purge_session(session_id) + except Exception as e: + import logging + logging.getLogger(__name__).warning( + "Failed to purge realtime history for session %s: %s", + session_id, e, + ) def get_last_message_timestamp(self, session_id: str, agent_id: str = None) -> Optional[float]: """Return the unix timestamp of the most recent message in a session. @@ -201,6 +210,15 @@ def delete_session(self, session_id: str, agent_id: str = None) -> bool: pass self._refresh_session_count(agent_id) self._remove_session_index(session_id) + try: + from backend.realtime_store import realtime_store + realtime_store.purge_session(session_id) + except Exception as e: + import logging + logging.getLogger(__name__).warning( + "Failed to purge realtime history for session %s: %s", + session_id, e, + ) # Wipe attachments tied to this session (rows + on-disk files) so # they don't linger unreachable after the conversation is gone. try: @@ -473,6 +491,14 @@ def clear_all_sessions(self): logging.getLogger(__name__).warning( "Failed to clear attachments during clear_all_sessions: %s", e ) + try: + from backend.realtime_store import realtime_store + realtime_store.purge_all() + except Exception as e: + import logging + logging.getLogger(__name__).warning( + "Failed to purge realtime history while clearing sessions: %s", e + ) # ---- Long-term Memory delegation ---- diff --git a/routes/agents.py b/routes/agents.py index 184b258f..f1d7ea9a 100644 --- a/routes/agents.py +++ b/routes/agents.py @@ -9,8 +9,9 @@ import uuid import queue import logging +from urllib.parse import urlencode from typing import Dict, Any, List, Optional -from flask import Blueprint, render_template, jsonify, request, Response, session, stream_with_context, g +from flask import Blueprint, render_template, jsonify, request, Response, session, stream_with_context, g, redirect from models.db import db from models.chatlog import chatlog_manager, _DISPLAY_TYPES from backend.agent_portability import AgentPortabilityError, export_agent, import_agent, preflight_import @@ -1646,6 +1647,7 @@ def api_chat(agent_id): if request.content_type and request.content_type.startswith('multipart/form-data'): message = (request.form.get('message') or '').strip() user_id = (request.form.get('user_id') or 'anonymous').strip() + client_message_id = (request.form.get('client_message_id') or '').strip() files = [f for f in request.files.getlist('files') if f and f.filename] if not files: legacy_file = request.files.get('file') @@ -1654,8 +1656,15 @@ def api_chat(agent_id): data = request.get_json() or {} message = data.get('message', '').strip() user_id = data.get('user_id', 'anonymous') + raw_client_message_id = data.get('client_message_id') + if raw_client_message_id is not None and not isinstance(raw_client_message_id, str): + return jsonify({'error': 'Invalid client_message_id'}), 400 + client_message_id = (raw_client_message_id or '').strip() files = [] + if client_message_id and not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', client_message_id): + return jsonify({'error': 'Invalid client_message_id'}), 400 + if not message and not files: return jsonify({'error': 'Message is required'}), 400 @@ -1708,9 +1717,9 @@ def api_chat(agent_id): image_url = image_urls[0] if image_urls else None attachment_info = attachment_infos[0] if attachment_infos else None - upload_meta = None + upload_meta = {'client_message_id': client_message_id} if client_message_id else None if attachment_infos: - upload_meta = {'attachment_infos': attachment_infos} + upload_meta = dict(upload_meta or {}, attachment_infos=attachment_infos) if len(attachment_infos) == 1: upload_meta['attachment_info'] = attachment_info @@ -1721,14 +1730,24 @@ def api_chat(agent_id): metadata=upload_meta, ) if result.get('buffered'): - resp = {'success': True, 'buffered': True} + resp = { + 'success': True, 'buffered': True, + 'message_id': result.get('message_id'), + 'client_message_id': result.get('client_message_id'), + 'turn_id': result.get('turn_id'), + } if attachment_infos: resp['attachment_infos'] = attachment_infos if len(attachment_infos) == 1: resp['attachment_info'] = attachment_info return jsonify(resp) if result.get('injected'): - resp = {'success': True, 'injected': True} + resp = { + 'success': True, 'injected': True, + 'message_id': result.get('message_id'), + 'client_message_id': result.get('client_message_id'), + 'turn_id': result.get('turn_id'), + } if attachment_infos: resp['attachment_infos'] = attachment_infos if len(attachment_infos) == 1: @@ -1742,6 +1761,10 @@ def api_chat(agent_id): 'slash_command': result.get('slash_command', False), 'bash_exec': result.get('bash_exec', False), 'clear_ui': result.get('clear_ui', False), + 'message_id': result.get('message_id'), + 'response_message_id': result.get('response_message_id'), + 'client_message_id': result.get('client_message_id'), + 'turn_id': result.get('turn_id'), } if attachment_infos: resp['attachment_infos'] = attachment_infos @@ -1768,6 +1791,8 @@ def api_chat_jsonl(agent_id): Response: {"entries": [...], "has_more": bool} has_more is true when exactly `limit` entries were returned. """ + from backend.realtime_store import realtime_store + realtime_cursor = realtime_store.high_water() user_id = request.args.get('user_id', 'anonymous') session_id = request.args.get('session_id') to_ts = request.args.get('to_ts', type=int) @@ -1794,15 +1819,21 @@ def api_chat_jsonl(agent_id): # fall through to tail_by_messages instead. all_entries = chatlog.get_entries_after_ts(after_ts, types=_DISPLAY_TYPES) entries = all_entries[:limit] - return jsonify({'entries': entries, 'has_more': len(all_entries) > limit}) + response = jsonify({'entries': entries, 'has_more': len(all_entries) > limit}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response # Backward (tail) scan: entries older than to_ts, counted by logical messages entries, has_more = chatlog.tail_by_messages(limit=limit, to_ts=to_ts) - return jsonify({'entries': entries, 'has_more': has_more}) + response = jsonify({'entries': entries, 'has_more': has_more}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response @agents_bp.route('/api/agents//chat/history', methods=['GET']) def api_chat_history(agent_id): + from backend.realtime_store import realtime_store + realtime_cursor = realtime_store.high_water() user_id = request.args.get('user_id', 'anonymous') session_id = db.get_session_id(agent_id, user_id) or db.get_or_create_session(agent_id, user_id) messages = db.get_session_messages(session_id, limit=50, agent_id=agent_id) @@ -1828,7 +1859,9 @@ def api_chat_history(agent_id): if m.get('metadata'): entry['metadata'] = m['metadata'] filtered.append(entry) - return jsonify({'messages': filtered}) + response = jsonify({'messages': filtered}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response @agents_bp.route('/api/agents//chat/poll', methods=['GET']) @@ -2359,240 +2392,18 @@ def api_chat_session(agent_id): @agents_bp.route('/api/agents//chat/stream', methods=['GET']) def api_chat_stream(agent_id): - """SSE endpoint — pushes live thinking/tool events for a session to the browser. - - Still the active chat transport: the chat UI bundle has not been migrated to the - unified GET /api/realtime/stream?chat=1 (the unified chat path needs its own - seq-correctness work first). Kept intentionally; do not remove.""" + """Compatibility redirect to the durable realtime chat gateway.""" session_id = request.args.get('session_id') if not session_id: return jsonify({'error': 'session_id required'}), 400 - - from backend.event_stream import event_stream - - # Release the thread-local DB connection acquired by enforce_auth. - # This SSE thread will block for 30+s; without close() it leaks an FD. - db.close() - - # SSE connection limiting — max 5 concurrent per user/IP (FINDING-004) - from flask import session as _flask_session - from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_id = ( - f"user:{_flask_session.get('_user_id', 'admin')}" - if _flask_session.get('authenticated') - else f"ip:{request.remote_addr or '0.0.0.0'}" - ) - _sse_allowed, _sse_count = sse_register(_sse_id) - if not _sse_allowed: - return jsonify({ - 'error': 'too_many_sse_connections', - 'message': f'Maximum {SSE_MAX_CONCURRENT} concurrent SSE connections allowed.', - 'retry_after': 30, - }), 429, {'Retry-After': '30'} - - q = queue.Queue(maxsize=200) - - _SENTINEL = object() - - def _make_handler(sse_event_name, transform): - def handler(data): - if data.get('session_id') != session_id: - return - try: - payload = transform(data) - if payload is not None: - payload['seq'] = data.get('_chat_seq') - q.put_nowait((sse_event_name, payload, data.get('_chat_seq'))) - except queue.Full: - pass - return handler - - _TRANSFORMS = { - 'turn_begin': ('turn_begin', lambda d: {'ts': d.get('ts', 0)}), - 'llm_thinking': ('thinking', lambda d: {'content': d.get('thinking', '')}), - 'tool_call_started': ('tool_call_started', lambda d: { - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'param_types': d.get('param_types', {}), + after = request.args.get('after', '0') + return redirect( + '/api/realtime/stream?' + urlencode({ + 'chat': 1, 'agent_id': agent_id, + 'session_id': session_id, 'after': after, + 'legacy': 'chat', 'snapshot': 1, }), - 'tool_executed': ('tool_executed', lambda d: { - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'result': d.get('tool_result', {}), - 'error': d.get('has_error', False), - }), - 'state:changed': ('state:changed', lambda d: { - key: d[key] for key in ('mode', 'plan_file', 'tasks', 'loaded_skills') - if key in d - }), - 'tasks:auto_transition': ('tasks:auto_transition', lambda d: { - key: d[key] for key in ('task_ids', 'tasks') if key in d - }), - 'tasks:stale': ('tasks:stale', lambda d: { - key: d[key] for key in ('task_ids', 'tasks') if key in d - }), - 'llm_response_chunk': ('response_chunk', lambda d: { - 'content': d.get('content', ''), - 'is_final': d.get('is_final', False), - 'send_as_message': d.get('send_as_message', False), - }), - 'turn_complete': ('done', lambda d: { - 'thinking_duration': d.get('thinking_duration'), - 'response': d.get('response', ''), - 'slash_command': d.get('slash_command', False), - }), - 'approval_required': ('approval_required', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'agent_id': d.get('agent_id', ''), - 'source_agent_id': d.get('source_agent_id', ''), - 'source_agent_name': d.get('source_agent_name', ''), - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'approval_info': d.get('approval_info', {}), - 'reasons': d.get('reasons', []), - 'score': d.get('score'), - }), - 'approval_resolved': ('approval_resolved', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'decision': d.get('decision', ''), - 'timed_out': d.get('timed_out', False), - }), - 'llm_retry': ('retry', lambda d: { - 'retry_count': d.get('retry_count', 0), - 'max_retries': d.get('max_retries', 0), - 'error_type': d.get('error_type', ''), - 'message': d.get('user_message', ''), - }), - 'message_injected': ('message_injected', lambda d: { - 'message': d.get('message', ''), - }), - 'message_injection_applied': ('message_injection_applied', lambda d: { - 'content': d.get('content', ''), - 'count': d.get('count', 1), - }), - 'message_received': ('message_received', lambda d: { - 'message': d.get('message', ''), - 'metadata': d.get('metadata', {}), - }), - 'session_clear': ('session_clear', lambda d: { - 'session_id': d.get('session_id', ''), - 'agent_id': d.get('agent_id', ''), - }), - 'turn_split': ('turn_split', lambda d: {}), - 'evonic:agent-state-changed': ('state_changed', lambda d: { - 'agent_id': d.get('agent_id', ''), - 'session_id': d.get('session_id', ''), - }), - } - - handlers = { - event_name: _make_handler(sse_name, transform) - for event_name, (sse_name, transform) in _TRANSFORMS.items() - } - - # Client passes ?after=N when it has already replayed events 1..N via /chat/events. - # We only pre-fill the gap (events N+1..M) that arrived between the client's replay - # fetch and this SSE subscription, avoiding duplicate delivery of already-seen events. - after_seq = request.args.get('after', 0, type=int) - - # Subscribe to live events BEFORE snapshotting the buffer. This ensures no events - # are lost in the window between snapshot and subscribe. Overlap (events captured - # by both snapshot and live handler) is safely deduplicated by seq on the client. - for event_name, handler in handlers.items(): - event_stream.on(event_name, handler) - - event_stream.register_web_listener(session_id) - - # Snapshot buffered events after subscribing — any event emitted after this point - # is caught by the live handler; events before are in the snapshot. - buffered_raw = event_stream.get_session_events(session_id, after_seq) - - # Only pre-fill events from the current in-progress turn. - # Treat turn_complete and session_clear as "boundary" events — discard everything - # up to and including the last one so a fresh SSE connection never replays a - # completed turn or a past session_clear that would wipe the UI. - # IMPORTANT: Only strip on fresh connections (after_seq == 0). On reconnections - # (after_seq > 0), the client hasn't seen these events yet and needs them — - # especially turn_complete which finalizes the thinking bubble. - if after_seq == 0: - last_complete = -1 - for i, e in enumerate(buffered_raw): - if e['event'] in ('turn_complete', 'session_clear'): - last_complete = i - if last_complete >= 0: - buffered_raw = buffered_raw[last_complete + 1:] - - # Prune resolved approval cycles — if an approval_required has already been - # followed by a matching approval_resolved, discard both. Only keep the most - # recent unresolved approval (if any) so a reconnecting client never re-shows - # an approval modal that was already handled. - active_approvals = {} - discard_set = set() - for i, e in enumerate(buffered_raw): - if e['event'] == 'approval_required': - d = e.get('data', {}) - if isinstance(d, dict): - aid = d.get('approval_id', '') - if aid: - # Replace any previous unresolved approval with the same id - # (shouldn't happen, but guard against duplicates) - if aid in active_approvals: - discard_set.add(active_approvals[aid]) - active_approvals[aid] = i - elif e['event'] == 'approval_resolved': - d = e.get('data', {}) - if isinstance(d, dict): - aid = d.get('approval_id', '') - if aid and aid in active_approvals: - discard_set.add(active_approvals[aid]) - discard_set.add(i) - del active_approvals[aid] - if discard_set: - buffered_raw = [e for i, e in enumerate(buffered_raw) if i not in discard_set] - - # Pre-fill the queue with buffered events so a reconnecting client immediately - # sees the in-progress reasoning trace without waiting for the next live event. - for entry in buffered_raw: - sse_name_transform = _TRANSFORMS.get(entry['event']) - if sse_name_transform: - sse_name, transform = sse_name_transform - payload = transform(entry['data']) - payload['seq'] = entry['chat_seq'] - try: - q.put_nowait((sse_name, payload, entry['chat_seq'])) - except queue.Full: - break - - def generate(): - try: - while True: - try: - item = q.get(timeout=30) - except queue.Empty: - # No events for 30s — send a heartbeat as a real SSE event so the - # client can detect it (EventSource ignores comment-only lines). - yield "event: heartbeat\ndata: {}\n\n" - continue - sse_event, payload, seq = item - id_line = f"id: {seq}\n" if seq is not None else '' - yield f"{id_line}event: {sse_event}\ndata: {json.dumps(payload)}\n\n" - if sse_event == 'done': - break - finally: - event_stream.unregister_web_listener(session_id) - for event_name, handler in handlers.items(): - event_stream.off(event_name, handler) - # Unregister SSE connection (FINDING-004) - sse_unregister(_sse_id) - - return Response( - stream_with_context(generate()), - mimetype='text/event-stream', - headers={ - 'Cache-Control': 'no-cache', - 'X-Accel-Buffering': 'no', - 'Connection': 'keep-alive', - } + code=307, ) @@ -2601,163 +2412,47 @@ def api_approvals_stream(): """Global SSE endpoint — pushes ALL approval events (any agent, any session) to every connected client. DEPRECATED: Use unified GET /api/realtime/stream?channels=approvals instead.""" - import logging as _log_depr - _log_depr.getLogger(__name__).warning( - "DEPRECATED endpoint /api/approvals/stream used — " - "migrate to /api/realtime/stream?channels=approvals") - from backend.event_stream import event_stream - - # Release the thread-local DB connection acquired by enforce_auth. - db.close() - - # SSE connection limiting — max 5 concurrent per user/IP (FINDING-004) - from flask import session as _flask_session - from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_id = ( - f"user:{_flask_session.get('_user_id', 'admin')}" - if _flask_session.get('authenticated') - else f"ip:{request.remote_addr or '0.0.0.0'}" - ) - _sse_allowed, _sse_count = sse_register(_sse_id) - if not _sse_allowed: - return jsonify({ - 'error': 'too_many_sse_connections', - 'message': f'Maximum {SSE_MAX_CONCURRENT} concurrent SSE connections allowed.', - 'retry_after': 30, - }), 429, {'Retry-After': '30'} - - q = queue.Queue(maxsize=200) - - _TRANSFORMS = { - 'approval_required': ('approval_required', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'agent_id': d.get('agent_id', ''), - 'source_agent_id': d.get('source_agent_id', ''), - 'source_agent_name': d.get('source_agent_name', ''), - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'approval_info': d.get('approval_info', {}), - 'reasons': d.get('reasons', []), - 'score': d.get('score'), - }), - 'approval_resolved': ('approval_resolved', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'decision': d.get('decision', ''), - 'timed_out': d.get('timed_out', False), - }), - } - - def _make_handler(sse_event_name, transform): - def handler(data): - try: - payload = transform(data) - if payload is not None: - payload['seq'] = data.get('_seq') - q.put_nowait((sse_event_name, payload, data.get('_seq'))) - except queue.Full: - pass - return handler - - handlers = {} - for event_name, (sse_name, transform) in _TRANSFORMS.items(): - h = _make_handler(sse_name, transform) - handlers[event_name] = h - event_stream.on(event_name, h) - - def generate(): - try: - while True: - try: - item = q.get(timeout=30) - except queue.Empty: - yield "event: heartbeat\ndata: {}\n\n" - continue - sse_event, payload, seq = item - id_line = f"id: {seq}\n" if seq is not None else '' - yield f"{id_line}event: {sse_event}\ndata: {json.dumps(payload)}\n\n" - finally: - for event_name, handler in handlers.items(): - event_stream.off(event_name, handler) - # Unregister SSE connection (FINDING-004) - sse_unregister(_sse_id) - - return Response( - stream_with_context(generate()), - mimetype='text/event-stream', - headers={ - 'Cache-Control': 'no-cache', - 'X-Accel-Buffering': 'no', - 'Connection': 'keep-alive', - } + return redirect( + '/api/realtime/stream?channels=approvals&legacy=approvals&snapshot=1', + code=307, ) @agents_bp.route('/api/agents//chat/events', methods=['GET']) def api_chat_events(agent_id): - """Fetch missed SSE events by sequence range for gap-detection recovery.""" + """Compatibility reader backed by the durable realtime journal.""" session_id = request.args.get('session_id') - after_seq = request.args.get('after', type=int) - up_to_seq = request.args.get('up_to', type=int) - if not session_id or after_seq is None: + if not session_id or request.args.get('after') is None: return jsonify({'error': 'session_id and after required'}), 400 + from backend.realtime_store import realtime_store + if request.args.get('cursor_version') != '2': + return jsonify({ + 'events': [], 'reset': True, + 'cursor': realtime_store.high_water(), + }) + try: + after_seq = int(request.args['after']) + raw_up_to = request.args.get('up_to') + up_to_seq = int(raw_up_to) if raw_up_to is not None else None + except (TypeError, ValueError): + return jsonify({'error': 'Invalid realtime cursor'}), 400 + max_cursor = 2**63 - 1 + if not 0 <= after_seq <= max_cursor or ( + up_to_seq is not None and not 0 <= up_to_seq <= max_cursor): + return jsonify({'error': 'Invalid realtime cursor'}), 400 if up_to_seq is not None and up_to_seq - after_seq > 200: return jsonify({'error': 'range too large (max 200)'}), 400 - from backend.event_stream import event_stream - - _TRANSFORM_MAP = { - 'turn_begin': ('turn_begin', lambda d: {'ts': d.get('ts', 0)}), - 'llm_thinking': ('thinking', lambda d: {'content': d.get('thinking', '')}), - 'tool_call_started': ('tool_call_started', lambda d: {'tool': d.get('tool_name', ''), 'args': d.get('tool_args', {}), 'param_types': d.get('param_types', {})}), - 'tool_executed': ('tool_executed', lambda d: {'tool': d.get('tool_name', ''), 'args': d.get('tool_args', {}), 'result': d.get('tool_result', {}), 'error': d.get('has_error', False)}), - 'state:changed': ('state:changed', lambda d: {key: d[key] for key in ('mode', 'plan_file', 'tasks', 'loaded_skills') if key in d}), - 'tasks:auto_transition': ('tasks:auto_transition', lambda d: {key: d[key] for key in ('task_ids', 'tasks') if key in d}), - 'tasks:stale': ('tasks:stale', lambda d: {key: d[key] for key in ('task_ids', 'tasks') if key in d}), - 'llm_response_chunk':('response_chunk', lambda d: {'content': d.get('content', ''), 'is_final': d.get('is_final', False), 'send_as_message': d.get('send_as_message', False)}), - 'turn_complete': ('done', lambda d: { - 'thinking_duration': d.get('thinking_duration'), - 'response': d.get('response', ''), - 'slash_command': d.get('slash_command', False), - }), - 'approval_required': ('approval_required', lambda d: {'approval_id': d.get('approval_id', ''), 'agent_id': d.get('agent_id', ''), 'source_agent_id': d.get('source_agent_id', ''), 'source_agent_name': d.get('source_agent_name', ''), 'tool': d.get('tool_name', ''), 'args': d.get('tool_args', {}), 'approval_info': d.get('approval_info', {}), 'reasons': d.get('reasons', []), 'score': d.get('score')}), - 'approval_resolved': ('approval_resolved', lambda d: {'approval_id': d.get('approval_id', ''), 'decision': d.get('decision', ''), 'timed_out': d.get('timed_out', False)}), - 'llm_retry': ('retry', lambda d: {'retry_count': d.get('retry_count', 0), 'max_retries': d.get('max_retries', 0), 'error_type': d.get('error_type', ''), 'message': d.get('user_message', '')}), - 'message_injected': ('message_injected', lambda d: {'message': d.get('message', '')}), - 'message_injection_applied': ('message_injection_applied', lambda d: {'content': d.get('content', ''), 'count': d.get('count', 1)}), - 'message_received': ('message_received', lambda d: {'message': d.get('message', ''), 'metadata': d.get('metadata', {})}), - 'session_clear': ('session_clear', lambda d: {'session_id': d.get('session_id', ''), 'agent_id': d.get('agent_id', '')}), - 'turn_split': ('turn_split', lambda d: {}), - 'evonic:agent-state-changed': ('state_changed', lambda d: {'agent_id': d.get('agent_id', ''), 'session_id': d.get('session_id', '')}), - } - - if up_to_seq is None: - raw = event_stream.get_session_events(session_id, after_seq) - else: - raw = event_stream.get_events_in_range(session_id, after_seq, up_to_seq) - - # Strip boundary events (turn_complete, session_clear) on fresh requests so - # restoreActiveReasoning() never replays completed turns or past session_clear - # events that would create a stale thinking bubble. Mirror the SSE stream logic - # at lines 1668-1674. Only strip when after_seq==0; on gap-fill reconnects - # (after_seq>0), the client hasn't seen these events and needs them. - if after_seq == 0: - last_boundary = -1 - for i, e in enumerate(raw): - if e['event'] in ('turn_complete', 'session_clear'): - last_boundary = i - if last_boundary >= 0: - raw = raw[last_boundary + 1:] - - events = [] - for entry in raw: - event_name = entry['event'] - if event_name in _TRANSFORM_MAP: - sse_name, transform = _TRANSFORM_MAP[event_name] - payload = transform(entry['data']) - payload['seq'] = entry['chat_seq'] - events.append({'event': sse_name, 'seq': entry['chat_seq'], 'data': payload}) - - return jsonify({'events': events}) + active_only = after_seq == 0 and up_to_seq is None + rows = realtime_store.events_after( + after_seq, {'chat'}, session_id=session_id, agent_id=agent_id, + up_to_id=up_to_seq, active_only=active_only, + limit=200 if up_to_seq is not None else 5000, + ) + return jsonify({'events': [ + {'event': row['event'], 'seq': row['id'], 'data': row['data']} + for row in rows + ]}) @agents_bp.route('/api/agents//chat/approve', methods=['POST']) @@ -2831,89 +2526,9 @@ def api_agents_status_stream(): data: {"agent_id": "...", "agent_name": "...", "response": "...", "external_user_id": "...", "session_id": "..."} """ - import logging as _log_depr - _log_depr.getLogger(__name__).warning( - "DEPRECATED endpoint /api/agents/status/stream used — " - "migrate to /api/realtime/stream?channels=status") - import queue as _queue - from backend.event_stream import event_stream - - # Release the thread-local DB connection acquired by enforce_auth. - db.close() - - # SSE connection limiting (max 5 concurrent per user/IP, FINDING-004) - from flask import session as _flsk_sess - from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_ident = ( - 'user:' + (_flsk_sess.get('_user_id', 'admin') if _flsk_sess.get('authenticated') else '') - if _flsk_sess.get('authenticated') - else 'ip:' + (request.remote_addr or '0.0.0.0') - ) - _ok, _cnt = sse_register(_sse_ident) - if not _ok: - return jsonify({ - 'error': 'too_many_sse_connections', - 'message': 'Maximum ' + str(SSE_MAX_CONCURRENT) + ' concurrent SSE connections allowed.', - 'retry_after': 30, - }), 429, {'Retry-After': '30'} - - q = _queue.Queue(maxsize=200) - - def busy_handler(data): - try: - payload = { - 'agent_id': data.get('agent_id', ''), - 'busy': data.get('busy', False), - 'session_id': data.get('session_id', ''), - } - q.put_nowait(('busy', payload)) - except _queue.Full: - pass - - def turn_handler(data): - try: - response = data.get('response', '') - if not response or data.get('is_error'): - return - payload = { - 'agent_id': data.get('agent_id', ''), - 'agent_name': data.get('agent_name', ''), - 'response': response, - 'session_id': data.get('session_id', ''), - 'external_user_id': data.get('external_user_id', ''), - } - q.put_nowait(('turn', payload)) - except _queue.Full: - pass - - event_stream.on('agent_busy_changed', busy_handler) - event_stream.on('turn_complete', turn_handler) - - def generate(): - try: - while True: - try: - kind, payload = q.get(timeout=30) - except _queue.Empty: - yield 'event: heartbeat\ndata: {}\n\n' - continue - if kind == 'busy': - yield f'event: agent_busy_changed\ndata: {json.dumps(payload)}\n\n' - elif kind == 'turn': - yield f'event: agent_turn_complete\ndata: {json.dumps(payload)}\n\n' - finally: - event_stream.off('agent_busy_changed', busy_handler) - event_stream.off('turn_complete', turn_handler) - sse_unregister(_sse_ident) - - return Response( - stream_with_context(generate()), - mimetype='text/event-stream', - headers={ - 'Cache-Control': 'no-cache', - 'X-Accel-Buffering': 'no', - 'Connection': 'keep-alive', - } + return redirect( + '/api/realtime/stream?channels=status&legacy=status&snapshot=1', + code=307, ) diff --git a/routes/realtime.py b/routes/realtime.py index c027777f..a2b4139a 100644 --- a/routes/realtime.py +++ b/routes/realtime.py @@ -1,1201 +1,367 @@ -""" -Unified Real-Time SSE Endpoint — consolidates 5 separate SSE connections -into 1 multiplexed connection with per-channel priority queuing. +"""Durable, multiplexed Server-Sent Events gateway.""" -Endpoint: GET /api/realtime/stream +from __future__ import annotations -Query parameters (opt-in channels): - channels — comma-separated: status,approvals,update - chat — 1 to include per-session chat events - session_id — chat session ID (required when chat=1) - agent_id — agent ID (required when chat=1) - after — chat event resume seq - workplace — workplace ID for connector events - chat_throttle — throttle interval ms for chat events (default 100) -""" - -import collections import json import logging -import math -import os -import queue -import random -import signal -import socket -import threading import time -from datetime import datetime, timedelta +from datetime import datetime from flask import Blueprint, Response, request, stream_with_context -log = logging.getLogger(__name__) - -realtime_bp = Blueprint('realtime', __name__) - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -RING_SIZES = { - 'chat': 256, - 'approvals': 8, - 'status': 32, - 'update': 16, - 'workplace': 16, -} - -RING_STRATEGIES = { - 'chat': 'drop_oldest', - 'approvals': 'drop_oldest', # on overflow keep the newest approval, never drop it - 'status': 'drop_oldest', - 'update': 'drop_oldest', - 'workplace': 'drop_oldest', -} - -CHANNEL_PRIORITY = { - 'update': 0, # highest — small, rare, must be fast - 'status': 0, - 'approvals': 1, # user-facing modal - 'chat': 2, # high throughput, tolerable delay - 'workplace': 2, # high throughput, tolerable delay -} - -# Weighted round-robin: 1 L0/L1 event per 5 L2 events -L2_WEIGHT = 5 - -HEARTBEAT_INTERVAL = 15 # seconds -HEARTBEAT_MAX_FAILURES = 3 -TCP_KEEPIDLE = 60 -TCP_KEEPINTVL = 10 -TCP_KEEPCNT = 3 - -CIRCUIT_BREAKER_WINDOW = 10 # seconds -CIRCUIT_BREAKER_THRESHOLD = 3 -CIRCUIT_BREAKER_COOLDOWN = 60 - -# Per-channel buffer for pause/resume -PAUSE_BUFFER = { - 'chat': 64, - 'workplace': 16, -} - -# --------------------------------------------------------------------------- -# Bounded Ring Buffer -# --------------------------------------------------------------------------- - -class BoundedRing: - """Thread-safe bounded queue with configurable overflow strategy.""" - - def __init__(self, channel: str, maxlen: int, strategy: str): - self.channel = channel - self.maxlen = maxlen - self.strategy = strategy - self._lock = threading.Lock() - self._q = collections.deque(maxlen=maxlen) - self._dropped_count = 0 - self._seq = 0 - - def put(self, item): - """Put an item. Returns (inserted, dropped_count_for_this_put).""" - dropped = 0 - with self._lock: - if len(self._q) >= self.maxlen: - if self.strategy == 'drop_oldest': - self._q.popleft() - dropped = 1 - elif self.strategy == 'drop_newest': - dropped = 1 - # don't actually enqueue — drop the new item - self._dropped_count += 1 - return (False, 1) - self._seq += 1 - self._q.append((self._seq, item)) - if dropped: - self._dropped_count += 1 - return (True, dropped) - - def get(self): - """Get the oldest item, or None if empty.""" - with self._lock: - if self._q: - return self._q.popleft() - return None - - def get_many(self, max_count: int): - """Get up to max_count items.""" - items = [] - with self._lock: - while self._q and len(items) < max_count: - items.append(self._q.popleft()) - return items - - def get_all(self): - """Get all items.""" - with self._lock: - items = list(self._q) - self._q.clear() - return items - - def drain_dropped(self) -> int: - """Atomically read and reset dropped count.""" - with self._lock: - c = self._dropped_count - self._dropped_count = 0 - return c - - def peek_all(self): - """Return all items without dequeuing (for snapshot).""" - with self._lock: - return list(self._q) - - def size(self): - with self._lock: - return len(self._q) - - -# --------------------------------------------------------------------------- -# Circuit Breaker -# --------------------------------------------------------------------------- +from backend.realtime_store import RETENTION_MS, realtime_store -class CircuitBreaker: - """Per-channel circuit breaker with sliding window crash tracking.""" - def __init__(self, channel: str): - self.channel = channel - self._lock = threading.Lock() - self._crashes = [] # list of crash timestamps - self._open_since = None - self._disabled = False - - def record_crash(self) -> bool: - """Record a crash. Returns True if circuit should open (stop restarting).""" - now = time.time() - with self._lock: - # Clean old entries outside window - cutoff = now - CIRCUIT_BREAKER_WINDOW - self._crashes = [t for t in self._crashes if t > cutoff] - self._crashes.append(now) - if len(self._crashes) >= CIRCUIT_BREAKER_THRESHOLD: - self._open_since = now - self._disabled = True - return True - return False - - def is_disabled(self) -> bool: - with self._lock: - if not self._disabled: - return False - # Check cooldown - if self._open_since and (time.time() - self._open_since) > CIRCUIT_BREAKER_COOLDOWN: - self._disabled = False - self._crashes = [] - self._open_since = None - return False - return True - - def reset(self): - with self._lock: - self._crashes = [] - self._open_since = None - self._disabled = False - - -# --------------------------------------------------------------------------- -# Connection State -# --------------------------------------------------------------------------- - -# Global registry of active connections (for pause/resume by session) -_connections: dict = {} # key: connection_id -> RealtimeConnection -_conn_lock = threading.Lock() - -# Heartbeat-aware web SSE delivery check: maximum age (seconds) of the -# last heartbeat before a connection is considered disconnected. -WEB_SSE_HEARTBEAT_MAX_AGE = 45 # 3 × HEARTBEAT_INTERVAL - - -def has_active_web_sse(session_id: str) -> bool: - """Return True if any SSE connection for *session_id* has had a - heartbeat within WEB_SSE_HEARTBEAT_MAX_AGE seconds. - - This is more reliable than has_web_listener() which only checks - listener registration, not actual delivery. - """ - now = time.monotonic() - with _conn_lock: - for conn in _connections.values(): - if conn.chat_session_id == session_id: - if (conn.last_heartbeat_time > 0 and - (now - conn.last_heartbeat_time) < WEB_SSE_HEARTBEAT_MAX_AGE): - return True - return False - - -class RealtimeConnection: - """Per-connection state for the unified SSE stream.""" - - def __init__(self, conn_id: str, channels: set, chat_session_id: str = None, - agent_id: str = None, after_seq: int = 0, - workplace_id: str = None, chat_throttle_ms: int = 100, - expires_at: float = None): - self.conn_id = conn_id - self.channels = channels - self.chat_session_id = chat_session_id - self.agent_id = agent_id - self.after_seq = after_seq - self.last_global_seq = after_seq - self.last_chat_seq = after_seq - self.workplace_id = workplace_id - self.chat_throttle_ms = chat_throttle_ms - self.expires_at = expires_at - self.paused = False - self._stop_event = threading.Event() - # Per-channel pause buffers - self._pause_buffers: dict[str, BoundedRing] = {} - self.last_write_ok = True - # Reference to per-channel rings (set by api_realtime_stream) - # so api_realtime_resume can flush pause buffers back into them. - self.rings: dict[str, BoundedRing] | None = None - # Last heartbeat timestamp (monotonic). Updated by the generator. - # Used by has_active_web_listener() to verify delivery. - self.last_heartbeat_time: float = 0.0 - - def stop(self): - self._stop_event.set() - - def is_stopped(self) -> bool: - return self._stop_event.is_set() - - def pause(self): - self.paused = True - - def resume(self): - self.paused = False - - def check_expired(self) -> bool: - if self.expires_at and time.time() > self.expires_at: - return True - return False +log = logging.getLogger(__name__) +realtime_bp = Blueprint('realtime', __name__) +HEARTBEAT_INTERVAL = 15 +SQLITE_MAX_ID = 2**63 - 1 +_ALLOWED_CHANNELS = {'chat', 'status', 'approvals', 'update', 'workplace'} -# --------------------------------------------------------------------------- -# SSE formatting helpers -# --------------------------------------------------------------------------- -def _json_default(obj): - """Custom JSON default handler: convert non-serializable types to strings.""" - if isinstance(obj, datetime): - return obj.isoformat() - log.warning("_format_sse_event: non-serializable type %s in SSE data", type(obj).__name__) - return str(obj) +def _json_default(value): + if isinstance(value, datetime): + return value.isoformat() + return str(value) -def _format_sse_event(event_name: str, data: dict, seq_id: str = None, - global_seq: int = None) -> str: - """Format a single SSE event with optional id and event fields.""" +def _format_sse_event(event_name: str, data: dict, + event_id: int | str | None = None) -> str: lines = [] - if seq_id: - lines.append(f"id: {seq_id}") - elif global_seq is not None: - lines.append(f"id: {global_seq}") + if event_id is not None: + lines.append(f'id: {event_id}') if event_name: - lines.append(f"event: {event_name}") - lines.append(f"data: {json.dumps(data, separators=(',', ':'), default=_json_default)}") - return "\n".join(lines) + "\n\n" - + lines.append(f'event: {event_name}') + lines.append('data: ' + json.dumps( + data, separators=(',', ':'), default=_json_default, + )) + return '\n'.join(lines) + '\n\n' -def _format_sse_comment(comment: str) -> str: - """Format an SSE comment line.""" - return f": {comment}\n\n" - -# --------------------------------------------------------------------------- -# State snapshot (atomic subscribe) -# --------------------------------------------------------------------------- - -# Agent busy-status snapshot, cached briefly so the multiple SSE connections -# a page opens at load share one db.get_agents() read. Staleness is bounded: -# changes after the snapshot arrive via agent_busy_changed events anyway. -_status_snapshot_cache = {'ts': 0.0, 'events': None} -_status_snapshot_lock = threading.Lock() -_STATUS_SNAPSHOT_TTL = 2.0 +def _parse_cursor(value) -> int | None: + text = str(value).strip() if value is not None else '' + if not text: + return None + try: + cursor = int(text) + except (TypeError, ValueError): + return None + return cursor if 0 <= cursor <= SQLITE_MAX_ID else None -def _get_status_snapshot_events() -> list: - from models.db import db - now = time.monotonic() - with _status_snapshot_lock: - cached = _status_snapshot_cache['events'] - if cached is not None and now - _status_snapshot_cache['ts'] < _STATUS_SNAPSHOT_TTL: - return cached - events = [] - for agent in db.get_agents(): - events.append(('agent_busy_changed', { - 'agent_id': agent['id'], - 'busy': agent.get('busy', False), - 'session_id': agent.get('current_session_id', ''), - })) - _status_snapshot_cache['ts'] = now - _status_snapshot_cache['events'] = events - return events +def _snapshot_payload(channel: str, payload: dict) -> dict: + result = dict(payload) + result.setdefault('timestamp', int(time.time() * 1000)) + result.setdefault('channel', channel) + result['snapshot'] = True + return result -def _build_snapshot(channels: set, agent_id: str = None, - session_id: str = None, - workplace_id: str = None) -> list: - """Capture current state snapshot for requested channels.""" - events = [] +def _build_snapshot(channels: set[str], session_id: str | None = None, + agent_id: str | None = None, + workplace_id: str | None = None, + skip_approval_ids: set[str] | None = None + ) -> list[tuple[str, str, dict]]: + events: list[tuple[str, str, dict]] = [] + skip_approval_ids = skip_approval_ids or set() - if 'status' in channels: + if 'status' in channels or session_id: try: - events.extend(_get_status_snapshot_events()) - except Exception as e: - log.warning("realtime snapshot: failed to get agent statuses: %s", e) + from models.db import db + busy = realtime_store.busy_agents() + for agent in db.get_agents(): + if session_id and 'status' not in channels and agent['id'] != agent_id: + continue + entry = busy.get(agent['id']) + events.append(('status', 'agent_busy_changed', { + 'agent_id': agent['id'], + 'busy': bool(entry), + 'session_id': entry.get('session_id', '') if entry else '', + 'session_ids': entry.get('session_ids', []) if entry else [], + 'active_count': entry.get('active_count', 0) if entry else 0, + 'state': entry.get('state', 'idle') if entry else 'idle', + })) + except Exception as exc: + log.warning('realtime status snapshot failed: %s', exc) if 'approvals' in channels or session_id: - from models.db import db try: - pending = db.get_pending_tool_approvals() - for app in (pending or []): - events.append(('approval_required', { - 'approval_id': app.get('id', ''), - 'agent_id': app.get('agent_id', ''), - 'source_agent_id': app.get('source_agent_id', ''), - 'source_agent_name': app.get('source_agent_name', ''), - 'tool': app.get('tool_name', ''), - 'args': app.get('tool_args', {}), - 'approval_info': app.get('approval_info', {}), - 'reasons': app.get('reasons', []), - 'score': app.get('score'), + from models.db import db + for approval in db.get_pending_tool_approvals() or []: + approval_session = approval.get('session_id') + if str(approval.get('id', '')) in skip_approval_ids: + continue + if session_id and 'approvals' not in channels \ + and approval_session != session_id: + continue + events.append(('approvals', 'approval_required', { + 'approval_id': approval.get('id', ''), + 'agent_id': approval.get('agent_id', ''), + 'source_agent_id': approval.get('source_agent_id', ''), + 'source_agent_name': approval.get('source_agent_name', ''), + 'tool': approval.get('tool_name', ''), + 'args': approval.get('tool_args', {}), + 'approval_info': approval.get('approval_info', {}), + 'reasons': approval.get('reasons', []), + 'score': approval.get('score'), })) - except Exception as e: - log.warning("realtime snapshot: failed to get pending approvals: %s", e) + except Exception as exc: + log.warning('realtime approval snapshot failed: %s', exc) if 'update' in channels: try: - from routes.update import update_manager - status = update_manager.get_status() - events.append(('update_status', status)) - except Exception as e: - log.warning("realtime snapshot: failed to get update status: %s", e) - - return events - - -# --------------------------------------------------------------------------- -# Producer factories (per-channel) — Task isolation -# --------------------------------------------------------------------------- - -def _producer_status(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event): - """Producer: listen to agent_busy_changed, turn_complete, whatsapp_bridge_status and panel_updated events.""" - from backend.event_stream import event_stream - - def busy_handler(data): - ring.put(('agent_busy_changed', { - 'agent_id': data.get('agent_id', ''), - 'busy': data.get('busy', False), - 'session_id': data.get('session_id', ''), - })) - - def turn_handler(data): - response = data.get('response', '') - if not response or data.get('is_error'): - return - ring.put(('agent_turn_complete', { - 'agent_id': data.get('agent_id', ''), - 'agent_name': data.get('agent_name', ''), - 'response': response, - 'session_id': data.get('session_id', ''), - 'external_user_id': data.get('external_user_id', ''), - })) - - def wa_bridge_handler(data): - ring.put(('whatsapp_bridge_status', { - 'agent_id': data.get('agent_id', ''), - 'channel_id': data.get('channel_id', ''), - 'status': data.get('status', ''), - })) - - def panel_handler(data): - ring.put(('panel_updated', { - 'agent_id': data.get('agent_id', ''), - })) - - event_stream.on('agent_busy_changed', busy_handler) - event_stream.on('turn_complete', turn_handler) - event_stream.on('whatsapp_bridge_status', wa_bridge_handler) - event_stream.on('panel_updated', panel_handler) - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - event_stream.off('agent_busy_changed', busy_handler) - event_stream.off('turn_complete', turn_handler) - event_stream.off('whatsapp_bridge_status', wa_bridge_handler) - event_stream.off('panel_updated', panel_handler) - - -def _producer_approval(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event): - """Producer: listen to approval_required and approval_resolved events.""" - from backend.event_stream import event_stream - - def approval_handler(data): - ring.put(('approval_required', { - 'approval_id': data.get('approval_id', ''), - 'agent_id': data.get('agent_id', ''), - 'source_agent_id': data.get('source_agent_id', ''), - 'source_agent_name': data.get('source_agent_name', ''), - 'tool': data.get('tool_name', ''), - 'args': data.get('tool_args', {}), - 'approval_info': data.get('approval_info', {}), - 'reasons': data.get('reasons', []), - 'score': data.get('score'), - })) - - def resolved_handler(data): - ring.put(('approval_resolved', { - 'approval_id': data.get('approval_id', ''), - 'decision': data.get('decision', ''), - 'timed_out': data.get('timed_out', False), - })) - - event_stream.on('approval_required', approval_handler) - event_stream.on('approval_resolved', resolved_handler) - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - event_stream.off('approval_required', approval_handler) - event_stream.off('approval_resolved', resolved_handler) - - -def _producer_chat(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event, session_id: str, - after_seq: int = 0): - """Producer: listen to per-session chat events.""" - from backend.event_stream import event_stream - - _TRANSFORMS = { - 'turn_begin': ('turn_begin', lambda d: {'ts': d.get('ts', 0)}), - 'llm_thinking': ('thinking', lambda d: {'content': d.get('thinking', '')}), - 'tool_call_started': ('tool_call_started', lambda d: { - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'param_types': d.get('param_types', {}), - }), - 'tool_executed': ('tool_executed', lambda d: { - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'result': d.get('tool_result', {}), - 'error': d.get('has_error', False), - }), - 'state:changed': ('state:changed', lambda d: { - key: d[key] for key in ('mode', 'plan_file', 'tasks', 'loaded_skills') - if key in d - }), - 'tasks:auto_transition': ('tasks:auto_transition', lambda d: { - key: d[key] for key in ('task_ids', 'tasks') if key in d - }), - 'tasks:stale': ('tasks:stale', lambda d: { - key: d[key] for key in ('task_ids', 'tasks') if key in d - }), - 'llm_response_chunk': ('response_chunk', lambda d: { - 'content': d.get('content', ''), - 'is_final': d.get('is_final', False), - 'send_as_message': d.get('send_as_message', False), - }), - 'turn_complete': ('done', lambda d: { - 'thinking_duration': d.get('thinking_duration'), - 'response': d.get('response', ''), - 'slash_command': d.get('slash_command', False), - 'attachment_info': d.get('attachment_info'), - }), - 'approval_required': ('approval_required', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'agent_id': d.get('agent_id', ''), - 'source_agent_id': d.get('source_agent_id', ''), - 'source_agent_name': d.get('source_agent_name', ''), - 'tool': d.get('tool_name', ''), - 'args': d.get('tool_args', {}), - 'approval_info': d.get('approval_info', {}), - 'reasons': d.get('reasons', []), - 'score': d.get('score'), - }), - 'approval_resolved': ('approval_resolved', lambda d: { - 'approval_id': d.get('approval_id', ''), - 'decision': d.get('decision', ''), - 'timed_out': d.get('timed_out', False), - }), - 'llm_retry': ('retry', lambda d: { - 'retry_count': d.get('retry_count', 0), - 'max_retries': d.get('max_retries', 0), - 'error_type': d.get('error_type', ''), - 'message': d.get('user_message', ''), - }), - 'message_injected': ('message_injected', lambda d: { - 'message': d.get('message', ''), - }), - 'message_injection_applied': ('message_injection_applied', lambda d: { - 'content': d.get('content', ''), - 'count': d.get('count', 1), - }), - 'message_received': ('message_received', lambda d: { - 'message': d.get('message', ''), - 'metadata': d.get('metadata', {}), - }), - 'whatsapp_restriction_warning': ('whatsapp_restriction_warning', lambda d: { - 'content': d.get('content', ''), - 'metadata': d.get('metadata', {}), - }), - 'session_clear': ('session_clear', lambda d: { - 'session_id': d.get('session_id', ''), - 'agent_id': d.get('agent_id', ''), - }), - 'turn_split': ('turn_split', lambda d: {}), - 'evonic:agent-state-changed': ('state_changed', lambda d: { - 'agent_id': d.get('agent_id', ''), - 'session_id': d.get('session_id', ''), - }), - } - - def make_handler(evt_name, sse_name, transform): - def handler(data): - if data.get('session_id') != session_id: - return - try: - payload = transform(data) if transform else data - if payload is not None: - # Use the contiguous per-session chat seq (not the global _seq) - # so the browser's gap detector sees a gap-free sequence and - # doesn't fire a phantom gap-fill on every event. Matches the - # legacy /chat/stream + /chat/events gap-fill endpoint. - payload['seq'] = data.get('_chat_seq') - ring.put((sse_name, payload)) - except Exception: - pass - return handler - - handlers = {} - for evt_name, (sse_name, transform) in _TRANSFORMS.items(): - h = make_handler(evt_name, sse_name, transform) - handlers[evt_name] = h - event_stream.on(evt_name, h) - - event_stream.register_web_listener(session_id) - - # Replay the in-progress session buffer so a client connecting at/after the - # POST that starts a turn still sees the turn's opening events (turn_begin, - # early thinking, first tool call). The legacy /chat/stream did this; without - # it the unified path loses those events and the UI shows only a spinner - # until a manual refresh. Subscribe-then-replay ordering (after event_stream.on - # above) means live events arriving during replay are also queued; the client - # dedups the overlap by the contiguous _chat_seq. - try: - buffered = event_stream.get_session_events(session_id, after_seq) - # On a fresh connect, drop everything up to and including the last - # completed turn / session_clear so we never replay a finished turn. - if after_seq == 0: - last_boundary = -1 - for i, e in enumerate(buffered): - if e['event'] in ('turn_complete', 'session_clear'): - last_boundary = i - if last_boundary >= 0: - buffered = buffered[last_boundary + 1:] - for entry in buffered: - st = _TRANSFORMS.get(entry['event']) - if not st: - continue - sse_name, transform = st - payload = transform(entry['data']) - payload['seq'] = entry.get('chat_seq') - ring.put((sse_name, payload)) - except Exception: - pass - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - event_stream.unregister_web_listener(session_id) - for evt_name, h in handlers.items(): - event_stream.off(evt_name, h) - - -def _producer_update(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event): - """Producer: listen to update manager status changes.""" - from routes.update import update_manager - - listener_q = update_manager.register_listener() - - try: - while not stop_event.is_set(): - try: - snapshot = listener_q.get(timeout=1) - ring.put(('update_status', snapshot)) - if snapshot.get('status') in ('success', 'failed'): - ring.put(('update_done', {'status': snapshot['status']})) - except queue.Empty: - pass - finally: - update_manager.unregister_listener(listener_q) - - -def _producer_workplace(ring: BoundedRing, breaker: CircuitBreaker, - stop_event: threading.Event, workplace_id: str): - """Producer: listen to workplace connector events for a specific workplace.""" - from backend.event_stream import event_stream - - _WATCHED = ('connector_connected', 'connector_disconnected', - 'connector_paired', 'workplace_status_changed') - - def handler(data): - if data.get('workplace_id') == workplace_id: - ring.put((data['_event'], dict(data))) - - for ev in _WATCHED: - event_stream.on(ev, handler) - - try: - while not stop_event.is_set(): - stop_event.wait(1) - finally: - for ev in _WATCHED: - event_stream.off(ev, handler) - - -# --------------------------------------------------------------------------- -# Priority-aware event scheduler -# --------------------------------------------------------------------------- - -def _priority_round_robin(rings: dict, conn: RealtimeConnection) -> list: - """Extract events from per-channel rings using weighted round-robin. - - Returns list of (channel, seq, sse_name, payload) tuples. - """ - result = [] - l2_count = 0 - - # First pass: L0 channels (update, status) — 1 event each - for ch in ('update', 'status'): - if ch not in rings: - continue - item = rings[ch].get() - if item: - seq, (sse_name, payload) = item - result.append((ch, seq, sse_name, payload)) - - # L1 channels (approvals) — 1 event - if 'approvals' in rings: - item = rings['approvals'].get() - if item: - seq, (sse_name, payload) = item - result.append(('approvals', seq, sse_name, payload)) - - # L2 channels (chat, workplace) — up to L2_WEIGHT events each - for ch in ('chat', 'workplace'): - if ch not in rings: - continue - for _ in range(L2_WEIGHT): - item = rings[ch].get() - if item: - seq, (sse_name, payload) = item - result.append((ch, seq, sse_name, payload)) - l2_count += 1 - else: - break - - return result - - -# --------------------------------------------------------------------------- -# Differential push for chat -# --------------------------------------------------------------------------- + from backend import update_manager + events.append(('update', 'update_status', update_manager.get_status())) + except Exception as exc: + log.warning('realtime update snapshot failed: %s', exc) -class ChatThrottle: - """Batches thinking chunks: push first chunk immediately, then batch - every throttle_ms, then push final event.""" - - def __init__(self, throttle_ms: int = 100): - self.throttle_ms = throttle_ms - self._batch = [] - self._first_sent = False - self._last_flush = 0 - # Highest chat seq among batched chunks — stamped on the merged event so - # the client's _lastSeq advances over the chunks folded into the batch, - # avoiding a spurious gap-fill (and duplicate CoT) per batch boundary. - self._batch_seq = None - - def _merged_thinking(self): - """Build the merged 'thinking' event from the current batch and reset it.""" - batched_content = ''.join(self._batch) - self._batch = [] - seq = self._batch_seq - self._batch_seq = None - if not batched_content: - return None - ev = {'content': batched_content} - if seq is not None: - ev['seq'] = seq - return ('thinking', ev) - - def feed(self, sse_name: str, payload: dict): - """Feed a chat event. Returns list of events to emit now (may be empty).""" - now_ms = time.monotonic() * 1000 - - # 'thinking' chunks get batched - if sse_name == 'thinking': - if not self._first_sent: - self._first_sent = True - self._last_flush = now_ms - return [('thinking', payload)] - - self._batch.append(payload.get('content', '')) - if payload.get('seq') is not None: - self._batch_seq = payload.get('seq') - - if (now_ms - self._last_flush) >= self.throttle_ms: - self._last_flush = now_ms - merged = self._merged_thinking() - return [merged] if merged else [] - return [] - - # Non-thinking event: flush any pending batch first - result = [] - merged = self._merged_thinking() - if merged: - result.append(merged) - - result.append((sse_name, payload)) - self._first_sent = False - return result - - def flush(self): - """Flush any remaining batched content. Returns list of events.""" - merged = self._merged_thinking() - return [merged] if merged else [] + if 'workplace' in channels and workplace_id: + try: + from backend.workplaces.manager import workplace_manager + events.append(( + 'workplace', 'workplace_status_changed', + workplace_manager.get_status(workplace_id), + )) + except Exception as exc: + log.warning('realtime workplace snapshot failed: %s', exc) + return events -# --------------------------------------------------------------------------- -# Main SSE endpoint -# --------------------------------------------------------------------------- @realtime_bp.route('/api/realtime/stream', methods=['GET']) def api_realtime_stream(): - """Unified multiplexed SSE endpoint. + channels = { + value.strip() for value in request.args.get('channels', '').split(',') + if value.strip() + } + unknown = channels - _ALLOWED_CHANNELS + if unknown: + return Response( + json.dumps({'error': 'unknown channels', 'channels': sorted(unknown)}), + status=400, mimetype='application/json', + ) - Consolidates 5 separate EventSource connections into 1: - - /api/agents/status/stream -> channels=status - - /api/approvals/stream -> channels=approvals - - /api/system/update/stream -> channels=update - - /api/agents//chat/stream -> chat=1&session_id=...&agent_id=... - - /api/workplaces//events -> workplace= - """ - # Parse query parameters - channels_str = request.args.get('channels', '') - channels = set(filter(None, [ch.strip() for ch in channels_str.split(',')])) chat_enabled = request.args.get('chat') == '1' session_id = request.args.get('session_id', '').strip() or None agent_id = request.args.get('agent_id', '').strip() or None - after_seq = request.args.get('after', 0, type=int) workplace_id = request.args.get('workplace', '').strip() or None - chat_throttle_ms = request.args.get('chat_throttle', 100, type=int) - - # Validate chat parameters - if chat_enabled and (not session_id or not agent_id): - return Response( - json.dumps({'error': 'session_id and agent_id required when chat=1'}), - status=400, - mimetype='application/json' - ) - - # Release thread-local DB connection (SSE thread is long-lived) - from models.db import db - db.close() - - # Build channel set - all_channels = set(channels) if chat_enabled: - all_channels.add('chat') + if not session_id or not agent_id: + return Response( + json.dumps({'error': 'session_id and agent_id required when chat=1'}), + status=400, mimetype='application/json', + ) + channels.add('chat') if workplace_id: - all_channels.add('workplace') - - if not all_channels: + channels.add('workplace') + if not channels: return Response( json.dumps({'error': 'At least one channel must be requested'}), - status=400, - mimetype='application/json' + status=400, mimetype='application/json', ) - # SSE connection limiting — max 5 concurrent per user/IP (FINDING-004) - from flask import session as _flask_session + raw_query_cursor = request.args.get('after') + raw_header_cursor = request.headers.get('Last-Event-ID') + versioned_cursor = request.args.get('cursor_version') == '2' + query_cursor = _parse_cursor(raw_query_cursor) if versioned_cursor else None + header_cursor = _parse_cursor(raw_header_cursor) if versioned_cursor else None + supplied_cursor = bool( + str(raw_query_cursor or '').strip() or str(raw_header_cursor or '').strip() + ) + valid_cursors = [value for value in (query_cursor, header_cursor) if value is not None] + cursor = max(valid_cursors) if valid_cursors else 0 + reset_cursor = not versioned_cursor or (supplied_cursor and not valid_cursors) + valid_header_cursor = header_cursor is not None and bool(str(raw_header_cursor or '').strip()) + snapshot_arg = request.args.get('snapshot') + send_snapshot = ( + snapshot_arg == '1' if snapshot_arg in {'0', '1'} + else not supplied_cursor + ) + if reset_cursor: + send_snapshot = True + if valid_header_cursor: + # Native EventSource reconnects reuse the original snapshot=1 URL but + # provide Last-Event-ID. Durable replay is sufficient on reconnect. + send_snapshot = False + legacy = request.args.get('legacy', '').strip() + + def public_event_name(event_name: str) -> str: + if legacy == 'update': + return { + 'update_status': 'status', + 'update_done': 'done', + }.get(event_name, event_name) + return event_name + + from flask import session as flask_session from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_id = ( - f"user:{_flask_session.get('_user_id', 'admin')}" - if _flask_session.get('authenticated') + sse_identity = ( + f"user:{flask_session.get('_user_id', 'admin')}" + if flask_session.get('authenticated') else f"ip:{request.remote_addr or '0.0.0.0'}" ) - _sse_allowed, _sse_count = sse_register(_sse_id) - if not _sse_allowed: + allowed, _count = sse_register(sse_identity) + if not allowed: return Response( json.dumps({ 'error': 'too_many_sse_connections', 'message': f'Maximum {SSE_MAX_CONCURRENT} concurrent SSE connections allowed.', 'retry_after': 30, }), - status=429, - headers={'Retry-After': '30'}, - mimetype='application/json' - ) - - # Thundering herd mitigation — check approximate connection count - with _conn_lock: - conn_count = len(_connections) - max_conn = int(os.environ.get('WORKER_CONNECTIONS', 512)) - if max_conn > 0 and conn_count >= max_conn * 0.8: - sse_unregister(_sse_id) - return Response( - json.dumps({'error': 'Server busy, please retry later'}), - status=503, - headers={'Retry-After': '10'}, - mimetype='application/json' + status=429, headers={'Retry-After': '30'}, + mimetype='application/json', ) - # Generate a connection ID - conn_id = f"{id(request)}:{time.monotonic()}" - - # Token expiry: 24h from now - expires_at = time.time() + 86400 - - conn = RealtimeConnection( - conn_id=conn_id, - channels=all_channels, - chat_session_id=session_id, - agent_id=agent_id, - after_seq=after_seq, - workplace_id=workplace_id, - chat_throttle_ms=chat_throttle_ms, - expires_at=expires_at, - ) - - # Register connection - with _conn_lock: - _connections[conn_id] = conn - - # Build per-channel bounded rings - rings: dict[str, BoundedRing] = {} - for ch in all_channels: - strategy = RING_STRATEGIES.get(ch, 'drop_oldest') - size = RING_SIZES.get(ch, 32) - rings[ch] = BoundedRing(ch, size, strategy) - - # Build circuit breakers - breakers: dict[str, CircuitBreaker] = {} - for ch in all_channels: - breakers[ch] = CircuitBreaker(ch) - - # Store rings ref on the connection so api_realtime_resume can flush - # pause buffers back into the rings after resume. - conn.rings = rings - - # Start producer threads (task isolation) - producers = {} - stop_event = conn._stop_event # shared stop signal - - _PRODUCERS = { - 'status': (_producer_status, {}), - 'approvals': (_producer_approval, {}), - 'update': (_producer_update, {}), - } - - for ch in all_channels: - if ch in _PRODUCERS: - fn, kwargs = _PRODUCERS[ch] - t = threading.Thread( - target=_start_producer, - args=(ch, fn, rings[ch], breakers[ch], stop_event, kwargs), - daemon=True, - name=f"realtime-producer-{ch}-{conn_id[:12]}" - ) - producers[ch] = t - t.start() - elif ch == 'chat' and session_id: - t = threading.Thread( - target=_start_producer, - args=(ch, _producer_chat, rings[ch], breakers[ch], - stop_event, {'session_id': session_id, 'after_seq': after_seq}), - daemon=True, - name=f"realtime-producer-{ch}-{conn_id[:12]}" - ) - producers[ch] = t - t.start() - elif ch == 'workplace' and workplace_id: - t = threading.Thread( - target=_start_producer, - args=(ch, _producer_workplace, rings[ch], breakers[ch], - stop_event, {'workplace_id': workplace_id}), - daemon=True, - name=f"realtime-producer-{ch}-{conn_id[:12]}" - ) - producers[ch] = t - t.start() - - # Chat throttler for differential push - chat_throttle = ChatThrottle(chat_throttle_ms) if 'chat' in all_channels else None - - # Global sequence counter - global_seq = after_seq - chat_seq = after_seq - - # Build state snapshot - snapshot_events = _build_snapshot(all_channels, agent_id, session_id, workplace_id) - - # Setup TCP_NODELAY for time-sensitive channels (approval, status, update) - # This is done during the first write in the generator + from models.db import db + db.close() + connected_at = time.time() - # SSE generator with priority scheduler @stream_with_context def generate(): - nonlocal global_seq, chat_seq - - # Set TCP keepalive and SIGPIPE handling on the socket - try: - # Ignore SIGPIPE at process level if not already done - try: - signal.signal(signal.SIGPIPE, signal.SIG_IGN) - except (ValueError, OSError): - pass # can only be set in main thread - except Exception: - pass - + nonlocal cursor, send_snapshot try: - # --- Phase 1: Emit retry with jitter --- - retry_ms = random.randint(3000, 8000) - yield f"retry: {retry_ms}\n" - - # --- Phase 2: Push state snapshot --- - for event_name, data in snapshot_events: - global_seq += 1 - yield _format_sse_event(event_name, data, - global_seq=global_seq) - - # --- Phase 3: Forward live events with priority scheduler --- - last_heartbeat = time.monotonic() - heartbeat_failures = 0 - - while not conn.is_stopped(): - # Check token expiry - if conn.check_expired(): - global_seq += 1 - yield _format_sse_event('auth_expired', - {'message': 'Token expired, please reconnect'}, - global_seq=global_seq) - conn.stop() + yield 'retry: 3000\n\n' + + high_water = realtime_store.high_water() + resync_reason = None + if chat_enabled and versioned_cursor and supplied_cursor: + if not valid_cursors: + resync_reason = 'invalid_cursor' + elif cursor > high_water: + resync_reason = 'cursor_ahead' + elif cursor < realtime_store.replay_floor(session_id): + resync_reason = 'cursor_expired' + if resync_reason: + yield _format_sse_event('history_resync_required', { + 'reason': resync_reason, + 'retention_seconds': RETENTION_MS // 1000, + 'event_id': high_water, + 'seq': high_water, + 'timestamp': int(time.time() * 1000), + 'channel': 'system', + }, high_water) + return + if reset_cursor or cursor > high_water: + cursor = high_water + send_snapshot = True + elif not supplied_cursor: + cursor = high_water + + replayed_approval_ids: set[str] = set() + + # Restore telemetry already represented by the HTTP chat history + # before sending current-state snapshots such as pending approval. + if send_snapshot and session_id: + snapshot_cursor = realtime_store.last_session_clear_id( + session_id, cursor, + ) + while snapshot_cursor < cursor: + active_events = realtime_store.events_after( + snapshot_cursor, {'chat'}, session_id=session_id, + agent_id=agent_id, up_to_id=cursor, + active_only=True, limit=500, + ) + if not active_events: + break + for event in active_events: + snapshot_cursor = event['id'] + data = dict(event['data']) + data['snapshot'] = True + data['source_event_id'] = event['id'] + data['event_id'] = 0 + data['seq'] = 0 + if event['event'] == 'approval_required': + approval_id = str(data.get('approval_id', '')) + if approval_id: + replayed_approval_ids.add(approval_id) + yield _format_sse_event( + public_event_name(event['event']), data, + ) + + # Close the history-to-stream handoff gap before taking state + # snapshots. Filtered global IDs may be skipped safely. + if send_snapshot and cursor < high_water: + while cursor < high_water: + events = realtime_store.events_after( + cursor, channels, session_id=session_id, agent_id=agent_id, + workplace_id=workplace_id, up_to_id=high_water, + ) + if not events: + break + for event in events: + cursor = event['id'] + if event['event'] == 'approval_required': + approval_id = str(event['data'].get('approval_id', '')) + if approval_id: + replayed_approval_ids.add(approval_id) + yield _format_sse_event( + public_event_name(event['event']), + event['data'], event['id'], + ) + cursor = high_water + + if send_snapshot: + for channel, event_name, payload in _build_snapshot( + channels, session_id, agent_id, workplace_id, + replayed_approval_ids): + yield _format_sse_event( + public_event_name(event_name), + _snapshot_payload(channel, payload), + ) + + yield _format_sse_event('ready', { + 'event_id': cursor, 'seq': cursor, + 'timestamp': int(time.time() * 1000), + 'channel': 'system', + }, cursor) + + while True: + if time.time() - connected_at >= 24 * 60 * 60: + yield _format_sse_event('auth_expired', { + 'message': 'Connection expired, please reconnect', + }, cursor) break - # Heartbeat - now = time.monotonic() - if now - last_heartbeat >= HEARTBEAT_INTERVAL: - try: - yield "event: heartbeat\ndata: {}\n\n" - heartbeat_failures = 0 - conn.last_heartbeat_time = now - except (BrokenPipeError, OSError): - heartbeat_failures += 1 - if heartbeat_failures >= HEARTBEAT_MAX_FAILURES: - conn.stop() - break - time.sleep(1) - continue - last_heartbeat = now - - # Priority-aware extraction from rings - events = _priority_round_robin(rings, conn) - - if not events: - # No events — short sleep to avoid busy-wait - time.sleep(0.05) + observed_high_water = realtime_store.high_water() + events = realtime_store.events_after( + cursor, channels, session_id=session_id, agent_id=agent_id, + workplace_id=workplace_id, up_to_id=observed_high_water, + ) + if events: + for event in events: + cursor = event['id'] + yield _format_sse_event( + public_event_name(event['event']), + event['data'], event['id'], + ) continue - for channel, seq, sse_name, payload in events: - if conn.is_stopped(): - break - - global_seq += 1 - - # Build composite id for per-channel resume - if channel == 'chat': - chat_seq += 1 - seq_id = f"chat:{chat_seq}" - else: - seq_id = f"{channel}:{global_seq}" - - # Differential push for chat - if channel == 'chat' and chat_throttle: - throttled = chat_throttle.feed(sse_name, payload) - if not throttled: - continue - for t_name, t_payload in throttled: - global_seq += 1 - chat_seq += 1 - yield _format_sse_event(t_name, t_payload, - seq_id=f"chat:{chat_seq}", - global_seq=global_seq) - else: - # Check if paused — buffer chat/workplace events - if conn.paused and channel in ('chat', 'workplace'): - buf = conn._pause_buffers.get(channel) - if buf is None: - max_buf = PAUSE_BUFFER.get(channel, 32) - buf = BoundedRing(channel, max_buf, 'drop_oldest') - conn._pause_buffers[channel] = buf - buf.put((sse_name, payload)) - continue - - try: - yield _format_sse_event(sse_name, payload, - seq_id=seq_id, - global_seq=global_seq) - conn.last_write_ok = True - except (BrokenPipeError, OSError) as e: - log.warning("realtime %s: write failed: %s", conn_id, e) - conn.stop() - break - - # Check for dropped events per channel - for ch_name, ring in rings.items(): - dropped = ring.drain_dropped() - if dropped > 0: - try: - yield _format_sse_comment(f"x-sse-dropped {ch_name}:{dropped}") - except (BrokenPipeError, OSError): - conn.stop() - break - - # Flush chat throttler on disconnect - if chat_throttle: - for t_name, t_payload in chat_throttle.flush(): - try: - global_seq += 1 - chat_seq += 1 - yield _format_sse_event(t_name, t_payload, - seq_id=f"chat:{chat_seq}", - global_seq=global_seq) - except (BrokenPipeError, OSError): - break - + realtime_store.wait_for_events(observed_high_water, HEARTBEAT_INTERVAL) + if realtime_store.high_water() <= observed_high_water: + yield 'event: heartbeat\ndata: {}\n\n' except GeneratorExit: pass finally: - # Cleanup: stop all producers - conn.stop() - for ch, t in producers.items(): - t.join(timeout=2) - - # Remove connection from registry - with _conn_lock: - _connections.pop(conn_id, None) - - # Unregister SSE connection (FINDING-004) - sse_unregister(_sse_id) - - # Check for circuit-breaker channel_disabled events - for ch_name in all_channels: - if breakers.get(ch_name) and breakers[ch_name].is_disabled(): - pass # Already disabled - - log.debug("realtime %s: connection closed", conn_id[:20]) + sse_unregister(sse_identity) return Response( - generate(), - mimetype='text/event-stream', + generate(), mimetype='text/event-stream', headers={ 'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no', 'Connection': 'keep-alive', - } + }, ) -def _start_producer(channel: str, producer_fn, ring: BoundedRing, - breaker: CircuitBreaker, stop_event: threading.Event, - kwargs: dict): - """Run a producer with task isolation and circuit breaker logic.""" - while not stop_event.is_set(): - if breaker.is_disabled(): - log.warning("realtime: channel %s disabled by circuit breaker", channel) - return - try: - producer_fn(ring, breaker, stop_event, **kwargs) - break # producer returned normally - except Exception as e: - log.error("realtime: producer %s crashed: %s", channel, e, exc_info=True) - should_stop = breaker.record_crash() - if should_stop: - log.error("realtime: channel %s circuit breaker open — stopping", channel) - return - # Wait before retry - stop_event.wait(1) - - -# --------------------------------------------------------------------------- -# Pause/Resume endpoint (internal — called by client) -# --------------------------------------------------------------------------- - @realtime_bp.route('/api/realtime/pause', methods=['POST']) def api_realtime_pause(): - """Pause chat+workplace event delivery for this session.""" - data = request.get_json() or {} - session_id = data.get('session_id', '').strip() - if not session_id: - return Response(json.dumps({'error': 'session_id required'}), status=400, - mimetype='application/json') - - with _conn_lock: - for conn in list(_connections.values()): - if conn.chat_session_id == session_id: - conn.pause() + # Delivery is durable now. Browser visibility may pause rendering locally; + # the server no longer needs a second per-connection buffer. return Response(json.dumps({'ok': True}), mimetype='application/json') @realtime_bp.route('/api/realtime/resume', methods=['POST']) def api_realtime_resume(): - """Resume chat+workplace event delivery and flush paused buffer.""" - data = request.get_json() or {} - session_id = data.get('session_id', '').strip() - if not session_id: - return Response(json.dumps({'error': 'session_id required'}), status=400, - mimetype='application/json') - - with _conn_lock: - for conn in list(_connections.values()): - if conn.chat_session_id == session_id: - conn.resume() - # Flush pause buffers — drain events that accumulated - # while paused and re-insert them into the per-channel - # rings so the generator yields them on the next pass. - if conn.rings: - for ch in ('chat', 'workplace'): - buf = conn._pause_buffers.get(ch) - if buf: - ring = conn.rings.get(ch) - if ring: - for _seq, item in buf.get_all(): - ring.put(item) return Response(json.dumps({'ok': True}), mimetype='application/json') - - -# --------------------------------------------------------------------------- -# Deprecated old-SSE endpoint wrappers (keep functional, log deprecation) -# --------------------------------------------------------------------------- - -_deprecated_logged: set = set() - -def _warn_deprecated_once(msg: str): - if msg not in _deprecated_logged: - log.warning("DEPRECATED: %s", msg) - _deprecated_logged.add(msg) diff --git a/routes/sessions.py b/routes/sessions.py index 6a7fb3ab..6a878bae 100644 --- a/routes/sessions.py +++ b/routes/sessions.py @@ -182,21 +182,25 @@ def api_list_sessions(): sessions, total = db.get_all_sessions(search=search, limit=limit, offset=offset, exclude_test=exclude_test) # Tag sessions that are currently being processed by an agent - from backend.agent_runtime import agent_runtime + from backend.realtime_store import realtime_store for s in sessions: - s['is_active'] = agent_runtime._is_busy(s['id']) + s['is_active'] = bool(realtime_store.active_turns(session_id=s['id'])) return jsonify({'sessions': sessions, 'total': total}) @sessions_bp.route('/api/sessions/') def api_get_session(session_id): + from backend.realtime_store import realtime_store + realtime_cursor = realtime_store.high_water() session = db.get_session_with_details(session_id) if not session: return jsonify({'error': 'Session not found'}), 404 limit = request.args.get('limit', 200, type=int) before_id = request.args.get('before_id', None, type=int) messages, has_more = db.get_session_messages_full(session_id, limit=limit, before_id=before_id) - return jsonify({'session': session, 'messages': messages, 'has_more': has_more}) + response = jsonify({'session': session, 'messages': messages, 'has_more': has_more}) + response.headers['X-Evonic-Realtime-Cursor'] = str(realtime_cursor) + return response @sessions_bp.route('/api/sessions//poll') @@ -213,13 +217,21 @@ def api_session_reply(session_id): if request.content_type and request.content_type.startswith('multipart/form-data'): text = (request.form.get('text') or '').strip() perspective = (request.form.get('perspective') or 'A').strip() + client_message_id = (request.form.get('client_message_id') or '').strip() file = request.files.get('file') else: - data = request.get_json() + data = request.get_json() or {} text = (data.get('text') or '').strip() perspective = (data.get('perspective') or 'A').strip() + raw_client_message_id = data.get('client_message_id') + if raw_client_message_id is not None and not isinstance(raw_client_message_id, str): + return jsonify({'error': 'Invalid client_message_id'}), 400 + client_message_id = (raw_client_message_id or '').strip() file = None + if client_message_id and not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', client_message_id): + return jsonify({'error': 'Invalid client_message_id'}), 400 + if not text and not file: return jsonify({'error': 'Text or file is required'}), 400 @@ -285,16 +297,22 @@ def api_session_reply(session_id): text = f"{info_line}\n\n{text}" if text else info_line if perspective == 'A': + upload_meta = dict(upload_meta or {}) + if client_message_id: + upload_meta['client_message_id'] = client_message_id result = agent_runtime.send_as_user(session_id, text, image_url=image_url, metadata=upload_meta) else: - result = agent_runtime.send_as_bot(session_id, text) + result = agent_runtime.send_as_bot( + session_id, text, + metadata={'client_message_id': client_message_id} if client_message_id else None, + ) if not result: return jsonify({'error': 'Session not found'}), 404 # Build response — for slash commands, include the response text directly - resp = {'success': True} + resp = {'success': True, 'client_message_id': client_message_id or None} if isinstance(result, str): # send_as_user returned the slash command response text resp['slash_command'] = True diff --git a/routes/update.py b/routes/update.py index d3b6f981..f0447048 100644 --- a/routes/update.py +++ b/routes/update.py @@ -1,9 +1,6 @@ """Routes for the system update UI and API.""" -import json -import queue - -from flask import Blueprint, Response, jsonify, render_template, request, stream_with_context +from flask import Blueprint, jsonify, render_template, request, redirect from backend import update_manager @@ -67,60 +64,7 @@ def api_update_restart(): def api_update_stream(): """SSE endpoint for real-time update progress. DEPRECATED: Use unified GET /api/realtime/stream?channels=update instead.""" - import logging as _log_depr - _log_depr.getLogger(__name__).warning( - "DEPRECATED endpoint /api/system/update/stream used — " - "migrate to /api/realtime/stream?channels=update") - # Release the thread-local DB connection — this SSE thread is long-lived. - from models.db import db - db.close() - - # SSE connection limiting (max 5 concurrent per user/IP, FINDING-004) - from flask import session as _flsk_sess - from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_ident = ( - 'user:' + (_flsk_sess.get('_user_id', 'admin') if _flsk_sess.get('authenticated') else '') - if _flsk_sess.get('authenticated') - else 'ip:' + (request.remote_addr or '0.0.0.0') - ) - _ok, _cnt = sse_register(_sse_ident) - if not _ok: - return jsonify({ - 'error': 'too_many_sse_connections', - 'message': 'Maximum ' + str(SSE_MAX_CONCURRENT) + ' concurrent SSE connections allowed.', - 'retry_after': 30, - }), 429, {'Retry-After': '30'} - - q = update_manager.register_listener() - - def generate(): - try: - # Send initial status immediately - status = update_manager.get_status() - yield f"event: status\ndata: {json.dumps(status)}\n\n" - - while True: - try: - snapshot = q.get(timeout=30) - except queue.Empty: - yield ": heartbeat\n\n" - continue - - yield f"event: status\ndata: {json.dumps(snapshot)}\n\n" - - # If terminal state, send done event and close - if snapshot.get('status') in ('success', 'failed'): - yield f"event: done\ndata: {json.dumps({'status': snapshot['status']})}\n\n" - finally: - update_manager.unregister_listener(q) - sse_unregister(_sse_ident) - - return Response( - stream_with_context(generate()), - mimetype='text/event-stream', - headers={ - 'Cache-Control': 'no-cache', - 'X-Accel-Buffering': 'no', - 'Connection': 'keep-alive', - }, + return redirect( + '/api/realtime/stream?channels=update&legacy=update&snapshot=1', + code=307, ) diff --git a/routes/workplaces.py b/routes/workplaces.py index c639bbb3..5fa96690 100644 --- a/routes/workplaces.py +++ b/routes/workplaces.py @@ -4,11 +4,9 @@ import secrets import string import uuid +from urllib.parse import urlencode -import queue -import threading - -from flask import Blueprint, Response, jsonify, render_template, request, stream_with_context +from flask import Blueprint, Response, jsonify, render_template, request, redirect from models.db import db @@ -154,70 +152,16 @@ def api_workplace_status(workplace_id): def api_workplace_events(workplace_id): """SSE stream for real-time workplace status changes (connector connect/disconnect, status). DEPRECATED: Use unified GET /api/realtime/stream?workplace= instead.""" - import logging as _log_depr - _log_depr.getLogger(__name__).warning( - "DEPRECATED endpoint /api/workplaces//events used — " - "migrate to /api/realtime/stream?workplace=") workplace = db.get_workplace(workplace_id) if not workplace: return jsonify({'error': 'Not found'}), 404 - - # Release the thread-local DB connection — this SSE thread is long-lived. - db.close() - - # SSE connection limiting (max 5 concurrent per user/IP, FINDING-004) - from flask import session as _flsk_sess - from models.api_rate_limit import sse_register, sse_unregister, SSE_MAX_CONCURRENT - _sse_ident = ( - 'user:' + (_flsk_sess.get('_user_id', 'admin') if _flsk_sess.get('authenticated') else '') - if _flsk_sess.get('authenticated') - else 'ip:' + (request.remote_addr or '0.0.0.0') - ) - _ok, _cnt = sse_register(_sse_ident) - if not _ok: - return jsonify({ - 'error': 'too_many_sse_connections', - 'message': 'Maximum ' + str(SSE_MAX_CONCURRENT) + ' concurrent SSE connections allowed.', - 'retry_after': 30, - }), 429, {'Retry-After': '30'} - - q = queue.Queue(maxsize=20) - - _WATCHED = ('connector_connected', 'connector_disconnected', 'connector_paired', 'workplace_status_changed') - - def handler(data): - if data.get('workplace_id') == workplace_id: - try: - q.put_nowait(data) - except queue.Full: - pass - - from backend.event_stream import event_stream - for ev in _WATCHED: - event_stream.on(ev, handler) - - @stream_with_context - def generate(): - try: - while True: - try: - data = q.get(timeout=30) - yield f"event: {data['_event']}\ndata: {json.dumps(data)}\n\n" - except queue.Empty: - yield ": heartbeat\n\n" - finally: - for ev in _WATCHED: - event_stream.off(ev, handler) - sse_unregister(_sse_ident) - - return Response( - generate(), - mimetype='text/event-stream', - headers={ - 'Cache-Control': 'no-cache', - 'X-Accel-Buffering': 'no', - 'Connection': 'keep-alive', - } + return redirect( + '/api/realtime/stream?' + urlencode({ + 'workplace': workplace_id, + 'legacy': 'workplace', + 'snapshot': 1, + }), + code=307, ) diff --git a/static/js/chat-ui.js b/static/js/chat-ui.js index 7337d743..dd33d126 100644 --- a/static/js/chat-ui.js +++ b/static/js/chat-ui.js @@ -2301,8 +2301,7 @@ const SSE_EVENTS = [ 'turn_begin', 'turn_split', 'thinking', 'tool_call_started', 'tool_executed', 'state:changed', 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', 'approval_required', 'approval_resolved', 'retry', 'message_injected', 'message_injection_applied', 'message_received', 'whatsapp_restriction_warning', 'session_clear', - 'state_changed', - 'heartbeat', + 'state_changed', 'turn_queued', 'ready', 'heartbeat', 'auth_expired', ]; // If no event (including heartbeats) arrives within this window, the connection @@ -2318,12 +2317,10 @@ class SSEAdapter { this._lastSeq = opts.afterSeq || 0; this._handler = null; this._es = null; - this._fillingGap = false; - this._pendingQueue = []; this._log = log('sse'); this._lastEventAt = 0; this._livenessInterval = null; - this._usingUnified = false; // true when using unified /api/realtime/stream + this._usingUnified = url.indexOf('/api/realtime/stream') !== -1; this._reconnectAttempts = 0; // consecutive immediate failures (for backoff) this._connectStartTime = 0; // when the current EventSource was opened } @@ -2353,7 +2350,7 @@ class SSEAdapter { const agentId = this._agentId || (url.match(/\/agents\/([^/?]+)\//) || [])[1] || ''; const sessionId = this._sessionId || u.searchParams.get('session_id') || ''; const after = this._lastSeq; - let newUrl = '/api/realtime/stream?chat=1'; + let newUrl = '/api/realtime/stream?chat=1&cursor_version=2&snapshot=1'; if (agentId) newUrl += '&agent_id=' + encodeURIComponent(agentId); if (sessionId) newUrl += '&session_id=' + encodeURIComponent(sessionId); if (after > 0) newUrl += '&after=' + after; @@ -2380,7 +2377,8 @@ class SSEAdapter { if (this._usingUnified) { const agentId = this._agentId || ''; const sessionId = this._sessionId || ''; - resumeUrl = '/api/realtime/stream?chat=1'; + resumeUrl = '/api/realtime/stream?chat=1&cursor_version=2&snapshot=' + + (this._lastSeq > 0 ? '0' : '1'); if (agentId) resumeUrl += '&agent_id=' + encodeURIComponent(agentId); if (sessionId) resumeUrl += '&session_id=' + encodeURIComponent(sessionId); if (this._lastSeq > 0) resumeUrl += '&after=' + this._lastSeq; @@ -2407,10 +2405,10 @@ class SSEAdapter { es.onerror = () => { this._log.warn('SSE error/closed', url); - console.warn('[sse] error/closed _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length); + console.warn('[sse] error/closed _lastSeq=', this._lastSeq); es.close(); if (this._es === es) this._es = null; - // Only reconnect if this was NOT an intentional stop (e.g. after 'done') + // Only reconnect if this adapter was not explicitly stopped. if (this._intentionallyStopped) { this._log.info('intentionally stopped — no reconnect'); return; @@ -2441,7 +2439,8 @@ class SSEAdapter { if (this._usingUnified) { const agentId = this._agentId || ''; const sessionId = this._sessionId || ''; - resumeUrl = '/api/realtime/stream?chat=1'; + resumeUrl = '/api/realtime/stream?chat=1&cursor_version=2&snapshot=' + + (this._lastSeq > 0 ? '0' : '1'); if (agentId) resumeUrl += '&agent_id=' + encodeURIComponent(agentId); if (sessionId) resumeUrl += '&session_id=' + encodeURIComponent(sessionId); if (this._lastSeq > 0) resumeUrl += '&after=' + this._lastSeq; @@ -2451,7 +2450,7 @@ class SSEAdapter { resumeUrl = u.pathname + u.search; } this._log.info('reconnecting from seq', this._lastSeq, resumeUrl); - console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length, 'url=', resumeUrl); + console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, 'url=', resumeUrl); this._connect(resumeUrl); }, delay); }; @@ -2459,80 +2458,21 @@ class SSEAdapter { _handleRaw(evtName, data) { const seq = data.seq || 0; - - if (this._fillingGap) { - this._log.debug('queued while filling gap', evtName, 'seq', seq, 'queueLen', this._pendingQueue.length); - if (this._pendingQueue.length >= 1 && this._pendingQueue.length % 10 === 0) { - console.warn('[sse] pendingQueue grew to', this._pendingQueue.length, 'while filling gap — possible reconnect storm'); - } - this._pendingQueue.push({ evtName, data }); - return; - } - if (seq && seq <= this._lastSeq) { this._log.debug('dedup skip', evtName, 'seq', seq, '≤ lastSeq', this._lastSeq); - console.log('[sse] dedup skip', evtName, 'seq=', seq, '_lastSeq=', this._lastSeq); return; } - - if (seq && this._lastSeq > 0 && seq > this._lastSeq + 1) { - this._log.warn('seq gap detected', this._lastSeq, '→', seq, '— filling'); - this._fillingGap = true; - this._pendingQueue.push({ evtName, data }); - this._fillGap(this._lastSeq, seq).then(() => { - this._fillingGap = false; - console.warn('[sse] draining pendingQueue len=', this._pendingQueue.length, '_lastSeq=', this._lastSeq); - this._drainQueue(); - }); - return; - } - if (seq) this._lastSeq = seq; + if (evtName === 'ready') return; this._dispatch(evtName, data); } - async _fillGap(afterSeq, upToSeq) { - try { - const agentId = this._agentId || this._url.match(/\/agents\/([^/?]+)\//)?.[1] || ''; - const res = await $.getJSON( - `/api/agents/${encodeURIComponent(agentId)}/chat/events?session_id=${encodeURIComponent(this._sessionId)}&after=${afterSeq}&up_to=${upToSeq}` - ); - const evts = res.events || []; - this._log.warn('gap-fill response: afterSeq=' + afterSeq + ' upToSeq=' + upToSeq + ' returned=' + evts.length + ' seqs=' + evts.map(e=>e.seq).join(',')); - console.warn('[gap-fill] returned', evts.length, 'events for after=', afterSeq, 'up_to=', upToSeq, 'seqs:', evts.map(e=>e.seq).join(',')); - for (const ev of evts) { - if (ev.seq <= this._lastSeq) continue; - this._lastSeq = ev.seq; - this._dispatch(ev.event, ev.data); - } - } catch (err) { - this._log.warn('gap-fill failed', err, '— skipping gap'); - this._lastSeq = upToSeq - 1; - } - } - - // Drain _pendingQueue asynchronously — one event per animation frame so we - // never block the main thread with a large synchronous burst. - _drainQueue() { - if (this._pendingQueue.length === 0) { - console.warn('[sse] queue drain done _lastSeq=', this._lastSeq); - return; - } - const item = this._pendingQueue.shift(); - const itemSeq = item.data.seq || 0; - if (itemSeq && itemSeq <= this._lastSeq) { - console.log('[sse] queue dedup skip', item.evtName, 'seq=', itemSeq); - // Skip but continue draining without waiting — dedup is cheap - this._drainQueue(); + _dispatch(evtName, data) { + if (evtName === 'auth_expired') { + this.stop(); + window.location.href = '/login'; return; } - if (itemSeq) this._lastSeq = itemSeq; - this._dispatch(item.evtName, item.data); - // Yield to the browser between each real event - requestAnimationFrame(() => this._drainQueue()); - } - - _dispatch(evtName, data) { if (evtName === 'state_changed') { // Not turn-scoped — bridge straight to the document-level event that // agent_detail.html / sessions.html already listen for (debounced refresh). @@ -2547,11 +2487,8 @@ class SSEAdapter { this._handler({ event: 'session_clear', data, seq: data.seq || 0 }); return; } - // done: stop reconnecting after this if (evtName === 'done') { - console.warn('[sse] _dispatch done _lastSeq=', this._lastSeq, 'data.seq=', data.seq); this._handler({ event: 'done', data, seq: data.seq || 0 }); - this.stop(); return; } this._handler({ event: evtName, data, seq: data.seq || 0 }); @@ -2678,8 +2615,6 @@ class ReplayAdapter { // ── turn.js ───────────────────────────────────────────────────── -const STALE_TIMEOUT_MS = 300_000; // 5 minutes — safety net for truly abandoned turns - const TERMINAL_PHASES = new Set(['final', 'done', 'aborted']); function reduceTurn(phase, eventKind, isFinal = false) { @@ -2755,7 +2690,6 @@ class Turn { this._lastSeq = 0; this._transports = []; this._timerInterval = null; - this._staleTimeout = null; this._scrollRAF = null; this._startTime = Date.now(); this._finalized = false; @@ -2769,7 +2703,6 @@ class Turn { this._buildDOM(); this._startTimer(); - this._armStaleTimeout(); } // ── DOM ────────────────────────────────────────────────────────────────── @@ -2831,16 +2764,8 @@ class Turn { }, 100); } - _armStaleTimeout() { - this._staleTimeout = setTimeout(() => { - this._log.warn('stale timeout reached, auto-finalizing', this.id); - this._finalizeBubble(null); - }, STALE_TIMEOUT_MS); - } - _clearTimers() { if (this._timerInterval) { clearInterval(this._timerInterval); this._timerInterval = null; } - if (this._staleTimeout) { clearTimeout(this._staleTimeout); this._staleTimeout = null; } if (this._scrollRAF) { cancelAnimationFrame(this._scrollRAF); this._scrollRAF = null; } } @@ -2865,15 +2790,6 @@ class Turn { } if (seq) this._lastSeq = seq; - // Reset stale timeout on every live event — turn is clearly still active - if (this._staleTimeout) { - clearTimeout(this._staleTimeout); - this._staleTimeout = setTimeout(() => { - this._log.warn('stale timeout reached, auto-finalizing', this.id); - this._finalizeBubble(null); - }, STALE_TIMEOUT_MS); - } - this._log.debug('ingest', evtName, seq, '→ phase was', this.phase, this.id); const nextPhase = reduceTurn(this.phase, evtName, data.is_final); @@ -2990,7 +2906,7 @@ class Turn { console.warn('[turn] done event turn=%s _finalized=%s _finalContent=%s', this.id, this._finalized, !!this._finalContent); this._finalizeBubble(data.thinking_duration); // Fire final:response so page-level code can render the response bubble - // synchronously — no dependency on pollForResponse JSONL poll. + // synchronously from the durable stream. if (this._finalContent) { console.warn('[turn] firing final:response turn=%s contentLen=%d', this.id, this._finalContent.length); this._onTrigger('final:response', { @@ -3003,9 +2919,12 @@ class Turn { } if (evtName === 'turn_split') { - this._finalizeBubble(null); + const splitDuration = data.timestamp && this._startTime + ? Math.max(0, (data.timestamp - this._startTime) / 1000) + : null; + this._finalizeBubble(splitDuration); // ChatUI will create a new Turn for the continuation - this._onTrigger('turn:split', { turnId: this.id }); + this._onTrigger('turn:split', { turnId: this.id, timestamp: data.timestamp }); return; } @@ -3787,6 +3706,7 @@ class ChatUI { const $lastUser = this.$container.find('[data-msg-role="user"]').last(); const $anchor = $lastUser.length ? $lastUser : (opts.userMsgEl ? $(opts.userMsgEl) : turn.$anchor); const newTurn = this.beginTurn($anchor); + if (data.timestamp) newTurn._startTime = data.timestamp; this._lastLiveTurnId = newTurn.id; this.markQueuedAsDelivered(); // Re-route the SSE adapter to the new turn so subsequent events diff --git a/static/js/chat-ui/index.js b/static/js/chat-ui/index.js index d80c734a..5566ddce 100644 --- a/static/js/chat-ui/index.js +++ b/static/js/chat-ui/index.js @@ -483,6 +483,7 @@ export class ChatUI { const $lastUser = this.$container.find('[data-msg-role="user"]').last(); const $anchor = $lastUser.length ? $lastUser : (opts.userMsgEl ? $(opts.userMsgEl) : turn.$anchor); const newTurn = this.beginTurn($anchor); + if (data.timestamp) newTurn._startTime = data.timestamp; this._lastLiveTurnId = newTurn.id; this.markQueuedAsDelivered(); // Re-route the SSE adapter to the new turn so subsequent events diff --git a/static/js/chat-ui/transport.js b/static/js/chat-ui/transport.js index 9b2de2dd..e0bbdbe7 100644 --- a/static/js/chat-ui/transport.js +++ b/static/js/chat-ui/transport.js @@ -14,8 +14,7 @@ const SSE_EVENTS = [ 'turn_begin', 'turn_split', 'thinking', 'tool_call_started', 'tool_executed', 'state:changed', 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', 'approval_required', 'approval_resolved', 'retry', 'message_injected', 'message_injection_applied', 'message_received', 'whatsapp_restriction_warning', 'session_clear', - 'state_changed', - 'heartbeat', + 'state_changed', 'turn_queued', 'ready', 'heartbeat', 'auth_expired', ]; // If no event (including heartbeats) arrives within this window, the connection @@ -28,7 +27,7 @@ export class SSEAdapter { * @param {object} [opts] * @param {string} [opts.agentId] * @param {string} [opts.sessionId] - * @param {number} [opts.afterSeq=0] - resume from this seq (gap-fill will request from here) + * @param {number} [opts.afterSeq=0] - durable journal resume cursor */ constructor(url, opts = {}) { this._url = url; @@ -37,12 +36,10 @@ export class SSEAdapter { this._lastSeq = opts.afterSeq || 0; this._handler = null; this._es = null; - this._fillingGap = false; - this._pendingQueue = []; this._log = log('sse'); this._lastEventAt = 0; this._livenessInterval = null; - this._usingUnified = false; // true when using unified /api/realtime/stream + this._usingUnified = url.indexOf('/api/realtime/stream') !== -1; this._reconnectAttempts = 0; // consecutive immediate failures (for backoff) this._connectStartTime = 0; // when the current EventSource was opened } @@ -72,7 +69,7 @@ export class SSEAdapter { const agentId = this._agentId || (url.match(/\/agents\/([^/?]+)\//) || [])[1] || ''; const sessionId = this._sessionId || u.searchParams.get('session_id') || ''; const after = this._lastSeq; - let newUrl = '/api/realtime/stream?chat=1'; + let newUrl = '/api/realtime/stream?chat=1&cursor_version=2&snapshot=1'; if (agentId) newUrl += '&agent_id=' + encodeURIComponent(agentId); if (sessionId) newUrl += '&session_id=' + encodeURIComponent(sessionId); if (after > 0) newUrl += '&after=' + after; @@ -99,7 +96,8 @@ export class SSEAdapter { if (this._usingUnified) { const agentId = this._agentId || ''; const sessionId = this._sessionId || ''; - resumeUrl = '/api/realtime/stream?chat=1'; + resumeUrl = '/api/realtime/stream?chat=1&cursor_version=2&snapshot=' + + (this._lastSeq > 0 ? '0' : '1'); if (agentId) resumeUrl += '&agent_id=' + encodeURIComponent(agentId); if (sessionId) resumeUrl += '&session_id=' + encodeURIComponent(sessionId); if (this._lastSeq > 0) resumeUrl += '&after=' + this._lastSeq; @@ -126,10 +124,10 @@ export class SSEAdapter { es.onerror = () => { this._log.warn('SSE error/closed', url); - console.warn('[sse] error/closed _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length); + console.warn('[sse] error/closed _lastSeq=', this._lastSeq); es.close(); if (this._es === es) this._es = null; - // Only reconnect if this was NOT an intentional stop (e.g. after 'done') + // Only reconnect if this adapter was not explicitly stopped. if (this._intentionallyStopped) { this._log.info('intentionally stopped — no reconnect'); return; @@ -160,7 +158,8 @@ export class SSEAdapter { if (this._usingUnified) { const agentId = this._agentId || ''; const sessionId = this._sessionId || ''; - resumeUrl = '/api/realtime/stream?chat=1'; + resumeUrl = '/api/realtime/stream?chat=1&cursor_version=2&snapshot=' + + (this._lastSeq > 0 ? '0' : '1'); if (agentId) resumeUrl += '&agent_id=' + encodeURIComponent(agentId); if (sessionId) resumeUrl += '&session_id=' + encodeURIComponent(sessionId); if (this._lastSeq > 0) resumeUrl += '&after=' + this._lastSeq; @@ -170,7 +169,7 @@ export class SSEAdapter { resumeUrl = u.pathname + u.search; } this._log.info('reconnecting from seq', this._lastSeq, resumeUrl); - console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, '_fillingGap=', this._fillingGap, '_pendingQueue=', this._pendingQueue.length, 'url=', resumeUrl); + console.warn('[sse] reconnecting _lastSeq=', this._lastSeq, 'url=', resumeUrl); this._connect(resumeUrl); }, delay); }; @@ -178,80 +177,21 @@ export class SSEAdapter { _handleRaw(evtName, data) { const seq = data.seq || 0; - - if (this._fillingGap) { - this._log.debug('queued while filling gap', evtName, 'seq', seq, 'queueLen', this._pendingQueue.length); - if (this._pendingQueue.length >= 1 && this._pendingQueue.length % 10 === 0) { - console.warn('[sse] pendingQueue grew to', this._pendingQueue.length, 'while filling gap — possible reconnect storm'); - } - this._pendingQueue.push({ evtName, data }); - return; - } - if (seq && seq <= this._lastSeq) { this._log.debug('dedup skip', evtName, 'seq', seq, '≤ lastSeq', this._lastSeq); - console.log('[sse] dedup skip', evtName, 'seq=', seq, '_lastSeq=', this._lastSeq); - return; - } - - if (seq && this._lastSeq > 0 && seq > this._lastSeq + 1) { - this._log.warn('seq gap detected', this._lastSeq, '→', seq, '— filling'); - this._fillingGap = true; - this._pendingQueue.push({ evtName, data }); - this._fillGap(this._lastSeq, seq).then(() => { - this._fillingGap = false; - console.warn('[sse] draining pendingQueue len=', this._pendingQueue.length, '_lastSeq=', this._lastSeq); - this._drainQueue(); - }); return; } - if (seq) this._lastSeq = seq; + if (evtName === 'ready') return; this._dispatch(evtName, data); } - async _fillGap(afterSeq, upToSeq) { - try { - const agentId = this._agentId || this._url.match(/\/agents\/([^/?]+)\//)?.[1] || ''; - const res = await $.getJSON( - `/api/agents/${encodeURIComponent(agentId)}/chat/events?session_id=${encodeURIComponent(this._sessionId)}&after=${afterSeq}&up_to=${upToSeq}` - ); - const evts = res.events || []; - this._log.warn('gap-fill response: afterSeq=' + afterSeq + ' upToSeq=' + upToSeq + ' returned=' + evts.length + ' seqs=' + evts.map(e=>e.seq).join(',')); - console.warn('[gap-fill] returned', evts.length, 'events for after=', afterSeq, 'up_to=', upToSeq, 'seqs:', evts.map(e=>e.seq).join(',')); - for (const ev of evts) { - if (ev.seq <= this._lastSeq) continue; - this._lastSeq = ev.seq; - this._dispatch(ev.event, ev.data); - } - } catch (err) { - this._log.warn('gap-fill failed', err, '— skipping gap'); - this._lastSeq = upToSeq - 1; - } - } - - // Drain _pendingQueue asynchronously — one event per animation frame so we - // never block the main thread with a large synchronous burst. - _drainQueue() { - if (this._pendingQueue.length === 0) { - console.warn('[sse] queue drain done _lastSeq=', this._lastSeq); - return; - } - const item = this._pendingQueue.shift(); - const itemSeq = item.data.seq || 0; - if (itemSeq && itemSeq <= this._lastSeq) { - console.log('[sse] queue dedup skip', item.evtName, 'seq=', itemSeq); - // Skip but continue draining without waiting — dedup is cheap - this._drainQueue(); + _dispatch(evtName, data) { + if (evtName === 'auth_expired') { + this.stop(); + window.location.href = '/login'; return; } - if (itemSeq) this._lastSeq = itemSeq; - this._dispatch(item.evtName, item.data); - // Yield to the browser between each real event - requestAnimationFrame(() => this._drainQueue()); - } - - _dispatch(evtName, data) { if (evtName === 'state_changed') { // Not turn-scoped — bridge straight to the document-level event that // agent_detail.html / sessions.html already listen for (debounced refresh). @@ -266,11 +206,8 @@ export class SSEAdapter { this._handler({ event: 'session_clear', data, seq: data.seq || 0 }); return; } - // done: stop reconnecting after this if (evtName === 'done') { - console.warn('[sse] _dispatch done _lastSeq=', this._lastSeq, 'data.seq=', data.seq); this._handler({ event: 'done', data, seq: data.seq || 0 }); - this.stop(); return; } this._handler({ event: evtName, data, seq: data.seq || 0 }); diff --git a/static/js/chat-ui/turn.js b/static/js/chat-ui/turn.js index ba5ab90e..72708217 100644 --- a/static/js/chat-ui/turn.js +++ b/static/js/chat-ui/turn.js @@ -23,8 +23,6 @@ import { log, assert } from './debug.js'; import { buildSavedArtifactsBlock } from './artifacts.js'; -const STALE_TIMEOUT_MS = 300_000; // 5 minutes — safety net for truly abandoned turns - const TERMINAL_PHASES = new Set(['final', 'done', 'aborted']); /** @@ -119,7 +117,6 @@ export class Turn { this._lastSeq = 0; this._transports = []; this._timerInterval = null; - this._staleTimeout = null; this._scrollRAF = null; this._startTime = Date.now(); this._finalized = false; @@ -133,7 +130,6 @@ export class Turn { this._buildDOM(); this._startTimer(); - this._armStaleTimeout(); } // ── DOM ────────────────────────────────────────────────────────────────── @@ -195,16 +191,8 @@ export class Turn { }, 100); } - _armStaleTimeout() { - this._staleTimeout = setTimeout(() => { - this._log.warn('stale timeout reached, auto-finalizing', this.id); - this._finalizeBubble(null); - }, STALE_TIMEOUT_MS); - } - _clearTimers() { if (this._timerInterval) { clearInterval(this._timerInterval); this._timerInterval = null; } - if (this._staleTimeout) { clearTimeout(this._staleTimeout); this._staleTimeout = null; } if (this._scrollRAF) { cancelAnimationFrame(this._scrollRAF); this._scrollRAF = null; } } @@ -232,15 +220,6 @@ export class Turn { } if (seq) this._lastSeq = seq; - // Reset stale timeout on every live event — turn is clearly still active - if (this._staleTimeout) { - clearTimeout(this._staleTimeout); - this._staleTimeout = setTimeout(() => { - this._log.warn('stale timeout reached, auto-finalizing', this.id); - this._finalizeBubble(null); - }, STALE_TIMEOUT_MS); - } - this._log.debug('ingest', evtName, seq, '→ phase was', this.phase, this.id); const nextPhase = reduceTurn(this.phase, evtName, data.is_final); @@ -357,7 +336,7 @@ export class Turn { console.warn('[turn] done event turn=%s _finalized=%s _finalContent=%s', this.id, this._finalized, !!this._finalContent); this._finalizeBubble(data.thinking_duration); // Fire final:response so page-level code can render the response bubble - // synchronously — no dependency on pollForResponse JSONL poll. + // synchronously from the durable stream. if (this._finalContent) { console.warn('[turn] firing final:response turn=%s contentLen=%d', this.id, this._finalContent.length); this._onTrigger('final:response', { @@ -370,9 +349,12 @@ export class Turn { } if (evtName === 'turn_split') { - this._finalizeBubble(null); + const splitDuration = data.timestamp && this._startTime + ? Math.max(0, (data.timestamp - this._startTime) / 1000) + : null; + this._finalizeBubble(splitDuration); // ChatUI will create a new Turn for the continuation - this._onTrigger('turn:split', { turnId: this.id }); + this._onTrigger('turn:split', { turnId: this.id, timestamp: data.timestamp }); return; } diff --git a/static/js/realtime.js b/static/js/realtime.js index 16f1a7ec..1bef7673 100644 --- a/static/js/realtime.js +++ b/static/js/realtime.js @@ -11,7 +11,6 @@ * sessionId: 'abc123', * agentId: 'my-agent', * workplace: 'wp-1', - * chatThrottle: 100, * }); * * rt.on('status', 'agent_busy_changed', (data) => { ... }); @@ -22,20 +21,6 @@ var RealtimeClient = (function () { 'use strict'; - // ---- Channel definitions ---- - var CHANNEL_PRIORITY = { - status: 0, update: 0, // Level 0: system/update - approvals: 1, // Level 1: user-facing - chat: 2, workplace: 2, // Level 2: high throughput - heartbeat: 0, - auth_expired: 0, - channel_disabled: 0, - }; - - // Per-channel resume sequence trackers - var _channelSeqs = {}; - var _channelIds = {}; // channel -> last SSE id - function RealtimeClient(opts) { opts = opts || {}; this._channels = (opts.channels || 'status,approvals,update').split(',').map(function (s) { return s.trim(); }); @@ -44,13 +29,11 @@ var RealtimeClient = (function () { this._agentId = opts.agentId || ''; this._after = opts.after || 0; this._workplace = opts.workplace || ''; - this._chatThrottle = opts.chatThrottle || 100; this._es = null; this._handlers = {}; // channel -> [handler] this._started = false; this._intentionallyStopped = false; this._paused = false; - this._pauseBuffer = {}; // channel -> [events] buffered during pause this._onAuthExpired = opts.onAuthExpired || function () { window.location.href = '/login'; }; this._visibilityBound = false; this._unloadHandlers = []; // cleanup hooks registered by consumers @@ -92,27 +75,13 @@ var RealtimeClient = (function () { RealtimeClient.prototype.pause = function () { if (this._paused) return; this._paused = true; - if (this._es && this._es.readyState === EventSource.OPEN) { - // Send pause signal via a separate fetch - this._sendCommand('pause'); - } + this._disconnect(); }; RealtimeClient.prototype.resume = function () { if (!this._paused) return; this._paused = false; - if (this._es && this._es.readyState === EventSource.OPEN) { - this._sendCommand('resume'); - } - // Replay buffered events - var self = this; - Object.keys(this._pauseBuffer).forEach(function (ch) { - var buf = self._pauseBuffer[ch]; - while (buf && buf.length) { - var item = buf.shift(); - self._dispatch(ch, item.evtName, item.data); - } - }); + if (this._started) this._connect(); }; // ---- Internal: Connection lifecycle ---- @@ -120,19 +89,20 @@ var RealtimeClient = (function () { RealtimeClient.prototype._buildUrl = function () { var params = []; params.push('channels=' + encodeURIComponent(this._channels.join(','))); + params.push('cursor_version=2'); + params.push('snapshot=' + (this._after ? '0' : '1')); if (this._chat) { params.push('chat=1'); if (this._sessionId) params.push('session_id=' + encodeURIComponent(this._sessionId)); if (this._agentId) params.push('agent_id=' + encodeURIComponent(this._agentId)); - if (this._after) params.push('after=' + this._after); } + if (this._after) params.push('after=' + this._after); if (this._workplace) params.push('workplace=' + encodeURIComponent(this._workplace)); - if (this._chatThrottle) params.push('chat_throttle=' + this._chatThrottle); return '/api/realtime/stream?' + params.join('&'); }; RealtimeClient.prototype._connect = function () { - if (this._intentionallyStopped) return; + if (this._intentionallyStopped || this._paused || this._es) return; var self = this; var url = this._buildUrl(); @@ -164,14 +134,17 @@ var RealtimeClient = (function () { // inner catch (in _dispatch) handles handler errors. var ALL_EVENTS = [ 'agent_busy_changed', 'agent_turn_complete', 'whatsapp_bridge_status', + 'panel_updated', 'approval_required', 'approval_resolved', 'update_status', 'update_done', 'turn_begin', 'thinking', 'tool_call_started', 'tool_executed', + 'state:changed', 'state_changed', 'tasks:auto_transition', 'tasks:stale', 'response_chunk', 'done', 'retry', 'message_injected', - 'message_injection_applied', 'whatsapp_restriction_warning', 'session_clear', 'turn_split', + 'message_injection_applied', 'message_received', 'turn_queued', + 'whatsapp_restriction_warning', 'session_clear', 'turn_split', 'connector_connected', 'connector_disconnected', 'connector_paired', 'workplace_status_changed', - 'heartbeat', 'auth_expired', 'channel_disabled', + 'ready', 'heartbeat', 'auth_expired', 'channel_disabled', ]; ALL_EVENTS.forEach(function (evtName) { @@ -189,7 +162,7 @@ var RealtimeClient = (function () { // Auto-reconnect with jitter var delay = 2000 + Math.floor(Math.random() * 5000); setTimeout(function () { - if (self._intentionallyStopped) return; + if (self._intentionallyStopped || self._paused) return; self._connect(); }, delay); }; @@ -202,24 +175,12 @@ var RealtimeClient = (function () { } }; - RealtimeClient.prototype._sendCommand = function (cmd) { - try { - var xhr = new XMLHttpRequest(); - xhr.open('POST', '/api/realtime/' + cmd, true); - xhr.send(); - } catch (_) {} - }; - // ---- Internal: Event routing ---- RealtimeClient.prototype._routeEvent = function (evtName, data, lastEventId) { - // Track per-channel seq from composite SSE id (e.g. "chat:892") + // Durable journal IDs are global and may legitimately skip after scope filtering. if (lastEventId) { - var parts = lastEventId.split(':'); - if (parts.length === 2) { - _channelIds[parts[0]] = lastEventId; - _channelSeqs[parts[0]] = parseInt(parts[1], 10) || 0; - } + this._after = Math.max(this._after, parseInt(lastEventId, 10) || 0); } // Map event name to channel @@ -241,15 +202,6 @@ var RealtimeClient = (function () { if (evtName === 'heartbeat') return; // no-op - // Pause buffering for chat/workplace events - if (this._paused && (channel === 'chat' || channel === 'workplace')) { - if (!this._pauseBuffer[channel]) this._pauseBuffer[channel] = []; - if (this._pauseBuffer[channel].length < 100) { - this._pauseBuffer[channel].push({ evtName: evtName, data: data }); - } - return; - } - this._dispatch(channel, evtName, data); }; @@ -279,7 +231,8 @@ var RealtimeClient = (function () { RealtimeClient.prototype._eventToChannel = function (evtName) { // Status channel events if (evtName === 'agent_busy_changed' || evtName === 'agent_turn_complete' || - evtName === 'whatsapp_bridge_status') { + evtName === 'turn_queued' || + evtName === 'whatsapp_bridge_status' || evtName === 'panel_updated') { return 'status'; } // Approval channel events @@ -293,9 +246,11 @@ var RealtimeClient = (function () { // Chat channel events if (evtName === 'turn_begin' || evtName === 'thinking' || evtName === 'tool_call_started' || evtName === 'tool_executed' || + evtName === 'state:changed' || evtName === 'state_changed' || + evtName === 'tasks:auto_transition' || evtName === 'tasks:stale' || evtName === 'response_chunk' || evtName === 'done' || evtName === 'retry' || evtName === 'message_injected' || - evtName === 'message_injection_applied' || + evtName === 'message_injection_applied' || evtName === 'message_received' || evtName === 'whatsapp_restriction_warning' || evtName === 'session_clear' || evtName === 'turn_split') { return 'chat'; @@ -330,24 +285,6 @@ var RealtimeClient = (function () { this._unloadHandlers.push(fn); }; - // ---- SSE comment handler (invoked by caller when EventSource - // comment events are intercepted — see x-sse-dropped below) ---- - - RealtimeClient.prototype._handleComment = function (comment) { - // :x-sse-dropped N — server lost N events on a channel - var match = comment.match(/^x-sse-dropped\s+(\d+)/); - if (match) { - var dropped = parseInt(match[1], 10); - console.warn('[realtime] stream thinned:', dropped, 'events dropped'); - this._dispatch('stream_thinned', { dropped: dropped }); - } - // :error channel= — producer error - var errMatch = comment.match(/^error\s+channel=(\S+)/); - if (errMatch) { - console.warn('[realtime] producer error on channel:', errMatch[1]); - } - }; - return RealtimeClient; })(); diff --git a/templates/agent_detail.html b/templates/agent_detail.html index e54b963f..33d78bb3 100644 --- a/templates/agent_detail.html +++ b/templates/agent_detail.html @@ -1696,7 +1696,7 @@

Allowed - + - - + + +