From 58f4b11d1d1649bf0560c3818c2daa8d5e0a32c4 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 20 Aug 2026 15:37:39 +0000 Subject: [PATCH] Stamp observability metadata/tags on preset automation conversations Preset scripts create their RemoteConversation directly against the sandbox agent server, bypassing the app-server start path, so their Laminar traces carry no origin signal and are indistinguishable from regular UI conversations. Both preset scripts (prompt and plugin) now pass observability_metadata and observability_tags to Conversation(): - trigger=automation metadata + trigger:automation tag, matching the app-server convention so automation vs. UI traffic splits uniformly - automation_trigger (cron/event), automation_id, automation_name and automation_run_id from AUTOMATION_EVENT_PAYLOAD / env for drill-down - plugin preset additionally records experiment_id / experiment_variant for A/B analysis Gated on an inspect.signature check (same pattern as the existing user_id gate) so sandboxes running older SDK versions silently skip the new kwargs instead of failing. Co-authored-by: openhands --- .../automation/presets/plugin/sdk_main.py | 41 +++++++++++++++++++ .../automation/presets/prompt/sdk_main.py | 37 +++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/openhands/automation/presets/plugin/sdk_main.py b/openhands/automation/presets/plugin/sdk_main.py index 7c259b15..5e8b45fe 100644 --- a/openhands/automation/presets/plugin/sdk_main.py +++ b/openhands/automation/presets/plugin/sdk_main.py @@ -148,6 +148,37 @@ def _conversation_supports_user_id() -> bool: return False +def _conversation_supports_observability() -> bool: + try: + params = inspect.signature(Conversation.__new__).parameters + return "observability_metadata" in params and "observability_tags" in params + except (TypeError, ValueError): + return False + + +def _build_observability_context(event_context, automation_run_id): + """Trace metadata and span tags marking this conversation as automation-run. + + Uses the same ``trigger`` metadata key and ``trigger:`` tag as the + app-server's observability context so Laminar queries can split automation + vs. UI-driven conversations uniformly. + """ + metadata = {"trigger": "automation"} + tags = ["trigger:automation"] + if isinstance(event_context, dict): + trigger_type = event_context.get("trigger") + if trigger_type: + metadata["automation_trigger"] = trigger_type + tags.append(f"automation_trigger:{trigger_type}") + for key in ("automation_id", "automation_name"): + value = event_context.get(key) + if value: + metadata[key] = value + if automation_run_id: + metadata["automation_run_id"] = automation_run_id + return metadata, tags + + def _normalize_mcp_config(raw_mcp_config): if not raw_mcp_config: return {} @@ -460,6 +491,16 @@ def event_callback(event) -> None: } if automation_user_id and _conversation_supports_user_id(): conversation_kwargs["user_id"] = automation_user_id + if _conversation_supports_observability(): + obs_metadata, obs_tags = _build_observability_context( + event_context, automation_run_id + ) + if experiment_id: + obs_metadata["experiment_id"] = experiment_id + if selected_variant: + obs_metadata["experiment_variant"] = selected_variant + conversation_kwargs["observability_metadata"] = obs_metadata + conversation_kwargs["observability_tags"] = obs_tags conversation = Conversation(**conversation_kwargs) assert isinstance(conversation, RemoteConversation) print(f" conversation created: {type(conversation).__name__}") diff --git a/openhands/automation/presets/prompt/sdk_main.py b/openhands/automation/presets/prompt/sdk_main.py index c0014af5..7fc181ae 100644 --- a/openhands/automation/presets/prompt/sdk_main.py +++ b/openhands/automation/presets/prompt/sdk_main.py @@ -151,6 +151,37 @@ def _conversation_supports_user_id() -> bool: return False +def _conversation_supports_observability() -> bool: + try: + params = inspect.signature(Conversation.__new__).parameters + return "observability_metadata" in params and "observability_tags" in params + except (TypeError, ValueError): + return False + + +def _build_observability_context(event_context, automation_run_id): + """Trace metadata and span tags marking this conversation as automation-run. + + Uses the same ``trigger`` metadata key and ``trigger:`` tag as the + app-server's observability context so Laminar queries can split automation + vs. UI-driven conversations uniformly. + """ + metadata = {"trigger": "automation"} + tags = ["trigger:automation"] + if isinstance(event_context, dict): + trigger_type = event_context.get("trigger") + if trigger_type: + metadata["automation_trigger"] = trigger_type + tags.append(f"automation_trigger:{trigger_type}") + for key in ("automation_id", "automation_name"): + value = event_context.get(key) + if value: + metadata[key] = value + if automation_run_id: + metadata["automation_run_id"] = automation_run_id + return metadata, tags + + def _normalize_mcp_config(raw_mcp_config): if not raw_mcp_config: return {} @@ -413,6 +444,12 @@ def event_callback(event) -> None: } if automation_user_id and _conversation_supports_user_id(): conversation_kwargs["user_id"] = automation_user_id + if _conversation_supports_observability(): + obs_metadata, obs_tags = _build_observability_context( + event_context, automation_run_id + ) + conversation_kwargs["observability_metadata"] = obs_metadata + conversation_kwargs["observability_tags"] = obs_tags conversation = Conversation(**conversation_kwargs) assert isinstance(conversation, RemoteConversation) print(f" conversation created: {type(conversation).__name__}")