fix(llmobs): make span data JSON-safe at span finish [backport 4.14]

ddtrace/llmobs/_llmobs.py

@@ -154,7 +154,7 @@
 from ddtrace.llmobs._utils import _get_parent_prompt
 from ddtrace.llmobs._utils import _normalize_wire_trace_id_to_hex
 from ddtrace.llmobs._utils import _resolve_parent_agent
-from ddtrace.llmobs._utils import _sanitize_span_event_depth
+from ddtrace.llmobs._utils import _sanitize_span_event_data
 from ddtrace.llmobs._utils import _stamp_agent_attribution
 from ddtrace.llmobs._utils import _trace_id_to_wire
 from ddtrace.llmobs._utils import _validate_prompt
@@ -713,7 +713,7 @@ def _prepare_llmobs_span_data(self, span: Span, span_kind: Optional[str]) -> boo
         # reaches every span in its block, but only agent spans carry the tags.
         agent_annotation = span._get_ctx_item(AGENT_ANNOTATION)
         if agent_annotation and span_kind == "agent":
-            llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = agent_annotation
+            llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = str(agent_annotation)
 
         llmobs_meta = llmobs_data.setdefault(LLMOBS_STRUCT.META, _Meta())
         llmobs_input = llmobs_meta.get(LLMOBS_STRUCT.INPUT) or _MetaIO()
@@ -738,7 +738,10 @@ def _prepare_llmobs_span_data(self, span: Span, span_kind: Optional[str]) -> boo
             output_type,
             export_to_llmobs=self._export_mode != LLMObsExportMode.APM_AGENTLESS,
         )
-        llmobs_data[LLMOBS_STRUCT.META] = _sanitize_span_event_depth(llmobs_meta)
+        llmobs_data[LLMOBS_STRUCT.META] = _sanitize_span_event_data(llmobs_meta)
+        config = llmobs_data.get(LLMOBS_STRUCT.CONFIG)
+        if config is not None:
+            llmobs_data[LLMOBS_STRUCT.CONFIG] = _sanitize_span_event_data(config)
         if self._export_mode == LLMObsExportMode.APM_AGENTLESS:
             # APM agentless ingestion treats dots in tag keys as nested-path separators;
             # replace them with underscores before encoding.

ddtrace/llmobs/_utils.py

@@ -252,16 +252,21 @@ def _unserializable_default_repr(obj):
 _MAX_NESTED_META_DEPTH = 12
 
 
-def _sanitize_span_event_depth(obj: Any) -> Any:
-    """Return a sanitized copy of obj with any container value that exceeds
-    _MAX_NESTED_META_DEPTH levels from the root replaced by its JSON string representation,
-    and every mapping key stringified. The original structure is never mutated.
-    A debug log is emitted for each stringified field, including its dotted path.
+def _sanitize_span_event_data(obj: Any) -> Any:
+    """Return a copy of obj that is safe to encode into a span event.
+
+    Three guarantees, applied to every node: any container that exceeds _MAX_NESTED_META_DEPTH
+    levels from the root is replaced by its JSON string representation, every mapping key is
+    stringified, and every leaf value is made JSON-serializable. The original structure is never
+    mutated. A debug log is emitted for each stringified over-depth field, including its dotted path.
+
+    This is the last line of defense before the agentless APM exporter, which drops the whole trace
+    payload on a single unserializable object. It runs after the user span processor for that reason.
     """
 
     def _walk(node: Any, depth: int, path: str) -> Any:
         if not isinstance(node, (dict, list)):
-            return node
+            return _sanitize_leaf(node, depth, path)
         if depth >= _MAX_NESTED_META_DEPTH:
             log.debug(
                 "LLMObs: span event field %r exceeds the maximum nested depth of %d and will be "
@@ -274,6 +279,21 @@ def _walk(node: Any, depth: int, path: str) -> Any:
             return {str(k): _walk(v, depth + 1, f"{path}.{k}" if path else str(k)) for k, v in node.items()}
         return [_walk(v, depth + 1, f"{path}[{i}]" if path else str(i)) for i, v in enumerate(node)]
 
+    def _sanitize_leaf(node: Any, depth: int, path: str) -> Any:
+        if node is None or isinstance(node, (str, int, float, bool)):
+            return node
+        # load_data_value keeps structure where it can (models become dicts) instead of flattening
+        # to a string, so nested objects stay queryable. Re-walk it to apply the depth limit.
+        try:
+            loaded = load_data_value(node)
+        except Exception:
+            log.debug("LLMObs: span event field %r could not be converted; falling back to str.", path)
+            try:
+                return str(node)
+            except Exception:
+                return "[Unserializable object of type {}]".format(type(node).__name__)
+        return _walk(loaded, depth, path) if isinstance(loaded, (dict, list)) else loaded
+
     return _walk(obj, 0, "")
 
 
@@ -310,13 +330,17 @@ def load_data_value(value):
     elif isinstance(value, type):
         return value.__name__
     elif hasattr(value, "model_dump"):
-        return value.model_dump(exclude_none=True)
+        # model_dump() can leave non-primitive field values (datetime, Enum, ...) unconverted.
+        return load_data_value(value.model_dump(exclude_none=True))
     elif is_dataclass(value):
-        return asdict(value)
+        # asdict() deep-copies non-dataclass field types as-is, leaving them unconverted.
+        return load_data_value(asdict(value))
     elif isinstance(value, (int, float, str, bool)) or value is None:
         return value
     else:
         value_str = safe_json(value)
+        if value_str is None:  # safe_json swallows its failure and returns None
+            return str(value)
         try:
             return json.loads(value_str)
         except json.JSONDecodeError:
@@ -678,9 +702,13 @@ def _sanitize_metric_key(key):
 
     LLMObs ingestion interprets dots in a metric key as nested-path separators, which breaks
     decoding of the flat numeric metrics map and causes the enclosing span batch to be dropped.
-    Non-string keys are returned unchanged (value validation happens upstream in ``annotate``).
+    Non-string keys are stringified first, since a metric key is serialized as a string either way
+    and the encoder has no representation for other key types -- and stringifying can itself
+    introduce a dot (1.5 -> "1.5"). (Value validation happens upstream in annotate.)
     """
-    if not isinstance(key, str) or "." not in key:
+    if not isinstance(key, str):
+        key = str(key)
+    if "." not in key:
         return key
     sanitized = key.replace(".", "_")
     log.warning(
@@ -770,7 +798,8 @@ def _annotate_llmobs_span_data(
         if model_provider is not None:
             meta[LLMOBS_STRUCT.MODEL_PROVIDER] = model_provider
         if metadata is not None:
-            # Metadata keys are serialized as strings, so coerce non-string keys here.
+            # Metadata keys are serialized as strings, so coerce non-string keys here. Values are
+            # left as-is for in-process consumers; _sanitize_span_event_data handles them at finish.
             meta[LLMOBS_STRUCT.METADATA].update({str(k): v for k, v in metadata.items()})
         if agent_manifest is not None or cost_tags is not None:
             # Initialize metadata_dd here to avoid unnecessary empty dict allocations in the top-level metadata dict.
@@ -785,11 +814,11 @@ def _annotate_llmobs_span_data(
         if metrics is not None:
             llmobs_span_data[LLMOBS_STRUCT.METRICS].update({_sanitize_metric_key(k): v for k, v in metrics.items()})
         if tags is not None:
-            # Tag values are serialized as strings, so coerce non-string values here.
-            llmobs_span_data[LLMOBS_STRUCT.TAGS].update({k: str(v) for k, v in tags.items()})
+            # Tag keys and values are both serialized as strings, so coerce non-string ones here.
+            llmobs_span_data[LLMOBS_STRUCT.TAGS].update({str(k): str(v) for k, v in tags.items()})
         if session_id is not None:
             llmobs_span_data[LLMOBS_STRUCT.SESSION_ID] = session_id
-            llmobs_span_data[LLMOBS_STRUCT.TAGS]["session_id"] = session_id
+            llmobs_span_data[LLMOBS_STRUCT.TAGS]["session_id"] = str(session_id)
             span._set_ctx_item(SESSION_ID, session_id)
         if span_links is not None:
             llmobs_span_data[LLMOBS_STRUCT.SPAN_LINKS] = span_links

releasenotes/notes/llmobs-metadata-json-safety-9c3f7a1e4b2d6f80.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    LLM Observability: Fixes an issue where the agentless exporter dropped traces due to non-JSON-serializable objects being stored on a span.
+    All LLM Observability span data is now made JSON-serializable before the span is submitted. Non-string span tag keys, span metric keys,
+    and agent versions supplied via ``LLMObs.annotate(agent=...)`` are also now coerced to strings.

tests/contrib/google_genai/utils.py

@@ -224,17 +224,31 @@ def get_current_weather(location: str, unit: str = "fahrenheit") -> dict[str, ob
 )
 
 
+def _as_submitted(value):
+    """Mirror the conversion a metadata value undergoes on its way onto the span.
+
+    Span data is made JSON-serializable at span finish, which turns pydantic config objects such as
+    SafetySetting into plain dicts. Dumping here rather than hardcoding the result keeps these
+    expectations correct across the google-genai versions in the test matrix.
+    """
+    if isinstance(value, list):
+        return [_as_submitted(v) for v in value]
+    if hasattr(value, "model_dump"):
+        return value.model_dump(exclude_none=True)
+    return value
+
+
 def get_expected_metadata():
     metadata = {}
     for param in GENERATE_METADATA_PARAMS:
-        metadata[param] = getattr(FULL_GENERATE_CONTENT_CONFIG, param, None)
+        metadata[param] = _as_submitted(getattr(FULL_GENERATE_CONTENT_CONFIG, param, None))
 
     return metadata
 
 
 def get_expected_tool_metadata():
     metadata = {}
     for param in GENERATE_METADATA_PARAMS:
-        metadata[param] = getattr(TOOL_GENERATE_CONTENT_CONFIG, param, None)
+        metadata[param] = _as_submitted(getattr(TOOL_GENERATE_CONTENT_CONFIG, param, None))
 
     return metadata

tests/contrib/litellm/utils.py

@@ -237,7 +237,11 @@ def parse_response(resp, is_completion=False):
 ]
 
 expected_router_settings = {
-    "router_general_settings": RouterGeneralSettings(async_only_mode=False, pass_through_all_models=False),
+    # Dumped rather than kept as a model: span data is made JSON-serializable at span finish, so
+    # what lands on the span is a plain dict.
+    "router_general_settings": RouterGeneralSettings(async_only_mode=False, pass_through_all_models=False).model_dump(
+        exclude_none=True
+    ),
     "routing_strategy": "simple-shuffle",
     "routing_strategy_args": {},
     "provider_budget_config": None,

tests/llmobs/test_llmobs.py

@@ -1,4 +1,5 @@
 import asyncio
+import json
 import os
 from textwrap import dedent
 from typing import Optional
@@ -9,6 +10,7 @@
 from ddtrace.ext import SpanTypes
 from ddtrace.internal.utils.formats import format_trace_id
 from ddtrace.llmobs import LLMObsSpan
+from ddtrace.llmobs._constants import AGENT_VERSION_TAG_KEY
 from ddtrace.llmobs._constants import LANGCHAIN_APM_SPAN_NAME
 from ddtrace.llmobs._constants import LLMOBS_APM_SHADOW_CACHE_READ_INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import LLMOBS_APM_SHADOW_CACHE_WRITE_INPUT_TOKENS_METRIC_KEY
@@ -17,6 +19,7 @@
 from ddtrace.llmobs._constants import LLMOBS_APM_SHADOW_OUTPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import LLMOBS_APM_SHADOW_SPAN_KIND_TAG_KEY
 from ddtrace.llmobs._constants import LLMOBS_APM_SHADOW_TOTAL_TOKENS_METRIC_KEY
+from ddtrace.llmobs._constants import LLMOBS_STRUCT
 from ddtrace.llmobs._constants import LLMOBS_SUBMITTED_TAG_KEY
 from ddtrace.llmobs._constants import ROOT_PARENT_ID
 from ddtrace.llmobs._constants import UNKNOWN_MODEL_PROVIDER
@@ -1432,3 +1435,83 @@ def test_sampling_decisions_follow_configured_rate():
     assert abs(sampled - expected) <= 25, (
         f"rate={configured_rate}: expected ~{expected} sampled out of {n}, got {sampled}"
     )
+
+
+class TestSpanEventJSONSafety:
+    """Span data must be JSON-safe by span finish, or agentless APM exporter will drop the entire trace.
+
+    Sanitizing at finish rather than at annotation time also covers what the user span processor adds.
+    """
+
+    class RawObject:
+        """Stands in for the live SDK objects integrations hand us (e.g. a safety-settings enum)."""
+
+        def __init__(self, value):
+            self.value = value
+
+        def __str__(self):
+            return "RawObject({})".format(self.value)
+
+    def test_agent_version_tag_is_stringified(self, llmobs):
+        """annotate(agent=...) is not type-validated, and the version is written into tags at finish,
+        after the string coercion in _annotate_llmobs_span_data has already run.
+        """
+        with llmobs.agent(name="my_agent") as span:
+            llmobs.annotate(span=span, agent={"version": self.RawObject("v2")})
+        tags = _get_llmobs_data_metastruct(span)[LLMOBS_STRUCT.TAGS]
+        assert tags[AGENT_VERSION_TAG_KEY] == "RawObject(v2)"
+
+    def test_metadata_is_sanitized_at_finish(self, llmobs, tracer):
+        with tracer.trace("root", span_type=SpanTypes.LLM) as span:
+            _annotate_llmobs_span_data(
+                span,
+                kind="llm",
+                metadata={"safety_settings": [self.RawObject("BLOCK_NONE")], "nested": {"cfg": self.RawObject("LOW")}},
+            )
+        assert get_llmobs_metadata(span) == {
+            "safety_settings": ["RawObject(BLOCK_NONE)"],
+            "nested": {"cfg": "RawObject(LOW)"},
+        }
+
+    def test_config_is_sanitized_at_finish(self, llmobs, tracer):
+        """config sits at the top level of the struct rather than under meta, so it gets its own pass."""
+        with tracer.trace("root", span_type=SpanTypes.LLM) as span:
+            _annotate_llmobs_span_data(span, kind="llm", config={"safety_settings": [self.RawObject("BLOCK_NONE")]})
+        data = _get_llmobs_data_metastruct(span)
+        assert data[LLMOBS_STRUCT.CONFIG] == {"safety_settings": ["RawObject(BLOCK_NONE)"]}
+
+    def test_metadata_from_user_span_processor_is_sanitized(self, llmobs, tracer):
+        """A user processor writing a raw object into metadata must not be able to poison the payload.
+
+        This is the case write-time enforcement in _annotate_llmobs_span_data cannot cover: the
+        processor runs at finish, after every annotation has already been stored.
+        """
+        raw = self.RawObject("from-processor")
+
+        def _sp(span):
+            span.metadata["injected"] = raw
+            return span
+
+        llmobs.register_processor(_sp)
+        try:
+            with tracer.trace("root", span_type=SpanTypes.LLM) as span:
+                _annotate_llmobs_span_data(span, kind="llm")
+            assert get_llmobs_metadata(span)["injected"] == "RawObject(from-processor)"
+        finally:
+            llmobs.register_processor(None)
+
+    def test_whole_struct_survives_json_encoding(self, llmobs, tracer):
+        """Reproduces the actual failure mode: json.dumps over the whole struct, the way the agentless
+        APM exporter encodes it. The per-field tests above assert sanitized values; this one asserts
+        that no other field (tags, metrics, _dd, ...) can still raise TypeError and drop the payload.
+        """
+        with tracer.trace("root", span_type=SpanTypes.LLM) as span:
+            _annotate_llmobs_span_data(
+                span,
+                kind="llm",
+                metadata={"raw": self.RawObject("m")},
+                config={"raw": self.RawObject("c")},
+                tags={self.RawObject("k"): self.RawObject("v")},
+                metrics={"input_tokens": 1},
+            )
+        json.dumps(_get_llmobs_data_metastruct(span))