From ab369771da7106628795c59aa3f6807e2d3db763 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Mon, 3 Aug 2026 22:58:25 +0200 Subject: [PATCH 01/11] Add `chat` instrumentation for the smolagents model classes Wrap every smolagents model class that defines its own `generate`, and emit a chat span through the public opentelemetry-util-genai API. The span records the request parameters, provider and endpoint, token usage, finish reason, tool definitions, and the input and output messages. Agent runs and tool calls stay uninstrumented. They follow in separate PRs. Tests run against smolagents 1.24.0 (the declared floor) and against the latest release, plus Weaver conformance scenarios for chat, tool-calling, image input, and reasoning output. Known gaps: - Model.generate_stream is not instrumented, so a call made with stream_outputs=True produces no chat span, no metrics, and no token usage. - A user-defined Model subclass that overrides generate shadows the patched base method and produces no chat span. --- .../.changelog/352.added | 1 + .../README.rst | 37 +- .../pyproject.toml | 2 +- .../genai/smolagents/__init__.py | 93 +- .../genai/smolagents/_messages.py | 316 ++++ .../genai/smolagents/package.py | 2 + .../instrumentation/genai/smolagents/patch.py | 436 ++++++ .../genai/smolagents/provider.py | 151 ++ .../tests/cassettes/litellm_reasoning.yaml | 20 + .../tests/cassettes/openai_model_basic.yaml | 25 + .../cassettes/openai_model_image_url.yaml | 26 + .../tests/cassettes/openai_model_tool.yaml | 32 + .../tests/conformance/__init__.py | 0 .../tests/conformance/_helpers.py | 34 + .../tests/conformance/inference.py | 113 ++ .../tests/conformance/multimodal.py | 163 +++ .../tests/conftest.py | 85 +- .../tests/requirements.latest.txt | 5 +- .../tests/requirements.oldest.txt | 14 +- .../tests/test_conformance.py | 46 + .../tests/test_instrumentor.py | 260 +++- .../tests/test_models.py | 1296 +++++++++++++++++ .../tests/test_utils.py | 222 +++ tox.ini | 4 + 24 files changed, 3339 insertions(+), 44 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added new file mode 100644 index 000000000..5aca70172 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added @@ -0,0 +1 @@ +Add ``chat`` instrumentation for the smolagents model classes. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index 932ad2c12..fa97a7c54 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -7,7 +7,31 @@ OpenTelemetry smolagents Instrumentation :target: https://pypi.org/project/opentelemetry-instrumentation-genai-smolagents/ This library provides OpenTelemetry instrumentation for `smolagents -`_. +`_. It wraps the smolagents model +classes and emits a GenAI semantic-convention ``chat`` span and the matching +metrics through ``opentelemetry-util-genai``. + +Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not +instrumented yet. A model call made inside an agent run still gets a ``chat`` +span, but no agent span sits above it. + +A streamed model call, whether it comes from ``stream_outputs=True`` on an agent +or from calling ``Model.generate_stream`` directly, gets a ``chat`` span that +stays open until the caller drains the deltas. The span carries +``gen_ai.request.stream``, and the call also records the +``gen_ai.client.operation.time_to_first_chunk`` and +``gen_ai.client.operation.time_per_output_chunk`` metrics. + +Known gaps: + +* The instrumentation patches ``generate`` and ``generate_stream`` on the model + classes that smolagents ships. A subclass that inherits either method is + instrumented. A subclass that overrides one is not: the override shadows the + patched method, so the call produces no ``chat`` span. +* A streamed ``chat`` span reports no ``gen_ai.response.id`` and no + ``gen_ai.response.model``, because a smolagents stream delta carries neither. + The span reports ``gen_ai.response.finish_reasons`` only when the model + requested tool calls, which is the one stop reason the deltas make visible. Installation ------------ @@ -24,10 +48,13 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import InferenceClientModel - # Instrument smolagents SmolagentsInstrumentor().instrument() + model = InferenceClientModel() + model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + Configuration ------------- @@ -71,6 +98,12 @@ environment variable: SmolagentsInstrumentor().instrument(completion_hook=my_hook) +Conformance +----------- + +The scenarios that check this package against the GenAI semantic conventions +live under ``tests/conformance/``. + References ---------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml index 9f7ed96ed..67140011a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "opentelemetry-api ~= 1.43", "opentelemetry-instrumentation >= 0.64b0, <1", "opentelemetry-semantic-conventions >= 0.64b0, <1", - "opentelemetry-util-genai >= 1.0b0, <2", + "opentelemetry-util-genai >= 1.1b0.dev, <2", ] [project.optional-dependencies] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index f882e1976..e43c85b5f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -7,6 +7,9 @@ Instrumentation for `smolagents `_. +Model calls are recorded as ``chat`` spans. Agent runs and tool calls are not +instrumented yet. + Usage ----- @@ -15,10 +18,13 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import InferenceClientModel - # Enable instrumentation SmolagentsInstrumentor().instrument() + model = InferenceClientModel() + model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + Configuration ------------- @@ -39,20 +45,63 @@ from __future__ import annotations from collections.abc import Collection +from types import ModuleType from typing import Any +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.util.genai.completion_hook import load_completion_hook from opentelemetry.util.genai.handler import TelemetryHandler from .package import _instruments +from .patch import model_generate, model_generate_stream __all__ = ["SmolagentsInstrumentor"] +def _model_classes_defining(smolagents: ModuleType, method: str) -> list[type]: + """The exported model classes whose ``method`` gets wrapped. + + Only classes that define ``method`` in their own ``__dict__`` are patched, + so a class that inherits it (``AzureOpenAIModel``, ``LiteLLMRouterModel``) + isn't wrapped a second time and can't produce duplicate ``chat`` spans. + A user-defined subclass that overrides the method shadows the patched base + method and emits no ``chat`` span; that limitation is documented in + ``README.rst``. + + Deduplicated by class object, because smolagents exports some classes under + two names (``OpenAIServerModel`` is ``OpenAIModel``) and wrapping the same + class twice would double every ``chat`` span. + """ + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + Model, + ) + + classes: dict[type, None] = {} + for obj in vars(smolagents).values(): + if ( + isinstance(obj, type) + and issubclass(obj, Model) + and method in obj.__dict__ + ): + classes.setdefault(obj, None) + return list(classes) + + class SmolagentsInstrumentor(BaseInstrumentor): """An instrumentor for smolagents.""" + # ``BaseInstrumentor.__new__`` returns a per-class singleton, but Python + # still runs ``__init__`` on every construction. Initializing this state in + # ``__init__`` would let the documented ``SmolagentsInstrumentor() + # .uninstrument()`` form wipe the live instance's bookkeeping and leave + # smolagents permanently patched, so these are class-level defaults that + # only ``_instrument`` / ``_uninstrument`` rebind. + _wrapped_generate_classes: list[type] = [] + _wrapped_generate_stream_classes: list[type] = [] + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -66,15 +115,51 @@ def _instrument(self, **kwargs: Any) -> None: - logger_provider: LoggerProvider instance - completion_hook: CompletionHook instance """ - TelemetryHandler( + import smolagents # pylint: disable=import-outside-toplevel # noqa: PLC0415 + + handler = TelemetryHandler( tracer_provider=kwargs.get("tracer_provider"), meter_provider=kwargs.get("meter_provider"), logger_provider=kwargs.get("logger_provider"), completion_hook=kwargs.get("completion_hook") or load_completion_hook(), ) - # Patching will be added in follow-up PRs. + + wrapped_generate_classes: list[type] = [] + self._wrapped_generate_classes = wrapped_generate_classes + wrapped_generate_stream_classes: list[type] = [] + self._wrapped_generate_stream_classes = wrapped_generate_stream_classes + try: + for model_cls in _model_classes_defining(smolagents, "generate"): + wrap_function_wrapper( + model_cls, + "generate", + model_generate(handler), + ) + wrapped_generate_classes.append(model_cls) + + for model_cls in _model_classes_defining( + smolagents, "generate_stream" + ): + wrap_function_wrapper( + model_cls, + "generate_stream", + model_generate_stream(handler), + ) + wrapped_generate_stream_classes.append(model_cls) + except Exception: + # BaseInstrumentor.instrument() doesn't mark the instrumentor as + # instrumented when _instrument raises, so uninstrument() would + # refuse to run and leave the patches applied with no way to undo. + self._uninstrument() + raise def _uninstrument(self, **kwargs: Any) -> None: """Disable smolagents instrumentation and restore patched originals.""" - # Unpatching will be added in follow-up PRs. + for model_cls in self._wrapped_generate_classes: + unwrap(model_cls, "generate") + self._wrapped_generate_classes = [] + + for model_cls in self._wrapped_generate_stream_classes: + unwrap(model_cls, "generate_stream") + self._wrapped_generate_stream_classes = [] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py new file mode 100644 index 000000000..797021372 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -0,0 +1,316 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Convert smolagents message and tool shapes into util-genai GenAI types. + +smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects +or plain dicts, and returns a ``ChatMessage``. This module maps those, and the +``tools_to_call_from`` tool objects, onto the types in +``opentelemetry.util.genai.types``. util-genai then serializes them into +``gen_ai.input.messages``, ``gen_ai.output.messages``, and +``gen_ai.tool.definitions``. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +from collections.abc import Mapping +from enum import Enum +from typing import Any + +from opentelemetry.util.genai.types import ( + Blob, + FunctionToolDefinition, + InputMessage, + OutputMessage, + Reasoning, + Text, + ToolCallRequest, + ToolDefinition, + Uri, +) + +_logger = logging.getLogger(__name__) + +_DEFAULT_IMAGE_MIME_TYPE = "image/png" +_DATA_URL_PREFIX = "data:" + +# smolagents-internal roles -> semconv ``gen_ai`` message roles. smolagents +# applies the same mapping (``models.tool_role_conversions``) inside +# ``generate``, after the wrapper has already read ``messages``, so the wrapper +# has to apply it too. Without this map the input messages would carry roles +# the spec doesn't define. A model configured with ``custom_role_conversions`` +# overrides the mapping and can send different roles. Roles that are already +# spec values pass through unchanged. +_ROLE_MAP: dict[str, str] = { + "tool-call": "assistant", + "tool-response": "user", +} + +# Amazon Bedrock reports the stop reason as ``stopReason`` using Anthropic's +# vocabulary. Normalize it the way the anthropic instrumentation does +# (``anthropic/utils.py``); anything unmapped passes through. +_STOP_REASON_MAP: dict[str, str] = { + "end_turn": "stop", + "stop_sequence": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", +} + + +def _unwrap_role(role: Any) -> str | None: + if role is None: + return None + if isinstance(role, Enum): + role = role.value + name = str(role) + return _ROLE_MAP.get(name, name) + + +def _decode_base64_image(image: str) -> tuple[bytes, str] | None: + """Decode a base64 payload or data URL into ``(bytes, mime_type)``.""" + mime_type = _DEFAULT_IMAGE_MIME_TYPE + if image.startswith(_DATA_URL_PREFIX): + header, _, image = image.partition(",") + media_type = header[len(_DATA_URL_PREFIX) :].split(";")[0] + if media_type: + mime_type = media_type + try: + return base64.b64decode(image, validate=True), mime_type + except (binascii.Error, ValueError): + _logger.debug("Failed to decode a base64 image", exc_info=True) + return None + + +def _encode_image_base64(image: Any) -> str | None: + try: + from smolagents.utils import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + encode_image_base64, + ) + except ImportError: + _logger.debug("smolagents.utils.encode_image_base64 is unavailable") + return None + try: + encoded = encode_image_base64(image) + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to encode image of type %s, dropping it from telemetry", + type(image).__name__, + exc_info=True, + ) + return None + return encoded if isinstance(encoded, str) else None + + +def _image_blob(image: Any) -> Blob | None: + """Build a ``Blob`` part from a base64 string, data URL, or PIL image.""" + if isinstance(image, str): + decoded = _decode_base64_image(image) + else: + encoded = _encode_image_base64(image) + decoded = ( + _decode_base64_image(encoded) if encoded is not None else None + ) + if decoded is None: + return None + content, mime_type = decoded + return Blob(mime_type=mime_type, modality="image", content=content) + + +def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: + content_type = element.get("type") + if content_type == "image_url": + image_url = element.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + if isinstance(url, str) and url: + return Uri(mime_type=None, modality="image", uri=url) + return None + if content_type == "image": + image = element.get("image") + if image is not None: + return _image_blob(image) + return None + + +def _parts_from_content(content: Any) -> list[Any]: + parts: list[Any] = [] + if isinstance(content, str): + parts.append(Text(content=content)) + return parts + if isinstance(content, list): + for element in content: + if not isinstance(element, dict): + _logger.debug( + "Unknown message content dropped from telemetry: %s", + type(element).__name__, + ) + continue + if element.get("type") == "text" and (text := element.get("text")): + parts.append(Text(content=text)) + continue + image_part = _image_part_from_element(element) + if image_part is not None: + parts.append(image_part) + else: + _logger.debug( + "Unknown message part dropped from telemetry: %s", + element.get("type"), + ) + return parts + + +def _get_role_and_content(message: Any) -> tuple[Any, Any]: + if isinstance(message, dict): + return message.get("role"), message.get("content") + return getattr(message, "role", None), getattr(message, "content", None) + + +def to_input_messages(messages: Any) -> list[InputMessage]: + """Map smolagents ``generate`` input messages to ``InputMessage`` objects.""" + result: list[InputMessage] = [] + if not isinstance(messages, list): + return result + for message in messages: + raw_role, content = _get_role_and_content(message) + role = _unwrap_role(raw_role) + if not role: + continue + result.append( + InputMessage(role=role, parts=_parts_from_content(content)) + ) + return result + + +def _raw_value(raw: Any, key: str) -> Any: + """Read ``key`` off a provider response that may be an object or a dict. + + ``ChatMessage.raw`` is whatever the provider handed back: an OpenAI-shaped + object for the API-backed models, and a dict for ``AmazonBedrockModel`` + (the boto3 ``converse`` response) and the local runtimes. + """ + if isinstance(raw, Mapping): + return raw.get(key) + return getattr(raw, key, None) + + +def _first_choice(output_message: Any) -> Any: + choices = _raw_value(getattr(output_message, "raw", None), "choices") + if isinstance(choices, list) and choices: + return choices[0] + return None + + +def _reasoning_from_raw(output_message: Any) -> str | None: + message = _raw_value(_first_choice(output_message), "message") + reasoning = _raw_value(message, "reasoning_content") + return reasoning if isinstance(reasoning, str) and reasoning else None + + +def _tool_call_requests(output_message: Any) -> list[ToolCallRequest]: + # ChatMessage.__post_init__ coerces every entry into a + # ChatMessageToolCall, so the id/function/name/arguments are all present. + tool_calls = getattr(output_message, "tool_calls", None) or [] + return [ + ToolCallRequest( + name=tool_call.function.name, + id=tool_call.id, + arguments=tool_call.function.arguments, + ) + for tool_call in tool_calls + ] + + +def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: + """Why the provider stopped generating, or ``None`` if it didn't say. + + The local runtimes (``TransformersModel``, ``VLLMModel``, ``MLXModel``) put + ``{"out": ..., "completion_kwargs": ...}`` on ``raw`` and report no finish + reason at all. Defaulting those to ``"stop"`` would make a generation cut + short by ``max_new_tokens`` look like a natural stop. A response carrying + tool calls is the one case where the reason follows without guessing. + """ + reason = _raw_value(_first_choice(output_message), "finish_reason") + if isinstance(reason, str) and reason: + return reason + raw = getattr(output_message, "raw", None) + stop_reason = _raw_value(raw, "stopReason") + if isinstance(stop_reason, str) and stop_reason: + return _STOP_REASON_MAP.get(stop_reason, stop_reason) + return "tool_calls" if has_tool_calls else None + + +def to_output_message(output_message: Any) -> OutputMessage: + """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``.""" + role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" + parts: list[Any] = _parts_from_content( + getattr(output_message, "content", None) + ) + if reasoning := _reasoning_from_raw(output_message): + parts.append(Reasoning(content=reasoning)) + tool_call_requests = _tool_call_requests(output_message) + parts.extend(tool_call_requests) + # OutputMessage requires the field; util-genai drops an empty value when it + # emits gen_ai.response.finish_reasons. + reason = _finish_reason(output_message, bool(tool_call_requests)) + return OutputMessage(role=role, parts=parts, finish_reason=reason or "") + + +def response_id(output_message: Any) -> str | None: + """Extract ``gen_ai.response.id`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "id") + return value if isinstance(value, str) and value else None + + +def response_model_name(output_message: Any) -> str | None: + """Extract ``gen_ai.response.model`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "model") + return value if isinstance(value, str) and value else None + + +def _tool_parameters(tool: Any) -> dict[str, Any] | None: + """Return the JSON Schema ``parameters`` object for a smolagents tool. + + A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it + in an object schema, derives ``required`` from ``nullable``, and rewrites its + non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the + schema the provider receives. + """ + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + get_tool_json_schema, + ) + except ImportError: + _logger.debug("smolagents.models.get_tool_json_schema is unavailable") + return None + try: + schema = get_tool_json_schema(tool) + parameters = schema["function"]["parameters"] + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to build a JSON Schema for tool %s", + getattr(tool, "name", None), + exc_info=True, + ) + return None + return parameters if isinstance(parameters, dict) else None + + +def to_tool_definitions(tools: Any) -> list[ToolDefinition] | None: + """Map smolagents tool objects to function tool definitions.""" + if not isinstance(tools, list) or not tools: + return None + definitions: list[ToolDefinition] = [] + for tool in tools: + name = getattr(tool, "name", None) + if not name: + continue + definitions.append( + FunctionToolDefinition( + name=name, + description=getattr(tool, "description", None), + parameters=_tool_parameters(tool), + ) + ) + return definitions or None diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py index b56883424..fa7223304 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 _instruments = ("smolagents >= 1.24.0",) + +_supports_metrics = True diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py new file mode 100644 index 000000000..4653108aa --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -0,0 +1,436 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""wrapt wrapper factories for smolagents instrumentation. + +Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper +suitable for :func:`wrapt.wrap_function_wrapper`: + +- :func:`model_generate` wraps each defining ``Model.generate`` -> ``chat`` span. +- :func:`model_generate_stream` wraps each defining ``Model.generate_stream`` + -> ``chat`` span, held open until the stream is drained. + +Original library exceptions are always re-raised unmodified; telemetry is +finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Generator, Mapping +from dataclasses import dataclass +from inspect import signature +from typing import Any, Callable + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import InferenceInvocation +from opentelemetry.util.genai.stream import SyncStreamWrapper +from opentelemetry.util.genai.types import OutputMessage, Text, ToolCallRequest + +from ._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from .provider import resolve_provider, resolve_server_address_port + +_logger = logging.getLogger(__name__) + +_Wrapper = Callable[ + [Callable[..., Any], Any, tuple[Any, ...], dict[str, Any]], Any +] + + +def _bind_arguments( + wrapped: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> dict[str, Any]: + """Bind call args to the wrapped callable's signature, applying defaults. + + smolagents passes the interesting arguments positionally + (``model.generate(input_messages)``), so binding is what makes them + readable by name. On a binding failure the keyword arguments are returned + on their own, without the positional ones and without the defaults. + """ + try: + bound = signature(wrapped).bind(*args, **kwargs) + bound.apply_defaults() + return dict(bound.arguments) + except (TypeError, ValueError): + _logger.debug( + "Failed to bind arguments of %s; falling back to keyword arguments", + getattr(wrapped, "__qualname__", wrapped), + exc_info=True, + ) + return dict(kwargs) + + +def _coerce_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def _coerce_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _remove_parameter_sentinel() -> Any: + """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + REMOVE_PARAMETER, + ) + except ImportError: + _logger.debug("smolagents.models.REMOVE_PARAMETER is unavailable") + return None + return REMOVE_PARAMETER + + +# Model classes that never forward ``stop_sequences``, whatever +# ``supports_stop_parameter`` answers. ``AmazonBedrockModel`` overrides +# ``_prepare_completion_kwargs`` and calls the base with a hardcoded +# ``stop_sequences=None`` (``models.py``), so its ``converse`` request carries +# no stop sequences at all. ``_forwards_stop_sequences`` matches these names +# along the MRO, so subclasses are covered too. +_MODELS_DROPPING_STOP_SEQUENCES = frozenset({"AmazonBedrockModel"}) + + +def _forwards_stop_sequences(instance: Any) -> bool: + if any( + cls.__name__ in _MODELS_DROPPING_STOP_SEQUENCES + for cls in type(instance).__mro__ + ): + return False + return bool(getattr(instance, "supports_stop_parameter", False)) + + +def _merged_request_kwargs( + instance: Any, bound: dict[str, Any] +) -> dict[str, Any]: + """Rebuild the request keyword arguments smolagents will send. + + ``Model._prepare_completion_kwargs`` seeds ``stop`` from the + ``stop_sequences`` argument and ``response_format`` from its own argument, + applies the per-call ``**kwargs`` on top and the model-level ``self.kwargs`` + last, so a key set at two levels reaches the provider with the model-level + value and a model-level ``REMOVE_PARAMETER`` drops it from the request + entirely. Following that order here is what keeps a removed key off the span + as well. + """ + merged: dict[str, Any] = {} + stop_sequences = bound.get("stop_sequences") + if stop_sequences is not None and _forwards_stop_sequences(instance): + merged["stop"] = stop_sequences + response_format = bound.get("response_format") + if response_format is not None: + merged["response_format"] = response_format + call_kwargs = bound.get("kwargs") + if isinstance(call_kwargs, dict): + merged.update(call_kwargs) + model_kwargs = getattr(instance, "kwargs", None) + if not isinstance(model_kwargs, dict): + return merged + remove = _remove_parameter_sentinel() + for name, value in model_kwargs.items(): + if remove is not None and value is remove: + merged.pop(name, None) + else: + merged[name] = value + return merged + + +def _stop_sequences(merged: dict[str, Any]) -> list[str] | None: + """Return the stop sequences the request carries, if any.""" + stop = merged.get("stop", merged.get("stop_sequences")) + if isinstance(stop, str): + return [stop] + if isinstance(stop, list): + return [str(item) for item in stop] + return None + + +# smolagents' ``response_format`` type -> ``gen_ai.output.type`` value, for the +# types whose provider spelling differs from the semconv one. The openai +# instrumentation maps the same two. +_OUTPUT_TYPE_MAP: dict[str, str] = { + "json_object": GenAI.GenAiOutputTypeValues.JSON.value, + "json_schema": GenAI.GenAiOutputTypeValues.JSON.value, +} + +_OUTPUT_TYPE_VALUES = frozenset( + value.value for value in GenAI.GenAiOutputTypeValues +) + + +def _output_type(merged: dict[str, Any]) -> str | None: + """Map the request's ``response_format`` to ``gen_ai.output.type``. + + smolagents forwards ``response_format`` to the provider unchanged, so its + ``type`` is whatever the provider accepts. Only the values the semconv + defines are recorded; a provider-specific one is dropped rather than put on + an enum attribute. + """ + response_format = merged.get("response_format") + if not isinstance(response_format, Mapping): + return None + format_type = response_format.get("type") + if not isinstance(format_type, str): + return None + output_type = _OUTPUT_TYPE_MAP.get(format_type, format_type) + if output_type not in _OUTPUT_TYPE_VALUES: + _logger.debug( + "No gen_ai.output.type value for response_format type %r", + format_type, + ) + return None + return output_type + + +def _apply_request_parameters( + invocation: InferenceInvocation, instance: Any, bound: dict[str, Any] +) -> None: + """Copy the request parameters smolagents will send onto the span.""" + merged = _merged_request_kwargs(instance, bound) + + invocation.temperature = _coerce_float(merged.get("temperature")) + invocation.top_p = _coerce_float(merged.get("top_p")) + invocation.top_k = _coerce_float(merged.get("top_k")) + invocation.frequency_penalty = _coerce_float( + merged.get("frequency_penalty") + ) + invocation.presence_penalty = _coerce_float(merged.get("presence_penalty")) + # TransformersModel takes the generation limit as max_new_tokens (which its + # constructor defaults to 4096) and treats max_tokens as an alias for it. + invocation.max_tokens = _coerce_int( + merged.get("max_tokens", merged.get("max_new_tokens")) + ) + invocation.seed = _coerce_int(merged.get("seed")) + invocation.stop_sequences = _stop_sequences(merged) + invocation.output_type = _output_type(merged) + + +def _apply_token_usage( + invocation: InferenceInvocation, output_message: Any +) -> None: + # ChatMessage.token_usage is the only source: the per-model + # last_input_token_count / last_output_token_count counters were removed + # before the oldest supported smolagents. It is None for the local runtimes + # (TransformersModel, VLLMModel, MLXModel), which report no usage. + token_usage = getattr(output_message, "token_usage", None) + if token_usage is None: + return + invocation.input_tokens = token_usage.input_tokens + invocation.output_tokens = token_usage.output_tokens + + +def _start_inference( + handler: TelemetryHandler, + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> InferenceInvocation: + """Start the ``chat`` span and record the request. + + ``generate`` and ``generate_stream`` take the same parameters. + """ + provider = resolve_provider(instance) + server_address, server_port = resolve_server_address_port(instance) + invocation = handler.inference( + provider, + request_model=getattr(instance, "model_id", None), + server_address=server_address, + server_port=server_port, + ) + bound = _bind_arguments(wrapped, args, kwargs) + _apply_request_parameters(invocation, instance, bound) + invocation.tool_definitions = to_tool_definitions( + bound.get("tools_to_call_from") + ) + if handler.should_capture_content(): + invocation.input_messages = to_input_messages(bound.get("messages")) + return invocation + + +def model_generate(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate`` to emit a ``chat`` span.""" + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + output_message = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + _apply_token_usage(invocation, output_message) + invocation.response_model_name = response_model_name(output_message) + invocation.response_id = response_id(output_message) + output = to_output_message(output_message) + # to_output_message leaves finish_reason empty when the provider + # reported none, and an empty value is dropped rather than guessed at. + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if handler.should_capture_content(): + invocation.output_messages = [output] + invocation.stop() + return output_message + + return wrapper + + +@dataclass +class _StreamedToolCall: + """A tool call assembled from stream deltas.""" + + id: str | None = None + name: str = "" + arguments: str = "" + + +class _ModelStreamWrapper(SyncStreamWrapper[Any]): + """Keep the ``chat`` span open until the delta stream is drained. + + Passing the invocation to ``super().__init__()`` turns on + ``gen_ai.request.stream`` and the per-chunk timing metrics. + """ + + def __init__( + self, + stream: Generator[Any, Any, Any], + invocation: InferenceInvocation, + handler: TelemetryHandler, + ) -> None: + super().__init__(stream, invocation=invocation) + self._self_inference = invocation + self._self_handler = handler + self._self_content: list[str] = [] + self._self_tool_calls: dict[int, _StreamedToolCall] = {} + self._self_input_tokens = 0 + self._self_output_tokens = 0 + self._self_saw_token_usage = False + + def _accumulate_tool_call(self, delta: Any) -> None: + index = getattr(delta, "index", None) + if not isinstance(index, int): + # agglomerate_stream_deltas raises here; telemetry must not. + _logger.debug("Dropping a tool call delta that carries no index") + return + tool_call = self._self_tool_calls.setdefault( + index, _StreamedToolCall() + ) + if delta.id: + tool_call.id = delta.id + function = getattr(delta, "function", None) + if function is None: + return + if function.name: + tool_call.name = function.name + if function.arguments: + tool_call.arguments += function.arguments + + def _process_chunk(self, chunk: Any) -> None: + content = getattr(chunk, "content", None) + if content: + self._self_content.append(content) + token_usage = getattr(chunk, "token_usage", None) + if token_usage is not None: + # Summed like agglomerate_stream_deltas, so the span agrees with the + # totals the agent's monitor reports. + self._self_saw_token_usage = True + self._self_input_tokens += token_usage.input_tokens + self._self_output_tokens += token_usage.output_tokens + for delta in getattr(chunk, "tool_calls", None) or []: + self._accumulate_tool_call(delta) + + def _output_message(self) -> OutputMessage | None: + parts: list[Any] = [] + content = "".join(self._self_content) + if content: + parts.append(Text(content=content)) + parts.extend( + ToolCallRequest( + name=tool_call.name, + id=tool_call.id, + arguments=tool_call.arguments or None, + ) + for tool_call in self._self_tool_calls.values() + ) + if not parts: + # Closed before it was drained, so there is no response to report. + return None + # Deltas carry no finish reason, so tool calls are the only evidence. + # Defaulting to "stop" would hide a generation cut short by a token + # limit. + finish_reason = "tool_calls" if self._self_tool_calls else "" + return OutputMessage( + role="assistant", parts=parts, finish_reason=finish_reason + ) + + def _finalize(self, error: BaseException | None = None) -> None: + invocation = self._self_inference + if self._self_saw_token_usage: + invocation.input_tokens = self._self_input_tokens + invocation.output_tokens = self._self_output_tokens + output = self._output_message() + if output is not None: + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if self._self_handler.should_capture_content(): + invocation.output_messages = [output] + if error is not None: + invocation.fail(error) + else: + invocation.stop() + + def _on_stream_end(self) -> None: + self._finalize() + + def _on_stream_error(self, error: BaseException) -> None: + # Records what was streamed before the failure. + self._finalize(error) + + +def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate_stream`` to emit a ``chat`` span. + + ``stream_outputs=True`` routes an agent's model calls here. The span stays + open until the caller drains the deltas. + """ + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + stream = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + return _ModelStreamWrapper(stream, invocation, handler) + + return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py new file mode 100644 index 000000000..f531606c1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -0,0 +1,151 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve a smolagents model instance to a ``gen_ai.provider.name`` value. + +Resolution order: + +1. For the LiteLLM model classes, the ``model_id`` vendor prefix + (``anthropic/claude-...`` -> ``anthropic``). +2. The model class name (e.g. ``OpenAIModel`` -> ``openai``), looked up along the + class hierarchy so a user subclass resolves to the provider of the base class + whose ``generate`` it inherits. +3. ``unknown``. + +``gen_ai.provider.name`` is a metric attribute as well as a span attribute, so +every value has to stay low cardinality: deployment-specific detail is reported +as ``server.address`` instead. ``TelemetryHandler.inference`` requires +``provider`` as a string, so this always returns a value rather than ``None``. +""" + +from __future__ import annotations + +import logging +from typing import Any +from urllib.parse import urlparse + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + +_logger = logging.getLogger(__name__) + +_PROVIDER = GenAI.GenAiProviderNameValues + +_UNKNOWN_PROVIDER = "unknown" + +# Model class name -> provider value. The GenAI registry has no value for the +# Hugging Face, vLLM, and MLX runtimes, so those use the product name; a class +# name would look like a provider without being one. +_CLASS_NAME_TO_PROVIDER: dict[str, str] = { + "OpenAIModel": _PROVIDER.OPENAI.value, + "AzureOpenAIModel": _PROVIDER.AZURE_AI_OPENAI.value, + "AmazonBedrockModel": _PROVIDER.AWS_BEDROCK.value, + "InferenceClientModel": "huggingface", + "TransformersModel": "huggingface", + "VLLMModel": "vllm", + "MLXModel": "mlx", +} + +# LiteLLM model_id prefix -> semconv provider value, for the prefixes whose +# LiteLLM vendor slug differs from the semconv value. Every other prefix is +# passed through as-is (``ollama/llama3`` -> ``ollama``). LiteLLM's slugs are a +# closed vocabulary, which keeps the cardinality bounded. +_LITELLM_PREFIX_TO_PROVIDER: dict[str, str] = { + "azure": _PROVIDER.AZURE_AI_OPENAI.value, + "azure_ai": _PROVIDER.AZURE_AI_INFERENCE.value, + "bedrock": _PROVIDER.AWS_BEDROCK.value, + "gemini": _PROVIDER.GCP_GEMINI.value, + "mistral": _PROVIDER.MISTRAL_AI.value, + "vertex_ai": _PROVIDER.GCP_VERTEX_AI.value, + "watsonx": _PROVIDER.IBM_WATSONX_AI.value, + "xai": _PROVIDER.X_AI.value, +} + +_LITELLM_CLASS_NAMES = frozenset({"LiteLLMModel", "LiteLLMRouterModel"}) + + +def _class_names(instance: Any) -> list[str]: + """The instance's class names, most derived first. + + Only the classes that define ``generate`` are patched, so an instrumented + model can be a user subclass of one of them. Matching the exact class name + alone would report ``unknown`` for every such subclass. + """ + return [cls.__name__ for cls in type(instance).__mro__] + + +def _provider_from_litellm(instance: Any) -> str | None: + model_id = getattr(instance, "model_id", None) + if not isinstance(model_id, str) or "/" not in model_id: + return None + prefix = model_id.split("/", 1)[0].lower() + return _LITELLM_PREFIX_TO_PROVIDER.get(prefix, prefix) + + +def _endpoint(instance: Any) -> str | None: + """The endpoint URL the model calls, or ``None`` if it exposes none. + + Three places can hold it, checked in this order: + + 1. ``api_base`` on the instance (``LiteLLMModel``). + 2. ``base_url`` or ``azure_endpoint`` in the SDK client kwargs + (``OpenAIModel``, ``AzureOpenAIModel``, ``InferenceClientModel``). + 3. ``base_url`` on the client the model constructed (e.g. openai's httpx + URL), which holds the effective URL when the caller left it at the + provider default. + """ + api_base = getattr(instance, "api_base", None) + if api_base: + return str(api_base) + client_kwargs = getattr(instance, "client_kwargs", None) + if isinstance(client_kwargs, dict): + for key in ("base_url", "azure_endpoint"): + value = client_kwargs.get(key) + if value: + return str(value) + client_base_url = getattr( + getattr(instance, "client", None), "base_url", None + ) + if client_base_url: + return str(client_base_url) + return None + + +def resolve_server_address_port( + instance: Any, +) -> tuple[str | None, int | None]: + """Return ``(server.address, server.port)`` from the model's endpoint URL. + + Models that don't expose an ``api_base`` / ``base_url`` / ``azure_endpoint`` + (e.g. ``LiteLLMModel`` resolving the host internally, local runtimes) + yield ``(None, None)`` and the caller omits the attributes. + """ + endpoint = _endpoint(instance) + if endpoint is None: + return None, None + parsed = urlparse(endpoint) + port = parsed.port + if port == 443: + port = None + return parsed.hostname or None, port + + +def resolve_provider(instance: Any) -> str: + """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" + class_names = _class_names(instance) + + if not _LITELLM_CLASS_NAMES.isdisjoint(class_names): + provider = _provider_from_litellm(instance) + if provider is not None: + return provider + + for class_name in class_names: + provider = _CLASS_NAME_TO_PROVIDER.get(class_name) + if provider is not None: + return provider + + _logger.debug( + "No known gen_ai.provider.name for model class %s", class_names[0] + ) + return _UNKNOWN_PROVIDER diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml new file mode 100644 index 000000000..6177ff059 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml @@ -0,0 +1,20 @@ +interactions: +- request: + body: '{"model": "claude-3-7-sonnet-20250219", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Who won the World Cup in 2018? Answer in one word + with no punctuation."}]}], "thinking": {"type": "enabled", "budget_tokens": + 4000}, "max_tokens": 8096}' + headers: {} + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id":"msg_011KX7d4TtALbugymC3Kb4oE","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[{"type":"thinking","thinking":"The + World Cup in 2018 was won by France. They defeated Croatia 4-2 in the final + match in Moscow, Russia.\n\nI need to answer in one word with no punctuation, + so my answer should simply be:\nFrance","signature":"ErUBCkYIBBgCIkAOLwDXty2UNgzsRPd4O0tNxqhaxPqqLw9isc2bDqyVS7Y87Cefkib8FVE9iTTjTSynEjrrHb+vei4K5pQrVOUrEgzCGfpi49gURMlFYTwaDGTEJyP22/PoNkje2CIwyiqcZqRj6rxDwzG0ayl3JQ2mi8x3iDIEuyP92Pw19EPRhATbpYiORzbaVoPUBuPEKh3gUYNSZwcjUAYNO01Qqo7GYFF08xO0MuULheotLRgC"},{"type":"text","text":"France"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":65,"service_tier":"standard"}}' + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml new file mode 100644 index 000000000..44f70cc65 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"Who won the + World Cup in 2018? Answer in one word with no punctuation."}]}],"model":"gpt-4o","max_tokens":4096}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb\",\n \"object\": + \"chat.completion\",\n \"created\": 1738649686,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"France\",\n \"refusal\": null\n + \ },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 25,\n \"completion_tokens\": + 2,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_4691090a87\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml new file mode 100644 index 000000000..9fde9a56e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml @@ -0,0 +1,26 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What breed + is this dog?"},{"type":"image_url","image_url":{"url":"https://fastly.picsum.photos/id/237/200/300.jpg?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U"}}]}],"model":"gpt-4o"}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DVJBfdzKCrLtPYM9ui8SZgIEQv857\",\n \"object\": + \"chat.completion\",\n \"created\": 1776354295,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"This looks like a Labrador Retriever + puppy. They are known for their friendly and outgoing nature.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 268,\n \"completion_tokens\": 18,\n \"total_tokens\": 286,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml new file mode 100644 index 000000000..a5a57c069 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml @@ -0,0 +1,32 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What is the + weather in Paris?"}]}],"model":"gpt-4o","max_tokens":4096,"tool_choice":"required","tools": + [{"type":"function","function":{"name":"get_weather","description":"Get the weather for a + given city","parameters":{"type":"object","properties":{"location":{"type":"string","description": + "The city to get the weather for"}},"required":["location"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax7BZfEQe2evzgqbTVXWo0ZqoMIUf\",\n \"object\": + \"chat.completion\",\n \"created\": 1738652337,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_EUuviydGIG5Jau3DLw4v4cue\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"location\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": + null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": + {\n \"prompt_tokens\": 61,\n \"completion_tokens\": 15,\n \"total_tokens\": + 76,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n + \ \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py new file mode 100644 index 000000000..be0d0071e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py @@ -0,0 +1,34 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for smolagents conformance scenarios.""" + +from __future__ import annotations + +import json +from typing import Any + + +def attr(span: dict[str, Any], name: str) -> Any: + for entry in span["attributes"]: + if entry["name"] == name: + return entry["value"] + return None + + +def chat_spans(report: Any) -> list[dict[str, Any]]: + return [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + + +def part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py new file mode 100644 index 000000000..f9e60fb0f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py @@ -0,0 +1,113 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for the ``chat`` operation (plain and tool-calling).""" + +from __future__ import annotations + +from typing import Any + +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +from ..test_utils import GetWeatherTool, openai_model # noqa: TID252 +from ._helpers import attr, chat_spans, part_fields + + +class ChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_basic.yaml"): + openai_model().generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer " + "in one word with no punctuation." + ), + } + ], + } + ] + ) + + +class ToolCallingScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_tool.yaml"): + openai_model().generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_part_types = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "tool_call" in output_part_types, ( + f"expected a tool_call output part, saw {output_part_types}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py new file mode 100644 index 000000000..5e8025c1b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py @@ -0,0 +1,163 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for non-text message parts: an image ``uri`` on a chat +input, and a ``reasoning`` part on a chat output.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +from ._helpers import attr, chat_spans, part_fields + +_IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_image_url.yaml"): + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What breed is this dog?", + }, + { + "type": "image_url", + "image_url": {"url": _IMAGE_URL}, + }, + ], + ) + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + input_parts = { + fields + for span in chat_spans(report) + for fields in part_fields(attr(span, "gen_ai.input.messages")) + } + assert ("uri", "image") in input_parts, ( + f"expected an image uri input part, saw {input_parts}" + ) + + +class ReasoningScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = ( + # LiteLLM routes to the provider host internally and a LiteLLMModel + # built without an explicit api_base exposes no endpoint URL, so there + # is nothing to derive server.address from on the chat span. + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + env = {"LITELLM_LOCAL_MODEL_COST_MAP": "True"} + with ( + mock.patch.dict(os.environ, env), + mock.patch("tiktoken.get_encoding") as get_encoding, + ): + get_encoding.return_value = mock.MagicMock( + encode=lambda *_: [1, 2, 3] + ) + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("litellm_reasoning.yaml"): + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? " + "Answer in one word with no " + "punctuation." + ), + } + ], + } + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_parts = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "reasoning" in output_parts, ( + f"expected a reasoning output part, saw {output_parts}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index da97f2987..baebcbf9c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -5,22 +5,103 @@ from __future__ import annotations +import os +from unittest.mock import MagicMock, patch + import pytest from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) from opentelemetry.test_util_genai.instrumentor import instrument +from opentelemetry.test_util_genai.vcr import ( + scrub_response_headers_overwrite, +) + +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("x-api-key", "test_anthropic_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers_overwrite( + { + "openai-organization": "test_openai_org_id", + "openai-project": "test_openai_project_id", + "Set-Cookie": "test_set_cookie", + } + ), + } + + +@pytest.fixture +def litellm_local_cost_map(): + """Use LiteLLM's bundled model-cost map so it doesn't fetch prices over the + network during cassette playback.""" + previous = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + try: + yield + finally: + if previous is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous + + +@pytest.fixture +def patch_tiktoken_encoding(): + """Patch ``tiktoken.get_encoding`` so LiteLLM doesn't download an encoding.""" + with patch("tiktoken.get_encoding") as mock_get_encoding: + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + mock_get_encoding.return_value = mock_encoding + yield + + +@pytest.fixture +def instrument_no_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="NO_CONTENT", + ) as instrumentor: + yield instrumentor -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] + +@pytest.fixture +def instrument_with_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ) as instrumentor: + yield instrumentor @pytest.fixture -def instrument_smolagents(tracer_provider, logger_provider, meter_provider): +def instrument_event_only(tracer_provider, logger_provider, meter_provider): with instrument( SmolagentsInstrumentor(), tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, + content_capture="EVENT_ONLY", + emit_event=True, ) as instrumentor: yield instrumentor diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index d833435b7..649ffc343 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -26,7 +26,10 @@ # This variant of the requirements aims to test the system using the newest # supported version of external dependencies. -smolagents +# openai/litellm are the model backends exercised by the VCR model tests; they +# are test-only (not declared in pyproject.toml) so they live here. +smolagents[openai,litellm] +wrapt>=2.2.2 -e util/opentelemetry-util-genai -e instrumentation/opentelemetry-instrumentation-genai-smolagents diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index 1d9b353f3..afb682e96 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -21,5 +21,15 @@ # pyproject.toml is the single source of truth. The OpenTelemetry SDK and test utilities # come transitively from opentelemetry-test-util-genai. # -# There is nothing to pin yet: the lifecycle tests need no model backend. The pins for -# the backends the model tests drive arrive with those tests. +# openai and litellm are the model backends the VCR model tests instantiate. They are +# test-only (not declared in pyproject.toml) and smolagents does not pull them in through +# the instruments extra, so pin them here to the versions smolagents 1.24.0 declares in its +# own openai/litellm extras. litellm resolves to its floor; openai resolves to 1.61.0 +# rather than 1.58.1 because litellm 1.60.2 itself requires openai>=1.61.0. +openai>=1.58.1 +litellm>=1.60.2 + +# The declared opentelemetry-util-genai floor carries the streaming metrics and the +# gen_ai.request.stream attribute, which are not in a published release yet. +# Drop this once opentelemetry-util-genai 1.1b0 is published. +-e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py new file mode 100644 index 000000000..41c60d2db --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -0,0 +1,46 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for smolagents.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check or OTLP exporters aren't installed +# (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") +pytest.importorskip("opentelemetry.exporter.otlp.proto.grpc") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.inference import ( # noqa: E402 + ChatScenario, + ToolCallingScenario, +) +from .conformance.multimodal import ( # noqa: E402 + MultimodalScenario, + ReasoningScenario, +) + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(ChatScenario()), + pytest.param(ToolCallingScenario()), + pytest.param(MultimodalScenario()), + pytest.param(ReasoningScenario()), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index 791f03237..ed759e5e8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -1,19 +1,40 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Entry-point and instrument/uninstrument lifecycle tests. - -The instrumentor patches nothing yet, so these cover the parts of its contract -that hold before any patching lands: the entry point resolves, the declared -dependency is reported, and repeated instrument/uninstrument cycles stay quiet. -""" +"""Lifecycle, entry-point, completion-hook, and restoration tests.""" from __future__ import annotations +from typing import Any +from unittest.mock import patch + +import pytest +import smolagents +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, + _model_classes_defining, ) +from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.util._importlib_metadata import entry_points +from opentelemetry.util.genai.completion_hook import CompletionHook + +from .test_utils import openai_model, stub_openai_client + + +class RecordingHook(CompletionHook): + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def on_completion(self, **kwargs: Any) -> None: + self.calls.append(kwargs) + + +def _generate_a_chat_span() -> None: + model = openai_model() + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) def test_entrypoint_loads_instrumentor() -> None: @@ -29,45 +50,42 @@ def test_instrumentation_dependencies() -> None: assert "smolagents >= 1.24.0" in dependencies -def test_instrument_uninstrument_cycle( +def test_instrument_uninstrument_restores_originals( tracer_provider, logger_provider, meter_provider ) -> None: + original_generate = smolagents.OpenAIModel.generate + original_base_generate = smolagents.Model.generate + original_generate_stream = smolagents.OpenAIModel.generate_stream + instrumentor = SmolagentsInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is not original_generate + assert smolagents.Model.generate is not original_base_generate + assert ( + smolagents.OpenAIModel.generate_stream is not original_generate_stream + ) + instrumentor.uninstrument() -def test_repeated_instrument_uninstrument( - tracer_provider, logger_provider, meter_provider -) -> None: - # BaseInstrumentor returns a per-class singleton, so the lifecycle has to - # survive being driven more than once. - instrumentor = SmolagentsInstrumentor() - for _ in range(2): - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.Model.generate is original_base_generate + assert smolagents.OpenAIModel.generate_stream is original_generate_stream def test_uninstrument_through_a_new_constructor_call( tracer_provider, logger_provider, meter_provider ) -> None: - # BaseInstrumentor.__new__ returns a per-class singleton but Python still - # runs __init__ on every construction, so the documented + # BaseInstrumentor is a per-class singleton, so the documented # SmolagentsInstrumentor().instrument() / SmolagentsInstrumentor() - # .uninstrument() form has to work on the live instance. + # .uninstrument() form must restore everything even though the second + # constructor call re-runs __init__ on the live instance. + original_generate = smolagents.OpenAIModel.generate + SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, @@ -75,23 +93,201 @@ def test_uninstrument_through_a_new_constructor_call( ) SmolagentsInstrumentor().uninstrument() - assert not SmolagentsInstrumentor().is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is original_generate + + +@pytest.mark.parametrize("method", ["generate", "generate_stream"]) +def test_model_classes_defining(method: str) -> None: + classes = _model_classes_defining(smolagents, method) + + # Each class object appears once, whatever names it is exported under. + assert len(classes) == len(set(classes)) + # Every entry owns the method, so no class is wrapped for one it only + # inherits. + for model_cls in classes: + assert method in model_cls.__dict__ + + # The API-backed model classes define both; the classes that inherit them + # are reached through the base they inherit from. + assert { + smolagents.OpenAIModel, + smolagents.LiteLLMModel, + smolagents.InferenceClientModel, + } <= set(classes) + assert smolagents.AzureOpenAIModel not in classes + assert smolagents.LiteLLMRouterModel not in classes + + +def test_only_generate_covers_the_base_class_and_bedrock() -> None: + # The base Model and AmazonBedrockModel define generate but no + # generate_stream, so streaming is patched on fewer classes. + generate = set(_model_classes_defining(smolagents, "generate")) + generate_stream = set( + _model_classes_defining(smolagents, "generate_stream") + ) + + assert {smolagents.Model, smolagents.AmazonBedrockModel} <= generate + assert generate_stream.isdisjoint( + {smolagents.Model, smolagents.AmazonBedrockModel} + ) + + +def test_repeated_instrument_uninstrument( + tracer_provider, logger_provider, meter_provider +) -> None: + # BaseInstrumentor returns a per-class singleton, so the wrapped-class + # bookkeeping has to survive being filled and drained more than once. + original_generate = smolagents.OpenAIModel.generate + + instrumentor = SmolagentsInstrumentor() + for _ in range(2): + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert smolagents.OpenAIModel.generate is not original_generate + instrumentor.uninstrument() + assert smolagents.OpenAIModel.generate is original_generate def test_uninstrument_without_instrument() -> None: # BaseInstrumentor.uninstrument() short-circuits, but _uninstrument() must - # also be a no-op on its own: the rollback path in _instrument() calls it - # after a partial patch. + # also be a no-op on unpatched attributes: the rollback in _instrument() + # calls it after a partial patch. + original_generate = smolagents.OpenAIModel.generate + SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() + assert smolagents.OpenAIModel.generate is original_generate + def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. + original_generate = smolagents.OpenAIModel.generate + instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: - assert instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is not original_generate finally: instrumentor.uninstrument() + + assert smolagents.OpenAIModel.generate is original_generate + + +def test_failed_instrument_rolls_back_partial_patches( + tracer_provider, logger_provider, meter_provider +) -> None: + # A failure part-way through must leave no class patched, because + # uninstrument() cannot clean up after a failed _instrument(). + model_classes = _model_classes_defining(smolagents, "generate") + assert len(model_classes) > 1, ( + "the rollback needs more than one class to patch" + ) + originals = { + model_cls: model_cls.__dict__["generate"] + for model_cls in model_classes + } + stream_classes = _model_classes_defining(smolagents, "generate_stream") + stream_originals = { + model_cls: model_cls.__dict__["generate_stream"] + for model_cls in stream_classes + } + + real_wrap = wrap_function_wrapper + calls = 0 + + def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("boom") + real_wrap(target, name, wrapper) + + with patch( + "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", + fail_on_the_second_class, + ): + with pytest.raises(RuntimeError, match="boom"): + SmolagentsInstrumentor().instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + assert calls == 2, "the first class was expected to be patched" + for model_cls, original in originals.items(): + assert model_cls.__dict__["generate"] is original + for model_cls, original in stream_originals.items(): + assert model_cls.__dict__["generate_stream"] is original + + +def test_inherited_generate_wrapped_only_on_defining_classes( + tracer_provider, logger_provider, meter_provider +) -> None: + # AzureOpenAIModel and LiteLLMRouterModel inherit generate; they must not be + # wrapped separately or they would emit duplicate chat spans. + assert "generate" not in smolagents.AzureOpenAIModel.__dict__ + assert "generate" not in smolagents.LiteLLMRouterModel.__dict__ + + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + # wrapt returns a fresh bound wrapper per attribute access, so the + # wrapper objects differ; the underlying wrapped function is shared, + # proving AzureOpenAIModel inherits the single wrapped generate. + assert ( + smolagents.AzureOpenAIModel.generate.__wrapped__ + is smolagents.OpenAIModel.generate.__wrapped__ + ) + finally: + instrumentor.uninstrument() + + +def test_explicit_completion_hook_takes_precedence( + tracer_provider, logger_provider, meter_provider, span_exporter +) -> None: + explicit_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook" + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + completion_hook=explicit_hook, + ): + _generate_a_chat_span() + + load_hook.assert_not_called() + assert explicit_hook.calls, "explicit completion hook was not invoked" + + +def test_env_completion_hook_used_when_no_explicit_hook( + tracer_provider, logger_provider, meter_provider +) -> None: + env_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook", + return_value=env_hook, + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + _generate_a_chat_span() + + load_hook.assert_called_once() + assert env_hook.calls, "env-resolved completion hook was not invoked" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py new file mode 100644 index 000000000..ca72d4e86 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -0,0 +1,1296 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Model (``chat``) instrumentation tests: VCR-backed runs of the real +smolagents model classes, provider/endpoint resolution, request-parameter +mapping, and message conversion. +""" + +from __future__ import annotations + +import inspect +import json +import sys +from collections.abc import Generator +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, + MessageRole, +) + +from opentelemetry.instrumentation.genai.smolagents._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from opentelemetry.instrumentation.genai.smolagents.provider import ( + resolve_provider, + resolve_server_address_port, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import ( + error_attributes, + server_attributes, +) +from opentelemetry.trace import StatusCode +from opentelemetry.util.genai.types import ( + Blob, + Reasoning, + Text, + ToolCallRequest, + Uri, +) + +from .test_utils import ( + GetWeatherTool, + attr, + data_point_attributes, + metrics_by_name, + openai_model, + parse_messages, + part_types, + spans_by_operation, + stub_openai_client, + stub_streaming_openai_client, + text_chunk, + tool_call_chunk, + usage_chunk, +) + +IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +def test_openai_model_basic( + instrument_with_content, span_exporter, metric_reader, vcr +) -> None: + model = openai_model() + text = "Who won the World Cup in 2018? Answer in one word with no punctuation." + with vcr.use_cassette("openai_model_basic.yaml"): + output = model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": text}]} + ] + ) + assert output.content == "France" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.name == "chat gpt-4o" + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 25 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + assert ( + attr(span, GenAI.GEN_AI_RESPONSE_ID) + == "chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb" + ) + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) == "gpt-4o-2024-08-06" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attr(span, server_attributes.SERVER_ADDRESS) == "api.openai.com" + assert attr(span, server_attributes.SERVER_PORT) is None + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["role"] == "user" + assert inputs[0]["parts"][0] == {"type": "text", "content": text} + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"][0] == {"type": "text", "content": "France"} + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert duration.unit == "s" + assert data_point_attributes(duration) == [ + { + GenAI.GEN_AI_OPERATION_NAME: "chat", + GenAI.GEN_AI_PROVIDER_NAME: "openai", + GenAI.GEN_AI_REQUEST_MODEL: "gpt-4o", + GenAI.GEN_AI_RESPONSE_MODEL: "gpt-4o-2024-08-06", + server_attributes.SERVER_ADDRESS: "api.openai.com", + } + ] + token_usage = metrics[gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE] + assert { + point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point.sum + for point in token_usage.data.data_points + } == {"input": 25, "output": 2} + + +def test_openai_model_no_content( + instrument_no_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_basic.yaml"): + model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hi"}]} + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert isinstance(attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS), int) + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_openai_model_image_url( + instrument_with_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_image_url.yaml"): + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + {"type": "text", "text": "What breed is this dog?"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + ], + ) + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + parts = inputs[0]["parts"] + assert parts[0] == {"type": "text", "content": "What breed is this dog?"} + assert parts[1]["type"] == "uri" + assert parts[1]["modality"] == "image" + assert parts[1]["uri"] == IMAGE_URL + + +def test_openai_model_with_tools( + instrument_with_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_tool.yaml"): + output = model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + assert output.tool_calls[0].function.name == "get_weather" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + }, + } + ] + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + tool_call_parts = [ + part for part in outputs[0]["parts"] if part["type"] == "tool_call" + ] + assert tool_call_parts[0]["name"] == "get_weather" + # smolagents hands the provider's raw argument payload through unparsed. + assert tool_call_parts[0]["arguments"] == '{"location":"Paris"}' + assert tool_call_parts[0]["id"] == "call_EUuviydGIG5Jau3DLw4v4cue" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def _litellm_supports_reasoning() -> bool: + """litellm surfaces Anthropic ``reasoning_content`` only from ~1.63 onward. + + The oldest supported litellm (smolagents' 1.60.2 floor) doesn't parse the + Anthropic thinking blocks into ``reasoning_content``, so the reasoning part + can't be mapped there. Gate the reasoning-specific assertion on the version. + """ + from importlib.metadata import version # noqa: PLC0415 + + parts = version("litellm").split(".") + try: + return (int(parts[0]), int(parts[1])) >= (1, 63) + except (IndexError, ValueError): + return True + + +def test_litellm_reasoning( + instrument_with_content, + span_exporter, + litellm_local_cost_map, + patch_tiktoken_encoding, + vcr, +) -> None: + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + with vcr.use_cassette("litellm_reasoning.yaml"): + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer in one " + "word with no punctuation." + ), + } + ], + } + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "anthropic" + assert ( + attr(span, GenAI.GEN_AI_REQUEST_MODEL) + == "anthropic/claude-3-7-sonnet-20250219" + ) + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + if _litellm_supports_reasoning(): + assert "reasoning" in part_types(outputs) + + +def test_model_generate_reraises_and_records_error( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model") + with pytest.raises(NotImplementedError): + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "NotImplementedError" + + +def test_inherited_azure_generate_emits_one_chat_span( + instrument_with_content, span_exporter +) -> None: + # AzureOpenAIModel inherits generate from OpenAIModel, so only the defining + # class is patched. Exercise the inherited method end to end to prove the + # single patch still produces exactly one span with the Azure provider. + from smolagents import AzureOpenAIModel # noqa: PLC0415 + + model = AzureOpenAIModel( + model_id="gpt-4o-deployment", + azure_endpoint="https://example-resource.openai.azure.com", + api_key="test_azure_api_key", + api_version="2024-10-21", + ) + model.client = stub_openai_client("Bonjour") + + output = model.generate(messages=[{"role": "user", "content": "Hi"}]) + assert output.content == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "azure.ai.openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o-deployment" + assert ( + attr(span, server_attributes.SERVER_ADDRESS) + == "example-resource.openai.azure.com" + ) + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 1 + + +def _fake_model(class_name: str, **attrs: Any) -> Any: + instance = type(class_name, (), {})() + for key, value in attrs.items(): + setattr(instance, key, value) + return instance + + +@pytest.mark.parametrize( + "class_name, attrs, expected", + [ + ("OpenAIModel", {}, "openai"), + ("AzureOpenAIModel", {}, "azure.ai.openai"), + ("AmazonBedrockModel", {}, "aws.bedrock"), + # No GenAI registry value exists for these runtimes, so the product + # name is used rather than the class name. + ("InferenceClientModel", {}, "huggingface"), + ("TransformersModel", {}, "huggingface"), + ("VLLMModel", {}, "vllm"), + ("MLXModel", {}, "mlx"), + # LiteLLM vendor prefixes: remapped where the slug differs from the + # semconv value, passed through otherwise. + ("LiteLLMModel", {"model_id": "anthropic/claude-3"}, "anthropic"), + ("LiteLLMModel", {"model_id": "mistral/large"}, "mistral_ai"), + ("LiteLLMModel", {"model_id": "xai/grok"}, "x_ai"), + ("LiteLLMModel", {"model_id": "gemini/gemini-2.0"}, "gcp.gemini"), + ( + "LiteLLMModel", + {"model_id": "vertex_ai/gemini-2.0"}, + "gcp.vertex_ai", + ), + ("LiteLLMModel", {"model_id": "watsonx/granite"}, "ibm.watsonx.ai"), + ( + "LiteLLMModel", + {"model_id": "azure_ai/phi-4"}, + "azure.ai.inference", + ), + ("LiteLLMModel", {"model_id": "ollama/llama3"}, "ollama"), + # LiteLLMRouterModel takes a model-group name, not a provider/model + # slug, so there is nothing to resolve. + ("LiteLLMRouterModel", {"model_id": "model-group-1"}, "unknown"), + # gen_ai.provider.name is also a metric attribute. An unmapped model + # must not fall back to the deployment-specific host or a class name. + ( + "CustomModel", + {"api_base": "https://llm.example.com/v1"}, + "unknown", + ), + ("CustomModel", {}, "unknown"), + ], +) +def test_resolve_provider( + class_name: str, attrs: dict[str, Any], expected: str +) -> None: + assert resolve_provider(_fake_model(class_name, **attrs)) == expected + + +@pytest.mark.parametrize( + "model_class, expected", + [ + ("OpenAIModel", "openai"), + ("AzureOpenAIModel", "azure.ai.openai"), + ("AmazonBedrockModel", "aws.bedrock"), + ("InferenceClientModel", "huggingface"), + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), + ("LiteLLMModel", "unknown"), + ("LiteLLMRouterModel", "unknown"), + ], +) +def test_resolve_provider_covers_every_real_model_class( + model_class: str, expected: str +) -> None: + # The mapping is keyed by class name, so pin it against the real classes + # rather than only against synthetic stand-ins. + import smolagents # noqa: PLC0415 + + instance = object.__new__(getattr(smolagents, model_class)) + assert resolve_provider(instance) == expected + + +@pytest.mark.parametrize( + "attrs, expected", + [ + ({"api_base": "https://api.openai.com/v1"}, ("api.openai.com", None)), + # The default HTTPS port is omitted per the semconv server.port guidance. + ( + {"api_base": "https://api.openai.com:443/v1"}, + ("api.openai.com", None), + ), + ({"api_base": "http://localhost:11434/v1"}, ("localhost", 11434)), + ( + { + "client_kwargs": { + "azure_endpoint": "https://x.openai.azure.com" + } + }, + ("x.openai.azure.com", None), + ), + ({}, (None, None)), + ], +) +def test_resolve_server_address_port( + attrs: dict[str, Any], expected: tuple[str | None, int | None] +) -> None: + assert resolve_server_address_port(_fake_model("M", **attrs)) == expected + + +def test_server_address_falls_back_to_the_sdk_client() -> None: + # The common configuration: no api_base, so the URL is only known to the + # client the model built for itself. + model = OpenAIModel(model_id="gpt-4o", api_key="test_openai_api_key") + assert resolve_server_address_port(model) == ("api.openai.com", None) + + +def test_request_parameters_recorded( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model( + model_id="broken-model", + temperature=0.5, + top_p=0.9, + top_k=40, + frequency_penalty=0.25, + presence_penalty=1, + max_tokens=256, + seed=7, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.5 + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.9 + # top_k is a float attribute in the spec even though callers pass an int. + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_K) == 40.0 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_TOP_K), float) + assert attr(span, GenAI.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.25 + assert attr(span, GenAI.GEN_AI_REQUEST_PRESENCE_PENALTY) == 1.0 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == 256 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS), int) + assert attr(span, GenAI.GEN_AI_REQUEST_SEED) == 7 + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == ("",) + + +def test_model_kwargs_win_over_call_kwargs( + instrument_with_content, span_exporter +) -> None: + # _prepare_completion_kwargs applies the call kwargs first and the model + # kwargs on top, and drops any key whose model-level value is the + # REMOVE_PARAMETER sentinel. + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model = Model( + model_id="broken-model", + temperature=0.1, + max_tokens=REMOVE_PARAMETER, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + temperature=0.9, + max_tokens=512, + top_p=0.5, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) is None + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.5 + + +@pytest.mark.parametrize( + "model_id, model_kwargs, expected", + [ + # gpt-5 doesn't accept `stop`, so smolagents truncates the generated + # text locally instead of sending the sequences. + ("gpt-5", {}, None), + ("gpt-4o", {}, ("",)), + # An explicit `stop` overrides the stop_sequences argument. + ("gpt-4o", {"stop": ["STOP"]}, ("STOP",)), + # The model-level sentinel pops the `stop` that + # _prepare_completion_kwargs seeded from stop_sequences, leaving the + # request with none. + ("gpt-4o", {"stop": "REMOVE"}, None), + ], +) +def test_stop_sequences_follow_what_is_sent( + instrument_with_content, + span_exporter, + model_id: str, + model_kwargs: dict[str, Any], + expected: tuple[str, ...] | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id=model_id, **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == expected + + +def _bedrock_model(response: dict[str, Any]) -> Any: + from smolagents import AmazonBedrockModel # noqa: PLC0415 + + # A caller-supplied client keeps boto3 out of the test. + return AmazonBedrockModel( + model_id="us.amazon.nova-pro-v1:0", + client=SimpleNamespace(converse=lambda **_: response), + ) + + +BEDROCK_RESPONSE: dict[str, Any] = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "done"}], + "tool_calls": None, + } + }, + "usage": {"inputTokens": 3, "outputTokens": 2}, + "stopReason": "end_turn", +} + + +def test_bedrock_stop_sequences_are_not_recorded( + instrument_with_content, span_exporter +) -> None: + # supports_stop_parameter says yes, but the prepared request carries no + # stop sequences, so the span must not claim any either. + model = _bedrock_model(BEDROCK_RESPONSE) + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + request = model._prepare_completion_kwargs( # noqa: SLF001 + messages=messages, stop_sequences=[""] + ) + assert model.supports_stop_parameter is True + assert "stop" not in request + + model.generate(messages=messages, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) is None + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "aws.bedrock" + # Bedrock's "end_turn" is normalized to the semconv "stop". + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + + +@pytest.mark.parametrize( + "model_kwargs, call_kwargs, expected", + [ + ({"max_tokens": 256}, {}, 256), + ({}, {"max_tokens": 256}, 256), + # max_new_tokens is the TransformersModel spelling of the same limit, + # and max_tokens wins when a caller sets both. + ({"max_new_tokens": 4096}, {}, 4096), + ({"max_new_tokens": 4096, "max_tokens": 256}, {}, 256), + ({}, {}, None), + ], +) +def test_max_tokens_covers_both_spellings( + instrument_with_content, + span_exporter, + model_kwargs: dict[str, Any], + call_kwargs: dict[str, Any], + expected: int | None, +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], **call_kwargs + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == expected + + +@pytest.mark.parametrize( + "response_format, model_kwargs, expected", + [ + ({"type": "json_object"}, {}, "json"), + ({"type": "json_schema", "json_schema": {}}, {}, "json"), + ({"type": "text"}, {}, "text"), + (None, {}, None), + # smolagents forwards response_format unchanged, so its type is + # whatever the provider accepts. An unknown one is dropped rather than + # recorded on an enum attribute. + ({"type": "xml"}, {}, None), + ({}, {}, None), + # The model-level kwargs win over the argument, and the sentinel drops + # the key from the request, the same as for every other parameter. + ( + {"type": "json_object"}, + {"response_format": {"type": "text"}}, + "text", + ), + ({"type": "json_object"}, {"response_format": "REMOVE"}, None), + (None, {"response_format": {"type": "json_object"}}, "json"), + ], +) +def test_output_type_follows_the_response_format( + instrument_with_content, + span_exporter, + response_format: dict[str, Any] | None, + model_kwargs: dict[str, Any], + expected: str | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + response_format=response_format, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == expected + + +def test_user_subclass_of_a_patched_model_keeps_its_provider( + instrument_with_content, span_exporter, metric_reader +) -> None: + # A subclass inherits the patched generate, so it is instrumented; resolving + # the provider by exact class name would report "unknown" on both the span + # and the metrics. + class TenantOpenAIModel(OpenAIModel): + pass + + model = TenantOpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="http://localhost:11434/v1", + ) + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + # A non-default port is part of the endpoint, unlike the HTTPS default. + assert attr(span, server_attributes.SERVER_ADDRESS) == "localhost" + assert attr(span, server_attributes.SERVER_PORT) == 11434 + duration = metrics_by_name(metric_reader)[ + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( + "openai" + ) + + +def test_provider_error_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_openai_client( + "", error=ConnectionError("connection reset") + ) + + with pytest.raises(ConnectionError, match="connection reset"): + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def _drain_stream(model: OpenAIModel, **kwargs: Any) -> list[Any]: + return list( + model.generate_stream( + messages=[{"role": "user", "content": "Hi"}], **kwargs + ) + ) + + +def test_generate_stream_is_lazy_and_records_the_drained_response( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + # A streamed response isn't finished until the caller drains it. + assert spans_by_operation(span_exporter.get_finished_spans(), "chat") == [] + + deltas = list(stream) + assert "".join(delta.content or "" for delta in deltas) == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_REQUEST_STREAM) is True + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bonjour"}] + # Deltas carry no finish reason, and "stop" would hide a truncation. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_generate_stream_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Instrumentation observes; it must not change what generate_stream returns. + model = openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + list(stream) + + +def test_generate_stream_accumulates_tool_calls( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [ + tool_call_chunk(0, call_id="call_1", name="get_weather"), + tool_call_chunk(0, arguments='{"location":'), + tool_call_chunk(0, arguments='"Paris"}'), + ] + ) + + _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + (part,) = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0]["parts"] + assert part == { + "type": "tool_call", + "id": "call_1", + "name": "get_weather", + "arguments": '{"location":"Paris"}', + } + # Tool calls are the only evidence of why a streamed generation stopped. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon")], error=ConnectionError("stream died") + ) + + with pytest.raises(ConnectionError, match="stream died"): + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + # What was streamed before the failure is still recorded. + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bon"}] + + +def test_generate_stream_close_before_drain_finalizes_once( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + stream.close() + stream.close() # idempotent + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_generate_stream_records_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK in metrics + ) + + +def test_generate_stream_no_content( + instrument_no_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bonjour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + + +def test_event_only_content_capture( + instrument_event_only, span_exporter, log_exporter +) -> None: + model = openai_model() + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + # Metadata still goes on the span. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.event_name == "gen_ai.client.inference.operation.details" + # Event attributes carry the messages as structured values, not JSON text. + attributes = record.attributes or {} + inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] + outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] + assert inputs[0]["parts"][0]["content"] == "Hi" + assert outputs[0]["parts"][0]["content"] == "Bonjour" + assert outputs[0]["finish_reason"] == "stop" + + +def test_to_tool_definitions_uses_json_schema() -> None: + (definition,) = to_tool_definitions([GetWeatherTool()]) + assert definition.name == "get_weather" + assert definition.description == "Get the weather for a given city" + # smolagents' raw ``inputs`` map is not a JSON Schema on its own. + assert definition.parameters == { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + } + + +def test_to_input_messages_maps_smolagents_only_roles() -> None: + # smolagents converts these roles itself inside generate(), after the + # wrapper has already read the messages. + messages = to_input_messages( + [ + {"role": MessageRole.TOOL_CALL, "content": "call"}, + {"role": MessageRole.TOOL_RESPONSE, "content": "response"}, + {"role": MessageRole.SYSTEM, "content": "sys"}, + ] + ) + assert [message.role for message in messages] == [ + "assistant", + "user", + "system", + ] + + +def test_to_input_messages_dict_and_chatmessage() -> None: + messages = to_input_messages( + [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ChatMessage(role=MessageRole.ASSISTANT, content="Hi there"), + ] + ) + assert messages[0].role == "user" + assert messages[0].parts == [Text(content="Hello")] + assert messages[1].role == "assistant" + assert messages[1].parts == [Text(content="Hi there")] + + +def test_to_input_messages_image_and_base64() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + } + ] + ) + parts = messages[0].parts + assert parts[0] == Uri(mime_type=None, modality="image", uri=IMAGE_URL) + assert isinstance(parts[1], Blob) + assert parts[1].modality == "image" + assert parts[1].mime_type == "image/png" + assert isinstance(parts[1].content, bytes) + + +def test_to_input_messages_data_url_keeps_media_type() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "data:image/jpeg;base64,aVZCT1J3MEtHZ28=", + } + ], + } + ] + ) + (part,) = messages[0].parts + assert isinstance(part, Blob) + assert part.mime_type == "image/jpeg" + + +@pytest.mark.parametrize( + "image", + [ + "not base64!!!!", + # A path is not base64, but a non-validating decode silently turns this + # one into 15 bytes of garbage after dropping the invalid characters. + "/tmp/photos/my-cat.png", + ], +) +def test_to_input_messages_drops_malformed_base64_image(image: str) -> None: + messages = to_input_messages( + [{"role": "user", "content": [{"type": "image", "image": image}]}] + ) + assert messages[0].parts == [] + + +def test_to_output_message_text_reasoning_and_tool_calls() -> None: + class _Msg: + role = MessageRole.ASSISTANT + content = "The answer" + tool_calls = [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments='{"location": "Paris"}' + ), + ) + ] + + class raw: # noqa: N801 + class _Choice: + class message: # noqa: N801 + reasoning_content = "thinking about it" + + choices = [_Choice()] + + output = to_output_message(_Msg()) + assert output.role == "assistant" + assert Text(content="The answer") in output.parts + assert Reasoning(content="thinking about it") in output.parts + tool_calls = [p for p in output.parts if isinstance(p, ToolCallRequest)] + assert tool_calls[0].name == "get_weather" + assert tool_calls[0].id == "call_1" + assert output.finish_reason == "tool_calls" + + +def test_to_output_message_from_a_dict_raw_response() -> None: + # AmazonBedrockModel, TransformersModel, VLLMModel and MLXModel put a plain + # dict on ChatMessage.raw rather than an SDK object. + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + raw={"stopReason": "max_tokens"}, + ) + output = to_output_message(message) + assert output.parts == [Text(content="done")] + # "max_tokens" is Bedrock's stopReason spelling for a length cutoff. + assert output.finish_reason == "length" + assert response_id(message) is None + assert response_model_name(message) is None + + +def test_to_output_message_unwraps_the_role_enum() -> None: + output = to_output_message( + ChatMessage(role=MessageRole.ASSISTANT, content="done") + ) + assert output.role == "assistant" + + +def test_to_output_message_maps_image_content() -> None: + # A response whose content is a list carries the same element shapes as a + # request, so an image in it maps the same way. + output = to_output_message( + ChatMessage( + role=MessageRole.ASSISTANT, + content=[ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + ) + ) + assert output.parts[0] == Text(content="Here it is") + assert output.parts[1] == Uri( + mime_type=None, modality="image", uri=IMAGE_URL + ) + blob = output.parts[2] + assert isinstance(blob, Blob) + assert blob.mime_type == "image/png" + assert blob.modality == "image" + + +LOCAL_RUNTIME_RAW = { + "out": "done", + "completion_kwargs": {"max_new_tokens": 4096}, +} + + +@pytest.mark.parametrize( + "raw, tool_calls, expected", + [ + # API-backed models: the provider's own value, whatever it is. + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="stop")]), + [], + "stop", + ), + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="length")]), + [], + "length", + ), + # Bedrock's stopReason values map onto the semconv vocabulary. + ({"stopReason": "max_tokens"}, [], "length"), + ({"stopReason": "tool_use"}, [], "tool_calls"), + # An unmapped value passes through rather than being guessed at. + ({"stopReason": "guardrail_intervened"}, [], "guardrail_intervened"), + # The local runtimes report no reason, so none is recorded and + # util-genai omits the empty value. + (LOCAL_RUNTIME_RAW, [], ""), + (None, [], ""), + # Tool calls in the response give the reason without guessing. + ( + LOCAL_RUNTIME_RAW, + [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments="{}" + ), + ) + ], + "tool_calls", + ), + ], +) +def test_finish_reason_follows_the_provider_response( + raw: Any, tool_calls: list[ChatMessageToolCall], expected: str +) -> None: + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + tool_calls=tool_calls or None, + raw=raw, + ) + assert to_output_message(message).finish_reason == expected + + +def test_model_reporting_no_finish_reason_omits_the_attribute( + instrument_with_content, span_exporter +) -> None: + # InferenceClientModel is a patched class whose provider response can come + # back without a finish reason; the span must then carry none. + from smolagents import InferenceClientModel # noqa: PLC0415 + + model = InferenceClientModel( + model_id="Qwen/Qwen2.5-Coder-32B-Instruct", token="hf_test" + ) + model.client = SimpleNamespace( + chat_completion=lambda **_: SimpleNamespace( + id="hf-1", + model="Qwen/Qwen2.5-Coder-32B-Instruct", + choices=[ + SimpleNamespace( + message=SimpleNamespace( + role="assistant", content="ok", tool_calls=None + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2), + ) + ) + + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["finish_reason"] == "" + + +# The local runtimes flatten a message's content as text, so the content has to +# be a list of parts rather than a bare string. +LOCAL_RUNTIME_MESSAGES: list[dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": "Where is the Louvre?"}], + } +] + + +def _mlx_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. + + ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` + over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from + ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real + ``generate``, and the runtime doesn't have to be installed. ``monkeypatch`` + is unused here; it keeps the factory signature uniform with the vllm one, + which does have modules to stub. + """ + from smolagents.models import MLXModel # noqa: PLC0415 + + model = object.__new__(MLXModel) + model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" + model.kwargs = {} + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.model = object() + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] + ) + model.stream_generate = lambda *_, **__: iter( + [SimpleNamespace(text="In "), SimpleNamespace(text="Paris")] + ) + return model + + +def _vllm_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """A ``VLLMModel`` with ``vllm`` itself stubbed. + + ``VLLMModel.generate`` imports ``SamplingParams`` and + ``StructuredOutputsParams`` from ``vllm`` when it runs, and neither test env + installs vllm, so both modules are faked for the duration of the test. + Everything the wrapper reads still comes from the real ``generate``. + """ + from smolagents.models import VLLMModel # noqa: PLC0415 + + def fake_params(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) + + vllm = ModuleType("vllm") + sampling_params = ModuleType("vllm.sampling_params") + setattr(vllm, "SamplingParams", fake_params) + setattr(sampling_params, "StructuredOutputsParams", fake_params) + setattr(vllm, "sampling_params", sampling_params) + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + + completion = SimpleNamespace( + prompt_token_ids=[1, 2, 3, 4], + outputs=[SimpleNamespace(text="In Paris", token_ids=[5, 6])], + ) + model = object.__new__(VLLMModel) + model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" + model.kwargs = {} + model.flatten_messages_as_text = True + model._is_vlm = False + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: "prompt" + ) + model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) + return model + + +@pytest.mark.parametrize( + "model_factory, provider, request_model, input_tokens, output_tokens", + [ + ( + _mlx_model, + "mlx", + "mlx-community/Qwen2.5-0.5B-Instruct-4bit", + 3, + 2, + ), + (_vllm_model, "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), + ], + ids=["mlx", "vllm"], +) +def test_local_runtime_response_is_recorded( + instrument_with_content, + span_exporter, + monkeypatch: pytest.MonkeyPatch, + model_factory: Any, + provider: str, + request_model: str, + input_tokens: int, + output_tokens: int, +) -> None: + output = model_factory(monkeypatch).generate( + messages=LOCAL_RUNTIME_MESSAGES + ) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens + # A local runtime returns no provider response envelope and listens on no + # socket, so it reports no finish reason, id or response model, and there is + # no endpoint to derive server.address from. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + assert attr(span, server_attributes.SERVER_PORT) is None + assert span.status.status_code == StatusCode.UNSET + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["parts"] == [ + {"type": "text", "content": "Where is the Louvre?"} + ] + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + assert outputs[0]["finish_reason"] == "" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py new file mode 100644 index 000000000..eeb8d78e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -0,0 +1,222 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers, tools, and model stubs for smolagents instrumentation tests.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any + +from smolagents import OpenAIModel, Tool + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + + +class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + } + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + +def openai_model() -> OpenAIModel: + return OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + + +def stub_openai_client( + content: str, finish_reason: str = "stop", error: Exception | None = None +) -> Any: + """An object shaped like the bits of ``openai.OpenAI`` that ``generate`` uses. + + Building a real ``ChatCompletion`` keeps the response shape honest (the + wrapper reads ``raw.model``, ``raw.id``, and ``raw.choices[0].finish_reason``) + without needing a cassette for a deployment we can't record against. + """ + from openai.types.chat import ( # noqa: PLC0415 + ChatCompletion, + ChatCompletionMessage, + ) + from openai.types.chat.chat_completion import Choice # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + completion = ChatCompletion( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion", + created=0, + choices=[ + Choice( + index=0, + finish_reason=finish_reason, + message=ChatCompletionMessage( + role="assistant", content=content + ), + ) + ], + usage=CompletionUsage( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def create(**_: Any) -> ChatCompletion: + if error is not None: + raise error + return completion + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def stub_streaming_openai_client( + chunks: list[Any], error: Exception | None = None +) -> Any: + """An ``openai.OpenAI`` stand-in whose ``create`` returns a chunk stream. + + ``OpenAIModel.generate_stream`` reads ``event.usage`` and + ``event.choices[0].delta``, so the chunks are real ``ChatCompletionChunk`` + objects. ``error`` is raised after the chunks are yielded, which is how a + provider failure part-way through a stream reaches the caller. + """ + + def create(**_: Any) -> Any: + def stream() -> Any: + yield from chunks + if error is not None: + raise error + + return stream() + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def text_chunk(content: str) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[Choice(index=0, delta=ChoiceDelta(content=content))], + ) + + +def tool_call_chunk( + index: int, + call_id: str | None = None, + name: str | None = None, + arguments: str | None = None, +) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ChoiceDeltaToolCall, + ChoiceDeltaToolCallFunction, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[ + Choice( + index=0, + delta=ChoiceDelta( + tool_calls=[ + ChoiceDeltaToolCall( + index=index, + id=call_id, + type="function", + function=ChoiceDeltaToolCallFunction( + name=name, arguments=arguments + ), + ) + ] + ), + ) + ], + ) + + +def usage_chunk(prompt_tokens: int, completion_tokens: int) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[], + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +def spans_by_operation( + spans: list[ReadableSpan], operation: str +) -> list[ReadableSpan]: + return [ + span + for span in spans + if (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + == operation + ] + + +def attr(span: ReadableSpan, name: str) -> Any: + return (span.attributes or {}).get(name) + + +def parse_messages(span: ReadableSpan, name: str) -> list[dict[str, Any]]: + raw = attr(span, name) + return json.loads(raw) if isinstance(raw, str) else [] + + +def part_types(messages: list[dict[str, Any]]) -> list[str]: + return [part["type"] for message in messages for part in message["parts"]] + + +def metrics_by_name(metric_reader: Any) -> dict[str, Any]: + data = metric_reader.get_metrics_data() + if data is None: + return {} + return { + metric.name: metric + for resource_metric in data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + +def data_point_attributes(metric: Any) -> list[dict[str, Any]]: + return [dict(point.attributes) for point in metric.data.data_points] diff --git a/tox.ini b/tox.ini index ce3694aa0..b202ba514 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ envlist = ; instrumentation-genai-smolagents py3{10,11,12,13,14}-test-instrumentation-genai-smolagents-latest py310-test-instrumentation-genai-smolagents-oldest + py314-test-instrumentation-genai-smolagents-conformance lint-instrumentation-genai-smolagents ; instrumentation-genai-anthropic @@ -157,6 +158,8 @@ deps = smolagents-latest: {[testenv]test_deps} smolagents-latest: {[testenv]pytest_deps} smolagents-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt + smolagents-conformance: {[testenv]pytest_deps} + smolagents-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt anthropic-oldest: {[testenv]pytest_deps} anthropic-oldest: -e {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic[instruments] @@ -257,6 +260,7 @@ commands = lint-instrumentation-genai-agno: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-agno" test-instrumentation-genai-smolagents-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests --vcr-record=none {posargs} + test-instrumentation-genai-smolagents-conformance: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py --vcr-record=none {posargs} lint-instrumentation-genai-smolagents: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-smolagents" test-instrumentation-genai-anthropic-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests --vcr-record=none {posargs} From 57afc6da2af61ad53a7bd5f717355f96bdb7be53 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Thu, 6 Aug 2026 20:01:56 +0200 Subject: [PATCH 02/11] review fixes --- .../genai/smolagents/__init__.py | 10 ++--- .../genai/smolagents/_messages.py | 12 +++--- .../instrumentation/genai/smolagents/patch.py | 42 ++++++++++--------- .../genai/smolagents/provider.py | 16 +++---- 4 files changed, 36 insertions(+), 44 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index e43c85b5f..1d847d678 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -125,10 +125,8 @@ def _instrument(self, **kwargs: Any) -> None: or load_completion_hook(), ) - wrapped_generate_classes: list[type] = [] - self._wrapped_generate_classes = wrapped_generate_classes - wrapped_generate_stream_classes: list[type] = [] - self._wrapped_generate_stream_classes = wrapped_generate_stream_classes + self._wrapped_generate_classes = [] + self._wrapped_generate_stream_classes = [] try: for model_cls in _model_classes_defining(smolagents, "generate"): wrap_function_wrapper( @@ -136,7 +134,7 @@ def _instrument(self, **kwargs: Any) -> None: "generate", model_generate(handler), ) - wrapped_generate_classes.append(model_cls) + self._wrapped_generate_classes.append(model_cls) for model_cls in _model_classes_defining( smolagents, "generate_stream" @@ -146,7 +144,7 @@ def _instrument(self, **kwargs: Any) -> None: "generate_stream", model_generate_stream(handler), ) - wrapped_generate_stream_classes.append(model_cls) + self._wrapped_generate_stream_classes.append(model_cls) except Exception: # BaseInstrumentor.instrument() doesn't mark the instrumentor as # instrumented when _instrument raises, so uninstrument() would diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index 797021372..f09046cd3 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -24,6 +24,7 @@ Blob, FunctionToolDefinition, InputMessage, + MessagePart, OutputMessage, Reasoning, Text, @@ -134,8 +135,8 @@ def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: return None -def _parts_from_content(content: Any) -> list[Any]: - parts: list[Any] = [] +def _parts_from_content(content: Any) -> list[MessagePart]: + parts: list[MessagePart] = [] if isinstance(content, str): parts.append(Text(content=content)) return parts @@ -150,8 +151,7 @@ def _parts_from_content(content: Any) -> list[Any]: if element.get("type") == "text" and (text := element.get("text")): parts.append(Text(content=text)) continue - image_part = _image_part_from_element(element) - if image_part is not None: + if image_part := _image_part_from_element(element): parts.append(image_part) else: _logger.debug( @@ -244,9 +244,7 @@ def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: def to_output_message(output_message: Any) -> OutputMessage: """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``.""" role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" - parts: list[Any] = _parts_from_content( - getattr(output_message, "content", None) - ) + parts = _parts_from_content(getattr(output_message, "content", None)) if reasoning := _reasoning_from_raw(output_message): parts.append(Reasoning(content=reasoning)) tool_call_requests = _tool_call_requests(output_message) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 4653108aa..0ed7da1e1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -28,7 +28,12 @@ from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import InferenceInvocation from opentelemetry.util.genai.stream import SyncStreamWrapper -from opentelemetry.util.genai.types import OutputMessage, Text, ToolCallRequest +from opentelemetry.util.genai.types import ( + MessagePart, + OutputMessage, + Text, + ToolCallRequest, +) from ._messages import ( response_id, @@ -275,25 +280,22 @@ def wrapper( kwargs: dict[str, Any], ) -> Any: invocation = _start_inference(handler, wrapped, instance, args, kwargs) - - try: + with invocation: output_message = wrapped(*args, **kwargs) - except Exception as error: # pylint: disable=broad-except - invocation.fail(error) - raise - - _apply_token_usage(invocation, output_message) - invocation.response_model_name = response_model_name(output_message) - invocation.response_id = response_id(output_message) - output = to_output_message(output_message) - # to_output_message leaves finish_reason empty when the provider - # reported none, and an empty value is dropped rather than guessed at. - if output.finish_reason: - invocation.finish_reasons = [output.finish_reason] - if handler.should_capture_content(): - invocation.output_messages = [output] - invocation.stop() - return output_message + _apply_token_usage(invocation, output_message) + invocation.response_model_name = response_model_name( + output_message + ) + invocation.response_id = response_id(output_message) + output = to_output_message(output_message) + # to_output_message leaves finish_reason empty when the provider + # reported none, and an empty value is dropped rather than guessed + # at. + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if handler.should_capture_content(): + invocation.output_messages = [output] + return output_message return wrapper @@ -363,7 +365,7 @@ def _process_chunk(self, chunk: Any) -> None: self._accumulate_tool_call(delta) def _output_message(self) -> OutputMessage | None: - parts: list[Any] = [] + parts: list[MessagePart] = [] content = "".join(self._self_content) if content: parts.append(Text(content=content)) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py index f531606c1..be54d6400 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -65,16 +65,6 @@ class hierarchy so a user subclass resolves to the provider of the base class _LITELLM_CLASS_NAMES = frozenset({"LiteLLMModel", "LiteLLMRouterModel"}) -def _class_names(instance: Any) -> list[str]: - """The instance's class names, most derived first. - - Only the classes that define ``generate`` are patched, so an instrumented - model can be a user subclass of one of them. Matching the exact class name - alone would report ``unknown`` for every such subclass. - """ - return [cls.__name__ for cls in type(instance).__mro__] - - def _provider_from_litellm(instance: Any) -> str | None: model_id = getattr(instance, "model_id", None) if not isinstance(model_id, str) or "/" not in model_id: @@ -133,7 +123,11 @@ def resolve_server_address_port( def resolve_provider(instance: Any) -> str: """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" - class_names = _class_names(instance) + # The instance's class names, most derived first. Only the classes that + # define ``generate`` are patched, so an instrumented model can be a user + # subclass of one of them. Matching the exact class name alone would report + # ``unknown`` for every such subclass. + class_names = [cls.__name__ for cls in type(instance).__mro__] if not _LITELLM_CLASS_NAMES.isdisjoint(class_names): provider = _provider_from_litellm(instance) From 5ba10c25fe81d93b2bee495c70e42c715f67479c Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Thu, 6 Aug 2026 20:02:48 +0200 Subject: [PATCH 03/11] skip building output content when capture is off --- .../genai/smolagents/_messages.py | 8 +++-- .../instrumentation/genai/smolagents/patch.py | 31 ++++++++++--------- .../tests/test_models.py | 14 +++++++-- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index f09046cd3..83743ccc0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -222,7 +222,7 @@ def _tool_call_requests(output_message: Any) -> list[ToolCallRequest]: ] -def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: +def finish_reason(output_message: Any) -> str | None: """Why the provider stopped generating, or ``None`` if it didn't say. The local runtimes (``TransformersModel``, ``VLLMModel``, ``MLXModel``) put @@ -238,7 +238,9 @@ def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: stop_reason = _raw_value(raw, "stopReason") if isinstance(stop_reason, str) and stop_reason: return _STOP_REASON_MAP.get(stop_reason, stop_reason) - return "tool_calls" if has_tool_calls else None + if getattr(output_message, "tool_calls", None): + return "tool_calls" + return None def to_output_message(output_message: Any) -> OutputMessage: @@ -251,7 +253,7 @@ def to_output_message(output_message: Any) -> OutputMessage: parts.extend(tool_call_requests) # OutputMessage requires the field; util-genai drops an empty value when it # emits gen_ai.response.finish_reasons. - reason = _finish_reason(output_message, bool(tool_call_requests)) + reason = finish_reason(output_message) return OutputMessage(role=role, parts=parts, finish_reason=reason or "") diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 0ed7da1e1..26ff1312f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -36,6 +36,7 @@ ) from ._messages import ( + finish_reason, response_id, response_model_name, to_input_messages, @@ -287,14 +288,12 @@ def wrapper( output_message ) invocation.response_id = response_id(output_message) - output = to_output_message(output_message) - # to_output_message leaves finish_reason empty when the provider - # reported none, and an empty value is dropped rather than guessed - # at. - if output.finish_reason: - invocation.finish_reasons = [output.finish_reason] + if reason := finish_reason(output_message): + invocation.finish_reasons = [reason] if handler.should_capture_content(): - invocation.output_messages = [output] + invocation.output_messages = [ + to_output_message(output_message) + ] return output_message return wrapper @@ -324,7 +323,7 @@ def __init__( ) -> None: super().__init__(stream, invocation=invocation) self._self_inference = invocation - self._self_handler = handler + self._self_capture_content = handler.should_capture_content() self._self_content: list[str] = [] self._self_tool_calls: dict[int, _StreamedToolCall] = {} self._self_input_tokens = 0 @@ -340,6 +339,10 @@ def _accumulate_tool_call(self, delta: Any) -> None: tool_call = self._self_tool_calls.setdefault( index, _StreamedToolCall() ) + if not self._self_capture_content: + # The finish reason only needs a tool call to have happened; its + # name and arguments are content. + return if delta.id: tool_call.id = delta.id function = getattr(delta, "function", None) @@ -352,7 +355,7 @@ def _accumulate_tool_call(self, delta: Any) -> None: def _process_chunk(self, chunk: Any) -> None: content = getattr(chunk, "content", None) - if content: + if content and self._self_capture_content: self._self_content.append(content) token_usage = getattr(chunk, "token_usage", None) if token_usage is not None: @@ -393,11 +396,11 @@ def _finalize(self, error: BaseException | None = None) -> None: if self._self_saw_token_usage: invocation.input_tokens = self._self_input_tokens invocation.output_tokens = self._self_output_tokens - output = self._output_message() - if output is not None: - if output.finish_reason: - invocation.finish_reasons = [output.finish_reason] - if self._self_handler.should_capture_content(): + if self._self_tool_calls: + invocation.finish_reasons = ["tool_calls"] + if self._self_capture_content: + output = self._output_message() + if output is not None: invocation.output_messages = [output] if error is not None: invocation.fail(error) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py index ca72d4e86..3260e65b4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -146,6 +146,8 @@ def test_openai_model_no_content( assert isinstance(attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS), int) assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + # The finish reason is metadata, so it survives without the content. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) def test_openai_model_image_url( @@ -861,15 +863,23 @@ def test_generate_stream_no_content( ) -> None: model = openai_model() model.client = stub_streaming_openai_client( - [text_chunk("Bonjour"), usage_chunk(3, 2)] + [ + text_chunk("Bonjour"), + tool_call_chunk( + 0, call_id="call_1", name="get_weather", arguments="{}" + ), + usage_chunk(3, 2), + ] ) - _drain_stream(model) + _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + # A tool call still drives the finish reason without its name or arguments. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) def test_event_only_content_capture( From 9e595a1dbb4af8f00eb32a468a81e41735d7bfcb Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Thu, 6 Aug 2026 20:03:56 +0200 Subject: [PATCH 04/11] only instrument the in-process model classes --- .../.changelog/352.added | 2 +- .../README.rst | 59 +- .../genai/smolagents/__init__.py | 76 +- .../genai/smolagents/_messages.py | 109 +- .../instrumentation/genai/smolagents/patch.py | 145 +-- .../genai/smolagents/provider.py | 109 +- .../tests/cassettes/litellm_reasoning.yaml | 20 - .../tests/cassettes/openai_model_basic.yaml | 25 - .../cassettes/openai_model_image_url.yaml | 26 - .../tests/cassettes/openai_model_tool.yaml | 32 - .../tests/conformance/inference.py | 138 ++- .../tests/conformance/multimodal.py | 115 +- .../tests/conftest.py | 59 +- .../tests/requirements.latest.txt | 6 +- .../tests/requirements.oldest.txt | 9 +- .../tests/test_conformance.py | 18 +- .../tests/test_instrumentor.py | 141 ++- .../tests/test_models.py | 1089 ++++------------- .../tests/test_utils.py | 261 ++-- 19 files changed, 749 insertions(+), 1690 deletions(-) delete mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml delete mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml delete mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml delete mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added index 5aca70172..e389f9e83 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added @@ -1 +1 @@ -Add ``chat`` instrumentation for the smolagents model classes. +Add ``chat`` instrumentation for the in-process smolagents model classes (``TransformersModel``, ``VLLMModel``, ``MLXModel``). diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index fa97a7c54..7c28e0e24 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -7,31 +7,56 @@ OpenTelemetry smolagents Instrumentation :target: https://pypi.org/project/opentelemetry-instrumentation-genai-smolagents/ This library provides OpenTelemetry instrumentation for `smolagents -`_. It wraps the smolagents model -classes and emits a GenAI semantic-convention ``chat`` span and the matching -metrics through ``opentelemetry-util-genai``. +`_. It wraps the model classes that +run inference in your own process and emits a GenAI semantic-convention ``chat`` +span and the matching metrics through ``opentelemetry-util-genai``: + +* ``TransformersModel`` +* ``VLLMModel`` +* ``MLXModel`` + +The API-backed model classes are not instrumented here. Each one calls a client +library that carries its own instrumentation. Emitting a span at the smolagents +layer as well would produce two ``chat`` spans for one model call, and would +count the token-usage and duration metrics twice. Install the instrumentation +for the client library instead: + +.. list-table:: + :header-rows: 1 + + * - smolagents model class + - Instrument this instead + * - ``OpenAIModel``, ``AzureOpenAIModel`` + - `opentelemetry-instrumentation-genai-openai + `_ + * - ``AmazonBedrockModel`` + - `opentelemetry-instrumentation-botocore + `_ + * - ``InferenceClientModel``, ``LiteLLMModel``, ``LiteLLMRouterModel`` + - the instrumentation or built-in telemetry of the client library the model + calls (``huggingface_hub``, ``litellm``) Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not instrumented yet. A model call made inside an agent run still gets a ``chat`` span, but no agent span sits above it. -A streamed model call, whether it comes from ``stream_outputs=True`` on an agent -or from calling ``Model.generate_stream`` directly, gets a ``chat`` span that -stays open until the caller drains the deltas. The span carries -``gen_ai.request.stream``, and the call also records the +``TransformersModel`` is the only instrumented class with a ``generate_stream``. +A streamed call gets a ``chat`` span that stays open until the caller drains the +deltas. This covers both ``stream_outputs=True`` on an agent and a direct +``generate_stream`` call. The span carries ``gen_ai.request.stream``, and the +call also records the ``gen_ai.client.operation.time_to_first_chunk`` and ``gen_ai.client.operation.time_per_output_chunk`` metrics. Known gaps: -* The instrumentation patches ``generate`` and ``generate_stream`` on the model - classes that smolagents ships. A subclass that inherits either method is - instrumented. A subclass that overrides one is not: the override shadows the - patched method, so the call produces no ``chat`` span. -* A streamed ``chat`` span reports no ``gen_ai.response.id`` and no - ``gen_ai.response.model``, because a smolagents stream delta carries neither. - The span reports ``gen_ai.response.finish_reasons`` only when the model - requested tool calls, which is the one stop reason the deltas make visible. +* A subclass that inherits ``generate`` or ``generate_stream`` from one of the + three classes above is instrumented. A subclass that overrides one is not: the + override shadows the patched method, so the call produces no ``chat`` span. +* A ``chat`` span reports no ``gen_ai.response.id``, no + ``gen_ai.response.model``, no ``gen_ai.response.finish_reasons`` and no + ``server.address``. A runtime in this process returns the generated text and + the token counts, nothing more. It also listens on no socket. Installation ------------ @@ -48,11 +73,11 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) - from smolagents import InferenceClientModel + from smolagents import TransformersModel SmolagentsInstrumentor().instrument() - model = InferenceClientModel() + model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) Configuration diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index 1d847d678..de4b7b655 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -7,8 +7,12 @@ Instrumentation for `smolagents `_. -Model calls are recorded as ``chat`` spans. Agent runs and tool calls are not -instrumented yet. +Calls to the in-process model classes (``TransformersModel``, ``VLLMModel`` and +``MLXModel``) are recorded as ``chat`` spans. The API-backed model classes are +not instrumented here: each one calls a client library that carries its own +instrumentation, and emitting a span at this layer as well would duplicate the +span and count the token-usage and duration metrics twice. Agent runs and tool +calls are not instrumented yet. Usage ----- @@ -18,12 +22,14 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) - from smolagents import InferenceClientModel + from smolagents import TransformersModel SmolagentsInstrumentor().instrument() - model = InferenceClientModel() - model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") + model.generate( + [{"role": "user", "content": "How many seconds are in a week?"}] + ) Configuration ------------- @@ -45,7 +51,6 @@ from __future__ import annotations from collections.abc import Collection -from types import ModuleType from typing import Any from wrapt import wrap_function_wrapper @@ -61,33 +66,40 @@ __all__ = ["SmolagentsInstrumentor"] -def _model_classes_defining(smolagents: ModuleType, method: str) -> list[type]: - """The exported model classes whose ``method`` gets wrapped. +# The model classes that run inference in the current process. They call no +# client library, so this instrumentation is the only place their model calls +# can be observed. +# +# The API-backed classes are left out on purpose. Each one calls a client +# library whose own instrumentation emits the ``chat`` span, so wrapping them +# here as well would produce two spans for one model call and count the +# token-usage and duration metrics twice. ``README.rst`` lists which +# instrumentation covers which class. +_IN_PROCESS_MODEL_CLASSES = ("MLXModel", "TransformersModel", "VLLMModel") + - Only classes that define ``method`` in their own ``__dict__`` are patched, - so a class that inherits it (``AzureOpenAIModel``, ``LiteLLMRouterModel``) - isn't wrapped a second time and can't produce duplicate ``chat`` spans. - A user-defined subclass that overrides the method shadows the patched base - method and emits no ``chat`` span; that limitation is documented in - ``README.rst``. +def _model_classes_defining(method: str) -> list[type]: + """The in-process model classes whose ``method`` gets wrapped. - Deduplicated by class object, because smolagents exports some classes under - two names (``OpenAIServerModel`` is ``OpenAIModel``) and wrapping the same - class twice would double every ``chat`` span. + Only classes that define ``method`` in their own ``__dict__`` are patched. + ``MLXModel`` and ``VLLMModel`` have no ``generate_stream``, and neither does + the base class, so wrapping it on them raises ``AttributeError``. The same + check keeps a method defined on a shared base from being wrapped once per + subclass. + + A user-defined subclass that overrides the method shadows the patched one and + emits no ``chat`` span. ``README.rst`` documents that limitation. """ - from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel - Model, + from smolagents import ( # pylint: disable=import-outside-toplevel + models, ) - classes: dict[type, None] = {} - for obj in vars(smolagents).values(): - if ( - isinstance(obj, type) - and issubclass(obj, Model) - and method in obj.__dict__ - ): - classes.setdefault(obj, None) - return list(classes) + classes: list[type] = [] + for name in _IN_PROCESS_MODEL_CLASSES: + model_cls = getattr(models, name, None) + if isinstance(model_cls, type) and method in model_cls.__dict__: + classes.append(model_cls) + return classes class SmolagentsInstrumentor(BaseInstrumentor): @@ -115,8 +127,6 @@ def _instrument(self, **kwargs: Any) -> None: - logger_provider: LoggerProvider instance - completion_hook: CompletionHook instance """ - import smolagents # pylint: disable=import-outside-toplevel # noqa: PLC0415 - handler = TelemetryHandler( tracer_provider=kwargs.get("tracer_provider"), meter_provider=kwargs.get("meter_provider"), @@ -128,7 +138,7 @@ def _instrument(self, **kwargs: Any) -> None: self._wrapped_generate_classes = [] self._wrapped_generate_stream_classes = [] try: - for model_cls in _model_classes_defining(smolagents, "generate"): + for model_cls in _model_classes_defining("generate"): wrap_function_wrapper( model_cls, "generate", @@ -136,9 +146,7 @@ def _instrument(self, **kwargs: Any) -> None: ) self._wrapped_generate_classes.append(model_cls) - for model_cls in _model_classes_defining( - smolagents, "generate_stream" - ): + for model_cls in _model_classes_defining("generate_stream"): wrap_function_wrapper( model_cls, "generate_stream", diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index 83743ccc0..d0bb17152 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -16,7 +16,6 @@ import base64 import binascii import logging -from collections.abc import Mapping from enum import Enum from typing import Any @@ -26,9 +25,7 @@ InputMessage, MessagePart, OutputMessage, - Reasoning, Text, - ToolCallRequest, ToolDefinition, Uri, ) @@ -50,16 +47,6 @@ "tool-response": "user", } -# Amazon Bedrock reports the stop reason as ``stopReason`` using Anthropic's -# vocabulary. Normalize it the way the anthropic instrumentation does -# (``anthropic/utils.py``); anything unmapped passes through. -_STOP_REASON_MAP: dict[str, str] = { - "end_turn": "stop", - "stop_sequence": "stop", - "max_tokens": "length", - "tool_use": "tool_calls", -} - def _unwrap_role(role: Any) -> str | None: if role is None: @@ -87,7 +74,7 @@ def _decode_base64_image(image: str) -> tuple[bytes, str] | None: def _encode_image_base64(image: Any) -> str | None: try: - from smolagents.utils import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + from smolagents.utils import ( # pylint: disable=import-outside-toplevel encode_image_base64, ) except ImportError: @@ -183,90 +170,20 @@ def to_input_messages(messages: Any) -> list[InputMessage]: return result -def _raw_value(raw: Any, key: str) -> Any: - """Read ``key`` off a provider response that may be an object or a dict. - - ``ChatMessage.raw`` is whatever the provider handed back: an OpenAI-shaped - object for the API-backed models, and a dict for ``AmazonBedrockModel`` - (the boto3 ``converse`` response) and the local runtimes. - """ - if isinstance(raw, Mapping): - return raw.get(key) - return getattr(raw, key, None) - - -def _first_choice(output_message: Any) -> Any: - choices = _raw_value(getattr(output_message, "raw", None), "choices") - if isinstance(choices, list) and choices: - return choices[0] - return None - - -def _reasoning_from_raw(output_message: Any) -> str | None: - message = _raw_value(_first_choice(output_message), "message") - reasoning = _raw_value(message, "reasoning_content") - return reasoning if isinstance(reasoning, str) and reasoning else None - - -def _tool_call_requests(output_message: Any) -> list[ToolCallRequest]: - # ChatMessage.__post_init__ coerces every entry into a - # ChatMessageToolCall, so the id/function/name/arguments are all present. - tool_calls = getattr(output_message, "tool_calls", None) or [] - return [ - ToolCallRequest( - name=tool_call.function.name, - id=tool_call.id, - arguments=tool_call.function.arguments, - ) - for tool_call in tool_calls - ] - - -def finish_reason(output_message: Any) -> str | None: - """Why the provider stopped generating, or ``None`` if it didn't say. - - The local runtimes (``TransformersModel``, ``VLLMModel``, ``MLXModel``) put - ``{"out": ..., "completion_kwargs": ...}`` on ``raw`` and report no finish - reason at all. Defaulting those to ``"stop"`` would make a generation cut - short by ``max_new_tokens`` look like a natural stop. A response carrying - tool calls is the one case where the reason follows without guessing. - """ - reason = _raw_value(_first_choice(output_message), "finish_reason") - if isinstance(reason, str) and reason: - return reason - raw = getattr(output_message, "raw", None) - stop_reason = _raw_value(raw, "stopReason") - if isinstance(stop_reason, str) and stop_reason: - return _STOP_REASON_MAP.get(stop_reason, stop_reason) - if getattr(output_message, "tool_calls", None): - return "tool_calls" - return None - - def to_output_message(output_message: Any) -> OutputMessage: - """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``.""" + """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``. + + The in-process runtimes return the generated text and nothing else: no tool + calls (the agent parses those out of the text afterwards), no reasoning + content, and no finish reason. ``OutputMessage`` requires + ``finish_reason``, and util-genai drops an empty value when it emits + ``gen_ai.response.finish_reasons``. Defaulting it to ``"stop"`` instead + would make a generation cut short by ``max_new_tokens`` look like a natural + stop. + """ role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" parts = _parts_from_content(getattr(output_message, "content", None)) - if reasoning := _reasoning_from_raw(output_message): - parts.append(Reasoning(content=reasoning)) - tool_call_requests = _tool_call_requests(output_message) - parts.extend(tool_call_requests) - # OutputMessage requires the field; util-genai drops an empty value when it - # emits gen_ai.response.finish_reasons. - reason = finish_reason(output_message) - return OutputMessage(role=role, parts=parts, finish_reason=reason or "") - - -def response_id(output_message: Any) -> str | None: - """Extract ``gen_ai.response.id`` from the provider response on ``.raw``.""" - value = _raw_value(getattr(output_message, "raw", None), "id") - return value if isinstance(value, str) and value else None - - -def response_model_name(output_message: Any) -> str | None: - """Extract ``gen_ai.response.model`` from the provider response on ``.raw``.""" - value = _raw_value(getattr(output_message, "raw", None), "model") - return value if isinstance(value, str) and value else None + return OutputMessage(role=role, parts=parts, finish_reason="") def _tool_parameters(tool: Any) -> dict[str, Any] | None: @@ -278,7 +195,7 @@ def _tool_parameters(tool: Any) -> dict[str, Any] | None: schema the provider receives. """ try: - from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + from smolagents.models import ( # pylint: disable=import-outside-toplevel get_tool_json_schema, ) except ImportError: diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 26ff1312f..e6fb218f6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -4,11 +4,12 @@ """wrapt wrapper factories for smolagents instrumentation. Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper -suitable for :func:`wrapt.wrap_function_wrapper`: +suitable for :func:`wrapt.wrap_function_wrapper`, applied to the in-process +model classes only (see ``_IN_PROCESS_MODEL_CLASSES``): -- :func:`model_generate` wraps each defining ``Model.generate`` -> ``chat`` span. -- :func:`model_generate_stream` wraps each defining ``Model.generate_stream`` - -> ``chat`` span, held open until the stream is drained. +- :func:`model_generate` wraps ``generate`` -> ``chat`` span. +- :func:`model_generate_stream` wraps ``generate_stream`` -> ``chat`` span, held + open until the stream is drained. Original library exceptions are always re-raised unmodified; telemetry is finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. @@ -17,10 +18,9 @@ from __future__ import annotations import logging -from collections.abc import Generator, Mapping -from dataclasses import dataclass +from collections.abc import Callable, Generator, Mapping from inspect import signature -from typing import Any, Callable +from typing import Any from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, @@ -28,22 +28,14 @@ from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import InferenceInvocation from opentelemetry.util.genai.stream import SyncStreamWrapper -from opentelemetry.util.genai.types import ( - MessagePart, - OutputMessage, - Text, - ToolCallRequest, -) +from opentelemetry.util.genai.types import OutputMessage, Text from ._messages import ( - finish_reason, - response_id, - response_model_name, to_input_messages, to_output_message, to_tool_definitions, ) -from .provider import resolve_provider, resolve_server_address_port +from .provider import resolve_provider _logger = logging.getLogger(__name__) @@ -96,7 +88,7 @@ def _coerce_int(value: Any) -> int | None: def _remove_parameter_sentinel() -> Any: """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" try: - from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + from smolagents.models import ( # pylint: disable=import-outside-toplevel REMOVE_PARAMETER, ) except ImportError: @@ -105,24 +97,6 @@ def _remove_parameter_sentinel() -> Any: return REMOVE_PARAMETER -# Model classes that never forward ``stop_sequences``, whatever -# ``supports_stop_parameter`` answers. ``AmazonBedrockModel`` overrides -# ``_prepare_completion_kwargs`` and calls the base with a hardcoded -# ``stop_sequences=None`` (``models.py``), so its ``converse`` request carries -# no stop sequences at all. ``_forwards_stop_sequences`` matches these names -# along the MRO, so subclasses are covered too. -_MODELS_DROPPING_STOP_SEQUENCES = frozenset({"AmazonBedrockModel"}) - - -def _forwards_stop_sequences(instance: Any) -> bool: - if any( - cls.__name__ in _MODELS_DROPPING_STOP_SEQUENCES - for cls in type(instance).__mro__ - ): - return False - return bool(getattr(instance, "supports_stop_parameter", False)) - - def _merged_request_kwargs( instance: Any, bound: dict[str, Any] ) -> dict[str, Any]: @@ -138,7 +112,9 @@ def _merged_request_kwargs( """ merged: dict[str, Any] = {} stop_sequences = bound.get("stop_sequences") - if stop_sequences is not None and _forwards_stop_sequences(instance): + if stop_sequences is not None and getattr( + instance, "supports_stop_parameter", False + ): merged["stop"] = stop_sequences response_format = bound.get("response_format") if response_format is not None: @@ -233,8 +209,8 @@ def _apply_token_usage( ) -> None: # ChatMessage.token_usage is the only source: the per-model # last_input_token_count / last_output_token_count counters were removed - # before the oldest supported smolagents. It is None for the local runtimes - # (TransformersModel, VLLMModel, MLXModel), which report no usage. + # before the oldest supported smolagents. The in-process runtimes count the + # prompt and generated tokens themselves and report them here. token_usage = getattr(output_message, "token_usage", None) if token_usage is None: return @@ -251,15 +227,14 @@ def _start_inference( ) -> InferenceInvocation: """Start the ``chat`` span and record the request. - ``generate`` and ``generate_stream`` take the same parameters. + ``generate`` and ``generate_stream`` take the same parameters. An in-process + runtime listens on no socket, so the span carries no ``server.address`` or + ``server.port``. """ provider = resolve_provider(instance) - server_address, server_port = resolve_server_address_port(instance) invocation = handler.inference( provider, request_model=getattr(instance, "model_id", None), - server_address=server_address, - server_port=server_port, ) bound = _bind_arguments(wrapped, args, kwargs) _apply_request_parameters(invocation, instance, bound) @@ -272,7 +247,13 @@ def _start_inference( def model_generate(handler: TelemetryHandler) -> _Wrapper: - """Wrap a defining ``Model.generate`` to emit a ``chat`` span.""" + """Wrap a defining ``Model.generate`` to emit a ``chat`` span. + + An in-process runtime returns the generated text, the token counts it made + itself, and no response envelope, so the span carries no + ``gen_ai.response.id``, ``gen_ai.response.model`` or + ``gen_ai.response.finish_reasons``. + """ def wrapper( wrapped: Callable[..., Any], @@ -284,12 +265,6 @@ def wrapper( with invocation: output_message = wrapped(*args, **kwargs) _apply_token_usage(invocation, output_message) - invocation.response_model_name = response_model_name( - output_message - ) - invocation.response_id = response_id(output_message) - if reason := finish_reason(output_message): - invocation.finish_reasons = [reason] if handler.should_capture_content(): invocation.output_messages = [ to_output_message(output_message) @@ -299,20 +274,15 @@ def wrapper( return wrapper -@dataclass -class _StreamedToolCall: - """A tool call assembled from stream deltas.""" - - id: str | None = None - name: str = "" - arguments: str = "" - - class _ModelStreamWrapper(SyncStreamWrapper[Any]): """Keep the ``chat`` span open until the delta stream is drained. Passing the invocation to ``super().__init__()`` turns on ``gen_ai.request.stream`` and the per-chunk timing metrics. + + ``TransformersModel`` is the only in-process runtime with a + ``generate_stream``. Its deltas carry the generated text and per-delta token + counts, and never any tool calls. """ def __init__( @@ -325,34 +295,10 @@ def __init__( self._self_inference = invocation self._self_capture_content = handler.should_capture_content() self._self_content: list[str] = [] - self._self_tool_calls: dict[int, _StreamedToolCall] = {} self._self_input_tokens = 0 self._self_output_tokens = 0 self._self_saw_token_usage = False - def _accumulate_tool_call(self, delta: Any) -> None: - index = getattr(delta, "index", None) - if not isinstance(index, int): - # agglomerate_stream_deltas raises here; telemetry must not. - _logger.debug("Dropping a tool call delta that carries no index") - return - tool_call = self._self_tool_calls.setdefault( - index, _StreamedToolCall() - ) - if not self._self_capture_content: - # The finish reason only needs a tool call to have happened; its - # name and arguments are content. - return - if delta.id: - tool_call.id = delta.id - function = getattr(delta, "function", None) - if function is None: - return - if function.name: - tool_call.name = function.name - if function.arguments: - tool_call.arguments += function.arguments - def _process_chunk(self, chunk: Any) -> None: content = getattr(chunk, "content", None) if content and self._self_capture_content: @@ -364,31 +310,16 @@ def _process_chunk(self, chunk: Any) -> None: self._self_saw_token_usage = True self._self_input_tokens += token_usage.input_tokens self._self_output_tokens += token_usage.output_tokens - for delta in getattr(chunk, "tool_calls", None) or []: - self._accumulate_tool_call(delta) def _output_message(self) -> OutputMessage | None: - parts: list[MessagePart] = [] content = "".join(self._self_content) - if content: - parts.append(Text(content=content)) - parts.extend( - ToolCallRequest( - name=tool_call.name, - id=tool_call.id, - arguments=tool_call.arguments or None, - ) - for tool_call in self._self_tool_calls.values() - ) - if not parts: + if not content: # Closed before it was drained, so there is no response to report. return None - # Deltas carry no finish reason, so tool calls are the only evidence. - # Defaulting to "stop" would hide a generation cut short by a token - # limit. - finish_reason = "tool_calls" if self._self_tool_calls else "" + # Deltas carry no finish reason, and defaulting to "stop" would hide a + # generation cut short by a token limit. return OutputMessage( - role="assistant", parts=parts, finish_reason=finish_reason + role="assistant", parts=[Text(content=content)], finish_reason="" ) def _finalize(self, error: BaseException | None = None) -> None: @@ -396,8 +327,6 @@ def _finalize(self, error: BaseException | None = None) -> None: if self._self_saw_token_usage: invocation.input_tokens = self._self_input_tokens invocation.output_tokens = self._self_output_tokens - if self._self_tool_calls: - invocation.finish_reasons = ["tool_calls"] if self._self_capture_content: output = self._output_message() if output is not None: @@ -429,13 +358,7 @@ def wrapper( kwargs: dict[str, Any], ) -> Any: invocation = _start_inference(handler, wrapped, instance, args, kwargs) - - try: - stream = wrapped(*args, **kwargs) - except Exception as error: # pylint: disable=broad-except - invocation.fail(error) - raise - + stream = wrapped(*args, **kwargs) return _ModelStreamWrapper(stream, invocation, handler) return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py index be54d6400..d640bb8ae 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -3,18 +3,11 @@ """Resolve a smolagents model instance to a ``gen_ai.provider.name`` value. -Resolution order: - -1. For the LiteLLM model classes, the ``model_id`` vendor prefix - (``anthropic/claude-...`` -> ``anthropic``). -2. The model class name (e.g. ``OpenAIModel`` -> ``openai``), looked up along the - class hierarchy so a user subclass resolves to the provider of the base class - whose ``generate`` it inherits. -3. ``unknown``. +Only the in-process model classes are instrumented here, so the value comes +from the model class name. ``gen_ai.provider.name`` is a metric attribute as well as a span attribute, so -every value has to stay low cardinality: deployment-specific detail is reported -as ``server.address`` instead. ``TelemetryHandler.inference`` requires +every value has to stay low cardinality. ``TelemetryHandler.inference`` requires ``provider`` as a string, so this always returns a value rather than ``None``. """ @@ -22,118 +15,28 @@ class hierarchy so a user subclass resolves to the provider of the base class import logging from typing import Any -from urllib.parse import urlparse - -from opentelemetry.semconv._incubating.attributes import ( - gen_ai_attributes as GenAI, -) _logger = logging.getLogger(__name__) -_PROVIDER = GenAI.GenAiProviderNameValues - _UNKNOWN_PROVIDER = "unknown" # Model class name -> provider value. The GenAI registry has no value for the # Hugging Face, vLLM, and MLX runtimes, so those use the product name; a class # name would look like a provider without being one. _CLASS_NAME_TO_PROVIDER: dict[str, str] = { - "OpenAIModel": _PROVIDER.OPENAI.value, - "AzureOpenAIModel": _PROVIDER.AZURE_AI_OPENAI.value, - "AmazonBedrockModel": _PROVIDER.AWS_BEDROCK.value, - "InferenceClientModel": "huggingface", "TransformersModel": "huggingface", "VLLMModel": "vllm", "MLXModel": "mlx", } -# LiteLLM model_id prefix -> semconv provider value, for the prefixes whose -# LiteLLM vendor slug differs from the semconv value. Every other prefix is -# passed through as-is (``ollama/llama3`` -> ``ollama``). LiteLLM's slugs are a -# closed vocabulary, which keeps the cardinality bounded. -_LITELLM_PREFIX_TO_PROVIDER: dict[str, str] = { - "azure": _PROVIDER.AZURE_AI_OPENAI.value, - "azure_ai": _PROVIDER.AZURE_AI_INFERENCE.value, - "bedrock": _PROVIDER.AWS_BEDROCK.value, - "gemini": _PROVIDER.GCP_GEMINI.value, - "mistral": _PROVIDER.MISTRAL_AI.value, - "vertex_ai": _PROVIDER.GCP_VERTEX_AI.value, - "watsonx": _PROVIDER.IBM_WATSONX_AI.value, - "xai": _PROVIDER.X_AI.value, -} - -_LITELLM_CLASS_NAMES = frozenset({"LiteLLMModel", "LiteLLMRouterModel"}) - - -def _provider_from_litellm(instance: Any) -> str | None: - model_id = getattr(instance, "model_id", None) - if not isinstance(model_id, str) or "/" not in model_id: - return None - prefix = model_id.split("/", 1)[0].lower() - return _LITELLM_PREFIX_TO_PROVIDER.get(prefix, prefix) - - -def _endpoint(instance: Any) -> str | None: - """The endpoint URL the model calls, or ``None`` if it exposes none. - - Three places can hold it, checked in this order: - - 1. ``api_base`` on the instance (``LiteLLMModel``). - 2. ``base_url`` or ``azure_endpoint`` in the SDK client kwargs - (``OpenAIModel``, ``AzureOpenAIModel``, ``InferenceClientModel``). - 3. ``base_url`` on the client the model constructed (e.g. openai's httpx - URL), which holds the effective URL when the caller left it at the - provider default. - """ - api_base = getattr(instance, "api_base", None) - if api_base: - return str(api_base) - client_kwargs = getattr(instance, "client_kwargs", None) - if isinstance(client_kwargs, dict): - for key in ("base_url", "azure_endpoint"): - value = client_kwargs.get(key) - if value: - return str(value) - client_base_url = getattr( - getattr(instance, "client", None), "base_url", None - ) - if client_base_url: - return str(client_base_url) - return None - - -def resolve_server_address_port( - instance: Any, -) -> tuple[str | None, int | None]: - """Return ``(server.address, server.port)`` from the model's endpoint URL. - - Models that don't expose an ``api_base`` / ``base_url`` / ``azure_endpoint`` - (e.g. ``LiteLLMModel`` resolving the host internally, local runtimes) - yield ``(None, None)`` and the caller omits the attributes. - """ - endpoint = _endpoint(instance) - if endpoint is None: - return None, None - parsed = urlparse(endpoint) - port = parsed.port - if port == 443: - port = None - return parsed.hostname or None, port - def resolve_provider(instance: Any) -> str: """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" - # The instance's class names, most derived first. Only the classes that - # define ``generate`` are patched, so an instrumented model can be a user - # subclass of one of them. Matching the exact class name alone would report - # ``unknown`` for every such subclass. + # An instrumented model can be a user subclass of a patched class. Matching + # the exact class name alone would report ``unknown`` for every subclass, so + # walk the hierarchy, most derived class first. class_names = [cls.__name__ for cls in type(instance).__mro__] - if not _LITELLM_CLASS_NAMES.isdisjoint(class_names): - provider = _provider_from_litellm(instance) - if provider is not None: - return provider - for class_name in class_names: provider = _CLASS_NAME_TO_PROVIDER.get(class_name) if provider is not None: diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml deleted file mode 100644 index 6177ff059..000000000 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml +++ /dev/null @@ -1,20 +0,0 @@ -interactions: -- request: - body: '{"model": "claude-3-7-sonnet-20250219", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Who won the World Cup in 2018? Answer in one word - with no punctuation."}]}], "thinking": {"type": "enabled", "budget_tokens": - 4000}, "max_tokens": 8096}' - headers: {} - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id":"msg_011KX7d4TtALbugymC3Kb4oE","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[{"type":"thinking","thinking":"The - World Cup in 2018 was won by France. They defeated Croatia 4-2 in the final - match in Moscow, Russia.\n\nI need to answer in one word with no punctuation, - so my answer should simply be:\nFrance","signature":"ErUBCkYIBBgCIkAOLwDXty2UNgzsRPd4O0tNxqhaxPqqLw9isc2bDqyVS7Y87Cefkib8FVE9iTTjTSynEjrrHb+vei4K5pQrVOUrEgzCGfpi49gURMlFYTwaDGTEJyP22/PoNkje2CIwyiqcZqRj6rxDwzG0ayl3JQ2mi8x3iDIEuyP92Pw19EPRhATbpYiORzbaVoPUBuPEKh3gUYNSZwcjUAYNO01Qqo7GYFF08xO0MuULheotLRgC"},{"type":"text","text":"France"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":65,"service_tier":"standard"}}' - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml deleted file mode 100644 index 44f70cc65..000000000 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml +++ /dev/null @@ -1,25 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":[{"type":"text","text":"Who won the - World Cup in 2018? Answer in one word with no punctuation."}]}],"model":"gpt-4o","max_tokens":4096}' - headers: {} - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: "{\n \"id\": \"chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb\",\n \"object\": - \"chat.completion\",\n \"created\": 1738649686,\n \"model\": \"gpt-4o-2024-08-06\",\n - \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"France\",\n \"refusal\": null\n - \ },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n - \ ],\n \"usage\": {\n \"prompt_tokens\": 25,\n \"completion_tokens\": - 2,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": - 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": - {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": - 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_4691090a87\"\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml deleted file mode 100644 index 9fde9a56e..000000000 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml +++ /dev/null @@ -1,26 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What breed - is this dog?"},{"type":"image_url","image_url":{"url":"https://fastly.picsum.photos/id/237/200/300.jpg?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U"}}]}],"model":"gpt-4o"}' - headers: {} - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: "{\n \"id\": \"chatcmpl-DVJBfdzKCrLtPYM9ui8SZgIEQv857\",\n \"object\": - \"chat.completion\",\n \"created\": 1776354295,\n \"model\": \"gpt-4o-2024-08-06\",\n - \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"This looks like a Labrador Retriever - puppy. They are known for their friendly and outgoing nature.\",\n \"refusal\": - null,\n \"annotations\": []\n },\n \"logprobs\": null,\n - \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 268,\n \"completion_tokens\": 18,\n \"total_tokens\": 286,\n \"prompt_tokens_details\": - {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": - {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": - 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml deleted file mode 100644 index a5a57c069..000000000 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml +++ /dev/null @@ -1,32 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What is the - weather in Paris?"}]}],"model":"gpt-4o","max_tokens":4096,"tool_choice":"required","tools": - [{"type":"function","function":{"name":"get_weather","description":"Get the weather for a - given city","parameters":{"type":"object","properties":{"location":{"type":"string","description": - "The city to get the weather for"}},"required":["location"]}}}]}' - headers: {} - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: "{\n \"id\": \"chatcmpl-Ax7BZfEQe2evzgqbTVXWo0ZqoMIUf\",\n \"object\": - \"chat.completion\",\n \"created\": 1738652337,\n \"model\": \"gpt-4o-2024-08-06\",\n - \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n - \ \"id\": \"call_EUuviydGIG5Jau3DLw4v4cue\",\n \"type\": - \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n - \ \"arguments\": \"{\\\"location\\\":\\\"Paris\\\"}\"\n }\n - \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": - null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": - {\n \"prompt_tokens\": 61,\n \"completion_tokens\": 15,\n \"total_tokens\": - 76,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": - 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": - 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n - \ \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py index f9e60fb0f..bad12d2eb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py @@ -1,14 +1,17 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Conformance scenarios for the ``chat`` operation (plain and tool-calling).""" +"""Conformance scenarios for the ``chat`` operation (plain, streamed, and with +tool definitions). + +Every scenario drives an in-process runtime over the stubs in ``test_utils``, +since those are the only instrumented model classes. +""" from __future__ import annotations from typing import Any -from smolagents.models import ChatMessage, MessageRole - from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) @@ -16,11 +19,35 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.test.weaver_live_check import LiveCheckReport -from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) from opentelemetry.test_util_genai.instrumentor import instrument -from ..test_utils import GetWeatherTool, openai_model # noqa: TID252 -from ._helpers import attr, chat_spans, part_fields +from ..test_utils import ( + MESSAGES, + GetWeatherTool, + transformers_model, +) +from ._helpers import attr, chat_spans + +# An in-process runtime returns only the generated text: no request id, no +# resolved model name, no stop reason. It also listens on no socket. semconv +# expects those four attributes on a chat span, so every scenario here carries +# the same gaps. +IN_PROCESS_GAPS = tuple( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring=attribute, + ) + for attribute in ( + "server.address", + "gen_ai.response.id", + "gen_ai.response.model", + "gen_ai.response.finish_reasons", + ) +) class ChatScenario(Scenario): @@ -29,6 +56,35 @@ class ChatScenario(Scenario): "gen_ai.client.operation.duration", "gen_ai.client.token.usage", ) + expected_violations = IN_PROCESS_GAPS + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + transformers_model().generate(messages=MESSAGES) + + +class StreamedChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + "gen_ai.client.operation.time_to_first_chunk", + "gen_ai.client.operation.time_per_output_chunk", + ) + expected_violations = IN_PROCESS_GAPS def run( self, @@ -45,28 +101,28 @@ def run( meter_provider=meter_provider, content_capture="SPAN_ONLY", ): - with vcr.use_cassette("openai_model_basic.yaml"): - openai_model().generate( - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": ( - "Who won the World Cup in 2018? Answer " - "in one word with no punctuation." - ), - } - ], - } - ] - ) - - -class ToolCallingScenario(Scenario): + model = transformers_model(stream_chunks=["In ", "Paris"]) + list(model.generate_stream(messages=MESSAGES)) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + for span in chat_spans(report): + assert attr(span, "gen_ai.request.stream") is True, ( + "expected gen_ai.request.stream on a streamed chat span" + ) + + +class ToolDefinitionsScenario(Scenario): + """A ``chat`` call that offers tools to the model. + + The in-process runtimes put the tool definitions in the prompt but never + return tool calls (the agent parses those out of the generated text), so + this covers the request side only. + """ + expected_spans = {"chat": 1} expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = IN_PROCESS_GAPS def run( self, @@ -83,31 +139,13 @@ def run( meter_provider=meter_provider, content_capture="SPAN_ONLY", ): - with vcr.use_cassette("openai_model_tool.yaml"): - openai_model().generate( - messages=[ - ChatMessage( - role=MessageRole.USER, - content=[ - { - "type": "text", - "text": "What is the weather in Paris?", - } - ], - ) - ], - tools_to_call_from=[GetWeatherTool()], - ) + transformers_model().generate( + messages=MESSAGES, tools_to_call_from=[GetWeatherTool()] + ) def validate(self, report: LiveCheckReport) -> None: super().validate(report) - output_part_types = { - part_type - for span in chat_spans(report) - for part_type, _ in part_fields( - attr(span, "gen_ai.output.messages") + for span in chat_spans(report): + assert attr(span, "gen_ai.tool.definitions"), ( + "expected gen_ai.tool.definitions on the chat span" ) - } - assert "tool_call" in output_part_types, ( - f"expected a tool_call output part, saw {output_part_types}" - ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py index 5e8025c1b..530cbf314 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py @@ -1,17 +1,13 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Conformance scenarios for non-text message parts: an image ``uri`` on a chat -input, and a ``reasoning`` part on a chat output.""" +"""Conformance scenario for an image ``uri`` part on a ``chat`` input.""" from __future__ import annotations -import os from typing import Any -from unittest import mock -from smolagents import LiteLLMModel, OpenAIModel -from smolagents.models import ChatMessage, MessageRole +import pytest from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, @@ -20,13 +16,12 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.test.weaver_live_check import LiveCheckReport -from opentelemetry.test_util_genai.conformance import ( - ExpectedViolation, - Scenario, -) +from opentelemetry.test_util_genai.conformance import Scenario from opentelemetry.test_util_genai.instrumentor import instrument +from ..test_utils import vllm_model from ._helpers import attr, chat_spans, part_fields +from .inference import IN_PROCESS_GAPS _IMAGE_URL = ( "https://fastly.picsum.photos/id/237/200/300.jpg" @@ -35,8 +30,15 @@ class MultimodalScenario(Scenario): + """A vision-capable ``VLLMModel`` taking an image alongside the text. + + A VLM keeps the message content as parts instead of flattening it to text, + which is what puts an image part on the span. + """ + expected_spans = {"chat": 1} expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = IN_PROCESS_GAPS def run( self, @@ -53,17 +55,16 @@ def run( meter_provider=meter_provider, content_capture="SPAN_ONLY", ): - with vcr.use_cassette("openai_model_image_url.yaml"): - model = OpenAIModel( - model_id="gpt-4o", - api_key="test_openai_api_key", - api_base="https://api.openai.com/v1", - ) + # vllm_model fakes the vllm modules for the duration of the call. + with pytest.MonkeyPatch.context() as monkeypatch: + model = vllm_model(monkeypatch) + model._is_vlm = True + model.flatten_messages_as_text = False model.generate( messages=[ - ChatMessage( - role=MessageRole.USER, - content=[ + { + "role": "user", + "content": [ { "type": "text", "text": "What breed is this dog?", @@ -73,7 +74,7 @@ def run( "image_url": {"url": _IMAGE_URL}, }, ], - ) + } ] ) @@ -87,77 +88,3 @@ def validate(self, report: LiveCheckReport) -> None: assert ("uri", "image") in input_parts, ( f"expected an image uri input part, saw {input_parts}" ) - - -class ReasoningScenario(Scenario): - expected_spans = {"chat": 1} - expected_metrics = ("gen_ai.client.operation.duration",) - expected_violations = ( - # LiteLLM routes to the provider host internally and a LiteLLMModel - # built without an explicit api_base exposes no endpoint URL, so there - # is nothing to derive server.address from on the chat span. - ExpectedViolation( - advice_id="genai_expected_attribute_missing", - message_substring="server.address", - ), - ) - - def run( - self, - *, - tracer_provider: TracerProvider, - meter_provider: MeterProvider, - logger_provider: LoggerProvider, - vcr: Any, - ) -> None: - env = {"LITELLM_LOCAL_MODEL_COST_MAP": "True"} - with ( - mock.patch.dict(os.environ, env), - mock.patch("tiktoken.get_encoding") as get_encoding, - ): - get_encoding.return_value = mock.MagicMock( - encode=lambda *_: [1, 2, 3] - ) - with instrument( - SmolagentsInstrumentor(), - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - content_capture="SPAN_ONLY", - ): - with vcr.use_cassette("litellm_reasoning.yaml"): - model = LiteLLMModel( - model_id="anthropic/claude-3-7-sonnet-20250219", - api_key="test_anthropic_api_key", - thinking={"type": "enabled", "budget_tokens": 4000}, - ) - model.generate( - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": ( - "Who won the World Cup in 2018? " - "Answer in one word with no " - "punctuation." - ), - } - ], - } - ] - ) - - def validate(self, report: LiveCheckReport) -> None: - super().validate(report) - output_parts = { - part_type - for span in chat_spans(report) - for part_type, _ in part_fields( - attr(span, "gen_ai.output.messages") - ) - } - assert "reasoning" in output_parts, ( - f"expected a reasoning output part, saw {output_parts}" - ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index baebcbf9c..01ae6a97d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -5,69 +5,16 @@ from __future__ import annotations -import os -from unittest.mock import MagicMock, patch - import pytest from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) from opentelemetry.test_util_genai.instrumentor import instrument -from opentelemetry.test_util_genai.vcr import ( - scrub_response_headers_overwrite, -) - -pytest_plugins = [ - "opentelemetry.test_util_genai.fixtures", - "opentelemetry.test_util_genai.vcr", -] - - -@pytest.fixture(scope="module") -def vcr_config(): - return { - "filter_headers": [ - ("cookie", "test_cookie"), - ("authorization", "Bearer test_openai_api_key"), - ("x-api-key", "test_anthropic_api_key"), - ("openai-organization", "test_openai_org_id"), - ("openai-project", "test_openai_project_id"), - ], - "decode_compressed_response": True, - "before_record_response": scrub_response_headers_overwrite( - { - "openai-organization": "test_openai_org_id", - "openai-project": "test_openai_project_id", - "Set-Cookie": "test_set_cookie", - } - ), - } - -@pytest.fixture -def litellm_local_cost_map(): - """Use LiteLLM's bundled model-cost map so it doesn't fetch prices over the - network during cassette playback.""" - previous = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - try: - yield - finally: - if previous is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous - - -@pytest.fixture -def patch_tiktoken_encoding(): - """Patch ``tiktoken.get_encoding`` so LiteLLM doesn't download an encoding.""" - with patch("tiktoken.get_encoding") as mock_get_encoding: - mock_encoding = MagicMock() - mock_encoding.encode.return_value = [1, 2, 3] - mock_get_encoding.return_value = mock_encoding - yield +# No VCR plugin: the instrumented model classes run inference in this process, +# so there is no HTTP traffic to record. +pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] @pytest.fixture diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index 649ffc343..3cdf75f33 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -26,9 +26,9 @@ # This variant of the requirements aims to test the system using the newest # supported version of external dependencies. -# openai/litellm are the model backends exercised by the VCR model tests; they -# are test-only (not declared in pyproject.toml) so they live here. -smolagents[openai,litellm] +# The instrumented model classes run inference in this process and the tests stub +# their runtimes, so no model backend extra is installed. +smolagents wrapt>=2.2.2 -e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index afb682e96..18cd3aaa7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -21,13 +21,8 @@ # pyproject.toml is the single source of truth. The OpenTelemetry SDK and test utilities # come transitively from opentelemetry-test-util-genai. # -# openai and litellm are the model backends the VCR model tests instantiate. They are -# test-only (not declared in pyproject.toml) and smolagents does not pull them in through -# the instruments extra, so pin them here to the versions smolagents 1.24.0 declares in its -# own openai/litellm extras. litellm resolves to its floor; openai resolves to 1.61.0 -# rather than 1.58.1 because litellm 1.60.2 itself requires openai>=1.61.0. -openai>=1.58.1 -litellm>=1.60.2 +# The instrumented model classes run inference in this process and the tests stub their +# runtimes, so there is no model backend to pin here. # The declared opentelemetry-util-genai floor carries the streaming metrics and the # gen_ai.request.stream attribute, which are not in a published release yet. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py index 41c60d2db..dc0ba1e7c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -14,29 +14,27 @@ pytest.importorskip("opentelemetry.test.weaver_live_check") pytest.importorskip("opentelemetry.exporter.otlp.proto.grpc") -from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 -from opentelemetry.test_util_genai.conformance import ( # noqa: E402 +from opentelemetry.test.weaver_live_check import WeaverLiveCheck +from opentelemetry.test_util_genai.conformance import ( Scenario, run_conformance, ) -from .conformance.inference import ( # noqa: E402 +from .conformance.inference import ( ChatScenario, - ToolCallingScenario, -) -from .conformance.multimodal import ( # noqa: E402 - MultimodalScenario, - ReasoningScenario, + StreamedChatScenario, + ToolDefinitionsScenario, ) +from .conformance.multimodal import MultimodalScenario @pytest.mark.parametrize( "scenario", [ pytest.param(ChatScenario()), - pytest.param(ToolCallingScenario()), + pytest.param(StreamedChatScenario()), + pytest.param(ToolDefinitionsScenario()), pytest.param(MultimodalScenario()), - pytest.param(ReasoningScenario()), ], ids=lambda s: type(s).__name__, ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index ed759e5e8..c42d08d78 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -20,7 +20,7 @@ from opentelemetry.util._importlib_metadata import entry_points from opentelemetry.util.genai.completion_hook import CompletionHook -from .test_utils import openai_model, stub_openai_client +from .test_utils import MESSAGES, transformers_model class RecordingHook(CompletionHook): @@ -32,9 +32,7 @@ def on_completion(self, **kwargs: Any) -> None: def _generate_a_chat_span() -> None: - model = openai_model() - model.client = stub_openai_client("Bonjour") - model.generate(messages=[{"role": "user", "content": "Hi"}]) + transformers_model().generate(messages=MESSAGES) def test_entrypoint_loads_instrumentor() -> None: @@ -53,9 +51,9 @@ def test_instrumentation_dependencies() -> None: def test_instrument_uninstrument_restores_originals( tracer_provider, logger_provider, meter_provider ) -> None: - original_generate = smolagents.OpenAIModel.generate - original_base_generate = smolagents.Model.generate - original_generate_stream = smolagents.OpenAIModel.generate_stream + original_generate = smolagents.TransformersModel.generate + original_mlx_generate = smolagents.MLXModel.generate + original_generate_stream = smolagents.TransformersModel.generate_stream instrumentor = SmolagentsInstrumentor() instrumentor.instrument( @@ -64,17 +62,56 @@ def test_instrument_uninstrument_restores_originals( meter_provider=meter_provider, ) - assert smolagents.OpenAIModel.generate is not original_generate - assert smolagents.Model.generate is not original_base_generate + assert smolagents.TransformersModel.generate is not original_generate + assert smolagents.MLXModel.generate is not original_mlx_generate assert ( - smolagents.OpenAIModel.generate_stream is not original_generate_stream + smolagents.TransformersModel.generate_stream + is not original_generate_stream ) instrumentor.uninstrument() - assert smolagents.OpenAIModel.generate is original_generate - assert smolagents.Model.generate is original_base_generate - assert smolagents.OpenAIModel.generate_stream is original_generate_stream + assert smolagents.TransformersModel.generate is original_generate + assert smolagents.MLXModel.generate is original_mlx_generate + assert ( + smolagents.TransformersModel.generate_stream + is original_generate_stream + ) + + +@pytest.mark.parametrize( + "model_class", + [ + "Model", + "OpenAIModel", + "AzureOpenAIModel", + "AmazonBedrockModel", + "InferenceClientModel", + "LiteLLMModel", + "LiteLLMRouterModel", + ], +) +def test_api_backed_classes_are_left_alone( + tracer_provider, logger_provider, meter_provider, model_class: str +) -> None: + # Each of these calls a client library that carries its own instrumentation, + # so patching them here would emit a second chat span for one model call and + # count the token-usage and duration metrics twice. + original = getattr(smolagents, model_class).__dict__.get("generate") + + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + assert ( + getattr(smolagents, model_class).__dict__.get("generate") + is original + ) + finally: + instrumentor.uninstrument() def test_uninstrument_through_a_new_constructor_call( @@ -84,7 +121,7 @@ def test_uninstrument_through_a_new_constructor_call( # SmolagentsInstrumentor().instrument() / SmolagentsInstrumentor() # .uninstrument() form must restore everything even though the second # constructor call re-runs __init__ on the live instance. - original_generate = smolagents.OpenAIModel.generate + original_generate = smolagents.TransformersModel.generate SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, @@ -93,43 +130,31 @@ def test_uninstrument_through_a_new_constructor_call( ) SmolagentsInstrumentor().uninstrument() - assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.TransformersModel.generate is original_generate @pytest.mark.parametrize("method", ["generate", "generate_stream"]) def test_model_classes_defining(method: str) -> None: - classes = _model_classes_defining(smolagents, method) + classes = _model_classes_defining(method) - # Each class object appears once, whatever names it is exported under. assert len(classes) == len(set(classes)) # Every entry owns the method, so no class is wrapped for one it only # inherits. for model_cls in classes: assert method in model_cls.__dict__ - # The API-backed model classes define both; the classes that inherit them - # are reached through the base they inherit from. - assert { - smolagents.OpenAIModel, - smolagents.LiteLLMModel, - smolagents.InferenceClientModel, - } <= set(classes) - assert smolagents.AzureOpenAIModel not in classes - assert smolagents.LiteLLMRouterModel not in classes - - -def test_only_generate_covers_the_base_class_and_bedrock() -> None: - # The base Model and AmazonBedrockModel define generate but no - # generate_stream, so streaming is patched on fewer classes. - generate = set(_model_classes_defining(smolagents, "generate")) - generate_stream = set( - _model_classes_defining(smolagents, "generate_stream") - ) - assert {smolagents.Model, smolagents.AmazonBedrockModel} <= generate - assert generate_stream.isdisjoint( - {smolagents.Model, smolagents.AmazonBedrockModel} - ) +def test_only_transformers_defines_generate_stream() -> None: + # All three in-process runtimes define generate, but only TransformersModel + # streams. + assert set(_model_classes_defining("generate")) == { + smolagents.TransformersModel, + smolagents.VLLMModel, + smolagents.MLXModel, + } + assert _model_classes_defining("generate_stream") == [ + smolagents.TransformersModel + ] def test_repeated_instrument_uninstrument( @@ -137,7 +162,7 @@ def test_repeated_instrument_uninstrument( ) -> None: # BaseInstrumentor returns a per-class singleton, so the wrapped-class # bookkeeping has to survive being filled and drained more than once. - original_generate = smolagents.OpenAIModel.generate + original_generate = smolagents.TransformersModel.generate instrumentor = SmolagentsInstrumentor() for _ in range(2): @@ -146,36 +171,36 @@ def test_repeated_instrument_uninstrument( logger_provider=logger_provider, meter_provider=meter_provider, ) - assert smolagents.OpenAIModel.generate is not original_generate + assert smolagents.TransformersModel.generate is not original_generate instrumentor.uninstrument() - assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.TransformersModel.generate is original_generate def test_uninstrument_without_instrument() -> None: # BaseInstrumentor.uninstrument() short-circuits, but _uninstrument() must # also be a no-op on unpatched attributes: the rollback in _instrument() # calls it after a partial patch. - original_generate = smolagents.OpenAIModel.generate + original_generate = smolagents.TransformersModel.generate SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() - assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.TransformersModel.generate is original_generate def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. - original_generate = smolagents.OpenAIModel.generate + original_generate = smolagents.TransformersModel.generate instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: - assert smolagents.OpenAIModel.generate is not original_generate + assert smolagents.TransformersModel.generate is not original_generate finally: instrumentor.uninstrument() - assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.TransformersModel.generate is original_generate def test_failed_instrument_rolls_back_partial_patches( @@ -183,7 +208,7 @@ def test_failed_instrument_rolls_back_partial_patches( ) -> None: # A failure part-way through must leave no class patched, because # uninstrument() cannot clean up after a failed _instrument(). - model_classes = _model_classes_defining(smolagents, "generate") + model_classes = _model_classes_defining("generate") assert len(model_classes) > 1, ( "the rollback needs more than one class to patch" ) @@ -191,7 +216,7 @@ def test_failed_instrument_rolls_back_partial_patches( model_cls: model_cls.__dict__["generate"] for model_cls in model_classes } - stream_classes = _model_classes_defining(smolagents, "generate_stream") + stream_classes = _model_classes_defining("generate_stream") stream_originals = { model_cls: model_cls.__dict__["generate_stream"] for model_cls in stream_classes @@ -225,13 +250,15 @@ def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: assert model_cls.__dict__["generate_stream"] is original -def test_inherited_generate_wrapped_only_on_defining_classes( +def test_a_user_subclass_inherits_the_patched_generate( tracer_provider, logger_provider, meter_provider ) -> None: - # AzureOpenAIModel and LiteLLMRouterModel inherit generate; they must not be - # wrapped separately or they would emit duplicate chat spans. - assert "generate" not in smolagents.AzureOpenAIModel.__dict__ - assert "generate" not in smolagents.LiteLLMRouterModel.__dict__ + # A subclass that doesn't override generate is instrumented through the base + # it inherits it from, and must not be wrapped a second time. + class TenantMLXModel(smolagents.MLXModel): + pass + + assert "generate" not in TenantMLXModel.__dict__ instrumentor = SmolagentsInstrumentor() instrumentor.instrument( @@ -242,10 +269,10 @@ def test_inherited_generate_wrapped_only_on_defining_classes( try: # wrapt returns a fresh bound wrapper per attribute access, so the # wrapper objects differ; the underlying wrapped function is shared, - # proving AzureOpenAIModel inherits the single wrapped generate. + # proving the subclass inherits the single wrapped generate. assert ( - smolagents.AzureOpenAIModel.generate.__wrapped__ - is smolagents.OpenAIModel.generate.__wrapped__ + TenantMLXModel.generate.__wrapped__ + is smolagents.MLXModel.generate.__wrapped__ ) finally: instrumentor.uninstrument() diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py index 3260e65b4..93af3e0c9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -1,39 +1,37 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Model (``chat``) instrumentation tests: VCR-backed runs of the real -smolagents model classes, provider/endpoint resolution, request-parameter -mapping, and message conversion. +"""Model (``chat``) instrumentation tests for the in-process runtimes: +provider resolution, request-parameter mapping, message conversion, streaming, +and errors. + +The three instrumented classes run inference in the current process, so there is +no HTTP traffic to record; each test drives smolagents' real ``generate`` over +the stubbed runtime pieces built in ``test_utils``. """ from __future__ import annotations import inspect import json -import sys from collections.abc import Generator -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace from typing import Any import pytest -from smolagents import LiteLLMModel, OpenAIModel -from smolagents.models import ( - ChatMessage, - ChatMessageToolCall, - ChatMessageToolCallFunction, - MessageRole, -) +from smolagents.models import ChatMessage, MessageRole from opentelemetry.instrumentation.genai.smolagents._messages import ( - response_id, - response_model_name, to_input_messages, to_output_message, to_tool_definitions, ) +from opentelemetry.instrumentation.genai.smolagents.patch import ( + _merged_request_kwargs, + _output_type, +) from opentelemetry.instrumentation.genai.smolagents.provider import ( resolve_provider, - resolve_server_address_port, ) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, @@ -44,28 +42,19 @@ server_attributes, ) from opentelemetry.trace import StatusCode -from opentelemetry.util.genai.types import ( - Blob, - Reasoning, - Text, - ToolCallRequest, - Uri, -) +from opentelemetry.util.genai.types import Blob, Text, Uri from .test_utils import ( + MESSAGES, GetWeatherTool, attr, data_point_attributes, metrics_by_name, - openai_model, + mlx_model, parse_messages, - part_types, spans_by_operation, - stub_openai_client, - stub_streaming_openai_client, - text_chunk, - tool_call_chunk, - usage_chunk, + transformers_model, + vllm_model, ) IMAGE_URL = ( @@ -74,42 +63,44 @@ ) -def test_openai_model_basic( - instrument_with_content, span_exporter, metric_reader, vcr +def test_transformers_generate_records_the_response( + instrument_with_content, span_exporter, metric_reader ) -> None: - model = openai_model() - text = "Who won the World Cup in 2018? Answer in one word with no punctuation." - with vcr.use_cassette("openai_model_basic.yaml"): - output = model.generate( - messages=[ - {"role": "user", "content": [{"type": "text", "text": text}]} - ] - ) - assert output.content == "France" + model = transformers_model( + prompt_ids=[1, 2, 3], generated_ids=[4, 5], text="In Paris" + ) + + output = model.generate(messages=MESSAGES) + assert output.content == "In Paris" (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert span.name == "chat gpt-4o" + assert span.name == "chat HuggingFaceTB/SmolLM2-135M-Instruct" assert span.status.status_code == StatusCode.UNSET - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" - assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" - assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 25 - assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" assert ( - attr(span, GenAI.GEN_AI_RESPONSE_ID) - == "chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb" + attr(span, GenAI.GEN_AI_REQUEST_MODEL) + == "HuggingFaceTB/SmolLM2-135M-Instruct" ) - assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) == "gpt-4o-2024-08-06" - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) - assert attr(span, server_attributes.SERVER_ADDRESS) == "api.openai.com" + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + # A runtime in this process returns no response envelope and listens on no + # socket, so none of these have a value to report. + assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None assert attr(span, server_attributes.SERVER_PORT) is None inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) assert inputs[0]["role"] == "user" - assert inputs[0]["parts"][0] == {"type": "text", "content": text} + assert inputs[0]["parts"] == [ + {"type": "text", "content": "Where is the Louvre?"} + ] outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) assert outputs[0]["role"] == "assistant" - assert outputs[0]["parts"][0] == {"type": "text", "content": "France"} + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + assert outputs[0]["finish_reason"] == "" metrics = metrics_by_name(metric_reader) duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] @@ -117,85 +108,102 @@ def test_openai_model_basic( assert data_point_attributes(duration) == [ { GenAI.GEN_AI_OPERATION_NAME: "chat", - GenAI.GEN_AI_PROVIDER_NAME: "openai", - GenAI.GEN_AI_REQUEST_MODEL: "gpt-4o", - GenAI.GEN_AI_RESPONSE_MODEL: "gpt-4o-2024-08-06", - server_attributes.SERVER_ADDRESS: "api.openai.com", + GenAI.GEN_AI_PROVIDER_NAME: "huggingface", + GenAI.GEN_AI_REQUEST_MODEL: "HuggingFaceTB/SmolLM2-135M-Instruct", } ] token_usage = metrics[gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE] assert { point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point.sum for point in token_usage.data.data_points - } == {"input": 25, "output": 2} + } == {"input": 3, "output": 2} -def test_openai_model_no_content( - instrument_no_content, span_exporter, vcr +@pytest.mark.parametrize( + "runtime, provider, request_model, input_tokens, output_tokens", + [ + ("mlx", "mlx", "mlx-community/Qwen2.5-0.5B-Instruct-4bit", 3, 2), + ("vllm", "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), + ], +) +def test_other_runtimes_record_the_response( + instrument_with_content, + span_exporter, + monkeypatch: pytest.MonkeyPatch, + runtime: str, + provider: str, + request_model: str, + input_tokens: int, + output_tokens: int, ) -> None: - model = openai_model() - with vcr.use_cassette("openai_model_basic.yaml"): - model.generate( - messages=[ - {"role": "user", "content": [{"type": "text", "text": "Hi"}]} - ] - ) + model = mlx_model() if runtime == "mlx" else vllm_model(monkeypatch) + + output = model.generate(messages=MESSAGES) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + + +def test_no_content(instrument_no_content, span_exporter) -> None: + transformers_model().generate(messages=MESSAGES) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" - assert isinstance(attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS), int) + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None - # The finish reason is metadata, so it survives without the content. - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + # Token usage is metadata, so it survives without the content. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 -def test_openai_model_image_url( - instrument_with_content, span_exporter, vcr +def test_event_only_content_capture( + instrument_event_only, span_exporter, log_exporter ) -> None: - model = openai_model() - with vcr.use_cassette("openai_model_image_url.yaml"): - model.generate( - messages=[ - ChatMessage( - role=MessageRole.USER, - content=[ - {"type": "text", "text": "What breed is this dog?"}, - {"type": "image_url", "image_url": {"url": IMAGE_URL}}, - ], - ) - ] - ) + transformers_model().generate(messages=MESSAGES) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) - parts = inputs[0]["parts"] - assert parts[0] == {"type": "text", "content": "What breed is this dog?"} - assert parts[1]["type"] == "uri" - assert parts[1]["modality"] == "image" - assert parts[1]["uri"] == IMAGE_URL + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.event_name == "gen_ai.client.inference.operation.details" + # Event attributes carry the messages as structured values, not JSON text. + attributes = record.attributes or {} + inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] + outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] + assert inputs[0]["parts"][0]["content"] == "Where is the Louvre?" + assert outputs[0]["parts"][0]["content"] == "In Paris" -def test_openai_model_with_tools( - instrument_with_content, span_exporter, vcr + +def test_runtime_error_is_recorded_and_reraised( + instrument_with_content, span_exporter ) -> None: - model = openai_model() - with vcr.use_cassette("openai_model_tool.yaml"): - output = model.generate( - messages=[ - ChatMessage( - role=MessageRole.USER, - content=[ - { - "type": "text", - "text": "What is the weather in Paris?", - } - ], - ) - ], - tools_to_call_from=[GetWeatherTool()], - ) - assert output.tool_calls[0].function.name == "get_weather" + model = transformers_model(error=RuntimeError("CUDA out of memory")) + + with pytest.raises(RuntimeError, match="CUDA out of memory"): + model.generate(messages=MESSAGES) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "RuntimeError" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_tool_definitions_recorded( + instrument_with_content, span_exporter +) -> None: + transformers_model().generate( + messages=MESSAGES, tools_to_call_from=[GetWeatherTool()] + ) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert json.loads(attr(span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ @@ -216,116 +224,33 @@ def test_openai_model_with_tools( } ] - outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) - tool_call_parts = [ - part for part in outputs[0]["parts"] if part["type"] == "tool_call" - ] - assert tool_call_parts[0]["name"] == "get_weather" - # smolagents hands the provider's raw argument payload through unparsed. - assert tool_call_parts[0]["arguments"] == '{"location":"Paris"}' - assert tool_call_parts[0]["id"] == "call_EUuviydGIG5Jau3DLw4v4cue" - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) - - -def _litellm_supports_reasoning() -> bool: - """litellm surfaces Anthropic ``reasoning_content`` only from ~1.63 onward. - - The oldest supported litellm (smolagents' 1.60.2 floor) doesn't parse the - Anthropic thinking blocks into ``reasoning_content``, so the reasoning part - can't be mapped there. Gate the reasoning-specific assertion on the version. - """ - from importlib.metadata import version # noqa: PLC0415 - - parts = version("litellm").split(".") - try: - return (int(parts[0]), int(parts[1])) >= (1, 63) - except (IndexError, ValueError): - return True - -def test_litellm_reasoning( - instrument_with_content, - span_exporter, - litellm_local_cost_map, - patch_tiktoken_encoding, - vcr, -) -> None: - model = LiteLLMModel( - model_id="anthropic/claude-3-7-sonnet-20250219", - api_key="test_anthropic_api_key", - thinking={"type": "enabled", "budget_tokens": 4000}, - ) - with vcr.use_cassette("litellm_reasoning.yaml"): - model.generate( - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": ( - "Who won the World Cup in 2018? Answer in one " - "word with no punctuation." - ), - } - ], - } - ] - ) - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "anthropic" - assert ( - attr(span, GenAI.GEN_AI_REQUEST_MODEL) - == "anthropic/claude-3-7-sonnet-20250219" - ) - outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) - if _litellm_supports_reasoning(): - assert "reasoning" in part_types(outputs) - - -def test_model_generate_reraises_and_records_error( - instrument_with_content, span_exporter +def test_user_subclass_keeps_its_provider( + instrument_with_content, span_exporter, metric_reader ) -> None: - from smolagents.models import Model # noqa: PLC0415 - - model = Model(model_id="broken-model") - with pytest.raises(NotImplementedError): - model.generate(messages=[{"role": "user", "content": "hi"}]) - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert span.status.status_code == StatusCode.ERROR - assert attr(span, error_attributes.ERROR_TYPE) == "NotImplementedError" + # A subclass inherits the patched generate, so it is instrumented; resolving + # the provider by exact class name would report "unknown" on both the span + # and the metrics. + from smolagents.models import MLXModel + class TenantMLXModel(MLXModel): + pass -def test_inherited_azure_generate_emits_one_chat_span( - instrument_with_content, span_exporter -) -> None: - # AzureOpenAIModel inherits generate from OpenAIModel, so only the defining - # class is patched. Exercise the inherited method end to end to prove the - # single patch still produces exactly one span with the Azure provider. - from smolagents import AzureOpenAIModel # noqa: PLC0415 - - model = AzureOpenAIModel( - model_id="gpt-4o-deployment", - azure_endpoint="https://example-resource.openai.azure.com", - api_key="test_azure_api_key", - api_version="2024-10-21", - ) - model.client = stub_openai_client("Bonjour") + model = mlx_model() + # The factory builds the base class; rebinding __class__ makes the stub an + # instance of the subclass without running __init__. + model.__class__ = TenantMLXModel - output = model.generate(messages=[{"role": "user", "content": "Hi"}]) - assert output.content == "Bonjour" + model.generate(messages=MESSAGES) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "azure.ai.openai" - assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o-deployment" - assert ( - attr(span, server_attributes.SERVER_ADDRESS) - == "example-resource.openai.azure.com" + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "mlx" + duration = metrics_by_name(metric_reader)[ + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( + "mlx" ) - assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 - assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 1 def _fake_model(class_name: str, **attrs: Any) -> Any: @@ -336,120 +261,45 @@ def _fake_model(class_name: str, **attrs: Any) -> Any: @pytest.mark.parametrize( - "class_name, attrs, expected", + "class_name, expected", [ - ("OpenAIModel", {}, "openai"), - ("AzureOpenAIModel", {}, "azure.ai.openai"), - ("AmazonBedrockModel", {}, "aws.bedrock"), # No GenAI registry value exists for these runtimes, so the product # name is used rather than the class name. - ("InferenceClientModel", {}, "huggingface"), - ("TransformersModel", {}, "huggingface"), - ("VLLMModel", {}, "vllm"), - ("MLXModel", {}, "mlx"), - # LiteLLM vendor prefixes: remapped where the slug differs from the - # semconv value, passed through otherwise. - ("LiteLLMModel", {"model_id": "anthropic/claude-3"}, "anthropic"), - ("LiteLLMModel", {"model_id": "mistral/large"}, "mistral_ai"), - ("LiteLLMModel", {"model_id": "xai/grok"}, "x_ai"), - ("LiteLLMModel", {"model_id": "gemini/gemini-2.0"}, "gcp.gemini"), - ( - "LiteLLMModel", - {"model_id": "vertex_ai/gemini-2.0"}, - "gcp.vertex_ai", - ), - ("LiteLLMModel", {"model_id": "watsonx/granite"}, "ibm.watsonx.ai"), - ( - "LiteLLMModel", - {"model_id": "azure_ai/phi-4"}, - "azure.ai.inference", - ), - ("LiteLLMModel", {"model_id": "ollama/llama3"}, "ollama"), - # LiteLLMRouterModel takes a model-group name, not a provider/model - # slug, so there is nothing to resolve. - ("LiteLLMRouterModel", {"model_id": "model-group-1"}, "unknown"), + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), # gen_ai.provider.name is also a metric attribute. An unmapped model - # must not fall back to the deployment-specific host or a class name. - ( - "CustomModel", - {"api_base": "https://llm.example.com/v1"}, - "unknown", - ), - ("CustomModel", {}, "unknown"), + # must not fall back to a class name. + ("CustomModel", "unknown"), ], ) -def test_resolve_provider( - class_name: str, attrs: dict[str, Any], expected: str -) -> None: - assert resolve_provider(_fake_model(class_name, **attrs)) == expected +def test_resolve_provider(class_name: str, expected: str) -> None: + assert resolve_provider(_fake_model(class_name)) == expected @pytest.mark.parametrize( "model_class, expected", [ - ("OpenAIModel", "openai"), - ("AzureOpenAIModel", "azure.ai.openai"), - ("AmazonBedrockModel", "aws.bedrock"), - ("InferenceClientModel", "huggingface"), ("TransformersModel", "huggingface"), ("VLLMModel", "vllm"), ("MLXModel", "mlx"), - ("LiteLLMModel", "unknown"), - ("LiteLLMRouterModel", "unknown"), ], ) -def test_resolve_provider_covers_every_real_model_class( +def test_resolve_provider_covers_every_instrumented_class( model_class: str, expected: str ) -> None: # The mapping is keyed by class name, so pin it against the real classes # rather than only against synthetic stand-ins. - import smolagents # noqa: PLC0415 + import smolagents instance = object.__new__(getattr(smolagents, model_class)) assert resolve_provider(instance) == expected -@pytest.mark.parametrize( - "attrs, expected", - [ - ({"api_base": "https://api.openai.com/v1"}, ("api.openai.com", None)), - # The default HTTPS port is omitted per the semconv server.port guidance. - ( - {"api_base": "https://api.openai.com:443/v1"}, - ("api.openai.com", None), - ), - ({"api_base": "http://localhost:11434/v1"}, ("localhost", 11434)), - ( - { - "client_kwargs": { - "azure_endpoint": "https://x.openai.azure.com" - } - }, - ("x.openai.azure.com", None), - ), - ({}, (None, None)), - ], -) -def test_resolve_server_address_port( - attrs: dict[str, Any], expected: tuple[str | None, int | None] -) -> None: - assert resolve_server_address_port(_fake_model("M", **attrs)) == expected - - -def test_server_address_falls_back_to_the_sdk_client() -> None: - # The common configuration: no api_base, so the URL is only known to the - # client the model built for itself. - model = OpenAIModel(model_id="gpt-4o", api_key="test_openai_api_key") - assert resolve_server_address_port(model) == ("api.openai.com", None) - - def test_request_parameters_recorded( instrument_with_content, span_exporter ) -> None: - from smolagents.models import Model # noqa: PLC0415 - - model = Model( - model_id="broken-model", + model = mlx_model( temperature=0.5, top_p=0.9, top_k=40, @@ -458,11 +308,8 @@ def test_request_parameters_recorded( max_tokens=256, seed=7, ) - with pytest.raises(NotImplementedError): - model.generate( - messages=[{"role": "user", "content": "hi"}], - stop_sequences=[""], - ) + + model.generate(messages=MESSAGES, stop_sequences=[""]) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.5 @@ -484,23 +331,13 @@ def test_model_kwargs_win_over_call_kwargs( # _prepare_completion_kwargs applies the call kwargs first and the model # kwargs on top, and drops any key whose model-level value is the # REMOVE_PARAMETER sentinel. - from smolagents.models import ( # noqa: PLC0415 - REMOVE_PARAMETER, - Model, - ) + from smolagents.models import REMOVE_PARAMETER + + model = mlx_model(temperature=0.1, max_tokens=REMOVE_PARAMETER) - model = Model( - model_id="broken-model", - temperature=0.1, - max_tokens=REMOVE_PARAMETER, + model.generate( + messages=MESSAGES, temperature=0.9, max_tokens=512, top_p=0.5 ) - with pytest.raises(NotImplementedError): - model.generate( - messages=[{"role": "user", "content": "hi"}], - temperature=0.9, - max_tokens=512, - top_p=0.5, - ) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.1 @@ -509,90 +346,52 @@ def test_model_kwargs_win_over_call_kwargs( @pytest.mark.parametrize( - "model_id, model_kwargs, expected", + "model_kwargs, expected", [ - # gpt-5 doesn't accept `stop`, so smolagents truncates the generated - # text locally instead of sending the sequences. - ("gpt-5", {}, None), - ("gpt-4o", {}, ("",)), + ({}, ("",)), # An explicit `stop` overrides the stop_sequences argument. - ("gpt-4o", {"stop": ["STOP"]}, ("STOP",)), + ({"stop": ["STOP"]}, ("STOP",)), # The model-level sentinel pops the `stop` that # _prepare_completion_kwargs seeded from stop_sequences, leaving the # request with none. - ("gpt-4o", {"stop": "REMOVE"}, None), + ({"stop": "REMOVE"}, None), ], ) def test_stop_sequences_follow_what_is_sent( instrument_with_content, span_exporter, - model_id: str, model_kwargs: dict[str, Any], expected: tuple[str, ...] | None, ) -> None: - from smolagents.models import ( # noqa: PLC0415 - REMOVE_PARAMETER, - Model, - ) + from smolagents.models import REMOVE_PARAMETER model_kwargs = { key: REMOVE_PARAMETER if value == "REMOVE" else value for key, value in model_kwargs.items() } - model = Model(model_id=model_id, **model_kwargs) - with pytest.raises(NotImplementedError): - model.generate( - messages=[{"role": "user", "content": "hi"}], - stop_sequences=[""], - ) + model = mlx_model(**model_kwargs) + + model.generate(messages=MESSAGES, stop_sequences=[""]) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == expected -def _bedrock_model(response: dict[str, Any]) -> Any: - from smolagents import AmazonBedrockModel # noqa: PLC0415 - - # A caller-supplied client keeps boto3 out of the test. - return AmazonBedrockModel( - model_id="us.amazon.nova-pro-v1:0", - client=SimpleNamespace(converse=lambda **_: response), - ) - - -BEDROCK_RESPONSE: dict[str, Any] = { - "output": { - "message": { - "role": "assistant", - "content": [{"text": "done"}], - "tool_calls": None, - } - }, - "usage": {"inputTokens": 3, "outputTokens": 2}, - "stopReason": "end_turn", -} - - -def test_bedrock_stop_sequences_are_not_recorded( +def test_stop_sequences_a_model_cannot_send_are_not_recorded( instrument_with_content, span_exporter ) -> None: - # supports_stop_parameter says yes, but the prepared request carries no - # stop sequences, so the span must not claim any either. - model = _bedrock_model(BEDROCK_RESPONSE) - messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] - request = model._prepare_completion_kwargs( # noqa: SLF001 - messages=messages, stop_sequences=[""] - ) - assert model.supports_stop_parameter is True - assert "stop" not in request + # supports_stop_parameter is False for the gpt-5 and o-series names, and + # then _prepare_completion_kwargs never seeds `stop`. The runtime truncates + # the generated text locally instead, so the request carries no stop + # sequences. + model = mlx_model() + model.model_id = "gpt-5" + assert model.supports_stop_parameter is False - model.generate(messages=messages, stop_sequences=[""]) + model.generate(messages=MESSAGES, stop_sequences=[""]) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) is None - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "aws.bedrock" - # Bedrock's "end_turn" is normalized to the semconv "stop". - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) @pytest.mark.parametrize( @@ -614,148 +413,123 @@ def test_max_tokens_covers_both_spellings( call_kwargs: dict[str, Any], expected: int | None, ) -> None: - from smolagents.models import Model # noqa: PLC0415 + model = transformers_model(**model_kwargs) - model = Model(model_id="broken-model", **model_kwargs) - with pytest.raises(NotImplementedError): - model.generate( - messages=[{"role": "user", "content": "hi"}], **call_kwargs - ) + model.generate(messages=MESSAGES, **call_kwargs) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == expected +def test_output_type_recorded_for_a_structured_request( + instrument_with_content, span_exporter, monkeypatch: pytest.MonkeyPatch +) -> None: + # VLLMModel is the only instrumented runtime that accepts a + # response_format; TransformersModel and MLXModel raise before generating. + model = vllm_model(monkeypatch) + + model.generate( + messages=MESSAGES, + response_format={ + "type": "json_schema", + "json_schema": {"schema": {"type": "object"}}, + }, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == "json" + + @pytest.mark.parametrize( - "response_format, model_kwargs, expected", + "response_format, expected", [ - ({"type": "json_object"}, {}, "json"), - ({"type": "json_schema", "json_schema": {}}, {}, "json"), - ({"type": "text"}, {}, "text"), - (None, {}, None), + ({"type": "json_object"}, "json"), + ({"type": "json_schema", "json_schema": {}}, "json"), + ({"type": "text"}, "text"), + (None, None), # smolagents forwards response_format unchanged, so its type is - # whatever the provider accepts. An unknown one is dropped rather than + # whatever the runtime accepts. An unknown one is dropped rather than # recorded on an enum attribute. - ({"type": "xml"}, {}, None), - ({}, {}, None), + ({"type": "xml"}, None), + ({}, None), + ], +) +def test_output_type_mapping( + response_format: dict[str, Any] | None, expected: str | None +) -> None: + # A unit test rather than a generate() call: no instrumented runtime accepts + # every response_format spelling, and the mapping table is what needs + # testing. + merged = ( + {} if response_format is None else {"response_format": response_format} + ) + assert _output_type(merged) == expected + + +@pytest.mark.parametrize( + "model_kwargs, response_format, expected", + [ # The model-level kwargs win over the argument, and the sentinel drops # the key from the request, the same as for every other parameter. ( - {"type": "json_object"}, {"response_format": {"type": "text"}}, + {"type": "json_object"}, "text", ), - ({"type": "json_object"}, {"response_format": "REMOVE"}, None), - (None, {"response_format": {"type": "json_object"}}, "json"), + ({"response_format": "REMOVE"}, {"type": "json_object"}, None), + ({}, {"type": "json_object"}, "json"), ], ) -def test_output_type_follows_the_response_format( - instrument_with_content, - span_exporter, - response_format: dict[str, Any] | None, +def test_response_format_precedence( model_kwargs: dict[str, Any], + response_format: dict[str, Any], expected: str | None, ) -> None: - from smolagents.models import ( # noqa: PLC0415 - REMOVE_PARAMETER, - Model, - ) + from smolagents.models import REMOVE_PARAMETER model_kwargs = { key: REMOVE_PARAMETER if value == "REMOVE" else value for key, value in model_kwargs.items() } - model = Model(model_id="broken-model", **model_kwargs) - with pytest.raises(NotImplementedError): - model.generate( - messages=[{"role": "user", "content": "hi"}], - response_format=response_format, - ) - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == expected - - -def test_user_subclass_of_a_patched_model_keeps_its_provider( - instrument_with_content, span_exporter, metric_reader -) -> None: - # A subclass inherits the patched generate, so it is instrumented; resolving - # the provider by exact class name would report "unknown" on both the span - # and the metrics. - class TenantOpenAIModel(OpenAIModel): - pass - - model = TenantOpenAIModel( - model_id="gpt-4o", - api_key="test_openai_api_key", - api_base="http://localhost:11434/v1", + merged = _merged_request_kwargs( + SimpleNamespace(kwargs=model_kwargs), + {"response_format": response_format}, ) - model.client = stub_openai_client("Bonjour") - model.generate(messages=[{"role": "user", "content": "Hi"}]) + assert _output_type(merged) == expected - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" - # A non-default port is part of the endpoint, unlike the HTTPS default. - assert attr(span, server_attributes.SERVER_ADDRESS) == "localhost" - assert attr(span, server_attributes.SERVER_PORT) == 11434 - duration = metrics_by_name(metric_reader)[ - gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION - ] - assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( - "openai" - ) +def _drain_stream(model: Any, **kwargs: Any) -> list[Any]: + return list(model.generate_stream(messages=MESSAGES, **kwargs)) -def test_provider_error_is_recorded_and_reraised( - instrument_with_content, span_exporter -) -> None: - model = openai_model() - model.client = stub_openai_client( - "", error=ConnectionError("connection reset") - ) - with pytest.raises(ConnectionError, match="connection reset"): - model.generate(messages=[{"role": "user", "content": "Hi"}]) - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert span.status.status_code == StatusCode.ERROR - assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None - - -def _drain_stream(model: OpenAIModel, **kwargs: Any) -> list[Any]: - return list( - model.generate_stream( - messages=[{"role": "user", "content": "Hi"}], **kwargs - ) - ) +def _failing_streamer( + chunks: list[str], error: Exception +) -> Generator[str, None, None]: + yield from chunks + raise error def test_generate_stream_is_lazy_and_records_the_drained_response( instrument_with_content, span_exporter ) -> None: - model = openai_model() - model.client = stub_streaming_openai_client( - [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] - ) + model = transformers_model(stream_chunks=["In ", "Paris"]) - stream = model.generate_stream( - messages=[{"role": "user", "content": "Hi"}] - ) + stream = model.generate_stream(messages=MESSAGES) # A streamed response isn't finished until the caller drains it. assert spans_by_operation(span_exporter.get_finished_spans(), "chat") == [] deltas = list(stream) - assert "".join(delta.content or "" for delta in deltas) == "Bonjour" + assert "".join(delta.content or "" for delta in deltas) == "In Paris" (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" - assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" assert attr(span, GenAI.GEN_AI_REQUEST_STREAM) is True + # TransformersModel reports the prompt tokens on the first delta only and + # one output token per delta. assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) - assert outputs[0]["parts"] == [{"type": "text", "content": "Bonjour"}] + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] # Deltas carry no finish reason, and "stop" would hide a truncation. assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None @@ -764,71 +538,49 @@ def test_generate_stream_stays_a_generator( instrument_with_content, span_exporter ) -> None: # Instrumentation observes; it must not change what generate_stream returns. - model = openai_model() - model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + model = transformers_model(stream_chunks=["In Paris"]) - stream = model.generate_stream( - messages=[{"role": "user", "content": "Hi"}] - ) + stream = model.generate_stream(messages=MESSAGES) assert isinstance(stream, Generator) assert inspect.isgenerator(stream) list(stream) -def test_generate_stream_accumulates_tool_calls( +def test_generate_stream_bad_call_emits_no_span( instrument_with_content, span_exporter ) -> None: - model = openai_model() - model.client = stub_streaming_openai_client( - [ - tool_call_chunk(0, call_id="call_1", name="get_weather"), - tool_call_chunk(0, arguments='{"location":'), - tool_call_chunk(0, arguments='"Paris"}'), - ] - ) + model = transformers_model(stream_chunks=["In Paris"]) - _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) + # messages is required, so the call fails before it reaches the runtime. + with pytest.raises(TypeError): + model.generate_stream() - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - (part,) = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0]["parts"] - assert part == { - "type": "tool_call", - "id": "call_1", - "name": "get_weather", - "arguments": '{"location":"Paris"}', - } - # Tool calls are the only evidence of why a streamed generation stopped. - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + assert span_exporter.get_finished_spans() == () def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( instrument_with_content, span_exporter ) -> None: - model = openai_model() - model.client = stub_streaming_openai_client( - [text_chunk("Bon")], error=ConnectionError("stream died") - ) + model = transformers_model() + model.streamer = _failing_streamer(["In "], RuntimeError("kernel crashed")) - with pytest.raises(ConnectionError, match="stream died"): + with pytest.raises(RuntimeError, match="kernel crashed"): _drain_stream(model) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert span.status.status_code == StatusCode.ERROR - assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + assert attr(span, error_attributes.ERROR_TYPE) == "RuntimeError" # What was streamed before the failure is still recorded. outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) - assert outputs[0]["parts"] == [{"type": "text", "content": "Bon"}] + assert outputs[0]["parts"] == [{"type": "text", "content": "In "}] def test_generate_stream_close_before_drain_finalizes_once( instrument_with_content, span_exporter ) -> None: - model = openai_model() - model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + model = transformers_model(stream_chunks=["In Paris"]) - stream = model.generate_stream( - messages=[{"role": "user", "content": "Hi"}] - ) + stream = model.generate_stream(messages=MESSAGES) stream.close() stream.close() # idempotent @@ -840,10 +592,7 @@ def test_generate_stream_close_before_drain_finalizes_once( def test_generate_stream_records_chunk_metrics( instrument_with_content, span_exporter, metric_reader ) -> None: - model = openai_model() - model.client = stub_streaming_openai_client( - [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] - ) + model = transformers_model(stream_chunks=["In ", "Paris"]) _drain_stream(model) @@ -858,53 +607,15 @@ def test_generate_stream_records_chunk_metrics( ) -def test_generate_stream_no_content( - instrument_no_content, span_exporter -) -> None: - model = openai_model() - model.client = stub_streaming_openai_client( - [ - text_chunk("Bonjour"), - tool_call_chunk( - 0, call_id="call_1", name="get_weather", arguments="{}" - ), - usage_chunk(3, 2), - ] - ) +def test_generate_stream_no_content(instrument_no_content, span_exporter): + model = transformers_model(stream_chunks=["In ", "Paris"]) - _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) + _drain_stream(model) (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 - # A tool call still drives the finish reason without its name or arguments. - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) - - -def test_event_only_content_capture( - instrument_event_only, span_exporter, log_exporter -) -> None: - model = openai_model() - model.client = stub_openai_client("Bonjour") - model.generate(messages=[{"role": "user", "content": "Hi"}]) - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None - assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None - # Metadata still goes on the span. - assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 - - (log,) = log_exporter.get_finished_logs() - record = log.log_record - assert record.event_name == "gen_ai.client.inference.operation.details" - # Event attributes carry the messages as structured values, not JSON text. - attributes = record.attributes or {} - inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] - outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] - assert inputs[0]["parts"][0]["content"] == "Hi" - assert outputs[0]["parts"][0]["content"] == "Bonjour" - assert outputs[0]["finish_reason"] == "stop" def test_to_tool_definitions_uses_json_schema() -> None: @@ -955,6 +666,8 @@ def test_to_input_messages_dict_and_chatmessage() -> None: def test_to_input_messages_image_and_base64() -> None: + # A vision-capable runtime (TransformersModel with a processor, VLLMModel + # with _is_vlm) takes image parts alongside the text. messages = to_input_messages( [ { @@ -1009,58 +722,15 @@ def test_to_input_messages_drops_malformed_base64_image(image: str) -> None: assert messages[0].parts == [] -def test_to_output_message_text_reasoning_and_tool_calls() -> None: - class _Msg: - role = MessageRole.ASSISTANT - content = "The answer" - tool_calls = [ - ChatMessageToolCall( - id="call_1", - type="function", - function=ChatMessageToolCallFunction( - name="get_weather", arguments='{"location": "Paris"}' - ), - ) - ] - - class raw: # noqa: N801 - class _Choice: - class message: # noqa: N801 - reasoning_content = "thinking about it" - - choices = [_Choice()] - - output = to_output_message(_Msg()) - assert output.role == "assistant" - assert Text(content="The answer") in output.parts - assert Reasoning(content="thinking about it") in output.parts - tool_calls = [p for p in output.parts if isinstance(p, ToolCallRequest)] - assert tool_calls[0].name == "get_weather" - assert tool_calls[0].id == "call_1" - assert output.finish_reason == "tool_calls" - - -def test_to_output_message_from_a_dict_raw_response() -> None: - # AmazonBedrockModel, TransformersModel, VLLMModel and MLXModel put a plain - # dict on ChatMessage.raw rather than an SDK object. - message = ChatMessage( - role=MessageRole.ASSISTANT, - content="done", - raw={"stopReason": "max_tokens"}, - ) - output = to_output_message(message) - assert output.parts == [Text(content="done")] - # "max_tokens" is Bedrock's stopReason spelling for a length cutoff. - assert output.finish_reason == "length" - assert response_id(message) is None - assert response_model_name(message) is None - - def test_to_output_message_unwraps_the_role_enum() -> None: output = to_output_message( ChatMessage(role=MessageRole.ASSISTANT, content="done") ) assert output.role == "assistant" + assert output.parts == [Text(content="done")] + # The in-process runtimes report no finish reason, and util-genai omits the + # empty value. + assert output.finish_reason == "" def test_to_output_message_maps_image_content() -> None: @@ -1072,7 +742,6 @@ def test_to_output_message_maps_image_content() -> None: content=[ {"type": "text", "text": "Here it is"}, {"type": "image_url", "image_url": {"url": IMAGE_URL}}, - {"type": "image", "image": "aVZCT1J3MEtHZ28="}, ], ) ) @@ -1080,227 +749,3 @@ def test_to_output_message_maps_image_content() -> None: assert output.parts[1] == Uri( mime_type=None, modality="image", uri=IMAGE_URL ) - blob = output.parts[2] - assert isinstance(blob, Blob) - assert blob.mime_type == "image/png" - assert blob.modality == "image" - - -LOCAL_RUNTIME_RAW = { - "out": "done", - "completion_kwargs": {"max_new_tokens": 4096}, -} - - -@pytest.mark.parametrize( - "raw, tool_calls, expected", - [ - # API-backed models: the provider's own value, whatever it is. - ( - SimpleNamespace(choices=[SimpleNamespace(finish_reason="stop")]), - [], - "stop", - ), - ( - SimpleNamespace(choices=[SimpleNamespace(finish_reason="length")]), - [], - "length", - ), - # Bedrock's stopReason values map onto the semconv vocabulary. - ({"stopReason": "max_tokens"}, [], "length"), - ({"stopReason": "tool_use"}, [], "tool_calls"), - # An unmapped value passes through rather than being guessed at. - ({"stopReason": "guardrail_intervened"}, [], "guardrail_intervened"), - # The local runtimes report no reason, so none is recorded and - # util-genai omits the empty value. - (LOCAL_RUNTIME_RAW, [], ""), - (None, [], ""), - # Tool calls in the response give the reason without guessing. - ( - LOCAL_RUNTIME_RAW, - [ - ChatMessageToolCall( - id="call_1", - type="function", - function=ChatMessageToolCallFunction( - name="get_weather", arguments="{}" - ), - ) - ], - "tool_calls", - ), - ], -) -def test_finish_reason_follows_the_provider_response( - raw: Any, tool_calls: list[ChatMessageToolCall], expected: str -) -> None: - message = ChatMessage( - role=MessageRole.ASSISTANT, - content="done", - tool_calls=tool_calls or None, - raw=raw, - ) - assert to_output_message(message).finish_reason == expected - - -def test_model_reporting_no_finish_reason_omits_the_attribute( - instrument_with_content, span_exporter -) -> None: - # InferenceClientModel is a patched class whose provider response can come - # back without a finish reason; the span must then carry none. - from smolagents import InferenceClientModel # noqa: PLC0415 - - model = InferenceClientModel( - model_id="Qwen/Qwen2.5-Coder-32B-Instruct", token="hf_test" - ) - model.client = SimpleNamespace( - chat_completion=lambda **_: SimpleNamespace( - id="hf-1", - model="Qwen/Qwen2.5-Coder-32B-Instruct", - choices=[ - SimpleNamespace( - message=SimpleNamespace( - role="assistant", content="ok", tool_calls=None - ) - ) - ], - usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2), - ) - ) - - model.generate(messages=[{"role": "user", "content": "hi"}]) - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None - outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) - assert outputs[0]["finish_reason"] == "" - - -# The local runtimes flatten a message's content as text, so the content has to -# be a list of parts rather than a bare string. -LOCAL_RUNTIME_MESSAGES: list[dict[str, Any]] = [ - { - "role": "user", - "content": [{"type": "text", "text": "Where is the Louvre?"}], - } -] - - -def _mlx_model(monkeypatch: pytest.MonkeyPatch) -> Any: - """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. - - ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` - over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from - ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real - ``generate``, and the runtime doesn't have to be installed. ``monkeypatch`` - is unused here; it keeps the factory signature uniform with the vllm one, - which does have modules to stub. - """ - from smolagents.models import MLXModel # noqa: PLC0415 - - model = object.__new__(MLXModel) - model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" - model.kwargs = {} - model.flatten_messages_as_text = True - model.apply_chat_template_kwargs = {} - model.model = object() - model.tokenizer = SimpleNamespace( - apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] - ) - model.stream_generate = lambda *_, **__: iter( - [SimpleNamespace(text="In "), SimpleNamespace(text="Paris")] - ) - return model - - -def _vllm_model(monkeypatch: pytest.MonkeyPatch) -> Any: - """A ``VLLMModel`` with ``vllm`` itself stubbed. - - ``VLLMModel.generate`` imports ``SamplingParams`` and - ``StructuredOutputsParams`` from ``vllm`` when it runs, and neither test env - installs vllm, so both modules are faked for the duration of the test. - Everything the wrapper reads still comes from the real ``generate``. - """ - from smolagents.models import VLLMModel # noqa: PLC0415 - - def fake_params(**kwargs: Any) -> SimpleNamespace: - return SimpleNamespace(**kwargs) - - vllm = ModuleType("vllm") - sampling_params = ModuleType("vllm.sampling_params") - setattr(vllm, "SamplingParams", fake_params) - setattr(sampling_params, "StructuredOutputsParams", fake_params) - setattr(vllm, "sampling_params", sampling_params) - monkeypatch.setitem(sys.modules, "vllm", vllm) - monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) - - completion = SimpleNamespace( - prompt_token_ids=[1, 2, 3, 4], - outputs=[SimpleNamespace(text="In Paris", token_ids=[5, 6])], - ) - model = object.__new__(VLLMModel) - model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" - model.kwargs = {} - model.flatten_messages_as_text = True - model._is_vlm = False - model.apply_chat_template_kwargs = {} - model.tokenizer = SimpleNamespace( - apply_chat_template=lambda messages, **_: "prompt" - ) - model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) - return model - - -@pytest.mark.parametrize( - "model_factory, provider, request_model, input_tokens, output_tokens", - [ - ( - _mlx_model, - "mlx", - "mlx-community/Qwen2.5-0.5B-Instruct-4bit", - 3, - 2, - ), - (_vllm_model, "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), - ], - ids=["mlx", "vllm"], -) -def test_local_runtime_response_is_recorded( - instrument_with_content, - span_exporter, - monkeypatch: pytest.MonkeyPatch, - model_factory: Any, - provider: str, - request_model: str, - input_tokens: int, - output_tokens: int, -) -> None: - output = model_factory(monkeypatch).generate( - messages=LOCAL_RUNTIME_MESSAGES - ) - assert output.content == "In Paris" - - (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") - assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider - assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model - assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens - assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens - # A local runtime returns no provider response envelope and listens on no - # socket, so it reports no finish reason, id or response model, and there is - # no endpoint to derive server.address from. - assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None - assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None - assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None - assert attr(span, server_attributes.SERVER_ADDRESS) is None - assert attr(span, server_attributes.SERVER_PORT) is None - assert span.status.status_code == StatusCode.UNSET - - inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) - assert inputs[0]["parts"] == [ - {"type": "text", "content": "Where is the Louvre?"} - ] - outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) - assert outputs[0]["role"] == "assistant" - assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] - assert outputs[0]["finish_reason"] == "" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py index eeb8d78e7..efec5fadb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -1,15 +1,25 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Shared helpers, tools, and model stubs for smolagents instrumentation tests.""" +"""Shared helpers, tools, and model stubs for smolagents instrumentation tests. + +Only the in-process model classes are instrumented, and none of them can be +recorded with VCR: they run inference in the current process instead of calling +a provider over HTTP. Each factory here bypasses ``__init__`` (which would load +gigabytes of weights) and stubs the runtime pieces the real ``generate`` drives, +so the code under test is smolagents' own ``generate`` and the wrapper around +it. +""" from __future__ import annotations import json -from types import SimpleNamespace +import sys +from types import ModuleType, SimpleNamespace from typing import Any -from smolagents import OpenAIModel, Tool +import pytest +from smolagents import Tool from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.semconv._incubating.attributes import ( @@ -32,154 +42,153 @@ def forward(self, location: str) -> str: return "sunny" -def openai_model() -> OpenAIModel: - return OpenAIModel( - model_id="gpt-4o", - api_key="test_openai_api_key", - api_base="https://api.openai.com/v1", - ) +# The in-process runtimes flatten a message's content as text, so the content +# has to be a list of parts rather than a bare string. +MESSAGES: list[dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": "Where is the Louvre?"}], + } +] -def stub_openai_client( - content: str, finish_reason: str = "stop", error: Exception | None = None -) -> Any: - """An object shaped like the bits of ``openai.OpenAI`` that ``generate`` uses. +class _PromptTokens: + """The part of a torch tensor that ``TransformersModel.generate`` uses. - Building a real ``ChatCompletion`` keeps the response shape honest (the - wrapper reads ``raw.model``, ``raw.id``, and ``raw.choices[0].finish_reason``) - without needing a cassette for a deployment we can't record against. + It reads ``inputs.shape[1]`` for the prompt length and slices the generated + tail out of the model's output with ``out[0, prompt_length:]``. Moving to a + device and the ``input_ids`` unwrap are part of the same path. """ - from openai.types.chat import ( # noqa: PLC0415 - ChatCompletion, - ChatCompletionMessage, - ) - from openai.types.chat.chat_completion import Choice # noqa: PLC0415 - from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 - - completion = ChatCompletion( - id="chatcmpl-stub", - model="gpt-4o-2024-08-06", - object="chat.completion", - created=0, - choices=[ - Choice( - index=0, - finish_reason=finish_reason, - message=ChatCompletionMessage( - role="assistant", content=content - ), - ) - ], - usage=CompletionUsage( - prompt_tokens=3, completion_tokens=1, total_tokens=4 - ), - ) - def create(**_: Any) -> ChatCompletion: - if error is not None: - raise error - return completion + def __init__(self, ids: list[int]) -> None: + self.ids = ids - return SimpleNamespace( - chat=SimpleNamespace(completions=SimpleNamespace(create=create)) - ) + @property + def shape(self) -> tuple[int, int]: + return (1, len(self.ids)) + + def to(self, _device: Any) -> _PromptTokens: + return self + def __getitem__(self, key: Any) -> list[int]: + _row, columns = key + return self.ids[columns] -def stub_streaming_openai_client( - chunks: list[Any], error: Exception | None = None + +def transformers_model( + prompt_ids: list[int] | None = None, + generated_ids: list[int] | None = None, + text: str = "In Paris", + error: Exception | None = None, + stream_chunks: list[str] | None = None, + **model_kwargs: Any, ) -> Any: - """An ``openai.OpenAI`` stand-in whose ``create`` returns a chunk stream. + """A ``TransformersModel`` whose tokenizer and weights are stubbed. - ``OpenAIModel.generate_stream`` reads ``event.usage`` and - ``event.choices[0].delta``, so the chunks are real ``ChatCompletionChunk`` - objects. ``error`` is raised after the chunks are yielded, which is how a - provider failure part-way through a stream reaches the caller. + ``generate`` builds the prompt through ``self.tokenizer + .apply_chat_template``, counts its tokens, calls ``self.model.generate`` and + decodes the tail. ``generate_stream`` instead runs ``self.model.generate`` + on a thread and iterates ``self.streamer``, so the stubbed streamer is what + yields the deltas. """ + from smolagents.models import TransformersModel - def create(**_: Any) -> Any: - def stream() -> Any: - yield from chunks - if error is not None: - raise error - - return stream() + prompt_ids = prompt_ids or [1, 2, 3] + generated_ids = generated_ids or [4, 5] - return SimpleNamespace( - chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + def generate(**_: Any) -> Any: + if error is not None: + raise error + return _PromptTokens(prompt_ids + generated_ids) + + model = object.__new__(TransformersModel) + model.model_id = "HuggingFaceTB/SmolLM2-135M-Instruct" + model.kwargs = dict(model_kwargs) + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: _PromptTokens(prompt_ids), + decode=lambda ids, **_: text, ) + model.model = SimpleNamespace(device="cpu", generate=generate) + model.streamer = iter(stream_chunks or []) + return model -def text_chunk(content: str) -> Any: - from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 - from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 - Choice, - ChoiceDelta, - ) +def mlx_model(text: str = "In Paris", **model_kwargs: Any) -> Any: + """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. - return ChatCompletionChunk( - id="chatcmpl-stub", - model="gpt-4o-2024-08-06", - object="chat.completion.chunk", - created=0, - choices=[Choice(index=0, delta=ChoiceDelta(content=content))], + ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` + over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from + ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real + ``generate`` without the runtime installed. + """ + from smolagents.models import MLXModel + + model = object.__new__(MLXModel) + model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" + model.kwargs = dict(model_kwargs) + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.model = object() + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] ) + # ``generate`` counts one output token per delta, so the deltas are the + # words of ``text`` and the output token count is the word count. + words = text.split(" ") + deltas = [ + SimpleNamespace(text=word if index == 0 else f" {word}") + for index, word in enumerate(words) + ] + model.stream_generate = lambda *_, **__: iter(deltas) + return model -def tool_call_chunk( - index: int, - call_id: str | None = None, - name: str | None = None, - arguments: str | None = None, +def vllm_model( + monkeypatch: pytest.MonkeyPatch, + text: str = "In Paris", + prompt_token_ids: list[int] | None = None, + output_token_ids: list[int] | None = None, + **model_kwargs: Any, ) -> Any: - from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 - from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 - Choice, - ChoiceDelta, - ChoiceDeltaToolCall, - ChoiceDeltaToolCallFunction, - ) + """A ``VLLMModel`` with ``vllm`` itself stubbed. - return ChatCompletionChunk( - id="chatcmpl-stub", - model="gpt-4o-2024-08-06", - object="chat.completion.chunk", - created=0, - choices=[ - Choice( - index=0, - delta=ChoiceDelta( - tool_calls=[ - ChoiceDeltaToolCall( - index=index, - id=call_id, - type="function", - function=ChoiceDeltaToolCallFunction( - name=name, arguments=arguments - ), - ) - ] - ), - ) + ``VLLMModel.generate`` imports ``SamplingParams`` and + ``StructuredOutputsParams`` from ``vllm`` when it runs, and no test env + installs vllm, so both modules are faked for the duration of the test. + Everything the wrapper reads still comes from the real ``generate``. + """ + from smolagents.models import VLLMModel + + def fake_params(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) + + vllm = ModuleType("vllm") + sampling_params = ModuleType("vllm.sampling_params") + setattr(vllm, "SamplingParams", fake_params) + setattr(sampling_params, "StructuredOutputsParams", fake_params) + setattr(vllm, "sampling_params", sampling_params) + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + + completion = SimpleNamespace( + prompt_token_ids=prompt_token_ids or [1, 2, 3, 4], + outputs=[ + SimpleNamespace(text=text, token_ids=output_token_ids or [5, 6]) ], ) - - -def usage_chunk(prompt_tokens: int, completion_tokens: int) -> Any: - from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 - from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 - - return ChatCompletionChunk( - id="chatcmpl-stub", - model="gpt-4o-2024-08-06", - object="chat.completion.chunk", - created=0, - choices=[], - usage=CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ), + model = object.__new__(VLLMModel) + model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" + model.kwargs = dict(model_kwargs) + model.flatten_messages_as_text = True + model._is_vlm = False + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: "prompt" ) + model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) + return model def spans_by_operation( From 1bdbe04aaf2dffaaa8c24f39328c8801cd412c0c Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Fri, 14 Aug 2026 23:04:19 +0200 Subject: [PATCH 05/11] fix types --- .../genai/smolagents/_messages.py | 71 +++++++++++-------- .../instrumentation/genai/smolagents/patch.py | 64 +++++++++-------- .../genai/smolagents/provider.py | 7 +- .../tests/test_models.py | 3 +- 4 files changed, 83 insertions(+), 62 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index d0bb17152..4e25c0d85 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -17,7 +17,7 @@ import binascii import logging from enum import Enum -from typing import Any +from typing import TYPE_CHECKING, Any from opentelemetry.util.genai.types import ( Blob, @@ -30,6 +30,11 @@ Uri, ) +if TYPE_CHECKING: + from PIL.Image import Image + from smolagents.models import ChatMessage, MessageRole + from smolagents.tools import Tool + _logger = logging.getLogger(__name__) _DEFAULT_IMAGE_MIME_TYPE = "image/png" @@ -48,7 +53,7 @@ } -def _unwrap_role(role: Any) -> str | None: +def _unwrap_role(role: MessageRole | str | None) -> str | None: if role is None: return None if isinstance(role, Enum): @@ -72,7 +77,7 @@ def _decode_base64_image(image: str) -> tuple[bytes, str] | None: return None -def _encode_image_base64(image: Any) -> str | None: +def _encode_image_base64(image: Image) -> str | None: try: from smolagents.utils import ( # pylint: disable=import-outside-toplevel encode_image_base64, @@ -92,7 +97,7 @@ def _encode_image_base64(image: Any) -> str | None: return encoded if isinstance(encoded, str) else None -def _image_blob(image: Any) -> Blob | None: +def _image_blob(image: Image | str) -> Blob | None: """Build a ``Blob`` part from a base64 string, data URL, or PIL image.""" if isinstance(image, str): decoded = _decode_base64_image(image) @@ -122,7 +127,9 @@ def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: return None -def _parts_from_content(content: Any) -> list[MessagePart]: +def _parts_from_content( + content: str | list[dict[str, Any]] | None, +) -> list[MessagePart]: parts: list[MessagePart] = [] if isinstance(content, str): parts.append(Text(content=content)) @@ -148,13 +155,19 @@ def _parts_from_content(content: Any) -> list[MessagePart]: return parts -def _get_role_and_content(message: Any) -> tuple[Any, Any]: +def _get_role_and_content( + message: ChatMessage | dict[str, Any], +) -> tuple[MessageRole | str | None, str | list[dict[str, Any]] | None]: + # smolagents reads a message the same way: a dict goes through + # ChatMessage.from_dict, anything else has its attributes read directly. if isinstance(message, dict): return message.get("role"), message.get("content") - return getattr(message, "role", None), getattr(message, "content", None) + return message.role, message.content -def to_input_messages(messages: Any) -> list[InputMessage]: +def to_input_messages( + messages: list[ChatMessage | dict[str, Any]] | None, +) -> list[InputMessage]: """Map smolagents ``generate`` input messages to ``InputMessage`` objects.""" result: list[InputMessage] = [] if not isinstance(messages, list): @@ -170,7 +183,7 @@ def to_input_messages(messages: Any) -> list[InputMessage]: return result -def to_output_message(output_message: Any) -> OutputMessage: +def to_output_message(output_message: ChatMessage) -> OutputMessage: """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``. The in-process runtimes return the generated text and nothing else: no tool @@ -181,12 +194,12 @@ def to_output_message(output_message: Any) -> OutputMessage: would make a generation cut short by ``max_new_tokens`` look like a natural stop. """ - role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" - parts = _parts_from_content(getattr(output_message, "content", None)) + role = _unwrap_role(output_message.role) or "assistant" + parts = _parts_from_content(output_message.content) return OutputMessage(role=role, parts=parts, finish_reason="") -def _tool_parameters(tool: Any) -> dict[str, Any] | None: +def _tool_parameters(tool: Tool) -> dict[str, Any] | None: """Return the JSON Schema ``parameters`` object for a smolagents tool. A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it @@ -207,27 +220,29 @@ def _tool_parameters(tool: Any) -> dict[str, Any] | None: except Exception: # pylint: disable=broad-except _logger.debug( "Failed to build a JSON Schema for tool %s", - getattr(tool, "name", None), + tool.name, exc_info=True, ) return None return parameters if isinstance(parameters, dict) else None -def to_tool_definitions(tools: Any) -> list[ToolDefinition] | None: - """Map smolagents tool objects to function tool definitions.""" - if not isinstance(tools, list) or not tools: +def to_tool_definitions( + tools: list[Tool] | None, +) -> list[ToolDefinition] | None: + """Map smolagents tool objects to function tool definitions. + + ``Tool.validate_arguments`` runs on every instantiation and requires a + non-empty ``name`` and a ``description``, so both are read directly. + """ + if not tools: return None - definitions: list[ToolDefinition] = [] - for tool in tools: - name = getattr(tool, "name", None) - if not name: - continue - definitions.append( - FunctionToolDefinition( - name=name, - description=getattr(tool, "description", None), - parameters=_tool_parameters(tool), - ) + definitions: list[ToolDefinition] = [ + FunctionToolDefinition( + name=tool.name, + description=tool.description, + parameters=_tool_parameters(tool), ) - return definitions or None + for tool in tools + ] + return definitions diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index e6fb218f6..19cee905c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -20,7 +20,7 @@ import logging from collections.abc import Callable, Generator, Mapping from inspect import signature -from typing import Any +from typing import TYPE_CHECKING, Any, TypeAlias from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, @@ -37,10 +37,19 @@ ) from .provider import resolve_provider +if TYPE_CHECKING: + from smolagents.models import ( + ChatMessage, + ChatMessageStreamDelta, + Model, + ) + _logger = logging.getLogger(__name__) -_Wrapper = Callable[ - [Callable[..., Any], Any, tuple[Any, ...], dict[str, Any]], Any +# ``Model`` is quoted because a type alias is evaluated at runtime, unlike an +# annotation. +_Wrapper: TypeAlias = Callable[ + [Callable[..., Any], "Model", tuple[Any, ...], dict[str, Any]], Any ] @@ -69,7 +78,7 @@ def _bind_arguments( return dict(kwargs) -def _coerce_float(value: Any) -> float | None: +def _coerce_float(value: object) -> float | None: if isinstance(value, bool): return None if isinstance(value, (int, float)): @@ -77,7 +86,7 @@ def _coerce_float(value: Any) -> float | None: return None -def _coerce_int(value: Any) -> int | None: +def _coerce_int(value: object) -> int | None: if isinstance(value, bool): return None if isinstance(value, int): @@ -85,7 +94,7 @@ def _coerce_int(value: Any) -> int | None: return None -def _remove_parameter_sentinel() -> Any: +def _remove_parameter_sentinel() -> object | None: """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" try: from smolagents.models import ( # pylint: disable=import-outside-toplevel @@ -98,7 +107,7 @@ def _remove_parameter_sentinel() -> Any: def _merged_request_kwargs( - instance: Any, bound: dict[str, Any] + instance: Model, bound: dict[str, Any] ) -> dict[str, Any]: """Rebuild the request keyword arguments smolagents will send. @@ -112,9 +121,7 @@ def _merged_request_kwargs( """ merged: dict[str, Any] = {} stop_sequences = bound.get("stop_sequences") - if stop_sequences is not None and getattr( - instance, "supports_stop_parameter", False - ): + if stop_sequences is not None and instance.supports_stop_parameter: merged["stop"] = stop_sequences response_format = bound.get("response_format") if response_format is not None: @@ -122,11 +129,8 @@ def _merged_request_kwargs( call_kwargs = bound.get("kwargs") if isinstance(call_kwargs, dict): merged.update(call_kwargs) - model_kwargs = getattr(instance, "kwargs", None) - if not isinstance(model_kwargs, dict): - return merged remove = _remove_parameter_sentinel() - for name, value in model_kwargs.items(): + for name, value in instance.kwargs.items(): if remove is not None and value is remove: merged.pop(name, None) else: @@ -182,7 +186,7 @@ def _output_type(merged: dict[str, Any]) -> str | None: def _apply_request_parameters( - invocation: InferenceInvocation, instance: Any, bound: dict[str, Any] + invocation: InferenceInvocation, instance: Model, bound: dict[str, Any] ) -> None: """Copy the request parameters smolagents will send onto the span.""" merged = _merged_request_kwargs(instance, bound) @@ -205,13 +209,13 @@ def _apply_request_parameters( def _apply_token_usage( - invocation: InferenceInvocation, output_message: Any + invocation: InferenceInvocation, output_message: ChatMessage ) -> None: # ChatMessage.token_usage is the only source: the per-model # last_input_token_count / last_output_token_count counters were removed # before the oldest supported smolagents. The in-process runtimes count the # prompt and generated tokens themselves and report them here. - token_usage = getattr(output_message, "token_usage", None) + token_usage = output_message.token_usage if token_usage is None: return invocation.input_tokens = token_usage.input_tokens @@ -221,7 +225,7 @@ def _apply_token_usage( def _start_inference( handler: TelemetryHandler, wrapped: Callable[..., Any], - instance: Any, + instance: Model, args: tuple[Any, ...], kwargs: dict[str, Any], ) -> InferenceInvocation: @@ -234,7 +238,7 @@ def _start_inference( provider = resolve_provider(instance) invocation = handler.inference( provider, - request_model=getattr(instance, "model_id", None), + request_model=instance.model_id, ) bound = _bind_arguments(wrapped, args, kwargs) _apply_request_parameters(invocation, instance, bound) @@ -256,11 +260,11 @@ def model_generate(handler: TelemetryHandler) -> _Wrapper: """ def wrapper( - wrapped: Callable[..., Any], - instance: Any, + wrapped: Callable[..., ChatMessage], + instance: Model, args: tuple[Any, ...], kwargs: dict[str, Any], - ) -> Any: + ) -> ChatMessage: invocation = _start_inference(handler, wrapped, instance, args, kwargs) with invocation: output_message = wrapped(*args, **kwargs) @@ -274,7 +278,7 @@ def wrapper( return wrapper -class _ModelStreamWrapper(SyncStreamWrapper[Any]): +class _ModelStreamWrapper(SyncStreamWrapper["ChatMessageStreamDelta"]): """Keep the ``chat`` span open until the delta stream is drained. Passing the invocation to ``super().__init__()`` turns on @@ -287,7 +291,7 @@ class _ModelStreamWrapper(SyncStreamWrapper[Any]): def __init__( self, - stream: Generator[Any, Any, Any], + stream: Generator[ChatMessageStreamDelta, None, None], invocation: InferenceInvocation, handler: TelemetryHandler, ) -> None: @@ -299,11 +303,11 @@ def __init__( self._self_output_tokens = 0 self._self_saw_token_usage = False - def _process_chunk(self, chunk: Any) -> None: - content = getattr(chunk, "content", None) + def _process_chunk(self, chunk: ChatMessageStreamDelta) -> None: + content = chunk.content if content and self._self_capture_content: self._self_content.append(content) - token_usage = getattr(chunk, "token_usage", None) + token_usage = chunk.token_usage if token_usage is not None: # Summed like agglomerate_stream_deltas, so the span agrees with the # totals the agent's monitor reports. @@ -352,11 +356,11 @@ def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: """ def wrapper( - wrapped: Callable[..., Any], - instance: Any, + wrapped: Callable[..., Generator[ChatMessageStreamDelta, None, None]], + instance: Model, args: tuple[Any, ...], kwargs: dict[str, Any], - ) -> Any: + ) -> _ModelStreamWrapper: invocation = _start_inference(handler, wrapped, instance, args, kwargs) stream = wrapped(*args, **kwargs) return _ModelStreamWrapper(stream, invocation, handler) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py index d640bb8ae..9c05d7c57 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -14,7 +14,10 @@ from __future__ import annotations import logging -from typing import Any +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from smolagents.models import Model _logger = logging.getLogger(__name__) @@ -30,7 +33,7 @@ } -def resolve_provider(instance: Any) -> str: +def resolve_provider(instance: Model) -> str: """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" # An instrumented model can be a user subclass of a patched class. Matching # the exact class name alone would report ``unknown`` for every subclass, so diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py index 93af3e0c9..1a2022624 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -15,7 +15,6 @@ import inspect import json from collections.abc import Generator -from types import SimpleNamespace from typing import Any import pytest @@ -492,7 +491,7 @@ def test_response_format_precedence( for key, value in model_kwargs.items() } merged = _merged_request_kwargs( - SimpleNamespace(kwargs=model_kwargs), + transformers_model(**model_kwargs), {"response_format": response_format}, ) assert _output_type(merged) == expected From a8f8c33f3acf36082c698a302d81e5ee25fd1d6d Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Fri, 14 Aug 2026 23:20:15 +0200 Subject: [PATCH 06/11] move imports to the module level --- .../genai/smolagents/__init__.py | 8 +++---- .../genai/smolagents/_messages.py | 21 +++++-------------- .../instrumentation/genai/smolagents/patch.py | 17 +++------------ 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index de4b7b655..3dcdd78d1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -53,6 +53,7 @@ from collections.abc import Collection from typing import Any +from smolagents import models from wrapt import wrap_function_wrapper from opentelemetry.instrumentation.instrumentor import BaseInstrumentor @@ -89,11 +90,10 @@ def _model_classes_defining(method: str) -> list[type]: A user-defined subclass that overrides the method shadows the patched one and emits no ``chat`` span. ``README.rst`` documents that limitation. - """ - from smolagents import ( # pylint: disable=import-outside-toplevel - models, - ) + A class is looked up by name so that a smolagents version without one of them + is skipped rather than raising. + """ classes: list[type] = [] for name in _IN_PROCESS_MODEL_CLASSES: model_cls = getattr(models, name, None) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index 4e25c0d85..ce995bbe6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -19,6 +19,9 @@ from enum import Enum from typing import TYPE_CHECKING, Any +from smolagents.models import get_tool_json_schema +from smolagents.utils import encode_image_base64 + from opentelemetry.util.genai.types import ( Blob, FunctionToolDefinition, @@ -77,14 +80,7 @@ def _decode_base64_image(image: str) -> tuple[bytes, str] | None: return None -def _encode_image_base64(image: Image) -> str | None: - try: - from smolagents.utils import ( # pylint: disable=import-outside-toplevel - encode_image_base64, - ) - except ImportError: - _logger.debug("smolagents.utils.encode_image_base64 is unavailable") - return None +def _encode_base64_image(image: Image) -> str | None: try: encoded = encode_image_base64(image) except Exception: # pylint: disable=broad-except @@ -102,7 +98,7 @@ def _image_blob(image: Image | str) -> Blob | None: if isinstance(image, str): decoded = _decode_base64_image(image) else: - encoded = _encode_image_base64(image) + encoded = _encode_base64_image(image) decoded = ( _decode_base64_image(encoded) if encoded is not None else None ) @@ -207,13 +203,6 @@ def _tool_parameters(tool: Tool) -> dict[str, Any] | None: non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the schema the provider receives. """ - try: - from smolagents.models import ( # pylint: disable=import-outside-toplevel - get_tool_json_schema, - ) - except ImportError: - _logger.debug("smolagents.models.get_tool_json_schema is unavailable") - return None try: schema = get_tool_json_schema(tool) parameters = schema["function"]["parameters"] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 19cee905c..d303939ba 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -22,6 +22,8 @@ from inspect import signature from typing import TYPE_CHECKING, Any, TypeAlias +from smolagents.models import REMOVE_PARAMETER + from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, ) @@ -94,18 +96,6 @@ def _coerce_int(value: object) -> int | None: return None -def _remove_parameter_sentinel() -> object | None: - """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" - try: - from smolagents.models import ( # pylint: disable=import-outside-toplevel - REMOVE_PARAMETER, - ) - except ImportError: - _logger.debug("smolagents.models.REMOVE_PARAMETER is unavailable") - return None - return REMOVE_PARAMETER - - def _merged_request_kwargs( instance: Model, bound: dict[str, Any] ) -> dict[str, Any]: @@ -129,9 +119,8 @@ def _merged_request_kwargs( call_kwargs = bound.get("kwargs") if isinstance(call_kwargs, dict): merged.update(call_kwargs) - remove = _remove_parameter_sentinel() for name, value in instance.kwargs.items(): - if remove is not None and value is remove: + if value is REMOVE_PARAMETER: merged.pop(name, None) else: merged[name] = value From fcce48b87a0b3c7bc54e0caa4966e921f193ef1c Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Fri, 14 Aug 2026 23:27:18 +0200 Subject: [PATCH 07/11] fix readme --- .../README.rst | 11 ++++++++++- .../instrumentation/genai/smolagents/__init__.py | 9 ++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index 7c28e0e24..edf44ac65 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -78,7 +78,16 @@ Usage SmolagentsInstrumentor().instrument() model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") - model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + model.generate( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "How many seconds are in a week?"} + ], + } + ] + ) Configuration ------------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index 3dcdd78d1..2a963e043 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -28,7 +28,14 @@ model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") model.generate( - [{"role": "user", "content": "How many seconds are in a week?"}] + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "How many seconds are in a week?"} + ], + } + ] ) Configuration From 387bce5123fe8ce23e474450f9e2d47b56f0ad04 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Fri, 14 Aug 2026 23:35:54 +0200 Subject: [PATCH 08/11] fix type of smolagents content element --- .../instrumentation/genai/smolagents/_messages.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index ce995bbe6..4ae6fdc12 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -17,7 +17,7 @@ import binascii import logging from enum import Enum -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeAlias from smolagents.models import get_tool_json_schema from smolagents.utils import encode_image_base64 @@ -40,6 +40,12 @@ _logger = logging.getLogger(__name__) +# One element of a message ``content`` list, typed as smolagents types it: +# ``ChatMessage.content`` is ``str | list[dict[str, Any]]``. The values within an +# element have mixed types, a ``str`` under ``text``, a nested dict under +# ``image_url``, and a PIL image under ``image``. +_ContentElement: TypeAlias = dict[str, Any] + _DEFAULT_IMAGE_MIME_TYPE = "image/png" _DATA_URL_PREFIX = "data:" @@ -108,7 +114,7 @@ def _image_blob(image: Image | str) -> Blob | None: return Blob(mime_type=mime_type, modality="image", content=content) -def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: +def _image_part_from_element(element: _ContentElement) -> Uri | Blob | None: content_type = element.get("type") if content_type == "image_url": image_url = element.get("image_url") @@ -124,7 +130,7 @@ def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: def _parts_from_content( - content: str | list[dict[str, Any]] | None, + content: str | list[_ContentElement] | None, ) -> list[MessagePart]: parts: list[MessagePart] = [] if isinstance(content, str): @@ -153,7 +159,7 @@ def _parts_from_content( def _get_role_and_content( message: ChatMessage | dict[str, Any], -) -> tuple[MessageRole | str | None, str | list[dict[str, Any]] | None]: +) -> tuple[MessageRole | str | None, str | list[_ContentElement] | None]: # smolagents reads a message the same way: a dict goes through # ChatMessage.from_dict, anything else has its attributes read directly. if isinstance(message, dict): From fedc6b1dbee97dfdfd8b2011c0cedca713cf5f13 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Sat, 15 Aug 2026 19:04:54 +0200 Subject: [PATCH 09/11] fail the invocation when the stream call raises --- .../instrumentation/genai/smolagents/patch.py | 8 ++- .../tests/test_models.py | 52 +++++++++++++++++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index d303939ba..5bbfea9db 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -351,7 +351,11 @@ def wrapper( kwargs: dict[str, Any], ) -> _ModelStreamWrapper: invocation = _start_inference(handler, wrapped, instance, args, kwargs) - stream = wrapped(*args, **kwargs) - return _ModelStreamWrapper(stream, invocation, handler) + try: + stream = wrapped(*args, **kwargs) + return _ModelStreamWrapper(stream, invocation, handler) + except Exception as error: + invocation.fail(error) + raise return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py index 1a2022624..4f9e1e070 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -20,6 +20,7 @@ import pytest from smolagents.models import ChatMessage, MessageRole +from opentelemetry.context import Context from opentelemetry.instrumentation.genai.smolagents._messages import ( to_input_messages, to_output_message, @@ -32,6 +33,7 @@ from opentelemetry.instrumentation.genai.smolagents.provider import ( resolve_provider, ) +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, ) @@ -197,6 +199,46 @@ def test_runtime_error_is_recorded_and_reraised( assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None +class _LifecycleRecorder(SpanProcessor): + """Records span starts and ends. + + The exporter only sees a span once it ends, so an unfinished span reads + there as no span at all. + """ + + def __init__(self) -> None: + self.started: list[str] = [] + self.ended: list[str] = [] + + def on_start( + self, span: Span, parent_context: Context | None = None + ) -> None: + self.started.append(span.name) + + def on_end(self, span: ReadableSpan) -> None: + self.ended.append(span.name) + + @property + def leaked(self) -> list[str]: + return self.started[len(self.ended) :] + + +@pytest.fixture +def lifecycle(tracer_provider) -> _LifecycleRecorder: + recorder = _LifecycleRecorder() + tracer_provider.add_span_processor(recorder) + return recorder + + +def test_lifecycle_recorder_sees_the_happy_path( + instrument_with_content, lifecycle +) -> None: + transformers_model().generate(messages=MESSAGES) + + assert len(lifecycle.started) == 1 + assert lifecycle.leaked == [] + + def test_tool_definitions_recorded( instrument_with_content, span_exporter ) -> None: @@ -545,16 +587,20 @@ def test_generate_stream_stays_a_generator( list(stream) -def test_generate_stream_bad_call_emits_no_span( - instrument_with_content, span_exporter +def test_generate_stream_bad_call_records_an_error_span( + instrument_with_content, span_exporter, lifecycle ) -> None: model = transformers_model(stream_chunks=["In Paris"]) # messages is required, so the call fails before it reaches the runtime. + # Nothing will drain the stream, so the span has to end here. with pytest.raises(TypeError): model.generate_stream() - assert span_exporter.get_finished_spans() == () + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "TypeError" + assert lifecycle.leaked == [] def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( From 58f3072547ba3b6ee722bc6ddf099f853a2784a8 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Sat, 15 Aug 2026 19:06:11 +0200 Subject: [PATCH 10/11] do not fail the model call from telemetry code --- .../instrumentation/genai/smolagents/patch.py | 61 +++++++++++++---- .../tests/test_models.py | 65 +++++++++++++++++++ 2 files changed, 114 insertions(+), 12 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 5bbfea9db..ff14ff493 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -211,6 +211,53 @@ def _apply_token_usage( invocation.output_tokens = token_usage.output_tokens +def _record_request( + handler: TelemetryHandler, + invocation: InferenceInvocation, + wrapped: Callable[..., Any], + instance: Model, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> None: + """Record the request on the invocation. Extraction errors are dropped. + + The messages, tools and keyword arguments come from the caller, so the + conversion can get a shape it does not handle. The span is already + started and the model has not been called yet, so an error raised here + would both break the call and leave the span unfinished. + """ + try: + bound = _bind_arguments(wrapped, args, kwargs) + _apply_request_parameters(invocation, instance, bound) + invocation.tool_definitions = to_tool_definitions( + bound.get("tools_to_call_from") + ) + if handler.should_capture_content(): + invocation.input_messages = to_input_messages( + bound.get("messages") + ) + except Exception: # pylint: disable=broad-except + _logger.debug("Failed to record the request", exc_info=True) + + +def _record_response( + handler: TelemetryHandler, + invocation: InferenceInvocation, + output_message: ChatMessage, +) -> None: + """Record the response on the invocation. Extraction errors are dropped. + + The model call has already succeeded at this point, so an error raised + here would turn a completed call into a failed one. + """ + try: + _apply_token_usage(invocation, output_message) + if handler.should_capture_content(): + invocation.output_messages = [to_output_message(output_message)] + except Exception: # pylint: disable=broad-except + _logger.debug("Failed to record the response", exc_info=True) + + def _start_inference( handler: TelemetryHandler, wrapped: Callable[..., Any], @@ -229,13 +276,7 @@ def _start_inference( provider, request_model=instance.model_id, ) - bound = _bind_arguments(wrapped, args, kwargs) - _apply_request_parameters(invocation, instance, bound) - invocation.tool_definitions = to_tool_definitions( - bound.get("tools_to_call_from") - ) - if handler.should_capture_content(): - invocation.input_messages = to_input_messages(bound.get("messages")) + _record_request(handler, invocation, wrapped, instance, args, kwargs) return invocation @@ -257,11 +298,7 @@ def wrapper( invocation = _start_inference(handler, wrapped, instance, args, kwargs) with invocation: output_message = wrapped(*args, **kwargs) - _apply_token_usage(invocation, output_message) - if handler.should_capture_content(): - invocation.output_messages = [ - to_output_message(output_message) - ] + _record_response(handler, invocation, output_message) return output_message return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py index 4f9e1e070..196d39979 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -21,6 +21,9 @@ from smolagents.models import ChatMessage, MessageRole from opentelemetry.context import Context +from opentelemetry.instrumentation.genai.smolagents import ( + patch as patch_module, +) from opentelemetry.instrumentation.genai.smolagents._messages import ( to_input_messages, to_output_message, @@ -239,6 +242,68 @@ def test_lifecycle_recorder_sees_the_happy_path( assert lifecycle.leaked == [] +def test_generate_bad_call_records_an_error_span( + instrument_with_content, span_exporter, lifecycle +) -> None: + model = transformers_model() + + # messages is required, so the call fails before it reaches the runtime. + with pytest.raises(TypeError): + model.generate() + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "TypeError" + assert lifecycle.leaked == [] + + +class _NotATool: + """tools_to_call_from is a plain runtime kwarg; the annotation is a + contract, not enforcement.""" + + +def test_bad_tool_does_not_leak_a_span( + instrument_with_content, span_exporter, lifecycle +) -> None: + model = transformers_model() + + # The AttributeError is the runtime's own, from building the tool schemas. + # Reading the tool for telemetry must not fail the call before that. + with pytest.raises(AttributeError, match="has no attribute 'inputs'"): + model.generate(messages=MESSAGES, tools_to_call_from=[_NotATool()]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "AttributeError" + assert lifecycle.leaked == [] + + +@pytest.mark.parametrize( + "conversion", ["to_input_messages", "to_output_message"] +) +def test_a_failed_conversion_does_not_break_the_call( + instrument_with_content, + span_exporter, + lifecycle, + monkeypatch: pytest.MonkeyPatch, + conversion: str, +) -> None: + # A shape the conversion cannot read drops the content from the span, and + # changes nothing else. + def raise_error(*args: Any, **kwargs: Any) -> Any: + raise ValueError("unexpected message shape") + + monkeypatch.setattr(patch_module, conversion, raise_error) + + output = transformers_model().generate(messages=MESSAGES) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert lifecycle.leaked == [] + + def test_tool_definitions_recorded( instrument_with_content, span_exporter ) -> None: From a9aeb07634cb16aacc593506678ac0fda93e2fff Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Sat, 15 Aug 2026 19:46:57 +0200 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Liudmila Molkova --- .../opentelemetry/instrumentation/genai/smolagents/__init__.py | 2 +- .../instrumentation/genai/smolagents/_messages.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index 2a963e043..fadfe4948 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -160,7 +160,7 @@ def _instrument(self, **kwargs: Any) -> None: model_generate_stream(handler), ) self._wrapped_generate_stream_classes.append(model_cls) - except Exception: + except BaseException: # BaseInstrumentor.instrument() doesn't mark the instrumentor as # instrumented when _instrument raises, so uninstrument() would # refuse to run and leave the patches applied with no way to undo. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index 4ae6fdc12..129617c01 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -71,6 +71,7 @@ def _unwrap_role(role: MessageRole | str | None) -> str | None: return _ROLE_MAP.get(name, name) +# TODO leverage util helpers def _decode_base64_image(image: str) -> tuple[bytes, str] | None: """Decode a base64 payload or data URL into ``(bytes, mime_type)``.""" mime_type = _DEFAULT_IMAGE_MIME_TYPE @@ -89,7 +90,7 @@ def _decode_base64_image(image: str) -> tuple[bytes, str] | None: def _encode_base64_image(image: Image) -> str | None: try: encoded = encode_image_base64(image) - except Exception: # pylint: disable=broad-except + except BaseException: # pylint: disable=broad-except _logger.debug( "Failed to encode image of type %s, dropping it from telemetry", type(image).__name__,