diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added new file mode 100644 index 000000000..e389f9e83 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added @@ -0,0 +1 @@ +Add ``chat`` instrumentation for the in-process smolagents model classes (``TransformersModel``, ``VLLMModel``, ``MLXModel``). diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index 932ad2c12..edf44ac65 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -7,7 +7,56 @@ OpenTelemetry smolagents Instrumentation :target: https://pypi.org/project/opentelemetry-instrumentation-genai-smolagents/ This library provides OpenTelemetry instrumentation for `smolagents -`_. +`_. It wraps the model classes that +run inference in your own process and emits a GenAI semantic-convention ``chat`` +span and the matching metrics through ``opentelemetry-util-genai``: + +* ``TransformersModel`` +* ``VLLMModel`` +* ``MLXModel`` + +The API-backed model classes are not instrumented here. Each one calls a client +library that carries its own instrumentation. Emitting a span at the smolagents +layer as well would produce two ``chat`` spans for one model call, and would +count the token-usage and duration metrics twice. Install the instrumentation +for the client library instead: + +.. list-table:: + :header-rows: 1 + + * - smolagents model class + - Instrument this instead + * - ``OpenAIModel``, ``AzureOpenAIModel`` + - `opentelemetry-instrumentation-genai-openai + `_ + * - ``AmazonBedrockModel`` + - `opentelemetry-instrumentation-botocore + `_ + * - ``InferenceClientModel``, ``LiteLLMModel``, ``LiteLLMRouterModel`` + - the instrumentation or built-in telemetry of the client library the model + calls (``huggingface_hub``, ``litellm``) + +Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not +instrumented yet. A model call made inside an agent run still gets a ``chat`` +span, but no agent span sits above it. + +``TransformersModel`` is the only instrumented class with a ``generate_stream``. +A streamed call gets a ``chat`` span that stays open until the caller drains the +deltas. This covers both ``stream_outputs=True`` on an agent and a direct +``generate_stream`` call. The span carries ``gen_ai.request.stream``, and the +call also records the +``gen_ai.client.operation.time_to_first_chunk`` and +``gen_ai.client.operation.time_per_output_chunk`` metrics. + +Known gaps: + +* A subclass that inherits ``generate`` or ``generate_stream`` from one of the + three classes above is instrumented. A subclass that overrides one is not: the + override shadows the patched method, so the call produces no ``chat`` span. +* A ``chat`` span reports no ``gen_ai.response.id``, no + ``gen_ai.response.model``, no ``gen_ai.response.finish_reasons`` and no + ``server.address``. A runtime in this process returns the generated text and + the token counts, nothing more. It also listens on no socket. Installation ------------ @@ -24,10 +73,22 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import TransformersModel - # Instrument smolagents SmolagentsInstrumentor().instrument() + model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") + model.generate( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "How many seconds are in a week?"} + ], + } + ] + ) + Configuration ------------- @@ -71,6 +132,12 @@ environment variable: SmolagentsInstrumentor().instrument(completion_hook=my_hook) +Conformance +----------- + +The scenarios that check this package against the GenAI semantic conventions +live under ``tests/conformance/``. + References ---------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml index 9f7ed96ed..67140011a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "opentelemetry-api ~= 1.43", "opentelemetry-instrumentation >= 0.64b0, <1", "opentelemetry-semantic-conventions >= 0.64b0, <1", - "opentelemetry-util-genai >= 1.0b0, <2", + "opentelemetry-util-genai >= 1.1b0.dev, <2", ] [project.optional-dependencies] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index f882e1976..fadfe4948 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -7,6 +7,13 @@ Instrumentation for `smolagents `_. +Calls to the in-process model classes (``TransformersModel``, ``VLLMModel`` and +``MLXModel``) are recorded as ``chat`` spans. The API-backed model classes are +not instrumented here: each one calls a client library that carries its own +instrumentation, and emitting a span at this layer as well would duplicate the +span and count the token-usage and duration metrics twice. Agent runs and tool +calls are not instrumented yet. + Usage ----- @@ -15,10 +22,22 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import TransformersModel - # Enable instrumentation SmolagentsInstrumentor().instrument() + model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") + model.generate( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "How many seconds are in a week?"} + ], + } + ] + ) + Configuration ------------- @@ -41,18 +60,67 @@ from collections.abc import Collection from typing import Any +from smolagents import models +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.util.genai.completion_hook import load_completion_hook from opentelemetry.util.genai.handler import TelemetryHandler from .package import _instruments +from .patch import model_generate, model_generate_stream __all__ = ["SmolagentsInstrumentor"] +# The model classes that run inference in the current process. They call no +# client library, so this instrumentation is the only place their model calls +# can be observed. +# +# The API-backed classes are left out on purpose. Each one calls a client +# library whose own instrumentation emits the ``chat`` span, so wrapping them +# here as well would produce two spans for one model call and count the +# token-usage and duration metrics twice. ``README.rst`` lists which +# instrumentation covers which class. +_IN_PROCESS_MODEL_CLASSES = ("MLXModel", "TransformersModel", "VLLMModel") + + +def _model_classes_defining(method: str) -> list[type]: + """The in-process model classes whose ``method`` gets wrapped. + + Only classes that define ``method`` in their own ``__dict__`` are patched. + ``MLXModel`` and ``VLLMModel`` have no ``generate_stream``, and neither does + the base class, so wrapping it on them raises ``AttributeError``. The same + check keeps a method defined on a shared base from being wrapped once per + subclass. + + A user-defined subclass that overrides the method shadows the patched one and + emits no ``chat`` span. ``README.rst`` documents that limitation. + + A class is looked up by name so that a smolagents version without one of them + is skipped rather than raising. + """ + classes: list[type] = [] + for name in _IN_PROCESS_MODEL_CLASSES: + model_cls = getattr(models, name, None) + if isinstance(model_cls, type) and method in model_cls.__dict__: + classes.append(model_cls) + return classes + + class SmolagentsInstrumentor(BaseInstrumentor): """An instrumentor for smolagents.""" + # ``BaseInstrumentor.__new__`` returns a per-class singleton, but Python + # still runs ``__init__`` on every construction. Initializing this state in + # ``__init__`` would let the documented ``SmolagentsInstrumentor() + # .uninstrument()`` form wipe the live instance's bookkeeping and leave + # smolagents permanently patched, so these are class-level defaults that + # only ``_instrument`` / ``_uninstrument`` rebind. + _wrapped_generate_classes: list[type] = [] + _wrapped_generate_stream_classes: list[type] = [] + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -66,15 +134,45 @@ def _instrument(self, **kwargs: Any) -> None: - logger_provider: LoggerProvider instance - completion_hook: CompletionHook instance """ - TelemetryHandler( + handler = TelemetryHandler( tracer_provider=kwargs.get("tracer_provider"), meter_provider=kwargs.get("meter_provider"), logger_provider=kwargs.get("logger_provider"), completion_hook=kwargs.get("completion_hook") or load_completion_hook(), ) - # Patching will be added in follow-up PRs. + + self._wrapped_generate_classes = [] + self._wrapped_generate_stream_classes = [] + try: + for model_cls in _model_classes_defining("generate"): + wrap_function_wrapper( + model_cls, + "generate", + model_generate(handler), + ) + self._wrapped_generate_classes.append(model_cls) + + for model_cls in _model_classes_defining("generate_stream"): + wrap_function_wrapper( + model_cls, + "generate_stream", + model_generate_stream(handler), + ) + self._wrapped_generate_stream_classes.append(model_cls) + except BaseException: + # BaseInstrumentor.instrument() doesn't mark the instrumentor as + # instrumented when _instrument raises, so uninstrument() would + # refuse to run and leave the patches applied with no way to undo. + self._uninstrument() + raise def _uninstrument(self, **kwargs: Any) -> None: """Disable smolagents instrumentation and restore patched originals.""" - # Unpatching will be added in follow-up PRs. + for model_cls in self._wrapped_generate_classes: + unwrap(model_cls, "generate") + self._wrapped_generate_classes = [] + + for model_cls in self._wrapped_generate_stream_classes: + unwrap(model_cls, "generate_stream") + self._wrapped_generate_stream_classes = [] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py new file mode 100644 index 000000000..129617c01 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -0,0 +1,244 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Convert smolagents message and tool shapes into util-genai GenAI types. + +smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects +or plain dicts, and returns a ``ChatMessage``. This module maps those, and the +``tools_to_call_from`` tool objects, onto the types in +``opentelemetry.util.genai.types``. util-genai then serializes them into +``gen_ai.input.messages``, ``gen_ai.output.messages``, and +``gen_ai.tool.definitions``. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +from enum import Enum +from typing import TYPE_CHECKING, Any, TypeAlias + +from smolagents.models import get_tool_json_schema +from smolagents.utils import encode_image_base64 + +from opentelemetry.util.genai.types import ( + Blob, + FunctionToolDefinition, + InputMessage, + MessagePart, + OutputMessage, + Text, + ToolDefinition, + Uri, +) + +if TYPE_CHECKING: + from PIL.Image import Image + from smolagents.models import ChatMessage, MessageRole + from smolagents.tools import Tool + +_logger = logging.getLogger(__name__) + +# One element of a message ``content`` list, typed as smolagents types it: +# ``ChatMessage.content`` is ``str | list[dict[str, Any]]``. The values within an +# element have mixed types, a ``str`` under ``text``, a nested dict under +# ``image_url``, and a PIL image under ``image``. +_ContentElement: TypeAlias = dict[str, Any] + +_DEFAULT_IMAGE_MIME_TYPE = "image/png" +_DATA_URL_PREFIX = "data:" + +# smolagents-internal roles -> semconv ``gen_ai`` message roles. smolagents +# applies the same mapping (``models.tool_role_conversions``) inside +# ``generate``, after the wrapper has already read ``messages``, so the wrapper +# has to apply it too. Without this map the input messages would carry roles +# the spec doesn't define. A model configured with ``custom_role_conversions`` +# overrides the mapping and can send different roles. Roles that are already +# spec values pass through unchanged. +_ROLE_MAP: dict[str, str] = { + "tool-call": "assistant", + "tool-response": "user", +} + + +def _unwrap_role(role: MessageRole | str | None) -> str | None: + if role is None: + return None + if isinstance(role, Enum): + role = role.value + name = str(role) + return _ROLE_MAP.get(name, name) + + +# TODO leverage util helpers +def _decode_base64_image(image: str) -> tuple[bytes, str] | None: + """Decode a base64 payload or data URL into ``(bytes, mime_type)``.""" + mime_type = _DEFAULT_IMAGE_MIME_TYPE + if image.startswith(_DATA_URL_PREFIX): + header, _, image = image.partition(",") + media_type = header[len(_DATA_URL_PREFIX) :].split(";")[0] + if media_type: + mime_type = media_type + try: + return base64.b64decode(image, validate=True), mime_type + except (binascii.Error, ValueError): + _logger.debug("Failed to decode a base64 image", exc_info=True) + return None + + +def _encode_base64_image(image: Image) -> str | None: + try: + encoded = encode_image_base64(image) + except BaseException: # pylint: disable=broad-except + _logger.debug( + "Failed to encode image of type %s, dropping it from telemetry", + type(image).__name__, + exc_info=True, + ) + return None + return encoded if isinstance(encoded, str) else None + + +def _image_blob(image: Image | str) -> Blob | None: + """Build a ``Blob`` part from a base64 string, data URL, or PIL image.""" + if isinstance(image, str): + decoded = _decode_base64_image(image) + else: + encoded = _encode_base64_image(image) + decoded = ( + _decode_base64_image(encoded) if encoded is not None else None + ) + if decoded is None: + return None + content, mime_type = decoded + return Blob(mime_type=mime_type, modality="image", content=content) + + +def _image_part_from_element(element: _ContentElement) -> Uri | Blob | None: + content_type = element.get("type") + if content_type == "image_url": + image_url = element.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + if isinstance(url, str) and url: + return Uri(mime_type=None, modality="image", uri=url) + return None + if content_type == "image": + image = element.get("image") + if image is not None: + return _image_blob(image) + return None + + +def _parts_from_content( + content: str | list[_ContentElement] | None, +) -> list[MessagePart]: + parts: list[MessagePart] = [] + if isinstance(content, str): + parts.append(Text(content=content)) + return parts + if isinstance(content, list): + for element in content: + if not isinstance(element, dict): + _logger.debug( + "Unknown message content dropped from telemetry: %s", + type(element).__name__, + ) + continue + if element.get("type") == "text" and (text := element.get("text")): + parts.append(Text(content=text)) + continue + if image_part := _image_part_from_element(element): + parts.append(image_part) + else: + _logger.debug( + "Unknown message part dropped from telemetry: %s", + element.get("type"), + ) + return parts + + +def _get_role_and_content( + message: ChatMessage | dict[str, Any], +) -> tuple[MessageRole | str | None, str | list[_ContentElement] | None]: + # smolagents reads a message the same way: a dict goes through + # ChatMessage.from_dict, anything else has its attributes read directly. + if isinstance(message, dict): + return message.get("role"), message.get("content") + return message.role, message.content + + +def to_input_messages( + messages: list[ChatMessage | dict[str, Any]] | None, +) -> list[InputMessage]: + """Map smolagents ``generate`` input messages to ``InputMessage`` objects.""" + result: list[InputMessage] = [] + if not isinstance(messages, list): + return result + for message in messages: + raw_role, content = _get_role_and_content(message) + role = _unwrap_role(raw_role) + if not role: + continue + result.append( + InputMessage(role=role, parts=_parts_from_content(content)) + ) + return result + + +def to_output_message(output_message: ChatMessage) -> OutputMessage: + """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``. + + The in-process runtimes return the generated text and nothing else: no tool + calls (the agent parses those out of the text afterwards), no reasoning + content, and no finish reason. ``OutputMessage`` requires + ``finish_reason``, and util-genai drops an empty value when it emits + ``gen_ai.response.finish_reasons``. Defaulting it to ``"stop"`` instead + would make a generation cut short by ``max_new_tokens`` look like a natural + stop. + """ + role = _unwrap_role(output_message.role) or "assistant" + parts = _parts_from_content(output_message.content) + return OutputMessage(role=role, parts=parts, finish_reason="") + + +def _tool_parameters(tool: Tool) -> dict[str, Any] | None: + """Return the JSON Schema ``parameters`` object for a smolagents tool. + + A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it + in an object schema, derives ``required`` from ``nullable``, and rewrites its + non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the + schema the provider receives. + """ + try: + schema = get_tool_json_schema(tool) + parameters = schema["function"]["parameters"] + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to build a JSON Schema for tool %s", + tool.name, + exc_info=True, + ) + return None + return parameters if isinstance(parameters, dict) else None + + +def to_tool_definitions( + tools: list[Tool] | None, +) -> list[ToolDefinition] | None: + """Map smolagents tool objects to function tool definitions. + + ``Tool.validate_arguments`` runs on every instantiation and requires a + non-empty ``name`` and a ``description``, so both are read directly. + """ + if not tools: + return None + definitions: list[ToolDefinition] = [ + FunctionToolDefinition( + name=tool.name, + description=tool.description, + parameters=_tool_parameters(tool), + ) + for tool in tools + ] + return definitions diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py index b56883424..fa7223304 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 _instruments = ("smolagents >= 1.24.0",) + +_supports_metrics = True diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py new file mode 100644 index 000000000..ff14ff493 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -0,0 +1,398 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""wrapt wrapper factories for smolagents instrumentation. + +Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper +suitable for :func:`wrapt.wrap_function_wrapper`, applied to the in-process +model classes only (see ``_IN_PROCESS_MODEL_CLASSES``): + +- :func:`model_generate` wraps ``generate`` -> ``chat`` span. +- :func:`model_generate_stream` wraps ``generate_stream`` -> ``chat`` span, held + open until the stream is drained. + +Original library exceptions are always re-raised unmodified; telemetry is +finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Generator, Mapping +from inspect import signature +from typing import TYPE_CHECKING, Any, TypeAlias + +from smolagents.models import REMOVE_PARAMETER + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import InferenceInvocation +from opentelemetry.util.genai.stream import SyncStreamWrapper +from opentelemetry.util.genai.types import OutputMessage, Text + +from ._messages import ( + to_input_messages, + to_output_message, + to_tool_definitions, +) +from .provider import resolve_provider + +if TYPE_CHECKING: + from smolagents.models import ( + ChatMessage, + ChatMessageStreamDelta, + Model, + ) + +_logger = logging.getLogger(__name__) + +# ``Model`` is quoted because a type alias is evaluated at runtime, unlike an +# annotation. +_Wrapper: TypeAlias = Callable[ + [Callable[..., Any], "Model", tuple[Any, ...], dict[str, Any]], Any +] + + +def _bind_arguments( + wrapped: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> dict[str, Any]: + """Bind call args to the wrapped callable's signature, applying defaults. + + smolagents passes the interesting arguments positionally + (``model.generate(input_messages)``), so binding is what makes them + readable by name. On a binding failure the keyword arguments are returned + on their own, without the positional ones and without the defaults. + """ + try: + bound = signature(wrapped).bind(*args, **kwargs) + bound.apply_defaults() + return dict(bound.arguments) + except (TypeError, ValueError): + _logger.debug( + "Failed to bind arguments of %s; falling back to keyword arguments", + getattr(wrapped, "__qualname__", wrapped), + exc_info=True, + ) + return dict(kwargs) + + +def _coerce_float(value: object) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def _coerce_int(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _merged_request_kwargs( + instance: Model, bound: dict[str, Any] +) -> dict[str, Any]: + """Rebuild the request keyword arguments smolagents will send. + + ``Model._prepare_completion_kwargs`` seeds ``stop`` from the + ``stop_sequences`` argument and ``response_format`` from its own argument, + applies the per-call ``**kwargs`` on top and the model-level ``self.kwargs`` + last, so a key set at two levels reaches the provider with the model-level + value and a model-level ``REMOVE_PARAMETER`` drops it from the request + entirely. Following that order here is what keeps a removed key off the span + as well. + """ + merged: dict[str, Any] = {} + stop_sequences = bound.get("stop_sequences") + if stop_sequences is not None and instance.supports_stop_parameter: + merged["stop"] = stop_sequences + response_format = bound.get("response_format") + if response_format is not None: + merged["response_format"] = response_format + call_kwargs = bound.get("kwargs") + if isinstance(call_kwargs, dict): + merged.update(call_kwargs) + for name, value in instance.kwargs.items(): + if value is REMOVE_PARAMETER: + merged.pop(name, None) + else: + merged[name] = value + return merged + + +def _stop_sequences(merged: dict[str, Any]) -> list[str] | None: + """Return the stop sequences the request carries, if any.""" + stop = merged.get("stop", merged.get("stop_sequences")) + if isinstance(stop, str): + return [stop] + if isinstance(stop, list): + return [str(item) for item in stop] + return None + + +# smolagents' ``response_format`` type -> ``gen_ai.output.type`` value, for the +# types whose provider spelling differs from the semconv one. The openai +# instrumentation maps the same two. +_OUTPUT_TYPE_MAP: dict[str, str] = { + "json_object": GenAI.GenAiOutputTypeValues.JSON.value, + "json_schema": GenAI.GenAiOutputTypeValues.JSON.value, +} + +_OUTPUT_TYPE_VALUES = frozenset( + value.value for value in GenAI.GenAiOutputTypeValues +) + + +def _output_type(merged: dict[str, Any]) -> str | None: + """Map the request's ``response_format`` to ``gen_ai.output.type``. + + smolagents forwards ``response_format`` to the provider unchanged, so its + ``type`` is whatever the provider accepts. Only the values the semconv + defines are recorded; a provider-specific one is dropped rather than put on + an enum attribute. + """ + response_format = merged.get("response_format") + if not isinstance(response_format, Mapping): + return None + format_type = response_format.get("type") + if not isinstance(format_type, str): + return None + output_type = _OUTPUT_TYPE_MAP.get(format_type, format_type) + if output_type not in _OUTPUT_TYPE_VALUES: + _logger.debug( + "No gen_ai.output.type value for response_format type %r", + format_type, + ) + return None + return output_type + + +def _apply_request_parameters( + invocation: InferenceInvocation, instance: Model, bound: dict[str, Any] +) -> None: + """Copy the request parameters smolagents will send onto the span.""" + merged = _merged_request_kwargs(instance, bound) + + invocation.temperature = _coerce_float(merged.get("temperature")) + invocation.top_p = _coerce_float(merged.get("top_p")) + invocation.top_k = _coerce_float(merged.get("top_k")) + invocation.frequency_penalty = _coerce_float( + merged.get("frequency_penalty") + ) + invocation.presence_penalty = _coerce_float(merged.get("presence_penalty")) + # TransformersModel takes the generation limit as max_new_tokens (which its + # constructor defaults to 4096) and treats max_tokens as an alias for it. + invocation.max_tokens = _coerce_int( + merged.get("max_tokens", merged.get("max_new_tokens")) + ) + invocation.seed = _coerce_int(merged.get("seed")) + invocation.stop_sequences = _stop_sequences(merged) + invocation.output_type = _output_type(merged) + + +def _apply_token_usage( + invocation: InferenceInvocation, output_message: ChatMessage +) -> None: + # ChatMessage.token_usage is the only source: the per-model + # last_input_token_count / last_output_token_count counters were removed + # before the oldest supported smolagents. The in-process runtimes count the + # prompt and generated tokens themselves and report them here. + token_usage = output_message.token_usage + if token_usage is None: + return + invocation.input_tokens = token_usage.input_tokens + invocation.output_tokens = token_usage.output_tokens + + +def _record_request( + handler: TelemetryHandler, + invocation: InferenceInvocation, + wrapped: Callable[..., Any], + instance: Model, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> None: + """Record the request on the invocation. Extraction errors are dropped. + + The messages, tools and keyword arguments come from the caller, so the + conversion can get a shape it does not handle. The span is already + started and the model has not been called yet, so an error raised here + would both break the call and leave the span unfinished. + """ + try: + bound = _bind_arguments(wrapped, args, kwargs) + _apply_request_parameters(invocation, instance, bound) + invocation.tool_definitions = to_tool_definitions( + bound.get("tools_to_call_from") + ) + if handler.should_capture_content(): + invocation.input_messages = to_input_messages( + bound.get("messages") + ) + except Exception: # pylint: disable=broad-except + _logger.debug("Failed to record the request", exc_info=True) + + +def _record_response( + handler: TelemetryHandler, + invocation: InferenceInvocation, + output_message: ChatMessage, +) -> None: + """Record the response on the invocation. Extraction errors are dropped. + + The model call has already succeeded at this point, so an error raised + here would turn a completed call into a failed one. + """ + try: + _apply_token_usage(invocation, output_message) + if handler.should_capture_content(): + invocation.output_messages = [to_output_message(output_message)] + except Exception: # pylint: disable=broad-except + _logger.debug("Failed to record the response", exc_info=True) + + +def _start_inference( + handler: TelemetryHandler, + wrapped: Callable[..., Any], + instance: Model, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> InferenceInvocation: + """Start the ``chat`` span and record the request. + + ``generate`` and ``generate_stream`` take the same parameters. An in-process + runtime listens on no socket, so the span carries no ``server.address`` or + ``server.port``. + """ + provider = resolve_provider(instance) + invocation = handler.inference( + provider, + request_model=instance.model_id, + ) + _record_request(handler, invocation, wrapped, instance, args, kwargs) + return invocation + + +def model_generate(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate`` to emit a ``chat`` span. + + An in-process runtime returns the generated text, the token counts it made + itself, and no response envelope, so the span carries no + ``gen_ai.response.id``, ``gen_ai.response.model`` or + ``gen_ai.response.finish_reasons``. + """ + + def wrapper( + wrapped: Callable[..., ChatMessage], + instance: Model, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> ChatMessage: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + with invocation: + output_message = wrapped(*args, **kwargs) + _record_response(handler, invocation, output_message) + return output_message + + return wrapper + + +class _ModelStreamWrapper(SyncStreamWrapper["ChatMessageStreamDelta"]): + """Keep the ``chat`` span open until the delta stream is drained. + + Passing the invocation to ``super().__init__()`` turns on + ``gen_ai.request.stream`` and the per-chunk timing metrics. + + ``TransformersModel`` is the only in-process runtime with a + ``generate_stream``. Its deltas carry the generated text and per-delta token + counts, and never any tool calls. + """ + + def __init__( + self, + stream: Generator[ChatMessageStreamDelta, None, None], + invocation: InferenceInvocation, + handler: TelemetryHandler, + ) -> None: + super().__init__(stream, invocation=invocation) + self._self_inference = invocation + self._self_capture_content = handler.should_capture_content() + self._self_content: list[str] = [] + self._self_input_tokens = 0 + self._self_output_tokens = 0 + self._self_saw_token_usage = False + + def _process_chunk(self, chunk: ChatMessageStreamDelta) -> None: + content = chunk.content + if content and self._self_capture_content: + self._self_content.append(content) + token_usage = chunk.token_usage + if token_usage is not None: + # Summed like agglomerate_stream_deltas, so the span agrees with the + # totals the agent's monitor reports. + self._self_saw_token_usage = True + self._self_input_tokens += token_usage.input_tokens + self._self_output_tokens += token_usage.output_tokens + + def _output_message(self) -> OutputMessage | None: + content = "".join(self._self_content) + if not content: + # Closed before it was drained, so there is no response to report. + return None + # Deltas carry no finish reason, and defaulting to "stop" would hide a + # generation cut short by a token limit. + return OutputMessage( + role="assistant", parts=[Text(content=content)], finish_reason="" + ) + + def _finalize(self, error: BaseException | None = None) -> None: + invocation = self._self_inference + if self._self_saw_token_usage: + invocation.input_tokens = self._self_input_tokens + invocation.output_tokens = self._self_output_tokens + if self._self_capture_content: + output = self._output_message() + if output is not None: + invocation.output_messages = [output] + if error is not None: + invocation.fail(error) + else: + invocation.stop() + + def _on_stream_end(self) -> None: + self._finalize() + + def _on_stream_error(self, error: BaseException) -> None: + # Records what was streamed before the failure. + self._finalize(error) + + +def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate_stream`` to emit a ``chat`` span. + + ``stream_outputs=True`` routes an agent's model calls here. The span stays + open until the caller drains the deltas. + """ + + def wrapper( + wrapped: Callable[..., Generator[ChatMessageStreamDelta, None, None]], + instance: Model, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> _ModelStreamWrapper: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + try: + stream = wrapped(*args, **kwargs) + return _ModelStreamWrapper(stream, invocation, handler) + except Exception as error: + invocation.fail(error) + raise + + return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py new file mode 100644 index 000000000..9c05d7c57 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -0,0 +1,51 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve a smolagents model instance to a ``gen_ai.provider.name`` value. + +Only the in-process model classes are instrumented here, so the value comes +from the model class name. + +``gen_ai.provider.name`` is a metric attribute as well as a span attribute, so +every value has to stay low cardinality. ``TelemetryHandler.inference`` requires +``provider`` as a string, so this always returns a value rather than ``None``. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from smolagents.models import Model + +_logger = logging.getLogger(__name__) + +_UNKNOWN_PROVIDER = "unknown" + +# Model class name -> provider value. The GenAI registry has no value for the +# Hugging Face, vLLM, and MLX runtimes, so those use the product name; a class +# name would look like a provider without being one. +_CLASS_NAME_TO_PROVIDER: dict[str, str] = { + "TransformersModel": "huggingface", + "VLLMModel": "vllm", + "MLXModel": "mlx", +} + + +def resolve_provider(instance: Model) -> str: + """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" + # An instrumented model can be a user subclass of a patched class. Matching + # the exact class name alone would report ``unknown`` for every subclass, so + # walk the hierarchy, most derived class first. + class_names = [cls.__name__ for cls in type(instance).__mro__] + + for class_name in class_names: + provider = _CLASS_NAME_TO_PROVIDER.get(class_name) + if provider is not None: + return provider + + _logger.debug( + "No known gen_ai.provider.name for model class %s", class_names[0] + ) + return _UNKNOWN_PROVIDER diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py new file mode 100644 index 000000000..be0d0071e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py @@ -0,0 +1,34 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for smolagents conformance scenarios.""" + +from __future__ import annotations + +import json +from typing import Any + + +def attr(span: dict[str, Any], name: str) -> Any: + for entry in span["attributes"]: + if entry["name"] == name: + return entry["value"] + return None + + +def chat_spans(report: Any) -> list[dict[str, Any]]: + return [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + + +def part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py new file mode 100644 index 000000000..bad12d2eb --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py @@ -0,0 +1,151 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for the ``chat`` operation (plain, streamed, and with +tool definitions). + +Every scenario drives an in-process runtime over the stubs in ``test_utils``, +since those are the only instrumented model classes. +""" + +from __future__ import annotations + +from typing import Any + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +from ..test_utils import ( + MESSAGES, + GetWeatherTool, + transformers_model, +) +from ._helpers import attr, chat_spans + +# An in-process runtime returns only the generated text: no request id, no +# resolved model name, no stop reason. It also listens on no socket. semconv +# expects those four attributes on a chat span, so every scenario here carries +# the same gaps. +IN_PROCESS_GAPS = tuple( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring=attribute, + ) + for attribute in ( + "server.address", + "gen_ai.response.id", + "gen_ai.response.model", + "gen_ai.response.finish_reasons", + ) +) + + +class ChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + expected_violations = IN_PROCESS_GAPS + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + transformers_model().generate(messages=MESSAGES) + + +class StreamedChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + "gen_ai.client.operation.time_to_first_chunk", + "gen_ai.client.operation.time_per_output_chunk", + ) + expected_violations = IN_PROCESS_GAPS + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + model = transformers_model(stream_chunks=["In ", "Paris"]) + list(model.generate_stream(messages=MESSAGES)) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + for span in chat_spans(report): + assert attr(span, "gen_ai.request.stream") is True, ( + "expected gen_ai.request.stream on a streamed chat span" + ) + + +class ToolDefinitionsScenario(Scenario): + """A ``chat`` call that offers tools to the model. + + The in-process runtimes put the tool definitions in the prompt but never + return tool calls (the agent parses those out of the generated text), so + this covers the request side only. + """ + + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = IN_PROCESS_GAPS + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + transformers_model().generate( + messages=MESSAGES, tools_to_call_from=[GetWeatherTool()] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + for span in chat_spans(report): + assert attr(span, "gen_ai.tool.definitions"), ( + "expected gen_ai.tool.definitions on the chat span" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py new file mode 100644 index 000000000..530cbf314 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py @@ -0,0 +1,90 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario for an image ``uri`` part on a ``chat`` input.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +from ..test_utils import vllm_model +from ._helpers import attr, chat_spans, part_fields +from .inference import IN_PROCESS_GAPS + +_IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +class MultimodalScenario(Scenario): + """A vision-capable ``VLLMModel`` taking an image alongside the text. + + A VLM keeps the message content as parts instead of flattening it to text, + which is what puts an image part on the span. + """ + + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = IN_PROCESS_GAPS + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + # vllm_model fakes the vllm modules for the duration of the call. + with pytest.MonkeyPatch.context() as monkeypatch: + model = vllm_model(monkeypatch) + model._is_vlm = True + model.flatten_messages_as_text = False + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What breed is this dog?", + }, + { + "type": "image_url", + "image_url": {"url": _IMAGE_URL}, + }, + ], + } + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + input_parts = { + fields + for span in chat_spans(report) + for fields in part_fields(attr(span, "gen_ai.input.messages")) + } + assert ("uri", "image") in input_parts, ( + f"expected an image uri input part, saw {input_parts}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index da97f2987..01ae6a97d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -12,15 +12,43 @@ ) from opentelemetry.test_util_genai.instrumentor import instrument +# No VCR plugin: the instrumented model classes run inference in this process, +# so there is no HTTP traffic to record. pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] @pytest.fixture -def instrument_smolagents(tracer_provider, logger_provider, meter_provider): +def instrument_no_content(tracer_provider, logger_provider, meter_provider): with instrument( SmolagentsInstrumentor(), tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, + content_capture="NO_CONTENT", + ) as instrumentor: + yield instrumentor + + +@pytest.fixture +def instrument_with_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ) as instrumentor: + yield instrumentor + + +@pytest.fixture +def instrument_event_only(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="EVENT_ONLY", + emit_event=True, ) as instrumentor: yield instrumentor diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index d833435b7..3cdf75f33 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -26,7 +26,10 @@ # This variant of the requirements aims to test the system using the newest # supported version of external dependencies. +# The instrumented model classes run inference in this process and the tests stub +# their runtimes, so no model backend extra is installed. smolagents +wrapt>=2.2.2 -e util/opentelemetry-util-genai -e instrumentation/opentelemetry-instrumentation-genai-smolagents diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index 1d9b353f3..18cd3aaa7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -21,5 +21,10 @@ # pyproject.toml is the single source of truth. The OpenTelemetry SDK and test utilities # come transitively from opentelemetry-test-util-genai. # -# There is nothing to pin yet: the lifecycle tests need no model backend. The pins for -# the backends the model tests drive arrive with those tests. +# The instrumented model classes run inference in this process and the tests stub their +# runtimes, so there is no model backend to pin here. + +# The declared opentelemetry-util-genai floor carries the streaming metrics and the +# gen_ai.request.stream attribute, which are not in a published release yet. +# Drop this once opentelemetry-util-genai 1.1b0 is published. +-e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py new file mode 100644 index 000000000..dc0ba1e7c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -0,0 +1,44 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for smolagents.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check or OTLP exporters aren't installed +# (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") +pytest.importorskip("opentelemetry.exporter.otlp.proto.grpc") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck +from opentelemetry.test_util_genai.conformance import ( + Scenario, + run_conformance, +) + +from .conformance.inference import ( + ChatScenario, + StreamedChatScenario, + ToolDefinitionsScenario, +) +from .conformance.multimodal import MultimodalScenario + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(ChatScenario()), + pytest.param(StreamedChatScenario()), + pytest.param(ToolDefinitionsScenario()), + pytest.param(MultimodalScenario()), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index 791f03237..c42d08d78 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -1,19 +1,38 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Entry-point and instrument/uninstrument lifecycle tests. - -The instrumentor patches nothing yet, so these cover the parts of its contract -that hold before any patching lands: the entry point resolves, the declared -dependency is reported, and repeated instrument/uninstrument cycles stay quiet. -""" +"""Lifecycle, entry-point, completion-hook, and restoration tests.""" from __future__ import annotations +from typing import Any +from unittest.mock import patch + +import pytest +import smolagents +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, + _model_classes_defining, ) +from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.util._importlib_metadata import entry_points +from opentelemetry.util.genai.completion_hook import CompletionHook + +from .test_utils import MESSAGES, transformers_model + + +class RecordingHook(CompletionHook): + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def on_completion(self, **kwargs: Any) -> None: + self.calls.append(kwargs) + + +def _generate_a_chat_span() -> None: + transformers_model().generate(messages=MESSAGES) def test_entrypoint_loads_instrumentor() -> None: @@ -29,45 +48,81 @@ def test_instrumentation_dependencies() -> None: assert "smolagents >= 1.24.0" in dependencies -def test_instrument_uninstrument_cycle( +def test_instrument_uninstrument_restores_originals( tracer_provider, logger_provider, meter_provider ) -> None: + original_generate = smolagents.TransformersModel.generate + original_mlx_generate = smolagents.MLXModel.generate + original_generate_stream = smolagents.TransformersModel.generate_stream + instrumentor = SmolagentsInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, ) - assert instrumentor.is_instrumented_by_opentelemetry + + assert smolagents.TransformersModel.generate is not original_generate + assert smolagents.MLXModel.generate is not original_mlx_generate + assert ( + smolagents.TransformersModel.generate_stream + is not original_generate_stream + ) instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.TransformersModel.generate is original_generate + assert smolagents.MLXModel.generate is original_mlx_generate + assert ( + smolagents.TransformersModel.generate_stream + is original_generate_stream + ) -def test_repeated_instrument_uninstrument( - tracer_provider, logger_provider, meter_provider + +@pytest.mark.parametrize( + "model_class", + [ + "Model", + "OpenAIModel", + "AzureOpenAIModel", + "AmazonBedrockModel", + "InferenceClientModel", + "LiteLLMModel", + "LiteLLMRouterModel", + ], +) +def test_api_backed_classes_are_left_alone( + tracer_provider, logger_provider, meter_provider, model_class: str ) -> None: - # BaseInstrumentor returns a per-class singleton, so the lifecycle has to - # survive being driven more than once. + # Each of these calls a client library that carries its own instrumentation, + # so patching them here would emit a second chat span for one model call and + # count the token-usage and duration metrics twice. + original = getattr(smolagents, model_class).__dict__.get("generate") + instrumentor = SmolagentsInstrumentor() - for _ in range(2): - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + assert ( + getattr(smolagents, model_class).__dict__.get("generate") + is original ) - assert instrumentor.is_instrumented_by_opentelemetry + finally: instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry def test_uninstrument_through_a_new_constructor_call( tracer_provider, logger_provider, meter_provider ) -> None: - # BaseInstrumentor.__new__ returns a per-class singleton but Python still - # runs __init__ on every construction, so the documented + # BaseInstrumentor is a per-class singleton, so the documented # SmolagentsInstrumentor().instrument() / SmolagentsInstrumentor() - # .uninstrument() form has to work on the live instance. + # .uninstrument() form must restore everything even though the second + # constructor call re-runs __init__ on the live instance. + original_generate = smolagents.TransformersModel.generate + SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, @@ -75,23 +130,191 @@ def test_uninstrument_through_a_new_constructor_call( ) SmolagentsInstrumentor().uninstrument() - assert not SmolagentsInstrumentor().is_instrumented_by_opentelemetry + assert smolagents.TransformersModel.generate is original_generate + + +@pytest.mark.parametrize("method", ["generate", "generate_stream"]) +def test_model_classes_defining(method: str) -> None: + classes = _model_classes_defining(method) + + assert len(classes) == len(set(classes)) + # Every entry owns the method, so no class is wrapped for one it only + # inherits. + for model_cls in classes: + assert method in model_cls.__dict__ + + +def test_only_transformers_defines_generate_stream() -> None: + # All three in-process runtimes define generate, but only TransformersModel + # streams. + assert set(_model_classes_defining("generate")) == { + smolagents.TransformersModel, + smolagents.VLLMModel, + smolagents.MLXModel, + } + assert _model_classes_defining("generate_stream") == [ + smolagents.TransformersModel + ] + + +def test_repeated_instrument_uninstrument( + tracer_provider, logger_provider, meter_provider +) -> None: + # BaseInstrumentor returns a per-class singleton, so the wrapped-class + # bookkeeping has to survive being filled and drained more than once. + original_generate = smolagents.TransformersModel.generate + + instrumentor = SmolagentsInstrumentor() + for _ in range(2): + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert smolagents.TransformersModel.generate is not original_generate + instrumentor.uninstrument() + assert smolagents.TransformersModel.generate is original_generate def test_uninstrument_without_instrument() -> None: # BaseInstrumentor.uninstrument() short-circuits, but _uninstrument() must - # also be a no-op on its own: the rollback path in _instrument() calls it - # after a partial patch. + # also be a no-op on unpatched attributes: the rollback in _instrument() + # calls it after a partial patch. + original_generate = smolagents.TransformersModel.generate + SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() + assert smolagents.TransformersModel.generate is original_generate + def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. + original_generate = smolagents.TransformersModel.generate + instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: - assert instrumentor.is_instrumented_by_opentelemetry + assert smolagents.TransformersModel.generate is not original_generate finally: instrumentor.uninstrument() + + assert smolagents.TransformersModel.generate is original_generate + + +def test_failed_instrument_rolls_back_partial_patches( + tracer_provider, logger_provider, meter_provider +) -> None: + # A failure part-way through must leave no class patched, because + # uninstrument() cannot clean up after a failed _instrument(). + model_classes = _model_classes_defining("generate") + assert len(model_classes) > 1, ( + "the rollback needs more than one class to patch" + ) + originals = { + model_cls: model_cls.__dict__["generate"] + for model_cls in model_classes + } + stream_classes = _model_classes_defining("generate_stream") + stream_originals = { + model_cls: model_cls.__dict__["generate_stream"] + for model_cls in stream_classes + } + + real_wrap = wrap_function_wrapper + calls = 0 + + def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("boom") + real_wrap(target, name, wrapper) + + with patch( + "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", + fail_on_the_second_class, + ): + with pytest.raises(RuntimeError, match="boom"): + SmolagentsInstrumentor().instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + assert calls == 2, "the first class was expected to be patched" + for model_cls, original in originals.items(): + assert model_cls.__dict__["generate"] is original + for model_cls, original in stream_originals.items(): + assert model_cls.__dict__["generate_stream"] is original + + +def test_a_user_subclass_inherits_the_patched_generate( + tracer_provider, logger_provider, meter_provider +) -> None: + # A subclass that doesn't override generate is instrumented through the base + # it inherits it from, and must not be wrapped a second time. + class TenantMLXModel(smolagents.MLXModel): + pass + + assert "generate" not in TenantMLXModel.__dict__ + + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + # wrapt returns a fresh bound wrapper per attribute access, so the + # wrapper objects differ; the underlying wrapped function is shared, + # proving the subclass inherits the single wrapped generate. + assert ( + TenantMLXModel.generate.__wrapped__ + is smolagents.MLXModel.generate.__wrapped__ + ) + finally: + instrumentor.uninstrument() + + +def test_explicit_completion_hook_takes_precedence( + tracer_provider, logger_provider, meter_provider, span_exporter +) -> None: + explicit_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook" + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + completion_hook=explicit_hook, + ): + _generate_a_chat_span() + + load_hook.assert_not_called() + assert explicit_hook.calls, "explicit completion hook was not invoked" + + +def test_env_completion_hook_used_when_no_explicit_hook( + tracer_provider, logger_provider, meter_provider +) -> None: + env_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook", + return_value=env_hook, + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + _generate_a_chat_span() + + load_hook.assert_called_once() + assert env_hook.calls, "env-resolved completion hook was not invoked" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py new file mode 100644 index 000000000..196d39979 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -0,0 +1,861 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Model (``chat``) instrumentation tests for the in-process runtimes: +provider resolution, request-parameter mapping, message conversion, streaming, +and errors. + +The three instrumented classes run inference in the current process, so there is +no HTTP traffic to record; each test drives smolagents' real ``generate`` over +the stubbed runtime pieces built in ``test_utils``. +""" + +from __future__ import annotations + +import inspect +import json +from collections.abc import Generator +from typing import Any + +import pytest +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.context import Context +from opentelemetry.instrumentation.genai.smolagents import ( + patch as patch_module, +) +from opentelemetry.instrumentation.genai.smolagents._messages import ( + to_input_messages, + to_output_message, + to_tool_definitions, +) +from opentelemetry.instrumentation.genai.smolagents.patch import ( + _merged_request_kwargs, + _output_type, +) +from opentelemetry.instrumentation.genai.smolagents.provider import ( + resolve_provider, +) +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import ( + error_attributes, + server_attributes, +) +from opentelemetry.trace import StatusCode +from opentelemetry.util.genai.types import Blob, Text, Uri + +from .test_utils import ( + MESSAGES, + GetWeatherTool, + attr, + data_point_attributes, + metrics_by_name, + mlx_model, + parse_messages, + spans_by_operation, + transformers_model, + vllm_model, +) + +IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +def test_transformers_generate_records_the_response( + instrument_with_content, span_exporter, metric_reader +) -> None: + model = transformers_model( + prompt_ids=[1, 2, 3], generated_ids=[4, 5], text="In Paris" + ) + + output = model.generate(messages=MESSAGES) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.name == "chat HuggingFaceTB/SmolLM2-135M-Instruct" + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert ( + attr(span, GenAI.GEN_AI_REQUEST_MODEL) + == "HuggingFaceTB/SmolLM2-135M-Instruct" + ) + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + # A runtime in this process returns no response envelope and listens on no + # socket, so none of these have a value to report. + assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + assert attr(span, server_attributes.SERVER_PORT) is None + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["role"] == "user" + assert inputs[0]["parts"] == [ + {"type": "text", "content": "Where is the Louvre?"} + ] + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + assert outputs[0]["finish_reason"] == "" + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert duration.unit == "s" + assert data_point_attributes(duration) == [ + { + GenAI.GEN_AI_OPERATION_NAME: "chat", + GenAI.GEN_AI_PROVIDER_NAME: "huggingface", + GenAI.GEN_AI_REQUEST_MODEL: "HuggingFaceTB/SmolLM2-135M-Instruct", + } + ] + token_usage = metrics[gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE] + assert { + point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point.sum + for point in token_usage.data.data_points + } == {"input": 3, "output": 2} + + +@pytest.mark.parametrize( + "runtime, provider, request_model, input_tokens, output_tokens", + [ + ("mlx", "mlx", "mlx-community/Qwen2.5-0.5B-Instruct-4bit", 3, 2), + ("vllm", "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), + ], +) +def test_other_runtimes_record_the_response( + instrument_with_content, + span_exporter, + monkeypatch: pytest.MonkeyPatch, + runtime: str, + provider: str, + request_model: str, + input_tokens: int, + output_tokens: int, +) -> None: + model = mlx_model() if runtime == "mlx" else vllm_model(monkeypatch) + + output = model.generate(messages=MESSAGES) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + + +def test_no_content(instrument_no_content, span_exporter) -> None: + transformers_model().generate(messages=MESSAGES) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + # Token usage is metadata, so it survives without the content. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + + +def test_event_only_content_capture( + instrument_event_only, span_exporter, log_exporter +) -> None: + transformers_model().generate(messages=MESSAGES) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.event_name == "gen_ai.client.inference.operation.details" + # Event attributes carry the messages as structured values, not JSON text. + attributes = record.attributes or {} + inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] + outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] + assert inputs[0]["parts"][0]["content"] == "Where is the Louvre?" + assert outputs[0]["parts"][0]["content"] == "In Paris" + + +def test_runtime_error_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = transformers_model(error=RuntimeError("CUDA out of memory")) + + with pytest.raises(RuntimeError, match="CUDA out of memory"): + model.generate(messages=MESSAGES) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "RuntimeError" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +class _LifecycleRecorder(SpanProcessor): + """Records span starts and ends. + + The exporter only sees a span once it ends, so an unfinished span reads + there as no span at all. + """ + + def __init__(self) -> None: + self.started: list[str] = [] + self.ended: list[str] = [] + + def on_start( + self, span: Span, parent_context: Context | None = None + ) -> None: + self.started.append(span.name) + + def on_end(self, span: ReadableSpan) -> None: + self.ended.append(span.name) + + @property + def leaked(self) -> list[str]: + return self.started[len(self.ended) :] + + +@pytest.fixture +def lifecycle(tracer_provider) -> _LifecycleRecorder: + recorder = _LifecycleRecorder() + tracer_provider.add_span_processor(recorder) + return recorder + + +def test_lifecycle_recorder_sees_the_happy_path( + instrument_with_content, lifecycle +) -> None: + transformers_model().generate(messages=MESSAGES) + + assert len(lifecycle.started) == 1 + assert lifecycle.leaked == [] + + +def test_generate_bad_call_records_an_error_span( + instrument_with_content, span_exporter, lifecycle +) -> None: + model = transformers_model() + + # messages is required, so the call fails before it reaches the runtime. + with pytest.raises(TypeError): + model.generate() + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "TypeError" + assert lifecycle.leaked == [] + + +class _NotATool: + """tools_to_call_from is a plain runtime kwarg; the annotation is a + contract, not enforcement.""" + + +def test_bad_tool_does_not_leak_a_span( + instrument_with_content, span_exporter, lifecycle +) -> None: + model = transformers_model() + + # The AttributeError is the runtime's own, from building the tool schemas. + # Reading the tool for telemetry must not fail the call before that. + with pytest.raises(AttributeError, match="has no attribute 'inputs'"): + model.generate(messages=MESSAGES, tools_to_call_from=[_NotATool()]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "AttributeError" + assert lifecycle.leaked == [] + + +@pytest.mark.parametrize( + "conversion", ["to_input_messages", "to_output_message"] +) +def test_a_failed_conversion_does_not_break_the_call( + instrument_with_content, + span_exporter, + lifecycle, + monkeypatch: pytest.MonkeyPatch, + conversion: str, +) -> None: + # A shape the conversion cannot read drops the content from the span, and + # changes nothing else. + def raise_error(*args: Any, **kwargs: Any) -> Any: + raise ValueError("unexpected message shape") + + monkeypatch.setattr(patch_module, conversion, raise_error) + + output = transformers_model().generate(messages=MESSAGES) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert lifecycle.leaked == [] + + +def test_tool_definitions_recorded( + instrument_with_content, span_exporter +) -> None: + transformers_model().generate( + messages=MESSAGES, tools_to_call_from=[GetWeatherTool()] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + }, + } + ] + + +def test_user_subclass_keeps_its_provider( + instrument_with_content, span_exporter, metric_reader +) -> None: + # A subclass inherits the patched generate, so it is instrumented; resolving + # the provider by exact class name would report "unknown" on both the span + # and the metrics. + from smolagents.models import MLXModel + + class TenantMLXModel(MLXModel): + pass + + model = mlx_model() + # The factory builds the base class; rebinding __class__ makes the stub an + # instance of the subclass without running __init__. + model.__class__ = TenantMLXModel + + model.generate(messages=MESSAGES) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "mlx" + duration = metrics_by_name(metric_reader)[ + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( + "mlx" + ) + + +def _fake_model(class_name: str, **attrs: Any) -> Any: + instance = type(class_name, (), {})() + for key, value in attrs.items(): + setattr(instance, key, value) + return instance + + +@pytest.mark.parametrize( + "class_name, expected", + [ + # No GenAI registry value exists for these runtimes, so the product + # name is used rather than the class name. + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), + # gen_ai.provider.name is also a metric attribute. An unmapped model + # must not fall back to a class name. + ("CustomModel", "unknown"), + ], +) +def test_resolve_provider(class_name: str, expected: str) -> None: + assert resolve_provider(_fake_model(class_name)) == expected + + +@pytest.mark.parametrize( + "model_class, expected", + [ + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), + ], +) +def test_resolve_provider_covers_every_instrumented_class( + model_class: str, expected: str +) -> None: + # The mapping is keyed by class name, so pin it against the real classes + # rather than only against synthetic stand-ins. + import smolagents + + instance = object.__new__(getattr(smolagents, model_class)) + assert resolve_provider(instance) == expected + + +def test_request_parameters_recorded( + instrument_with_content, span_exporter +) -> None: + model = mlx_model( + temperature=0.5, + top_p=0.9, + top_k=40, + frequency_penalty=0.25, + presence_penalty=1, + max_tokens=256, + seed=7, + ) + + model.generate(messages=MESSAGES, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.5 + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.9 + # top_k is a float attribute in the spec even though callers pass an int. + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_K) == 40.0 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_TOP_K), float) + assert attr(span, GenAI.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.25 + assert attr(span, GenAI.GEN_AI_REQUEST_PRESENCE_PENALTY) == 1.0 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == 256 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS), int) + assert attr(span, GenAI.GEN_AI_REQUEST_SEED) == 7 + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == ("",) + + +def test_model_kwargs_win_over_call_kwargs( + instrument_with_content, span_exporter +) -> None: + # _prepare_completion_kwargs applies the call kwargs first and the model + # kwargs on top, and drops any key whose model-level value is the + # REMOVE_PARAMETER sentinel. + from smolagents.models import REMOVE_PARAMETER + + model = mlx_model(temperature=0.1, max_tokens=REMOVE_PARAMETER) + + model.generate( + messages=MESSAGES, temperature=0.9, max_tokens=512, top_p=0.5 + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) is None + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.5 + + +@pytest.mark.parametrize( + "model_kwargs, expected", + [ + ({}, ("",)), + # An explicit `stop` overrides the stop_sequences argument. + ({"stop": ["STOP"]}, ("STOP",)), + # The model-level sentinel pops the `stop` that + # _prepare_completion_kwargs seeded from stop_sequences, leaving the + # request with none. + ({"stop": "REMOVE"}, None), + ], +) +def test_stop_sequences_follow_what_is_sent( + instrument_with_content, + span_exporter, + model_kwargs: dict[str, Any], + expected: tuple[str, ...] | None, +) -> None: + from smolagents.models import REMOVE_PARAMETER + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = mlx_model(**model_kwargs) + + model.generate(messages=MESSAGES, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == expected + + +def test_stop_sequences_a_model_cannot_send_are_not_recorded( + instrument_with_content, span_exporter +) -> None: + # supports_stop_parameter is False for the gpt-5 and o-series names, and + # then _prepare_completion_kwargs never seeds `stop`. The runtime truncates + # the generated text locally instead, so the request carries no stop + # sequences. + model = mlx_model() + model.model_id = "gpt-5" + assert model.supports_stop_parameter is False + + model.generate(messages=MESSAGES, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) is None + + +@pytest.mark.parametrize( + "model_kwargs, call_kwargs, expected", + [ + ({"max_tokens": 256}, {}, 256), + ({}, {"max_tokens": 256}, 256), + # max_new_tokens is the TransformersModel spelling of the same limit, + # and max_tokens wins when a caller sets both. + ({"max_new_tokens": 4096}, {}, 4096), + ({"max_new_tokens": 4096, "max_tokens": 256}, {}, 256), + ({}, {}, None), + ], +) +def test_max_tokens_covers_both_spellings( + instrument_with_content, + span_exporter, + model_kwargs: dict[str, Any], + call_kwargs: dict[str, Any], + expected: int | None, +) -> None: + model = transformers_model(**model_kwargs) + + model.generate(messages=MESSAGES, **call_kwargs) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == expected + + +def test_output_type_recorded_for_a_structured_request( + instrument_with_content, span_exporter, monkeypatch: pytest.MonkeyPatch +) -> None: + # VLLMModel is the only instrumented runtime that accepts a + # response_format; TransformersModel and MLXModel raise before generating. + model = vllm_model(monkeypatch) + + model.generate( + messages=MESSAGES, + response_format={ + "type": "json_schema", + "json_schema": {"schema": {"type": "object"}}, + }, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == "json" + + +@pytest.mark.parametrize( + "response_format, expected", + [ + ({"type": "json_object"}, "json"), + ({"type": "json_schema", "json_schema": {}}, "json"), + ({"type": "text"}, "text"), + (None, None), + # smolagents forwards response_format unchanged, so its type is + # whatever the runtime accepts. An unknown one is dropped rather than + # recorded on an enum attribute. + ({"type": "xml"}, None), + ({}, None), + ], +) +def test_output_type_mapping( + response_format: dict[str, Any] | None, expected: str | None +) -> None: + # A unit test rather than a generate() call: no instrumented runtime accepts + # every response_format spelling, and the mapping table is what needs + # testing. + merged = ( + {} if response_format is None else {"response_format": response_format} + ) + assert _output_type(merged) == expected + + +@pytest.mark.parametrize( + "model_kwargs, response_format, expected", + [ + # The model-level kwargs win over the argument, and the sentinel drops + # the key from the request, the same as for every other parameter. + ( + {"response_format": {"type": "text"}}, + {"type": "json_object"}, + "text", + ), + ({"response_format": "REMOVE"}, {"type": "json_object"}, None), + ({}, {"type": "json_object"}, "json"), + ], +) +def test_response_format_precedence( + model_kwargs: dict[str, Any], + response_format: dict[str, Any], + expected: str | None, +) -> None: + from smolagents.models import REMOVE_PARAMETER + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + merged = _merged_request_kwargs( + transformers_model(**model_kwargs), + {"response_format": response_format}, + ) + assert _output_type(merged) == expected + + +def _drain_stream(model: Any, **kwargs: Any) -> list[Any]: + return list(model.generate_stream(messages=MESSAGES, **kwargs)) + + +def _failing_streamer( + chunks: list[str], error: Exception +) -> Generator[str, None, None]: + yield from chunks + raise error + + +def test_generate_stream_is_lazy_and_records_the_drained_response( + instrument_with_content, span_exporter +) -> None: + model = transformers_model(stream_chunks=["In ", "Paris"]) + + stream = model.generate_stream(messages=MESSAGES) + # A streamed response isn't finished until the caller drains it. + assert spans_by_operation(span_exporter.get_finished_spans(), "chat") == [] + + deltas = list(stream) + assert "".join(delta.content or "" for delta in deltas) == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert attr(span, GenAI.GEN_AI_REQUEST_STREAM) is True + # TransformersModel reports the prompt tokens on the first delta only and + # one output token per delta. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + # Deltas carry no finish reason, and "stop" would hide a truncation. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_generate_stream_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Instrumentation observes; it must not change what generate_stream returns. + model = transformers_model(stream_chunks=["In Paris"]) + + stream = model.generate_stream(messages=MESSAGES) + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + list(stream) + + +def test_generate_stream_bad_call_records_an_error_span( + instrument_with_content, span_exporter, lifecycle +) -> None: + model = transformers_model(stream_chunks=["In Paris"]) + + # messages is required, so the call fails before it reaches the runtime. + # Nothing will drain the stream, so the span has to end here. + with pytest.raises(TypeError): + model.generate_stream() + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "TypeError" + assert lifecycle.leaked == [] + + +def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = transformers_model() + model.streamer = _failing_streamer(["In "], RuntimeError("kernel crashed")) + + with pytest.raises(RuntimeError, match="kernel crashed"): + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "RuntimeError" + # What was streamed before the failure is still recorded. + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "In "}] + + +def test_generate_stream_close_before_drain_finalizes_once( + instrument_with_content, span_exporter +) -> None: + model = transformers_model(stream_chunks=["In Paris"]) + + stream = model.generate_stream(messages=MESSAGES) + stream.close() + stream.close() # idempotent + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_generate_stream_records_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + model = transformers_model(stream_chunks=["In ", "Paris"]) + + _drain_stream(model) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK in metrics + ) + + +def test_generate_stream_no_content(instrument_no_content, span_exporter): + model = transformers_model(stream_chunks=["In ", "Paris"]) + + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + + +def test_to_tool_definitions_uses_json_schema() -> None: + (definition,) = to_tool_definitions([GetWeatherTool()]) + assert definition.name == "get_weather" + assert definition.description == "Get the weather for a given city" + # smolagents' raw ``inputs`` map is not a JSON Schema on its own. + assert definition.parameters == { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + } + + +def test_to_input_messages_maps_smolagents_only_roles() -> None: + # smolagents converts these roles itself inside generate(), after the + # wrapper has already read the messages. + messages = to_input_messages( + [ + {"role": MessageRole.TOOL_CALL, "content": "call"}, + {"role": MessageRole.TOOL_RESPONSE, "content": "response"}, + {"role": MessageRole.SYSTEM, "content": "sys"}, + ] + ) + assert [message.role for message in messages] == [ + "assistant", + "user", + "system", + ] + + +def test_to_input_messages_dict_and_chatmessage() -> None: + messages = to_input_messages( + [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ChatMessage(role=MessageRole.ASSISTANT, content="Hi there"), + ] + ) + assert messages[0].role == "user" + assert messages[0].parts == [Text(content="Hello")] + assert messages[1].role == "assistant" + assert messages[1].parts == [Text(content="Hi there")] + + +def test_to_input_messages_image_and_base64() -> None: + # A vision-capable runtime (TransformersModel with a processor, VLLMModel + # with _is_vlm) takes image parts alongside the text. + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + } + ] + ) + parts = messages[0].parts + assert parts[0] == Uri(mime_type=None, modality="image", uri=IMAGE_URL) + assert isinstance(parts[1], Blob) + assert parts[1].modality == "image" + assert parts[1].mime_type == "image/png" + assert isinstance(parts[1].content, bytes) + + +def test_to_input_messages_data_url_keeps_media_type() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "data:image/jpeg;base64,aVZCT1J3MEtHZ28=", + } + ], + } + ] + ) + (part,) = messages[0].parts + assert isinstance(part, Blob) + assert part.mime_type == "image/jpeg" + + +@pytest.mark.parametrize( + "image", + [ + "not base64!!!!", + # A path is not base64, but a non-validating decode silently turns this + # one into 15 bytes of garbage after dropping the invalid characters. + "/tmp/photos/my-cat.png", + ], +) +def test_to_input_messages_drops_malformed_base64_image(image: str) -> None: + messages = to_input_messages( + [{"role": "user", "content": [{"type": "image", "image": image}]}] + ) + assert messages[0].parts == [] + + +def test_to_output_message_unwraps_the_role_enum() -> None: + output = to_output_message( + ChatMessage(role=MessageRole.ASSISTANT, content="done") + ) + assert output.role == "assistant" + assert output.parts == [Text(content="done")] + # The in-process runtimes report no finish reason, and util-genai omits the + # empty value. + assert output.finish_reason == "" + + +def test_to_output_message_maps_image_content() -> None: + # A response whose content is a list carries the same element shapes as a + # request, so an image in it maps the same way. + output = to_output_message( + ChatMessage( + role=MessageRole.ASSISTANT, + content=[ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + ], + ) + ) + assert output.parts[0] == Text(content="Here it is") + assert output.parts[1] == Uri( + mime_type=None, modality="image", uri=IMAGE_URL + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py new file mode 100644 index 000000000..efec5fadb --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -0,0 +1,231 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers, tools, and model stubs for smolagents instrumentation tests. + +Only the in-process model classes are instrumented, and none of them can be +recorded with VCR: they run inference in the current process instead of calling +a provider over HTTP. Each factory here bypasses ``__init__`` (which would load +gigabytes of weights) and stubs the runtime pieces the real ``generate`` drives, +so the code under test is smolagents' own ``generate`` and the wrapper around +it. +""" + +from __future__ import annotations + +import json +import sys +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import Tool + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + + +class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + } + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + +# The in-process runtimes flatten a message's content as text, so the content +# has to be a list of parts rather than a bare string. +MESSAGES: list[dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": "Where is the Louvre?"}], + } +] + + +class _PromptTokens: + """The part of a torch tensor that ``TransformersModel.generate`` uses. + + It reads ``inputs.shape[1]`` for the prompt length and slices the generated + tail out of the model's output with ``out[0, prompt_length:]``. Moving to a + device and the ``input_ids`` unwrap are part of the same path. + """ + + def __init__(self, ids: list[int]) -> None: + self.ids = ids + + @property + def shape(self) -> tuple[int, int]: + return (1, len(self.ids)) + + def to(self, _device: Any) -> _PromptTokens: + return self + + def __getitem__(self, key: Any) -> list[int]: + _row, columns = key + return self.ids[columns] + + +def transformers_model( + prompt_ids: list[int] | None = None, + generated_ids: list[int] | None = None, + text: str = "In Paris", + error: Exception | None = None, + stream_chunks: list[str] | None = None, + **model_kwargs: Any, +) -> Any: + """A ``TransformersModel`` whose tokenizer and weights are stubbed. + + ``generate`` builds the prompt through ``self.tokenizer + .apply_chat_template``, counts its tokens, calls ``self.model.generate`` and + decodes the tail. ``generate_stream`` instead runs ``self.model.generate`` + on a thread and iterates ``self.streamer``, so the stubbed streamer is what + yields the deltas. + """ + from smolagents.models import TransformersModel + + prompt_ids = prompt_ids or [1, 2, 3] + generated_ids = generated_ids or [4, 5] + + def generate(**_: Any) -> Any: + if error is not None: + raise error + return _PromptTokens(prompt_ids + generated_ids) + + model = object.__new__(TransformersModel) + model.model_id = "HuggingFaceTB/SmolLM2-135M-Instruct" + model.kwargs = dict(model_kwargs) + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: _PromptTokens(prompt_ids), + decode=lambda ids, **_: text, + ) + model.model = SimpleNamespace(device="cpu", generate=generate) + model.streamer = iter(stream_chunks or []) + return model + + +def mlx_model(text: str = "In Paris", **model_kwargs: Any) -> Any: + """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. + + ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` + over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from + ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real + ``generate`` without the runtime installed. + """ + from smolagents.models import MLXModel + + model = object.__new__(MLXModel) + model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" + model.kwargs = dict(model_kwargs) + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.model = object() + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] + ) + # ``generate`` counts one output token per delta, so the deltas are the + # words of ``text`` and the output token count is the word count. + words = text.split(" ") + deltas = [ + SimpleNamespace(text=word if index == 0 else f" {word}") + for index, word in enumerate(words) + ] + model.stream_generate = lambda *_, **__: iter(deltas) + return model + + +def vllm_model( + monkeypatch: pytest.MonkeyPatch, + text: str = "In Paris", + prompt_token_ids: list[int] | None = None, + output_token_ids: list[int] | None = None, + **model_kwargs: Any, +) -> Any: + """A ``VLLMModel`` with ``vllm`` itself stubbed. + + ``VLLMModel.generate`` imports ``SamplingParams`` and + ``StructuredOutputsParams`` from ``vllm`` when it runs, and no test env + installs vllm, so both modules are faked for the duration of the test. + Everything the wrapper reads still comes from the real ``generate``. + """ + from smolagents.models import VLLMModel + + def fake_params(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) + + vllm = ModuleType("vllm") + sampling_params = ModuleType("vllm.sampling_params") + setattr(vllm, "SamplingParams", fake_params) + setattr(sampling_params, "StructuredOutputsParams", fake_params) + setattr(vllm, "sampling_params", sampling_params) + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + + completion = SimpleNamespace( + prompt_token_ids=prompt_token_ids or [1, 2, 3, 4], + outputs=[ + SimpleNamespace(text=text, token_ids=output_token_ids or [5, 6]) + ], + ) + model = object.__new__(VLLMModel) + model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" + model.kwargs = dict(model_kwargs) + model.flatten_messages_as_text = True + model._is_vlm = False + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: "prompt" + ) + model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) + return model + + +def spans_by_operation( + spans: list[ReadableSpan], operation: str +) -> list[ReadableSpan]: + return [ + span + for span in spans + if (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + == operation + ] + + +def attr(span: ReadableSpan, name: str) -> Any: + return (span.attributes or {}).get(name) + + +def parse_messages(span: ReadableSpan, name: str) -> list[dict[str, Any]]: + raw = attr(span, name) + return json.loads(raw) if isinstance(raw, str) else [] + + +def part_types(messages: list[dict[str, Any]]) -> list[str]: + return [part["type"] for message in messages for part in message["parts"]] + + +def metrics_by_name(metric_reader: Any) -> dict[str, Any]: + data = metric_reader.get_metrics_data() + if data is None: + return {} + return { + metric.name: metric + for resource_metric in data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + +def data_point_attributes(metric: Any) -> list[dict[str, Any]]: + return [dict(point.attributes) for point in metric.data.data_points] diff --git a/tox.ini b/tox.ini index ce3694aa0..b202ba514 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ envlist = ; instrumentation-genai-smolagents py3{10,11,12,13,14}-test-instrumentation-genai-smolagents-latest py310-test-instrumentation-genai-smolagents-oldest + py314-test-instrumentation-genai-smolagents-conformance lint-instrumentation-genai-smolagents ; instrumentation-genai-anthropic @@ -157,6 +158,8 @@ deps = smolagents-latest: {[testenv]test_deps} smolagents-latest: {[testenv]pytest_deps} smolagents-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt + smolagents-conformance: {[testenv]pytest_deps} + smolagents-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt anthropic-oldest: {[testenv]pytest_deps} anthropic-oldest: -e {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic[instruments] @@ -257,6 +260,7 @@ commands = lint-instrumentation-genai-agno: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-agno" test-instrumentation-genai-smolagents-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests --vcr-record=none {posargs} + test-instrumentation-genai-smolagents-conformance: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py --vcr-record=none {posargs} lint-instrumentation-genai-smolagents: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-smolagents" test-instrumentation-genai-anthropic-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests --vcr-record=none {posargs}