diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index d31e23aa..32646aae 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -26,12 +26,35 @@ jobs: - name: Check formatting with Black run: tox -e black + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e .[dev] + python -m pip install -e "src/_langchain/opentelemetry-instrumentation-langchain[instruments,test]" - - name: Lint with Pylint + - name: Check formatting with Black + run: black --check src tests samples + + - name: Lint with Pylint (root) + run: pylint src tests + + - name: Lint with Pylint (langchain) + working-directory: src/_langchain/opentelemetry-instrumentation-langchain run: tox -e pylint - - name: Typecheck with Mypy - run: tox -e mypy + - name: Typecheck with Mypy (root) + run: mypy src + + - name: Typecheck with Mypy (langchain) + working-directory: src/_langchain/opentelemetry-instrumentation-langchain + run: mypy src --ignore-missing-imports --allow-untyped-defs + + - name: Typecheck with Pyright (root) + run: pyright src + + - name: Typecheck with Pyright (langchain) + working-directory: src/_langchain/opentelemetry-instrumentation-langchain + run: pyright src tests: name: Pytest (${{ matrix.python-version }}) diff --git a/mypy.ini b/mypy.ini index 2864dd03..e75b7aa7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,6 +6,7 @@ disallow_untyped_defs = true namespace_packages = true explicit_package_bases = true mypy_path = src +exclude = (?x)(src/_langchain/ | src/_a365/) [mypy-microsoft.azureMonitor.*] disallow_untyped_defs = false diff --git a/pyproject.toml b/pyproject.toml index d21628a8..43573a07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ target-version = ["py310"] [tool.pylint.main] py-version = "3.10" +ignore-paths = ["tests/azureMonitor", "src/_langchain"] [tool.pylint.format] max-line-length = 120 @@ -65,5 +66,34 @@ disable = [ "invalid-name", ] +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +namespace_packages = true +explicit_package_bases = true +mypy_path = "src" +exclude = ["src/_langchain"] + +[[tool.mypy.overrides]] +module = "microsoft.azureMonitor.*" +disallow_untyped_defs = false +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = [ + "django.*", + "azure.core.*", + "opentelemetry.instrumentation.*", +] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] + +[tool.pyright] +pythonVersion = "3.10" +typeCheckingMode = "basic" +ignore = ["src/microsoft/azureMonitor", "tests/azureMonitor"] +exclude = ["src/_langchain"] diff --git a/samples/langchain/sample_langchain_azure.py b/samples/langchain/sample_langchain_azure.py index a43f5a71..caf6aa7a 100644 --- a/samples/langchain/sample_langchain_azure.py +++ b/samples/langchain/sample_langchain_azure.py @@ -117,8 +117,7 @@ def find_activities(destination: str) -> str: def main(): session_id = str(uuid4()) user_request = ( - "Plan a weekend trip to Paris from Seattle next month. " - + "Find flights, a boutique hotel, and fun activities." + "Plan a weekend trip to Paris from Seattle next month. Find flights, a boutique hotel, and fun activities." ) print("Travel Planner (Nested Agents)") diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt b/src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt new file mode 100644 index 00000000..d9f1b3b9 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt @@ -0,0 +1,2 @@ +pytest>=8.0 +pytest-recording diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml new file mode 100644 index 00000000..5cdb51f3 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml @@ -0,0 +1,78 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-instrumentation-langchain" +dynamic = ["version"] +description = "OpenTelemetry Official Langchain instrumentation" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "opentelemetry-api ~= 1.40", + "opentelemetry-instrumentation ~= 0.61b0", + "opentelemetry-semantic-conventions ~= 0.61b0", +] + +[project.optional-dependencies] +instruments = [ + "langchain >= 0.3.21", + "langchain-core >= 0.3", +] +test = [ + "opentelemetry-sdk ~= 1.40", + "langchain-openai >= 0.3", + "openai >= 1.0", + "PyYAML >= 6.0", + "vcrpy >= 6.0", + "pytest >= 8.0", + "pytest-recording", +] + +[project.entry-points.opentelemetry_instrumentor] +langchain = "opentelemetry.instrumentation.langchain:LangChainInstrumentor" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/langchain/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] + +[tool.pylint.format] +max-line-length = 120 + +[tool.pylint.messages_control] +disable = [ + "missing-function-docstring", + "missing-module-docstring", + "import-error", + "no-name-in-module", + "too-many-positional-arguments", +] + +[tool.pyright] +pythonVersion = "3.9" +typeCheckingMode = "basic" +reportMissingImports = false diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py new file mode 100644 index 00000000..3ce3b978 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -0,0 +1,122 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Langchain instrumentation supporting `ChatOpenAI` and `ChatBedrock`, it can be enabled by +using ``LangChainInstrumentor``. Other providers/LLMs may be supported in the future and +telemetry for them is skipped for now. + +Usage +----- +.. code:: python + from opentelemetry.instrumentation.langchain import LangChainInstrumentor + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_openai import ChatOpenAI + + LangChainInstrumentor().instrument() + llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, max_tokens=1000) + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + result = llm.invoke(messages) + LangChainInstrumentor().uninstrument() + +API +--- +""" + +from typing import Any, Callable, Collection + +from langchain_core.callbacks import BaseCallbackHandler +from wrapt import wrap_function_wrapper # type: ignore + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore[attr-defined] +from opentelemetry.instrumentation.langchain.callback_handler import ( + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.instrumentation.langchain.package import _instruments +from opentelemetry.instrumentation.utils import unwrap +from opentelemetry.util.genai.handler import get_telemetry_handler + + +class LangChainInstrumentor(BaseInstrumentor): + """ + OpenTelemetry instrumentor for LangChain. + This adds a custom callback handler to the LangChain callback manager + to capture LLM telemetry. + """ + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments # type: ignore[no-any-return] + + def _instrument(self, **kwargs: Any): + """ + Enable Langchain instrumentation. + """ + tracer_provider = kwargs.get("tracer_provider") + meter_provider = kwargs.get("meter_provider") + logger_provider = kwargs.get("logger_provider") + + telemetry_handler = get_telemetry_handler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + otel_callback_handler = OpenTelemetryLangChainCallbackHandler( + telemetry_handler=telemetry_handler, + ) + + wrap_function_wrapper( + module="langchain_core.callbacks", + name="BaseCallbackManager.__init__", + wrapper=_BaseCallbackManagerInitWrapper(otel_callback_handler), + ) + + def _uninstrument(self, **kwargs: Any): + """ + Cleanup instrumentation (unwrap). + """ + unwrap("langchain_core.callbacks.base.BaseCallbackManager", "__init__") + # Clear the TelemetryHandler singleton so the next instrument() uses + # the provided tracer_provider/meter_provider/logger_provider instead + # of reusing the previous handler. + if getattr(get_telemetry_handler, "_default_handler", None) is not None: + delattr(get_telemetry_handler, "_default_handler") + + +class _BaseCallbackManagerInitWrapper: # pylint: disable=too-few-public-methods + """ + Wrap the BaseCallbackManager __init__ to insert custom callback handler in the manager's handlers list. + """ + + def __init__( # pylint: disable=too-few-public-methods + self, callback_handler: OpenTelemetryLangChainCallbackHandler + ): + self._otel_handler = callback_handler + + def __call__( + self, + wrapped: Callable[..., None], + instance: BaseCallbackHandler, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ): + wrapped(*args, **kwargs) + # Ensure our OTel callback is present if not already. + for handler in instance.inheritable_handlers: # type: ignore + if isinstance(handler, type(self._otel_handler)): + break + else: + instance.add_handler(self._otel_handler, inherit=True) # type: ignore diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py new file mode 100644 index 00000000..15936ed7 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -0,0 +1,237 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any, Optional, cast +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.messages import BaseMessage +from langchain_core.outputs import LLMResult + +from opentelemetry.instrumentation.langchain.invocation_manager import ( + _InvocationManager, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.types import ( + Error, + InputMessage, + LLMInvocation, + MessagePart, + OutputMessage, + Text, +) + + +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches, unused-argument +class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): + """ + A callback handler for LangChain that uses OpenTelemetry to create spans for LLM calls and + chains, tools etc,. in future. + """ + + def __init__(self, telemetry_handler: TelemetryHandler) -> None: + super().__init__() + self._telemetry_handler = telemetry_handler + self._invocation_manager = _InvocationManager() + + def on_chat_model_start( # pylint: disable=too-many-statements + self, + serialized: dict[str, Any], + messages: list[list[BaseMessage]], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + metadata: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + # Other providers/LLMs may be supported in the future and telemetry for them is skipped for now. + if serialized.get("name") not in ("ChatOpenAI", "ChatBedrock"): + return + + if "invocation_params" in kwargs: + params = kwargs["invocation_params"].get("params") or kwargs["invocation_params"] + else: + params = kwargs + + request_model = "unknown" + for model_tag in ( + "model_name", # ChatOpenAI + "model_id", # ChatBedrock + ): + if (model := (params or {}).get(model_tag)) is not None: # pylint: disable=no-else-break + request_model = model + break + elif (model := (metadata or {}).get(model_tag)) is not None: + request_model = model + break + + # Skip telemetry for unsupported request models + if request_model == "unknown": + return + + # Initialize variables with default values to avoid "possibly unbound" errors + top_p = None + frequency_penalty = None + presence_penalty = None + stop_sequences = None + seed = None + temperature = None + max_tokens = None + + if params is not None: + top_p = params.get("top_p") + frequency_penalty = params.get("frequency_penalty") + presence_penalty = params.get("presence_penalty") + stop_sequences = params.get("stop") + seed = params.get("seed") + temperature = params.get("temperature") + max_tokens = params.get("max_completion_tokens") + + provider = "unknown" + if metadata is not None: + provider = metadata.get("ls_provider", "unknown") + + # Override with ChatBedrock values if present + if "ls_temperature" in metadata: + temperature = metadata.get("ls_temperature") + if "ls_max_tokens" in metadata: + max_tokens = metadata.get("ls_max_tokens") + + input_messages: list[InputMessage] = [] + for sub_messages in messages: # pylint: disable=too-many-nested-blocks + for message in sub_messages: + # Cast to Any to avoid type checking issues with LangChain's complex content type + raw_content: Any = message.content # type: ignore[misc] + role = message.type + parts: list[Text] = [] + + if isinstance(raw_content, str): + parts = [Text(content=raw_content, type="text")] + elif isinstance(raw_content, list): + for item in raw_content: # type: ignore[misc] + if isinstance(item, str): + parts.append(Text(content=item, type="text")) + elif isinstance(item, dict): + # Safely extract text content from dict + text_value = item.get("text") # type: ignore[misc] + if isinstance(text_value, str) and text_value: + parts.append(Text(content=text_value, type="text")) + + input_messages.append(InputMessage(parts=cast(list[MessagePart], parts), role=role)) + + llm_invocation = LLMInvocation( + request_model=request_model, + input_messages=input_messages, + provider=provider, + top_p=top_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + stop_sequences=stop_sequences, + seed=seed, + temperature=temperature, + max_tokens=max_tokens, + ) + llm_invocation = self._telemetry_handler.start_llm(invocation=llm_invocation) + self._invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=parent_run_id, + invocation=llm_invocation, + ) + + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) + if llm_invocation is None or not isinstance(llm_invocation, LLMInvocation): + # If the invocation does not exist, we cannot set attributes or end it + return + + output_messages: list[OutputMessage] = [] + for generation in getattr(response, "generations", []): + for chat_generation in generation: + # Get finish reason + finish_reason = "unknown" # Default value + generation_info = getattr(chat_generation, "generation_info", None) + if generation_info is not None: + finish_reason = generation_info.get("finish_reason", "unknown") + + if chat_generation.message: + # Get finish reason if generation_info is None above + if generation_info is None and chat_generation.message.response_metadata: + finish_reason = chat_generation.message.response_metadata.get("stopReason", "unknown") + + # Get message content + parts = [ + Text( + content=chat_generation.message.content, + type="text", + ) + ] + role = chat_generation.message.type + output_message = OutputMessage( + role=role, + parts=cast(list[MessagePart], parts), + finish_reason=finish_reason, + ) + output_messages.append(output_message) + + # Get token usage if available + if chat_generation.message.usage_metadata: + input_tokens = chat_generation.message.usage_metadata.get("input_tokens", 0) + llm_invocation.input_tokens = input_tokens + + output_tokens = chat_generation.message.usage_metadata.get("output_tokens", 0) + llm_invocation.output_tokens = output_tokens + + llm_invocation.output_messages = output_messages + + llm_output = getattr(response, "llm_output", None) + if llm_output is not None: + response_model = llm_output.get("model_name") or llm_output.get("model") + if response_model is not None: + llm_invocation.response_model_name = str(response_model) + + response_id = llm_output.get("id") + if response_id is not None: + llm_invocation.response_id = str(response_id) + + llm_invocation = self._telemetry_handler.stop_llm(invocation=llm_invocation) + if llm_invocation.span and not llm_invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id=run_id) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) + if llm_invocation is None or not isinstance(llm_invocation, LLMInvocation): + # If the invocation does not exist, we cannot set attributes or end it + return + + error_otel = Error(message=str(error), type=type(error)) + llm_invocation = self._telemetry_handler.fail_llm(invocation=llm_invocation, error=error_otel) + if llm_invocation.span and not llm_invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id=run_id) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py new file mode 100644 index 00000000..9b55056d --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -0,0 +1,61 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from uuid import UUID + +from opentelemetry.util.genai.types import GenAIInvocation + +__all__ = ["_InvocationManager"] + + +@dataclass +class _InvocationState: + invocation: GenAIInvocation + children: List[UUID] = field(default_factory=lambda: []) # pylint: disable=unnecessary-lambda + + +class _InvocationManager: + def __init__( + self, + ) -> None: + # Map from run_id -> _InvocationState, to keep track of invocations and parent/child relationships + # TODO: TTL cache to avoid memory leaks in long-running processes. # pylint: disable=fixme + self._invocations: Dict[UUID, _InvocationState] = {} + + def add_invocation_state( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + invocation: GenAIInvocation, + ): + invocation_state = _InvocationState(invocation=invocation) + self._invocations[run_id] = invocation_state + + if parent_run_id is not None and parent_run_id in self._invocations: + parent_invocation_state = self._invocations[parent_run_id] + parent_invocation_state.children.append(run_id) + + def get_invocation(self, run_id: UUID) -> Optional[GenAIInvocation]: + invocation_state = self._invocations.get(run_id) + return invocation_state.invocation if invocation_state else None + + def delete_invocation_state(self, run_id: UUID) -> None: + invocation_state = self._invocations.get(run_id) + if not invocation_state: + return + for child_id in list(invocation_state.children): + self._invocations.pop(child_id, None) + self._invocations.pop(run_id, None) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py new file mode 100644 index 00000000..1aaafae8 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +_instruments = ("langchain >= 0.3.21",) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py new file mode 100644 index 00000000..61ae9b7c --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "2.0b0.dev" diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/_init_.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/_init_.py new file mode 100644 index 00000000..e69de29b diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml new file mode 100644 index 00000000..911b44b1 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml @@ -0,0 +1,161 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "frequency_penalty": 0.5, + "max_completion_tokens": 100, + "presence_penalty": 0.5, + "seed": 100, + "stop": [ + "\n", + "Human:", + "AI:" + ], + "stream": false, + "temperature": 0.1, + "top_p": 0.9 + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '316' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.21.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.21.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.5 + authorization: + - Bearer test_openai_api_key + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-DB5VKnX0DBvthDkYqZDgfZVAqhZUk", + "object": "chat.completion", + "created": 1771535138, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 7, + "total_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null + } + headers: + CF-RAY: + - 9d08b930cee533ed-SJC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 19 Feb 2026 21:05:38 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '822' + openai-organization: test_openai_org_id + openai-processing-ms: + - '170' + openai-project: + - proj_GLiYlAc06hF0Fm06IMReZLy4 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=tYjXaOj8KokJGc56XWDShqpm2zPWeq3Q19yhnJcqKqU-1771535137.4012983-1.0.1.1-k50ojuWx4sSPmgdu8QLKD3OmGYDMjKIDSqQcANN1wo1SBWNjIE.yAsOYOgaalvDiPrXObJN80xhnlcC9mXOXnOfGb9HRyT5un2ASAG29wbujzo_b9a3zQg9hHU024Nkz; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 19 Feb 2026 + 21:35:38 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199982' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 5ms + x-request-id: + - req_5f037f577c544db292661f8bbf12d569 + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml new file mode 100644 index 00000000..43d099b5 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml @@ -0,0 +1,115 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "frequency_penalty": 0.5, + "max_completion_tokens": 100, + "presence_penalty": 0.5, + "seed": 100, + "stop": [ + "\n", + "Human:", + "AI:" + ], + "stream": false, + "temperature": 0.1, + "top_p": 0.9 + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '316' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.21.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.21.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.5 + authorization: + - Bearer test_openai_api_key + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "error": { + "message": "Incorrect API key provided: test_ope*******_key. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": null, + "code": "invalid_api_key" + } + } + headers: + CF-RAY: + - 9d08b96c8d2b67d7-SJC + Connection: + - keep-alive + Content-Length: + - '269' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 19 Feb 2026 21:05:46 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + set-cookie: + - __cf_bm=wRTe.yg21t4Hmlnym6umYHo41PFme8jHV65z7Ai6_04-1771535146.9666395-1.0.1.1-V3j6uPp1b7mudGWLLSB5Vk_zC3O8FBmCjRrguc_.hixN.Zy67xOtEeE__VwyjmXrWSnJnjHz2VO2RUr.6C04qr2J8dPfV5._bQfv4thKMonsCohocrVcwmwwfgF6GtYD; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 19 Feb 2026 + 21:35:46 GMT + - _cfuvid=FiOO9id9BeHCp.jp23WGFYuUJAqS3RdB2T1lLaroTEs-1771535146.9666395-1.0.1.1-GwFV1AQJkfqXaVwhyb_WBpNzyDw3zVtMUGJb8rtOMFg; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com + vary: + - Origin + x-openai-proxy-wasm: + - v0.1 + x-request-id: + - req_71913c60957d49b29348e7966f3616d6 + status: + code: 401 + message: Unauthorized +version: 1 \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py new file mode 100644 index 00000000..5dbf8797 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -0,0 +1,230 @@ +"""Unit tests configuration module.""" + +import json +import os + +import pytest +import yaml +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogRecordExporter, + SimpleLogRecordProcessor, +) +from opentelemetry.sdk.metrics import ( + MeterProvider, +) +from opentelemetry.sdk.metrics.export import ( + InMemoryMetricReader, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +# pylint: disable=protected-access, redefined-outer-name +@pytest.fixture(scope="module", name="chat_openai_gpt_3_5_turbo_model") +def fixture_chat_openai_gpt_3_5_turbo_model(): + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + yield llm + + +@pytest.fixture(name="span_exporter") +def fixture_span_exporter(): + exporter = InMemorySpanExporter() + yield exporter + + +@pytest.fixture(name="log_exporter") +def fixture_log_exporter(): + exporter = InMemoryLogRecordExporter() + yield exporter + + +@pytest.fixture(name="metric_reader") +def fixture_metric_reader(): + exporter = InMemoryMetricReader() + yield exporter + + +@pytest.fixture(name="tracer_provider") +def fixture_tracer_provider(span_exporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + return provider + + +@pytest.fixture(name="logger_provider") +def fixture_logger_provider(log_exporter): + provider = LoggerProvider() + provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + return provider + + +@pytest.fixture(name="meter_provider") +def fixture_meter_provider(metric_reader): + meter_provider = MeterProvider( + metric_readers=[metric_reader], + ) + return meter_provider + + +@pytest.fixture() +def start_instrumentation( + tracer_provider, + meter_provider, + logger_provider, +): + # pylint: disable=import-outside-toplevel + from opentelemetry.instrumentation._semconv import ( + _OpenTelemetrySemanticConventionStability, + ) + + # Save prior state + prev_env = os.environ.get("OTEL_SEMCONV_STABILITY_OPT_IN") + prev_initialized = _OpenTelemetrySemanticConventionStability._initialized + prev_mapping = _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING.copy() + + # Ensure stability opt-in is set before the first call to _initialize() + os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = "gen_ai_latest_experimental" + # Reset the cached initialization so the env var is picked up + _OpenTelemetrySemanticConventionStability._initialized = False + _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING = {} + _OpenTelemetrySemanticConventionStability._initialize() + + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + + yield instrumentor + instrumentor.uninstrument() + + # Restore prior state + if prev_env is None: + os.environ.pop("OTEL_SEMCONV_STABILITY_OPT_IN", None) + else: + os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = prev_env + _OpenTelemetrySemanticConventionStability._initialized = prev_initialized + _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING = prev_mapping + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = "test_openai_api_key" + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } + + +class LiteralBlockScalar(str): + """Formats the string as a literal block scalar, preserving whitespace and + without interpreting escape characters""" + + +def literal_block_scalar_presenter(dumper, data): + """Represents a scalar string as a literal block, via '|' syntax""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.add_representer(LiteralBlockScalar, literal_block_scalar_presenter) + + +def process_string_value(string_value): + """Pretty-prints JSON or returns long strings as a LiteralBlockScalar""" + try: + json_data = json.loads(string_value) + return LiteralBlockScalar(json.dumps(json_data, indent=2)) + except (ValueError, TypeError): + if len(string_value) > 80: + return LiteralBlockScalar(string_value) + return string_value + + +def convert_body_to_literal(data): + """Searches the data for body strings, attempting to pretty-print JSON""" + if isinstance(data, dict): + for key, value in data.items(): + # Handle response body case (e.g., response.body.string) + if key == "body" and isinstance(value, dict) and "string" in value: + value["string"] = process_string_value(value["string"]) + + # Handle request body case (e.g., request.body) + elif key == "body" and isinstance(value, str): + data[key] = process_string_value(value) + + else: + convert_body_to_literal(value) + + elif isinstance(data, list): + for idx, choice in enumerate(data): + data[idx] = convert_body_to_literal(choice) + + return data + + +class PrettyPrintJSONBody: + """This makes request and response body recordings more readable.""" + + @staticmethod + def serialize(cassette_dict): + cassette_dict = convert_body_to_literal(cassette_dict) + return yaml.dump(cassette_dict, default_flow_style=False, allow_unicode=True) + + @staticmethod + def deserialize(cassette_string): + return yaml.safe_load(cassette_string) + + +@pytest.fixture(scope="module", autouse=True) +def fixture_vcr(vcr_config): + import vcr as vcrpy # pylint: disable=import-outside-toplevel + + my_vcr = vcrpy.VCR(**vcr_config) + my_vcr.register_serializer("yaml", PrettyPrintJSONBody) + my_vcr.record_mode = "none" + my_vcr.serializer = "yaml" + my_vcr.cassette_library_dir = os.path.join(os.path.dirname(__file__), "cassettes") + return my_vcr + + +@pytest.fixture() +def vcr(fixture_vcr): + """Function-scoped vcr fixture that replaces pytest-recording's default one.""" + return fixture_vcr + + +def scrub_response_headers(response): + """ + This scrubs sensitive response headers. Note they are case-sensitive! + """ + response["headers"]["openai-organization"] = "test_openai_org_id" + response["headers"]["Set-Cookie"] = "test_set_cookie" + return response diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py new file mode 100644 index 00000000..a99e05b6 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py @@ -0,0 +1,139 @@ +# tests/test_invocation_manager.py +# pylint: disable=redefined-outer-name +import uuid +from unittest import mock + +import pytest + +from opentelemetry.instrumentation.langchain.invocation_manager import ( + _InvocationManager, +) +from opentelemetry.util.genai.types import GenAIInvocation # pylint: disable=no-name-in-module + + +@pytest.fixture +def invocation_manager(): + return _InvocationManager() + + +@pytest.fixture +def mock_invocation(): + return mock.Mock(spec=GenAIInvocation) + + +# pylint: disable=protected-access +def test_add_invocation_state_without_parent(invocation_manager, mock_invocation): + run_id = uuid.uuid4() + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=None, + invocation=mock_invocation, + ) + + assert invocation_manager.get_invocation(run_id) == mock_invocation + assert len(invocation_manager._invocations) == 1 + assert invocation_manager._invocations[run_id].children == [] + + +def test_add_invocation_state_with_parent(invocation_manager): + parent_id = uuid.uuid4() + child_id = uuid.uuid4() + parent_invocation = mock.Mock(spec=GenAIInvocation) + child_invocation = mock.Mock(spec=GenAIInvocation) + + # Add parent first + invocation_manager.add_invocation_state( + run_id=parent_id, + parent_run_id=None, + invocation=parent_invocation, + ) + + # Then add child with parent reference + invocation_manager.add_invocation_state( + run_id=child_id, + parent_run_id=parent_id, + invocation=child_invocation, + ) + + # Check that parent has child in its children list + assert child_id in invocation_manager._invocations[parent_id].children + assert invocation_manager.get_invocation(child_id) == child_invocation + assert invocation_manager.get_invocation(parent_id) == parent_invocation + + +def test_add_invocation_state_with_nonexistent_parent(invocation_manager, mock_invocation): + run_id = uuid.uuid4() + nonexistent_parent_id = uuid.uuid4() + + # Adding with a parent that doesn't exist should still add the child without error + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=nonexistent_parent_id, + invocation=mock_invocation, + ) + + assert invocation_manager.get_invocation(run_id) == mock_invocation + assert len(invocation_manager._invocations) == 1 + + +def test_get_nonexistent_invocation(invocation_manager): + nonexistent_id = uuid.uuid4() + assert invocation_manager.get_invocation(nonexistent_id) is None + + +def test_delete_invocation_state(invocation_manager, mock_invocation): + run_id = uuid.uuid4() + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=None, + invocation=mock_invocation, + ) + + # Verify it was added + assert invocation_manager.get_invocation(run_id) == mock_invocation + + # Delete it + invocation_manager.delete_invocation_state(run_id) + + # Verify it was removed + assert run_id not in invocation_manager._invocations + + +def test_delete_invocation_state_with_children(invocation_manager): + parent_id = uuid.uuid4() + child1_id = uuid.uuid4() + child2_id = uuid.uuid4() + + parent_invocation = mock.Mock(spec=GenAIInvocation) + child1_invocation = mock.Mock(spec=GenAIInvocation) + child2_invocation = mock.Mock(spec=GenAIInvocation) + + # Add parent and children + invocation_manager.add_invocation_state( + run_id=parent_id, + parent_run_id=None, + invocation=parent_invocation, + ) + invocation_manager.add_invocation_state( + run_id=child1_id, + parent_run_id=parent_id, + invocation=child1_invocation, + ) + invocation_manager.add_invocation_state( + run_id=child2_id, + parent_run_id=parent_id, + invocation=child2_invocation, + ) + + # Verify initial state + assert len(invocation_manager._invocations) == 3 + assert len(invocation_manager._invocations[parent_id].children) == 2 + + # Delete parent + invocation_manager.delete_invocation_state(parent_id) + + # Verify parent and all children were removed + assert parent_id not in invocation_manager._invocations + assert child1_id not in invocation_manager._invocations + assert child2_id not in invocation_manager._invocations + assert len(invocation_manager._invocations) == 0 diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py new file mode 100644 index 00000000..29946366 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py @@ -0,0 +1,442 @@ +from typing import Optional + +# pylint: disable=no-name-in-module +import pytest +from langchain_core.messages import HumanMessage, SystemMessage +from openai import AuthenticationError + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + event_attributes as EventAttributes, +) +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import error_attributes + + +# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming +# from fixtures defined in conftest.py +@pytest.mark.vcr() +@pytest.mark.parametrize( + "capture_content", + ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], +) + +# pylint: disable=too-many-arguments, too-many-locals, unused-argument +def test_chat_openai_gpt_3_5_turbo_model_llm_call( + span_exporter, + metric_reader, + log_exporter, + start_instrumentation, + chat_openai_gpt_3_5_turbo_model, + monkeypatch, + capture_content, + vcr, +): + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + with vcr.use_cassette("test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml"): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + assert response.content == "The capital of France is Paris." + + # verify spans + spans = span_exporter.get_finished_spans() + + verify_content = capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") + assert_openai_completion_attributes( + spans[0] if len(spans) > 0 else None, + response, + verify_content=verify_content, + ) + + # verify metrics + metrics_data = metric_reader.get_metrics_data() + assert metrics_data is not None + resource_metrics = metrics_data.resource_metrics + assert resource_metrics is not None + assert len(resource_metrics) == 1 + + metrics = resource_metrics[0].scope_metrics[0].metrics + assert len(metrics) == 2 + for metric in metrics: + if metric.name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION: + assert_duration_metric(metric, spans[0]) + if metric.name == gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE: + assert_token_usage_metric(metric, spans[0]) + + # verify logs + logs = log_exporter.get_finished_logs() + if capture_content in ("SPAN_AND_EVENT", "EVENT_ONLY"): + assert len(logs) == 1 + log_record = logs[0].log_record + assert_log_record(log_record, spans[0]) + elif capture_content in ("SPAN_ONLY", "NO_CONTENT"): + assert len(logs) == 0 + + +# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are +# coming from fixtures defined in conftest.py +@pytest.mark.vcr() +@pytest.mark.parametrize( + "capture_content", + ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], +) + +# pylint: disable=too-many-arguments, too-many-locals, unused-argument +def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( + span_exporter, + metric_reader, + log_exporter, + start_instrumentation, + chat_openai_gpt_3_5_turbo_model, + monkeypatch, + capture_content, + vcr, +): + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + response = None + try: + with vcr.use_cassette("test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml"): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + except Exception as e: # pylint: disable=broad-exception-caught + # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. + assert isinstance(e, AuthenticationError) + + assert response is None + + # verify spans + spans = span_exporter.get_finished_spans() + + verify_content = capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") + assert_openai_completion_attributes_with_error(spans[0] if len(spans) > 0 else None, verify_content=verify_content) + + # verify metrics + metrics_data = metric_reader.get_metrics_data() + assert metrics_data is not None + resource_metrics = metrics_data.resource_metrics + assert len(resource_metrics) == 1 + metrics = resource_metrics[0].scope_metrics[0].metrics + assert len(metrics) == 1 + assert metrics[0].name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + assert_duration_metric_when_error(metrics[0], spans[0]) + + # verify logs + logs = log_exporter.get_finished_logs() + if capture_content in ("SPAN_AND_EVENT", "EVENT_ONLY"): + assert len(logs) == 1 + log_record = logs[0].log_record + assert_log_record_when_error(log_record, spans[0]) + elif capture_content in ("SPAN_ONLY", "NO_CONTENT"): + assert len(logs) == 0 + + +def assert_openai_completion_attributes(span: ReadableSpan, response: Optional, verify_content: bool = True): + assert span is not None + attributes = span.attributes + assert span.name == "chat gpt-3.5-turbo" + assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" + assert attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo-0125" + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 + assert attributes["gen_ai.provider.name"] == "openai" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID in attributes + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.5 + stop_sequences = attributes.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES) + assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 + + input_tokens = response.response_metadata.get("token_usage").get("prompt_tokens") + if input_tokens: + assert input_tokens == attributes[gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS] + else: + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attributes + + output_tokens = response.response_metadata.get("token_usage").get("completion_tokens") + if output_tokens: + assert output_tokens == attributes[gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS] + else: + assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attributes + + if verify_content: + input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] + assert input_message is not None + assert '"role":"system"' in input_message + assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"human"' in input_message + assert '"content":"What is the capital of France?"' in input_message + + # Assert output message + output_message = attributes[gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES] + assert output_message is not None + assert '"role":"ai"' in output_message + assert '"content":"The capital of France is Paris."' in output_message + assert '"finish_reason":"stop"' in output_message + else: + assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + + +def assert_openai_completion_attributes_with_error(span: ReadableSpan, verify_content: bool = True): + assert span is not None + assert span.name == "chat gpt-3.5-turbo" + attributes = span.attributes + assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" + assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" + assert gen_ai_attributes.GEN_AI_RESPONSE_MODEL not in attributes + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 + assert attributes["gen_ai.provider.name"] == "openai" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID not in attributes + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.5 + stop_sequences = attributes.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES) + assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 + + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attributes + + assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attributes + + if verify_content: + input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] + assert input_message is not None + assert '"role":"system"' in input_message + assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"human"' in input_message + assert '"content":"What is the capital of France?"' in input_message + + # Assert output message + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + else: + assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + + +def assert_duration_metric(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 1 + assert metric.data.data_points[0].sum > 0 + + assert_duration_metric_attributes(metric.data.data_points[0].attributes, parent_span) + assert_exemplars( + metric.data.data_points[0].exemplars, + metric.data.data_points[0].sum, + parent_span, + ) + + +def assert_duration_metric_attributes(attributes, parent_span): + assert len(attributes) == 4 + assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert ( + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] + ) + + +def assert_duration_metric_when_error(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 1 + assert metric.data.data_points[0].sum > 0 + + assert_duration_metric_attributes_when_error(metric.data.data_points[0].attributes, parent_span) + assert_exemplars( + metric.data.data_points[0].exemplars, + metric.data.data_points[0].sum, + parent_span, + ) + + +def assert_duration_metric_attributes_when_error(attributes, parent_span): + assert len(attributes) == 4 + assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" + assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert ( + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + ) + + +def assert_token_usage_metric(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 2 + + assert metric.data.data_points[0].sum > 0 + assert_token_usage_metric_attributes(metric.data.data_points[0].attributes, parent_span, "input") + assert_exemplars( + metric.data.data_points[0].exemplars, + metric.data.data_points[0].sum, + parent_span, + ) + + assert metric.data.data_points[1].sum > 0 + assert_token_usage_metric_attributes(metric.data.data_points[1].attributes, parent_span, "output") + assert_exemplars( + metric.data.data_points[1].exemplars, + metric.data.data_points[1].sum, + parent_span, + ) + + +def assert_token_usage_metric_attributes(attributes, parent_span, token_type): + assert len(attributes) == 5 + assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert ( + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] + ) + assert attributes.get(gen_ai_attributes.GEN_AI_TOKEN_TYPE) == token_type + + +def assert_exemplars(exemplars, datapoint_sum, parent_span): + assert len(exemplars) == 1 + assert exemplars[0].value == datapoint_sum + assert exemplars[0].span_id == parent_span.get_span_context().span_id + assert exemplars[0].trace_id == parent_span.get_span_context().trace_id + + +def assert_log_record(log_record, parent_span): + # Event name (support both .event_name and attributes for SDK differences) + event_name = getattr(log_record, "event_name", None) or log_record.attributes.get(EventAttributes.EVENT_NAME) + assert event_name == "gen_ai.client.inference.operation.details" + + attrs = log_record.attributes + assert attrs.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == "chat" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == "gpt-3.5-turbo" + assert attrs.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TOP_P) == 0.9 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) == 100 + stop_seqs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, [])) + assert stop_seqs == ["\n", "Human:", "AI:"] + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_SEED) == 100 + assert attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) == "gpt-3.5-turbo-0125" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID in attrs + assert attrs.get(gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS) == 24 + assert attrs.get(gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS) == 7 + + # Input/output messages: normalize list/tuple and compare structure + input_msgs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, [])) + expected_input = [ + { + "parts": [{"content": "You are a helpful assistant!", "type": "text"}], + "role": "system", + }, + { + "parts": [{"content": "What is the capital of France?", "type": "text"}], + "role": "human", + }, + ] + assert len(input_msgs) == 2 + for i, exp in enumerate(expected_input): + got = _normalize_to_dict(input_msgs[i]) + assert got["role"] == exp["role"] + assert _normalize_to_list(got["parts"]) == exp["parts"] + + output_msgs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, [])) + assert len(output_msgs) == 1 + out = _normalize_to_dict(output_msgs[0]) + assert out["role"] == "ai" + assert out["finish_reason"] == "stop" + assert _normalize_to_list(out["parts"]) == [{"content": "The capital of France is Paris.", "type": "text"}] + assert_log_parent(log_record, parent_span) + + +def assert_log_record_when_error(log_record, parent_span): + # Event name (support both .event_name and attributes for SDK differences) + event_name = getattr(log_record, "event_name", None) or log_record.attributes.get(EventAttributes.EVENT_NAME) + assert event_name == "gen_ai.client.inference.operation.details" + + attrs = log_record.attributes + assert attrs.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == "chat" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == "gpt-3.5-turbo" + assert attrs.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TOP_P) == 0.9 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) == 100 + stop_seqs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, [])) + assert stop_seqs == ["\n", "Human:", "AI:"] + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_SEED) == 100 + assert gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS not in attrs + assert gen_ai_attributes.GEN_AI_RESPONSE_MODEL not in attrs + assert gen_ai_attributes.GEN_AI_RESPONSE_ID not in attrs + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attrs + assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attrs + + # Input/output messages: normalize list/tuple and compare structure + input_msgs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, [])) + expected_input = [ + { + "parts": [{"content": "You are a helpful assistant!", "type": "text"}], + "role": "system", + }, + { + "parts": [{"content": "What is the capital of France?", "type": "text"}], + "role": "human", + }, + ] + assert len(input_msgs) == 2 + for i, exp in enumerate(expected_input): + got = _normalize_to_dict(input_msgs[i]) + assert got["role"] == exp["role"] + assert _normalize_to_list(got["parts"]) == exp["parts"] + + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attrs + assert_log_parent(log_record, parent_span) + + +def _normalize_to_list(value): + """Normalize tuple or list to list for OpenTelemetry compatibility.""" + return list(value) if isinstance(value, tuple) else value + + +def _normalize_to_dict(value): + """Normalize tuple or dict to dict for OpenTelemetry compatibility.""" + return dict(value) if isinstance(value, tuple) else value + + +def assert_log_parent(log_record, span): + if span: + assert log_record.trace_id == span.get_span_context().trace_id + assert log_record.span_id == span.get_span_context().span_id + assert log_record.trace_flags == span.get_span_context().trace_flags