From f1ca02c3cb50aa5b65a09892e9d617cde2920c18 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 13:01:41 -0700 Subject: [PATCH 01/20] Add the internal otel langchain instrumentation --- .../instrumentation/langchain/__init__.py | 129 ++++++++ .../langchain/callback_handler.py | 275 ++++++++++++++++++ .../langchain/invocation_manager.py | 61 ++++ .../instrumentation/langchain/package.py | 16 + .../instrumentation/langchain/span_manager.py | 117 ++++++++ .../instrumentation/langchain/version.py | 15 + 6 files changed, 613 insertions(+) create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py new file mode 100644 index 00000000..5eddc34f --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -0,0 +1,129 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Langchain instrumentation supporting `ChatOpenAI` and `ChatBedrock`, it can be enabled by +using ``LangChainInstrumentor``. Other providers/LLMs may be supported in the future and telemetry for them is skipped for now. + +Usage +----- +.. code:: python + from opentelemetry.instrumentation.langchain import LangChainInstrumentor + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_openai import ChatOpenAI + + LangChainInstrumentor().instrument() + llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, max_tokens=1000) + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + result = llm.invoke(messages) + LangChainInstrumentor().uninstrument() + +API +--- +""" + +from typing import Any, Callable, Collection + +from langchain_core.callbacks import BaseCallbackHandler +from wrapt import wrap_function_wrapper # type: ignore + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.langchain.callback_handler import ( + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.instrumentation.langchain.package import _instruments +from opentelemetry.instrumentation.utils import unwrap +from opentelemetry.util.genai.handler import get_telemetry_handler + + +class LangChainInstrumentor(BaseInstrumentor): + """ + OpenTelemetry instrumentor for LangChain. + This adds a custom callback handler to the LangChain callback manager + to capture LLM telemetry. + """ + + def __init__( + self, + ): + super().__init__() + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any): + """ + Enable Langchain instrumentation. + """ + tracer_provider = kwargs.get("tracer_provider") + meter_provider = kwargs.get("meter_provider") + logger_provider = kwargs.get("logger_provider") + + telemetry_handler = get_telemetry_handler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + otel_callback_handler = OpenTelemetryLangChainCallbackHandler( + telemetry_handler=telemetry_handler, + ) + + wrap_function_wrapper( + module="langchain_core.callbacks", + name="BaseCallbackManager.__init__", + wrapper=_BaseCallbackManagerInitWrapper(otel_callback_handler), + ) + + def _uninstrument(self, **kwargs: Any): + """ + Cleanup instrumentation (unwrap). + """ + unwrap("langchain_core.callbacks.base.BaseCallbackManager", "__init__") + # Clear the TelemetryHandler singleton so the next instrument() uses + # the provided tracer_provider/meter_provider/logger_provider instead + # of reusing the previous handler. + if ( + getattr(get_telemetry_handler, "_default_handler", None) + is not None + ): + delattr(get_telemetry_handler, "_default_handler") + + +class _BaseCallbackManagerInitWrapper: + """ + Wrap the BaseCallbackManager __init__ to insert custom callback handler in the manager's handlers list. + """ + + def __init__( + self, callback_handler: OpenTelemetryLangChainCallbackHandler + ): + self._otel_handler = callback_handler + + def __call__( + self, + wrapped: Callable[..., None], + instance: BaseCallbackHandler, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ): + wrapped(*args, **kwargs) + # Ensure our OTel callback is present if not already. + for handler in instance.inheritable_handlers: # type: ignore + if isinstance(handler, type(self._otel_handler)): + break + else: + instance.add_handler(self._otel_handler, inherit=True) # type: ignore \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py new file mode 100644 index 00000000..c8340b36 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -0,0 +1,275 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any, Optional, cast +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.messages import BaseMessage +from langchain_core.outputs import LLMResult + +from opentelemetry.instrumentation.langchain.invocation_manager import ( + _InvocationManager, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.types import ( + Error, + InputMessage, + LLMInvocation, + MessagePart, + OutputMessage, + Text, +) + + +class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): + """ + A callback handler for LangChain that uses OpenTelemetry to create spans for LLM calls and chains, tools etc,. in future. + """ + + def __init__(self, telemetry_handler: TelemetryHandler) -> None: + super().__init__() + self._telemetry_handler = telemetry_handler + self._invocation_manager = _InvocationManager() + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list[BaseMessage]], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + metadata: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + # Other providers/LLMs may be supported in the future and telemetry for them is skipped for now. + if serialized.get("name") not in ("ChatOpenAI", "ChatBedrock"): + return + + if "invocation_params" in kwargs: + params = ( + kwargs["invocation_params"].get("params") + or kwargs["invocation_params"] + ) + else: + params = kwargs + + request_model = "unknown" + for model_tag in ( + "model_name", # ChatOpenAI + "model_id", # ChatBedrock + ): + if (model := (params or {}).get(model_tag)) is not None: + request_model = model + break + elif (model := (metadata or {}).get(model_tag)) is not None: + request_model = model + break + + # Skip telemetry for unsupported request models + if request_model == "unknown": + return + + # Initialize variables with default values to avoid "possibly unbound" errors + top_p = None + frequency_penalty = None + presence_penalty = None + stop_sequences = None + seed = None + temperature = None + max_tokens = None + + if params is not None: + top_p = params.get("top_p") + frequency_penalty = params.get("frequency_penalty") + presence_penalty = params.get("presence_penalty") + stop_sequences = params.get("stop") + seed = params.get("seed") + temperature = params.get("temperature") + max_tokens = params.get("max_completion_tokens") + + provider = "unknown" + if metadata is not None: + provider = metadata.get("ls_provider", "unknown") + + # Override with ChatBedrock values if present + if "ls_temperature" in metadata: + temperature = metadata.get("ls_temperature") + if "ls_max_tokens" in metadata: + max_tokens = metadata.get("ls_max_tokens") + + input_messages: list[InputMessage] = [] + for sub_messages in messages: + for message in sub_messages: + # Cast to Any to avoid type checking issues with LangChain's complex content type + raw_content: Any = message.content # type: ignore[misc] + role = message.type + parts: list[Text] = [] + + if isinstance(raw_content, str): + parts = [Text(content=raw_content, type="text")] + elif isinstance(raw_content, list): + for item in raw_content: # type: ignore[misc] + if isinstance(item, str): + parts.append(Text(content=item, type="text")) + elif isinstance(item, dict): + # Safely extract text content from dict + text_value = item.get("text") # type: ignore[misc] + if isinstance(text_value, str) and text_value: + parts.append( + Text(content=text_value, type="text") + ) + + input_messages.append( + InputMessage( + parts=cast(list[MessagePart], parts), role=role + ) + ) + + llm_invocation = LLMInvocation( + request_model=request_model, + input_messages=input_messages, + provider=provider, + top_p=top_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + stop_sequences=stop_sequences, + seed=seed, + temperature=temperature, + max_tokens=max_tokens, + ) + llm_invocation = self._telemetry_handler.start_llm( + invocation=llm_invocation + ) + self._invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=parent_run_id, + invocation=llm_invocation, + ) + + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) + if llm_invocation is None or not isinstance( + llm_invocation, LLMInvocation + ): + # If the invocation does not exist, we cannot set attributes or end it + return + + output_messages: list[OutputMessage] = [] + for generation in getattr(response, "generations", []): + for chat_generation in generation: + # Get finish reason + finish_reason = "unknown" # Default value + generation_info = getattr( + chat_generation, "generation_info", None + ) + if generation_info is not None: + finish_reason = generation_info.get( + "finish_reason", "unknown" + ) + + if chat_generation.message: + # Get finish reason if generation_info is None above + if ( + generation_info is None + and chat_generation.message.response_metadata + ): + finish_reason = ( + chat_generation.message.response_metadata.get( + "stopReason", "unknown" + ) + ) + + # Get message content + parts = [ + Text( + content=chat_generation.message.content, + type="text", + ) + ] + role = chat_generation.message.type + output_message = OutputMessage( + role=role, + parts=cast(list[MessagePart], parts), + finish_reason=finish_reason, + ) + output_messages.append(output_message) + + # Get token usage if available + if chat_generation.message.usage_metadata: + input_tokens = ( + chat_generation.message.usage_metadata.get( + "input_tokens", 0 + ) + ) + llm_invocation.input_tokens = input_tokens + + output_tokens = ( + chat_generation.message.usage_metadata.get( + "output_tokens", 0 + ) + ) + llm_invocation.output_tokens = output_tokens + + llm_invocation.output_messages = output_messages + + llm_output = getattr(response, "llm_output", None) + if llm_output is not None: + response_model = llm_output.get("model_name") or llm_output.get( + "model" + ) + if response_model is not None: + llm_invocation.response_model_name = str(response_model) + + response_id = llm_output.get("id") + if response_id is not None: + llm_invocation.response_id = str(response_id) + + llm_invocation = self._telemetry_handler.stop_llm( + invocation=llm_invocation + ) + if llm_invocation.span and not llm_invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id=run_id) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) + if llm_invocation is None or not isinstance( + llm_invocation, LLMInvocation + ): + # If the invocation does not exist, we cannot set attributes or end it + return + + error_otel = Error(message=str(error), type=type(error)) + llm_invocation = self._telemetry_handler.fail_llm( + invocation=llm_invocation, error=error_otel + ) + if llm_invocation.span and not llm_invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id=run_id) \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py new file mode 100644 index 00000000..51492053 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -0,0 +1,61 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from uuid import UUID + +from opentelemetry.util.genai.types import GenAIInvocation + +__all__ = ["_InvocationManager"] + + +@dataclass +class _InvocationState: + invocation: GenAIInvocation + children: List[UUID] = field(default_factory=lambda: list()) + + +class _InvocationManager: + def __init__( + self, + ) -> None: + # Map from run_id -> _InvocationState, to keep track of invocations and parent/child relationships + # TODO: TTL cache to avoid memory leaks in long-running processes. + self._invocations: Dict[UUID, _InvocationState] = {} + + def add_invocation_state( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + invocation: GenAIInvocation, + ): + invocation_state = _InvocationState(invocation=invocation) + self._invocations[run_id] = invocation_state + + if parent_run_id is not None and parent_run_id in self._invocations: + parent_invocation_state = self._invocations[parent_run_id] + parent_invocation_state.children.append(run_id) + + def get_invocation(self, run_id: UUID) -> Optional[GenAIInvocation]: + invocation_state = self._invocations.get(run_id) + return invocation_state.invocation if invocation_state else None + + def delete_invocation_state(self, run_id: UUID) -> None: + invocation_state = self._invocations.get(run_id) + if not invocation_state: + return + for child_id in list(invocation_state.children): + self._invocations.pop(child_id, None) + self._invocations.pop(run_id, None) \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py new file mode 100644 index 00000000..08c9e49e --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +_instruments = ("langchain >= 0.3.21",) \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py new file mode 100644 index 00000000..be71280e --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py @@ -0,0 +1,117 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from uuid import UUID + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import ( + error_attributes, +) +from opentelemetry.trace import Span, SpanKind, Tracer, set_span_in_context +from opentelemetry.trace.status import Status, StatusCode + +__all__ = ["_SpanManager"] + + +@dataclass +class _SpanState: + span: Span + children: List[UUID] = field(default_factory=lambda: list()) + + +class _SpanManager: + def __init__( + self, + tracer: Tracer, + ) -> None: + self._tracer = tracer + + # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships + # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 + self.spans: Dict[UUID, _SpanState] = {} + + def _create_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + span_name: str, + kind: SpanKind = SpanKind.INTERNAL, + ) -> Span: + if parent_run_id is not None and parent_run_id in self.spans: + parent_state = self.spans[parent_run_id] + parent_span = parent_state.span + ctx = set_span_in_context(parent_span) + span = self._tracer.start_span( + name=span_name, kind=kind, context=ctx + ) + parent_state.children.append(run_id) + else: + # top-level or missing parent + span = self._tracer.start_span(name=span_name, kind=kind) + set_span_in_context(span) + + span_state = _SpanState(span=span) + self.spans[run_id] = span_state + + return span + + def create_chat_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + request_model: str, + ) -> Span: + span = self._create_span( + run_id=run_id, + parent_run_id=parent_run_id, + span_name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}", + kind=SpanKind.CLIENT, + ) + span.set_attribute( + GenAI.GEN_AI_OPERATION_NAME, + GenAI.GenAiOperationNameValues.CHAT.value, + ) + if request_model: + span.set_attribute(GenAI.GEN_AI_REQUEST_MODEL, request_model) + + return span + + def end_span(self, run_id: UUID) -> None: + state = self.spans[run_id] + for child_id in state.children: + child_state = self.spans.get(child_id) + if child_state: + child_state.span.end() + del self.spans[child_id] + state.span.end() + del self.spans[run_id] + + def get_span(self, run_id: UUID) -> Optional[Span]: + state = self.spans.get(run_id) + return state.span if state else None + + def handle_error(self, error: BaseException, run_id: UUID): + span = self.get_span(run_id) + if span is None: + # If the span does not exist, we cannot set the error status + return + span.set_status(Status(StatusCode.ERROR, str(error))) + span.set_attribute( + error_attributes.ERROR_TYPE, type(error).__qualname__ + ) + self.end_span(run_id) \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py new file mode 100644 index 00000000..d3cc86dd --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "2.0b0.dev" \ No newline at end of file From 8b0c36595e7e0b344d7d53270d36275891c4885f Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 13:22:19 -0700 Subject: [PATCH 02/20] Add test suite --- .../tests/_init_.py | 0 ...t_openai_gpt_3_5_turbo_model_llm_call.yaml | 161 ++++++ ...t_3_5_turbo_model_llm_call_with_error.yaml | 115 ++++ .../tests/conftest.py | 218 ++++++++ .../tests/test_invocation_manager.py | 141 +++++ .../tests/test_llm_call.py | 522 ++++++++++++++++++ 6 files changed, 1157 insertions(+) create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/tests/_init_.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/_init_.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/_init_.py new file mode 100644 index 00000000..e69de29b diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml new file mode 100644 index 00000000..911b44b1 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml @@ -0,0 +1,161 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "frequency_penalty": 0.5, + "max_completion_tokens": 100, + "presence_penalty": 0.5, + "seed": 100, + "stop": [ + "\n", + "Human:", + "AI:" + ], + "stream": false, + "temperature": 0.1, + "top_p": 0.9 + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '316' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.21.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.21.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.5 + authorization: + - Bearer test_openai_api_key + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-DB5VKnX0DBvthDkYqZDgfZVAqhZUk", + "object": "chat.completion", + "created": 1771535138, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 7, + "total_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null + } + headers: + CF-RAY: + - 9d08b930cee533ed-SJC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 19 Feb 2026 21:05:38 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '822' + openai-organization: test_openai_org_id + openai-processing-ms: + - '170' + openai-project: + - proj_GLiYlAc06hF0Fm06IMReZLy4 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=tYjXaOj8KokJGc56XWDShqpm2zPWeq3Q19yhnJcqKqU-1771535137.4012983-1.0.1.1-k50ojuWx4sSPmgdu8QLKD3OmGYDMjKIDSqQcANN1wo1SBWNjIE.yAsOYOgaalvDiPrXObJN80xhnlcC9mXOXnOfGb9HRyT5un2ASAG29wbujzo_b9a3zQg9hHU024Nkz; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 19 Feb 2026 + 21:35:38 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199982' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 5ms + x-request-id: + - req_5f037f577c544db292661f8bbf12d569 + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml new file mode 100644 index 00000000..43d099b5 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/cassettes/test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml @@ -0,0 +1,115 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "frequency_penalty": 0.5, + "max_completion_tokens": 100, + "presence_penalty": 0.5, + "seed": 100, + "stop": [ + "\n", + "Human:", + "AI:" + ], + "stream": false, + "temperature": 0.1, + "top_p": 0.9 + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '316' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.21.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.21.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.5 + authorization: + - Bearer test_openai_api_key + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "error": { + "message": "Incorrect API key provided: test_ope*******_key. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": null, + "code": "invalid_api_key" + } + } + headers: + CF-RAY: + - 9d08b96c8d2b67d7-SJC + Connection: + - keep-alive + Content-Length: + - '269' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 19 Feb 2026 21:05:46 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + set-cookie: + - __cf_bm=wRTe.yg21t4Hmlnym6umYHo41PFme8jHV65z7Ai6_04-1771535146.9666395-1.0.1.1-V3j6uPp1b7mudGWLLSB5Vk_zC3O8FBmCjRrguc_.hixN.Zy67xOtEeE__VwyjmXrWSnJnjHz2VO2RUr.6C04qr2J8dPfV5._bQfv4thKMonsCohocrVcwmwwfgF6GtYD; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 19 Feb 2026 + 21:35:46 GMT + - _cfuvid=FiOO9id9BeHCp.jp23WGFYuUJAqS3RdB2T1lLaroTEs-1771535146.9666395-1.0.1.1-GwFV1AQJkfqXaVwhyb_WBpNzyDw3zVtMUGJb8rtOMFg; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com + vary: + - Origin + x-openai-proxy-wasm: + - v0.1 + x-request-id: + - req_71913c60957d49b29348e7966f3616d6 + status: + code: 401 + message: Unauthorized +version: 1 \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py new file mode 100644 index 00000000..b67c0b09 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -0,0 +1,218 @@ +"""Unit tests configuration module.""" + +import json +import os + +import pytest +import yaml +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogRecordExporter, + SimpleLogRecordProcessor, +) +from opentelemetry.sdk.metrics import ( + MeterProvider, +) +from opentelemetry.sdk.metrics.export import ( + InMemoryMetricReader, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +@pytest.fixture(scope="module", name="chat_openai_gpt_3_5_turbo_model") +def fixture_chat_openai_gpt_3_5_turbo_model(): + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + yield llm + + +@pytest.fixture(name="span_exporter") +def fixture_span_exporter(): + exporter = InMemorySpanExporter() + yield exporter + + +@pytest.fixture(name="log_exporter") +def fixture_log_exporter(): + exporter = InMemoryLogRecordExporter() + yield exporter + + +@pytest.fixture(name="metric_reader") +def fixture_metric_reader(): + exporter = InMemoryMetricReader() + yield exporter + + +@pytest.fixture(name="tracer_provider") +def fixture_tracer_provider(span_exporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + return provider + + +@pytest.fixture(name="logger_provider") +def fixture_logger_provider(log_exporter): + provider = LoggerProvider() + provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + return provider + + +@pytest.fixture(name="meter_provider") +def fixture_meter_provider(metric_reader): + meter_provider = MeterProvider( + metric_readers=[metric_reader], + ) + return meter_provider + + +@pytest.fixture() +def start_instrumentation( + tracer_provider, + meter_provider, + logger_provider, +): + # Ensure stability opt-in is set before the first call to _initialize() + os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = "gen_ai_latest_experimental" + # Reset the cached initialization so the env var is picked up + from opentelemetry.instrumentation._semconv import ( + _OpenTelemetrySemanticConventionStability, + ) + _OpenTelemetrySemanticConventionStability._initialized = False + _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING = {} + _OpenTelemetrySemanticConventionStability._initialize() + + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + + yield instrumentor + instrumentor.uninstrument() + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = "test_openai_api_key" + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } + + +class LiteralBlockScalar(str): + """Formats the string as a literal block scalar, preserving whitespace and + without interpreting escape characters""" + + +def literal_block_scalar_presenter(dumper, data): + """Represents a scalar string as a literal block, via '|' syntax""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.add_representer(LiteralBlockScalar, literal_block_scalar_presenter) + + +def process_string_value(string_value): + """Pretty-prints JSON or returns long strings as a LiteralBlockScalar""" + try: + json_data = json.loads(string_value) + return LiteralBlockScalar(json.dumps(json_data, indent=2)) + except (ValueError, TypeError): + if len(string_value) > 80: + return LiteralBlockScalar(string_value) + return string_value + + +def convert_body_to_literal(data): + """Searches the data for body strings, attempting to pretty-print JSON""" + if isinstance(data, dict): + for key, value in data.items(): + # Handle response body case (e.g., response.body.string) + if key == "body" and isinstance(value, dict) and "string" in value: + value["string"] = process_string_value(value["string"]) + + # Handle request body case (e.g., request.body) + elif key == "body" and isinstance(value, str): + data[key] = process_string_value(value) + + else: + convert_body_to_literal(value) + + elif isinstance(data, list): + for idx, choice in enumerate(data): + data[idx] = convert_body_to_literal(choice) + + return data + + +class PrettyPrintJSONBody: + """This makes request and response body recordings more readable.""" + + @staticmethod + def serialize(cassette_dict): + cassette_dict = convert_body_to_literal(cassette_dict) + return yaml.dump( + cassette_dict, default_flow_style=False, allow_unicode=True + ) + + @staticmethod + def deserialize(cassette_string): + return yaml.load(cassette_string, Loader=yaml.Loader) + + +@pytest.fixture(scope="module", autouse=True) +def fixture_vcr(vcr_config): + import vcr as vcrpy + + my_vcr = vcrpy.VCR(**vcr_config) + my_vcr.register_serializer("yaml", PrettyPrintJSONBody) + my_vcr.record_mode = "none" + my_vcr.serializer = "yaml" + my_vcr.cassette_library_dir = os.path.join( + os.path.dirname(__file__), "cassettes" + ) + return my_vcr + + +@pytest.fixture() +def vcr(fixture_vcr): + """Function-scoped vcr fixture that replaces pytest-recording's default one.""" + return fixture_vcr + + +def scrub_response_headers(response): + """ + This scrubs sensitive response headers. Note they are case-sensitive! + """ + response["headers"]["openai-organization"] = "test_openai_org_id" + response["headers"]["Set-Cookie"] = "test_set_cookie" + return response \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py new file mode 100644 index 00000000..851bb32a --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py @@ -0,0 +1,141 @@ +# tests/test_invocation_manager.py +import uuid +from unittest import mock + +import pytest + +from opentelemetry.instrumentation.langchain.invocation_manager import ( + _InvocationManager, +) +from opentelemetry.util.genai.types import GenAIInvocation + + +@pytest.fixture +def invocation_manager(): + return _InvocationManager() + + +@pytest.fixture +def mock_invocation(): + return mock.Mock(spec=GenAIInvocation) + + +def test_add_invocation_state_without_parent( + invocation_manager, mock_invocation +): + run_id = uuid.uuid4() + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=None, + invocation=mock_invocation, + ) + + assert invocation_manager.get_invocation(run_id) == mock_invocation + assert len(invocation_manager._invocations) == 1 + assert invocation_manager._invocations[run_id].children == [] + + +def test_add_invocation_state_with_parent(invocation_manager): + parent_id = uuid.uuid4() + child_id = uuid.uuid4() + parent_invocation = mock.Mock(spec=GenAIInvocation) + child_invocation = mock.Mock(spec=GenAIInvocation) + + # Add parent first + invocation_manager.add_invocation_state( + run_id=parent_id, + parent_run_id=None, + invocation=parent_invocation, + ) + + # Then add child with parent reference + invocation_manager.add_invocation_state( + run_id=child_id, + parent_run_id=parent_id, + invocation=child_invocation, + ) + + # Check that parent has child in its children list + assert child_id in invocation_manager._invocations[parent_id].children + assert invocation_manager.get_invocation(child_id) == child_invocation + assert invocation_manager.get_invocation(parent_id) == parent_invocation + + +def test_add_invocation_state_with_nonexistent_parent( + invocation_manager, mock_invocation +): + run_id = uuid.uuid4() + nonexistent_parent_id = uuid.uuid4() + + # Adding with a parent that doesn't exist should still add the child without error + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=nonexistent_parent_id, + invocation=mock_invocation, + ) + + assert invocation_manager.get_invocation(run_id) == mock_invocation + assert len(invocation_manager._invocations) == 1 + + +def test_get_nonexistent_invocation(invocation_manager): + nonexistent_id = uuid.uuid4() + assert invocation_manager.get_invocation(nonexistent_id) is None + + +def test_delete_invocation_state(invocation_manager, mock_invocation): + run_id = uuid.uuid4() + invocation_manager.add_invocation_state( + run_id=run_id, + parent_run_id=None, + invocation=mock_invocation, + ) + + # Verify it was added + assert invocation_manager.get_invocation(run_id) == mock_invocation + + # Delete it + invocation_manager.delete_invocation_state(run_id) + + # Verify it was removed + assert run_id not in invocation_manager._invocations + + +def test_delete_invocation_state_with_children(invocation_manager): + parent_id = uuid.uuid4() + child1_id = uuid.uuid4() + child2_id = uuid.uuid4() + + parent_invocation = mock.Mock(spec=GenAIInvocation) + child1_invocation = mock.Mock(spec=GenAIInvocation) + child2_invocation = mock.Mock(spec=GenAIInvocation) + + # Add parent and children + invocation_manager.add_invocation_state( + run_id=parent_id, + parent_run_id=None, + invocation=parent_invocation, + ) + invocation_manager.add_invocation_state( + run_id=child1_id, + parent_run_id=parent_id, + invocation=child1_invocation, + ) + invocation_manager.add_invocation_state( + run_id=child2_id, + parent_run_id=parent_id, + invocation=child2_invocation, + ) + + # Verify initial state + assert len(invocation_manager._invocations) == 3 + assert len(invocation_manager._invocations[parent_id].children) == 2 + + # Delete parent + invocation_manager.delete_invocation_state(parent_id) + + # Verify parent and all children were removed + assert parent_id not in invocation_manager._invocations + assert child1_id not in invocation_manager._invocations + assert child2_id not in invocation_manager._invocations + assert len(invocation_manager._invocations) == 0 \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py new file mode 100644 index 00000000..25ab240a --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py @@ -0,0 +1,522 @@ +from typing import Optional + +import pytest +from langchain_core.messages import HumanMessage, SystemMessage +from openai import AuthenticationError + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + event_attributes as EventAttributes, +) +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import error_attributes + + +# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py +@pytest.mark.vcr() +@pytest.mark.parametrize( + "capture_content", + ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], +) +def test_chat_openai_gpt_3_5_turbo_model_llm_call( + span_exporter, + metric_reader, + log_exporter, + start_instrumentation, + chat_openai_gpt_3_5_turbo_model, + monkeypatch, + capture_content, + vcr, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content + ) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + with vcr.use_cassette( + "test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml" + ): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + assert response.content == "The capital of France is Paris." + + # verify spans + spans = span_exporter.get_finished_spans() + + verify_content: bool = ( + True if capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") else False + ) + assert_openai_completion_attributes( + spans[0] if len(spans) > 0 else None, + response, + verify_content=verify_content, + ) + + # verify metrics + metrics_data = metric_reader.get_metrics_data() + assert metrics_data is not None + resource_metrics = metrics_data.resource_metrics + assert resource_metrics is not None + assert len(resource_metrics) == 1 + + metrics = resource_metrics[0].scope_metrics[0].metrics + assert len(metrics) == 2 + for metric in metrics: + if metric.name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION: + assert_duration_metric(metric, spans[0]) + if metric.name == gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE: + assert_token_usage_metric(metric, spans[0]) + + # verify logs + logs = log_exporter.get_finished_logs() + if capture_content in ("SPAN_AND_EVENT", "EVENT_ONLY"): + assert len(logs) == 1 + log_record = logs[0].log_record + assert_log_record(log_record, spans[0]) + elif capture_content in ("SPAN_ONLY", "NO_CONTENT"): + assert len(logs) == 0 + + +# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py +@pytest.mark.vcr() +@pytest.mark.parametrize( + "capture_content", + ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], +) +def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( + span_exporter, + metric_reader, + log_exporter, + start_instrumentation, + chat_openai_gpt_3_5_turbo_model, + monkeypatch, + capture_content, + vcr, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content + ) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + response = None + try: + with vcr.use_cassette( + "test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml" + ): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + except Exception as e: + # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. + assert isinstance(e, AuthenticationError) + + assert response is None + + # verify spans + spans = span_exporter.get_finished_spans() + + verify_content: bool = ( + True if capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") else False + ) + assert_openai_completion_attributes_with_error( + spans[0] if len(spans) > 0 else None, verify_content=verify_content + ) + + # verify metrics + metrics_data = metric_reader.get_metrics_data() + assert metrics_data is not None + resource_metrics = metrics_data.resource_metrics + assert len(resource_metrics) == 1 + metrics = resource_metrics[0].scope_metrics[0].metrics + assert len(metrics) == 1 + assert metrics[0].name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + assert_duration_metric_when_error(metrics[0], spans[0]) + + # verify logs + logs = log_exporter.get_finished_logs() + if capture_content in ("SPAN_AND_EVENT", "EVENT_ONLY"): + assert len(logs) == 1 + log_record = logs[0].log_record + assert_log_record_when_error(log_record, spans[0]) + elif capture_content in ("SPAN_ONLY", "NO_CONTENT"): + assert len(logs) == 0 + + +def assert_openai_completion_attributes( + span: ReadableSpan, response: Optional, verify_content: bool = True +): + assert span is not None + attributes = span.attributes + assert span.name == "chat gpt-3.5-turbo" + assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" + assert ( + attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" + ) + assert ( + attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo-0125" + ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 + assert attributes["gen_ai.provider.name"] == "openai" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID in attributes + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert ( + attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 + ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.5 + stop_sequences = attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES + ) + assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 + + input_tokens = response.response_metadata.get("token_usage").get( + "prompt_tokens" + ) + if input_tokens: + assert ( + input_tokens + == attributes[gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS] + ) + else: + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attributes + + output_tokens = response.response_metadata.get("token_usage").get( + "completion_tokens" + ) + if output_tokens: + assert ( + output_tokens + == attributes[gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS] + ) + else: + assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attributes + + if verify_content: + input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] + assert input_message is not None + assert '"role":"system"' in input_message + assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"human"' in input_message + assert '"content":"What is the capital of France?"' in input_message + + # Assert output message + output_message = attributes[gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES] + assert output_message is not None + assert '"role":"ai"' in output_message + assert '"content":"The capital of France is Paris."' in output_message + assert '"finish_reason":"stop"' in output_message + else: + assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + + +def assert_openai_completion_attributes_with_error( + span: ReadableSpan, verify_content: bool = True +): + assert span is not None + assert span.name == "chat gpt-3.5-turbo" + attributes = span.attributes + assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" + assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" + assert ( + attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" + ) + assert gen_ai_attributes.GEN_AI_RESPONSE_MODEL not in attributes + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 + assert attributes["gen_ai.provider.name"] == "openai" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID not in attributes + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert ( + attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 + ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.5 + stop_sequences = attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES + ) + assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 + + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attributes + + assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attributes + + if verify_content: + input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] + assert input_message is not None + assert '"role":"system"' in input_message + assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"human"' in input_message + assert '"content":"What is the capital of France?"' in input_message + + # Assert output message + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + else: + assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + + +def assert_duration_metric(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 1 + assert metric.data.data_points[0].sum > 0 + + assert_duration_metric_attributes( + metric.data.data_points[0].attributes, parent_span + ) + assert_exemplars( + metric.data.data_points[0].exemplars, + metric.data.data_points[0].sum, + parent_span, + ) + + +def assert_duration_metric_attributes(attributes, parent_span): + assert len(attributes) == 4 + assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert ( + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] + ) + + +def assert_duration_metric_when_error(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 1 + assert metric.data.data_points[0].sum > 0 + + assert_duration_metric_attributes_when_error( + metric.data.data_points[0].attributes, parent_span + ) + assert_exemplars( + metric.data.data_points[0].exemplars, + metric.data.data_points[0].sum, + parent_span, + ) + + +def assert_duration_metric_attributes_when_error(attributes, parent_span): + assert len(attributes) == 4 + assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" + assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert ( + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + ) + + +def assert_token_usage_metric(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 2 + + assert metric.data.data_points[0].sum > 0 + assert_token_usage_metric_attributes( + metric.data.data_points[0].attributes, parent_span, "input" + ) + assert_exemplars( + metric.data.data_points[0].exemplars, + metric.data.data_points[0].sum, + parent_span, + ) + + assert metric.data.data_points[1].sum > 0 + assert_token_usage_metric_attributes( + metric.data.data_points[1].attributes, parent_span, "output" + ) + assert_exemplars( + metric.data.data_points[1].exemplars, + metric.data.data_points[1].sum, + parent_span, + ) + + +def assert_token_usage_metric_attributes(attributes, parent_span, token_type): + assert len(attributes) == 5 + assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert ( + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + ) + assert ( + attributes.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) + == parent_span.attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] + ) + assert attributes.get(gen_ai_attributes.GEN_AI_TOKEN_TYPE) == token_type + + +def assert_exemplars(exemplars, datapoint_sum, parent_span): + assert len(exemplars) == 1 + assert exemplars[0].value == datapoint_sum + assert exemplars[0].span_id == parent_span.get_span_context().span_id + assert exemplars[0].trace_id == parent_span.get_span_context().trace_id + + +def assert_log_record(log_record, parent_span): + # Event name (support both .event_name and attributes for SDK differences) + event_name = getattr( + log_record, "event_name", None + ) or log_record.attributes.get(EventAttributes.EVENT_NAME) + assert event_name == "gen_ai.client.inference.operation.details" + + attrs = log_record.attributes + assert attrs.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == "chat" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == "gpt-3.5-turbo" + assert attrs.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TOP_P) == 0.9 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) == 100 + stop_seqs = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, []) + ) + assert stop_seqs == ["\n", "Human:", "AI:"] + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_SEED) == 100 + assert attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS) == ( + "stop", + ) + assert ( + attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) + == "gpt-3.5-turbo-0125" + ) + assert gen_ai_attributes.GEN_AI_RESPONSE_ID in attrs + assert attrs.get(gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS) == 24 + assert attrs.get(gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS) == 7 + + # Input/output messages: normalize list/tuple and compare structure + input_msgs = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) + ) + expected_input = [ + { + "parts": [ + {"content": "You are a helpful assistant!", "type": "text"} + ], + "role": "system", + }, + { + "parts": [ + {"content": "What is the capital of France?", "type": "text"} + ], + "role": "human", + }, + ] + assert len(input_msgs) == 2 + for i, exp in enumerate(expected_input): + got = _normalize_to_dict(input_msgs[i]) + assert got["role"] == exp["role"] + assert _normalize_to_list(got["parts"]) == exp["parts"] + + output_msgs = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, []) + ) + assert len(output_msgs) == 1 + out = _normalize_to_dict(output_msgs[0]) + assert out["role"] == "ai" + assert out["finish_reason"] == "stop" + assert _normalize_to_list(out["parts"]) == [ + {"content": "The capital of France is Paris.", "type": "text"} + ] + assert_log_parent(log_record, parent_span) + + +def assert_log_record_when_error(log_record, parent_span): + # Event name (support both .event_name and attributes for SDK differences) + event_name = getattr( + log_record, "event_name", None + ) or log_record.attributes.get(EventAttributes.EVENT_NAME) + assert event_name == "gen_ai.client.inference.operation.details" + + attrs = log_record.attributes + assert attrs.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == "chat" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == "gpt-3.5-turbo" + assert attrs.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_TOP_P) == 0.9 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY) == 0.5 + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) == 100 + stop_seqs = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, []) + ) + assert stop_seqs == ["\n", "Human:", "AI:"] + assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_SEED) == 100 + assert gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS not in attrs + assert gen_ai_attributes.GEN_AI_RESPONSE_MODEL not in attrs + assert gen_ai_attributes.GEN_AI_RESPONSE_ID not in attrs + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attrs + assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attrs + + # Input/output messages: normalize list/tuple and compare structure + input_msgs = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) + ) + expected_input = [ + { + "parts": [ + {"content": "You are a helpful assistant!", "type": "text"} + ], + "role": "system", + }, + { + "parts": [ + {"content": "What is the capital of France?", "type": "text"} + ], + "role": "human", + }, + ] + assert len(input_msgs) == 2 + for i, exp in enumerate(expected_input): + got = _normalize_to_dict(input_msgs[i]) + assert got["role"] == exp["role"] + assert _normalize_to_list(got["parts"]) == exp["parts"] + + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attrs + assert_log_parent(log_record, parent_span) + + +def _normalize_to_list(value): + """Normalize tuple or list to list for OpenTelemetry compatibility.""" + return list(value) if isinstance(value, tuple) else value + + +def _normalize_to_dict(value): + """Normalize tuple or dict to dict for OpenTelemetry compatibility.""" + return dict(value) if isinstance(value, tuple) else value + + +def assert_log_parent(log_record, span): + if span: + assert log_record.trace_id == span.get_span_context().trace_id + assert log_record.span_id == span.get_span_context().span_id + assert log_record.trace_flags == span.get_span_context().trace_flags \ No newline at end of file From 168738edbb7c7d6a5319a8a5f93766a100d5ad82 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 15:58:49 -0700 Subject: [PATCH 03/20] Fix lint and format --- .../dev_requirements.txt | 2 + .../pyproject.toml | 60 ++++++ .../instrumentation/langchain/__init__.py | 19 +- .../langchain/callback_handler.py | 82 ++------ .../langchain/invocation_manager.py | 6 +- .../instrumentation/langchain/package.py | 2 +- .../instrumentation/langchain/span_manager.py | 14 +- .../instrumentation/langchain/version.py | 2 +- .../tests/conftest.py | 15 +- .../tests/test_invocation_manager.py | 14 +- .../tests/test_llm_call.py | 188 +++++------------- 11 files changed, 167 insertions(+), 237 deletions(-) create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt b/src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt new file mode 100644 index 00000000..d9f1b3b9 --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/dev_requirements.txt @@ -0,0 +1,2 @@ +pytest>=8.0 +pytest-recording diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml new file mode 100644 index 00000000..7cc0f34c --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-instrumentation-langchain" +dynamic = ["version"] +description = "OpenTelemetry Official Langchain instrumentation" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "opentelemetry-instrumentation ~= 0.57b0", +] + +[project.optional-dependencies] +instruments = [ + "langchain >= 0.3.21", +] + +[project.entry-points.opentelemetry_instrumentor] +langchain = "opentelemetry.instrumentation.langchain:LangChainInstrumentor" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/langchain/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] + +[tool.pylint.format] +max-line-length = 120 + +[tool.pylint.messages_control] +disable = [ + "missing-function-docstring", + "missing-module-docstring", + "import-error", + "no-name-in-module", +] \ No newline at end of file diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py index 5eddc34f..ca056106 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -14,7 +14,8 @@ """ Langchain instrumentation supporting `ChatOpenAI` and `ChatBedrock`, it can be enabled by -using ``LangChainInstrumentor``. Other providers/LLMs may be supported in the future and telemetry for them is skipped for now. +using ``LangChainInstrumentor``. Other providers/LLMs may be supported in the future and +telemetry for them is skipped for now. Usage ----- @@ -57,11 +58,6 @@ class LangChainInstrumentor(BaseInstrumentor): to capture LLM telemetry. """ - def __init__( - self, - ): - super().__init__() - def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -96,19 +92,16 @@ def _uninstrument(self, **kwargs: Any): # Clear the TelemetryHandler singleton so the next instrument() uses # the provided tracer_provider/meter_provider/logger_provider instead # of reusing the previous handler. - if ( - getattr(get_telemetry_handler, "_default_handler", None) - is not None - ): + if getattr(get_telemetry_handler, "_default_handler", None) is not None: delattr(get_telemetry_handler, "_default_handler") -class _BaseCallbackManagerInitWrapper: +class _BaseCallbackManagerInitWrapper: # pylint: disable=too-few-public-methods """ Wrap the BaseCallbackManager __init__ to insert custom callback handler in the manager's handlers list. """ - def __init__( + def __init__( # pylint: disable=too-few-public-methods self, callback_handler: OpenTelemetryLangChainCallbackHandler ): self._otel_handler = callback_handler @@ -126,4 +119,4 @@ def __call__( if isinstance(handler, type(self._otel_handler)): break else: - instance.add_handler(self._otel_handler, inherit=True) # type: ignore \ No newline at end of file + instance.add_handler(self._otel_handler, inherit=True) # type: ignore diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index c8340b36..15936ed7 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -35,9 +35,11 @@ ) +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches, unused-argument class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): """ - A callback handler for LangChain that uses OpenTelemetry to create spans for LLM calls and chains, tools etc,. in future. + A callback handler for LangChain that uses OpenTelemetry to create spans for LLM calls and + chains, tools etc,. in future. """ def __init__(self, telemetry_handler: TelemetryHandler) -> None: @@ -45,7 +47,7 @@ def __init__(self, telemetry_handler: TelemetryHandler) -> None: self._telemetry_handler = telemetry_handler self._invocation_manager = _InvocationManager() - def on_chat_model_start( + def on_chat_model_start( # pylint: disable=too-many-statements self, serialized: dict[str, Any], messages: list[list[BaseMessage]], @@ -61,10 +63,7 @@ def on_chat_model_start( return if "invocation_params" in kwargs: - params = ( - kwargs["invocation_params"].get("params") - or kwargs["invocation_params"] - ) + params = kwargs["invocation_params"].get("params") or kwargs["invocation_params"] else: params = kwargs @@ -73,7 +72,7 @@ def on_chat_model_start( "model_name", # ChatOpenAI "model_id", # ChatBedrock ): - if (model := (params or {}).get(model_tag)) is not None: + if (model := (params or {}).get(model_tag)) is not None: # pylint: disable=no-else-break request_model = model break elif (model := (metadata or {}).get(model_tag)) is not None: @@ -113,7 +112,7 @@ def on_chat_model_start( max_tokens = metadata.get("ls_max_tokens") input_messages: list[InputMessage] = [] - for sub_messages in messages: + for sub_messages in messages: # pylint: disable=too-many-nested-blocks for message in sub_messages: # Cast to Any to avoid type checking issues with LangChain's complex content type raw_content: Any = message.content # type: ignore[misc] @@ -130,15 +129,9 @@ def on_chat_model_start( # Safely extract text content from dict text_value = item.get("text") # type: ignore[misc] if isinstance(text_value, str) and text_value: - parts.append( - Text(content=text_value, type="text") - ) + parts.append(Text(content=text_value, type="text")) - input_messages.append( - InputMessage( - parts=cast(list[MessagePart], parts), role=role - ) - ) + input_messages.append(InputMessage(parts=cast(list[MessagePart], parts), role=role)) llm_invocation = LLMInvocation( request_model=request_model, @@ -152,9 +145,7 @@ def on_chat_model_start( temperature=temperature, max_tokens=max_tokens, ) - llm_invocation = self._telemetry_handler.start_llm( - invocation=llm_invocation - ) + llm_invocation = self._telemetry_handler.start_llm(invocation=llm_invocation) self._invocation_manager.add_invocation_state( run_id=run_id, parent_run_id=parent_run_id, @@ -170,9 +161,7 @@ def on_llm_end( **kwargs: Any, ) -> None: llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) - if llm_invocation is None or not isinstance( - llm_invocation, LLMInvocation - ): + if llm_invocation is None or not isinstance(llm_invocation, LLMInvocation): # If the invocation does not exist, we cannot set attributes or end it return @@ -181,25 +170,14 @@ def on_llm_end( for chat_generation in generation: # Get finish reason finish_reason = "unknown" # Default value - generation_info = getattr( - chat_generation, "generation_info", None - ) + generation_info = getattr(chat_generation, "generation_info", None) if generation_info is not None: - finish_reason = generation_info.get( - "finish_reason", "unknown" - ) + finish_reason = generation_info.get("finish_reason", "unknown") if chat_generation.message: # Get finish reason if generation_info is None above - if ( - generation_info is None - and chat_generation.message.response_metadata - ): - finish_reason = ( - chat_generation.message.response_metadata.get( - "stopReason", "unknown" - ) - ) + if generation_info is None and chat_generation.message.response_metadata: + finish_reason = chat_generation.message.response_metadata.get("stopReason", "unknown") # Get message content parts = [ @@ -218,27 +196,17 @@ def on_llm_end( # Get token usage if available if chat_generation.message.usage_metadata: - input_tokens = ( - chat_generation.message.usage_metadata.get( - "input_tokens", 0 - ) - ) + input_tokens = chat_generation.message.usage_metadata.get("input_tokens", 0) llm_invocation.input_tokens = input_tokens - output_tokens = ( - chat_generation.message.usage_metadata.get( - "output_tokens", 0 - ) - ) + output_tokens = chat_generation.message.usage_metadata.get("output_tokens", 0) llm_invocation.output_tokens = output_tokens llm_invocation.output_messages = output_messages llm_output = getattr(response, "llm_output", None) if llm_output is not None: - response_model = llm_output.get("model_name") or llm_output.get( - "model" - ) + response_model = llm_output.get("model_name") or llm_output.get("model") if response_model is not None: llm_invocation.response_model_name = str(response_model) @@ -246,9 +214,7 @@ def on_llm_end( if response_id is not None: llm_invocation.response_id = str(response_id) - llm_invocation = self._telemetry_handler.stop_llm( - invocation=llm_invocation - ) + llm_invocation = self._telemetry_handler.stop_llm(invocation=llm_invocation) if llm_invocation.span and not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) @@ -261,15 +227,11 @@ def on_llm_error( **kwargs: Any, ) -> None: llm_invocation = self._invocation_manager.get_invocation(run_id=run_id) - if llm_invocation is None or not isinstance( - llm_invocation, LLMInvocation - ): + if llm_invocation is None or not isinstance(llm_invocation, LLMInvocation): # If the invocation does not exist, we cannot set attributes or end it return error_otel = Error(message=str(error), type=type(error)) - llm_invocation = self._telemetry_handler.fail_llm( - invocation=llm_invocation, error=error_otel - ) + llm_invocation = self._telemetry_handler.fail_llm(invocation=llm_invocation, error=error_otel) if llm_invocation.span and not llm_invocation.span.is_recording(): - self._invocation_manager.delete_invocation_state(run_id=run_id) \ No newline at end of file + self._invocation_manager.delete_invocation_state(run_id=run_id) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py index 51492053..9b55056d 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/invocation_manager.py @@ -24,7 +24,7 @@ @dataclass class _InvocationState: invocation: GenAIInvocation - children: List[UUID] = field(default_factory=lambda: list()) + children: List[UUID] = field(default_factory=lambda: []) # pylint: disable=unnecessary-lambda class _InvocationManager: @@ -32,7 +32,7 @@ def __init__( self, ) -> None: # Map from run_id -> _InvocationState, to keep track of invocations and parent/child relationships - # TODO: TTL cache to avoid memory leaks in long-running processes. + # TODO: TTL cache to avoid memory leaks in long-running processes. # pylint: disable=fixme self._invocations: Dict[UUID, _InvocationState] = {} def add_invocation_state( @@ -58,4 +58,4 @@ def delete_invocation_state(self, run_id: UUID) -> None: return for child_id in list(invocation_state.children): self._invocations.pop(child_id, None) - self._invocations.pop(run_id, None) \ No newline at end of file + self._invocations.pop(run_id, None) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py index 08c9e49e..1aaafae8 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py @@ -13,4 +13,4 @@ # limitations under the License. -_instruments = ("langchain >= 0.3.21",) \ No newline at end of file +_instruments = ("langchain >= 0.3.21",) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py index be71280e..c718a52c 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py @@ -31,7 +31,7 @@ @dataclass class _SpanState: span: Span - children: List[UUID] = field(default_factory=lambda: list()) + children: List[UUID] = field(default_factory=lambda: []) # pylint: disable=unnecessary-lambda class _SpanManager: @@ -42,7 +42,7 @@ def __init__( self._tracer = tracer # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships - # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 + # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 # pylint: disable=fixme self.spans: Dict[UUID, _SpanState] = {} def _create_span( @@ -56,9 +56,7 @@ def _create_span( parent_state = self.spans[parent_run_id] parent_span = parent_state.span ctx = set_span_in_context(parent_span) - span = self._tracer.start_span( - name=span_name, kind=kind, context=ctx - ) + span = self._tracer.start_span(name=span_name, kind=kind, context=ctx) parent_state.children.append(run_id) else: # top-level or missing parent @@ -111,7 +109,5 @@ def handle_error(self, error: BaseException, run_id: UUID): # If the span does not exist, we cannot set the error status return span.set_status(Status(StatusCode.ERROR, str(error))) - span.set_attribute( - error_attributes.ERROR_TYPE, type(error).__qualname__ - ) - self.end_span(run_id) \ No newline at end of file + span.set_attribute(error_attributes.ERROR_TYPE, type(error).__qualname__) + self.end_span(run_id) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py index d3cc86dd..61ae9b7c 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0b0.dev" \ No newline at end of file +__version__ = "2.0b0.dev" diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py index b67c0b09..18546db4 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -26,6 +26,7 @@ ) +# pylint: disable=protected-access, redefined-outer-name @pytest.fixture(scope="module", name="chat_openai_gpt_3_5_turbo_model") def fixture_chat_openai_gpt_3_5_turbo_model(): llm = ChatOpenAI( @@ -87,12 +88,14 @@ def start_instrumentation( meter_provider, logger_provider, ): + # pylint: disable=import-outside-toplevel # Ensure stability opt-in is set before the first call to _initialize() os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = "gen_ai_latest_experimental" # Reset the cached initialization so the env var is picked up from opentelemetry.instrumentation._semconv import ( _OpenTelemetrySemanticConventionStability, ) + _OpenTelemetrySemanticConventionStability._initialized = False _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING = {} _OpenTelemetrySemanticConventionStability._initialize() @@ -180,9 +183,7 @@ class PrettyPrintJSONBody: @staticmethod def serialize(cassette_dict): cassette_dict = convert_body_to_literal(cassette_dict) - return yaml.dump( - cassette_dict, default_flow_style=False, allow_unicode=True - ) + return yaml.dump(cassette_dict, default_flow_style=False, allow_unicode=True) @staticmethod def deserialize(cassette_string): @@ -191,15 +192,13 @@ def deserialize(cassette_string): @pytest.fixture(scope="module", autouse=True) def fixture_vcr(vcr_config): - import vcr as vcrpy + import vcr as vcrpy # pylint: disable=import-outside-toplevel my_vcr = vcrpy.VCR(**vcr_config) my_vcr.register_serializer("yaml", PrettyPrintJSONBody) my_vcr.record_mode = "none" my_vcr.serializer = "yaml" - my_vcr.cassette_library_dir = os.path.join( - os.path.dirname(__file__), "cassettes" - ) + my_vcr.cassette_library_dir = os.path.join(os.path.dirname(__file__), "cassettes") return my_vcr @@ -215,4 +214,4 @@ def scrub_response_headers(response): """ response["headers"]["openai-organization"] = "test_openai_org_id" response["headers"]["Set-Cookie"] = "test_set_cookie" - return response \ No newline at end of file + return response diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py index 851bb32a..a99e05b6 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_invocation_manager.py @@ -1,4 +1,5 @@ # tests/test_invocation_manager.py +# pylint: disable=redefined-outer-name import uuid from unittest import mock @@ -7,7 +8,7 @@ from opentelemetry.instrumentation.langchain.invocation_manager import ( _InvocationManager, ) -from opentelemetry.util.genai.types import GenAIInvocation +from opentelemetry.util.genai.types import GenAIInvocation # pylint: disable=no-name-in-module @pytest.fixture @@ -20,9 +21,8 @@ def mock_invocation(): return mock.Mock(spec=GenAIInvocation) -def test_add_invocation_state_without_parent( - invocation_manager, mock_invocation -): +# pylint: disable=protected-access +def test_add_invocation_state_without_parent(invocation_manager, mock_invocation): run_id = uuid.uuid4() invocation_manager.add_invocation_state( run_id=run_id, @@ -61,9 +61,7 @@ def test_add_invocation_state_with_parent(invocation_manager): assert invocation_manager.get_invocation(parent_id) == parent_invocation -def test_add_invocation_state_with_nonexistent_parent( - invocation_manager, mock_invocation -): +def test_add_invocation_state_with_nonexistent_parent(invocation_manager, mock_invocation): run_id = uuid.uuid4() nonexistent_parent_id = uuid.uuid4() @@ -138,4 +136,4 @@ def test_delete_invocation_state_with_children(invocation_manager): assert parent_id not in invocation_manager._invocations assert child1_id not in invocation_manager._invocations assert child2_id not in invocation_manager._invocations - assert len(invocation_manager._invocations) == 0 \ No newline at end of file + assert len(invocation_manager._invocations) == 0 diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py index 25ab240a..29946366 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/test_llm_call.py @@ -1,5 +1,6 @@ from typing import Optional +# pylint: disable=no-name-in-module import pytest from langchain_core.messages import HumanMessage, SystemMessage from openai import AuthenticationError @@ -13,12 +14,15 @@ from opentelemetry.semconv.attributes import error_attributes -# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py +# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming +# from fixtures defined in conftest.py @pytest.mark.vcr() @pytest.mark.parametrize( "capture_content", ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], ) + +# pylint: disable=too-many-arguments, too-many-locals, unused-argument def test_chat_openai_gpt_3_5_turbo_model_llm_call( span_exporter, metric_reader, @@ -29,30 +33,22 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call( capture_content, vcr, ): - monkeypatch.setenv( - "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" - ) - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content - ) + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content) messages = [ SystemMessage(content="You are a helpful assistant!"), HumanMessage(content="What is the capital of France?"), ] - with vcr.use_cassette( - "test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml" - ): + with vcr.use_cassette("test_chat_openai_gpt_3_5_turbo_model_llm_call.yaml"): response = chat_openai_gpt_3_5_turbo_model.invoke(messages) assert response.content == "The capital of France is Paris." # verify spans spans = span_exporter.get_finished_spans() - verify_content: bool = ( - True if capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") else False - ) + verify_content = capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") assert_openai_completion_attributes( spans[0] if len(spans) > 0 else None, response, @@ -84,12 +80,15 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call( assert len(logs) == 0 -# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py +# span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are +# coming from fixtures defined in conftest.py @pytest.mark.vcr() @pytest.mark.parametrize( "capture_content", ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], ) + +# pylint: disable=too-many-arguments, too-many-locals, unused-argument def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( span_exporter, metric_reader, @@ -100,12 +99,8 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( capture_content, vcr, ): - monkeypatch.setenv( - "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" - ) - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content - ) + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content) messages = [ SystemMessage(content="You are a helpful assistant!"), @@ -114,11 +109,9 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( response = None try: - with vcr.use_cassette( - "test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml" - ): + with vcr.use_cassette("test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error.yaml"): response = chat_openai_gpt_3_5_turbo_model.invoke(messages) - except Exception as e: + except Exception as e: # pylint: disable=broad-exception-caught # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. assert isinstance(e, AuthenticationError) @@ -127,12 +120,8 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( # verify spans spans = span_exporter.get_finished_spans() - verify_content: bool = ( - True if capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") else False - ) - assert_openai_completion_attributes_with_error( - spans[0] if len(spans) > 0 else None, verify_content=verify_content - ) + verify_content = capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") + assert_openai_completion_attributes_with_error(spans[0] if len(spans) > 0 else None, verify_content=verify_content) # verify metrics metrics_data = metric_reader.get_metrics_data() @@ -154,54 +143,33 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( assert len(logs) == 0 -def assert_openai_completion_attributes( - span: ReadableSpan, response: Optional, verify_content: bool = True -): +def assert_openai_completion_attributes(span: ReadableSpan, response: Optional, verify_content: bool = True): assert span is not None attributes = span.attributes assert span.name == "chat gpt-3.5-turbo" assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" - assert ( - attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" - ) - assert ( - attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo-0125" - ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" + assert attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo-0125" assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 assert attributes["gen_ai.provider.name"] == "openai" assert gen_ai_attributes.GEN_AI_RESPONSE_ID in attributes assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 - assert ( - attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 - ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 assert attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.5 - stop_sequences = attributes.get( - gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES - ) + stop_sequences = attributes.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES) assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) assert attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 - input_tokens = response.response_metadata.get("token_usage").get( - "prompt_tokens" - ) + input_tokens = response.response_metadata.get("token_usage").get("prompt_tokens") if input_tokens: - assert ( - input_tokens - == attributes[gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS] - ) + assert input_tokens == attributes[gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS] else: assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in attributes - output_tokens = response.response_metadata.get("token_usage").get( - "completion_tokens" - ) + output_tokens = response.response_metadata.get("token_usage").get("completion_tokens") if output_tokens: - assert ( - output_tokens - == attributes[gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS] - ) + assert output_tokens == attributes[gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS] else: assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attributes @@ -224,30 +192,22 @@ def assert_openai_completion_attributes( assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes -def assert_openai_completion_attributes_with_error( - span: ReadableSpan, verify_content: bool = True -): +def assert_openai_completion_attributes_with_error(span: ReadableSpan, verify_content: bool = True): assert span is not None assert span.name == "chat gpt-3.5-turbo" attributes = span.attributes assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" - assert ( - attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" - ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" assert gen_ai_attributes.GEN_AI_RESPONSE_MODEL not in attributes assert attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 assert attributes["gen_ai.provider.name"] == "openai" assert gen_ai_attributes.GEN_AI_RESPONSE_ID not in attributes assert attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 - assert ( - attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 - ) + assert attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.5 assert attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.5 - stop_sequences = attributes.get( - gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES - ) + stop_sequences = attributes.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES) assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) assert attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 @@ -275,9 +235,7 @@ def assert_duration_metric(metric, parent_span): assert len(metric.data.data_points) == 1 assert metric.data.data_points[0].sum > 0 - assert_duration_metric_attributes( - metric.data.data_points[0].attributes, parent_span - ) + assert_duration_metric_attributes(metric.data.data_points[0].attributes, parent_span) assert_exemplars( metric.data.data_points[0].exemplars, metric.data.data_points[0].sum, @@ -289,8 +247,7 @@ def assert_duration_metric_attributes(attributes, parent_span): assert len(attributes) == 4 assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" assert ( - attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) - == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value ) assert ( attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) @@ -307,9 +264,7 @@ def assert_duration_metric_when_error(metric, parent_span): assert len(metric.data.data_points) == 1 assert metric.data.data_points[0].sum > 0 - assert_duration_metric_attributes_when_error( - metric.data.data_points[0].attributes, parent_span - ) + assert_duration_metric_attributes_when_error(metric.data.data_points[0].attributes, parent_span) assert_exemplars( metric.data.data_points[0].exemplars, metric.data.data_points[0].sum, @@ -322,8 +277,7 @@ def assert_duration_metric_attributes_when_error(attributes, parent_span): assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" assert ( - attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) - == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value ) assert ( attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) @@ -336,9 +290,7 @@ def assert_token_usage_metric(metric, parent_span): assert len(metric.data.data_points) == 2 assert metric.data.data_points[0].sum > 0 - assert_token_usage_metric_attributes( - metric.data.data_points[0].attributes, parent_span, "input" - ) + assert_token_usage_metric_attributes(metric.data.data_points[0].attributes, parent_span, "input") assert_exemplars( metric.data.data_points[0].exemplars, metric.data.data_points[0].sum, @@ -346,9 +298,7 @@ def assert_token_usage_metric(metric, parent_span): ) assert metric.data.data_points[1].sum > 0 - assert_token_usage_metric_attributes( - metric.data.data_points[1].attributes, parent_span, "output" - ) + assert_token_usage_metric_attributes(metric.data.data_points[1].attributes, parent_span, "output") assert_exemplars( metric.data.data_points[1].exemplars, metric.data.data_points[1].sum, @@ -360,8 +310,7 @@ def assert_token_usage_metric_attributes(attributes, parent_span, token_type): assert len(attributes) == 5 assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" assert ( - attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) - == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value ) assert ( attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) @@ -383,9 +332,7 @@ def assert_exemplars(exemplars, datapoint_sum, parent_span): def assert_log_record(log_record, parent_span): # Event name (support both .event_name and attributes for SDK differences) - event_name = getattr( - log_record, "event_name", None - ) or log_record.attributes.get(EventAttributes.EVENT_NAME) + event_name = getattr(log_record, "event_name", None) or log_record.attributes.get(EventAttributes.EVENT_NAME) assert event_name == "gen_ai.client.inference.operation.details" attrs = log_record.attributes @@ -397,37 +344,24 @@ def assert_log_record(log_record, parent_span): assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.5 assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY) == 0.5 assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) == 100 - stop_seqs = _normalize_to_list( - attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, []) - ) + stop_seqs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, [])) assert stop_seqs == ["\n", "Human:", "AI:"] assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_SEED) == 100 - assert attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS) == ( - "stop", - ) - assert ( - attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) - == "gpt-3.5-turbo-0125" - ) + assert attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attrs.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) == "gpt-3.5-turbo-0125" assert gen_ai_attributes.GEN_AI_RESPONSE_ID in attrs assert attrs.get(gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS) == 24 assert attrs.get(gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS) == 7 # Input/output messages: normalize list/tuple and compare structure - input_msgs = _normalize_to_list( - attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) - ) + input_msgs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, [])) expected_input = [ { - "parts": [ - {"content": "You are a helpful assistant!", "type": "text"} - ], + "parts": [{"content": "You are a helpful assistant!", "type": "text"}], "role": "system", }, { - "parts": [ - {"content": "What is the capital of France?", "type": "text"} - ], + "parts": [{"content": "What is the capital of France?", "type": "text"}], "role": "human", }, ] @@ -437,24 +371,18 @@ def assert_log_record(log_record, parent_span): assert got["role"] == exp["role"] assert _normalize_to_list(got["parts"]) == exp["parts"] - output_msgs = _normalize_to_list( - attrs.get(gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, []) - ) + output_msgs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, [])) assert len(output_msgs) == 1 out = _normalize_to_dict(output_msgs[0]) assert out["role"] == "ai" assert out["finish_reason"] == "stop" - assert _normalize_to_list(out["parts"]) == [ - {"content": "The capital of France is Paris.", "type": "text"} - ] + assert _normalize_to_list(out["parts"]) == [{"content": "The capital of France is Paris.", "type": "text"}] assert_log_parent(log_record, parent_span) def assert_log_record_when_error(log_record, parent_span): # Event name (support both .event_name and attributes for SDK differences) - event_name = getattr( - log_record, "event_name", None - ) or log_record.attributes.get(EventAttributes.EVENT_NAME) + event_name = getattr(log_record, "event_name", None) or log_record.attributes.get(EventAttributes.EVENT_NAME) assert event_name == "gen_ai.client.inference.operation.details" attrs = log_record.attributes @@ -466,9 +394,7 @@ def assert_log_record_when_error(log_record, parent_span): assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.5 assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY) == 0.5 assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) == 100 - stop_seqs = _normalize_to_list( - attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, []) - ) + stop_seqs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES, [])) assert stop_seqs == ["\n", "Human:", "AI:"] assert attrs.get(gen_ai_attributes.GEN_AI_REQUEST_SEED) == 100 assert gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS not in attrs @@ -478,20 +404,14 @@ def assert_log_record_when_error(log_record, parent_span): assert gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in attrs # Input/output messages: normalize list/tuple and compare structure - input_msgs = _normalize_to_list( - attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) - ) + input_msgs = _normalize_to_list(attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, [])) expected_input = [ { - "parts": [ - {"content": "You are a helpful assistant!", "type": "text"} - ], + "parts": [{"content": "You are a helpful assistant!", "type": "text"}], "role": "system", }, { - "parts": [ - {"content": "What is the capital of France?", "type": "text"} - ], + "parts": [{"content": "What is the capital of France?", "type": "text"}], "role": "human", }, ] @@ -519,4 +439,4 @@ def assert_log_parent(log_record, span): if span: assert log_record.trace_id == span.get_span_context().trace_id assert log_record.span_id == span.get_span_context().span_id - assert log_record.trace_flags == span.get_span_context().trace_flags \ No newline at end of file + assert log_record.trace_flags == span.get_span_context().trace_flags From 422b56837bc00c8dae12c2a88c90a219366255e6 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:35:50 -0700 Subject: [PATCH 04/20] Add dependency --- .github/workflows/pr-validation.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index d31e23aa..f4aed84b 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -26,6 +26,14 @@ jobs: - name: Check formatting with Black run: tox -e black + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e .[dev] + python -m pip install -e src/_langchain/opentelemetry-instrumentation-langchain + + - name: Check formatting with Black + run: black --check src tests samples - name: Lint with Pylint run: tox -e pylint From 05ba8c886931884b7bea34c5829d7df682c04733 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:40:50 -0700 Subject: [PATCH 05/20] Fix format on langchain samples --- samples/langchain/sample_langchain_azure.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/langchain/sample_langchain_azure.py b/samples/langchain/sample_langchain_azure.py index a43f5a71..fc10c911 100644 --- a/samples/langchain/sample_langchain_azure.py +++ b/samples/langchain/sample_langchain_azure.py @@ -117,8 +117,7 @@ def find_activities(destination: str) -> str: def main(): session_id = str(uuid4()) user_request = ( - "Plan a weekend trip to Paris from Seattle next month. " - + "Find flights, a boutique hotel, and fun activities." + "Plan a weekend trip to Paris from Seattle next month. " "Find flights, a boutique hotel, and fun activities." ) print("Travel Planner (Nested Agents)") From 553308d49d141f554575eeeeaf1491b655155f6b Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:36:21 -0700 Subject: [PATCH 06/20] Update src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../opentelemetry-instrumentation-langchain/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py index 18546db4..02f60002 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -187,7 +187,7 @@ def serialize(cassette_dict): @staticmethod def deserialize(cassette_string): - return yaml.load(cassette_string, Loader=yaml.Loader) + return yaml.safe_load(cassette_string) @pytest.fixture(scope="module", autouse=True) From 4c93064818b75fa0c04b87f1fdaae414c01b7161 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:45:01 -0700 Subject: [PATCH 07/20] Remove span manager for now --- .../instrumentation/langchain/span_manager.py | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py deleted file mode 100644 index c718a52c..00000000 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from dataclasses import dataclass, field -from typing import Dict, List, Optional -from uuid import UUID - -from opentelemetry.semconv._incubating.attributes import ( - gen_ai_attributes as GenAI, -) -from opentelemetry.semconv.attributes import ( - error_attributes, -) -from opentelemetry.trace import Span, SpanKind, Tracer, set_span_in_context -from opentelemetry.trace.status import Status, StatusCode - -__all__ = ["_SpanManager"] - - -@dataclass -class _SpanState: - span: Span - children: List[UUID] = field(default_factory=lambda: []) # pylint: disable=unnecessary-lambda - - -class _SpanManager: - def __init__( - self, - tracer: Tracer, - ) -> None: - self._tracer = tracer - - # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships - # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 # pylint: disable=fixme - self.spans: Dict[UUID, _SpanState] = {} - - def _create_span( - self, - run_id: UUID, - parent_run_id: Optional[UUID], - span_name: str, - kind: SpanKind = SpanKind.INTERNAL, - ) -> Span: - if parent_run_id is not None and parent_run_id in self.spans: - parent_state = self.spans[parent_run_id] - parent_span = parent_state.span - ctx = set_span_in_context(parent_span) - span = self._tracer.start_span(name=span_name, kind=kind, context=ctx) - parent_state.children.append(run_id) - else: - # top-level or missing parent - span = self._tracer.start_span(name=span_name, kind=kind) - set_span_in_context(span) - - span_state = _SpanState(span=span) - self.spans[run_id] = span_state - - return span - - def create_chat_span( - self, - run_id: UUID, - parent_run_id: Optional[UUID], - request_model: str, - ) -> Span: - span = self._create_span( - run_id=run_id, - parent_run_id=parent_run_id, - span_name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}", - kind=SpanKind.CLIENT, - ) - span.set_attribute( - GenAI.GEN_AI_OPERATION_NAME, - GenAI.GenAiOperationNameValues.CHAT.value, - ) - if request_model: - span.set_attribute(GenAI.GEN_AI_REQUEST_MODEL, request_model) - - return span - - def end_span(self, run_id: UUID) -> None: - state = self.spans[run_id] - for child_id in state.children: - child_state = self.spans.get(child_id) - if child_state: - child_state.span.end() - del self.spans[child_id] - state.span.end() - del self.spans[run_id] - - def get_span(self, run_id: UUID) -> Optional[Span]: - state = self.spans.get(run_id) - return state.span if state else None - - def handle_error(self, error: BaseException, run_id: UUID): - span = self.get_span(run_id) - if span is None: - # If the span does not exist, we cannot set the error status - return - span.set_status(Status(StatusCode.ERROR, str(error))) - span.set_attribute(error_attributes.ERROR_TYPE, type(error).__qualname__) - self.end_span(run_id) From ec2ce218378bba2088e8b915d930ec09ae99ce65 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:53:30 -0700 Subject: [PATCH 08/20] Add dependency and fix tests --- .../pyproject.toml | 13 ++++++++++++- .../tests/conftest.py | 19 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml index 7cc0f34c..fd27b2f2 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml @@ -25,12 +25,23 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "opentelemetry-instrumentation ~= 0.57b0", + "opentelemetry-api ~= 1.40", + "opentelemetry-instrumentation ~= 0.61b0", + "opentelemetry-semantic-conventions ~= 0.61b0", ] [project.optional-dependencies] instruments = [ "langchain >= 0.3.21", + "langchain-core >= 0.3", +] +test = [ + "opentelemetry-sdk ~= 1.40", + "langchain-openai >= 0.3", + "openai >= 1.0", + "vcrpy >= 6.0", + "pytest >= 8.0", + "pytest-recording", ] [project.entry-points.opentelemetry_instrumentor] diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py index 02f60002..5dbf8797 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -89,13 +89,18 @@ def start_instrumentation( logger_provider, ): # pylint: disable=import-outside-toplevel - # Ensure stability opt-in is set before the first call to _initialize() - os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = "gen_ai_latest_experimental" - # Reset the cached initialization so the env var is picked up from opentelemetry.instrumentation._semconv import ( _OpenTelemetrySemanticConventionStability, ) + # Save prior state + prev_env = os.environ.get("OTEL_SEMCONV_STABILITY_OPT_IN") + prev_initialized = _OpenTelemetrySemanticConventionStability._initialized + prev_mapping = _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING.copy() + + # Ensure stability opt-in is set before the first call to _initialize() + os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = "gen_ai_latest_experimental" + # Reset the cached initialization so the env var is picked up _OpenTelemetrySemanticConventionStability._initialized = False _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING = {} _OpenTelemetrySemanticConventionStability._initialize() @@ -110,6 +115,14 @@ def start_instrumentation( yield instrumentor instrumentor.uninstrument() + # Restore prior state + if prev_env is None: + os.environ.pop("OTEL_SEMCONV_STABILITY_OPT_IN", None) + else: + os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = prev_env + _OpenTelemetrySemanticConventionStability._initialized = prev_initialized + _OpenTelemetrySemanticConventionStability._OTEL_SEMCONV_STABILITY_SIGNAL_MAPPING = prev_mapping + @pytest.fixture(autouse=True) def environment(): From 99870ebecde21bc6fda3a8c979897b24a5254b11 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:58:39 -0700 Subject: [PATCH 09/20] Fix dependency issues --- .github/workflows/pr-validation.yml | 8 ++++++-- pyproject.toml | 2 ++ .../pyproject.toml | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f4aed84b..42b21b0d 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -30,12 +30,16 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e .[dev] - python -m pip install -e src/_langchain/opentelemetry-instrumentation-langchain + python -m pip install -e "src/_langchain/opentelemetry-instrumentation-langchain[instruments,test]" - name: Check formatting with Black run: black --check src tests samples - - name: Lint with Pylint + - name: Lint with Pylint (root) + run: pylint src tests + + - name: Lint with Pylint (langchain) + working-directory: src/_langchain/opentelemetry-instrumentation-langchain run: tox -e pylint - name: Typecheck with Mypy diff --git a/pyproject.toml b/pyproject.toml index d21628a8..0b8f20db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,8 @@ target-version = ["py310"] [tool.pylint.main] py-version = "3.10" +ignore-paths = ["tests/azureMonitor"] +ignore-paths = ["src/_langchain"] [tool.pylint.format] max-line-length = 120 diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml index fd27b2f2..8f231fa9 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml @@ -39,6 +39,7 @@ test = [ "opentelemetry-sdk ~= 1.40", "langchain-openai >= 0.3", "openai >= 1.0", + "PyYAML >= 6.0", "vcrpy >= 6.0", "pytest >= 8.0", "pytest-recording", @@ -68,4 +69,5 @@ disable = [ "missing-module-docstring", "import-error", "no-name-in-module", + "too-many-positional-arguments", ] \ No newline at end of file From b9aeffbe32976a8877d7e645d5c75f7e5d3be371 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 17:01:34 -0700 Subject: [PATCH 10/20] Fix mypy dependency issues --- .github/workflows/pr-validation.yml | 15 +++++++++++++-- pyproject.toml | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 42b21b0d..becb83df 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -42,8 +42,19 @@ jobs: working-directory: src/_langchain/opentelemetry-instrumentation-langchain run: tox -e pylint - - name: Typecheck with Mypy - run: tox -e mypy + - name: Typecheck with Mypy (root) + run: mypy src + + - name: Typecheck with Mypy (langchain) + working-directory: src/_langchain/opentelemetry-instrumentation-langchain + run: mypy src --ignore-missing-imports --no-strict + + - name: Typecheck with Pyright (root) + run: pyright src + + - name: Typecheck with Pyright (langchain) + working-directory: src/_langchain/opentelemetry-instrumentation-langchain + run: pyright src tests: name: Pytest (${{ matrix.python-version }}) diff --git a/pyproject.toml b/pyproject.toml index 0b8f20db..e0a046d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,5 +67,34 @@ disable = [ "invalid-name", ] +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +namespace_packages = true +explicit_package_bases = true +mypy_path = "src" + +[[tool.mypy.overrides]] +module = "microsoft.azureMonitor.*" +disallow_untyped_defs = false +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = [ + "django.*", + "azure.core.*", + "opentelemetry.instrumentation.*", +] +ignore_missing_imports = true +exclude = ["src/_langchain"] + [tool.pytest.ini_options] testpaths = ["tests"] + +[tool.pyright] +pythonVersion = "3.10" +typeCheckingMode = "basic" +ignore = ["src/microsoft/azureMonitor", "tests/azureMonitor"] +exclude = ["src/_langchain"] From 5ada14db0b4d908ac78bc19d8a93720c151e5e94 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 17:06:22 -0700 Subject: [PATCH 11/20] fix flag --- .github/workflows/pr-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index becb83df..32646aae 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -47,7 +47,7 @@ jobs: - name: Typecheck with Mypy (langchain) working-directory: src/_langchain/opentelemetry-instrumentation-langchain - run: mypy src --ignore-missing-imports --no-strict + run: mypy src --ignore-missing-imports --allow-untyped-defs - name: Typecheck with Pyright (root) run: pyright src From 570830c5e6a28097e6450c995fcc9052a684e61c Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 17:10:58 -0700 Subject: [PATCH 12/20] Fix mypy errors --- .../src/opentelemetry/instrumentation/langchain/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py index ca056106..3ce3b978 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -42,7 +42,7 @@ from langchain_core.callbacks import BaseCallbackHandler from wrapt import wrap_function_wrapper # type: ignore -from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore[attr-defined] from opentelemetry.instrumentation.langchain.callback_handler import ( OpenTelemetryLangChainCallbackHandler, ) @@ -59,7 +59,7 @@ class LangChainInstrumentor(BaseInstrumentor): """ def instrumentation_dependencies(self) -> Collection[str]: - return _instruments + return _instruments # type: ignore[no-any-return] def _instrument(self, **kwargs: Any): """ From bcaf1b9d40f5d45c07cf9ff297dbdf1075ec8802 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 17:13:49 -0700 Subject: [PATCH 13/20] Fix pyright errors --- .../opentelemetry-instrumentation-langchain/pyproject.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml index 8f231fa9..16a63818 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml @@ -70,4 +70,9 @@ disable = [ "import-error", "no-name-in-module", "too-many-positional-arguments", -] \ No newline at end of file +] + +[tool.pyright] +pythonVersion = "3.9" +typeCheckingMode = "basic" +reportMissingImports = false \ No newline at end of file From 01a6b632ab48ddcb345dab323fb7607cd053d976 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 13:01:41 -0700 Subject: [PATCH 14/20] Add the internal otel langchain instrumentation --- .../instrumentation/langchain/span_manager.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py new file mode 100644 index 00000000..be71280e --- /dev/null +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py @@ -0,0 +1,117 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from uuid import UUID + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import ( + error_attributes, +) +from opentelemetry.trace import Span, SpanKind, Tracer, set_span_in_context +from opentelemetry.trace.status import Status, StatusCode + +__all__ = ["_SpanManager"] + + +@dataclass +class _SpanState: + span: Span + children: List[UUID] = field(default_factory=lambda: list()) + + +class _SpanManager: + def __init__( + self, + tracer: Tracer, + ) -> None: + self._tracer = tracer + + # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships + # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 + self.spans: Dict[UUID, _SpanState] = {} + + def _create_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + span_name: str, + kind: SpanKind = SpanKind.INTERNAL, + ) -> Span: + if parent_run_id is not None and parent_run_id in self.spans: + parent_state = self.spans[parent_run_id] + parent_span = parent_state.span + ctx = set_span_in_context(parent_span) + span = self._tracer.start_span( + name=span_name, kind=kind, context=ctx + ) + parent_state.children.append(run_id) + else: + # top-level or missing parent + span = self._tracer.start_span(name=span_name, kind=kind) + set_span_in_context(span) + + span_state = _SpanState(span=span) + self.spans[run_id] = span_state + + return span + + def create_chat_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + request_model: str, + ) -> Span: + span = self._create_span( + run_id=run_id, + parent_run_id=parent_run_id, + span_name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}", + kind=SpanKind.CLIENT, + ) + span.set_attribute( + GenAI.GEN_AI_OPERATION_NAME, + GenAI.GenAiOperationNameValues.CHAT.value, + ) + if request_model: + span.set_attribute(GenAI.GEN_AI_REQUEST_MODEL, request_model) + + return span + + def end_span(self, run_id: UUID) -> None: + state = self.spans[run_id] + for child_id in state.children: + child_state = self.spans.get(child_id) + if child_state: + child_state.span.end() + del self.spans[child_id] + state.span.end() + del self.spans[run_id] + + def get_span(self, run_id: UUID) -> Optional[Span]: + state = self.spans.get(run_id) + return state.span if state else None + + def handle_error(self, error: BaseException, run_id: UUID): + span = self.get_span(run_id) + if span is None: + # If the span does not exist, we cannot set the error status + return + span.set_status(Status(StatusCode.ERROR, str(error))) + span.set_attribute( + error_attributes.ERROR_TYPE, type(error).__qualname__ + ) + self.end_span(run_id) \ No newline at end of file From 110766e7606deb0dc482bc1ecb80f463bfbb867a Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 15:58:49 -0700 Subject: [PATCH 15/20] Fix lint and format --- .../pyproject.toml | 2 +- .../instrumentation/langchain/span_manager.py | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml index 16a63818..5cdb51f3 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/src/_langchain/opentelemetry-instrumentation-langchain/pyproject.toml @@ -75,4 +75,4 @@ disable = [ [tool.pyright] pythonVersion = "3.9" typeCheckingMode = "basic" -reportMissingImports = false \ No newline at end of file +reportMissingImports = false diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py index be71280e..c718a52c 100644 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py +++ b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py @@ -31,7 +31,7 @@ @dataclass class _SpanState: span: Span - children: List[UUID] = field(default_factory=lambda: list()) + children: List[UUID] = field(default_factory=lambda: []) # pylint: disable=unnecessary-lambda class _SpanManager: @@ -42,7 +42,7 @@ def __init__( self._tracer = tracer # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships - # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 + # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 # pylint: disable=fixme self.spans: Dict[UUID, _SpanState] = {} def _create_span( @@ -56,9 +56,7 @@ def _create_span( parent_state = self.spans[parent_run_id] parent_span = parent_state.span ctx = set_span_in_context(parent_span) - span = self._tracer.start_span( - name=span_name, kind=kind, context=ctx - ) + span = self._tracer.start_span(name=span_name, kind=kind, context=ctx) parent_state.children.append(run_id) else: # top-level or missing parent @@ -111,7 +109,5 @@ def handle_error(self, error: BaseException, run_id: UUID): # If the span does not exist, we cannot set the error status return span.set_status(Status(StatusCode.ERROR, str(error))) - span.set_attribute( - error_attributes.ERROR_TYPE, type(error).__qualname__ - ) - self.end_span(run_id) \ No newline at end of file + span.set_attribute(error_attributes.ERROR_TYPE, type(error).__qualname__) + self.end_span(run_id) From b1cfab692715592325a4b2c4d3ccee769f8a56fa Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 6 Apr 2026 16:45:01 -0700 Subject: [PATCH 16/20] Remove span manager for now --- .../instrumentation/langchain/span_manager.py | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py diff --git a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py deleted file mode 100644 index c718a52c..00000000 --- a/src/_langchain/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from dataclasses import dataclass, field -from typing import Dict, List, Optional -from uuid import UUID - -from opentelemetry.semconv._incubating.attributes import ( - gen_ai_attributes as GenAI, -) -from opentelemetry.semconv.attributes import ( - error_attributes, -) -from opentelemetry.trace import Span, SpanKind, Tracer, set_span_in_context -from opentelemetry.trace.status import Status, StatusCode - -__all__ = ["_SpanManager"] - - -@dataclass -class _SpanState: - span: Span - children: List[UUID] = field(default_factory=lambda: []) # pylint: disable=unnecessary-lambda - - -class _SpanManager: - def __init__( - self, - tracer: Tracer, - ) -> None: - self._tracer = tracer - - # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships - # TODO: Use weak references or a TTL cache to avoid memory leaks in long-running processes. See #3735 # pylint: disable=fixme - self.spans: Dict[UUID, _SpanState] = {} - - def _create_span( - self, - run_id: UUID, - parent_run_id: Optional[UUID], - span_name: str, - kind: SpanKind = SpanKind.INTERNAL, - ) -> Span: - if parent_run_id is not None and parent_run_id in self.spans: - parent_state = self.spans[parent_run_id] - parent_span = parent_state.span - ctx = set_span_in_context(parent_span) - span = self._tracer.start_span(name=span_name, kind=kind, context=ctx) - parent_state.children.append(run_id) - else: - # top-level or missing parent - span = self._tracer.start_span(name=span_name, kind=kind) - set_span_in_context(span) - - span_state = _SpanState(span=span) - self.spans[run_id] = span_state - - return span - - def create_chat_span( - self, - run_id: UUID, - parent_run_id: Optional[UUID], - request_model: str, - ) -> Span: - span = self._create_span( - run_id=run_id, - parent_run_id=parent_run_id, - span_name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}", - kind=SpanKind.CLIENT, - ) - span.set_attribute( - GenAI.GEN_AI_OPERATION_NAME, - GenAI.GenAiOperationNameValues.CHAT.value, - ) - if request_model: - span.set_attribute(GenAI.GEN_AI_REQUEST_MODEL, request_model) - - return span - - def end_span(self, run_id: UUID) -> None: - state = self.spans[run_id] - for child_id in state.children: - child_state = self.spans.get(child_id) - if child_state: - child_state.span.end() - del self.spans[child_id] - state.span.end() - del self.spans[run_id] - - def get_span(self, run_id: UUID) -> Optional[Span]: - state = self.spans.get(run_id) - return state.span if state else None - - def handle_error(self, error: BaseException, run_id: UUID): - span = self.get_span(run_id) - if span is None: - # If the span does not exist, we cannot set the error status - return - span.set_status(Status(StatusCode.ERROR, str(error))) - span.set_attribute(error_attributes.ERROR_TYPE, type(error).__qualname__) - self.end_span(run_id) From 8c2d5deb6ba433a5b87bed890d4e319995d2c83d Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 7 Apr 2026 14:20:29 -0700 Subject: [PATCH 17/20] Fix line --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e0a046d1..83d562e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,7 @@ target-version = ["py310"] [tool.pylint.main] py-version = "3.10" -ignore-paths = ["tests/azureMonitor"] -ignore-paths = ["src/_langchain"] +ignore-paths = ["tests/azureMonitor", "src/_langchain"] [tool.pylint.format] max-line-length = 120 From f20a9f5dde19449246cf82a8a68e104c2ea52218 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 7 Apr 2026 14:32:44 -0700 Subject: [PATCH 18/20] Fix line --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 83d562e7..43573a07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ disallow_untyped_defs = true namespace_packages = true explicit_package_bases = true mypy_path = "src" +exclude = ["src/_langchain"] [[tool.mypy.overrides]] module = "microsoft.azureMonitor.*" @@ -87,7 +88,6 @@ module = [ "opentelemetry.instrumentation.*", ] ignore_missing_imports = true -exclude = ["src/_langchain"] [tool.pytest.ini_options] testpaths = ["tests"] From b42387cfe623b458c769f3f3aee67d71fa0125b2 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 7 Apr 2026 17:16:25 -0700 Subject: [PATCH 19/20] Fix lint --- samples/langchain/sample_langchain_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/langchain/sample_langchain_azure.py b/samples/langchain/sample_langchain_azure.py index fc10c911..caf6aa7a 100644 --- a/samples/langchain/sample_langchain_azure.py +++ b/samples/langchain/sample_langchain_azure.py @@ -117,7 +117,7 @@ def find_activities(destination: str) -> str: def main(): session_id = str(uuid4()) user_request = ( - "Plan a weekend trip to Paris from Seattle next month. " "Find flights, a boutique hotel, and fun activities." + "Plan a weekend trip to Paris from Seattle next month. Find flights, a boutique hotel, and fun activities." ) print("Travel Planner (Nested Agents)") From c0661b707db6cae8e829fd2ee2d2fc0568790135 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 7 Apr 2026 17:22:22 -0700 Subject: [PATCH 20/20] Add exclusion from mypy root --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index 2864dd03..e75b7aa7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,6 +6,7 @@ disallow_untyped_defs = true namespace_packages = true explicit_package_bases = true mypy_path = src +exclude = (?x)(src/_langchain/ | src/_a365/) [mypy-microsoft.azureMonitor.*] disallow_untyped_defs = false