diff --git a/A365_DOCUMENTATION.md b/A365_DOCUMENTATION.md index 17aa49ac..723aafea 100644 --- a/A365_DOCUMENTATION.md +++ b/A365_DOCUMENTATION.md @@ -356,22 +356,59 @@ Top-level agent invocation — wraps the entire request/response cycle: ```python from microsoft.opentelemetry.a365.core import ( - AgentDetails, CallerDetails, Channel, InvokeAgentScope, - InvokeAgentScopeDetails, Request, ServiceEndpoint, UserDetails, + AgentDetails, CallerDetails, Channel, GenAiRequestParameters, + GenAiResponseParameters, InvokeAgentScope, InvokeAgentScopeDetails, + Request, ServiceEndpoint, TextPart, UserDetails, ) agent = AgentDetails(agent_id="agent-001", agent_name="My Agent", tenant_id="t1") with InvokeAgentScope.start( request=Request(content="Hello", session_id="s1", conversation_id="c1", channel=Channel(name="msteams")), - scope_details=InvokeAgentScopeDetails(endpoint=ServiceEndpoint(hostname="agent.contoso.com")), + scope_details=InvokeAgentScopeDetails( + endpoint=ServiceEndpoint(hostname="agent.contoso.com"), + request_parameters=GenAiRequestParameters( + model="gpt-4o", + max_tokens=256, + temperature=0.2, + stop_sequences=["END"], + output_type="text", + system_instructions=[TextPart(content="Be concise.")], + ), + ), agent_details=agent, caller_details=CallerDetails(user_details=UserDetails(user_id="u1", user_email="u@contoso.com")), ) as scope: # ... do work ... + scope.record_response_parameters( + GenAiResponseParameters( + finish_reasons=["stop"], + input_tokens=42, + output_tokens=18, + cache_write_input_tokens=4, + ) + ) scope.record_response("Here is the answer.") ``` +`GenAiRequestParameters` values are recorded when the scope starts. Use +`scope.record_response_parameters()` for response values that are only known +after completion. Fields left as `None` are omitted from the span. Sequence +fields such as `stop_sequences` and `finish_reasons` are emitted as +OpenTelemetry string arrays. + +Supported semantic attributes: + +- Request: `gen_ai.request.model`, `gen_ai.request.seed`, + `gen_ai.request.choice.count`, `gen_ai.request.frequency_penalty`, + `gen_ai.request.max_tokens`, `gen_ai.request.presence_penalty`, + `gen_ai.request.stop_sequences`, `gen_ai.request.temperature`, + `gen_ai.request.top_p`, `gen_ai.data_source.id`, `gen_ai.output.type`, + `gen_ai.system_instructions` +- Response: `gen_ai.response.finish_reasons`, `gen_ai.usage.input_tokens`, + `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_write.input_tokens`, + `gen_ai.usage.cache_read.input_tokens` + ### ExecuteToolScope ```python diff --git a/CHANGELOG.md b/CHANGELOG.md index 9da53e9e..b518c6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ # Release History +# Unreleased +### Features Added +- Add Python-native `InvokeAgentScope` request and response parameter models + that emit OpenTelemetry GenAI semantic attributes, including structured + system instructions and cache read/write token counts, introduced by .NET + PR #120. + # 1.3.9 (2026-09-09) ### Features Added - Update OpenTelemetry dependencies to latest versions, bump `langchain-core` minimum version to address S360, and support the new `httpx2` entry point exposed by `opentelemetry-instrumentation-httpx`. diff --git a/README.md b/README.md index edae89b4..4875e67d 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,49 @@ See the [A365 guide](https://github.com/microsoft/opentelemetry-distro-python/bl > For A365 token resolver patterns, baggage, and scope classes, see the [A365 guide](https://github.com/microsoft/opentelemetry-distro-python/blob/main/A365_DOCUMENTATION.md). +### InvokeAgent semantic parameters + +Manual `InvokeAgentScope` instrumentation accepts Python-native request and response parameter models: + +```python +from microsoft.opentelemetry.a365.core import ( + AgentDetails, + GenAiRequestParameters, + GenAiResponseParameters, + InvokeAgentScope, + InvokeAgentScopeDetails, + Request, + TextPart, +) + +request = Request(content="Hello") +agent_details = AgentDetails(agent_id="agent-001") +details = InvokeAgentScopeDetails( + request_parameters=GenAiRequestParameters( + model="gpt-4o", + max_tokens=256, + stop_sequences=["END"], + system_instructions=[TextPart(content="Be concise.")], + ) +) + +with InvokeAgentScope.start(request, details, agent_details) as scope: + # Invoke your agent here. + scope.record_response_parameters( + GenAiResponseParameters( + finish_reasons=["stop"], + output_tokens=18, + cache_write_input_tokens=4, + ) + ) +``` + +Unset fields are omitted. Supported attributes include `gen_ai.request.*`, +`gen_ai.data_source.id`, `gen_ai.output.type`, `gen_ai.system_instructions`, +`gen_ai.response.finish_reasons`, and `gen_ai.usage.*` token counts. See the +[A365 guide](https://github.com/microsoft/opentelemetry-distro-python/blob/main/A365_DOCUMENTATION.md#invokeagentscope) +for the full list. + ### Sampling Configured via standard OpenTelemetry environment variables: diff --git a/src/microsoft/opentelemetry/a365/README.md b/src/microsoft/opentelemetry/a365/README.md index 16cdae73..dce127ba 100644 --- a/src/microsoft/opentelemetry/a365/README.md +++ b/src/microsoft/opentelemetry/a365/README.md @@ -21,11 +21,13 @@ Core tracing primitives — scopes, configuration, data models, and internal uti | `agent_details.py` | `AgentDetails` dataclass — metadata about an AI agent (ID, name, description, blueprint/platform IDs, tenant, version). | | `channel.py` | `Channel` dataclass — channel context (name, link) for agent execution. | | `constants.py` | Core-level constants for span operations, OTel conventions, feature switches, and error types. | +| `gen_ai_request_parameters.py` | GenAI request parameter models for generation settings and related semantic attributes. | +| `gen_ai_response_parameters.py` | GenAI response parameter models for finish reasons and token usage attributes. | | `execute_tool_scope.py` | `ExecuteToolScope` — tracing scope for AI tool executions. Records tool name, arguments, call ID, type, and endpoint. | | `inference_call_details.py` | `InferenceCallDetails` dataclass — LLM call metadata (model, provider, token counts, finish reasons, endpoint). | | `inference_operation_type.py` | `InferenceOperationType` enum — Chat, TextCompletion, GenerateContent. | | `inference_scope.py` | `InferenceScope` — tracing scope for LLM/AI inference operations. Records input/output messages, model details, token usage, and user info. | -| `invoke_agent_details.py` | `InvokeAgentScopeDetails` dataclass — configuration for agent invocation tracing (endpoint). | +| `invoke_agent_details.py` | `InvokeAgentScopeDetails` dataclass — configuration for agent invocation tracing (endpoint, optional request parameters, optional response parameters). | | `invoke_agent_scope.py` | `InvokeAgentScope` — tracing scope for agent invocations. Records request/response, caller details (human and agent-to-agent), channel, and endpoint info. | | `message_utils.py` | Conversion and serialization helpers for OTel gen-ai message format. Normalizes strings/lists to structured `InputMessages`/`OutputMessages`. | | `opentelemetry_scope.py` | `OpenTelemetryScope` — base class for all tracing scopes. Manages span creation, attribute setting, context management, and baggage building. | diff --git a/src/microsoft/opentelemetry/a365/constants.py b/src/microsoft/opentelemetry/a365/constants.py index a5af7008..90ae6351 100644 --- a/src/microsoft/opentelemetry/a365/constants.py +++ b/src/microsoft/opentelemetry/a365/constants.py @@ -36,8 +36,15 @@ GEN_AI_CLIENT_OPERATION_DURATION_METRIC_NAME = "gen_ai.client.operation.duration" GEN_AI_CLIENT_TOKEN_USAGE_METRIC_NAME = "gen_ai.client.token.usage" GEN_AI_OPERATION_NAME_KEY = "gen_ai.operation.name" +GEN_AI_DATA_SOURCE_ID_KEY = "gen_ai.data_source.id" +GEN_AI_OUTPUT_TYPE_KEY = "gen_ai.output.type" +GEN_AI_REQUEST_CHOICE_COUNT_KEY = "gen_ai.request.choice.count" +GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY = "gen_ai.request.frequency_penalty" GEN_AI_REQUEST_MAX_TOKENS_KEY = "gen_ai.request.max_tokens" GEN_AI_REQUEST_MODEL_KEY = "gen_ai.request.model" +GEN_AI_REQUEST_PRESENCE_PENALTY_KEY = "gen_ai.request.presence_penalty" +GEN_AI_REQUEST_SEED_KEY = "gen_ai.request.seed" +GEN_AI_REQUEST_STOP_SEQUENCES_KEY = "gen_ai.request.stop_sequences" GEN_AI_REQUEST_TEMPERATURE_KEY = "gen_ai.request.temperature" GEN_AI_REQUEST_TOP_P_KEY = "gen_ai.request.top_p" GEN_AI_RESPONSE_FINISH_REASONS_KEY = "gen_ai.response.finish_reasons" @@ -52,6 +59,8 @@ GEN_AI_CONVERSATION_ID_KEY = "gen_ai.conversation.id" GEN_AI_CONVERSATION_ITEM_LINK_KEY = "microsoft.conversation.item.link" GEN_AI_TOKEN_TYPE_KEY = "gen_ai.token.type" +GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY = "gen_ai.usage.cache_write.input_tokens" +GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY = "gen_ai.usage.cache_read.input_tokens" GEN_AI_USAGE_INPUT_TOKENS_KEY = "gen_ai.usage.input_tokens" GEN_AI_USAGE_OUTPUT_TOKENS_KEY = "gen_ai.usage.output_tokens" GEN_AI_CHOICE = "gen_ai.choice" diff --git a/src/microsoft/opentelemetry/a365/core/__init__.py b/src/microsoft/opentelemetry/a365/core/__init__.py index a3e5347f..5f033c32 100644 --- a/src/microsoft/opentelemetry/a365/core/__init__.py +++ b/src/microsoft/opentelemetry/a365/core/__init__.py @@ -15,6 +15,8 @@ from microsoft.opentelemetry.a365.core.guardrail_finding import GuardrailFinding from microsoft.opentelemetry.a365.core.guardrail_risk_severity import GuardrailRiskSeverity from microsoft.opentelemetry.a365.core.guardrail_target_type import GuardrailTargetType +from microsoft.opentelemetry.a365.core.gen_ai_request_parameters import GenAiRequestParameters +from microsoft.opentelemetry.a365.core.gen_ai_response_parameters import GenAiResponseParameters from microsoft.opentelemetry.a365.core.inference_call_details import InferenceCallDetails from microsoft.opentelemetry.a365.core.models.service_endpoint import ServiceEndpoint from microsoft.opentelemetry.a365.core.inference_operation_type import InferenceOperationType @@ -84,6 +86,8 @@ "SpanDetails", "InferenceCallDetails", "ServiceEndpoint", + "GenAiRequestParameters", + "GenAiResponseParameters", # Enums "InferenceOperationType", "ToolType", diff --git a/src/microsoft/opentelemetry/a365/core/constants.py b/src/microsoft/opentelemetry/a365/core/constants.py index 6644ed03..ca24cb87 100644 --- a/src/microsoft/opentelemetry/a365/core/constants.py +++ b/src/microsoft/opentelemetry/a365/core/constants.py @@ -33,8 +33,15 @@ GEN_AI_CLIENT_OPERATION_DURATION_METRIC_NAME = "gen_ai.client.operation.duration" GEN_AI_CLIENT_TOKEN_USAGE_METRIC_NAME = "gen_ai.client.token.usage" GEN_AI_OPERATION_NAME_KEY = "gen_ai.operation.name" +GEN_AI_DATA_SOURCE_ID_KEY = "gen_ai.data_source.id" +GEN_AI_OUTPUT_TYPE_KEY = "gen_ai.output.type" +GEN_AI_REQUEST_CHOICE_COUNT_KEY = "gen_ai.request.choice.count" +GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY = "gen_ai.request.frequency_penalty" GEN_AI_REQUEST_MAX_TOKENS_KEY = "gen_ai.request.max_tokens" GEN_AI_REQUEST_MODEL_KEY = "gen_ai.request.model" +GEN_AI_REQUEST_PRESENCE_PENALTY_KEY = "gen_ai.request.presence_penalty" +GEN_AI_REQUEST_SEED_KEY = "gen_ai.request.seed" +GEN_AI_REQUEST_STOP_SEQUENCES_KEY = "gen_ai.request.stop_sequences" GEN_AI_REQUEST_TEMPERATURE_KEY = "gen_ai.request.temperature" GEN_AI_REQUEST_TOP_P_KEY = "gen_ai.request.top_p" GEN_AI_RESPONSE_FINISH_REASONS_KEY = "gen_ai.response.finish_reasons" @@ -49,6 +56,8 @@ GEN_AI_CONVERSATION_ID_KEY = "gen_ai.conversation.id" GEN_AI_CONVERSATION_ITEM_LINK_KEY = "microsoft.conversation.item.link" GEN_AI_TOKEN_TYPE_KEY = "gen_ai.token.type" +GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY = "gen_ai.usage.cache_write.input_tokens" +GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY = "gen_ai.usage.cache_read.input_tokens" GEN_AI_USAGE_INPUT_TOKENS_KEY = "gen_ai.usage.input_tokens" GEN_AI_USAGE_OUTPUT_TOKENS_KEY = "gen_ai.usage.output_tokens" GEN_AI_CHOICE = "gen_ai.choice" diff --git a/src/microsoft/opentelemetry/a365/core/gen_ai_request_parameters.py b/src/microsoft/opentelemetry/a365/core/gen_ai_request_parameters.py new file mode 100644 index 00000000..126cce9d --- /dev/null +++ b/src/microsoft/opentelemetry/a365/core/gen_ai_request_parameters.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""GenAI request parameters for InvokeAgent telemetry.""" + +from dataclasses import dataclass +from typing import Sequence + +from microsoft.opentelemetry.a365.core.models.messages import TextPart + + +@dataclass +class GenAiRequestParameters: + """Optional GenAI request parameters recorded on InvokeAgent spans.""" + + model: str | None = None + seed: int | None = None + choice_count: int | None = None + frequency_penalty: float | None = None + max_tokens: int | None = None + presence_penalty: float | None = None + stop_sequences: Sequence[str] | None = None + temperature: float | None = None + top_p: float | None = None + data_source_id: str | None = None + output_type: str | None = None + system_instructions: Sequence[TextPart] | None = None diff --git a/src/microsoft/opentelemetry/a365/core/gen_ai_response_parameters.py b/src/microsoft/opentelemetry/a365/core/gen_ai_response_parameters.py new file mode 100644 index 00000000..f7747721 --- /dev/null +++ b/src/microsoft/opentelemetry/a365/core/gen_ai_response_parameters.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""GenAI response parameters for InvokeAgent telemetry.""" + +from dataclasses import dataclass +from typing import Sequence + + +@dataclass +class GenAiResponseParameters: + """Optional GenAI response parameters recorded on InvokeAgent spans.""" + + finish_reasons: Sequence[str] | None = None + input_tokens: int | None = None + output_tokens: int | None = None + cache_write_input_tokens: int | None = None + cache_read_input_tokens: int | None = None diff --git a/src/microsoft/opentelemetry/a365/core/invoke_agent_details.py b/src/microsoft/opentelemetry/a365/core/invoke_agent_details.py index 2014b6af..e86abd38 100644 --- a/src/microsoft/opentelemetry/a365/core/invoke_agent_details.py +++ b/src/microsoft/opentelemetry/a365/core/invoke_agent_details.py @@ -5,6 +5,8 @@ from dataclasses import dataclass +from microsoft.opentelemetry.a365.core.gen_ai_request_parameters import GenAiRequestParameters +from microsoft.opentelemetry.a365.core.gen_ai_response_parameters import GenAiResponseParameters from microsoft.opentelemetry.a365.core.models.service_endpoint import ServiceEndpoint @@ -13,3 +15,5 @@ class InvokeAgentScopeDetails: """Scope-level configuration for agent invocation tracing.""" endpoint: ServiceEndpoint | None = None + request_parameters: GenAiRequestParameters | None = None + response_parameters: GenAiResponseParameters | None = None diff --git a/src/microsoft/opentelemetry/a365/core/invoke_agent_scope.py b/src/microsoft/opentelemetry/a365/core/invoke_agent_scope.py index b6a04071..9ca2dea6 100644 --- a/src/microsoft/opentelemetry/a365/core/invoke_agent_scope.py +++ b/src/microsoft/opentelemetry/a365/core/invoke_agent_scope.py @@ -20,8 +20,25 @@ GEN_AI_CALLER_AGENT_VERSION_KEY, GEN_AI_CALLER_CLIENT_IP_KEY, GEN_AI_CONVERSATION_ID_KEY, + GEN_AI_DATA_SOURCE_ID_KEY, GEN_AI_INPUT_MESSAGES_KEY, GEN_AI_OUTPUT_MESSAGES_KEY, + GEN_AI_OUTPUT_TYPE_KEY, + GEN_AI_REQUEST_CHOICE_COUNT_KEY, + GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY, + GEN_AI_REQUEST_MAX_TOKENS_KEY, + GEN_AI_REQUEST_MODEL_KEY, + GEN_AI_REQUEST_PRESENCE_PENALTY_KEY, + GEN_AI_REQUEST_SEED_KEY, + GEN_AI_REQUEST_STOP_SEQUENCES_KEY, + GEN_AI_REQUEST_TEMPERATURE_KEY, + GEN_AI_REQUEST_TOP_P_KEY, + GEN_AI_RESPONSE_FINISH_REASONS_KEY, + GEN_AI_SYSTEM_INSTRUCTIONS_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + GEN_AI_USAGE_INPUT_TOKENS_KEY, + GEN_AI_USAGE_OUTPUT_TOKENS_KEY, INVOKE_AGENT_OPERATION_NAME, SERVER_ADDRESS_KEY, SERVER_PORT_KEY, @@ -30,11 +47,14 @@ USER_ID_KEY, USER_NAME_KEY, ) +from microsoft.opentelemetry.a365.core.gen_ai_request_parameters import GenAiRequestParameters +from microsoft.opentelemetry.a365.core.gen_ai_response_parameters import GenAiResponseParameters from microsoft.opentelemetry.a365.core.invoke_agent_details import InvokeAgentScopeDetails from microsoft.opentelemetry.a365.core.message_utils import ( normalize_input_messages, normalize_output_messages, serialize_messages, + serialize_system_instructions, ) from microsoft.opentelemetry.a365.core.models.caller_details import CallerDetails from microsoft.opentelemetry.a365.core.models.messages import InputMessagesParam, OutputMessagesParam @@ -129,6 +149,11 @@ def __init__( if endpoint.port and endpoint.port != 443: self.set_tag_maybe(SERVER_PORT_KEY, endpoint.port) + if scope_details.request_parameters: + self._record_request_parameters(scope_details.request_parameters) + if scope_details.response_parameters: + self._record_response_parameters(scope_details.response_parameters) + # Set request metadata if request.channel: self.set_tag_maybe(CHANNEL_NAME_KEY, request.channel.name) @@ -182,6 +207,10 @@ def record_response(self, response: str) -> None: """ self.record_output_messages([response]) + def record_response_parameters(self, parameters: GenAiResponseParameters) -> None: + """Record GenAI response parameters that are only known after completion.""" + self._record_response_parameters(parameters) + def record_input_messages(self, messages: InputMessagesParam) -> None: """Record the input messages for telemetry tracking. @@ -205,3 +234,40 @@ def record_output_messages(self, messages: OutputMessagesParam) -> None: """ wrapper = normalize_output_messages(messages) self.set_tag_maybe(GEN_AI_OUTPUT_MESSAGES_KEY, serialize_messages(wrapper)) + + def _record_request_parameters(self, parameters: GenAiRequestParameters) -> None: + self.set_tag_maybe(GEN_AI_REQUEST_MODEL_KEY, parameters.model) + self.set_tag_maybe(GEN_AI_REQUEST_SEED_KEY, parameters.seed) + self.set_tag_maybe(GEN_AI_REQUEST_CHOICE_COUNT_KEY, parameters.choice_count) + self.set_tag_maybe(GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY, parameters.frequency_penalty) + self.set_tag_maybe(GEN_AI_REQUEST_MAX_TOKENS_KEY, parameters.max_tokens) + self.set_tag_maybe(GEN_AI_REQUEST_PRESENCE_PENALTY_KEY, parameters.presence_penalty) + self.set_tag_maybe( + GEN_AI_REQUEST_STOP_SEQUENCES_KEY, + tuple(parameters.stop_sequences) if parameters.stop_sequences is not None else None, + ) + self.set_tag_maybe(GEN_AI_REQUEST_TEMPERATURE_KEY, parameters.temperature) + self.set_tag_maybe(GEN_AI_REQUEST_TOP_P_KEY, parameters.top_p) + self.set_tag_maybe(GEN_AI_DATA_SOURCE_ID_KEY, parameters.data_source_id) + self.set_tag_maybe(GEN_AI_OUTPUT_TYPE_KEY, parameters.output_type) + if parameters.system_instructions is not None: + self.set_tag_maybe( + GEN_AI_SYSTEM_INSTRUCTIONS_KEY, + serialize_system_instructions(parameters.system_instructions), + ) + + def _record_response_parameters(self, parameters: GenAiResponseParameters) -> None: + self.set_tag_maybe( + GEN_AI_RESPONSE_FINISH_REASONS_KEY, + tuple(parameters.finish_reasons) if parameters.finish_reasons is not None else None, + ) + self.set_tag_maybe(GEN_AI_USAGE_INPUT_TOKENS_KEY, parameters.input_tokens) + self.set_tag_maybe(GEN_AI_USAGE_OUTPUT_TOKENS_KEY, parameters.output_tokens) + self.set_tag_maybe( + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + parameters.cache_write_input_tokens, + ) + self.set_tag_maybe( + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + parameters.cache_read_input_tokens, + ) diff --git a/src/microsoft/opentelemetry/a365/core/message_utils.py b/src/microsoft/opentelemetry/a365/core/message_utils.py index 21f86565..71064161 100644 --- a/src/microsoft/opentelemetry/a365/core/message_utils.py +++ b/src/microsoft/opentelemetry/a365/core/message_utils.py @@ -13,7 +13,7 @@ import logging from dataclasses import asdict from enum import Enum -from typing import Union +from typing import Any, Sequence, Union from microsoft.opentelemetry.a365.core.models.messages import ( ChatMessage, @@ -105,6 +105,15 @@ def _message_dict_factory(items: list[tuple[str, object]]) -> dict[str, object]: return {k: (v.value if isinstance(v, Enum) else v) for k, v in items if v is not None} +def _serialize_dataclasses(items: Sequence[Any]) -> str: + """Serialize dataclass instances as a JSON array.""" + return json.dumps( + [asdict(item, dict_factory=_message_dict_factory) for item in items], + default=str, + ensure_ascii=False, + ) + + def serialize_messages( wrapper: Union[InputMessages, OutputMessages], ) -> str: @@ -117,12 +126,7 @@ def serialize_messages( message parts contain non-JSON-serializable values. """ try: - serialized_list = [asdict(msg, dict_factory=_message_dict_factory) for msg in wrapper.messages] - return json.dumps( - serialized_list, - default=str, - ensure_ascii=False, - ) + return _serialize_dataclasses(wrapper.messages) except Exception: logger.warning("Failed to serialize messages; using fallback.", exc_info=True) messages = getattr(wrapper, "messages", []) @@ -140,3 +144,14 @@ def serialize_messages( if isinstance(wrapper, OutputMessages): fallback_msg["finish_reason"] = "error" return json.dumps([fallback_msg], ensure_ascii=False) + + +def serialize_system_instructions(parts: Sequence[TextPart]) -> str: + """Serialize OTel system-instruction parts as a JSON array.""" + try: + return _serialize_dataclasses(parts) + except Exception: + logger.warning("Failed to serialize system instructions; using fallback.", exc_info=True) + count = len(parts) + noun = "instruction part" if count == 1 else "instruction parts" + return _serialize_dataclasses([TextPart(content=f"[serialization failed: {count} {noun}]")]) diff --git a/tests/a365/test_invoke_agent_scope.py b/tests/a365/test_invoke_agent_scope.py new file mode 100644 index 00000000..c387cf55 --- /dev/null +++ b/tests/a365/test_invoke_agent_scope.py @@ -0,0 +1,175 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from typing import Any, cast + +from microsoft.opentelemetry.a365.core import ( + AgentDetails, + GenAiRequestParameters, + GenAiResponseParameters, + InvokeAgentScope, + InvokeAgentScopeDetails, + Request, + TextPart, +) +from microsoft.opentelemetry.a365.core.constants import SOURCE_NAME +from microsoft.opentelemetry.a365.core.opentelemetry_scope import OpenTelemetryScope +from opentelemetry.sdk.trace import TracerProvider + + +def _start_invoke_agent_scope(scope_details: InvokeAgentScopeDetails) -> InvokeAgentScope: + return InvokeAgentScope.start( + request=Request(), + scope_details=scope_details, + agent_details=AgentDetails(agent_id="agent-1"), + ) + + +def _span_attributes(scope: InvokeAgentScope) -> dict[str, object]: + assert scope._span is not None + span = cast(Any, scope._span) + return dict(span.attributes) + + +def _string_attribute(attrs: dict[str, object], key: str) -> str: + value = attrs[key] + assert isinstance(value, str) + return value + + +LEGACY_CACHE_CREATION_INPUT_TOKENS_KEY = ".".join(("gen_ai", "usage", "cache_creation", "input_tokens")) + + +def setup_function() -> None: + import os + + os.environ["ENABLE_OBSERVABILITY"] = "true" + provider = TracerProvider() + OpenTelemetryScope._tracer = provider.get_tracer(SOURCE_NAME) + setup_function.provider = provider # type: ignore[attr-defined] + + +def teardown_function() -> None: + import os + + provider = getattr(setup_function, "provider", None) + if provider: + provider.shutdown() + OpenTelemetryScope._tracer = None + os.environ.pop("ENABLE_OBSERVABILITY", None) + + +def test_invoke_agent_details_accepts_semantic_parameters() -> None: + request = GenAiRequestParameters(model="gpt-4o", max_tokens=128) + response = GenAiResponseParameters(input_tokens=10, output_tokens=4) + details = InvokeAgentScopeDetails( + request_parameters=request, + response_parameters=response, + ) + assert details.request_parameters is request + assert details.response_parameters is response + + +def test_invoke_agent_scope_records_request_parameters() -> None: + parameters = GenAiRequestParameters( + model="gpt-4o", + seed=42, + choice_count=2, + frequency_penalty=0.5, + max_tokens=128, + presence_penalty=0.25, + stop_sequences=["stop", "halt"], + temperature=0.7, + top_p=0.9, + data_source_id="data-source-1", + output_type="json", + system_instructions=[ + TextPart(content="Be concise."), + TextPart(content="Return JSON."), + ], + ) + scope = _start_invoke_agent_scope(InvokeAgentScopeDetails(request_parameters=parameters)) + try: + attrs = _span_attributes(scope) + assert attrs["gen_ai.request.model"] == "gpt-4o" + assert attrs["gen_ai.request.seed"] == 42 + assert attrs["gen_ai.request.choice.count"] == 2 + assert attrs["gen_ai.request.frequency_penalty"] == 0.5 + assert attrs["gen_ai.request.max_tokens"] == 128 + assert attrs["gen_ai.request.presence_penalty"] == 0.25 + assert attrs["gen_ai.request.stop_sequences"] == ("stop", "halt") + assert attrs["gen_ai.request.temperature"] == 0.7 + assert attrs["gen_ai.request.top_p"] == 0.9 + assert attrs["gen_ai.data_source.id"] == "data-source-1" + assert attrs["gen_ai.output.type"] == "json" + assert json.loads(_string_attribute(attrs, "gen_ai.system_instructions")) == [ + {"content": "Be concise.", "type": "text"}, + {"content": "Return JSON.", "type": "text"}, + ] + finally: + scope.dispose() + + +def test_invoke_agent_scope_records_response_parameters_after_completion() -> None: + scope = _start_invoke_agent_scope(InvokeAgentScopeDetails()) + try: + scope.record_response_parameters( + GenAiResponseParameters( + finish_reasons=["stop"], + input_tokens=10, + output_tokens=4, + cache_write_input_tokens=2, + cache_read_input_tokens=1, + ) + ) + attrs = _span_attributes(scope) + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.usage.input_tokens"] == 10 + assert attrs["gen_ai.usage.output_tokens"] == 4 + assert attrs["gen_ai.usage.cache_write.input_tokens"] == 2 + assert LEGACY_CACHE_CREATION_INPUT_TOKENS_KEY not in attrs + assert attrs["gen_ai.usage.cache_read.input_tokens"] == 1 + finally: + scope.dispose() + + +def test_invoke_agent_scope_omits_none_semantic_parameters() -> None: + scope = _start_invoke_agent_scope( + InvokeAgentScopeDetails( + request_parameters=GenAiRequestParameters(model="gpt-4o"), + response_parameters=GenAiResponseParameters(input_tokens=10), + ) + ) + try: + attrs = _span_attributes(scope) + assert attrs["gen_ai.request.model"] == "gpt-4o" + assert attrs["gen_ai.usage.input_tokens"] == 10 + assert "gen_ai.request.seed" not in attrs + assert "gen_ai.request.stop_sequences" not in attrs + assert "gen_ai.response.finish_reasons" not in attrs + assert "gen_ai.usage.output_tokens" not in attrs + assert "gen_ai.usage.cache_write.input_tokens" not in attrs + assert LEGACY_CACHE_CREATION_INPUT_TOKENS_KEY not in attrs + assert "gen_ai.usage.cache_read.input_tokens" not in attrs + assert "gen_ai.system_instructions" not in attrs + finally: + scope.dispose() + + +def test_invoke_agent_scope_system_instructions_serialization_is_non_throwing() -> None: + parameters = GenAiRequestParameters( + system_instructions=[object()], # type: ignore[list-item] + ) + + scope = _start_invoke_agent_scope(InvokeAgentScopeDetails(request_parameters=parameters)) + try: + attrs = _span_attributes(scope) + assert json.loads(_string_attribute(attrs, "gen_ai.system_instructions")) == [ + { + "content": "[serialization failed: 1 instruction part]", + "type": "text", + } + ] + finally: + scope.dispose() diff --git a/tests/a365/test_message_utils.py b/tests/a365/test_message_utils.py new file mode 100644 index 00000000..bc7fab41 --- /dev/null +++ b/tests/a365/test_message_utils.py @@ -0,0 +1,74 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json + +from microsoft.opentelemetry.a365.core.message_utils import serialize_messages +from microsoft.opentelemetry.a365.core.models.messages import ( + ChatMessage, + InputMessages, + MessageRole, + OutputMessage, + OutputMessages, + TextPart, +) + + +def test_serialize_messages_preserves_input_message_shape() -> None: + payload = InputMessages( + messages=[ + ChatMessage( + role=MessageRole.USER, + parts=[TextPart(content="hello")], + ) + ] + ) + + assert json.loads(serialize_messages(payload)) == [ + { + "role": "user", + "parts": [{"content": "hello", "type": "text"}], + } + ] + + +def test_serialize_messages_preserves_output_message_fallback_shape() -> None: + payload = OutputMessages(messages=[object()]) # type: ignore[list-item] + + assert json.loads(serialize_messages(payload)) == [ + { + "role": "system", + "parts": [{"type": "text", "content": "[serialization failed: 1 message]"}], + "finish_reason": "error", + } + ] + + +def test_serialize_messages_preserves_input_message_fallback_shape() -> None: + payload = InputMessages(messages=[object()]) # type: ignore[list-item] + + assert json.loads(serialize_messages(payload)) == [ + { + "role": "system", + "parts": [{"type": "text", "content": "[serialization failed: 1 message]"}], + } + ] + + +def test_serialize_messages_preserves_output_message_shape() -> None: + payload = OutputMessages( + messages=[ + OutputMessage( + role=MessageRole.ASSISTANT, + parts=[TextPart(content="hello back")], + ) + ] + ) + + assert json.loads(serialize_messages(payload)) == [ + { + "role": "assistant", + "parts": [{"content": "hello back", "type": "text"}], + "finish_reason": "stop", + } + ]