From 65fb43314383f84870cf937e1fc7f346f93403de Mon Sep 17 00:00:00 2001 From: Hector Hernandez <39923391+hectorhdzg@users.noreply.github.com> Date: Thu, 9 Apr 2026 15:31:50 -0700 Subject: [PATCH 1/2] Add langchain instrumentation --- package-lock.json | 243 ++++++++ package.json | 1 + src/genai/index.ts | 4 + src/genai/instrumentations/langchain/index.ts | 5 + .../langchain/langchainTraceInstrumentor.ts | 204 +++++++ .../instrumentations/langchain/tracer.ts | 195 +++++++ src/genai/instrumentations/langchain/utils.ts | 340 +++++++++++ src/genai/semconv.ts | 51 ++ src/genai/utils.ts | 38 ++ test/internal/functional/langchain.test.ts | 339 +++++++++++ .../langchainTraceInstrumentor.test.ts | 159 ++++++ .../unit/genai/langchain/tracer.test.ts | 308 ++++++++++ .../unit/genai/langchain/utils.test.ts | 539 ++++++++++++++++++ 13 files changed, 2426 insertions(+) create mode 100644 src/genai/index.ts create mode 100644 src/genai/instrumentations/langchain/index.ts create mode 100644 src/genai/instrumentations/langchain/langchainTraceInstrumentor.ts create mode 100644 src/genai/instrumentations/langchain/tracer.ts create mode 100644 src/genai/instrumentations/langchain/utils.ts create mode 100644 src/genai/semconv.ts create mode 100644 src/genai/utils.ts create mode 100644 test/internal/functional/langchain.test.ts create mode 100644 test/internal/unit/genai/langchain/langchainTraceInstrumentor.test.ts create mode 100644 test/internal/unit/genai/langchain/tracer.test.ts create mode 100644 test/internal/unit/genai/langchain/utils.test.ts diff --git a/package-lock.json b/package-lock.json index d67ded3a..ac307198 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ }, "devDependencies": { "@azure/functions": "^4.9.0", + "@langchain/core": "^1.1.39", "@types/node": "^22.0.0", "@typescript-eslint/eslint-plugin": "^8.0.0", "@typescript-eslint/parser": "^8.0.0", @@ -58,6 +59,14 @@ }, "engines": { "node": ">=20.0.0" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.0" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + } } }, "node_modules/@azure-rest/core-client": { @@ -718,6 +727,13 @@ "node": ">=6.9.0" } }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "dev": true, + "license": "MIT" + }, "node_modules/@colors/colors": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", @@ -1628,6 +1644,42 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@langchain/core": { + "version": "1.1.39", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.39.tgz", + "integrity": "sha512-DP9c7TREy6iA7HnywstmUAsNyJNYTFpRg2yBfQ+6H0l1HnvQzei9GsQ36GeOLxgRaD3vm9K8urCcawSC7yQpCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "uuid": "^11.1.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/core/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/@microsoft/applicationinsights-web-snippet": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@microsoft/applicationinsights-web-snippet/-/applicationinsights-web-snippet-1.2.3.tgz", @@ -3462,6 +3514,13 @@ "win32" ] }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/bunyan": { "version": "1.8.11", "resolved": "https://registry.npmjs.org/@types/bunyan/-/bunyan-1.8.11.tgz", @@ -4048,6 +4107,27 @@ "node": "18 || 20 || >=22" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/baseline-browser-mapping": { "version": "2.10.16", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.16.tgz", @@ -4128,6 +4208,19 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001786", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001786.tgz", @@ -4287,6 +4380,16 @@ } } }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/deep-eql": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", @@ -4639,6 +4742,13 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -5106,6 +5216,16 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -5183,6 +5303,55 @@ "json-buffer": "3.0.1" } }, + "node_modules/langsmith": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.5.18.tgz", + "integrity": "sha512-3zuZUWffTHQ+73EAwnodADtf534VNEZUpXr9jC12qyG8/IQuJET7PRsCpTb9wX2lmBspakwLUpqpj3tNm/0bVA==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2", + "uuid": "10.0.0" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, + "node_modules/langsmith/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5342,6 +5511,16 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -5393,6 +5572,16 @@ "node": ">= 0.8.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -5425,6 +5614,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -6349,6 +6568,20 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/vite": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", @@ -6677,6 +6910,16 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 7a6d5e4c..1fb42ca2 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ }, "devDependencies": { "@azure/functions": "^4.9.0", + "@langchain/core": "^1.1.39", "@types/node": "^22.0.0", "@typescript-eslint/eslint-plugin": "^8.0.0", "@typescript-eslint/parser": "^8.0.0", diff --git a/src/genai/index.ts b/src/genai/index.ts new file mode 100644 index 00000000..992102d5 --- /dev/null +++ b/src/genai/index.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export * from './semconv.js'; diff --git a/src/genai/instrumentations/langchain/index.ts b/src/genai/instrumentations/langchain/index.ts new file mode 100644 index 00000000..df4b0de4 --- /dev/null +++ b/src/genai/instrumentations/langchain/index.ts @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export { LangChainTraceInstrumentor } from './langchainTraceInstrumentor.js'; +export { LangChainTracer } from './tracer.js'; diff --git a/src/genai/instrumentations/langchain/langchainTraceInstrumentor.ts b/src/genai/instrumentations/langchain/langchainTraceInstrumentor.ts new file mode 100644 index 00000000..2221adb8 --- /dev/null +++ b/src/genai/instrumentations/langchain/langchainTraceInstrumentor.ts @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Vendored from microsoft/Agent365-nodejs packages/agents-a365-observability-extensions-langchain +// Adapted: removed ObservabilityManager dependency, uses diag logger instead of A365 logger + +import type * as CallbackManagerModule from "@langchain/core/callbacks/manager"; +import { diag, trace, Tracer } from "@opentelemetry/api"; +import { + InstrumentationBase, + InstrumentationConfig, + InstrumentationModuleDefinition, + isWrapped +} from "@opentelemetry/instrumentation"; +import { LangChainTracer } from "./tracer.js"; + +type CallbackManagerModuleType = typeof CallbackManagerModule; + +class LangChainTraceInstrumentorImpl extends InstrumentationBase { + private static _instance: LangChainTraceInstrumentorImpl | null = null; + private _hasBeenEnabled = false; + private _isPatched = false; + protected otelTracer: Tracer; + private isContentRecordingEnabled: boolean; + + private constructor(options?: { isContentRecordingEnabled?: boolean }) { + if (LangChainTraceInstrumentorImpl._instance !== null) { + throw new Error("LangChainTraceInstrumentor can only be instantiated once."); + } + + super("microsoft-otel-langchain-instrumentor", "1.0.0", { + enabled: true + }); + + this.otelTracer = trace.getTracer( + "microsoft-otel-langchain", + "1.0.0" + ); + this.isContentRecordingEnabled = options?.isContentRecordingEnabled ?? false; + + LangChainTraceInstrumentorImpl._instance = this; + diag.info("[LangChainTraceInstrumentor] Initialized and automatically enabled"); + } + + static getInstance(options?: { isContentRecordingEnabled?: boolean }): LangChainTraceInstrumentorImpl { + if (!LangChainTraceInstrumentorImpl._instance) { + LangChainTraceInstrumentorImpl._instance = new LangChainTraceInstrumentorImpl(options); + } + return LangChainTraceInstrumentorImpl._instance; + } + + static hasInstance(): boolean { + return LangChainTraceInstrumentorImpl._instance !== null; + } + + static resetInstance(): void { + if (LangChainTraceInstrumentorImpl._instance) { + LangChainTraceInstrumentorImpl._instance._isPatched = false; + } + LangChainTraceInstrumentorImpl._instance = null; + } + + protected init(): InstrumentationModuleDefinition { + return { + name: "@langchain/core/callbacks/manager", + supportedVersions: [">=0.2.0"], + files: [], + patch: this.patch.bind(this), + unpatch: this.unpatch.bind(this) + }; + } + + private unpatch(moduleExports: Record): void { + const CallbackManager = moduleExports?.CallbackManager as typeof CallbackManagerModule.CallbackManager; + if (!CallbackManager || !isWrapped(CallbackManager._configureSync)) { + return; + } + + this._unwrap(CallbackManager, "_configureSync"); + this._isPatched = false; + diag.info("[LangChainTraceInstrumentor] Unpatched OTEL LangChain instrumentation"); + } + + patch(module: CallbackManagerModuleType): CallbackManagerModuleType { + if (this._isPatched) { + return module; + } + + const { CallbackManager } = module as CallbackManagerModuleType; + if (!CallbackManager || !("_configureSync" in CallbackManager)) { + return module; + } + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const instrumentor = this; + this._wrap(CallbackManager, "_configureSync", (original) => { + return function ( + this: CallbackManagerModuleType, + ...args: Parameters + ) { + args[0] = addTracerToHandlers(instrumentor.otelTracer, args[0], { isContentRecordingEnabled: instrumentor.isContentRecordingEnabled }); + diag.debug("[LangChainTraceInstrumentor] _configureSync wrapped to add LangChainTracer"); + return original.apply(this, args); + }; + }); + + diag.info("[LangChainTraceInstrumentor] Patched OTEL LangChain instrumentation"); + this._isPatched = true; + return module; + } + + manuallyInstrumentImpl(module: CallbackManagerModuleType): void { + diag.info("[LangChainTraceInstrumentor] Manually instrumenting CallbackManagerModule"); + this.patch(module); + } + + public instrumentationDependencies(): readonly string[] { + return ["@langchain/core >= 0.2.0"] as const; + } + + public override enable(): void { + if (this._hasBeenEnabled) { + return; + } + this._hasBeenEnabled = true; + diag.info("[LangChainTraceInstrumentor] Enabled LangChain instrumentation"); + super.enable(); + } + + public override disable(): void { + this._hasBeenEnabled = false; + diag.info("[LangChainTraceInstrumentor] Disabled LangChain instrumentation"); + super.disable(); + } +} + +/** + * Static wrapper for LangChain tracing instrumentation + */ +export class LangChainTraceInstrumentor { + private static throwNotInitialized(): never { + throw new Error( + "LangChainTraceInstrumentor must be initialized first. " + + "Call LangChainTraceInstrumentor.instrument() before using enable/disable." + ); + } + + /** + * Initialize and auto-instrument for LangChain + * @param module The CallbackManager module to instrument + * @param options Optional configuration options + */ + static instrument(module: CallbackManagerModuleType, options?: { isContentRecordingEnabled?: boolean }): void { + LangChainTraceInstrumentorImpl.getInstance(options).manuallyInstrumentImpl(module); + } + + /** + * Enable LangChain instrumentation + */ + static enable(): void { + if (!LangChainTraceInstrumentorImpl.hasInstance()) { + this.throwNotInitialized(); + } + LangChainTraceInstrumentorImpl.getInstance().enable(); + } + + /** + * Disable LangChain instrumentation + */ + static disable(): void { + if (!LangChainTraceInstrumentorImpl.hasInstance()) { + this.throwNotInitialized(); + } + LangChainTraceInstrumentorImpl.getInstance().disable(); + } + + /** + * Reset the instrumentor instance (for testing) + */ + static resetInstance(): void { + LangChainTraceInstrumentorImpl.resetInstance(); + } +} + +export function addTracerToHandlers( + tracer: Tracer, + handlers: CallbackManagerModule.Callbacks | undefined, + options?: { isContentRecordingEnabled?: boolean } +): CallbackManagerModule.Callbacks { + if (handlers == null) { + return [new LangChainTracer(tracer, options)]; + } + + if (Array.isArray(handlers)) { + if (!handlers.some((h) => h instanceof LangChainTracer)) { + handlers.push(new LangChainTracer(tracer, options)); + } + return handlers; + } + + if (!handlers.inheritableHandlers.some((h) => h instanceof LangChainTracer)) { + handlers.addHandler(new LangChainTracer(tracer, options), true); + } + return handlers; +} diff --git a/src/genai/instrumentations/langchain/tracer.ts b/src/genai/instrumentations/langchain/tracer.ts new file mode 100644 index 00000000..d899c7e1 --- /dev/null +++ b/src/genai/instrumentations/langchain/tracer.ts @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Vendored from microsoft/Agent365-nodejs packages/agents-a365-observability-extensions-langchain + +import { context, trace, Span, SpanKind, SpanStatusCode, Tracer } from "@opentelemetry/api"; +import { BaseTracer, Run } from "@langchain/core/tracers/base"; +import { isTracingSuppressed } from "@opentelemetry/core"; +import { diag } from "@opentelemetry/api"; +import { ATTR_ERROR_MESSAGE, ATTR_GEN_AI_PROVIDER_NAME } from "../../index.js"; +import * as Utils from "./utils.js"; + +type RunWithSpan = { run: Run; span: Span; startTime: number; lastAccessTime: number }; + +/** + * OpenTelemetry-based tracer for LangChain / LangGraph applications. + * + * Extends LangChain's `BaseTracer` callback handler so it can be injected into + * the LangChain callback system (via `LangChainTraceInstrumentor`). Every + * LangChain "run" (agent invocation, tool execution, or LLM call) is mapped to + * an OTel span with GenAI semantic convention attributes. + * + * Key behaviors: + * - Creates a span on `onRunCreate` and ends it on `_endTrace`. + * - Maintains parent–child span relationships by tracking run IDs and walking + * up the parent chain to find the nearest span context. + * - Skips LangChain-internal runs (tagged `langsmith:hidden`, `Branch*`, or + * unmapped run types) to avoid noisy traces. + * - Guards against unbounded memory with a hard cap of {@link MAX_RUNS}. + * - Content-sensitive attributes (messages, tool args, system instructions) + * are only recorded when `isContentRecordingEnabled` is true. + */ +export class LangChainTracer extends BaseTracer { + /** Hard cap on concurrent tracked runs to prevent memory leaks. */ + private static readonly MAX_RUNS = 10_000; + private tracer: Tracer; + private isContentRecordingEnabled: boolean; + /** Active runs keyed by LangChain run ID. */ + private runs = new Map(); + /** Maps each run ID → its parent run ID for parent-span-context lookup. */ + private parentByRunId = new Map(); + + constructor(tracer: Tracer, options?: { isContentRecordingEnabled?: boolean }) { + super(); + this.tracer = tracer; + this.isContentRecordingEnabled = options?.isContentRecordingEnabled ?? false; + } + + name = "OpenTelemetryLangChainTracer"; + + protected persistRun(_run: Run): Promise { + return Promise.resolve(); + } + + /** + * Called by LangChain when a new run starts. Records the parent mapping + * and opens a span via {@link startTracing}. + */ + async onRunCreate(run: Run) { + this.parentByRunId.set(run.id, run.parent_run_id); + if (super.onRunCreate) await super.onRunCreate(run); + this.startTracing(run); + } + + /** + * Opens an OTel span for the given run. The span name is derived from the + * operation type (invoke_agent, execute_tool, chat) and the run/model name. + * Internal or unknown runs are silently skipped. + */ + protected startTracing(run: Run) { + if (isTracingSuppressed(context.active())) { + return; + } + + const operation = Utils.getOperationType(run); + + // Skip internal runs (LangSmith hidden, Branch nodes, unknown operations) + if (run.tags?.includes("langsmith:hidden") || run.name?.startsWith("Branch") || operation === "unknown") { + diag.debug(`[LangChainTracer] Skipping internal run: ${run.name} (parent: ${run.parent_run_id})`); + return; + } + + // Attach to parent span if one exists in the run hierarchy + const parentCtx = this.getNearestParentSpanContext(run); + const activeContext = parentCtx + ? trace.setSpanContext(context.active(), parentCtx) + : context.active(); + + // Build span name: " " + let spanName = run.name; + if (operation === "invoke_agent") { + spanName = `${operation} ${run.name}`; + } else if (operation === "execute_tool") { + spanName = `${operation} ${run.name}`; + } else if (operation === "chat") { + spanName = `${operation} ${Utils.getModel(run) || run.name}`.trim(); + } + + if (this.runs.size >= LangChainTracer.MAX_RUNS) { + diag.warn(`[LangChainTracer] Max runs (${LangChainTracer.MAX_RUNS}) reached, skipping span`); + this.parentByRunId.delete(run.id); + return; + } + + const startTime = run.start_time ?? Date.now(); + const span = this.tracer.startSpan(spanName, { + kind: SpanKind.INTERNAL, + startTime, + attributes: { [ATTR_GEN_AI_PROVIDER_NAME]: "langchain" }, + }, activeContext); + + this.runs.set(run.id, { run, span, startTime, lastAccessTime: startTime }); + } + + /** + * Called by LangChain when a run finishes. Sets status, enriches the span + * with GenAI attributes, and ends it. If content recording is enabled, + * message bodies, tool arguments, and system instructions are also attached. + */ + protected async _endTrace(run: Run) { + if (isTracingSuppressed(context.active())) { + // End any span that was started before suppression to avoid leaks. + const suppressedEntry = this.runs.get(run.id); + if (suppressedEntry) { + suppressedEntry.span.end(run.end_time ?? undefined); + } + this.parentByRunId.delete(run.id); + this.runs.delete(run.id); + return; + } + + const operation = Utils.getOperationType(run); + if (run.tags?.includes("langsmith:hidden") || run.name?.startsWith("Branch") || operation === "unknown") { + diag.debug(`[LangChainTracer] Skipping internal run: ${run.name} (parent: ${run.parent_run_id})`); + return; + } + + const entry = this.runs.get(run.id); + if (!entry) { + return; + } + + const { span } = entry; + try { + entry.lastAccessTime = Date.now(); + + if (run.error) { + span.setStatus({ code: SpanStatusCode.ERROR }); + span.setAttribute(ATTR_ERROR_MESSAGE, String(run.error)); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + } + + // Always-on attributes: operation type, agent info, model, provider, session, tokens + Utils.setOperationTypeAttribute(operation, span); + Utils.setAgentAttributes(run, span); + Utils.setModelAttribute(run, span); + Utils.setProviderNameAttribute(run, span); + Utils.setSessionIdAttribute(run, span); + Utils.setTokenAttributes(run, span); + + // Opt-in content attributes (may contain PII / large payloads) + if (this.isContentRecordingEnabled) { + Utils.setToolAttributes(run, span); + Utils.setInputMessagesAttribute(run, span); + Utils.setOutputMessagesAttribute(run, span); + Utils.setSystemInstructionsAttribute(run, span); + } + + } catch (error) { + diag.error(`[LangChainTracer] Error setting span attributes for run ${run.name}: ${error instanceof Error ? error.message : String(error)}`); + span.setStatus({ code: SpanStatusCode.ERROR }); + } finally { + span.end(run.end_time ?? undefined); + this.runs.delete(run.id); + this.parentByRunId.delete(run.id); + await super._endTrace(run); + } + } + + /** + * Walks up the parent run chain to find the nearest ancestor that has an + * active span, returning its `SpanContext` so the new span can be linked + * as a child. + */ + private getNearestParentSpanContext(run: Run) { + let pid = run.parent_run_id; + + while (pid) { + const entry = this.runs.get(pid); + if (entry) return entry.span.spanContext(); + pid = this.parentByRunId.get(pid); + } + return undefined; + } +} diff --git a/src/genai/instrumentations/langchain/utils.ts b/src/genai/instrumentations/langchain/utils.ts new file mode 100644 index 00000000..8069cc61 --- /dev/null +++ b/src/genai/instrumentations/langchain/utils.ts @@ -0,0 +1,340 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Vendored from microsoft/Agent365-nodejs packages/agents-a365-observability-extensions-langchain + +import { Run } from "@langchain/core/tracers/base"; +import { Span } from "@opentelemetry/api"; +import { + ATTR_GEN_AI_AGENT_NAME, + ATTR_GEN_AI_INPUT_MESSAGES, + ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_OUTPUT_MESSAGES, + ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, + ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, + ATTR_GEN_AI_TOOL_CALL_ID, + ATTR_GEN_AI_TOOL_CALL_RESULT, + ATTR_GEN_AI_TOOL_NAME, + ATTR_GEN_AI_TOOL_TYPE, + ATTR_GEN_AI_USAGE_INPUT_TOKENS, + ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, + ATTR_MICROSOFT_SESSION_ID, + GEN_AI_OPERATION_CHAT, + GEN_AI_OPERATION_EXECUTE_TOOL, + GEN_AI_OPERATION_INVOKE_AGENT, +} from "../../index.js"; + +// Type guards +export function isString(value: unknown): value is string { + return typeof value === "string"; +} + +// Operation type mapping +export function getOperationType(run: Run): string { + let operation = "unknown"; + + if (run.run_type === "chain" && isLangGraphAgentInvoke(run)) { + operation = GEN_AI_OPERATION_INVOKE_AGENT; + } else if (run.run_type === "tool") { + operation = GEN_AI_OPERATION_EXECUTE_TOOL; + } else if (run.run_type === "llm") { + operation = GEN_AI_OPERATION_CHAT; + } + return operation; +} + +// Operation type mapping +export function setOperationTypeAttribute(operation: string, span: Span) { + span.setAttribute(ATTR_GEN_AI_OPERATION_NAME, operation); +} + +// Agent attributes +export function setAgentAttributes(run: Run, span: Span) { + if (isLangGraphAgentInvoke(run)) { + const agentName = run.name; + if (isString(agentName)) { + span.setAttribute(ATTR_GEN_AI_AGENT_NAME, agentName); + } + } +} + +// Tool attributes +export function setToolAttributes(run: Run, span: Span) { + if (run.run_type !== "tool") { + return; + } + if (!run.serialized || typeof run.serialized !== "object" || Array.isArray(run.serialized)) { + return; + } + + if (isString(run.name)) { + span.setAttribute(ATTR_GEN_AI_TOOL_NAME, run.name); + } + if (run.inputs) span.setAttribute(ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, JSON.stringify(run.inputs?.input ?? run.inputs)); + if (run.outputs?.output?.kwargs?.content) + span.setAttribute(ATTR_GEN_AI_TOOL_CALL_RESULT, JSON.stringify(run.outputs?.output?.kwargs?.content)); + span.setAttribute(ATTR_GEN_AI_TOOL_TYPE, "extension"); + + if (run.outputs?.output?.tool_call_id) + span.setAttribute(ATTR_GEN_AI_TOOL_CALL_ID, run.outputs?.output?.tool_call_id); +} + +export function setInputMessagesAttribute(run: Run, span: Span) { + const messages = run.inputs?.messages; + if (!Array.isArray(messages)) { + return; + } + + const preprocess = getScopeType(run) === "inference" && messages.length > 0 ? messages[0] : messages; + const processed = preprocess?.map((msg: Record) => { + const content = extractMessageContent(msg); + if (!content) return null; + + const msgType = getMessageType(msg); + if (shouldIncludeInputMessage(msgType)) { + return content; + } + return null; + }) + .filter(Boolean); + + if (processed.length > 0) { + span.setAttribute(ATTR_GEN_AI_INPUT_MESSAGES, JSON.stringify(processed)); + } +} + +// Helper: Extract message content from various formats +function extractMessageContent(msg: Record): string | null { + // Simple format: {role: "user", content} + if (isString(msg.content)) { + return msg.content; + } + + // LangChain format: {lc_type: "human", lc_kwargs: {content}} + if (msg.lc_kwargs && typeof msg.lc_kwargs === "object" && !Array.isArray(msg.lc_kwargs)) { + const kwargs = msg.lc_kwargs as Record; + if (isString(kwargs.content)) return kwargs.content; + } + + // New LangChain format: {lc: 1, type: "constructor", kwargs: {content}} + if (msg.lc === 1 && msg.type === "constructor" && msg.kwargs && typeof msg.kwargs === "object" && !Array.isArray(msg.kwargs)) { + const kwargs = msg.kwargs as Record; + if (isString(kwargs.content)) return kwargs.content; + } + return null; +} + +// Helper: Determine message type +function getMessageType(msg: Record): string { + // Simple format + if (isString(msg.role)) return msg.role; + // LangChain old format + if (isString(msg.lc_type)) return msg.lc_type; + if (isString(msg.type)) return msg.type; + // LangChain new format - check id array for message type + if (Array.isArray(msg.id)) { + const lastId = msg.id[msg.id.length - 1]; + if (isString(lastId)) { + if (lastId.includes("Human")) return "human"; + if (lastId.includes("AI")) return "ai"; + if (lastId.includes("System")) return "system"; + } + } + return "unknown"; +} + +// Helper: Determine scope type from run +function getScopeType(run: Run): "agent" | "tool" | "inference" | "unknown" { + if (run.run_type === "chain" && isLangGraphAgentInvoke(run)) { + return "agent"; + } else if (run.run_type === "tool") { + return "tool"; + } else if (run.run_type === "llm") { + return "inference"; + } + return "unknown"; +} + +// Helper: Check if input message should be included based on scope and message type +function shouldIncludeInputMessage(msgType: string): boolean { + // For input messages: all scopes want user/human messages only + return msgType === "user" || msgType === "human"; +} + +// Helper: Check if output message should be included based on scope and message type +function shouldIncludeOutputMessage(scopeType: string, msgType: string): boolean { + if (scopeType === "agent" || scopeType === "inference") { + // Agent and Inference scopes want assistant/AI messages only + return msgType === "ai" || msgType === "assistant"; + } else if (scopeType === "tool") { + // Tool scope wants all output messages + return true; + } + // Default: all messages + return true; +} + +export function setOutputMessagesAttribute(run: Run, span: Span) { + const outputs = run.outputs; + if (!outputs) { + return; + } + + const scopeType = getScopeType(run); + const messages: string[] = []; + + // Direct messages array (used in agent/chain outputs) + if (Array.isArray(outputs.messages)) { + outputs.messages.forEach((msg: Record) => { + const content = extractMessageContent(msg); + if (!content) return; + + const msgType = getMessageType(msg); + if (shouldIncludeOutputMessage(scopeType, msgType)) { + messages.push(content); + } + }); + } + + // LangChain generations format (used in LLM/inference outputs) + if (Array.isArray(outputs.generations)) { + outputs.generations.forEach((gen: unknown) => { + if (Array.isArray(gen)) { + gen.forEach((item: Record) => { + // Try message property + if (item.message && typeof item.message === "object" && !Array.isArray(item.message)) { + const msg = item.message as Record; + const content = extractMessageContent(msg); + if (!content) { + return; + } + + const msgType = getMessageType(msg); + if (shouldIncludeOutputMessage(scopeType, msgType)) { + messages.push(content); + } + } + // Try direct text property (for generation items) + else if (isString(item.text) && scopeType === "inference") { + messages.push(item.text); + } + }); + } + }); + } + + // Check for direct message object (some models return this) + if (outputs.message && typeof outputs.message === "object" && !Array.isArray(outputs.message)) { + const msg = outputs.message as Record; + const content = extractMessageContent(msg); + if (content) { + const msgType = getMessageType(msg); + if (shouldIncludeOutputMessage(scopeType, msgType)) { + messages.push(content); + } + } + } + + if (messages.length > 0) { + span.setAttribute(ATTR_GEN_AI_OUTPUT_MESSAGES, JSON.stringify(messages)); + } +} + +// Model - Helper to extract model name from run +export function getModel(run: Run): string | undefined { + return [run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata?.model_name, + run.extra?.metadata?.ls_model_name, + run.extra?.invocation_params?.model, + run.extra?.invocation_params?.model_name] + .map((v) => (v != null ? String(v).trim() : "")) + .find((v) => v.length > 0); +} + +// Model - Set model attribute on span +export function setModelAttribute(run: Run, span: Span) { + const model = getModel(run); + if (model) span.setAttribute(ATTR_GEN_AI_REQUEST_MODEL, model); +} + +// Provider +export function setProviderNameAttribute(run: Run, span: Span) { + const provider = (run.extra?.metadata as Record | undefined)?.ls_provider; + if (isString(provider)) + span.setAttribute(ATTR_GEN_AI_PROVIDER_NAME, provider.toLowerCase()); +} + +export function setSessionIdAttribute(run: Run, span: Span): void { + const metadata = run.extra?.metadata as Record | undefined; + if (!metadata) return; + + const sessionId = + metadata.session_id ?? + metadata.conversation_id ?? + metadata.thread_id; + + if (typeof sessionId === "string" && sessionId.length > 0) { + span.setAttribute(ATTR_MICROSOFT_SESSION_ID, sessionId); + } +} + +// System instructions +export function setSystemInstructionsAttribute(run: Run, span: Span) { + const inputs = run.inputs as Record | undefined; + if (!inputs) { + return; + } + + const prompts = Array.isArray(inputs.prompts) ? inputs.prompts.map(p => String(p ?? "").trim()).filter(Boolean).join("\n") : ""; + if (prompts) + return span.setAttribute(ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, prompts); + + const messages = Array.isArray(inputs.messages) ? inputs.messages : []; + const systemText = messages + .filter((m: Record) => m.lc_type === "system") + .map((m: Record) => String((m.lc_kwargs as Record | undefined)?.content ?? "").trim()) + .filter(Boolean) + .join("\n"); + if (systemText) + span.setAttribute(ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, systemText); +} + +// Tokens (input and output) +export function setTokenAttributes(run: Run, span: Span) { + // Try multiple paths to find usage metadata (LLM direct/kwargs/response_metadata, agent calls, and chain/model_request outputs) + const usage = + run.outputs?.generations?.[0]?.[0]?.message?.usage_metadata || + run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.usage_metadata || + run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata?.tokenUsage || + run.outputs?.messages?.[1]?.usage_metadata || + run.outputs?.message?.response_metadata?.usage || + run.outputs?.message?.response_metadata?.tokenUsage || + run.outputs?.messages + ?.map((msg: Record) => (msg.response_metadata as Record | undefined)?.tokenUsage) + .filter(Boolean)[0]; + + if (!usage || typeof usage !== "object") { + return; + } + + const usageObj = usage as Record; + if (typeof usageObj.input_tokens === "number") { + span.setAttribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usageObj.input_tokens); + } + if (typeof usageObj.output_tokens === "number") { + span.setAttribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usageObj.output_tokens); + } +} + +// LangGraph agent check +function isLangGraphAgentInvoke(run: Run): boolean { + if (run.run_type !== "chain") { + return false; + } + if (!run.serialized || typeof run.serialized !== "object" || Array.isArray(run.serialized)) { + return false; + } + const serialized = run.serialized as Record; + const id = serialized.id; + return Array.isArray(id) && id.includes("langgraph") && id.includes("CompiledStateGraph"); +} diff --git a/src/genai/semconv.ts b/src/genai/semconv.ts new file mode 100644 index 00000000..13776cc6 --- /dev/null +++ b/src/genai/semconv.ts @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * GenAI semantic conventions (incubating/unstable). + * + * Per OTel recommendation, unstable conventions are copied locally rather than + * imported from @opentelemetry/semantic-conventions/incubating. + * See: https://opentelemetry.io/docs/specs/semconv/non-normative/code-generation/#stability-and-versioning + * + * Sourced from OTel semantic conventions + microsoft/Agent365-nodejs. + */ + +// --- Span operation names --- +export const GEN_AI_OPERATION_INVOKE_AGENT = 'invoke_agent' as const; +export const GEN_AI_OPERATION_EXECUTE_TOOL = 'execute_tool' as const; +export const GEN_AI_OPERATION_OUTPUT_MESSAGES = 'output_messages' as const; +export const GEN_AI_OPERATION_CHAT = 'chat' as const; + +// --- Attributes (ATTR_ prefix, following OTel convention) --- + +// Error +export const ATTR_ERROR_TYPE = 'error.type' as const; +export const ATTR_ERROR_MESSAGE = 'error.message' as const; + +// GenAI core +export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name' as const; +export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model' as const; +export const ATTR_GEN_AI_RESPONSE_MODEL = 'gen_ai.response.model' as const; +export const ATTR_GEN_AI_PROVIDER_NAME = 'gen_ai.provider.name' as const; +export const ATTR_GEN_AI_SYSTEM_INSTRUCTIONS = 'gen_ai.system_instructions' as const; +export const ATTR_GEN_AI_INPUT_MESSAGES = 'gen_ai.input.messages' as const; +export const ATTR_GEN_AI_OUTPUT_MESSAGES = 'gen_ai.output.messages' as const; + +// GenAI usage +export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens' as const; +export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens' as const; + +// GenAI agent +export const ATTR_GEN_AI_AGENT_ID = 'gen_ai.agent.id' as const; +export const ATTR_GEN_AI_AGENT_NAME = 'gen_ai.agent.name' as const; + +// GenAI tool +export const ATTR_GEN_AI_TOOL_CALL_ID = 'gen_ai.tool.call.id' as const; +export const ATTR_GEN_AI_TOOL_NAME = 'gen_ai.tool.name' as const; +export const ATTR_GEN_AI_TOOL_CALL_ARGUMENTS = 'gen_ai.tool.call.arguments' as const; +export const ATTR_GEN_AI_TOOL_CALL_RESULT = 'gen_ai.tool.call.result' as const; +export const ATTR_GEN_AI_TOOL_TYPE = 'gen_ai.tool.type' as const; + +// Microsoft-specific (not in OTel semconv) +export const ATTR_MICROSOFT_SESSION_ID = 'microsoft.session.id' as const; diff --git a/src/genai/utils.ts b/src/genai/utils.ts new file mode 100644 index 00000000..3e669c67 --- /dev/null +++ b/src/genai/utils.ts @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// TODO: Align truncation strategy across exporters. Currently there are two +// separate truncation mechanisms vendored from A365: +// 1. truncateValue() here — 8,192 char limit on individual span attribute values, +// used by GenAI instrumentations (LangChain, OpenAI) before setting attributes. +// 2. truncateSpan() in _a365/exporter/utils.ts — 250KB limit on entire serialized +// spans, used by Agent365Exporter before posting to the A365 service. +// These serve different purposes (attribute-level vs span-level) and target +// different exporter backends. Need to determine: +// - What truncation Azure Monitor exporter expects/handles natively +// - What truncation OTLP exporter expects/handles natively +// - Whether attribute-level truncation should live in a shared location or +// remain exporter-specific under _a365/ +// For now, keeping this here since GenAI instrumentations need it regardless +// of which exporter is active. + +/** + * Maximum length for span attribute values. + * Values exceeding this limit will be truncated with a suffix. + */ +export const MAX_ATTRIBUTE_LENGTH = 8_192; + +const TRUNCATION_SUFFIX = '...[truncated]'; + +/** + * Truncate a string value to {@link MAX_ATTRIBUTE_LENGTH} characters. + * If the value exceeds the limit, it is trimmed and a truncation suffix is appended. + * @param value The string to truncate + * @returns The original string if within limits, otherwise the truncated string + */ +export function truncateValue(value: string): string { + if (value.length > MAX_ATTRIBUTE_LENGTH) { + return value.substring(0, MAX_ATTRIBUTE_LENGTH - TRUNCATION_SUFFIX.length) + TRUNCATION_SUFFIX; + } + return value; +} diff --git a/test/internal/functional/langchain.test.ts b/test/internal/functional/langchain.test.ts new file mode 100644 index 00000000..7f4c1652 --- /dev/null +++ b/test/internal/functional/langchain.test.ts @@ -0,0 +1,339 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Functional test for LangChain instrumentation. + * + * Exercises the full pipeline: LangChainTracer receives LangChain Run callbacks + * and produces OTel spans captured by an InMemorySpanExporter. Validates that + * the correct spans, attributes, and parent-child relationships are emitted. + */ + +import { afterEach, assert, describe, it } from "vitest"; +import { + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import type { Run } from "@langchain/core/tracers/base"; +import { LangChainTracer } from "../../../src/genai/instrumentations/langchain/tracer.js"; +import { + ATTR_GEN_AI_AGENT_NAME, + ATTR_GEN_AI_INPUT_MESSAGES, + ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_OUTPUT_MESSAGES, + ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_TOOL_NAME, + ATTR_GEN_AI_USAGE_INPUT_TOKENS, + ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, + ATTR_MICROSOFT_SESSION_ID, + GEN_AI_OPERATION_CHAT, + GEN_AI_OPERATION_EXECUTE_TOOL, + GEN_AI_OPERATION_INVOKE_AGENT, +} from "../../../src/genai/index.js"; + +let provider: BasicTracerProvider; +let exporter: InMemorySpanExporter; +let langchainTracer: LangChainTracer; + +function setup(options?: { isContentRecordingEnabled?: boolean }) { + exporter = new InMemorySpanExporter(); + provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + + const tracer = provider.getTracer("test-langchain", "1.0.0"); + langchainTracer = new LangChainTracer(tracer, options); +} + +function makeRun(overrides: Partial = {}): Run { + return { + id: `run-${Math.random().toString(36).slice(2, 8)}`, + name: "test-run", + run_type: "llm", + start_time: Date.now(), + end_time: Date.now() + 100, + serialized: {}, + inputs: {}, + outputs: {}, + execution_order: 1, + child_execution_order: 1, + child_runs: [], + tags: [], + events: [], + ...overrides, + } as unknown as Run; +} + +afterEach(async () => { + if (provider) { + await provider.forceFlush(); + await provider.shutdown(); + } + exporter?.reset(); +}); + +describe("LangChain Instrumentation Functional Tests", () => { + describe("single LLM call", () => { + it("produces a span with chat operation and model attributes", async () => { + setup(); + const run = makeRun({ + id: "llm-1", + name: "ChatOpenAI", + run_type: "llm", + extra: { + metadata: { ls_model_name: "gpt-4o", ls_provider: "OpenAI" }, + }, + outputs: { + generations: [ + [ + { + message: { + usage_metadata: { input_tokens: 50, output_tokens: 20 }, + kwargs: { response_metadata: { model_name: "gpt-4o" } }, + }, + }, + ], + ], + }, + }); + + await langchainTracer.onRunCreate(run); + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(run); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + + const span = spans[0]; + assert.ok(span.name.includes("chat"), `span name "${span.name}" should contain "chat"`); + assert.strictEqual(span.kind, SpanKind.INTERNAL); + assert.strictEqual(span.status.code, SpanStatusCode.OK); + assert.strictEqual(span.attributes[ATTR_GEN_AI_OPERATION_NAME], GEN_AI_OPERATION_CHAT); + assert.strictEqual(span.attributes[ATTR_GEN_AI_REQUEST_MODEL], "gpt-4o"); + assert.strictEqual(span.attributes[ATTR_GEN_AI_PROVIDER_NAME], "openai"); + assert.strictEqual(span.attributes[ATTR_GEN_AI_USAGE_INPUT_TOKENS], 50); + assert.strictEqual(span.attributes[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS], 20); + }); + }); + + describe("agent with tool and LLM children", () => { + it("produces parent-child spans matching the LangGraph execution flow", async () => { + setup({ isContentRecordingEnabled: true }); + + // 1. Agent (parent) run + const agentRun = makeRun({ + id: "agent-1", + name: "ResearchAgent", + run_type: "chain", + serialized: { + id: ["langchain", "langgraph", "pregel", "CompiledStateGraph"], + }, + extra: { metadata: { session_id: "sess-abc" } }, + outputs: { + messages: [{ role: "assistant", content: "Here is my research." }], + }, + }); + + // 2. Tool run (child of agent) + const toolRun = makeRun({ + id: "tool-1", + parent_run_id: "agent-1", + name: "web_search", + run_type: "tool", + serialized: { name: "web_search" }, + inputs: { input: "opentelemetry langchain" }, + outputs: { + output: { + kwargs: { content: "search results..." }, + tool_call_id: "tc-1", + }, + }, + }); + + // 3. LLM run (child of agent) + const llmRun = makeRun({ + id: "llm-1", + parent_run_id: "agent-1", + name: "ChatOpenAI", + run_type: "llm", + extra: { + metadata: { ls_model_name: "gpt-4o", ls_provider: "OpenAI" }, + }, + inputs: { + messages: [ + [ + { role: "system", content: "You are a research assistant." }, + { role: "user", content: "Find info about OTel" }, + ], + ], + }, + outputs: { + generations: [ + [ + { + text: "Here is the information...", + message: { + role: "assistant", + content: "Here is the information...", + usage_metadata: { input_tokens: 200, output_tokens: 80 }, + kwargs: { response_metadata: { model_name: "gpt-4o" } }, + }, + }, + ], + ], + }, + }); + + // Simulate LangChain callback order: agent starts, then tool, then llm + await langchainTracer.onRunCreate(agentRun); + await langchainTracer.onRunCreate(toolRun); + await langchainTracer.onRunCreate(llmRun); + + // End in reverse order (children end before parent) + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(llmRun); + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(toolRun); + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(agentRun); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 3, `expected 3 spans, got ${spans.length}`); + + // Find spans by operation + const agentSpan = spans.find((s) => s.attributes[ATTR_GEN_AI_OPERATION_NAME] === GEN_AI_OPERATION_INVOKE_AGENT); + const toolSpan = spans.find((s) => s.attributes[ATTR_GEN_AI_OPERATION_NAME] === GEN_AI_OPERATION_EXECUTE_TOOL); + const llmSpan = spans.find((s) => s.attributes[ATTR_GEN_AI_OPERATION_NAME] === GEN_AI_OPERATION_CHAT); + + assert.ok(agentSpan, "agent span should exist"); + assert.ok(toolSpan, "tool span should exist"); + assert.ok(llmSpan, "llm span should exist"); + + // Verify agent span attributes + assert.ok(agentSpan!.name.includes("invoke_agent")); + assert.strictEqual(agentSpan!.attributes[ATTR_GEN_AI_AGENT_NAME], "ResearchAgent"); + assert.strictEqual(agentSpan!.attributes[ATTR_MICROSOFT_SESSION_ID], "sess-abc"); + + // Verify tool span attributes + assert.ok(toolSpan!.name.includes("execute_tool")); + assert.strictEqual(toolSpan!.attributes[ATTR_GEN_AI_TOOL_NAME], "web_search"); + + // Verify LLM span attributes + assert.ok(llmSpan!.name.includes("chat")); + assert.strictEqual(llmSpan!.attributes[ATTR_GEN_AI_REQUEST_MODEL], "gpt-4o"); + assert.strictEqual(llmSpan!.attributes[ATTR_GEN_AI_USAGE_INPUT_TOKENS], 200); + assert.strictEqual(llmSpan!.attributes[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS], 80); + + // Verify parent-child: tool and LLM spans should have the agent as parent + assert.strictEqual( + toolSpan!.parentSpanContext?.spanId, + agentSpan!.spanContext().spanId, + "tool span should be child of agent span" + ); + assert.strictEqual( + llmSpan!.parentSpanContext?.spanId, + agentSpan!.spanContext().spanId, + "llm span should be child of agent span" + ); + }); + }); + + describe("content recording gating", () => { + it("does not record message content when disabled", async () => { + setup({ isContentRecordingEnabled: false }); + + const run = makeRun({ + id: "llm-no-content", + name: "ChatOpenAI", + run_type: "llm", + inputs: { + messages: [[{ role: "user", content: "Secret question" }]], + }, + outputs: { + generations: [[{ text: "Secret answer" }]], + }, + }); + + await langchainTracer.onRunCreate(run); + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(run); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + + const span = spans[0]; + assert.strictEqual(span.attributes[ATTR_GEN_AI_INPUT_MESSAGES], undefined, "should not record input messages"); + assert.strictEqual(span.attributes[ATTR_GEN_AI_OUTPUT_MESSAGES], undefined, "should not record output messages"); + }); + + it("records message content when enabled", async () => { + setup({ isContentRecordingEnabled: true }); + + const run = makeRun({ + id: "llm-with-content", + name: "ChatOpenAI", + run_type: "llm", + inputs: { + messages: [[{ role: "user", content: "What is 2+2?" }]], + }, + outputs: { + generations: [[{ text: "4" }]], + }, + }); + + await langchainTracer.onRunCreate(run); + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(run); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + + const span = spans[0]; + assert.ok(span.attributes[ATTR_GEN_AI_INPUT_MESSAGES], "should record input messages"); + assert.ok(span.attributes[ATTR_GEN_AI_OUTPUT_MESSAGES], "should record output messages"); + }); + }); + + describe("error handling", () => { + it("sets error status and message on failed runs", async () => { + setup(); + + const run = makeRun({ + id: "llm-error", + name: "ChatOpenAI", + run_type: "llm", + error: "Rate limit exceeded", + }); + + await langchainTracer.onRunCreate(run); + await (langchainTracer as unknown as { _endTrace(r: Run): Promise })._endTrace(run); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + + const span = spans[0]; + assert.strictEqual(span.status.code, SpanStatusCode.ERROR); + assert.strictEqual(span.attributes["error.message"], "Rate limit exceeded"); + }); + }); + + describe("internal run filtering", () => { + it("does not create spans for langsmith:hidden or Branch runs", async () => { + setup(); + + const hiddenRun = makeRun({ id: "hidden-1", tags: ["langsmith:hidden"] }); + const branchRun = makeRun({ + id: "branch-1", + name: "BranchDecision", + run_type: "chain", + serialized: {}, + }); + + await langchainTracer.onRunCreate(hiddenRun); + await langchainTracer.onRunCreate(branchRun); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 0, "no spans should be created for internal runs"); + }); + }); +}); diff --git a/test/internal/unit/genai/langchain/langchainTraceInstrumentor.test.ts b/test/internal/unit/genai/langchain/langchainTraceInstrumentor.test.ts new file mode 100644 index 00000000..acce63c0 --- /dev/null +++ b/test/internal/unit/genai/langchain/langchainTraceInstrumentor.test.ts @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, assert, describe, it, vi } from "vitest"; +import { Tracer } from "@opentelemetry/api"; +import { + LangChainTraceInstrumentor, + addTracerToHandlers, +} from "../../../../../src/genai/instrumentations/langchain/langchainTraceInstrumentor.js"; +import { LangChainTracer } from "../../../../../src/genai/instrumentations/langchain/tracer.js"; + +function createMockTracer(): Tracer { + return { + startSpan: vi.fn(), + } as unknown as Tracer; +} + +afterEach(() => { + LangChainTraceInstrumentor.resetInstance(); + vi.restoreAllMocks(); +}); + +describe("LangChainTraceInstrumentor", () => { + describe("instrument", () => { + it("patches CallbackManager._configureSync", () => { + const configureSyncOriginal = vi.fn(); + const mockModule = { + CallbackManager: { + _configureSync: configureSyncOriginal, + }, + }; + + LangChainTraceInstrumentor.instrument(mockModule as any); + + // After instrumenting, _configureSync should be wrapped + assert.notStrictEqual( + mockModule.CallbackManager._configureSync, + configureSyncOriginal, + "_configureSync should be wrapped" + ); + }); + + it("does nothing when CallbackManager is missing", () => { + const mockModule = {} as any; + // Should not throw + LangChainTraceInstrumentor.instrument(mockModule); + }); + }); + + describe("enable / disable", () => { + it("throws when not initialized", () => { + assert.throws( + () => LangChainTraceInstrumentor.enable(), + /must be initialized first/ + ); + }); + + it("throws disable when not initialized", () => { + assert.throws( + () => LangChainTraceInstrumentor.disable(), + /must be initialized first/ + ); + }); + + it("enable and disable do not throw after initialization", () => { + const mockModule = { + CallbackManager: { + _configureSync: vi.fn(), + }, + }; + LangChainTraceInstrumentor.instrument(mockModule as any); + + // Should not throw + LangChainTraceInstrumentor.enable(); + LangChainTraceInstrumentor.disable(); + }); + }); + + describe("resetInstance", () => { + it("allows re-initialization after reset", () => { + const mockModule = { + CallbackManager: { + _configureSync: vi.fn(), + }, + }; + LangChainTraceInstrumentor.instrument(mockModule as any); + LangChainTraceInstrumentor.resetInstance(); + + // Should be able to instrument again + const mockModule2 = { + CallbackManager: { + _configureSync: vi.fn(), + }, + }; + LangChainTraceInstrumentor.instrument(mockModule2 as any); + }); + }); +}); + +describe("addTracerToHandlers", () => { + it("creates a new array with LangChainTracer when handlers is null", () => { + const tracer = createMockTracer(); + const result = addTracerToHandlers(tracer, undefined); + assert.ok(Array.isArray(result)); + assert.strictEqual(result.length, 1); + assert.ok(result[0] instanceof LangChainTracer); + }); + + it("appends LangChainTracer to existing array handlers", () => { + const tracer = createMockTracer(); + const existingHandler = { handleLLMStart: vi.fn() }; + const handlers = [existingHandler] as any; + const result = addTracerToHandlers(tracer, handlers); + assert.ok(Array.isArray(result)); + assert.strictEqual(result.length, 2); + assert.ok(result[1] instanceof LangChainTracer); + }); + + it("does not add duplicate LangChainTracer to array handlers", () => { + const tracer = createMockTracer(); + const existingTracer = new LangChainTracer(tracer); + const handlers = [existingTracer] as any; + const result = addTracerToHandlers(tracer, handlers); + assert.ok(Array.isArray(result)); + assert.strictEqual(result.length, 1, "should not duplicate"); + }); + + it("adds LangChainTracer to CallbackManager-style handlers", () => { + const tracer = createMockTracer(); + const addHandlerSpy = vi.fn(); + const handlers = { + inheritableHandlers: [] as any[], + addHandler: addHandlerSpy, + } as any; + addTracerToHandlers(tracer, handlers); + assert.ok(addHandlerSpy.mock.calls.length === 1); + assert.ok(addHandlerSpy.mock.calls[0][0] instanceof LangChainTracer); + assert.strictEqual(addHandlerSpy.mock.calls[0][1], true, "should be inheritable"); + }); + + it("does not add duplicate to CallbackManager-style handlers", () => { + const tracer = createMockTracer(); + const existingTracer = new LangChainTracer(tracer); + const addHandlerSpy = vi.fn(); + const handlers = { + inheritableHandlers: [existingTracer], + addHandler: addHandlerSpy, + } as any; + addTracerToHandlers(tracer, handlers); + assert.strictEqual(addHandlerSpy.mock.calls.length, 0, "should not add duplicate"); + }); + + it("passes content recording option through", () => { + const tracer = createMockTracer(); + const result = addTracerToHandlers(tracer, undefined, { isContentRecordingEnabled: true }); + assert.ok(Array.isArray(result)); + assert.ok(result[0] instanceof LangChainTracer); + }); +}); diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts new file mode 100644 index 00000000..6a1b039a --- /dev/null +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, assert, describe, it, vi } from "vitest"; +import { Span, SpanContext, SpanKind, SpanStatusCode, Tracer, TraceFlags } from "@opentelemetry/api"; +import type { Run } from "@langchain/core/tracers/base"; +import { LangChainTracer } from "../../../../../src/genai/instrumentations/langchain/tracer.js"; +import { + ATTR_ERROR_MESSAGE, + ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_PROVIDER_NAME, +} from "../../../../../src/genai/index.js"; + +function makeRun(overrides: Partial = {}): Run { + return { + id: `run-${Math.random().toString(36).slice(2, 8)}`, + name: "test-run", + run_type: "llm", + start_time: Date.now(), + end_time: Date.now() + 100, + serialized: {}, + inputs: {}, + outputs: {}, + execution_order: 1, + child_execution_order: 1, + child_runs: [], + tags: [], + events: [], + ...overrides, + } as unknown as Run; +} + +function makeLangGraphRun(overrides: Partial = {}): Run { + return makeRun({ + run_type: "chain", + name: "MyAgent", + serialized: { + id: ["langchain", "langgraph", "pregel", "CompiledStateGraph"], + }, + ...overrides, + }); +} + +function createMockSpan(): Span & { attrs: Record; ended: boolean; endTime?: unknown; statusObj?: { code: SpanStatusCode } } { + const spanCtx: SpanContext = { + traceId: "aaaa0000bbbb0000cccc0000dddd0000", + spanId: "1111000022220000", + traceFlags: TraceFlags.SAMPLED, + }; + const mockSpan = { + attrs: {} as Record, + ended: false, + endTime: undefined as unknown, + statusObj: undefined as { code: SpanStatusCode } | undefined, + setAttribute: vi.fn(function (this: typeof mockSpan, key: string, val: unknown) { + this.attrs[key] = val; + return this; + }), + setStatus: vi.fn(function (this: typeof mockSpan, status: { code: SpanStatusCode }) { + this.statusObj = status; + }), + end: vi.fn(function (this: typeof mockSpan, endTime?: unknown) { + this.ended = true; + this.endTime = endTime; + }), + spanContext: vi.fn(() => spanCtx), + recordException: vi.fn(), + addEvent: vi.fn(), + isRecording: vi.fn(() => true), + updateName: vi.fn(), + addLink: vi.fn(), + addLinks: vi.fn(), + }; + return mockSpan as unknown as Span & { attrs: Record; ended: boolean; endTime?: unknown; statusObj?: { code: SpanStatusCode } }; +} + +function createMockTracer(): Tracer & { lastSpan: ReturnType | undefined; spans: ReturnType[] } { + const mockTracer = { + lastSpan: undefined as ReturnType | undefined, + spans: [] as ReturnType[], + startSpan: vi.fn(function (this: typeof mockTracer, _name: string, _options?: unknown, _ctx?: unknown) { + const span = createMockSpan(); + this.lastSpan = span; + this.spans.push(span); + return span; + }), + }; + return mockTracer as unknown as Tracer & { lastSpan: ReturnType | undefined; spans: ReturnType[] }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("LangChainTracer", () => { + describe("constructor", () => { + it("creates a tracer with default options", () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + assert.strictEqual(lct.name, "OpenTelemetryLangChainTracer"); + }); + }); + + describe("onRunCreate / startTracing", () => { + it("creates a span for an LLM run", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ name: "gpt-4o" }); + await lct.onRunCreate(run); + assert.ok(tracer.lastSpan, "should have created a span"); + assert.ok( + (tracer.startSpan as ReturnType).mock.calls[0][0].includes("chat"), + "span name should include operation type" + ); + }); + + it("creates a span for a tool run", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ run_type: "tool", name: "search", serialized: { name: "search" } }); + await lct.onRunCreate(run); + const spanName = (tracer.startSpan as ReturnType).mock.calls[0][0]; + assert.ok(spanName.includes("execute_tool"), "span name should include execute_tool"); + assert.ok(spanName.includes("search"), "span name should include tool name"); + }); + + it("creates a span for a LangGraph agent run", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeLangGraphRun({ name: "WeatherBot" }); + await lct.onRunCreate(run); + const spanName = (tracer.startSpan as ReturnType).mock.calls[0][0]; + assert.ok(spanName.includes("invoke_agent"), "span name should include invoke_agent"); + assert.ok(spanName.includes("WeatherBot"), "span name should include agent name"); + }); + + it("skips internal runs tagged langsmith:hidden", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ tags: ["langsmith:hidden"] }); + await lct.onRunCreate(run); + assert.strictEqual(tracer.lastSpan, undefined); + }); + + it("skips Branch-prefixed runs", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ name: "BranchDecision", run_type: "chain", serialized: {} }); + await lct.onRunCreate(run); + assert.strictEqual(tracer.lastSpan, undefined); + }); + + it("skips unknown operation types", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ run_type: "retriever" as Run["run_type"] }); + await lct.onRunCreate(run); + assert.strictEqual(tracer.lastSpan, undefined); + }); + + it("sets langchain as provider name attribute", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun(); + await lct.onRunCreate(run); + const attrs = (tracer.startSpan as ReturnType).mock.calls[0][1]?.attributes; + assert.strictEqual(attrs?.[ATTR_GEN_AI_PROVIDER_NAME], "langchain"); + }); + + it("sets span kind to INTERNAL", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun(); + await lct.onRunCreate(run); + const kind = (tracer.startSpan as ReturnType).mock.calls[0][1]?.kind; + assert.strictEqual(kind, SpanKind.INTERNAL); + }); + }); + + describe("_endTrace", () => { + it("ends the span with OK status on success", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun(); + await lct.onRunCreate(run); + const span = tracer.lastSpan!; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + assert.strictEqual(span.ended, true); + assert.strictEqual(span.statusObj?.code, SpanStatusCode.OK); + }); + + it("sets ERROR status when run has an error", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ error: "Something went wrong" }); + await lct.onRunCreate(run); + const span = tracer.lastSpan!; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + assert.strictEqual(span.statusObj?.code, SpanStatusCode.ERROR); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_ERROR_MESSAGE && c[1] === "Something went wrong" + ) + ); + }); + + it("sets operation type attribute", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun({ run_type: "llm" }); + await lct.onRunCreate(run); + const span = tracer.lastSpan!; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_OPERATION_NAME && c[1] === "chat" + ) + ); + }); + + it("does not set content attributes when content recording is disabled", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer, { isContentRecordingEnabled: false }); + const run = makeRun({ + run_type: "tool", + name: "my_tool", + serialized: { name: "my_tool" }, + inputs: { input: "test" }, + }); + await lct.onRunCreate(run); + const span = tracer.lastSpan!; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + const attrKeys = (span.setAttribute as ReturnType).mock.calls.map((c: unknown[]) => c[0]); + assert.ok(!attrKeys.includes("gen_ai.tool.call.arguments"), "should not set tool arguments"); + assert.ok(!attrKeys.includes("gen_ai.input.messages"), "should not set input messages"); + }); + + it("sets content attributes when content recording is enabled", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer, { isContentRecordingEnabled: true }); + const run = makeRun({ + run_type: "tool", + name: "my_tool", + serialized: { name: "my_tool" }, + inputs: { input: "test-input" }, + }); + await lct.onRunCreate(run); + const span = tracer.lastSpan!; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + const attrKeys = (span.setAttribute as ReturnType).mock.calls.map((c: unknown[]) => c[0]); + assert.ok(attrKeys.includes("gen_ai.tool.name"), "should set tool name"); + }); + + it("cleans up run from tracking after end", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + const run = makeRun(); + await lct.onRunCreate(run); + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + // A second _endTrace should be a no-op (no span found) + const spanCount = tracer.spans.length; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(run); + // Should not end any additional spans + assert.strictEqual(tracer.spans.length, spanCount); + }); + }); + + describe("parent-child span linking", () => { + it("links child spans to parent spans", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + + const parentRun = makeLangGraphRun({ id: "parent-1" }); + await lct.onRunCreate(parentRun); + + const childRun = makeRun({ id: "child-1", parent_run_id: "parent-1" }); + await lct.onRunCreate(childRun); + + // The child span should have been started with a context that includes the parent span + const startSpanCalls = (tracer.startSpan as ReturnType).mock.calls; + assert.strictEqual(startSpanCalls.length, 2); + // The 3rd argument to startSpan is the context — it should not be the bare active context + const childCtxArg = startSpanCalls[1][2]; + assert.ok(childCtxArg, "child span should receive a parent context"); + }); + }); + + describe("MAX_RUNS cap", () => { + it("stops creating spans after MAX_RUNS is reached", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + + // Access the private MAX_RUNS value — it's 10_000 but we can't create that many. + // Instead, fill the runs map directly and test the guard. + const runsMap = (lct as unknown as { runs: Map }).runs; + for (let i = 0; i < 10_000; i++) { + runsMap.set(`fill-${i}`, {}); + } + + const run = makeRun({ id: "overflow-run" }); + await lct.onRunCreate(run); + + // After reaching MAX_RUNS, startSpan should not be called for the overflow run + const startSpanCalls = (tracer.startSpan as ReturnType).mock.calls; + assert.strictEqual(startSpanCalls.length, 0, "should not create span when MAX_RUNS exceeded"); + }); + }); +}); diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts new file mode 100644 index 00000000..2ea98da1 --- /dev/null +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -0,0 +1,539 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, assert, describe, it, vi } from "vitest"; +import type { Span } from "@opentelemetry/api"; +import type { Run } from "@langchain/core/tracers/base"; +import { + getOperationType, + setOperationTypeAttribute, + setAgentAttributes, + setToolAttributes, + setInputMessagesAttribute, + setOutputMessagesAttribute, + setModelAttribute, + setProviderNameAttribute, + setSessionIdAttribute, + setSystemInstructionsAttribute, + setTokenAttributes, + isString, +} from "../../../../../src/genai/instrumentations/langchain/utils.js"; +import { + ATTR_GEN_AI_AGENT_NAME, + ATTR_GEN_AI_INPUT_MESSAGES, + ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_OUTPUT_MESSAGES, + ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, + ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, + ATTR_GEN_AI_TOOL_CALL_ID, + ATTR_GEN_AI_TOOL_CALL_RESULT, + ATTR_GEN_AI_TOOL_NAME, + ATTR_GEN_AI_TOOL_TYPE, + ATTR_GEN_AI_USAGE_INPUT_TOKENS, + ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, + ATTR_MICROSOFT_SESSION_ID, + GEN_AI_OPERATION_CHAT, + GEN_AI_OPERATION_EXECUTE_TOOL, + GEN_AI_OPERATION_INVOKE_AGENT, +} from "../../../../../src/genai/index.js"; + +function makeSpan(): Span & { attrs: Record } { + const attrs: Record = {}; + return { + attrs, + setAttribute: vi.fn((key: string, value: unknown) => { + attrs[key] = value; + return this; + }), + setStatus: vi.fn(), + end: vi.fn(), + recordException: vi.fn(), + addEvent: vi.fn(), + isRecording: vi.fn(() => true), + spanContext: vi.fn(), + updateName: vi.fn(), + addLink: vi.fn(), + addLinks: vi.fn(), + } as unknown as Span & { attrs: Record }; +} + +function makeRun(overrides: Partial = {}): Run { + return { + id: "run-1", + name: "test-run", + run_type: "llm", + start_time: Date.now(), + serialized: {}, + inputs: {}, + execution_order: 1, + child_execution_order: 1, + child_runs: [], + tags: [], + events: [], + ...overrides, + } as unknown as Run; +} + +function makeLangGraphRun(overrides: Partial = {}): Run { + return makeRun({ + run_type: "chain", + name: "MyAgent", + serialized: { + id: ["langchain", "langgraph", "pregel", "CompiledStateGraph"], + }, + ...overrides, + }); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("isString", () => { + it("returns true for strings", () => { + assert.strictEqual(isString("hello"), true); + assert.strictEqual(isString(""), true); + }); + + it("returns false for non-strings", () => { + assert.strictEqual(isString(123), false); + assert.strictEqual(isString(null), false); + assert.strictEqual(isString(undefined), false); + assert.strictEqual(isString({}), false); + }); +}); + +describe("getOperationType", () => { + it("returns invoke_agent for LangGraph chain runs", () => { + const run = makeLangGraphRun(); + assert.strictEqual(getOperationType(run), GEN_AI_OPERATION_INVOKE_AGENT); + }); + + it("returns execute_tool for tool runs", () => { + const run = makeRun({ run_type: "tool" }); + assert.strictEqual(getOperationType(run), GEN_AI_OPERATION_EXECUTE_TOOL); + }); + + it("returns chat for llm runs", () => { + const run = makeRun({ run_type: "llm" }); + assert.strictEqual(getOperationType(run), GEN_AI_OPERATION_CHAT); + }); + + it("returns unknown for unrecognized run types", () => { + const run = makeRun({ run_type: "retriever" as Run["run_type"] }); + assert.strictEqual(getOperationType(run), "unknown"); + }); + + it("returns unknown for chain runs that are not LangGraph agents", () => { + const run = makeRun({ run_type: "chain", serialized: {} }); + assert.strictEqual(getOperationType(run), "unknown"); + }); +}); + +describe("setOperationTypeAttribute", () => { + it("sets the operation name attribute on the span", () => { + const span = makeSpan(); + setOperationTypeAttribute("chat", span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_OPERATION_NAME && c[1] === "chat" + ) + ); + }); +}); + +describe("setAgentAttributes", () => { + it("sets agent name for LangGraph agent runs", () => { + const span = makeSpan(); + const run = makeLangGraphRun({ name: "WeatherAgent" }); + setAgentAttributes(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_AGENT_NAME && c[1] === "WeatherAgent" + ) + ); + }); + + it("does not set agent name for non-agent runs", () => { + const span = makeSpan(); + const run = makeRun({ run_type: "llm", name: "SomeModel" }); + setAgentAttributes(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setToolAttributes", () => { + it("sets tool attributes for tool runs", () => { + const span = makeSpan(); + const run = makeRun({ + run_type: "tool", + name: "search_web", + serialized: { name: "search_web" }, + inputs: { input: "query text" }, + outputs: { + output: { + kwargs: { content: "result content" }, + tool_call_id: "tc-123", + }, + }, + }); + setToolAttributes(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_TOOL_NAME && c[1] === "search_web")); + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_TOOL_CALL_ARGUMENTS)); + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_TOOL_CALL_RESULT)); + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_TOOL_TYPE && c[1] === "extension")); + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_TOOL_CALL_ID && c[1] === "tc-123")); + }); + + it("does nothing for non-tool runs", () => { + const span = makeSpan(); + const run = makeRun({ run_type: "llm" }); + setToolAttributes(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); + + it("does nothing when serialized is missing", () => { + const span = makeSpan(); + const run = makeRun({ run_type: "tool", serialized: undefined as unknown as Record }); + setToolAttributes(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setInputMessagesAttribute", () => { + it("extracts simple format user messages", () => { + const span = makeSpan(); + const run = makeRun({ + run_type: "llm", + inputs: { + messages: [ + [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there" }, + ], + ], + }, + }); + setInputMessagesAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + const msgCall = calls.find((c: unknown[]) => c[0] === ATTR_GEN_AI_INPUT_MESSAGES); + assert.ok(msgCall, "should set input messages"); + const parsed = JSON.parse(msgCall![1] as string); + assert.ok(parsed.includes("Hello")); + // Assistant messages should be filtered out for input + assert.ok(!parsed.includes("Hi there")); + }); + + it("extracts LangChain lc_kwargs format", () => { + const span = makeSpan(); + const run = makeRun({ + run_type: "llm", + inputs: { + messages: [ + [ + { lc_type: "human", lc_kwargs: { content: "What is 2+2?" } }, + ], + ], + }, + }); + setInputMessagesAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + const msgCall = calls.find((c: unknown[]) => c[0] === ATTR_GEN_AI_INPUT_MESSAGES); + assert.ok(msgCall); + assert.ok(JSON.parse(msgCall![1] as string).includes("What is 2+2?")); + }); + + it("extracts messages using id array-based type detection", () => { + const span = makeSpan(); + // When role/lc_type/type are absent, getMessageType falls through to the id array check. + // Content is extracted via lc_kwargs. + const run = makeRun({ + run_type: "llm", + inputs: { + messages: [ + [ + { lc_kwargs: { content: "Build this" }, id: ["langchain", "schema", "messages", "HumanMessage"] }, + ], + ], + }, + }); + setInputMessagesAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + const msgCall = calls.find((c: unknown[]) => c[0] === ATTR_GEN_AI_INPUT_MESSAGES); + assert.ok(msgCall); + assert.ok(JSON.parse(msgCall![1] as string).includes("Build this")); + }); + + it("does nothing when messages is not an array", () => { + const span = makeSpan(); + const run = makeRun({ inputs: { messages: "not-an-array" } }); + setInputMessagesAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setOutputMessagesAttribute", () => { + it("extracts output from messages array", () => { + const span = makeSpan(); + const run = makeRun({ + run_type: "llm", + outputs: { + messages: [ + { role: "assistant", content: "Here is the answer" }, + ], + }, + }); + setOutputMessagesAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + const msgCall = calls.find((c: unknown[]) => c[0] === ATTR_GEN_AI_OUTPUT_MESSAGES); + assert.ok(msgCall); + assert.ok(JSON.parse(msgCall![1] as string).includes("Here is the answer")); + }); + + it("extracts output from generations format", () => { + const span = makeSpan(); + const run = makeRun({ + run_type: "llm", + outputs: { + generations: [ + [ + { + text: "Generated text", + }, + ], + ], + }, + }); + setOutputMessagesAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + const msgCall = calls.find((c: unknown[]) => c[0] === ATTR_GEN_AI_OUTPUT_MESSAGES); + assert.ok(msgCall); + assert.ok(JSON.parse(msgCall![1] as string).includes("Generated text")); + }); + + it("extracts output from single message object", () => { + const span = makeSpan(); + const run = makeLangGraphRun({ + outputs: { + message: { role: "assistant", content: "Single response" }, + }, + }); + setOutputMessagesAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + const msgCall = calls.find((c: unknown[]) => c[0] === ATTR_GEN_AI_OUTPUT_MESSAGES); + assert.ok(msgCall); + assert.ok(JSON.parse(msgCall![1] as string).includes("Single response")); + }); + + it("does nothing when outputs is undefined", () => { + const span = makeSpan(); + const run = makeRun({ outputs: undefined }); + setOutputMessagesAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setModelAttribute", () => { + it("extracts model from response_metadata", () => { + const span = makeSpan(); + const run = makeRun({ + outputs: { + generations: [[{ message: { kwargs: { response_metadata: { model_name: "gpt-4o" } } } }]], + }, + }); + setModelAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "gpt-4o" + ) + ); + }); + + it("extracts model from extra.metadata.ls_model_name", () => { + const span = makeSpan(); + const run = makeRun({ + extra: { metadata: { ls_model_name: "claude-3" } }, + }); + setModelAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "claude-3" + ) + ); + }); + + it("extracts model from extra.invocation_params.model", () => { + const span = makeSpan(); + const run = makeRun({ + extra: { invocation_params: { model: "llama-3" } }, + }); + setModelAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "llama-3" + ) + ); + }); + + it("does nothing when no model is found", () => { + const span = makeSpan(); + const run = makeRun(); + setModelAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setProviderNameAttribute", () => { + it("sets provider name from metadata", () => { + const span = makeSpan(); + const run = makeRun({ extra: { metadata: { ls_provider: "OpenAI" } } }); + setProviderNameAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_PROVIDER_NAME && c[1] === "openai" + ) + ); + }); + + it("does nothing when no provider metadata", () => { + const span = makeSpan(); + const run = makeRun(); + setProviderNameAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setSessionIdAttribute", () => { + it("extracts session_id from metadata", () => { + const span = makeSpan(); + const run = makeRun({ extra: { metadata: { session_id: "sess-123" } } }); + setSessionIdAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_MICROSOFT_SESSION_ID && c[1] === "sess-123" + ) + ); + }); + + it("falls back to conversation_id", () => { + const span = makeSpan(); + const run = makeRun({ extra: { metadata: { conversation_id: "conv-456" } } }); + setSessionIdAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_MICROSOFT_SESSION_ID && c[1] === "conv-456" + ) + ); + }); + + it("falls back to thread_id", () => { + const span = makeSpan(); + const run = makeRun({ extra: { metadata: { thread_id: "thread-789" } } }); + setSessionIdAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => c[0] === ATTR_MICROSOFT_SESSION_ID && c[1] === "thread-789" + ) + ); + }); + + it("does nothing for empty session id", () => { + const span = makeSpan(); + const run = makeRun({ extra: { metadata: { session_id: "" } } }); + setSessionIdAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); + + it("does nothing when no metadata", () => { + const span = makeSpan(); + const run = makeRun(); + setSessionIdAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setSystemInstructionsAttribute", () => { + it("extracts from prompts array", () => { + const span = makeSpan(); + const run = makeRun({ + inputs: { prompts: ["You are a helpful assistant.", "Be concise."] }, + }); + setSystemInstructionsAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => + c[0] === ATTR_GEN_AI_SYSTEM_INSTRUCTIONS && + (c[1] as string).includes("You are a helpful assistant.") + ) + ); + }); + + it("extracts from system messages with lc_type", () => { + const span = makeSpan(); + const run = makeRun({ + inputs: { + messages: [ + { lc_type: "system", lc_kwargs: { content: "System prompt here" } }, + { lc_type: "human", lc_kwargs: { content: "User message" } }, + ], + }, + }); + setSystemInstructionsAttribute(run, span); + assert.ok( + (span.setAttribute as ReturnType).mock.calls.some( + (c: unknown[]) => + c[0] === ATTR_GEN_AI_SYSTEM_INSTRUCTIONS && + (c[1] as string).includes("System prompt here") + ) + ); + }); + + it("does nothing when no inputs", () => { + const span = makeSpan(); + const run = makeRun({ inputs: undefined as unknown as Record }); + setSystemInstructionsAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); + +describe("setTokenAttributes", () => { + it("extracts token usage from usage_metadata", () => { + const span = makeSpan(); + const run = makeRun({ + outputs: { + generations: [ + [{ message: { usage_metadata: { input_tokens: 100, output_tokens: 50 } } }], + ], + }, + }); + setTokenAttributes(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_USAGE_INPUT_TOKENS && c[1] === 100)); + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_USAGE_OUTPUT_TOKENS && c[1] === 50)); + }); + + it("extracts from response_metadata.tokenUsage", () => { + const span = makeSpan(); + const run = makeRun({ + outputs: { + generations: [ + [{ message: { kwargs: { response_metadata: { tokenUsage: { input_tokens: 10, output_tokens: 5 } } } } }], + ], + }, + }); + setTokenAttributes(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_USAGE_INPUT_TOKENS && c[1] === 10)); + assert.ok(calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_USAGE_OUTPUT_TOKENS && c[1] === 5)); + }); + + it("does nothing when no usage data", () => { + const span = makeSpan(); + const run = makeRun({ outputs: {} }); + setTokenAttributes(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); +}); From 2e35ace62646e8d26792aee55f2943d1c2a404c9 Mon Sep 17 00:00:00 2001 From: Hector Hernandez <39923391+hectorhdzg@users.noreply.github.com> Date: Fri, 10 Apr 2026 09:23:08 -0700 Subject: [PATCH 2/2] Add changelog and sample --- CHANGELOG.md | 1 + samples/README.md | 2 + samples/package.json | 4 +- samples/src/langchainInstrumentation.ts | 50 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 samples/src/langchainInstrumentation.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index dc628723..76d1517b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [0.1.0] - Unreleased ### Features Added +- Add langchain isntrumentation.([#10](https://github.com/microsoft/opentelemetry-distro-javascript/pull/10)) ### Breaking Changes diff --git a/samples/README.md b/samples/README.md index 4fa46b9a..c05102ff 100644 --- a/samples/README.md +++ b/samples/README.md @@ -22,6 +22,7 @@ These sample programs show how to use the `@microsoft/opentelemetry` distributio | [otlpExporter.ts][otlpexporter] | Demonstrates how to enable the OTLP exporter alongside Azure Monitor to send telemetry to two locations. | | [redactQueryStrings.ts][redactquerystrings] | Demonstrates how to redact URL query strings from telemetry to protect sensitive information. | | [sampling.ts][sampling] | Demonstrates how to enable sampling to reduce data ingestion volume and control costs. | +| [langchainInstrumentation.ts][langchaininstrumentation] | Demonstrates how to enable LangChain instrumentation to trace GenAI operations. | ## Prerequisites @@ -70,3 +71,4 @@ APPLICATIONINSIGHTS_CONNECTION_STRING="" node dist/basic [otlpexporter]: https://github.com/Azure/opentelemetry-distro-javascript/blob/main/samples/src/otlpExporter.ts [redactquerystrings]: https://github.com/Azure/opentelemetry-distro-javascript/blob/main/samples/src/redactQueryStrings.ts [sampling]: https://github.com/Azure/opentelemetry-distro-javascript/blob/main/samples/src/sampling.ts +[langchaininstrumentation]: https://github.com/Azure/opentelemetry-distro-javascript/blob/main/samples/src/langchainInstrumentation.ts diff --git a/samples/package.json b/samples/package.json index 5899caa0..72982afd 100644 --- a/samples/package.json +++ b/samples/package.json @@ -23,7 +23,9 @@ "@opentelemetry/resources": "^2.2.0", "@opentelemetry/semantic-conventions": "^1.38.0", "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/exporter-trace-otlp-http": "^0.208.0" + "@opentelemetry/exporter-trace-otlp-http": "^0.208.0", + "@langchain/openai": "^1.4.4", + "@langchain/core": "^1.1.39" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/samples/src/langchainInstrumentation.ts b/samples/src/langchainInstrumentation.ts new file mode 100644 index 00000000..3108d04a --- /dev/null +++ b/samples/src/langchainInstrumentation.ts @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @summary Demonstrates how to enable LangChain instrumentation to trace GenAI operations. + */ + +import { + useMicrosoftOpenTelemetry, + shutdownMicrosoftOpenTelemetry, +} from "@microsoft/opentelemetry"; +import { ChatOpenAI } from "@langchain/openai"; +import { HumanMessage } from "@langchain/core/messages"; +import "dotenv/config"; + +async function main(): Promise { + useMicrosoftOpenTelemetry({ + azureMonitor: { + azureMonitorExporterOptions: { + connectionString: + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING || "", + }, + }, + instrumentationOptions: { + langchain: { + isContentRecordingEnabled: true, + }, + }, + }); + + // Create a LangChain chat model + const model = new ChatOpenAI({ + modelName: "gpt-4o", + temperature: 0, + azureOpenAIApiKey: process.env.AZURE_OPENAI_API_KEY, + azureOpenAIApiInstanceName: process.env.AZURE_OPENAI_INSTANCE_NAME, + azureOpenAIApiDeploymentName: process.env.AZURE_OPENAI_DEPLOYMENT_NAME, + azureOpenAIApiVersion: process.env.AZURE_OPENAI_API_VERSION || "2024-06-01", + }); + + // Invoke the model — this call is automatically traced + const response = await model.invoke([new HumanMessage("What is OpenTelemetry?")]); + + console.log("Response:", response.content); + console.log("LangChain traces sent to Azure Monitor"); + + await shutdownMicrosoftOpenTelemetry(); +} + +main().catch(console.error);