diff --git a/.claude/skills/ruby-integration/SKILL.md b/.claude/skills/ruby-integration/SKILL.md index 4bc068aa..a52d6d78 100644 --- a/.claude/skills/ruby-integration/SKILL.md +++ b/.claude/skills/ruby-integration/SKILL.md @@ -168,12 +168,11 @@ Follow existing example patterns: **Do in this order:** - [ ] **Appraisals FIRST**: Add to `Appraisals` file (latest + 2 recent + uninstalled), run `bundle exec appraisal generate` -- [ ] **Tests**: `test/braintrust/trace/your_provider_test.rb` -- [ ] **Integration**: `lib/braintrust/trace/contrib/your_provider.rb` -- [ ] **VCR cassettes**: `test/fixtures/vcr_cassettes/your_provider/` (record as you write tests) -- [ ] **Auto-load**: Add to `lib/braintrust/trace.rb` with `begin/rescue LoadError` -- [ ] **Example**: `examples/your_provider.rb` -- [ ] **Example**: `examples/internal/your_provider.rb` (comprehensive internal example) +- [ ] **Tests**: `test/braintrust/contrib/your_provider/` +- [ ] **Integration**: `lib/braintrust/contrib/your_provider/` +- [ ] **VCR cassettes**: Record with `VCR_MODE=all bundle exec appraisal ruby -Ilib:test test/...` — **never hand-craft cassettes** +- [ ] **Example**: `examples/contrib/your_provider.rb` — run it and verify the permalink works +- [ ] **Example**: `examples/internal/contrib/your_provider/basic.rb` (comprehensive internal example) - [ ] **Env var**: Add to `.env.example` if needed ## Test Coverage (LLM Providers) @@ -324,12 +323,26 @@ Use shared `TokenParser.parse_usage_tokens(usage)` in `lib/braintrust/trace/toke ## VCR Cassettes +**CRITICAL: Never hand-craft cassette YAML files.** Cassettes must be recorded from real API responses so they contain authentic request/response data. Hand-crafted cassettes produce tests that pass against fake data but fail against real APIs. + +To record cassettes (requires real API keys in env): + ```bash -VCR_MODE=all bundle exec rake test # Re-record all -VCR_MODE=new_episodes bundle exec rake test # Record new only -VCR_OFF=true bundle exec rake test # Skip VCR +# Record cassettes for a specific appraisal (preferred — targets only your new tests) +VCR_MODE=all bundle exec appraisal ruby -Ilib:test test/braintrust/contrib//... + +# Re-record all cassettes for an appraisal +VCR_MODE=all bundle exec appraisal rake test + +# Record only new cassettes (keep existing) +VCR_MODE=new_episodes bundle exec appraisal rake test + +# Skip VCR entirely (useful for local debugging with a real key) +VCR_OFF=true bundle exec rake test ``` +An `AGENTS.md` file in `test/fixtures/vcr_cassettes/` explains this to future contributors. + ## Reference Files - Integrations: `lib/braintrust/trace/contrib/{openai,anthropic}.rb` diff --git a/Appraisals b/Appraisals index d74f4180..8af01753 100644 --- a/Appraisals +++ b/Appraisals @@ -26,6 +26,10 @@ OPTIONAL_GEMS = { "1.8" => {constraint: "~> 1.8.0", deps: {}}, "1.9" => {constraint: "~> 1.9.0", deps: {}}, "latest" => {constraint: ">= 1.9", deps: {}} + }, + "llm.rb" => { + "4.11" => {constraint: "~> 4.11.0", deps: {}}, + "latest" => {constraint: ">= 4.11", deps: {}} } } diff --git a/examples/contrib/llm_rb.rb b/examples/contrib/llm_rb.rb new file mode 100644 index 00000000..1666fa13 --- /dev/null +++ b/examples/contrib/llm_rb.rb @@ -0,0 +1,51 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require "braintrust" +require "llm" +require "opentelemetry/sdk" + +# Example: Basic llm.rb chat with Braintrust tracing +# +# Usage: +# OPENAI_API_KEY=your-key bundle exec appraisal llm.rb ruby examples/contrib/llm_rb.rb + +unless ENV["OPENAI_API_KEY"] + puts "Error: OPENAI_API_KEY environment variable is required" + exit 1 +end + +# Initialize Braintrust (with blocking login) +# +# NOTE: blocking_login is only necessary for this short-lived example. +# In most production apps, you can omit this. +Braintrust.init(blocking_login: true) + +llm = LLM.openai(key: ENV["OPENAI_API_KEY"]) +ctx = LLM::Context.new(llm) + +# Instrument this context instance to produce Braintrust spans +Braintrust.instrument!(:llm_rb, target: ctx) + +# Get a tracer and wrap the conversation in a root span +tracer = OpenTelemetry.tracer_provider.tracer("llm-rb-example") + +root_span = nil +tracer.in_span("examples/contrib/llm_rb.rb") do |span| + root_span = span + + # Each ctx.talk call is automatically traced as a child span + ctx.talk("What is the capital of France?") + ctx.talk("And what is the population of that city?") +end + +# Print permalink to view this trace in Braintrust +puts "\nView this trace in Braintrust:" +puts " #{Braintrust::Trace.permalink(root_span)}" + +# Shutdown to flush spans to Braintrust +# +# NOTE: shutdown is only necessary for this short-lived example. +# In most production apps, you can omit this. +OpenTelemetry.tracer_provider.shutdown diff --git a/gemfiles/llm.rb.gemfile b/gemfiles/llm.rb.gemfile new file mode 100644 index 00000000..9a39e1c0 --- /dev/null +++ b/gemfiles/llm.rb.gemfile @@ -0,0 +1,19 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "appraisal", "~> 2.5" +gem "climate_control", "~> 1.2" +gem "kramdown", "~> 2.0" +gem "minitest-reporters", "~> 1.6" +gem "minitest-stub-const", "~> 0.6" +gem "minitest", "~> 5.0" +gem "rake", "~> 13.0" +gem "simplecov", "~> 0.22" +gem "standard", "~> 1.0" +gem "vcr", "~> 6.0" +gem "webmock", "~> 3.0" +gem "yard", "~> 0.9" +gem "llm.rb", ">= 4.11" + +gemspec path: "../" diff --git a/gemfiles/llm.rb_4_11.gemfile b/gemfiles/llm.rb_4_11.gemfile new file mode 100644 index 00000000..c85b593a --- /dev/null +++ b/gemfiles/llm.rb_4_11.gemfile @@ -0,0 +1,19 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "appraisal", "~> 2.5" +gem "climate_control", "~> 1.2" +gem "kramdown", "~> 2.0" +gem "minitest-reporters", "~> 1.6" +gem "minitest-stub-const", "~> 0.6" +gem "minitest", "~> 5.0" +gem "rake", "~> 13.0" +gem "simplecov", "~> 0.22" +gem "standard", "~> 1.0" +gem "vcr", "~> 6.0" +gem "webmock", "~> 3.0" +gem "yard", "~> 0.9" +gem "llm.rb", "~> 4.11.0" + +gemspec path: "../" diff --git a/gemfiles/llm.rb_uninstalled.gemfile b/gemfiles/llm.rb_uninstalled.gemfile new file mode 100644 index 00000000..6c953f98 --- /dev/null +++ b/gemfiles/llm.rb_uninstalled.gemfile @@ -0,0 +1,18 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "appraisal", "~> 2.5" +gem "climate_control", "~> 1.2" +gem "kramdown", "~> 2.0" +gem "minitest-reporters", "~> 1.6" +gem "minitest-stub-const", "~> 0.6" +gem "minitest", "~> 5.0" +gem "rake", "~> 13.0" +gem "simplecov", "~> 0.22" +gem "standard", "~> 1.0" +gem "vcr", "~> 6.0" +gem "webmock", "~> 3.0" +gem "yard", "~> 0.9" + +gemspec path: "../" diff --git a/lib/braintrust/contrib.rb b/lib/braintrust/contrib.rb index feef9c28..c6b91400 100644 --- a/lib/braintrust/contrib.rb +++ b/lib/braintrust/contrib.rb @@ -197,9 +197,11 @@ def tracer_for(target, name: "braintrust") require_relative "contrib/ruby_openai/integration" require_relative "contrib/ruby_llm/integration" require_relative "contrib/anthropic/integration" +require_relative "contrib/llm_rb/integration" # Register integrations Braintrust::Contrib::OpenAI::Integration.register! Braintrust::Contrib::RubyOpenAI::Integration.register! Braintrust::Contrib::RubyLLM::Integration.register! Braintrust::Contrib::Anthropic::Integration.register! +Braintrust::Contrib::LlmRb::Integration.register! diff --git a/lib/braintrust/contrib/llm_rb/instrumentation/common.rb b/lib/braintrust/contrib/llm_rb/instrumentation/common.rb new file mode 100644 index 00000000..badb248e --- /dev/null +++ b/lib/braintrust/contrib/llm_rb/instrumentation/common.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Braintrust + module Contrib + module LlmRb + module Instrumentation + # Common utilities for llm.rb instrumentation. + module Common + # Parse LLM::Usage into normalized Braintrust metrics. + # LLM::Usage has: input_tokens, output_tokens, reasoning_tokens, total_tokens + # + # @param usage [LLM::Usage, nil] usage struct from llm.rb response + # @return [Hash] normalized metrics for Braintrust + def self.parse_usage_tokens(usage) + return {} unless usage + + input = usage.respond_to?(:input_tokens) ? usage.input_tokens : nil + output = usage.respond_to?(:output_tokens) ? usage.output_tokens : nil + reasoning = usage.respond_to?(:reasoning_tokens) ? usage.reasoning_tokens : nil + total = usage.respond_to?(:total_tokens) ? usage.total_tokens : nil + + metrics = {} + metrics["prompt_tokens"] = input.to_i if input + metrics["completion_tokens"] = output.to_i if output + metrics["completion_reasoning_tokens"] = reasoning.to_i if reasoning && reasoning.to_i > 0 + + if metrics.key?("prompt_tokens") && metrics.key?("completion_tokens") + metrics["tokens"] = metrics["prompt_tokens"] + metrics["completion_tokens"] + elsif total + metrics["tokens"] = total.to_i + end + + metrics + end + end + end + end + end +end diff --git a/lib/braintrust/contrib/llm_rb/instrumentation/context.rb b/lib/braintrust/contrib/llm_rb/instrumentation/context.rb new file mode 100644 index 00000000..850beec6 --- /dev/null +++ b/lib/braintrust/contrib/llm_rb/instrumentation/context.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require "opentelemetry/sdk" +require "json" + +require_relative "common" +require_relative "../../support/otel" + +module Braintrust + module Contrib + module LlmRb + module Instrumentation + # Context instrumentation for llm.rb. + # Wraps LLM::Context#talk to create Braintrust spans for chat completions. + module Context + def self.included(base) + base.prepend(InstanceMethods) unless applied?(base) + end + + def self.applied?(base) + base.ancestors.include?(InstanceMethods) + end + + module InstanceMethods + # Wrap talk() to trace chat completions. + # Captures input messages, output, token usage, and timing. + # NOTE: super must be called from within this method (not a helper) + # because Ruby's super keyword resolves the method chain at the call site. + def talk(prompt, params = {}) + return super unless tracing_enabled? + + tracer = Braintrust::Contrib.tracer_for(self) + + tracer.in_span("llm_rb.chat") do |span| + # Capture inputs BEFORE calling super (before @messages is updated) + input_messages = build_input_messages(prompt, params) + Support::OTel.set_json_attr(span, "braintrust.input_json", input_messages) if input_messages.any? + + metadata = extract_metadata(params) + + begin + res = super(prompt, params) + + # Capture output from response + output = capture_output(res) + Support::OTel.set_json_attr(span, "braintrust.output_json", output) unless output.empty? + + # Update metadata with actual model from response + if res.respond_to?(:model) && res.model + metadata["model"] = res.model + end + Support::OTel.set_json_attr(span, "braintrust.metadata", metadata) + + # Capture token metrics + usage = res.respond_to?(:usage) ? res.usage : nil + metrics = Common.parse_usage_tokens(usage) + Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty? + + res + rescue => e + span.record_exception(e) + span.status = ::OpenTelemetry::Trace::Status.error("llm.rb error: #{e.message}") + raise + end + end + end + + private + + # Checks if tracing is enabled via Braintrust::Contrib context. + def tracing_enabled? + ctx = Braintrust::Contrib.context_for(self) + ctx&.[](:enabled) != false + end + + # Build input messages array from existing history + new prompt. + # Called BEFORE super so we capture the state before @messages is updated. + def build_input_messages(prompt, params) + existing = @messages.to_a.map { |m| format_message_for_input(m) } + + new_msgs = if defined?(::LLM::Prompt) && ::LLM::Prompt === prompt + prompt.to_a.map { |m| format_message_for_input(m) } + elsif prompt.is_a?(Array) + prompt.flat_map do |m| + if m.respond_to?(:role) + [format_message_for_input(m)] + else + [{"role" => "user", "content" => m.to_s}] + end + end + else + role = (params[:role] || @params[:role] || @llm.user_role).to_s + [{"role" => role, "content" => prompt.to_s}] + end + + existing + new_msgs + end + + # Format an LLM::Message into OpenAI-compatible hash. + def format_message_for_input(msg) + return {"role" => "user", "content" => msg.to_s} unless msg.respond_to?(:role) + + formatted = {"role" => msg.role.to_s} + + content = msg.content + content = content.to_s if content && !content.is_a?(String) + formatted["content"] = content + + # Tool calls on assistant messages + if msg.respond_to?(:extra) && (tcs = msg.extra&.tool_calls)&.respond_to?(:any?) && tcs.any? + formatted["tool_calls"] = tcs.map { |tc| format_tool_call_for_input(tc) } + formatted["content"] = nil + end + + formatted.compact + end + + # Format a tool call into OpenAI-compatible format. + def format_tool_call_for_input(tc) + id = tc.respond_to?(:[]) ? (tc["id"] || tc[:id]) : nil + name = tc.respond_to?(:[]) ? (tc["name"] || tc[:name]) : nil + args = tc.respond_to?(:[]) ? (tc["arguments"] || tc[:arguments]) : nil + args_str = args.is_a?(String) ? args : args.to_json + + { + "id" => id, + "type" => "function", + "function" => { + "name" => name, + "arguments" => args_str + } + }.compact + end + + # Extract metadata from the context (provider name, model). + def extract_metadata(params) + provider_name = @llm.respond_to?(:name) ? @llm.name.to_s : @llm.class.name.split("::").last.downcase + merged = @params.merge(params) + model = merged[:model] + + { + "provider" => "llm_rb", + "llm_provider" => provider_name, + "model" => model + }.compact + end + + # Capture output messages from the response. + def capture_output(res) + return [] unless res.respond_to?(:choices) + + res.choices.map { |msg| format_message_for_input(msg) } + rescue + [] + end + end + end + end + end + end +end diff --git a/lib/braintrust/contrib/llm_rb/integration.rb b/lib/braintrust/contrib/llm_rb/integration.rb new file mode 100644 index 00000000..f6cfb1b2 --- /dev/null +++ b/lib/braintrust/contrib/llm_rb/integration.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require_relative "../integration" + +module Braintrust + module Contrib + module LlmRb + # llm.rb integration for automatic instrumentation. + # Instruments the 0x-r/llm.rb gem (gem name: "llm.rb"). + class Integration + include Braintrust::Contrib::Integration + + MINIMUM_VERSION = "4.11.0" + + GEM_NAMES = ["llm.rb"].freeze + REQUIRE_PATHS = ["llm"].freeze + + # @return [Symbol] Unique identifier for this integration + def self.integration_name + :llm_rb + end + + # @return [Array] Gem names this integration supports + def self.gem_names + GEM_NAMES + end + + # @return [Array] Require paths for auto-instrument detection + def self.require_paths + REQUIRE_PATHS + end + + # @return [String] Minimum compatible version + def self.minimum_version + MINIMUM_VERSION + end + + # @return [Boolean] true if llm.rb gem is available + def self.loaded? + defined?(::LLM::Context) ? true : false + end + + # Lazy-load the patcher only when actually patching. + # @return [Array] The patcher classes + def self.patchers + require_relative "patcher" + [ContextPatcher] + end + end + end + end +end diff --git a/lib/braintrust/contrib/llm_rb/patcher.rb b/lib/braintrust/contrib/llm_rb/patcher.rb new file mode 100644 index 00000000..3f619c6d --- /dev/null +++ b/lib/braintrust/contrib/llm_rb/patcher.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative "../patcher" +require_relative "instrumentation/context" + +module Braintrust + module Contrib + module LlmRb + # Patcher for llm.rb chat context. + # Instruments LLM::Context#talk to trace chat completions. + class ContextPatcher < Braintrust::Contrib::Patcher + class << self + def applicable? + defined?(::LLM::Context) + end + + def patched?(**options) + target_class = options[:target]&.singleton_class || ::LLM::Context + Instrumentation::Context.applied?(target_class) + end + + # Perform the actual patching. + # @param options [Hash] Configuration options + # @option options [LLM::Context] :target Optional context instance to patch + # @option options [OpenTelemetry::SDK::Trace::TracerProvider] :tracer_provider Optional + # @return [void] + def perform_patch(**options) + return unless applicable? + + if options[:target] + unless options[:target].is_a?(::LLM::Context) + raise ArgumentError, "target must be a kind of ::LLM::Context" + end + + options[:target].singleton_class.include(Instrumentation::Context) + else + ::LLM::Context.include(Instrumentation::Context) + end + end + end + end + end + end +end diff --git a/test/braintrust/contrib/llm_rb/instrumentation/context_test.rb b/test/braintrust/contrib/llm_rb/instrumentation/context_test.rb new file mode 100644 index 00000000..38c5065c --- /dev/null +++ b/test/braintrust/contrib/llm_rb/instrumentation/context_test.rb @@ -0,0 +1,253 @@ +# frozen_string_literal: true + +require "test_helper" +require_relative "../integration_helper" +require "braintrust/contrib/llm_rb/instrumentation/context" +require "braintrust/contrib/llm_rb/patcher" + +class Braintrust::Contrib::LlmRb::Instrumentation::ContextTest < Minitest::Test + include Braintrust::Contrib::LlmRb::IntegrationHelper + + def setup + skip_unless_llm_rb! + end + + # --- .included --- + + def test_included_prepends_instance_methods + mod = Braintrust::Contrib::LlmRb::Instrumentation::Context + + base = Class.new + instance_methods_included = false + base.define_singleton_method(:prepend) do |m| + instance_methods_included = true if m == mod::InstanceMethods + end + base.define_singleton_method(:ancestors) { [] } + + mod.included(base) + + assert instance_methods_included + end + + def test_included_skips_prepend_when_already_applied + base = Class.new do + include Braintrust::Contrib::LlmRb::Instrumentation::Context + end + + # Should not raise or double-prepend + Braintrust::Contrib::LlmRb::Instrumentation::Context.included(base) + + count = base.ancestors.count { |a| a == Braintrust::Contrib::LlmRb::Instrumentation::Context::InstanceMethods } + assert_equal 1, count + end + + # --- .applied? --- + + def test_applied_returns_false_when_not_included + base = Class.new + refute Braintrust::Contrib::LlmRb::Instrumentation::Context.applied?(base) + end + + def test_applied_returns_true_when_included + base = Class.new do + include Braintrust::Contrib::LlmRb::Instrumentation::Context + end + assert Braintrust::Contrib::LlmRb::Instrumentation::Context.applied?(base) + end +end + +# E2E tests for Context instrumentation +class Braintrust::Contrib::LlmRb::Instrumentation::ContextE2ETest < Minitest::Test + include Braintrust::Contrib::LlmRb::IntegrationHelper + + def setup + skip_unless_llm_rb! + # Reset the patcher state to avoid cross-test pollution + Braintrust::Contrib::LlmRb::ContextPatcher.reset! + end + + def teardown + Braintrust::Contrib::LlmRb::ContextPatcher.reset! + end + + def make_llm + LLM.openai(key: get_openai_key) + end + + # --- basic chat --- + + def test_talk_creates_span_with_correct_name + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + + res = ctx.talk("Say 'test'") + refute_nil res + + span = rig.drain_one + assert_equal "llm_rb.chat", span.name + end + end + + def test_talk_captures_input_json + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + ctx.talk("Say 'test'") + + span = rig.drain_one + assert span.attributes.key?("braintrust.input_json"), "Should have braintrust.input_json" + + input = JSON.parse(span.attributes["braintrust.input_json"]) + assert input.is_a?(Array) + assert_equal 1, input.length + assert_equal "user", input[0]["role"] + assert_equal "Say 'test'", input[0]["content"] + end + end + + def test_talk_captures_output_json + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + ctx.talk("Say 'test'") + + span = rig.drain_one + assert span.attributes.key?("braintrust.output_json"), "Should have braintrust.output_json" + + output = JSON.parse(span.attributes["braintrust.output_json"]) + assert output.is_a?(Array) + assert output.length > 0 + assert_equal "assistant", output[0]["role"] + refute_nil output[0]["content"] + end + end + + def test_talk_captures_metadata + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + ctx.talk("Say 'test'") + + span = rig.drain_one + assert span.attributes.key?("braintrust.metadata"), "Should have braintrust.metadata" + + metadata = JSON.parse(span.attributes["braintrust.metadata"]) + assert_equal "llm_rb", metadata["provider"] + assert_equal "openai", metadata["llm_provider"] + refute_nil metadata["model"] + end + end + + def test_talk_captures_token_metrics + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + ctx.talk("Say 'test'") + + span = rig.drain_one + assert span.attributes.key?("braintrust.metrics"), "Should have braintrust.metrics" + + metrics = JSON.parse(span.attributes["braintrust.metrics"]) + assert metrics["prompt_tokens"] > 0, "Should have prompt_tokens" + assert metrics["completion_tokens"] > 0, "Should have completion_tokens" + assert metrics["tokens"] > 0, "Should have total tokens" + end + end + + def test_talk_does_not_change_return_value + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + + res = ctx.talk("Say 'test'") + + refute_nil res + assert res.respond_to?(:choices), "Response should have choices method" + assert res.respond_to?(:usage), "Response should have usage method" + end + end + + # --- multi-turn conversation --- + + def test_multi_turn_includes_history_in_input + VCR.use_cassette("contrib/llm_rb/multi_turn_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + + ctx.talk("Hello") + ctx.talk("What is 2+2?") + + spans = rig.drain + assert_equal 2, spans.length + + # Second span should include the first turn's messages + second_span = spans.last + input = JSON.parse(second_span.attributes["braintrust.input_json"]) + assert input.length >= 3, "Second turn should include history: #{input.inspect}" + + roles = input.map { |m| m["role"] } + assert_includes roles, "user" + assert_includes roles, "assistant" + end + end + + # --- error handling --- + + def test_talk_records_exception_on_error + VCR.use_cassette("contrib/llm_rb/error_chat") do + rig = setup_otel_test_rig + llm = LLM.openai(key: "invalid-key") + ctx = LLM::Context.new(llm) + + Braintrust.instrument!(:llm_rb, target: ctx, tracer_provider: rig.tracer_provider) + + assert_raises(LLM::Error) { ctx.talk("Hello") } + + span = rig.drain_one + assert_equal "llm_rb.chat", span.name + + # Span should have error status + assert_equal ::OpenTelemetry::Trace::Status::ERROR, span.status.code + end + end + + # --- class-level vs instance-level patching --- + + def test_instance_level_patching_only_affects_target + VCR.use_cassette("contrib/llm_rb/basic_chat") do + rig = setup_otel_test_rig + llm = make_llm + ctx1 = LLM::Context.new(llm) + + # Only instrument ctx1 + Braintrust.instrument!(:llm_rb, target: ctx1, tracer_provider: rig.tracer_provider) + + ctx1.talk("Say 'test'") + + spans = rig.drain + assert_equal 1, spans.length, "Only instrumented context should produce spans" + end + end +end diff --git a/test/braintrust/contrib/llm_rb/integration_helper.rb b/test/braintrust/contrib/llm_rb/integration_helper.rb new file mode 100644 index 00000000..2b8f3420 --- /dev/null +++ b/test/braintrust/contrib/llm_rb/integration_helper.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Test helpers for llm.rb integration tests. +module Braintrust + module Contrib + module LlmRb + module IntegrationHelper + # Skip test unless llm.rb gem is available. + def skip_unless_llm_rb! + unless Gem.loaded_specs["llm.rb"] + skip "llm.rb gem not available" + end + + require "llm" unless defined?(::LLM) + end + end + end + end +end diff --git a/test/braintrust/contrib/llm_rb/integration_test.rb b/test/braintrust/contrib/llm_rb/integration_test.rb new file mode 100644 index 00000000..146b1c3a --- /dev/null +++ b/test/braintrust/contrib/llm_rb/integration_test.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "test_helper" +require_relative "integration_helper" + +class Braintrust::Contrib::LlmRb::IntegrationTest < Minitest::Test + include Braintrust::Contrib::LlmRb::IntegrationHelper + + def test_integration_name + assert_equal :llm_rb, Braintrust::Contrib::LlmRb::Integration.integration_name + end + + def test_gem_names + assert_equal ["llm.rb"], Braintrust::Contrib::LlmRb::Integration.gem_names + end + + def test_require_paths + assert_equal ["llm"], Braintrust::Contrib::LlmRb::Integration.require_paths + end + + def test_minimum_version + assert_equal "4.11.0", Braintrust::Contrib::LlmRb::Integration.minimum_version + end + + def test_loaded_returns_true_when_llm_context_defined + skip_unless_llm_rb! + assert Braintrust::Contrib::LlmRb::Integration.loaded? + end + + def test_loaded_returns_false_when_llm_context_not_defined + # Simulate unloaded state + was_defined = defined?(::LLM::Context) + if was_defined + skip "Cannot undefine LLM::Context in this test environment" + end + refute Braintrust::Contrib::LlmRb::Integration.loaded? + end + + def test_integration_registered_in_registry + registry = Braintrust::Contrib::Registry.instance + integration = registry[:llm_rb] + refute_nil integration, "Expected :llm_rb integration to be registered" + assert_equal :llm_rb, integration.integration_name + end + + def test_five_integrations_now_registered + names = Braintrust::Contrib::Registry.instance.all.map(&:integration_name) + assert_includes names, :llm_rb, "Expected :llm_rb to be in registry" + assert_includes names, :openai + assert_includes names, :anthropic + assert_includes names, :ruby_llm + assert_includes names, :ruby_openai + end + + def test_require_path_llm_detected + skip_unless_llm_rb! + registry = Braintrust::Contrib::Registry.instance + matches = registry.integrations_for_require_path("llm") + assert_includes matches.map(&:integration_name), :llm_rb + end +end diff --git a/test/braintrust/contrib/llm_rb/patcher_test.rb b/test/braintrust/contrib/llm_rb/patcher_test.rb new file mode 100644 index 00000000..a2732ecb --- /dev/null +++ b/test/braintrust/contrib/llm_rb/patcher_test.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require "test_helper" +require_relative "integration_helper" +require "braintrust/contrib/llm_rb/patcher" + +class Braintrust::Contrib::LlmRb::ContextPatcherTest < Minitest::Test + include Braintrust::Contrib::LlmRb::IntegrationHelper + + def setup + skip_unless_llm_rb! + end + + def test_applicable_when_llm_context_defined + assert Braintrust::Contrib::LlmRb::ContextPatcher.applicable? + end + + # --- .patched? --- + + def test_patched_returns_false_for_unpatched_class + fake_ctx_class = Class.new + + ::LLM.stub_const(:Context, fake_ctx_class) do + refute Braintrust::Contrib::LlmRb::ContextPatcher.patched? + end + end + + def test_patched_returns_true_when_module_included + fake_ctx_class = Class.new do + include Braintrust::Contrib::LlmRb::Instrumentation::Context + end + + ::LLM.stub_const(:Context, fake_ctx_class) do + assert Braintrust::Contrib::LlmRb::ContextPatcher.patched? + end + end + + def test_patched_returns_false_for_unpatched_instance + fake_singleton = Class.new + + mock_chain(:singleton_class, returns: fake_singleton) do |ctx| + refute Braintrust::Contrib::LlmRb::ContextPatcher.patched?(target: ctx) + end + end + + def test_patched_returns_true_for_patched_instance + fake_singleton = Class.new do + include Braintrust::Contrib::LlmRb::Instrumentation::Context + end + + mock_chain(:singleton_class, returns: fake_singleton) do |ctx| + assert Braintrust::Contrib::LlmRb::ContextPatcher.patched?(target: ctx) + end + end + + # --- .perform_patch --- + + def test_perform_patch_includes_module_for_class_level + fake_ctx_class = Minitest::Mock.new + fake_ctx_class.expect(:include, true, [Braintrust::Contrib::LlmRb::Instrumentation::Context]) + + ::LLM.stub_const(:Context, fake_ctx_class) do + Braintrust::Contrib::LlmRb::ContextPatcher.perform_patch + fake_ctx_class.verify + end + end + + def test_perform_patch_includes_module_for_instance_level + terminal = Minitest::Mock.new + terminal.expect(:include, true, [Braintrust::Contrib::LlmRb::Instrumentation::Context]) + + mock_chain(:singleton_class, returns: terminal) do |ctx| + ctx.expect(:is_a?, true, [::LLM::Context]) + Braintrust::Contrib::LlmRb::ContextPatcher.perform_patch(target: ctx) + end + terminal.verify + end + + def test_perform_patch_raises_for_invalid_target + fake_ctx = Minitest::Mock.new + fake_ctx.expect(:is_a?, false, [::LLM::Context]) + + assert_raises(ArgumentError) do + Braintrust::Contrib::LlmRb::ContextPatcher.perform_patch(target: fake_ctx) + end + + fake_ctx.verify + end +end diff --git a/test/braintrust/contrib/llm_rb_gap_test.rb b/test/braintrust/contrib/llm_rb_gap_test.rb new file mode 100644 index 00000000..0130a4c1 --- /dev/null +++ b/test/braintrust/contrib/llm_rb_gap_test.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require "test_helper" + +# Historical note: This file previously tested the GAP described in issue #140 +# where the llm.rb gem had no Braintrust integration. +# +# The integration has since been implemented as :llm_rb. These tests now verify +# that the integration is registered and working correctly at the registry level. +class Braintrust::Contrib::LlmRbRegistryTest < Minitest::Test + def setup + @registry = Braintrust::Contrib::Registry.instance + end + + def test_integration_registered_for_llm_rb + assert_equal :llm_rb, @registry[:llm_rb]&.integration_name, + "Expected :llm_rb integration to be registered" + end + + def test_five_integrations_registered + names = @registry.all.map(&:integration_name) + assert_equal [:openai, :ruby_openai, :ruby_llm, :anthropic, :llm_rb], names, + "Expected exactly 5 integrations: openai, ruby_openai, ruby_llm, anthropic, llm_rb" + end + + def test_require_path_llm_detected + matches = @registry.integrations_for_require_path("llm") + names = matches.map(&:integration_name) + assert_includes names, :llm_rb, + "Expected :llm_rb to be detected when require path 'llm' is seen" + end + + def test_instrument_llm_rb_returns_truthy_when_loaded + skip "llm.rb gem not available" unless Gem.loaded_specs["llm.rb"] + require "llm" unless defined?(::LLM) + + result = Braintrust::Contrib.instrument!(:llm_rb) + assert result, "Expected instrument!(:llm_rb) to return truthy when gem is loaded" + end +end diff --git a/test/braintrust/contrib/llm_rb_spans_test.rb b/test/braintrust/contrib/llm_rb_spans_test.rb new file mode 100644 index 00000000..079218b9 --- /dev/null +++ b/test/braintrust/contrib/llm_rb_spans_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "test_helper" + +# Verifies that after instrumenting with :llm_rb, LLM::Context#talk produces +# Braintrust-enriched spans. Companion to the integration tests in +# test/braintrust/contrib/llm_rb/. +class Braintrust::Contrib::LlmRbSpansTest < Minitest::Test + def setup + @llm_available = begin + require "llm" + true + rescue LoadError + false + end + end + + def teardown + # Reset patcher state after each test + Braintrust::Contrib::LlmRb::ContextPatcher.reset! if defined?(Braintrust::Contrib::LlmRb::ContextPatcher) + end + + def test_llm_rb_context_patched_after_instrument + skip "llm.rb gem not available" unless @llm_available + + ancestors_before = ::LLM::Context.ancestors.map(&:to_s) + bt_before = ancestors_before.select { |a| a.include?("Braintrust") } + assert_empty bt_before, "LLM::Context should start unpatched" + + Braintrust.instrument!(:llm_rb) + + ancestors_after = ::LLM::Context.ancestors.map(&:to_s) + bt_after = ancestors_after.select { |a| a.include?("Braintrust") } + refute_empty bt_after, "LLM::Context should be patched after instrument!(:llm_rb)" + end + + def test_llm_rb_loaded_detected_by_llm_rb_integration + skip "llm.rb gem not available" unless @llm_available + + assert defined?(::LLM::Context), "LLM::Context should be defined" + + integration = Braintrust::Contrib::Registry.instance[:llm_rb] + refute_nil integration + assert integration.loaded?, "llm_rb integration should report loaded? = true when llm.rb is loaded" + end +end diff --git a/test/fixtures/vcr_cassettes/AGENTS.md b/test/fixtures/vcr_cassettes/AGENTS.md new file mode 100644 index 00000000..2cef3779 --- /dev/null +++ b/test/fixtures/vcr_cassettes/AGENTS.md @@ -0,0 +1,49 @@ +# VCR Cassettes + +This directory contains VCR cassette files that record and replay HTTP interactions for tests. + +## Do NOT hand-craft cassette YAML files + +Cassette files must be **recorded from real API responses**, not written by hand. Hand-crafted cassettes contain plausible but fake data — they produce tests that pass locally against invented responses but silently diverge from what real APIs actually return. This defeats the purpose of integration testing. + +## How to record cassettes + +You need a real API key in the environment. Check `.env.example` for which keys are needed. + +```bash +# Record cassettes for a specific appraisal + test file (fastest) +VCR_MODE=all bundle exec appraisal ruby -Ilib:test test/braintrust/contrib//instrumentation/..._test.rb + +# Re-record all cassettes for an appraisal scenario +VCR_MODE=all bundle exec appraisal rake test + +# Record only new cassettes, keep existing ones +VCR_MODE=new_episodes bundle exec appraisal rake test +``` + +VCR filters sensitive data automatically (see `test/test_helper.rb` — API keys are replaced with placeholder strings before saving). + +## Cassette lifecycle + +| Mode | Behavior | +|------|----------| +| `:once` (default) | Record if missing, replay if present. Never re-records. | +| `VCR_MODE=all` | Always re-record, overwriting existing cassettes. | +| `VCR_MODE=new_episodes` | Record new interactions, replay existing ones. | +| `VCR_OFF=true` | Disable VCR entirely — makes real HTTP requests every run. | + +## Directory structure + +Cassettes mirror the test directory structure under `contrib/`: + +``` +vcr_cassettes/ + contrib/ + openai/ + anthropic/ + ruby_llm/ + llm_rb/ + ... +``` + +Each subdirectory corresponds to one integration. Cassette filenames match the `VCR.use_cassette(...)` call in the test. diff --git a/test/fixtures/vcr_cassettes/contrib/llm_rb/basic_chat.yml b/test/fixtures/vcr_cassettes/contrib/llm_rb/basic_chat.yml new file mode 100644 index 00000000..d33fc039 --- /dev/null +++ b/test/fixtures/vcr_cassettes/contrib/llm_rb/basic_chat.yml @@ -0,0 +1,117 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"messages":[{"role":"user","content":[{"type":"text","text":"Say ''test''"}]}],"model":"gpt-4.1"}' + headers: + User-Agent: + - llm.rb v4.11.0 + Content-Type: + - application/json + Authorization: + - Bearer + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + Transfer-Encoding: + - chunked + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 06 Apr 2026 14:42:52 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Cf-Ray: + - 9e818fc49be7e88e-CMH + Cf-Cache-Status: + - DYNAMIC + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - braintrust-data + Openai-Processing-Ms: + - '508' + Openai-Project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + Openai-Version: + - '2020-10-01' + X-Openai-Proxy-Wasm: + - v0.1 + X-Ratelimit-Limit-Requests: + - '10000' + X-Ratelimit-Limit-Tokens: + - '30000000' + X-Ratelimit-Remaining-Requests: + - '9999' + X-Ratelimit-Remaining-Tokens: + - '29999995' + X-Ratelimit-Reset-Requests: + - 6ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - req_7da6782a9dbe46aab9017262ed02b1e4 + Set-Cookie: + - __cf_bm=H7n6bGyaiH7gcjN2_RElDpkkNxioD022bA_rYWNOsig-1775486572.258481-1.0.1.1-4RVkAPN7ytRr9Id7pFPLpEj1lS7GDo13VfhPNSYuxuumezOTWi5WptCU0BINhIZdacya3uSxxWlslYD5RxwkiIj07DnKRkCCNUEoYpozuvfFDtDdlKtb3gDISE_vADGO; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Mon, 06 Apr 2026 + 15:12:52 GMT + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-DRfS8YJp2wVWLuHaPm9ebOgzoUP4N", + "object": "chat.completion", + "created": 1775486572, + "model": "gpt-4.1-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "test", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 11, + "completion_tokens": 1, + "total_tokens": 12, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_15eef0a82c" + } + recorded_at: Mon, 06 Apr 2026 14:42:52 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/contrib/llm_rb/error_chat.yml b/test/fixtures/vcr_cassettes/contrib/llm_rb/error_chat.yml new file mode 100644 index 00000000..ff5f7d30 --- /dev/null +++ b/test/fixtures/vcr_cassettes/contrib/llm_rb/error_chat.yml @@ -0,0 +1,71 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"messages":[{"role":"user","content":[{"type":"text","text":"Hello"}]}],"model":"gpt-4.1"}' + headers: + User-Agent: + - llm.rb v4.11.0 + Content-Type: + - application/json + Authorization: + - Bearer invalid-key + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + Transfer-Encoding: + - chunked + response: + status: + code: 401 + message: Unauthorized + headers: + Date: + - Mon, 06 Apr 2026 14:42:48 GMT + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '261' + Connection: + - keep-alive + Set-Cookie: + - __cf_bm=UYoIffiH.sMPnPvXAWP6xcVOUVQZO45gC8teTA2zmNs-1775486568.1962547-1.0.1.1-VD0Gc.D53tPjKBVxx33zayJnKj3bAzN3K.En1GXQ4xttLOE_UcTjuxyGixRUGEU30rqd4vK_JZ.f9.6ocVq.8OfxFSh0p_vx2dcKaPufq1qXMt7cfnHxwurLSvpOzYns; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Mon, 06 Apr 2026 + 15:12:48 GMT + - _cfuvid=_6V5phFjZ.johROOHDd2co64NIKrbV0h8IsxTu9lJxI-1775486568.1962547-1.0.1.1-o32CYPo8.dH0vrHfMne3cFWc5.pi3e5MUSVCxQd72iM; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com + Cf-Ray: + - 9e818fab3d91cf53-CMH + Cf-Cache-Status: + - DYNAMIC + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Openai-Proxy-Wasm: + - v0.1 + X-Request-Id: + - req_1ce40bc30b34431b905892f98c478aee + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: UTF-8 + string: | + { + "error": { + "message": "Incorrect API key provided: invalid-key. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": null, + "code": "invalid_api_key" + } + } + recorded_at: Mon, 06 Apr 2026 14:42:48 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/contrib/llm_rb/multi_turn_chat.yml b/test/fixtures/vcr_cassettes/contrib/llm_rb/multi_turn_chat.yml new file mode 100644 index 00000000..5474df89 --- /dev/null +++ b/test/fixtures/vcr_cassettes/contrib/llm_rb/multi_turn_chat.yml @@ -0,0 +1,233 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"messages":[{"role":"user","content":[{"type":"text","text":"Hello"}]}],"model":"gpt-4.1"}' + headers: + User-Agent: + - llm.rb v4.11.0 + Content-Type: + - application/json + Authorization: + - Bearer + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + Transfer-Encoding: + - chunked + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 06 Apr 2026 14:42:49 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Cf-Ray: + - 9e818fac1ad5725b-CMH + Cf-Cache-Status: + - DYNAMIC + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - braintrust-data + Openai-Processing-Ms: + - '571' + Openai-Project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + Openai-Version: + - '2020-10-01' + X-Openai-Proxy-Wasm: + - v0.1 + X-Ratelimit-Limit-Requests: + - '10000' + X-Ratelimit-Limit-Tokens: + - '30000000' + X-Ratelimit-Remaining-Requests: + - '9999' + X-Ratelimit-Remaining-Tokens: + - '29999996' + X-Ratelimit-Reset-Requests: + - 6ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - req_6492da2b8a9c451fa125d54363b2face + Set-Cookie: + - __cf_bm=ckT_76wYC4uBSf6xOgKgHN5OozxWjqbHQqcwlbidFjs-1775486568.340096-1.0.1.1-uvjPuBCgMmYtovE9ibMmkEdjXezwduySN1oT2CL2C0KzSYbfPr9qrZ2F3n8xBRsd4D0HaleeIFJ2ZtWFeFJ8mnhUNNk3ON_lOQUdkioY.dTYciD2bVWMO716rThcIXai; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Mon, 06 Apr 2026 + 15:12:49 GMT + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-DRfS4lpSqxLALYUXhAGGgDCAgqkkp", + "object": "chat.completion", + "created": 1775486568, + "model": "gpt-4.1-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I help you today?", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 9, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_15eef0a82c" + } + recorded_at: Mon, 06 Apr 2026 14:42:49 GMT +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"messages":[{"role":"user","content":[{"type":"text","text":"Hello"}]},{"role":"assistant","content":[{"type":"text","text":"Hello! + How can I help you today?"}]},{"role":"user","content":[{"type":"text","text":"What + is 2+2?"}]}],"model":"gpt-4.1"}' + headers: + User-Agent: + - llm.rb v4.11.0 + Content-Type: + - application/json + Authorization: + - Bearer + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + Transfer-Encoding: + - chunked + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 06 Apr 2026 14:42:49 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Cf-Ray: + - 9e818fb0792bcf7f-CMH + Cf-Cache-Status: + - DYNAMIC + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - braintrust-data + Openai-Processing-Ms: + - '623' + Openai-Project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + Openai-Version: + - '2020-10-01' + X-Openai-Proxy-Wasm: + - v0.1 + X-Ratelimit-Limit-Requests: + - '10000' + X-Ratelimit-Limit-Tokens: + - '30000000' + X-Ratelimit-Remaining-Requests: + - '9999' + X-Ratelimit-Remaining-Tokens: + - '29999983' + X-Ratelimit-Reset-Requests: + - 6ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - req_4b914bd2eb8f469083919be04d449722 + Set-Cookie: + - __cf_bm=pgSrJEY0MENcnAseVwcANxsnrWBnEsVsbDjLp8zC2W0-1775486569.0400677-1.0.1.1-bngLZ9x2VKgP6truHOZNB8iDK40kV2lVDZgw3ac.Jc0IACWhH8Pl5bpxNkNCz6mZIGn3kr8Bqi36bPF7vS3YqCMMrL2e.j2pWaiS7gEF.eq7SnZ0U6.om2ypK_EF8F3p; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Mon, 06 Apr 2026 + 15:12:49 GMT + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-DRfS5WcOW78aGtv0zoiueODSNUDK1", + "object": "chat.completion", + "created": 1775486569, + "model": "gpt-4.1-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "2 + 2 = 4", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 32, + "completion_tokens": 7, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_15eef0a82c" + } + recorded_at: Mon, 06 Apr 2026 14:42:49 GMT +recorded_with: VCR 6.4.0