diff --git a/lib/chat_models/chat_anthropic.ex b/lib/chat_models/chat_anthropic.ex index 8ef40d05..3481df36 100644 --- a/lib/chat_models/chat_anthropic.ex +++ b/lib/chat_models/chat_anthropic.ex @@ -1463,8 +1463,6 @@ defmodule LangChain.ChatModels.ChatAnthropic do }) end - defp get_token_usage(_usage_data), do: nil - @doc """ Generate a config map that can later restore the model's configuration. """ diff --git a/lib/chat_models/chat_google_ai.ex b/lib/chat_models/chat_google_ai.ex index 355c5ee2..37addc4a 100644 --- a/lib/chat_models/chat_google_ai.ex +++ b/lib/chat_models/chat_google_ai.ex @@ -187,8 +187,18 @@ defmodule LangChain.ChatModels.ChatGoogleAI do nil -> nil - %Message{role: :system, content: content} -> + %Message{role: :system, content: content} when is_binary(content) -> %{"parts" => [%{"text" => content}]} + + %Message{role: :system, content: content} when is_list(content) -> + # Extract text from ContentPart structures + text_content = + content + |> Enum.filter(&match?(%ContentPart{type: :text}, &1)) + |> Enum.map(& &1.content) + |> Enum.join(" ") + + %{"parts" => [%{"text" => text_content}]} end messages_for_api = diff --git a/lib/chat_models/chat_mistral_ai.ex b/lib/chat_models/chat_mistral_ai.ex index 55e36300..07b33257 100644 --- a/lib/chat_models/chat_mistral_ai.ex +++ b/lib/chat_models/chat_mistral_ai.ex @@ -56,6 +56,12 @@ defmodule LangChain.ChatModels.ChatMistralAI do # For choosing a specific tool call (like forcing a function execution). field :tool_choice, :map + # JSON Schema to validate the output format (for structured JSON output) + field :json_schema, :map + + # Whether to force a JSON response format + field :json_response, :boolean, default: false + # A list of callback handlers field :callbacks, {:array, :map}, default: [] end @@ -73,7 +79,9 @@ defmodule LangChain.ChatModels.ChatMistralAI do :safe_prompt, :random_seed, :stream, - :tool_choice + :tool_choice, + :json_schema, + :json_response ] @required_fields [ :model @@ -126,6 +134,38 @@ defmodule LangChain.ChatModels.ChatMistralAI do |> Utils.conditionally_add_to_map(:max_tokens, mistral.max_tokens) |> Utils.conditionally_add_to_map(:tools, get_tools_for_api(mistral, tools)) |> Utils.conditionally_add_to_map(:tool_choice, get_tool_choice(mistral)) + |> Utils.conditionally_add_to_map(:response_format, set_response_format(mistral)) + end + + # Creates the response_format field for JSON output when json_response is true. + # If json_schema is provided, it will be included in the response format. + # + # For Mistral, the format is as follows: + # https://docs.mistral.ai/capabilities/structured-output/custom_structured_output/ + # { + # "type": "json_schema", + # "json_schema": { + # "schema": { ... }, + # "name": "output", + # "strict": true + # } + # } + @spec set_response_format(t()) :: map() | nil + defp set_response_format(%ChatMistralAI{json_response: true, json_schema: schema}) + when is_map(schema) and map_size(schema) > 0 do + # The schema should already be in the correct format + schema + end + + defp set_response_format(%ChatMistralAI{json_response: true}) do + # For Mistral, when no schema is provided, we use json_object type + %{ + "type" => "json_object" + } + end + + defp set_response_format(%ChatMistralAI{}) do + nil end # Add a more complete function to map tools. This mirrors ChatOpenAI approach. @@ -179,23 +219,50 @@ defmodule LangChain.ChatModels.ChatMistralAI do def for_api(%_{} = model, %Message{role: :assistant, tool_calls: tool_calls} = msg) when is_list(tool_calls) do + content = + case msg.content do + content when is_binary(content) -> content + content when is_list(content) -> ContentPart.parts_to_string(content) + nil -> nil + end + %{ "role" => :assistant, - "content" => msg.content + "content" => content } |> Utils.conditionally_add_to_map("tool_calls", Enum.map(tool_calls, &for_api(model, &1))) end - def for_api(%_{} = model, %Message{role: :user, content: content} = msg) + def for_api(%_{} = _model, %Message{role: :user, content: content} = msg) when is_list(content) do # A user message can hold an array of ContentParts %{ "role" => msg.role, - "content" => Enum.map(content, &for_api(model, &1)) + "content" => ContentPart.parts_to_string(content) } |> Utils.conditionally_add_to_map("name", msg.name) end + # Handle messages with ContentPart content for non-user roles + def for_api(%_{} = model, %Message{content: content} = msg) when is_list(content) do + role = get_message_role(model, msg.role) + + %{ + "role" => role, + "content" => ContentPart.parts_to_string(content) + } + |> Utils.conditionally_add_to_map("name", msg.name) + |> Utils.conditionally_add_to_map( + "tool_calls", + Enum.map(msg.tool_calls || [], &for_api(model, &1)) + ) + end + + # Handle ContentPart structures + def for_api(%_{} = _model, %ContentPart{type: :text, content: content}) do + content + end + # ToolResult => stand-alone message with "role: :tool" def for_api(%_{} = _model, %ToolResult{type: :function} = result) do %{ @@ -649,7 +716,9 @@ defmodule LangChain.ChatModels.ChatMistralAI do :max_tokens, :safe_prompt, :random_seed, - :stream + :stream, + :json_schema, + :json_response ], @current_config_version ) diff --git a/lib/chat_models/chat_ollama_ai.ex b/lib/chat_models/chat_ollama_ai.ex index 8a1196d4..f97a16bc 100644 --- a/lib/chat_models/chat_ollama_ai.ex +++ b/lib/chat_models/chat_ollama_ai.ex @@ -42,6 +42,7 @@ defmodule LangChain.ChatModels.ChatOllamaAI do alias LangChain.ChatModels.ChatModel alias LangChain.ChatModels.ChatOpenAI alias LangChain.Message + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.Message.ToolResult alias LangChain.MessageDelta @@ -242,9 +243,16 @@ defmodule LangChain.ChatModels.ChatOllamaAI do def for_api(%Message{role: :assistant, tool_calls: tool_calls} = msg) when is_list(tool_calls) do + content = + case msg.content do + content when is_binary(content) -> content + content when is_list(content) -> ContentPart.parts_to_string(content) + nil -> nil + end + %{ "role" => :assistant, - "content" => msg.content + "content" => content } |> Utils.conditionally_add_to_map("tool_calls", Enum.map(tool_calls, &for_api(&1))) end @@ -292,11 +300,25 @@ defmodule LangChain.ChatModels.ChatOllamaAI do def for_api(%Message{role: :user, content: content} = msg) when is_list(content) do %{ "role" => msg.role, - "content" => Enum.map(content, &for_api(&1)) + "content" => ContentPart.parts_to_string(content) + } + |> Utils.conditionally_add_to_map("name", msg.name) + end + + # Handle messages with ContentPart content for non-user roles + def for_api(%Message{content: content} = msg) when is_list(content) do + %{ + "role" => msg.role, + "content" => ContentPart.parts_to_string(content) } |> Utils.conditionally_add_to_map("name", msg.name) end + # Handle ContentPart structures + def for_api(%ContentPart{type: :text, content: content}) do + content + end + defp get_tools_for_api(nil), do: [] defp get_tools_for_api(tools) do diff --git a/lib/chat_models/chat_open_ai.ex b/lib/chat_models/chat_open_ai.ex index 6084ce38..66409403 100644 --- a/lib/chat_models/chat_open_ai.ex +++ b/lib/chat_models/chat_open_ai.ex @@ -536,6 +536,11 @@ defmodule LangChain.ChatModels.ChatOpenAI do raise LangChainError, "PromptTemplates must be converted to messages." end + # Handle ContentPart structures directly + def for_api(%_{} = model, %ContentPart{} = part) do + content_part_for_api(model, part) + end + @doc """ Convert a list of ContentParts to the expected map of data for the OpenAI API. """ diff --git a/lib/chat_models/chat_perplexity.ex b/lib/chat_models/chat_perplexity.ex index 046c0428..5b18e0a3 100644 --- a/lib/chat_models/chat_perplexity.ex +++ b/lib/chat_models/chat_perplexity.ex @@ -45,6 +45,7 @@ defmodule LangChain.ChatModels.ChatPerplexity do alias LangChain.ChatModels.ChatModel alias LangChain.Message alias LangChain.MessageDelta + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.TokenUsage alias LangChain.LangChainError @@ -283,9 +284,16 @@ defmodule LangChain.ChatModels.ChatPerplexity do """ @spec for_api(t(), Message.t()) :: %{String.t() => any()} def for_api(%ChatPerplexity{}, %Message{} = msg) do + content = + case msg.content do + content when is_binary(content) -> content + content when is_list(content) -> ContentPart.parts_to_string(content) + nil -> nil + end + %{ "role" => msg.role, - "content" => msg.content + "content" => content } end diff --git a/lib/chat_models/chat_vertex_ai.ex b/lib/chat_models/chat_vertex_ai.ex index ab2a464c..25112eb6 100644 --- a/lib/chat_models/chat_vertex_ai.ex +++ b/lib/chat_models/chat_vertex_ai.ex @@ -216,20 +216,26 @@ defmodule LangChain.ChatModels.ChatVertexAI do end defp for_api(%Message{role: :system} = message) do - %{"parts" => %{"text" => message.content}} + # System messages should return a single text part, not a list + case get_message_contents(message) do + [%{"text" => text}] -> %{"parts" => %{"text" => text}} + _ -> %{"parts" => %{"text" => message.content}} + end end defp for_api(%Message{role: :user, content: content}) when is_list(content) do %{ - "role" => "user", + "role" => map_role(:user), "parts" => Enum.map(content, &for_api(&1)) } end defp for_api(%Message{} = message) do + content_parts = get_message_contents(message) || [] + %{ "role" => map_role(message.role), - "parts" => [%{"text" => message.content}] + "parts" => content_parts } end @@ -257,9 +263,9 @@ defmodule LangChain.ChatModels.ChatVertexAI do defp for_api(%ContentPart{type: :file_url} = part) do %{ - "file_data" => %{ + "fileData" => %{ "mimeType" => Keyword.fetch!(part.options, :media), - "file_uri" => part.content + "fileUri" => part.content } } end diff --git a/test/chat_models/chat_mistral_ai_test.exs b/test/chat_models/chat_mistral_ai_test.exs index f729a75e..50b736c5 100644 --- a/test/chat_models/chat_mistral_ai_test.exs +++ b/test/chat_models/chat_mistral_ai_test.exs @@ -4,6 +4,7 @@ defmodule LangChain.ChatModels.ChatMistralAITest do alias LangChain.ChatModels.ChatMistralAI alias LangChain.Message alias LangChain.MessageDelta + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.LangChainError alias LangChain.TokenUsage @@ -113,7 +114,7 @@ defmodule LangChain.ChatModels.ChatMistralAITest do assert [%Message{} = msg] = ChatMistralAI.do_process_response(model, response) assert msg.role == :assistant - assert msg.content == "Hello User!" + assert msg.content == [ContentPart.text!("Hello User!")] assert msg.index == 0 assert msg.status == :complete end @@ -315,8 +316,8 @@ defmodule LangChain.ChatModels.ChatMistralAITest do result = ChatMistralAI.do_process_response(model, response) - assert [%Message{role: :assistant, content: "Hello from Mistral!", status: :complete}] = - result + assert [%Message{role: :assistant, status: :complete} = message] = result + assert message.content == [ContentPart.text!("Hello from Mistral!")] end end @@ -353,7 +354,9 @@ defmodule LangChain.ChatModels.ChatMistralAITest do "random_seed" => 42, "safe_prompt" => true, "top_p" => 1.0, - "version" => 1 + "version" => 1, + "json_response" => false, + "json_schema" => nil } end end @@ -369,6 +372,160 @@ defmodule LangChain.ChatModels.ChatMistralAITest do end end + describe "structured output support" do + test "new/1 accepts json_response and json_schema fields" do + schema = %{ + "type" => "json_schema", + "json_schema" => %{ + "schema" => %{ + "properties" => %{ + "name" => %{"title" => "Name", "type" => "string"}, + "authors" => %{ + "items" => %{"type" => "string"}, + "title" => "Authors", + "type" => "array" + } + }, + "required" => ["name", "authors"], + "title" => "Book", + "type" => "object", + "additionalProperties" => false + }, + "name" => "book", + "strict" => true + } + } + + assert {:ok, %ChatMistralAI{} = mistral_ai} = + ChatMistralAI.new(%{ + "model" => "ministral-8b-latest", + "json_response" => true, + "json_schema" => schema + }) + + assert mistral_ai.json_response == true + assert mistral_ai.json_schema == schema + end + + test "for_api/3 includes response_format when json_response is true with schema" do + schema = %{ + "type" => "json_schema", + "json_schema" => %{ + "schema" => %{ + "properties" => %{ + "name" => %{"title" => "Name", "type" => "string"}, + "authors" => %{ + "items" => %{"type" => "string"}, + "title" => "Authors", + "type" => "array" + } + }, + "required" => ["name", "authors"], + "title" => "Book", + "type" => "object", + "additionalProperties" => false + }, + "name" => "book", + "strict" => true + } + } + + mistral_ai = + ChatMistralAI.new!(%{ + "model" => "ministral-8b-latest", + "json_response" => true, + "json_schema" => schema + }) + + data = ChatMistralAI.for_api(mistral_ai, [], []) + + assert data.response_format == schema + end + + test "for_api/3 includes response_format when json_response is true without schema" do + mistral_ai = + ChatMistralAI.new!(%{ + "model" => "ministral-8b-latest", + "json_response" => true + }) + + data = ChatMistralAI.for_api(mistral_ai, [], []) + + assert data.response_format == %{"type" => "json_object"} + end + + test "for_api/3 does not include response_format when json_response is false" do + mistral_ai = + ChatMistralAI.new!(%{ + "model" => "ministral-8b-latest", + "json_response" => false + }) + + data = ChatMistralAI.for_api(mistral_ai, [], []) + + refute Map.has_key?(data, :response_format) + end + + test "serialize_config/1 includes json_response and json_schema fields" do + schema = %{ + "type" => "json_schema", + "json_schema" => %{ + "schema" => %{ + "properties" => %{ + "name" => %{"title" => "Name", "type" => "string"} + }, + "required" => ["name"], + "title" => "Book", + "type" => "object" + }, + "name" => "book", + "strict" => true + } + } + + model = + ChatMistralAI.new!(%{ + "model" => "ministral-8b-latest", + "json_response" => true, + "json_schema" => schema + }) + + result = ChatMistralAI.serialize_config(model) + + assert result["json_response"] == true + assert result["json_schema"] == schema + end + + test "restore_from_map/1 restores json_response and json_schema fields" do + schema = %{ + "type" => "json_schema", + "json_schema" => %{ + "schema" => %{ + "properties" => %{ + "name" => %{"title" => "Name", "type" => "string"} + }, + "required" => ["name"], + "title" => "Book", + "type" => "object" + }, + "name" => "book", + "strict" => true + } + } + + config = %{ + "version" => 1, + "model" => "ministral-8b-latest", + "json_response" => true, + "json_schema" => schema + } + + assert {:ok, %ChatMistralAI{} = restored} = ChatMistralAI.restore_from_map(config) + assert restored.json_response == true + assert restored.json_schema == schema + end + end + describe "live tests and token usage information" do @tag live_call: true, live_mistral_ai: true test "basic non-streamed response works and fires token usage callback" do @@ -401,7 +558,7 @@ defmodule LangChain.ChatModels.ChatMistralAITest do # returns a list of MessageDeltas. A list of a list because it's "n" choices. assert result == [ %Message{ - content: "Colorful Threads", + content: [ContentPart.text!("Colorful Threads")], status: :complete, role: :assistant, index: 0, @@ -410,7 +567,9 @@ defmodule LangChain.ChatModels.ChatMistralAITest do ] assert_received {:fired_token_usage, usage} - assert %TokenUsage{input: 18, output: 4} = usage + assert %TokenUsage{input: 18} = usage + # Allow for slight variation in token count + assert usage.output in [4, 5] end @tag live_call: true, live_mistral_ai: true @@ -452,7 +611,9 @@ defmodule LangChain.ChatModels.ChatMistralAITest do assert result_string == "Colorful Threads" assert_received {:fired_token_usage, usage} - assert %TokenUsage{input: 18, output: 4} = usage + assert %TokenUsage{input: 18} = usage + # Allow for slight variation in token count + assert usage.output in [4, 5] end @tag live_call: true, live_mistral_ai: true @@ -483,5 +644,92 @@ defmodule LangChain.ChatModels.ChatMistralAITest do tool_call_msg = Enum.find(result, fn [msg] -> msg.tool_calls != nil end) assert [%MessageDelta{tool_calls: [%ToolCall{name: "current_time"}]}] = tool_call_msg end + + @tag live_call: true, live_mistral_ai: true + test "structured output with JSON schema works" do + schema = %{ + "type" => "json_schema", + "json_schema" => %{ + "schema" => %{ + "properties" => %{ + "name" => %{"title" => "Name", "type" => "string"}, + "authors" => %{ + "items" => %{"type" => "string"}, + "title" => "Authors", + "type" => "array" + } + }, + "required" => ["name", "authors"], + "title" => "Book", + "type" => "object", + "additionalProperties" => false + }, + "name" => "book", + "strict" => true + } + } + + chat = + ChatMistralAI.new!(%{ + temperature: 0, + model: "ministral-8b-latest", + stream: false, + json_response: true, + json_schema: schema + }) + + {:ok, result} = + ChatMistralAI.call( + chat, + [ + Message.new_system!("Extract the books information."), + Message.new_user!("I recently read To Kill a Mockingbird by Harper Lee.") + ], + [] + ) + + assert [%Message{content: content, status: :complete, role: :assistant}] = result + + # The content should be a valid JSON string that matches our schema + assert is_list(content) + assert [%ContentPart{type: :text, content: json_content}] = content + assert is_binary(json_content) + {:ok, parsed_json} = Jason.decode(json_content) + + # Verify the structure matches our schema + assert %{"name" => name, "authors" => authors} = parsed_json + assert is_binary(name) + assert is_list(authors) + assert Enum.all?(authors, &is_binary/1) + end + + @tag live_call: true, live_mistral_ai: true + test "structured output with json_object type works" do + chat = + ChatMistralAI.new!(%{ + temperature: 0, + model: "ministral-8b-latest", + stream: false, + json_response: true + }) + + {:ok, result} = + ChatMistralAI.call( + chat, + [ + Message.new_system!("Extract the books information and return as JSON."), + Message.new_user!("I recently read To Kill a Mockingbird by Harper Lee.") + ], + [] + ) + + assert [%Message{content: content, status: :complete, role: :assistant}] = result + + # The content should be a valid JSON string + assert is_list(content) + assert [%ContentPart{type: :text, content: json_content}] = content + assert is_binary(json_content) + assert {:ok, _parsed_json} = Jason.decode(json_content) + end end end diff --git a/test/chat_models/chat_ollama_ai_test.exs b/test/chat_models/chat_ollama_ai_test.exs index 99ea0f96..a8e6a3e8 100644 --- a/test/chat_models/chat_ollama_ai_test.exs +++ b/test/chat_models/chat_ollama_ai_test.exs @@ -6,6 +6,7 @@ defmodule ChatModels.ChatOllamaAITest do alias LangChain.ChatModels.ChatOllamaAI alias LangChain.Function alias LangChain.FunctionParam + alias LangChain.Message.ContentPart use Mimic @@ -600,7 +601,7 @@ defmodule ChatModels.ChatOllamaAITest do assert %Message{} = struct = ChatOllamaAI.do_process_response(model, response) assert struct.role == :assistant - assert struct.content == "Greetings!" + assert struct.content == [ContentPart.text!("Greetings!")] assert struct.index == nil end diff --git a/test/chat_models/chat_open_ai_test.exs b/test/chat_models/chat_open_ai_test.exs index b3dc1949..bfbe0243 100644 --- a/test/chat_models/chat_open_ai_test.exs +++ b/test/chat_models/chat_open_ai_test.exs @@ -702,7 +702,7 @@ defmodule LangChain.ChatModels.ChatOpenAITest do } result = - ChatOpenAI.for_api( + ChatOpenAI.content_part_for_api( ChatOpenAI.new!(), ContentPart.file!(file_base64_data, media: :pdf, type: :base64, filename: filename) ) @@ -721,7 +721,7 @@ defmodule LangChain.ChatModels.ChatOpenAITest do } result = - ChatOpenAI.for_api( + ChatOpenAI.content_part_for_api( ChatOpenAI.new!(), ContentPart.file!(file_id, media: :pdf, type: :file_id) ) diff --git a/test/chat_models/chat_perplexity_test.exs b/test/chat_models/chat_perplexity_test.exs index c60afc51..4e3027c9 100644 --- a/test/chat_models/chat_perplexity_test.exs +++ b/test/chat_models/chat_perplexity_test.exs @@ -6,6 +6,7 @@ defmodule LangChain.ChatModels.ChatPerplexityTest do alias LangChain.ChatModels.ChatPerplexity alias LangChain.Message alias LangChain.MessageDelta + alias LangChain.Message.ContentPart alias LangChain.TokenUsage alias LangChain.LangChainError alias LangChain.Function @@ -430,7 +431,7 @@ defmodule LangChain.ChatModels.ChatPerplexityTest do assert %Message{} = message = ChatPerplexity.do_process_response(model, response) assert message.role == :assistant - assert message.content == "Hello!" + assert message.content == [ContentPart.text!("Hello!")] assert message.index == 1 assert message.status == :complete end @@ -451,7 +452,7 @@ defmodule LangChain.ChatModels.ChatPerplexityTest do assert %Message{} = struct = ChatPerplexity.do_process_response(model, response) assert struct.role == :assistant - assert struct.content == "Some of the response that was abruptly" + assert struct.content == [ContentPart.text!("Some of the response that was abruptly")] assert struct.index == 0 assert struct.status == :length end diff --git a/test/chat_models/chat_vertex_ai_test.exs b/test/chat_models/chat_vertex_ai_test.exs index fe0d689a..9dcad7d4 100644 --- a/test/chat_models/chat_vertex_ai_test.exs +++ b/test/chat_models/chat_vertex_ai_test.exs @@ -130,9 +130,9 @@ defmodule ChatModels.ChatVertexAITest do "parts" => [ %{"text" => "User prompt"}, %{ - "file_data" => %{ - "file_uri" => "example.com/test.pdf", - "mime_type" => "application/pdf" + "fileData" => %{ + "fileUri" => "example.com/test.pdf", + "mimeType" => "application/pdf" } } ],