diff --git a/lib/chat_models/chat_google_ai.ex b/lib/chat_models/chat_google_ai.ex index 355c5ee2..a093b0eb 100644 --- a/lib/chat_models/chat_google_ai.ex +++ b/lib/chat_models/chat_google_ai.ex @@ -187,8 +187,17 @@ defmodule LangChain.ChatModels.ChatGoogleAI do nil -> nil - %Message{role: :system, content: content} -> + %Message{role: :system, content: content} when is_binary(content) -> %{"parts" => [%{"text" => content}]} + + %Message{role: :system, content: content} when is_list(content) -> + # Extract text from ContentPart structures + text_content = + content + |> Enum.filter(&match?(%ContentPart{type: :text}, &1)) + |> Enum.map(& &1.content) + |> Enum.join(" ") + %{"parts" => [%{"text" => text_content}]} end messages_for_api = diff --git a/lib/chat_models/chat_mistral_ai.ex b/lib/chat_models/chat_mistral_ai.ex index 55e36300..fc2a11dd 100644 --- a/lib/chat_models/chat_mistral_ai.ex +++ b/lib/chat_models/chat_mistral_ai.ex @@ -179,23 +179,49 @@ defmodule LangChain.ChatModels.ChatMistralAI do def for_api(%_{} = model, %Message{role: :assistant, tool_calls: tool_calls} = msg) when is_list(tool_calls) do + content = case msg.content do + content when is_binary(content) -> content + content when is_list(content) -> ContentPart.parts_to_string(content) + nil -> nil + end + %{ "role" => :assistant, - "content" => msg.content + "content" => content } |> Utils.conditionally_add_to_map("tool_calls", Enum.map(tool_calls, &for_api(model, &1))) end - def for_api(%_{} = model, %Message{role: :user, content: content} = msg) + def for_api(%_{} = _model, %Message{role: :user, content: content} = msg) when is_list(content) do # A user message can hold an array of ContentParts %{ "role" => msg.role, - "content" => Enum.map(content, &for_api(model, &1)) + "content" => ContentPart.parts_to_string(content) } |> Utils.conditionally_add_to_map("name", msg.name) end + # Handle messages with ContentPart content for non-user roles + def for_api(%_{} = model, %Message{content: content} = msg) when is_list(content) do + role = get_message_role(model, msg.role) + + %{ + "role" => role, + "content" => ContentPart.parts_to_string(content) + } + |> Utils.conditionally_add_to_map("name", msg.name) + |> Utils.conditionally_add_to_map( + "tool_calls", + Enum.map(msg.tool_calls || [], &for_api(model, &1)) + ) + end + + # Handle ContentPart structures + def for_api(%_{} = _model, %ContentPart{type: :text, content: content}) do + content + end + # ToolResult => stand-alone message with "role: :tool" def for_api(%_{} = _model, %ToolResult{type: :function} = result) do %{ diff --git a/lib/chat_models/chat_ollama_ai.ex b/lib/chat_models/chat_ollama_ai.ex index 8a1196d4..50cf2329 100644 --- a/lib/chat_models/chat_ollama_ai.ex +++ b/lib/chat_models/chat_ollama_ai.ex @@ -42,6 +42,7 @@ defmodule LangChain.ChatModels.ChatOllamaAI do alias LangChain.ChatModels.ChatModel alias LangChain.ChatModels.ChatOpenAI alias LangChain.Message + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.Message.ToolResult alias LangChain.MessageDelta @@ -242,9 +243,15 @@ defmodule LangChain.ChatModels.ChatOllamaAI do def for_api(%Message{role: :assistant, tool_calls: tool_calls} = msg) when is_list(tool_calls) do + content = case msg.content do + content when is_binary(content) -> content + content when is_list(content) -> ContentPart.parts_to_string(content) + nil -> nil + end + %{ "role" => :assistant, - "content" => msg.content + "content" => content } |> Utils.conditionally_add_to_map("tool_calls", Enum.map(tool_calls, &for_api(&1))) end @@ -292,11 +299,25 @@ defmodule LangChain.ChatModels.ChatOllamaAI do def for_api(%Message{role: :user, content: content} = msg) when is_list(content) do %{ "role" => msg.role, - "content" => Enum.map(content, &for_api(&1)) + "content" => ContentPart.parts_to_string(content) + } + |> Utils.conditionally_add_to_map("name", msg.name) + end + + # Handle messages with ContentPart content for non-user roles + def for_api(%Message{content: content} = msg) when is_list(content) do + %{ + "role" => msg.role, + "content" => ContentPart.parts_to_string(content) } |> Utils.conditionally_add_to_map("name", msg.name) end + # Handle ContentPart structures + def for_api(%ContentPart{type: :text, content: content}) do + content + end + defp get_tools_for_api(nil), do: [] defp get_tools_for_api(tools) do diff --git a/lib/chat_models/chat_open_ai.ex b/lib/chat_models/chat_open_ai.ex index 6084ce38..66409403 100644 --- a/lib/chat_models/chat_open_ai.ex +++ b/lib/chat_models/chat_open_ai.ex @@ -536,6 +536,11 @@ defmodule LangChain.ChatModels.ChatOpenAI do raise LangChainError, "PromptTemplates must be converted to messages." end + # Handle ContentPart structures directly + def for_api(%_{} = model, %ContentPart{} = part) do + content_part_for_api(model, part) + end + @doc """ Convert a list of ContentParts to the expected map of data for the OpenAI API. """ diff --git a/lib/chat_models/chat_perplexity.ex b/lib/chat_models/chat_perplexity.ex index 046c0428..9e965a87 100644 --- a/lib/chat_models/chat_perplexity.ex +++ b/lib/chat_models/chat_perplexity.ex @@ -45,6 +45,7 @@ defmodule LangChain.ChatModels.ChatPerplexity do alias LangChain.ChatModels.ChatModel alias LangChain.Message alias LangChain.MessageDelta + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.TokenUsage alias LangChain.LangChainError @@ -283,9 +284,15 @@ defmodule LangChain.ChatModels.ChatPerplexity do """ @spec for_api(t(), Message.t()) :: %{String.t() => any()} def for_api(%ChatPerplexity{}, %Message{} = msg) do + content = case msg.content do + content when is_binary(content) -> content + content when is_list(content) -> ContentPart.parts_to_string(content) + nil -> nil + end + %{ "role" => msg.role, - "content" => msg.content + "content" => content } end diff --git a/lib/chat_models/chat_vertex_ai.ex b/lib/chat_models/chat_vertex_ai.ex index ab2a464c..069a0d0c 100644 --- a/lib/chat_models/chat_vertex_ai.ex +++ b/lib/chat_models/chat_vertex_ai.ex @@ -216,20 +216,25 @@ defmodule LangChain.ChatModels.ChatVertexAI do end defp for_api(%Message{role: :system} = message) do - %{"parts" => %{"text" => message.content}} + # System messages should return a single text part, not a list + case get_message_contents(message) do + [%{"text" => text}] -> %{"parts" => %{"text" => text}} + _ -> %{"parts" => %{"text" => message.content}} + end end defp for_api(%Message{role: :user, content: content}) when is_list(content) do %{ - "role" => "user", + "role" => map_role(:user), "parts" => Enum.map(content, &for_api(&1)) } end defp for_api(%Message{} = message) do + content_parts = get_message_contents(message) || [] %{ "role" => map_role(message.role), - "parts" => [%{"text" => message.content}] + "parts" => content_parts } end @@ -257,9 +262,9 @@ defmodule LangChain.ChatModels.ChatVertexAI do defp for_api(%ContentPart{type: :file_url} = part) do %{ - "file_data" => %{ + "fileData" => %{ "mimeType" => Keyword.fetch!(part.options, :media), - "file_uri" => part.content + "fileUri" => part.content } } end diff --git a/test/chat_models/chat_mistral_ai_test.exs b/test/chat_models/chat_mistral_ai_test.exs index f729a75e..ad9abefd 100644 --- a/test/chat_models/chat_mistral_ai_test.exs +++ b/test/chat_models/chat_mistral_ai_test.exs @@ -4,6 +4,7 @@ defmodule LangChain.ChatModels.ChatMistralAITest do alias LangChain.ChatModels.ChatMistralAI alias LangChain.Message alias LangChain.MessageDelta + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.LangChainError alias LangChain.TokenUsage @@ -113,7 +114,7 @@ defmodule LangChain.ChatModels.ChatMistralAITest do assert [%Message{} = msg] = ChatMistralAI.do_process_response(model, response) assert msg.role == :assistant - assert msg.content == "Hello User!" + assert msg.content == [ContentPart.text!("Hello User!")] assert msg.index == 0 assert msg.status == :complete end @@ -315,8 +316,8 @@ defmodule LangChain.ChatModels.ChatMistralAITest do result = ChatMistralAI.do_process_response(model, response) - assert [%Message{role: :assistant, content: "Hello from Mistral!", status: :complete}] = - result + assert [%Message{role: :assistant, status: :complete} = message] = result + assert message.content == [ContentPart.text!("Hello from Mistral!")] end end diff --git a/test/chat_models/chat_ollama_ai_test.exs b/test/chat_models/chat_ollama_ai_test.exs index 99ea0f96..a8e6a3e8 100644 --- a/test/chat_models/chat_ollama_ai_test.exs +++ b/test/chat_models/chat_ollama_ai_test.exs @@ -6,6 +6,7 @@ defmodule ChatModels.ChatOllamaAITest do alias LangChain.ChatModels.ChatOllamaAI alias LangChain.Function alias LangChain.FunctionParam + alias LangChain.Message.ContentPart use Mimic @@ -600,7 +601,7 @@ defmodule ChatModels.ChatOllamaAITest do assert %Message{} = struct = ChatOllamaAI.do_process_response(model, response) assert struct.role == :assistant - assert struct.content == "Greetings!" + assert struct.content == [ContentPart.text!("Greetings!")] assert struct.index == nil end diff --git a/test/chat_models/chat_perplexity_test.exs b/test/chat_models/chat_perplexity_test.exs index c60afc51..4e3027c9 100644 --- a/test/chat_models/chat_perplexity_test.exs +++ b/test/chat_models/chat_perplexity_test.exs @@ -6,6 +6,7 @@ defmodule LangChain.ChatModels.ChatPerplexityTest do alias LangChain.ChatModels.ChatPerplexity alias LangChain.Message alias LangChain.MessageDelta + alias LangChain.Message.ContentPart alias LangChain.TokenUsage alias LangChain.LangChainError alias LangChain.Function @@ -430,7 +431,7 @@ defmodule LangChain.ChatModels.ChatPerplexityTest do assert %Message{} = message = ChatPerplexity.do_process_response(model, response) assert message.role == :assistant - assert message.content == "Hello!" + assert message.content == [ContentPart.text!("Hello!")] assert message.index == 1 assert message.status == :complete end @@ -451,7 +452,7 @@ defmodule LangChain.ChatModels.ChatPerplexityTest do assert %Message{} = struct = ChatPerplexity.do_process_response(model, response) assert struct.role == :assistant - assert struct.content == "Some of the response that was abruptly" + assert struct.content == [ContentPart.text!("Some of the response that was abruptly")] assert struct.index == 0 assert struct.status == :length end diff --git a/test/chat_models/chat_vertex_ai_test.exs b/test/chat_models/chat_vertex_ai_test.exs index fe0d689a..9dcad7d4 100644 --- a/test/chat_models/chat_vertex_ai_test.exs +++ b/test/chat_models/chat_vertex_ai_test.exs @@ -130,9 +130,9 @@ defmodule ChatModels.ChatVertexAITest do "parts" => [ %{"text" => "User prompt"}, %{ - "file_data" => %{ - "file_uri" => "example.com/test.pdf", - "mime_type" => "application/pdf" + "fileData" => %{ + "fileUri" => "example.com/test.pdf", + "mimeType" => "application/pdf" } } ],