From 78d4fb9b52a13ac945a82467661105f8898534c1 Mon Sep 17 00:00:00 2001 From: Damian Momot Date: Thu, 16 Jul 2026 03:37:44 -0700 Subject: [PATCH] feat(mlkit): use ML Kit GenAI beta3 API in GenaiPrompt Upgrade GenaiPrompt to genai-prompt 1.0.0-beta3 and use its newer request API: - Map each turn to a Content so all images are sent, not just the first. - For multi-turn requests, prefix each turn's text with a `[role]:` marker and add a default system instruction so the model does not echo the marker back. - Pass the system instruction via the SystemInstruction field. PiperOrigin-RevId: 948874190 --- .../utils/mlkit/GenaiPromptConversionsTest.kt | 91 ++++++++++++- .../google/adk/kt/models/mlkit/GenaiPrompt.kt | 15 +-- .../adk/kt/models/mlkit/GenaiPromptTracing.kt | 12 +- .../kt/utils/mlkit/GenaiPromptConversions.kt | 123 ++++++++++-------- gradle/libs.versions.toml | 2 +- 5 files changed, 169 insertions(+), 74 deletions(-) diff --git a/core/src/androidHostTest/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversionsTest.kt b/core/src/androidHostTest/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversionsTest.kt index 54eed635..c627b83b 100644 --- a/core/src/androidHostTest/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversionsTest.kt +++ b/core/src/androidHostTest/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversionsTest.kt @@ -33,6 +33,7 @@ import com.google.adk.kt.utils.mlkit.GenaiPromptConversions.toLlmResponse import com.google.common.truth.Truth.assertThat import com.google.mlkit.common.sdkinternal.MlKitContext import com.google.mlkit.genai.prompt.Candidate.FinishReason as MlKitFinishReason +import com.google.mlkit.genai.prompt.TextPart import java.io.ByteArrayOutputStream import java.io.File import org.junit.Before @@ -81,6 +82,86 @@ class GenaiPromptConversionsTest { assertThat(generateContentRequest.promptPrefix).isNull() } + @Test + fun toGenerateContentRequest_singleTurn_noRoleMarker() { + val request = + LlmRequest(contents = listOf(Content(role = "user", parts = listOf(Part(text = "Hello"))))) + + val generateContentRequest = request.toGenerateContentRequest() + + val texts = + generateContentRequest.contents + .flatMap { it.parts } + .filterIsInstance() + .map { it.textString } + assertThat(texts).containsExactly("Hello") + // Single-turn requests get no default multi-turn system instruction. + assertThat(generateContentRequest.systemInstruction).isNull() + } + + @Test + fun toGenerateContentRequest_multiTurn_addsRoleMarkers() { + val request = + LlmRequest( + contents = + listOf( + Content(role = "user", parts = listOf(Part(text = "Hello"))), + Content(role = "model", parts = listOf(Part(text = "Hi there"))), + Content(role = "user", parts = listOf(Part(text = "How are you?"))), + ) + ) + + val generateContentRequest = request.toGenerateContentRequest() + + val texts = + generateContentRequest.contents + .flatMap { it.parts } + .filterIsInstance() + .map { it.textString } + assertThat(texts) + .containsExactly("[user]: Hello", "[model]: Hi there", "[user]: How are you?") + .inOrder() + } + + @Test + fun toGenerateContentRequest_multiTurn_addsDefaultSystemInstruction() { + val request = + LlmRequest( + contents = + listOf( + Content(role = "user", parts = listOf(Part(text = "Hello"))), + Content(role = "model", parts = listOf(Part(text = "Hi there"))), + ) + ) + + val generateContentRequest = request.toGenerateContentRequest() + + assertThat(generateContentRequest.systemInstruction?.textString) + .contains("Do not prefix your own response with a role marker") + } + + @Test + fun toGenerateContentRequest_multiTurn_combinesWithUserSystemInstruction() { + val request = + LlmRequest( + contents = + listOf( + Content(role = "user", parts = listOf(Part(text = "Hello"))), + Content(role = "model", parts = listOf(Part(text = "Hi there"))), + ), + config = + GenerateContentConfig( + systemInstruction = Content(parts = listOf(Part(text = "Be concise."))) + ), + ) + + val generateContentRequest = request.toGenerateContentRequest() + + val systemInstruction = generateContentRequest.systemInstruction?.textString + assertThat(systemInstruction).contains("Do not prefix your own response with a role marker") + assertThat(systemInstruction).contains("Be concise.") + } + @Ignore("throws java.lang.VerifyError") fun toGenerateContentRequest_textAndImage_success() { val request = @@ -184,13 +265,13 @@ class GenaiPromptConversionsTest { ) val generateContentRequest = request.toGenerateContentRequest() assertThat(generateContentRequest.image?.bitmap).isNotNull() - // Only the first image is used. It has a size of 1x1. + // All images are sent; the deprecated `image` getter still returns the first one (1x1). assertThat(generateContentRequest.image?.width).isEqualTo(1) assertThat(generateContentRequest.image?.height).isEqualTo(1) } @Test - fun toGenerateContentRequest_textAndSystemInstruction_successWithPromptPrefix() { + fun toGenerateContentRequest_textAndSystemInstruction_usesSystemInstructionField() { val request = LlmRequest( contents = listOf(Content(role = "user", parts = listOf(Part(text = "Hello World")))), @@ -200,7 +281,7 @@ class GenaiPromptConversionsTest { Content( parts = listOf( - Part(text = "Test prompt prefix"), + Part(text = "Test system instruction"), Part(text = "Another system instruction"), ) ) @@ -209,8 +290,8 @@ class GenaiPromptConversionsTest { val generateContentRequest = request.toGenerateContentRequest() assertThat(generateContentRequest.text.textString).isEqualTo("Hello World") assertThat(generateContentRequest.image).isNull() - assertThat(generateContentRequest.promptPrefix?.textString) - .isEqualTo("Test prompt prefix\n\nAnother system instruction") + assertThat(generateContentRequest.systemInstruction?.textString) + .isEqualTo("Test system instruction\n\nAnother system instruction") } @Ignore("throws java.lang.VerifyError") diff --git a/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPrompt.kt b/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPrompt.kt index ec6f49b4..81f02b71 100644 --- a/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPrompt.kt +++ b/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPrompt.kt @@ -32,16 +32,11 @@ import kotlinx.coroutines.flow.flow /** * A [Model] implementation that uses the ML Kit GenAI API to generate content. * - * The underlying ML Kit GenAI `GenerateContentRequest` is a single-prompt API, so this - * implementation currently has the following limitations: - * - Multi-turn conversations are not truly supported. The whole [LlmRequest.contents] history is - * flattened into a single prompt: all text parts (across every turn) are concatenated with "\n\n" - * and the role of each turn is discarded, so the model cannot distinguish user turns from model - * turns. - * - Only text and a single image are sent. Text parts are concatenated; if multiple images are - * present, only the first is used and the rest are ignored. - * - Tool use is not supported. Only text parts are read; `functionCall` and `functionResponse` - * parts are dropped. + * ML Kit has no per-turn role, so this implementation has these limitations: + * - Multi-turn is approximated: for multi-turn requests each turn's text is prefixed with a + * `[role]:` marker (e.g. `[user]:`/`[model]:`), and a default system instruction tells the model + * not to echo the marker back. + * - Tool use is unsupported: `functionCall`/`functionResponse` parts are dropped. * - Only the first response candidate is used. * * @param generativeModel The [GenerativeModel] to use for generation. diff --git a/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPromptTracing.kt b/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPromptTracing.kt index 174166cb..1c29d92a 100644 --- a/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPromptTracing.kt +++ b/core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPromptTracing.kt @@ -25,6 +25,8 @@ import com.google.adk.kt.types.Part import com.google.adk.kt.utils.mlkit.AggregatedResponse import com.google.mlkit.genai.prompt.GenerateContentRequest import com.google.mlkit.genai.prompt.GenerateContentResponse +import com.google.mlkit.genai.prompt.ImagePart +import com.google.mlkit.genai.prompt.TextPart /** * Helpers that format ML Kit GenAI and ADK request/response objects for tracing. @@ -41,10 +43,12 @@ import com.google.mlkit.genai.prompt.GenerateContentResponse internal object GenaiPromptTracing { internal fun format(generateContentRequest: GenerateContentRequest): String { - val imageTrace = - generateContentRequest.image?.bitmap?.let { "${it.width}x${it.height}" } ?: "none" - return "generateContentRequest: text: ${redactedText(generateContentRequest.text.textString.length)}, " + - "promptPrefix: ${redactedText(generateContentRequest.promptPrefix?.textString?.length ?: 0)}, image: $imageTrace" + val parts = generateContentRequest.contents.flatMap { it.parts } + val textLength = parts.filterIsInstance().sumOf { it.textString.length } + val imageCount = parts.filterIsInstance().size + val systemInstructionLength = generateContentRequest.systemInstruction?.textString?.length ?: 0 + return "generateContentRequest: text: ${redactedText(textLength)}, " + + "systemInstruction: ${redactedText(systemInstructionLength)}, images: $imageCount" } internal fun format(generateContentResponse: GenerateContentResponse): String { diff --git a/core/src/androidMain/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversions.kt b/core/src/androidMain/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversions.kt index 0dbb86e6..d3b58d34 100644 --- a/core/src/androidMain/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversions.kt +++ b/core/src/androidMain/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversions.kt @@ -24,10 +24,11 @@ import com.google.adk.kt.types.FinishReason import com.google.adk.kt.types.Part import com.google.adk.kt.types.Role import com.google.mlkit.genai.prompt.Candidate +import com.google.mlkit.genai.prompt.Content as MlKitContent import com.google.mlkit.genai.prompt.GenerateContentRequest import com.google.mlkit.genai.prompt.GenerateContentResponse import com.google.mlkit.genai.prompt.ImagePart -import com.google.mlkit.genai.prompt.PromptPrefix +import com.google.mlkit.genai.prompt.SystemInstruction import com.google.mlkit.genai.prompt.TextPart /** Utility functions for converting between ADK and ML Kit request and response formats. */ @@ -40,80 +41,94 @@ internal object GenaiPromptConversions { private val instructionSeparator = "\n\n" + /** Guidance prepended to the system instruction for multi-turn requests. */ + private val multiTurnSystemInstruction = + "You are given a multi-turn conversation history where each turn is prefixed with a role " + + "marker such as \"[user]:\" or \"[model]:\". Do not prefix your own response with a role " + + "marker such as \"[model]:\"." + /** * Converts an [LlmRequest] to a [GenerateContentRequest]. * - * The ML Kit request can only contain one text part and one image part. If multiple text parts - * are present, they are concatenated together separated by "\n\n". Image parts except for the - * first one are ignored. - * - * Certain parameters from the [LlmRequest.config] are used to configure the ML Kit request. - * - * @return The [GenerateContentRequest] to be used for the ML Kit API call. + * Each ADK [Content] (turn) maps to an ML Kit [MlKitContent], keeping all images and turn order. + * Since ML Kit has no per-turn role, multi-turn requests prefix each turn's text with a `[role]:` + * marker and prepend a default [multiTurnSystemInstruction] explaining the markers. The system + * instruction is passed through the [SystemInstruction] field. */ internal fun LlmRequest.toGenerateContentRequest(): GenerateContentRequest { - val allParts = contents.flatMap { it.parts } - val allText = allParts.mapNotNull { it.text }.joinToString(instructionSeparator) - val imageCount = allParts.count { - it.inlineData?.mimeType.isImageMimeType() || it.fileData?.mimeType.isImageMimeType() - } + val isMultiTurn = contents.size > 1 + val mlKitContents = contents.mapNotNull { it.toMlKitContent(includeRoleMarkers = isMultiTurn) } - if (imageCount > 1) { - logger.warn { - "Multiple images found in the LlmRequest. Only the first image will be used in the GenerateContentRequest." - } - } - - val imagePart = allParts.firstNotNullOfOrNull { part -> - val inlineData = part.inlineData - val fileData = part.fileData - when { - inlineData?.mimeType.isImageMimeType() -> { - inlineData?.data?.let { ImagePart(it) } - } - fileData?.mimeType.isImageMimeType() -> { - fileData?.fileUri?.let { ImagePart(it.toUri()) } - } - else -> null - } + // ML Kit requires at least one content; fall back to an empty text prompt. + val requestContents = mlKitContents.ifEmpty { + listOf(MlKitContent.builder().addPart(TextPart("")).build()) } + // For multi-turn requests, prepend the guidance that explains the `[role]:` markers. val systemText = - config.systemInstruction?.parts?.mapNotNull { it.text }?.joinToString(instructionSeparator) - ?: "" - - // ML Kit GenerateContentRequest doesn't support promptPrefix with image input. - // Prepend system text to the main text if image is present, otherwise use promptPrefix - // for caching benefits. - val shouldUsePromptPrefix = systemText.isNotEmpty() && imagePart == null - val promptPrefix = if (shouldUsePromptPrefix) PromptPrefix(systemText) else null - val textPart = - TextPart( - if (shouldUsePromptPrefix || systemText.isEmpty()) { - allText - } else { - "$systemText$instructionSeparator$allText".trim() - } - ) + listOfNotNull(multiTurnSystemInstruction.takeIf { isMultiTurn }, systemInstructionText()) + .joinToString(instructionSeparator) + .takeIf { it.isNotEmpty() } - val builder = - if (imagePart != null) { - GenerateContentRequest.builder(imagePart, textPart) - } else { - GenerateContentRequest.builder(textPart) - } + val builder = GenerateContentRequest.Builder(requestContents) builder.apply { config.temperature?.let { temperature = it } config.topK?.let { topK = it } config.candidateCount?.let { candidateCount = it } config.maxOutputTokens?.let { maxOutputTokens = it } - promptPrefix?.let { this.promptPrefix = it } + systemText?.let { systemInstruction = SystemInstruction(it) } } return builder.build() } + /** + * Maps an ADK [Content] (turn) to an ML Kit [MlKitContent], prefixing text with a `[role]:` + * marker when [includeRoleMarkers] is true. Returns `null` if the turn has no text or image. + */ + private fun Content.toMlKitContent(includeRoleMarkers: Boolean): MlKitContent? { + val text = parts.mapNotNull { it.text }.joinToString(instructionSeparator) + val imageParts = parts.mapNotNull { it.toImagePartOrNull() } + if (text.isEmpty() && imageParts.isEmpty()) { + return null + } + + val markedText = + when { + !includeRoleMarkers -> text + else -> "[${role ?: Role.USER}]:" + if (text.isNotEmpty()) " $text" else "" + } + + val builder = MlKitContent.builder() + if (markedText.isNotEmpty()) { + builder.addPart(TextPart(markedText)) + } + imageParts.forEach { builder.addPart(it) } + return builder.build() + } + + /** Converts an ADK [Part] to an ML Kit [ImagePart], or `null` if it is not an image. */ + private fun Part.toImagePartOrNull(): ImagePart? { + val inlineData = inlineData + val fileData = fileData + return when { + inlineData != null && inlineData.mimeType.isImageMimeType() -> + inlineData.data?.let { ImagePart(it) } + fileData != null && fileData.mimeType.isImageMimeType() -> + fileData.fileUri?.let { ImagePart(it.toUri()) } + else -> null + } + } + + /** Returns the request's own system instruction text, or `null` if none is set. */ + private fun LlmRequest.systemInstructionText(): String? = + config.systemInstruction + ?.parts + ?.mapNotNull { it.text } + ?.joinToString(instructionSeparator) + ?.takeIf { it.isNotEmpty() } + /** * Converts a [GenerateContentResponse] to an [LlmResponse]. * diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6266e2a6..6be9eb2a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ google-cloud-storage = "2.35.0" google-firebase-platform = "34.16.0" google-genai-kotlin = "0.2.0" google-gson = "2.10.1" -google-mlkit-genai = "1.0.0-beta2" +google-mlkit-genai = "1.0.0-beta3" google-protobuf-javalite = "3.25.5" google-truth = "1.4.5" gradle-test-retry = "1.6.5"