feat(mlkit): use ML Kit GenAI beta3 API in GenaiPrompt

core/src/androidHostTest/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversionsTest.kt

@@ -33,6 +33,7 @@ import com.google.adk.kt.utils.mlkit.GenaiPromptConversions.toLlmResponse
 import com.google.common.truth.Truth.assertThat
 import com.google.mlkit.common.sdkinternal.MlKitContext
 import com.google.mlkit.genai.prompt.Candidate.FinishReason as MlKitFinishReason
+import com.google.mlkit.genai.prompt.TextPart
 import java.io.ByteArrayOutputStream
 import java.io.File
 import org.junit.Before
@@ -81,6 +82,86 @@ class GenaiPromptConversionsTest {
     assertThat(generateContentRequest.promptPrefix).isNull()
   }
 
+  @Test
+  fun toGenerateContentRequest_singleTurn_noRoleMarker() {
+    val request =
+      LlmRequest(contents = listOf(Content(role = "user", parts = listOf(Part(text = "Hello")))))
+
+    val generateContentRequest = request.toGenerateContentRequest()
+
+    val texts =
+      generateContentRequest.contents
+        .flatMap { it.parts }
+        .filterIsInstance<TextPart>()
+        .map { it.textString }
+    assertThat(texts).containsExactly("Hello")
+    // Single-turn requests get no default multi-turn system instruction.
+    assertThat(generateContentRequest.systemInstruction).isNull()
+  }
+
+  @Test
+  fun toGenerateContentRequest_multiTurn_addsRoleMarkers() {
+    val request =
+      LlmRequest(
+        contents =
+          listOf(
+            Content(role = "user", parts = listOf(Part(text = "Hello"))),
+            Content(role = "model", parts = listOf(Part(text = "Hi there"))),
+            Content(role = "user", parts = listOf(Part(text = "How are you?"))),
+          )
+      )
+
+    val generateContentRequest = request.toGenerateContentRequest()
+
+    val texts =
+      generateContentRequest.contents
+        .flatMap { it.parts }
+        .filterIsInstance<TextPart>()
+        .map { it.textString }
+    assertThat(texts)
+      .containsExactly("[user]: Hello", "[model]: Hi there", "[user]: How are you?")
+      .inOrder()
+  }
+
+  @Test
+  fun toGenerateContentRequest_multiTurn_addsDefaultSystemInstruction() {
+    val request =
+      LlmRequest(
+        contents =
+          listOf(
+            Content(role = "user", parts = listOf(Part(text = "Hello"))),
+            Content(role = "model", parts = listOf(Part(text = "Hi there"))),
+          )
+      )
+
+    val generateContentRequest = request.toGenerateContentRequest()
+
+    assertThat(generateContentRequest.systemInstruction?.textString)
+      .contains("Do not prefix your own response with a role marker")
+  }
+
+  @Test
+  fun toGenerateContentRequest_multiTurn_combinesWithUserSystemInstruction() {
+    val request =
+      LlmRequest(
+        contents =
+          listOf(
+            Content(role = "user", parts = listOf(Part(text = "Hello"))),
+            Content(role = "model", parts = listOf(Part(text = "Hi there"))),
+          ),
+        config =
+          GenerateContentConfig(
+            systemInstruction = Content(parts = listOf(Part(text = "Be concise.")))
+          ),
+      )
+
+    val generateContentRequest = request.toGenerateContentRequest()
+
+    val systemInstruction = generateContentRequest.systemInstruction?.textString
+    assertThat(systemInstruction).contains("Do not prefix your own response with a role marker")
+    assertThat(systemInstruction).contains("Be concise.")
+  }
+
   @Ignore("throws java.lang.VerifyError")
   fun toGenerateContentRequest_textAndImage_success() {
     val request =
@@ -184,13 +265,13 @@ class GenaiPromptConversionsTest {
       )
     val generateContentRequest = request.toGenerateContentRequest()
     assertThat(generateContentRequest.image?.bitmap).isNotNull()
-    // Only the first image is used. It has a size of 1x1.
+    // All images are sent; the deprecated `image` getter still returns the first one (1x1).
     assertThat(generateContentRequest.image?.width).isEqualTo(1)
     assertThat(generateContentRequest.image?.height).isEqualTo(1)
   }
 
   @Test
-  fun toGenerateContentRequest_textAndSystemInstruction_successWithPromptPrefix() {
+  fun toGenerateContentRequest_textAndSystemInstruction_usesSystemInstructionField() {
     val request =
       LlmRequest(
         contents = listOf(Content(role = "user", parts = listOf(Part(text = "Hello World")))),
@@ -200,7 +281,7 @@ class GenaiPromptConversionsTest {
               Content(
                 parts =
                   listOf(
-                    Part(text = "Test prompt prefix"),
+                    Part(text = "Test system instruction"),
                     Part(text = "Another system instruction"),
                   )
               )
@@ -209,8 +290,8 @@ class GenaiPromptConversionsTest {
     val generateContentRequest = request.toGenerateContentRequest()
     assertThat(generateContentRequest.text.textString).isEqualTo("Hello World")
     assertThat(generateContentRequest.image).isNull()
-    assertThat(generateContentRequest.promptPrefix?.textString)
-      .isEqualTo("Test prompt prefix\n\nAnother system instruction")
+    assertThat(generateContentRequest.systemInstruction?.textString)
+      .isEqualTo("Test system instruction\n\nAnother system instruction")
   }
 
   @Ignore("throws java.lang.VerifyError")

core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPrompt.kt

@@ -32,16 +32,11 @@ import kotlinx.coroutines.flow.flow
 /**
  * A [Model] implementation that uses the ML Kit GenAI API to generate content.
  *
- * The underlying ML Kit GenAI `GenerateContentRequest` is a single-prompt API, so this
- * implementation currently has the following limitations:
- * - Multi-turn conversations are not truly supported. The whole [LlmRequest.contents] history is
- *   flattened into a single prompt: all text parts (across every turn) are concatenated with "\n\n"
- *   and the role of each turn is discarded, so the model cannot distinguish user turns from model
- *   turns.
- * - Only text and a single image are sent. Text parts are concatenated; if multiple images are
- *   present, only the first is used and the rest are ignored.
- * - Tool use is not supported. Only text parts are read; `functionCall` and `functionResponse`
- *   parts are dropped.
+ * ML Kit has no per-turn role, so this implementation has these limitations:
+ * - Multi-turn is approximated: for multi-turn requests each turn's text is prefixed with a
+ *   `[role]:` marker (e.g. `[user]:`/`[model]:`), and a default system instruction tells the model
+ *   not to echo the marker back.
+ * - Tool use is unsupported: `functionCall`/`functionResponse` parts are dropped.
  * - Only the first response candidate is used.
  *
  * @param generativeModel The [GenerativeModel] to use for generation.

core/src/androidMain/kotlin/com/google/adk/kt/models/mlkit/GenaiPromptTracing.kt

@@ -25,6 +25,8 @@ import com.google.adk.kt.types.Part
 import com.google.adk.kt.utils.mlkit.AggregatedResponse
 import com.google.mlkit.genai.prompt.GenerateContentRequest
 import com.google.mlkit.genai.prompt.GenerateContentResponse
+import com.google.mlkit.genai.prompt.ImagePart
+import com.google.mlkit.genai.prompt.TextPart
 
 /**
  * Helpers that format ML Kit GenAI and ADK request/response objects for tracing.
@@ -41,10 +43,12 @@ import com.google.mlkit.genai.prompt.GenerateContentResponse
 internal object GenaiPromptTracing {
 
   internal fun format(generateContentRequest: GenerateContentRequest): String {
-    val imageTrace =
-      generateContentRequest.image?.bitmap?.let { "${it.width}x${it.height}" } ?: "none"
-    return "generateContentRequest: text: ${redactedText(generateContentRequest.text.textString.length)}, " +
-      "promptPrefix: ${redactedText(generateContentRequest.promptPrefix?.textString?.length ?: 0)}, image: $imageTrace"
+    val parts = generateContentRequest.contents.flatMap { it.parts }
+    val textLength = parts.filterIsInstance<TextPart>().sumOf { it.textString.length }
+    val imageCount = parts.filterIsInstance<ImagePart>().size
+    val systemInstructionLength = generateContentRequest.systemInstruction?.textString?.length ?: 0
+    return "generateContentRequest: text: ${redactedText(textLength)}, " +
+      "systemInstruction: ${redactedText(systemInstructionLength)}, images: $imageCount"
   }
 
   internal fun format(generateContentResponse: GenerateContentResponse): String {

core/src/androidMain/kotlin/com/google/adk/kt/utils/mlkit/GenaiPromptConversions.kt

@@ -24,10 +24,11 @@ import com.google.adk.kt.types.FinishReason
 import com.google.adk.kt.types.Part
 import com.google.adk.kt.types.Role
 import com.google.mlkit.genai.prompt.Candidate
+import com.google.mlkit.genai.prompt.Content as MlKitContent
 import com.google.mlkit.genai.prompt.GenerateContentRequest
 import com.google.mlkit.genai.prompt.GenerateContentResponse
 import com.google.mlkit.genai.prompt.ImagePart
-import com.google.mlkit.genai.prompt.PromptPrefix
+import com.google.mlkit.genai.prompt.SystemInstruction
 import com.google.mlkit.genai.prompt.TextPart
 
 /** Utility functions for converting between ADK and ML Kit request and response formats. */
@@ -40,80 +41,94 @@ internal object GenaiPromptConversions {
 
   private val instructionSeparator = "\n\n"
 
+  /** Guidance prepended to the system instruction for multi-turn requests. */
+  private val multiTurnSystemInstruction =
+    "You are given a multi-turn conversation history where each turn is prefixed with a role " +
+      "marker such as \"[user]:\" or \"[model]:\". Do not prefix your own response with a role " +
+      "marker such as \"[model]:\"."
+
   /**
    * Converts an [LlmRequest] to a [GenerateContentRequest].
    *
-   * The ML Kit request can only contain one text part and one image part. If multiple text parts
-   * are present, they are concatenated together separated by "\n\n". Image parts except for the
-   * first one are ignored.
-   *
-   * Certain parameters from the [LlmRequest.config] are used to configure the ML Kit request.
-   *
-   * @return The [GenerateContentRequest] to be used for the ML Kit API call.
+   * Each ADK [Content] (turn) maps to an ML Kit [MlKitContent], keeping all images and turn order.
+   * Since ML Kit has no per-turn role, multi-turn requests prefix each turn's text with a `[role]:`
+   * marker and prepend a default [multiTurnSystemInstruction] explaining the markers. The system
+   * instruction is passed through the [SystemInstruction] field.
    */
   internal fun LlmRequest.toGenerateContentRequest(): GenerateContentRequest {
-    val allParts = contents.flatMap { it.parts }
-    val allText = allParts.mapNotNull { it.text }.joinToString(instructionSeparator)
-    val imageCount = allParts.count {
-      it.inlineData?.mimeType.isImageMimeType() || it.fileData?.mimeType.isImageMimeType()
-    }
+    val isMultiTurn = contents.size > 1
+    val mlKitContents = contents.mapNotNull { it.toMlKitContent(includeRoleMarkers = isMultiTurn) }
 
-    if (imageCount > 1) {
-      logger.warn {
-        "Multiple images found in the LlmRequest. Only the first image will be used in the GenerateContentRequest."
-      }
-    }
-
-    val imagePart = allParts.firstNotNullOfOrNull { part ->
-      val inlineData = part.inlineData
-      val fileData = part.fileData
-      when {
-        inlineData?.mimeType.isImageMimeType() -> {
-          inlineData?.data?.let { ImagePart(it) }
-        }
-        fileData?.mimeType.isImageMimeType() -> {
-          fileData?.fileUri?.let { ImagePart(it.toUri()) }
-        }
-        else -> null
-      }
+    // ML Kit requires at least one content; fall back to an empty text prompt.
+    val requestContents = mlKitContents.ifEmpty {
+      listOf(MlKitContent.builder().addPart(TextPart("")).build())
     }
 
+    // For multi-turn requests, prepend the guidance that explains the `[role]:` markers.
     val systemText =
-      config.systemInstruction?.parts?.mapNotNull { it.text }?.joinToString(instructionSeparator)
-        ?: ""
-
-    // ML Kit GenerateContentRequest doesn't support promptPrefix with image input.
-    // Prepend system text to the main text if image is present, otherwise use promptPrefix
-    // for caching benefits.
-    val shouldUsePromptPrefix = systemText.isNotEmpty() && imagePart == null
-    val promptPrefix = if (shouldUsePromptPrefix) PromptPrefix(systemText) else null
-    val textPart =
-      TextPart(
-        if (shouldUsePromptPrefix || systemText.isEmpty()) {
-          allText
-        } else {
-          "$systemText$instructionSeparator$allText".trim()
-        }
-      )
+      listOfNotNull(multiTurnSystemInstruction.takeIf { isMultiTurn }, systemInstructionText())
+        .joinToString(instructionSeparator)
+        .takeIf { it.isNotEmpty() }
 
-    val builder =
-      if (imagePart != null) {
-        GenerateContentRequest.builder(imagePart, textPart)
-      } else {
-        GenerateContentRequest.builder(textPart)
-      }
+    val builder = GenerateContentRequest.Builder(requestContents)
 
     builder.apply {
       config.temperature?.let { temperature = it }
       config.topK?.let { topK = it }
       config.candidateCount?.let { candidateCount = it }
       config.maxOutputTokens?.let { maxOutputTokens = it }
-      promptPrefix?.let { this.promptPrefix = it }
+      systemText?.let { systemInstruction = SystemInstruction(it) }
     }
 
     return builder.build()
   }
 
+  /**
+   * Maps an ADK [Content] (turn) to an ML Kit [MlKitContent], prefixing text with a `[role]:`
+   * marker when [includeRoleMarkers] is true. Returns `null` if the turn has no text or image.
+   */
+  private fun Content.toMlKitContent(includeRoleMarkers: Boolean): MlKitContent? {
+    val text = parts.mapNotNull { it.text }.joinToString(instructionSeparator)
+    val imageParts = parts.mapNotNull { it.toImagePartOrNull() }
+    if (text.isEmpty() && imageParts.isEmpty()) {
+      return null
+    }
+
+    val markedText =
+      when {
+        !includeRoleMarkers -> text
+        else -> "[${role ?: Role.USER}]:" + if (text.isNotEmpty()) " $text" else ""
+      }
+
+    val builder = MlKitContent.builder()
+    if (markedText.isNotEmpty()) {
+      builder.addPart(TextPart(markedText))
+    }
+    imageParts.forEach { builder.addPart(it) }
+    return builder.build()
+  }
+
+  /** Converts an ADK [Part] to an ML Kit [ImagePart], or `null` if it is not an image. */
+  private fun Part.toImagePartOrNull(): ImagePart? {
+    val inlineData = inlineData
+    val fileData = fileData
+    return when {
+      inlineData != null && inlineData.mimeType.isImageMimeType() ->
+        inlineData.data?.let { ImagePart(it) }
+      fileData != null && fileData.mimeType.isImageMimeType() ->
+        fileData.fileUri?.let { ImagePart(it.toUri()) }
+      else -> null
+    }
+  }
+
+  /** Returns the request's own system instruction text, or `null` if none is set. */
+  private fun LlmRequest.systemInstructionText(): String? =
+    config.systemInstruction
+      ?.parts
+      ?.mapNotNull { it.text }
+      ?.joinToString(instructionSeparator)
+      ?.takeIf { it.isNotEmpty() }
+
   /**
    * Converts a [GenerateContentResponse] to an [LlmResponse].
    *

gradle/libs.versions.toml

@@ -18,7 +18,7 @@ google-cloud-storage = "2.35.0"
 google-firebase-platform = "34.16.0"
 google-genai-kotlin = "0.2.0"
 google-gson = "2.10.1"
-google-mlkit-genai = "1.0.0-beta2"
+google-mlkit-genai = "1.0.0-beta3"
 google-protobuf-javalite = "3.25.5"
 google-truth = "1.4.5"
 gradle-test-retry = "1.6.5"