Skip to content

Commit 08a857e

Browse files
author
Android PowerUser
committed
fix: correctly suppress screenshot capture/toast/MediaProjection for text-only models across all code paths
- Revert broken Android.triggerNextTurnWithoutScreenshot() call in index.html (method never existed natively); WebView now always calls Bridge.requestScreenshot() as before, since the native side is the correct place to decide whether a real screenshot is needed. - ScreenOperatorAccessibilityService.executeTakeScreenshotCommand now correctly resolves screenshot support for all three model categories: custom JSON-defined models, JS-only online models (WebView dropdown, via js_model_prefs SharedPreferences), and native built-in/offline models - not just the two that were previously handled. - Extracted currentModelSupportsScreenshot() as the single source of truth, used by: - the immediate capture/MediaProjection-request decision - the delayed-screenshot toast (previously unconditionally said 'before screenshot' even for text-only models) - Command.Wait's toast (previously unconditionally said 'Delaying next screenshot' even for text-only models) - Removed the misleading 'Capturing screen info...' toast for text-only models entirely (no screenshot is taken or sent, so no toast should suggest otherwise).
1 parent 590a404 commit 08a857e

2 files changed

Lines changed: 63 additions & 27 deletions

File tree

app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,11 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
287287
.coerceAtLeast(0L)
288288
.coerceAtMost(Long.MAX_VALUE / 1000L) * 1000L
289289
Log.d(TAG, "Command.Wait: Delaying the next takeScreenshot command by ${command.seconds} seconds.")
290-
showToast("Delaying next screenshot by ${command.seconds} seconds", false)
290+
if (currentModelSupportsScreenshot()) {
291+
showToast("Delaying next screenshot by ${command.seconds} seconds", false)
292+
} else {
293+
showToast("Waiting ${command.seconds} seconds", false)
294+
}
291295
false
292296
}
293297
is Command.PressHomeButton -> {
@@ -501,18 +505,50 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
501505
}
502506
}
503507

508+
/**
509+
* Resolves whether the currently active model (whichever of the three model
510+
* categories - custom JSON-defined model, JS-only online model, or native built-in
511+
* model/offline model - is active) actually supports screenshots/vision input.
512+
*
513+
* This is the single source of truth used everywhere a screenshot-related toast or
514+
* action might otherwise leak the impression that a screenshot was or will be taken
515+
* for a text-only model: Command.Wait's toast, the delayed-screenshot toast, and the
516+
* actual capture/MediaProjection-request decision in executeTakeScreenshotCommand.
517+
*/
518+
private fun currentModelSupportsScreenshot(): Boolean {
519+
com.google.ai.sample.util.CustomModelRegistry.getActiveModel()?.let { customModel ->
520+
return customModel.supportsScreenshot
521+
}
522+
523+
val currentModel = GenerativeAiViewModelFactory.getCurrentModel()
524+
val jsModelPrefs = applicationContext.getSharedPreferences("js_model_prefs", android.content.Context.MODE_PRIVATE)
525+
val jsOnlyModelId = jsModelPrefs.getString("js_only_model_id", null)
526+
527+
// JS-only online models (the normal WebView model dropdown) are a third, separate
528+
// case: they are not in CustomModelRegistry, and ModelOption.ONLINE_MODEL always
529+
// reports supportsScreenshot=true. Their real capability is persisted by the WebView
530+
// on model selection (dispatch("setJsOnlyModelSupportsScreenshot", ...)) - the same
531+
// flag PhotoReasoningViewModel already reads to decide whether to attach the image to
532+
// the outgoing payload. jsOnlyModelId being non-null is itself the reliable signal
533+
// that a JS-only model is active (it is cleared whenever a custom or native built-in
534+
// model is selected instead - see WebViewBridge.setSelectedModel) - it is checked on
535+
// its own, without also requiring currentModel == ONLINE_MODEL, because currentModel
536+
// is only updated by native built-in model selections and can otherwise still hold a
537+
// stale value (e.g. a previously selected offline model) while a JS-only model is the
538+
// one actually active.
539+
if (jsOnlyModelId != null) {
540+
return jsModelPrefs.getBoolean("js_only_supports_screenshot", true)
541+
}
542+
543+
return currentModel.supportsScreenshot
544+
}
545+
504546
private fun executeTakeScreenshotCommand(): Boolean {
505547
val delayMillis = pendingScreenshotDelayMillis
506548
pendingScreenshotDelayMillis = 0L
507549

508550
val captureAndRequestScreenshot = {
509-
val currentModel = GenerativeAiViewModelFactory.getCurrentModel()
510-
// A custom (JSON-defined) model, if active, overrides the stale native ModelOption's
511-
// flag here - otherwise the autonomous screenshot loop would silently never send
512-
// real screenshots to a custom vision model (it would fall back to text-only screen
513-
// info every time, regardless of "supportsScreenshot" in custom-models.json).
514-
val effectiveSupportsScreenshot = com.google.ai.sample.util.CustomModelRegistry.getActiveModel()
515-
?.supportsScreenshot ?: currentModel.supportsScreenshot
551+
val effectiveSupportsScreenshot = currentModelSupportsScreenshot()
516552

517553
val termuxOutput = TermuxOutputPreferences.consumeOutput(applicationContext)?.trim().orEmpty()
518554
val isTermuxOutputOnly = termuxOutput.isNotBlank()
@@ -522,8 +558,11 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
522558
Log.i(TAG, "executeTakeScreenshotCommand: Sending Termux output only without screenshot. chars=${termuxOutput.length}")
523559
"Termux output:\n$termuxOutput"
524560
} else {
561+
// Text-only model: no screenshot is taken, no MediaProjection is requested,
562+
// and no image is sent to the model - only plain screen-element text. No
563+
// toast is shown here: a toast would incorrectly suggest to the user that a
564+
// screenshot was captured and handed to the model.
525565
Log.d(TAG, "Command.TakeScreenshot: Model has no screenshot support, capturing screen info only.")
526-
showToast("Capturing screen info...", false)
527566
captureScreenInformation()
528567
}
529568

@@ -556,7 +595,11 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
556595
}
557596

558597
Log.d(TAG, "Command.TakeScreenshot: Waiting ${delayMillis}ms before capturing screen info and screenshot.")
559-
showToast("Waiting ${delayMillis / 1000L} seconds before screenshot...", false)
598+
if (currentModelSupportsScreenshot()) {
599+
showToast("Waiting ${delayMillis / 1000L} seconds before screenshot...", false)
600+
} else {
601+
showToast("Waiting ${delayMillis / 1000L} seconds...", false)
602+
}
560603
val delayedScreenshotRunnable = Runnable {
561604
pendingDelayedScreenshotRunnable = null
562605
captureAndRequestScreenshot()

index.html

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3700,24 +3700,17 @@
37003700
// No screenshot here – handled by auto-appended SCREENSHOT.
37013701
break;
37023702
}
3703-
case 'SCREENSHOT': {
3704-
// Kotlin pipeline: captures screen → addScreenshotToConversation → reason() → onCustomModelRequest
3705-
// Only take a screenshot if the current model supports it; text-only models
3706-
// must never trigger a screenshot (no toast, no capture, no image sent).
3707-
const _screenshotModelDef = MODELS.find(m => m.id === currentModelId);
3708-
const _screenshotSupported = _screenshotModelDef ? _screenshotModelDef.supportsScreenshot !== false : true;
3709-
if (_screenshotSupported) {
3710-
Bridge.requestScreenshot();
3711-
} else {
3712-
// Text-only model: trigger the next AI turn without a screenshot.
3713-
// Call onCustomModelRequest directly with no image payload so the
3714-
// conversation continues normally.
3715-
if (getInAndroid() && typeof Android.triggerNextTurnWithoutScreenshot === 'function') {
3716-
Android.triggerNextTurnWithoutScreenshot();
3717-
}
3718-
}
3703+
case 'SCREENSHOT':
3704+
// Kotlin pipeline: captures screen → addScreenshotToConversation → reason() → onCustomModelRequest.
3705+
// Always request here, for every model (text-only or not): the native side
3706+
// (executeTakeScreenshotCommand in ScreenOperatorAccessibilityService) resolves the
3707+
// model's real supportsScreenshot flag itself - including for JS-only online models,
3708+
// via the "js_only_supports_screenshot" SharedPreferences flag - and takes the
3709+
// no-capture/no-toast/no-MediaProjection branch for text-only models. The turn is still
3710+
// continued in that branch (addScreenshotToConversation with Uri.EMPTY -> reason()), so
3711+
// the conversation keeps going normally without ever taking a real screenshot.
3712+
Bridge.requestScreenshot();
37193713
break;
3720-
}
37213714
case 'COMPLETED':
37223715
Bridge.markCompleted();
37233716
_stopExecution = true;

0 commit comments

Comments
 (0)