From 408aef618293fa82d9dfd9930776bf3f9b63b31f Mon Sep 17 00:00:00 2001 From: drown0315 Date: Sun, 12 Jul 2026 15:42:02 +0800 Subject: [PATCH 1/4] fix(runner): wait for finder matches after UI updates --- docs-internal/flutter-pilot-prd.md | 4 +- lib/src/execution/scenario_runner.dart | 69 ++++++---- lib/src/runtime/pilot_runtime_adapter.dart | 13 ++ lib/src/runtime/runtime_contract.dart | 10 +- .../lib/src/pilot_runtime_binding.dart | 36 +++++ .../lib/src/pilot_runtime_client.dart | 13 ++ .../lib/src/pilot_runtime_protocol.dart | 8 ++ .../test/pilot_runtime_binding_test.dart | 27 ++++ .../test/pilot_runtime_client_test.dart | 27 ++++ test/execution/scenario_runner_test.dart | 124 +++++++++++++++++- test/runtime/pilot_runtime_adapter_test.dart | 21 +++ test/support/fake_runtime_adapter.dart | 22 ++++ 12 files changed, 341 insertions(+), 33 deletions(-) diff --git a/docs-internal/flutter-pilot-prd.md b/docs-internal/flutter-pilot-prd.md index 2e25db8..b183acc 100644 --- a/docs-internal/flutter-pilot-prd.md +++ b/docs-internal/flutter-pilot-prd.md @@ -144,8 +144,8 @@ The result is a reproducible bug report package that can be consumed by humans, - A Finder must resolve to exactly one widget before an action can execute. Zero matches fail the step as "Finder matched no widgets"; multiple matches fail the step as "Finder matched multiple widgets." Flutter Pilot does not automatically choose the first match. - The initial action set includes `tap`, `type`, `scroll`, `waitFor`, and `capture`. - The `type` action means replacing text in a widget: clear existing text directly, then enter the configured text one character at a time. It is distinct from the `byType` Finder constraint. -- The `waitFor` action waits for a Finder to produce exactly one match before its timeout. Zero matches keep waiting until timeout, one match succeeds, and multiple matches fail the step. The first version does not support waiting for disappearance, enabled state, or disabled state. -- `waitFor.timeoutMs` defaults to `3000` when omitted. The first version supports per-step timeout overrides but no global timeout defaults in the Scenario. +- Finder-backed `tap`, `type`, and targeted `scroll` actions wait for a Finder to produce exactly one match within a default `3000ms` budget. Before polling, the runner waits up to `500ms` for the current or next Flutter frame; that frame wait consumes the same action budget. Zero matches keep polling every `50ms` until timeout, one match executes, and multiple matches fail the Step immediately. +- The `waitFor` action uses the same frame synchronization and Finder polling behavior. `waitFor.timeoutMs` defaults to `3000` when omitted and replaces that Step's default budget when specified; it does not add another timeout. The first version does not support waiting for disappearance, enabled state, or disabled state, and exposes no global timeout defaults in the Scenario. - The `scroll` action accepts `deltaX` and `deltaY` as gesture drag deltas in logical pixels. Omitted deltas default to `0`. For example, `deltaY: -500` means dragging upward by 500 logical pixels, which usually reveals lower content. A Finder is optional for `scroll`; when omitted, Flutter Pilot scrolls the primary scrollable. When provided, the Finder must resolve to exactly one scrollable target. At least one of `deltaX` or `deltaY` must be non-zero, so `scroll: {}` and zero-delta scrolls are invalid. - Capture directives support screenshots, semantic snapshots, widget summaries, logs, and labels. Runtime errors are collected as part of logs in the first version. - Failed steps automatically trigger diagnostic capture even if the YAML did not request a capture at that point. diff --git a/lib/src/execution/scenario_runner.dart b/lib/src/execution/scenario_runner.dart index cf951ff..344bbf5 100644 --- a/lib/src/execution/scenario_runner.dart +++ b/lib/src/execution/scenario_runner.dart @@ -28,14 +28,17 @@ class ScenarioRunner { required this.adapter, this.recordingController, this.targetDevice, + this.finderTimeout = const Duration(seconds: 3), required this.outputDirectory, }); static const Duration _waitForPollInterval = Duration(milliseconds: 50); + static const Duration _endOfFrameTimeout = Duration(milliseconds: 500); final RuntimeAdapter adapter; final RecordingController? recordingController; final TargetDevice? targetDevice; + final Duration finderTimeout; final Directory outputDirectory; /// Execute Scenario Steps and write a run report. @@ -577,6 +580,7 @@ class ScenarioRunner { final FinderMatch match = await _resolveUniqueMatch( finder, actionName: actionName, + timeout: finderTimeout, ); await operation(match); return const _ActionExecutionResult(); @@ -586,43 +590,30 @@ class ScenarioRunner { Future _resolveUniqueMatch( Finder finder, { required String actionName, - }) async { - final List matches = await adapter.resolveFinder(finder); - if (matches.isEmpty) { - throw _StepFailureException( - actionName: actionName, - message: 'Finder matched no widgets.', - ); - } - if (matches.length > 1) { - throw _StepFailureException( - actionName: actionName, - message: 'Finder matched multiple widgets.', - ); - } - return matches.single; - } - - /// Poll a Finder until it has one unique match or the timeout expires. - Future<_ActionExecutionResult> _waitFor( - Finder finder, { required Duration timeout, }) async { final Stopwatch stopwatch = Stopwatch()..start(); + await _waitForEndOfFrame(stopwatch: stopwatch, timeout: timeout); while (true) { + if (stopwatch.elapsed >= timeout) { + throw _StepFailureException( + actionName: actionName, + message: 'Finder matched no widgets before timeout.', + ); + } final List matches = await adapter.resolveFinder(finder); if (matches.length == 1) { - return const _ActionExecutionResult(); + return matches.single; } if (matches.length > 1) { throw _StepFailureException( - actionName: 'waitFor', + actionName: actionName, message: 'Finder matched multiple widgets.', ); } if (stopwatch.elapsed >= timeout) { throw _StepFailureException( - actionName: 'waitFor', + actionName: actionName, message: 'Finder matched no widgets before timeout.', ); } @@ -630,6 +621,32 @@ class ScenarioRunner { } } + /// Wait for one frame without exceeding the Finder Action's total budget. + Future _waitForEndOfFrame({ + required Stopwatch stopwatch, + required Duration timeout, + }) async { + final Duration remaining = timeout - stopwatch.elapsed; + if (remaining <= Duration.zero) { + return; + } + final Duration frameTimeout = remaining < _endOfFrameTimeout + ? remaining + : _endOfFrameTimeout; + await adapter + .waitForEndOfFrame(timeout: frameTimeout) + .timeout(frameTimeout, onTimeout: () {}); + } + + /// Poll a Finder until it has one unique match or the timeout expires. + Future<_ActionExecutionResult> _waitFor( + Finder finder, { + required Duration timeout, + }) async { + await _resolveUniqueMatch(finder, actionName: 'waitFor', timeout: timeout); + return const _ActionExecutionResult(); + } + /// Execute a scroll action, resolving its optional Finder when provided. Future<_ActionExecutionResult> _executeScroll({ required Finder? finder, @@ -638,7 +655,11 @@ class ScenarioRunner { }) async { FinderMatch? match; if (finder != null) { - match = await _resolveUniqueMatch(finder, actionName: 'scroll'); + match = await _resolveUniqueMatch( + finder, + actionName: 'scroll', + timeout: finderTimeout, + ); } await adapter.performScroll(match: match, deltaX: deltaX, deltaY: deltaY); return const _ActionExecutionResult(); diff --git a/lib/src/runtime/pilot_runtime_adapter.dart b/lib/src/runtime/pilot_runtime_adapter.dart index bd9da69..1cfa6af 100644 --- a/lib/src/runtime/pilot_runtime_adapter.dart +++ b/lib/src/runtime/pilot_runtime_adapter.dart @@ -50,6 +50,19 @@ class PilotRuntimeAdapter implements RuntimeAdapter { await _disposeClient?.call(); } + @override + Future waitForEndOfFrame({required Duration timeout}) async { + try { + await _client.waitForEndOfFrame(timeout: timeout); + } catch (error) { + throw RuntimeOperationException( + operation: RuntimeOperation.waitForEndOfFrame, + message: error.toString(), + cause: error, + ); + } + } + @override Future> resolveFinder(Finder finder) async { final List matches = await _client.resolveFinder( diff --git a/lib/src/runtime/runtime_contract.dart b/lib/src/runtime/runtime_contract.dart index 240ef37..e9740ed 100644 --- a/lib/src/runtime/runtime_contract.dart +++ b/lib/src/runtime/runtime_contract.dart @@ -40,6 +40,12 @@ abstract interface class RuntimeAdapter { /// diagnostic based on whether the run had already failed. Future dispose(); + /// Wait for the current or next Flutter frame to finish. + /// + /// `timeout` bounds this synchronization attempt. Timing out is not a + /// Runtime operation failure; callers may continue with condition polling. + Future waitForEndOfFrame({required Duration timeout}); + /// Return all widgets that satisfy a Flutter Pilot Finder. /// /// Args: @@ -48,7 +54,8 @@ abstract interface class RuntimeAdapter { /// /// Returns: /// A complete list of Finder Matches. The runner applies cardinality rules: - /// zero matches fail, one match executes, and multiple matches fail. + /// zero matches keep polling until the Step budget expires, one match + /// executes, and multiple matches fail immediately. Future> resolveFinder(Finder finder); /// Tap the widget represented by a Finder Match from the current Step. @@ -177,6 +184,7 @@ class LogsCapture { /// The enum keeps failure reporting stable and lets runner/report code group /// failures without comparing string literals. enum RuntimeOperation { + waitForEndOfFrame, resolveFinder, performTap, clearText, diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart index c01c46f..ce1c874 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart @@ -4,6 +4,7 @@ import 'dart:developer'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; import 'finder_resolver.dart'; import 'pilot_runtime_protocol.dart'; @@ -75,6 +76,7 @@ class PilotRuntimeBinding { PilotRuntimeProtocol.resolveFinderExtension, _handleResolveFinder, ); + registrar(PilotRuntimeProtocol.endOfFrameExtension, _handleEndOfFrame); registrar(PilotRuntimeProtocol.tapExtension, _handleTap); registrar(PilotRuntimeProtocol.clearTextExtension, _handleClearText); registrar(PilotRuntimeProtocol.enterTextExtension, _handleEnterText); @@ -122,6 +124,20 @@ class PilotRuntimeBinding { ); } + static Future> _handleEndOfFrame( + Map parameters, + ) async { + final int timeoutMs = _requiredInt(parameters, 'timeoutMs', 'endOfFrame'); + bool timedOut = false; + await WidgetsBinding.instance.endOfFrame.timeout( + Duration(milliseconds: timeoutMs), + onTimeout: () { + timedOut = true; + }, + ); + return {'ok': true, 'timedOut': timedOut}; + } + static Future> _handleTap( Map parameters, ) async { @@ -274,6 +290,26 @@ class PilotRuntimeBinding { throw FormatException('$operation parameter $field must be a number.'); } + static int _requiredInt( + Map parameters, + String field, + String operation, + ) { + final Object? value = parameters[field]; + if (value is int && value > 0) { + return value; + } + if (value is String) { + final int? parsed = int.tryParse(value); + if (parsed != null && parsed > 0) { + return parsed; + } + } + throw FormatException( + '$operation parameter $field must be a positive integer.', + ); + } + static void _registerVmServiceExtension( String extensionName, PilotRuntimeExtensionHandler handler, diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart index 3c4f79b..0adb52b 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart @@ -616,6 +616,19 @@ class PilotRuntimeClient { return List.unmodifiable(matches); } + /// Wait for the Runtime Target's current or next Flutter frame to finish. + /// + /// `timeout` bounds the app-side wait. A frame timeout is a successful + /// synchronization attempt so callers can continue with condition polling. + Future waitForEndOfFrame({required Duration timeout}) async { + await _vmService.callServiceExtension( + PilotRuntimeProtocol.endOfFrameExtension, + parameters: { + 'timeoutMs': timeout.inMilliseconds.clamp(1, 0x7fffffff), + }, + ); + } + /// Tap one Runtime Handle returned by Finder resolution. /// /// Args: diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart b/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart index a4931de..95c6765 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart @@ -16,6 +16,10 @@ class PilotRuntimeProtocol { static const String resolveFinderExtension = 'ext.flutter_pilot.runtime.resolveFinder'; + /// VM Service extension used to wait for the current or next frame. + static const String endOfFrameExtension = + 'ext.flutter_pilot.runtime.endOfFrame'; + /// VM Service extension used to tap one resolved Runtime Handle. static const String tapExtension = 'ext.flutter_pilot.runtime.tap'; @@ -40,6 +44,9 @@ class PilotRuntimeProtocol { /// Capability name reported when visible Finder resolution is available. static const String resolveFinderCapability = 'runtime.finder.resolve'; + /// Capability name reported when frame synchronization is available. + static const String endOfFrameCapability = 'runtime.frame.end'; + /// Capability name reported when tap replay is available. static const String tapCapability = 'runtime.action.tap'; @@ -59,6 +66,7 @@ class PilotRuntimeProtocol { static const Set requiredCapabilities = { handshakeCapability, resolveFinderCapability, + endOfFrameCapability, tapCapability, clearTextCapability, enterTextCapability, diff --git a/packages/pilot_runtime/test/pilot_runtime_binding_test.dart b/packages/pilot_runtime/test/pilot_runtime_binding_test.dart index 3d12da1..b10c85e 100644 --- a/packages/pilot_runtime/test/pilot_runtime_binding_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_binding_test.dart @@ -24,6 +24,7 @@ void main() { expect(registeredExtensions, [ PilotRuntimeProtocol.handshakeExtension, PilotRuntimeProtocol.resolveFinderExtension, + PilotRuntimeProtocol.endOfFrameExtension, PilotRuntimeProtocol.tapExtension, PilotRuntimeProtocol.clearTextExtension, PilotRuntimeProtocol.enterTextExtension, @@ -42,6 +43,7 @@ void main() { 'runtime.action.scroll', 'runtime.action.tap', 'runtime.finder.resolve', + 'runtime.frame.end', 'runtime.handshake', 'runtime.logs.collect', ], @@ -49,6 +51,31 @@ void main() { ); }); + testWidgets('bounds end-of-frame waiting by timeoutMs', ( + WidgetTester tester, + ) async { + final Map registeredHandlers = + {}; + PilotRuntimeBinding.ensureInitialized( + debugMode: true, + registerExtension: + (String extensionName, PilotRuntimeExtensionHandler handler) { + registeredHandlers[extensionName] = handler; + }, + ); + + final Map response = + await tester.runAsync( + () => registeredHandlers[PilotRuntimeProtocol.endOfFrameExtension]!( + {'timeoutMs': '1'}, + ), + ) ?? + {}; + + expect(response['ok'], true); + expect(response['timedOut'], true); + }); + test('is a no-op outside debug mode', () { final List registeredExtensions = []; diff --git a/packages/pilot_runtime/test/pilot_runtime_client_test.dart b/packages/pilot_runtime/test/pilot_runtime_client_test.dart index f9bf25b..7937e0e 100644 --- a/packages/pilot_runtime/test/pilot_runtime_client_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_client_test.dart @@ -14,6 +14,7 @@ void main() { 'runtime.action.scroll', 'runtime.action.tap', 'runtime.finder.resolve', + 'runtime.frame.end', 'runtime.handshake', 'runtime.logs.collect', ], @@ -26,6 +27,7 @@ void main() { expect(session.protocolVersion, 1); expect(session.capabilities, contains('runtime.handshake')); expect(session.capabilities, contains('runtime.finder.resolve')); + expect(session.capabilities, contains('runtime.frame.end')); expect(session.capabilities, contains('runtime.action.tap')); expect(session.capabilities, contains('runtime.action.clearText')); expect(session.capabilities, contains('runtime.action.enterText')); @@ -121,6 +123,31 @@ void main() { }); }); + group('PilotRuntimeClient frame synchronization', () { + test('passes the frame timeout to the runtime extension', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + extensionResponses: >{ + PilotRuntimeProtocol.endOfFrameExtension: { + 'ok': true, + 'timedOut': false, + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await client.waitForEndOfFrame( + timeout: const Duration(milliseconds: 375), + ); + + expect(vmService.calledExtensions, [ + PilotRuntimeProtocol.endOfFrameExtension, + ]); + expect(vmService.calledParameters.single, { + 'timeoutMs': 375, + }); + }); + }); + group('PilotRuntimeClient Finder resolution', () { test('decodes Finder Matches from the runtime extension', () async { final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( diff --git a/test/execution/scenario_runner_test.dart b/test/execution/scenario_runner_test.dart index 2a769d2..faf69d0 100644 --- a/test/execution/scenario_runner_test.dart +++ b/test/execution/scenario_runner_test.dart @@ -267,6 +267,7 @@ void main() { final ScenarioRunReport report = await ScenarioRunner( adapter: adapter, + finderTimeout: const Duration(milliseconds: 1), outputDirectory: outputDirectory, ).run(scenario, onProgress: progressEvents.add); @@ -1027,6 +1028,7 @@ steps: await FileTestkit.runZoned(() async { final Directory outputDirectory = Directory('wait_for_success_output'); final FakeRuntimeAdapter adapter = FakeRuntimeAdapter( + recordEndOfFrameWaits: true, finderResultSequences: >>{ 'loading_done': >[ const [], @@ -1060,6 +1062,7 @@ steps: adapter.events.map((FakeRuntimeEvent event) => event.operation), [ RuntimeOperation.initialize, + RuntimeOperation.waitForEndOfFrame, RuntimeOperation.resolveFinder, RuntimeOperation.resolveFinder, RuntimeOperation.dispose, @@ -1068,6 +1071,46 @@ steps: }); }); + test('waits for a unique Finder Match before tapping', () async { + await FileTestkit.runZoned(() async { + final Directory outputDirectory = Directory('tap_wait_output'); + final FakeRuntimeAdapter adapter = FakeRuntimeAdapter( + finderResultSequences: >>{ + 'continue_button': >[ + const [], + const [FinderMatch(id: 'continue-match')], + ], + }, + ); + final Scenario scenario = Scenario( + name: 'tap_after_widget_appears', + steps: const [ + ScenarioStep( + index: 1, + action: TapAction(finder: Finder(byText: 'continue_button')), + ), + ], + ); + + final ScenarioRunReport report = await ScenarioRunner( + adapter: adapter, + outputDirectory: outputDirectory, + ).run(scenario); + + expect(report.status, ScenarioRunStatus.passed); + expect( + adapter.events.map((FakeRuntimeEvent event) => event.operation), + [ + RuntimeOperation.initialize, + RuntimeOperation.resolveFinder, + RuntimeOperation.resolveFinder, + RuntimeOperation.performTap, + RuntimeOperation.dispose, + ], + ); + }); + }); + test( 'stops after the selected Step number and reports later Steps skipped', () async { @@ -1389,6 +1432,54 @@ steps: }); }); + test('endOfFrame consumes the waitFor timeout budget', () async { + await FileTestkit.runZoned(() async { + final Directory outputDirectory = Directory('frame_budget_output'); + final FakeRuntimeAdapter adapter = FakeRuntimeAdapter( + recordEndOfFrameWaits: true, + endOfFrameDelay: const Duration(milliseconds: 50), + ); + final Scenario scenario = Scenario( + name: 'frame_consumes_wait_budget', + steps: const [ + ScenarioStep( + index: 1, + action: WaitForAction( + finder: Finder(byText: 'home'), + timeoutMs: 10, + ), + ), + ], + ); + + final ScenarioRunReport report = await ScenarioRunner( + adapter: adapter, + outputDirectory: outputDirectory, + ).run(scenario); + + expect(report.status, ScenarioRunStatus.failed); + expect( + report.steps.single.failureReason, + 'Finder matched no widgets before timeout.', + ); + expect( + adapter.events.map((FakeRuntimeEvent event) => event.operation), + [ + RuntimeOperation.initialize, + RuntimeOperation.waitForEndOfFrame, + RuntimeOperation.captureScreenshot, + RuntimeOperation.captureWidgetTree, + RuntimeOperation.collectLogs, + RuntimeOperation.dispose, + ], + ); + expect( + adapter.events[1].duration, + lessThanOrEqualTo(const Duration(milliseconds: 10)), + ); + }); + }); + test('fails waitFor when multiple widgets match', () async { await FileTestkit.runZoned(() async { final Directory outputDirectory = Directory('wait_for_multiple_output'); @@ -1454,12 +1545,16 @@ steps: final ScenarioRunReport report = await ScenarioRunner( adapter: adapter, + finderTimeout: const Duration(milliseconds: 1), outputDirectory: outputDirectory, ).run(scenario); expect(report.status, ScenarioRunStatus.failed); expect(report.steps.single.status, StepStatus.failed); - expect(report.steps.single.failureReason, 'Finder matched no widgets.'); + expect( + report.steps.single.failureReason, + 'Finder matched no widgets before timeout.', + ); expect( adapter.events.map((FakeRuntimeEvent event) => event.operation), [ @@ -1476,7 +1571,9 @@ steps: expect(reportFile.readAsStringSync(), contains('"status": "failed"')); expect( reportFile.readAsStringSync(), - contains('"failureReason": "Finder matched no widgets."'), + contains( + '"failureReason": "Finder matched no widgets before timeout."', + ), ); }); }); @@ -1534,13 +1631,17 @@ steps: final ScenarioRunReport report = await ScenarioRunner( adapter: adapter, + finderTimeout: const Duration(milliseconds: 1), outputDirectory: outputDirectory, ).run(scenario); expect(report.status, ScenarioRunStatus.failed); expect(report.steps, hasLength(1)); expect(report.steps.single.status, StepStatus.failed); - expect(report.steps.single.failureReason, 'Finder matched no widgets.'); + expect( + report.steps.single.failureReason, + 'Finder matched no widgets before timeout.', + ); expect( report.steps.single.artifacts.map( (ArtifactReport artifact) => artifact.type, @@ -1625,11 +1726,15 @@ steps: final ScenarioRunReport report = await ScenarioRunner( adapter: adapter, + finderTimeout: const Duration(milliseconds: 1), outputDirectory: outputDirectory, ).run(scenario); expect(report.status, ScenarioRunStatus.failed); - expect(report.steps.single.failureReason, 'Finder matched no widgets.'); + expect( + report.steps.single.failureReason, + 'Finder matched no widgets before timeout.', + ); expect( report.steps.single.diagnosticFailureReason, 'Screenshot RPC failed.', @@ -1644,7 +1749,9 @@ steps: final String reportJson = _runReportFile(report).readAsStringSync(); expect( reportJson, - contains('"failureReason": "Finder matched no widgets."'), + contains( + '"failureReason": "Finder matched no widgets before timeout."', + ), ); expect( reportJson, @@ -1677,6 +1784,7 @@ steps: final ScenarioRunReport report = await ScenarioRunner( adapter: adapter, + finderTimeout: const Duration(milliseconds: 1), outputDirectory: outputDirectory, ).run(scenario); @@ -1725,11 +1833,15 @@ steps: final ScenarioRunReport report = await ScenarioRunner( adapter: adapter, + finderTimeout: const Duration(milliseconds: 1), outputDirectory: outputDirectory, ).run(scenario); expect(report.status, ScenarioRunStatus.failed); - expect(report.steps.single.failureReason, 'Finder matched no widgets.'); + expect( + report.steps.single.failureReason, + 'Finder matched no widgets before timeout.', + ); final File reportFile = _runReportFile(report); expect( diff --git a/test/runtime/pilot_runtime_adapter_test.dart b/test/runtime/pilot_runtime_adapter_test.dart index 12476fe..ef10648 100644 --- a/test/runtime/pilot_runtime_adapter_test.dart +++ b/test/runtime/pilot_runtime_adapter_test.dart @@ -75,6 +75,20 @@ void main() { }, ); + test('forwards end-of-frame timeout to pilot_runtime', () async { + final _FakePilotRuntimeClient client = _FakePilotRuntimeClient(); + final PilotRuntimeAdapter adapter = PilotRuntimeAdapter( + client: client, + projectRoot: '/target/app', + ); + + await adapter.waitForEndOfFrame(timeout: const Duration(milliseconds: 375)); + + expect(client.endOfFrameTimeouts, [ + const Duration(milliseconds: 375), + ]); + }); + test('returns Widget Tree capture data from pilot_runtime', () async { final _FakePilotRuntimeClient client = _FakePilotRuntimeClient( widgetTree: { @@ -502,6 +516,7 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { <({String? byText, String? byType, String? byKey, String? byWidget})>[]; final List tapHandles = []; final List clearTextHandles = []; + final List endOfFrameTimeouts = []; final List<({String handle, String text})> enterTextRequests = <({String handle, String text})>[]; final List<({String? handle, double dx, double dy})> scrollRequests = @@ -521,6 +536,7 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { 'runtime.action.scroll', 'runtime.action.tap', 'runtime.finder.resolve', + 'runtime.frame.end', 'runtime.handshake', 'runtime.logs.collect', }, @@ -541,6 +557,11 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { return logs; } + @override + Future waitForEndOfFrame({required Duration timeout}) async { + endOfFrameTimeouts.add(timeout); + } + @override Future hotReload() async { return const PilotRuntimeReloadResult( diff --git a/test/support/fake_runtime_adapter.dart b/test/support/fake_runtime_adapter.dart index aea0a49..58c15f1 100644 --- a/test/support/fake_runtime_adapter.dart +++ b/test/support/fake_runtime_adapter.dart @@ -16,6 +16,8 @@ class FakeRuntimeAdapter implements RuntimeAdapter { WidgetTreeCapture? widgetTree, LogsCapture? logs, Map? failures, + this.recordEndOfFrameWaits = false, + this.endOfFrameDelay = Duration.zero, }) : finderResults = finderResults ?? >{}, finderResultSequences = finderResultSequences ?? >>{}, @@ -45,6 +47,8 @@ class FakeRuntimeAdapter implements RuntimeAdapter { final WidgetTreeCapture widgetTree; final LogsCapture logs; final Map failures; + final bool recordEndOfFrameWaits; + final Duration endOfFrameDelay; final List events = []; final Map _finderSequenceOffsets = {}; @@ -60,6 +64,22 @@ class FakeRuntimeAdapter implements RuntimeAdapter { events.add(const FakeRuntimeEvent(operation: RuntimeOperation.dispose)); } + @override + Future waitForEndOfFrame({required Duration timeout}) async { + _throwIfConfigured(RuntimeOperation.waitForEndOfFrame); + if (recordEndOfFrameWaits) { + events.add( + FakeRuntimeEvent( + operation: RuntimeOperation.waitForEndOfFrame, + duration: timeout, + ), + ); + } + if (endOfFrameDelay > Duration.zero) { + await Future.delayed(endOfFrameDelay); + } + } + @override Future> resolveFinder(Finder finder) async { _throwIfConfigured(RuntimeOperation.resolveFinder); @@ -196,6 +216,7 @@ class FakeRuntimeEvent { this.text, this.deltaX, this.deltaY, + this.duration, }); final RuntimeOperation operation; @@ -204,4 +225,5 @@ class FakeRuntimeEvent { final String? text; final double? deltaX; final double? deltaY; + final Duration? duration; } From 2759320caa897f3fc7f61310fdf02e4017850052 Mon Sep 17 00:00:00 2001 From: drown0315 Date: Sun, 12 Jul 2026 17:15:54 +0800 Subject: [PATCH 2/4] fix(runtime): select outer primary scrollable --- docs-internal/flutter-pilot-prd.md | 2 +- .../lib/src/scroll_performer.dart | 86 ++++++++++++--- .../test/pilot_runtime_finder_test.dart | 103 ++++++++++++++++++ 3 files changed, 175 insertions(+), 16 deletions(-) diff --git a/docs-internal/flutter-pilot-prd.md b/docs-internal/flutter-pilot-prd.md index b183acc..95e5ac3 100644 --- a/docs-internal/flutter-pilot-prd.md +++ b/docs-internal/flutter-pilot-prd.md @@ -146,7 +146,7 @@ The result is a reproducible bug report package that can be consumed by humans, - The `type` action means replacing text in a widget: clear existing text directly, then enter the configured text one character at a time. It is distinct from the `byType` Finder constraint. - Finder-backed `tap`, `type`, and targeted `scroll` actions wait for a Finder to produce exactly one match within a default `3000ms` budget. Before polling, the runner waits up to `500ms` for the current or next Flutter frame; that frame wait consumes the same action budget. Zero matches keep polling every `50ms` until timeout, one match executes, and multiple matches fail the Step immediately. - The `waitFor` action uses the same frame synchronization and Finder polling behavior. `waitFor.timeoutMs` defaults to `3000` when omitted and replaces that Step's default budget when specified; it does not add another timeout. The first version does not support waiting for disappearance, enabled state, or disabled state, and exposes no global timeout defaults in the Scenario. -- The `scroll` action accepts `deltaX` and `deltaY` as gesture drag deltas in logical pixels. Omitted deltas default to `0`. For example, `deltaY: -500` means dragging upward by 500 logical pixels, which usually reveals lower content. A Finder is optional for `scroll`; when omitted, Flutter Pilot scrolls the primary scrollable. When provided, the Finder must resolve to exactly one scrollable target. At least one of `deltaX` or `deltaY` must be non-zero, so `scroll: {}` and zero-delta scrolls are invalid. +- The `scroll` action accepts `deltaX` and `deltaY` as gesture drag deltas in logical pixels. Omitted deltas default to `0`. For example, `deltaY: -500` means dragging upward by 500 logical pixels, which usually reveals lower content. A Finder is optional for `scroll`; when omitted, Flutter Pilot selects the unique outermost visible scrollable matching the dominant drag axis and ignores nested scrollables on that axis. Multiple peer candidates remain ambiguous and require a Finder. When provided, the Finder must resolve to exactly one scrollable target. At least one of `deltaX` or `deltaY` must be non-zero, so `scroll: {}` and zero-delta scrolls are invalid. - Capture directives support screenshots, semantic snapshots, widget summaries, logs, and labels. Runtime errors are collected as part of logs in the first version. - Failed steps automatically trigger diagnostic capture even if the YAML did not request a capture at that point. - Scenario execution produces a run directory containing a structured run report, an HTML timeline report, aggregated Step metadata, and capture artifacts. diff --git a/packages/pilot_runtime/lib/src/scroll_performer.dart b/packages/pilot_runtime/lib/src/scroll_performer.dart index e11dd3e..aba9e89 100644 --- a/packages/pilot_runtime/lib/src/scroll_performer.dart +++ b/packages/pilot_runtime/lib/src/scroll_performer.dart @@ -29,7 +29,7 @@ class PilotRuntimeScrollPerformer { required double deltaY, }) async { final Element? scrollableElement = handle == null - ? _primaryScrollable() + ? _primaryScrollable(deltaX: deltaX, deltaY: deltaY) : _scrollableForHandle(handle); if (scrollableElement == null) { return handle == null @@ -45,8 +45,8 @@ class PilotRuntimeScrollPerformer { ); } - final Offset? center = _centerFor(scrollableElement); - if (center == null) { + final Offset? dragStart = _dragStartFor(scrollableElement); + if (dragStart == null) { return _failure( code: handle == null ? 'primaryScrollableUnavailable' : 'notScrollable', message: handle == null @@ -57,33 +57,33 @@ class PilotRuntimeScrollPerformer { final int pointer = _nextPointer++; GestureBinding.instance.handlePointerEvent( - PointerAddedEvent(pointer: pointer, position: center), + PointerAddedEvent(pointer: pointer, position: dragStart), ); GestureBinding.instance.handlePointerEvent( PointerDownEvent( pointer: pointer, - position: center, + position: dragStart, buttons: kPrimaryButton, ), ); final Offset firstDelta = Offset(deltaX, deltaY) / 2.0; final Offset secondDelta = Offset(deltaX, deltaY) - firstDelta; - _movePointer(pointer: pointer, from: center, delta: firstDelta); + _movePointer(pointer: pointer, from: dragStart, delta: firstDelta); _movePointer( pointer: pointer, - from: center + firstDelta, + from: dragStart + firstDelta, delta: secondDelta, ); GestureBinding.instance.handlePointerEvent( PointerUpEvent( pointer: pointer, - position: center + Offset(deltaX, deltaY), + position: dragStart + Offset(deltaX, deltaY), ), ); GestureBinding.instance.handlePointerEvent( PointerRemovedEvent( pointer: pointer, - position: center + Offset(deltaX, deltaY), + position: dragStart + Offset(deltaX, deltaY), ), ); return {'ok': true, 'method': 'pointer'}; @@ -132,20 +132,43 @@ class PilotRuntimeScrollPerformer { return scrollable; } - static Element? _primaryScrollable() { + static Element? _primaryScrollable({ + required double deltaX, + required double deltaY, + }) { + final Axis preferredAxis = deltaY.abs() >= deltaX.abs() + ? Axis.vertical + : Axis.horizontal; final List scrollables = []; PilotRuntimeFinderResolver.visitVisibleElements((Element element) { - if (element.widget is Scrollable) { + final Widget widget = element.widget; + if (widget is Scrollable && + axisDirectionToAxis(widget.axisDirection) == preferredAxis) { scrollables.add(element); } }); - if (scrollables.length != 1) { + final Set candidates = Set.identity() + ..addAll(scrollables); + final List outermostScrollables = scrollables + .where((Element element) { + bool hasScrollableAncestor = false; + element.visitAncestorElements((Element ancestor) { + if (candidates.contains(ancestor)) { + hasScrollableAncestor = true; + return false; + } + return true; + }); + return !hasScrollableAncestor; + }) + .toList(growable: false); + if (outermostScrollables.length != 1) { return null; } - return scrollables.single; + return outermostScrollables.single; } - static Offset? _centerFor(Element element) { + static Offset? _dragStartFor(Element element) { final RenderObject? renderObject = element.renderObject; if (renderObject is! RenderBox || !renderObject.hasSize) { return null; @@ -153,7 +176,40 @@ class PilotRuntimeScrollPerformer { if (renderObject.size.isEmpty) { return null; } - return renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + final Rect bounds = + renderObject.localToGlobal(Offset.zero) & renderObject.size; + final List nestedScrollableBounds = []; + element.visitChildren((Element child) { + _collectScrollableBounds(child, nestedScrollableBounds); + }); + final List candidates = [ + bounds.center, + Offset(bounds.center.dx, bounds.top + bounds.height * 0.25), + Offset(bounds.center.dx, bounds.top + bounds.height * 0.75), + Offset(bounds.left + bounds.width * 0.25, bounds.center.dy), + Offset(bounds.left + bounds.width * 0.75, bounds.center.dy), + ]; + for (final Offset candidate in candidates) { + if (!nestedScrollableBounds.any( + (Rect nested) => nested.contains(candidate), + )) { + return candidate; + } + } + return null; + } + + static void _collectScrollableBounds(Element element, List bounds) { + final RenderObject? renderObject = element.renderObject; + if (element.widget is Scrollable && + renderObject is RenderBox && + renderObject.hasSize && + !renderObject.size.isEmpty) { + bounds.add(renderObject.localToGlobal(Offset.zero) & renderObject.size); + } + element.visitChildren((Element child) { + _collectScrollableBounds(child, bounds); + }); } static Map _failure({ diff --git a/packages/pilot_runtime/test/pilot_runtime_finder_test.dart b/packages/pilot_runtime/test/pilot_runtime_finder_test.dart index 67faef1..e55e986 100644 --- a/packages/pilot_runtime/test/pilot_runtime_finder_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_finder_test.dart @@ -692,6 +692,109 @@ void main() { expect(controller.offset, greaterThan(0)); }); + testWidgets( + 'untargeted scroll chooses an outer scrollable over a nested one', + (WidgetTester tester) async { + final Map extensions = + _registerRuntimeExtensions(); + final ScrollController outerController = ScrollController(); + final ScrollController innerController = ScrollController(); + addTearDown(outerController.dispose); + addTearDown(innerController.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SizedBox( + height: 300, + child: ListView( + controller: outerController, + children: [ + const SizedBox(height: 120, child: Text('Header')), + SizedBox( + height: 140, + child: ListView.builder( + controller: innerController, + itemCount: 20, + itemBuilder: (BuildContext context, int index) { + return SizedBox( + height: 40, + child: Text('Nested $index'), + ); + }, + ), + ), + const SizedBox(height: 800, child: Text('Page tail')), + ], + ), + ), + ), + ), + ); + + final Map scrollResponse = await _scroll( + extensions, + deltaX: 0, + deltaY: -120, + ); + await tester.pumpAndSettle(); + + expect(scrollResponse['ok'], true); + expect(outerController.offset, greaterThan(0)); + expect(innerController.offset, 0); + }, + ); + + testWidgets('untargeted scroll selects the scrollable matching its axis', ( + WidgetTester tester, + ) async { + final Map extensions = + _registerRuntimeExtensions(); + final ScrollController horizontalController = ScrollController(); + final ScrollController verticalController = ScrollController(); + addTearDown(horizontalController.dispose); + addTearDown(verticalController.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: Row( + children: [ + Expanded( + child: ListView.builder( + controller: horizontalController, + scrollDirection: Axis.horizontal, + itemCount: 20, + itemBuilder: (BuildContext context, int index) { + return SizedBox(width: 80, child: Text('Across $index')); + }, + ), + ), + Expanded( + child: ListView.builder( + controller: verticalController, + itemCount: 20, + itemBuilder: (BuildContext context, int index) { + return SizedBox(height: 80, child: Text('Down $index')); + }, + ), + ), + ], + ), + ), + ), + ); + + final Map scrollResponse = await _scroll( + extensions, + deltaX: 0, + deltaY: -120, + ); + await tester.pumpAndSettle(); + + expect(scrollResponse['ok'], true); + expect(horizontalController.offset, 0); + expect(verticalController.offset, greaterThan(0)); + }); + testWidgets('scroll fails when primary scrollable is ambiguous', ( WidgetTester tester, ) async { From 0258f7b05cb72b793c577fea17d0c8802b9ff3cf Mon Sep 17 00:00:00 2001 From: drown0315 Date: Sun, 12 Jul 2026 18:25:30 +0800 Subject: [PATCH 3/4] no-mistakes(review): Allow cross-axis nested scroll drag starts --- .../lib/src/scroll_performer.dart | 33 +++++++------ .../test/pilot_runtime_finder_test.dart | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/packages/pilot_runtime/lib/src/scroll_performer.dart b/packages/pilot_runtime/lib/src/scroll_performer.dart index aba9e89..b55a1cd 100644 --- a/packages/pilot_runtime/lib/src/scroll_performer.dart +++ b/packages/pilot_runtime/lib/src/scroll_performer.dart @@ -28,8 +28,11 @@ class PilotRuntimeScrollPerformer { required double deltaX, required double deltaY, }) async { + final Axis dragAxis = deltaY.abs() >= deltaX.abs() + ? Axis.vertical + : Axis.horizontal; final Element? scrollableElement = handle == null - ? _primaryScrollable(deltaX: deltaX, deltaY: deltaY) + ? _primaryScrollable(dragAxis) : _scrollableForHandle(handle); if (scrollableElement == null) { return handle == null @@ -45,7 +48,7 @@ class PilotRuntimeScrollPerformer { ); } - final Offset? dragStart = _dragStartFor(scrollableElement); + final Offset? dragStart = _dragStartFor(scrollableElement, dragAxis); if (dragStart == null) { return _failure( code: handle == null ? 'primaryScrollableUnavailable' : 'notScrollable', @@ -132,18 +135,12 @@ class PilotRuntimeScrollPerformer { return scrollable; } - static Element? _primaryScrollable({ - required double deltaX, - required double deltaY, - }) { - final Axis preferredAxis = deltaY.abs() >= deltaX.abs() - ? Axis.vertical - : Axis.horizontal; + static Element? _primaryScrollable(Axis dragAxis) { final List scrollables = []; PilotRuntimeFinderResolver.visitVisibleElements((Element element) { final Widget widget = element.widget; if (widget is Scrollable && - axisDirectionToAxis(widget.axisDirection) == preferredAxis) { + axisDirectionToAxis(widget.axisDirection) == dragAxis) { scrollables.add(element); } }); @@ -168,7 +165,7 @@ class PilotRuntimeScrollPerformer { return outermostScrollables.single; } - static Offset? _dragStartFor(Element element) { + static Offset? _dragStartFor(Element element, Axis dragAxis) { final RenderObject? renderObject = element.renderObject; if (renderObject is! RenderBox || !renderObject.hasSize) { return null; @@ -180,7 +177,7 @@ class PilotRuntimeScrollPerformer { renderObject.localToGlobal(Offset.zero) & renderObject.size; final List nestedScrollableBounds = []; element.visitChildren((Element child) { - _collectScrollableBounds(child, nestedScrollableBounds); + _collectScrollableBounds(child, dragAxis, nestedScrollableBounds); }); final List candidates = [ bounds.center, @@ -199,16 +196,22 @@ class PilotRuntimeScrollPerformer { return null; } - static void _collectScrollableBounds(Element element, List bounds) { + static void _collectScrollableBounds( + Element element, + Axis dragAxis, + List bounds, + ) { final RenderObject? renderObject = element.renderObject; - if (element.widget is Scrollable && + final Widget widget = element.widget; + if (widget is Scrollable && + axisDirectionToAxis(widget.axisDirection) == dragAxis && renderObject is RenderBox && renderObject.hasSize && !renderObject.size.isEmpty) { bounds.add(renderObject.localToGlobal(Offset.zero) & renderObject.size); } element.visitChildren((Element child) { - _collectScrollableBounds(child, bounds); + _collectScrollableBounds(child, dragAxis, bounds); }); } diff --git a/packages/pilot_runtime/test/pilot_runtime_finder_test.dart b/packages/pilot_runtime/test/pilot_runtime_finder_test.dart index e55e986..a6d1dc5 100644 --- a/packages/pilot_runtime/test/pilot_runtime_finder_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_finder_test.dart @@ -744,6 +744,54 @@ void main() { }, ); + testWidgets('untargeted scroll allows a cross-axis nested scrollable', ( + WidgetTester tester, + ) async { + final Map extensions = + _registerRuntimeExtensions(); + final ScrollController outerController = ScrollController(); + final ScrollController innerController = ScrollController(); + addTearDown(outerController.dispose); + addTearDown(innerController.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SizedBox( + height: 300, + child: ListView( + controller: outerController, + children: [ + SizedBox( + height: 300, + child: ListView.builder( + controller: innerController, + scrollDirection: Axis.horizontal, + itemCount: 20, + itemBuilder: (BuildContext context, int index) { + return SizedBox(width: 80, child: Text('Page $index')); + }, + ), + ), + const SizedBox(height: 800, child: Text('Page tail')), + ], + ), + ), + ), + ), + ); + + final Map scrollResponse = await _scroll( + extensions, + deltaX: 0, + deltaY: -120, + ); + await tester.pumpAndSettle(); + + expect(scrollResponse['ok'], true); + expect(outerController.offset, greaterThan(0)); + expect(innerController.offset, 0); + }); + testWidgets('untargeted scroll selects the scrollable matching its axis', ( WidgetTester tester, ) async { From 7673816f09c957c31cdde44e6f26aba54d7ba227 Mon Sep 17 00:00:00 2001 From: drown0315 Date: Sun, 12 Jul 2026 18:38:03 +0800 Subject: [PATCH 4/4] no-mistakes(document): Synchronize runtime stability documentation --- CONTEXT.md | 6 +++++- docs-internal/flutter-pilot-prd.md | 2 +- docs-internal/pilot-runtime-grill-notes.md | 6 ++++-- docs-internal/pilot-runtime-prd.md | 10 ++++++---- docs-internal/scenario-yaml.md | 19 +++++++++++++------ docs/guide/write-scenario.md | 5 +++++ docs/reference/scenario-dsl.md | 12 ++++++++++++ lib/src/execution/scenario_runner.dart | 1 + lib/src/scenario/scenario_parser.dart | 6 +++--- .../lib/src/finder_resolver.dart | 2 +- .../lib/src/pilot_runtime_binding.dart | 10 +++++++--- .../lib/src/pilot_runtime_client.dart | 7 ++++--- .../lib/src/scroll_performer.dart | 11 +++++++---- 13 files changed, 69 insertions(+), 28 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 33ce6f3..b72d5d7 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -76,6 +76,10 @@ _Avoid_: Selector, locator, query A Runtime Target match produced by applying a Finder during a Scenario run. A valid Finder resolution requires exactly one Finder Match; zero matches or multiple matches fail the step, and action-specific capabilities such as tapping, typing, or scrolling are validated when the action executes. _Avoid_: First match, best match +**Finder Action Budget**: +The total time available for a Finder-backed action to synchronize with a Flutter frame and resolve exactly one Finder Match. `tap`, `type`, and targeted `scroll` use the runner's default 3000ms budget; `waitFor.timeoutMs` supplies the budget for that WaitFor Action. Frame synchronization consumes the same budget rather than adding a separate timeout. +_Avoid_: Finder timeout plus frame timeout, retry count + **Runtime Handle**: An opaque runtime token returned with a Finder Match and accepted back by the Runtime Adapter for the immediately following action. Flutter Pilot may record it for diagnostics, but must not parse it, construct it, or treat it as stable identity. _Avoid_: Widget id, key, Inspector id, stable reference @@ -129,7 +133,7 @@ An action that waits until a Finder produces exactly one match. It does not wait _Avoid_: Wait assertion, sleep **Scroll Action**: -An action that moves a scrollable area by configured gesture drag deltas. It may target a specific scrollable with a Finder, or use the primary scrollable when no Finder is provided. +An action that moves a scrollable area by configured gesture drag deltas. It may target a specific scrollable with a Finder, or select the unique outermost visible scrollable on the dominant drag axis when no Finder is provided. _Avoid_: Swipe **Screenshot**: diff --git a/docs-internal/flutter-pilot-prd.md b/docs-internal/flutter-pilot-prd.md index 95e5ac3..a1a4a35 100644 --- a/docs-internal/flutter-pilot-prd.md +++ b/docs-internal/flutter-pilot-prd.md @@ -141,7 +141,7 @@ The result is a reproducible bug report package that can be consumed by humans, - `byKey` is not part of the current Scenario DSL because the calibrated `pilot_runtime` semantic Snapshot path does not expose Flutter key values reliably. Key-based Finders may be added later if the Runtime Adapter can obtain stable key data. - `byType` accepts the `pilot_runtime` semantic Snapshot node type, such as `textField`, `button`, `text`, `scrollable`, or `header`. It does not accept Dart widget class names such as `TextField`, `FilledButton`, or app-defined wrapper widget classes. - `byText` matches exact visible text. It does not perform contains, fuzzy, or regular expression matching in the first version. -- A Finder must resolve to exactly one widget before an action can execute. Zero matches fail the step as "Finder matched no widgets"; multiple matches fail the step as "Finder matched multiple widgets." Flutter Pilot does not automatically choose the first match. +- A Finder must resolve to exactly one widget before an action can execute. Zero matches keep polling until the Finder Action Budget expires; multiple matches fail the Step as "Finder matched multiple widgets." Flutter Pilot does not automatically choose the first match. - The initial action set includes `tap`, `type`, `scroll`, `waitFor`, and `capture`. - The `type` action means replacing text in a widget: clear existing text directly, then enter the configured text one character at a time. It is distinct from the `byType` Finder constraint. - Finder-backed `tap`, `type`, and targeted `scroll` actions wait for a Finder to produce exactly one match within a default `3000ms` budget. Before polling, the runner waits up to `500ms` for the current or next Flutter frame; that frame wait consumes the same action budget. Zero matches keep polling every `50ms` until timeout, one match executes, and multiple matches fail the Step immediately. diff --git a/docs-internal/pilot-runtime-grill-notes.md b/docs-internal/pilot-runtime-grill-notes.md index 13922d9..3a0a89f 100644 --- a/docs-internal/pilot-runtime-grill-notes.md +++ b/docs-internal/pilot-runtime-grill-notes.md @@ -118,8 +118,10 @@ design memory, not a final PRD. - `scroll` v1 uses pointer drag gestures and Flutter logical pixel deltas. It does not use semantic scroll actions. - Scroll with a Finder validates that the unique match can be scrolled. -- Scroll without a Finder targets the primary scrollable. If the primary - scrollable cannot be uniquely determined, the action fails. +- Scroll without a Finder targets the unique outermost visible scrollable on + the dominant drag axis. Nested scrollables on that axis are excluded from + primary selection and avoided when choosing the drag start; multiple peer + candidates fail as ambiguous. ## Widget Tree And Capture Artifacts diff --git a/docs-internal/pilot-runtime-prd.md b/docs-internal/pilot-runtime-prd.md index 5eac1a6..dc13630 100644 --- a/docs-internal/pilot-runtime-prd.md +++ b/docs-internal/pilot-runtime-prd.md @@ -59,8 +59,8 @@ structured UI artifact exposed through `widgetTree`. 22. As a Flutter developer, I want tap to fall back to pointer center taps on calibrated platforms, so that targets without semantic tap actions can still be exercised. 23. As a Flutter developer, I want `type` to clear editable text directly and then enter the configured text character by character without simulating platform keyboard input, so that text entry is deterministic while still exercising per-character text changes. 24. As a Flutter developer, I want scroll deltas to remain Flutter logical pixel drag deltas, so that existing Scenario scroll semantics remain intact. -25. As a Flutter developer, I want untargeted scroll to use the primary scrollable, so that simple scrolling Scenarios remain concise. -26. As a Flutter developer, I want untargeted scroll to fail when the primary scrollable is ambiguous, so that Flutter Pilot does not pick an arbitrary scrollable. +25. As a Flutter developer, I want untargeted scroll to use the unique outermost visible scrollable on the dominant drag axis, so that simple scrolling Scenarios remain concise and nested views do not capture the gesture accidentally. +26. As a Flutter developer, I want untargeted scroll to fail when multiple outermost scrollables remain on the dominant drag axis, so that Flutter Pilot does not pick an arbitrary scrollable. 27. As a Flutter developer, I want Widget Tree capture to use Flutter Inspector summary tree data, so that artifacts preserve useful hierarchy without raw dump noise. 28. As a Flutter developer, I want Widget Tree JSON to be normalized, so that artifacts are stable and not tied to raw Inspector response envelopes. 29. As a Flutter developer, I want Widget Tree artifacts to include schema and source metadata, so that tools can version and interpret them safely. @@ -114,7 +114,9 @@ structured UI artifact exposed through `widgetTree`. - Tap execution prefers semantic tap actions and falls back to pointer center tap on calibrated platforms. - Text entry supports editable text targets only and replaces existing text. It does not simulate keyboard or IME input. - Scroll execution uses pointer drag gestures and Flutter logical pixel deltas. It does not use semantic scroll actions. -- Untargeted scroll resolves the primary scrollable and fails when that target cannot be uniquely determined. +- Untargeted scroll chooses the dominant drag axis, resolves the unique outermost visible scrollable on that axis, and fails when multiple peer candidates remain. +- Scroll gesture start selection avoids the bounds of nested scrollables on the same axis so the intended outer scrollable receives the drag. +- Finder-backed actions synchronize with the current or next Flutter frame before polling for a unique Finder Match. Frame synchronization and polling share one action budget; multiple matches fail immediately. - Remove Snapshot from the new Scenario capture contract, Runtime Adapter contract, print diagnostics, and artifact language. - Use `widgetTree` as the structured UI capture field, print diagnostic, and report artifact type. - Write Widget Tree artifacts with a `widget_tree` filename suffix. @@ -161,7 +163,7 @@ Major modules to build or modify: - Runtime client tests should use fake VM Service responses for handshake, protocol mismatch, capability missing, Widget Tree capture, and error mapping. - Runtime protocol tests should cover response decoding, version validation, capability validation, and structured runtime failures. - Finder resolution tests inside `pilot_runtime` should use Flutter widget tests for visible matching, offstage exclusion, `byText`, semantic `byType`, `ValueKey`, `byWidget`, strict AND combinations, and wrapper-child subtree evidence. -- Action execution tests inside `pilot_runtime` should use Flutter widget tests for semantic tap, pointer fallback, editable text clear/entry, targeted scroll, and primary scrollable resolution. +- Action execution tests inside `pilot_runtime` should use Flutter widget tests for semantic tap, pointer fallback, editable text clear/entry, targeted scroll, axis-aware outermost scrollable selection, nested-scrollable avoidance, and ambiguous peer scrollables. - Widget Tree normalizer tests should use recorded Inspector summary tree fixtures and verify normalized schema, source, node fields, child structure, missing optional fields, and rejection of invalid required shape. - Screenshot tests should verify returned MIME type and bytes shape where the chosen screenshot path can be faked; real screenshot quality should be covered by calibration smoke tests. - Logs tests should verify the not-implemented payload and that `logs: true` does not fail a capture Step. diff --git a/docs-internal/scenario-yaml.md b/docs-internal/scenario-yaml.md index 4f353e0..1024988 100644 --- a/docs-internal/scenario-yaml.md +++ b/docs-internal/scenario-yaml.md @@ -233,7 +233,8 @@ Flutter Pilot supports five Scenario actions: `tap`, `type`, `scroll`, ``` The Finder must resolve to exactly one Finder Match before the tap can execute. -Zero matches fail the Step. Multiple matches fail the Step. +Zero matches keep polling within the default `3000ms` budget. Multiple matches +fail the Step immediately. ### type @@ -260,9 +261,10 @@ The action clears existing text and enters the configured text. `deltaX` and `deltaY` default to `0`, but at least one must be non-zero. -A Finder is optional. When omitted, Flutter Pilot targets the primary -scrollable. When provided, the Finder must resolve to exactly one scrollable -target. +A Finder is optional. When omitted, Flutter Pilot selects the unique outermost +visible scrollable on the dominant drag axis and ignores nested scrollables on +that axis. Multiple peer candidates are ambiguous. When provided, the Finder +must resolve to exactly one scrollable target. ```yaml - label: scroll_results @@ -284,8 +286,13 @@ target. `timeoutMs` is optional and defaults to `3000`. -`waitFor` waits until the Finder produces exactly one match. Zero matches keep -waiting until timeout. Multiple matches fail the Step. +`waitFor` first waits up to `500ms` for the current or next Flutter frame, then +polls every `50ms` until the Finder produces exactly one match. Frame +synchronization and polling share the `timeoutMs` budget. Zero matches keep +waiting until timeout. Multiple matches fail the Step immediately. + +Finder-backed `tap`, `type`, and targeted `scroll` use the same synchronization +and polling behavior with a default `3000ms` budget. ### capture diff --git a/docs/guide/write-scenario.md b/docs/guide/write-scenario.md index 93cd6f4..0a67a2e 100644 --- a/docs/guide/write-scenario.md +++ b/docs/guide/write-scenario.md @@ -48,6 +48,11 @@ steps: When a Finder has multiple fields, every configured field must match. +Finder-backed actions tolerate UI that appears asynchronously: Flutter Pilot +waits for a Flutter frame and polls for one unique match within the action's +timeout budget. Multiple matches fail immediately instead of selecting the +first match. + ## Capture diagnostics `capture` is a Step action. Use it as its own Step when you want the default diff --git a/docs/reference/scenario-dsl.md b/docs/reference/scenario-dsl.md index 729457d..54aee09 100644 --- a/docs/reference/scenario-dsl.md +++ b/docs/reference/scenario-dsl.md @@ -115,6 +115,12 @@ steps: When several Finder fields are present, all constraints must match. Finder fields are single strings. +Finder-backed actions first wait up to 500ms for the current or next Flutter +frame, then poll every 50ms until exactly one match is available. The frame wait +and polling share one total budget. `tap`, `type`, and targeted `scroll` use a +3000ms budget; `waitFor.timeoutMs` sets the budget for that `waitFor` Step. +Multiple matches fail immediately. + ## Actions Flutter Pilot supports these Scenario actions: @@ -155,6 +161,12 @@ steps: deltaY: -500 ``` +The dominant drag axis is vertical when `abs(deltaY) >= abs(deltaX)` and +horizontal otherwise. Without a Finder, Flutter Pilot selects the unique +outermost visible scrollable on that axis and avoids starting the gesture over +a nested scrollable on the same axis. Multiple peer scrollables are ambiguous; +add a Finder to choose one explicitly. + ### waitFor ```yaml scenario diff --git a/lib/src/execution/scenario_runner.dart b/lib/src/execution/scenario_runner.dart index 344bbf5..24cf13f 100644 --- a/lib/src/execution/scenario_runner.dart +++ b/lib/src/execution/scenario_runner.dart @@ -12,6 +12,7 @@ import '../target/target_device.dart'; /// /// It contains: /// - the Runtime Adapter used for UI operations and captures +/// - the shared Finder Action Budget for tap, type, and targeted scroll /// - the output directory where run artifacts are written /// /// During `run`, it: diff --git a/lib/src/scenario/scenario_parser.dart b/lib/src/scenario/scenario_parser.dart index f53df14..c98ded3 100644 --- a/lib/src/scenario/scenario_parser.dart +++ b/lib/src/scenario/scenario_parser.dart @@ -685,9 +685,9 @@ class ScenarioParser { /// Parse a scroll action using gesture drag deltas. /// - /// A Finder is optional. Without a Finder, the future runner should target - /// the primary scrollable. At least one of `deltaX` or `deltaY` must be - /// non-zero. + /// A Finder is optional. Without a Finder, the runner selects the unique + /// outermost visible scrollable on the dominant drag axis. At least one of + /// `deltaX` or `deltaY` must be non-zero. static ScrollAction? _parseScroll( YamlMap yaml, String path, diff --git a/packages/pilot_runtime/lib/src/finder_resolver.dart b/packages/pilot_runtime/lib/src/finder_resolver.dart index 3cb9b2f..121dcff 100644 --- a/packages/pilot_runtime/lib/src/finder_resolver.dart +++ b/packages/pilot_runtime/lib/src/finder_resolver.dart @@ -93,7 +93,7 @@ class PilotRuntimeFinderResolver { /// /// Runtime action performers use this to preserve the same visible-target /// rules as Finder resolution when they need to locate related widgets, such - /// as the primary scrollable for an untargeted Scroll Action. + /// as candidates for an untargeted Scroll Action. static void visitVisibleElements(void Function(Element element) visitor) { final Element? rootElement = WidgetsBinding.instance.rootElement; if (rootElement == null) { diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart index ce1c874..630310c 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart @@ -31,9 +31,9 @@ typedef PilotRuntimeExtensionRegistrar = /// App-side hook that exposes Flutter Pilot runtime service extensions. /// /// Target App Packages call `ensureInitialized()` from app startup while -/// running in debug mode. The binding registers the protocol handshake -/// extension once per isolate and returns without side effects when debug mode -/// is disabled. +/// running in debug mode. The binding registers the handshake, frame +/// synchronization, Finder, action, and log extensions once per isolate and +/// returns without side effects when debug mode is disabled. class PilotRuntimeBinding { PilotRuntimeBinding._(); @@ -124,6 +124,10 @@ class PilotRuntimeBinding { ); } + /// Wait for the current or next Flutter frame within the requested timeout. + /// + /// The response reports timeout as data because Finder polling may continue + /// after the synchronization attempt reaches its bound. static Future> _handleEndOfFrame( Map parameters, ) async { diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart index 0adb52b..4d7af27 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart @@ -159,7 +159,7 @@ enum PilotRuntimeActionFailure { /// A resolved Runtime Handle cannot receive scroll drag gestures. notScrollable, - /// The primary scrollable is missing or ambiguous for untargeted scroll. + /// No unique outermost scrollable exists for untargeted scroll. primaryScrollableUnavailable, /// The app-side runtime returned an action failure code this client does not @@ -676,11 +676,12 @@ class PilotRuntimeClient { _checkActionResponse(response, actionName: 'enterText'); } - /// Scroll a Runtime Handle or the primary scrollable by drag deltas. + /// Scroll a Runtime Handle or an automatically selected scrollable. /// /// Args: /// - `handle`: Optional opaque Runtime Handle from a Finder Match. When - /// omitted, the app-side runtime resolves the primary scrollable. + /// omitted, the app-side runtime selects the unique outermost visible + /// scrollable on the dominant drag axis. /// - `deltaX`: Horizontal drag distance in logical pixels. /// - `deltaY`: Vertical drag distance in logical pixels. Future performScroll({ diff --git a/packages/pilot_runtime/lib/src/scroll_performer.dart b/packages/pilot_runtime/lib/src/scroll_performer.dart index b55a1cd..99aaacf 100644 --- a/packages/pilot_runtime/lib/src/scroll_performer.dart +++ b/packages/pilot_runtime/lib/src/scroll_performer.dart @@ -3,7 +3,7 @@ import 'package:flutter/material.dart'; import 'finder_resolver.dart'; -/// Performs scroll actions for targeted and primary scrollables. +/// Performs scroll actions for targeted and automatically selected scrollables. /// /// The performer uses pointer drag gestures with Flutter logical-pixel deltas. /// It does not use semantic scroll actions, so Scenario `deltaX` and `deltaY` @@ -13,16 +13,19 @@ class PilotRuntimeScrollPerformer { static int _nextPointer = 1; - /// Drag a targeted scrollable or the primary visible scrollable. + /// Drag a targeted scrollable or the unique outermost visible scrollable. /// /// Args: /// - `handle`: Optional Runtime Handle. When present, the handle must identify /// a visible scrollable or a widget subtree that contains one scrollable. /// - `deltaX`: Horizontal drag distance in logical pixels. /// - `deltaY`: Vertical drag distance in logical pixels. + /// The larger absolute delta selects the scroll axis; ties are vertical. /// - /// Returns a structured VM Service response. Missing, ambiguous, or - /// non-scrollable targets return `ok: false` instead of throwing. + /// Without a handle, selection ignores nested scrollables on the chosen axis + /// and fails when multiple outermost candidates remain. Returns a structured + /// VM Service response. Missing, ambiguous, or non-scrollable targets return + /// `ok: false` instead of throwing. static Future> scroll({ String? handle, required double deltaX,