diff --git a/examples/smoke_app/README.md b/examples/smoke_app/README.md index cb8498e..a47ee6e 100644 --- a/examples/smoke_app/README.md +++ b/examples/smoke_app/README.md @@ -105,3 +105,55 @@ Expected result: - the Scenario fails at `scroll_read_only_text` - the failure reason contains `does not identify a scrollable` - failure diagnostics are still written under the run directory + +## PilotRuntime Replacement Calibration + +The `pilot/calibration` Project Scenarios are the live replacement checks for +the `pilot_runtime` path. They run against one debug Target App Package target +and cover `byText`, semantic `byType`, `byKey`, `byWidget`, tap, type, targeted +scroll, untargeted scroll, Screenshot, Widget Tree, Logs, and Project Run hot +restart between Scenarios. + +The calibration app routes `package:logging` records through `debugPrint`, so a +capture step should write a non-empty `.log` artifact. + +Run the macOS desktop debug calibration from `examples/smoke_app`: + +```bash +dart run ../../bin/flutter_pilot.dart test pilot/calibration \ + --target lib/pilot_runtime_calibration_app.dart \ + --device macos +``` + +Run the Android debug calibration by replacing the device id with a connected +debug device: + +```bash +dart run ../../bin/flutter_pilot.dart test pilot/calibration \ + --target lib/pilot_runtime_calibration_app.dart \ + --device +``` + +Expected Project Run result: + +- `01_interact.yaml` passes and writes Screenshot, Widget Tree, and Logs + artifacts. +- Flutter Pilot hot restarts the Target App Package before + `02_after_restart.yaml`. +- `02_after_restart.yaml` sees `Calibration taps: 0`, proving the restart reset + app state while `PilotRuntimeBinding` and capture still work. +- The Project Run writes `project_run_report.json`; each Scenario Run writes + `run_report.json` and `timeline.html`. + +Calibration output should record: + +- platform: macOS desktop debug or Android debug +- Flutter SDK version +- selected Target Device id +- exact command +- observed Project Run result and artifact paths +- any unsupported capability or platform-specific difference + +Web, profile, release, and iOS are not claimed by v1. These examples do not +change Flutter Pilot runtime selection behavior; they only provide a focused +Project Run for live `pilot_runtime` calibration. diff --git a/examples/smoke_app/lib/pilot_runtime_calibration_app.dart b/examples/smoke_app/lib/pilot_runtime_calibration_app.dart new file mode 100644 index 0000000..a9ba2de --- /dev/null +++ b/examples/smoke_app/lib/pilot_runtime_calibration_app.dart @@ -0,0 +1,182 @@ +import 'package:flutter/material.dart'; +import 'package:logging/logging.dart'; +import 'package:pilot_runtime/pilot_runtime.dart'; + +final Logger _log = Logger('pilot_runtime_calibration'); + +/// Start the replacement-calibration app with `pilot_runtime` installed. +/// +/// This target is intentionally broader than the focused tap and scroll demos: +/// it exercises Finder constraints, replay actions, capture artifacts, and +/// Project Run hot restart behavior against one real debug Runtime Target. +void main() { + WidgetsFlutterBinding.ensureInitialized(); + PilotRuntimeBinding.ensureInitialized(); + _configureLogging(); + _log.info('Pilot Runtime calibration app started.'); + runApp(const PilotRuntimeCalibrationApp()); +} + +/// Route package logging records through Flutter debug output. +/// +/// Flutter Pilot captures `debugPrint` through `PilotRuntimeBinding`, so this +/// keeps the target on the public `logging` API while producing a non-empty +/// runtime Logs artifact during calibration capture. +void _configureLogging() { + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((LogRecord record) { + debugPrint('${record.level.name} ${record.loggerName}: ${record.message}'); + }); +} + +/// Smoke app used by the live `pilot_runtime` replacement calibration. +class PilotRuntimeCalibrationApp extends StatelessWidget { + const PilotRuntimeCalibrationApp({super.key}); + + @override + Widget build(BuildContext context) { + return MaterialApp( + title: 'Pilot Runtime Calibration', + theme: ThemeData(colorSchemeSeed: Colors.teal), + home: const PilotRuntimeCalibrationPage(), + ); + } +} + +/// Single target page that exposes stable Finder and action evidence. +class PilotRuntimeCalibrationPage extends StatefulWidget { + const PilotRuntimeCalibrationPage({super.key}); + + @override + State createState() => + _PilotRuntimeCalibrationPageState(); +} + +class _PilotRuntimeCalibrationPageState + extends State { + final TextEditingController _emailController = TextEditingController(); + int _tapCount = 0; + int _chipTapCount = 0; + + @override + void dispose() { + _emailController.dispose(); + super.dispose(); + } + + /// Record a semantic button tap for the calibration Scenario. + void _recordTap() { + setState(() { + _tapCount += 1; + }); + _log.info('Calibration button tapped $_tapCount time(s).'); + } + + /// Record a custom-widget tap for the byKey plus byWidget calibration path. + void _recordChipTap() { + setState(() { + _chipTapCount += 1; + }); + _log.info('Calibration chip tapped $_chipTapCount time(s).'); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar(title: const Text('Pilot Runtime Calibration')), + body: ListView( + padding: const EdgeInsets.all(24), + children: [ + const Text( + 'Pilot Runtime Calibration Ready', + key: ValueKey('calibration_ready_text'), + ), + const SizedBox(height: 16), + TextField( + key: const ValueKey('calibration_email_field'), + controller: _emailController, + decoration: const InputDecoration(labelText: 'Calibration email'), + ), + const SizedBox(height: 16), + FilledButton( + key: const ValueKey('calibration_button'), + onPressed: _recordTap, + child: const Text('Calibration tap'), + ), + const SizedBox(height: 12), + Text( + 'Calibration taps: $_tapCount', + key: const ValueKey('calibration_tap_count'), + ), + const SizedBox(height: 16), + CalibrationChip( + key: const ValueKey('calibration_chip'), + taps: _chipTapCount, + onTap: _recordChipTap, + ), + const SizedBox(height: 12), + Text( + 'Calibration chip taps: $_chipTapCount', + key: const ValueKey('calibration_chip_count'), + ), + const SizedBox(height: 24), + SizedBox( + height: 180, + child: ListView( + key: ValueKey('calibration_target_scrollable'), + padding: EdgeInsets.all(12), + children: [ + Text('Calibration nested row 0'), + SizedBox(height: 56), + Text('Calibration nested row 1'), + SizedBox(height: 56), + Text('Calibration nested row 2'), + SizedBox(height: 56), + Text('Calibration nested row 3'), + SizedBox(height: 56), + Text('Calibration nested row 4'), + SizedBox(height: 56), + Text('Calibration nested row 5'), + SizedBox(height: 56), + Text('Calibration nested row 6'), + SizedBox(height: 56), + Text('Calibration nested row 7'), + ], + ), + ), + const SizedBox(height: 24), + for (int index = 0; index < 28; index += 1) + Padding( + padding: const EdgeInsets.symmetric(vertical: 14), + child: Text('Calibration page row $index'), + ), + ], + ), + ); + } +} + +/// Custom tap target used to calibrate exact Dart widget type matching. +class CalibrationChip extends StatelessWidget { + const CalibrationChip({required this.taps, required this.onTap, super.key}); + + /// Number of taps recorded by the parent page. + final int taps; + + /// Callback invoked when this custom widget is tapped. + final VoidCallback onTap; + + @override + Widget build(BuildContext context) { + return GestureDetector( + behavior: HitTestBehavior.opaque, + onTap: onTap, + child: Container( + height: 52, + alignment: Alignment.center, + color: Colors.teal.shade100, + child: Text('Calibration chip target ($taps)'), + ), + ); + } +} diff --git a/examples/smoke_app/lib/pilot_runtime_scroll_demo.dart b/examples/smoke_app/lib/pilot_runtime_scroll_demo.dart index 9bd1fef..5feb40e 100644 --- a/examples/smoke_app/lib/pilot_runtime_scroll_demo.dart +++ b/examples/smoke_app/lib/pilot_runtime_scroll_demo.dart @@ -1,6 +1,9 @@ import 'package:flutter/material.dart'; +import 'package:logging/logging.dart'; import 'package:pilot_runtime/pilot_runtime.dart'; +final Logger _log = Logger('pilot_runtime_scroll_demo'); + /// Start the scroll replay demo with the real `pilot_runtime` binding installed. /// /// This target is intended for manual Flutter Pilot acceptance runs. It exposes @@ -9,9 +12,23 @@ import 'package:pilot_runtime/pilot_runtime.dart'; void main() { WidgetsFlutterBinding.ensureInitialized(); PilotRuntimeBinding.ensureInitialized(); + _configureLogging(); + _log.info('Pilot Runtime scroll demo started.'); runApp(const PilotRuntimeScrollDemoApp()); } +/// Route package logging records through Flutter debug output. +/// +/// Flutter Pilot captures `debugPrint` through `PilotRuntimeBinding`, so this +/// keeps the smoke app on the public `logging` API while still producing a +/// runtime Logs artifact during Scenario capture. +void _configureLogging() { + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((LogRecord record) { + debugPrint('${record.level.name} ${record.loggerName}: ${record.message}'); + }); +} + /// Demo app used to verify scroll replay through `PilotRuntimeAdapter`. class PilotRuntimeScrollDemoApp extends StatelessWidget { const PilotRuntimeScrollDemoApp({super.key}); diff --git a/examples/smoke_app/pilot/calibration/01_interact.yaml b/examples/smoke_app/pilot/calibration/01_interact.yaml new file mode 100644 index 0000000..f6a20e6 --- /dev/null +++ b/examples/smoke_app/pilot/calibration/01_interact.yaml @@ -0,0 +1,64 @@ +scenario: + name: pilot_runtime_calibration_01_interact + description: | + Exercises the live pilot_runtime replacement checks before Project Run hot + restart resets the Target App Package for the next Scenario. + +steps: + - label: wait_for_calibration_ready + waitFor: + byText: Pilot Runtime Calibration Ready + timeoutMs: 5000 + + - label: tap_calibration_button_by_text_and_type + tap: + byText: Calibration tap + byType: button + + - label: wait_for_tap_count + waitFor: + byText: 'Calibration taps: 1' + byKey: calibration_tap_count + timeoutMs: 5000 + + - label: type_calibration_field_by_semantic_type + type: + byType: textField + text: calibrated@example.com + + - label: tap_calibration_chip_by_key_and_widget + tap: + byKey: calibration_chip + byWidget: CalibrationChip + + - label: wait_for_chip_count + waitFor: + byText: 'Calibration chip taps: 1' + byKey: calibration_chip_count + timeoutMs: 5000 + + - label: targeted_scroll_calibration_list + scroll: + byKey: calibration_target_scrollable + byType: scrollable + deltaY: -260 + + - label: wait_for_nested_scroll_result + waitFor: + byText: Calibration nested row 6 + timeoutMs: 5000 + + - label: primary_scroll_calibration_page + scroll: + deltaY: -900 + + - label: wait_for_primary_scroll_result + waitFor: + byText: Calibration page row 24 + timeoutMs: 5000 + + - label: capture_calibration_artifacts + capture: + screenshot: true + widgetTree: true + logs: true diff --git a/examples/smoke_app/pilot/calibration/02_after_restart.yaml b/examples/smoke_app/pilot/calibration/02_after_restart.yaml new file mode 100644 index 0000000..3b70ad2 --- /dev/null +++ b/examples/smoke_app/pilot/calibration/02_after_restart.yaml @@ -0,0 +1,24 @@ +scenario: + name: pilot_runtime_calibration_02_after_restart + description: | + Verifies the Project Run hot restart boundary by checking that the runtime + binding, Finder resolution, and capture artifacts still work after the + previous Scenario changed app state. + +steps: + - label: verify_hot_restart_reset_state + waitFor: + byText: 'Calibration taps: 0' + byKey: calibration_tap_count + timeoutMs: 5000 + + - label: verify_runtime_binding_after_restart + waitFor: + byText: Pilot Runtime Calibration Ready + timeoutMs: 5000 + + - label: capture_after_restart_artifacts + capture: + screenshot: true + widgetTree: true + logs: true diff --git a/examples/smoke_app/pilot/pilot_runtime_scroll.yaml b/examples/smoke_app/pilot/pilot_runtime_scroll.yaml index 942f996..46f84da 100644 --- a/examples/smoke_app/pilot/pilot_runtime_scroll.yaml +++ b/examples/smoke_app/pilot/pilot_runtime_scroll.yaml @@ -4,6 +4,11 @@ scenario: Verifies targeted scroll replay through PilotRuntimeAdapter. steps: + - label: tt + waitFor: + byText: Scroll demo row 1 + timeoutMs: 5000 + - label: scroll_target_list scroll: byKey: target_scroll_list @@ -14,3 +19,8 @@ steps: waitFor: byText: Scroll demo row 18 timeoutMs: 5000 + + - label: capture_logs + capture: + logs: true + widgetTree: false diff --git a/examples/smoke_app/pubspec.lock b/examples/smoke_app/pubspec.lock index 71c0f48..84016b5 100644 --- a/examples/smoke_app/pubspec.lock +++ b/examples/smoke_app/pubspec.lock @@ -107,6 +107,14 @@ packages: url: "https://pub.dev" source: hosted version: "6.1.0" + logging: + dependency: "direct main" + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" matcher: dependency: transitive description: diff --git a/examples/smoke_app/pubspec.yaml b/examples/smoke_app/pubspec.yaml index 4b7d8b2..3625f49 100644 --- a/examples/smoke_app/pubspec.yaml +++ b/examples/smoke_app/pubspec.yaml @@ -36,6 +36,7 @@ dependencies: cupertino_icons: ^1.0.8 pilot_runtime: path: ../../packages/pilot_runtime + logging: ^1.3.0 dev_dependencies: flutter_test: diff --git a/issues/0.1.1-pilot-runtime.md b/issues/0.1.1-pilot-runtime.md index 335b9ab..d47ab0a 100644 --- a/issues/0.1.1-pilot-runtime.md +++ b/issues/0.1.1-pilot-runtime.md @@ -339,7 +339,9 @@ Verification: - 3. Expose PilotRuntimeAdapter behind hidden runtime switch -## 11. Return not-implemented logs artifact without failing capture +## 11. Capture Flutter runtime logs as a `.log` artifact + +Status: completed in working tree. ## Parent @@ -347,14 +349,33 @@ https://github.com/drown0315/flutter_pilot/issues/110 ## What to build -Keep Logs in the capture contract without implementing real log collection in `pilot_runtime` v1. A logs capture should return a structured not-implemented payload, write an artifact, and allow the capture Step to pass. +Implement real Logs capture for Flutter runtime logs. A `logs: true` capture should write a `*_logs.log` artifact and record it in `run_report.json` with `type: logs`. + +Implementation note: the current slice collects logs through the app-side `pilot_runtime` hook instead of VM Service log stream subscription. `PilotRuntimeBinding.ensureInitialized()` registers `ext.flutter_pilot.runtime.collectLogs`, installs a bounded in-memory buffer, and records `debugPrint`, `FlutterError.onError`, and `PlatformDispatcher.instance.onError` entries. `PilotRuntimeClient.collectLogs()` calls that extension, and `PilotRuntimeAdapter.collectLogs()` maps the payload into Flutter Pilot's `LogsCapture`. + +The `.log` file currently contains an indented structured payload with schema `pilot_runtime.logs.v1`. This keeps the artifact easy for Flutter Pilot's diagnostic reducer and Run Diff to parse while using a log-specific file extension for humans and tools. Logs emitted before `PilotRuntimeBinding.ensureInitialized()` installs capture are not guaranteed to appear. Flutter Pilot progress output and Flutter build or launch logs are not part of this artifact. ## Acceptance criteria -- [ ] `pilot_runtime` collect logs returns a structured not-implemented payload. -- [ ] `PilotRuntimeAdapter.collectLogs()` maps that payload to Flutter Pilot `LogsCapture`. -- [ ] A `logs: true` capture writes a logs artifact and does not fail because logs are not implemented. -- [ ] Tests cover payload shape, artifact writing, and capture Step success. +- [x] `pilot_runtime` exposes `ext.flutter_pilot.runtime.collectLogs` and reports `runtime.logs.collect` in the handshake capabilities. +- [x] `PilotRuntimeBinding` captures `debugPrint`, Flutter framework errors, and platform dispatcher errors into a bounded runtime log buffer. +- [x] `PilotRuntimeClient.collectLogs()` returns the structured runtime log payload from the app-side hook. +- [x] `PilotRuntimeAdapter.collectLogs()` returns Flutter Pilot `LogsCapture` data from `pilot_runtime` instead of throwing not implemented. +- [x] A `logs: true` capture writes a `*_logs.log` artifact with `type: logs` and does not fail when the buffer is empty. +- [x] Run reports, timeline links, Run Diff fallback fixtures, and artifact store tests use the `.log` path. +- [x] Tests cover handshake capability, app-side log capture, client extension calls, adapter mapping, `.log` artifact writing, empty-log capture success, and capture Step success. + +Verification: + +- `dart format .` +- `dart analyze lib test` +- `dart test` +- `flutter analyze` from `packages/pilot_runtime` +- `flutter test` from `packages/pilot_runtime` +- `flutter analyze` from `examples/smoke_app` +- `flutter test` from `examples/smoke_app` +- Manual smoke run from `examples/smoke_app`: + - `dart run ~/.treehouse/flutter_pilot-b6aa0b/5/flutter_pilot/bin/flutter_pilot.dart test pilot/pilot_runtime_scroll.yaml --target lib/pilot_runtime_scroll_demo.dart -d Pixel` ## Blocked by @@ -362,6 +383,8 @@ Keep Logs in the capture contract without implementing real log collection in `p ## 12. Implement hot reload and hot restart client capabilities +Status: completed in working tree. + ## Parent https://github.com/drown0315/flutter_pilot/issues/110 @@ -370,12 +393,24 @@ https://github.com/drown0315/flutter_pilot/issues/110 Add hot reload and hot restart capabilities to the `pilot_runtime` client using VM Service or Flutter runtime paths. These capabilities should not be Scenario Step actions and should not be app-side hook extensions. +Implementation note: `PilotRuntimeClient.hotReload()` and +`PilotRuntimeClient.hotRestart()` call VM Service `reloadSources` through the +client VM Service abstraction. Hot reload uses `force: false`; hot restart uses +`force: true`. Results are normalized into `PilotRuntimeReloadResult`, and VM +Service errors or unsuccessful reload reports are mapped to +`PilotRuntimeReloadException`. + ## Acceptance criteria -- [ ] `pilot_runtime` client exposes hot reload and hot restart operations. -- [ ] The operations use VM Service or Flutter runtime capabilities rather than `ext.flutter_pilot.runtime.*` hook extensions. -- [ ] Success and failure responses are normalized into Flutter Pilot-owned result or error types. -- [ ] Tests cover successful hot reload, successful hot restart, and failure mapping using fake runtime responses. +- [x] `pilot_runtime` client exposes hot reload and hot restart operations. +- [x] The operations use VM Service or Flutter runtime capabilities rather than `ext.flutter_pilot.runtime.*` hook extensions. +- [x] Success and failure responses are normalized into Flutter Pilot-owned result or error types. +- [x] Tests cover successful hot reload, successful hot restart, and failure mapping using fake runtime responses. + +Verification: + +- `flutter test test/pilot_runtime_client_test.dart test/pilot_runtime_widget_tree_test.dart` from `packages/pilot_runtime` +- `dart test test/runtime/pilot_runtime_adapter_test.dart` ## Blocked by @@ -393,12 +428,64 @@ Create the live calibration slice that proves `pilot_runtime` is ready to replac ## Acceptance criteria -- [ ] A smoke target initializes `PilotRuntimeBinding` in debug mode. -- [ ] macOS desktop debug calibration verifies `byText`, semantic `byType`, `byKey`, `byWidget`, tap, type, targeted scroll, untargeted scroll, Screenshot, Widget Tree, hot reload, and hot restart. -- [ ] Android debug calibration verifies the same required capabilities where available. -- [ ] Calibration output records platform, Flutter SDK version, device target, commands, and observed results. -- [ ] The calibration result explicitly states that Web, profile, release, and iOS are not claimed by v1. -- [ ] `mcp_toolkit` remains the default runtime after this issue; replacement is not performed here. +- [x] A smoke target initializes `PilotRuntimeBinding` in debug mode. +- [x] The calibration smoke target routes `package:logging` records through + `debugPrint` so runtime Logs capture can produce a non-empty `.log` artifact. +- [ ] macOS desktop debug calibration verifies `byText`, semantic `byType`, + `byKey`, `byWidget`, tap, type, targeted scroll, untargeted scroll, + Screenshot, Widget Tree, hot reload, and hot restart. +- [ ] Android debug calibration verifies the same required capabilities where + available. +- [x] Calibration instructions state that output should record platform, + Flutter SDK version, device target, commands, and observed results. +- [x] The calibration instructions explicitly state that Web, profile, release, + and iOS are not claimed by v1. +- [x] The examples do not change Flutter Pilot runtime selection behavior. + +## Implementation status + +- Added `examples/smoke_app/lib/pilot_runtime_calibration_app.dart` as the + live calibration Target App Package entrypoint. It initializes + `PilotRuntimeBinding`, exposes visible Finder targets, records tap/type/scroll + state, and routes `package:logging` through `debugPrint`. +- Added Project Scenarios under `examples/smoke_app/pilot/calibration/`: + - `01_interact.yaml` covers `byText`, semantic `byType`, `byKey`, + `byWidget`, tap, type, targeted scroll, untargeted scroll, Screenshot, + Widget Tree, and Logs. + - `02_after_restart.yaml` verifies Project Run hot restart by checking reset + state and capture after the first Scenario. +- Updated `examples/smoke_app/README.md` with macOS desktop debug and Android + debug calibration commands, expected artifacts, required result fields, and + unsupported v1 targets. +- Android HITL runs exposed two runtime issues that are now fixed: + - Flutter Inspector Widget Tree responses can arrive as VM Service + `Response.json` wrappers; `PilotRuntimeClient.captureWidgetTree()` unwraps + them before normalization. + - Finder Matches that resolve to custom wrapper widgets can contain a tappable + descendant; `PilotRuntimeTapPerformer` now chooses an actionable descendant + rather than requiring the matched wrapper itself to be a button or + `GestureDetector`. + +## Verification + +- `dart format .` +- `dart analyze` +- `dart test` +- `flutter analyze` from `packages/pilot_runtime` +- `flutter test` from `packages/pilot_runtime` +- `dart run bin/flutter_pilot.dart validate examples/smoke_app/pilot/calibration/01_interact.yaml` +- `dart run bin/flutter_pilot.dart validate examples/smoke_app/pilot/calibration/02_after_restart.yaml` + +## Remaining HITL calibration + +- Run the macOS desktop debug calibration from `examples/smoke_app` and record + platform, Flutter SDK version, Target Device, command, run report paths, and + observed results. +- Re-run the Android debug calibration from `examples/smoke_app` after commits + `a20bd4c` and `9fe5f80`, then record the same result fields. +- Hot reload is a client/runtime capability, not a Scenario Step action. Record + the manual hot reload calibration evidence separately from the Project Run hot + restart evidence. ## Blocked by @@ -407,5 +494,5 @@ Create the live calibration slice that proves `pilot_runtime` is ready to replac - 8. Implement type action for editable text targets - 9. Implement scroll action with targeted and primary scrollables - 10. Implement Flutter-layer screenshot capture -- 11. Return not-implemented logs artifact without failing capture +- 11. Capture Flutter runtime logs as a `.log` artifact - 12. Implement hot reload and hot restart client capabilities diff --git a/lib/src/artifacts/artifact_store.dart b/lib/src/artifacts/artifact_store.dart index 8d57673..3a01041 100644 --- a/lib/src/artifacts/artifact_store.dart +++ b/lib/src/artifacts/artifact_store.dart @@ -434,14 +434,16 @@ class RunArtifactWriter { ); } - /// Write one structured Logs capture produced by a Step. + /// Write one Flutter runtime Logs capture produced by a Step. /// /// Args: /// `index` is the 1-based Step number. /// `label` is the optional Step label used in the artifact file name. /// `data` is the JSON-compatible Logs payload returned by the Runtime /// Adapter. Runtime errors are represented inside this payload when the - /// adapter exposes them. + /// adapter exposes them. The file uses a `.log` extension so humans can + /// identify it as runtime log output while tools can still decode the JSON + /// payload. /// `purpose` explains whether these Logs came from an explicit capture Step /// or from an automatic failure bundle. /// @@ -460,7 +462,7 @@ class RunArtifactWriter { index: index, label: label, suffix: 'logs', - extension: 'json', + extension: 'log', ), ); final File logsFile = File(p.join(runDirectory.path, relativePath)); diff --git a/lib/src/runtime/pilot_runtime_adapter.dart b/lib/src/runtime/pilot_runtime_adapter.dart index d578353..bd9da69 100644 --- a/lib/src/runtime/pilot_runtime_adapter.dart +++ b/lib/src/runtime/pilot_runtime_adapter.dart @@ -209,8 +209,9 @@ class PilotRuntimeAdapter implements RuntimeAdapter { } @override - Future collectLogs() { - throw _notImplemented(RuntimeOperation.collectLogs); + Future collectLogs() async { + final Map data = await _client.collectLogs(); + return LogsCapture(data: data); } RuntimeOperationException _notImplemented(RuntimeOperation operation) { diff --git a/lib/src/runtime/pilot_runtime_vm_service.dart b/lib/src/runtime/pilot_runtime_vm_service.dart index 3a03a27..6b7c270 100644 --- a/lib/src/runtime/pilot_runtime_vm_service.dart +++ b/lib/src/runtime/pilot_runtime_vm_service.dart @@ -39,6 +39,17 @@ class PilotRuntimeVmServiceConnection implements PilotRuntimeVmService { } } + @override + Future> reloadSources({required bool force}) async { + final vm_service.VmService service = await _connectedService(); + final String isolateId = await _selectedIsolateId(service); + final vm_service.ReloadReport response = await service.reloadSources( + isolateId, + force: force, + ); + return response.toJson(); + } + /// Close the underlying VM Service connection when one was opened. Future dispose() async { await _service?.dispose(); diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart index 153f152..c01c46f 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:developer'; +import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; @@ -36,6 +37,12 @@ class PilotRuntimeBinding { PilotRuntimeBinding._(); static bool _initialized = false; + static const int _maxLogEntries = 200; + static final List> _logEntries = + >[]; + static DebugPrintCallback? _previousDebugPrint; + static FlutterExceptionHandler? _previousFlutterErrorHandler; + static ui.ErrorCallback? _previousPlatformErrorHandler; /// Register the Flutter Pilot debug runtime hook when debug mode is enabled. /// @@ -45,12 +52,16 @@ class PilotRuntimeBinding { /// `dart:developer.registerExtension`. /// - `debugMode`: Optional debug-mode override used by tests. When omitted, /// Flutter's `kDebugMode` decides whether registration should happen. + /// - `captureLogs`: Optional log-capture override used by tests. When + /// omitted, log capture is installed only for the production VM Service + /// registrar so fake registrar tests do not mutate Flutter debug globals. /// /// Returns without registering anything when debug mode is false. Repeated /// calls in the same isolate are idempotent. static void ensureInitialized({ PilotRuntimeExtensionRegistrar? registerExtension, bool? debugMode, + bool? captureLogs, }) { final bool shouldRegister = debugMode ?? kDebugMode; if (!shouldRegister || _initialized) { @@ -68,6 +79,11 @@ class PilotRuntimeBinding { registrar(PilotRuntimeProtocol.clearTextExtension, _handleClearText); registrar(PilotRuntimeProtocol.enterTextExtension, _handleEnterText); registrar(PilotRuntimeProtocol.scrollExtension, _handleScroll); + registrar(PilotRuntimeProtocol.collectLogsExtension, _handleCollectLogs); + final bool shouldCaptureLogs = captureLogs ?? registerExtension == null; + if (shouldCaptureLogs) { + _installLogCapture(); + } _initialized = true; } @@ -77,6 +93,15 @@ class PilotRuntimeBinding { /// fake registrar when they need repeated isolated assertions. @visibleForTesting static void debugResetForTesting() { + if (_previousDebugPrint != null) { + debugPrint = _previousDebugPrint!; + } + FlutterError.onError = _previousFlutterErrorHandler; + ui.PlatformDispatcher.instance.onError = _previousPlatformErrorHandler; + _previousDebugPrint = null; + _previousFlutterErrorHandler = null; + _previousPlatformErrorHandler = null; + _logEntries.clear(); _initialized = false; } @@ -132,6 +157,76 @@ class PilotRuntimeBinding { ); } + static Future> _handleCollectLogs( + Map parameters, + ) async { + return { + 'schema': 'pilot_runtime.logs.v1', + 'entries': [ + for (final Map entry in _logEntries) + Map.from(entry), + ], + }; + } + + static void _installLogCapture() { + _previousDebugPrint = debugPrint; + debugPrint = (String? message, {int? wrapWidth}) { + final String? text = message; + if (text != null && text.isNotEmpty) { + _appendLogEntry(level: 'info', message: text, source: 'debugPrint'); + } + _previousDebugPrint?.call(message, wrapWidth: wrapWidth); + }; + + _previousFlutterErrorHandler = FlutterError.onError; + FlutterError.onError = (FlutterErrorDetails details) { + _appendLogEntry( + level: 'error', + message: details.exceptionAsString(), + error: details.context?.toDescription(), + stackTrace: details.stack?.toString(), + source: 'FlutterError', + ); + _previousFlutterErrorHandler?.call(details); + }; + + _previousPlatformErrorHandler = ui.PlatformDispatcher.instance.onError; + ui.PlatformDispatcher.instance.onError = (Object error, StackTrace stack) { + _appendLogEntry( + level: 'error', + message: error.toString(), + stackTrace: stack.toString(), + source: 'PlatformDispatcher', + ); + final ui.ErrorCallback? previousHandler = _previousPlatformErrorHandler; + if (previousHandler != null) { + return previousHandler(error, stack); + } + return false; + }; + } + + static void _appendLogEntry({ + required String level, + required String message, + required String source, + String? error, + String? stackTrace, + }) { + _logEntries.add({ + 'timestamp': DateTime.now().toUtc().toIso8601String(), + 'level': level, + 'message': message, + 'source': source, + if (error != null && error.isNotEmpty) 'error': error, + if (stackTrace != null && stackTrace.isNotEmpty) 'stackTrace': stackTrace, + }); + if (_logEntries.length > _maxLogEntries) { + _logEntries.removeRange(0, _logEntries.length - _maxLogEntries); + } + } + static String? _optionalString( Map parameters, String field, diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_binding_stub.dart b/packages/pilot_runtime/lib/src/pilot_runtime_binding_stub.dart index 263e694..8deb7d4 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_binding_stub.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_binding_stub.dart @@ -10,6 +10,7 @@ class PilotRuntimeBinding { static void ensureInitialized({ PilotRuntimeExtensionRegistrar? registerExtension, bool? debugMode, + bool? captureLogs, }) { throw UnsupportedError( 'PilotRuntimeBinding.ensureInitialized() is only available in Flutter.', diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart index e187189..3c4f79b 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'pilot_runtime_protocol.dart'; import 'widget_tree_normalizer.dart'; @@ -23,6 +25,17 @@ abstract interface class PilotRuntimeVmService { String extensionName, { Map parameters = const {}, }); + + /// Request VM Service source reload for the selected Runtime Target isolate. + /// + /// Args: + /// - `force`: `false` performs Flutter hot reload semantics. `true` forces a + /// full source reload and is used by Flutter Pilot as the hot restart + /// client capability. + /// + /// Returns the decoded VM Service reload report so the client can normalize + /// success and failure into Flutter Pilot-owned result types. + Future> reloadSources({required bool force}); } /// Signals that a required Flutter Pilot service extension is not registered. @@ -181,6 +194,62 @@ class PilotRuntimeActionException implements Exception { } } +/// Runtime lifecycle operation requested through VM Service source reload. +enum PilotRuntimeReloadOperation { + /// Hot reload updates modified source without forcing a full restart. + hotReload, + + /// Hot restart forces source reload for the selected Runtime Target isolate. + hotRestart, +} + +/// Normalized result for a runtime hot reload or hot restart request. +/// +/// The result keeps the VM Service response available for diagnostics while +/// exposing operation and success fields that belong to Flutter Pilot's client +/// contract. +class PilotRuntimeReloadResult { + /// Create one normalized VM Service reload result. + const PilotRuntimeReloadResult({ + required this.operation, + required this.success, + required this.response, + }); + + /// Runtime lifecycle operation that produced this result. + final PilotRuntimeReloadOperation operation; + + /// Whether the VM Service reported a successful reload. + final bool success; + + /// Decoded VM Service reload response retained for diagnostics. + final Map response; +} + +/// Failure thrown when a VM Service hot reload or hot restart request fails. +class PilotRuntimeReloadException implements Exception { + /// Create a runtime reload failure. + const PilotRuntimeReloadException({ + required this.operation, + required this.message, + this.cause, + }); + + /// Runtime lifecycle operation that failed. + final PilotRuntimeReloadOperation operation; + + /// Human-readable explanation suitable for run reports or calibration logs. + final String message; + + /// Original VM Service error or response when available. + final Object? cause; + + @override + String toString() { + return 'PilotRuntimeReloadException: $message'; + } +} + /// Logical-pixel rectangle reported for a resolved Finder Match. class PilotRuntimeBounds { /// Create one visible target bounds rectangle. @@ -429,9 +498,9 @@ class PilotRuntimeClient { ); } - final Map rawTree; + final Map rawResponse; try { - rawTree = await _vmService.callServiceExtension( + rawResponse = await _vmService.callServiceExtension( PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension, parameters: PilotRuntimeInspectorProtocol.summaryTreeParameters, ); @@ -446,6 +515,9 @@ class PilotRuntimeClient { } try { + final Map rawTree = _unwrapWidgetTreeResponse( + rawResponse, + ); return PilotRuntimeWidgetTreeNormalizer.normalize(rawTree); } on FormatException catch (error) { throw PilotRuntimeWidgetTreeCaptureException( @@ -458,6 +530,42 @@ class PilotRuntimeClient { } } + /// Extract the Inspector diagnostics node from VM Service response wrappers. + /// + /// Some Flutter Inspector service extension calls return the diagnostics tree + /// directly while others return a `Response` object with a JSON-encoded + /// payload. Normalize both shapes before Widget Tree validation. + static Map _unwrapWidgetTreeResponse( + Map response, + ) { + final Object? jsonPayload = response['json']; + if (jsonPayload is String) { + final Object? decoded = jsonDecode(jsonPayload); + if (decoded is Map) { + return decoded; + } + throw const FormatException( + 'Widget Tree response json must decode to an object.', + ); + } + + final Object? result = response['result']; + if (result is Map) { + return result; + } + if (result is String) { + final Object? decoded = jsonDecode(result); + if (decoded is Map) { + return decoded; + } + throw const FormatException( + 'Widget Tree response result must decode to an object.', + ); + } + + return response; + } + /// Resolve one Finder through the app-side runtime extension. /// /// Args: @@ -581,6 +689,39 @@ class PilotRuntimeClient { _checkActionResponse(response, actionName: 'scroll'); } + /// Collect buffered runtime logs from the app-side runtime hook. + /// + /// Returns the structured Logs payload exposed by the Runtime Target. The + /// payload includes debug print messages and Flutter runtime errors captured + /// since `PilotRuntimeBinding.ensureInitialized()` was called. + Future> collectLogs() async { + return _vmService.callServiceExtension( + PilotRuntimeProtocol.collectLogsExtension, + ); + } + + /// Request a hot reload through VM Service source reload. + /// + /// This operation is a client capability, not a Scenario Step action and not + /// an app-side `ext.flutter_pilot.runtime.*` hook extension. + Future hotReload() { + return _reloadSources( + operation: PilotRuntimeReloadOperation.hotReload, + force: false, + ); + } + + /// Request a hot restart through VM Service source reload. + /// + /// This operation is a client capability, not a Scenario Step action and not + /// an app-side `ext.flutter_pilot.runtime.*` hook extension. + Future hotRestart() { + return _reloadSources( + operation: PilotRuntimeReloadOperation.hotRestart, + force: true, + ); + } + void _checkActionResponse( Map response, { required String actionName, @@ -645,4 +786,52 @@ class PilotRuntimeClient { ); } } + + Future _reloadSources({ + required PilotRuntimeReloadOperation operation, + required bool force, + }) async { + final Map response; + try { + response = await _vmService.reloadSources(force: force); + } catch (error) { + throw PilotRuntimeReloadException( + operation: operation, + message: '${_reloadOperationLabel(operation)} failed: $error', + cause: error, + ); + } + + final Object? successValue = response['success']; + if (successValue is! bool) { + throw PilotRuntimeReloadException( + operation: operation, + message: + '${_reloadOperationLabel(operation)} returned an invalid VM ' + 'Service reload report.', + cause: response, + ); + } + + if (!successValue) { + throw PilotRuntimeReloadException( + operation: operation, + message: '${_reloadOperationLabel(operation)} failed.', + cause: response, + ); + } + + return PilotRuntimeReloadResult( + operation: operation, + success: successValue, + response: response, + ); + } + + String _reloadOperationLabel(PilotRuntimeReloadOperation operation) { + return switch (operation) { + PilotRuntimeReloadOperation.hotReload => 'Hot reload', + PilotRuntimeReloadOperation.hotRestart => 'Hot restart', + }; + } } diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart b/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart index d496d29..a4931de 100644 --- a/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart +++ b/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart @@ -1,7 +1,7 @@ /// Protocol constants shared by the app-side hook and VM Service client. /// /// Version 1 exposes runtime handshake, visible Finder resolution, tap, -/// editable text entry, and scroll replay. +/// editable text entry, scroll replay, and runtime log collection. class PilotRuntimeProtocol { PilotRuntimeProtocol._(); @@ -30,6 +30,10 @@ class PilotRuntimeProtocol { /// VM Service extension used to drag one scrollable Runtime Handle. static const String scrollExtension = 'ext.flutter_pilot.runtime.scroll'; + /// VM Service extension used to collect buffered Flutter runtime logs. + static const String collectLogsExtension = + 'ext.flutter_pilot.runtime.collectLogs'; + /// Capability name reported when the handshake extension is available. static const String handshakeCapability = 'runtime.handshake'; @@ -48,6 +52,9 @@ class PilotRuntimeProtocol { /// Capability name reported when scroll replay is available. static const String scrollCapability = 'runtime.action.scroll'; + /// Capability name reported when runtime logs can be collected. + static const String collectLogsCapability = 'runtime.logs.collect'; + /// Capabilities that this client requires before Scenario execution. static const Set requiredCapabilities = { handshakeCapability, @@ -56,6 +63,7 @@ class PilotRuntimeProtocol { clearTextCapability, enterTextCapability, scrollCapability, + collectLogsCapability, }; } diff --git a/packages/pilot_runtime/lib/src/tap_performer.dart b/packages/pilot_runtime/lib/src/tap_performer.dart index 0ae4c31..310f0e2 100644 --- a/packages/pilot_runtime/lib/src/tap_performer.dart +++ b/packages/pilot_runtime/lib/src/tap_performer.dart @@ -33,14 +33,23 @@ class PilotRuntimeTapPerformer { ); } - final SemanticsNode? semanticsNode = element.renderObject?.debugSemantics; + final Element? actionElement = _actionElementFor(element); + if (actionElement == null) { + return _failure( + code: 'notTappable', + message: 'Runtime Handle $handle cannot be tapped.', + ); + } + + final SemanticsNode? semanticsNode = + actionElement.renderObject?.debugSemantics; if (semanticsNode != null && semanticsNode.getSemanticsData().hasAction(SemanticsAction.tap)) { semanticsNode.owner?.performAction(semanticsNode.id, SemanticsAction.tap); return {'ok': true, 'method': 'semantic'}; } - final Offset? center = _centerFor(element); + final Offset? center = _centerFor(actionElement); if (center == null) { return _failure( code: 'notTappable', @@ -48,12 +57,6 @@ class PilotRuntimeTapPerformer { 'Runtime Handle $handle does not have usable bounds for pointer tap.', ); } - if (!_canReceivePointerTap(element)) { - return _failure( - code: 'notTappable', - message: 'Runtime Handle $handle cannot be tapped.', - ); - } final int pointer = DateTime.now().microsecondsSinceEpoch; GestureBinding.instance.handlePointerEvent( @@ -87,6 +90,24 @@ class PilotRuntimeTapPerformer { return renderObject.localToGlobal(renderObject.size.center(Offset.zero)); } + static Element? _actionElementFor(Element element) { + if (_hasSemanticTap(element) || _canReceivePointerTap(element)) { + return element; + } + + Element? match; + element.visitChildren((Element child) { + match ??= _actionElementFor(child); + }); + return match; + } + + static bool _hasSemanticTap(Element element) { + final SemanticsNode? semanticsNode = element.renderObject?.debugSemantics; + return semanticsNode != null && + semanticsNode.getSemanticsData().hasAction(SemanticsAction.tap); + } + static bool _canReceivePointerTap(Element element) { final Widget widget = element.widget; if (widget is GestureDetector && widget.onTap != null) { diff --git a/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart b/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart index e48cbbc..6eda373 100644 --- a/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart +++ b/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart @@ -40,12 +40,14 @@ class PilotRuntimeWidgetTreeNormalizer { Map node, String path, ) { - final String description = _requiredString(node, 'description', path); final String widgetRuntimeType = _requiredString( node, 'widgetRuntimeType', path, ); + final String description = path == 'root' + ? _rootDescription(node, widgetRuntimeType) + : _requiredString(node, 'description', path); final String inspectorValueId = _requiredString(node, 'valueId', path); final Map normalized = { @@ -99,6 +101,22 @@ class PilotRuntimeWidgetTreeNormalizer { return value; } + /// Return the root node description, accepting Android Inspector omissions. + /// + /// Some real debug Runtime Targets return a root summary node without a + /// usable `description`. The root still has `widgetRuntimeType`, so use that + /// as the display label while preserving strict validation for child nodes. + static String _rootDescription( + Map node, + String widgetRuntimeType, + ) { + final Object? value = node['description']; + if (value is String && value.isNotEmpty) { + return value; + } + return widgetRuntimeType; + } + /// Normalize an Inspector node's child list. /// /// Args: diff --git a/packages/pilot_runtime/test/pilot_runtime_binding_test.dart b/packages/pilot_runtime/test/pilot_runtime_binding_test.dart index f3abb56..3d12da1 100644 --- a/packages/pilot_runtime/test/pilot_runtime_binding_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_binding_test.dart @@ -28,6 +28,7 @@ void main() { PilotRuntimeProtocol.clearTextExtension, PilotRuntimeProtocol.enterTextExtension, PilotRuntimeProtocol.scrollExtension, + PilotRuntimeProtocol.collectLogsExtension, ]); expect( await registeredHandlers[PilotRuntimeProtocol.handshakeExtension]!( @@ -42,6 +43,7 @@ void main() { 'runtime.action.tap', 'runtime.finder.resolve', 'runtime.handshake', + 'runtime.logs.collect', ], }, ); @@ -73,6 +75,7 @@ void main() { child: SingleChildScrollView(child: SizedBox(height: 1000)), ), ); + addTearDown(PilotRuntimeBinding.debugResetForTesting); PilotRuntimeBinding.ensureInitialized( debugMode: true, registerExtension: @@ -85,8 +88,81 @@ void main() { await registeredHandlers[PilotRuntimeProtocol.scrollExtension]!( {'deltaX': '0.0', 'deltaY': '-120.5'}, ); + PilotRuntimeBinding.debugResetForTesting(); expect(response['ok'], true); }); + + testWidgets('captures debug prints and Flutter errors as runtime logs', ( + WidgetTester tester, + ) async { + final Map registeredHandlers = + {}; + final void Function(FlutterErrorDetails)? originalFlutterErrorHandler = + FlutterError.onError; + + FlutterError.onError = (FlutterErrorDetails details) {}; + addTearDown(PilotRuntimeBinding.debugResetForTesting); + PilotRuntimeBinding.ensureInitialized( + debugMode: true, + captureLogs: true, + registerExtension: + (String extensionName, PilotRuntimeExtensionHandler handler) { + registeredHandlers[extensionName] = handler; + }, + ); + + debugPrint('Submitting checkout form'); + FlutterError.reportError( + FlutterErrorDetails( + exception: StateError('Checkout failed'), + stack: StackTrace.current, + library: 'flutter_pilot_test', + context: ErrorDescription('while submitting checkout'), + ), + ); + + final Map response = + await registeredHandlers[PilotRuntimeProtocol.collectLogsExtension]!( + const {}, + ); + PilotRuntimeBinding.debugResetForTesting(); + FlutterError.onError = originalFlutterErrorHandler; + + expect(response['schema'], 'pilot_runtime.logs.v1'); + final List entries = response['entries']! as List; + expect( + entries, + contains( + isA>() + .having( + (Map entry) => entry['level'], + 'level', + 'info', + ) + .having( + (Map entry) => entry['message'], + 'message', + 'Submitting checkout form', + ), + ), + ); + expect( + entries, + contains( + isA>() + .having( + (Map entry) => entry['level'], + 'level', + 'error', + ) + .having( + (Map entry) => entry['message'], + 'message', + contains('Checkout failed'), + ), + ), + ); + }); }); } diff --git a/packages/pilot_runtime/test/pilot_runtime_client_test.dart b/packages/pilot_runtime/test/pilot_runtime_client_test.dart index 8f7f735..f9bf25b 100644 --- a/packages/pilot_runtime/test/pilot_runtime_client_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_client_test.dart @@ -15,6 +15,7 @@ void main() { 'runtime.action.tap', 'runtime.finder.resolve', 'runtime.handshake', + 'runtime.logs.collect', ], }, ); @@ -29,6 +30,7 @@ void main() { expect(session.capabilities, contains('runtime.action.clearText')); expect(session.capabilities, contains('runtime.action.enterText')); expect(session.capabilities, contains('runtime.action.scroll')); + expect(session.capabilities, contains('runtime.logs.collect')); expect(vmService.calledExtensions, [ PilotRuntimeProtocol.handshakeExtension, ]); @@ -319,6 +321,107 @@ void main() { }, ); }); + + group('PilotRuntimeClient logs', () { + test('returns structured runtime logs from the logs extension', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + extensionResponses: >{ + PilotRuntimeProtocol.collectLogsExtension: { + 'schema': 'pilot_runtime.logs.v1', + 'entries': [ + { + 'level': 'info', + 'message': 'Submitting checkout form', + }, + ], + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final Map logs = await client.collectLogs(); + + expect(vmService.calledExtensions, [ + PilotRuntimeProtocol.collectLogsExtension, + ]); + expect(logs, { + 'schema': 'pilot_runtime.logs.v1', + 'entries': [ + { + 'level': 'info', + 'message': 'Submitting checkout form', + }, + ], + }); + }); + }); + + group('PilotRuntimeClient reload lifecycle', () { + test('uses VM Service reloadSources for hot reload', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + reloadResponses: >{ + false: {'type': 'ReloadReport', 'success': true}, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final PilotRuntimeReloadResult result = await client.hotReload(); + + expect(result.operation, PilotRuntimeReloadOperation.hotReload); + expect(result.success, isTrue); + expect(result.response, { + 'type': 'ReloadReport', + 'success': true, + }); + expect(vmService.reloadForces, [false]); + }); + + test('uses forced VM Service reloadSources for hot restart', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + reloadResponses: >{ + true: {'type': 'ReloadReport', 'success': true}, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final PilotRuntimeReloadResult result = await client.hotRestart(); + + expect(result.operation, PilotRuntimeReloadOperation.hotRestart); + expect(result.success, isTrue); + expect(vmService.reloadForces, [true]); + }); + + test('maps failed VM Service reload report to typed failure', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + reloadResponses: >{ + false: {'type': 'ReloadReport', 'success': false}, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.hotReload(), + throwsA( + isA() + .having( + (PilotRuntimeReloadException error) => error.operation, + 'operation', + PilotRuntimeReloadOperation.hotReload, + ) + .having( + (PilotRuntimeReloadException error) => error.message, + 'message', + contains('Hot reload failed'), + ) + .having( + (PilotRuntimeReloadException error) => error.cause, + 'cause', + {'type': 'ReloadReport', 'success': false}, + ), + ), + ); + }); + }); } class FakePilotRuntimeVmService implements PilotRuntimeVmService { @@ -327,14 +430,19 @@ class FakePilotRuntimeVmService implements PilotRuntimeVmService { this.handshakeResponse = const {}, this.missingExtension = false, Map>? extensionResponses, + Map>? reloadResponses, }) : extensionResponses = - extensionResponses ?? const >{}; + extensionResponses ?? const >{}, + reloadResponses = + reloadResponses ?? const >{}; final Map handshakeResponse; final bool missingExtension; final Map> extensionResponses; + final Map> reloadResponses; final List calledExtensions = []; final List> calledParameters = >[]; + final List reloadForces = []; @override Future> callServiceExtension( @@ -353,4 +461,11 @@ class FakePilotRuntimeVmService implements PilotRuntimeVmService { } return handshakeResponse; } + + @override + Future> reloadSources({required bool force}) async { + reloadForces.add(force); + return reloadResponses[force] ?? + {'type': 'ReloadReport', 'success': true}; + } } diff --git a/packages/pilot_runtime/test/pilot_runtime_finder_test.dart b/packages/pilot_runtime/test/pilot_runtime_finder_test.dart index 1a664c0..67faef1 100644 --- a/packages/pilot_runtime/test/pilot_runtime_finder_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_finder_test.dart @@ -423,6 +423,38 @@ void main() { }, ); + testWidgets('tap delegates from a custom wrapper to a tappable child', ( + WidgetTester tester, + ) async { + final Map extensions = + _registerRuntimeExtensions(); + int taps = 0; + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: _CustomTapWrapper( + key: const ValueKey('custom_wrapper_target'), + onTap: () { + taps += 1; + }, + ), + ), + ), + ); + + final Map response = await _resolveFinder( + extensions, + byKey: 'custom_wrapper_target', + byWidget: '_CustomTapWrapper', + ); + final Map match = _singleMatch(response); + + await _tap(extensions, handle: match['handle']! as String); + await tester.pump(); + + expect(taps, 1); + }); + testWidgets('tap falls back to pointer center tap for Material buttons', ( WidgetTester tester, ) async { @@ -699,6 +731,21 @@ void main() { }); } +class _CustomTapWrapper extends StatelessWidget { + const _CustomTapWrapper({required this.onTap, super.key}); + + final VoidCallback onTap; + + @override + Widget build(BuildContext context) { + return GestureDetector( + behavior: HitTestBehavior.opaque, + onTap: onTap, + child: const SizedBox(width: 80, height: 40), + ); + } +} + Map _registerRuntimeExtensions() { final Map extensions = {}; diff --git a/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart b/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart index fc1d6d0..0d7e56b 100644 --- a/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart +++ b/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart @@ -160,6 +160,68 @@ void main() { }, ); + test( + 'falls back to widget runtime type when root description is absent', + () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{ + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + {'result': 'ok'}, + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension: + { + 'widgetRuntimeType': 'RootWidget', + 'valueId': 'inspector-1', + 'children': [ + { + 'description': 'Text', + 'widgetRuntimeType': 'Text', + 'valueId': 'inspector-2', + }, + ], + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final Map widgetTree = await client.captureWidgetTree( + projectRoot: '/tmp/smoke_app', + ); + final Map root = + widgetTree['root']! as Map; + + expect(root['description'], 'RootWidget'); + expect(root['widgetRuntimeType'], 'RootWidget'); + }, + ); + + test( + 'unwraps VM Service response JSON before normalizing Widget Tree', + () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{ + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + {'result': 'ok'}, + PilotRuntimeInspectorProtocol + .getRootWidgetTreeExtension: { + 'type': 'Response', + 'json': + '{"description":"Root","widgetRuntimeType":"RootWidget","valueId":"inspector-1"}', + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final Map widgetTree = await client.captureWidgetTree( + projectRoot: '/tmp/smoke_app', + ); + final Map root = + widgetTree['root']! as Map; + + expect(root['description'], 'Root'); + expect(root['widgetRuntimeType'], 'RootWidget'); + }, + ); + test( 'fails clearly when Inspector cannot set pub root directories', () async { @@ -257,6 +319,11 @@ class FakePilotRuntimeVmService implements PilotRuntimeVmService { } return responses[extensionName] ?? {}; } + + @override + Future> reloadSources({required bool force}) async { + return {'type': 'ReloadReport', 'success': true}; + } } class FakeVmServiceCall { diff --git a/test/artifacts/artifact_store_test.dart b/test/artifacts/artifact_store_test.dart index 457c004..de1739b 100644 --- a/test/artifacts/artifact_store_test.dart +++ b/test/artifacts/artifact_store_test.dart @@ -386,11 +386,11 @@ void main() { 'captures/0001_checkpoint_widget_tree.json', ); expect(logsArtifact.type, ArtifactType.logs); - expect(logsArtifact.path, 'captures/0001_checkpoint_logs.json'); + expect(logsArtifact.path, 'captures/0001_checkpoint_logs.log'); expect(failureLogsArtifact.purpose, ArtifactPurpose.failure); expect(failureLogsArtifact.toJson(), { 'type': 'logs', - 'path': 'captures/0002_failed_submit_logs.json', + 'path': 'captures/0002_failed_submit_logs.log', 'purpose': 'failure', }); expect( @@ -415,6 +415,10 @@ void main() { ) as Map; expect(logsJson['entries'], hasLength(1)); + expect( + File('${writer.runDirectory.path}/${logsArtifact.path}').path, + endsWith('.log'), + ); }); }); } diff --git a/test/diff/run_diff_diagnostics_test.dart b/test/diff/run_diff_diagnostics_test.dart index bf3952d..0899410 100644 --- a/test/diff/run_diff_diagnostics_test.dart +++ b/test/diff/run_diff_diagnostics_test.dart @@ -71,7 +71,7 @@ void main() { ); writeJsonArtifact( runs.beforeRun, - 'captures/before_logs.json', + 'captures/before_logs.log', { 'entries': [ { @@ -93,7 +93,7 @@ void main() { ); writeJsonArtifact( runs.afterRun, - 'captures/after_logs.json', + 'captures/after_logs.log', { 'entries': [ { @@ -110,7 +110,7 @@ void main() { type: 'snapshot', path: 'captures/before_snapshot.json', ), - artifactReport(type: 'logs', path: 'captures/before_logs.json'), + artifactReport(type: 'logs', path: 'captures/before_logs.log'), ], ); writeRunReport( @@ -120,7 +120,7 @@ void main() { type: 'snapshot', path: 'captures/after_snapshot.json', ), - artifactReport(type: 'logs', path: 'captures/after_logs.json'), + artifactReport(type: 'logs', path: 'captures/after_logs.log'), ], ); @@ -154,7 +154,7 @@ void main() { type: 'snapshot', path: 'captures/missing_snapshot.json', ), - artifactReport(type: 'logs', path: 'captures/missing_logs.json'), + artifactReport(type: 'logs', path: 'captures/missing_logs.log'), ], ); writeRunReport(runs.afterRun); @@ -171,7 +171,7 @@ void main() { expect(output, contains('Missing snapshot artifact')); expect(output, contains('captures/missing_snapshot.json')); expect(output, contains('Missing logs artifact')); - expect(output, contains('captures/missing_logs.json')); + expect(output, contains('captures/missing_logs.log')); expect(json['outcome'], 'changed'); expect(json['warnings'], contains(contains('Missing snapshot artifact'))); expect(json['warnings'], contains(contains('Missing logs artifact'))); diff --git a/test/execution/scenario_runner_test.dart b/test/execution/scenario_runner_test.dart index f56d473..2a769d2 100644 --- a/test/execution/scenario_runner_test.dart +++ b/test/execution/scenario_runner_test.dart @@ -835,7 +835,7 @@ steps: widgetTreeArtifact.path, 'captures/0001_after_submit_widget_tree.json', ); - expect(logsArtifact.path, 'captures/0001_after_submit_logs.json'); + expect(logsArtifact.path, 'captures/0001_after_submit_logs.log'); expect( File( '${report.runDirectoryPath}/${screenshotArtifact.path}', diff --git a/test/fixtures/run_diff/partial_artifact/before/run_report.json b/test/fixtures/run_diff/partial_artifact/before/run_report.json index 509f0bd..34d9b9b 100644 --- a/test/fixtures/run_diff/partial_artifact/before/run_report.json +++ b/test/fixtures/run_diff/partial_artifact/before/run_report.json @@ -13,7 +13,7 @@ }, { "type": "logs", - "path": "captures/missing_logs.json" + "path": "captures/missing_logs.log" } ], "steps": [ diff --git a/test/reports/html_timeline_report_test.dart b/test/reports/html_timeline_report_test.dart index f6a9359..5e7f988 100644 --- a/test/reports/html_timeline_report_test.dart +++ b/test/reports/html_timeline_report_test.dart @@ -65,7 +65,7 @@ void main() { contains('{ + 'schema': 'pilot_runtime.logs.v1', + 'entries': [ + { + 'level': 'info', + 'message': 'Submitting checkout form', + }, + ], + }, + ); + final PilotRuntimeAdapter adapter = PilotRuntimeAdapter( + client: client, + projectRoot: '/target/app', + ); + + final LogsCapture capture = await adapter.collectLogs(); + + expect(client.collectedLogs, isTrue); + expect(capture.data, { + 'schema': 'pilot_runtime.logs.v1', + 'entries': [ + { + 'level': 'info', + 'message': 'Submitting checkout form', + }, + ], + }); + }); + test( 'maps pilot_runtime Finder Matches to Runtime Adapter matches', () async { @@ -449,8 +480,10 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { this.clearTextFailure, this.scrollFailure, Map? widgetTree, + Map? logs, List? finderMatches, }) : widgetTree = widgetTree ?? {}, + logs = logs ?? {}, finderMatches = finderMatches ?? const []; final PilotRuntimeInitializationException? initializeFailure; @@ -458,8 +491,10 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { final Object? clearTextFailure; final Object? scrollFailure; final Map widgetTree; + final Map logs; final List finderMatches; final List projectRoots = []; + bool collectedLogs = false; final List< ({String? byText, String? byType, String? byKey, String? byWidget}) > @@ -487,6 +522,7 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { 'runtime.action.tap', 'runtime.finder.resolve', 'runtime.handshake', + 'runtime.logs.collect', }, ); } @@ -499,6 +535,30 @@ class _FakePilotRuntimeClient implements PilotRuntimeClient { return widgetTree; } + @override + Future> collectLogs() async { + collectedLogs = true; + return logs; + } + + @override + Future hotReload() async { + return const PilotRuntimeReloadResult( + operation: PilotRuntimeReloadOperation.hotReload, + success: true, + response: {'type': 'ReloadReport', 'success': true}, + ); + } + + @override + Future hotRestart() async { + return const PilotRuntimeReloadResult( + operation: PilotRuntimeReloadOperation.hotRestart, + success: true, + response: {'type': 'ReloadReport', 'success': true}, + ); + } + @override Future> resolveFinder({ String? byText, diff --git a/test/smoke/smoke_scenario_contract_test.dart b/test/smoke/smoke_scenario_contract_test.dart index 4a69cef..ea3f920 100644 --- a/test/smoke/smoke_scenario_contract_test.dart +++ b/test/smoke/smoke_scenario_contract_test.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + import 'package:flutter_pilot/flutter_pilot.dart'; import 'package:test/test.dart'; @@ -43,6 +45,128 @@ void main() { expect(scenario.steps, hasLength(4)); }, ); + + test( + 'pilot_runtime calibration Project Scenarios cover replacement checks', + () { + final List scenarios = + ProjectScenarioDiscovery.discoverInDirectory( + 'examples/smoke_app/pilot', + ) + .where( + (ProjectScenarioFile scenario) => + scenario.relativePath.startsWith('calibration/'), + ) + .toList(); + + expect( + scenarios.map((ProjectScenarioFile scenario) => scenario.relativePath), + [ + 'calibration/01_interact.yaml', + 'calibration/02_after_restart.yaml', + ], + ); + + final Scenario interact = scenarios.first.scenario; + expect(interact.name, 'pilot_runtime_calibration_01_interact'); + + final TapAction tapByText = _actionWithLabel( + interact, + 'tap_calibration_button_by_text_and_type', + ); + expect(tapByText.finder.byText, 'Calibration tap'); + expect(tapByText.finder.byType, 'button'); + + final TypeAction typeBySemanticType = _actionWithLabel( + interact, + 'type_calibration_field_by_semantic_type', + ); + expect(typeBySemanticType.finder.byType, 'textField'); + expect(typeBySemanticType.text, 'calibrated@example.com'); + + final TapAction tapByKeyAndWidget = _actionWithLabel( + interact, + 'tap_calibration_chip_by_key_and_widget', + ); + expect(tapByKeyAndWidget.finder.byKey, 'calibration_chip'); + expect(tapByKeyAndWidget.finder.byWidget, 'CalibrationChip'); + + final ScrollAction targetedScroll = _actionWithLabel( + interact, + 'targeted_scroll_calibration_list', + ); + expect(targetedScroll.finder?.byKey, 'calibration_target_scrollable'); + + final ScrollAction primaryScroll = _actionWithLabel( + interact, + 'primary_scroll_calibration_page', + ); + expect(primaryScroll.finder, isNull); + + final CaptureAction capture = _actionWithLabel( + interact, + 'capture_calibration_artifacts', + ); + expect(capture.screenshot, isTrue); + expect(capture.widgetTree, isTrue); + expect(capture.logs, isTrue); + + final Scenario afterRestart = scenarios.last.scenario; + expect(afterRestart.name, 'pilot_runtime_calibration_02_after_restart'); + + final WaitForAction resetCheck = _actionWithLabel( + afterRestart, + 'verify_hot_restart_reset_state', + ); + expect(resetCheck.finder.byText, 'Calibration taps: 0'); + expect(resetCheck.finder.byKey, 'calibration_tap_count'); + + final CaptureAction restartCapture = _actionWithLabel( + afterRestart, + 'capture_after_restart_artifacts', + ); + expect(restartCapture.screenshot, isTrue); + expect(restartCapture.widgetTree, isTrue); + expect(restartCapture.logs, isTrue); + }, + ); + + test( + 'pilot_runtime calibration target initializes runtime and routes logs', + () { + final String source = File( + 'examples/smoke_app/lib/pilot_runtime_calibration_app.dart', + ).readAsStringSync(); + + expect(source, contains('PilotRuntimeBinding.ensureInitialized()')); + expect(source, contains("Logger('pilot_runtime_calibration')")); + expect(source, contains('Logger.root.onRecord.listen')); + expect(source, contains('debugPrint(')); + expect(source, contains('CalibrationChip')); + }, + ); + + test( + 'pilot_runtime calibration README states claimed and unclaimed targets', + () { + final String readme = File( + 'examples/smoke_app/README.md', + ).readAsStringSync(); + + expect(readme, contains('PilotRuntime Replacement Calibration')); + expect(readme, contains('macOS desktop debug')); + expect(readme, contains('Android debug')); + expect( + readme, + contains('Web, profile, release, and iOS are not claimed'), + ); + expect(readme, contains('These examples do not')); + expect( + readme, + contains('change Flutter Pilot runtime selection behavior'), + ); + }, + ); } /// Return the typed action for a labeled Step in the smoke Scenario.