diff --git a/.github/workflows/dart-checks.yml b/.github/workflows/dart-checks.yml deleted file mode 100644 index 0f09fb7..0000000 --- a/.github/workflows/dart-checks.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Dart Checks - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - dart: - name: Dart checks - runs-on: ubuntu-latest - - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set up Dart - uses: dart-lang/setup-dart@v1 - with: - sdk: stable - - - name: Install root dependencies - run: dart pub get - - - name: Install screen_recorder dependencies - working-directory: packages/screen_recorder - run: dart pub get - - - name: Check formatting - run: dart format --output=none --set-exit-if-changed . - - - name: Analyze - run: dart analyze --fatal-infos --fatal-warnings - - - name: Run unit tests - run: dart test diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index ad5ec24..2b528b1 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -24,10 +24,10 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Dart - uses: dart-lang/setup-dart@v1 + - name: Set up Flutter + uses: subosito/flutter-action@v2 with: - sdk: stable + channel: stable - name: Set up Node uses: actions/setup-node@v4 @@ -36,7 +36,11 @@ jobs: cache: npm - name: Install Dart dependencies - run: dart pub get + run: flutter pub get + + - name: Install pilot_runtime dependencies + working-directory: packages/pilot_runtime + run: flutter pub get - name: Install Node dependencies run: npm ci diff --git a/.github/workflows/flutter-dart-checks.yml b/.github/workflows/flutter-dart-checks.yml new file mode 100644 index 0000000..78512b0 --- /dev/null +++ b/.github/workflows/flutter-dart-checks.yml @@ -0,0 +1,59 @@ +name: Flutter/Dart Checks + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + dart: + name: Flutter/Dart checks + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Flutter + uses: subosito/flutter-action@v2 + with: + channel: stable + + - name: Install root dependencies + run: flutter pub get + + - name: Install pilot_runtime dependencies + working-directory: packages/pilot_runtime + run: flutter pub get + + - name: Install screen_recorder dependencies + working-directory: packages/screen_recorder + run: dart pub get + + - name: Check formatting + run: dart format --output=none --set-exit-if-changed . + + - name: Analyze + run: dart analyze --fatal-infos --fatal-warnings bin lib test tool + + - name: Analyze pilot_runtime + working-directory: packages/pilot_runtime + run: flutter analyze --fatal-infos --fatal-warnings + + - name: Analyze screen_recorder + working-directory: packages/screen_recorder + run: dart analyze --fatal-infos --fatal-warnings + + - name: Run unit tests + run: dart test + + - name: Run pilot_runtime tests + working-directory: packages/pilot_runtime + run: flutter test + + - name: Run screen_recorder tests + working-directory: packages/screen_recorder + run: dart test diff --git a/issues/0.1.1-pilot-runtime.md b/issues/0.1.1-pilot-runtime.md index 1828ea2..d4494d8 100644 --- a/issues/0.1.1-pilot-runtime.md +++ b/issues/0.1.1-pilot-runtime.md @@ -6,6 +6,8 @@ Publishing status: skipped by user request; local issue breakdown only. ## 1. Scaffold pilot_runtime with protocol handshake +Status: completed in `91f9d21 feat(runtime): scaffold pilot runtime handshake`. + ## Parent https://github.com/drown0315/flutter_pilot/issues/110 @@ -16,11 +18,18 @@ Create the first usable `pilot_runtime` package slice: a Flutter package with a ## Acceptance criteria -- [ ] `pilot_runtime` exists as an independent Flutter package with public app-side hook and client APIs. -- [ ] `PilotRuntimeBinding.ensureInitialized()` can register a debug runtime handshake and is a no-op outside debug mode. -- [ ] The client validates protocol version 1 and required capabilities through a handshake. -- [ ] Missing hook, missing capability, and protocol mismatch produce clear initialization failures. -- [ ] Unit tests cover successful handshake and each initialization failure path without requiring a live Flutter app. +- [x] `pilot_runtime` exists as an independent Flutter package with public app-side hook and client APIs. +- [x] `PilotRuntimeBinding.ensureInitialized()` can register a debug runtime handshake and is a no-op outside debug mode. +- [x] The client validates protocol version 1 and required capabilities through a handshake. +- [x] Missing hook, missing capability, and protocol mismatch produce clear initialization failures. +- [x] Unit tests cover successful handshake and each initialization failure path without requiring a live Flutter app. + +Verification: + +- `flutter analyze` from `packages/pilot_runtime` +- `flutter test` from `packages/pilot_runtime` +- `dart analyze` +- `dart test` ## Blocked by @@ -28,6 +37,8 @@ None - can start immediately ## 2. Capture normalized Widget Tree through pilot_runtime +Status: completed in working tree. + ## Parent https://github.com/drown0315/flutter_pilot/issues/110 @@ -38,12 +49,19 @@ Implement the first diagnostic capability in `pilot_runtime`: capture a normaliz ## Acceptance criteria -- [ ] `captureWidgetTree()` sets Inspector pub root directories using the runtime target project root. -- [ ] The client requests the root widget tree as a summary tree with previews and without full details. -- [ ] The returned JSON includes `schema`, `source`, and a normalized root node. -- [ ] Nodes normalize description, widget type, Inspector value id as `inspectorValueId`, text preview, local project marker, and children. -- [ ] The normalizer does not parse keys from Inspector description strings. -- [ ] Tests cover valid normalization, optional missing fields, invalid required shape, and Inspector call failures. +- [x] `captureWidgetTree()` sets Inspector pub root directories using the runtime target project root. +- [x] The client requests the root widget tree as a summary tree with previews and without full details. +- [x] The returned JSON includes `schema`, `source`, and a normalized root node. +- [x] Nodes normalize description, widget type, Inspector value id as `inspectorValueId`, text preview, local project marker, and children. +- [x] The normalizer does not parse keys from Inspector description strings. +- [x] Tests cover valid normalization, optional missing fields, invalid required shape, and Inspector call failures. + +Verification: + +- `flutter analyze` from `packages/pilot_runtime` +- `flutter test` from `packages/pilot_runtime` +- `dart analyze` +- `dart test` ## Blocked by @@ -51,6 +69,8 @@ Implement the first diagnostic capability in `pilot_runtime`: capture a normaliz ## 3. Expose PilotRuntimeAdapter behind hidden runtime switch +Status: completed in `7261c6c feat(runtime): expose pilot runtime adapter`. + ## Parent https://github.com/drown0315/flutter_pilot/issues/110 @@ -61,12 +81,20 @@ Wire Flutter Pilot to the new runtime without replacing the default bridge. Add ## Acceptance criteria -- [ ] Flutter Pilot can select `PilotRuntimeAdapter` through a hidden environment switch. -- [ ] The default runtime path still uses `McpFlutterRuntimeAdapter`. -- [ ] Invalid hidden runtime switch values fail clearly without appearing in public CLI help. -- [ ] `PilotRuntimeAdapter.initialize()` maps missing hook and protocol failures to run-level initialization failures. -- [ ] `PilotRuntimeAdapter.captureWidgetTree()` returns Flutter Pilot `WidgetTreeCapture` data from `pilot_runtime`. -- [ ] Tests cover default adapter selection, hidden switch selection, invalid switch values, and initialization failure behavior. +- [x] Flutter Pilot can select `PilotRuntimeAdapter` through a hidden environment switch. +- [x] The default runtime path still uses `McpFlutterRuntimeAdapter`. +- [x] Invalid hidden runtime switch values fail clearly without appearing in public CLI help. +- [x] `PilotRuntimeAdapter.initialize()` maps missing hook and protocol failures to run-level initialization failures. +- [x] `PilotRuntimeAdapter.captureWidgetTree()` returns Flutter Pilot `WidgetTreeCapture` data from `pilot_runtime`. +- [x] Tests cover default adapter selection, hidden switch selection, invalid switch values, and initialization failure behavior. + +Verification: + +- `dart format .` +- `dart analyze` +- `dart test` +- `flutter analyze` from `packages/pilot_runtime` +- `flutter test` from `packages/pilot_runtime` ## Blocked by diff --git a/lib/flutter_pilot.dart b/lib/flutter_pilot.dart index 8e18c42..4e24abc 100644 --- a/lib/flutter_pilot.dart +++ b/lib/flutter_pilot.dart @@ -18,6 +18,9 @@ export 'src/project_scenario_discovery.dart'; export 'src/project_run_report.dart'; export 'src/runtime/fake_runtime_adapter.dart'; export 'src/runtime/mcp_flutter_runtime_adapter.dart'; +export 'src/runtime/pilot_runtime_adapter.dart'; +export 'src/runtime/pilot_runtime_vm_service.dart'; +export 'src/runtime/runtime_adapter_selector.dart'; export 'src/runtime/runtime_contract.dart'; export 'src/run_diff.dart'; export 'src/scenario.dart'; @@ -28,4 +31,5 @@ export 'src/step_progress_renderer.dart'; export 'src/target_app_launch_progress_renderer.dart'; export 'src/target_app_launcher.dart'; export 'src/target_device.dart'; +export 'src/test_command_support.dart'; export 'src/terminal_style.dart'; diff --git a/lib/src/cli.dart b/lib/src/cli.dart index cb808ad..8748946 100644 --- a/lib/src/cli.dart +++ b/lib/src/cli.dart @@ -1,30 +1,13 @@ -import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'package:args/command_runner.dart'; -import 'package:path/path.dart' as p; -import 'package:screen_recorder/screen_recorder.dart' as screen_recorder; -import 'package:yaml/yaml.dart'; -import 'app_setup.dart'; -import 'artifacts/artifact_store.dart'; -import 'diagnostic_text_renderer.dart'; -import 'html_timeline_report.dart'; -import 'recording/recording_contract.dart'; -import 'recording/screen_recorder_recording_controller.dart'; -import 'project_scenario_discovery.dart'; -import 'project_run_report.dart'; -import 'runtime/mcp_flutter_runtime_adapter.dart'; -import 'runtime/runtime_contract.dart'; -import 'run_diff.dart'; -import 'scenario.dart'; -import 'scenario_parser.dart'; -import 'scenario_runner.dart'; -import 'step_progress_renderer.dart'; -import 'target_app_launch_progress_renderer.dart'; -import 'target_app_launcher.dart'; -import 'target_device.dart'; +import 'commands/app_setup_commands.dart'; +import 'commands/diff_command.dart'; +import 'commands/report_command.dart'; +import 'commands/test_command.dart'; +import 'commands/validate_command.dart'; +import 'test_command_support.dart'; /// Command-line entry point for Flutter Pilot. /// @@ -64,17 +47,17 @@ class FlutterPilotCli { 'flutter_pilot', 'Replay Flutter UI scenarios and collect debugging artifacts.', ) - ..addCommand(_ValidateCommand()) + ..addCommand(ValidateCommand()) ..addCommand( - _TestCommand( + TestCommand( executor: _testCommandExecutor, projectRunExecutor: _projectRunCommandExecutor, ), ) - ..addCommand(_ReportCommand()) - ..addCommand(_DiffCommand()) - ..addCommand(_DoctorCommand()) - ..addCommand(_InitCommand()); + ..addCommand(ReportCommand()) + ..addCommand(DiffCommand()) + ..addCommand(DoctorCommand()) + ..addCommand(InitCommand()); try { return await runner.run(arguments) ?? 0; @@ -86,1470 +69,3 @@ class FlutterPilotCli { } } } - -/// `doctor` command for checking Flutter Pilot setup in a Target App Package. -/// -/// It inspects the current working directory without modifying files. Missing -/// setup is reported as diagnostic output, while unreadable or unsupported -/// package shapes return an execution failure. -class _DoctorCommand extends Command { - @override - String get description => 'Check Flutter Pilot setup in a Flutter package.'; - - @override - String get name => 'doctor'; - - @override - Future run() async { - if (argResults!.rest.isNotEmpty) { - throw UsageException('Expected no arguments.', usage); - } - try { - final AppSetupStatus status = AppSetupChecker.check(Directory.current); - if (!status.isFlutterPackage) { - stderr.writeln(_flutterPackageOnlyMessage); - return 1; - } - stdout.writeln('Flutter Pilot doctor'); - stdout.writeln(''); - if (status.isComplete) { - stdout.writeln('✅ Flutter Pilot app setup is complete.'); - } else { - if (!status.hasMcpToolkitDependency) { - stdout.writeln( - '❌ MCP Toolkit dependency missing: run `flutter pub add mcp_toolkit`', - ); - } - if (!status.hasBootstrapFlutter) { - _writeBootstrapGuidance(); - } - } - return 0; - } on FileSystemException catch (error) { - stderr.writeln(error.message); - return 1; - } on YamlException catch (error) { - stderr.writeln(error.message); - return 1; - } - } -} - -/// `init` command for adding safe Flutter Pilot setup to a Target App Package. -/// -/// It can add the `mcp_toolkit` dependency through Flutter tooling, then -/// reports whether the app entrypoint still needs manual bootstrap code. -class _InitCommand extends Command { - @override - String get description => - 'Initialize Flutter Pilot setup in a Flutter package.'; - - @override - String get name => 'init'; - - @override - Future run() async { - if (argResults!.rest.isNotEmpty) { - throw UsageException('Expected no arguments.', usage); - } - try { - final AppSetupStatus status = AppSetupChecker.check(Directory.current); - if (!status.isFlutterPackage) { - stderr.writeln(_flutterPackageOnlyMessage); - return 1; - } - final AppSetupInitResult result = await AppSetupInitializer.initialize( - Directory.current, - addMcpToolkitDependency: _addMcpToolkitDependency, - ); - stdout.writeln('Flutter Pilot init'); - stdout.writeln(''); - if (result.addedMcpToolkitDependency) { - stdout.writeln('✅ Added MCP Toolkit dependency.'); - } else { - stdout.writeln('✅ MCP Toolkit dependency already exists.'); - } - if (result.status.hasBootstrapFlutter) { - stdout.writeln('✅ bootstrapFlutter already exists.'); - } else { - _writeBootstrapGuidance(); - } - return 0; - } on AppSetupInstallException catch (error) { - stderr.writeln('Failed to add MCP Toolkit dependency.'); - if (error.result.stderr.isNotEmpty) { - stderr.writeln(''); - stderr.writeln('flutter pub add output:'); - stderr.write(error.result.stderr); - } - stderr.writeln(''); - stderr.writeln('Run this command manually from the Flutter package:'); - stderr.writeln('flutter pub add mcp_toolkit'); - return 1; - } on FileSystemException catch (error) { - stderr.writeln(error.message); - return 1; - } on YamlException catch (error) { - stderr.writeln(error.message); - return 1; - } - } -} - -/// Run Flutter tooling to add the runtime dependency. -Future _addMcpToolkitDependency( - Directory packageDirectory, -) async { - try { - final ProcessResult result = await Process.run('flutter', [ - 'pub', - 'add', - 'mcp_toolkit', - ], workingDirectory: packageDirectory.path); - if (result.exitCode == 0) { - return const AppSetupInstallResult.success(); - } - return AppSetupInstallResult.failure( - exitCode: result.exitCode, - stderr: result.stderr.toString(), - ); - } on ProcessException catch (error) { - return AppSetupInstallResult.failure(exitCode: 1, stderr: error.message); - } -} - -/// Print the manual app entrypoint change required for MCP Toolkit. -void _writeBootstrapGuidance() { - stdout.writeln( - '❌ bootstrapFlutter missing: add ' - 'MCPToolkitBinding.instance.bootstrapFlutter in lib/main.dart', - ); - stdout.writeln(''); - stdout.writeln('Add the MCP Toolkit import:'); - stdout.writeln("import 'package:mcp_toolkit/mcp_toolkit.dart';"); - stdout.writeln(''); - stdout.writeln('Wrap runApp with MCPToolkitBinding:'); - stdout.writeln('Future main() async {'); - stdout.writeln(' await MCPToolkitBinding.instance.bootstrapFlutter('); - stdout.writeln(' runApp: () => runApp(const MyApp()),'); - stdout.writeln(' );'); - stdout.writeln('}'); -} - -const String _flutterPackageOnlyMessage = - 'Flutter Pilot only supports Flutter packages. Run this command from a ' - 'directory with a pubspec.yaml that declares dependencies.flutter.sdk: ' - 'flutter.'; - -/// `diff` command for comparing two existing Scenario Run directories. -/// -/// It reads each directory's `run_report.json`, generates a Step-focused Run -/// Diff, and prints either human-readable output or `--json` output. -/// Regressions are report content, not process failures, so successful diff -/// generation exits `0`. -class _DiffCommand extends Command { - _DiffCommand() { - argParser.addFlag( - 'json', - negatable: false, - help: 'Print machine-readable Run Diff output.', - ); - } - - @override - String get description => 'Compare two Scenario Run directories.'; - - @override - String get name => 'diff'; - - @override - Future run() async { - final bool jsonOutput = argResults!.flag('json'); - if (argResults!.rest.length != 2) { - throw UsageException('Expected before and after run directories.', usage); - } - final Directory beforeRunDirectory = Directory(argResults!.rest[0]); - final Directory afterRunDirectory = Directory(argResults!.rest[1]); - try { - final RunDiff diff = RunDiffEngine.diffDirectories( - beforeRunDirectory: beforeRunDirectory, - afterRunDirectory: afterRunDirectory, - ); - if (jsonOutput) { - stdout.writeln(RunDiffJsonRenderer.render(diff)); - } else { - stdout.writeln(RunDiffTextRenderer.render(diff)); - } - return 0; - } on RunDiffException catch (error) { - stderr.writeln(error.message); - return 1; - } - } -} - -/// `report` command for regenerating HTML from an existing run directory. -/// -/// It reads `/run_report.json` and writes -/// `/timeline.html` without connecting to a Flutter runtime. -class _ReportCommand extends Command { - @override - String get description => - 'Generate an HTML timeline report from an existing run directory.'; - - @override - String get name => 'report'; - - @override - Future run() async { - if (argResults!.rest.length != 1) { - throw UsageException('Expected exactly one run directory.', usage); - } - final Directory runDirectory = Directory(argResults!.rest.single); - if (!runDirectory.existsSync()) { - stderr.writeln('Run directory does not exist: ${runDirectory.path}'); - return 1; - } - try { - HtmlTimelineReport.generateFromRunDirectory(runDirectory); - stdout.writeln('HTML report: ${runDirectory.path}/timeline.html'); - return 0; - } on FileSystemException catch (error) { - stderr.writeln(error.message); - return 1; - } on FormatException catch (error) { - stderr.writeln(error.message); - return 1; - } - } -} - -/// `validate` command for checking Scenario YAML without connecting to Flutter. -/// -/// It reads exactly one file path. By default it prints human-readable output; -/// with `--json` it prints a stable object containing `valid` and `errors`. -class _ValidateCommand extends Command { - _ValidateCommand() { - argParser.addFlag( - 'json', - negatable: false, - help: 'Print machine-readable validation output.', - ); - } - - @override - String get description => 'Validate a scenario YAML file.'; - - @override - String get name => 'validate'; - - @override - Future run() async { - final bool jsonOutput = argResults!.flag('json'); - if (argResults!.rest.length != 1) { - throw UsageException('Expected exactly one scenario file.', usage); - } - - try { - ScenarioParser.parseFile(argResults!.rest.single); - if (jsonOutput) { - _writeValidationJson(valid: true, errors: const []); - } else { - stdout.writeln('Scenario is valid.'); - } - return 0; - } on ScenarioValidationException catch (error) { - if (jsonOutput) { - _writeValidationJson(valid: false, errors: error.errors); - } else { - _writeValidationErrors(error.errors); - } - return 1; - } - } - - /// Print validation status in the `validate --json` response shape. - void _writeValidationJson({ - required bool valid, - required List errors, - }) { - stdout.writeln( - const JsonEncoder.withIndent(' ').convert({ - 'valid': valid, - 'errors': [ - for (final ScenarioValidationError error in errors) - {'path': error.path, 'message': error.message}, - ], - }), - ); - } - - /// Print validation errors as `path: message` lines for humans. - void _writeValidationErrors(List errors) { - for (final ScenarioValidationError error in errors) { - stderr.writeln('${error.path}: ${error.message}'); - } - } -} - -/// `test` command shell for validating CLI arguments before app launch. -/// -/// It checks Scenario YAML and validates `--until` / `--print` relationships -/// before the Target App Package is launched. -class _TestCommand extends Command { - _TestCommand({ - required TestCommandExecutor executor, - required ProjectRunCommandExecutor projectRunExecutor, - }) : _executor = executor, - _projectRunExecutor = projectRunExecutor { - argParser - ..addOption( - 'device', - abbr: 'd', - help: 'Target Device id, exact name, or unique id/name prefix.', - ) - ..addOption('flavor', help: 'Flutter flavor passed to flutter run.') - ..addOption( - 'target', - abbr: 't', - help: 'Flutter app entrypoint file passed to flutter run.', - ) - ..addOption( - 'until', - help: 'Run through a 1-based step number or step label.', - ) - ..addMultiOption( - 'print', - allowed: ['snapshot', 'widget-tree', 'errors'], - help: 'Print diagnostics after --until.', - ) - ..addFlag( - 'json', - negatable: false, - help: 'Print raw diagnostics as indented JSON.', - ); - } - - final TestCommandExecutor _executor; - final ProjectRunCommandExecutor _projectRunExecutor; - - @override - String get description => 'Launch the Target App Package and run a Scenario.'; - - @override - String get name => 'test'; - - @override - Future run() async { - if (argResults!.rest.length > 1) { - throw UsageException('Expected zero or one scenario path.', usage); - } - final String? scenarioPath = argResults!.rest.isEmpty - ? null - : argResults!.rest.single; - if (scenarioPath == null) { - return _runProjectRun( - ProjectScenarioDiscovery.defaultPilotDirectory, - defaultDiscovery: true, - ); - } - if (FileSystemEntity.isDirectorySync(scenarioPath)) { - return _runProjectRun(scenarioPath); - } - if (!FileSystemEntity.isFileSync(scenarioPath)) { - throw UsageException( - 'Scenario path does not exist: $scenarioPath', - usage, - ); - } - if (argResults!.multiOption('print').isNotEmpty && - argResults!.option('until') == null) { - throw UsageException('--print must be used with --until.', usage); - } - - final Scenario parsedScenario; - try { - parsedScenario = ScenarioParser.parseFile(scenarioPath); - } on ScenarioValidationException catch (error) { - _writeValidationErrors(error.errors); - return 1; - } - - final String? until = argResults!.option('until'); - RunStopPoint? stopPoint; - if (until != null) { - final String? error = _validateUntil(until, parsedScenario); - if (error != null) { - stderr.writeln(error); - return 64; - } - stopPoint = _stopPointFromUntil(until); - } - - final String? device = argResults!.option('device')?.trim(); - if (device != null && device.isEmpty) { - stderr.writeln('Target Device selector must not be empty.'); - return 64; - } - final String? flavor = argResults!.option('flavor')?.trim(); - if (flavor != null && flavor.isEmpty) { - stderr.writeln('--flavor must not be empty.'); - return 64; - } - final String? target = argResults!.option('target')?.trim(); - if (target != null && target.isEmpty) { - stderr.writeln('--target must not be empty.'); - return 64; - } - final TestCommandOptions options = TestCommandOptions( - scenario: parsedScenario, - device: device, - flavor: flavor, - target: target, - stopPoint: stopPoint, - printDiagnostics: _printDiagnosticsFromOptions( - argResults!.multiOption('print'), - ), - jsonOutput: argResults!.flag('json'), - ); - try { - final StepProgressRenderer? progressRenderer = - TestCommandOutput.stepProgressRenderer( - sink: stderr, - jsonOutput: options.jsonOutput, - stderrHasTerminal: stderr.hasTerminal, - ); - final TargetAppLaunchProgressRenderer? launchProgressRenderer = - TestCommandOutput.targetAppLaunchProgressRenderer( - sink: stderr, - jsonOutput: options.jsonOutput, - stderrHasTerminal: stderr.hasTerminal, - ); - final ScenarioRunReport report = await _executor.run( - options, - onLaunchProgress: launchProgressRenderer?.render, - launchHeartbeatEnabled: launchProgressRenderer != null, - onProgress: progressRenderer?.render, - ); - if (report.printedDiagnostics.isNotEmpty) { - if (options.jsonOutput) { - stdout.writeln( - const JsonEncoder.withIndent( - ' ', - ).convert(_printDiagnosticsJson(report)), - ); - } else { - stdout.writeln(DiagnosticTextRenderer.render(report)); - } - } - stdout.writeln('Run report: ${report.runDirectoryPath}/run_report.json'); - stdout.writeln('HTML report: ${report.runDirectoryPath}/timeline.html'); - return report.status == ScenarioRunStatus.passed ? 0 : 1; - } on TestCommandException catch (error) { - if (!error.alreadyRendered) { - stderr.writeln(error.message); - } - return error.exitCode; - } - } - - /// Discover Project Scenarios and delegate Project Run execution. - Future _runProjectRun( - String discoveryRootPath, { - bool defaultDiscovery = false, - }) async { - if (argResults!.option('until') != null) { - throw UsageException( - '--until is only supported for one Scenario file.', - usage, - ); - } - if (argResults!.multiOption('print').isNotEmpty) { - throw UsageException( - '--print is only supported for one Scenario file.', - usage, - ); - } - final String? device = argResults!.option('device')?.trim(); - if (device != null && device.isEmpty) { - stderr.writeln('--device must not be empty.'); - return 64; - } - final String? flavor = argResults!.option('flavor')?.trim(); - if (flavor != null && flavor.isEmpty) { - stderr.writeln('--flavor must not be empty.'); - return 64; - } - final String? target = argResults!.option('target')?.trim(); - if (target != null && target.isEmpty) { - stderr.writeln('--target must not be empty.'); - return 64; - } - final List scenarios; - try { - scenarios = defaultDiscovery - ? ProjectScenarioDiscovery.discoverDefault() - : ProjectScenarioDiscovery.discoverInDirectory(discoveryRootPath); - } on ProjectScenarioDiscoveryException catch (error) { - stderr.writeln(error.message); - return error.usageError ? 64 : 1; - } on ScenarioValidationException catch (error) { - _writeValidationErrors(error.errors); - return 1; - } - final ProjectRunCommandOptions options = ProjectRunCommandOptions( - discoveryRootPath: discoveryRootPath, - scenarios: scenarios, - device: device, - flavor: flavor, - target: target, - jsonOutput: argResults!.flag('json'), - ); - try { - final StepProgressRenderer? progressRenderer = - TestCommandOutput.stepProgressRenderer( - sink: stderr, - jsonOutput: options.jsonOutput, - stderrHasTerminal: stderr.hasTerminal, - ); - final TargetAppLaunchProgressRenderer? launchProgressRenderer = - TestCommandOutput.targetAppLaunchProgressRenderer( - sink: stderr, - jsonOutput: options.jsonOutput, - stderrHasTerminal: stderr.hasTerminal, - ); - final ProjectRunCommandReport report = await _projectRunExecutor.run( - options, - onLaunchProgress: launchProgressRenderer?.render, - launchHeartbeatEnabled: launchProgressRenderer != null, - onProgress: progressRenderer?.render, - ); - stdout.write(TestCommandOutput.renderProjectRunSummary(report)); - return report.passed ? 0 : 1; - } on TestCommandException catch (error) { - if (!error.alreadyRendered) { - stderr.writeln(error.message); - } - return error.exitCode; - } - } - - /// Return the runner stop point selected by a validated `--until` value. - RunStopPoint _stopPointFromUntil(String until) { - final int? stepNumber = int.tryParse(until); - if (stepNumber != null) { - return RunStopPoint.stepNumber(stepNumber); - } - return RunStopPoint.stepLabel(until); - } - - /// Return the runner diagnostics selected by repeated CLI `--print` options. - /// - /// Args: - /// `printValues` are the already-validated option values from `args`. - /// - /// Returns: - /// The diagnostic requests to pass to the runner. Duplicate options collapse - /// to one request. - Set _printDiagnosticsFromOptions(List printValues) { - final Set printDiagnostics = {}; - for (final String printValue in printValues) { - final PrintDiagnostic? printDiagnostic = switch (printValue) { - 'snapshot' => PrintDiagnostic.snapshot, - 'widget-tree' => PrintDiagnostic.widgetTree, - 'errors' => PrintDiagnostic.errors, - _ => null, - }; - if (printDiagnostic != null) { - printDiagnostics.add(printDiagnostic); - } - } - return printDiagnostics; - } - - /// Return the stdout JSON object for printable diagnostics. - Map _printDiagnosticsJson(ScenarioRunReport report) { - return { - for (final PrintedDiagnostic diagnostic in report.printedDiagnostics) - _printDiagnosticJsonKey(diagnostic.type): diagnostic.data, - }; - } - - /// Return the public JSON key for a printable diagnostic type. - String _printDiagnosticJsonKey(PrintDiagnostic printDiagnostic) { - return switch (printDiagnostic) { - PrintDiagnostic.snapshot => 'snapshot', - PrintDiagnostic.widgetTree => 'widgetTree', - PrintDiagnostic.errors => 'errors', - }; - } - - /// Validate that `--until` names an existing stop point in the Scenario. - /// - /// Args: - /// `until` is either a 1-based step number or a step label. - /// `scenario` provides the step count and available labels. - /// - /// Returns: - /// `null` when valid, otherwise a human-readable CLI usage error. - String? _validateUntil(String until, Scenario scenario) { - final int? stepNumber = int.tryParse(until); - if (stepNumber != null) { - if (stepNumber < 1 || stepNumber > scenario.steps.length) { - return '--until step number must be between 1 and ${scenario.steps.length}.'; - } - return null; - } - - final Iterable labels = scenario.steps - .map((ScenarioStep step) => step.label) - .whereType(); - if (!labels.contains(until)) { - return '--until must be a 1-based step number or an existing step label.'; - } - return null; - } - - /// Print validation errors as `path: message` lines for humans. - void _writeValidationErrors(List errors) { - for (final ScenarioValidationError error in errors) { - stderr.writeln('${error.path}: ${error.message}'); - } - } -} - -/// Parsed `test` command inputs that are validated before app launch. -class TestCommandOptions { - /// Creates validated inputs for executing one Scenario through `test`. - const TestCommandOptions({ - required this.scenario, - required this.device, - required this.flavor, - required this.target, - required this.stopPoint, - required this.printDiagnostics, - required this.jsonOutput, - }); - - final Scenario scenario; - final String? device; - final String? flavor; - final String? target; - final RunStopPoint? stopPoint; - final Set printDiagnostics; - final bool jsonOutput; -} - -/// Parsed Project Run inputs selected by the `test` command. -class ProjectRunCommandOptions { - /// Creates validated Project Run command inputs. - const ProjectRunCommandOptions({ - required this.discoveryRootPath, - required this.scenarios, - required this.device, - required this.flavor, - required this.target, - required this.jsonOutput, - }); - - /// Directory used to discover Project Scenarios. - final String discoveryRootPath; - - /// Validated Entry Scenario files selected for the Project Run. - final List scenarios; - - final String? device; - final String? flavor; - final String? target; - final bool jsonOutput; -} - -/// Project Run command result used by mode selection and stdout rendering. -class ProjectRunCommandReport { - const ProjectRunCommandReport({ - required this.passed, - required this.status, - required this.projectRunReportPath, - required this.scenarioReports, - }); - - /// Whether the selected Project Scenarios all passed. - final bool passed; - - /// Overall Project Run status. - final ProjectRunStatus status; - - /// Path to the batch-level `project_run_report.json`. - final String projectRunReportPath; - - /// Per-Scenario report paths to print after the Project Run finishes. - final List scenarioReports; -} - -/// Report paths for one Scenario inside Project Run stdout. -class ProjectRunScenarioOutputReport { - const ProjectRunScenarioOutputReport({ - required this.scenarioPath, - required this.status, - required this.runReportPath, - required this.htmlReportPath, - }); - - /// Project Scenario path relative to the discovery root. - final String scenarioPath; - - /// Scenario status in the Project Run summary. - final ProjectScenarioRunStatus status; - - /// Path to the child Scenario Run report. - final String runReportPath; - - /// Path to the child Scenario HTML timeline report. - final String htmlReportPath; -} - -/// Executes a validated Project Run selected by `test`. -abstract interface class ProjectRunCommandExecutor { - /// Run the Project Scenarios described by [options]. - /// - /// `onLaunchProgress`, when provided, receives the batch-level Target App - /// Launch Progress events before any Project Scenario executes. - Future run( - ProjectRunCommandOptions options, { - void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, - bool launchHeartbeatEnabled = false, - void Function(StepProgressEvent event)? onProgress, - }); -} - -/// Default Project Run executor for launch reuse and batch Scenario execution. -class DefaultProjectRunCommandExecutor implements ProjectRunCommandExecutor { - const DefaultProjectRunCommandExecutor({ - this.deviceDiscovery = const DefaultTestDeviceDiscovery(), - this.launcher = const TargetAppLauncher(), - this.runnerFactory = const DefaultTestScenarioRunnerFactory(), - this.interruptSignals, - this.outputDirectory, - this.clock = DateTime.now, - this.launchHeartbeatTicks, - }); - - /// Discovers Flutter and Recording Devices before app launch when needed. - final TestDeviceDiscovery deviceDiscovery; - - /// Starts the Target App Package and exposes hot restart. - final TargetAppLauncher launcher; - - /// Creates Scenario runners for each selected Project Scenario. - final TestScenarioRunnerFactory runnerFactory; - - /// Optional interrupt stream used by tests and Ctrl-C handling. - final Stream? interruptSignals; - - /// Directory where Project Run artifacts are written. - final Directory? outputDirectory; - - /// Clock used for report timestamps, durations, and tests. - final TargetAppLaunchClock clock; - - /// Optional heartbeat stream used by launch progress tests. - final Stream? launchHeartbeatTicks; - - @override - Future run( - ProjectRunCommandOptions options, { - void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, - bool launchHeartbeatEnabled = false, - void Function(StepProgressEvent event)? onProgress, - }) async { - final DateTime startedAt = clock().toUtc(); - final ProjectRunArtifactWriter projectRunWriter = RunArtifactStore( - outputDirectory ?? Directory.current, - ).createProjectRun(startedAt: startedAt); - final Stopwatch stopwatch = Stopwatch()..start(); - final List scenarioResults = - []; - ProjectRunEnvironmentFailure? environmentFailure; - TargetAppLaunch? launch; - TargetDevice? targetDevice; - final bool recordingRequired = options.scenarios.any( - (ProjectScenarioFile file) => file.scenario.recording?.enabled == true, - ); - if (options.device != null || recordingRequired) { - try { - final List flutterDevices = await deviceDiscovery - .listFlutterDevices(); - final List recordingDevices = recordingRequired - ? await deviceDiscovery.listRecordingDevices() - : const []; - targetDevice = TargetDeviceResolver.resolve( - selector: options.device, - recordingRequired: recordingRequired, - flutterDevices: flutterDevices, - recordingDevices: recordingDevices, - ); - } on TargetDeviceResolutionException catch (error) { - environmentFailure = ProjectRunEnvironmentFailure( - phase: ProjectRunEnvironmentFailurePhase.targetDeviceResolution, - message: error.message, - ); - } on DeviceDiscoveryException catch (error) { - environmentFailure = ProjectRunEnvironmentFailure( - phase: ProjectRunEnvironmentFailurePhase.targetDeviceResolution, - message: error.message, - ); - } - } - if (environmentFailure != null) { - stopwatch.stop(); - final ProjectRunReport projectReport = - ProjectRunReport.environmentFailure( - discoveryRootPath: options.discoveryRootPath, - startedAt: startedAt, - durationMs: stopwatch.elapsedMilliseconds, - commandInputs: ProjectRunCommandInputs( - device: options.device, - flavor: options.flavor, - target: options.target, - ), - failure: environmentFailure, - scenarioResults: scenarioResults, - ); - projectRunWriter.writeProjectRunReport(projectReport.toJson()); - return ProjectRunCommandReport( - passed: false, - status: ProjectRunStatus.environmentFailed, - projectRunReportPath: p.join( - projectRunWriter.runDirectory.path, - 'project_run_report.json', - ), - scenarioReports: const [], - ); - } - final TargetAppLaunchChoices launchChoices = TargetAppLaunchChoices( - targetDevice: targetDevice, - selectionReason: _targetDeviceSelectionReason( - deviceSelector: options.device, - recordingRequired: recordingRequired, - ), - flavor: options.flavor, - target: options.target, - ); - final TargetAppLaunchStartedEvent launchStartedEvent = - TargetAppLaunchStartedEvent( - startedAt: startedAt, - choices: launchChoices, - ); - onLaunchProgress?.call(launchStartedEvent); - TargetAppLaunchHeartbeat? launchHeartbeat; - if (onLaunchProgress != null && launchHeartbeatEnabled) { - launchHeartbeat = TargetAppLaunchHeartbeat( - ticks: - launchHeartbeatTicks ?? - Stream.periodic(const Duration(seconds: 10)), - onProgress: onLaunchProgress, - clock: clock, - ); - launchHeartbeat.start(launchStartedEvent); - } - try { - launch = await launcher.launch( - TargetAppLaunchCommand( - flavor: options.flavor, - target: options.target, - deviceId: targetDevice?.id, - ), - ); - onLaunchProgress?.call( - TargetAppLaunchSucceededEvent( - startedAt: startedAt, - finishedAt: clock().toUtc(), - choices: launchChoices, - ), - ); - await launchHeartbeat?.stop(); - for (int index = 0; index < options.scenarios.length; index++) { - final ProjectScenarioFile scenarioFile = options.scenarios[index]; - if (index > 0) { - try { - await launch.hotRestart(); - } on TargetAppLaunchException catch (error) { - environmentFailure = ProjectRunEnvironmentFailure( - phase: ProjectRunEnvironmentFailurePhase.hotRestart, - message: error.message, - ); - break; - } - } - final RunArtifactWriter childRun = projectRunWriter.createScenarioRun( - scenario: scenarioFile.scenario, - startedAt: clock().toUtc(), - ); - final TestScenarioRunner runner = runnerFactory.create( - runtimeTarget: RuntimeTarget(vmServiceUri: launch.runtimeTargetUri), - targetDevice: targetDevice, - recordingController: scenarioFile.scenario.recording?.enabled == true - ? ScreenRecorderRecordingController( - recorder: screen_recorder.ScreenRecorder.defaultRecorder(), - deviceSelector: targetDevice!.id, - outputDirectory: Directory.current, - ) - : null, - ); - final Future runFuture = runner.run( - scenarioFile.scenario, - runArtifactWriter: childRun, - onProgress: onProgress, - ); - final ScenarioRunReport scenarioReport = - await _runScenarioWithInterrupt(runFuture); - final ProjectScenarioRunStatus scenarioStatus = - scenarioReport.status == ScenarioRunStatus.passed - ? ProjectScenarioRunStatus.passed - : ProjectScenarioRunStatus.failed; - scenarioResults.add( - ProjectScenarioRunReport( - scenarioPath: scenarioFile.relativePath, - status: scenarioStatus, - runReportPath: projectRunWriter.relativePathFor( - childRun, - 'run_report.json', - ), - htmlReportPath: projectRunWriter.relativePathFor( - childRun, - 'timeline.html', - ), - ), - ); - } - } on TargetAppLaunchException catch (error) { - environmentFailure = ProjectRunEnvironmentFailure( - phase: ProjectRunEnvironmentFailurePhase.launch, - message: error.message, - ); - onLaunchProgress?.call( - TargetAppLaunchFailedEvent( - startedAt: startedAt, - failedAt: clock().toUtc(), - message: error.message, - stderrLines: error.stderrLines, - choices: launchChoices, - ), - ); - await launchHeartbeat?.stop(); - } on TestCommandException catch (error) { - if (error.exitCode == 130) { - rethrow; - } - environmentFailure = ProjectRunEnvironmentFailure( - phase: ProjectRunEnvironmentFailurePhase.validation, - message: error.message, - ); - } finally { - stopwatch.stop(); - await launchHeartbeat?.stop(); - await launch?.cleanup(); - } - final bool allPassed = - environmentFailure == null && - scenarioResults.every( - (ProjectScenarioRunReport report) => - report.status == ProjectScenarioRunStatus.passed, - ); - final ProjectRunStatus projectStatus = environmentFailure != null - ? ProjectRunStatus.environmentFailed - : allPassed - ? ProjectRunStatus.passed - : ProjectRunStatus.failed; - final ProjectRunReport projectReport = environmentFailure == null - ? ProjectRunReport( - discoveryRootPath: options.discoveryRootPath, - scenarioResults: scenarioResults, - status: projectStatus, - startedAt: startedAt, - durationMs: stopwatch.elapsedMilliseconds, - commandInputs: ProjectRunCommandInputs( - device: options.device, - flavor: options.flavor, - target: options.target, - ), - ) - : ProjectRunReport.environmentFailure( - discoveryRootPath: options.discoveryRootPath, - startedAt: startedAt, - durationMs: stopwatch.elapsedMilliseconds, - commandInputs: ProjectRunCommandInputs( - device: options.device, - flavor: options.flavor, - target: options.target, - ), - failure: environmentFailure, - scenarioResults: scenarioResults, - ); - projectRunWriter.writeProjectRunReport(projectReport.toJson()); - return ProjectRunCommandReport( - passed: projectStatus == ProjectRunStatus.passed, - status: projectStatus, - projectRunReportPath: p.join( - projectRunWriter.runDirectory.path, - 'project_run_report.json', - ), - scenarioReports: [ - for (final ProjectScenarioRunReport scenarioReport in scenarioResults) - ProjectRunScenarioOutputReport( - scenarioPath: scenarioReport.scenarioPath, - status: scenarioReport.status, - runReportPath: p.join( - projectRunWriter.runDirectory.path, - scenarioReport.runReportPath, - ), - htmlReportPath: p.join( - projectRunWriter.runDirectory.path, - scenarioReport.htmlReportPath, - ), - ), - ], - ); - } - - Future _runScenarioWithInterrupt( - Future runFuture, - ) async { - StreamSubscription? interruptSub; - try { - final Completer interruptCompleter = - Completer(); - interruptSub = - (interruptSignals ?? - ProcessSignal.sigint.watch().map((ProcessSignal _) {})) - .listen((_) { - if (!interruptCompleter.isCompleted) { - interruptCompleter.completeError( - const TestCommandException( - message: 'test command interrupted.', - exitCode: 130, - ), - ); - } - }); - return await Future.any(>[ - runFuture, - interruptCompleter.future, - ]); - } finally { - runFuture.ignore(); - await interruptSub?.cancel(); - } - } -} - -/// Terminal output helpers for the `test` command. -class TestCommandOutput { - TestCommandOutput._(); - - /// Return the Step progress renderer for human-readable `test` output. - /// - /// JSON output is machine-oriented and suppresses Step progress. Interactive - /// rendering is used only when stderr is a terminal; redirected stderr gets - /// deterministic plain-text progress lines for CI logs. - static StepProgressRenderer? stepProgressRenderer({ - required IOSink sink, - required bool jsonOutput, - required bool stderrHasTerminal, - }) { - if (jsonOutput) { - return null; - } - return StepProgressRenderer(sink: sink, interactive: stderrHasTerminal); - } - - /// Return the Target App Launch Progress renderer for human-readable output. - /// - /// Launch progress is stderr-only status. JSON output suppresses it so stdout - /// remains machine-oriented. - static TargetAppLaunchProgressRenderer? targetAppLaunchProgressRenderer({ - required IOSink sink, - required bool jsonOutput, - required bool stderrHasTerminal, - }) { - if (jsonOutput) { - return null; - } - return TargetAppLaunchProgressRenderer( - sink: sink, - interactive: stderrHasTerminal, - ); - } - - /// Return deterministic stdout summary lines for a completed Project Run. - /// - /// The summary keeps the batch-level report path first, then prints each - /// Scenario's existing report paths in execution order. - static String renderProjectRunSummary(ProjectRunCommandReport report) { - final StringBuffer buffer = StringBuffer() - ..writeln('Project Run: ${report.status.name}') - ..writeln('Project Run report: ${report.projectRunReportPath}'); - for (final ProjectRunScenarioOutputReport scenarioReport - in report.scenarioReports) { - buffer - ..writeln( - 'Scenario: ${scenarioReport.scenarioPath} ' - '(${scenarioReport.status.name})', - ) - ..writeln('Run report: ${scenarioReport.runReportPath}') - ..writeln('HTML report: ${scenarioReport.htmlReportPath}'); - } - return buffer.toString(); - } -} - -/// Return the user-facing reason for the selected Target Device. -TargetDeviceSelectionReason? _targetDeviceSelectionReason({ - required String? deviceSelector, - required bool recordingRequired, -}) { - if (deviceSelector != null) { - return TargetDeviceSelectionReason.explicit(selector: deviceSelector); - } - if (recordingRequired) { - return const TargetDeviceSelectionReason.autoSelectedForRecording(); - } - return null; -} - -/// Executes a validated `test` command. -abstract interface class TestCommandExecutor { - /// Run the Scenario described by `options` and return its run report. - /// - /// `onLaunchProgress`, when provided, receives Target App Launch Progress - /// events before Scenario execution starts. `launchHeartbeatEnabled` controls - /// whether long pending launches emit heartbeat events. `onProgress`, when - /// provided, receives Scenario Step progress events during execution. - Future run( - TestCommandOptions options, { - void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, - bool launchHeartbeatEnabled = false, - void Function(StepProgressEvent event)? onProgress, - }); -} - -/// Default `test` command executor for launching and running one Scenario. -class DefaultTestCommandExecutor implements TestCommandExecutor { - /// Creates an executor with injectable launch and discovery boundaries. - /// - /// `launchHeartbeatTicks` and `launchClock` make Target App Launch Progress - /// deterministic in tests without waiting on real time. - const DefaultTestCommandExecutor({ - this.deviceDiscovery = const DefaultTestDeviceDiscovery(), - this.launcher = const TargetAppLauncher(), - this.runnerFactory = const DefaultTestScenarioRunnerFactory(), - this.interruptSignals, - this.launchHeartbeatTicks, - this.launchClock = DateTime.now, - }); - - final TestDeviceDiscovery deviceDiscovery; - final TargetAppLauncher launcher; - final TestScenarioRunnerFactory runnerFactory; - final Stream? interruptSignals; - final Stream? launchHeartbeatTicks; - final TargetAppLaunchClock launchClock; - - @override - Future run( - TestCommandOptions options, { - void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, - bool launchHeartbeatEnabled = false, - void Function(StepProgressEvent event)? onProgress, - }) async { - final bool recordingRequired = options.scenario.recording?.enabled == true; - TargetDevice? targetDevice; - if (options.device != null || recordingRequired) { - try { - final List flutterDevices = await deviceDiscovery - .listFlutterDevices(); - final List recordingDevices = recordingRequired - ? await deviceDiscovery.listRecordingDevices() - : const []; - targetDevice = TargetDeviceResolver.resolve( - selector: options.device, - recordingRequired: recordingRequired, - flutterDevices: flutterDevices, - recordingDevices: recordingDevices, - ); - } on TargetDeviceResolutionException catch (error) { - throw TestCommandException(message: error.message, exitCode: 64); - } on DeviceDiscoveryException catch (error) { - throw TestCommandException(message: error.message, exitCode: 1); - } - } - - TargetAppLaunch launch; - final DateTime launchStartedAt = launchClock(); - final TargetAppLaunchChoices launchChoices = TargetAppLaunchChoices( - targetDevice: targetDevice, - selectionReason: _targetDeviceSelectionReason( - deviceSelector: options.device, - recordingRequired: recordingRequired, - ), - flavor: options.flavor, - target: options.target, - ); - onLaunchProgress?.call( - TargetAppLaunchStartedEvent( - startedAt: launchStartedAt, - choices: launchChoices, - ), - ); - TargetAppLaunchHeartbeat? launchHeartbeat; - if (onLaunchProgress != null && launchHeartbeatEnabled) { - launchHeartbeat = TargetAppLaunchHeartbeat( - ticks: - launchHeartbeatTicks ?? - Stream.periodic(const Duration(seconds: 10)), - onProgress: onLaunchProgress, - clock: launchClock, - ); - launchHeartbeat.start( - TargetAppLaunchStartedEvent( - startedAt: launchStartedAt, - choices: launchChoices, - ), - ); - } - try { - launch = await launcher.launch( - TargetAppLaunchCommand( - deviceId: targetDevice?.id, - flavor: options.flavor, - target: options.target, - ), - ); - onLaunchProgress?.call( - TargetAppLaunchSucceededEvent( - startedAt: launchStartedAt, - finishedAt: launchClock(), - choices: launchChoices, - ), - ); - await launchHeartbeat?.stop(); - } on TargetAppLaunchException catch (error) { - onLaunchProgress?.call( - TargetAppLaunchFailedEvent( - startedAt: launchStartedAt, - failedAt: launchClock(), - message: error.message, - stderrLines: error.stderrLines, - choices: launchChoices, - ), - ); - await launchHeartbeat?.stop(); - final String stderrContext = - onLaunchProgress == null && error.stderrLines.isNotEmpty - ? '\n${error.stderrLines.join('\n')}' - : ''; - throw TestCommandException( - message: '${error.message}$stderrContext', - exitCode: 1, - alreadyRendered: onLaunchProgress != null, - ); - } - - try { - final TestScenarioRunner runner = runnerFactory.create( - runtimeTarget: RuntimeTarget(vmServiceUri: launch.runtimeTargetUri), - targetDevice: targetDevice, - recordingController: recordingRequired - ? ScreenRecorderRecordingController( - recorder: screen_recorder.ScreenRecorder.defaultRecorder(), - deviceSelector: targetDevice!.id, - outputDirectory: Directory.current, - ) - : null, - ); - final Future runFuture = runner.run( - options.scenario, - stopPoint: options.stopPoint, - printDiagnostics: options.printDiagnostics, - onProgress: onProgress, - ); - StreamSubscription? interruptSub; - try { - final Completer interruptCompleter = - Completer(); - interruptSub = - (interruptSignals ?? - ProcessSignal.sigint.watch().map( - (ProcessSignal _) {}, - )) - .listen((_) { - if (!interruptCompleter.isCompleted) { - interruptCompleter.completeError( - const TestCommandException( - message: 'test command interrupted.', - exitCode: 130, - ), - ); - } - }); - return await Future.any(>[ - runFuture, - interruptCompleter.future, - ]); - } finally { - runFuture.ignore(); - await interruptSub?.cancel(); - } - } on RuntimeOperationException catch (error) { - throw TestCommandException(message: error.message, exitCode: 1); - } finally { - await launch.cleanup(); - } - } -} - -/// Discovers Flutter Devices and Recording Devices for `test`. -abstract interface class TestDeviceDiscovery { - /// Return Flutter Devices from `flutter devices --machine`. - Future> listFlutterDevices(); - - /// Return Recording Device identities available for Scenario Recording. - Future> listRecordingDevices(); -} - -/// Default device discovery backed by Flutter CLI and `screen_recorder`. -class DefaultTestDeviceDiscovery implements TestDeviceDiscovery { - /// Creates default Target Device discovery. - const DefaultTestDeviceDiscovery(); - - @override - Future> listFlutterDevices() async { - final ProcessResult result; - try { - result = await Process.run('flutter', ['devices', '--machine']); - } on ProcessException catch (error) { - throw DeviceDiscoveryException(error.message); - } - if (result.exitCode != 0) { - throw DeviceDiscoveryException(result.stderr.toString()); - } - try { - return TargetDeviceParser.parseMachineJson(result.stdout.toString()); - } on FormatException catch (e) { - throw DeviceDiscoveryException(e.message); - } - } - - @override - Future> listRecordingDevices() async { - final screen_recorder.ScreenRecorder recorder = - screen_recorder.ScreenRecorder.defaultRecorder(); - final List devices = await recorder - .listDevices(); - return [ - for (final screen_recorder.RecordingDevice device in devices) - RecordingDeviceIdentity(id: device.id), - ]; - } -} - -/// Creates a Scenario runner for one launched Runtime Target. -abstract interface class TestScenarioRunnerFactory { - /// Create a runner bound to the launched Runtime Target. - TestScenarioRunner create({ - required RuntimeTarget runtimeTarget, - required TargetDevice? targetDevice, - required RecordingController? recordingController, - }); -} - -/// Narrow Scenario runner interface used by the `test` command executor. -abstract interface class TestScenarioRunner { - /// Run `scenario` with optional stop, diagnostic, and Step progress controls. - Future run( - Scenario scenario, { - RunStopPoint? stopPoint, - Set printDiagnostics, - void Function(StepProgressEvent event)? onProgress, - RunArtifactWriter? runArtifactWriter, - }); -} - -/// Default Scenario runner factory backed by `McpFlutterRuntimeAdapter`. -class DefaultTestScenarioRunnerFactory implements TestScenarioRunnerFactory { - /// Creates the default runner factory. - const DefaultTestScenarioRunnerFactory(); - - @override - TestScenarioRunner create({ - required RuntimeTarget runtimeTarget, - required TargetDevice? targetDevice, - required RecordingController? recordingController, - }) { - return _ScenarioRunnerAdapter( - ScenarioRunner( - adapter: McpFlutterRuntimeAdapter(target: runtimeTarget), - recordingController: recordingController, - targetDevice: targetDevice, - outputDirectory: Directory.current, - ), - ); - } -} - -class _ScenarioRunnerAdapter implements TestScenarioRunner { - const _ScenarioRunnerAdapter(this._runner); - - final ScenarioRunner _runner; - - @override - Future run( - Scenario scenario, { - RunStopPoint? stopPoint, - Set printDiagnostics = const {}, - void Function(StepProgressEvent event)? onProgress, - RunArtifactWriter? runArtifactWriter, - }) { - return _runner.run( - scenario, - stopPoint: stopPoint, - printDiagnostics: printDiagnostics, - onProgress: onProgress, - runArtifactWriter: runArtifactWriter, - ); - } -} - -/// Failure raised when device discovery fails. -class DeviceDiscoveryException implements Exception { - /// Creates a device discovery failure. - const DeviceDiscoveryException(this.message); - - /// Human-readable failure reason. - final String message; - - @override - String toString() => message; -} - -/// Failure from the `test` command executor. -class TestCommandException implements Exception { - /// Creates a command execution failure. - const TestCommandException({ - required this.message, - required this.exitCode, - this.alreadyRendered = false, - }); - - /// Human-readable error message. - final String message; - - /// CLI exit code. - final int exitCode; - - /// Whether `message` was already written by command-specific output. - /// - /// The `test` command uses this for launch failures rendered by Target App - /// Launch Progress so the same message is not printed twice. - final bool alreadyRendered; -} diff --git a/lib/src/commands/app_setup_commands.dart b/lib/src/commands/app_setup_commands.dart new file mode 100644 index 0000000..46e1e8e --- /dev/null +++ b/lib/src/commands/app_setup_commands.dart @@ -0,0 +1,160 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:yaml/yaml.dart'; + +import '../app_setup.dart'; + +/// `doctor` command for checking Flutter Pilot setup in a Target App Package. +/// +/// It inspects the current working directory without modifying files. Missing +/// setup is reported as diagnostic output, while unreadable or unsupported +/// package shapes return an execution failure. +class DoctorCommand extends Command { + @override + String get description => 'Check Flutter Pilot setup in a Flutter package.'; + + @override + String get name => 'doctor'; + + @override + Future run() async { + if (argResults!.rest.isNotEmpty) { + throw UsageException('Expected no arguments.', usage); + } + try { + final AppSetupStatus status = AppSetupChecker.check(Directory.current); + if (!status.isFlutterPackage) { + stderr.writeln(_flutterPackageOnlyMessage); + return 1; + } + stdout.writeln('Flutter Pilot doctor'); + stdout.writeln(''); + if (status.isComplete) { + stdout.writeln('✅ Flutter Pilot app setup is complete.'); + } else { + if (!status.hasMcpToolkitDependency) { + stdout.writeln( + '❌ MCP Toolkit dependency missing: run `flutter pub add mcp_toolkit`', + ); + } + if (!status.hasBootstrapFlutter) { + _writeBootstrapGuidance(); + } + } + return 0; + } on FileSystemException catch (error) { + stderr.writeln(error.message); + return 1; + } on YamlException catch (error) { + stderr.writeln(error.message); + return 1; + } + } +} + +/// `init` command for adding safe Flutter Pilot setup to a Target App Package. +/// +/// It can add the `mcp_toolkit` dependency through Flutter tooling, then +/// reports whether the app entrypoint still needs manual bootstrap code. +class InitCommand extends Command { + @override + String get description => + 'Initialize Flutter Pilot setup in a Flutter package.'; + + @override + String get name => 'init'; + + @override + Future run() async { + if (argResults!.rest.isNotEmpty) { + throw UsageException('Expected no arguments.', usage); + } + try { + final AppSetupStatus status = AppSetupChecker.check(Directory.current); + if (!status.isFlutterPackage) { + stderr.writeln(_flutterPackageOnlyMessage); + return 1; + } + final AppSetupInitResult result = await AppSetupInitializer.initialize( + Directory.current, + addMcpToolkitDependency: _addMcpToolkitDependency, + ); + stdout.writeln('Flutter Pilot init'); + stdout.writeln(''); + if (result.addedMcpToolkitDependency) { + stdout.writeln('✅ Added MCP Toolkit dependency.'); + } else { + stdout.writeln('✅ MCP Toolkit dependency already exists.'); + } + if (result.status.hasBootstrapFlutter) { + stdout.writeln('✅ bootstrapFlutter already exists.'); + } else { + _writeBootstrapGuidance(); + } + return 0; + } on AppSetupInstallException catch (error) { + stderr.writeln('Failed to add MCP Toolkit dependency.'); + if (error.result.stderr.isNotEmpty) { + stderr.writeln(''); + stderr.writeln('flutter pub add output:'); + stderr.write(error.result.stderr); + } + stderr.writeln(''); + stderr.writeln('Run this command manually from the Flutter package:'); + stderr.writeln('flutter pub add mcp_toolkit'); + return 1; + } on FileSystemException catch (error) { + stderr.writeln(error.message); + return 1; + } on YamlException catch (error) { + stderr.writeln(error.message); + return 1; + } + } +} + +/// Run Flutter tooling to add the runtime dependency. +Future _addMcpToolkitDependency( + Directory packageDirectory, +) async { + try { + final ProcessResult result = await Process.run('flutter', [ + 'pub', + 'add', + 'mcp_toolkit', + ], workingDirectory: packageDirectory.path); + if (result.exitCode == 0) { + return const AppSetupInstallResult.success(); + } + return AppSetupInstallResult.failure( + exitCode: result.exitCode, + stderr: result.stderr.toString(), + ); + } on ProcessException catch (error) { + return AppSetupInstallResult.failure(exitCode: 1, stderr: error.message); + } +} + +/// Print the manual app entrypoint change required for MCP Toolkit. +void _writeBootstrapGuidance() { + stdout.writeln( + '❌ bootstrapFlutter missing: add ' + 'MCPToolkitBinding.instance.bootstrapFlutter in lib/main.dart', + ); + stdout.writeln(''); + stdout.writeln('Add the MCP Toolkit import:'); + stdout.writeln("import 'package:mcp_toolkit/mcp_toolkit.dart';"); + stdout.writeln(''); + stdout.writeln('Wrap runApp with MCPToolkitBinding:'); + stdout.writeln('Future main() async {'); + stdout.writeln(' await MCPToolkitBinding.instance.bootstrapFlutter('); + stdout.writeln(' runApp: () => runApp(const MyApp()),'); + stdout.writeln(' );'); + stdout.writeln('}'); +} + +const String _flutterPackageOnlyMessage = + 'Flutter Pilot only supports Flutter packages. Run this command from a ' + 'directory with a pubspec.yaml that declares dependencies.flutter.sdk: ' + 'flutter.'; diff --git a/lib/src/commands/diff_command.dart b/lib/src/commands/diff_command.dart new file mode 100644 index 0000000..b549e25 --- /dev/null +++ b/lib/src/commands/diff_command.dart @@ -0,0 +1,51 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; + +import '../run_diff.dart'; + +/// `diff` command for comparing two existing Scenario Run directories. +/// +/// It reads each directory's `run_report.json`, generates a Step-focused Run +/// Diff, and prints either human-readable output or `--json` output. +/// Regressions are report content, not process failures, so successful diff +class DiffCommand extends Command { + DiffCommand() { + argParser.addFlag( + 'json', + negatable: false, + help: 'Print machine-readable Run Diff output.', + ); + } + + @override + String get description => 'Compare two Scenario Run directories.'; + + @override + String get name => 'diff'; + + @override + Future run() async { + final bool jsonOutput = argResults!.flag('json'); + if (argResults!.rest.length != 2) { + throw UsageException('Expected before and after run directories.', usage); + } + final Directory beforeRunDirectory = Directory(argResults!.rest[0]); + final Directory afterRunDirectory = Directory(argResults!.rest[1]); + try { + final RunDiff diff = RunDiffEngine.diffDirectories( + beforeRunDirectory: beforeRunDirectory, + afterRunDirectory: afterRunDirectory, + ); + if (jsonOutput) { + stdout.writeln(RunDiffJsonRenderer.render(diff)); + } else { + stdout.writeln(RunDiffTextRenderer.render(diff)); + } + return 0; + } on RunDiffException catch (error) { + stderr.writeln(error.message); + return 1; + } + } +} diff --git a/lib/src/commands/report_command.dart b/lib/src/commands/report_command.dart new file mode 100644 index 0000000..4e81918 --- /dev/null +++ b/lib/src/commands/report_command.dart @@ -0,0 +1,41 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; + +import '../html_timeline_report.dart'; + +/// `report` command for regenerating HTML from an existing run directory. +/// +/// It reads `/run_report.json` and writes `timeline.html` beside +/// the existing report artifacts. +class ReportCommand extends Command { + @override + String get description => + 'Generate an HTML timeline report from an existing run directory.'; + + @override + String get name => 'report'; + + @override + Future run() async { + if (argResults!.rest.length != 1) { + throw UsageException('Expected exactly one run directory.', usage); + } + final Directory runDirectory = Directory(argResults!.rest.single); + if (!runDirectory.existsSync()) { + stderr.writeln('Run directory does not exist: ${runDirectory.path}'); + return 1; + } + try { + HtmlTimelineReport.generateFromRunDirectory(runDirectory); + stdout.writeln('HTML report: ${runDirectory.path}/timeline.html'); + return 0; + } on FileSystemException catch (error) { + stderr.writeln(error.message); + return 1; + } on FormatException catch (error) { + stderr.writeln(error.message); + return 1; + } + } +} diff --git a/lib/src/commands/test_command.dart b/lib/src/commands/test_command.dart new file mode 100644 index 0000000..6a4b898 --- /dev/null +++ b/lib/src/commands/test_command.dart @@ -0,0 +1,338 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; + +import '../diagnostic_text_renderer.dart'; +import '../project_scenario_discovery.dart'; +import '../scenario.dart'; +import '../scenario_parser.dart'; +import '../scenario_runner.dart'; +import '../step_progress_renderer.dart'; +import '../target_app_launch_progress_renderer.dart'; +import '../test_command_support.dart'; + +/// `test` command shell for validating CLI arguments before app launch. +/// +/// It checks Scenario YAML and validates `--until` / `--print` relationships +class TestCommand extends Command { + TestCommand({ + required TestCommandExecutor executor, + required ProjectRunCommandExecutor projectRunExecutor, + }) : _executor = executor, + _projectRunExecutor = projectRunExecutor { + argParser + ..addOption( + 'device', + abbr: 'd', + help: 'Target Device id, exact name, or unique id/name prefix.', + ) + ..addOption('flavor', help: 'Flutter flavor passed to flutter run.') + ..addOption( + 'target', + abbr: 't', + help: 'Flutter app entrypoint file passed to flutter run.', + ) + ..addOption( + 'until', + help: 'Run through a 1-based step number or step label.', + ) + ..addMultiOption( + 'print', + allowed: ['snapshot', 'widget-tree', 'errors'], + help: 'Print diagnostics after --until.', + ) + ..addFlag( + 'json', + negatable: false, + help: 'Print raw diagnostics as indented JSON.', + ); + } + + final TestCommandExecutor _executor; + final ProjectRunCommandExecutor _projectRunExecutor; + + @override + String get description => 'Launch the Target App Package and run a Scenario.'; + + @override + String get name => 'test'; + + @override + Future run() async { + if (argResults!.rest.length > 1) { + throw UsageException('Expected zero or one scenario path.', usage); + } + final String? scenarioPath = argResults!.rest.isEmpty + ? null + : argResults!.rest.single; + if (scenarioPath == null) { + return _runProjectRun( + ProjectScenarioDiscovery.defaultPilotDirectory, + defaultDiscovery: true, + ); + } + if (FileSystemEntity.isDirectorySync(scenarioPath)) { + return _runProjectRun(scenarioPath); + } + if (!FileSystemEntity.isFileSync(scenarioPath)) { + throw UsageException( + 'Scenario path does not exist: $scenarioPath', + usage, + ); + } + if (argResults!.multiOption('print').isNotEmpty && + argResults!.option('until') == null) { + throw UsageException('--print must be used with --until.', usage); + } + + final Scenario parsedScenario; + try { + parsedScenario = ScenarioParser.parseFile(scenarioPath); + } on ScenarioValidationException catch (error) { + _writeValidationErrors(error.errors); + return 1; + } + + final String? until = argResults!.option('until'); + RunStopPoint? stopPoint; + if (until != null) { + final String? error = _validateUntil(until, parsedScenario); + if (error != null) { + stderr.writeln(error); + return 64; + } + stopPoint = _stopPointFromUntil(until); + } + + final String? device = argResults!.option('device')?.trim(); + if (device != null && device.isEmpty) { + stderr.writeln('Target Device selector must not be empty.'); + return 64; + } + final String? flavor = argResults!.option('flavor')?.trim(); + if (flavor != null && flavor.isEmpty) { + stderr.writeln('--flavor must not be empty.'); + return 64; + } + final String? target = argResults!.option('target')?.trim(); + if (target != null && target.isEmpty) { + stderr.writeln('--target must not be empty.'); + return 64; + } + final TestCommandOptions options = TestCommandOptions( + scenario: parsedScenario, + device: device, + flavor: flavor, + target: target, + stopPoint: stopPoint, + printDiagnostics: _printDiagnosticsFromOptions( + argResults!.multiOption('print'), + ), + jsonOutput: argResults!.flag('json'), + ); + try { + final StepProgressRenderer? progressRenderer = + TestCommandOutput.stepProgressRenderer( + sink: stderr, + jsonOutput: options.jsonOutput, + stderrHasTerminal: stderr.hasTerminal, + ); + final TargetAppLaunchProgressRenderer? launchProgressRenderer = + TestCommandOutput.targetAppLaunchProgressRenderer( + sink: stderr, + jsonOutput: options.jsonOutput, + stderrHasTerminal: stderr.hasTerminal, + ); + final ScenarioRunReport report = await _executor.run( + options, + onLaunchProgress: launchProgressRenderer?.render, + launchHeartbeatEnabled: launchProgressRenderer != null, + onProgress: progressRenderer?.render, + ); + if (report.printedDiagnostics.isNotEmpty) { + if (options.jsonOutput) { + stdout.writeln( + const JsonEncoder.withIndent( + ' ', + ).convert(_printDiagnosticsJson(report)), + ); + } else { + stdout.writeln(DiagnosticTextRenderer.render(report)); + } + } + stdout.writeln('Run report: ${report.runDirectoryPath}/run_report.json'); + stdout.writeln('HTML report: ${report.runDirectoryPath}/timeline.html'); + return report.status == ScenarioRunStatus.passed ? 0 : 1; + } on TestCommandException catch (error) { + if (!error.alreadyRendered) { + stderr.writeln(error.message); + } + return error.exitCode; + } + } + + /// Discover Project Scenarios and delegate Project Run execution. + Future _runProjectRun( + String discoveryRootPath, { + bool defaultDiscovery = false, + }) async { + if (argResults!.option('until') != null) { + throw UsageException( + '--until is only supported for one Scenario file.', + usage, + ); + } + if (argResults!.multiOption('print').isNotEmpty) { + throw UsageException( + '--print is only supported for one Scenario file.', + usage, + ); + } + final String? device = argResults!.option('device')?.trim(); + if (device != null && device.isEmpty) { + stderr.writeln('--device must not be empty.'); + return 64; + } + final String? flavor = argResults!.option('flavor')?.trim(); + if (flavor != null && flavor.isEmpty) { + stderr.writeln('--flavor must not be empty.'); + return 64; + } + final String? target = argResults!.option('target')?.trim(); + if (target != null && target.isEmpty) { + stderr.writeln('--target must not be empty.'); + return 64; + } + final List scenarios; + try { + scenarios = defaultDiscovery + ? ProjectScenarioDiscovery.discoverDefault() + : ProjectScenarioDiscovery.discoverInDirectory(discoveryRootPath); + } on ProjectScenarioDiscoveryException catch (error) { + stderr.writeln(error.message); + return error.usageError ? 64 : 1; + } on ScenarioValidationException catch (error) { + _writeValidationErrors(error.errors); + return 1; + } + final ProjectRunCommandOptions options = ProjectRunCommandOptions( + discoveryRootPath: discoveryRootPath, + scenarios: scenarios, + device: device, + flavor: flavor, + target: target, + jsonOutput: argResults!.flag('json'), + ); + try { + final StepProgressRenderer? progressRenderer = + TestCommandOutput.stepProgressRenderer( + sink: stderr, + jsonOutput: options.jsonOutput, + stderrHasTerminal: stderr.hasTerminal, + ); + final TargetAppLaunchProgressRenderer? launchProgressRenderer = + TestCommandOutput.targetAppLaunchProgressRenderer( + sink: stderr, + jsonOutput: options.jsonOutput, + stderrHasTerminal: stderr.hasTerminal, + ); + final ProjectRunCommandReport report = await _projectRunExecutor.run( + options, + onLaunchProgress: launchProgressRenderer?.render, + launchHeartbeatEnabled: launchProgressRenderer != null, + onProgress: progressRenderer?.render, + ); + stdout.write(TestCommandOutput.renderProjectRunSummary(report)); + return report.passed ? 0 : 1; + } on TestCommandException catch (error) { + if (!error.alreadyRendered) { + stderr.writeln(error.message); + } + return error.exitCode; + } + } + + /// Return the runner stop point selected by a validated `--until` value. + RunStopPoint _stopPointFromUntil(String until) { + final int? stepNumber = int.tryParse(until); + if (stepNumber != null) { + return RunStopPoint.stepNumber(stepNumber); + } + return RunStopPoint.stepLabel(until); + } + + /// Return the runner diagnostics selected by repeated CLI `--print` options. + /// + /// Args: + /// `printValues` are the already-validated option values from `args`. + /// + /// Returns: + /// The diagnostic requests to pass to the runner. Duplicate options collapse + /// to one request. + Set _printDiagnosticsFromOptions(List printValues) { + final Set printDiagnostics = {}; + for (final String printValue in printValues) { + final PrintDiagnostic? printDiagnostic = switch (printValue) { + 'snapshot' => PrintDiagnostic.snapshot, + 'widget-tree' => PrintDiagnostic.widgetTree, + 'errors' => PrintDiagnostic.errors, + _ => null, + }; + if (printDiagnostic != null) { + printDiagnostics.add(printDiagnostic); + } + } + return printDiagnostics; + } + + /// Return the stdout JSON object for printable diagnostics. + Map _printDiagnosticsJson(ScenarioRunReport report) { + return { + for (final PrintedDiagnostic diagnostic in report.printedDiagnostics) + _printDiagnosticJsonKey(diagnostic.type): diagnostic.data, + }; + } + + /// Return the public JSON key for a printable diagnostic type. + String _printDiagnosticJsonKey(PrintDiagnostic printDiagnostic) { + return switch (printDiagnostic) { + PrintDiagnostic.snapshot => 'snapshot', + PrintDiagnostic.widgetTree => 'widgetTree', + PrintDiagnostic.errors => 'errors', + }; + } + + /// Validate that `--until` names an existing stop point in the Scenario. + /// + /// Args: + /// `until` is either a 1-based step number or a step label. + /// `scenario` provides the step count and available labels. + /// + /// Returns: + /// `null` when valid, otherwise a human-readable CLI usage error. + String? _validateUntil(String until, Scenario scenario) { + final int? stepNumber = int.tryParse(until); + if (stepNumber != null) { + if (stepNumber < 1 || stepNumber > scenario.steps.length) { + return '--until step number must be between 1 and ${scenario.steps.length}.'; + } + return null; + } + + final Iterable labels = scenario.steps + .map((ScenarioStep step) => step.label) + .whereType(); + if (!labels.contains(until)) { + return '--until must be a 1-based step number or an existing step label.'; + } + return null; + } + + /// Print validation errors as `path: message` lines for humans. + void _writeValidationErrors(List errors) { + for (final ScenarioValidationError error in errors) { + stderr.writeln('${error.path}: ${error.message}'); + } + } +} diff --git a/lib/src/commands/validate_command.dart b/lib/src/commands/validate_command.dart new file mode 100644 index 0000000..47f5d4f --- /dev/null +++ b/lib/src/commands/validate_command.dart @@ -0,0 +1,74 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; + +import '../scenario_parser.dart'; + +/// `validate` command for checking Scenario YAML without connecting to Flutter. +/// +/// It reads exactly one file path. By default it prints human-readable output; +/// with `--json` it prints a stable object containing `valid` and `errors`. +class ValidateCommand extends Command { + ValidateCommand() { + argParser.addFlag( + 'json', + negatable: false, + help: 'Print machine-readable validation output.', + ); + } + + @override + String get description => 'Validate a scenario YAML file.'; + + @override + String get name => 'validate'; + + @override + Future run() async { + final bool jsonOutput = argResults!.flag('json'); + if (argResults!.rest.length != 1) { + throw UsageException('Expected exactly one scenario file.', usage); + } + + try { + ScenarioParser.parseFile(argResults!.rest.single); + if (jsonOutput) { + _writeValidationJson(valid: true, errors: const []); + } else { + stdout.writeln('Scenario is valid.'); + } + return 0; + } on ScenarioValidationException catch (error) { + if (jsonOutput) { + _writeValidationJson(valid: false, errors: error.errors); + } else { + _writeValidationErrors(error.errors); + } + return 1; + } + } + + /// Print validation status in the `validate --json` response shape. + void _writeValidationJson({ + required bool valid, + required List errors, + }) { + stdout.writeln( + const JsonEncoder.withIndent(' ').convert({ + 'valid': valid, + 'errors': [ + for (final ScenarioValidationError error in errors) + {'path': error.path, 'message': error.message}, + ], + }), + ); + } + + /// Print validation errors as `path: message` lines for humans. + void _writeValidationErrors(List errors) { + for (final ScenarioValidationError error in errors) { + stderr.writeln('${error.path}: ${error.message}'); + } + } +} diff --git a/lib/src/project_run_command_executor.dart b/lib/src/project_run_command_executor.dart new file mode 100644 index 0000000..d759fe4 --- /dev/null +++ b/lib/src/project_run_command_executor.dart @@ -0,0 +1,365 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:screen_recorder/screen_recorder.dart' as screen_recorder; + +import 'artifacts/artifact_store.dart'; +import 'project_scenario_discovery.dart'; +import 'project_run_report.dart'; +import 'recording/screen_recorder_recording_controller.dart'; +import 'runtime/runtime_adapter_selector.dart'; +import 'runtime/runtime_contract.dart'; +import 'scenario_runner.dart'; +import 'target_app_launch_progress_renderer.dart'; +import 'target_app_launcher.dart'; +import 'target_device.dart'; +import 'test_command_models.dart'; +import 'test_device_discovery.dart'; +import 'test_scenario_runner_factory.dart'; +import 'test_target_device_selection.dart'; + +/// Executes a validated Project Run selected by `test`. +abstract interface class ProjectRunCommandExecutor { + /// Run the Project Scenarios described by [options]. + /// + /// `onLaunchProgress`, when provided, receives the batch-level Target App + /// Launch Progress events before any Project Scenario executes. + Future run( + ProjectRunCommandOptions options, { + void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, + bool launchHeartbeatEnabled = false, + void Function(StepProgressEvent event)? onProgress, + }); +} + +/// Default Project Run executor for launch reuse and batch Scenario execution. +class DefaultProjectRunCommandExecutor implements ProjectRunCommandExecutor { + const DefaultProjectRunCommandExecutor({ + this.deviceDiscovery = const DefaultTestDeviceDiscovery(), + this.launcher = const TargetAppLauncher(), + this.runnerFactory = const DefaultTestScenarioRunnerFactory(), + this.interruptSignals, + this.outputDirectory, + this.clock = DateTime.now, + this.launchHeartbeatTicks, + }); + + /// Discovers Flutter and Recording Devices before app launch when needed. + final TestDeviceDiscovery deviceDiscovery; + + /// Starts the Target App Package and exposes hot restart. + final TargetAppLauncher launcher; + + /// Creates Scenario runners for each selected Project Scenario. + final TestScenarioRunnerFactory runnerFactory; + + /// Optional interrupt stream used by tests and Ctrl-C handling. + final Stream? interruptSignals; + + /// Directory where Project Run artifacts are written. + final Directory? outputDirectory; + + /// Clock used for report timestamps, durations, and tests. + final TargetAppLaunchClock clock; + + /// Optional heartbeat stream used by launch progress tests. + final Stream? launchHeartbeatTicks; + + @override + Future run( + ProjectRunCommandOptions options, { + void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, + bool launchHeartbeatEnabled = false, + void Function(StepProgressEvent event)? onProgress, + }) async { + final DateTime startedAt = clock().toUtc(); + final ProjectRunArtifactWriter projectRunWriter = RunArtifactStore( + outputDirectory ?? Directory.current, + ).createProjectRun(startedAt: startedAt); + final Stopwatch stopwatch = Stopwatch()..start(); + final List scenarioResults = + []; + ProjectRunEnvironmentFailure? environmentFailure; + TargetAppLaunch? launch; + TargetDevice? targetDevice; + final bool recordingRequired = options.scenarios.any( + (ProjectScenarioFile file) => file.scenario.recording?.enabled == true, + ); + if (options.device != null || recordingRequired) { + try { + final List flutterDevices = await deviceDiscovery + .listFlutterDevices(); + final List recordingDevices = recordingRequired + ? await deviceDiscovery.listRecordingDevices() + : const []; + targetDevice = TargetDeviceResolver.resolve( + selector: options.device, + recordingRequired: recordingRequired, + flutterDevices: flutterDevices, + recordingDevices: recordingDevices, + ); + } on TargetDeviceResolutionException catch (error) { + environmentFailure = ProjectRunEnvironmentFailure( + phase: ProjectRunEnvironmentFailurePhase.targetDeviceResolution, + message: error.message, + ); + } on DeviceDiscoveryException catch (error) { + environmentFailure = ProjectRunEnvironmentFailure( + phase: ProjectRunEnvironmentFailurePhase.targetDeviceResolution, + message: error.message, + ); + } + } + if (environmentFailure != null) { + stopwatch.stop(); + final ProjectRunReport projectReport = + ProjectRunReport.environmentFailure( + discoveryRootPath: options.discoveryRootPath, + startedAt: startedAt, + durationMs: stopwatch.elapsedMilliseconds, + commandInputs: ProjectRunCommandInputs( + device: options.device, + flavor: options.flavor, + target: options.target, + ), + failure: environmentFailure, + scenarioResults: scenarioResults, + ); + projectRunWriter.writeProjectRunReport(projectReport.toJson()); + return ProjectRunCommandReport( + passed: false, + status: ProjectRunStatus.environmentFailed, + projectRunReportPath: p.join( + projectRunWriter.runDirectory.path, + 'project_run_report.json', + ), + scenarioReports: const [], + ); + } + final TargetAppLaunchChoices launchChoices = TargetAppLaunchChoices( + targetDevice: targetDevice, + selectionReason: targetDeviceSelectionReason( + deviceSelector: options.device, + recordingRequired: recordingRequired, + ), + flavor: options.flavor, + target: options.target, + ); + final TargetAppLaunchStartedEvent launchStartedEvent = + TargetAppLaunchStartedEvent( + startedAt: startedAt, + choices: launchChoices, + ); + onLaunchProgress?.call(launchStartedEvent); + TargetAppLaunchHeartbeat? launchHeartbeat; + if (onLaunchProgress != null && launchHeartbeatEnabled) { + launchHeartbeat = TargetAppLaunchHeartbeat( + ticks: + launchHeartbeatTicks ?? + Stream.periodic(const Duration(seconds: 10)), + onProgress: onLaunchProgress, + clock: clock, + ); + launchHeartbeat.start(launchStartedEvent); + } + try { + launch = await launcher.launch( + TargetAppLaunchCommand( + flavor: options.flavor, + target: options.target, + deviceId: targetDevice?.id, + ), + ); + onLaunchProgress?.call( + TargetAppLaunchSucceededEvent( + startedAt: startedAt, + finishedAt: clock().toUtc(), + choices: launchChoices, + ), + ); + await launchHeartbeat?.stop(); + for (int index = 0; index < options.scenarios.length; index++) { + final ProjectScenarioFile scenarioFile = options.scenarios[index]; + if (index > 0) { + try { + await launch.hotRestart(); + } on TargetAppLaunchException catch (error) { + environmentFailure = ProjectRunEnvironmentFailure( + phase: ProjectRunEnvironmentFailurePhase.hotRestart, + message: error.message, + ); + break; + } + } + final RunArtifactWriter childRun = projectRunWriter.createScenarioRun( + scenario: scenarioFile.scenario, + startedAt: clock().toUtc(), + ); + final TestScenarioRunner runner; + try { + runner = runnerFactory.create( + runtimeTarget: RuntimeTarget(vmServiceUri: launch.runtimeTargetUri), + targetDevice: targetDevice, + recordingController: + scenarioFile.scenario.recording?.enabled == true + ? ScreenRecorderRecordingController( + recorder: screen_recorder.ScreenRecorder.defaultRecorder(), + deviceSelector: targetDevice!.id, + outputDirectory: Directory.current, + ) + : null, + ); + } on RuntimeAdapterSelectionException catch (error) { + environmentFailure = ProjectRunEnvironmentFailure( + phase: ProjectRunEnvironmentFailurePhase.runtimeSelection, + message: error.message, + ); + break; + } + final Future runFuture = runner.run( + scenarioFile.scenario, + runArtifactWriter: childRun, + onProgress: onProgress, + ); + final ScenarioRunReport scenarioReport = + await _runScenarioWithInterrupt(runFuture); + final ProjectScenarioRunStatus scenarioStatus = + scenarioReport.status == ScenarioRunStatus.passed + ? ProjectScenarioRunStatus.passed + : ProjectScenarioRunStatus.failed; + scenarioResults.add( + ProjectScenarioRunReport( + scenarioPath: scenarioFile.relativePath, + status: scenarioStatus, + runReportPath: projectRunWriter.relativePathFor( + childRun, + 'run_report.json', + ), + htmlReportPath: projectRunWriter.relativePathFor( + childRun, + 'timeline.html', + ), + ), + ); + } + } on TargetAppLaunchException catch (error) { + environmentFailure = ProjectRunEnvironmentFailure( + phase: ProjectRunEnvironmentFailurePhase.launch, + message: error.message, + ); + onLaunchProgress?.call( + TargetAppLaunchFailedEvent( + startedAt: startedAt, + failedAt: clock().toUtc(), + message: error.message, + stderrLines: error.stderrLines, + choices: launchChoices, + ), + ); + await launchHeartbeat?.stop(); + } on TestCommandException catch (error) { + if (error.exitCode == 130) { + rethrow; + } + environmentFailure = ProjectRunEnvironmentFailure( + phase: ProjectRunEnvironmentFailurePhase.validation, + message: error.message, + ); + } finally { + stopwatch.stop(); + await launchHeartbeat?.stop(); + await launch?.cleanup(); + } + final bool allPassed = + environmentFailure == null && + scenarioResults.every( + (ProjectScenarioRunReport report) => + report.status == ProjectScenarioRunStatus.passed, + ); + final ProjectRunStatus projectStatus = environmentFailure != null + ? ProjectRunStatus.environmentFailed + : allPassed + ? ProjectRunStatus.passed + : ProjectRunStatus.failed; + final ProjectRunReport projectReport = environmentFailure == null + ? ProjectRunReport( + discoveryRootPath: options.discoveryRootPath, + scenarioResults: scenarioResults, + status: projectStatus, + startedAt: startedAt, + durationMs: stopwatch.elapsedMilliseconds, + commandInputs: ProjectRunCommandInputs( + device: options.device, + flavor: options.flavor, + target: options.target, + ), + ) + : ProjectRunReport.environmentFailure( + discoveryRootPath: options.discoveryRootPath, + startedAt: startedAt, + durationMs: stopwatch.elapsedMilliseconds, + commandInputs: ProjectRunCommandInputs( + device: options.device, + flavor: options.flavor, + target: options.target, + ), + failure: environmentFailure, + scenarioResults: scenarioResults, + ); + projectRunWriter.writeProjectRunReport(projectReport.toJson()); + return ProjectRunCommandReport( + passed: projectStatus == ProjectRunStatus.passed, + status: projectStatus, + projectRunReportPath: p.join( + projectRunWriter.runDirectory.path, + 'project_run_report.json', + ), + scenarioReports: [ + for (final ProjectScenarioRunReport scenarioReport in scenarioResults) + ProjectRunScenarioOutputReport( + scenarioPath: scenarioReport.scenarioPath, + status: scenarioReport.status, + runReportPath: p.join( + projectRunWriter.runDirectory.path, + scenarioReport.runReportPath, + ), + htmlReportPath: p.join( + projectRunWriter.runDirectory.path, + scenarioReport.htmlReportPath, + ), + ), + ], + ); + } + + Future _runScenarioWithInterrupt( + Future runFuture, + ) async { + StreamSubscription? interruptSub; + try { + final Completer interruptCompleter = + Completer(); + interruptSub = + (interruptSignals ?? + ProcessSignal.sigint.watch().map((ProcessSignal _) {})) + .listen((_) { + if (!interruptCompleter.isCompleted) { + interruptCompleter.completeError( + const TestCommandException( + message: 'test command interrupted.', + exitCode: 130, + ), + ); + } + }); + return await Future.any(>[ + runFuture, + interruptCompleter.future, + ]); + } finally { + runFuture.ignore(); + await interruptSub?.cancel(); + } + } +} diff --git a/lib/src/project_run_report.dart b/lib/src/project_run_report.dart index 8cab5b8..8a78aca 100644 --- a/lib/src/project_run_report.dart +++ b/lib/src/project_run_report.dart @@ -150,5 +150,6 @@ enum ProjectRunEnvironmentFailurePhase { validation, targetDeviceResolution, launch, + runtimeSelection, hotRestart, } diff --git a/lib/src/runtime/pilot_runtime_adapter.dart b/lib/src/runtime/pilot_runtime_adapter.dart new file mode 100644 index 0000000..01723aa --- /dev/null +++ b/lib/src/runtime/pilot_runtime_adapter.dart @@ -0,0 +1,104 @@ +import 'package:pilot_runtime/pilot_runtime_client.dart'; + +import '../scenario.dart'; +import 'runtime_contract.dart'; + +/// Runtime Adapter backed by the `pilot_runtime` package. +/// +/// This first adapter slice verifies the app-side runtime handshake and exposes +/// Widget Tree capture through Flutter Pilot's existing Runtime Adapter +/// contract. User actions are implemented in later runtime slices. +class PilotRuntimeAdapter implements RuntimeAdapter { + /// Create an adapter backed by a checked `pilot_runtime` client. + const PilotRuntimeAdapter({ + required PilotRuntimeClient client, + required String projectRoot, + Future Function()? disposeClient, + }) : _client = client, + _projectRoot = projectRoot, + _disposeClient = disposeClient; + + final PilotRuntimeClient _client; + final String _projectRoot; + final Future Function()? _disposeClient; + + @override + Future initialize() async { + try { + await _client.initialize(); + } on PilotRuntimeInitializationException catch (error) { + throw RuntimeOperationException( + operation: RuntimeOperation.initialize, + message: error.message, + cause: error, + ); + } + } + + @override + Future dispose() async { + await _disposeClient?.call(); + } + + @override + Future> resolveFinder(Finder finder) { + throw _notImplemented(RuntimeOperation.resolveFinder); + } + + @override + Future performTap(FinderMatch match) { + throw _notImplemented(RuntimeOperation.performTap); + } + + @override + Future replaceText(FinderMatch match, String text) { + throw _notImplemented(RuntimeOperation.replaceText); + } + + @override + Future performScroll({ + FinderMatch? match, + required double deltaX, + required double deltaY, + }) { + throw _notImplemented(RuntimeOperation.performScroll); + } + + @override + Future captureScreenshot() { + throw _notImplemented(RuntimeOperation.captureScreenshot); + } + + @override + Future captureSnapshot() { + throw _notImplemented(RuntimeOperation.captureSnapshot); + } + + @override + Future captureWidgetTree() async { + try { + final Map data = await _client.captureWidgetTree( + projectRoot: _projectRoot, + ); + return WidgetTreeCapture(data: data); + } on PilotRuntimeWidgetTreeCaptureException catch (error) { + throw RuntimeOperationException( + operation: RuntimeOperation.captureWidgetTree, + message: error.message, + cause: error, + ); + } + } + + @override + Future collectLogs() { + throw _notImplemented(RuntimeOperation.collectLogs); + } + + RuntimeOperationException _notImplemented(RuntimeOperation operation) { + return RuntimeOperationException( + operation: operation, + message: '${operation.name} is not implemented for pilot_runtime yet.', + ); + } +} diff --git a/lib/src/runtime/pilot_runtime_vm_service.dart b/lib/src/runtime/pilot_runtime_vm_service.dart new file mode 100644 index 0000000..a03a2d3 --- /dev/null +++ b/lib/src/runtime/pilot_runtime_vm_service.dart @@ -0,0 +1,75 @@ +import 'package:pilot_runtime/pilot_runtime_client.dart'; +import 'package:vm_service/vm_service.dart' as vm_service; +import 'package:vm_service/vm_service_io.dart' as vm_service_io; + +/// VM Service caller used by `PilotRuntimeClient` in the Flutter Pilot CLI. +/// +/// The caller connects lazily to the Runtime Target, selects the first running +/// isolate, and forwards service extension calls with JSON-compatible +/// arguments. +class PilotRuntimeVmServiceConnection implements PilotRuntimeVmService { + /// Create a VM Service caller for one Runtime Target URI. + PilotRuntimeVmServiceConnection({required Uri vmServiceUri}) + : _vmServiceUri = vmServiceUri; + + final Uri _vmServiceUri; + vm_service.VmService? _service; + String? _isolateId; + + @override + Future> callServiceExtension( + String extensionName, { + Map parameters = const {}, + }) async { + final vm_service.VmService service = await _connectedService(); + final String isolateId = await _selectedIsolateId(service); + try { + final vm_service.Response response = await service.callServiceExtension( + extensionName, + isolateId: isolateId, + args: {...parameters}, + ); + return response.toJson(); + } on vm_service.RPCError catch (error) { + if (error.code == vm_service.RPCErrorKind.kMethodNotFound.code || + error.code == vm_service.RPCErrorKind.kServiceDisappeared.code) { + throw PilotRuntimeServiceExtensionMissingException(extensionName); + } + rethrow; + } + } + + /// Close the underlying VM Service connection when one was opened. + Future dispose() async { + await _service?.dispose(); + _service = null; + _isolateId = null; + } + + Future _connectedService() async { + final vm_service.VmService? existingService = _service; + if (existingService != null) { + return existingService; + } + final vm_service.VmService service = await vm_service_io + .vmServiceConnectUri(_vmServiceUri.toString()); + _service = service; + return service; + } + + Future _selectedIsolateId(vm_service.VmService service) async { + final String? existingIsolateId = _isolateId; + if (existingIsolateId != null) { + return existingIsolateId; + } + final vm_service.VM vm = await service.getVM(); + for (final vm_service.IsolateRef isolate in vm.isolates ?? const []) { + final String? isolateId = isolate.id; + if (isolateId != null) { + _isolateId = isolateId; + return isolateId; + } + } + throw StateError('Runtime Target VM Service has no running isolates.'); + } +} diff --git a/lib/src/runtime/runtime_adapter_selector.dart b/lib/src/runtime/runtime_adapter_selector.dart new file mode 100644 index 0000000..1c31011 --- /dev/null +++ b/lib/src/runtime/runtime_adapter_selector.dart @@ -0,0 +1,63 @@ +import 'dart:io'; + +import 'package:pilot_runtime/pilot_runtime_client.dart'; + +import 'mcp_flutter_runtime_adapter.dart'; +import 'pilot_runtime_adapter.dart'; +import 'pilot_runtime_vm_service.dart'; +import 'runtime_contract.dart'; + +/// Selects the Runtime Adapter for a launched Runtime Target. +/// +/// The default path stays on `mcp_flutter`. A hidden environment switch is +/// reserved for experimental runtime adapters without adding public CLI flags. +class RuntimeAdapterSelector { + RuntimeAdapterSelector._(); + + /// Environment variable that selects a non-default runtime adapter. + static const String environmentKey = 'FLUTTER_PILOT_RUNTIME'; + + /// Return the Runtime Adapter selected by the hidden environment switch. + /// + /// Args: + /// `target` is the launched Runtime Target passed to the adapter. + /// `environment` defaults to `Platform.environment`; tests pass explicit + /// maps so selection can be verified without changing process state. + static RuntimeAdapter select({ + required RuntimeTarget target, + Map? environment, + String? projectRoot, + }) { + final Map effectiveEnvironment = + environment ?? Platform.environment; + final String? runtime = effectiveEnvironment[environmentKey]; + if (runtime == null || runtime.isEmpty || runtime == 'mcp_flutter') { + return McpFlutterRuntimeAdapter(target: target); + } + if (runtime == 'pilot_runtime') { + final PilotRuntimeVmServiceConnection vmService = + PilotRuntimeVmServiceConnection(vmServiceUri: target.vmServiceUri); + return PilotRuntimeAdapter( + client: PilotRuntimeClient(vmService), + projectRoot: projectRoot ?? Directory.current.path, + disposeClient: vmService.dispose, + ); + } + throw RuntimeAdapterSelectionException( + 'Invalid $environmentKey value "$runtime". ' + 'Expected "mcp_flutter" or "pilot_runtime".', + ); + } +} + +/// Failure raised when the hidden Runtime Adapter switch is invalid. +class RuntimeAdapterSelectionException implements Exception { + /// Create a selection failure with a user-facing message. + const RuntimeAdapterSelectionException(this.message); + + /// Human-readable explanation of the invalid switch value. + final String message; + + @override + String toString() => message; +} diff --git a/lib/src/test_command_executor.dart b/lib/src/test_command_executor.dart new file mode 100644 index 0000000..24ea09b --- /dev/null +++ b/lib/src/test_command_executor.dart @@ -0,0 +1,212 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:screen_recorder/screen_recorder.dart' as screen_recorder; + +import 'recording/screen_recorder_recording_controller.dart'; +import 'runtime/runtime_adapter_selector.dart'; +import 'runtime/runtime_contract.dart'; +import 'scenario_runner.dart'; +import 'target_app_launch_progress_renderer.dart'; +import 'target_app_launcher.dart'; +import 'target_device.dart'; +import 'test_command_models.dart'; +import 'test_device_discovery.dart'; +import 'test_scenario_runner_factory.dart'; +import 'test_target_device_selection.dart'; + +/// Executes a validated `test` command. +abstract interface class TestCommandExecutor { + /// Run the Scenario described by `options` and return its run report. + /// + /// `onLaunchProgress`, when provided, receives Target App Launch Progress + /// events before Scenario execution starts. `launchHeartbeatEnabled` controls + /// whether long pending launches emit heartbeat events. `onProgress`, when + /// provided, receives Scenario Step progress events during execution. + Future run( + TestCommandOptions options, { + void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, + bool launchHeartbeatEnabled = false, + void Function(StepProgressEvent event)? onProgress, + }); +} + +/// Default `test` command executor for launching and running one Scenario. +class DefaultTestCommandExecutor implements TestCommandExecutor { + /// Creates an executor with injectable launch and discovery boundaries. + /// + /// `launchHeartbeatTicks` and `launchClock` make Target App Launch Progress + /// deterministic in tests without waiting on real time. + const DefaultTestCommandExecutor({ + this.deviceDiscovery = const DefaultTestDeviceDiscovery(), + this.launcher = const TargetAppLauncher(), + this.runnerFactory = const DefaultTestScenarioRunnerFactory(), + this.interruptSignals, + this.launchHeartbeatTicks, + this.launchClock = DateTime.now, + }); + + final TestDeviceDiscovery deviceDiscovery; + final TargetAppLauncher launcher; + final TestScenarioRunnerFactory runnerFactory; + final Stream? interruptSignals; + final Stream? launchHeartbeatTicks; + final TargetAppLaunchClock launchClock; + + @override + Future run( + TestCommandOptions options, { + void Function(TargetAppLaunchProgressEvent event)? onLaunchProgress, + bool launchHeartbeatEnabled = false, + void Function(StepProgressEvent event)? onProgress, + }) async { + final bool recordingRequired = options.scenario.recording?.enabled == true; + TargetDevice? targetDevice; + if (options.device != null || recordingRequired) { + try { + final List flutterDevices = await deviceDiscovery + .listFlutterDevices(); + final List recordingDevices = recordingRequired + ? await deviceDiscovery.listRecordingDevices() + : const []; + targetDevice = TargetDeviceResolver.resolve( + selector: options.device, + recordingRequired: recordingRequired, + flutterDevices: flutterDevices, + recordingDevices: recordingDevices, + ); + } on TargetDeviceResolutionException catch (error) { + throw TestCommandException(message: error.message, exitCode: 64); + } on DeviceDiscoveryException catch (error) { + throw TestCommandException(message: error.message, exitCode: 1); + } + } + + TargetAppLaunch launch; + final DateTime launchStartedAt = launchClock(); + final TargetAppLaunchChoices launchChoices = TargetAppLaunchChoices( + targetDevice: targetDevice, + selectionReason: targetDeviceSelectionReason( + deviceSelector: options.device, + recordingRequired: recordingRequired, + ), + flavor: options.flavor, + target: options.target, + ); + onLaunchProgress?.call( + TargetAppLaunchStartedEvent( + startedAt: launchStartedAt, + choices: launchChoices, + ), + ); + TargetAppLaunchHeartbeat? launchHeartbeat; + if (onLaunchProgress != null && launchHeartbeatEnabled) { + launchHeartbeat = TargetAppLaunchHeartbeat( + ticks: + launchHeartbeatTicks ?? + Stream.periodic(const Duration(seconds: 10)), + onProgress: onLaunchProgress, + clock: launchClock, + ); + launchHeartbeat.start( + TargetAppLaunchStartedEvent( + startedAt: launchStartedAt, + choices: launchChoices, + ), + ); + } + try { + launch = await launcher.launch( + TargetAppLaunchCommand( + deviceId: targetDevice?.id, + flavor: options.flavor, + target: options.target, + ), + ); + onLaunchProgress?.call( + TargetAppLaunchSucceededEvent( + startedAt: launchStartedAt, + finishedAt: launchClock(), + choices: launchChoices, + ), + ); + await launchHeartbeat?.stop(); + } on TargetAppLaunchException catch (error) { + onLaunchProgress?.call( + TargetAppLaunchFailedEvent( + startedAt: launchStartedAt, + failedAt: launchClock(), + message: error.message, + stderrLines: error.stderrLines, + choices: launchChoices, + ), + ); + await launchHeartbeat?.stop(); + final String stderrContext = + onLaunchProgress == null && error.stderrLines.isNotEmpty + ? '\n${error.stderrLines.join('\n')}' + : ''; + throw TestCommandException( + message: '${error.message}$stderrContext', + exitCode: 1, + alreadyRendered: onLaunchProgress != null, + ); + } + + try { + final TestScenarioRunner runner; + try { + runner = runnerFactory.create( + runtimeTarget: RuntimeTarget(vmServiceUri: launch.runtimeTargetUri), + targetDevice: targetDevice, + recordingController: recordingRequired + ? ScreenRecorderRecordingController( + recorder: screen_recorder.ScreenRecorder.defaultRecorder(), + deviceSelector: targetDevice!.id, + outputDirectory: Directory.current, + ) + : null, + ); + } on RuntimeAdapterSelectionException catch (error) { + throw TestCommandException(message: error.message, exitCode: 1); + } + final Future runFuture = runner.run( + options.scenario, + stopPoint: options.stopPoint, + printDiagnostics: options.printDiagnostics, + onProgress: onProgress, + ); + StreamSubscription? interruptSub; + try { + final Completer interruptCompleter = + Completer(); + interruptSub = + (interruptSignals ?? + ProcessSignal.sigint.watch().map( + (ProcessSignal _) {}, + )) + .listen((_) { + if (!interruptCompleter.isCompleted) { + interruptCompleter.completeError( + const TestCommandException( + message: 'test command interrupted.', + exitCode: 130, + ), + ); + } + }); + return await Future.any(>[ + runFuture, + interruptCompleter.future, + ]); + } finally { + runFuture.ignore(); + await interruptSub?.cancel(); + } + } on RuntimeOperationException catch (error) { + throw TestCommandException(message: error.message, exitCode: 1); + } finally { + await launch.cleanup(); + } + } +} diff --git a/lib/src/test_command_models.dart b/lib/src/test_command_models.dart new file mode 100644 index 0000000..303b1c2 --- /dev/null +++ b/lib/src/test_command_models.dart @@ -0,0 +1,128 @@ +import 'project_scenario_discovery.dart'; +import 'project_run_report.dart'; +import 'scenario.dart'; +import 'scenario_runner.dart'; + +/// Parsed `test` command inputs that are validated before app launch. +class TestCommandOptions { + /// Creates validated inputs for executing one Scenario through `test`. + const TestCommandOptions({ + required this.scenario, + required this.device, + required this.flavor, + required this.target, + required this.stopPoint, + required this.printDiagnostics, + required this.jsonOutput, + }); + + final Scenario scenario; + final String? device; + final String? flavor; + final String? target; + final RunStopPoint? stopPoint; + final Set printDiagnostics; + final bool jsonOutput; +} + +/// Parsed Project Run inputs selected by the `test` command. +class ProjectRunCommandOptions { + /// Creates validated Project Run command inputs. + const ProjectRunCommandOptions({ + required this.discoveryRootPath, + required this.scenarios, + required this.device, + required this.flavor, + required this.target, + required this.jsonOutput, + }); + + /// Directory used to discover Project Scenarios. + final String discoveryRootPath; + + /// Validated Entry Scenario files selected for the Project Run. + final List scenarios; + + final String? device; + final String? flavor; + final String? target; + final bool jsonOutput; +} + +/// Project Run command result used by mode selection and stdout rendering. +class ProjectRunCommandReport { + const ProjectRunCommandReport({ + required this.passed, + required this.status, + required this.projectRunReportPath, + required this.scenarioReports, + }); + + /// Whether the selected Project Scenarios all passed. + final bool passed; + + /// Overall Project Run status. + final ProjectRunStatus status; + + /// Path to the batch-level `project_run_report.json`. + final String projectRunReportPath; + + /// Per-Scenario report paths to print after the Project Run finishes. + final List scenarioReports; +} + +/// Report paths for one Scenario inside Project Run stdout. +class ProjectRunScenarioOutputReport { + const ProjectRunScenarioOutputReport({ + required this.scenarioPath, + required this.status, + required this.runReportPath, + required this.htmlReportPath, + }); + + /// Project Scenario path relative to the discovery root. + final String scenarioPath; + + /// Scenario status in the Project Run summary. + final ProjectScenarioRunStatus status; + + /// Path to the child Scenario Run report. + final String runReportPath; + + /// Path to the child Scenario HTML timeline report. + final String htmlReportPath; +} + +/// Failure raised when device discovery fails. +class DeviceDiscoveryException implements Exception { + /// Creates a device discovery failure. + const DeviceDiscoveryException(this.message); + + /// Human-readable failure reason. + final String message; + + @override + String toString() => message; +} + +/// Failure from the `test` command executor. +class TestCommandException implements Exception { + /// Creates a command execution failure. + const TestCommandException({ + required this.message, + required this.exitCode, + this.alreadyRendered = false, + }); + + /// Human-readable error message. + final String message; + + /// CLI exit code. + final int exitCode; + + /// Whether `message` was already written by command-specific output. + /// + /// The `test` command uses this for launch failures rendered by Target App + /// Launch Progress so the same message is not printed twice. + final bool alreadyRendered; +} diff --git a/lib/src/test_command_output.dart b/lib/src/test_command_output.dart new file mode 100644 index 0000000..5946056 --- /dev/null +++ b/lib/src/test_command_output.dart @@ -0,0 +1,65 @@ +import 'dart:io'; + +import 'step_progress_renderer.dart'; +import 'target_app_launch_progress_renderer.dart'; +import 'test_command_models.dart'; + +/// Terminal output helpers for the `test` command. +class TestCommandOutput { + TestCommandOutput._(); + + /// Return the Step progress renderer for human-readable `test` output. + /// + /// JSON output is machine-oriented and suppresses Step progress. Interactive + /// rendering is used only when stderr is a terminal; redirected stderr gets + /// deterministic plain-text progress lines for CI logs. + static StepProgressRenderer? stepProgressRenderer({ + required IOSink sink, + required bool jsonOutput, + required bool stderrHasTerminal, + }) { + if (jsonOutput) { + return null; + } + return StepProgressRenderer(sink: sink, interactive: stderrHasTerminal); + } + + /// Return the Target App Launch Progress renderer for human-readable output. + /// + /// Launch progress is stderr-only status. JSON output suppresses it so stdout + /// remains machine-oriented. + static TargetAppLaunchProgressRenderer? targetAppLaunchProgressRenderer({ + required IOSink sink, + required bool jsonOutput, + required bool stderrHasTerminal, + }) { + if (jsonOutput) { + return null; + } + return TargetAppLaunchProgressRenderer( + sink: sink, + interactive: stderrHasTerminal, + ); + } + + /// Return deterministic stdout summary lines for a completed Project Run. + /// + /// The summary keeps the batch-level report path first, then prints each + /// Scenario's existing report paths in execution order. + static String renderProjectRunSummary(ProjectRunCommandReport report) { + final StringBuffer buffer = StringBuffer() + ..writeln('Project Run: ${report.status.name}') + ..writeln('Project Run report: ${report.projectRunReportPath}'); + for (final ProjectRunScenarioOutputReport scenarioReport + in report.scenarioReports) { + buffer + ..writeln( + 'Scenario: ${scenarioReport.scenarioPath} ' + '(${scenarioReport.status.name})', + ) + ..writeln('Run report: ${scenarioReport.runReportPath}') + ..writeln('HTML report: ${scenarioReport.htmlReportPath}'); + } + return buffer.toString(); + } +} diff --git a/lib/src/test_command_support.dart b/lib/src/test_command_support.dart new file mode 100644 index 0000000..3d61db8 --- /dev/null +++ b/lib/src/test_command_support.dart @@ -0,0 +1,13 @@ +/// Compatibility export for the `test` command execution support modules. +/// +/// The public package API originally exposed these types through `cli.dart`. +/// Keeping this barrel lets callers import one stable support surface while the +/// implementations live in focused files. +library; + +export 'project_run_command_executor.dart'; +export 'test_command_executor.dart'; +export 'test_command_models.dart'; +export 'test_command_output.dart'; +export 'test_device_discovery.dart'; +export 'test_scenario_runner_factory.dart'; diff --git a/lib/src/test_device_discovery.dart b/lib/src/test_device_discovery.dart new file mode 100644 index 0000000..b72d1e3 --- /dev/null +++ b/lib/src/test_device_discovery.dart @@ -0,0 +1,51 @@ +import 'dart:io'; + +import 'package:screen_recorder/screen_recorder.dart' as screen_recorder; + +import 'target_device.dart'; +import 'test_command_models.dart'; + +/// Discovers Flutter Devices and Recording Devices for `test`. +abstract interface class TestDeviceDiscovery { + /// Return Flutter Devices from `flutter devices --machine`. + Future> listFlutterDevices(); + + /// Return Recording Device identities available for Scenario Recording. + Future> listRecordingDevices(); +} + +/// Default device discovery backed by Flutter CLI and `screen_recorder`. +class DefaultTestDeviceDiscovery implements TestDeviceDiscovery { + /// Creates default Target Device discovery. + const DefaultTestDeviceDiscovery(); + + @override + Future> listFlutterDevices() async { + final ProcessResult result; + try { + result = await Process.run('flutter', ['devices', '--machine']); + } on ProcessException catch (error) { + throw DeviceDiscoveryException(error.message); + } + if (result.exitCode != 0) { + throw DeviceDiscoveryException(result.stderr.toString()); + } + try { + return TargetDeviceParser.parseMachineJson(result.stdout.toString()); + } on FormatException catch (e) { + throw DeviceDiscoveryException(e.message); + } + } + + @override + Future> listRecordingDevices() async { + final screen_recorder.ScreenRecorder recorder = + screen_recorder.ScreenRecorder.defaultRecorder(); + final List devices = await recorder + .listDevices(); + return [ + for (final screen_recorder.RecordingDevice device in devices) + RecordingDeviceIdentity(id: device.id), + ]; + } +} diff --git a/lib/src/test_scenario_runner_factory.dart b/lib/src/test_scenario_runner_factory.dart new file mode 100644 index 0000000..0dbd6d1 --- /dev/null +++ b/lib/src/test_scenario_runner_factory.dart @@ -0,0 +1,79 @@ +import 'dart:io'; + +import 'artifacts/artifact_store.dart'; +import 'recording/recording_contract.dart'; +import 'runtime/runtime_adapter_selector.dart'; +import 'runtime/runtime_contract.dart'; +import 'scenario.dart'; +import 'scenario_runner.dart'; +import 'target_device.dart'; + +/// Creates a Scenario runner for one launched Runtime Target. +abstract interface class TestScenarioRunnerFactory { + /// Create a runner bound to the launched Runtime Target. + TestScenarioRunner create({ + required RuntimeTarget runtimeTarget, + required TargetDevice? targetDevice, + required RecordingController? recordingController, + }); +} + +/// Narrow Scenario runner interface used by the `test` command executor. +abstract interface class TestScenarioRunner { + /// Run `scenario` with optional stop, diagnostic, and Step progress controls. + Future run( + Scenario scenario, { + RunStopPoint? stopPoint, + Set printDiagnostics, + void Function(StepProgressEvent event)? onProgress, + RunArtifactWriter? runArtifactWriter, + }); +} + +/// Default Scenario runner factory backed by `McpFlutterRuntimeAdapter`. +class DefaultTestScenarioRunnerFactory implements TestScenarioRunnerFactory { + /// Creates the default runner factory. + const DefaultTestScenarioRunnerFactory(); + + @override + TestScenarioRunner create({ + required RuntimeTarget runtimeTarget, + required TargetDevice? targetDevice, + required RecordingController? recordingController, + }) { + final RuntimeAdapter adapter = RuntimeAdapterSelector.select( + target: runtimeTarget, + ); + return _ScenarioRunnerAdapter( + ScenarioRunner( + adapter: adapter, + recordingController: recordingController, + targetDevice: targetDevice, + outputDirectory: Directory.current, + ), + ); + } +} + +class _ScenarioRunnerAdapter implements TestScenarioRunner { + const _ScenarioRunnerAdapter(this._runner); + + final ScenarioRunner _runner; + + @override + Future run( + Scenario scenario, { + RunStopPoint? stopPoint, + Set printDiagnostics = const {}, + void Function(StepProgressEvent event)? onProgress, + RunArtifactWriter? runArtifactWriter, + }) { + return _runner.run( + scenario, + stopPoint: stopPoint, + printDiagnostics: printDiagnostics, + onProgress: onProgress, + runArtifactWriter: runArtifactWriter, + ); + } +} diff --git a/lib/src/test_target_device_selection.dart b/lib/src/test_target_device_selection.dart new file mode 100644 index 0000000..a468bd3 --- /dev/null +++ b/lib/src/test_target_device_selection.dart @@ -0,0 +1,15 @@ +import 'target_app_launch_progress_renderer.dart'; + +/// Return the user-facing reason for the selected Target Device. +TargetDeviceSelectionReason? targetDeviceSelectionReason({ + required String? deviceSelector, + required bool recordingRequired, +}) { + if (deviceSelector != null) { + return TargetDeviceSelectionReason.explicit(selector: deviceSelector); + } + if (recordingRequired) { + return const TargetDeviceSelectionReason.autoSelectedForRecording(); + } + return null; +} diff --git a/packages/pilot_runtime/lib/pilot_runtime.dart b/packages/pilot_runtime/lib/pilot_runtime.dart new file mode 100644 index 0000000..847beff --- /dev/null +++ b/packages/pilot_runtime/lib/pilot_runtime.dart @@ -0,0 +1,11 @@ +/// Public API for app-side Flutter Pilot runtime access and client handshakes. +/// +/// Target App Packages call `PilotRuntimeBinding.ensureInitialized()` in debug +/// mode. Flutter Pilot tools use `PilotRuntimeClient` to verify that the debug +/// Runtime Target exposes the expected runtime protocol before running +/// Scenarios. +library; + +export 'src/pilot_runtime_binding.dart'; +export 'src/pilot_runtime_client.dart'; +export 'src/pilot_runtime_protocol.dart'; diff --git a/packages/pilot_runtime/lib/pilot_runtime_client.dart b/packages/pilot_runtime/lib/pilot_runtime_client.dart new file mode 100644 index 0000000..0633bbe --- /dev/null +++ b/packages/pilot_runtime/lib/pilot_runtime_client.dart @@ -0,0 +1,8 @@ +/// Public client-side API for Flutter Pilot runtime access. +/// +/// Dart CLI tools can import this library without loading the Flutter app-side +/// binding API from `pilot_runtime.dart`. +library; + +export 'src/pilot_runtime_client.dart'; +export 'src/pilot_runtime_protocol.dart'; diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart new file mode 100644 index 0000000..921eac3 --- /dev/null +++ b/packages/pilot_runtime/lib/src/pilot_runtime_binding.dart @@ -0,0 +1,86 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:developer'; + +import 'package:flutter/foundation.dart'; + +import 'pilot_runtime_protocol.dart'; + +/// Handles one app-side Flutter Pilot service extension request. +/// +/// The first runtime slice only needs a zero-argument handshake request. Later +/// capabilities can add their own typed request handlers without changing the +/// handshake registration contract. +typedef PilotRuntimeExtensionHandler = Future> Function(); + +/// Registers one Flutter Pilot service extension with the debug VM Service. +/// +/// Tests can provide a fake registrar to inspect the extension name and invoke +/// the handler without launching a Flutter app. Production callers omit the +/// registrar so the binding registers through `dart:developer`. +typedef PilotRuntimeExtensionRegistrar = + void Function(String extensionName, PilotRuntimeExtensionHandler handler); + +/// App-side hook that exposes Flutter Pilot runtime service extensions. +/// +/// Target App Packages call `ensureInitialized()` from app startup while +/// running in debug mode. The binding registers the protocol handshake +/// extension once per isolate and returns without side effects when debug mode +/// is disabled. +class PilotRuntimeBinding { + PilotRuntimeBinding._(); + + static bool _initialized = false; + + /// Register the Flutter Pilot debug runtime hook when debug mode is enabled. + /// + /// Args: + /// - `registerExtension`: Optional registrar used by tests to avoid touching + /// the VM Service. When omitted, the binding registers through + /// `dart:developer.registerExtension`. + /// - `debugMode`: Optional debug-mode override used by tests. When omitted, + /// Flutter's `kDebugMode` decides whether registration should happen. + /// + /// Returns without registering anything when debug mode is false. Repeated + /// calls in the same isolate are idempotent. + static void ensureInitialized({ + PilotRuntimeExtensionRegistrar? registerExtension, + bool? debugMode, + }) { + final bool shouldRegister = debugMode ?? kDebugMode; + if (!shouldRegister || _initialized) { + return; + } + + final PilotRuntimeExtensionRegistrar registrar = + registerExtension ?? _registerVmServiceExtension; + registrar(PilotRuntimeProtocol.handshakeExtension, _handleHandshake); + _initialized = true; + } + + /// Clear registration state for tests that verify binding behavior. + /// + /// This method does not unregister VM Service extensions. Tests should use a + /// fake registrar when they need repeated isolated assertions. + @visibleForTesting + static void debugResetForTesting() { + _initialized = false; + } + + static Future> _handleHandshake() async { + return PilotRuntimeHandshakeResponse.current().toJson(); + } + + static void _registerVmServiceExtension( + String extensionName, + PilotRuntimeExtensionHandler handler, + ) { + registerExtension(extensionName, ( + String method, + Map parameters, + ) async { + final Map payload = await handler(); + return ServiceExtensionResponse.result(jsonEncode(payload)); + }); + } +} diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_client.dart b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart new file mode 100644 index 0000000..6ba448d --- /dev/null +++ b/packages/pilot_runtime/lib/src/pilot_runtime_client.dart @@ -0,0 +1,296 @@ +import 'pilot_runtime_protocol.dart'; +import 'widget_tree_normalizer.dart'; + +/// Minimal VM Service surface used by `PilotRuntimeClient`. +/// +/// A concrete adapter calls service extensions on the target isolate and +/// returns decoded JSON maps. Unit tests provide fakes so the client can verify +/// protocol behavior without launching a Flutter app. +abstract interface class PilotRuntimeVmService { + /// Call a Flutter Pilot service extension on the Runtime Target. + /// + /// Args: + /// - `extensionName`: Full VM Service extension name, such as + /// `ext.flutter_pilot.runtime.handshake`. + /// - `parameters`: JSON-compatible VM Service extension arguments. Flutter + /// Inspector expects string values for its boolean-like flags, so callers + /// pass normalized strings instead of Dart booleans. + /// + /// Returns the decoded JSON object returned by the extension. Implementations + /// throw `PilotRuntimeServiceExtensionMissingException` when the extension is + /// not registered on the Runtime Target. + Future> callServiceExtension( + String extensionName, { + Map parameters = const {}, + }); +} + +/// Signals that a required Flutter Pilot service extension is not registered. +/// +/// VM Service adapters throw this exception when a Runtime Target does not have +/// `PilotRuntimeBinding.ensureInitialized()` active in the app isolate. +class PilotRuntimeServiceExtensionMissingException implements Exception { + /// Create a missing-extension failure for one VM Service extension name. + const PilotRuntimeServiceExtensionMissingException(this.extensionName); + + /// Full VM Service extension name that could not be called. + final String extensionName; + + @override + String toString() { + return 'PilotRuntimeServiceExtensionMissingException: ' + '$extensionName is not registered.'; + } +} + +/// Initialization failure category reported by `PilotRuntimeClient`. +/// +/// Flutter Pilot maps these values to run-level initialization failures before +/// any Scenario Step executes. +enum PilotRuntimeInitializationFailure { + /// The Runtime Target does not expose the Flutter Pilot runtime hook. + missingHook, + + /// The handshake response omits a capability required by this client. + missingCapability, + + /// The Runtime Target speaks a protocol version this client does not accept. + protocolVersionMismatch, + + /// The handshake response is not shaped like a Flutter Pilot protocol reply. + invalidHandshake, +} + +/// Clear initialization failure thrown by `PilotRuntimeClient.initialize()`. +/// +/// The exception identifies the failure category, includes a user-facing +/// message, and can carry the original VM Service error when one caused the +/// failure. +class PilotRuntimeInitializationException implements Exception { + /// Create a runtime initialization failure. + const PilotRuntimeInitializationException({ + required this.failure, + required this.message, + this.cause, + }); + + /// Machine-readable failure category for adapter and runner mapping. + final PilotRuntimeInitializationFailure failure; + + /// Human-readable explanation of the initialization failure. + final String message; + + /// Original lower-level error when the VM Service call failed. + final Object? cause; + + @override + String toString() { + return 'PilotRuntimeInitializationException: $message'; + } +} + +/// Widget Tree capture failure category reported by `PilotRuntimeClient`. +/// +/// Flutter Pilot can map these values to capture-step failures while preserving +/// the underlying VM Service or normalization error as context. +enum PilotRuntimeWidgetTreeCaptureFailure { + /// Inspector failed while configuring Target App Package pub root directories. + setPubRootDirectoriesFailed, + + /// Inspector failed while returning the root summary Widget Tree. + getRootWidgetTreeFailed, + + /// Inspector returned a tree shape that cannot be normalized safely. + invalidResponse, +} + +/// Clear failure thrown by `PilotRuntimeClient.captureWidgetTree()`. +/// +/// The exception identifies which capture stage failed and includes a message +/// suitable for run reports. The original lower-level error is retained when +/// one caused the failure. +class PilotRuntimeWidgetTreeCaptureException implements Exception { + /// Create a Widget Tree capture failure. + const PilotRuntimeWidgetTreeCaptureException({ + required this.failure, + required this.message, + this.cause, + }); + + /// Machine-readable Widget Tree capture failure category. + final PilotRuntimeWidgetTreeCaptureFailure failure; + + /// Human-readable explanation of the capture failure. + final String message; + + /// Original VM Service or normalization error when available. + final Object? cause; + + @override + String toString() { + return 'PilotRuntimeWidgetTreeCaptureException: $message'; + } +} + +/// Verified runtime session returned after a successful protocol handshake. +/// +/// The session records the accepted protocol version and the capabilities +/// reported by the Runtime Target. Flutter Pilot uses it as proof that the +/// target has the required debug hook before executing Scenario Steps. +class PilotRuntimeSession { + /// Create an initialized session from one accepted handshake response. + const PilotRuntimeSession({ + required this.protocolVersion, + required this.capabilities, + }); + + /// Runtime protocol version accepted by this client. + final int protocolVersion; + + /// Capabilities reported by the Runtime Target during handshake. + final Set capabilities; +} + +/// VM Service client that validates the Flutter Pilot runtime protocol. +/// +/// The client owns runtime initialization checks: it calls the app-side +/// handshake extension, accepts protocol version 1, and verifies required +/// capabilities before returning a session. +class PilotRuntimeClient { + /// Create a client backed by a VM Service extension caller. + const PilotRuntimeClient(this._vmService); + + final PilotRuntimeVmService _vmService; + + /// Initialize the Runtime Target through the protocol handshake. + /// + /// Returns a `PilotRuntimeSession` when the target exposes protocol version 1 + /// and all required capabilities. Throws + /// `PilotRuntimeInitializationException` for missing hooks, unsupported + /// protocol versions, missing capabilities, or malformed handshake data. + Future initialize() async { + final Map response = await _callHandshake(); + final PilotRuntimeHandshakeResponse handshake = _parseHandshakeResponse( + response, + ); + + if (handshake.protocolVersion != PilotRuntimeProtocol.version) { + throw PilotRuntimeInitializationException( + failure: PilotRuntimeInitializationFailure.protocolVersionMismatch, + message: + 'pilot_runtime protocol version ' + '${handshake.protocolVersion} is incompatible with client ' + 'version ${PilotRuntimeProtocol.version}.', + ); + } + + final Set missingCapabilities = PilotRuntimeProtocol + .requiredCapabilities + .difference(handshake.capabilities); + if (missingCapabilities.isNotEmpty) { + throw PilotRuntimeInitializationException( + failure: PilotRuntimeInitializationFailure.missingCapability, + message: + 'pilot_runtime handshake is missing required capabilities: ' + '${missingCapabilities.join(', ')}.', + ); + } + + return PilotRuntimeSession( + protocolVersion: handshake.protocolVersion, + capabilities: handshake.capabilities, + ); + } + + /// Capture a normalized Widget Tree from Flutter Inspector summary data. + /// + /// This method: + /// 1. configures Flutter Inspector pub root directories from `projectRoot` + /// 2. requests the root Widget Tree as a summary tree with previews + /// 3. normalizes Inspector node fields into Flutter Pilot Widget Tree v1 JSON + /// + /// Args: + /// - `projectRoot`: Target App Package root used by Inspector to mark local + /// project widgets. It is passed to + /// `ext.flutter.inspector.setPubRootDirectories` as `arg0`. + /// + /// Returns JSON with `schema`, `source`, and a normalized `root` node. Missing + /// child lists are returned as empty `children` arrays. + Future> captureWidgetTree({ + required String projectRoot, + }) async { + try { + await _vmService.callServiceExtension( + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension, + parameters: {'arg0': projectRoot}, + ); + } catch (error) { + throw PilotRuntimeWidgetTreeCaptureException( + failure: + PilotRuntimeWidgetTreeCaptureFailure.setPubRootDirectoriesFailed, + message: + 'Flutter Inspector could not set pub root directories for ' + 'Widget Tree capture: $error', + cause: error, + ); + } + + final Map rawTree; + try { + rawTree = await _vmService.callServiceExtension( + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension, + parameters: PilotRuntimeInspectorProtocol.summaryTreeParameters, + ); + } catch (error) { + throw PilotRuntimeWidgetTreeCaptureException( + failure: PilotRuntimeWidgetTreeCaptureFailure.getRootWidgetTreeFailed, + message: + 'Flutter Inspector could not return the root summary ' + 'Widget Tree: $error', + cause: error, + ); + } + + try { + return PilotRuntimeWidgetTreeNormalizer.normalize(rawTree); + } on FormatException catch (error) { + throw PilotRuntimeWidgetTreeCaptureException( + failure: PilotRuntimeWidgetTreeCaptureFailure.invalidResponse, + message: + 'Flutter Inspector returned an invalid Widget Tree: ' + '${error.message}', + cause: error, + ); + } + } + + Future> _callHandshake() async { + try { + return await _vmService.callServiceExtension( + PilotRuntimeProtocol.handshakeExtension, + ); + } on PilotRuntimeServiceExtensionMissingException catch (error) { + throw PilotRuntimeInitializationException( + failure: PilotRuntimeInitializationFailure.missingHook, + message: + 'PilotRuntimeBinding.ensureInitialized() is not registered ' + 'on the debug Runtime Target.', + cause: error, + ); + } + } + + PilotRuntimeHandshakeResponse _parseHandshakeResponse( + Map response, + ) { + try { + return PilotRuntimeHandshakeResponse.fromJson(response); + } on FormatException catch (error) { + throw PilotRuntimeInitializationException( + failure: PilotRuntimeInitializationFailure.invalidHandshake, + message: error.message, + cause: error, + ); + } + } +} diff --git a/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart b/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart new file mode 100644 index 0000000..1c94b5d --- /dev/null +++ b/packages/pilot_runtime/lib/src/pilot_runtime_protocol.dart @@ -0,0 +1,125 @@ +/// Protocol constants shared by the app-side hook and VM Service client. +/// +/// Version 1 exposes only the runtime handshake. Later runtime capabilities add +/// capability names while keeping the version gate explicit. +class PilotRuntimeProtocol { + PilotRuntimeProtocol._(); + + /// First accepted protocol version for `pilot_runtime`. + static const int version = 1; + + /// VM Service extension used to verify that the app-side hook is installed. + static const String handshakeExtension = + 'ext.flutter_pilot.runtime.handshake'; + + /// Capability name reported when the handshake extension is available. + static const String handshakeCapability = 'runtime.handshake'; + + /// Capabilities that this client requires before Scenario execution. + static const Set requiredCapabilities = {handshakeCapability}; +} + +/// Flutter Inspector service extension names and arguments used by the client. +/// +/// Widget Tree capture first configures the Target App Package pub root and +/// then requests the compact root Widget Tree with text previews. Flutter +/// Inspector expects boolean-like options as strings in VM Service parameters. +class PilotRuntimeInspectorProtocol { + PilotRuntimeInspectorProtocol._(); + + /// Extension that tells Inspector which package roots count as local project. + static const String setPubRootDirectoriesExtension = + 'ext.flutter.inspector.setPubRootDirectories'; + + /// Extension that returns the root diagnostics Widget Tree. + static const String getRootWidgetTreeExtension = + 'ext.flutter.inspector.getRootWidgetTree'; + + /// Deterministic Inspector group for Widget Tree capture objects. + static const String widgetTreeGroupName = 'pilot_runtime_widget_tree'; + + /// Parameters for the normalized Widget Tree source request. + static const Map summaryTreeParameters = { + 'groupName': widgetTreeGroupName, + 'isSummaryTree': 'true', + 'withPreviews': 'true', + 'fullDetails': 'false', + }; +} + +/// Versioned response returned by the app-side runtime handshake. +/// +/// The response contains the protocol version spoken by the Runtime Target and +/// the capability names that the client may use after initialization. +class PilotRuntimeHandshakeResponse { + /// Create a handshake response from decoded protocol fields. + const PilotRuntimeHandshakeResponse({ + required this.protocolVersion, + required this.capabilities, + }); + + /// Create the current app-side handshake response. + factory PilotRuntimeHandshakeResponse.current() { + return const PilotRuntimeHandshakeResponse( + protocolVersion: PilotRuntimeProtocol.version, + capabilities: PilotRuntimeProtocol.requiredCapabilities, + ); + } + + /// Decode one VM Service handshake payload. + /// + /// Args: + /// - `json`: Decoded JSON object returned by the Runtime Target. + /// + /// Returns a typed handshake response when `protocolVersion` is an integer + /// and `capabilities` is a list of strings. Throws `FormatException` when a + /// required field is missing or cannot be parsed. + factory PilotRuntimeHandshakeResponse.fromJson(Map json) { + final Object? protocolVersionValue = json['protocolVersion']; + if (protocolVersionValue is! int) { + throw const FormatException( + 'pilot_runtime handshake must include integer protocolVersion.', + ); + } + + final Object? capabilitiesValue = json['capabilities']; + if (capabilitiesValue is! List) { + throw const FormatException( + 'pilot_runtime handshake must include string capabilities.', + ); + } + + final Set capabilities = {}; + for (final Object? capability in capabilitiesValue) { + if (capability is! String) { + throw const FormatException( + 'pilot_runtime handshake capabilities must be strings.', + ); + } + capabilities.add(capability); + } + + return PilotRuntimeHandshakeResponse( + protocolVersion: protocolVersionValue, + capabilities: Set.unmodifiable(capabilities), + ); + } + + /// Runtime protocol version spoken by the Runtime Target. + final int protocolVersion; + + /// Capability names reported by the Runtime Target. + final Set capabilities; + + /// Encode this response for a VM Service extension result. + /// + /// Returns a JSON-compatible map with `protocolVersion` and `capabilities` + /// fields. The capability list is sorted for deterministic tests and logs. + Map toJson() { + final List sortedCapabilities = capabilities.toList()..sort(); + return { + 'protocolVersion': protocolVersion, + 'capabilities': sortedCapabilities, + }; + } +} diff --git a/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart b/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart new file mode 100644 index 0000000..e48cbbc --- /dev/null +++ b/packages/pilot_runtime/lib/src/widget_tree_normalizer.dart @@ -0,0 +1,132 @@ +/// Converts Flutter Inspector summary diagnostics into Widget Tree v1 JSON. +/// +/// The normalizer preserves only fields proven by calibration: +/// - display `description` +/// - `widgetRuntimeType` +/// - Inspector `valueId` renamed to `inspectorValueId` +/// - optional `createdByLocalProject` +/// - optional `textPreview` +/// - recursive `children` +class PilotRuntimeWidgetTreeNormalizer { + PilotRuntimeWidgetTreeNormalizer._(); + + /// Normalize one Inspector summary tree response. + /// + /// Args: + /// - `root`: Decoded root diagnostics node returned by + /// `ext.flutter.inspector.getRootWidgetTree`. + /// + /// Returns a JSON-compatible Widget Tree v1 map with `schema`, `source`, and + /// `root`. Throws `FormatException` when a required node field is missing or + /// has the wrong type. + static Map normalize(Map root) { + return { + 'schema': 'flutter_pilot.widget_tree.v1', + 'source': 'flutter_inspector_summary_tree', + 'root': _normalizeNode(root, 'root'), + }; + } + + /// Normalize one Inspector diagnostics node and its descendants. + /// + /// Args: + /// - `node`: Decoded Inspector node to convert. + /// - `path`: Human-readable path used in validation errors. + /// + /// Returns a node map with required normalized fields and always-present + /// `children`. Throws `FormatException` when a required field is absent or an + /// optional field has the wrong type. + static Map _normalizeNode( + Map node, + String path, + ) { + final String description = _requiredString(node, 'description', path); + final String widgetRuntimeType = _requiredString( + node, + 'widgetRuntimeType', + path, + ); + final String inspectorValueId = _requiredString(node, 'valueId', path); + + final Map normalized = { + 'description': description, + 'widgetRuntimeType': widgetRuntimeType, + 'inspectorValueId': inspectorValueId, + }; + + final Object? createdByLocalProject = node['createdByLocalProject']; + if (createdByLocalProject != null) { + if (createdByLocalProject is! bool) { + throw FormatException( + '$path.createdByLocalProject must be a boolean when present.', + ); + } + normalized['createdByLocalProject'] = createdByLocalProject; + } + + final Object? textPreview = node['textPreview']; + if (textPreview != null) { + if (textPreview is! String) { + throw FormatException( + '$path.textPreview must be a string when present.', + ); + } + normalized['textPreview'] = textPreview; + } + + normalized['children'] = _normalizeChildren(node, path); + return normalized; + } + + /// Read one required string field from an Inspector node. + /// + /// Args: + /// - `node`: Inspector node that should contain the field. + /// - `field`: Field name to read, such as `description` or `valueId`. + /// - `path`: Node path included in the failure message. + /// + /// Returns the non-empty string value. Throws `FormatException` when the + /// field is missing, empty, or not a string. + static String _requiredString( + Map node, + String field, + String path, + ) { + final Object? value = node[field]; + if (value is! String || value.isEmpty) { + throw FormatException('$path.$field must be a non-empty string.'); + } + return value; + } + + /// Normalize an Inspector node's child list. + /// + /// Args: + /// - `node`: Inspector node whose `children` field may be absent. + /// - `path`: Node path included in child validation errors. + /// + /// Returns a list of normalized child nodes. Missing `children` returns an + /// empty list so consumers can traverse one stable shape. + static List _normalizeChildren( + Map node, + String path, + ) { + final Object? rawChildren = node['children']; + if (rawChildren == null) { + return []; + } + if (rawChildren is! List) { + throw FormatException('$path.children must be a list when present.'); + } + + final List children = []; + for (int index = 0; index < rawChildren.length; index += 1) { + final Object? child = rawChildren[index]; + if (child is! Map) { + throw FormatException('$path.children[$index] must be an object.'); + } + children.add(_normalizeNode(child, '$path.children[$index]')); + } + return children; + } +} diff --git a/packages/pilot_runtime/pubspec.yaml b/packages/pilot_runtime/pubspec.yaml new file mode 100644 index 0000000..3fc0cc2 --- /dev/null +++ b/packages/pilot_runtime/pubspec.yaml @@ -0,0 +1,15 @@ +name: pilot_runtime +description: Flutter Pilot runtime hook and VM Service client package. +version: 0.1.0 +publish_to: none + +environment: + sdk: ^3.9.0 + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter diff --git a/packages/pilot_runtime/test/pilot_runtime_binding_test.dart b/packages/pilot_runtime/test/pilot_runtime_binding_test.dart new file mode 100644 index 0000000..11cae44 --- /dev/null +++ b/packages/pilot_runtime/test/pilot_runtime_binding_test.dart @@ -0,0 +1,45 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pilot_runtime/pilot_runtime.dart'; + +/// Verifies app-side registration behavior without touching the real VM Service. +void main() { + group('PilotRuntimeBinding', () { + setUp(PilotRuntimeBinding.debugResetForTesting); + + test('registers the debug runtime handshake', () async { + final List registeredExtensions = []; + PilotRuntimeExtensionHandler? registeredHandler; + + PilotRuntimeBinding.ensureInitialized( + debugMode: true, + registerExtension: + (String extensionName, PilotRuntimeExtensionHandler handler) { + registeredExtensions.add(extensionName); + registeredHandler = handler; + }, + ); + + expect(registeredExtensions, [ + PilotRuntimeProtocol.handshakeExtension, + ]); + expect(await registeredHandler!(), { + 'protocolVersion': 1, + 'capabilities': ['runtime.handshake'], + }); + }); + + test('is a no-op outside debug mode', () { + final List registeredExtensions = []; + + PilotRuntimeBinding.ensureInitialized( + debugMode: false, + registerExtension: + (String extensionName, PilotRuntimeExtensionHandler handler) { + registeredExtensions.add(extensionName); + }, + ); + + expect(registeredExtensions, isEmpty); + }); + }); +} diff --git a/packages/pilot_runtime/test/pilot_runtime_client_test.dart b/packages/pilot_runtime/test/pilot_runtime_client_test.dart new file mode 100644 index 0000000..5afa086 --- /dev/null +++ b/packages/pilot_runtime/test/pilot_runtime_client_test.dart @@ -0,0 +1,127 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pilot_runtime/pilot_runtime.dart'; + +/// Verifies runtime initialization through the public client handshake API. +void main() { + group('PilotRuntimeClient handshake', () { + test('accepts protocol version 1 with required capabilities', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + handshakeResponse: { + 'protocolVersion': 1, + 'capabilities': ['runtime.handshake'], + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final PilotRuntimeSession session = await client.initialize(); + + expect(session.protocolVersion, 1); + expect(session.capabilities, contains('runtime.handshake')); + expect(vmService.calledExtensions, [ + PilotRuntimeProtocol.handshakeExtension, + ]); + }); + + test('fails clearly when the runtime hook is missing', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + missingExtension: true, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.initialize(), + throwsA( + isA() + .having( + (PilotRuntimeInitializationException error) => error.failure, + 'failure', + PilotRuntimeInitializationFailure.missingHook, + ) + .having( + (PilotRuntimeInitializationException error) => error.message, + 'message', + contains('PilotRuntimeBinding.ensureInitialized()'), + ), + ), + ); + }); + + test('fails clearly when required capabilities are missing', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + handshakeResponse: { + 'protocolVersion': 1, + 'capabilities': [], + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.initialize(), + throwsA( + isA() + .having( + (PilotRuntimeInitializationException error) => error.failure, + 'failure', + PilotRuntimeInitializationFailure.missingCapability, + ) + .having( + (PilotRuntimeInitializationException error) => error.message, + 'message', + contains('runtime.handshake'), + ), + ), + ); + }); + + test('fails clearly when protocol versions do not match', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + handshakeResponse: { + 'protocolVersion': 2, + 'capabilities': ['runtime.handshake'], + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.initialize(), + throwsA( + isA() + .having( + (PilotRuntimeInitializationException error) => error.failure, + 'failure', + PilotRuntimeInitializationFailure.protocolVersionMismatch, + ) + .having( + (PilotRuntimeInitializationException error) => error.message, + 'message', + allOf(contains('2'), contains('1')), + ), + ), + ); + }); + }); +} + +class FakePilotRuntimeVmService implements PilotRuntimeVmService { + /// Create a fake VM Service that returns one handshake or simulates no hook. + FakePilotRuntimeVmService({ + this.handshakeResponse = const {}, + this.missingExtension = false, + }); + + final Map handshakeResponse; + final bool missingExtension; + final List calledExtensions = []; + + @override + Future> callServiceExtension( + String extensionName, { + Map parameters = const {}, + }) async { + calledExtensions.add(extensionName); + if (missingExtension) { + throw PilotRuntimeServiceExtensionMissingException(extensionName); + } + return handshakeResponse; + } +} diff --git a/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart b/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart new file mode 100644 index 0000000..fc1d6d0 --- /dev/null +++ b/packages/pilot_runtime/test/pilot_runtime_widget_tree_test.dart @@ -0,0 +1,300 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pilot_runtime/pilot_runtime.dart'; + +/// Verifies Widget Tree capture through the public `PilotRuntimeClient` API. +void main() { + group('PilotRuntimeClient Widget Tree capture', () { + test('sets pub roots and returns normalized summary tree JSON', () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{ + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + {'result': 'ok'}, + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension: + { + 'description': '[root]', + 'widgetRuntimeType': 'RootWidget', + 'valueId': 'inspector-1', + 'children': [ + { + 'description': "FilledButton-[<'submit-smoke'>]", + 'widgetRuntimeType': 'FilledButton', + 'valueId': 'inspector-2', + 'createdByLocalProject': true, + 'children': [ + { + 'description': 'Text', + 'widgetRuntimeType': 'Text', + 'valueId': 'inspector-3', + 'textPreview': 'Submit smoke', + }, + ], + }, + ], + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final Map widgetTree = await client.captureWidgetTree( + projectRoot: '/tmp/smoke_app', + ); + + expect(vmService.calls, [ + const FakeVmServiceCall( + extensionName: + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension, + parameters: {'arg0': '/tmp/smoke_app'}, + ), + const FakeVmServiceCall( + extensionName: + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension, + parameters: { + 'groupName': 'pilot_runtime_widget_tree', + 'isSummaryTree': 'true', + 'withPreviews': 'true', + 'fullDetails': 'false', + }, + ), + ]); + expect(widgetTree, { + 'schema': 'flutter_pilot.widget_tree.v1', + 'source': 'flutter_inspector_summary_tree', + 'root': { + 'description': '[root]', + 'widgetRuntimeType': 'RootWidget', + 'inspectorValueId': 'inspector-1', + 'children': [ + { + 'description': "FilledButton-[<'submit-smoke'>]", + 'widgetRuntimeType': 'FilledButton', + 'inspectorValueId': 'inspector-2', + 'createdByLocalProject': true, + 'children': [ + { + 'description': 'Text', + 'widgetRuntimeType': 'Text', + 'inspectorValueId': 'inspector-3', + 'textPreview': 'Submit smoke', + 'children': [], + }, + ], + }, + ], + }, + }); + expect( + (widgetTree['root']! as Map).containsKey('key'), + isFalse, + ); + }); + + test( + 'fails clearly when Inspector returns an invalid tree shape', + () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{ + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + {'result': 'ok'}, + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension: + { + 'description': 'Broken', + 'widgetRuntimeType': 'BrokenWidget', + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.captureWidgetTree(projectRoot: '/tmp/smoke_app'), + throwsA( + isA() + .having( + (PilotRuntimeWidgetTreeCaptureException error) => + error.failure, + 'failure', + PilotRuntimeWidgetTreeCaptureFailure.invalidResponse, + ) + .having( + (PilotRuntimeWidgetTreeCaptureException error) => + error.message, + 'message', + contains('valueId'), + ), + ), + ); + }, + ); + + test( + 'normalizes optional missing fields without adding key data', + () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{ + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + {'result': 'ok'}, + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension: + { + 'description': "FilledButton-[<'submit-smoke'>]", + 'widgetRuntimeType': 'FilledButton', + 'valueId': 'inspector-2', + }, + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + final Map widgetTree = await client.captureWidgetTree( + projectRoot: '/tmp/smoke_app', + ); + final Map root = + widgetTree['root']! as Map; + + expect(root, { + 'description': "FilledButton-[<'submit-smoke'>]", + 'widgetRuntimeType': 'FilledButton', + 'inspectorValueId': 'inspector-2', + 'children': [], + }); + expect(root.containsKey('key'), isFalse); + expect(root.containsKey('createdByLocalProject'), isFalse); + expect(root.containsKey('textPreview'), isFalse); + }, + ); + + test( + 'fails clearly when Inspector cannot set pub root directories', + () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{}, + failures: { + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + StateError('pub root rejected'), + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.captureWidgetTree(projectRoot: '/tmp/smoke_app'), + throwsA( + isA() + .having( + (PilotRuntimeWidgetTreeCaptureException error) => + error.failure, + 'failure', + PilotRuntimeWidgetTreeCaptureFailure + .setPubRootDirectoriesFailed, + ) + .having( + (PilotRuntimeWidgetTreeCaptureException error) => + error.message, + 'message', + contains('pub root'), + ), + ), + ); + }, + ); + + test( + 'fails clearly when Inspector cannot return the summary tree', + () async { + final FakePilotRuntimeVmService vmService = FakePilotRuntimeVmService( + responses: >{ + PilotRuntimeInspectorProtocol.setPubRootDirectoriesExtension: + {'result': 'ok'}, + }, + failures: { + PilotRuntimeInspectorProtocol.getRootWidgetTreeExtension: + StateError('tree unavailable'), + }, + ); + final PilotRuntimeClient client = PilotRuntimeClient(vmService); + + await expectLater( + client.captureWidgetTree(projectRoot: '/tmp/smoke_app'), + throwsA( + isA() + .having( + (PilotRuntimeWidgetTreeCaptureException error) => + error.failure, + 'failure', + PilotRuntimeWidgetTreeCaptureFailure.getRootWidgetTreeFailed, + ) + .having( + (PilotRuntimeWidgetTreeCaptureException error) => + error.message, + 'message', + contains('root summary Widget Tree'), + ), + ), + ); + }, + ); + }); +} + +class FakePilotRuntimeVmService implements PilotRuntimeVmService { + /// Create a fake VM Service that returns responses by extension name. + FakePilotRuntimeVmService({ + required this.responses, + this.failures = const {}, + }); + + final Map> responses; + final Map failures; + final List calls = []; + + @override + Future> callServiceExtension( + String extensionName, { + Map parameters = const {}, + }) async { + calls.add( + FakeVmServiceCall(extensionName: extensionName, parameters: parameters), + ); + final Object? failure = failures[extensionName]; + if (failure != null) { + throw failure; + } + return responses[extensionName] ?? {}; + } +} + +class FakeVmServiceCall { + /// Create a recorded VM Service extension call for assertions. + const FakeVmServiceCall({ + required this.extensionName, + required this.parameters, + }); + + final String extensionName; + final Map parameters; + + @override + bool operator ==(Object other) { + return other is FakeVmServiceCall && + other.extensionName == extensionName && + _mapsEqual(other.parameters, parameters); + } + + @override + int get hashCode { + return Object.hash(extensionName, _mapHash(parameters)); + } +} + +int _mapHash(Map map) { + final List keys = map.keys.toList()..sort(); + return Object.hashAll(keys.map((String key) => Object.hash(key, map[key]))); +} + +bool _mapsEqual(Map left, Map right) { + if (left.length != right.length) { + return false; + } + for (final MapEntry entry in left.entries) { + if (right[entry.key] != entry.value) { + return false; + } + } + return true; +} diff --git a/pubspec.lock b/pubspec.lock index 3ef720a..bcc277e 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -41,6 +41,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.2" + characters: + dependency: transitive + description: + name: characters + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b + url: "https://pub.dev" + source: hosted + version: "1.4.1" cli_config: dependency: transitive description: @@ -97,6 +105,11 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.0" + flutter: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" frontend_server_client: dependency: transitive description: @@ -161,14 +174,22 @@ packages: url: "https://pub.dev" source: hosted version: "0.12.19" + material_color_utilities: + dependency: transitive + description: + name: material_color_utilities + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" + url: "https://pub.dev" + source: hosted + version: "0.13.0" meta: dependency: transitive description: name: meta - sha256: c82594181e3312f3d0695fc95aaaf7758d75b8d4ae2bbecf223b9fd5109a059d + sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349" url: "https://pub.dev" source: hosted - version: "1.18.3" + version: "1.18.0" mime: dependency: transitive description: @@ -201,6 +222,13 @@ packages: url: "https://pub.dev" source: hosted version: "1.9.1" + pilot_runtime: + dependency: "direct main" + description: + path: "packages/pilot_runtime" + relative: true + source: path + version: "0.1.0" pool: dependency: transitive description: @@ -256,6 +284,11 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.0" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" source_map_stack_trace: dependency: transitive description: @@ -344,8 +377,16 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.0" - vm_service: + vector_math: dependency: transitive + description: + name: vector_math + sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b + url: "https://pub.dev" + source: hosted + version: "2.2.0" + vm_service: + dependency: "direct main" description: name: vm_service sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360" diff --git a/pubspec.yaml b/pubspec.yaml index 773986f..687994b 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,8 +12,11 @@ dependencies: args: ^2.7.0 crypto: ^3.0.7 path: ^1.9.1 + pilot_runtime: + path: packages/pilot_runtime screen_recorder: path: packages/screen_recorder + vm_service: ^15.2.0 yaml: ^3.1.3 dev_dependencies: file_testkit: ^1.0.0 diff --git a/test/cli_test.dart b/test/cli_test.dart index e052108..d9cdc82 100644 --- a/test/cli_test.dart +++ b/test/cli_test.dart @@ -185,6 +185,7 @@ steps: expect(result.stdout, contains('--print')); expect(result.stdout, contains('--json')); expect(result.stdout, isNot(contains('--html'))); + expect(result.stdout, isNot(contains('FLUTTER_PILOT_RUNTIME'))); }); test('test rejects unknown --until values before app launch', () async { diff --git a/test/pilot_runtime_adapter_test.dart b/test/pilot_runtime_adapter_test.dart new file mode 100644 index 0000000..93ff440 --- /dev/null +++ b/test/pilot_runtime_adapter_test.dart @@ -0,0 +1,128 @@ +import 'package:flutter_pilot/flutter_pilot.dart'; +import 'package:pilot_runtime/pilot_runtime_client.dart'; +import 'package:test/test.dart'; + +/// Verifies Flutter Pilot's Runtime Adapter backed by `pilot_runtime`. +void main() { + test( + 'maps pilot_runtime initialization failures to run initialization failures', + () async { + final PilotRuntimeAdapter adapter = PilotRuntimeAdapter( + client: _FakePilotRuntimeClient( + initializeFailure: const PilotRuntimeInitializationException( + failure: PilotRuntimeInitializationFailure.missingHook, + message: + 'PilotRuntimeBinding.ensureInitialized() is not registered ' + 'on the debug Runtime Target.', + ), + ), + projectRoot: '/target/app', + ); + + await expectLater( + adapter.initialize(), + throwsA( + isA() + .having( + (RuntimeOperationException error) => error.operation, + 'operation', + RuntimeOperation.initialize, + ) + .having( + (RuntimeOperationException error) => error.message, + 'message', + contains('PilotRuntimeBinding.ensureInitialized()'), + ), + ), + ); + }, + ); + + test( + 'maps pilot_runtime protocol failures to run initialization failures', + () async { + final PilotRuntimeAdapter adapter = PilotRuntimeAdapter( + client: _FakePilotRuntimeClient( + initializeFailure: const PilotRuntimeInitializationException( + failure: PilotRuntimeInitializationFailure.protocolVersionMismatch, + message: + 'pilot_runtime protocol version 2 is incompatible with ' + 'client version 1.', + ), + ), + projectRoot: '/target/app', + ); + + await expectLater( + adapter.initialize(), + throwsA( + isA() + .having( + (RuntimeOperationException error) => error.operation, + 'operation', + RuntimeOperation.initialize, + ) + .having( + (RuntimeOperationException error) => error.message, + 'message', + contains('protocol version 2'), + ), + ), + ); + }, + ); + + test('returns Widget Tree capture data from pilot_runtime', () async { + final _FakePilotRuntimeClient client = _FakePilotRuntimeClient( + widgetTree: { + 'schema': 'flutter_pilot.widget_tree.v1', + 'source': 'flutter_inspector.summary_tree', + 'root': {'widgetType': 'MaterialApp'}, + }, + ); + final PilotRuntimeAdapter adapter = PilotRuntimeAdapter( + client: client, + projectRoot: '/target/app', + ); + + final WidgetTreeCapture capture = await adapter.captureWidgetTree(); + + expect(client.projectRoots, ['/target/app']); + expect(capture.data, { + 'schema': 'flutter_pilot.widget_tree.v1', + 'source': 'flutter_inspector.summary_tree', + 'root': {'widgetType': 'MaterialApp'}, + }); + }); +} + +class _FakePilotRuntimeClient implements PilotRuntimeClient { + _FakePilotRuntimeClient({ + this.initializeFailure, + Map? widgetTree, + }) : widgetTree = widgetTree ?? {}; + + final PilotRuntimeInitializationException? initializeFailure; + final Map widgetTree; + final List projectRoots = []; + + @override + Future initialize() async { + final PilotRuntimeInitializationException? failure = initializeFailure; + if (failure != null) { + throw failure; + } + return const PilotRuntimeSession( + protocolVersion: 1, + capabilities: {'runtime.handshake'}, + ); + } + + @override + Future> captureWidgetTree({ + required String projectRoot, + }) async { + projectRoots.add(projectRoot); + return widgetTree; + } +} diff --git a/test/runtime_adapter_selection_test.dart b/test/runtime_adapter_selection_test.dart new file mode 100644 index 0000000..79355e1 --- /dev/null +++ b/test/runtime_adapter_selection_test.dart @@ -0,0 +1,49 @@ +import 'package:flutter_pilot/flutter_pilot.dart'; +import 'package:test/test.dart'; + +/// Verifies hidden Runtime Adapter selection without launching a Flutter app. +void main() { + test('uses mcp_flutter adapter when hidden runtime switch is omitted', () { + final RuntimeAdapter adapter = RuntimeAdapterSelector.select( + target: RuntimeTarget( + vmServiceUri: Uri.parse('ws://127.0.0.1:1234/example=/ws'), + ), + environment: const {}, + ); + + expect(adapter, isA()); + }); + + test('uses PilotRuntimeAdapter when hidden runtime switch selects it', () { + final RuntimeAdapter adapter = RuntimeAdapterSelector.select( + target: RuntimeTarget( + vmServiceUri: Uri.parse('ws://127.0.0.1:1234/example=/ws'), + ), + environment: const { + RuntimeAdapterSelector.environmentKey: 'pilot_runtime', + }, + ); + + expect(adapter, isA()); + }); + + test('rejects invalid hidden runtime switch values clearly', () { + expect( + () => RuntimeAdapterSelector.select( + target: RuntimeTarget( + vmServiceUri: Uri.parse('ws://127.0.0.1:1234/example=/ws'), + ), + environment: const { + RuntimeAdapterSelector.environmentKey: 'other_runtime', + }, + ), + throwsA( + isA().having( + (RuntimeAdapterSelectionException error) => error.message, + 'message', + contains('FLUTTER_PILOT_RUNTIME'), + ), + ), + ); + }); +} diff --git a/test/test_command_executor_test.dart b/test/test_command_executor_test.dart index a458e61..989ef71 100644 --- a/test/test_command_executor_test.dart +++ b/test/test_command_executor_test.dart @@ -768,6 +768,68 @@ steps: }); }); + test( + 'default executor reports invalid hidden runtime switch values', + () async { + await FileTestkit.runZoned(() async { + final FakeTargetAppProcess process = FakeTargetAppProcess(); + final FakeTargetAppProcessStarter starter = FakeTargetAppProcessStarter( + process, + ); + final DefaultTestCommandExecutor executor = DefaultTestCommandExecutor( + deviceDiscovery: FakeDeviceDiscovery(), + launcher: TargetAppLauncher(starter: starter), + runnerFactory: const ThrowingScenarioRunnerFactory( + RuntimeAdapterSelectionException( + 'Invalid FLUTTER_PILOT_RUNTIME value "other_runtime".', + ), + ), + ); + final Scenario scenario = Scenario( + name: 'invalid_runtime_switch', + steps: const [ + ScenarioStep( + index: 1, + action: TapAction(finder: Finder(byText: 'Continue')), + ), + ], + ); + + final Future reportFuture = executor.run( + TestCommandOptions( + scenario: scenario, + device: null, + flavor: null, + target: null, + stopPoint: null, + printDiagnostics: const {}, + jsonOutput: false, + ), + ); + process.emitStdout( + jsonEncode({ + 'event': 'app.debugPort', + 'params': { + 'wsUri': 'ws://127.0.0.1:1234/token=/ws', + }, + }), + ); + + await expectLater( + reportFuture, + throwsA( + isA().having( + (TestCommandException error) => error.message, + 'message', + contains('FLUTTER_PILOT_RUNTIME'), + ), + ), + ); + expect(process.stdinWrites, ['q\n']); + }); + }, + ); + test( 'default executor forwards Step progress events to the runner', () async { @@ -2591,6 +2653,21 @@ class FakeScenarioRunnerFactory implements TestScenarioRunnerFactory { } } +class ThrowingScenarioRunnerFactory implements TestScenarioRunnerFactory { + const ThrowingScenarioRunnerFactory(this.exception); + + final Exception exception; + + @override + TestScenarioRunner create({ + required RuntimeTarget runtimeTarget, + required TargetDevice? targetDevice, + required RecordingController? recordingController, + }) { + throw exception; + } +} + class QueueScenarioRunnerFactory implements TestScenarioRunnerFactory { QueueScenarioRunnerFactory(this.runners);