From ec69292fd8e05a3cee5dc68fd069980ab35f8145 Mon Sep 17 00:00:00 2001 From: drown0315 Date: Thu, 9 Jul 2026 20:47:38 +0800 Subject: [PATCH] fix: sync select mode widget selection --- .../inspector/flutter_inspector_client.dart | 111 ++++++++++++++++++ .../sessions/bridge_session_event_stream.dart | 17 +++ .../test/flutter_app_controller_test.dart | 9 ++ .../test/flutter_inspector_client_test.dart | 77 ++++++++++++ .../server/bridge_server_test_harness.dart | 14 +++ .../bridge_session_event_stream_test.dart | 27 ++++- .../src/app/useWorkbenchSelectionComments.ts | 28 ++++- .../bridge-events/bridgeSessionEvents.test.ts | 21 ++++ .../src/bridge-events/bridgeSessionEvents.ts | 9 ++ .../widget-tree/WidgetTreePanel.tsx | 35 +++++- .../widget-tree/widgetTreeSearch.test.ts | 10 ++ .../widget-tree/widgetTreeSearch.ts | 19 +++ apps/web/src/services/bridgeTypes.ts | 14 ++- 13 files changed, 384 insertions(+), 7 deletions(-) diff --git a/apps/bridge/lib/inspector/flutter_inspector_client.dart b/apps/bridge/lib/inspector/flutter_inspector_client.dart index 25ed2ec..91ce29f 100644 --- a/apps/bridge/lib/inspector/flutter_inspector_client.dart +++ b/apps/bridge/lib/inspector/flutter_inspector_client.dart @@ -66,6 +66,10 @@ abstract class FlutterInspectorClient { Stream watchSelectWidgetModeStatus( BridgeSession session, ); + + Stream watchWidgetSelectionStatus( + BridgeSession session, + ); } /// Flutter Inspector client backed by Dart VM Service extension calls. @@ -93,6 +97,8 @@ class VmServiceFlutterInspectorClient implements FlutterInspectorClient { final _selectWidgetMonitorServices = {}; final _selectWidgetModeControllers = >{}; + final _widgetSelectionControllers = + >{}; @override Future fetchRootWidgetTree(BridgeSession session) async { @@ -261,6 +267,57 @@ class VmServiceFlutterInspectorClient implements FlutterInspectorClient { 'monitor_update extension=${change.extension} enabled=$enabled', ); }); + _logger.info( + 'widget_selection session=${session.id} monitor_listen_inspect_stream start', + ); + try { + await vmService.listenToInspectorSelectionChanges(() async { + _logger.info( + 'widget_selection session=${session.id} inspect_event received', + ); + try { + final isolateId = await vmService.findMainIsolateId(); + final response = await vmService.callServiceExtension( + 'ext.flutter.inspector.getSelectedSummaryWidget', + isolateId: isolateId, + args: { + 'objectGroup': 'ask_ui_widget_tree', + }, + ); + final result = _decodeInspectorResult(response); + if (result is! Map) { + _logger.info( + 'widget_selection session=${session.id} monitor_ignore reason=selected_summary_not_object', + ); + return; + } + + final widgetId = result['valueId']?.toString().trim(); + if (widgetId == null || widgetId.isEmpty) { + _logger.info( + 'widget_selection session=${session.id} monitor_ignore reason=selected_summary_missing_value_id', + ); + return; + } + + _publishWidgetSelectionStatus(session.id, widgetId); + _logger.info( + 'widget_selection session=${session.id} monitor_update widget=$widgetId', + ); + } catch (error) { + _logger.info( + 'widget_selection session=${session.id} monitor_failed error=$error', + ); + } + }); + _logger.info( + 'widget_selection session=${session.id} monitor_listen_inspect_stream ready', + ); + } catch (error) { + _logger.info( + 'widget_selection session=${session.id} monitor_listen_inspect_stream failed error=$error', + ); + } }); } @@ -308,8 +365,36 @@ class VmServiceFlutterInspectorClient implements FlutterInspectorClient { controller.add(SelectWidgetModeStatus(enabled: enabled)); } + + @override + Stream watchWidgetSelectionStatus( + BridgeSession session, + ) { + unawaited(_ensureSelectWidgetModeMonitor(session)); + return _widgetSelectionController(session.id).stream; + } + + StreamController _widgetSelectionController( + String sessionId, + ) { + return _widgetSelectionControllers.putIfAbsent( + sessionId, + () => StreamController.broadcast(), + ); + } + + void _publishWidgetSelectionStatus(String sessionId, String widgetId) { + final controller = _widgetSelectionControllers[sessionId]; + if (controller == null || controller.isClosed) { + return; + } + + controller.add(WidgetSelectionStatus(widgetId: widgetId)); + } } +const flutterInspectorSelectionEventStream = EventStreams.kDebug; + Future> _fetchWidgetBoundsById( FlutterInspectorVmService vmService, { required String isolateId, @@ -530,6 +615,14 @@ class WidgetSelectionResult { } } +class WidgetSelectionStatus { + const WidgetSelectionStatus({ + required this.widgetId, + }); + + final String widgetId; +} + /// Cached Select Widget mode state observed from Flutter Inspector. /// /// It contains the last known `enabled` value reported by the app. When the @@ -657,6 +750,10 @@ abstract class FlutterInspectorVmService { void Function(FlutterServiceExtensionStateChange change) onChange, ); + Future listenToInspectorSelectionChanges( + FutureOr Function() onSelectionChanged, + ); + Future dispose(); } @@ -810,6 +907,20 @@ class VmServiceAdapter implements FlutterInspectorVmService { await _vmService.streamListen(EventStreams.kExtension); } + @override + Future listenToInspectorSelectionChanges( + FutureOr Function() onSelectionChanged, + ) async { + _vmService.onEvent(flutterInspectorSelectionEventStream).listen((event) { + if (event.kind != EventKind.kInspect) { + return; + } + unawaited(Future.sync(onSelectionChanged)); + }); + + await _vmService.streamListen(flutterInspectorSelectionEventStream); + } + @override Future dispose() => _vmService.dispose(); } diff --git a/apps/bridge/lib/sessions/bridge_session_event_stream.dart b/apps/bridge/lib/sessions/bridge_session_event_stream.dart index 41d4d8a..75c67b6 100644 --- a/apps/bridge/lib/sessions/bridge_session_event_stream.dart +++ b/apps/bridge/lib/sessions/bridge_session_event_stream.dart @@ -39,6 +39,7 @@ class BridgeSessionEventStream { Timer? heartbeat; StreamSubscription? selectWidgetSubscription; + StreamSubscription? widgetSelectionSubscription; StreamSubscription? chatSubscription; var streamClosed = false; var writeQueue = Future.value(); @@ -50,6 +51,7 @@ class BridgeSessionEventStream { streamClosed = true; heartbeat?.cancel(); await selectWidgetSubscription?.cancel(); + await widgetSelectionSubscription?.cancel(); await chatSubscription?.cancel(); _logger.info('events stream session=$sessionId close'); } @@ -121,6 +123,21 @@ class BridgeSessionEventStream { ); })); }); + widgetSelectionSubscription = + _inspectorClient.watchWidgetSelectionStatus(session).listen((status) { + unawaited(enqueueSseWrite(() { + transport.writeEvent( + event: 'bridge_session_event', + data: { + 'type': 'widget_selection_changed', + 'sessionId': sessionId, + 'payload': { + 'widgetId': status.widgetId, + }, + }, + ); + })); + }); chatSubscription = session.chat.events.listen((event) { unawaited(enqueueSseWrite(() { transport.writeEvent( diff --git a/apps/bridge/test/flutter_app_controller_test.dart b/apps/bridge/test/flutter_app_controller_test.dart index 393e831..4b886f3 100644 --- a/apps/bridge/test/flutter_app_controller_test.dart +++ b/apps/bridge/test/flutter_app_controller_test.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:ask_ui_bridge/app_controller/flutter_app_controller.dart'; import 'package:ask_ui_bridge/inspector/flutter_inspector_client.dart'; import 'package:ask_ui_bridge/logging/bridge_logger.dart'; @@ -334,6 +336,13 @@ class RecordingHotReloadVmService implements FlutterInspectorVmService { ) { throw UnimplementedError(); } + + @override + Future listenToInspectorSelectionChanges( + FutureOr Function() onSelectionChanged, + ) { + throw UnimplementedError(); + } } class RecordedRegisteredServiceWait { diff --git a/apps/bridge/test/flutter_inspector_client_test.dart b/apps/bridge/test/flutter_inspector_client_test.dart index 6538a0f..31cd798 100644 --- a/apps/bridge/test/flutter_inspector_client_test.dart +++ b/apps/bridge/test/flutter_inspector_client_test.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:ask_ui_bridge/inspector/flutter_inspector_client.dart'; import 'package:ask_ui_bridge/logging/bridge_logger.dart'; import 'package:ask_ui_bridge/sessions/session_store.dart'; @@ -390,6 +392,67 @@ void main() { ), ); }); + + test('publishes Widget Selection status from Inspector inspect events', + () async { + final vmService = RecordingFlutterInspectorVmService( + {'result': null}, + serviceExtensionResponses: { + 'ext.flutter.inspector.getSelectedSummaryWidget': [ + { + 'result': { + 'valueId': 'inspector-2', + 'description': 'PrimaryButton', + 'children': [], + }, + }, + ], + }, + ); + final logs = []; + final client = VmServiceFlutterInspectorClient( + vmServiceFactory: RecordingFlutterInspectorVmServiceFactory(vmService), + logger: BridgeLogger(write: logs.add), + ); + final session = BridgeSession( + id: 'session-1', + vmServiceUri: 'ws://127.0.0.1:12345/ws', + projectRoot: '/Users/example/app', + deviceId: '19271FDF6007TY', + ); + final emittedSelections = []; + final subscription = client + .watchWidgetSelectionStatus(session) + .listen(emittedSelections.add); + + await Future.delayed(Duration.zero); + await vmService.emitInspectorSelectionChange(); + await Future.delayed(Duration.zero); + await subscription.cancel(); + + expect( + emittedSelections.map((status) => status.widgetId), + ['inspector-2'], + ); + expect( + vmService.calls, + contains( + const FlutterInspectorVmServiceCall( + method: 'ext.flutter.inspector.getSelectedSummaryWidget', + isolateId: 'isolates/main', + args: { + 'objectGroup': 'ask_ui_widget_tree', + }, + ), + ), + ); + expect( + logs, + contains( + '[ask_ui_bridge] widget_selection session=session-1 monitor_update widget=inspector-2', + ), + ); + }); }); } @@ -423,6 +486,7 @@ class RecordingFlutterInspectorVmService implements FlutterInspectorVmService { final calls = []; final serviceExtensionStateListeners = []; + final inspectorSelectionListeners = Function()>[]; bool didFindMainIsolateId = false; bool disposed = false; @@ -493,6 +557,13 @@ class RecordingFlutterInspectorVmService implements FlutterInspectorVmService { serviceExtensionStateListeners.add(onChange); } + @override + Future listenToInspectorSelectionChanges( + FutureOr Function() onSelectionChanged, + ) async { + inspectorSelectionListeners.add(onSelectionChanged); + } + @override Future dispose() async { disposed = true; @@ -505,6 +576,12 @@ class RecordingFlutterInspectorVmService implements FlutterInspectorVmService { listener(change); } } + + Future emitInspectorSelectionChange() async { + for (final listener in inspectorSelectionListeners) { + await listener(); + } + } } class FlutterInspectorVmServiceCall { diff --git a/apps/bridge/test/server/bridge_server_test_harness.dart b/apps/bridge/test/server/bridge_server_test_harness.dart index 09cb90f..e59997f 100644 --- a/apps/bridge/test/server/bridge_server_test_harness.dart +++ b/apps/bridge/test/server/bridge_server_test_harness.dart @@ -313,6 +313,8 @@ class RecordingFlutterInspectorClient implements FlutterInspectorClient { final selectedWidgets = []; final _selectWidgetModeControllers = >{}; + final _widgetSelectionControllers = + >{}; Exception? failure; bool? selectWidgetModeStatus; @@ -394,6 +396,18 @@ class RecordingFlutterInspectorClient implements FlutterInspectorClient { ) .stream; } + + @override + Stream watchWidgetSelectionStatus( + BridgeSession session, + ) { + return _widgetSelectionControllers + .putIfAbsent( + session.id, + () => StreamController.broadcast(), + ) + .stream; + } } class RecordingFlutterAppController implements FlutterAppController { diff --git a/apps/bridge/test/sessions/bridge_session_event_stream_test.dart b/apps/bridge/test/sessions/bridge_session_event_stream_test.dart index ec13c65..96a6926 100644 --- a/apps/bridge/test/sessions/bridge_session_event_stream_test.dart +++ b/apps/bridge/test/sessions/bridge_session_event_stream_test.dart @@ -69,14 +69,16 @@ void main() { ]); }); - test('forwards Select Widget and Chat updates', () async { + test('forwards Select Widget, Widget Selection, and Chat updates', + () async { await eventStream().open(session: session, transport: transport); transport.events.clear(); inspectorClient.emitSelectWidgetMode(true); + inspectorClient.emitWidgetSelection('inspector-2'); session.chat.setAgentStatus(AgentStatus.agentReady); - await waitForEventCount(transport, 2); + await waitForEventCount(transport, 3); expect(transport.events, [ SseWrite( @@ -87,6 +89,14 @@ void main() { 'payload': {'enabled': true}, }, ), + SseWrite( + event: 'bridge_session_event', + data: { + 'type': 'widget_selection_changed', + 'sessionId': 'session-1', + 'payload': {'widgetId': 'inspector-2'}, + }, + ), SseWrite( event: 'bridge_session_event', data: { @@ -241,6 +251,8 @@ class SseWrite { class RecordingInspectorClient implements FlutterInspectorClient { final _controller = StreamController.broadcast(); + final _widgetSelectionController = + StreamController.broadcast(); bool? selectWidgetModeStatus; Object? failure; @@ -266,6 +278,17 @@ class RecordingInspectorClient implements FlutterInspectorClient { _controller.add(SelectWidgetModeStatus(enabled: enabled)); } + @override + Stream watchWidgetSelectionStatus( + BridgeSession session, + ) { + return _widgetSelectionController.stream; + } + + void emitWidgetSelection(String widgetId) { + _widgetSelectionController.add(WidgetSelectionStatus(widgetId: widgetId)); + } + @override Future fetchRootWidgetTree(BridgeSession session) { throw UnimplementedError(); diff --git a/apps/web/src/app/useWorkbenchSelectionComments.ts b/apps/web/src/app/useWorkbenchSelectionComments.ts index 904b845..ee8b53b 100644 --- a/apps/web/src/app/useWorkbenchSelectionComments.ts +++ b/apps/web/src/app/useWorkbenchSelectionComments.ts @@ -1,4 +1,4 @@ -import { useCallback, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { getInitialSelectionCommentState, getLocatableWidgetBoundsById, @@ -8,7 +8,10 @@ import { getSelectionCommentOverlayMarkers, type SelectionCommentAttachmentToken, } from '../selection-comments/selectionCommentState'; -import { selectWidgetById } from '../services/askUiBridgeClient'; +import { + selectWidgetById, + subscribeToBridgeSessionEvents, +} from '../services/askUiBridgeClient'; import type { WidgetTreeLoadState } from '../types/bridgeSession'; type UseWorkbenchSelectionCommentsOptions = { @@ -78,6 +81,27 @@ export function useWorkbenchSelectionComments({ setActiveSelectionCommentId(null); }, []); + useEffect(() => { + if (sessionId === null) { + return; + } + + const subscription = subscribeToBridgeSessionEvents(sessionId, (event) => { + if ( + event.sessionId !== sessionId || + event.type !== 'widget_selection_changed' + ) { + return; + } + + handleSelectedWidgetIdChange(event.payload.widgetId); + }); + + return () => { + subscription.close(); + }; + }, [handleSelectedWidgetIdChange, sessionId]); + const handleAttachmentTokenClick = useCallback( (token: SelectionCommentAttachmentToken) => { setActiveSelectionCommentId(token.id); diff --git a/apps/web/src/bridge-events/bridgeSessionEvents.test.ts b/apps/web/src/bridge-events/bridgeSessionEvents.test.ts index bb36524..74b2a5a 100644 --- a/apps/web/src/bridge-events/bridgeSessionEvents.test.ts +++ b/apps/web/src/bridge-events/bridgeSessionEvents.test.ts @@ -92,6 +92,27 @@ test('parses Chat bridge session events from EventSource', () => { ); }); +test('parses Widget Selection bridge session events from EventSource', () => { + assert.deepEqual( + parseBridgeSessionEvent( + JSON.stringify({ + type: 'widget_selection_changed', + sessionId: 'session-1', + payload: { + widgetId: 'inspector-2', + }, + }), + ), + { + type: 'widget_selection_changed', + sessionId: 'session-1', + payload: { + widgetId: 'inspector-2', + }, + }, + ); +}); + test('reports invalid bridge session events without dispatching them', () => { const invalidEvents: string[] = []; diff --git a/apps/web/src/bridge-events/bridgeSessionEvents.ts b/apps/web/src/bridge-events/bridgeSessionEvents.ts index 44001ca..df75e68 100644 --- a/apps/web/src/bridge-events/bridgeSessionEvents.ts +++ b/apps/web/src/bridge-events/bridgeSessionEvents.ts @@ -9,6 +9,7 @@ type LegacyBridgeSessionEvent = { type: | 'select_widget_mode_snapshot' | 'select_widget_mode_changed' + | 'widget_selection_changed' | 'chat_snapshot' | 'agent_status_changed' | 'chat_history_changed'; @@ -86,6 +87,7 @@ function isBridgeSessionEventType( return ( type === 'select_widget_mode_snapshot' || type === 'select_widget_mode_changed' || + type === 'widget_selection_changed' || type === 'chat_snapshot' || type === 'agent_status_changed' || type === 'chat_history_changed' @@ -100,6 +102,13 @@ function validateBridgeSessionEventPayload(event: LegacyBridgeSessionEvent) { return; } + if ( + event.type === 'widget_selection_changed' && + typeof event.payload.widgetId !== 'string' + ) { + throw new Error('Widget Selection event did not include widgetId'); + } + if ( (event.type === 'chat_snapshot' || event.type === 'agent_status_changed') && diff --git a/apps/web/src/components/widget-tree/WidgetTreePanel.tsx b/apps/web/src/components/widget-tree/WidgetTreePanel.tsx index 8f56494..46723e5 100644 --- a/apps/web/src/components/widget-tree/WidgetTreePanel.tsx +++ b/apps/web/src/components/widget-tree/WidgetTreePanel.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, + useRef, useState, type CSSProperties, type KeyboardEvent, @@ -16,6 +17,7 @@ import { collectExpandableNodeIds, } from './widgetTreeRows'; import { + collectAncestorNodeIdsForWidget, collectAncestorNodeIds, findWidgetTreeMatches, getNextMatchIndex, @@ -168,6 +170,7 @@ function WidgetTreeRows({ onSelectWidget: (widgetId: string) => void; }) { const allExpandableNodeIds = useMemo(() => collectExpandableNodeIds(root), [root]); + const selectedRowRef = useRef(null); const [expandedNodeIds, setExpandedNodeIds] = useState( () => new Set(allExpandableNodeIds), ); @@ -200,6 +203,32 @@ function WidgetTreeRows({ }); }, [searchExpandedAncestorIds]); + useEffect(() => { + const selectedAncestorNodeIds = collectAncestorNodeIdsForWidget( + root, + selectedWidgetId, + ); + if (selectedAncestorNodeIds.length === 0) { + return; + } + + setExpandedNodeIds((current) => { + const next = new Set(current); + + for (const nodeId of selectedAncestorNodeIds) { + next.add(nodeId); + } + + return next; + }); + }, [root, selectedWidgetId]); + + useEffect(() => { + selectedRowRef.current?.scrollIntoView({ + block: 'nearest', + }); + }, [selectedWidgetId, rows]); + function toggleNode(nodeId: string) { setExpandedNodeIds((current) => { const next = new Set(current); @@ -218,15 +247,16 @@ function WidgetTreeRows({
{rows.map((row) => { const isExpanded = row.hasChildren && expandedNodeIds.has(row.node.id); + const isSelected = row.node.id === selectedWidgetId; const icon = getWidgetTreeIcon(row.node.label); return (