diff --git a/client/components/xero/emulator-inspector-overlay.tsx b/client/components/xero/emulator-inspector-overlay.tsx new file mode 100644 index 00000000..119248b2 --- /dev/null +++ b/client/components/xero/emulator-inspector-overlay.tsx @@ -0,0 +1,266 @@ +"use client" + +import { useCallback, useRef } from "react" +import { Crosshair, ExternalLink, Search, X } from "lucide-react" +import { cn } from "@/lib/utils" +import type { ElementInfo, UseInspector } from "@/src/features/emulator/use-inspector" + +interface InspectorOverlayProps { + /** Device dimensions (in device pixels). */ + deviceWidth: number + deviceHeight: number + /** The inspector state from useInspector(). */ + inspector: UseInspector + /** Called when the user clicks an element to open source (RN only). */ + onOpenSource?: (file: string, line: number, column: number) => void + /** Called to search the project for a given AX identifier or label. */ + onSearchProject?: (query: string) => void +} + +/** + * Transparent overlay rendered on top of the emulator frame when inspect + * mode is active. Captures ALL pointer events (suppresses touch input to + * the device). Queries element-at-point via Metro or AXUIElement, renders + * highlight rectangle and info tooltip. + */ +export function InspectorOverlay({ + deviceWidth, + deviceHeight, + inspector, + onOpenSource, + onSearchProject, +}: InspectorOverlayProps) { + const overlayRef = useRef(null) + + // Convert pointer position to device pixels. + const toDeviceCoords = useCallback( + (e: React.PointerEvent) => { + const rect = e.currentTarget.getBoundingClientRect() + if (rect.width === 0 || rect.height === 0) return null + const nx = (e.clientX - rect.left) / rect.width + const ny = (e.clientY - rect.top) / rect.height + return { + x: Math.round(nx * deviceWidth), + y: Math.round(ny * deviceHeight), + } + }, + [deviceWidth, deviceHeight], + ) + + const handlePointerMove = useCallback( + (e: React.PointerEvent) => { + // Suppress event propagation so the device doesn't receive touch input. + e.stopPropagation() + e.preventDefault() + const coords = toDeviceCoords(e) + if (!coords) return + inspector.elementAt(coords.x, coords.y) + }, + [toDeviceCoords, inspector], + ) + + const handlePointerDown = useCallback( + (e: React.PointerEvent) => { + // Suppress all pointer events — inspect mode owns the viewport. + e.stopPropagation() + e.preventDefault() + }, + [], + ) + + const handleClick = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation() + e.preventDefault() + const el = inspector.hoveredElement + if (!el) return + + // RN apps: open source file. + if (el.source && onOpenSource) { + onOpenSource(el.source.file, el.source.line, el.source.column) + return + } + + // Native apps: search project for AX identifier or label. + const searchTerm = el.componentName || el.nativeType + if (searchTerm && onSearchProject) { + onSearchProject(searchTerm) + } + }, + [inspector.hoveredElement, onOpenSource, onSearchProject], + ) + + const el = inspector.hoveredElement + const isNativeMode = el != null && el.source == null + + return ( +
+ {/* Highlight rectangle */} + {el && deviceWidth > 0 && ( + + )} + + {/* Tooltip */} + {el && ( + + )} + + {/* Inspect mode badge */} +
+
+ + Inspect +
+ {isNativeMode && ( +
+ Accessibility view — source correlation is best-effort +
+ )} +
+
+ ) +} + +// MARK: - Subcomponents + +function HighlightBox({ + bounds, + deviceWidth, + deviceHeight, +}: { + bounds: { x: number; y: number; w: number; h: number } + deviceWidth: number + deviceHeight: number +}) { + // Convert device coords to percentage-based positioning. + const left = `${(bounds.x / deviceWidth) * 100}%` + const top = `${(bounds.y / deviceHeight) * 100}%` + const width = `${(bounds.w / deviceWidth) * 100}%` + const height = `${(bounds.h / deviceHeight) * 100}%` + + return ( +
+ ) +} + +function ElementTooltip({ + element, + hasSource, + hasSearch, + isNativeMode, +}: { + element: ElementInfo + hasSource: boolean + hasSearch: boolean + isNativeMode: boolean +}) { + return ( +
+ {/* Element name + native type */} +
+ + {isNativeMode ? ( + // Native: show AX role/type + <>{element.nativeType || "Unknown"} + ) : ( + // RN: show component name + <>{"<"}{element.componentName || "Unknown"}{" />"} + )} + + {!isNativeMode && element.nativeType && ( + ({element.nativeType}) + )} +
+ + {/* Label (for native AX elements) */} + {isNativeMode && element.componentName && element.componentName !== element.nativeType && ( +
+ Label: “{element.componentName}” +
+ )} + + {/* Bounds */} +
+ {element.bounds.w}×{element.bounds.h} at ({element.bounds.x}, {element.bounds.y}) +
+ + {/* Source location (RN only) */} + {element.source && ( +
+ + + {element.source.file.split("/").pop()}:{element.source.line} + + {hasSource && ( + (click to open) + )} +
+ )} + + {/* Search project (native AX — best-effort source correlation) */} + {isNativeMode && hasSearch && (element.componentName || element.nativeType) && ( +
+ + + Search project for “{element.componentName || element.nativeType}” + + (click) +
+ )} +
+ ) +} + +// MARK: - Inspect mode toggle button (for use in toolbar) + +export function InspectModeButton({ + active, + connected, + disabled, + onClick, +}: { + active: boolean + connected: boolean + disabled?: boolean + onClick: () => void +}) { + return ( + + ) +} diff --git a/client/components/xero/emulator-missing-sdk.tsx b/client/components/xero/emulator-missing-sdk.tsx index 99f459af..82c7b04c 100644 --- a/client/components/xero/emulator-missing-sdk.tsx +++ b/client/components/xero/emulator-missing-sdk.tsx @@ -22,6 +22,8 @@ interface SdkStatus { idbCompanionPresent: boolean supported: boolean axPermissionGranted: boolean + screenRecordingPermissionGranted: boolean + helperPresent: boolean } } @@ -194,6 +196,15 @@ export function EmulatorMissingSdk({ active = true, platform, onDismiss }: Props ) } + // Screen Recording permission is needed for the Swift helper's + // ScreenCaptureKit frame capture. Show this card when the helper + // binary is present but permission hasn't been granted yet. + if (status.ios.present && status.ios.helperPresent && !status.ios.screenRecordingPermissionGranted) { + return ( + + ) + } + // AX-permission case takes precedence when Xcode is fine but macOS // hasn't granted us the Accessibility right. We render a dedicated // card because it needs invoke-backed action buttons, not just hrefs. @@ -323,6 +334,105 @@ function IosAxPermissionCard({ ) } +function IosScreenRecordingPermissionCard({ + isProbing, + onDismiss, + onProbe, +}: { + isProbing: boolean + onDismiss?: () => void + onProbe: () => void +}) { + const [busy, setBusy] = useState(false) + + // Poll while this banner is mounted so it disappears within a second + // or two of the user granting the permission. + useEffect(() => { + if (!isTauri()) return + const handle = window.setInterval(() => { + onProbe() + }, 1500) + return () => window.clearInterval(handle) + }, [onProbe]) + + const handlePrompt = useCallback(async () => { + if (!isTauri()) return + setBusy(true) + try { + await invoke("emulator_ios_request_screen_recording_permission") + } finally { + setBusy(false) + onProbe() + } + }, [onProbe]) + + const handleOpenSettings = useCallback(async () => { + if (!isTauri()) return + await invoke("emulator_ios_open_screen_recording_settings") + }, []) + + return ( +
+
Screen Recording permission needed
+
+ Xero captures the iOS Simulator window for a smooth preview using + ScreenCaptureKit — macOS requires Screen Recording permission for this. + Without it, Xero falls back to slower screenshot polling. Enable Xero in + System Settings → Privacy & Security → Screen Recording. +
+
+ + + + {onDismiss ? ( + + ) : null} +
+
+ ) +} + function errorMessage(err: unknown): string { if (err && typeof err === "object" && "message" in err) { const message = (err as { message?: unknown }).message diff --git a/client/components/xero/emulator-sidebar.tsx b/client/components/xero/emulator-sidebar.tsx index 8cdc97a2..77c0f0dc 100644 --- a/client/components/xero/emulator-sidebar.tsx +++ b/client/components/xero/emulator-sidebar.tsx @@ -30,7 +30,9 @@ import { type EmulatorPlatform, } from "@/src/features/emulator/use-emulator-session" import { EmulatorHardwareStrip } from "./emulator-hardware-strip" +import { InspectorOverlay, InspectModeButton } from "./emulator-inspector-overlay" import { EmulatorMissingSdk } from "./emulator-missing-sdk" +import { useInspector } from "@/src/features/emulator/use-inspector" interface EmulatorSidebarProps { open: boolean @@ -233,6 +235,20 @@ export function EmulatorSidebar({ open, platform }: EmulatorSidebarProps) { const [selectedDeviceId, setSelectedDeviceId] = useState(null) const session = useEmulatorSession({ platform, active: sessionActive }) + const inspector = useInspector() + + // Auto-connect Metro inspector when streaming an iOS session. + useEffect(() => { + if (session.status.phase === "streaming" && platform === "ios" && !inspector.metroConnected) { + inspector.connect().catch(() => { + // Metro not running — silent, inspect button stays available. + }) + } + if (session.status.phase !== "streaming" && inspector.metroConnected) { + inspector.disconnect() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [session.status.phase, platform]) useEffect(() => { if (typeof window === "undefined") return @@ -317,8 +333,14 @@ export function EmulatorSidebar({ open, platform }: EmulatorSidebarProps) { const Icon = platform === "ios" ? Apple : Smartphone const handleStart = useCallback(() => { - if (!selectedDeviceId) return - void session.start(selectedDeviceId) + // Use selected device, or auto-pick first available device. + const deviceId = selectedDeviceId ?? session.devices[0]?.id + if (!deviceId) { + // Force refresh devices then retry. + void session.refreshDevices() + return + } + void session.start(deviceId) }, [selectedDeviceId, session]) const handleStop = useCallback(() => { @@ -441,7 +463,7 @@ export function EmulatorSidebar({ open, platform }: EmulatorSidebarProps) { {meta.label} -
+
{session.devices.length > 0 ? (