diff --git a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs index 850d35e..979415a 100644 --- a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs @@ -263,11 +263,16 @@ private void RegisterRoutes() _server.MapPost("/api/action/navigate", HandleNavigate); _server.MapPost("/api/action/resize", HandleResize); _server.MapPost("/api/action/scroll", HandleScroll); + _server.MapPost("/api/action/highlight", HandleHighlight); _server.MapGet("/api/logs", HandleLogs); _server.MapPost("/api/cdp", HandleCdp); _server.MapGet("/api/cdp/webviews", HandleCdpWebViews); _server.MapGet("/api/cdp/source", HandleCdpSource); + // Accessibility + _server.MapGet("/api/accessibility", HandleAccessibility); + _server.MapGet("/api/a11y/native-tree", HandleNativeA11yTree); + // Network monitoring _server.MapGet("/api/network", HandleNetworkList); _server.MapGet("/api/network/{id}", HandleNetworkDetail); @@ -332,6 +337,105 @@ private async Task HandleTree(HttpRequest request) return HttpResponse.Json(tree); } + private async Task HandleAccessibility(HttpRequest request) + { + if (_app == null) return HttpResponse.Error("Agent not bound to app"); + + var windowIndex = ParseWindowIndex(request); + var tree = await DispatchAsync(() => _treeWalker.WalkTree(_app, 0, windowIndex)); + + // Flatten the tree and extract only elements with accessibility info + var accessibilityElements = new List(); + var order = 0; + FlattenAccessibilityTree(tree, accessibilityElements, ref order); + + return HttpResponse.Json(new + { + totalElements = CountElements(tree), + accessibilityElements, + }); + } + + private static void FlattenAccessibilityTree(List elements, List result, ref int order) + { + foreach (var el in elements) + { + if (el.IsVisible && el.Accessibility != null) + { + var a11y = el.Accessibility; + // Include elements that a screen reader would visit: + // - explicitly marked as accessibility elements + // - have a label (announced text) + // - have a role (interactive or meaningful control) + // - have text content + if (a11y.IsAccessibilityElement + || !string.IsNullOrEmpty(a11y.Label) + || !string.IsNullOrEmpty(a11y.Role) + || !string.IsNullOrEmpty(el.Text)) + { + // Use WindowBounds, fall back to Bounds + var bounds = el.WindowBounds ?? el.Bounds; + a11y.Order = order++; + result.Add(new + { + el.Id, + el.Type, + el.AutomationId, + el.Text, + windowBounds = bounds, + accessibility = a11y, + }); + } + } + if (el.Children != null) + FlattenAccessibilityTree(el.Children, result, ref order); + } + } + + /// + /// Returns the native accessibility tree in the exact order the platform screen reader + /// (VoiceOver, TalkBack, Narrator) would visit elements. More accurate than the DFS + /// heuristic used by /api/accessibility, which assumes visual tree order == reading order. + /// + private async Task HandleNativeA11yTree(HttpRequest request) + { + if (_app == null) return HttpResponse.Error("Agent not bound to app"); + + var windowIndex = ParseWindowIndex(request) ?? 0; + + // Bounded MAUI tree walk (depth 100 is ample for any app) so we never + // block the main thread indefinitely on a large/complex visual tree. + // The walk only populates _elementIdToExternalId for native↔MAUI correlation. + var dispatchTask = DispatchAsync(() => + { + _treeWalker.WalkTree(_app, 100, windowIndex); + return _treeWalker.GetNativeA11yTree(_app, windowIndex); + }); + + if (await Task.WhenAny(dispatchTask, Task.Delay(TimeSpan.FromSeconds(15))) != dispatchTask) + return HttpResponse.Error("Native tree walk timed out (15s). The app UI thread may be blocked."); + + var entries = await dispatchTask; + return HttpResponse.Json(new + { + platform = DeviceInfo.Current.Platform.ToString(), + count = entries.Count, + entries, + }); + } + + private static int CountElements(List elements) + { + var count = 0; + foreach (var el in elements) + { + count++; + if (el.Children != null) + count += CountElements(el.Children); + } + return count; + } + private async Task HandleElement(HttpRequest request) { if (_app == null) return HttpResponse.Error("Agent not bound to app"); @@ -1223,6 +1327,100 @@ private async Task HandleFocus(HttpRequest request) return success ? HttpResponse.Ok("Focused") : HttpResponse.Error("Cannot focus element"); } + // Active highlight cleanup timer — cancelled when a new highlight replaces it + private CancellationTokenSource? _highlightCts; + + private async Task HandleHighlight(HttpRequest request) + { + if (_app == null) return HttpResponse.Error("Agent not bound to app"); + + var body = request.BodyAs(); + + // Cancel any previous highlight + _highlightCts?.Cancel(); + _highlightCts = null; + + // elementId = null, no fallback bounds → just clear + if (string.IsNullOrEmpty(body?.ElementId) && body?.X == null) + { + await DispatchAsync(() => { ClearNativeHighlight(); return true; }); + return HttpResponse.Ok("Cleared"); + } + + BoundsInfo? bounds = null; + + if (!string.IsNullOrEmpty(body?.ElementId)) + { + bounds = await DispatchAsync(async () => + { + var el = _treeWalker.GetElementById(body.ElementId, _app); + if (el is not VisualElement ve) return null; + + // Scroll element into view first (no animation), then resolve bounds — + // bounds must be read AFTER scroll so they reflect the post-scroll position. + if (body.ScrollIntoView) + { + try + { + Element? parent = ve.Parent; + while (parent != null) + { + if (parent is ScrollView sv) + { + await sv.ScrollToAsync(ve, ScrollToPosition.Center, animated: false); + break; + } + parent = parent.Parent; + } + } + catch { /* scroll is best-effort */ } + } + + return _treeWalker.ResolveWindowBoundsPublic(ve); + }); + } + + // Fallback: use raw window bounds supplied by caller (e.g. native-only tab bar items) + if (bounds == null && body?.X != null && body.Width != null && body.Height != null) + { + bounds = new BoundsInfo + { + X = body.X.Value, Y = body.Y ?? 0, + Width = body.Width.Value, Height = body.Height.Value, + }; + } + + if (bounds == null) + return HttpResponse.Error("Element not found or has no bounds"); + + var color = body?.Color ?? "#FF6B2FFF"; // default: accessible orange with full alpha + await DispatchAsync(() => { ShowNativeHighlight(bounds, color); return true; }); + + // Auto-clear after durationMs (default 3s) + var cts = new CancellationTokenSource(); + _highlightCts = cts; + var durationMs = body?.DurationMs > 0 ? body.DurationMs : 3000; + _ = Task.Delay(durationMs, cts.Token).ContinueWith(async t => + { + if (!t.IsCanceled) + await DispatchAsync(() => { ClearNativeHighlight(); return true; }); + }); + + return HttpResponse.Ok("Highlighted"); + } + + /// + /// Override in platform-specific subclasses to draw a visible highlight overlay + /// at the given window-relative bounds. Called on the UI thread. + /// + protected virtual void ShowNativeHighlight(BoundsInfo bounds, string color) { } + + /// + /// Override in platform-specific subclasses to remove the highlight overlay. + /// Called on the UI thread. + /// + protected virtual void ClearNativeHighlight() { } + private async Task HandleNavigate(HttpRequest request) { if (_app == null) return HttpResponse.Error("Agent not bound to app"); @@ -1904,6 +2102,26 @@ public class ActionRequest public string? ElementId { get; set; } } +public class HighlightRequest +{ + /// Element to highlight. Null or empty to just clear the current highlight. + public string? ElementId { get; set; } + /// Border color in #AARRGGBB or #RRGGBB format. Defaults to accessible orange. + public string? Color { get; set; } + /// Auto-clear after this many milliseconds. 0 = use default (2000ms). + public int DurationMs { get; set; } + /// + /// Fallback window-logical bounds used when ElementId is null or not found. + /// Allows highlighting native-only elements (e.g. tab bar buttons) that have no MAUI element ID. + /// + public double? X { get; set; } + public double? Y { get; set; } + public double? Width { get; set; } + public double? Height { get; set; } + /// When true, also scroll the element into view before highlighting. + public bool ScrollIntoView { get; set; } = true; +} + public class FillRequest { public string? ElementId { get; set; } diff --git a/src/MauiDevFlow.Agent.Core/ElementInfo.cs b/src/MauiDevFlow.Agent.Core/ElementInfo.cs index 77aa42b..1e0f867 100644 --- a/src/MauiDevFlow.Agent.Core/ElementInfo.cs +++ b/src/MauiDevFlow.Agent.Core/ElementInfo.cs @@ -64,6 +64,28 @@ public class ElementInfo [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? NativeProperties { get; set; } + /// + /// Effective text color as rendered by the platform (after theme/style resolution). + /// Format: #AARRGGBB hex string. Only populated for text-rendering elements. + /// Use this instead of fetching TextColor via /api/property — MAUI TextColor is often + /// null (theme default), while this reflects the actual rendered color. + /// + [JsonPropertyName("effectiveTextColor")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? EffectiveTextColor { get; set; } + + /// + /// Effective background color as rendered by the platform (after theme/style resolution). + /// Format: #AARRGGBB hex string. + /// + [JsonPropertyName("effectiveBackgroundColor")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? EffectiveBackgroundColor { get; set; } + + [JsonPropertyName("accessibility")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public AccessibilityInfo? Accessibility { get; set; } + [JsonPropertyName("children")] public List? Children { get; set; } } @@ -86,6 +108,104 @@ public class BoundsInfo public double Height { get; set; } } +/// +/// Native accessibility properties extracted from the platform accessibility APIs. +/// +public class AccessibilityInfo +{ + [JsonPropertyName("isAccessibilityElement")] + public bool IsAccessibilityElement { get; set; } + + [JsonPropertyName("label")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Label { get; set; } + + [JsonPropertyName("hint")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Hint { get; set; } + + [JsonPropertyName("value")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Value { get; set; } + + [JsonPropertyName("role")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Role { get; set; } + + [JsonPropertyName("traits")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Traits { get; set; } + + [JsonPropertyName("isEnabled")] + public bool IsEnabled { get; set; } = true; + + [JsonPropertyName("isFocusable")] + public bool IsFocusable { get; set; } + + [JsonPropertyName("isFocused")] + public bool IsFocused { get; set; } + + [JsonPropertyName("isHeading")] + public bool IsHeading { get; set; } + + [JsonPropertyName("order")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Order { get; set; } + + [JsonPropertyName("childCount")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? ChildCount { get; set; } + + [JsonPropertyName("liveRegion")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? LiveRegion { get; set; } +} + +/// +/// An element in the native screen reader traversal order, as the platform accessibility +/// framework would present it (VoiceOver on iOS, TalkBack on Android, etc.). +/// +public class NativeScreenReaderEntry +{ + [JsonPropertyName("order")] + public int Order { get; set; } + + [JsonPropertyName("label")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Label { get; set; } + + [JsonPropertyName("hint")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Hint { get; set; } + + [JsonPropertyName("value")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Value { get; set; } + + [JsonPropertyName("role")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Role { get; set; } + + [JsonPropertyName("traits")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Traits { get; set; } + + [JsonPropertyName("isHeading")] + public bool IsHeading { get; set; } + + [JsonPropertyName("windowBounds")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public BoundsInfo? WindowBounds { get; set; } + + [JsonPropertyName("elementId")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ElementId { get; set; } + + [JsonPropertyName("nativeType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? NativeType { get; set; } +} + /// /// Metadata for a registered CDP-capable WebView. /// diff --git a/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs b/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs index edf0316..7f1cc94 100644 --- a/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs +++ b/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs @@ -283,6 +283,44 @@ protected virtual void PopulateSyntheticNativeInfo(ElementInfo info, object mark /// public BoundsInfo? ResolveWindowBoundsPublic(VisualElement ve) => ResolveWindowBounds(ve); + /// + /// Walks the native platform accessibility tree (VoiceOver on iOS, TalkBack on Android, + /// Narrator on Windows, VoiceOver on macOS) and returns elements in the exact order + /// a screen reader would visit them. Override in platform-specific subclasses. + /// Must be called after WalkTree so that _elementIdToExternalId is populated. + /// + public virtual List GetNativeA11yTree(Application app, int windowIndex) + => new(); + + /// + /// Builds a map from native platform view object → MAUI external element ID. + /// Uses the _elementIdToExternalId map populated by a prior WalkTree call. + /// Call this from GetNativeA11yTree implementations to correlate native elements back to MAUI IDs. + /// + protected Dictionary BuildNativeViewToIdMap(Application app, int windowIndex) + { + var map = new Dictionary(ReferenceEqualityComparer.Instance); + var targetWindow = windowIndex >= 0 && windowIndex < app.Windows.Count + ? app.Windows[windowIndex] + : app.Windows.FirstOrDefault(); + if (targetWindow is IVisualTreeElement windowElement) + BuildNativeViewMapRecursive(windowElement, map); + return map; + } + + private void BuildNativeViewMapRecursive(IVisualTreeElement element, Dictionary map, int depth = 0) + { + if (depth > 120) return; + if (element is VisualElement ve) + { + var platformView = ve.Handler?.PlatformView; + if (platformView != null && _elementIdToExternalId.TryGetValue(ve.Id, out var externalId)) + map[platformView] = externalId; + } + foreach (var child in element.GetVisualChildren()) + BuildNativeViewMapRecursive(child, map, depth + 1); + } + /// /// Walks the visual tree starting from the application's windows. /// When windowIndex is null, walks all windows. Otherwise walks only the specified window. @@ -1222,6 +1260,9 @@ private ElementInfo CreateElementInfo(IVisualTreeElement element, string id, str // Populate native view info from handler PopulateNativeInfo(info, ve); + // Populate native accessibility info + PopulateAccessibilityInfo(info, ve); + // Resolve window-absolute bounds via platform-native APIs info.WindowBounds = ResolveWindowBounds(ve); } @@ -1315,6 +1356,115 @@ protected virtual void PopulateNativeInfo(ElementInfo info, VisualElement ve) } } + /// + /// Populates native accessibility properties from platform APIs. + /// Base implementation uses MAUI SemanticProperties; platform overrides add native data. + /// + protected virtual void PopulateAccessibilityInfo(ElementInfo info, VisualElement ve) + { + try + { + var a11y = new AccessibilityInfo(); + var hasData = false; + + // MAUI SemanticProperties (cross-platform) + var desc = SemanticProperties.GetDescription(ve); + if (!string.IsNullOrEmpty(desc)) + { + a11y.Label = desc; + hasData = true; + } + + var hint = SemanticProperties.GetHint(ve); + if (!string.IsNullOrEmpty(hint)) + { + a11y.Hint = hint; + hasData = true; + } + + var headingLevel = SemanticProperties.GetHeadingLevel(ve); + if (headingLevel != SemanticHeadingLevel.None) + { + a11y.IsHeading = true; + hasData = true; + } + + // Determine focusability from element type (interactive elements are focusable) + if (ve is View) + { + var isFocusable = ve is Button or ImageButton or Entry or Editor + or Switch or CheckBox or RadioButton or Slider or Stepper + or Picker or DatePicker or TimePicker or SearchBar; + a11y.IsFocusable = isFocusable; + if (isFocusable) hasData = true; + } + + a11y.IsEnabled = ve.IsEnabled; + a11y.IsFocused = ve.IsFocused; + + // Determine role from element type + a11y.Role = ve switch + { + Button => "Button", + ImageButton => "Button", + Label => "StaticText", + Entry => "TextField", + Editor => "TextArea", + Switch => "Switch", + CheckBox => "CheckBox", + RadioButton => "RadioButton", + Slider => "Slider", + Stepper => "Stepper", + Picker => "ComboBox", + DatePicker => "DatePicker", + TimePicker => "TimePicker", + SearchBar => "SearchField", + Image => "Image", + ProgressBar => "ProgressBar", + ActivityIndicator => "BusyIndicator", + _ => null + }; + if (a11y.Role != null) hasData = true; + + // Determine if this is an accessibility element (leaf that screen reader should visit) + a11y.IsAccessibilityElement = IsAccessibilityLeaf(ve); + if (a11y.IsAccessibilityElement) hasData = true; + + if (hasData) + info.Accessibility = a11y; + } + catch + { + // Accessibility info is best-effort + } + } + + /// + /// Determines whether an element is a leaf accessibility element (one the screen reader would stop on). + /// + private static bool IsAccessibilityLeaf(VisualElement ve) + { + // Elements with explicit semantic description are accessibility elements + if (!string.IsNullOrEmpty(SemanticProperties.GetDescription(ve))) + return true; + + // Interactive controls are accessibility elements + if (ve is Button or ImageButton or Entry or Editor or Switch or CheckBox + or RadioButton or Slider or Stepper or Picker or DatePicker or TimePicker + or SearchBar) + return true; + + // Labels with text + if (ve is Label label && !string.IsNullOrEmpty(label.Text)) + return true; + + // Images with AutomationId (likely meaningful) + if (ve is Image && !string.IsNullOrEmpty(ve.AutomationId)) + return true; + + return false; + } + /// /// Queries elements matching a CSS selector string. /// Walks the full tree, then runs the CSS selector engine against it. diff --git a/src/MauiDevFlow.Agent/DevFlowAgentService.cs b/src/MauiDevFlow.Agent/DevFlowAgentService.cs index c991b0b..59cdcaa 100644 --- a/src/MauiDevFlow.Agent/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent/DevFlowAgentService.cs @@ -450,4 +450,200 @@ await wv2.CoreWebView2.CapturePreviewAsync( return null; } #endif + + // ----------------------------------------------------------------------- + // Native Highlight Overlay + // Drawn directly on the native window layer — pixel-perfect because it uses + // the same coordinate origin as ResolveWindowBounds. + // ----------------------------------------------------------------------- + +#if IOS || MACCATALYST + private UIKit.UIView? _highlightView; + + protected override void ShowNativeHighlight(BoundsInfo bounds, string color) + { + ClearNativeHighlight(); + var window = UIKit.UIApplication.SharedApplication.KeyWindow + ?? Microsoft.Maui.Controls.Application.Current?.Windows.FirstOrDefault()?.Handler?.PlatformView as UIKit.UIWindow; + var root = window?.RootViewController?.View; + if (root == null) return; + + var parsed = ParseHexColor(color); + var highlight = new UIKit.UIView(new CoreGraphics.CGRect(bounds.X, bounds.Y, bounds.Width, bounds.Height)) + { + BackgroundColor = UIKit.UIColor.Clear, + UserInteractionEnabled = false, + AccessibilityElementsHidden = true, + }; + highlight.Layer.BorderColor = new CoreGraphics.CGColor( + (nfloat)((parsed.r) / 255.0), + (nfloat)((parsed.g) / 255.0), + (nfloat)((parsed.b) / 255.0), + (nfloat)((parsed.a) / 255.0)); + highlight.Layer.BorderWidth = 3f; + // Subtle fill + highlight.BackgroundColor = UIKit.UIColor.FromRGBA( + (byte)parsed.r, (byte)parsed.g, (byte)parsed.b, (byte)Math.Min(40, parsed.a)); + + root.AddSubview(highlight); + _highlightView = highlight; + } + + protected override void ClearNativeHighlight() + { + _highlightView?.RemoveFromSuperview(); + _highlightView = null; + } +#elif ANDROID + private Android.Views.View? _highlightView; + + protected override void ShowNativeHighlight(BoundsInfo bounds, string color) + { + ClearNativeHighlight(); + var activity = Microsoft.Maui.Controls.Application.Current?.Windows.FirstOrDefault()?.Handler?.PlatformView as Android.App.Activity; + if (activity?.Window?.DecorView is not Android.Views.ViewGroup decorView) return; + + var density = activity.Resources?.DisplayMetrics?.Density ?? 1f; + var parsed = ParseHexColor(color); + var borderColor = Android.Graphics.Color.Argb(parsed.a, parsed.r, parsed.g, parsed.b); + var fillColor = Android.Graphics.Color.Argb(Math.Min(40, parsed.a), parsed.r, parsed.g, parsed.b); + + var drawable = new Android.Graphics.Drawables.GradientDrawable(); + drawable.SetShape(Android.Graphics.Drawables.ShapeType.Rectangle); + drawable.SetColor(fillColor); + drawable.SetStroke((int)(3 * density), borderColor); + + var view = new Android.Views.View(activity) { Background = drawable }; + view.Focusable = false; + view.ImportantForAccessibility = Android.Views.ImportantForAccessibility.No; + + var lp = new Android.Widget.FrameLayout.LayoutParams( + (int)(bounds.Width * density), + (int)(bounds.Height * density)) + { + LeftMargin = (int)(bounds.X * density), + TopMargin = (int)(bounds.Y * density), + }; + decorView.AddView(view, lp); + _highlightView = view; + } + + protected override void ClearNativeHighlight() + { + if (_highlightView?.Parent is Android.Views.ViewGroup parent) + parent.RemoveView(_highlightView); + _highlightView = null; + } +#elif WINDOWS + private Microsoft.UI.Xaml.Shapes.Rectangle? _highlightView; + + protected override void ShowNativeHighlight(BoundsInfo bounds, string color) + { + ClearNativeHighlight(); + var window = Microsoft.Maui.Controls.Application.Current?.Windows.FirstOrDefault()?.Handler?.PlatformView as Microsoft.UI.Xaml.Window; + if (window?.Content is not Microsoft.UI.Xaml.FrameworkElement root) return; + + // Find or create a Canvas overlay at root level + var canvas = FindOrCreateHighlightCanvas(root); + if (canvas == null) return; + + var parsed = ParseHexColor(color); + var rect = new Microsoft.UI.Xaml.Shapes.Rectangle + { + Width = bounds.Width, + Height = bounds.Height, + Fill = new Microsoft.UI.Xaml.Media.SolidColorBrush( + Windows.UI.Color.FromArgb((byte)Math.Min(40, parsed.a), (byte)parsed.r, (byte)parsed.g, (byte)parsed.b)), + Stroke = new Microsoft.UI.Xaml.Media.SolidColorBrush( + Windows.UI.Color.FromArgb((byte)parsed.a, (byte)parsed.r, (byte)parsed.g, (byte)parsed.b)), + StrokeThickness = 3, + IsHitTestVisible = false, + }; + Microsoft.UI.Xaml.Controls.Canvas.SetLeft(rect, bounds.X); + Microsoft.UI.Xaml.Controls.Canvas.SetTop(rect, bounds.Y); + canvas.Children.Add(rect); + _highlightView = rect; + } + + protected override void ClearNativeHighlight() + { + if (_highlightView?.Parent is Microsoft.UI.Xaml.Controls.Canvas canvas) + canvas.Children.Remove(_highlightView); + _highlightView = null; + } + + private static Microsoft.UI.Xaml.Controls.Canvas? FindOrCreateHighlightCanvas(Microsoft.UI.Xaml.FrameworkElement root) + { + // Look for existing highlight canvas in the visual tree root + if (root is Microsoft.UI.Xaml.Controls.Panel panel) + { + foreach (var child in panel.Children) + { + if (child is Microsoft.UI.Xaml.Controls.Canvas c && c.Name == "__devflow_highlight") + return c; + } + var canvas = new Microsoft.UI.Xaml.Controls.Canvas + { + Name = "__devflow_highlight", + IsHitTestVisible = false, + Width = root.ActualWidth, + Height = root.ActualHeight, + }; + panel.Children.Add(canvas); + return canvas; + } + return null; + } +#elif MACOS + private AppKit.NSView? _highlightView; + + protected override void ShowNativeHighlight(BoundsInfo bounds, string color) + { + ClearNativeHighlight(); + var nsWindow = AppKit.NSApplication.SharedApplication.KeyWindow + ?? (Microsoft.Maui.Controls.Application.Current?.Windows.FirstOrDefault()?.Handler?.PlatformView as AppKit.NSWindow); + if (nsWindow?.ContentView == null) return; + + var parsed = ParseHexColor(color); + var highlight = new AppKit.NSView(new CoreGraphics.CGRect( + bounds.X, + nsWindow.ContentView.Bounds.Height - bounds.Y - bounds.Height, // flip Y + bounds.Width, + bounds.Height)) + { + WantsLayer = true, + }; + highlight.Layer!.BorderColor = new CoreGraphics.CGColor( + (nfloat)(parsed.r / 255.0), (nfloat)(parsed.g / 255.0), + (nfloat)(parsed.b / 255.0), (nfloat)(parsed.a / 255.0)); + highlight.Layer.BorderWidth = 3f; + highlight.Layer.BackgroundColor = new CoreGraphics.CGColor( + (nfloat)(parsed.r / 255.0), (nfloat)(parsed.g / 255.0), + (nfloat)(parsed.b / 255.0), (nfloat)(Math.Min(40, parsed.a) / 255.0)); + + nsWindow.ContentView.AddSubview(highlight); + _highlightView = highlight; + } + + protected override void ClearNativeHighlight() + { + _highlightView?.RemoveFromSuperview(); + _highlightView = null; + } +#endif + + /// Parses #AARRGGBB or #RRGGBB hex color string into components. + private static (int a, int r, int g, int b) ParseHexColor(string hex) + { + hex = hex.TrimStart('#'); + if (hex.Length == 6) + hex = "FF" + hex; + if (hex.Length != 8) + return (255, 0x10, 0x7C, 0x10); // fallback green + return ( + Convert.ToInt32(hex[..2], 16), + Convert.ToInt32(hex[2..4], 16), + Convert.ToInt32(hex[4..6], 16), + Convert.ToInt32(hex[6..8], 16)); + } } diff --git a/src/MauiDevFlow.Agent/VisualTreeWalker.cs b/src/MauiDevFlow.Agent/VisualTreeWalker.cs index 952cd73..28d8e8e 100644 --- a/src/MauiDevFlow.Agent/VisualTreeWalker.cs +++ b/src/MauiDevFlow.Agent/VisualTreeWalker.cs @@ -38,6 +38,35 @@ protected override void PopulateNativeInfo(ElementInfo info, VisualElement ve) props["isSecureTextEntry"] = textField.SecureTextEntry.ToString(); if (props.Count > 0) info.NativeProperties = props; + + // Effective rendered colors — resolved after theme/styles. + // For background: walk up the superview chain because most text views + // have UIColor.Clear as their own background; the real background is on + // a parent container (UIView, UIScrollView, window background). + if (uiView is UIKit.UILabel uiLabel) + { + info.EffectiveTextColor = IosColorToHex(uiLabel.TextColor); + info.EffectiveBackgroundColor = IosColorToHex(uiView.BackgroundColor) + ?? IosEffectiveBackgroundColor(uiView.Superview); + } + else if (uiView is UIKit.UITextField uiTextField2) + { + info.EffectiveTextColor = IosColorToHex(uiTextField2.TextColor); + info.EffectiveBackgroundColor = IosColorToHex(uiView.BackgroundColor) + ?? IosEffectiveBackgroundColor(uiView.Superview); + } + else if (uiView is UIKit.UITextView uiTextView) + { + info.EffectiveTextColor = IosColorToHex(uiTextView.TextColor); + info.EffectiveBackgroundColor = IosColorToHex(uiView.BackgroundColor) + ?? IosEffectiveBackgroundColor(uiView.Superview); + } + else if (uiView is UIKit.UIButton uiButton) + { + info.EffectiveTextColor = IosColorToHex(uiButton.TitleLabel?.TextColor); + info.EffectiveBackgroundColor = IosColorToHex(uiView.BackgroundColor) + ?? IosEffectiveBackgroundColor(uiView.Superview); + } } #elif ANDROID if (platformView is Android.Views.View androidView) @@ -51,6 +80,15 @@ protected override void PopulateNativeInfo(ElementInfo info, VisualElement ve) props["clickable"] = "true"; if (props.Count > 0) info.NativeProperties = props; + + // Effective rendered colors — after theme/style resolution + if (androidView is Android.Widget.TextView textView) + { + var argb = textView.CurrentTextColor; + info.EffectiveTextColor = $"#{(argb >> 24) & 0xFF:X2}{(argb >> 16) & 0xFF:X2}{(argb >> 8) & 0xFF:X2}{argb & 0xFF:X2}"; + // Walk up parent chain for background — text views often have transparent bg + info.EffectiveBackgroundColor = AndroidEffectiveBackgroundColor(androidView); + } } #elif MACOS if (platformView is NSView nsView) @@ -71,6 +109,8 @@ protected override void PopulateNativeInfo(ElementInfo info, VisualElement ve) { props["stringValue"] = nsTextField.StringValue; props["isEditable"] = nsTextField.Editable.ToString(); + info.EffectiveTextColor = MacColorToHex(nsTextField.TextColor); + info.EffectiveBackgroundColor = MacColorToHex(nsTextField.BackgroundColor); } props["isHidden"] = nsView.Hidden.ToString(); props["alphaValue"] = nsView.AlphaValue.ToString("F2"); @@ -112,6 +152,24 @@ protected override void PopulateNativeInfo(ElementInfo info, VisualElement ve) props["isPassword"] = "true"; if (props.Count > 0) info.NativeProperties = props; + + // Effective rendered colors + Windows.UI.Color? fg = null; + Windows.UI.Color? bg = null; + if (frameworkElement is Microsoft.UI.Xaml.Controls.TextBlock tb + && tb.Foreground is Microsoft.UI.Xaml.Media.SolidColorBrush tbBrush) + fg = tbBrush.Color; + else if (frameworkElement is Microsoft.UI.Xaml.Controls.Control ctrl + && ctrl.Foreground is Microsoft.UI.Xaml.Media.SolidColorBrush ctrlBrush) + fg = ctrlBrush.Color; + var bgBrushRaw = (frameworkElement as Microsoft.UI.Xaml.Controls.Control)?.Background + ?? (frameworkElement as Microsoft.UI.Xaml.Controls.Panel)?.Background; + if (bgBrushRaw is Microsoft.UI.Xaml.Media.SolidColorBrush bgBrush) + bg = bgBrush.Color; + if (fg.HasValue) + info.EffectiveTextColor = $"#{fg.Value.A:X2}{fg.Value.R:X2}{fg.Value.G:X2}{fg.Value.B:X2}"; + if (bg.HasValue) + info.EffectiveBackgroundColor = $"#{bg.Value.A:X2}{bg.Value.R:X2}{bg.Value.G:X2}{bg.Value.B:X2}"; } #endif } @@ -121,6 +179,258 @@ protected override void PopulateNativeInfo(ElementInfo info, VisualElement ve) } } + protected override void PopulateAccessibilityInfo(ElementInfo info, VisualElement ve) + { + // Let base populate MAUI-level semantics first + base.PopulateAccessibilityInfo(info, ve); + + try + { + var platformView = ve.Handler?.PlatformView; + if (platformView == null) return; + + info.Accessibility ??= new AccessibilityInfo(); + var a11y = info.Accessibility; + +#if IOS || MACCATALYST + if (platformView is UIKit.UIView uiView) + { + // Native accessibility label (resolved by UIKit, includes any overrides) + var nativeLabel = uiView.AccessibilityLabel; + if (!string.IsNullOrEmpty(nativeLabel)) + a11y.Label = nativeLabel; + + var nativeHint = uiView.AccessibilityHint; + if (!string.IsNullOrEmpty(nativeHint)) + a11y.Hint = nativeHint; + + var nativeValue = uiView.AccessibilityValue; + if (!string.IsNullOrEmpty(nativeValue)) + a11y.Value = nativeValue; + + // UIAccessibilityTraits → readable trait list + var traits = uiView.AccessibilityTraits; + var traitNames = new List(); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Button)) traitNames.Add("Button"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Link)) traitNames.Add("Link"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Header)) { traitNames.Add("Header"); a11y.IsHeading = true; } + if (traits.HasFlag(UIKit.UIAccessibilityTrait.SearchField)) traitNames.Add("SearchField"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Image)) traitNames.Add("Image"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Selected)) traitNames.Add("Selected"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.PlaysSound)) traitNames.Add("PlaysSound"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.KeyboardKey)) traitNames.Add("KeyboardKey"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.StaticText)) traitNames.Add("StaticText"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.SummaryElement)) traitNames.Add("SummaryElement"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.NotEnabled)) { traitNames.Add("NotEnabled"); a11y.IsEnabled = false; } + if (traits.HasFlag(UIKit.UIAccessibilityTrait.UpdatesFrequently)) traitNames.Add("UpdatesFrequently"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Adjustable)) traitNames.Add("Adjustable"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.AllowsDirectInteraction)) traitNames.Add("AllowsDirectInteraction"); + if (traitNames.Count > 0) + a11y.Traits = traitNames; + + // Map traits to role if not already set + if (a11y.Role == null) + { + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Button)) a11y.Role = "Button"; + else if (traits.HasFlag(UIKit.UIAccessibilityTrait.Link)) a11y.Role = "Link"; + else if (traits.HasFlag(UIKit.UIAccessibilityTrait.SearchField)) a11y.Role = "SearchField"; + else if (traits.HasFlag(UIKit.UIAccessibilityTrait.Image)) a11y.Role = "Image"; + else if (traits.HasFlag(UIKit.UIAccessibilityTrait.Header)) a11y.Role = "Header"; + else if (traits.HasFlag(UIKit.UIAccessibilityTrait.StaticText)) a11y.Role = "StaticText"; + else if (traits.HasFlag(UIKit.UIAccessibilityTrait.Adjustable)) a11y.Role = "Adjustable"; + } + + // Native IsAccessibilityElement can upgrade but not downgrade + // (UIKit returns false for views grouped into containers, but they're still relevant) + if (uiView.IsAccessibilityElement) + a11y.IsAccessibilityElement = true; + if (uiView.IsAccessibilityElement) + a11y.IsFocusable = true; + + // Accessibility container child count + if (uiView is UIKit.IUIAccessibilityContainer container) + { + var count = container.AccessibilityElementCount(); + if (count > 0) + a11y.ChildCount = (int)count; + } + } +#elif ANDROID + if (platformView is Android.Views.View androidView) + { + var nodeInfo = androidView.CreateAccessibilityNodeInfo(); + if (nodeInfo != null) + { + try + { + if (!string.IsNullOrEmpty(nodeInfo.ContentDescription?.ToString())) + a11y.Label = nodeInfo.ContentDescription!.ToString(); + + if (!string.IsNullOrEmpty(nodeInfo.Text?.ToString())) + { + // If no label, text serves as announced text + if (string.IsNullOrEmpty(a11y.Label)) + a11y.Label = nodeInfo.Text!.ToString(); + a11y.Value = nodeInfo.Text!.ToString(); + } + + if (!string.IsNullOrEmpty(nodeInfo.HintText?.ToString())) + a11y.Hint = nodeInfo.HintText!.ToString(); + + // Map Android className to role + var className = nodeInfo.ClassName?.ToString() ?? ""; + a11y.Role ??= className switch + { + var c when c.Contains("Button") => "Button", + var c when c.Contains("EditText") => "TextField", + var c when c.Contains("TextView") => "StaticText", + var c when c.Contains("ImageView") => "Image", + var c when c.Contains("CheckBox") => "CheckBox", + var c when c.Contains("RadioButton") => "RadioButton", + var c when c.Contains("Switch") || c.Contains("ToggleButton") => "Switch", + var c when c.Contains("SeekBar") || c.Contains("ProgressBar") => "Slider", + var c when c.Contains("Spinner") => "ComboBox", + var c when c.Contains("ScrollView") || c.Contains("RecyclerView") => "ScrollView", + _ => a11y.Role + }; + + // Traits from Android node info + var traits = new List(); + if (nodeInfo.Clickable) traits.Add("Clickable"); + if (nodeInfo.LongClickable) traits.Add("LongClickable"); + if (nodeInfo.Checkable) traits.Add("Checkable"); + if (nodeInfo.Checked) traits.Add("Checked"); + if (nodeInfo.Selected) traits.Add("Selected"); + if (nodeInfo.Scrollable) traits.Add("Scrollable"); + if (nodeInfo.Focusable) traits.Add("Focusable"); + if (nodeInfo.Editable) traits.Add("Editable"); + if (nodeInfo.Heading) { traits.Add("Heading"); a11y.IsHeading = true; } + if (traits.Count > 0) + a11y.Traits = traits; + + a11y.IsEnabled = nodeInfo.Enabled; + a11y.IsFocusable = nodeInfo.Focusable || nodeInfo.AccessibilityFocused; + a11y.IsFocused = nodeInfo.AccessibilityFocused || nodeInfo.Focused; + a11y.IsAccessibilityElement = nodeInfo.VisibleToUser && + (nodeInfo.Clickable || nodeInfo.Focusable || !string.IsNullOrEmpty(nodeInfo.ContentDescription?.ToString()) + || !string.IsNullOrEmpty(nodeInfo.Text?.ToString())); + + // Live region + if ((int)nodeInfo.LiveRegion != 0) + a11y.LiveRegion = nodeInfo.LiveRegion.ToString(); + + a11y.ChildCount = nodeInfo.ChildCount; + } + finally + { + nodeInfo.Recycle(); + } + } + } +#elif WINDOWS + if (platformView is Microsoft.UI.Xaml.FrameworkElement fe) + { + var name = Microsoft.UI.Xaml.Automation.AutomationProperties.GetName(fe); + if (!string.IsNullOrEmpty(name)) + a11y.Label = name; + + var helpText = Microsoft.UI.Xaml.Automation.AutomationProperties.GetHelpText(fe); + if (!string.IsNullOrEmpty(helpText)) + a11y.Hint = helpText; + + var headingLevel = Microsoft.UI.Xaml.Automation.AutomationProperties.GetHeadingLevel(fe); + if (headingLevel != Microsoft.UI.Xaml.Automation.Peers.AutomationHeadingLevel.None) + { + a11y.IsHeading = true; + a11y.Traits ??= new List(); + a11y.Traits.Add($"Heading:{headingLevel}"); + } + + var liveRegion = Microsoft.UI.Xaml.Automation.AutomationProperties.GetLiveSetting(fe); + if (liveRegion != Microsoft.UI.Xaml.Automation.Peers.AutomationLiveSetting.Off) + a11y.LiveRegion = liveRegion.ToString(); + + var isRequired = Microsoft.UI.Xaml.Automation.AutomationProperties.GetIsRequiredForForm(fe); + if (isRequired) + { + a11y.Traits ??= new List(); + a11y.Traits.Add("RequiredForForm"); + } + + var accessKey = Microsoft.UI.Xaml.Automation.AutomationProperties.GetAccessKey(fe); + if (!string.IsNullOrEmpty(accessKey)) + { + a11y.Traits ??= new List(); + a11y.Traits.Add($"AccessKey:{accessKey}"); + } + + if (fe is Microsoft.UI.Xaml.Controls.Control control) + { + a11y.IsEnabled = control.IsEnabled; + a11y.IsFocusable = control.IsTabStop; + a11y.IsFocused = control.FocusState != Microsoft.UI.Xaml.FocusState.Unfocused; + } + + // Try to get the AutomationPeer for role + try + { + var peer = Microsoft.UI.Xaml.Automation.Peers.FrameworkElementAutomationPeer.FromElement(fe); + if (peer != null) + { + var automationRole = peer.GetAutomationControlType(); + a11y.Role ??= automationRole.ToString(); + a11y.IsAccessibilityElement = !peer.IsOffscreen(); + + // Get announced name from peer if label not set + if (string.IsNullOrEmpty(a11y.Label)) + { + var peerName = peer.GetName(); + if (!string.IsNullOrEmpty(peerName)) + a11y.Label = peerName; + } + } + } + catch { /* AutomationPeer may not be available for all elements */ } + } +#elif MACOS + if (platformView is AppKit.NSView nsView) + { + var nativeLabel = nsView.AccessibilityLabel; + if (!string.IsNullOrEmpty(nativeLabel)) + a11y.Label = nativeLabel; + + var nativeTitle = nsView.AccessibilityTitle; + if (!string.IsNullOrEmpty(nativeTitle) && string.IsNullOrEmpty(a11y.Label)) + a11y.Label = nativeTitle; + + var nativeHelp = nsView.AccessibilityHelp; + if (!string.IsNullOrEmpty(nativeHelp)) + a11y.Hint = nativeHelp; + + var nativeValue = nsView.AccessibilityValue?.ToString(); + if (!string.IsNullOrEmpty(nativeValue)) + a11y.Value = nativeValue; + + var nativeRole = nsView.AccessibilityRole; + if (nativeRole != null) + a11y.Role = nativeRole.ToString(); + + a11y.IsAccessibilityElement = nsView.AccessibilityElement; + a11y.IsFocusable = nsView.CanBecomeKeyView; + a11y.IsFocused = nsView.Window?.FirstResponder == nsView; + + var children = nsView.AccessibilityChildren; + if (children != null) + a11y.ChildCount = children.Length; + } +#endif + } + catch + { + // Accessibility info is best-effort + } + } + protected override BoundsInfo? ResolveSyntheticBounds(object marker) { try @@ -633,4 +943,506 @@ private void PopulateNativeInfoWindows(ElementInfo info, object marker) } } #endif + + // ----------------------------------------------------------------------- + // Native Accessibility Tree Walk + // Returns elements in the exact order the platform screen reader visits them. + // ----------------------------------------------------------------------- + + public override List GetNativeA11yTree(Application app, int windowIndex) + { + var results = new List(); + try + { + var nativeToId = BuildNativeViewToIdMap(app, windowIndex); + var order = 0; + +#if IOS || MACCATALYST + var window = windowIndex >= 0 && windowIndex < app.Windows.Count + ? app.Windows[windowIndex] : app.Windows.FirstOrDefault(); + if (window?.Handler?.PlatformView is UIKit.UIWindow uiWindow + && uiWindow.RootViewController?.View is UIKit.UIView rootView) + { + var visited = new HashSet(); + WalkIosA11yTree(rootView, uiWindow, nativeToId, results, ref order, visited, 0); + } +#elif ANDROID + var window = windowIndex >= 0 && windowIndex < app.Windows.Count + ? app.Windows[windowIndex] : app.Windows.FirstOrDefault(); + if (window?.Handler?.PlatformView is Android.App.Activity activity + && activity.Window?.DecorView is Android.Views.ViewGroup decorView) + { + var density = activity.Resources?.DisplayMetrics?.Density ?? 1f; + var visited = new HashSet(); + WalkAndroidA11yTree(decorView, nativeToId, results, ref order, density, visited, 0); + } +#elif WINDOWS + var window = windowIndex >= 0 && windowIndex < app.Windows.Count + ? app.Windows[windowIndex] : app.Windows.FirstOrDefault(); + if (window?.Handler?.PlatformView is Microsoft.UI.Xaml.Window winWindow + && winWindow.Content is Microsoft.UI.Xaml.UIElement rootContent) + { + var visited = new HashSet(); + WalkWindowsA11yTree(rootContent, nativeToId, results, ref order, visited, 0); + } +#elif MACOS + var nsWindow = AppKit.NSApplication.SharedApplication.KeyWindow + ?? (app.Windows.ElementAtOrDefault(windowIndex)?.Handler?.PlatformView as AppKit.NSWindow); + if (nsWindow?.ContentView is AppKit.NSView contentView) + { + var visited = new HashSet(); + WalkMacA11yTree(contentView, nsWindow.ContentView!, nativeToId, results, ref order, visited, 0); + } +#endif + } + catch { } + return results; + } + +#if IOS || MACCATALYST + private const int MaxA11yDepth = 60; + private const int MaxA11yResults = 800; + + private static void WalkIosA11yTree( + UIKit.UIView view, + UIKit.UIWindow window, + Dictionary nativeToId, + List results, + ref int order, + HashSet visited, + int depth) + { + if (results.Count >= MaxA11yResults || depth > MaxA11yDepth) return; + + // Cycle guard — some containers return themselves or ancestors + if (!visited.Add(view.Handle)) return; + + // Hidden from accessibility — skip entirely + if (view.AccessibilityElementsHidden || view.Hidden || view.Alpha < 0.01f) + return; + + // Views whose accessibility trees involve IPC / expensive async work that + // can deadlock the main thread — skip them as opaque nodes. + var typeName = view.GetType().Name; + if (view is WebKit.WKWebView + || typeName.Contains("MKMapView", StringComparison.Ordinal) + || typeName.Contains("GLKView", StringComparison.Ordinal) + || typeName.Contains("MTKView", StringComparison.Ordinal) + || typeName.Contains("SCNView", StringComparison.Ordinal)) + return; + + // This view is an accessibility leaf — VoiceOver stops here and announces it + if (view.IsAccessibilityElement) + { + var rootView = window.RootViewController?.View ?? window; + var rect = view.ConvertRectToView(view.Bounds, rootView); + var traits = view.AccessibilityTraits; + results.Add(new NativeScreenReaderEntry + { + Order = order++, + Label = view.AccessibilityLabel ?? HarvestIosLabel(view), + Hint = view.AccessibilityHint, + Value = view.AccessibilityValue?.ToString(), + Role = GetIosRole(traits), + Traits = GetIosTraits(traits), + IsHeading = traits.HasFlag(UIKit.UIAccessibilityTrait.Header), + WindowBounds = new BoundsInfo { X = rect.X, Y = rect.Y, Width = rect.Width, Height = rect.Height }, + ElementId = nativeToId.TryGetValue(view, out var eid) ? eid : null, + NativeType = typeName, + }); + return; // Don't recurse — VoiceOver treats children as part of this element + } + + // Explicit ordering via IUIAccessibilityContainer overrides visual subview order. + // IMPORTANT: every UIView conforms to UIAccessibilityContainer. Views that do NOT + // explicitly override it return NSNotFound (== nint.MaxValue on 64-bit) from + // AccessibilityElementCount(). Without a guard, `count > 0` passes for MaxValue and + // the loop runs forever while GetAccessibilityElementAt keeps returning null. + // We treat counts > MaxA11yResults as "not explicitly set" (NSNotFound sentinel). + if (view is UIKit.IUIAccessibilityContainer container) + { + nint count; + try { count = container.AccessibilityElementCount(); } + catch { count = -1; } + + if (count > 0 && count <= MaxA11yResults) + { + for (nint i = 0; i < count && results.Count < MaxA11yResults; i++) + { + Foundation.NSObject? child; + try { child = container.GetAccessibilityElementAt(i); } + catch { continue; } + if (child is UIKit.UIView childView) + WalkIosA11yTree(childView, window, nativeToId, results, ref order, visited, depth + 1); + } + return; + } + } + + // Walk subviews — VoiceOver visits them in subview order (which matches layout order) + foreach (var subview in view.Subviews) + WalkIosA11yTree(subview, window, nativeToId, results, ref order, visited, depth + 1); + } + + /// + /// When AccessibilityLabel is null (e.g. UITabBarButton synthesises its label + /// dynamically via ObjC override), recursively scan subviews for any text content. + /// Depth-limited to avoid expensive traversal. + /// + private static string? HarvestIosLabel(UIKit.UIView view, int maxDepth = 3) + { + if (maxDepth <= 0) return null; + foreach (var sub in view.Subviews) + { + var text = sub switch + { + UIKit.UILabel lbl when !string.IsNullOrEmpty(lbl.Text) => lbl.Text, + UIKit.UIButton btn when !string.IsNullOrEmpty(btn.CurrentTitle) => btn.CurrentTitle, + _ => null, + }; + if (text != null) return text; + text = HarvestIosLabel(sub, maxDepth - 1); + if (text != null) return text; + } + return null; + } + + private static string? GetIosRole(UIKit.UIAccessibilityTrait traits) + { + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Button)) return "Button"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Link)) return "Link"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Header)) return "Header"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.SearchField))return "SearchField"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Image)) return "Image"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Adjustable)) return "Slider"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.KeyboardKey))return "Key"; + if (traits.HasFlag(UIKit.UIAccessibilityTrait.StaticText)) return "Text"; + return null; + } + + private static List? GetIosTraits(UIKit.UIAccessibilityTrait traits) + { + var list = new List(); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Button)) list.Add("Button"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Link)) list.Add("Link"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Header)) list.Add("Header"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.SearchField)) list.Add("SearchField"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Image)) list.Add("Image"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Selected)) list.Add("Selected"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.NotEnabled)) list.Add("Disabled"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.Adjustable)) list.Add("Adjustable"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.UpdatesFrequently)) list.Add("LiveRegion"); + if (traits.HasFlag(UIKit.UIAccessibilityTrait.StaticText)) list.Add("StaticText"); + return list.Count > 0 ? list : null; + } +#endif + +#if ANDROID + private const int MaxA11yDepth = 60; + private const int MaxA11yResults = 800; + + private static void WalkAndroidA11yTree( + Android.Views.View view, + Dictionary nativeToId, + List results, + ref int order, + float density, + HashSet visited, + int depth) + { + if (results.Count >= MaxA11yResults || depth > MaxA11yDepth) return; + + // Cycle guard using object identity + if (!visited.Add(System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(view))) return; + + if (!view.IsShown) return; + + // Skip views whose a11y trees live in another process + if (view is Android.Webkit.WebView + || view.GetType().Name.Contains("MapView", StringComparison.Ordinal)) + return; + + // NoHideDescendants hides this view AND its entire subtree from accessibility + if (view.ImportantForAccessibility == Android.Views.ImportantForAccessibility.NoHideDescendants) + return; + + // Per-view AccessibilityNodeInfo: replicate TalkBack's focusability predicate. + // + // Key rules: + // 1. Do NOT filter by VisibleToUser — TalkBack navigates to off-screen elements + // in scrollable containers (it scrolls the container to bring them into view). + // view.IsShown above already excludes truly GONE/INVISIBLE views. + // 2. Do NOT use Focusable (keyboard focus) — TalkBack has its own focus concept. + // TalkBack focuses an element when it is Clickable, LongClickable, Checkable, + // or has an announced label/text. + var nodeInfo = view.CreateAccessibilityNodeInfo(); + if (nodeInfo != null) + { + try + { + var hasLabel = !string.IsNullOrEmpty(nodeInfo.ContentDescription?.ToString()) + || !string.IsNullOrEmpty(nodeInfo.Text?.ToString()); + var isA11yFocusable = nodeInfo.Clickable + || nodeInfo.LongClickable + || nodeInfo.Checkable + || hasLabel; + + if (isA11yFocusable) + { + var location = new int[2]; + view.GetLocationInWindow(location); + bool isHeading = false; + try { isHeading = nodeInfo.Heading; } catch { } + + results.Add(new NativeScreenReaderEntry + { + Order = order++, + Label = nodeInfo.ContentDescription?.ToString() ?? nodeInfo.Text?.ToString(), + Role = GetAndroidRoleFromNode(nodeInfo), + IsHeading = isHeading, + WindowBounds = new BoundsInfo + { + X = location[0] / density, + Y = location[1] / density, + Width = view.Width / density, + Height = view.Height / density, + }, + ElementId = nativeToId.TryGetValue(view, out var eid) ? eid : null, + NativeType = view.GetType().Name, + }); + } + } + finally + { + nodeInfo.Recycle(); + } + } + + if (view is not Android.Views.ViewGroup viewGroup) return; + + // Sort children top-to-bottom, left-to-right — TalkBack reading order + var children = Enumerable.Range(0, viewGroup.ChildCount) + .Select(i => viewGroup.GetChildAt(i)) + .Where(c => c != null) + .ToList(); + children.Sort((a, b) => + { + var aLoc = new int[2]; a!.GetLocationInWindow(aLoc); + var bLoc = new int[2]; b!.GetLocationInWindow(bLoc); + var yDiff = aLoc[1] - bLoc[1]; + return yDiff != 0 ? yDiff : (aLoc[0] - bLoc[0]); + }); + + foreach (var child in children) + WalkAndroidA11yTree(child!, nativeToId, results, ref order, density, visited, depth + 1); + } + + private static string? AndroidEffectiveBackgroundColor(Android.Views.View? view, int maxDepth = 20) + { + var current = view; + var depth = 0; + while (current != null && depth < maxDepth) + { + if (current.Background is Android.Graphics.Drawables.ColorDrawable cd) + { + var argb = cd.Color.ToArgb(); + var a = (argb >> 24) & 0xFF; + if (a > 10) // non-transparent + return $"#{a:X2}{(argb >> 16) & 0xFF:X2}{(argb >> 8) & 0xFF:X2}{argb & 0xFF:X2}"; + } + current = current.Parent as Android.Views.View; + depth++; + } + return null; + } + + private static string? GetAndroidRoleFromNode(Android.Views.Accessibility.AccessibilityNodeInfo node) + { + var cn = node.ClassName?.ToString() ?? string.Empty; + // More specific class names must come before their base classes + if (cn.EndsWith("CheckBox", StringComparison.Ordinal)) return "Checkbox"; + if (cn.EndsWith("Switch", StringComparison.Ordinal)) return "Switch"; + if (cn.EndsWith("RadioButton", StringComparison.Ordinal)) return "RadioButton"; + if (cn.EndsWith("SeekBar", StringComparison.Ordinal)) return "Slider"; + if (cn.EndsWith("EditText", StringComparison.Ordinal)) return "TextField"; + if (cn.EndsWith("ImageButton", StringComparison.Ordinal)) return "Button"; + if (cn.EndsWith("Button", StringComparison.Ordinal)) return "Button"; + if (cn.EndsWith("RecyclerView",StringComparison.Ordinal)) return "List"; + if (node.Clickable) return "Button"; + return null; + } +#endif + +#if WINDOWS + private const int MaxA11yDepth = 60; + private const int MaxA11yResults = 800; + + private static void WalkWindowsA11yTree( + Microsoft.UI.Xaml.UIElement element, + Dictionary nativeToId, + List results, + ref int order, + HashSet visited, + int depth) + { + if (results.Count >= MaxA11yResults || depth > MaxA11yDepth) return; + if (!visited.Add(System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(element))) return; + + if (element.Visibility != Microsoft.UI.Xaml.Visibility.Visible) return; + + // Skip WebView2 — its a11y tree is backed by the browser process + if (element.GetType().Name.Contains("WebView", StringComparison.Ordinal) + || element.GetType().Name.Contains("MapControl", StringComparison.Ordinal)) + return; + + // Try to get AutomationPeer — if it reports content, treat as leaf + if (element is Microsoft.UI.Xaml.FrameworkElement fe) + { + var peer = Microsoft.UI.Xaml.Automation.Peers.FrameworkElementAutomationPeer.FromElement(fe); + if (peer != null && peer.IsContentElement()) + { + var name = peer.GetName(); + var controlType = peer.GetAutomationControlType(); + if (!string.IsNullOrEmpty(name) || controlType != Microsoft.UI.Xaml.Automation.Peers.AutomationControlType.Custom) + { + var transform = fe.TransformToVisual(null); + var pt = transform.TransformPoint(new Windows.Foundation.Point(0, 0)); + results.Add(new NativeScreenReaderEntry + { + Order = order++, + Label = name, + Role = controlType.ToString(), + WindowBounds = new BoundsInfo { X = pt.X, Y = pt.Y, Width = fe.ActualWidth, Height = fe.ActualHeight }, + ElementId = nativeToId.TryGetValue(element, out var eid) ? eid : null, + NativeType = element.GetType().Name, + }); + return; + } + } + } + + var count = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetChildrenCount(element); + for (var i = 0; i < count; i++) + { + if (Microsoft.UI.Xaml.Media.VisualTreeHelper.GetChild(element, i) is Microsoft.UI.Xaml.UIElement child) + WalkWindowsA11yTree(child, nativeToId, results, ref order, visited, depth + 1); + } + } +#endif + +#if MACOS + private const int MaxA11yDepth = 60; + private const int MaxA11yResults = 800; + + private static void WalkMacA11yTree( + AppKit.NSView view, + AppKit.NSView rootView, + Dictionary nativeToId, + List results, + ref int order, + HashSet visited, + int depth) + { + if (results.Count >= MaxA11yResults || depth > MaxA11yDepth) return; + if (!visited.Add(view.Handle)) return; + + if (view.Hidden) return; + + // WKWebView / MapKit — AccessibilityChildren involves IPC, skip + if (view is WebKit.WKWebView + || view.GetType().Name.Contains("MKMapView", StringComparison.Ordinal)) + return; + + var a11yChildren = view.AccessibilityChildren; + bool hasChildren = a11yChildren != null && a11yChildren.Length > 0; + var label = view.AccessibilityLabel ?? view.AccessibilityTitle; + + // Treat as leaf when no accessible children and has meaningful content + if (!hasChildren && !string.IsNullOrEmpty(label)) + { + var windowRect = view.ConvertRectToView(view.Bounds, rootView); + var contentHeight = rootView.Bounds.Height; + results.Add(new NativeScreenReaderEntry + { + Order = order++, + Label = label, + Hint = view.AccessibilityHelp, + Value = view.AccessibilityValue?.ToString(), + Role = view.AccessibilityRole, + WindowBounds = new BoundsInfo + { + X = windowRect.X, + Y = contentHeight - windowRect.Y - windowRect.Height, // flip Y (NSView is bottom-left) + Width = windowRect.Width, + Height = windowRect.Height, + }, + ElementId = nativeToId.TryGetValue(view, out var eid) ? eid : null, + NativeType = view.GetType().Name, + }); + return; + } + + if (hasChildren) + { + foreach (var child in a11yChildren!) + { + if (child is AppKit.NSView childView) + WalkMacA11yTree(childView, rootView, nativeToId, results, ref order, visited, depth + 1); + } + } + else + { + foreach (var subview in view.Subviews) + WalkMacA11yTree(subview, rootView, nativeToId, results, ref order, visited, depth + 1); + } + } +#endif + + // ----------------------------------------------------------------------- + // Color helpers + // ----------------------------------------------------------------------- + +#if IOS || MACCATALYST + /// + /// Walks up the superview chain to find the first non-transparent background color. + /// Most UILabels/UIButtons have UIColor.Clear; the visible background is on a parent. + /// + private static string? IosEffectiveBackgroundColor(UIKit.UIView? view, int maxDepth = 20) + { + var current = view; + var depth = 0; + while (current != null && depth < maxDepth) + { + var hex = IosColorToHex(current.BackgroundColor); + if (hex != null) return hex; + current = current.Superview; + depth++; + } + return null; + } + + private static string? IosColorToHex(UIKit.UIColor? color) + { + if (color == null) return null; + color.GetRGBA(out var r, out var g, out var b, out var a); + if (a < 0.01f) return null; // fully transparent — not useful for contrast + return $"#{(int)(a * 255):X2}{(int)(r * 255):X2}{(int)(g * 255):X2}{(int)(b * 255):X2}"; + } +#endif + +#if MACOS + private static string? MacColorToHex(AppKit.NSColor? color) + { + if (color == null) return null; + try + { + var rgb = color.UsingColorSpace(AppKit.NSColorSpace.DeviceRGB); + if (rgb == null) return null; + rgb.GetRgba(out var r, out var g, out var b, out var a); + if (a < 0.01f) return null; + return $"#{(int)(a * 255):X2}{(int)(r * 255):X2}{(int)(g * 255):X2}{(int)(b * 255):X2}"; + } + catch { return null; } + } +#endif }