feat(marquee): macOS adapters resolve a drawn frame to an element (VRT-cne0.7)

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: angusbezzina

Sources/AnnotKit/macOS/AXIntrospection.swift

@@ -265,12 +265,45 @@ enum AXIntrospection {
         let candidates = chain.map { candidate(for: $0, windowFrame: windowFrame) }
         guard let targetIndex = AnnotationTargetRule.targetIndex(in: candidates) else { return [] }
         let target = chain[targetIndex]
-        let targetArea = area(of: target)
 
+        return [element(for: target, ancestorChain: chain)]
+            + enclosingComponents(of: target, containing: point, in: chain)
+                .map { element(for: $0, ancestorChain: ancestorChain(from: $0)) }
+    }
+
+    /// The identified components that geometrically ENCLOSE `target` at `point`,
+    /// smallest-first — the widening rungs above a bound target. Shared verbatim
+    /// by the point path (``componentLadder(for:)``) and the frame path
+    /// (``marqueeLadder(for:)``) so a click and a drag onto the same element can
+    /// never widen through different components; duplicating this scan is exactly
+    /// how the two paths would silently drift apart.
+    ///
+    /// The scan is GEOMETRIC (DECISIONS.md): a `.axCardSurface` card hangs its
+    /// identifier on a clear background leaf that is a SIBLING of the card's
+    /// content, so it never appears in an ancestor chain. Scanning each ancestor
+    /// PLUS its direct children reaches those surfaces without a full-tree walk.
+    /// Deduped by identifier (the same surface is reachable from several
+    /// ancestors), and the `>= targetArea` floor keeps a smaller identified
+    /// sibling that merely happens to cover the point out of the widening ladder.
+    ///
+    /// Container ROOTS are excluded, matching ``AnnotationTargetRule/wideningLadder(in:)``
+    /// stopping at the window. Load-bearing, not tidiness: the chain climbs to
+    /// `AXApplication`, whose direct children are the app's WINDOWS — including our
+    /// own overlay panel, which carries ``overlayWindowIdentifier`` and encloses
+    /// every point in the host. Without this the top rung of every ladder is
+    /// AnnotKit's own overlay, so widening would bind the user's note to our UI.
+    private static func enclosingComponents(
+        of target: AXUIElement,
+        containing point: CGPoint,
+        in rootFirstChain: [AXUIElement]
+    ) -> [AXUIElement] {
+        let targetArea = area(of: target)
         var containers: [(node: AXUIElement, area: CGFloat)] = []
         var seen = Set<String>()
-        for ancestor in chain {
+        for ancestor in rootFirstChain {
             for node in [ancestor] + elementArray(ancestor, kAXChildrenAttribute) {
+                let role = string(node, kAXRoleAttribute) ?? ""
+                guard role != "AXWindow", role != "AXApplication" else { continue }
                 let id = string(node, kAXIdentifierAttribute) ?? ""
                 guard !id.isEmpty, !seen.contains(id), !CFEqual(node, target) else { continue }
                 let frame = frameScreen(of: node)
@@ -282,9 +315,107 @@ enum AXIntrospection {
             }
         }
         containers.sort { $0.area < $1.area }
+        return containers.map(\.node)
+    }
+
+    // MARK: - Marquee (drawn frame -> element)
+
+    /// The component-widening ladder for a frame the user DREW: the element the
+    /// frame binds to per ``MarqueeTargetRule`` first, then each enclosing
+    /// identified component, broadest last. Same shape as
+    /// ``componentLadder(for:)``, because the session reuses its ladder machinery
+    /// verbatim — widening and the note's `component` field both assume
+    /// `ladder[0]` is the bound target. Empty when the frame resolves to nothing
+    /// (the session then captures a region note instead).
+    ///
+    /// Cost: this walks the whole window subtree ONCE, on drag RELEASE only —
+    /// never during the drag and never on hover. A full walk is affordable at that
+    /// rate; it would not be on the hover path, which is why the point path still
+    /// uses the ancestor-chain scan instead.
+    static func marqueeLadder(for rect: CGRect) -> [Element] {
+        // Standardize before anything geometric: a right-to-left / bottom-to-top
+        // drag arrives with negative extents, where `contains` degenerates and the
+        // window lookup below would silently find nothing.
+        let marquee = rect.standardized
+        let center = CGPoint(x: marquee.midX, y: marquee.midY)
+        let app = appElement()
+        // Same window pick as ``regionAnchor(for:)`` / ``hitBeneathOverlay(_:)``:
+        // `kAXWindows` is front-to-back, so the first non-overlay window containing
+        // the frame's center is the frontmost real target.
+        let windows = elementArray(app, kAXWindowsAttribute).filter { !isOverlayWindow($0) }
+        guard let window = windows.first(where: { frameScreen(of: $0).contains(center) }) else { return [] }
+        let windowFrame = frameScreen(of: window)
+
+        // ONE recursive walk, so every candidate's depth is measured from the SAME
+        // root (window = 0). Depth is the rule's tie-break between geometrically
+        // indistinguishable candidates; assembling the array from several
+        // differently-rooted traversals would turn that tie-break into noise.
+        //
+        // The subtree is collected WHOLE — deliberately not pre-filtered to what
+        // intersects the drawn frame. The rule's second pass needs the candidates
+        // whose frames CONTAIN the frame (the user drew inside something), and an
+        // intersects-the-marquee filter is precisely what discards them.
+        var nodes: [AXUIElement] = []
+        var candidates: [MarqueeCandidate] = []
+        collectMarqueeCandidates(
+            window, windowFrame: windowFrame, depth: 0, nodes: &nodes, candidates: &candidates
+        )
 
+        guard let resolution = MarqueeTargetRule.resolve(marquee: marquee, in: candidates) else { return [] }
+        let target = nodes[resolution.index]
+        let chain = ancestorChain(from: target)
+
+        // The widening rungs are anchored at the TARGET's frame center, not the
+        // drawn frame's: a sloppy marquee can spill outside the element it bound
+        // to, and a container that does not contain the target is not a component
+        // the user could widen to. This is also the value the point path passes.
+        let targetFrame = frameScreen(of: target)
+        let targetCenter = CGPoint(x: targetFrame.midX, y: targetFrame.midY)
         return [element(for: target, ancestorChain: chain)]
-            + containers.map { element(for: $0.node, ancestorChain: ancestorChain(from: $0.node)) }
+            + enclosingComponents(of: target, containing: targetCenter, in: chain)
+                .map { element(for: $0, ancestorChain: ancestorChain(from: $0)) }
+    }
+
+    /// Depth-first walk collecting a PARALLEL pair per node: the live
+    /// `AXUIElement` and its pure ``MarqueeCandidate``, so
+    /// ``MarqueeTargetRule/Resolution/index`` maps straight back to a live handle.
+    ///
+    /// The candidate is built with the same ``candidate(for:windowFrame:)`` the
+    /// point path uses, so chrome / container-root / window-ghost classification —
+    /// which is what the rule's eligibility filter reads — is identical for a click
+    /// and a drag by construction.
+    private static func collectMarqueeCandidates(
+        _ element: AXUIElement,
+        windowFrame: CGRect,
+        depth: Int,
+        nodes: inout [AXUIElement],
+        candidates: inout [MarqueeCandidate]
+    ) {
+        nodes.append(element)
+        candidates.append(
+            MarqueeCandidate(
+                element: candidate(for: element, windowFrame: windowFrame),
+                frame: frameScreen(of: element),
+                depth: depth
+            )
+        )
+        guard depth < maxDepth else { return }
+        for child in elementArray(element, kAXChildrenAttribute) {
+            // Starting from a non-overlay window should already put the overlay out
+            // of reach, but AppKit exposes an attached child PANEL through some
+            // parents' `kAXChildren`, so verify rather than assume: a marquee that
+            // swept the overlay's own hosting view would bind the note to our UI.
+            if isOverlayWindow(child) { continue }
+            // Chrome's whole subtree is skipped, not just the button: the traffic
+            // lights' inner glyph groups carry no chrome subrole of their own, so
+            // the rule's `isChrome` filter alone would let a marquee over the title
+            // bar bind to a glyph. The point path rejects chrome geometrically for
+            // the same reason.
+            if isChrome(child) { continue }
+            collectMarqueeCandidates(
+                child, windowFrame: windowFrame, depth: depth + 1, nodes: &nodes, candidates: &candidates
+            )
+        }
     }
 
     /// Resolve `point` to a root-first ancestor chain of the deepest host element

Sources/AnnotKit/macOS/MacElementSource.swift

@@ -39,4 +39,10 @@ extension MacElementSource: ComponentLadderSource {
         AXIntrospection.componentLadder(for: point)
     }
 }
+
+extension MacElementSource: MarqueeTargetSource {
+    public func marqueeLadder(in rect: CGRect) -> [Element] {
+        AXIntrospection.marqueeLadder(for: rect)
+    }
+}
 #endif

Sources/AnnotKit/macOS/MacViewTreeElementSource.swift

@@ -79,7 +79,7 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc
 
     public func hitTest(_ point: CGPoint) -> Element? {
         guard let chain = Self.resolvedChain(at: point) else { return nil }
-        let candidates = chain.map(Self.candidate(for:))
+        let candidates = chain.map { Self.candidate(for: $0) }
         guard let index = AnnotationTargetRule.targetIndex(in: candidates) else { return nil }
         return Self.element(for: chain[index])
     }
@@ -88,7 +88,7 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc
     /// views), matching the AX and iOS sources so all three widen identically.
     public func componentLadder(at point: CGPoint) -> [Element] {
         guard let chain = Self.resolvedChain(at: point) else { return [] }
-        let candidates = chain.map(Self.candidate(for:))
+        let candidates = chain.map { Self.candidate(for: $0) }
         return AnnotationTargetRule.wideningLadder(in: candidates).map { Self.element(for: chain[$0]) }
     }
 
@@ -216,13 +216,21 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc
 
     /// Read one view's target-relevant facts into a pure ``TargetCandidate``. The
     /// NSView tree has no displayed value text, so `value` is always empty.
-    private static func candidate(for view: NSView) -> TargetCandidate {
+    ///
+    /// `isContainerRoot` defaults to false because on the POINT path the chain is
+    /// an ancestor chain rooted at the hit's window content view, and flagging it
+    /// there would change which element a click resolves to. The marquee walk
+    /// passes true for the content view: a whole-tree walk offers it as a real
+    /// candidate, and without the flag a large drag would surround it and bind the
+    /// note to the app's entire content view instead of the card inside it.
+    private static func candidate(for view: NSView, isContainerRoot: Bool = false) -> TargetCandidate {
         TargetCandidate(
             role: String(describing: Swift.type(of: view)),
             identifier: view.accessibilityIdentifier(),
             label: view.accessibilityLabel() ?? "",
             value: "",
-            isActionable: view is NSControl
+            isActionable: view is NSControl,
+            isContainerRoot: isContainerRoot
         )
     }
 
@@ -237,4 +245,110 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc
         NSScreen.screens.first?.frame.height ?? 0
     }
 }
+
+// MARK: - Marquee (drawn frame -> view)
+
+extension MacViewTreeElementSource: MarqueeTargetSource {
+    /// The ladder for a frame the user DREW, over the `NSView` tree: the view the
+    /// frame binds to per ``MarqueeTargetRule`` first, then its enclosing
+    /// identified views, broadest last — the same contract as
+    /// ``componentLadder(at:)``, so the session's widening works from a framed
+    /// selection unchanged.
+    ///
+    /// Cost: one full walk of the hit window's view tree per drag RELEASE. Never
+    /// during the drag and never on hover, which is what makes a whole-tree walk
+    /// affordable here where the hover path must stay on the ancestor chain.
+    public func marqueeLadder(in rect: CGRect) -> [Element] {
+        // Standardize first: a right-to-left / bottom-to-top drag arrives with
+        // negative extents, where the window lookup's `contains` degenerates.
+        let marquee = rect.standardized
+        guard let content = Self.marqueeRoot(containing: CGPoint(x: marquee.midX, y: marquee.midY)) else {
+            return []
+        }
+
+        // ONE recursive walk from the content view, so every candidate's depth is
+        // measured from the SAME root (content view = 0) — depth is the rule's
+        // tie-break between geometrically indistinguishable candidates, and mixing
+        // differently-rooted numbering would make it noise. The subtree is
+        // collected WHOLE, deliberately not pre-filtered to what intersects the
+        // frame: the rule's enclosing pass needs the views that CONTAIN the frame,
+        // which such a filter is exactly what discards.
+        var views: [NSView] = []
+        var candidates: [MarqueeCandidate] = []
+        Self.collectMarqueeCandidates(content, isRoot: true, depth: 0, views: &views, candidates: &candidates)
+
+        guard let resolution = MarqueeTargetRule.resolve(marquee: marquee, in: candidates) else { return [] }
+        let target = views[resolution.index]
+        return [Self.element(for: target)]
+            + Self.enclosingIdentifiedViews(of: target, upTo: content).map { Self.element(for: $0) }
+    }
+
+    /// The content view of the frontmost visible non-overlay window under `point`.
+    /// `NSApp.orderedWindows` is front-to-back, mirroring how the AX source reads
+    /// `kAXWindows`, so both sources pick the same window for the same drag. The
+    /// overlay panel is excluded by the identifier ``OverlayController`` stamps on
+    /// it — the drawn frame always lies over the expanded overlay, so without this
+    /// every marquee would walk our own hosting view.
+    private static func marqueeRoot(containing point: CGPoint) -> NSView? {
+        for window in NSApp.orderedWindows {
+            guard window.isVisible,
+                  window.accessibilityIdentifier() != AXIntrospection.overlayWindowIdentifier,
+                  let content = window.contentView,
+                  screenFrame(of: content).contains(point)
+            else { continue }
+            return content
+        }
+        return nil
+    }
+
+    /// Depth-first walk collecting a PARALLEL pair per view — the live `NSView`
+    /// and its pure ``MarqueeCandidate`` — so
+    /// ``MarqueeTargetRule/Resolution/index`` maps straight back to a view.
+    ///
+    /// Hidden and fully transparent subtrees are skipped: they keep real frames, so
+    /// a marquee would happily "surround" a hidden view the user cannot even see,
+    /// and being the largest such frame it would win pass 1 outright. The point
+    /// path gets this for free from `NSView.hitTest`, which a whole-tree walk does
+    /// not go through.
+    private static func collectMarqueeCandidates(
+        _ view: NSView,
+        isRoot: Bool,
+        depth: Int,
+        views: inout [NSView],
+        candidates: inout [MarqueeCandidate]
+    ) {
+        views.append(view)
+        candidates.append(
+            MarqueeCandidate(
+                element: candidate(for: view, isContainerRoot: isRoot),
+                frame: screenFrame(of: view),
+                depth: depth
+            )
+        )
+        guard depth < maxDepth else { return }
+        for subview in view.subviews where !subview.isHidden && subview.alphaValue > 0.01 {
+            collectMarqueeCandidates(
+                subview, isRoot: false, depth: depth + 1, views: &views, candidates: &candidates
+            )
+        }
+    }
+
+    /// The identified superviews of `target`, nearest first (so broadest last),
+    /// stopping BEFORE `root` — the content view is the container root and is never
+    /// a widening rung, for the same reason it is never a target. Pure ancestry,
+    /// matching this source's point ladder; the AX source's extra geometric scan
+    /// exists for SwiftUI `.background` surfaces, which are AX-only artifacts with
+    /// no counterpart in the `NSView` tree.
+    private static func enclosingIdentifiedViews(of target: NSView, upTo root: NSView) -> [NSView] {
+        var out: [NSView] = []
+        var current = target.superview
+        var depth = 0
+        while let view = current, view !== root, depth < maxDepth {
+            if !view.accessibilityIdentifier().isEmpty { out.append(view) }
+            current = view.superview
+            depth += 1
+        }
+        return out
+    }
+}
 #endif