Skip to content

Commit bdeb24b

Browse files
angusbezzinaclaude
andcommitted
fix(overlay): even idle pill inset, chrome-proof hit-test, hover-exit clearing, container surfaces
Four fixes from the first real dogfood on VirgilHUD (all probe-covered): - Idle pill: EVEN 8pt inset around the single pencil button (concentric 44x44 capsule) instead of the 4-button row's 6/8pt padding, which read as lopsided. - Hit-test: window chrome is never an annotation target. The traffic lights are actionable AXButtons that passed the resolvers; rejected via subrole at chain level PLUS a geometric backstop (the zoom button nests a glyph AXGroup whose reported AX parent is the WINDOW, so a chain scan alone misses it). Also: descend to the SMALLEST containing child (AX order does not encode z-order), and never fall back to an unidentified structural group spanning the whole window (NSHostingView's root). - Hover: clear the highlight when the cursor leaves the catcher (HoverPhase .ended) — the last-hovered element used to stay highlighted after the cursor left the app. New AnnotationSession.clearHover() keeps selection untouched. - Containers: probed that a children-.contain group reports the UNION of its children as its AX frame (card padding dead to hover); the working pattern is a full-size clear background leaf carrying the identifier, validated by the new overlay-probe Phase 5. Probe: +Phase 4 (chrome exclusion through the expanded overlay, incl. positive control) and +Phase 5 (container surface body-hover), plus W1 window hygiene. Tests: +clearHover, +chrome-subrole predicate (47 total). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 3b57702 commit bdeb24b

6 files changed

Lines changed: 330 additions & 11 deletions

File tree

Sources/AnnotKit/Overlay/AnnotationSession.swift

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,12 @@ public final class AnnotationSession: ObservableObject {
7777
hovered = source.hitTest(point)
7878
}
7979

80+
/// Drop the hover highlight — the cursor left the annotatable surface, so
81+
/// nothing is hovered. `selected` (an open composer) is deliberately kept.
82+
public func clearHover() {
83+
hovered = nil
84+
}
85+
8086
/// Select the element under a screen point (AX top-left coordinates).
8187
@discardableResult
8288
public func select(atAXPoint point: CGPoint) -> Element? {

Sources/AnnotKit/Overlay/OverlayView.swift

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,17 @@ struct OverlayView: View {
9696
Color.clear
9797
.contentShape(Rectangle())
9898
.onContinuousHover { phase in
99-
if case .active(let point) = phase {
99+
switch phase {
100+
case .active(let point):
100101
session.hover(atAXPoint: CGPoint(x: point.x + axOrigin.x, y: point.y + axOrigin.y))
102+
case .ended:
103+
// The cursor left the catcher (it covers the host's full
104+
// frame, so this is "left the window" — or moved onto the
105+
// toolbar/composer, which consume hover). Nothing is
106+
// hovered: drop the highlight instead of freezing on the
107+
// last element. An open composer is unaffected (the
108+
// highlight renders selected ?? hovered).
109+
session.clearHover()
101110
}
102111
}
103112
.gesture(
@@ -437,7 +446,11 @@ private struct ToolbarView: View {
437446
}
438447
}
439448
}
440-
.padding(.horizontal, 6)
449+
// Idle shows ONE 28pt button, so the inset must be EVEN (8pt all around ->
450+
// a concentric 44x44 capsule hugging the hover wash). The 6pt horizontal
451+
// inset is for the annotate-mode 4-button ROW only, where the buttons'
452+
// own spacing makes the tighter ends read as balanced.
453+
.padding(.horizontal, annotating ? 6 : 8)
441454
.padding(.vertical, 8) // 28pt buttons + 8*2 -> 44pt pill height
442455
.background(
443456
Capsule(style: .continuous)

Sources/AnnotKit/macOS/AXIntrospection.swift

Lines changed: 68 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,18 @@ enum AXIntrospection {
250250

251251
let chain = ancestorChain(from: deepest)
252252
guard !chain.isEmpty else { return nil }
253+
// A hit through window CHROME (the traffic lights) is not annotatable at
254+
// ANY level: the hit itself may be the button or one of its inner glyph
255+
// groups, and everything above it is the title bar / window. Two checks,
256+
// because the glyph groups report the WINDOW (not the button) as their
257+
// AX parent, so the chain scan alone can miss them — the geometric
258+
// backstop catches any query point inside a chrome button's frame.
259+
if chain.contains(where: isChrome) { return nil }
260+
if let window = chain.first(where: { string($0, kAXRoleAttribute) == "AXWindow" }),
261+
elementArray(window, kAXChildrenAttribute)
262+
.contains(where: { isChrome($0) && frameScreen(of: $0).contains(point) }) {
263+
return nil
264+
}
253265
// Never fall back to the window or application container: escalating to
254266
// AXWindow is what made a background click resolve to the whole app.
255267
guard let target = nearestIdentified(in: chain) ?? deepestNonContainer(in: chain) else {
@@ -273,17 +285,27 @@ enum AXIntrospection {
273285
}
274286

275287
/// Deepest descendant of `element` whose (non-empty) frame contains `point`.
276-
/// Later children are drawn on top, so a topmost match wins; returns
277-
/// `element` itself when no child contains the point.
288+
/// Among children containing the point, descend into the SMALLEST (most
289+
/// specific) one: AX child order does not reliably encode z-order, so a
290+
/// full-size sibling layer (a window's title-bar strip, or a card's clear
291+
/// accessibility surface in `.background`) must not swallow the more
292+
/// specific content that visually sits above it. Returns `element` itself
293+
/// when no child contains the point.
278294
private static func deepestChild(of element: AXUIElement, containing point: CGPoint, depth: Int) -> AXUIElement {
279295
guard depth < maxDepth else { return element }
280-
for child in elementArray(element, kAXChildrenAttribute).reversed() {
281-
let frame = frameScreen(of: child)
282-
if frame.width > 0, frame.height > 0, frame.contains(point) {
283-
return deepestChild(of: child, containing: point, depth: depth + 1)
296+
let candidate = elementArray(element, kAXChildrenAttribute)
297+
.filter {
298+
let frame = frameScreen(of: $0)
299+
return frame.width > 0 && frame.height > 0 && frame.contains(point)
284300
}
285-
}
286-
return element
301+
.min { area(of: $0) < area(of: $1) }
302+
guard let candidate else { return element }
303+
return deepestChild(of: candidate, containing: point, depth: depth + 1)
304+
}
305+
306+
private static func area(of element: AXUIElement) -> CGFloat {
307+
let frame = frameScreen(of: element)
308+
return frame.width * frame.height
287309
}
288310

289311
/// Climb `kAXParentAttribute` from `element` up to the window, returning the
@@ -307,6 +329,29 @@ enum AXIntrospection {
307329
/// to the whole window (and show its title in the composer header). Structural
308330
/// `AXGroup`s are skipped unless they are actionable or identified, so a
309331
/// near-miss lands on the nearest meaningful control rather than a wrapper.
332+
/// Window-chrome subroles: the traffic lights (+ the full-screen affordance).
333+
/// They are real, actionable `AXButton`s, but they are the WINDOW's chrome,
334+
/// not app content — their selectors locate no app code, so the hit-test must
335+
/// never offer them as annotation targets. Checked in BOTH resolvers below:
336+
/// `deepestNonContainer` is the fallback path, so filtering only
337+
/// `nearestIdentified` would just hand the rejected button back via the
338+
/// fallback.
339+
static let chromeSubroles: Set<String> = [
340+
kAXCloseButtonSubrole as String,
341+
kAXMinimizeButtonSubrole as String,
342+
kAXZoomButtonSubrole as String,
343+
kAXFullScreenButtonSubrole as String,
344+
]
345+
346+
/// True when `subrole` marks window chrome (see ``chromeSubroles``).
347+
static func isWindowChrome(subrole: String) -> Bool {
348+
chromeSubroles.contains(subrole)
349+
}
350+
351+
private static func isChrome(_ element: AXUIElement) -> Bool {
352+
isWindowChrome(subrole: string(element, kAXSubroleAttribute) ?? "")
353+
}
354+
310355
private static func nearestIdentified(in rootFirstChain: [AXUIElement]) -> AXUIElement? {
311356
for element in rootFirstChain.reversed() {
312357
let role = string(element, kAXRoleAttribute) ?? ""
@@ -328,9 +373,24 @@ enum AXIntrospection {
328373
/// to that leaf rather than escalating to the whole window (which would then
329374
/// show the window title in the composer header).
330375
private static func deepestNonContainer(in rootFirstChain: [AXUIElement]) -> AXUIElement? {
376+
// A structural, unidentified group that spans (nearly) the whole window is
377+
// the window in disguise — NSHostingView's root AXGroup covers the full
378+
// window frame, so falling back to it is the same "background click
379+
// highlights the whole app" bug the window/application skip guards
380+
// against. Detect it geometrically: the group's frame swallows the
381+
// window's frame minus a small inset.
382+
let windowFrame = rootFirstChain.first { string($0, kAXRoleAttribute) == "AXWindow" }
383+
.map(frameScreen(of:))
331384
for element in rootFirstChain.reversed() {
332385
let role = string(element, kAXRoleAttribute) ?? ""
333386
if role == "AXWindow" || role == "AXApplication" { continue }
387+
if role == "AXGroup",
388+
(string(element, kAXIdentifierAttribute) ?? "").isEmpty,
389+
labelText(element).isEmpty,
390+
let windowFrame,
391+
frameScreen(of: element).contains(windowFrame.insetBy(dx: 8, dy: 8)) {
392+
continue
393+
}
334394
return element
335395
}
336396
return nil

Sources/AnnotKitOverlayProbe/main.swift

Lines changed: 202 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -896,17 +896,193 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate {
896896

897897
self.resizeController?.unmount()
898898
host.orderOut(nil)
899+
self.phase4Chrome()
900+
}
901+
}
902+
903+
// ---- Phase 4: window CHROME is never an annotation target ---------------
904+
// The traffic lights are real, actionable AXButtons, so before the subrole
905+
// filter the hit-test offered them as targets (user bug: hovering close/
906+
// minimize/zoom highlighted them). While annotating, the native point query
907+
// hits the expanded overlay and is DISCARDED, so chrome resolves via the
908+
// geometric hitBeneathOverlay path — this phase exercises that real path
909+
// end-to-end (filter must cover deepestNonContainer too, or the fallback
910+
// hands the rejected button straight back), plus a positive control proving
911+
// real content still resolves.
912+
var chromeController: OverlayController?
913+
var chromeSession: AnnotationSession?
914+
var chromeHost: NSWindow?
915+
var passChrome = true
916+
func check4(_ cond: Bool, _ msg: String) {
917+
passChrome = passChrome && cond
918+
print(" " + (cond ? "ok " : "FAIL ") + msg)
919+
}
920+
921+
func phase4Chrome() {
922+
print("\n--- Phase 4: window chrome (traffic lights) is never an annotation target ---")
923+
// Retire Phase 1's host: titled windows are constrained ON-SCREEN by
924+
// AppKit, and a still-ordered-in W1 overlapping this phase's host makes
925+
// the geometric hit-test descend the WRONG window.
926+
h1?.orderOut(nil)
927+
// .miniaturizable + .resizable so all three traffic lights exist (the
928+
// other probe hosts are only titled+closable).
929+
let window = NSWindow(
930+
contentRect: NSRect(x: 0, y: 0, width: 480, height: 320),
931+
styleMask: [.titled, .closable, .miniaturizable, .resizable],
932+
backing: .buffered,
933+
defer: false
934+
)
935+
window.title = "AnnotKit Harness W4 (chrome)"
936+
window.contentView = NSHostingView(
937+
rootView: Button("Probe Target") {}
938+
.accessibilityIdentifier("Probe.ChromeContent")
939+
.padding(40)
940+
)
941+
window.makeKeyAndOrderFront(nil)
942+
chromeHost = window
943+
944+
let session = AnnotationSession(
945+
source: MacElementSource(),
946+
sink: NotesFileSink(path: NSTemporaryDirectory() + "annotkit-chrome.md")
947+
)
948+
let controller = OverlayController(session: session)
949+
controller.mount(on: window)
950+
controller.start() // expand, so hits resolve through the geometric path
951+
chromeController = controller
952+
chromeSession = session
953+
954+
DispatchQueue.main.asyncAfter(deadline: .now() + 0.9) { [weak self] in
955+
guard let self else { return }
956+
let source = MacElementSource()
957+
// Chrome buttons via raw AX (subrole is not part of the public
958+
// Element); the content control via the public snapshot.
959+
let app = AXUIElementCreateApplication(ProcessInfo.processInfo.processIdentifier)
960+
let axWindow = AX.windows(app).first {
961+
AX.string($0, kAXTitleAttribute) == "AnnotKit Harness W4 (chrome)"
962+
}
963+
let chromeSubroles: Set<String> = [
964+
kAXCloseButtonSubrole as String,
965+
kAXMinimizeButtonSubrole as String,
966+
kAXZoomButtonSubrole as String,
967+
kAXFullScreenButtonSubrole as String,
968+
]
969+
let chrome = axWindow.map { self.axFindAll(in: $0, subroles: chromeSubroles, depth: 0) } ?? []
970+
check4(chrome.count >= 3, "host exposes >=3 traffic-light AXButtons (found \(chrome.count))")
971+
for button in chrome {
972+
let frame = AX.frame(button)
973+
let subrole = AX.string(button, kAXSubroleAttribute)
974+
let hit = source.hitTest(CGPoint(x: frame.midX, y: frame.midY))
975+
print(" \(subrole) center=\(fmt(frame)) -> \(hit.map { "#\($0.id)" } ?? "nil")")
976+
check4(hit == nil, "\(subrole) is NOT an annotation target")
977+
}
978+
if let content = self.findElement(id: "Probe.ChromeContent", in: source.snapshot().map(\.root)) {
979+
let hit = source.hitTest(CGPoint(x: content.frame.midX, y: content.frame.midY))
980+
check4(hit?.id == "Probe.ChromeContent",
981+
"content control still resolves through the expanded overlay (got \(hit.map { "#\($0.id)" } ?? "nil"))")
982+
} else {
983+
check4(false, "content control Probe.ChromeContent present in the snapshot")
984+
}
985+
self.chromeController?.unmount()
986+
window.orderOut(nil)
987+
self.phase5Card()
988+
}
989+
}
990+
991+
// ---- Phase 5: a seeded CONTAINER resolves when its BODY is hovered -------
992+
// Mirrors the HUD-card fix: cards get .accessibilityElement(children:
993+
// .contain) + an identifier, and hovering the card body (inside the card,
994+
// outside any child) must resolve to the CARD, while hovering a child still
995+
// resolves the child. Validates nearestIdentified's identified-container
996+
// branch through the expanded overlay.
997+
var cardController: OverlayController?
998+
var cardSession: AnnotationSession?
999+
var cardHost: NSWindow?
1000+
var passCard = true
1001+
func check5(_ cond: Bool, _ msg: String) {
1002+
passCard = passCard && cond
1003+
print(" " + (cond ? "ok " : "FAIL ") + msg)
1004+
}
1005+
1006+
func phase5Card() {
1007+
print("\n--- Phase 5: seeded container (card) resolves on body hover ---")
1008+
let window = NSWindow(
1009+
contentRect: NSRect(x: 0, y: 0, width: 480, height: 360),
1010+
styleMask: [.titled, .closable],
1011+
backing: .buffered,
1012+
defer: false
1013+
)
1014+
window.title = "AnnotKit Harness W5 (card)"
1015+
window.contentView = NSHostingView(rootView: ProbeCardView())
1016+
window.makeKeyAndOrderFront(nil)
1017+
cardHost = window
1018+
1019+
let session = AnnotationSession(
1020+
source: MacElementSource(),
1021+
sink: NotesFileSink(path: NSTemporaryDirectory() + "annotkit-card.md")
1022+
)
1023+
let controller = OverlayController(session: session)
1024+
controller.mount(on: window)
1025+
controller.start()
1026+
cardController = controller
1027+
cardSession = session
1028+
1029+
DispatchQueue.main.asyncAfter(deadline: .now() + 0.9) { [weak self] in
1030+
guard let self else { return }
1031+
let source = MacElementSource()
1032+
let roots = source.snapshot().map(\.root)
1033+
guard let card = self.findElement(id: "Probe.Card", in: roots),
1034+
let text = self.findElement(id: "Probe.CardText", in: roots) else {
1035+
check5(false, "snapshot exposes Probe.Card + Probe.CardText")
1036+
self.cardController?.unmount()
1037+
window.orderOut(nil)
1038+
self.finish()
1039+
return
1040+
}
1041+
// A body point: inside the card, left of the (centered) text child.
1042+
let body = CGPoint(x: card.frame.minX + 16, y: card.frame.midY)
1043+
check5(!text.frame.contains(body), "sanity: the body point is outside the text child")
1044+
let bodyHit = source.hitTest(body)
1045+
print(" card=\(fmt(card.frame)) text=\(fmt(text.frame)) body-hit -> \(bodyHit.map { "#\($0.id)" } ?? "nil")")
1046+
check5(bodyHit?.id == "Probe.Card",
1047+
"hovering the card BODY resolves to the seeded container (got \(bodyHit.map { "#\($0.id)" } ?? "nil"))")
1048+
let textHit = source.hitTest(CGPoint(x: text.frame.midX, y: text.frame.midY))
1049+
check5(textHit?.id == "Probe.CardText",
1050+
"hovering a child still resolves the CHILD, not the container (got \(textHit.map { "#\($0.id)" } ?? "nil"))")
1051+
self.cardController?.unmount()
1052+
window.orderOut(nil)
8991053
self.finish()
9001054
}
9011055
}
9021056

1057+
/// Recursive raw-AX search for elements matching one of `subroles`.
1058+
func axFindAll(in element: AXUIElement, subroles: Set<String>, depth: Int) -> [AXUIElement] {
1059+
guard depth < 12 else { return [] }
1060+
var out: [AXUIElement] = []
1061+
if subroles.contains(AX.string(element, kAXSubroleAttribute)) { out.append(element) }
1062+
for child in AX.children(element) {
1063+
out.append(contentsOf: axFindAll(in: child, subroles: subroles, depth: depth + 1))
1064+
}
1065+
return out
1066+
}
1067+
1068+
/// Depth-first search of the public snapshot tree by element id.
1069+
func findElement(id: String, in elements: [Element]) -> Element? {
1070+
for element in elements {
1071+
if element.id == id { return element }
1072+
if let found = findElement(id: id, in: element.children) { return found }
1073+
}
1074+
return nil
1075+
}
1076+
9031077
func finish() {
9041078
print("\n issue-2 (per-control hit-test through the expanded overlay): \(passIssue2 ? "PASS" : "FAIL")")
9051079
print(" issue-1 (retention / copy / export / pill persistence): \(pass1 ? "PASS" : "FAIL")")
9061080
print(" Feature 1 (numbered-pin MODEL: anchors + update + delete/reflow): \(passPins ? "PASS" : "FAIL")")
9071081
print(" Phase 3 (mis-placed-pill regression: pill + axOrigin track FINAL frame): \(passResize ? "PASS" : "FAIL")")
1082+
print(" Phase 4 (window chrome excluded from the hit-test): \(passChrome ? "PASS" : "FAIL")")
1083+
print(" Phase 5 (seeded container resolves on body hover): \(passCard ? "PASS" : "FAIL")")
9081084
print("\n=== AnnotKitOverlayProbe complete ===")
909-
exit(pass1 && passIssue2 && passPins && passResize ? 0 : 1)
1085+
exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard ? 0 : 1)
9101086
}
9111087

9121088
func collectIDs(_ elements: [Element]) -> [String] {
@@ -919,6 +1095,31 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate {
9191095
}
9201096
}
9211097

1098+
/// Phase 5 host content: the HUD-card pattern under test. NOTE a plain
1099+
/// `.accessibilityElement(children: .contain)` container is NOT enough: SwiftUI
1100+
/// reports the contained group's AX frame as the UNION OF ITS CHILDREN (probed:
1101+
/// card frame == text frame), so the card's visual padding is dead to the AX
1102+
/// hit-test. The working pattern is an explicit full-size accessibility SURFACE:
1103+
/// a clear background carrying the card's identifier, whose AX frame is the
1104+
/// card's real visual bounds. Hovering the body hits the surface; hovering a
1105+
/// child still hits the child (the surface sits below content in AX z-order).
1106+
struct ProbeCardView: View {
1107+
var body: some View {
1108+
VStack {
1109+
Text("Card body probe")
1110+
.accessibilityIdentifier("Probe.CardText")
1111+
}
1112+
.padding(60)
1113+
.background(
1114+
Color.clear
1115+
.contentShape(Rectangle())
1116+
.accessibilityElement(children: .ignore)
1117+
.accessibilityIdentifier("Probe.Card")
1118+
)
1119+
.padding(40)
1120+
}
1121+
}
1122+
9221123
let app = NSApplication.shared
9231124
app.setActivationPolicy(.accessory)
9241125
let delegate = OverlayProbeDelegate()

0 commit comments

Comments
 (0)