Skip to content

Notch grab: fn+drag that owns the drag — multi-capture, typeable note, top-anchored #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sameeeeeeep merged 3 commits into from
Aug 15, 2026
Merged

Notch grab: fn+drag that owns the drag — multi-capture, typeable note, top-anchored #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b8d8030
feat(notch): fn+drag grab that owns the drag — multi-capture, typeabl…
sameeeeeeep Aug 14, 2026
896aef6
feat(notch): direct-grab mode (mode:"grab") — /screen + /reference sk…
sameeeeeeep Aug 15, 2026
19d4ff4
feat(notch): grab chips as a compact stack + count (no title crowding)
sameeeeeeep Aug 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 43 additions & 9 deletions packages/menubar/CursorGuide.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import ApplicationServices // AXIsProcessTrusted — the permission strip chec

// MARK: - Model

enum GuideMode: String { case tour, test, teach }
enum GuideMode: String { case tour, test, teach, grab }

// A picture or looping GIF shown in a step's media zone (show-don't-tell). `src` is an absolute file
// path or an http(s) URL; loaded lazily when the step appears (docs/GUIDE-CARD-SPEC §6 media states).
Expand Down Expand Up @@ -158,9 +158,10 @@ indirect enum Predicate {
// A screenshot and/or a note the human left on a step during a guide run. Both optional and
// independent — a step may have a shot, a note, both, or (the common case) neither.
struct StepFeedback {
var screenshot: String? // absolute path to an fn-drag jpg (in NSTemporaryDirectory)
var note: String? // raw typed + dictated text, no cleanup
var isEmpty: Bool { (screenshot?.isEmpty ?? true) && (note?.isEmpty ?? true) }
var screenshot: String? // FIRST fn-drag jpg (back-compat; = screenshots.first)
var screenshots: [String] = [] // ALL grabs on this step — multiple fn-drags accumulate here
var note: String? // raw typed + dictated text, no cleanup
var isEmpty: Bool { screenshots.isEmpty && (screenshot?.isEmpty ?? true) && (note?.isEmpty ?? true) }
}

struct GuideResult {
Expand Down Expand Up @@ -992,26 +993,51 @@ final class CursorGuide {
return ("In your own words", "text.bubble.fill", false)
}

// Direct grab (mode:"grab") — the /screen + /reference path. No guide card, no pill, no ⌥↓ to arm:
// set up ONE synthetic result and drop straight into the feedback grab + note panel (fn+drag captures,
// multiple accumulate, note field owns the top of the screen). ↵ saves → the run finishes and writes
// guide-result.json with feedback.screenshots + note, exactly like the normal feedback path.
private func beginGrab(title: String, source: String?, project: String?) {
self.steps = [GuideStep(id: "grab", text: title, hint: nil)]
self.idx = 0
self.results = [GuideResult(id: "grab", text: title, verdict: "done", notedAt: nil)]
self.startedAt = Date()
self.isActive = true
self.autoClipboard = false
clipboardSaved = false; savedClipboard = nil
model.source = source
model.project = project
model.mode = .grab
model.done = nil; model.target = nil
model.visible = false // never render a card — the note panel is the only surface
ensureOverlay() // the overlay panel exists (invisible) so keys/monitors work
installMonitors()
beginFeedback() // straight into the grab + note
}

// Enter capture for the CURRENT step. The verdict is already set by the time this runs, so it
// never changes it — it just opens capture.
private func beginFeedback() {
guard isActive, idx < results.count, !capturingFeedback else { return }
capturingFeedback = true
feedbackIdx = idx
if results[idx].feedback == nil { results[idx].feedback = StepFeedback() }
// Collapse the card to its pill so it vacates the notch — the feedback note drops BELOW it instead of
// being hidden behind it. Restored on endFeedback.
// Hide the guide card/pill ENTIRELY while capturing, so the feedback note panel owns the top of the
// screen (no collapsed pill sitting above it). Restored on endFeedback.
wasCollapsedBeforeFeedback = model.collapsed
model.collapsed = true
wasVisibleBeforeFeedback = model.visible
model.visible = false
onFeedbackBegin?(results[idx].id)
}
private var wasCollapsedBeforeFeedback = false
private var wasVisibleBeforeFeedback = true

// RelayController pushes the fn-drag jpg here as soon as a region is grabbed.
func attachFeedbackScreenshot(_ path: String) {
guard capturingFeedback, let i = feedbackIdx, i < results.count else { return }
var fb = results[i].feedback ?? StepFeedback()
fb.screenshot = path
fb.screenshots.append(path) // ACCUMULATE — multiple grabs per step
if fb.screenshot == nil { fb.screenshot = path } // legacy single = the first grab
results[i].feedback = fb
}

Expand Down Expand Up @@ -1047,6 +1073,7 @@ final class CursorGuide {
capturingFeedback = false
feedbackIdx = nil
model.collapsed = wasCollapsedBeforeFeedback // re-expand the card (unless it was already collapsed)
model.visible = wasVisibleBeforeFeedback // re-show the guide card (hidden during capture)
onFeedbackEnd?()
}

Expand Down Expand Up @@ -1160,6 +1187,12 @@ final class CursorGuide {
guard let obj = raw as? [String: Any] else { logMalformed(); return }
let m = GuideMode(rawValue: (obj["mode"] as? String) ?? "") ?? defaultMode
let title = (obj["title"] as? String) ?? "Untitled"
// Direct-grab mode (/screen + /reference): NO guide card at all — go straight to the fn+drag grab +
// note panel. Needs no `steps`, so branch out before the steps guard below.
if m == .grab {
beginGrab(title: title, source: obj["source"] as? String, project: obj["project"] as? String)
return
}
guard let rawSteps = obj["steps"] as? [[String: Any]], !rawSteps.isEmpty else { logMalformed(); return }

// Run-level: shot describes the pixel space the step `point`s live in; autoClipboard opts into
Expand Down Expand Up @@ -1747,7 +1780,8 @@ final class CursorGuide {
]
if let fb = r.feedback, !fb.isEmpty {
var fbo: [String: Any] = [:]
if let s = fb.screenshot, !s.isEmpty { fbo["screenshot"] = s }
if let s = fb.screenshot, !s.isEmpty { fbo["screenshot"] = s } // first (back-compat)
if !fb.screenshots.isEmpty { fbo["screenshots"] = fb.screenshots } // ALL grabs
if let n = fb.note, !n.isEmpty { fbo["note"] = n }
d["feedback"] = fbo
}
Expand Down
67 changes: 51 additions & 16 deletions packages/menubar/RelayMenuBar.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3612,7 +3612,7 @@ struct FeedbackNoteDrop: View {
var title: String = "In your own words" // neutral by default — ⌥↓ is a general note, not always an error
var icon: String = "text.bubble.fill"
var danger: Bool = false // true only when the current step actually failed
var shotThumb: NSImage? // the fn-drag grab's chip, nil until grabbed
var shotThumbs: [NSImage] = [] // the fn-drag grabs, in order — each new grab ACCUMULATES (a row of chips)
var onCommit: () -> Void // ↵ / Save
var onCancel: () -> Void // esc / Discard
@FocusState private var focused: Bool
Expand All @@ -3622,10 +3622,27 @@ struct FeedbackNoteDrop: View {
Image(systemName: icon).font(.system(size: 13)).foregroundColor(danger ? .danger : .lime)
Text(title).font(.hanken(13, .semibold)).foregroundColor(.ink)
Spacer(minLength: 0)
if let t = shotThumb {
Image(nsImage: t).resizable().aspectRatio(contentMode: .fit)
.frame(width: 40, height: 26).cornerRadius(4)
.overlay(RoundedRectangle(cornerRadius: 4).stroke(Color.lime.opacity(0.5), lineWidth: 1))
// Grab chips as a compact STACK (+ count) rather than a wide row — a FIXED footprint so the
// title never gets crowded/wrapped no matter how many you grab.
if !shotThumbs.isEmpty {
ZStack(alignment: .topTrailing) {
ZStack {
ForEach(Array(shotThumbs.suffix(3).enumerated()), id: \.offset) { i, t in
Image(nsImage: t).resizable().aspectRatio(contentMode: .fill)
.frame(width: 34, height: 24).clipped().cornerRadius(4)
.overlay(RoundedRectangle(cornerRadius: 4).stroke(Color.lime.opacity(0.5), lineWidth: 1))
.rotationEffect(.degrees(Double(i - 1) * 3))
.offset(x: CGFloat(i) * 5)
}
}
if shotThumbs.count > 1 {
Text("\(shotThumbs.count)").font(.splMono(9.5)).foregroundColor(.page)
.padding(.horizontal, 4).padding(.vertical, 1)
.background(Capsule().fill(Color.lime))
.offset(x: 9, y: -6)
}
}
.frame(width: 52, height: 28)
}
}
TextField("type a note — or hold ⌃⌥ to dictate", text: $note, axis: .vertical)
Expand All @@ -3642,7 +3659,7 @@ struct FeedbackNoteDrop: View {
Text("esc discard").font(.splMono(9)).foregroundColor(.inkDim)
}
}
.padding(.horizontal, 20).padding(.top, 12).padding(.bottom, 12) // sits below the notch now (dropped panel)
.padding(.horizontal, 20).padding(.top, 40).padding(.bottom, 12) // top-pad clears the menu bar/notch (panel drops from the very top edge)
.frame(width: 300)
.padding(.horizontal, 14) // notch ears
.background(Color.page)
Expand Down Expand Up @@ -3940,7 +3957,7 @@ struct ActionConsentDrop: View {
private var lastToolInput: [String: String] = [:] // the query per tool id, to re-run after a key is set
private struct PendingCredential { let l: SBListing; let binding: SBMcpBinding; let tool: String; let input: String?; let env: String; let label: String; let hint: String }
private var feedbackNote = ""
private var feedbackShotThumb: NSImage?
private var feedbackShotThumbs: [NSImage] = []
private var feedbackKeyMonitor: Any?
private var fnCaptureActive = false // an fn+click/fn+drag capture gesture is in progress
private var lastGodCaptureIntentional = false // the last ⌃⌃ did an explicit fn capture → usable as an image reference
Expand Down Expand Up @@ -5638,7 +5655,7 @@ struct ActionConsentDrop: View {
// Raised from CursorGuide.onFeedbackBegin: shows a focused note field + arms the fn-drag screenshot grab.
// ⌃⌥ dictation stays live during a guide, and finishDictation routes its transcript here (not the app).
@MainActor private func showFeedbackNote() {
feedbackNote = ""; feedbackShotThumb = nil
feedbackNote = ""; feedbackShotThumbs = []
guard let screen = statusItem?.button?.window?.screen ?? NSScreen.main else { return }
rebuildFeedbackPanel(screen)
// The panel is key → it owns ↵ / esc → CursorGuide commits/cancels. CursorGuide.onKey no-ops while
Expand All @@ -5665,7 +5682,7 @@ struct ActionConsentDrop: View {
Task { @MainActor in
guard let self else { return }
CursorGuide.shared.attachFeedbackScreenshot(path)
self.feedbackShotThumb = NSImage(contentsOfFile: path)
if let img = NSImage(contentsOfFile: path) { self.feedbackShotThumbs.append(img) }
if let scr = self.statusItem?.button?.window?.screen ?? NSScreen.main { self.rebuildFeedbackPanel(scr) } // show the chip
}
}
Expand All @@ -5677,7 +5694,7 @@ struct ActionConsentDrop: View {
note: Binding(get: { [weak self] in self?.feedbackNote ?? "" },
set: { [weak self] in self?.feedbackNote = $0 }),
title: fp.title, icon: fp.icon, danger: fp.danger,
shotThumb: feedbackShotThumb,
shotThumbs: feedbackShotThumbs,
onCommit: { [weak self] in Task { @MainActor in self?.commitFeedbackFromField(cancel: false) } },
onCancel: { [weak self] in Task { @MainActor in self?.commitFeedbackFromField(cancel: true) } })
let host = NoInsetHostingView(rootView: view)
Expand All @@ -5694,7 +5711,11 @@ struct ActionConsentDrop: View {
feedbackPanel!.setContentSize(size)
// Drop the note BELOW the notch (not pinned to the very top) so it never hides behind the guide card
// or any notch content — the guide card collapses to its pill while capturing, and this sits under it.
feedbackPanel!.setFrameTopLeftPoint(NSPoint(x: screen.frame.midX - size.width / 2, y: screen.frame.maxY - feedbackNotchDrop))
// Anchor right below the menu bar (top of the usable area) — as high as possible while the text field
// stays CLICKABLE. Pinning at screen.frame.maxY tucked it under the menu bar/notch and made the field
// untypeable; visibleFrame.maxY is the menu-bar bottom, so it's "at the top" but reachable.
feedbackPanel!.setFrameTopLeftPoint(NSPoint(x: screen.frame.midX - size.width / 2, y: screen.frame.maxY + notchTopBleed)) // notch-top aligned to the top edge of the menu bar (drops from the very top); content is top-padded to clear it
feedbackPanel!.makeKeyAndOrderFront(nil) // regain key focus so typing works after a rebuild/grab
feedbackPanel!.makeKeyAndOrderFront(nil)
orb?.orderOut(nil)
presentFromNotch(feedbackPanel!)
Expand All @@ -5716,7 +5737,7 @@ struct ActionConsentDrop: View {
disarmFeedbackRegionCapture()
if let m = feedbackKeyMonitor { NSEvent.removeMonitor(m); feedbackKeyMonitor = nil }
if let p = feedbackPanel { dismissToNotch(p) }
feedbackNote = ""; feedbackShotThumb = nil
feedbackNote = ""; feedbackShotThumbs = []
}

// ── BUNDLED WEB SERVER — a packaged app ships examples/apps in Resources/webapps + the node binary, and
Expand Down Expand Up @@ -7237,9 +7258,21 @@ struct ActionConsentDrop: View {
let btnDown = (NSEvent.pressedMouseButtons & 0x1) != 0
let fn = NSEvent.modifierFlags.contains(.function)
let p = toView(NSEvent.mouseLocation)
// The overlay must be present + OWNING the mouse the instant fn is held — BEFORE the mouse-down —
// or that first down leaks to the app and starts a text selection. So arm on fn-hold, not on
// mouse-down. While fn is held the overlay is opaque to the mouse (swallows the whole drag); when
// fn is released and no drag is in flight it's click-through so normal clicking still works. This
// is the fix for the "fn+drag selects text / blank grab" bug.
if fn {
self.ensureRegionOverlay(screen)
self.regionOverlay?.ignoresMouseEvents = false
} else if self.feedbackRegionStart == nil {
// fn released and not mid-drag → REMOVE the overlay entirely (don't just make it click-through)
// so it can never sit above the note panel and block clicks/typing into the field.
if self.regionOverlay != nil { self.regionOverlay?.orderOut(nil); self.regionOverlay = nil; self.regionView = nil }
}
if btnDown && !self.feedbackPrevBtnDown { // ── DOWN edge
if fn { self.feedbackRegionStart = p; self.feedbackRegionMoved = false
self.ensureRegionOverlay(screen); self.regionView?.setSel(.zero) }
if fn { self.feedbackRegionStart = p; self.feedbackRegionMoved = false; self.regionView?.setSel(.zero) }
} else if btnDown, self.feedbackRegionStart != nil, let s = self.feedbackRegionStart { // ── dragging
if hypot(p.x - s.x, p.y - s.y) > 6 { self.feedbackRegionMoved = true }
self.regionView?.setSel(NSRect(x: min(s.x, p.x), y: min(s.y, p.y), width: abs(p.x - s.x), height: abs(p.y - s.y)))
Expand All @@ -7253,10 +7286,12 @@ struct ActionConsentDrop: View {
self.regionOverlay?.orderOut(nil); self.regionOverlay = nil; self.regionView = nil
let shot = NSTemporaryDirectory() + "guide-feedback-\(UUID().uuidString).jpg"
self.captureShot(pick, to: shot)
self.disarmFeedbackRegionCapture()
// STAY ARMED for the next grab (multiple screenshots per note, like God's chip-accumulating
// grab) — reset the per-grab state instead of disarming. The note panel accumulates thumbs.
self.feedbackRegionStart = nil; self.feedbackRegionMoved = false
if FileManager.default.fileExists(atPath: shot) {
NSSound(named: "Morse")?.play()
self.onFeedbackShot?(shot) // → CursorGuide.attachFeedbackScreenshot
self.onFeedbackShot?(shot) // → CursorGuide.attachFeedbackScreenshot (appends)
}
// if the file is missing (Screen-Recording ungranted) we simply don't attach a shot.
}
Expand Down