Hold synthesized iPad presses long enough for the host to see them

KVMConsoleiPad/Input/KeyboardCaptureView.swift

@@ -14,11 +14,13 @@ struct KeyboardCaptureView: UIViewRepresentable {
     let extraModifierByte: UInt8
     let pendingVirtualKey: VirtualKeyTap?
     let onKeyboardReport: @MainActor (HIDKeyboardReport) -> Void
+    let onKeyboardRelease: @MainActor (HIDKeyboardReport) -> Void
     let onMomentaryModifiersConsumed: @MainActor () -> Void
 
     func makeUIView(context: Context) -> KeyboardCaptureUIView {
         let view = KeyboardCaptureUIView()
         view.onKeyboardReport = onKeyboardReport
+        view.onKeyboardRelease = onKeyboardRelease
         view.onMomentaryModifiersConsumed = onMomentaryModifiersConsumed
         view.isCaptureEnabled = isEnabled
         view.extraModifierByte = extraModifierByte
@@ -27,6 +29,7 @@ struct KeyboardCaptureView: UIViewRepresentable {
 
     func updateUIView(_ uiView: KeyboardCaptureUIView, context: Context) {
         uiView.onKeyboardReport = onKeyboardReport
+        uiView.onKeyboardRelease = onKeyboardRelease
         uiView.onMomentaryModifiersConsumed = onMomentaryModifiersConsumed
         uiView.isCaptureEnabled = isEnabled
         uiView.extraModifierByte = extraModifierByte
@@ -53,12 +56,20 @@ struct KeyboardCaptureView: UIViewRepresentable {
 }
 
 final class KeyboardCaptureUIView: UIView, UIKeyInput {
-    var isCaptureEnabled = false
+    var isCaptureEnabled = false {
+        didSet {
+            if !isCaptureEnabled {
+                virtualKeySequencer.cancelAll()
+            }
+        }
+    }
     var extraModifierByte: UInt8 = 0
     var onKeyboardReport: (@MainActor (HIDKeyboardReport) -> Void)?
+    var onKeyboardRelease: (@MainActor (HIDKeyboardReport) -> Void)?
     var onMomentaryModifiersConsumed: (@MainActor () -> Void)?
 
     private let builder = HIDKeyboardReportBuilder()
+    private let virtualKeySequencer = SynthesizedTapSequencer()
 
     override var canBecomeFirstResponder: Bool { true }
     var hasText: Bool { false }
@@ -84,6 +95,7 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput {
             super.pressesBegan(presses, with: event)
             return
         }
+        virtualKeySequencer.flushPendingRelease()
         for press in presses {
             guard let usage = press.key?.keyCode.rawValue, let keyUsage = UInt8(exactly: usage) else { continue }
             if let bit = HIDModifierBit.bit(forHIDUsage: keyUsage) {
@@ -115,7 +127,8 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput {
             super.pressesCancelled(presses, with: event)
             return
         }
-        emit(builder.reset())
+        virtualKeySequencer.flushPendingRelease()
+        emitRelease(builder.reset())
         super.pressesCancelled(presses, with: event)
     }
 
@@ -138,14 +151,23 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput {
         // Combine synthesized modifier bits with whatever real modifiers the builder is
         // currently tracking, then on release drop only the synthesized bits — this keeps a
         // hardware modifier (e.g. held BT-keyboard Shift) asserted on the host and prevents
-        // the synthesized one from sticking.
-        let down = HIDKeyboardReport(
-            modifier: builder.modifierByte | transientModifier,
-            keycodes: [usage]
+        // the synthesized one from sticking. The release re-reads the builder rather than
+        // capturing it, so a hardware modifier pressed during the hold survives too.
+        virtualKeySequencer.tap(
+            down: { [weak self] in
+                guard let self else { return }
+                self.emit(HIDKeyboardReport(
+                    modifier: self.builder.modifierByte | transientModifier,
+                    keycodes: [usage]
+                ))
+            },
+            up: { [weak self] in
+                guard let self else { return }
+                // The builder's current state, not an empty report: a hardware key pressed during
+                // the hold is still down and must not be released along with the synthesized one.
+                self.emitRelease(self.builder.currentReport)
+            }
         )
-        let up = HIDKeyboardReport(modifier: builder.modifierByte, keycodes: [])
-        emit(down)
-        emit(up)
     }
 
     private func withExtraModifiers(_ report: HIDKeyboardReport, eventModifiers: UIKeyModifierFlags) -> HIDKeyboardReport {
@@ -166,6 +188,15 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput {
         }
     }
 
+    /// Releases go through their own channel: by the time capture is switched off the guarded
+    /// path drops everything, which would leave a key asserted — and auto-repeating — on the host.
+    private func emitRelease(_ report: HIDKeyboardReport) {
+        guard let onKeyboardRelease else { return }
+        MainActor.assumeIsolated {
+            onKeyboardRelease(report)
+        }
+    }
+
     private func consumeMomentary() {
         guard let onMomentaryModifiersConsumed else { return }
         MainActor.assumeIsolated {

KVMConsoleiPad/Input/PointerCaptureView.swift

@@ -8,10 +8,12 @@ struct PointerCaptureView: UIViewRepresentable {
     let videoSize: CGSize?
     let zoom: ViewerZoomState
     let onMouseReport: @MainActor (HIDMouseAbsoluteReport) -> Void
+    let onMouseRelease: @MainActor (HIDMouseAbsoluteReport) -> Void
 
     func makeUIView(context: Context) -> PointerCaptureUIView {
         let view = PointerCaptureUIView()
         view.onMouseReport = onMouseReport
+        view.onMouseRelease = onMouseRelease
         view.isCaptureEnabled = isEnabled
         view.isScrollInverted = isScrollInverted
         view.videoSize = videoSize
@@ -21,6 +23,7 @@ struct PointerCaptureView: UIViewRepresentable {
 
     func updateUIView(_ uiView: PointerCaptureUIView, context: Context) {
         uiView.onMouseReport = onMouseReport
+        uiView.onMouseRelease = onMouseRelease
         uiView.isCaptureEnabled = isEnabled
         uiView.isScrollInverted = isScrollInverted
         uiView.videoSize = videoSize
@@ -32,18 +35,20 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate {
     var isCaptureEnabled = false {
         didSet {
             if !isCaptureEnabled {
-                releaseActiveDragIfNeeded()
+                releaseHeldButtons()
             }
         }
     }
     var isScrollInverted = true
     var videoSize: CGSize?
     var zoom: ViewerZoomState?
     var onMouseReport: (@MainActor (HIDMouseAbsoluteReport) -> Void)?
+    var onMouseRelease: (@MainActor (HIDMouseAbsoluteReport) -> Void)?
 
     private let mouseReportBuilder = HIDMouseAbsoluteReportBuilder()
     private var scrollAccumulator = MouseScrollAccumulator()
     private var activeDragButtonNumber: Int?
+    private let clickSequencer = SynthesizedTapSequencer()
     private weak var pinchRecognizer: UIPinchGestureRecognizer?
     private weak var pointerPanRecognizer: UIPanGestureRecognizer?
     private var pinchAnchorVideo: CGPoint?
@@ -83,6 +88,9 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate {
 
     @objc private func handleHover(_ recognizer: UIHoverGestureRecognizer) {
         guard isCaptureEnabled else { return }
+        // `move` keeps the buttons byte, so emitting one while a synthesized click is still held
+        // would read on the host as a drag. The pointer catches up when the click releases.
+        guard !clickSequencer.isHoldingPress else { return }
         let location = recognizer.location(in: self)
         let effective = effectiveRect()
         let normalized = MouseCoordinateMapper.normalizedPoint(clientPoint: location, effectiveRect: effective)
@@ -94,7 +102,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate {
 
     @objc private func handlePan(_ recognizer: UIPanGestureRecognizer) {
         guard isCaptureEnabled else {
-            releaseActiveDragIfNeeded()
+            releaseHeldButtons()
             return
         }
         let location = recognizer.location(in: self)
@@ -110,13 +118,15 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate {
         switch recognizer.state {
         case .began:
             if let dragButtonNumber {
+                clickSequencer.cancelAll()
                 activeDragButtonNumber = dragButtonNumber
                 emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y))
             } else {
                 emit(mouseReportBuilder.move(x: point.x, y: point.y))
             }
         case .changed:
             if activeDragButtonNumber == nil, let dragButtonNumber {
+                clickSequencer.cancelAll()
                 activeDragButtonNumber = dragButtonNumber
                 emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y))
             }
@@ -217,15 +227,24 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate {
         let effective = effectiveRect()
         let normalized = MouseCoordinateMapper.normalizedPoint(clientPoint: location, effectiveRect: effective)
         let point = MouseCoordinateMapper.absolutePoint(clientPoint: location, effectiveRect: effective)
-        emit(mouseReportBuilder.buttonDown(buttonNumber: buttonNumber, x: point.x, y: point.y))
-        emit(mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y))
+        clickSequencer.tap(
+            down: { [weak self] in
+                guard let self else { return }
+                self.emit(self.mouseReportBuilder.buttonDown(buttonNumber: buttonNumber, x: point.x, y: point.y))
+            },
+            up: { [weak self] in
+                guard let self else { return }
+                self.emitRelease(self.mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y))
+            }
+        )
         zoom?.cursorNormalized = normalized
     }
 
-    private func releaseActiveDragIfNeeded() {
+    private func releaseHeldButtons() {
+        clickSequencer.cancelAll()
         guard activeDragButtonNumber != nil else { return }
         activeDragButtonNumber = nil
-        emit(mouseReportBuilder.reset())
+        emitRelease(mouseReportBuilder.reset())
     }
 
     // UIGestureRecognizer callbacks run on the main thread; assuming
@@ -239,6 +258,15 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate {
         }
     }
 
+    /// Releases go through their own channel: by the time capture is switched off the guarded
+    /// path drops everything, which would strand a held button on the host.
+    private func emitRelease(_ report: HIDMouseAbsoluteReport) {
+        guard let onMouseRelease else { return }
+        MainActor.assumeIsolated {
+            onMouseRelease(report)
+        }
+    }
+
     private func effectiveRect() -> CGRect {
         let baseRect = MouseCoordinateMapper.aspectFitRect(for: videoSize, in: bounds)
         guard let zoom else { return baseRect }

KVMConsoleiPad/Input/SynthesizedTapSequencer.swift

@@ -0,0 +1,93 @@
+import Foundation
+
+/// Gives a synthesized press — a tap-to-click, an on-screen key — a duration.
+///
+/// A tap has no dwell of its own, so the down and up reports would otherwise be emitted in the
+/// same instant. HID reports carry *state*, not events: the KVM holds the most recent report and
+/// the attached host only samples it when it polls, every few milliseconds. A press and release
+/// that land inside one polling window collapse into the released state, so the host sees
+/// nothing happen at all. Holding for `pressDuration` guarantees at least one poll observes the
+/// press.
+///
+/// Taps arriving while one is still held are queued rather than merged — `insertText` synthesizes
+/// a whole string's keystrokes in one pass, and each of them needs its own hold. The first `down`
+/// is emitted synchronously so a tap never pays a run-loop hop before the host sees it.
+///
+/// Real presses need none of this: a held finger, a physical key, and a drag all supply their own
+/// duration.
+@MainActor
+final class SynthesizedTapSequencer {
+    nonisolated static let defaultPressDuration = Duration.milliseconds(50)
+
+    private typealias Tap = (down: @MainActor () -> Void, up: @MainActor () -> Void)
+
+    private let pressDuration: Duration
+    private let sleep: @Sendable (Duration) async -> Void
+    private var queue: [Tap] = []
+    private var pendingRelease: (@MainActor () -> Void)?
+    private var holdTask: Task<Void, Never>?
+
+    init(
+        pressDuration: Duration = SynthesizedTapSequencer.defaultPressDuration,
+        sleep: @escaping @Sendable (Duration) async -> Void = { try? await Task.sleep(for: $0) }
+    ) {
+        self.pressDuration = pressDuration
+        self.sleep = sleep
+    }
+
+    /// Whether a press is currently being held. Callers suppress pointer moves while it is true,
+    /// so a click can't turn into a drag just because the pointer drifted during the hold.
+    var isHoldingPress: Bool { pendingRelease != nil }
+
+    /// Emits `down` now and `up` once the press has been held long enough to be observed.
+    func tap(down: @escaping @MainActor () -> Void, up: @escaping @MainActor () -> Void) {
+        queue.append((down, up))
+        startNextTap()
+    }
+
+    /// Emits a still-held release immediately, leaving anything queued behind it to drain. Use
+    /// this when the current press has to end early but the remaining input is still wanted — an
+    /// interrupted paste, say, whose remaining characters would otherwise vanish.
+    func flushPendingRelease() {
+        holdTask?.cancel()
+        holdTask = nil
+        releasePendingIfNeeded()
+        startNextTap()
+    }
+
+    /// Releases the held press and discards everything queued behind it — for when the pending
+    /// input is no longer wanted at all: a drag taking the button over, or capture switching off.
+    func cancelAll() {
+        queue.removeAll()
+        holdTask?.cancel()
+        holdTask = nil
+        releasePendingIfNeeded()
+    }
+
+    private func startNextTap() {
+        guard holdTask == nil, !queue.isEmpty else { return }
+        let tap = queue.removeFirst()
+        // Arm the release before running `down`, so a re-entrant flush from inside it sees a press
+        // to release rather than silently leaving one armed behind its back.
+        pendingRelease = tap.up
+        holdTask = Task { [weak self, pressDuration, sleep] in
+            await sleep(pressDuration)
+            // A flush already emitted this release.
+            guard !Task.isCancelled else { return }
+            self?.finishTap()
+        }
+        tap.down()
+    }
+
+    private func finishTap() {
+        holdTask = nil
+        releasePendingIfNeeded()
+        startNextTap()
+    }
+
+    private func releasePendingIfNeeded() {
+        guard let pendingRelease else { return }
+        self.pendingRelease = nil
+        pendingRelease()
+    }
+}

KVMConsoleiPad/UI/ViewerView.swift

@@ -113,7 +113,8 @@ struct ViewerView: View {
                 isScrollInverted: model.isScrollInverted,
                 videoSize: model.videoSize,
                 zoom: model.zoom,
-                onMouseReport: { report in model.sendMouseReport(report) }
+                onMouseReport: { report in model.sendMouseReport(report) },
+                onMouseRelease: { report in model.sendInputRelease(mouse: report) }
             )
             .frame(maxWidth: .infinity, maxHeight: .infinity)
 
@@ -123,6 +124,7 @@ struct ViewerView: View {
                 extraModifierByte: modifierState.activeModifierByte,
                 pendingVirtualKey: pendingVirtualKey,
                 onKeyboardReport: { report in model.sendKeyboardReport(report) },
+                onKeyboardRelease: { report in model.sendInputRelease(keyboard: report) },
                 onMomentaryModifiersConsumed: { modifierState.consumeMomentary() }
             )
             .frame(width: 1, height: 1)