From 8b65d9b4db576fe46b74ec4ed278e62850c58692 Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Fri, 21 Aug 2026 20:58:26 +0800 Subject: [PATCH 1/3] Hold synthesized iPad presses long enough for the host to see them Tapping to click moved the pointer but never clicked, and modifier-bar keys behaved the same way. Both paths emitted their down and up reports in the same instant. HID reports carry state rather than events: the KVM holds the most recent report and the attached host only samples it when it polls, every few milliseconds, so a press and release landing inside one polling window collapse into the released state. The position survives; the press never happened. macOS is unaffected because it emits from mouseDown/mouseUp, which carry real human dwell. iPad is the only platform that synthesizes a press, and it synthesized one of zero duration. SynthesizedTapSequencer holds each press for 50ms before releasing it. It queues rather than merges, because insertText synthesizes a whole string's keystrokes in one pass and every one of them needs its own hold. The first down stays synchronous, so press latency is unchanged. The key release now re-reads the builder instead of capturing it at press time, so a hardware modifier pressed during the hold survives, and pressesCancelled flushes a pending release so an interrupted sequence can't leave a key held. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XU7Ck7AbZ9KpRhdPbRzNj8 --- .../Input/KeyboardCaptureView.swift | 23 ++-- KVMConsoleiPad/Input/PointerCaptureView.swift | 22 +++- .../Input/SynthesizedTapSequencer.swift | 78 +++++++++++++ KVMConsoleiPadTests/NanoKVMiPadTests.swift | 108 ++++++++++++++++++ 4 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 KVMConsoleiPad/Input/SynthesizedTapSequencer.swift diff --git a/KVMConsoleiPad/Input/KeyboardCaptureView.swift b/KVMConsoleiPad/Input/KeyboardCaptureView.swift index becd0bc..b0c7485 100644 --- a/KVMConsoleiPad/Input/KeyboardCaptureView.swift +++ b/KVMConsoleiPad/Input/KeyboardCaptureView.swift @@ -59,6 +59,7 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { var onMomentaryModifiersConsumed: (@MainActor () -> Void)? private let builder = HIDKeyboardReportBuilder() + private let virtualKeySequencer = SynthesizedTapSequencer() override var canBecomeFirstResponder: Bool { true } var hasText: Bool { false } @@ -115,6 +116,7 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { super.pressesCancelled(presses, with: event) return } + virtualKeySequencer.flushPendingRelease() emit(builder.reset()) super.pressesCancelled(presses, with: event) } @@ -138,14 +140,21 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { // Combine synthesized modifier bits with whatever real modifiers the builder is // currently tracking, then on release drop only the synthesized bits — this keeps a // hardware modifier (e.g. held BT-keyboard Shift) asserted on the host and prevents - // the synthesized one from sticking. - let down = HIDKeyboardReport( - modifier: builder.modifierByte | transientModifier, - keycodes: [usage] + // the synthesized one from sticking. The release re-reads the builder rather than + // capturing it, so a hardware modifier pressed during the hold survives too. + virtualKeySequencer.tap( + down: { [weak self] in + guard let self else { return } + self.emit(HIDKeyboardReport( + modifier: self.builder.modifierByte | transientModifier, + keycodes: [usage] + )) + }, + up: { [weak self] in + guard let self else { return } + self.emit(HIDKeyboardReport(modifier: self.builder.modifierByte, keycodes: [])) + } ) - let up = HIDKeyboardReport(modifier: builder.modifierByte, keycodes: []) - emit(down) - emit(up) } private func withExtraModifiers(_ report: HIDKeyboardReport, eventModifiers: UIKeyModifierFlags) -> HIDKeyboardReport { diff --git a/KVMConsoleiPad/Input/PointerCaptureView.swift b/KVMConsoleiPad/Input/PointerCaptureView.swift index 3cc6d6e..e0fba54 100644 --- a/KVMConsoleiPad/Input/PointerCaptureView.swift +++ b/KVMConsoleiPad/Input/PointerCaptureView.swift @@ -32,7 +32,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { var isCaptureEnabled = false { didSet { if !isCaptureEnabled { - releaseActiveDragIfNeeded() + releaseHeldButtons() } } } @@ -44,6 +44,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { private let mouseReportBuilder = HIDMouseAbsoluteReportBuilder() private var scrollAccumulator = MouseScrollAccumulator() private var activeDragButtonNumber: Int? + private let clickSequencer = SynthesizedTapSequencer() private weak var pinchRecognizer: UIPinchGestureRecognizer? private weak var pointerPanRecognizer: UIPanGestureRecognizer? private var pinchAnchorVideo: CGPoint? @@ -94,7 +95,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { @objc private func handlePan(_ recognizer: UIPanGestureRecognizer) { guard isCaptureEnabled else { - releaseActiveDragIfNeeded() + releaseHeldButtons() return } let location = recognizer.location(in: self) @@ -110,6 +111,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { switch recognizer.state { case .began: if let dragButtonNumber { + clickSequencer.flushPendingRelease() activeDragButtonNumber = dragButtonNumber emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y)) } else { @@ -117,6 +119,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { } case .changed: if activeDragButtonNumber == nil, let dragButtonNumber { + clickSequencer.flushPendingRelease() activeDragButtonNumber = dragButtonNumber emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y)) } @@ -217,12 +220,21 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { let effective = effectiveRect() let normalized = MouseCoordinateMapper.normalizedPoint(clientPoint: location, effectiveRect: effective) let point = MouseCoordinateMapper.absolutePoint(clientPoint: location, effectiveRect: effective) - emit(mouseReportBuilder.buttonDown(buttonNumber: buttonNumber, x: point.x, y: point.y)) - emit(mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y)) + clickSequencer.tap( + down: { [weak self] in + guard let self else { return } + self.emit(self.mouseReportBuilder.buttonDown(buttonNumber: buttonNumber, x: point.x, y: point.y)) + }, + up: { [weak self] in + guard let self else { return } + self.emit(self.mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y)) + } + ) zoom?.cursorNormalized = normalized } - private func releaseActiveDragIfNeeded() { + private func releaseHeldButtons() { + clickSequencer.flushPendingRelease() guard activeDragButtonNumber != nil else { return } activeDragButtonNumber = nil emit(mouseReportBuilder.reset()) diff --git a/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift b/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift new file mode 100644 index 0000000..1f9ac02 --- /dev/null +++ b/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift @@ -0,0 +1,78 @@ +import Foundation + +/// Gives a synthesized press — a tap-to-click, an on-screen key — a duration. +/// +/// A tap has no dwell of its own, so the down and up reports would otherwise be emitted in the +/// same instant. HID reports carry *state*, not events: the KVM holds the most recent report and +/// the attached host only samples it when it polls, every few milliseconds. A press and release +/// that land inside one polling window collapse into the released state, so the host sees +/// nothing happen at all. Holding for `pressDuration` guarantees at least one poll observes the +/// press. +/// +/// Taps arriving while one is still held are queued rather than merged — `insertText` synthesizes +/// a whole string's keystrokes in one pass, and each of them needs its own hold. The first `down` +/// is emitted synchronously so a tap never pays a run-loop hop before the host sees it. +/// +/// Real presses need none of this: a held finger, a physical key, and a drag all supply their own +/// duration. +@MainActor +final class SynthesizedTapSequencer { + nonisolated static let defaultPressDuration = Duration.milliseconds(50) + + private typealias Tap = (down: @MainActor () -> Void, up: @MainActor () -> Void) + + private let pressDuration: Duration + private let sleep: @Sendable (Duration) async -> Void + private var queue: [Tap] = [] + private var pendingRelease: (@MainActor () -> Void)? + private var holdTask: Task? + + init( + pressDuration: Duration = SynthesizedTapSequencer.defaultPressDuration, + sleep: @escaping @Sendable (Duration) async -> Void = { try? await Task.sleep(for: $0) } + ) { + self.pressDuration = pressDuration + self.sleep = sleep + } + + /// Emits `down` now and `up` once the press has been held long enough to be observed. + func tap(down: @escaping @MainActor () -> Void, up: @escaping @MainActor () -> Void) { + queue.append((down, up)) + guard holdTask == nil else { return } + startNextTap() + } + + /// Emits a still-held release immediately and drops anything queued behind it — used when a + /// drag takes the button over, and when capture is turned off. + func flushPendingRelease() { + holdTask?.cancel() + holdTask = nil + queue.removeAll() + releasePendingIfNeeded() + } + + private func startNextTap() { + guard !queue.isEmpty else { return } + let tap = queue.removeFirst() + tap.down() + pendingRelease = tap.up + holdTask = Task { [weak self, pressDuration, sleep] in + await sleep(pressDuration) + // A flush already emitted this release and cleared the queue. + guard !Task.isCancelled else { return } + self?.finishTap() + } + } + + private func finishTap() { + holdTask = nil + releasePendingIfNeeded() + startNextTap() + } + + private func releasePendingIfNeeded() { + guard let pendingRelease else { return } + self.pendingRelease = nil + pendingRelease() + } +} diff --git a/KVMConsoleiPadTests/NanoKVMiPadTests.swift b/KVMConsoleiPadTests/NanoKVMiPadTests.swift index 2baf3ba..a1f7ec5 100644 --- a/KVMConsoleiPadTests/NanoKVMiPadTests.swift +++ b/KVMConsoleiPadTests/NanoKVMiPadTests.swift @@ -76,4 +76,112 @@ final class KVMConsoleiPadTests: XCTestCase { ModifierBarVisibility.shouldShow(userEnabled: false, physicalKeyboardConnected: false) ) } + + // MARK: Synthesized tap dwell + + @MainActor + func test_synthesizedTapHoldsButtonBeforeReleasing() async { + let sleeper = RecordingSleeper() + let sequencer = SynthesizedTapSequencer(sleep: { await sleeper.sleep($0) }) + let log = TapLog() + let released = expectation(description: "release emitted") + + sequencer.tap( + down: { log.append("down") }, + up: { log.append("up"); released.fulfill() } + ) + + XCTAssertEqual(log.events, ["down"], "release must not be emitted in the same instant as the press") + + await fulfillment(of: [released], timeout: 2) + + XCTAssertEqual(log.events, ["down", "up"]) + XCTAssertEqual( + sleeper.durations, + [SynthesizedTapSequencer.defaultPressDuration], + "the button has to be held long enough for the host to poll it" + ) + } + + /// `insertText` synthesizes a whole string's keystrokes in one pass, so taps queued while + /// another is still held must each get their own hold rather than collapsing. + @MainActor + func test_queuedTapsEachGetTheirOwnHold() async { + let sleeper = RecordingSleeper() + let sequencer = SynthesizedTapSequencer(sleep: { await sleeper.sleep($0) }) + let log = TapLog() + let finished = expectation(description: "second release emitted") + + sequencer.tap(down: { log.append("down1") }, up: { log.append("up1") }) + sequencer.tap(down: { log.append("down2") }, up: { log.append("up2"); finished.fulfill() }) + + XCTAssertEqual(log.events, ["down1"], "the queued tap must wait for the held one to be released") + + await fulfillment(of: [finished], timeout: 2) + + XCTAssertEqual(log.events, ["down1", "up1", "down2", "up2"]) + XCTAssertEqual(sleeper.durations.count, 2, "each tap gets its own hold") + } + + @MainActor + func test_flushPendingReleaseReleasesHeldTapAndDropsTheQueue() { + let sleeper = RecordingSleeper() + let sequencer = SynthesizedTapSequencer(sleep: { await sleeper.sleep($0) }) + let log = TapLog() + + sequencer.tap(down: { log.append("down1") }, up: { log.append("up1") }) + sequencer.tap(down: { log.append("down2") }, up: { log.append("up2") }) + sequencer.flushPendingRelease() + sequencer.flushPendingRelease() + + XCTAssertEqual(log.events, ["down1", "up1"]) + } + + // MARK: Virtual keys + + @MainActor + func test_virtualKeyIsHeldRatherThanReleasedInTheSameInstant() async { + let view = KeyboardCaptureUIView() + let log = TapLog() + let released = expectation(description: "key release emitted") + view.onKeyboardReport = { report in + log.append(report.keycodes.isEmpty ? "up" : "down") + if report.keycodes.isEmpty { + released.fulfill() + } + } + + view.sendVirtualKey(usage: 0x04, transientModifier: HIDModifierBit.leftShift.rawValue) + + XCTAssertEqual(log.events, ["down"], "the key release must not land in the same instant as the press") + + await fulfillment(of: [released], timeout: 2) + + XCTAssertEqual(log.events, ["down", "up"]) + } +} + +/// Collects emitted tap phases by reference so the escaping down/up closures don't have to +/// capture a mutable local. +private final class TapLog { + private(set) var events: [String] = [] + + func append(_ event: String) { + events.append(event) + } +} + +/// Stands in for `Task.sleep`: records the dwell that was asked for and returns immediately, so +/// the tests assert ordering and duration without waiting on the clock. +private final class RecordingSleeper: @unchecked Sendable { + private let lock = NSLock() + private var storedDurations: [Duration] = [] + + var durations: [Duration] { + lock.withLock { storedDurations } + } + + func sleep(_ duration: Duration) async { + lock.withLock { storedDurations.append(duration) } + } } From 5794fa80422b2ebe5a8e2f47a4d01c6a9a1fdb23 Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Fri, 21 Aug 2026 20:58:26 +0800 Subject: [PATCH 2/3] Bump marketing version to 1.0.4 1.0.3 is already tagged and released, so shipping this fix needs a new version for the release tag to match. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XU7Ck7AbZ9KpRhdPbRzNj8 --- project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.yml b/project.yml index 0222292..53afa40 100644 --- a/project.yml +++ b/project.yml @@ -20,7 +20,7 @@ settings: # Single source of truth for app version, shared by both targets via # $(MARKETING_VERSION) / $(CURRENT_PROJECT_VERSION) references in their # Info.plist. CI overrides CURRENT_PROJECT_VERSION per release (timestamp). - MARKETING_VERSION: "1.0.3" + MARKETING_VERSION: "1.0.4" CURRENT_PROJECT_VERSION: "1" targets: KVMConsole: From 44b5210984fae8e71167a53dcf54223d0ed0693c Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Fri, 21 Aug 2026 21:18:17 +0800 Subject: [PATCH 3/3] Close the hazards the deferred release opened Deferring the release by 50ms created a window that the previous same-instant emission made structurally impossible. Six issues lived in it: - A release emitted after capture was switched off was dropped by the capture guards in ViewerViewModel, stranding a held button or key on the host forever. Releases now travel on their own channel, sendInputRelease, which bypasses the guard by design. This also covers the pre-existing case of toggling capture off mid-drag. - KeyboardCaptureUIView had no flush hook at all, so disabling keyboard capture during a hold left the key asserted and auto-repeating. It now mirrors the pointer view's didSet. - The synthesized key release emitted an empty report, releasing any hardware key held during the hold and causing a spurious repeat when the next key arrived. It now emits the builder's current state, and pressesBegan flushes so real and synthesized presses can't interleave. - A hover during the hold emitted a move that kept the button bit set, turning a trackpad click into a drag. Hover is suppressed while a press is held. - flushPendingRelease discarded the whole queue, silently losing the rest of a multi-character insertText. Releasing the held press and dropping the queue are now separate operations, and only the drag-takeover and capture-off paths drop. - startNextTap armed the release after running down(), so a re-entrant flush could leave a press armed behind its back. State is now armed first and the entry guarded on holdTask. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XU7Ck7AbZ9KpRhdPbRzNj8 --- .../Input/KeyboardCaptureView.swift | 28 +++++++++++-- KVMConsoleiPad/Input/PointerCaptureView.swift | 26 ++++++++++--- .../Input/SynthesizedTapSequencer.swift | 27 ++++++++++--- KVMConsoleiPad/UI/ViewerView.swift | 4 +- KVMConsoleiPadTests/NanoKVMiPadTests.swift | 39 ++++++++++--------- .../KVMCore/Input/HIDKeyboardReport.swift | 17 ++++---- .../Sources/KVMCore/UI/ViewerViewModel.swift | 13 +++++++ .../KVMCoreTests/HIDKeyboardReportTests.swift | 11 ++++++ 8 files changed, 124 insertions(+), 41 deletions(-) diff --git a/KVMConsoleiPad/Input/KeyboardCaptureView.swift b/KVMConsoleiPad/Input/KeyboardCaptureView.swift index b0c7485..c12e310 100644 --- a/KVMConsoleiPad/Input/KeyboardCaptureView.swift +++ b/KVMConsoleiPad/Input/KeyboardCaptureView.swift @@ -14,11 +14,13 @@ struct KeyboardCaptureView: UIViewRepresentable { let extraModifierByte: UInt8 let pendingVirtualKey: VirtualKeyTap? let onKeyboardReport: @MainActor (HIDKeyboardReport) -> Void + let onKeyboardRelease: @MainActor (HIDKeyboardReport) -> Void let onMomentaryModifiersConsumed: @MainActor () -> Void func makeUIView(context: Context) -> KeyboardCaptureUIView { let view = KeyboardCaptureUIView() view.onKeyboardReport = onKeyboardReport + view.onKeyboardRelease = onKeyboardRelease view.onMomentaryModifiersConsumed = onMomentaryModifiersConsumed view.isCaptureEnabled = isEnabled view.extraModifierByte = extraModifierByte @@ -27,6 +29,7 @@ struct KeyboardCaptureView: UIViewRepresentable { func updateUIView(_ uiView: KeyboardCaptureUIView, context: Context) { uiView.onKeyboardReport = onKeyboardReport + uiView.onKeyboardRelease = onKeyboardRelease uiView.onMomentaryModifiersConsumed = onMomentaryModifiersConsumed uiView.isCaptureEnabled = isEnabled uiView.extraModifierByte = extraModifierByte @@ -53,9 +56,16 @@ struct KeyboardCaptureView: UIViewRepresentable { } final class KeyboardCaptureUIView: UIView, UIKeyInput { - var isCaptureEnabled = false + var isCaptureEnabled = false { + didSet { + if !isCaptureEnabled { + virtualKeySequencer.cancelAll() + } + } + } var extraModifierByte: UInt8 = 0 var onKeyboardReport: (@MainActor (HIDKeyboardReport) -> Void)? + var onKeyboardRelease: (@MainActor (HIDKeyboardReport) -> Void)? var onMomentaryModifiersConsumed: (@MainActor () -> Void)? private let builder = HIDKeyboardReportBuilder() @@ -85,6 +95,7 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { super.pressesBegan(presses, with: event) return } + virtualKeySequencer.flushPendingRelease() for press in presses { guard let usage = press.key?.keyCode.rawValue, let keyUsage = UInt8(exactly: usage) else { continue } if let bit = HIDModifierBit.bit(forHIDUsage: keyUsage) { @@ -117,7 +128,7 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { return } virtualKeySequencer.flushPendingRelease() - emit(builder.reset()) + emitRelease(builder.reset()) super.pressesCancelled(presses, with: event) } @@ -152,7 +163,9 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { }, up: { [weak self] in guard let self else { return } - self.emit(HIDKeyboardReport(modifier: self.builder.modifierByte, keycodes: [])) + // The builder's current state, not an empty report: a hardware key pressed during + // the hold is still down and must not be released along with the synthesized one. + self.emitRelease(self.builder.currentReport) } ) } @@ -175,6 +188,15 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { } } + /// Releases go through their own channel: by the time capture is switched off the guarded + /// path drops everything, which would leave a key asserted — and auto-repeating — on the host. + private func emitRelease(_ report: HIDKeyboardReport) { + guard let onKeyboardRelease else { return } + MainActor.assumeIsolated { + onKeyboardRelease(report) + } + } + private func consumeMomentary() { guard let onMomentaryModifiersConsumed else { return } MainActor.assumeIsolated { diff --git a/KVMConsoleiPad/Input/PointerCaptureView.swift b/KVMConsoleiPad/Input/PointerCaptureView.swift index e0fba54..998ca6f 100644 --- a/KVMConsoleiPad/Input/PointerCaptureView.swift +++ b/KVMConsoleiPad/Input/PointerCaptureView.swift @@ -8,10 +8,12 @@ struct PointerCaptureView: UIViewRepresentable { let videoSize: CGSize? let zoom: ViewerZoomState let onMouseReport: @MainActor (HIDMouseAbsoluteReport) -> Void + let onMouseRelease: @MainActor (HIDMouseAbsoluteReport) -> Void func makeUIView(context: Context) -> PointerCaptureUIView { let view = PointerCaptureUIView() view.onMouseReport = onMouseReport + view.onMouseRelease = onMouseRelease view.isCaptureEnabled = isEnabled view.isScrollInverted = isScrollInverted view.videoSize = videoSize @@ -21,6 +23,7 @@ struct PointerCaptureView: UIViewRepresentable { func updateUIView(_ uiView: PointerCaptureUIView, context: Context) { uiView.onMouseReport = onMouseReport + uiView.onMouseRelease = onMouseRelease uiView.isCaptureEnabled = isEnabled uiView.isScrollInverted = isScrollInverted uiView.videoSize = videoSize @@ -40,6 +43,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { var videoSize: CGSize? var zoom: ViewerZoomState? var onMouseReport: (@MainActor (HIDMouseAbsoluteReport) -> Void)? + var onMouseRelease: (@MainActor (HIDMouseAbsoluteReport) -> Void)? private let mouseReportBuilder = HIDMouseAbsoluteReportBuilder() private var scrollAccumulator = MouseScrollAccumulator() @@ -84,6 +88,9 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { @objc private func handleHover(_ recognizer: UIHoverGestureRecognizer) { guard isCaptureEnabled else { return } + // `move` keeps the buttons byte, so emitting one while a synthesized click is still held + // would read on the host as a drag. The pointer catches up when the click releases. + guard !clickSequencer.isHoldingPress else { return } let location = recognizer.location(in: self) let effective = effectiveRect() let normalized = MouseCoordinateMapper.normalizedPoint(clientPoint: location, effectiveRect: effective) @@ -111,7 +118,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { switch recognizer.state { case .began: if let dragButtonNumber { - clickSequencer.flushPendingRelease() + clickSequencer.cancelAll() activeDragButtonNumber = dragButtonNumber emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y)) } else { @@ -119,7 +126,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { } case .changed: if activeDragButtonNumber == nil, let dragButtonNumber { - clickSequencer.flushPendingRelease() + clickSequencer.cancelAll() activeDragButtonNumber = dragButtonNumber emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y)) } @@ -227,17 +234,17 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { }, up: { [weak self] in guard let self else { return } - self.emit(self.mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y)) + self.emitRelease(self.mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y)) } ) zoom?.cursorNormalized = normalized } private func releaseHeldButtons() { - clickSequencer.flushPendingRelease() + clickSequencer.cancelAll() guard activeDragButtonNumber != nil else { return } activeDragButtonNumber = nil - emit(mouseReportBuilder.reset()) + emitRelease(mouseReportBuilder.reset()) } // UIGestureRecognizer callbacks run on the main thread; assuming @@ -251,6 +258,15 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { } } + /// Releases go through their own channel: by the time capture is switched off the guarded + /// path drops everything, which would strand a held button on the host. + private func emitRelease(_ report: HIDMouseAbsoluteReport) { + guard let onMouseRelease else { return } + MainActor.assumeIsolated { + onMouseRelease(report) + } + } + private func effectiveRect() -> CGRect { let baseRect = MouseCoordinateMapper.aspectFitRect(for: videoSize, in: bounds) guard let zoom else { return baseRect } diff --git a/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift b/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift index 1f9ac02..a414eb0 100644 --- a/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift +++ b/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift @@ -35,33 +35,48 @@ final class SynthesizedTapSequencer { self.sleep = sleep } + /// Whether a press is currently being held. Callers suppress pointer moves while it is true, + /// so a click can't turn into a drag just because the pointer drifted during the hold. + var isHoldingPress: Bool { pendingRelease != nil } + /// Emits `down` now and `up` once the press has been held long enough to be observed. func tap(down: @escaping @MainActor () -> Void, up: @escaping @MainActor () -> Void) { queue.append((down, up)) - guard holdTask == nil else { return } startNextTap() } - /// Emits a still-held release immediately and drops anything queued behind it — used when a - /// drag takes the button over, and when capture is turned off. + /// Emits a still-held release immediately, leaving anything queued behind it to drain. Use + /// this when the current press has to end early but the remaining input is still wanted — an + /// interrupted paste, say, whose remaining characters would otherwise vanish. func flushPendingRelease() { holdTask?.cancel() holdTask = nil + releasePendingIfNeeded() + startNextTap() + } + + /// Releases the held press and discards everything queued behind it — for when the pending + /// input is no longer wanted at all: a drag taking the button over, or capture switching off. + func cancelAll() { queue.removeAll() + holdTask?.cancel() + holdTask = nil releasePendingIfNeeded() } private func startNextTap() { - guard !queue.isEmpty else { return } + guard holdTask == nil, !queue.isEmpty else { return } let tap = queue.removeFirst() - tap.down() + // Arm the release before running `down`, so a re-entrant flush from inside it sees a press + // to release rather than silently leaving one armed behind its back. pendingRelease = tap.up holdTask = Task { [weak self, pressDuration, sleep] in await sleep(pressDuration) - // A flush already emitted this release and cleared the queue. + // A flush already emitted this release. guard !Task.isCancelled else { return } self?.finishTap() } + tap.down() } private func finishTap() { diff --git a/KVMConsoleiPad/UI/ViewerView.swift b/KVMConsoleiPad/UI/ViewerView.swift index 452810f..a2f4bfc 100644 --- a/KVMConsoleiPad/UI/ViewerView.swift +++ b/KVMConsoleiPad/UI/ViewerView.swift @@ -113,7 +113,8 @@ struct ViewerView: View { isScrollInverted: model.isScrollInverted, videoSize: model.videoSize, zoom: model.zoom, - onMouseReport: { report in model.sendMouseReport(report) } + onMouseReport: { report in model.sendMouseReport(report) }, + onMouseRelease: { report in model.sendInputRelease(mouse: report) } ) .frame(maxWidth: .infinity, maxHeight: .infinity) @@ -123,6 +124,7 @@ struct ViewerView: View { extraModifierByte: modifierState.activeModifierByte, pendingVirtualKey: pendingVirtualKey, onKeyboardReport: { report in model.sendKeyboardReport(report) }, + onKeyboardRelease: { report in model.sendInputRelease(keyboard: report) }, onMomentaryModifiersConsumed: { modifierState.consumeMomentary() } ) .frame(width: 1, height: 1) diff --git a/KVMConsoleiPadTests/NanoKVMiPadTests.swift b/KVMConsoleiPadTests/NanoKVMiPadTests.swift index a1f7ec5..22bc6b6 100644 --- a/KVMConsoleiPadTests/NanoKVMiPadTests.swift +++ b/KVMConsoleiPadTests/NanoKVMiPadTests.swift @@ -123,20 +123,6 @@ final class KVMConsoleiPadTests: XCTestCase { XCTAssertEqual(sleeper.durations.count, 2, "each tap gets its own hold") } - @MainActor - func test_flushPendingReleaseReleasesHeldTapAndDropsTheQueue() { - let sleeper = RecordingSleeper() - let sequencer = SynthesizedTapSequencer(sleep: { await sleeper.sleep($0) }) - let log = TapLog() - - sequencer.tap(down: { log.append("down1") }, up: { log.append("up1") }) - sequencer.tap(down: { log.append("down2") }, up: { log.append("up2") }) - sequencer.flushPendingRelease() - sequencer.flushPendingRelease() - - XCTAssertEqual(log.events, ["down1", "up1"]) - } - // MARK: Virtual keys @MainActor @@ -144,11 +130,10 @@ final class KVMConsoleiPadTests: XCTestCase { let view = KeyboardCaptureUIView() let log = TapLog() let released = expectation(description: "key release emitted") - view.onKeyboardReport = { report in - log.append(report.keycodes.isEmpty ? "up" : "down") - if report.keycodes.isEmpty { - released.fulfill() - } + view.onKeyboardReport = { _ in log.append("down") } + view.onKeyboardRelease = { _ in + log.append("up") + released.fulfill() } view.sendVirtualKey(usage: 0x04, transientModifier: HIDModifierBit.leftShift.rawValue) @@ -159,6 +144,22 @@ final class KVMConsoleiPadTests: XCTestCase { XCTAssertEqual(log.events, ["down", "up"]) } + + @MainActor + func test_disablingCaptureReleasesAHeldVirtualKey() { + let view = KeyboardCaptureUIView() + let log = TapLog() + view.onKeyboardReport = { _ in log.append("down") } + view.onKeyboardRelease = { _ in log.append("up") } + view.isCaptureEnabled = true + + view.sendVirtualKey(usage: 0x04, transientModifier: 0) + XCTAssertEqual(log.events, ["down"]) + + view.isCaptureEnabled = false + + XCTAssertEqual(log.events, ["down", "up"], "a key held when capture is switched off must be released") + } } /// Collects emitted tap phases by reference so the escaping down/up closures don't have to diff --git a/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift b/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift index 582df36..2a2692d 100644 --- a/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift +++ b/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift @@ -33,16 +33,22 @@ public final class HIDKeyboardReportBuilder { public var modifierByte: UInt8 { modifier } + /// The keys and modifiers currently held. Callers that synthesize a keystroke release it by + /// emitting this rather than an empty report, so real keys held at the same time survive. + public var currentReport: HIDKeyboardReport { + HIDKeyboardReport(modifier: modifier, keycodes: pressedKeys) + } + public func keyDown(usage: UInt8) -> HIDKeyboardReport { if !pressedKeys.contains(usage), pressedKeys.count < HIDKeyboardReport.maxKeys { pressedKeys.append(usage) } - return currentReport() + return currentReport } public func keyUp(usage: UInt8) -> HIDKeyboardReport { pressedKeys.removeAll { $0 == usage } - return currentReport() + return currentReport } public func modifierChanged(bit: UInt8, isPressed: Bool) -> HIDKeyboardReport { @@ -51,16 +57,13 @@ public final class HIDKeyboardReportBuilder { } else { modifier &= ~bit } - return currentReport() + return currentReport } public func reset() -> HIDKeyboardReport { modifier = 0 pressedKeys.removeAll() - return currentReport() + return currentReport } - private func currentReport() -> HIDKeyboardReport { - HIDKeyboardReport(modifier: modifier, keycodes: pressedKeys) - } } diff --git a/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift b/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift index 2c9f4e0..f46bb0e 100644 --- a/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift +++ b/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift @@ -160,6 +160,19 @@ public final class ViewerViewModel: ObservableObject { session.sendMouseReport(report) } + /// Releases input that is still held, bypassing the capture guard on purpose. + /// + /// A button or key held when capture is switched off would otherwise stay pressed on the host + /// forever: every later report — including the release itself — is dropped by the guard above. + /// Only pass reports that clear held state. + public func sendInputRelease(mouse report: HIDMouseAbsoluteReport) { + session.sendMouseReport(report) + } + + public func sendInputRelease(keyboard report: HIDKeyboardReport) { + session.sendKeyboardReport(report) + } + public func powerOn() { performPowerAction { try await $0.powerOn() } } diff --git a/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift b/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift index f9197c3..1c025e9 100644 --- a/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift +++ b/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift @@ -34,4 +34,15 @@ final class HIDKeyboardReportTests: XCTestCase { _ = builder.keyDown(usage: 0x04) XCTAssertEqual(builder.keyDown(usage: 0x04), HIDKeyboardReport(keycodes: [0x04])) } + + func test_currentReportReflectsHeldKeysAndModifiers() { + let builder = HIDKeyboardReportBuilder() + _ = builder.modifierChanged(bit: HIDModifierBit.leftControl.rawValue, isPressed: true) + _ = builder.keyDown(usage: 0x04) + + let report = builder.currentReport + + XCTAssertEqual(report.modifier, HIDModifierBit.leftControl.rawValue) + XCTAssertEqual(report.keycodes, [0x04]) + } }