diff --git a/KVMConsoleiPad/Input/KeyboardCaptureView.swift b/KVMConsoleiPad/Input/KeyboardCaptureView.swift index becd0bc..c12e310 100644 --- a/KVMConsoleiPad/Input/KeyboardCaptureView.swift +++ b/KVMConsoleiPad/Input/KeyboardCaptureView.swift @@ -14,11 +14,13 @@ struct KeyboardCaptureView: UIViewRepresentable { let extraModifierByte: UInt8 let pendingVirtualKey: VirtualKeyTap? let onKeyboardReport: @MainActor (HIDKeyboardReport) -> Void + let onKeyboardRelease: @MainActor (HIDKeyboardReport) -> Void let onMomentaryModifiersConsumed: @MainActor () -> Void func makeUIView(context: Context) -> KeyboardCaptureUIView { let view = KeyboardCaptureUIView() view.onKeyboardReport = onKeyboardReport + view.onKeyboardRelease = onKeyboardRelease view.onMomentaryModifiersConsumed = onMomentaryModifiersConsumed view.isCaptureEnabled = isEnabled view.extraModifierByte = extraModifierByte @@ -27,6 +29,7 @@ struct KeyboardCaptureView: UIViewRepresentable { func updateUIView(_ uiView: KeyboardCaptureUIView, context: Context) { uiView.onKeyboardReport = onKeyboardReport + uiView.onKeyboardRelease = onKeyboardRelease uiView.onMomentaryModifiersConsumed = onMomentaryModifiersConsumed uiView.isCaptureEnabled = isEnabled uiView.extraModifierByte = extraModifierByte @@ -53,12 +56,20 @@ struct KeyboardCaptureView: UIViewRepresentable { } final class KeyboardCaptureUIView: UIView, UIKeyInput { - var isCaptureEnabled = false + var isCaptureEnabled = false { + didSet { + if !isCaptureEnabled { + virtualKeySequencer.cancelAll() + } + } + } var extraModifierByte: UInt8 = 0 var onKeyboardReport: (@MainActor (HIDKeyboardReport) -> Void)? + var onKeyboardRelease: (@MainActor (HIDKeyboardReport) -> Void)? var onMomentaryModifiersConsumed: (@MainActor () -> Void)? private let builder = HIDKeyboardReportBuilder() + private let virtualKeySequencer = SynthesizedTapSequencer() override var canBecomeFirstResponder: Bool { true } var hasText: Bool { false } @@ -84,6 +95,7 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { super.pressesBegan(presses, with: event) return } + virtualKeySequencer.flushPendingRelease() for press in presses { guard let usage = press.key?.keyCode.rawValue, let keyUsage = UInt8(exactly: usage) else { continue } if let bit = HIDModifierBit.bit(forHIDUsage: keyUsage) { @@ -115,7 +127,8 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { super.pressesCancelled(presses, with: event) return } - emit(builder.reset()) + virtualKeySequencer.flushPendingRelease() + emitRelease(builder.reset()) super.pressesCancelled(presses, with: event) } @@ -138,14 +151,23 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { // Combine synthesized modifier bits with whatever real modifiers the builder is // currently tracking, then on release drop only the synthesized bits — this keeps a // hardware modifier (e.g. held BT-keyboard Shift) asserted on the host and prevents - // the synthesized one from sticking. - let down = HIDKeyboardReport( - modifier: builder.modifierByte | transientModifier, - keycodes: [usage] + // the synthesized one from sticking. The release re-reads the builder rather than + // capturing it, so a hardware modifier pressed during the hold survives too. + virtualKeySequencer.tap( + down: { [weak self] in + guard let self else { return } + self.emit(HIDKeyboardReport( + modifier: self.builder.modifierByte | transientModifier, + keycodes: [usage] + )) + }, + up: { [weak self] in + guard let self else { return } + // The builder's current state, not an empty report: a hardware key pressed during + // the hold is still down and must not be released along with the synthesized one. + self.emitRelease(self.builder.currentReport) + } ) - let up = HIDKeyboardReport(modifier: builder.modifierByte, keycodes: []) - emit(down) - emit(up) } private func withExtraModifiers(_ report: HIDKeyboardReport, eventModifiers: UIKeyModifierFlags) -> HIDKeyboardReport { @@ -166,6 +188,15 @@ final class KeyboardCaptureUIView: UIView, UIKeyInput { } } + /// Releases go through their own channel: by the time capture is switched off the guarded + /// path drops everything, which would leave a key asserted — and auto-repeating — on the host. + private func emitRelease(_ report: HIDKeyboardReport) { + guard let onKeyboardRelease else { return } + MainActor.assumeIsolated { + onKeyboardRelease(report) + } + } + private func consumeMomentary() { guard let onMomentaryModifiersConsumed else { return } MainActor.assumeIsolated { diff --git a/KVMConsoleiPad/Input/PointerCaptureView.swift b/KVMConsoleiPad/Input/PointerCaptureView.swift index 3cc6d6e..998ca6f 100644 --- a/KVMConsoleiPad/Input/PointerCaptureView.swift +++ b/KVMConsoleiPad/Input/PointerCaptureView.swift @@ -8,10 +8,12 @@ struct PointerCaptureView: UIViewRepresentable { let videoSize: CGSize? let zoom: ViewerZoomState let onMouseReport: @MainActor (HIDMouseAbsoluteReport) -> Void + let onMouseRelease: @MainActor (HIDMouseAbsoluteReport) -> Void func makeUIView(context: Context) -> PointerCaptureUIView { let view = PointerCaptureUIView() view.onMouseReport = onMouseReport + view.onMouseRelease = onMouseRelease view.isCaptureEnabled = isEnabled view.isScrollInverted = isScrollInverted view.videoSize = videoSize @@ -21,6 +23,7 @@ struct PointerCaptureView: UIViewRepresentable { func updateUIView(_ uiView: PointerCaptureUIView, context: Context) { uiView.onMouseReport = onMouseReport + uiView.onMouseRelease = onMouseRelease uiView.isCaptureEnabled = isEnabled uiView.isScrollInverted = isScrollInverted uiView.videoSize = videoSize @@ -32,7 +35,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { var isCaptureEnabled = false { didSet { if !isCaptureEnabled { - releaseActiveDragIfNeeded() + releaseHeldButtons() } } } @@ -40,10 +43,12 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { var videoSize: CGSize? var zoom: ViewerZoomState? var onMouseReport: (@MainActor (HIDMouseAbsoluteReport) -> Void)? + var onMouseRelease: (@MainActor (HIDMouseAbsoluteReport) -> Void)? private let mouseReportBuilder = HIDMouseAbsoluteReportBuilder() private var scrollAccumulator = MouseScrollAccumulator() private var activeDragButtonNumber: Int? + private let clickSequencer = SynthesizedTapSequencer() private weak var pinchRecognizer: UIPinchGestureRecognizer? private weak var pointerPanRecognizer: UIPanGestureRecognizer? private var pinchAnchorVideo: CGPoint? @@ -83,6 +88,9 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { @objc private func handleHover(_ recognizer: UIHoverGestureRecognizer) { guard isCaptureEnabled else { return } + // `move` keeps the buttons byte, so emitting one while a synthesized click is still held + // would read on the host as a drag. The pointer catches up when the click releases. + guard !clickSequencer.isHoldingPress else { return } let location = recognizer.location(in: self) let effective = effectiveRect() let normalized = MouseCoordinateMapper.normalizedPoint(clientPoint: location, effectiveRect: effective) @@ -94,7 +102,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { @objc private func handlePan(_ recognizer: UIPanGestureRecognizer) { guard isCaptureEnabled else { - releaseActiveDragIfNeeded() + releaseHeldButtons() return } let location = recognizer.location(in: self) @@ -110,6 +118,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { switch recognizer.state { case .began: if let dragButtonNumber { + clickSequencer.cancelAll() activeDragButtonNumber = dragButtonNumber emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y)) } else { @@ -117,6 +126,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { } case .changed: if activeDragButtonNumber == nil, let dragButtonNumber { + clickSequencer.cancelAll() activeDragButtonNumber = dragButtonNumber emit(mouseReportBuilder.buttonDown(buttonNumber: dragButtonNumber, x: point.x, y: point.y)) } @@ -217,15 +227,24 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { let effective = effectiveRect() let normalized = MouseCoordinateMapper.normalizedPoint(clientPoint: location, effectiveRect: effective) let point = MouseCoordinateMapper.absolutePoint(clientPoint: location, effectiveRect: effective) - emit(mouseReportBuilder.buttonDown(buttonNumber: buttonNumber, x: point.x, y: point.y)) - emit(mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y)) + clickSequencer.tap( + down: { [weak self] in + guard let self else { return } + self.emit(self.mouseReportBuilder.buttonDown(buttonNumber: buttonNumber, x: point.x, y: point.y)) + }, + up: { [weak self] in + guard let self else { return } + self.emitRelease(self.mouseReportBuilder.buttonUp(buttonNumber: buttonNumber, x: point.x, y: point.y)) + } + ) zoom?.cursorNormalized = normalized } - private func releaseActiveDragIfNeeded() { + private func releaseHeldButtons() { + clickSequencer.cancelAll() guard activeDragButtonNumber != nil else { return } activeDragButtonNumber = nil - emit(mouseReportBuilder.reset()) + emitRelease(mouseReportBuilder.reset()) } // UIGestureRecognizer callbacks run on the main thread; assuming @@ -239,6 +258,15 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { } } + /// Releases go through their own channel: by the time capture is switched off the guarded + /// path drops everything, which would strand a held button on the host. + private func emitRelease(_ report: HIDMouseAbsoluteReport) { + guard let onMouseRelease else { return } + MainActor.assumeIsolated { + onMouseRelease(report) + } + } + private func effectiveRect() -> CGRect { let baseRect = MouseCoordinateMapper.aspectFitRect(for: videoSize, in: bounds) guard let zoom else { return baseRect } diff --git a/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift b/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift new file mode 100644 index 0000000..a414eb0 --- /dev/null +++ b/KVMConsoleiPad/Input/SynthesizedTapSequencer.swift @@ -0,0 +1,93 @@ +import Foundation + +/// Gives a synthesized press — a tap-to-click, an on-screen key — a duration. +/// +/// A tap has no dwell of its own, so the down and up reports would otherwise be emitted in the +/// same instant. HID reports carry *state*, not events: the KVM holds the most recent report and +/// the attached host only samples it when it polls, every few milliseconds. A press and release +/// that land inside one polling window collapse into the released state, so the host sees +/// nothing happen at all. Holding for `pressDuration` guarantees at least one poll observes the +/// press. +/// +/// Taps arriving while one is still held are queued rather than merged — `insertText` synthesizes +/// a whole string's keystrokes in one pass, and each of them needs its own hold. The first `down` +/// is emitted synchronously so a tap never pays a run-loop hop before the host sees it. +/// +/// Real presses need none of this: a held finger, a physical key, and a drag all supply their own +/// duration. +@MainActor +final class SynthesizedTapSequencer { + nonisolated static let defaultPressDuration = Duration.milliseconds(50) + + private typealias Tap = (down: @MainActor () -> Void, up: @MainActor () -> Void) + + private let pressDuration: Duration + private let sleep: @Sendable (Duration) async -> Void + private var queue: [Tap] = [] + private var pendingRelease: (@MainActor () -> Void)? + private var holdTask: Task? + + init( + pressDuration: Duration = SynthesizedTapSequencer.defaultPressDuration, + sleep: @escaping @Sendable (Duration) async -> Void = { try? await Task.sleep(for: $0) } + ) { + self.pressDuration = pressDuration + self.sleep = sleep + } + + /// Whether a press is currently being held. Callers suppress pointer moves while it is true, + /// so a click can't turn into a drag just because the pointer drifted during the hold. + var isHoldingPress: Bool { pendingRelease != nil } + + /// Emits `down` now and `up` once the press has been held long enough to be observed. + func tap(down: @escaping @MainActor () -> Void, up: @escaping @MainActor () -> Void) { + queue.append((down, up)) + startNextTap() + } + + /// Emits a still-held release immediately, leaving anything queued behind it to drain. Use + /// this when the current press has to end early but the remaining input is still wanted — an + /// interrupted paste, say, whose remaining characters would otherwise vanish. + func flushPendingRelease() { + holdTask?.cancel() + holdTask = nil + releasePendingIfNeeded() + startNextTap() + } + + /// Releases the held press and discards everything queued behind it — for when the pending + /// input is no longer wanted at all: a drag taking the button over, or capture switching off. + func cancelAll() { + queue.removeAll() + holdTask?.cancel() + holdTask = nil + releasePendingIfNeeded() + } + + private func startNextTap() { + guard holdTask == nil, !queue.isEmpty else { return } + let tap = queue.removeFirst() + // Arm the release before running `down`, so a re-entrant flush from inside it sees a press + // to release rather than silently leaving one armed behind its back. + pendingRelease = tap.up + holdTask = Task { [weak self, pressDuration, sleep] in + await sleep(pressDuration) + // A flush already emitted this release. + guard !Task.isCancelled else { return } + self?.finishTap() + } + tap.down() + } + + private func finishTap() { + holdTask = nil + releasePendingIfNeeded() + startNextTap() + } + + private func releasePendingIfNeeded() { + guard let pendingRelease else { return } + self.pendingRelease = nil + pendingRelease() + } +} diff --git a/KVMConsoleiPad/UI/ViewerView.swift b/KVMConsoleiPad/UI/ViewerView.swift index 452810f..a2f4bfc 100644 --- a/KVMConsoleiPad/UI/ViewerView.swift +++ b/KVMConsoleiPad/UI/ViewerView.swift @@ -113,7 +113,8 @@ struct ViewerView: View { isScrollInverted: model.isScrollInverted, videoSize: model.videoSize, zoom: model.zoom, - onMouseReport: { report in model.sendMouseReport(report) } + onMouseReport: { report in model.sendMouseReport(report) }, + onMouseRelease: { report in model.sendInputRelease(mouse: report) } ) .frame(maxWidth: .infinity, maxHeight: .infinity) @@ -123,6 +124,7 @@ struct ViewerView: View { extraModifierByte: modifierState.activeModifierByte, pendingVirtualKey: pendingVirtualKey, onKeyboardReport: { report in model.sendKeyboardReport(report) }, + onKeyboardRelease: { report in model.sendInputRelease(keyboard: report) }, onMomentaryModifiersConsumed: { modifierState.consumeMomentary() } ) .frame(width: 1, height: 1) diff --git a/KVMConsoleiPadTests/NanoKVMiPadTests.swift b/KVMConsoleiPadTests/NanoKVMiPadTests.swift index 2baf3ba..22bc6b6 100644 --- a/KVMConsoleiPadTests/NanoKVMiPadTests.swift +++ b/KVMConsoleiPadTests/NanoKVMiPadTests.swift @@ -76,4 +76,113 @@ final class KVMConsoleiPadTests: XCTestCase { ModifierBarVisibility.shouldShow(userEnabled: false, physicalKeyboardConnected: false) ) } + + // MARK: Synthesized tap dwell + + @MainActor + func test_synthesizedTapHoldsButtonBeforeReleasing() async { + let sleeper = RecordingSleeper() + let sequencer = SynthesizedTapSequencer(sleep: { await sleeper.sleep($0) }) + let log = TapLog() + let released = expectation(description: "release emitted") + + sequencer.tap( + down: { log.append("down") }, + up: { log.append("up"); released.fulfill() } + ) + + XCTAssertEqual(log.events, ["down"], "release must not be emitted in the same instant as the press") + + await fulfillment(of: [released], timeout: 2) + + XCTAssertEqual(log.events, ["down", "up"]) + XCTAssertEqual( + sleeper.durations, + [SynthesizedTapSequencer.defaultPressDuration], + "the button has to be held long enough for the host to poll it" + ) + } + + /// `insertText` synthesizes a whole string's keystrokes in one pass, so taps queued while + /// another is still held must each get their own hold rather than collapsing. + @MainActor + func test_queuedTapsEachGetTheirOwnHold() async { + let sleeper = RecordingSleeper() + let sequencer = SynthesizedTapSequencer(sleep: { await sleeper.sleep($0) }) + let log = TapLog() + let finished = expectation(description: "second release emitted") + + sequencer.tap(down: { log.append("down1") }, up: { log.append("up1") }) + sequencer.tap(down: { log.append("down2") }, up: { log.append("up2"); finished.fulfill() }) + + XCTAssertEqual(log.events, ["down1"], "the queued tap must wait for the held one to be released") + + await fulfillment(of: [finished], timeout: 2) + + XCTAssertEqual(log.events, ["down1", "up1", "down2", "up2"]) + XCTAssertEqual(sleeper.durations.count, 2, "each tap gets its own hold") + } + + // MARK: Virtual keys + + @MainActor + func test_virtualKeyIsHeldRatherThanReleasedInTheSameInstant() async { + let view = KeyboardCaptureUIView() + let log = TapLog() + let released = expectation(description: "key release emitted") + view.onKeyboardReport = { _ in log.append("down") } + view.onKeyboardRelease = { _ in + log.append("up") + released.fulfill() + } + + view.sendVirtualKey(usage: 0x04, transientModifier: HIDModifierBit.leftShift.rawValue) + + XCTAssertEqual(log.events, ["down"], "the key release must not land in the same instant as the press") + + await fulfillment(of: [released], timeout: 2) + + XCTAssertEqual(log.events, ["down", "up"]) + } + + @MainActor + func test_disablingCaptureReleasesAHeldVirtualKey() { + let view = KeyboardCaptureUIView() + let log = TapLog() + view.onKeyboardReport = { _ in log.append("down") } + view.onKeyboardRelease = { _ in log.append("up") } + view.isCaptureEnabled = true + + view.sendVirtualKey(usage: 0x04, transientModifier: 0) + XCTAssertEqual(log.events, ["down"]) + + view.isCaptureEnabled = false + + XCTAssertEqual(log.events, ["down", "up"], "a key held when capture is switched off must be released") + } +} + +/// Collects emitted tap phases by reference so the escaping down/up closures don't have to +/// capture a mutable local. +private final class TapLog { + private(set) var events: [String] = [] + + func append(_ event: String) { + events.append(event) + } +} + +/// Stands in for `Task.sleep`: records the dwell that was asked for and returns immediately, so +/// the tests assert ordering and duration without waiting on the clock. +private final class RecordingSleeper: @unchecked Sendable { + private let lock = NSLock() + private var storedDurations: [Duration] = [] + + var durations: [Duration] { + lock.withLock { storedDurations } + } + + func sleep(_ duration: Duration) async { + lock.withLock { storedDurations.append(duration) } + } } diff --git a/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift b/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift index 582df36..2a2692d 100644 --- a/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift +++ b/KVMCore/Sources/KVMCore/Input/HIDKeyboardReport.swift @@ -33,16 +33,22 @@ public final class HIDKeyboardReportBuilder { public var modifierByte: UInt8 { modifier } + /// The keys and modifiers currently held. Callers that synthesize a keystroke release it by + /// emitting this rather than an empty report, so real keys held at the same time survive. + public var currentReport: HIDKeyboardReport { + HIDKeyboardReport(modifier: modifier, keycodes: pressedKeys) + } + public func keyDown(usage: UInt8) -> HIDKeyboardReport { if !pressedKeys.contains(usage), pressedKeys.count < HIDKeyboardReport.maxKeys { pressedKeys.append(usage) } - return currentReport() + return currentReport } public func keyUp(usage: UInt8) -> HIDKeyboardReport { pressedKeys.removeAll { $0 == usage } - return currentReport() + return currentReport } public func modifierChanged(bit: UInt8, isPressed: Bool) -> HIDKeyboardReport { @@ -51,16 +57,13 @@ public final class HIDKeyboardReportBuilder { } else { modifier &= ~bit } - return currentReport() + return currentReport } public func reset() -> HIDKeyboardReport { modifier = 0 pressedKeys.removeAll() - return currentReport() + return currentReport } - private func currentReport() -> HIDKeyboardReport { - HIDKeyboardReport(modifier: modifier, keycodes: pressedKeys) - } } diff --git a/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift b/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift index 2c9f4e0..f46bb0e 100644 --- a/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift +++ b/KVMCore/Sources/KVMCore/UI/ViewerViewModel.swift @@ -160,6 +160,19 @@ public final class ViewerViewModel: ObservableObject { session.sendMouseReport(report) } + /// Releases input that is still held, bypassing the capture guard on purpose. + /// + /// A button or key held when capture is switched off would otherwise stay pressed on the host + /// forever: every later report — including the release itself — is dropped by the guard above. + /// Only pass reports that clear held state. + public func sendInputRelease(mouse report: HIDMouseAbsoluteReport) { + session.sendMouseReport(report) + } + + public func sendInputRelease(keyboard report: HIDKeyboardReport) { + session.sendKeyboardReport(report) + } + public func powerOn() { performPowerAction { try await $0.powerOn() } } diff --git a/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift b/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift index f9197c3..1c025e9 100644 --- a/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift +++ b/KVMCore/Tests/KVMCoreTests/HIDKeyboardReportTests.swift @@ -34,4 +34,15 @@ final class HIDKeyboardReportTests: XCTestCase { _ = builder.keyDown(usage: 0x04) XCTAssertEqual(builder.keyDown(usage: 0x04), HIDKeyboardReport(keycodes: [0x04])) } + + func test_currentReportReflectsHeldKeysAndModifiers() { + let builder = HIDKeyboardReportBuilder() + _ = builder.modifierChanged(bit: HIDModifierBit.leftControl.rawValue, isPressed: true) + _ = builder.keyDown(usage: 0x04) + + let report = builder.currentReport + + XCTAssertEqual(report.modifier, HIDModifierBit.leftControl.rawValue) + XCTAssertEqual(report.keycodes, [0x04]) + } } diff --git a/project.yml b/project.yml index 0222292..53afa40 100644 --- a/project.yml +++ b/project.yml @@ -20,7 +20,7 @@ settings: # Single source of truth for app version, shared by both targets via # $(MARKETING_VERSION) / $(CURRENT_PROJECT_VERSION) references in their # Info.plist. CI overrides CURRENT_PROJECT_VERSION per release (timestamp). - MARKETING_VERSION: "1.0.3" + MARKETING_VERSION: "1.0.4" CURRENT_PROJECT_VERSION: "1" targets: KVMConsole: