From c51ff80dd5f60f7bc61e22fba1c459ea59183184 Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Fri, 21 Aug 2026 23:27:35 +0800 Subject: [PATCH 1/2] Stop the two-finger tap losing its gesture to the pinch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-finger right click added in 1.0.6 never fired. Pinch and pan both claim two touches as soon as they land, and whichever begins first takes the gesture, so the tap was read as a zoom of roughly no magnitude — nothing visibly happened and no click was sent. Pinch and pan now wait for the two-finger tap to fail before they can begin, expressed through shouldRequireFailureOf rather than require(toFail:) because the delegate method can be exercised directly in a test. The cost is negligible: any real zoom or scroll moves the fingers, which fails the tap at about the same threshold those recognizers need to begin anyway. Only holding two fingers still defers them, and that is the tap itself. A one-finger tap is deliberately excluded from the requirement, since it can never block a pinch and waiting on it would only add latency. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XU7Ck7AbZ9KpRhdPbRzNj8 --- KVMConsoleiPad/Input/PointerCaptureView.swift | 18 ++++++++++++++-- KVMConsoleiPadTests/NanoKVMiPadTests.swift | 21 +++++++++++++++++++ project.yml | 2 +- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/KVMConsoleiPad/Input/PointerCaptureView.swift b/KVMConsoleiPad/Input/PointerCaptureView.swift index f4e671e..83e373a 100644 --- a/KVMConsoleiPad/Input/PointerCaptureView.swift +++ b/KVMConsoleiPad/Input/PointerCaptureView.swift @@ -51,6 +51,7 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { private let clickSequencer = SynthesizedTapSequencer() private weak var pinchRecognizer: UIPinchGestureRecognizer? private weak var pointerPanRecognizer: UIPanGestureRecognizer? + private weak var twoFingerTapRecognizer: UITapGestureRecognizer? private var pinchAnchorVideo: CGPoint? override init(frame: CGRect) { @@ -88,12 +89,12 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { // The only way to right-click from a bare touchscreen. A trackpad's two-finger tap already // arrives as an indirect secondary click, so this one is restricted to direct touches to - // keep the two from both firing. A stationary two-finger tap starts neither the pan nor the - // pinch — both need movement — so wheel scrolling and zoom are unaffected. + // keep the two from both firing. let twoFingerTap = UITapGestureRecognizer(target: self, action: #selector(handleSecondaryTap(_:))) twoFingerTap.numberOfTouchesRequired = 2 twoFingerTap.allowedTouchTypes = [NSNumber(value: UITouch.TouchType.direct.rawValue)] addGestureRecognizer(twoFingerTap) + twoFingerTapRecognizer = twoFingerTap } required init?(coder: NSCoder) { @@ -297,6 +298,19 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { // the pan's wheel-scroll tracking (we suppress wheel emission while pinch is active). return true } + + func gestureRecognizer( + _ gestureRecognizer: UIGestureRecognizer, + shouldRequireFailureOf other: UIGestureRecognizer + ) -> Bool { + // Pinch and pan both claim two touches the moment they land, and whichever begins first + // takes the gesture — which is why a two-finger tap registered as a zoom that went + // nowhere instead of a right click. Making them wait for the tap to fail costs nothing in + // practice: any real zoom or scroll moves the fingers, which fails the tap immediately. + // Only holding two fingers still defers them, and that is the tap itself. + guard let twoFingerTapRecognizer, other === twoFingerTapRecognizer else { return false } + return gestureRecognizer === pinchRecognizer || gestureRecognizer === pointerPanRecognizer + } } enum PointerDragButtonResolver { diff --git a/KVMConsoleiPadTests/NanoKVMiPadTests.swift b/KVMConsoleiPadTests/NanoKVMiPadTests.swift index ac71bf2..271b4b0 100644 --- a/KVMConsoleiPadTests/NanoKVMiPadTests.swift +++ b/KVMConsoleiPadTests/NanoKVMiPadTests.swift @@ -116,6 +116,27 @@ final class KVMConsoleiPadTests: XCTestCase { ) } + /// Pinch and pan both claim two touches as soon as they land, so without this the two-finger + /// tap loses the gesture to a zoom that goes nowhere and no right click is ever sent. + @MainActor + func test_pinchAndPanWaitForTheTwoFingerTapToFail() { + let view = PointerCaptureUIView() + let recognizers = view.gestureRecognizers ?? [] + let taps = recognizers.compactMap { $0 as? UITapGestureRecognizer } + let twoFinger = try! XCTUnwrap(taps.first { $0.numberOfTouchesRequired == 2 }) + let singleTap = try! XCTUnwrap(taps.first { $0.numberOfTouchesRequired == 1 }) + let pinch = try! XCTUnwrap(recognizers.compactMap { $0 as? UIPinchGestureRecognizer }.first) + let pan = try! XCTUnwrap(recognizers.compactMap { $0 as? UIPanGestureRecognizer }.first) + + XCTAssertTrue(view.gestureRecognizer(pinch, shouldRequireFailureOf: twoFinger)) + XCTAssertTrue(view.gestureRecognizer(pan, shouldRequireFailureOf: twoFinger)) + + XCTAssertFalse( + view.gestureRecognizer(pinch, shouldRequireFailureOf: singleTap), + "a one-finger tap can never block a pinch, so waiting on it would only add latency" + ) + } + // MARK: Synthesized tap dwell @MainActor diff --git a/project.yml b/project.yml index ed772ee..b3a1e23 100644 --- a/project.yml +++ b/project.yml @@ -20,7 +20,7 @@ settings: # Single source of truth for app version, shared by both targets via # $(MARKETING_VERSION) / $(CURRENT_PROJECT_VERSION) references in their # Info.plist. CI overrides CURRENT_PROJECT_VERSION per release (timestamp). - MARKETING_VERSION: "1.0.6" + MARKETING_VERSION: "1.0.7" CURRENT_PROJECT_VERSION: "1" targets: KVMConsole: From 0d7d9e72e218372836b6b77619ff9c85b4de8f99 Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Fri, 21 Aug 2026 23:44:25 +0800 Subject: [PATCH 2/2] Add a gesture UI-test harness and correct the two-finger tap arbitration GestureHarness is a UI-test host that puts the real PointerCaptureView on screen with capture forced on and surfaces emitted reports in a label. GestureHarnessUITests drives real multi-touch against it. Recognizer arbitration cannot be reached from a unit test, and it is where both of the recent gesture bugs lived. It immediately paid for itself twice, and once against expectation: - The first version of this fix made both pan and pinch wait for the tap to fail. That starves the pinch completely: a spread fired a right click and did not zoom. Caught before shipping. - Running the harness against the 1.0.6 code showed a pinch there emits a right click as well as zooming, because the delegate allowed the tap and the pinch to recognize simultaneously. That is a real defect in the shipped build, and separate from the reported one. So the arbitration is now asymmetric: the pan waits for the tap to fail, which is what makes the tap fire at all; the pinch does not wait and is kept off the tap by mutual exclusion instead. Honest limitation: the harness does NOT reproduce the reported bug. The two-finger tap test passes against the 1.0.6 code, because XCUITest synthesizes two perfectly simultaneous touches with no drift and real fingers have plenty. The pan-waits-for-tap fix therefore still rests on inference and needs hardware confirmation. The harness has its own scheme and stays out of CI, so PR runs keep their current shape. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XU7Ck7AbZ9KpRhdPbRzNj8 --- GestureHarness/App/GestureHarnessApp.swift | 71 +++++++++++++++++++ .../PointerGestureTests.swift | 67 +++++++++++++++++ KVMConsoleiPad/Input/PointerCaptureView.swift | 21 ++++-- KVMConsoleiPadTests/NanoKVMiPadTests.swift | 42 +++++++---- project.yml | 42 +++++++++++ 5 files changed, 225 insertions(+), 18 deletions(-) create mode 100644 GestureHarness/App/GestureHarnessApp.swift create mode 100644 GestureHarnessUITests/PointerGestureTests.swift diff --git a/GestureHarness/App/GestureHarnessApp.swift b/GestureHarness/App/GestureHarnessApp.swift new file mode 100644 index 0000000..2f96be4 --- /dev/null +++ b/GestureHarness/App/GestureHarnessApp.swift @@ -0,0 +1,71 @@ +import KVMCore +import SwiftUI + +/// A host for UI tests that need real multi-touch against `PointerCaptureView`. +/// +/// The gesture recognizers in `PointerCaptureUIView` arbitrate against each other, and that +/// arbitration cannot be exercised from a unit test — synthesizing touches needs a running app. +/// This target exists only so `GestureHarnessUITests` has somewhere to tap. It is never shipped. +@main +struct GestureHarnessApp: App { + var body: some Scene { + WindowGroup { + GestureHarnessView() + } + } +} + +struct GestureHarnessView: View { + @StateObject private var zoom = ViewerZoomState() + // A click is a press followed by a release 50ms later, so the *last* report is always the + // release. The presses are what the tests are asking about, so keep those. + @State private var presses: [Int] = [] + @State private var wheelNotches = 0 + @State private var reportCount = 0 + + /// Square so the aspect-fit rect is predictable regardless of the simulator's screen size. + private let videoSize = CGSize(width: 1000, height: 1000) + + var body: some View { + ZStack { + Color.black + + PointerCaptureView( + isEnabled: true, + isScrollInverted: false, + videoSize: videoSize, + zoom: zoom, + onMouseReport: { report in record(report) }, + onMouseRelease: { report in record(report) } + ) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .accessibilityIdentifier("pointerCapture") + } + .ignoresSafeArea() + .overlay(alignment: .top) { + // The UI test reads this rather than the view hierarchy: gesture output is a stream of + // reports, not view state, so it has to be surfaced somewhere XCUITest can see it. + Text(stateDescription) + .accessibilityIdentifier("state") + .foregroundStyle(.white) + .font(.caption) + .padding(6) + } + } + + private func record(_ report: HIDMouseAbsoluteReport) { + reportCount += 1 + if report.buttons != 0 { + presses.append(Int(report.buttons)) + } + if report.wheel != 0 { + wheelNotches += 1 + } + } + + private var stateDescription: String { + let pressList = presses.map(String.init).joined(separator: ",") + let scale = String(format: "%.2f", zoom.scale) + return "presses=[\(pressList)] wheel=\(wheelNotches) count=\(reportCount) scale=\(scale)" + } +} diff --git a/GestureHarnessUITests/PointerGestureTests.swift b/GestureHarnessUITests/PointerGestureTests.swift new file mode 100644 index 0000000..c877ab1 --- /dev/null +++ b/GestureHarnessUITests/PointerGestureTests.swift @@ -0,0 +1,67 @@ +import XCTest + +/// Real multi-touch against the real recognizers. Unit tests can assert how `PointerCaptureUIView` +/// is *configured*, but not which recognizer wins when several want the same touches — and that +/// arbitration is what broke the two-finger right click in 1.0.6. +final class PointerGestureTests: XCTestCase { + private var app: XCUIApplication! + + override func setUp() { + super.setUp() + continueAfterFailure = false + app = XCUIApplication() + app.launch() + } + + func test_oneFingerTapSendsThePrimaryButton() { + let surface = app.otherElements["pointerCapture"] + XCTAssertTrue(surface.waitForExistence(timeout: 10)) + + surface.tap() + + XCTAssertTrue(waitForState(containing: "presses=[1]"), "one-finger tap should press the left button, got \(stateText)") + } + + func test_twoFingerTapSendsTheSecondaryButton() { + let surface = app.otherElements["pointerCapture"] + XCTAssertTrue(surface.waitForExistence(timeout: 10)) + + surface.tap(withNumberOfTaps: 1, numberOfTouches: 2) + + XCTAssertTrue( + waitForState(containing: "presses=[2]"), + "two-finger tap should press the right button, got \(stateText)" + ) + } + + func test_pinchZoomsWithoutSendingAClick() { + let surface = app.otherElements["pointerCapture"] + XCTAssertTrue(surface.waitForExistence(timeout: 10)) + + surface.pinch(withScale: 3, velocity: 3) + + XCTAssertTrue(waitForState(satisfying: { !$0.contains("scale=1.00") }), "pinch should zoom, got \(stateText)") + XCTAssertTrue( + stateText.contains("presses=[]"), + "a pinch must not also fire a click, got \(stateText)" + ) + } + + private var stateText: String { + app.staticTexts["state"].label + } + + private func waitForState(containing needle: String) -> Bool { + waitForState(satisfying: { $0.contains(needle) }) + } + + private func waitForState(satisfying predicate: @escaping (String) -> Bool) -> Bool { + let element = app.staticTexts["state"] + let matches = NSPredicate { object, _ in + guard let element = object as? XCUIElement else { return false } + return predicate(element.label) + } + let expectation = XCTNSPredicateExpectation(predicate: matches, object: element) + return XCTWaiter().wait(for: [expectation], timeout: 5) == .completed + } +} diff --git a/KVMConsoleiPad/Input/PointerCaptureView.swift b/KVMConsoleiPad/Input/PointerCaptureView.swift index 83e373a..27c2942 100644 --- a/KVMConsoleiPad/Input/PointerCaptureView.swift +++ b/KVMConsoleiPad/Input/PointerCaptureView.swift @@ -294,6 +294,14 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { _ gestureRecognizer: UIGestureRecognizer, shouldRecognizeSimultaneouslyWith other: UIGestureRecognizer ) -> Bool { + // A pinch and the two-finger tap must be mutually exclusive, or spreading two fingers + // zooms *and* fires a right click — the tap's movement tolerance is measured loosely + // enough that a symmetric spread still looks stationary to it. A pinch begins as soon as + // the fingers move, which cancels the tap; a tap that stays still never starts a pinch. + if let twoFingerTapRecognizer, + gestureRecognizer === twoFingerTapRecognizer || other === twoFingerTapRecognizer { + return false + } // Pinch should coexist with the pointer pan so two-finger zoom/pan doesn't fail-cancel // the pan's wheel-scroll tracking (we suppress wheel emission while pinch is active). return true @@ -303,13 +311,14 @@ final class PointerCaptureUIView: UIView, UIGestureRecognizerDelegate { _ gestureRecognizer: UIGestureRecognizer, shouldRequireFailureOf other: UIGestureRecognizer ) -> Bool { - // Pinch and pan both claim two touches the moment they land, and whichever begins first - // takes the gesture — which is why a two-finger tap registered as a zoom that went - // nowhere instead of a right click. Making them wait for the tap to fail costs nothing in - // practice: any real zoom or scroll moves the fingers, which fails the tap immediately. - // Only holding two fingers still defers them, and that is the tap itself. + // The pan claims two touches the moment they land and begins before the tap can complete, + // which is why the two-finger tap never fired. Making it wait for the tap to fail costs + // nothing in practice: any real scroll moves the fingers, which fails the tap at about the + // threshold the pan needs anyway. The pinch is deliberately NOT here — making it wait + // starves it entirely, so a spread fires a right click instead of zooming. Pinch is kept + // off the tap by the mutual-exclusion rule above instead. guard let twoFingerTapRecognizer, other === twoFingerTapRecognizer else { return false } - return gestureRecognizer === pinchRecognizer || gestureRecognizer === pointerPanRecognizer + return gestureRecognizer === pointerPanRecognizer } } diff --git a/KVMConsoleiPadTests/NanoKVMiPadTests.swift b/KVMConsoleiPadTests/NanoKVMiPadTests.swift index 271b4b0..040d4f0 100644 --- a/KVMConsoleiPadTests/NanoKVMiPadTests.swift +++ b/KVMConsoleiPadTests/NanoKVMiPadTests.swift @@ -116,24 +116,42 @@ final class KVMConsoleiPadTests: XCTestCase { ) } - /// Pinch and pan both claim two touches as soon as they land, so without this the two-finger - /// tap loses the gesture to a zoom that goes nowhere and no right click is ever sent. + /// The two-finger tap has to be arbitrated against pan and pinch differently, and the + /// difference was established by running real multi-touch through `GestureHarnessUITests`: + /// + /// - The **pan** claims two touches the moment they land and begins before the tap can + /// complete, so it must wait for the tap to fail. That is what makes the tap fire at all. + /// - The **pinch** must NOT wait: making it wait starves it completely, so a spread fires a + /// right click instead of zooming. It is kept off the tap by mutual exclusion instead — + /// a pinch begins as soon as the fingers move, which cancels the tap. @MainActor - func test_pinchAndPanWaitForTheTwoFingerTapToFail() { + func test_twoFingerTapArbitratesAgainstPanAndPinchDifferently() { let view = PointerCaptureUIView() let recognizers = view.gestureRecognizers ?? [] let taps = recognizers.compactMap { $0 as? UITapGestureRecognizer } - let twoFinger = try! XCTUnwrap(taps.first { $0.numberOfTouchesRequired == 2 }) - let singleTap = try! XCTUnwrap(taps.first { $0.numberOfTouchesRequired == 1 }) - let pinch = try! XCTUnwrap(recognizers.compactMap { $0 as? UIPinchGestureRecognizer }.first) - let pan = try! XCTUnwrap(recognizers.compactMap { $0 as? UIPanGestureRecognizer }.first) - - XCTAssertTrue(view.gestureRecognizer(pinch, shouldRequireFailureOf: twoFinger)) - XCTAssertTrue(view.gestureRecognizer(pan, shouldRequireFailureOf: twoFinger)) + guard + let twoFinger = taps.first(where: { $0.numberOfTouchesRequired == 2 }), + let pinch = recognizers.compactMap({ $0 as? UIPinchGestureRecognizer }).first, + let pan = recognizers.compactMap({ $0 as? UIPanGestureRecognizer }).first + else { + return XCTFail("expected a two-finger tap, a pinch and a pan on the capture view") + } + XCTAssertTrue( + view.gestureRecognizer(pan, shouldRequireFailureOf: twoFinger), + "without this the pan begins first and the two-finger tap never fires" + ) XCTAssertFalse( - view.gestureRecognizer(pinch, shouldRequireFailureOf: singleTap), - "a one-finger tap can never block a pinch, so waiting on it would only add latency" + view.gestureRecognizer(pinch, shouldRequireFailureOf: twoFinger), + "making the pinch wait starves it: a spread fires a right click instead of zooming" + ) + XCTAssertFalse( + view.gestureRecognizer(pinch, shouldRecognizeSimultaneouslyWith: twoFinger), + "otherwise a spread both zooms and fires a right click" + ) + XCTAssertTrue( + view.gestureRecognizer(pinch, shouldRecognizeSimultaneouslyWith: pan), + "pinch and pan still coexist so zooming doesn't fail-cancel wheel tracking" ) } diff --git a/project.yml b/project.yml index b3a1e23..a28d45f 100644 --- a/project.yml +++ b/project.yml @@ -122,6 +122,38 @@ targets: SWIFT_STRICT_CONCURRENCY: complete ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon GENERATE_INFOPLIST_FILE: NO + GestureHarness: + # UI-test host only — never shipped. Exists so GestureHarnessUITests can drive real + # multi-touch against PointerCaptureView; recognizer arbitration cannot be unit tested. + type: application + platform: iOS + sources: + - GestureHarness + # The real files under test, compiled into the harness — not a copy of them. + - KVMConsoleiPad/Input/PointerCaptureView.swift + - KVMConsoleiPad/Input/SynthesizedTapSequencer.swift + dependencies: + - package: KVMCore + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: io.lyx.KVMConsole.GestureHarness + TARGETED_DEVICE_FAMILY: "2" + IPHONEOS_DEPLOYMENT_TARGET: "26.0" + SWIFT_STRICT_CONCURRENCY: complete + GENERATE_INFOPLIST_FILE: YES + INFOPLIST_KEY_UILaunchScreen_Generation: YES + GestureHarnessUITests: + type: bundle.ui-testing + platform: iOS + sources: + - GestureHarnessUITests + dependencies: + - target: GestureHarness + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: io.lyx.GestureHarnessUITests + GENERATE_INFOPLIST_FILE: YES + TEST_TARGET_NAME: GestureHarness KVMConsoleiPadTests: type: bundle.unit-test platform: iOS @@ -151,3 +183,13 @@ schemes: test: targets: - KVMConsoleiPadTests + GestureHarness: + # Deliberately separate from the KVMConsoleiPad scheme: UI tests are slow and flakier than + # unit tests, so they run on demand rather than on every PR. + build: + targets: + GestureHarness: all + GestureHarnessUITests: [test] + test: + targets: + - GestureHarnessUITests