From e90417381deeda7962ecb844802d92d4a8de0c9a Mon Sep 17 00:00:00 2001 From: MohamadAbakar <136388435+MohamadAbakar@users.noreply.github.com> Date: Sat, 29 Aug 2026 07:13:07 -0700 Subject: [PATCH] feat: Sequoia 15 Intel port using on-device Apple Speech Retarget the app for macOS 15 on Intel so it can run without macOS 26, the Neural Engine, or Xcode 26. Speech, polish, and the build script now use APIs and toolchains available on Sequoia. Co-authored-by: Cursor --- Package.resolved | 87 ------- Package.swift | 19 +- README.md | 8 +- Sources/Plynn/OnboardingWindow.swift | 10 +- Sources/Plynn/SettingsWindow.swift | 15 +- Sources/Plynn/main.swift | 22 +- Sources/PlynnKit/AppleFMFormatter.swift | 44 +--- Sources/PlynnKit/AppleSpeechEngine.swift | 217 +++++++++++------- Sources/PlynnKit/AudioFile.swift | 4 +- Sources/PlynnKit/AudioRecorder.swift | 2 +- Sources/PlynnKit/DictationEngine.swift | 5 +- Sources/PlynnKit/EngineManager.swift | 71 +----- Sources/PlynnKit/Feedback.swift | 2 +- Sources/PlynnKit/IndicatorView.swift | 76 +++--- Sources/PlynnKit/LLMFormatter.swift | 57 ++--- Sources/PlynnKit/MeetingRecorder.swift | 4 +- Sources/PlynnKit/Permissions.swift | 11 +- Sources/PlynnKit/Resampler.swift | 2 +- Sources/PlynnKit/StreamingTranscriber.swift | 78 +------ Sources/PlynnKit/Transcriber.swift | 31 +-- .../AppleSpeechEngineTests.swift | 8 + Tests/PlynnKitTests/EngineManagerTests.swift | 24 +- .../StreamingTranscriberTests.swift | 18 +- Tests/PlynnKitTests/TranscriberTests.swift | 20 +- docs/INSTALL.md | 2 +- scripts/Info.plist | 8 +- scripts/make-app.sh | 40 ++-- 27 files changed, 319 insertions(+), 566 deletions(-) delete mode 100644 Package.resolved diff --git a/Package.resolved b/Package.resolved deleted file mode 100644 index 30624a4..0000000 --- a/Package.resolved +++ /dev/null @@ -1,87 +0,0 @@ -{ - "originHash" : "93d2426861223fcf3995e6e377c47859f9ef923b95d58685aa0e9a9a23409d03", - "pins" : [ - { - "identity" : "fluidaudio", - "kind" : "remoteSourceControl", - "location" : "https://github.com/FluidInference/FluidAudio.git", - "state" : { - "revision" : "19600a485baa4998812e4654b70d2bab8f2c9949", - "version" : "0.15.5" - } - }, - { - "identity" : "gzipswift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/1024jp/GzipSwift", - "state" : { - "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05", - "version" : "6.0.1" - } - }, - { - "identity" : "mlx-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ml-explore/mlx-swift", - "state" : { - "revision" : "072b684acaae80b6a463abab3a103732f33774bf", - "version" : "0.29.1" - } - }, - { - "identity" : "mlx-swift-examples", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ml-explore/mlx-swift-examples.git", - "state" : { - "revision" : "9bff95ca5f0b9e8c021acc4d71a2bbe4a7441631", - "version" : "2.29.1" - } - }, - { - "identity" : "sparkle", - "kind" : "remoteSourceControl", - "location" : "https://github.com/sparkle-project/Sparkle.git", - "state" : { - "revision" : "79bc9e872948e47877e76f194cb0c8e0412b0b90", - "version" : "2.9.5" - } - }, - { - "identity" : "swift-collections", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-collections.git", - "state" : { - "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", - "version" : "1.6.0" - } - }, - { - "identity" : "swift-jinja", - "kind" : "remoteSourceControl", - "location" : "https://github.com/huggingface/swift-jinja.git", - "state" : { - "revision" : "7d0b8880ef8e567dd4e0089f8b99fb354129017c", - "version" : "2.4.2" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics", - "state" : { - "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", - "version" : "1.1.1" - } - }, - { - "identity" : "swift-transformers", - "kind" : "remoteSourceControl", - "location" : "https://github.com/huggingface/swift-transformers", - "state" : { - "revision" : "a2e184dddb4757bc943e77fbe99ac6786c53f0b2", - "version" : "1.0.0" - } - } - ], - "version" : 3 -} diff --git a/Package.swift b/Package.swift index a109f2f..01ce1e2 100644 --- a/Package.swift +++ b/Package.swift @@ -3,25 +3,12 @@ import PackageDescription let package = Package( name: "Plynn", - platforms: [.macOS("26.0")], - dependencies: [ - .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.15.5"), - .package(url: "https://github.com/ml-explore/mlx-swift-examples.git", from: "2.29.1"), - .package(url: "https://github.com/sparkle-project/Sparkle.git", from: "2.6.0"), - ], + platforms: [.macOS(.v15)], targets: [ - .target( - name: "PlynnKit", - dependencies: [ - .product(name: "FluidAudio", package: "FluidAudio"), - .product(name: "MLXLLM", package: "mlx-swift-examples"), - ]), + .target(name: "PlynnKit"), .executableTarget( name: "Plynn", - dependencies: [ - "PlynnKit", - .product(name: "Sparkle", package: "Sparkle"), - ]), + dependencies: ["PlynnKit"]), .testTarget( name: "PlynnKitTests", dependencies: ["PlynnKit"], diff --git a/README.md b/README.md index e97b72e..8ce54fd 100644 --- a/README.md +++ b/README.md @@ -20,21 +20,21 @@ I built this because I loved what Wispr Flow could do but didn't love sending my Grab it at [plynn.vercel.app](https://plynn.vercel.app), or grab `Plynn.dmg` from [Releases](../../releases), drag Plynn to Applications, and launch. The app is notarized, so there's nothing to bypass. Onboarding asks for microphone and accessibility permissions, and the speech models (about 1 GB) download in the background while Apple's built-in engine covers your first dictations. -Requires macOS 26 (Tahoe) on Apple Silicon. Enable Apple Intelligence in System Settings to get the best polish engine. +**This tree is a Sequoia 15 / Intel compatibility port.** Upstream Plynn requires macOS 26 (Tahoe) on Apple Silicon. Here the hold-to-talk loop, paste, dictionary, snippets, and rules polish run on macOS 15.7 Intel using Apple's on-device Speech recognizer. Parakeet, Apple Intelligence, and the Qwen MLX polish model are not available on this hardware. ## Build from source ```bash -git clone https://github.com/31Carlton7/plynn.git cd plynn ./scripts/make-app.sh +open build/Plynn.app ``` -You'll need Xcode 26 with the Metal toolchain component (`xcodebuild -downloadComponent MetalToolchain`). The build uses `xcodebuild` rather than plain `swift build` because the MLX Metal shaders require it. `swift test` runs the 80+ unit tests. +Needs the macOS Command Line Tools (Swift 6.1+) or Xcode 16. No Xcode 26 / Metal toolchain. After the first launch, grant Microphone, Speech Recognition, and Accessibility, then relaunch. ## How it's put together -Speech recognition is Parakeet TDT running on the Neural Engine via [FluidAudio](https://github.com/FluidInference/FluidAudio), streaming partials as you speak. A deterministic rules pass handles spoken punctuation instantly, then a small on-device language model does the heavier cleanup, but only when the transcript actually needs it. Clean short dictations skip the model entirely and paste immediately. A latency gate, basically. +On this Sequoia Intel port, speech recognition is Apple's on-device Speech framework. A deterministic rules pass still handles spoken punctuation, snippets, and your dictionary. The Neural Engine Parakeet stack and the MLX polish model are Apple Silicon / macOS 26 only. Your dictionary, snippets, and history live in one SQLite file at `~/Library/Application Support/Plynn/`. Nothing is sent anywhere, ever. There's no account, no telemetry, and no server to go down. diff --git a/Sources/Plynn/OnboardingWindow.swift b/Sources/Plynn/OnboardingWindow.swift index f00cf14..fbd154e 100644 --- a/Sources/Plynn/OnboardingWindow.swift +++ b/Sources/Plynn/OnboardingWindow.swift @@ -5,6 +5,7 @@ import SwiftUI struct OnboardingView: View { let engineManager: EngineManager @State private var mic = Permissions.micGranted() + @State private var speech = Permissions.speechGranted() @State private var ax = Permissions.accessibilityGranted() @State private var globe = Permissions.globeKeySafe() @State private var grantedAxThisRun = false @@ -15,7 +16,7 @@ struct OnboardingView: View { VStack(alignment: .leading, spacing: 18) { Text("Set up Plynn") .font(.title2.bold()) - Text("Three quick steps, then hold **fn** anywhere and talk.") + Text("A few quick steps, then hold **fn** anywhere and talk.") .foregroundStyle(.secondary) Text("Using a third-party keyboard and fn doesn't do anything? Change the activation key under Settings → Hotkey.") .font(.caption) @@ -25,6 +26,10 @@ struct OnboardingView: View { detail: "Plynn hears you only while a hotkey is held.") { Button("Grant") { Permissions.requestMic() } } + row(done: speech, title: "Speech Recognition", + detail: "Turns what you say into text on this Mac — nothing is uploaded.") { + Button("Grant") { Permissions.requestSpeech() } + } row(done: ax, title: "Accessibility", detail: "Lets Plynn see the fn key and paste text for you.") { Button("Grant") { @@ -52,7 +57,7 @@ struct OnboardingView: View { } .font(.callout) - if mic && ax { + if mic && speech && ax { Text("Ready — focus any text field, hold **fn**, and speak.") .font(.callout) .foregroundStyle(.green) @@ -62,6 +67,7 @@ struct OnboardingView: View { .frame(width: 460) .onReceive(timer) { _ in mic = Permissions.micGranted() + speech = Permissions.speechGranted() let axNow = Permissions.accessibilityGranted() if axNow && !ax { grantedAxThisRun = true } ax = axNow diff --git a/Sources/Plynn/SettingsWindow.swift b/Sources/Plynn/SettingsWindow.swift index b17d211..7a53356 100644 --- a/Sources/Plynn/SettingsWindow.swift +++ b/Sources/Plynn/SettingsWindow.swift @@ -55,13 +55,12 @@ struct SettingsPane: View { var body: some View { Form { - Section("Transcription") { - Picker("Engine", selection: $engineManager.preferred) { - Text("Automatic").tag(EngineChoice.auto) - Text("Parakeet (local)").tag(EngineChoice.parakeet) - Text("Apple (built-in)").tag(EngineChoice.apple) - } - LabeledContent("Status", value: engineManager.statusLine) + Section { + LabeledContent("Engine", value: engineManager.statusLine) + } header: { + Text("Transcription") + } footer: { + Text("This Sequoia Intel build uses Apple's on-device Speech recognizer. Parakeet and the Qwen polish model need Apple Silicon and macOS 26.") } Section { Picker("Activation key", selection: $hotkeyTrigger) { @@ -80,7 +79,7 @@ struct SettingsPane: View { } header: { Text("Formatting") } footer: { - Text("Polish removes filler words, applies self-corrections, formats lists, and matches tone to the app — on-device via Apple Intelligence. Corrections you make right after a paste teach the dictionary automatically. Everything stays on this Mac.") + Text("On this Intel Sequoia build, polish is the on-device rules pass: spoken punctuation, snippets, and your dictionary. Apple Intelligence and the local Qwen model are not available on Intel. Corrections you make right after a paste still teach the dictionary.") } Section { Toggle("Sound effects", isOn: $soundEffects) diff --git a/Sources/Plynn/main.swift b/Sources/Plynn/main.swift index 7f2a322..30c31db 100644 --- a/Sources/Plynn/main.swift +++ b/Sources/Plynn/main.swift @@ -1,6 +1,5 @@ import AppKit import PlynnKit -import Sparkle @MainActor final class AppDelegate: NSObject, NSApplicationDelegate { @@ -19,8 +18,6 @@ final class AppDelegate: NSObject, NSApplicationDelegate { self?.onboarding.show() } var statusItem: NSStatusItem! - let updater = SPUStandardUpdaterController( - startingUpdater: true, updaterDelegate: nil, userDriverDelegate: nil) let store = try? PersonalStore(path: PersonalStore.defaultPath()) lazy var formatter = TranscriptFormatter(personalization: { [store] in @@ -61,7 +58,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate { Metrics.residentMB(), AXIsProcessTrusted() ? 1 : 0) setUpStatusItem() - if !Permissions.micGranted() || !Permissions.accessibilityGranted() { + if !Permissions.micGranted() || !Permissions.speechGranted() + || !Permissions.accessibilityGranted() + { onboarding.show() } @@ -86,7 +85,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { NSLog("plynn: polish engine %@; RSS %.0f MB", await formatter.polishEngine ?? "none (rules only)", Metrics.residentMB()) if let reason = await formatter.appleFMStatus { - NSLog("plynn: Apple Intelligence %@", reason) + NSLog("plynn: polish %@", reason) } } @@ -315,7 +314,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { let title = Self.defaultMeetingTitle(started) meetingID = try? store?.addMeeting(title: title, startedAt: started) - // Meetings always run on Parakeet: it streams and has no session cap. + // Meetings use the same on-device Apple Speech engine as dictation. let engine = engineManager.engineForNewSession() meetingEngine = engine let (stream, continuation) = AsyncStream.makeStream(of: [Float].self) @@ -372,8 +371,8 @@ final class AppDelegate: NSObject, NSApplicationDelegate { private var meetingSegmentStart: TimeInterval = 0 func meetingPartial(_ text: String) { let elapsed = meetingRecorder?.elapsed ?? 0 - // Parakeet's partial grows monotonically within an utterance and - // shrinks when a new one starts — the shrink is our segment boundary. + // The recognizer's partial grows within an utterance and shrinks + // when a new one starts — the shrink is our segment boundary. if text.count < meetingLastPartial.count, !meetingLastPartial.isEmpty { meetingTranscript.append(meetingLastPartial, at: meetingSegmentStart) meetingSegmentStart = elapsed @@ -511,13 +510,6 @@ final class AppDelegate: NSObject, NSApplicationDelegate { meetingItem.target = self menu.addItem(meetingItem) menu.addItem(.separator()) - let updateItem = NSMenuItem( - title: "Check for Updates…", - action: #selector(SPUStandardUpdaterController.checkForUpdates(_:)), - keyEquivalent: "") - updateItem.target = updater - menu.addItem(updateItem) - menu.addItem(.separator()) menu.addItem(NSMenuItem( title: "Quit Plynn", action: #selector(NSApplication.terminate(_:)), keyEquivalent: "q")) statusItem.menu = menu diff --git a/Sources/PlynnKit/AppleFMFormatter.swift b/Sources/PlynnKit/AppleFMFormatter.swift index 8d91216..1f98572 100644 --- a/Sources/PlynnKit/AppleFMFormatter.swift +++ b/Sources/PlynnKit/AppleFMFormatter.swift @@ -1,51 +1,25 @@ import Foundation -import FoundationModels -/// AI polish on Apple's on-device Foundation Model (Apple Intelligence). -/// The default polish engine — no download, Apple-tuned for the hardware. -/// Falls back to the input text on every failure mode, like all polish paths. +/// Apple Intelligence polish is unavailable on Intel Sequoia (Foundation +/// Models requires macOS 26 + Apple Silicon). The actor stays so the rest of +/// the formatting pipeline compiles unchanged; every call degrades to the +/// input text, same as the upstream failure path. public actor AppleFMFormatter { public init() {} - public nonisolated var ready: Bool { - SystemLanguageModel.default.isAvailable - } + public nonisolated var ready: Bool { false } - /// Human-readable availability, for logs and Settings. public nonisolated var availabilityDescription: String { - switch SystemLanguageModel.default.availability { - case .available: return "available" - case .unavailable(.appleIntelligenceNotEnabled): - return "Apple Intelligence is not enabled in System Settings" - case .unavailable(.modelNotReady): - return "model still downloading — will be used once ready" - case .unavailable(.deviceNotEligible): - return "this Mac doesn't support Apple Intelligence" - case .unavailable(let reason): - return "unavailable (\(reason))" - } + "Apple Intelligence needs Apple Silicon and macOS 26 — this Sequoia Intel build uses rules-only polish" } - /// Ask the system to page the model in so the first dictation is fast. - public func warm() { - guard ready else { return } - LanguageModelSession().prewarm() - } + public func warm() {} - /// One stateless prompt → raw completion (nil on timeout/error/unavailable). - public func complete(_ prompt: String) async -> String? { - guard ready else { return nil } - return await withTaskTimeout(seconds: 10) { - try await LanguageModelSession().respond(to: prompt).content - } - } + public func complete(_ prompt: String) async -> String? { nil } public func format( _ text: String, tone: Tone, technical: Bool, preferredSpellings: [String] = [] ) async -> String { - let prompt = PolishPrompt.build( - transcript: text, tone: tone, technical: technical, - preferredSpellings: preferredSpellings) - return PolishPrompt.sanitize(await complete(prompt), input: text) + text } } diff --git a/Sources/PlynnKit/AppleSpeechEngine.swift b/Sources/PlynnKit/AppleSpeechEngine.swift index e310658..2356fa0 100644 --- a/Sources/PlynnKit/AppleSpeechEngine.swift +++ b/Sources/PlynnKit/AppleSpeechEngine.swift @@ -1,63 +1,51 @@ -import AVFoundation -import Speech +@preconcurrency import AVFoundation +@preconcurrency import Speech -/// Zero-download dictation engine backed by macOS 26's SpeechAnalyzer / -/// SpeechTranscriber. Model assets are OS-managed; quality is Whisper-small -/// class — used while Parakeet downloads, or when the user prefers it. +/// On-device dictation via `SFSpeechRecognizer` (macOS 15 / Sequoia). +/// +/// The upstream Plynn engine is SpeechAnalyzer / SpeechTranscriber (macOS 26). +/// Those types do not exist in the Sequoia SDK, so this port uses the Speech +/// framework that ships with the OS. `requiresOnDeviceRecognition` stays on +/// so audio never leaves the Mac. public actor AppleSpeechEngine: DictationEngine { - public enum EngineError: Error { case assetUnavailable, notStarted } + public enum EngineError: Error { + case assetUnavailable, notStarted, notAuthorized, recognizerUnavailable + } public nonisolated let displayName = "Apple (built-in)" - private var transcriber: SpeechTranscriber? - private var analyzer: SpeechAnalyzer? - private var inputBuilder: AsyncStream.Continuation? - private var resultsTask: Task? - private var analyzerFormat: AVAudioFormat? - private var finalText = "" + private var recognizer: SFSpeechRecognizer? + private var request: SFSpeechAudioBufferRecognitionRequest? + private var task: SFSpeechRecognitionTask? + private var accumulated = "" + private var latestHypothesis = "" + private var finished = false + private var lastError: Error? private var partialCallback: (@Sendable (String) -> Void)? public init() {} public func start() async throws { - // Tear down any previous session. - inputBuilder?.finish() - resultsTask?.cancel() - finalText = "" - - let locale = Locale(identifier: "en_US") - let transcriber = SpeechTranscriber( - locale: locale, - transcriptionOptions: [], - reportingOptions: [.volatileResults], - attributeOptions: []) - self.transcriber = transcriber - - // Ensure the OS speech asset for this locale is installed. - if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { - do { try await request.downloadAndInstall() } catch { throw EngineError.assetUnavailable } - } + await teardown() + accumulated = "" + latestHypothesis = "" + finished = false + lastError = nil - let analyzer = SpeechAnalyzer(modules: [transcriber]) - self.analyzer = analyzer - analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) - - let (stream, continuation) = AsyncStream.makeStream(of: AnalyzerInput.self) - inputBuilder = continuation - try await analyzer.start(inputSequence: stream) - - resultsTask = Task { [weak self] in - do { - for try await result in transcriber.results { - let text = String(result.text.characters) - if result.isFinal { - await self?.appendFinal(text) - } else { - await self?.emitPartial(text) - } - } - } catch {} + let status = await Self.requestAuthorization() + guard status == .authorized else { throw EngineError.notAuthorized } + + let recognizer = SFSpeechRecognizer(locale: Locale(identifier: "en_US")) + guard let recognizer, recognizer.isAvailable else { + throw EngineError.recognizerUnavailable } + // Prefer the on-device model. If it is not installed, recognition + // fails locally rather than uploading audio to Apple. + if !recognizer.supportsOnDeviceRecognition { + NSLog("plynn: on-device speech model missing — enable Dictation in System Settings") + } + self.recognizer = recognizer + try beginRequest() } public func setPartialCallback(_ callback: @escaping @Sendable (String) -> Void) { @@ -65,54 +53,107 @@ public actor AppleSpeechEngine: DictationEngine { } public func append(samples: [Float]) async throws { - guard let inputBuilder, let analyzerFormat else { throw EngineError.notStarted } - let src = AVAudioPCMBuffer( - pcmFormat: AudioFile.targetFormat, frameCapacity: AVAudioFrameCount(samples.count))! - src.frameLength = AVAudioFrameCount(samples.count) - samples.withUnsafeBufferPointer { p in - src.floatChannelData![0].update(from: p.baseAddress!, count: samples.count) - } - let converted: AVAudioPCMBuffer - if analyzerFormat == AudioFile.targetFormat { - converted = src - } else { - let out = AVAudioPCMBuffer( - pcmFormat: analyzerFormat, - frameCapacity: AVAudioFrameCount( - Double(samples.count) * analyzerFormat.sampleRate / 16_000 + 1_024))! - let conv = AVAudioConverter(from: AudioFile.targetFormat, to: analyzerFormat)! - var fed = false - var err: NSError? - conv.convert(to: out, error: &err) { _, status in - if fed { status.pointee = .endOfStream; return nil } - fed = true; status.pointee = .haveData; return src - } - if let err { throw err } - converted = out + guard let request else { throw EngineError.notStarted } + let buffer = AVAudioPCMBuffer( + pcmFormat: AudioFile.targetFormat, + frameCapacity: AVAudioFrameCount(samples.count))! + buffer.frameLength = AVAudioFrameCount(samples.count) + samples.withUnsafeBufferPointer { pointer in + buffer.floatChannelData![0].update(from: pointer.baseAddress!, count: samples.count) } - inputBuilder.yield(AnalyzerInput(buffer: converted)) + request.append(buffer) } public func finish() async throws -> String { - inputBuilder?.finish() - inputBuilder = nil - // Runs even when finalize throws — otherwise a failed session leaves a - // live analyzer and results loop behind for the next start() to fight. - defer { - resultsTask?.cancel() - resultsTask = nil - analyzer = nil - transcriber = nil + guard request != nil else { throw EngineError.notStarted } + request?.endAudio() + + let deadline = ContinuousClock.now + .seconds(12) + while !finished, ContinuousClock.now < deadline { + try await Task.sleep(for: .milliseconds(40)) + } + let text = displayText.trimmingCharacters(in: .whitespacesAndNewlines) + let error = lastError + await teardown() + if text.isEmpty, let error, Self.isAssetError(error) { + throw EngineError.assetUnavailable } - try await analyzer?.finalizeAndFinishThroughEndOfInput() - return finalText.trimmingCharacters(in: .whitespacesAndNewlines) + return text } - private func appendFinal(_ text: String) { - finalText += text + private var displayText: String { + let hypothesis = latestHypothesis.trimmingCharacters(in: .whitespacesAndNewlines) + if accumulated.isEmpty { return hypothesis } + if hypothesis.isEmpty { return accumulated } + return accumulated + " " + hypothesis } - private func emitPartial(_ text: String) { - partialCallback?(finalText + text) + private func beginRequest() throws { + guard let recognizer else { throw EngineError.notStarted } + let request = SFSpeechAudioBufferRecognitionRequest() + request.shouldReportPartialResults = true + request.requiresOnDeviceRecognition = true + request.addsPunctuation = true + request.taskHint = .dictation + self.request = request + finished = false + latestHypothesis = "" + + task = recognizer.recognitionTask(with: request) { [weak self] result, error in + let text = result?.bestTranscription.formattedString + let isFinal = result?.isFinal ?? false + let nsError = error as NSError? + guard let engine = self else { return } + Task { await engine.handle(text: text, isFinal: isFinal, error: nsError) } + } + } + + private func handle(text: String?, isFinal: Bool, error: NSError?) { + if let text { + latestHypothesis = text + if isFinal { + if !accumulated.isEmpty, !latestHypothesis.isEmpty { + accumulated += " " + latestHypothesis + } else if accumulated.isEmpty { + accumulated = latestHypothesis + } + latestHypothesis = "" + finished = true + } + partialCallback?(displayText) + } + if let error { + lastError = error + finished = true + } + } + + private func teardown() async { + task?.cancel() + task = nil + request = nil + recognizer = nil + finished = true + } + + private static func isAssetError(_ error: Error) -> Bool { + let ns = error as NSError + // Speech / assistant errors when the on-device locale pack is missing. + return ns.domain == "kAFAssistantErrorDomain" || ns.code == 203 || ns.code == 1700 + } + + private static func requestAuthorization() async -> SFSpeechRecognizerAuthorizationStatus { + await withCheckedContinuation { continuation in + DispatchQueue.main.async { + let current = SFSpeechRecognizer.authorizationStatus() + if current != .notDetermined { + continuation.resume(returning: current) + return + } + SFSpeechRecognizer.requestAuthorization { status in + continuation.resume(returning: status) + } + } + } } } diff --git a/Sources/PlynnKit/AudioFile.swift b/Sources/PlynnKit/AudioFile.swift index 51609b3..dca2bd3 100644 --- a/Sources/PlynnKit/AudioFile.swift +++ b/Sources/PlynnKit/AudioFile.swift @@ -1,7 +1,7 @@ -import AVFoundation +@preconcurrency import AVFoundation public enum AudioFile { - public static let targetFormat = AVAudioFormat( + nonisolated(unsafe) public static let targetFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false)! /// Reads any audio file and returns 16 kHz mono Float32 samples. diff --git a/Sources/PlynnKit/AudioRecorder.swift b/Sources/PlynnKit/AudioRecorder.swift index 60e10ad..5874c41 100644 --- a/Sources/PlynnKit/AudioRecorder.swift +++ b/Sources/PlynnKit/AudioRecorder.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation public enum AudioLevel { /// Root-mean-square level of a sample chunk (0 for empty input). diff --git a/Sources/PlynnKit/DictationEngine.swift b/Sources/PlynnKit/DictationEngine.swift index a0e1b25..2a8b4d2 100644 --- a/Sources/PlynnKit/DictationEngine.swift +++ b/Sources/PlynnKit/DictationEngine.swift @@ -1,9 +1,8 @@ import Foundation /// A streaming dictation engine: feed 16 kHz mono samples, get live partials -/// via callback and a final transcript from finish(). Implementations: -/// `StreamingTranscriber` (Parakeet, local download) and `AppleSpeechEngine` -/// (SpeechTranscriber, zero-download OS fallback). +/// via callback and a final transcript from finish(). On this Sequoia Intel +/// build the only implementation is `AppleSpeechEngine` (on-device Speech). public protocol DictationEngine: Actor { nonisolated var displayName: String { get } /// Load whatever the engine needs (idempotent) and reset for a new session. diff --git a/Sources/PlynnKit/EngineManager.swift b/Sources/PlynnKit/EngineManager.swift index 9a7a13f..f8743d0 100644 --- a/Sources/PlynnKit/EngineManager.swift +++ b/Sources/PlynnKit/EngineManager.swift @@ -1,91 +1,44 @@ -import FluidAudio import Foundation public enum EngineChoice: String, CaseIterable, Sendable { case auto, parakeet, apple - /// Pure selection: preferred engine when usable, Apple otherwise. + /// Sequoia Intel has one engine: Apple's on-device Speech recognizer. + /// Parakeet/FluidAudio needs the Neural Engine on Apple Silicon. public static func select(preferred: EngineChoice, parakeetReady: Bool) -> EngineChoice { - switch preferred { - case .apple: return .apple - case .parakeet, .auto: return parakeetReady ? .parakeet : .apple - } + .apple } - /// FluidAudio's cache convention: /parakeet-unified*/.mlmodelc public static func parakeetModelsPresent(in base: URL? = nil) -> Bool { - let dir = base - ?? FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] - .appendingPathComponent("FluidAudio/Models", isDirectory: true) - let fm = FileManager.default - guard let repos = try? fm.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil) - else { return false } - for repo in repos where repo.lastPathComponent.hasPrefix("parakeet-unified") { - if let files = try? fm.contentsOfDirectory(at: repo, includingPropertiesForKeys: nil), - files.contains(where: { $0.pathExtension == "mlmodelc" }) { - return true - } - } - return false + false } } -/// Owns both engines; picks per-session (never mid-session), and downloads -/// Parakeet in the background when absent, publishing progress for the UI. +/// Owns the dictation engine. Upstream this also downloaded Parakeet; that +/// path is compiled out on Intel because FluidAudio/ANE are unavailable. @MainActor @Observable public final class EngineManager { - public let parakeet = StreamingTranscriber() public let apple = AppleSpeechEngine() public var preferred: EngineChoice { didSet { UserDefaults.standard.set(preferred.rawValue, forKey: "engineChoice") } } - public private(set) var parakeetReady: Bool - /// 0…1 while the background download runs; nil when idle/complete. + public private(set) var parakeetReady: Bool = false public private(set) var downloadProgress: Double? public init() { preferred = EngineChoice( - rawValue: UserDefaults.standard.string(forKey: "engineChoice") ?? "auto") ?? .auto - parakeetReady = EngineChoice.parakeetModelsPresent() - if !parakeetReady { downloadParakeet() } + rawValue: UserDefaults.standard.string(forKey: "engineChoice") ?? "apple") ?? .apple + preferred = .apple } - public var activeChoice: EngineChoice { - EngineChoice.select(preferred: preferred, parakeetReady: parakeetReady) - } + public var activeChoice: EngineChoice { .apple } - /// Engine to use for a session that starts now. Warm it via start() upstream. public func engineForNewSession() -> any DictationEngine { - activeChoice == .parakeet ? parakeet : apple + apple } public var statusLine: String { - if let p = downloadProgress { - return "Apple engine — Parakeet downloading \(Int(p * 100))%" - } - return activeChoice == .parakeet ? "Parakeet (local)" : "Apple (built-in)" - } - - private func downloadParakeet() { - downloadProgress = 0 - let parakeet = parakeet - Task { - // StreamingTranscriber.start() triggers the model download via - // FluidAudio. Poll the cache for coarse progress (file count based - // progress handlers aren't exposed through the streaming manager). - let poll = Task { @MainActor [weak self] in - while self?.downloadProgress != nil { - try? await Task.sleep(for: .seconds(2)) - if EngineChoice.parakeetModelsPresent() { break } - } - } - try? await parakeet.start() - poll.cancel() - await MainActor.run { [weak self] in - self?.downloadProgress = nil - self?.parakeetReady = EngineChoice.parakeetModelsPresent() - } - } + "Apple Speech (on-device, Sequoia)" } } diff --git a/Sources/PlynnKit/Feedback.swift b/Sources/PlynnKit/Feedback.swift index eec22be..c0304bc 100644 --- a/Sources/PlynnKit/Feedback.swift +++ b/Sources/PlynnKit/Feedback.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import AppKit /// Sound and haptic cues for the dictation lifecycle. diff --git a/Sources/PlynnKit/IndicatorView.swift b/Sources/PlynnKit/IndicatorView.swift index 2ff40c2..d60bd61 100644 --- a/Sources/PlynnKit/IndicatorView.swift +++ b/Sources/PlynnKit/IndicatorView.swift @@ -23,55 +23,53 @@ enum IndicatorMetrics { static let bottomMargin: CGFloat = 14 } -/// The floating capsule: Liquid Glass surface whose bottom third is a live -/// waveform blended into the glass, centered text that fades up and slides +/// The floating capsule: Sequoia material surface whose bottom third is a live +/// waveform blended into the pill, centered text that fades up and slides /// as it overflows, and a close-in checkmark on success. struct IndicatorView: View { @Bindable var model: IndicatorModel - @Namespace private var glassNS private var isCompact: Bool { model.phase == .done } var body: some View { - GlassEffectContainer { - ZStack { - // Bottom-blended visualizer — always present (inert when - // hidden) so phase changes never reset its motion. - VStack(spacing: 0) { - Spacer(minLength: 0) - BottomWave(levels: model.levels, idle: model.phase == .transcribing) - .frame(height: IndicatorMetrics.waveHeight) - } - .opacity(waveOpacity) + ZStack { + Capsule() + .fill(.ultraThinMaterial) + Capsule() + .fill(Color.black.opacity(0.32)) - centerContent + // Bottom-blended visualizer — always present (inert when + // hidden) so phase changes never reset its motion. + VStack(spacing: 0) { + Spacer(minLength: 0) + BottomWave(levels: model.levels, idle: model.phase == .transcribing) + .frame(height: IndicatorMetrics.waveHeight) } - .frame( - width: isCompact ? IndicatorMetrics.compactWidth : IndicatorMetrics.width, - height: IndicatorMetrics.height) - .clipShape(Capsule()) - .glassEffect(.regular.tint(.black.opacity(0.18)).interactive(), in: .capsule) - .glassEffectID("capsule", in: glassNS) - // Explicit glass character — a bright rim light along the top edge - // and a soft specular sheen, visible on any background. - .overlay( - Capsule().strokeBorder( - LinearGradient( - stops: [ - .init(color: .white.opacity(0.5), location: 0), - .init(color: .white.opacity(0.06), location: 0.35), - .init(color: .white.opacity(0.18), location: 1), - ], - startPoint: .top, endPoint: .bottom), - lineWidth: 1)) - .overlay( - Capsule() - .fill( - LinearGradient( - colors: [.white.opacity(0.10), .clear], - startPoint: .top, endPoint: .center)) - .allowsHitTesting(false)) + .opacity(waveOpacity) + + centerContent } + .frame( + width: isCompact ? IndicatorMetrics.compactWidth : IndicatorMetrics.width, + height: IndicatorMetrics.height) + .clipShape(Capsule()) + .overlay( + Capsule().strokeBorder( + LinearGradient( + stops: [ + .init(color: .white.opacity(0.5), location: 0), + .init(color: .white.opacity(0.06), location: 0.35), + .init(color: .white.opacity(0.18), location: 1), + ], + startPoint: .top, endPoint: .bottom), + lineWidth: 1)) + .overlay( + Capsule() + .fill( + LinearGradient( + colors: [.white.opacity(0.10), .clear], + startPoint: .top, endPoint: .center)) + .allowsHitTesting(false)) .contentShape(Capsule()) .onTapGesture { model.onTap?() } .animation(.spring(response: 0.38, dampingFraction: 0.82), value: model.phase) diff --git a/Sources/PlynnKit/LLMFormatter.swift b/Sources/PlynnKit/LLMFormatter.swift index 2746687..83c5b97 100644 --- a/Sources/PlynnKit/LLMFormatter.swift +++ b/Sources/PlynnKit/LLMFormatter.swift @@ -1,64 +1,34 @@ import Foundation -import MLXLLM -import MLXLMCommon -/// AI polish: filler removal, backtrack self-correction, list formatting, -/// tone matching — Qwen3-4B 4-bit via MLX on the GPU (never contends with the -/// ANE-resident ASR). Every failure mode falls back to the input text. +/// Local Qwen polish via MLX is Apple Silicon + Metal only. On Intel the +/// formatter stays as a no-op so Settings / command mode / meeting notes +/// degrade cleanly instead of linking an unusable GPU runtime. public actor LLMFormatter { - public static let modelID = "mlx-community/Qwen3-4B-Instruct-2507-4bit" - - private var model: ModelContainer? - private var loading = false + public static let modelID = "unavailable-on-intel" public init() {} - public var ready: Bool { model != nil } + public var ready: Bool { false } - /// Download (first run, ~2.3 GB), load, and warm the model. Idempotent. - public func ensureLoaded() async throws { - guard model == nil, !loading else { return } - loading = true - defer { loading = false } - let container = try await loadModelContainer(id: Self.modelID) - // One-token warm-up: Metal kernel JIT + weight page-in happen here at - // launch, not on the user's first dictation. - let warm = ChatSession( - container, generateParameters: GenerateParameters(maxTokens: 1, temperature: 0)) - _ = try? await warm.respond(to: "hi") - model = container - } + public func ensureLoaded() async throws {} - /// One stateless prompt → raw completion (nil on timeout/error/not loaded). - public func complete(_ prompt: String) async -> String? { - guard let model else { return nil } - return await withTaskTimeout(seconds: 10) { - let session = ChatSession( - model, - generateParameters: GenerateParameters(maxTokens: 1024, temperature: 0)) - return try await session.respond(to: prompt) - } - } + public func complete(_ prompt: String) async -> String? { nil } public func format( _ text: String, tone: Tone, technical: Bool, preferredSpellings: [String] = [] ) async -> String { - let prompt = PolishPrompt.build( - transcript: text, tone: tone, technical: technical, - preferredSpellings: preferredSpellings) - return PolishPrompt.sanitize(await complete(prompt), input: text) + text } - } /// Run an async operation with a wall-clock timeout; nil on timeout or error. /// /// The operation runs *unstructured* on purpose. A task group awaits every /// child before it returns, and `cancelAll()` only requests cancellation — so -/// one operation that doesn't honour it (FoundationModels can block well past -/// its deadline) would hold the "timeout" open indefinitely and strand the -/// caller. Racing an abandoned task against the sleep means the deadline -/// always wins on time; a straggler finishes into the void and is discarded. +/// one operation that doesn't honour it would hold the "timeout" open +/// indefinitely and strand the caller. Racing an abandoned task against the +/// sleep means the deadline always wins on time; a straggler finishes into +/// the void and is discarded. func withTaskTimeout( seconds: Double, _ operation: @escaping @Sendable () async throws -> T ) async -> T? { @@ -69,7 +39,7 @@ func withTaskTimeout( await box.settle(nil) } let value = await box.value() - work.cancel() // best effort — honoured only if the operation checks + work.cancel() timer.cancel() return value } @@ -93,7 +63,6 @@ private actor FirstResult { func value() async -> T? { if settled { return value } return await withCheckedContinuation { continuation in - // Runs synchronously on this actor, so `settled` cannot flip here. if settled { continuation.resume(returning: value) } else { waiter = continuation } } diff --git a/Sources/PlynnKit/MeetingRecorder.swift b/Sources/PlynnKit/MeetingRecorder.swift index 00fc95f..3ead115 100644 --- a/Sources/PlynnKit/MeetingRecorder.swift +++ b/Sources/PlynnKit/MeetingRecorder.swift @@ -1,5 +1,5 @@ -import AVFoundation -import ScreenCaptureKit +@preconcurrency import AVFoundation +@preconcurrency import ScreenCaptureKit /// Captures a meeting: your microphone AND the other side of the call /// (system audio), via one ScreenCaptureKit stream. Both feeds are already diff --git a/Sources/PlynnKit/Permissions.swift b/Sources/PlynnKit/Permissions.swift index e53e1f5..c6457fb 100644 --- a/Sources/PlynnKit/Permissions.swift +++ b/Sources/PlynnKit/Permissions.swift @@ -1,6 +1,7 @@ -import AVFoundation +@preconcurrency import AVFoundation import AppKit import ApplicationServices +@preconcurrency import Speech /// Permission checks and actions. All checks are poll-based — macOS has no /// grant notifications for Accessibility. @@ -14,6 +15,10 @@ public enum Permissions { AXIsProcessTrusted() } + public static func speechGranted() -> Bool { + SFSpeechRecognizer.authorizationStatus() == .authorized + } + /// Globe key set to "Do Nothing" (0) so it can't fight Plynn's fn hotkey. public static func globeKeySafe() -> Bool { let defaults = UserDefaults(suiteName: "com.apple.HIToolbox") @@ -25,6 +30,10 @@ public enum Permissions { AVCaptureDevice.requestAccess(for: .audio) { _ in } } + public static func requestSpeech() { + SFSpeechRecognizer.requestAuthorization { _ in } + } + /// Shows the system Accessibility prompt (once per TCC state). public static func promptAccessibility() { // kAXTrustedCheckOptionPrompt is a C global Swift 6 won't touch across diff --git a/Sources/PlynnKit/Resampler.swift b/Sources/PlynnKit/Resampler.swift index b39b456..6b9bd03 100644 --- a/Sources/PlynnKit/Resampler.swift +++ b/Sources/PlynnKit/Resampler.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation public enum Resampler { public enum ResampleError: Error { case noConverter } diff --git a/Sources/PlynnKit/StreamingTranscriber.swift b/Sources/PlynnKit/StreamingTranscriber.swift index 30aba5c..4f9d46e 100644 --- a/Sources/PlynnKit/StreamingTranscriber.swift +++ b/Sources/PlynnKit/StreamingTranscriber.swift @@ -1,75 +1,5 @@ -import AVFoundation -import FluidAudio +import Foundation -/// Streaming ASR over FluidAudio's Parakeet Unified streaming manager, with a -/// VAD silence gate at finish() so silence-only sessions can never hallucinate -/// text. Reusable across sessions: call start() before each dictation. -public actor StreamingTranscriber: DictationEngine { - public nonisolated let displayName = "Parakeet (local)" - private let variant: StreamingModelVariant - private var manager: (any StreamingAsrManager)? - private var vad: VadManager? - private var sessionSamples: [Float] = [] - - /// parakeetUnified1120ms shares its model repo with the offline batch path — - /// one download covers both — and beats the 2080ms tier on WER and latency. - public init(variant: StreamingModelVariant = .parakeetUnified1120ms) { - self.variant = variant - } - - /// Load models (idempotent) and reset for a new session. - public func start() async throws { - if manager == nil { - manager = variant.createManager() - try await manager!.loadModels() - } - try await manager!.reset() - sessionSamples.removeAll(keepingCapacity: true) - } - - public func setPartialCallback(_ callback: @escaping @Sendable (String) -> Void) async { - await manager?.setPartialTranscriptCallback(callback) - } - - /// Feed 16 kHz mono Float32 samples; partial transcripts fire via the callback. - public func append(samples: [Float]) async throws { - guard let manager else { return } - sessionSamples.append(contentsOf: samples) - let buffer = AVAudioPCMBuffer( - pcmFormat: AudioFile.targetFormat, frameCapacity: AVAudioFrameCount(samples.count))! - buffer.frameLength = AVAudioFrameCount(samples.count) - samples.withUnsafeBufferPointer { src in - buffer.floatChannelData![0].update(from: src.baseAddress!, count: samples.count) - } - try await manager.appendAudio(buffer) - try await manager.processBufferedAudio() - } - - /// Trailing silence fed before finish(): flushes the streaming encoder's - /// look-ahead window so words spoken right before release aren't dropped — - /// the same short-utterance fix as the batch path, and also pads very short - /// sessions up to the 2 s floor the model needs to emit anything at all. - private static let flushPadSamples = 20_000 // 1.25 s - private static let minSessionSamples = 32_000 // 2 s - - /// Flush and return the final transcript — empty string if VAD saw no speech. - public func finish() async throws -> String { - guard let manager else { return "" } - let spoken = sessionSamples // VAD judges only real mic audio, not padding - var pad = Self.flushPadSamples - if spoken.count + pad < Self.minSessionSamples { pad = Self.minSessionSamples - spoken.count } - try await append(samples: [Float](repeating: 0, count: pad)) - sessionSamples = spoken - let text = try await manager.finish() - guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return "" } - if try await hasSpeech(in: sessionSamples) { return text } - return "" - } - - private func hasSpeech(in samples: [Float]) async throws -> Bool { - guard !samples.isEmpty else { return false } - if vad == nil { vad = try await VadManager() } - let results = try await vad!.process(samples) - return results.contains { $0.isVoiceActive } - } -} +/// Upstream this wrapped FluidAudio's Parakeet streaming manager. On Intel +/// Sequoia the same `DictationEngine` surface is Apple's on-device recognizer. +public typealias StreamingTranscriber = AppleSpeechEngine diff --git a/Sources/PlynnKit/Transcriber.swift b/Sources/PlynnKit/Transcriber.swift index d5def59..bea30fe 100644 --- a/Sources/PlynnKit/Transcriber.swift +++ b/Sources/PlynnKit/Transcriber.swift @@ -1,34 +1,15 @@ -import FluidAudio import Foundation -/// Wraps FluidAudio's Parakeet Unified (EN) ASR — int8 encoder on the ANE. -/// First call downloads models (~0.6 GB) to Application Support — allow time + network. +/// Batch helper used by tests. Upstream this was FluidAudio/Parakeet; here it +/// is a one-shot session on Apple's on-device Speech engine. public actor Transcriber { - private var manager: UnifiedAsrManager? + private let engine = AppleSpeechEngine() public init() {} - private func loadedManager() async throws -> UnifiedAsrManager { - if let manager { return manager } - let m = UnifiedAsrManager() - try await m.loadModels() - manager = m - return m - } - - /// Sub-2s clips come back empty from the model (the classic dictation-app - /// short-utterance bug) — pad with leading/trailing silence to a 2 s floor. - private static let minSamples = 32_000 - private static let leadInSamples = 1_600 // 0.1 s - - /// 16 kHz mono Float32 samples in, transcript out. public func transcribe(samples: [Float]) async throws -> String { - let m = try await loadedManager() - var padded = samples - if padded.count < Self.minSamples { - padded = [Float](repeating: 0, count: Self.leadInSamples) + padded - padded += [Float](repeating: 0, count: Self.minSamples - padded.count) - } - return try await m.transcribe(padded) + try await engine.start() + try await engine.append(samples: samples) + return try await engine.finish() } } diff --git a/Tests/PlynnKitTests/AppleSpeechEngineTests.swift b/Tests/PlynnKitTests/AppleSpeechEngineTests.swift index d54c8c2..2c30155 100644 --- a/Tests/PlynnKitTests/AppleSpeechEngineTests.swift +++ b/Tests/PlynnKitTests/AppleSpeechEngineTests.swift @@ -6,6 +6,10 @@ final class AppleSpeechEngineTests: XCTestCase { let engine = AppleSpeechEngine() do { try await engine.start() } catch AppleSpeechEngine.EngineError.assetUnavailable { throw XCTSkip("English speech asset unavailable on this machine") + } catch AppleSpeechEngine.EngineError.notAuthorized { + throw XCTSkip("Speech recognition not authorized in this test environment") + } catch AppleSpeechEngine.EngineError.recognizerUnavailable { + throw XCTSkip("Speech recognizer unavailable") } nonisolated(unsafe) var partials: [String] = [] await engine.setPartialCallback { partials.append($0) } @@ -22,6 +26,10 @@ final class AppleSpeechEngineTests: XCTestCase { let engine = AppleSpeechEngine() do { try await engine.start() } catch AppleSpeechEngine.EngineError.assetUnavailable { throw XCTSkip("English speech asset unavailable on this machine") + } catch AppleSpeechEngine.EngineError.notAuthorized { + throw XCTSkip("Speech recognition not authorized in this test environment") + } catch AppleSpeechEngine.EngineError.recognizerUnavailable { + throw XCTSkip("Speech recognizer unavailable") } let samples = try AudioFile.loadSamples16kMono( url: Bundle.module.url(forResource: "Fixtures/short.wav", withExtension: nil)!) diff --git a/Tests/PlynnKitTests/EngineManagerTests.swift b/Tests/PlynnKitTests/EngineManagerTests.swift index 1e7696d..19c6093 100644 --- a/Tests/PlynnKitTests/EngineManagerTests.swift +++ b/Tests/PlynnKitTests/EngineManagerTests.swift @@ -2,32 +2,18 @@ import XCTest @testable import PlynnKit final class EngineManagerTests: XCTestCase { - func testSelection() { + func testSelectionAlwaysAppleOnThisPort() { XCTAssertEqual(EngineChoice.select(preferred: .auto, parakeetReady: false), .apple) - XCTAssertEqual(EngineChoice.select(preferred: .auto, parakeetReady: true), .parakeet) + XCTAssertEqual(EngineChoice.select(preferred: .auto, parakeetReady: true), .apple) XCTAssertEqual(EngineChoice.select(preferred: .parakeet, parakeetReady: false), .apple) - XCTAssertEqual(EngineChoice.select(preferred: .parakeet, parakeetReady: true), .parakeet) + XCTAssertEqual(EngineChoice.select(preferred: .parakeet, parakeetReady: true), .apple) XCTAssertEqual(EngineChoice.select(preferred: .apple, parakeetReady: true), .apple) XCTAssertEqual(EngineChoice.select(preferred: .apple, parakeetReady: false), .apple) } - func testParakeetModelsAbsentInEmptyDir() { + func testParakeetModelsNeverPresentOnIntelPort() { let dir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) XCTAssertFalse(EngineChoice.parakeetModelsPresent(in: dir)) - } - - func testParakeetModelsPresentWhenEncoderExists() throws { - let dir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) - let modelDir = dir.appendingPathComponent("parakeet-unified-0.6b-coreml/foo.mlmodelc") - try FileManager.default.createDirectory(at: modelDir, withIntermediateDirectories: true) - XCTAssertTrue(EngineChoice.parakeetModelsPresent(in: dir)) - try? FileManager.default.removeItem(at: dir) - } - - func testDefaultCacheDirDetectionMatchesRealState() { - // On this dev machine models are downloaded; the check must agree with reality. - // (Weak assertion by design: just verifies the path logic doesn't crash and - // returns a Bool consistent with the FluidAudio cache convention.) - _ = EngineChoice.parakeetModelsPresent() + XCTAssertFalse(EngineChoice.parakeetModelsPresent()) } } diff --git a/Tests/PlynnKitTests/StreamingTranscriberTests.swift b/Tests/PlynnKitTests/StreamingTranscriberTests.swift index 8ee408d..269ac5c 100644 --- a/Tests/PlynnKitTests/StreamingTranscriberTests.swift +++ b/Tests/PlynnKitTests/StreamingTranscriberTests.swift @@ -4,7 +4,11 @@ import XCTest final class StreamingTranscriberTests: XCTestCase { func testStreamedFixtureProducesPartialsAndFinal() async throws { let st = StreamingTranscriber() - try await st.start() + do { try await st.start() } catch AppleSpeechEngine.EngineError.assetUnavailable { + throw XCTSkip("English on-device speech asset unavailable on this machine") + } catch AppleSpeechEngine.EngineError.notAuthorized { + throw XCTSkip("Speech recognition not authorized in this test environment") + } nonisolated(unsafe) var partials: [String] = [] await st.setPartialCallback { partials.append($0) } let samples = try AudioFile.loadSamples16kMono( @@ -19,7 +23,11 @@ final class StreamingTranscriberTests: XCTestCase { func testSilenceProducesEmptyTranscript() async throws { let st = StreamingTranscriber() - try await st.start() + do { try await st.start() } catch AppleSpeechEngine.EngineError.assetUnavailable { + throw XCTSkip("English on-device speech asset unavailable on this machine") + } catch AppleSpeechEngine.EngineError.notAuthorized { + throw XCTSkip("Speech recognition not authorized in this test environment") + } try await st.append(samples: [Float](repeating: 0, count: 48_000)) // 3 s silence let final = try await st.finish() XCTAssertEqual(final.trimmingCharacters(in: .whitespacesAndNewlines), "", @@ -28,7 +36,11 @@ final class StreamingTranscriberTests: XCTestCase { func testReuseAcrossSessions() async throws { let st = StreamingTranscriber() - try await st.start() + do { try await st.start() } catch AppleSpeechEngine.EngineError.assetUnavailable { + throw XCTSkip("English on-device speech asset unavailable on this machine") + } catch AppleSpeechEngine.EngineError.notAuthorized { + throw XCTSkip("Speech recognition not authorized in this test environment") + } let samples = try AudioFile.loadSamples16kMono( url: Bundle.module.url(forResource: "Fixtures/short.wav", withExtension: nil)!) try await st.append(samples: samples) diff --git a/Tests/PlynnKitTests/TranscriberTests.swift b/Tests/PlynnKitTests/TranscriberTests.swift index fb8f31e..1ffa689 100644 --- a/Tests/PlynnKitTests/TranscriberTests.swift +++ b/Tests/PlynnKitTests/TranscriberTests.swift @@ -14,7 +14,20 @@ final class TranscriberTests: XCTestCase { url: Bundle.module.url(forResource: "Fixtures/\(name)", withExtension: nil)!) } + func startOrSkip() async throws { + do { + _ = try await Self.transcriber.transcribe(samples: [Float](repeating: 0, count: 1_600)) + } catch AppleSpeechEngine.EngineError.assetUnavailable { + throw XCTSkip("English on-device speech asset unavailable on this machine") + } catch AppleSpeechEngine.EngineError.notAuthorized { + throw XCTSkip("Speech recognition not authorized in this test environment") + } catch AppleSpeechEngine.EngineError.recognizerUnavailable { + throw XCTSkip("Speech recognizer unavailable") + } + } + func testTranscribesSentenceFixture() async throws { + try await startOrSkip() let text = try await Self.transcriber.transcribe(samples: try fixture("hello.wav")) let lower = text.lowercased() XCTAssertTrue(lower.contains("hello"), "got: \(text)") @@ -28,11 +41,6 @@ final class TranscriberTests: XCTestCase { } func testLatencyBudgetOnLongUtterance() async throws { - let samples = try fixture("long.wav") // ~20 s of speech - _ = try await Self.transcriber.transcribe(samples: samples) // warm-up - let start = ContinuousClock.now - _ = try await Self.transcriber.transcribe(samples: samples) - let elapsed = start.duration(to: .now) - XCTAssertLessThan(elapsed, .seconds(2), "Parakeet on M4 Pro should be ~100x RT; got \(elapsed)") + throw XCTSkip("ANE latency budget is Parakeet-specific; Apple Speech on Intel is not realtime") } } diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 3eac4e0..7b59a93 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -15,7 +15,7 @@ Updates arrive through Sparkle: the app tells you when a new version exists and ## Release engineering -- `make-app.sh` builds with `xcodebuild` (the MLX Metal shaders require it) and produces a signed .app with SPM resource bundles in `Contents/Resources`. +- `make-app.sh` builds with `swift build` (Command Line Tools on Sequoia 15 / Intel) and ad-hoc signs `build/Plynn.app`. - `notarize.sh` submits to Apple with `notarytool` and staples the ticket. - `make-dmg.sh` packages the drag to Applications DMG. Pass `SKIP_BUILD=1` after notarizing so the stapled app is packaged as is. - `make-release.sh` runs the whole chain and publishes the GitHub release with a signed Sparkle appcast. diff --git a/scripts/Info.plist b/scripts/Info.plist index c3ea1bf..d404af4 100644 --- a/scripts/Info.plist +++ b/scripts/Info.plist @@ -30,14 +30,12 @@ CFBundleVersion 9 LSMinimumSystemVersion - 26.0 + 15.0 LSUIElement NSMicrophoneUsageDescription Plynn records your voice while you hold the fn key to transcribe it on-device. - SUFeedURL - https://github.com/31Carlton7/plynn/releases/latest/download/appcast.xml - SUPublicEDKey - RUQWPEDzgPh9Qj1GH959liET1qvj5CDLeJR6DYrSaYQ= + NSSpeechRecognitionUsageDescription + Plynn turns your speech into text on this Mac using Apple's on-device recognizer. diff --git a/scripts/make-app.sh b/scripts/make-app.sh index 8b0621c..276e6d6 100755 --- a/scripts/make-app.sh +++ b/scripts/make-app.sh @@ -1,39 +1,29 @@ #!/bin/bash +# Build a Sequoia 15 / Intel .app with Command Line Tools (no Xcode 26, no MLX). set -euo pipefail cd "$(dirname "$0")/.." -IDENTITY="${IDENTITY:-Developer ID Application: Carlton Aikins (FY9QB79VAP)}" -# MLX's Metal shaders only compile under xcodebuild — SwiftPM CLI builds ship -# no metallib and the LLM dies at runtime (see mlx-swift README). -xcodebuild build -scheme Plynn -configuration Release \ - -destination 'platform=macOS' \ - -derivedDataPath build/DerivedData -quiet +swift build -c release --product Plynn -BUILT="build/DerivedData/Build/Products/Release" +BIN="$(swift build -c release --show-bin-path)/Plynn" APP="build/Plynn.app" rm -rf "$APP" mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources" -cp "$BUILT/Plynn" "$APP/Contents/MacOS/Plynn" -# SPM resource bundles (incl. mlx-swift_Cmlx.bundle with mlx.metallib) resolve -# via Bundle.main.resourceURL inside an .app — they belong in Contents/Resources. -for b in "$BUILT"/*.bundle; do - cp -R "$b" "$APP/Contents/Resources/" -done -# Sparkle ships as a binary framework; it must live in Contents/Frameworks. -mkdir -p "$APP/Contents/Frameworks" -SPARKLE=$(find build/DerivedData -type d -name "Sparkle.framework" -not -path "*dSYM*" | head -1) -ditto "$SPARKLE" "$APP/Contents/Frameworks/Sparkle.framework" +cp "$BIN" "$APP/Contents/MacOS/Plynn" cp scripts/Info.plist "$APP/Contents/Info.plist" -cp scripts/AppIcon.icns "$APP/Contents/Resources/AppIcon.icns" -install_name_tool -add_rpath "@executable_path/../Frameworks" "$APP/Contents/MacOS/Plynn" 2>/dev/null || true -codesign --force --options runtime \ - --sign "$IDENTITY" "$APP/Contents/Frameworks/Sparkle.framework" -codesign --force --options runtime --deep \ +if [[ -f scripts/AppIcon.icns ]]; then + cp scripts/AppIcon.icns "$APP/Contents/Resources/AppIcon.icns" +fi + +# Ad-hoc sign so TCC (mic / speech / accessibility) can attach to the bundle. +# Skip --options runtime: hardened runtime + ad-hoc often blocks the event tap +# on Sequoia without a Developer ID. +codesign --force --deep \ --entitlements scripts/plynn.entitlements \ - --sign "$IDENTITY" "$APP" -echo "Built and signed $APP" + --sign - "$APP" + +echo "Built $APP (Intel / macOS 15, Apple Speech on-device)" -# --install: replace the copy in /Applications (relaunch is the caller's job). if [[ "${1:-}" == "--install" ]]; then rm -rf /Applications/Plynn.app ditto "$APP" /Applications/Plynn.app