Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 20 additions & 30 deletions Sources/TranscriberCore/Engines/CohereMLXBackend.swift
Original file line number Diff line number Diff line change
Expand Up @@ -69,49 +69,39 @@ public protocol CohereMLXAudioDurationReading: Sendable {
func durationSeconds(for audioURL: URL) async throws -> Double
}

/// Terminal errors surfaced by `CohereMLXBackend.transcribe(_:)` that the
/// recovery layer must NOT treat as transient. The MLX path runs locally with
/// deterministic params, so a degenerate decode is a true failure — retrying
/// without changing inputs would loop again. `TranscriptionWorker.isTransient`
/// does not classify these, so they reach the user as a clear failed status.
/// Terminal errors surfaced by `CohereMLXBackend.transcribe(_:)`. Marked
/// terminal so `TranscriptionWorker.isTransient` (which classifies by type)
/// does not retry — deterministic local inference would just loop again.
public enum CohereMLXBackendError: Error, Sendable, Equatable {
case degenerateOutput(reason: String, sample: String)
}

/// Heuristic guard against the Cohere/MLX decode-loop failure mode where the
/// model spends the entire 583-second recording emitting the same phrase
/// dozens of times. The two checks below are pure-Swift and cheap; either one
/// firing means the output is unusable as a transcript.
///
/// Compression-ratio (gzip) is intentionally omitted — it would require a new
/// dependency to catch a failure mode already covered by the two checks here.
/// Add it later only if the field shows misses.
/// Heuristic guard against decode-loop output (model emits the same phrase
/// repeatedly instead of transcribing). Tri-gram density + unique-word
/// fraction are both cheap and either firing means the output is unusable.
enum DegenerateOutputDetector {
/// Returns `nil` when the text looks like a healthy transcript; otherwise
/// returns a short human-readable reason that callers can put in an error
/// payload or log line. Texts shorter than 30 words always return `nil` —
/// the worker has a separate "No speech detected" terminal path for those.
/// returns a short human-readable reason for logs and error payloads.
/// Inputs under 30 words always return `nil` — the worker handles the
/// "no speech detected" path separately.
static func evaluate(_ text: String) -> String? {
let words = text
.split(whereSeparator: { $0.isWhitespace })
.map(String.init)
.map { $0.lowercased() }
guard words.count >= 30 else { return nil }

if words.count >= 3 {
var counts: [String: Int] = [:]
let total = words.count - 2
for i in 0..<total {
let key = "\(words[i].lowercased()) \(words[i + 1].lowercased()) \(words[i + 2].lowercased())"
counts[key, default: 0] += 1
}
if let (top, n) = counts.max(by: { $0.value < $1.value }),
Double(n) / Double(total) > 0.08 {
return "tri-gram \"\(top)\" repeats \(n)/\(total) times"
}
var counts: [String: Int] = [:]
let total = words.count - 2
for i in 0..<total {
let key = "\(words[i]) \(words[i + 1]) \(words[i + 2])"
counts[key, default: 0] += 1
}
if let (top, n) = counts.max(by: { $0.value < $1.value }),
Double(n) / Double(total) > 0.08 {
return "tri-gram \"\(top)\" repeats \(n)/\(total) times"
}

let unique = Set(words.map { $0.lowercased() }).count
let fraction = Double(unique) / Double(words.count)
let fraction = Double(Set(words).count) / Double(words.count)
if fraction < 0.10 {
return "unique-word fraction \(String(format: "%.3f", fraction)) below 0.10"
}
Expand Down
20 changes: 13 additions & 7 deletions Tests/TranscriberCoreTests/Engines/CohereRustBackendTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ final class CohereMLXBackendTests: XCTestCase {
XCTAssertEqual(parameters.maxTokens, 1024, "maxTokens is the only value taken from model defaults")
}

func testDegenerateOutputDetectorFlagsRepetitiveTranscripts() {
func testDegenerateOutputDetectorFlagsRepetitiveTranscripts() throws {
let looped = String(repeating: "I think that's what I'm hearing ", count: 50)
let loopedReason = DegenerateOutputDetector.evaluate(looped)
XCTAssertNotNil(loopedReason, "Detector must catch the observed 'I think that's what I'm hearing' loop")
Expand All @@ -327,8 +327,8 @@ final class CohereMLXBackendTests: XCTestCase {
let fixtureURL = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent()
.appendingPathComponent("Fixtures/elevenlabs-success.json")
let fixtureData = try? Data(contentsOf: fixtureURL)
let fixtureText = (try? JSONSerialization.jsonObject(with: fixtureData ?? Data())) as? [String: Any]
let fixtureData = try Data(contentsOf: fixtureURL)
let fixtureText = try JSONSerialization.jsonObject(with: fixtureData) as? [String: Any]
let cloudText = (fixtureText?["text"] as? String) ?? ""
XCTAssertFalse(cloudText.isEmpty, "Fixture must still be present")
XCTAssertNil(DegenerateOutputDetector.evaluate(cloudText), "Detector must not fire on cloud fixture text")
Expand All @@ -347,12 +347,18 @@ final class CohereMLXBackendTests: XCTestCase {
"Integration test; set SCRIBE_RUN_MLX_INTEGRATION=1 to run"
)

let audioURL = FileManager.default
.homeDirectoryForCurrentUser
.appendingPathComponent("Scribe/2026-05-11-1756/audio.m4a")
let env = ProcessInfo.processInfo.environment
let audioURL: URL
if let override = env["SCRIBE_MLX_INTEGRATION_AUDIO"], !override.isEmpty {
audioURL = URL(fileURLWithPath: (override as NSString).expandingTildeInPath)
} else {
audioURL = FileManager.default
.homeDirectoryForCurrentUser
.appendingPathComponent("Scribe/2026-05-11-1756/audio.m4a")
}
try XCTSkipUnless(
FileManager.default.fileExists(atPath: audioURL.path),
"Real recording fixture missing at \(audioURL.path)"
"Real recording fixture missing at \(audioURL.path); set SCRIBE_MLX_INTEGRATION_AUDIO to point at one"
)

let modelDir = CohereMLXBackend.defaultModelDirectoryURL
Expand Down
Loading