Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions Sources/TranscriberCore/Engines/CohereMLXBackend.swift
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,14 @@ public protocol CohereMLXAudioDurationReading: Sendable {
/// Terminal errors surfaced by `CohereMLXBackend.transcribe(_:)`. Marked
/// terminal so `TranscriptionWorker.isTransient` (which classifies by type)
/// does not retry — deterministic local inference would just loop again.
///
/// `reason` is a structural summary only (tri-gram stats, unique-word fraction)
/// and never contains transcript content. `TranscriptionWorker` persists
/// `String(describing: error)` into `transcript.md`/`metadata.json` on
/// failure, so anything embedded in this case lands on disk — keep
/// transcript text out of the payload.
public enum CohereMLXBackendError: Error, Sendable, Equatable {
case degenerateOutput(reason: String, sample: String)
case degenerateOutput(reason: String)
}

/// Heuristic guard against decode-loop output (model emits the same phrase
Expand All @@ -96,7 +102,12 @@ enum DegenerateOutputDetector {
let key = "\(words[i]) \(words[i + 1]) \(words[i + 2])"
counts[key, default: 0] += 1
}
// Require BOTH an absolute minimum repeat count AND a high fraction.
// The fraction alone would flag a 30-word transcript with three
// repeats of "thank you very" (3/28 ≈ 11%) as degenerate; an
// observed real-world loop has 12+ repeats, well above the floor.
if let (top, n) = counts.max(by: { $0.value < $1.value }),
n >= 5,
Double(n) / Double(total) > 0.08 {
return "tri-gram \"\(top)\" repeats \(n)/\(total) times"
}
Expand Down Expand Up @@ -168,9 +179,12 @@ public final class CohereMLXBackend: TranscriptionEngine, @unchecked Sendable {
)
let output = try await adapter.transcribe(localRequest)
if let reason = DegenerateOutputDetector.evaluate(output.text) {
let sample = String(output.text.prefix(120))
Log.engine.error("Cohere MLX produced degenerate output: \(reason, privacy: .public)")
throw CohereMLXBackendError.degenerateOutput(reason: reason, sample: sample)
// Sample stays in the unified log at `.private` for live debugging
// and is intentionally NOT carried on the thrown error — the
// worker persists `String(describing:)` to disk and meeting
// content must not land in the failure artifact.
Log.engine.error("Cohere MLX produced degenerate output: \(reason, privacy: .public); sample: \(output.text.prefix(120), privacy: .private)")
throw CohereMLXBackendError.degenerateOutput(reason: reason)
}
let utterances: [EngineResponse.Utterance]
if output.segments.isEmpty {
Expand Down
28 changes: 25 additions & 3 deletions Tests/TranscriberCoreTests/Engines/CohereRustBackendTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -407,16 +407,38 @@ final class CohereMLXBackendTests: XCTestCase {
XCTFail("Backend must throw on degenerate output, not silently return")
} catch let error as CohereMLXBackendError {
switch error {
case .degenerateOutput(let reason, let sample):
case .degenerateOutput(let reason):
XCTAssertTrue(reason.contains("tri-gram") || reason.contains("unique-word"),
"Reason should identify the failure mode, got: \(reason)")
XCTAssertFalse(sample.isEmpty, "Sample should include a snippet of the failing transcript")
XCTAssertLessThanOrEqual(sample.count, 120, "Sample is capped at 120 chars")
let serialized = String(describing: error)
XCTAssertFalse(
serialized.contains("I think that's what I'm hearing"),
"String(describing:) must not embed transcript content; TranscriptionWorker persists this verbatim to disk"
)
}
} catch {
XCTFail("Expected CohereMLXBackendError.degenerateOutput, got \(error)")
}
}

func testDetectorDoesNotFireOnShortTranscriptWithFewRepeats() {
// 30+ words, "thank you very" tri-gram appears 3 times. Before the
// `n >= 5` floor was added this fired (3/28 ≈ 11% > 8%) and the
// backend threw, costing the user a usable transcript.
let words: [String] = [
"thank", "you", "very", "much", "for",
"joining", "the", "call", "today",
"thank", "you", "very", "much", "again",
"before", "we", "wrap",
"thank", "you", "very", "kindly",
"and", "have", "a", "wonderful", "rest", "of", "your", "afternoon", "everyone"
]
XCTAssertGreaterThanOrEqual(words.count, 30)
XCTAssertNil(
DegenerateOutputDetector.evaluate(words.joined(separator: " ")),
"Three repeats of a common tri-gram in a short transcript is not degenerate"
)
}
}

private final class RecordingLocalAdapter: CohereMLXTranscribing, @unchecked Sendable {
Expand Down
Loading