From 7f8278685c9fa9d5c54202c92203d358f2dcf997 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 07:47:07 -0400 Subject: [PATCH 01/20] feat(voice): estimate speech delivered before interruption --- src-tauri/native/siri_tts_bridge.h | 1 + src-tauri/native/siri_tts_bridge.m | 73 +++++- src-tauri/src/commands/pocket_voice.rs | 236 +++++++++++++++--- src-tauri/src/commands/siri_voice.rs | 85 ++++++- src/app/AppShell.navigation.test.tsx | 1 - .../transcript/projection/messageRevisions.ts | 29 ++- .../transcriptProjectionCache.test.ts | 31 +++ src/features/chat/ui/AgentWorkPanel.tsx | 13 +- src/features/chat/ui/MessageBubble.tsx | 15 +- .../chat/ui/__tests__/MessageBubble.test.tsx | 29 ++- .../voice-conversation/api/pocketVoice.ts | 13 +- .../voice-conversation/api/siriVoice.ts | 4 +- .../lib/nativeAssistantSpeech.test.ts | 61 +++++ .../lib/nativeAssistantSpeech.ts | 222 +++++++++++++++- src/shared/types/messages.ts | 5 + 15 files changed, 744 insertions(+), 74 deletions(-) diff --git a/src-tauri/native/siri_tts_bridge.h b/src-tauri/native/siri_tts_bridge.h index 92b98e1c6..b388bcf09 100644 --- a/src-tauri/native/siri_tts_bridge.h +++ b/src-tauri/native/siri_tts_bridge.h @@ -54,6 +54,7 @@ bool berd_siri_tts_stream_enqueue(void *stream, const char *text, char **error_o void berd_siri_tts_stream_finish(void *stream); bool berd_siri_tts_stream_is_finished(void *stream); uint64_t berd_siri_tts_stream_progress(void *stream); +char *berd_siri_tts_stream_copy_delivery_json(void *stream); char *berd_siri_tts_stream_copy_error(void *stream); void berd_siri_tts_stream_cancel(void *stream); void berd_siri_tts_stream_release(void *stream); diff --git a/src-tauri/native/siri_tts_bridge.m b/src-tauri/native/siri_tts_bridge.m index e0558a71f..66d24d608 100644 --- a/src-tauri/native/siri_tts_bridge.m +++ b/src-tauri/native/siri_tts_bridge.m @@ -3,6 +3,7 @@ #import #import #import +#import #import #import #import @@ -273,13 +274,22 @@ - (void)cancel { - (void)dealloc { [self.connection invalidate]; } @end +@interface BerdSiriDeliverySegment : NSObject +@property(nonatomic, copy) NSString *text; +@property(nonatomic, assign) uint64_t totalFrames; +@end + +@implementation BerdSiriDeliverySegment +@end + @interface BerdSiriSpeechPlayer : NSObject @property(nonatomic, strong) dispatch_queue_t queue; @property(nonatomic, strong) AVAudioEngine *engine; @property(nonatomic, strong) AVAudioPlayerNode *player; @property(nonatomic, strong) AVAudioConverter *converter; @property(nonatomic, strong) BerdSiriSynthesisSession *session; -@property(nonatomic, strong) NSMutableArray *pendingTexts; +@property(nonatomic, strong) NSMutableArray *pendingTexts; +@property(nonatomic, strong) NSMutableArray *deliverySegments; @property(nonatomic, strong) dispatch_semaphore_t completionSemaphore; @property(nonatomic, strong) NSError *error; @property(nonatomic, assign) NSInteger pendingBuffers; @@ -287,6 +297,7 @@ @interface BerdSiriSpeechPlayer : NSObject @property(nonatomic, assign) BOOL playbackStarted; @property(nonatomic, assign) BOOL finished; @property(nonatomic, assign) uint64_t progressGeneration; +@property(nonatomic, assign) double playbackSampleRate; @property(nonatomic, assign) BerdSiriTTSPlaybackStarted startedCallback; @property(nonatomic, assign) void *callbackContext; @property(nonatomic, copy) NSString *language; @@ -295,6 +306,7 @@ @interface BerdSiriSpeechPlayer : NSObject - (void)enqueueText:(NSString *)text; - (void)finishInput; - (void)cancel; +- (NSString *)deliveryJSON; @end @implementation BerdSiriSpeechPlayer @@ -306,6 +318,7 @@ - (instancetype)init { BerdSiriSpeechQueueKey, NULL); _completionSemaphore = dispatch_semaphore_create(0); _pendingTexts = [NSMutableArray array]; + _deliverySegments = [NSMutableArray array]; } return self; } @@ -405,7 +418,8 @@ - (BOOL)ensurePlayer:(AVAudioFormat *)format error:(NSError **)error { return YES; } - (void)enqueueData:(NSData *)data format:(AudioStreamBasicDescription)format - packetCount:(UInt32)packetCount packetDescriptions:(NSData *)packetDescriptions { + packetCount:(UInt32)packetCount packetDescriptions:(NSData *)packetDescriptions + deliverySegment:(BerdSiriDeliverySegment *)deliverySegment { if (self.finished || !data.length) return; self.progressGeneration += 1; NSError *error = nil; @@ -419,6 +433,10 @@ - (void)enqueueData:(NSData *)data format:(AudioStreamBasicDescription)format [self finish:error]; return; } + if (self.playbackSampleRate == 0) { + self.playbackSampleRate = buffer.format.sampleRate; + } + deliverySegment.totalFrames += buffer.frameLength; self.pendingBuffers += 1; [self.player scheduleBuffer:buffer completionCallbackType:AVAudioPlayerNodeCompletionDataPlayedBack completionHandler:^(__unused AVAudioPlayerNodeCompletionCallbackType type) { @@ -439,7 +457,8 @@ - (void)startNextSynthesis { [self finishIfReady]; return; } - NSString *text = self.pendingTexts.firstObject; + BerdSiriDeliverySegment *deliverySegment = self.pendingTexts.firstObject; + NSString *text = deliverySegment.text; [self.pendingTexts removeObjectAtIndex:0]; self.progressGeneration += 1; __weak typeof(self) weakSelf = self; @@ -448,7 +467,7 @@ - (void)startNextSynthesis { UInt32 packetCount, NSData *descriptions) { dispatch_async(weakSelf.queue, ^{ [weakSelf enqueueData:data format:format packetCount:packetCount - packetDescriptions:descriptions]; + packetDescriptions:descriptions deliverySegment:deliverySegment]; }); }]; [self.session synthesizeText:text language:self.language voiceName:self.voiceName rate:self.rate @@ -467,7 +486,10 @@ - (void)startNextSynthesis { - (void)enqueueText:(NSString *)text { dispatch_async(self.queue, ^{ if (self.finished || self.inputFinished || !text.length) return; - [self.pendingTexts addObject:text]; + BerdSiriDeliverySegment *segment = [BerdSiriDeliverySegment new]; + segment.text = text; + [self.pendingTexts addObject:segment]; + [self.deliverySegments addObject:segment]; self.progressGeneration += 1; [self startNextSynthesis]; }); @@ -480,6 +502,41 @@ - (void)finishInput { [self finishIfReady]; }); } +- (NSString *)deliveryJSON { + __block NSString *json = @"{\"segments\":[]}"; + void (^snapshot)(void) = ^{ + uint64_t playedFrames = 0; + if (self.player && self.player.lastRenderTime) { + AVAudioTime *playerTime = [self.player playerTimeForNodeTime:self.player.lastRenderTime]; + if (playerTime && playerTime.sampleTime > 0) { + playedFrames = (uint64_t)playerTime.sampleTime; + } + } + uint64_t latencyFrames = self.playbackSampleRate > 0 + ? (uint64_t)llround(self.playbackSampleRate * 0.1) + : 0; + playedFrames = playedFrames > latencyFrames ? playedFrames - latencyFrames : 0; + uint64_t segmentStart = 0; + NSMutableArray *> *segments = [NSMutableArray array]; + for (BerdSiriDeliverySegment *segment in self.deliverySegments) { + uint64_t played = playedFrames > segmentStart + ? MIN(segment.totalFrames, playedFrames - segmentStart) + : 0; + [segments addObject:@{ + @"text": segment.text ?: @"", + @"playedFrames": @(played), + @"totalFrames": @(segment.totalFrames), + }]; + segmentStart += segment.totalFrames; + } + NSData *data = [NSJSONSerialization dataWithJSONObject:@{ @"segments": segments } + options:0 error:nil]; + if (data) json = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; + }; + if (dispatch_get_specific(BerdSiriSpeechQueueKey)) snapshot(); + else dispatch_sync(self.queue, snapshot); + return json; +} - (void)cancel { void (^cancelWork)(void) = ^{ if (self.finished) return; @@ -1006,6 +1063,12 @@ uint64_t berd_siri_tts_stream_progress(void *stream) { return progress; } +char *berd_siri_tts_stream_copy_delivery_json(void *stream) { + if (!stream) return strdup("{\"segments\":[]}"); + NSString *json = [(__bridge BerdSiriSpeechPlayer *)stream deliveryJSON]; + return strdup((json ?: @"{\"segments\":[]}").UTF8String); +} + char *berd_siri_tts_stream_copy_error(void *stream) { if (!stream) return strdup("Siri stream is unavailable"); BerdSiriSpeechPlayer *player = (__bridge BerdSiriSpeechPlayer *)stream; diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index 32dd45fa5..0f9cb1403 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -12,7 +12,7 @@ use std::sync::mpsc; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime}; -#[cfg(target_os = "macos")] +#[cfg(any(test, target_os = "macos"))] use berd_voice::SAMPLE_RATE; #[cfg(target_os = "macos")] use berd_voice::{load_text_to_speech, load_voice_style, PocketTts, VoiceStyle}; @@ -42,6 +42,10 @@ const DOWNLOAD_READ_TIMEOUT: Duration = Duration::from_secs(30); const DOWNLOAD_TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60); #[cfg(target_os = "macos")] const STREAMING_EMIT_FRAMES: usize = 12; +#[cfg(target_os = "macos")] +const PLAYBACK_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(100); +#[cfg(any(test, target_os = "macos"))] +const PLAYBACK_LATENCY_SAFETY_FRAMES: u64 = SAMPLE_RATE as u64 / 10; const PARAKEET_ARCHIVE: Artifact = Artifact { filename: "parakeet.tar.bz2", size: 104_337_827, @@ -158,6 +162,7 @@ enum PocketStreamCommand { #[serde(rename_all = "camelCase")] enum PocketStreamEventState { Started, + Progress, Completed, Interrupted, Failed, @@ -170,6 +175,84 @@ struct PocketStreamEvent { stream_id: String, state: PocketStreamEventState, error: Option, + delivery: Option, +} + +#[cfg(any(test, target_os = "macos"))] +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDeliverySegment { + text: String, + played_frames: u64, + total_frames: u64, +} + +#[cfg(any(test, target_os = "macos"))] +#[derive(Clone, Debug, Serialize)] +struct VoiceDeliveryProgress { + segments: Vec, +} + +#[cfg(any(test, target_os = "macos"))] +#[derive(Debug, Default)] +struct PlaybackDeliveryLedger { + segments: Vec<(String, u64)>, + pieces: Vec, +} + +#[cfg(any(test, target_os = "macos"))] +impl PlaybackDeliveryLedger { + fn begin_segment(&mut self, text: String) { + self.segments.push((text, 0)); + } + + fn append_frames(&mut self, frames: usize) { + let frames = frames as u64; + if frames == 0 { + return; + } + if let Some((_, total)) = self.segments.last_mut() { + *total = total.saturating_add(frames); + self.pieces.push(frames); + } + } + + fn snapshot(&self, queued_pieces: usize, current_piece_frames: u64) -> VoiceDeliveryProgress { + let completed_pieces = self.pieces.len().saturating_sub(queued_pieces); + let completed_frames = self + .pieces + .iter() + .take(completed_pieces) + .copied() + .sum::(); + let current_total = self.pieces.get(completed_pieces).copied().unwrap_or(0); + let consumed_frames = completed_frames + .saturating_add(current_piece_frames.min(current_total)) + .saturating_sub(PLAYBACK_LATENCY_SAFETY_FRAMES); + let mut segment_start = 0_u64; + let segments = self + .segments + .iter() + .map(|(text, total_frames)| { + let played_frames = consumed_frames + .saturating_sub(segment_start) + .min(*total_frames); + segment_start = segment_start.saturating_add(*total_frames); + VoiceDeliverySegment { + text: text.clone(), + played_frames, + total_frames: *total_frames, + } + }) + .collect(); + VoiceDeliveryProgress { segments } + } +} + +#[cfg(target_os = "macos")] +struct PocketStreamOutcome { + state: PocketStreamEventState, + delivery: Option, } #[derive(Clone, Debug, Default)] @@ -755,19 +838,15 @@ pub fn start_pocket_voice_stream( speed, receiver, ); - let (event_state, error) = match result { - Ok(PocketStreamEventState::Completed) => (PocketStreamEventState::Completed, None), - Ok(PocketStreamEventState::Interrupted) => { - (PocketStreamEventState::Interrupted, None) - } - Ok(other) => (other, None), + let (event_state, error, delivery) = match result { + Ok(outcome) => (outcome.state, None, outcome.delivery), Err(error) if !active.load(Ordering::SeqCst) => { log::debug!("Pocket voice stream stopped after error: {error}"); - (PocketStreamEventState::Interrupted, None) + (PocketStreamEventState::Interrupted, None, None) } - Err(error) => (PocketStreamEventState::Failed, Some(error)), + Err(error) => (PocketStreamEventState::Failed, Some(error), None), }; - emit_pocket_stream_event(&app, &stream_id, event_state, error); + emit_pocket_stream_event(&app, &stream_id, event_state, error, delivery); finish_playback(&playback, &playback_active); }); Ok(()) @@ -1812,6 +1891,7 @@ fn emit_pocket_stream_event( stream_id: &str, state: PocketStreamEventState, error: Option, + delivery: Option, ) { let _ = app.emit( POCKET_STREAM_EVENT, @@ -1819,6 +1899,7 @@ fn emit_pocket_stream_event( stream_id: stream_id.to_string(), state, error, + delivery, }, ); } @@ -1834,7 +1915,7 @@ fn run_pocket_voice_stream( active: Arc, speed: f32, receiver: mpsc::Receiver, -) -> Result { +) -> Result { use std::num::NonZero; use rodio::cpal::traits::HostTrait; @@ -1880,13 +1961,19 @@ fn run_pocket_voice_stream( let mut pending = String::new(); let mut first_chunk_pending = true; let mut playback_started = false; + let mut delivery_ledger = PlaybackDeliveryLedger::default(); + let mut last_progress_emit = Instant::now(); loop { if !active.load(Ordering::SeqCst) { + let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); player.stop(); - return Ok(PocketStreamEventState::Interrupted); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(delivery), + }); } - match receiver.recv() { + match receiver.recv_timeout(Duration::from_millis(20)) { Ok(PocketStreamCommand::Append(text)) => { pending.push_str(&text); if !synthesize_pocket_stream_ready( @@ -1902,9 +1989,14 @@ fn run_pocket_voice_stream( &mut pending, &mut first_chunk_pending, &mut playback_started, + &mut delivery_ledger, + &mut last_progress_emit, false, )? { - return Ok(PocketStreamEventState::Interrupted); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + }); } } Ok(PocketStreamCommand::Flush) => { @@ -1921,12 +2013,18 @@ fn run_pocket_voice_stream( &mut pending, &mut first_chunk_pending, &mut playback_started, + &mut delivery_ledger, + &mut last_progress_emit, true, )? { - return Ok(PocketStreamEventState::Interrupted); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + }); } let tail = speed_processor.drain_and_reset()?; if !tail.is_empty() { + delivery_ledger.append_frames(tail.len()); player.append(SamplesBuffer::new(channels, rate, tail)); if !playback_started { playback_started = true; @@ -1935,6 +2033,7 @@ fn run_pocket_voice_stream( stream_id, PocketStreamEventState::Started, None, + None, ); println!("VOICE_CONVERSATION_PLAYBACK_STARTED"); std::io::stdout() @@ -1957,12 +2056,18 @@ fn run_pocket_voice_stream( &mut pending, &mut first_chunk_pending, &mut playback_started, + &mut delivery_ledger, + &mut last_progress_emit, true, )? { - return Ok(PocketStreamEventState::Interrupted); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + }); } let tail = speed_processor.finish()?; if !tail.is_empty() { + delivery_ledger.append_frames(tail.len()); player.append(SamplesBuffer::new(channels, rate, tail)); if !playback_started { emit_pocket_stream_event( @@ -1970,27 +2075,67 @@ fn run_pocket_voice_stream( stream_id, PocketStreamEventState::Started, None, + None, ); } } while !player.empty() { if !active.load(Ordering::SeqCst) { + let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); player.stop(); - return Ok(PocketStreamEventState::Interrupted); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(delivery), + }); } std::thread::sleep(Duration::from_millis(10)); } - return Ok(PocketStreamEventState::Completed); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Completed, + delivery: None, + }); } - Ok(PocketStreamCommand::Stop) | Err(_) => { + Ok(PocketStreamCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { + let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); active.store(false, Ordering::SeqCst); player.stop(); - return Ok(PocketStreamEventState::Interrupted); + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(delivery), + }); + } + Err(mpsc::RecvTimeoutError::Timeout) => { + if playback_started + && last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL + { + emit_pocket_stream_event( + app, + stream_id, + PocketStreamEventState::Progress, + None, + Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + ); + last_progress_emit = Instant::now(); + } } } } } +#[cfg(target_os = "macos")] +fn pocket_delivery_snapshot( + ledger: &PlaybackDeliveryLedger, + player: &Player, +) -> VoiceDeliveryProgress { + // Read the queue depth first. If the player advances to the next source + // before get_pos(), pairing the newer (smaller) position with the older + // (larger) queue depth can only undercount delivery. + let queued_pieces = player.len(); + let current_piece_frames = + (player.get_pos().as_secs_f64() * f64::from(SAMPLE_RATE)).round() as u64; + ledger.snapshot(queued_pieces, current_piece_frames) +} + #[cfg(target_os = "macos")] #[allow(clippy::too_many_arguments)] fn synthesize_pocket_stream_ready( @@ -2006,6 +2151,8 @@ fn synthesize_pocket_stream_ready( pending: &mut String, first_chunk_pending: &mut bool, playback_started: &mut bool, + delivery_ledger: &mut PlaybackDeliveryLedger, + last_progress_emit: &mut Instant, flush: bool, ) -> Result { let split = engine.take_streaming_text_chunks(pending, *first_chunk_pending, flush)?; @@ -2016,12 +2163,11 @@ fn synthesize_pocket_stream_ready( player.stop(); return Ok(false); } + let text = text.trim().to_string(); + delivery_ledger.begin_segment(text.clone()); let mut callback_error = None; - let completed = engine.synth_chunk_streaming( - text.trim(), - style, - STREAMING_EMIT_FRAMES, - &mut |samples| { + let completed = + engine.synth_chunk_streaming(&text, style, STREAMING_EMIT_FRAMES, &mut |samples| { if !active.load(Ordering::SeqCst) { return false; } @@ -2038,19 +2184,35 @@ fn synthesize_pocket_stream_ready( if delta.is_empty() { return true; } + delivery_ledger.append_frames(delta.len()); player.append(SamplesBuffer::new(channels, rate, delta)); if !*playback_started { *playback_started = true; - emit_pocket_stream_event(app, stream_id, PocketStreamEventState::Started, None); + emit_pocket_stream_event( + app, + stream_id, + PocketStreamEventState::Started, + None, + None, + ); println!("VOICE_CONVERSATION_PLAYBACK_STARTED"); if let Err(error) = std::io::stdout().flush() { callback_error = Some(format!("signal Pocket playback start: {error}")); return false; } } + if last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { + emit_pocket_stream_event( + app, + stream_id, + PocketStreamEventState::Progress, + None, + Some(pocket_delivery_snapshot(delivery_ledger, player)), + ); + *last_progress_emit = Instant::now(); + } true - }, - )?; + })?; if let Some(error) = callback_error { player.stop(); return Err(error); @@ -2213,6 +2375,24 @@ fn synthesize_and_stream( mod tests { use super::*; + #[test] + fn playback_ledger_maps_consumed_frames_to_text_segments_conservatively() { + let mut ledger = PlaybackDeliveryLedger::default(); + ledger.begin_segment("First sentence.".to_string()); + ledger.append_frames(4_800); + ledger.begin_segment("Second sentence.".to_string()); + ledger.append_frames(4_800); + + // One source has completed and the next is 50 ms in. The 100 ms + // output-latency allowance leaves 3,600 safely delivered frames in + // the first segment and none in the second. + let progress = ledger.snapshot(1, 1_200); + assert_eq!(progress.segments[0].played_frames, 3_600); + assert_eq!(progress.segments[0].total_frames, 4_800); + assert_eq!(progress.segments[1].played_frames, 0); + assert_eq!(progress.segments[1].total_frames, 4_800); + } + #[test] fn window_destroy_cancels_active_pocket_playback() { let state = PocketVoiceState::default(); diff --git a/src-tauri/src/commands/siri_voice.rs b/src-tauri/src/commands/siri_voice.rs index 8595c0735..d9223631f 100644 --- a/src-tauri/src/commands/siri_voice.rs +++ b/src-tauri/src/commands/siri_voice.rs @@ -56,6 +56,7 @@ enum SiriStreamCommand { #[serde(rename_all = "camelCase")] enum SiriStreamEventState { Started, + Progress, Completed, Interrupted, Failed, @@ -68,12 +69,36 @@ struct SiriStreamEvent { stream_id: String, state: SiriStreamEventState, error: Option, + delivery: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDeliverySegment { + text: String, + played_frames: u64, + total_frames: u64, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Deserialize, Serialize)] +struct VoiceDeliveryProgress { + segments: Vec, +} + +#[cfg(target_os = "macos")] +struct SiriStreamOutcome { + state: SiriStreamEventState, + delivery: Option, } #[cfg(target_os = "macos")] const SIRI_STREAM_EVENT: &str = "siri-voice:stream-event"; #[cfg(target_os = "macos")] const SIRI_STREAM_STALL_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(target_os = "macos")] +const PLAYBACK_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(100); const MIN_PLAYBACK_SPEED: f32 = 0.5; const MAX_PLAYBACK_SPEED: f32 = 2.0; static SIRI_SETTINGS_LOCK: Mutex<()> = Mutex::new(()); @@ -221,6 +246,7 @@ unsafe extern "C" { fn berd_siri_tts_stream_finish(stream: *mut std::ffi::c_void); fn berd_siri_tts_stream_is_finished(stream: *mut std::ffi::c_void) -> bool; fn berd_siri_tts_stream_progress(stream: *mut std::ffi::c_void) -> u64; + fn berd_siri_tts_stream_copy_delivery_json(stream: *mut std::ffi::c_void) -> *mut c_char; fn berd_siri_tts_stream_copy_error(stream: *mut std::ffi::c_void) -> *mut c_char; fn berd_siri_tts_stream_cancel(stream: *mut std::ffi::c_void); fn berd_siri_tts_stream_release(stream: *mut std::ffi::c_void); @@ -336,6 +362,7 @@ unsafe extern "C" fn siri_playback_started(context: *mut std::ffi::c_void) { stream_id: context.stream_id.clone(), state: SiriStreamEventState::Started, error: None, + delivery: None, }, ); } @@ -346,6 +373,7 @@ fn emit_stream_event( stream_id: &str, state: SiriStreamEventState, error: Option, + delivery: Option, ) { let _ = app.emit( SIRI_STREAM_EVENT, @@ -353,6 +381,7 @@ fn emit_stream_event( stream_id: stream_id.to_string(), state, error, + delivery, }, ); } @@ -554,6 +583,12 @@ fn enqueue_native_stream(stream: *mut std::ffi::c_void, text: &str) -> Result<() .ok_or_else(|| bridge_error(error, "Siri stream rejected text")) } +#[cfg(target_os = "macos")] +fn siri_delivery_progress(stream: *mut std::ffi::c_void) -> Option { + let json = take_bridge_string(unsafe { berd_siri_tts_stream_copy_delivery_json(stream) })?; + serde_json::from_str(&json).ok() +} + #[cfg(target_os = "macos")] #[allow(clippy::too_many_arguments)] fn run_siri_stream( @@ -563,7 +598,7 @@ fn run_siri_stream( speed: f32, active: Arc, receiver: mpsc::Receiver, -) -> Result { +) -> Result { let language = CString::new(selection.language) .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; let name = CString::new(selection.name) @@ -597,15 +632,27 @@ fn run_siri_stream( let mut first_chunk_pending = true; let mut finishing = false; let mut watchdog: Option = None; + let mut last_progress_emit = Instant::now(); + let mut last_delivery_json = String::new(); loop { if !active.load(Ordering::SeqCst) { + let delivery = siri_delivery_progress(stream); unsafe { berd_siri_tts_stream_cancel(stream) }; - return Ok(SiriStreamEventState::Interrupted); + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Interrupted, + delivery, + }); } if finishing && unsafe { berd_siri_tts_stream_is_finished(stream) } { let native_error = take_bridge_string(unsafe { berd_siri_tts_stream_copy_error(stream) }); - return native_error.map_or(Ok(SiriStreamEventState::Completed), Err); + return native_error.map_or( + Ok(SiriStreamOutcome { + state: SiriStreamEventState::Completed, + delivery: None, + }), + Err, + ); } if let Some(watchdog) = watchdog.as_mut() { let progress = unsafe { berd_siri_tts_stream_progress(stream) }; @@ -614,6 +661,22 @@ fn run_siri_stream( return Err("Siri synthesis stopped making progress".to_string()); } } + if last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { + if let Some(delivery) = siri_delivery_progress(stream) { + let delivery_json = serde_json::to_string(&delivery).unwrap_or_default(); + if delivery_json != last_delivery_json { + emit_stream_event( + &app, + &stream_id, + SiriStreamEventState::Progress, + None, + Some(delivery), + ); + last_delivery_json = delivery_json; + } + } + last_progress_emit = Instant::now(); + } let command = match receiver.recv_timeout(Duration::from_millis(10)) { Ok(command) => command, @@ -664,9 +727,13 @@ fn run_siri_stream( )); } SiriStreamCommand::Stop => { + let delivery = siri_delivery_progress(stream); active.store(false, Ordering::SeqCst); unsafe { berd_siri_tts_stream_cancel(stream) }; - return Ok(SiriStreamEventState::Interrupted); + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Interrupted, + delivery, + }); } _ => {} } @@ -734,14 +801,14 @@ pub fn start_siri_voice_stream( active.clone(), receiver, ); - let (event_state, error) = match result { - Ok(state) => (state, None), + let (event_state, error, delivery) = match result { + Ok(outcome) => (outcome.state, None, outcome.delivery), Err(_error) if !active.load(Ordering::SeqCst) => { - (SiriStreamEventState::Interrupted, None) + (SiriStreamEventState::Interrupted, None, None) } - Err(error) => (SiriStreamEventState::Failed, Some(error)), + Err(error) => (SiriStreamEventState::Failed, Some(error), None), }; - emit_stream_event(&app, &stream_id, event_state, error); + emit_stream_event(&app, &stream_id, event_state, error, delivery); finish_playback(&playback_state, &playback_active); }); Ok(()) diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index 6926f031e..161e125ac 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -26,7 +26,6 @@ import type { Message } from "@/shared/types/messages"; import type { GitState } from "@/shared/types/git"; import { setMultiWorkspaceEnabled } from "@/features/workspaces/multiWorkspacePreference"; import { OPEN_SETTINGS_EVENT } from "@/features/settings/lib/settingsEvents"; -import { useVoiceConversationStore } from "@/features/voice-conversation/stores/voiceConversationStore"; import { SHORTCUT_PREFERENCES_STORAGE_KEY } from "@/features/shortcuts/lib/shortcutRegistry"; import { useShortcutsDialogStore } from "@/features/shortcuts/stores/shortcutsDialogStore"; import { useProjectStore } from "@/features/projects/stores/projectStore"; diff --git a/src/features/chat/transcript/projection/messageRevisions.ts b/src/features/chat/transcript/projection/messageRevisions.ts index 46593b442..17c79745d 100644 --- a/src/features/chat/transcript/projection/messageRevisions.ts +++ b/src/features/chat/transcript/projection/messageRevisions.ts @@ -57,12 +57,12 @@ function buildSingleTextMessageRevisions( message.created, )}:${renderMetadataRevision(message.metadata)}:text:${revision}:${annotationsRevision( content.annotations, - )}:${content.speech?.status ?? ""}`; + )}:${speechRevision(content)}`; const heightRevision = `message-height:${message.id}:${ message.role - }:${heightMetadataRevision(message.metadata)}:text-height:${revision}:${ - content.speech?.status ?? "" - }`; + }:${heightMetadataRevision(message.metadata)}:text-height:${revision}:${speechRevision( + content, + )}`; return { renderRevision, @@ -106,13 +106,11 @@ function buildSingleContentRevisionParts( "text", revision, annotationsRevision(content.annotations), - content.speech?.status ?? "", - ].join(":"), - heightRevision: [ - "text-height", - revision, - content.speech?.status ?? "", + speechRevision(content), ].join(":"), + heightRevision: ["text-height", revision, speechRevision(content)].join( + ":", + ), }; } case "image": @@ -133,6 +131,17 @@ function buildSingleContentRevisionParts( } } +function speechRevision(content: TextContent): string { + const speech = content.speech; + if (!speech) return ""; + return [ + speech.status, + speech.spokenText === undefined ? "" : textRevision(speech.spokenText), + speech.unspokenText === undefined ? "" : textRevision(speech.unspokenText), + speech.confidence ?? "", + ].join(":"); +} + export function buildContentRenderRevision(content: MessageContent): string { switch (content.type) { case "text": diff --git a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts index 16892cc9b..987059440 100644 --- a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts +++ b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts @@ -1539,6 +1539,37 @@ describe("transcript projection cache", () => { expect(second.heightRevision).not.toBe(first.heightRevision); }); + it("invalidates render and height revisions when the interruption cutoff changes", () => { + const original = message( + "assistant-1", + "assistant", + "One. Two. Three.", + utc(2026, 6, 4, 10), + ); + const interrupted = (spokenText: string, unspokenText: string) => ({ + ...original, + content: original.content.map((content) => + content.type === "text" + ? { + ...content, + speech: { + status: "interrupted" as const, + spokenText, + unspokenText, + confidence: "medium" as const, + }, + } + : content, + ), + }); + + const first = buildMessageRevisions(interrupted("One.", " Two. Three.")); + const second = buildMessageRevisions(interrupted("One. Two.", " Three.")); + + expect(second.renderRevision).not.toBe(first.renderRevision); + expect(second.heightRevision).not.toBe(first.heightRevision); + }); + it("includes user message origin in render and height revisions", () => { const original = message("user-1", "user", "same", utc(2026, 6, 4, 10)); const withOrigin = { diff --git a/src/features/chat/ui/AgentWorkPanel.tsx b/src/features/chat/ui/AgentWorkPanel.tsx index b78801793..5e1e0bc48 100644 --- a/src/features/chat/ui/AgentWorkPanel.tsx +++ b/src/features/chat/ui/AgentWorkPanel.tsx @@ -307,6 +307,15 @@ function AgentWorkItemRow({ if (item.kind === "progress") { const speechStatus = item.content.speech?.status; + const speechDisplayText = + speechStatus === "interrupted" && + item.content.speech?.spokenText !== undefined && + item.content.speech.unspokenText !== undefined + ? `${item.content.speech.spokenText}${item.content.speech.unspokenText + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">")}` + : item.content.text; const speechLabel = speechStatus ? { speaking: t("message.voiceSpeechSpeakingLabel"), @@ -324,7 +333,7 @@ function AgentWorkItemRow({ className={cn( "min-w-0 flex-1 pb-2 text-sm leading-relaxed", usePrimaryText ? "text-foreground" : "text-muted-foreground", - speechStatus === "interrupted" && "line-through opacity-70", + speechStatus === "interrupted" && "opacity-80", )} > {speechStatus && speechLabel ? ( @@ -333,7 +342,7 @@ function AgentWorkItemRow({ label={speechLabel} /> ) : null} - {item.content.text} + {speechDisplayText} ); diff --git a/src/features/chat/ui/MessageBubble.tsx b/src/features/chat/ui/MessageBubble.tsx index 58b290dde..40ea1c253 100644 --- a/src/features/chat/ui/MessageBubble.tsx +++ b/src/features/chat/ui/MessageBubble.tsx @@ -532,6 +532,15 @@ function renderContentBlock( options.resolveProviderErrorNotice?.(tc.text) ?? null; const displayText = providerErrorNotice ?? tc.text; const speechStatus = tc.speech?.status; + const speechDisplayText = + speechStatus === "interrupted" && + tc.speech?.spokenText !== undefined && + tc.speech.unspokenText !== undefined + ? `${tc.speech.spokenText}${tc.speech.unspokenText + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">")}` + : displayText; const speechLabel = speechStatus ? { speaking: options.voiceSpeechSpeakingLabel, @@ -545,9 +554,7 @@ function renderContentBlock(
{speechStatus && speechLabel ? ( - {displayText} + {speechDisplayText}
); diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index 38bfcea59..2f6851807 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -708,9 +708,32 @@ describe("MessageBubble", () => { expect(screen.getAllByText("One visible assistant response.")).toHaveLength( 1, ); - if (status === "interrupted") { - expect(block).toHaveClass("line-through"); - } + }); + + it("strikes only the estimated unspoken suffix after barge-in", () => { + const { container } = render( + , + ); + + const block = container.querySelector( + '[data-voice-speech-status="interrupted"]', + ); + expect(block).toHaveTextContent("One. Two. Three."); + expect(block?.querySelector("del")).toHaveTextContent(". Three."); + expect(block?.querySelector("del")).not.toHaveTextContent("One. Two"); }); it("renders multiple content blocks", () => { diff --git a/src/features/voice-conversation/api/pocketVoice.ts b/src/features/voice-conversation/api/pocketVoice.ts index b8152a8b5..342722658 100644 --- a/src/features/voice-conversation/api/pocketVoice.ts +++ b/src/features/voice-conversation/api/pocketVoice.ts @@ -36,8 +36,19 @@ export interface PocketVoiceStatus { export interface PocketVoiceStreamEvent { streamId: string; - state: "started" | "completed" | "interrupted" | "failed"; + state: "started" | "progress" | "completed" | "interrupted" | "failed"; error: string | null; + delivery?: VoiceDeliveryProgress | null; +} + +export interface VoiceDeliverySegment { + text: string; + playedFrames: number; + totalFrames: number; +} + +export interface VoiceDeliveryProgress { + segments: VoiceDeliverySegment[]; } export type VoiceModelKind = "pocket" | "parakeet"; diff --git a/src/features/voice-conversation/api/siriVoice.ts b/src/features/voice-conversation/api/siriVoice.ts index 2b72bbc56..0454b25a1 100644 --- a/src/features/voice-conversation/api/siriVoice.ts +++ b/src/features/voice-conversation/api/siriVoice.ts @@ -1,5 +1,6 @@ import { invoke } from "@tauri-apps/api/core"; import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import type { VoiceDeliveryProgress } from "./pocketVoice"; export interface SiriVoice { name: string; @@ -58,8 +59,9 @@ export function setSiriPlaybackSpeed(speed: number): Promise { export interface SiriVoiceStreamEvent { streamId: string; - state: "started" | "completed" | "interrupted" | "failed"; + state: "started" | "progress" | "completed" | "interrupted" | "failed"; error: string | null; + delivery?: VoiceDeliveryProgress | null; } export function startSiriVoiceStream(streamId: string): Promise { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 2b4ac5f11..5d27f8069 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -493,4 +493,65 @@ describe("native assistant speech stream", () => { ); expect(takeVoicePlaybackNotices("session-1")).toBeNull(); }); + + it("uses playback progress to report and decorate only the unspoken suffix", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "One. Two. Three." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + emit("started"); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + mocks.streamHandler?.({ + streamId, + state: "progress", + error: null, + delivery: { + segments: [ + { + text: "One. Two. Three.", + playedFrames: 600, + totalFrames: 1_000, + }, + ], + }, + }); + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { + status: "interrupted", + spokenText: "One. Two", + unspokenText: ". Three.", + confidence: "medium", + }, + }); + const notice = takeVoicePlaybackNotices("session-1"); + expect(notice).toContain('"spokenText":"One. Two"'); + expect(notice).toContain('"unspokenText":". Three."'); + expect(notice).toContain('"confidence":"medium"'); + + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .appendStreamingText("session-1", "assistant-1", " Four."); + await vi.waitFor(() => { + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { + status: "interrupted", + spokenText: "One. Two", + unspokenText: ". Three. Four.", + }, + }); + }); + expect(mocks.append).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 042eb749b..59d8af02b 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -1,4 +1,5 @@ import { useChatStore } from "@/features/chat/stores/chatStore"; +import type { VoiceSpeechState } from "@/shared/types/messages"; import { appendPocketVoiceStream, finishPocketVoiceStream, @@ -6,6 +7,7 @@ import { listenToPocketVoiceStream, startPocketVoiceStream, stopPocketVoice, + type VoiceDeliveryProgress, type PocketVoiceStreamEvent, } from "../api/pocketVoice"; import { @@ -23,15 +25,25 @@ import { useVoiceConversationStore } from "../stores/voiceConversationStore"; type SpeechFailureHandler = (text: string, error: unknown) => void; type SpeechTarget = { messageId: string; textOrdinal: number }; +type SpeechTargetSpan = SpeechTarget & { start: number; end: number }; +type SpeechDeliveryEstimate = { + cutoff: number; + spokenText: string; + unspokenText: string; + confidence: "low" | "medium"; +}; type ActiveUtterance = { id: string; sessionId: string; voiceRevision: number; targets: SpeechTarget[]; + targetSpans: SpeechTargetSpan[]; text: string; finishing: boolean; + latestDelivery: VoiceDeliveryProgress | null; status: SpeechStatus | null; onFailure: SpeechFailureHandler; + onInterrupted: () => void; onTerminal: () => void; }; type SpeechStatus = @@ -85,6 +97,7 @@ function recordPlaybackNotice( key: string, text: string, status: "interrupted" | "notSpoken" | "failed", + estimate?: SpeechDeliveryEstimate, ) { const noticeKey = `${sessionId}\0${key}\0${status}`; if (recordedNoticeKeys.has(noticeKey)) return; @@ -96,8 +109,17 @@ function recordPlaybackNotice( : status === "notSpoken" ? "TTS delivery was blocked because the user was speaking; the assistant reply was not spoken." : "Native TTS could not deliver the assistant reply."; + const estimateLine = estimate + ? `\nDelivery estimate: ${JSON.stringify({ + spokenText: estimate.spokenText, + unspokenText: estimate.unspokenText, + cutoff: estimate.cutoff, + confidence: estimate.confidence, + estimated: true, + })}` + : ""; const notice = - `[voice: tts-delivery-failed]\n${outcome}\nOriginal text: ${excerpt}\n` + + `[voice: tts-delivery-failed]\n${outcome}\nOriginal text: ${excerpt}${estimateLine}\n` + "This is TTS delivery state, not live user voice input. Do not respond to this control message or repeat the reply unless re-delivery is still appropriate."; pendingNotices.set(sessionId, [ ...(pendingNotices.get(sessionId) ?? []), @@ -115,10 +137,131 @@ function targetKey(target: SpeechTarget): string { return `${target.messageId}\0text:${target.textOrdinal}`; } -function setTargetStatus( +function completedWordCutoff(text: string, playedRatio: number): number { + const approximateCutoff = Math.floor( + text.length * Math.max(0, Math.min(1, playedRatio)), + ); + if (approximateCutoff >= text.length) return text.length; + const segmenter = new Intl.Segmenter(undefined, { granularity: "word" }); + let cutoff = 0; + for (const part of segmenter.segment(text)) { + const end = part.index + part.segment.length; + if (end > approximateCutoff) break; + if (part.isWordLike) cutoff = end; + } + return cutoff; +} + +function estimateSpeechDelivery( + text: string, + delivery: VoiceDeliveryProgress | null, +): SpeechDeliveryEstimate { + if (!delivery?.segments.length) { + return { + cutoff: 0, + spokenText: "", + unspokenText: text, + confidence: "low", + }; + } + + let searchFrom = 0; + let cutoff = 0; + let matchedSegment = false; + for (const segment of delivery.segments) { + const segmentStart = text.indexOf(segment.text, searchFrom); + if (segmentStart === -1) continue; + matchedSegment = true; + const totalFrames = Math.max(0, segment.totalFrames); + const playedFrames = Math.max( + 0, + Math.min(totalFrames, segment.playedFrames), + ); + if (totalFrames === 0 || playedFrames === 0) break; + if (playedFrames >= totalFrames) { + cutoff = segmentStart + segment.text.length; + searchFrom = cutoff; + continue; + } + cutoff = + segmentStart + + completedWordCutoff(segment.text, playedFrames / totalFrames); + break; + } + + return { + cutoff, + spokenText: text.slice(0, cutoff), + unspokenText: text.slice(cutoff), + confidence: matchedSegment ? "medium" : "low", + }; +} + +function targetText(sessionId: string, target: SpeechTarget): string { + const message = + useChatStore + .getState() + .messagesBySession[sessionId]?.find( + (candidate) => candidate.id === target.messageId, + ) ?? null; + if (!message) return ""; + let textOrdinal = 0; + for (const content of message.content) { + if (content.type !== "text") continue; + if (textOrdinal === target.textOrdinal) return content.text; + textOrdinal += 1; + } + return ""; +} + +function applyInterruptionEstimate( + utterance: ActiveUtterance, + estimate: SpeechDeliveryEstimate, +) { + const firstTargetKey = utterance.targets[0] + ? targetKey(utterance.targets[0]) + : null; + for (const target of utterance.targets) { + const spans = utterance.targetSpans.filter( + (span) => targetKey(span) === targetKey(target), + ); + const start = spans.at(0)?.start ?? 0; + const end = spans.at(-1)?.end ?? start; + const text = targetText(utterance.sessionId, target); + if (estimate.cutoff >= end && end > start) { + setTargetSpeech(utterance.sessionId, target, { status: "spoken" }); + continue; + } + if (estimate.cutoff <= start) { + if (targetKey(target) === firstTargetKey) { + setTargetSpeech(utterance.sessionId, target, { + status: "interrupted", + spokenText: "", + unspokenText: text, + confidence: estimate.confidence, + }); + continue; + } + setTargetSpeech(utterance.sessionId, target, { status: "notSpoken" }); + continue; + } + const localCutoff = Math.max( + 0, + Math.min(text.length, estimate.cutoff - start), + ); + setTargetSpeech(utterance.sessionId, target, { + status: "interrupted", + spokenText: text.slice(0, localCutoff), + unspokenText: text.slice(localCutoff), + confidence: estimate.confidence, + }); + } +} + +function setTargetSpeech( sessionId: string, target: SpeechTarget, - status: SpeechStatus, + speech: VoiceSpeechState, ) { useChatStore .getState() @@ -130,7 +273,7 @@ function setTargetStatus( if (content.type !== "text") return content; const matches = textOrdinal === target.textOrdinal; textOrdinal += 1; - return matches ? { ...content, speech: { status } } : content; + return matches ? { ...content, speech } : content; }), }; }); @@ -139,7 +282,7 @@ function setTargetStatus( function setUtteranceStatus(utterance: ActiveUtterance, status: SpeechStatus) { utterance.status = status; for (const target of utterance.targets) { - setTargetStatus(utterance.sessionId, target, status); + setTargetSpeech(utterance.sessionId, target, { status }); } } @@ -188,6 +331,9 @@ function handleStreamEvent( const voice = useVoiceConversationStore.getState(); switch (event.state) { + case "progress": + utterance.latestDelivery = event.delivery ?? null; + break; case "started": setUtteranceStatus(utterance, "speaking"); voice.setUiState("agent-speaking"); @@ -208,13 +354,20 @@ function handleStreamEvent( ); utterance.onTerminal(); break; - case "interrupted": - setUtteranceStatus(utterance, "interrupted"); + case "interrupted": { + utterance.latestDelivery = event.delivery ?? utterance.latestDelivery; + const estimate = estimateSpeechDelivery( + utterance.text, + utterance.latestDelivery, + ); + applyInterruptionEstimate(utterance, estimate); + utterance.onInterrupted(); recordPlaybackNotice( utterance.sessionId, utterance.id, utterance.text, "interrupted", + estimate, ); voice.setUiState("listening"); activeUtterance = null; @@ -225,6 +378,7 @@ function handleStreamEvent( ); utterance.onTerminal(); break; + } case "failed": setUtteranceStatus(utterance, "failed"); recordPlaybackNotice( @@ -254,12 +408,18 @@ function interruptActiveUtterance(): boolean { commandEpoch += 1; activeUtterance = null; if (utterance) { - setUtteranceStatus(utterance, "interrupted"); + const estimate = estimateSpeechDelivery( + utterance.text, + utterance.latestDelivery, + ); + applyInterruptionEstimate(utterance, estimate); + utterance.onInterrupted(); recordPlaybackNotice( utterance.sessionId, utterance.id, utterance.text, "interrupted", + estimate, ); reportAssistantActivity( utterance.sessionId, @@ -338,6 +498,7 @@ export function startNativeAssistantSpeech( const toolCountByMessage = new Map(); const consumedTextBySlot = new Map(); const completedMessages = new Set(); + const interruptedMessages = new Set(); for (const message of initialMessages) { toolCountByMessage.set( message.id, @@ -367,7 +528,9 @@ export function startNativeAssistantSpeech( ) { activeUtterance.targets.push(target); if (activeUtterance.status) { - setTargetStatus(sessionId, target, activeUtterance.status); + setTargetSpeech(sessionId, target, { + status: activeUtterance.status, + }); } } return activeUtterance; @@ -379,10 +542,17 @@ export function startNativeAssistantSpeech( activeSpeechRevision ?? useVoiceConversationStore.getState().status.revision, targets: [target], + targetSpans: [], text: "", finishing: false, + latestDelivery: null, status: null, onFailure, + onInterrupted: () => { + for (const utteranceTarget of utterance.targets) { + interruptedMessages.add(utteranceTarget.messageId); + } + }, onTerminal: () => queueMicrotask(inspect), }; activeUtterance = utterance; @@ -444,15 +614,47 @@ export function startNativeAssistantSpeech( consumedTextBySlot.set(slot, content.text); if (!delta) continue; + if (interruptedMessages.has(message.id)) { + const currentSpeech = content.speech; + if ( + currentSpeech?.status === "interrupted" && + currentSpeech.spokenText !== undefined + ) { + setTargetSpeech(sessionId, target, { + ...currentSpeech, + unspokenText: content.text.slice(currentSpeech.spokenText.length), + }); + } else { + setTargetSpeech(sessionId, target, { status: "notSpoken" }); + } + recordPlaybackNotice(sessionId, slot, content.text, "notSpoken"); + continue; + } + if (voice.userSpeaking) { - setTargetStatus(sessionId, target, "notSpoken"); + setTargetSpeech(sessionId, target, { status: "notSpoken" }); recordPlaybackNotice(sessionId, slot, content.text, "notSpoken"); continue; } const utterance = ensureUtterance(target); if (utterance.finishing) continue; + const spanStart = utterance.text.length; utterance.text += delta; + const previousSpan = utterance.targetSpans.at(-1); + if ( + previousSpan && + targetKey(previousSpan) === targetKey(target) && + previousSpan.end === spanStart + ) { + previousSpan.end = utterance.text.length; + } else { + utterance.targetSpans.push({ + ...target, + start: spanStart, + end: utterance.text.length, + }); + } queueStreamCommand( utterance, () => streamBackend.append(utterance.id, delta), diff --git a/src/shared/types/messages.ts b/src/shared/types/messages.ts index 68b25e773..e83922119 100644 --- a/src/shared/types/messages.ts +++ b/src/shared/types/messages.ts @@ -30,6 +30,11 @@ export type VoiceSpeechStatus = export interface VoiceSpeechState { status: VoiceSpeechStatus; + /** Ephemeral playback estimate; never serialized into ACP history. */ + spokenText?: string; + /** Ephemeral playback estimate; never serialized into ACP history. */ + unspokenText?: string; + confidence?: "low" | "medium"; } /** ACP TextContent with discriminator and local voice playback state. */ From 70973ad92fd5cfcb53be39240682d54f808ac7ab Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 14:12:17 -0400 Subject: [PATCH 02/20] fix(voice): finalize interruption delivery from playback --- src/features/chat/ui/MessageBubble.tsx | 1 + .../chat/ui/__tests__/MessageBubble.test.tsx | 31 ++++++++++ .../lib/nativeAssistantSpeech.test.ts | 56 ++++++++++++++++++- .../lib/nativeAssistantSpeech.ts | 56 ++++++++++++++----- 4 files changed, 130 insertions(+), 14 deletions(-) diff --git a/src/features/chat/ui/MessageBubble.tsx b/src/features/chat/ui/MessageBubble.tsx index 40ea1c253..785ade33a 100644 --- a/src/features/chat/ui/MessageBubble.tsx +++ b/src/features/chat/ui/MessageBubble.tsx @@ -533,6 +533,7 @@ function renderContentBlock( const displayText = providerErrorNotice ?? tc.text; const speechStatus = tc.speech?.status; const speechDisplayText = + providerErrorNotice === null && speechStatus === "interrupted" && tc.speech?.spokenText !== undefined && tc.speech.unspokenText !== undefined diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index 2f6851807..8939ca5be 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -736,6 +736,37 @@ describe("MessageBubble", () => { expect(block?.querySelector("del")).not.toHaveTextContent("One. Two"); }); + it("preserves provider-error presentation after interrupted delivery", () => { + const rawError = + "Ran into this error: thinking blocks in the latest assistant message cannot be modified"; + const { container } = render( + , + ); + + const block = container.querySelector( + '[data-voice-speech-status="interrupted"]', + ); + expect(block).toHaveTextContent( + "This chat can't continue with a Claude model", + ); + expect(block?.querySelector("del")).toBeNull(); + expect(block).not.toHaveTextContent(rawError); + }); + it("renders multiple content blocks", () => { const msg = assistantMessage([ { type: "text", text: "first block" }, diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 5d27f8069..6f41c0957 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -485,6 +485,7 @@ describe("native assistant speech stream", () => { useVoiceConversationStore.setState({ userSpeaking: true }); await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + emit("interrupted"); expect( useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], ).toMatchObject({ speech: { status: "interrupted" } }); @@ -512,7 +513,7 @@ describe("native assistant speech stream", () => { segments: [ { text: "One. Two. Three.", - playedFrames: 600, + playedFrames: 300, totalFrames: 1_000, }, ], @@ -521,6 +522,23 @@ describe("native assistant speech stream", () => { useVoiceConversationStore.setState({ userSpeaking: true }); await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ speech: { status: "speaking" } }); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "One. Two. Three.", + playedFrames: 600, + totalFrames: 1_000, + }, + ], + }, + }); expect( useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], @@ -554,4 +572,40 @@ describe("native assistant speech stream", () => { }); expect(mocks.append).toHaveBeenCalledTimes(1); }); + + it("bounds spoken and unspoken excerpts in the model delivery notice", async () => { + const text = `${"spoken ".repeat(100)}${"unspoken ".repeat(100)}`; + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [assistant([{ type: "text", text }])]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + emit("started"); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [{ text, playedFrames: 1_000, totalFrames: 2_000 }], + }, + }); + + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + const estimate = JSON.parse( + notice.match(/Delivery estimate: (\{.*\})/)?.[1] ?? "{}", + ) as { + spokenText: string; + unspokenText: string; + spokenTextTruncated: boolean; + unspokenTextTruncated: boolean; + }; + expect(estimate.spokenText.length).toBeLessThanOrEqual(250); + expect(estimate.unspokenText.length).toBeLessThanOrEqual(250); + expect(estimate.spokenTextTruncated).toBe(true); + expect(estimate.unspokenTextTruncated).toBe(true); + }); }); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 59d8af02b..a7278e85e 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -40,6 +40,7 @@ type ActiveUtterance = { targetSpans: SpeechTargetSpan[]; text: string; finishing: boolean; + interruptionRequested: boolean; latestDelivery: VoiceDeliveryProgress | null; status: SpeechStatus | null; onFailure: SpeechFailureHandler; @@ -67,6 +68,25 @@ let stopActiveVoice: () => Promise = stopPocketVoice; let activityReportQueue = Promise.resolve(); const pendingNotices = new Map(); const recordedNoticeKeys = new Set(); +const DELIVERY_NOTICE_TEXT_LIMIT = 250; + +function boundedDeliveryText( + text: string, + side: "start" | "end", +): { text: string; truncated: boolean } { + if (text.length <= DELIVERY_NOTICE_TEXT_LIMIT) { + return { text, truncated: false }; + } + return side === "start" + ? { + text: `${text.slice(0, DELIVERY_NOTICE_TEXT_LIMIT - 1)}…`, + truncated: true, + } + : { + text: `…${text.slice(-(DELIVERY_NOTICE_TEXT_LIMIT - 1))}`, + truncated: true, + }; +} function reportAssistantActivity( sessionId: string, @@ -109,15 +129,20 @@ function recordPlaybackNotice( : status === "notSpoken" ? "TTS delivery was blocked because the user was speaking; the assistant reply was not spoken." : "Native TTS could not deliver the assistant reply."; - const estimateLine = estimate - ? `\nDelivery estimate: ${JSON.stringify({ - spokenText: estimate.spokenText, - unspokenText: estimate.unspokenText, - cutoff: estimate.cutoff, - confidence: estimate.confidence, - estimated: true, - })}` - : ""; + const estimateLine = (() => { + if (!estimate) return ""; + const spoken = boundedDeliveryText(estimate.spokenText, "end"); + const unspoken = boundedDeliveryText(estimate.unspokenText, "start"); + return `\nDelivery estimate: ${JSON.stringify({ + spokenText: spoken.text, + unspokenText: unspoken.text, + spokenTextTruncated: spoken.truncated, + unspokenTextTruncated: unspoken.truncated, + cutoff: estimate.cutoff, + confidence: estimate.confidence, + estimated: true, + })}`; + })(); const notice = `[voice: tts-delivery-failed]\n${outcome}\nOriginal text: ${excerpt}${estimateLine}\n` + "This is TTS delivery state, not live user voice input. Do not respond to this control message or repeat the reply unless re-delivery is still appropriate."; @@ -403,11 +428,15 @@ function handleStreamEvent( } } -function interruptActiveUtterance(): boolean { +function interruptActiveUtterance(awaitTerminalDelivery = false): boolean { const utterance = activeUtterance; commandEpoch += 1; - activeUtterance = null; - if (utterance) { + if (utterance && !utterance.interruptionRequested) { + utterance.interruptionRequested = true; + if (awaitTerminalDelivery) utterance.onInterrupted(); + } + if (utterance && !awaitTerminalDelivery) { + activeUtterance = null; const estimate = estimateSpeechDelivery( utterance.text, utterance.latestDelivery, @@ -545,6 +574,7 @@ export function startNativeAssistantSpeech( targetSpans: [], text: "", finishing: false, + interruptionRequested: false, latestDelivery: null, status: null, onFailure, @@ -722,7 +752,7 @@ export function startNativeAssistantSpeech( const becameUserSpeaking = voice.userSpeaking && !wasUserSpeaking; wasUserSpeaking = voice.userSpeaking; if (!becameUserSpeaking || activeGeneration !== generation) return; - interruptActiveUtterance(); + interruptActiveUtterance(true); }); queueMicrotask(inspect); } From 87e3af6110c25352f67227b3680de87b7ea9a82b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 14:18:10 -0400 Subject: [PATCH 03/20] fix(voice): finalize pre-playback interruptions --- .../lib/nativeAssistantSpeech.test.ts | 25 +++++++++++++++++++ .../lib/nativeAssistantSpeech.ts | 6 +++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 6f41c0957..0cb3f7a47 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -495,6 +495,31 @@ describe("native assistant speech stream", () => { expect(takeVoicePlaybackNotices("session-1")).toBeNull(); }); + it("finalizes an interruption before native playback starts", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Queued reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { + status: "interrupted", + spokenText: "", + unspokenText: "Queued reply.", + confidence: "low", + }, + }); + expect(takeVoicePlaybackNotices("session-1")).toContain('"spokenText":""'); + }); + it("uses playback progress to report and decorate only the unspoken suffix", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index a7278e85e..3e46b6b1a 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -430,12 +430,14 @@ function handleStreamEvent( function interruptActiveUtterance(awaitTerminalDelivery = false): boolean { const utterance = activeUtterance; + const terminalEventExpected = + awaitTerminalDelivery && utterance?.status === "speaking"; commandEpoch += 1; if (utterance && !utterance.interruptionRequested) { utterance.interruptionRequested = true; - if (awaitTerminalDelivery) utterance.onInterrupted(); + if (terminalEventExpected) utterance.onInterrupted(); } - if (utterance && !awaitTerminalDelivery) { + if (utterance && !terminalEventExpected) { activeUtterance = null; const estimate = estimateSpeechDelivery( utterance.text, From 7f3ba3c306496918b94bf95b26720ef9cdd92315 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 14:22:58 -0400 Subject: [PATCH 04/20] fix(voice): track native interruption readiness --- .../lib/nativeAssistantSpeech.test.ts | 41 ++++++++++++++++++- .../lib/nativeAssistantSpeech.ts | 5 ++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 0cb3f7a47..9911563e7 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -496,13 +496,20 @@ describe("native assistant speech stream", () => { }); it("finalizes an interruption before native playback starts", async () => { + let resolveStart: (() => void) | undefined; + mocks.start.mockImplementation( + () => + new Promise((resolve) => { + resolveStart = resolve; + }), + ); startNativeAssistantSpeech("session-1", vi.fn()); useChatStore .getState() .setMessages("session-1", [ assistant([{ type: "text", text: "Queued reply." }]), ]); - await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + await vi.waitFor(() => expect(mocks.start).toHaveBeenCalled()); useVoiceConversationStore.setState({ userSpeaking: true }); await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); @@ -518,6 +525,38 @@ describe("native assistant speech stream", () => { }, }); expect(takeVoicePlaybackNotices("session-1")).toContain('"spokenText":""'); + resolveStart?.(); + }); + + it("waits for terminal delivery once the native stream exists", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Native reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).not.toHaveProperty("speech"); + + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { text: "Native reply.", playedFrames: 400, totalFrames: 1_000 }, + ], + }, + }); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ speech: { status: "interrupted" } }); }); it("uses playback progress to report and decorate only the unspoken suffix", async () => { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 3e46b6b1a..9f8fbf8d8 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -40,6 +40,7 @@ type ActiveUtterance = { targetSpans: SpeechTargetSpan[]; text: string; finishing: boolean; + nativeStreamStarted: boolean; interruptionRequested: boolean; latestDelivery: VoiceDeliveryProgress | null; status: SpeechStatus | null; @@ -431,7 +432,7 @@ function handleStreamEvent( function interruptActiveUtterance(awaitTerminalDelivery = false): boolean { const utterance = activeUtterance; const terminalEventExpected = - awaitTerminalDelivery && utterance?.status === "speaking"; + awaitTerminalDelivery && utterance?.nativeStreamStarted === true; commandEpoch += 1; if (utterance && !utterance.interruptionRequested) { utterance.interruptionRequested = true; @@ -576,6 +577,7 @@ export function startNativeAssistantSpeech( targetSpans: [], text: "", finishing: false, + nativeStreamStarted: false, interruptionRequested: false, latestDelivery: null, status: null, @@ -593,6 +595,7 @@ export function startNativeAssistantSpeech( async () => { await streamListenerReady; await streamBackend.start(utterance.id); + utterance.nativeStreamStarted = true; }, onFailure, ); From 3f8008acdb3fb9f89dd142538e8b2af5193b0ef3 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 20:33:13 -0400 Subject: [PATCH 05/20] fix(voice): distinguish hang-up delivery notices --- .../lib/nativeAssistantSpeech.test.ts | 28 +++++++++++++++++++ .../lib/nativeAssistantSpeech.ts | 18 ++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 9911563e7..c4405dfee 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -559,6 +559,34 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted" } }); }); + it("describes a hang-up as stopping the voice conversation", async () => { + takeVoicePlaybackNotices("session-1"); + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Goodbye." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + emit("started"); + + useVoiceConversationStore.setState((voice) => ({ + status: { + ...voice.status, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: voice.status.revision + 1, + }, + uiState: "off", + })); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + + const notice = takeVoicePlaybackNotices("session-1"); + expect(notice).toContain("because the voice conversation stopped"); + expect(notice).not.toContain("because the user started speaking"); + }); + it("uses playback progress to report and decorate only the unspoken suffix", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 9f8fbf8d8..4a9451492 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -32,6 +32,7 @@ type SpeechDeliveryEstimate = { unspokenText: string; confidence: "low" | "medium"; }; +type InterruptionCause = "userSpeaking" | "voiceStopped"; type ActiveUtterance = { id: string; sessionId: string; @@ -42,6 +43,7 @@ type ActiveUtterance = { finishing: boolean; nativeStreamStarted: boolean; interruptionRequested: boolean; + interruptionCause: InterruptionCause | null; latestDelivery: VoiceDeliveryProgress | null; status: SpeechStatus | null; onFailure: SpeechFailureHandler; @@ -119,6 +121,7 @@ function recordPlaybackNotice( text: string, status: "interrupted" | "notSpoken" | "failed", estimate?: SpeechDeliveryEstimate, + interruptionCause: InterruptionCause = "voiceStopped", ) { const noticeKey = `${sessionId}\0${key}\0${status}`; if (recordedNoticeKeys.has(noticeKey)) return; @@ -126,7 +129,9 @@ function recordPlaybackNotice( const excerpt = text.length > 500 ? `${text.slice(0, 497).trimEnd()}…` : text; const outcome = status === "interrupted" - ? "TTS delivery was interrupted because the user started speaking; the assistant reply was not fully spoken." + ? interruptionCause === "userSpeaking" + ? "TTS delivery was interrupted because the user started speaking; the assistant reply was not fully spoken." + : "TTS delivery was interrupted because the voice conversation stopped; the assistant reply was not fully spoken." : status === "notSpoken" ? "TTS delivery was blocked because the user was speaking; the assistant reply was not spoken." : "Native TTS could not deliver the assistant reply."; @@ -394,6 +399,7 @@ function handleStreamEvent( utterance.text, "interrupted", estimate, + utterance.interruptionCause ?? "voiceStopped", ); voice.setUiState("listening"); activeUtterance = null; @@ -429,13 +435,17 @@ function handleStreamEvent( } } -function interruptActiveUtterance(awaitTerminalDelivery = false): boolean { +function interruptActiveUtterance( + awaitTerminalDelivery = false, + cause: InterruptionCause = "voiceStopped", +): boolean { const utterance = activeUtterance; const terminalEventExpected = awaitTerminalDelivery && utterance?.nativeStreamStarted === true; commandEpoch += 1; if (utterance && !utterance.interruptionRequested) { utterance.interruptionRequested = true; + utterance.interruptionCause = cause; if (terminalEventExpected) utterance.onInterrupted(); } if (utterance && !terminalEventExpected) { @@ -452,6 +462,7 @@ function interruptActiveUtterance(awaitTerminalDelivery = false): boolean { utterance.text, "interrupted", estimate, + utterance.interruptionCause ?? cause, ); reportAssistantActivity( utterance.sessionId, @@ -579,6 +590,7 @@ export function startNativeAssistantSpeech( finishing: false, nativeStreamStarted: false, interruptionRequested: false, + interruptionCause: null, latestDelivery: null, status: null, onFailure, @@ -757,7 +769,7 @@ export function startNativeAssistantSpeech( const becameUserSpeaking = voice.userSpeaking && !wasUserSpeaking; wasUserSpeaking = voice.userSpeaking; if (!becameUserSpeaking || activeGeneration !== generation) return; - interruptActiveUtterance(true); + interruptActiveUtterance(true, "userSpeaking"); }); queueMicrotask(inspect); } From 1b3ab60864066f595bc2683f9fe011120a195055 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 20:42:40 -0400 Subject: [PATCH 06/20] fix(voice): strike all unspoken paragraphs --- src/features/chat/ui/AgentWorkPanel.tsx | 13 ++++---- src/features/chat/ui/MessageBubble.tsx | 13 ++++---- .../chat/ui/VoiceSpeechStatusIndicator.tsx | 20 +++++++++++++ .../chat/ui/__tests__/MessageBubble.test.tsx | 30 +++++++++++++++++++ 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/features/chat/ui/AgentWorkPanel.tsx b/src/features/chat/ui/AgentWorkPanel.tsx index 5e1e0bc48..f2c2604e6 100644 --- a/src/features/chat/ui/AgentWorkPanel.tsx +++ b/src/features/chat/ui/AgentWorkPanel.tsx @@ -30,7 +30,10 @@ import type { import type { TranscriptAgentWorkPayload } from "@/features/chat/transcript/projection/transcriptItemTypes"; import { useTranscriptRowStateAdapter } from "@/features/chat/transcript/row-state"; import { ToolCallAdapter } from "./ToolCallAdapter"; -import { VoiceSpeechStatusIndicator } from "./VoiceSpeechStatusIndicator"; +import { + formatInterruptedSpeechMarkdown, + VoiceSpeechStatusIndicator, +} from "./VoiceSpeechStatusIndicator"; interface ToolTimelineItem { kind: "tool"; @@ -311,10 +314,10 @@ function AgentWorkItemRow({ speechStatus === "interrupted" && item.content.speech?.spokenText !== undefined && item.content.speech.unspokenText !== undefined - ? `${item.content.speech.spokenText}${item.content.speech.unspokenText - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">")}` + ? formatInterruptedSpeechMarkdown( + item.content.speech.spokenText, + item.content.speech.unspokenText, + ) : item.content.text; const speechLabel = speechStatus ? { diff --git a/src/features/chat/ui/MessageBubble.tsx b/src/features/chat/ui/MessageBubble.tsx index 785ade33a..60b186bfa 100644 --- a/src/features/chat/ui/MessageBubble.tsx +++ b/src/features/chat/ui/MessageBubble.tsx @@ -63,7 +63,10 @@ import { UserMessageClamp, } from "./UserMessageClamp"; import { ImageLightbox } from "@/shared/ui/ImageLightbox"; -import { VoiceSpeechStatusIndicator } from "./VoiceSpeechStatusIndicator"; +import { + formatInterruptedSpeechMarkdown, + VoiceSpeechStatusIndicator, +} from "./VoiceSpeechStatusIndicator"; interface MessageAttachmentPreviewItem { key: string; @@ -537,10 +540,10 @@ function renderContentBlock( speechStatus === "interrupted" && tc.speech?.spokenText !== undefined && tc.speech.unspokenText !== undefined - ? `${tc.speech.spokenText}${tc.speech.unspokenText - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">")}` + ? formatInterruptedSpeechMarkdown( + tc.speech.spokenText, + tc.speech.unspokenText, + ) : displayText; const speechLabel = speechStatus ? { diff --git a/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx b/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx index 40c32774a..ab0b53384 100644 --- a/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx +++ b/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx @@ -3,6 +3,26 @@ import { Volume2 } from "lucide-react"; import { cn } from "@/shared/lib/cn"; import type { VoiceSpeechStatus } from "@/shared/types/messages"; +function escapeHtml(text: string): string { + return text + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +export function formatInterruptedSpeechMarkdown( + spokenText: string, + unspokenText: string, +): string { + const struckBlocks = escapeHtml(unspokenText) + .split(/(\n\s*\n)/) + .map((part, index) => + index % 2 === 0 && part ? `${part}` : part, + ) + .join(""); + return `${spokenText}${struckBlocks}`; +} + export function VoiceSpeechStatusIndicator({ status, label, diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index 8939ca5be..107efb624 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -736,6 +736,36 @@ describe("MessageBubble", () => { expect(block?.querySelector("del")).not.toHaveTextContent("One. Two"); }); + it("strikes every paragraph after the estimated interruption cutoff", () => { + const { container } = render( + , + ); + + const block = container.querySelector( + '[data-voice-speech-status="interrupted"]', + ); + const struckBlocks = block?.querySelectorAll("del"); + expect(struckBlocks).toHaveLength(3); + expect(struckBlocks?.[0]).toHaveTextContent("Unheard first paragraph."); + expect(struckBlocks?.[1]).toHaveTextContent("Unheard second paragraph."); + expect(struckBlocks?.[2]).toHaveTextContent("Unheard third paragraph."); + expect(struckBlocks?.[0]).not.toHaveTextContent("Heard text."); + }); + it("preserves provider-error presentation after interrupted delivery", () => { const rawError = "Ran into this error: thinking blocks in the latest assistant message cannot be modified"; From 5f74c73b2826aa17f8f22083d46c05717ac06794 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 21:24:33 -0400 Subject: [PATCH 07/20] fix(voice): keep interruption estimates conservative --- src-tauri/native/siri_tts_bridge.m | 3 + src-tauri/src/commands/pocket_voice.rs | 28 ++++++- src-tauri/src/commands/siri_voice.rs | 1 + .../transcript/projection/messageRevisions.ts | 4 +- .../transcriptProjectionCache.test.ts | 9 +- src/features/chat/ui/AgentWorkPanel.tsx | 23 +++-- src/features/chat/ui/MessageBubble.tsx | 22 ++--- .../chat/ui/VoiceSpeechStatusIndicator.tsx | 20 ----- .../chat/ui/__tests__/MessageBubble.test.tsx | 69 +++++++++++---- .../voice-conversation/api/pocketVoice.ts | 1 + .../lib/nativeAssistantSpeech.test.ts | 72 +++++++++++++--- .../lib/nativeAssistantSpeech.ts | 83 +++++++++---------- src/shared/types/messages.ts | 7 +- src/shared/ui/ai-elements/message.tsx | 82 +++++++++++++++++- 14 files changed, 290 insertions(+), 134 deletions(-) diff --git a/src-tauri/native/siri_tts_bridge.m b/src-tauri/native/siri_tts_bridge.m index 66d24d608..09d1dc55c 100644 --- a/src-tauri/native/siri_tts_bridge.m +++ b/src-tauri/native/siri_tts_bridge.m @@ -277,6 +277,7 @@ - (void)dealloc { [self.connection invalidate]; } @interface BerdSiriDeliverySegment : NSObject @property(nonatomic, copy) NSString *text; @property(nonatomic, assign) uint64_t totalFrames; +@property(nonatomic, assign) BOOL synthesisComplete; @end @implementation BerdSiriDeliverySegment @@ -479,6 +480,7 @@ - (void)startNextSynthesis { [weakSelf finish:error]; return; } + if (!error) deliverySegment.synthesisComplete = YES; [weakSelf startNextSynthesis]; }); }]; @@ -526,6 +528,7 @@ - (NSString *)deliveryJSON { @"text": segment.text ?: @"", @"playedFrames": @(played), @"totalFrames": @(segment.totalFrames), + @"synthesisComplete": @(segment.synthesisComplete), }]; segmentStart += segment.totalFrames; } diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index 0f9cb1403..c2408b670 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -185,6 +185,7 @@ struct VoiceDeliverySegment { text: String, played_frames: u64, total_frames: u64, + synthesis_complete: bool, } #[cfg(any(test, target_os = "macos"))] @@ -196,14 +197,14 @@ struct VoiceDeliveryProgress { #[cfg(any(test, target_os = "macos"))] #[derive(Debug, Default)] struct PlaybackDeliveryLedger { - segments: Vec<(String, u64)>, + segments: Vec<(String, u64, bool)>, pieces: Vec, } #[cfg(any(test, target_os = "macos"))] impl PlaybackDeliveryLedger { fn begin_segment(&mut self, text: String) { - self.segments.push((text, 0)); + self.segments.push((text, 0, false)); } fn append_frames(&mut self, frames: usize) { @@ -211,12 +212,24 @@ impl PlaybackDeliveryLedger { if frames == 0 { return; } - if let Some((_, total)) = self.segments.last_mut() { + // The speed processor can retain a tail between text segments. The + // previous segment is final only once output for its successor arrives. + if self.segments.len() > 1 { + let previous = self.segments.len() - 2; + self.segments[previous].2 = true; + } + if let Some((_, total, _)) = self.segments.last_mut() { *total = total.saturating_add(frames); self.pieces.push(frames); } } + fn complete_segment(&mut self) { + if let Some((_, _, synthesis_complete)) = self.segments.last_mut() { + *synthesis_complete = true; + } + } + fn snapshot(&self, queued_pieces: usize, current_piece_frames: u64) -> VoiceDeliveryProgress { let completed_pieces = self.pieces.len().saturating_sub(queued_pieces); let completed_frames = self @@ -233,7 +246,7 @@ impl PlaybackDeliveryLedger { let segments = self .segments .iter() - .map(|(text, total_frames)| { + .map(|(text, total_frames, synthesis_complete)| { let played_frames = consumed_frames .saturating_sub(segment_start) .min(*total_frames); @@ -242,6 +255,7 @@ impl PlaybackDeliveryLedger { text: text.clone(), played_frames, total_frames: *total_frames, + synthesis_complete: *synthesis_complete, } }) .collect(); @@ -2041,6 +2055,7 @@ fn run_pocket_voice_stream( .map_err(|error| format!("signal Pocket playback start: {error}"))?; } } + delivery_ledger.complete_segment(); } Ok(PocketStreamCommand::Finish) => { if !synthesize_pocket_stream_ready( @@ -2079,6 +2094,7 @@ fn run_pocket_voice_stream( ); } } + delivery_ledger.complete_segment(); while !player.empty() { if !active.load(Ordering::SeqCst) { let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); @@ -2381,6 +2397,8 @@ mod tests { ledger.begin_segment("First sentence.".to_string()); ledger.append_frames(4_800); ledger.begin_segment("Second sentence.".to_string()); + let before_second_audio = ledger.snapshot(1, 0); + assert!(!before_second_audio.segments[0].synthesis_complete); ledger.append_frames(4_800); // One source has completed and the next is 50 ms in. The 100 ms @@ -2389,8 +2407,10 @@ mod tests { let progress = ledger.snapshot(1, 1_200); assert_eq!(progress.segments[0].played_frames, 3_600); assert_eq!(progress.segments[0].total_frames, 4_800); + assert!(progress.segments[0].synthesis_complete); assert_eq!(progress.segments[1].played_frames, 0); assert_eq!(progress.segments[1].total_frames, 4_800); + assert!(!progress.segments[1].synthesis_complete); } #[test] diff --git a/src-tauri/src/commands/siri_voice.rs b/src-tauri/src/commands/siri_voice.rs index d9223631f..050b1d243 100644 --- a/src-tauri/src/commands/siri_voice.rs +++ b/src-tauri/src/commands/siri_voice.rs @@ -79,6 +79,7 @@ struct VoiceDeliverySegment { text: String, played_frames: u64, total_frames: u64, + synthesis_complete: bool, } #[cfg(target_os = "macos")] diff --git a/src/features/chat/transcript/projection/messageRevisions.ts b/src/features/chat/transcript/projection/messageRevisions.ts index 17c79745d..7b7deeb7c 100644 --- a/src/features/chat/transcript/projection/messageRevisions.ts +++ b/src/features/chat/transcript/projection/messageRevisions.ts @@ -136,9 +136,9 @@ function speechRevision(content: TextContent): string { if (!speech) return ""; return [ speech.status, - speech.spokenText === undefined ? "" : textRevision(speech.spokenText), - speech.unspokenText === undefined ? "" : textRevision(speech.unspokenText), + speech.spokenThrough ?? "", speech.confidence ?? "", + speech.interruptionCause ?? "", ].join(":"); } diff --git a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts index 987059440..bd49e8582 100644 --- a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts +++ b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts @@ -1546,7 +1546,7 @@ describe("transcript projection cache", () => { "One. Two. Three.", utc(2026, 6, 4, 10), ); - const interrupted = (spokenText: string, unspokenText: string) => ({ + const interrupted = (spokenThrough: number) => ({ ...original, content: original.content.map((content) => content.type === "text" @@ -1554,8 +1554,7 @@ describe("transcript projection cache", () => { ...content, speech: { status: "interrupted" as const, - spokenText, - unspokenText, + spokenThrough, confidence: "medium" as const, }, } @@ -1563,8 +1562,8 @@ describe("transcript projection cache", () => { ), }); - const first = buildMessageRevisions(interrupted("One.", " Two. Three.")); - const second = buildMessageRevisions(interrupted("One. Two.", " Three.")); + const first = buildMessageRevisions(interrupted("One.".length)); + const second = buildMessageRevisions(interrupted("One. Two.".length)); expect(second.renderRevision).not.toBe(first.renderRevision); expect(second.heightRevision).not.toBe(first.heightRevision); diff --git a/src/features/chat/ui/AgentWorkPanel.tsx b/src/features/chat/ui/AgentWorkPanel.tsx index f2c2604e6..de66b9bcd 100644 --- a/src/features/chat/ui/AgentWorkPanel.tsx +++ b/src/features/chat/ui/AgentWorkPanel.tsx @@ -30,10 +30,7 @@ import type { import type { TranscriptAgentWorkPayload } from "@/features/chat/transcript/projection/transcriptItemTypes"; import { useTranscriptRowStateAdapter } from "@/features/chat/transcript/row-state"; import { ToolCallAdapter } from "./ToolCallAdapter"; -import { - formatInterruptedSpeechMarkdown, - VoiceSpeechStatusIndicator, -} from "./VoiceSpeechStatusIndicator"; +import { VoiceSpeechStatusIndicator } from "./VoiceSpeechStatusIndicator"; interface ToolTimelineItem { kind: "tool"; @@ -310,15 +307,13 @@ function AgentWorkItemRow({ if (item.kind === "progress") { const speechStatus = item.content.speech?.status; - const speechDisplayText = + const strikethroughFrom = speechStatus === "interrupted" && - item.content.speech?.spokenText !== undefined && - item.content.speech.unspokenText !== undefined - ? formatInterruptedSpeechMarkdown( - item.content.speech.spokenText, - item.content.speech.unspokenText, - ) - : item.content.text; + item.content.speech?.spokenThrough !== undefined + ? item.content.speech.spokenThrough + : speechStatus === "notSpoken" + ? 0 + : undefined; const speechLabel = speechStatus ? { speaking: t("message.voiceSpeechSpeakingLabel"), @@ -345,7 +340,9 @@ function AgentWorkItemRow({ label={speechLabel} /> ) : null} - {speechDisplayText} + + {item.content.text} + ); diff --git a/src/features/chat/ui/MessageBubble.tsx b/src/features/chat/ui/MessageBubble.tsx index 60b186bfa..1477a83b2 100644 --- a/src/features/chat/ui/MessageBubble.tsx +++ b/src/features/chat/ui/MessageBubble.tsx @@ -63,10 +63,7 @@ import { UserMessageClamp, } from "./UserMessageClamp"; import { ImageLightbox } from "@/shared/ui/ImageLightbox"; -import { - formatInterruptedSpeechMarkdown, - VoiceSpeechStatusIndicator, -} from "./VoiceSpeechStatusIndicator"; +import { VoiceSpeechStatusIndicator } from "./VoiceSpeechStatusIndicator"; interface MessageAttachmentPreviewItem { key: string; @@ -535,16 +532,14 @@ function renderContentBlock( options.resolveProviderErrorNotice?.(tc.text) ?? null; const displayText = providerErrorNotice ?? tc.text; const speechStatus = tc.speech?.status; - const speechDisplayText = + const strikethroughFrom = providerErrorNotice === null && speechStatus === "interrupted" && - tc.speech?.spokenText !== undefined && - tc.speech.unspokenText !== undefined - ? formatInterruptedSpeechMarkdown( - tc.speech.spokenText, - tc.speech.unspokenText, - ) - : displayText; + tc.speech?.spokenThrough !== undefined + ? tc.speech.spokenThrough + : providerErrorNotice === null && speechStatus === "notSpoken" + ? 0 + : undefined; const speechLabel = speechStatus ? { speaking: options.voiceSpeechSpeakingLabel, @@ -573,8 +568,9 @@ function renderContentBlock( options.onRunShellCommand ? options.runItCodeRenderers : undefined } imageRenderer={MarkdownImage} + strikethroughFrom={strikethroughFrom} > - {speechDisplayText} + {displayText} ); diff --git a/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx b/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx index ab0b53384..40c32774a 100644 --- a/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx +++ b/src/features/chat/ui/VoiceSpeechStatusIndicator.tsx @@ -3,26 +3,6 @@ import { Volume2 } from "lucide-react"; import { cn } from "@/shared/lib/cn"; import type { VoiceSpeechStatus } from "@/shared/types/messages"; -function escapeHtml(text: string): string { - return text - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">"); -} - -export function formatInterruptedSpeechMarkdown( - spokenText: string, - unspokenText: string, -): string { - const struckBlocks = escapeHtml(unspokenText) - .split(/(\n\s*\n)/) - .map((part, index) => - index % 2 === 0 && part ? `${part}` : part, - ) - .join(""); - return `${spokenText}${struckBlocks}`; -} - export function VoiceSpeechStatusIndicator({ status, label, diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index 107efb624..a65fae984 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -719,8 +719,7 @@ describe("MessageBubble", () => { text: "One. Two. Three.", speech: { status: "interrupted", - spokenText: "One. Two", - unspokenText: ". Three.", + spokenThrough: "One. Two".length, confidence: "medium", }, }, @@ -745,9 +744,7 @@ describe("MessageBubble", () => { text: "Heard text. Unheard first paragraph.\n\nUnheard second paragraph.\n\nUnheard third paragraph.", speech: { status: "interrupted", - spokenText: "Heard text.", - unspokenText: - " Unheard first paragraph.\n\nUnheard second paragraph.\n\nUnheard third paragraph.", + spokenThrough: "Heard text.".length, confidence: "medium", }, }, @@ -758,12 +755,58 @@ describe("MessageBubble", () => { const block = container.querySelector( '[data-voice-speech-status="interrupted"]', ); - const struckBlocks = block?.querySelectorAll("del"); - expect(struckBlocks).toHaveLength(3); - expect(struckBlocks?.[0]).toHaveTextContent("Unheard first paragraph."); - expect(struckBlocks?.[1]).toHaveTextContent("Unheard second paragraph."); - expect(struckBlocks?.[2]).toHaveTextContent("Unheard third paragraph."); - expect(struckBlocks?.[0]).not.toHaveTextContent("Heard text."); + const paragraphs = block?.querySelectorAll("p"); + expect(paragraphs).toHaveLength(3); + expect(paragraphs?.[0]?.querySelector("del")).toHaveTextContent( + "Unheard first paragraph.", + ); + expect(paragraphs?.[0]?.querySelector("del")).not.toHaveTextContent( + "Heard text.", + ); + expect(paragraphs?.[1]?.closest("del")).toBeTruthy(); + expect(paragraphs?.[2]?.closest("del")).toBeTruthy(); + }); + + it("preserves Markdown structure while striking the unspoken range", async () => { + const spoken = "Heard. "; + const text = `${spoken}**bold** [link](https://example.com)\n\n- list item\n\n\`inline\`\n\n\`\`\`ts\nconst value = 1;\n\`\`\``; + const { container } = render( + , + ); + + const block = container.querySelector( + '[data-voice-speech-status="interrupted"]', + ); + await waitFor(() => { + expect( + block?.querySelector('del [data-streamdown="strong"]'), + ).toHaveTextContent("bold"); + expect( + block?.querySelector('del a[href="https://example.com/"]'), + ).toHaveTextContent("link"); + expect(block?.querySelector("del li")).toHaveTextContent("list item"); + expect( + block?.querySelector('del [data-streamdown="inline-code"]'), + ).toHaveTextContent("inline"); + expect( + block?.querySelector('del [data-streamdown="code-block"]'), + ).toBeTruthy(); + expect(block?.querySelector("pre code")).toHaveTextContent( + "const value = 1;", + ); + }); }); it("preserves provider-error presentation after interrupted delivery", () => { @@ -777,9 +820,7 @@ describe("MessageBubble", () => { text: rawError, speech: { status: "interrupted", - spokenText: "Ran into this error:", - unspokenText: - " thinking blocks in the latest assistant message cannot be modified", + spokenThrough: "Ran into this error:".length, confidence: "medium", }, }, diff --git a/src/features/voice-conversation/api/pocketVoice.ts b/src/features/voice-conversation/api/pocketVoice.ts index 342722658..6f4d0999c 100644 --- a/src/features/voice-conversation/api/pocketVoice.ts +++ b/src/features/voice-conversation/api/pocketVoice.ts @@ -45,6 +45,7 @@ export interface VoiceDeliverySegment { text: string; playedFrames: number; totalFrames: number; + synthesisComplete: boolean; } export interface VoiceDeliveryProgress { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index c4405dfee..2fc5ac1f0 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -519,8 +519,7 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted", - spokenText: "", - unspokenText: "Queued reply.", + spokenThrough: 0, confidence: "low", }, }); @@ -550,7 +549,12 @@ describe("native assistant speech stream", () => { error: null, delivery: { segments: [ - { text: "Native reply.", playedFrames: 400, totalFrames: 1_000 }, + { + text: "Native reply.", + playedFrames: 400, + totalFrames: 1_000, + synthesisComplete: true, + }, ], }, }); @@ -607,6 +611,7 @@ describe("native assistant speech stream", () => { text: "One. Two. Three.", playedFrames: 300, totalFrames: 1_000, + synthesisComplete: true, }, ], }, @@ -627,6 +632,7 @@ describe("native assistant speech stream", () => { text: "One. Two. Three.", playedFrames: 600, totalFrames: 1_000, + synthesisComplete: true, }, ], }, @@ -637,16 +643,10 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted", - spokenText: "One. Two", - unspokenText: ". Three.", + spokenThrough: "One. Two".length, confidence: "medium", }, }); - const notice = takeVoicePlaybackNotices("session-1"); - expect(notice).toContain('"spokenText":"One. Two"'); - expect(notice).toContain('"unspokenText":". Three."'); - expect(notice).toContain('"confidence":"medium"'); - useVoiceConversationStore.setState({ userSpeaking: false }); useChatStore .getState() @@ -657,12 +657,51 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted", - spokenText: "One. Two", - unspokenText: ". Three. Four.", + spokenThrough: "One. Two".length, }, }); }); expect(mocks.append).toHaveBeenCalledTimes(1); + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice.match(/\[voice: tts-delivery-failed\]/g)).toHaveLength(1); + expect(notice).toContain('"spokenText":"One. Two"'); + expect(notice).toContain('"unspokenText":". Three. Four."'); + expect(notice).toContain('"confidence":"medium"'); + }); + + it("does not treat generated-but-incomplete audio as fully spoken", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "One. Two. Three." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "One. Two. Three.", + playedFrames: 1_000, + totalFrames: 1_000, + synthesisComplete: false, + }, + ], + }, + }); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { status: "interrupted", spokenThrough: 0 }, + }); }); it("bounds spoken and unspoken excerpts in the model delivery notice", async () => { @@ -682,7 +721,14 @@ describe("native assistant speech stream", () => { state: "interrupted", error: null, delivery: { - segments: [{ text, playedFrames: 1_000, totalFrames: 2_000 }], + segments: [ + { + text, + playedFrames: 1_000, + totalFrames: 2_000, + synthesisComplete: true, + }, + ], }, }); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 4a9451492..537cb2835 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -69,8 +69,7 @@ let activeSpeechRevision: number | null = null; let activeUtterance: ActiveUtterance | null = null; let stopActiveVoice: () => Promise = stopPocketVoice; let activityReportQueue = Promise.resolve(); -const pendingNotices = new Map(); -const recordedNoticeKeys = new Set(); +const pendingNotices = new Map>(); const DELIVERY_NOTICE_TEXT_LIMIT = 250; function boundedDeliveryText( @@ -124,8 +123,6 @@ function recordPlaybackNotice( interruptionCause: InterruptionCause = "voiceStopped", ) { const noticeKey = `${sessionId}\0${key}\0${status}`; - if (recordedNoticeKeys.has(noticeKey)) return; - recordedNoticeKeys.add(noticeKey); const excerpt = text.length > 500 ? `${text.slice(0, 497).trimEnd()}…` : text; const outcome = status === "interrupted" @@ -152,16 +149,15 @@ function recordPlaybackNotice( const notice = `[voice: tts-delivery-failed]\n${outcome}\nOriginal text: ${excerpt}${estimateLine}\n` + "This is TTS delivery state, not live user voice input. Do not respond to this control message or repeat the reply unless re-delivery is still appropriate."; - pendingNotices.set(sessionId, [ - ...(pendingNotices.get(sessionId) ?? []), - notice, - ]); + const notices = pendingNotices.get(sessionId) ?? new Map(); + notices.set(noticeKey, notice); + pendingNotices.set(sessionId, notices); } export function takeVoicePlaybackNotices(sessionId: string): string | null { const notices = pendingNotices.get(sessionId); pendingNotices.delete(sessionId); - return notices?.join("\n") ?? null; + return notices ? [...notices.values()].join("\n") : null; } function targetKey(target: SpeechTarget): string { @@ -209,6 +205,9 @@ function estimateSpeechDelivery( Math.min(totalFrames, segment.playedFrames), ); if (totalFrames === 0 || playedFrames === 0) break; + if (!segment.synthesisComplete) { + break; + } if (playedFrames >= totalFrames) { cutoff = segmentStart + segment.text.length; searchFrom = cutoff; @@ -228,23 +227,6 @@ function estimateSpeechDelivery( }; } -function targetText(sessionId: string, target: SpeechTarget): string { - const message = - useChatStore - .getState() - .messagesBySession[sessionId]?.find( - (candidate) => candidate.id === target.messageId, - ) ?? null; - if (!message) return ""; - let textOrdinal = 0; - for (const content of message.content) { - if (content.type !== "text") continue; - if (textOrdinal === target.textOrdinal) return content.text; - textOrdinal += 1; - } - return ""; -} - function applyInterruptionEstimate( utterance: ActiveUtterance, estimate: SpeechDeliveryEstimate, @@ -258,7 +240,6 @@ function applyInterruptionEstimate( ); const start = spans.at(0)?.start ?? 0; const end = spans.at(-1)?.end ?? start; - const text = targetText(utterance.sessionId, target); if (estimate.cutoff >= end && end > start) { setTargetSpeech(utterance.sessionId, target, { status: "spoken" }); continue; @@ -267,9 +248,9 @@ function applyInterruptionEstimate( if (targetKey(target) === firstTargetKey) { setTargetSpeech(utterance.sessionId, target, { status: "interrupted", - spokenText: "", - unspokenText: text, + spokenThrough: 0, confidence: estimate.confidence, + interruptionCause: utterance.interruptionCause ?? "voiceStopped", }); continue; } @@ -278,13 +259,13 @@ function applyInterruptionEstimate( } const localCutoff = Math.max( 0, - Math.min(text.length, estimate.cutoff - start), + Math.min(end - start, estimate.cutoff - start), ); setTargetSpeech(utterance.sessionId, target, { status: "interrupted", - spokenText: text.slice(0, localCutoff), - unspokenText: text.slice(localCutoff), + spokenThrough: localCutoff, confidence: estimate.confidence, + interruptionCause: utterance.interruptionCause ?? "voiceStopped", }); } } @@ -395,7 +376,7 @@ function handleStreamEvent( utterance.onInterrupted(); recordPlaybackNotice( utterance.sessionId, - utterance.id, + utterance.targets[0] ? targetKey(utterance.targets[0]) : utterance.id, utterance.text, "interrupted", estimate, @@ -458,7 +439,7 @@ function interruptActiveUtterance( utterance.onInterrupted(); recordPlaybackNotice( utterance.sessionId, - utterance.id, + utterance.targets[0] ? targetKey(utterance.targets[0]) : utterance.id, utterance.text, "interrupted", estimate, @@ -542,6 +523,7 @@ export function startNativeAssistantSpeech( const consumedTextBySlot = new Map(); const completedMessages = new Set(); const interruptedMessages = new Set(); + const interruptionCauseByMessage = new Map(); for (const message of initialMessages) { toolCountByMessage.set( message.id, @@ -597,6 +579,10 @@ export function startNativeAssistantSpeech( onInterrupted: () => { for (const utteranceTarget of utterance.targets) { interruptedMessages.add(utteranceTarget.messageId); + interruptionCauseByMessage.set( + utteranceTarget.messageId, + utterance.interruptionCause ?? "voiceStopped", + ); } }, onTerminal: () => queueMicrotask(inspect), @@ -663,18 +649,27 @@ export function startNativeAssistantSpeech( if (interruptedMessages.has(message.id)) { const currentSpeech = content.speech; - if ( - currentSpeech?.status === "interrupted" && - currentSpeech.spokenText !== undefined - ) { - setTargetSpeech(sessionId, target, { - ...currentSpeech, - unspokenText: content.text.slice(currentSpeech.spokenText.length), - }); - } else { + const interruptionCause = + currentSpeech?.interruptionCause ?? + interruptionCauseByMessage.get(message.id) ?? + "voiceStopped"; + if (currentSpeech?.status !== "interrupted") { setTargetSpeech(sessionId, target, { status: "notSpoken" }); } - recordPlaybackNotice(sessionId, slot, content.text, "notSpoken"); + const spokenThrough = currentSpeech?.spokenThrough ?? 0; + recordPlaybackNotice( + sessionId, + slot, + content.text, + "interrupted", + { + cutoff: spokenThrough, + spokenText: content.text.slice(0, spokenThrough), + unspokenText: content.text.slice(spokenThrough), + confidence: currentSpeech?.confidence ?? "low", + }, + interruptionCause, + ); continue; } diff --git a/src/shared/types/messages.ts b/src/shared/types/messages.ts index e83922119..b0aedb915 100644 --- a/src/shared/types/messages.ts +++ b/src/shared/types/messages.ts @@ -30,11 +30,10 @@ export type VoiceSpeechStatus = export interface VoiceSpeechState { status: VoiceSpeechStatus; - /** Ephemeral playback estimate; never serialized into ACP history. */ - spokenText?: string; - /** Ephemeral playback estimate; never serialized into ACP history. */ - unspokenText?: string; + /** Ephemeral source-text cutoff for completed speech; never serialized. */ + spokenThrough?: number; confidence?: "low" | "medium"; + interruptionCause?: "userSpeaking" | "voiceStopped"; } /** ACP TextContent with discriminator and local voice playback state. */ diff --git a/src/shared/ui/ai-elements/message.tsx b/src/shared/ui/ai-elements/message.tsx index 2349fc5aa..8243fb70a 100644 --- a/src/shared/ui/ai-elements/message.tsx +++ b/src/shared/ui/ai-elements/message.tsx @@ -341,6 +341,8 @@ export const MessageBranchPage = ({ export type MessageResponseProps = ComponentProps & { codeRenderers?: CustomRenderer[]; + /** Source-text offset after which rendered Markdown is struck through. */ + strikethroughFrom?: number; /** * Optional feature-aware Markdown image renderer. Chat injects one that can * resolve local files through the asset scheme; when omitted, images render @@ -599,6 +601,13 @@ function isReservedBerdSessionLinkPrefix(href: string | undefined): boolean { type MarkdownHastNode = { children?: MarkdownHastNode[]; properties?: Record; + position?: { + start: { offset?: number }; + end: { offset?: number }; + }; + tagName?: string; + type?: string; + value?: string; }; function hasControlCharacter(value: string): boolean { @@ -694,6 +703,62 @@ const berdRehypePlugins: NonNullable< restoreBerdMarkdownDestinations, ]; +function strikethroughFromPlugin(cutoff: number) { + const wrap = (node: MarkdownHastNode): MarkdownHastNode => ({ + type: "element", + tagName: "del", + properties: {}, + children: [node], + position: node.position, + }); + + const decorate = (node: MarkdownHastNode) => { + if (!node.children) return; + const children: MarkdownHastNode[] = []; + for (const child of node.children) { + const start = child.position?.start.offset; + const end = child.position?.end.offset; + if (child.type === "element" && start !== undefined && cutoff <= start) { + children.push(wrap(child)); + continue; + } + if ( + child.type === "text" && + child.value !== undefined && + start !== undefined && + end !== undefined + ) { + if (cutoff <= start) { + children.push(wrap(child)); + continue; + } + if (cutoff < end) { + const sourceLength = Math.max(1, end - start); + const valueOffset = Math.max( + 0, + Math.min( + child.value.length, + Math.round( + ((cutoff - start) / sourceLength) * child.value.length, + ), + ), + ); + const spoken = child.value.slice(0, valueOffset); + const unspoken = child.value.slice(valueOffset); + if (spoken) children.push({ ...child, value: spoken }); + if (unspoken) children.push(wrap({ ...child, value: unspoken })); + continue; + } + } + decorate(child); + children.push(child); + } + node.children = children; + }; + + return (tree: MarkdownHastNode) => decorate(tree); +} + const linkSafetyConfig: ComponentProps["linkSafety"] = { enabled: false, }; @@ -708,6 +773,7 @@ export const MessageResponse = memo( mode, onAnimationEnd, onAnimationStart, + strikethroughFrom, ...props }: MessageResponseProps) => { const { t } = useTranslation("common"); @@ -716,6 +782,18 @@ export const MessageResponse = memo( () => buildStreamdownComponents(imageRenderer), [imageRenderer], ); + const rehypePlugins = useMemo< + NonNullable["rehypePlugins"]> + >( + () => + strikethroughFrom === undefined + ? berdRehypePlugins + : [ + ...berdRehypePlugins, + [strikethroughFromPlugin, strikethroughFrom], + ], + [strikethroughFrom], + ); const streamdownRootRef = useRef(null); const streamdownLayoutPending = useVirtualLayoutPendingForStreamdown({ contentKey: children, @@ -780,10 +858,10 @@ export const MessageResponse = memo( components={streamdownComponents} isAnimating={isAnimating} linkSafety={linkSafetyConfig} - mode={mode} + mode={strikethroughFrom === undefined ? mode : "static"} onAnimationEnd={streamdownLayoutPending.onAnimationEnd} onAnimationStart={streamdownLayoutPending.onAnimationStart} - rehypePlugins={berdRehypePlugins} + rehypePlugins={rehypePlugins} plugins={ codeRenderers ? { ...streamdownPlugins, renderers: codeRenderers } From bccfd24c40ac8cda9fa8592d7299469e6cc305be Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 21:33:33 -0400 Subject: [PATCH 08/20] fix(voice): preserve interruption boundaries --- .../src/commands/pocket_playback_speed_dsp.rs | 4 ++ src-tauri/src/commands/pocket_voice.rs | 31 ++++----- .../chat/ui/__tests__/MessageBubble.test.tsx | 66 +++++++++++++++++++ .../lib/nativeAssistantSpeech.test.ts | 49 ++++++++++++++ .../lib/nativeAssistantSpeech.ts | 20 +++++- src/shared/ui/ai-elements/message.tsx | 21 +++++- 6 files changed, 172 insertions(+), 19 deletions(-) diff --git a/src-tauri/src/commands/pocket_playback_speed_dsp.rs b/src-tauri/src/commands/pocket_playback_speed_dsp.rs index 221768342..25cfb90af 100644 --- a/src-tauri/src/commands/pocket_playback_speed_dsp.rs +++ b/src-tauri/src/commands/pocket_playback_speed_dsp.rs @@ -91,6 +91,10 @@ impl StreamingSpeedProcessor { Ok(self.trim_and_count(output[0].as_slice())) } + pub(super) fn expected_output_frames(&self) -> usize { + stretched_len(self.total_input, self.speed) + } + pub(super) fn finish(&mut self) -> Result, String> { if self.stretch.is_none() { return Ok(Vec::new()); diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index c2408b670..c90fa362e 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -212,20 +212,17 @@ impl PlaybackDeliveryLedger { if frames == 0 { return; } - // The speed processor can retain a tail between text segments. The - // previous segment is final only once output for its successor arrives. - if self.segments.len() > 1 { - let previous = self.segments.len() - 2; - self.segments[previous].2 = true; - } - if let Some((_, total, _)) = self.segments.last_mut() { - *total = total.saturating_add(frames); + if let Some((_, total, synthesis_complete)) = self.segments.last_mut() { + if !*synthesis_complete { + *total = total.saturating_add(frames); + } self.pieces.push(frames); } } - fn complete_segment(&mut self) { - if let Some((_, _, synthesis_complete)) = self.segments.last_mut() { + fn complete_segment(&mut self, final_total_frames: u64) { + if let Some((_, total, synthesis_complete)) = self.segments.last_mut() { + *total = (*total).max(final_total_frames); *synthesis_complete = true; } } @@ -2055,7 +2052,6 @@ fn run_pocket_voice_stream( .map_err(|error| format!("signal Pocket playback start: {error}"))?; } } - delivery_ledger.complete_segment(); } Ok(PocketStreamCommand::Finish) => { if !synthesize_pocket_stream_ready( @@ -2094,7 +2090,6 @@ fn run_pocket_voice_stream( ); } } - delivery_ledger.complete_segment(); while !player.empty() { if !active.load(Ordering::SeqCst) { let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); @@ -2181,6 +2176,7 @@ fn synthesize_pocket_stream_ready( } let text = text.trim().to_string(); delivery_ledger.begin_segment(text.clone()); + let output_start = speed_processor.expected_output_frames(); let mut callback_error = None; let completed = engine.synth_chunk_streaming(&text, style, STREAMING_EMIT_FRAMES, &mut |samples| { @@ -2237,6 +2233,10 @@ fn synthesize_pocket_stream_ready( player.stop(); return Ok(false); } + let final_total_frames = speed_processor + .expected_output_frames() + .saturating_sub(output_start) as u64; + delivery_ledger.complete_segment(final_total_frames); } Ok(true) } @@ -2396,10 +2396,11 @@ mod tests { let mut ledger = PlaybackDeliveryLedger::default(); ledger.begin_segment("First sentence.".to_string()); ledger.append_frames(4_800); + assert!(!ledger.snapshot(1, 0).segments[0].synthesis_complete); + ledger.complete_segment(4_800); ledger.begin_segment("Second sentence.".to_string()); - let before_second_audio = ledger.snapshot(1, 0); - assert!(!before_second_audio.segments[0].synthesis_complete); ledger.append_frames(4_800); + ledger.complete_segment(4_800); // One source has completed and the next is 50 ms in. The 100 ms // output-latency allowance leaves 3,600 safely delivered frames in @@ -2410,7 +2411,7 @@ mod tests { assert!(progress.segments[0].synthesis_complete); assert_eq!(progress.segments[1].played_frames, 0); assert_eq!(progress.segments[1].total_frames, 4_800); - assert!(!progress.segments[1].synthesis_complete); + assert!(progress.segments[1].synthesis_complete); } #[test] diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index a65fae984..4c5fbf772 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -735,6 +735,72 @@ describe("MessageBubble", () => { expect(block?.querySelector("del")).not.toHaveTextContent("One. Two"); }); + it("updates the strike when unchanged text becomes interrupted", () => { + const text = "One. Two. Three."; + const { container, rerender } = render( + , + ); + expect(container.querySelector("del")).toBeNull(); + + rerender( + , + ); + + expect(container.querySelector("del")).toHaveTextContent(". Three."); + }); + + it("keeps required list and table children structurally valid", () => { + const text = + "- Heard item\n- Unheard item\n\n| Name |\n| --- |\n| Heard |\n| Unheard |"; + const { container } = render( + , + ); + + const list = container.querySelector("ul"); + expect([...list!.children].every((child) => child.tagName === "LI")).toBe( + true, + ); + expect(list?.children[1]?.querySelector("del")).toHaveTextContent( + "Unheard item", + ); + const table = container.querySelector("table"); + expect(table).toBeInTheDocument(); + expect(table?.querySelector("tbody")?.parentElement).toBe(table); + expect( + [...(table?.querySelector("tbody")?.children ?? [])].every( + (child) => child.tagName === "TR", + ), + ).toBe(true); + }); + it("strikes every paragraph after the estimated interruption cutoff", () => { const { container } = render( { }); }); + it("preserves a fully spoken prefix when text arrives after interruption", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "One. Two." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "One. Two.", + playedFrames: 1_000, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); + + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .appendStreamingText("session-1", "assistant-1", " Three."); + await vi.waitFor(() => { + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { + status: "interrupted", + spokenThrough: "One. Two.".length, + }, + }); + }); + + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice.match(/\[voice: tts-delivery-failed\]/g)).toHaveLength(1); + expect(notice).toContain('"spokenText":"One. Two."'); + expect(notice).toContain('"unspokenText":" Three."'); + }); + it("bounds spoken and unspoken excerpts in the model delivery notice", async () => { const text = `${"spoken ".repeat(100)}${"unspoken ".repeat(100)}`; startNativeAssistantSpeech("session-1", vi.fn()); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 537cb2835..f364c0a40 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -241,7 +241,10 @@ function applyInterruptionEstimate( const start = spans.at(0)?.start ?? 0; const end = spans.at(-1)?.end ?? start; if (estimate.cutoff >= end && end > start) { - setTargetSpeech(utterance.sessionId, target, { status: "spoken" }); + setTargetSpeech(utterance.sessionId, target, { + status: "spoken", + spokenThrough: end - start, + }); continue; } if (estimate.cutoff <= start) { @@ -653,10 +656,21 @@ export function startNativeAssistantSpeech( currentSpeech?.interruptionCause ?? interruptionCauseByMessage.get(message.id) ?? "voiceStopped"; + const spokenThrough = currentSpeech?.spokenThrough ?? 0; if (currentSpeech?.status !== "interrupted") { - setTargetSpeech(sessionId, target, { status: "notSpoken" }); + setTargetSpeech( + sessionId, + target, + spokenThrough > 0 + ? { + status: "interrupted", + spokenThrough, + confidence: currentSpeech?.confidence ?? "medium", + interruptionCause, + } + : { status: "notSpoken" }, + ); } - const spokenThrough = currentSpeech?.spokenThrough ?? 0; recordPlaybackNotice( sessionId, slot, diff --git a/src/shared/ui/ai-elements/message.tsx b/src/shared/ui/ai-elements/message.tsx index 8243fb70a..4a33f2376 100644 --- a/src/shared/ui/ai-elements/message.tsx +++ b/src/shared/ui/ai-elements/message.tsx @@ -704,6 +704,18 @@ const berdRehypePlugins: NonNullable< ]; function strikethroughFromPlugin(cutoff: number) { + const structureParents = new Set([ + "dl", + "menu", + "ol", + "select", + "table", + "tbody", + "tfoot", + "thead", + "tr", + "ul", + ]); const wrap = (node: MarkdownHastNode): MarkdownHastNode => ({ type: "element", tagName: "del", @@ -718,7 +730,12 @@ function strikethroughFromPlugin(cutoff: number) { for (const child of node.children) { const start = child.position?.start.offset; const end = child.position?.end.offset; - if (child.type === "element" && start !== undefined && cutoff <= start) { + if ( + child.type === "element" && + start !== undefined && + cutoff <= start && + !structureParents.has(node.tagName ?? "") + ) { children.push(wrap(child)); continue; } @@ -851,6 +868,7 @@ export const MessageResponse = memo( {...streamdownLayoutPending.layoutPendingAttributes} > *:first-child]:mt-0 [&>*:last-child]:mb-0", className, @@ -887,6 +905,7 @@ export const MessageResponse = memo( prevProps.children === nextProps.children && nextProps.isAnimating === prevProps.isAnimating && nextProps.mode === prevProps.mode && + nextProps.strikethroughFrom === prevProps.strikethroughFrom && nextProps.codeRenderers === prevProps.codeRenderers, ); From 905050e05450595c9fa1571898d9264b278c53f8 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 21:34:16 -0400 Subject: [PATCH 09/20] test(voice): avoid unsafe list assertion --- src/features/chat/ui/__tests__/MessageBubble.test.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index 4c5fbf772..cb784b3ae 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -785,9 +785,10 @@ describe("MessageBubble", () => { ); const list = container.querySelector("ul"); - expect([...list!.children].every((child) => child.tagName === "LI")).toBe( - true, - ); + expect(list).toBeInTheDocument(); + expect( + [...(list?.children ?? [])].every((child) => child.tagName === "LI"), + ).toBe(true); expect(list?.children[1]?.querySelector("del")).toHaveTextContent( "Unheard item", ); From 7fb78912dd5f4d330657176be444331d97185bb0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 21:40:26 -0400 Subject: [PATCH 10/20] fix(voice): keep delivery notices coherent --- .../lib/nativeAssistantSpeech.test.ts | 64 +++++++++++++++++ .../lib/nativeAssistantSpeech.ts | 72 ++++++++++++++++--- 2 files changed, 125 insertions(+), 11 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 3b0a77853..42ea72281 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -97,6 +97,7 @@ function emit( describe("native assistant speech stream", () => { beforeEach(() => { + takeVoicePlaybackNotices("session-1"); mocks.backend = "pocket"; mocks.start.mockReset().mockResolvedValue(); mocks.append.mockReset().mockResolvedValue(); @@ -422,6 +423,69 @@ describe("native assistant speech stream", () => { expect(content?.[2]).toMatchObject({ speech: { status: "spoken" } }); }); + it("replaces the interrupted tool-suffix notice when more text arrives", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore.getState().setMessages("session-1", [ + assistant([ + { type: "text", text: "Before the tool." }, + { + type: "toolRequest", + id: "tool-1", + name: "Read", + arguments: {}, + status: "completed", + }, + { type: "text", text: "After the tool." }, + ]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalledTimes(2)); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + emit("started"); + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "Before the tool.", + playedFrames: 1_000, + totalFrames: 1_000, + synthesisComplete: true, + }, + { + text: "After the tool.", + playedFrames: 500, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); + + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .appendStreamingText("session-1", "assistant-1", " More."); + await vi.waitFor(() => { + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[2], + ).toMatchObject({ + text: "After the tool. More.", + speech: { status: "interrupted", spokenThrough: "After".length }, + }); + }); + + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice.match(/\[voice: tts-delivery-failed\]/g)).toHaveLength(1); + expect(notice).toContain('"spokenText":"After"'); + expect(notice).toContain('"unspokenText":" the tool. More."'); + expect(notice).not.toContain("Before the tool."); + }); + it("queues the next reply until the finishing stream completes", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index f364c0a40..439293adb 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -1,5 +1,5 @@ import { useChatStore } from "@/features/chat/stores/chatStore"; -import type { VoiceSpeechState } from "@/shared/types/messages"; +import type { TextContent, VoiceSpeechState } from "@/shared/types/messages"; import { appendPocketVoiceStream, finishPocketVoiceStream, @@ -273,6 +273,62 @@ function applyInterruptionEstimate( } } +function targetContent( + sessionId: string, + target: SpeechTarget, +): TextContent | null { + const message = useChatStore + .getState() + .messagesBySession[sessionId]?.find( + (candidate) => candidate.id === target.messageId, + ); + if (!message) return null; + let textOrdinal = 0; + for (const content of message.content) { + if (content.type !== "text") continue; + if (textOrdinal === target.textOrdinal) return content; + textOrdinal += 1; + } + return null; +} + +function recordInterruptionNotices( + utterance: ActiveUtterance, + fallbackEstimate: SpeechDeliveryEstimate, + cause: InterruptionCause, +) { + let recorded = false; + for (const target of utterance.targets) { + const content = targetContent(utterance.sessionId, target); + if (!content || content.speech?.status === "spoken") continue; + const spokenThrough = content.speech?.spokenThrough ?? 0; + recordPlaybackNotice( + utterance.sessionId, + targetKey(target), + content.text, + "interrupted", + { + cutoff: spokenThrough, + spokenText: content.text.slice(0, spokenThrough), + unspokenText: content.text.slice(spokenThrough), + confidence: content.speech?.confidence ?? fallbackEstimate.confidence, + }, + cause, + ); + recorded = true; + } + if (!recorded && utterance.targets.length === 0) { + recordPlaybackNotice( + utterance.sessionId, + utterance.id, + utterance.text, + "interrupted", + fallbackEstimate, + cause, + ); + } +} + function setTargetSpeech( sessionId: string, target: SpeechTarget, @@ -377,11 +433,8 @@ function handleStreamEvent( ); applyInterruptionEstimate(utterance, estimate); utterance.onInterrupted(); - recordPlaybackNotice( - utterance.sessionId, - utterance.targets[0] ? targetKey(utterance.targets[0]) : utterance.id, - utterance.text, - "interrupted", + recordInterruptionNotices( + utterance, estimate, utterance.interruptionCause ?? "voiceStopped", ); @@ -440,11 +493,8 @@ function interruptActiveUtterance( ); applyInterruptionEstimate(utterance, estimate); utterance.onInterrupted(); - recordPlaybackNotice( - utterance.sessionId, - utterance.targets[0] ? targetKey(utterance.targets[0]) : utterance.id, - utterance.text, - "interrupted", + recordInterruptionNotices( + utterance, estimate, utterance.interruptionCause ?? cause, ); From 8f0f5a0455ac30387705014f4db5f755c7784a32 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:25:08 -0400 Subject: [PATCH 11/20] fix(voice): snapshot Pocket delivery before stopping --- src-tauri/src/commands/pocket_voice.rs | 57 +++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index c90fa362e..c6f873058 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -2004,9 +2004,13 @@ fn run_pocket_voice_stream( &mut last_progress_emit, false, )? { + let delivery = capture_before_stop( + || pocket_delivery_snapshot(&delivery_ledger, &player), + || player.stop(), + ); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + delivery: Some(delivery), }); } } @@ -2028,9 +2032,13 @@ fn run_pocket_voice_stream( &mut last_progress_emit, true, )? { + let delivery = capture_before_stop( + || pocket_delivery_snapshot(&delivery_ledger, &player), + || player.stop(), + ); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + delivery: Some(delivery), }); } let tail = speed_processor.drain_and_reset()?; @@ -2071,9 +2079,13 @@ fn run_pocket_voice_stream( &mut last_progress_emit, true, )? { + let delivery = capture_before_stop( + || pocket_delivery_snapshot(&delivery_ledger, &player), + || player.stop(), + ); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + delivery: Some(delivery), }); } let tail = speed_processor.finish()?; @@ -2147,6 +2159,16 @@ fn pocket_delivery_snapshot( ledger.snapshot(queued_pieces, current_piece_frames) } +#[cfg(any(test, target_os = "macos"))] +fn capture_before_stop( + snapshot: impl FnOnce() -> VoiceDeliveryProgress, + stop: impl FnOnce(), +) -> VoiceDeliveryProgress { + let delivery = snapshot(); + stop(); + delivery +} + #[cfg(target_os = "macos")] #[allow(clippy::too_many_arguments)] fn synthesize_pocket_stream_ready( @@ -2171,7 +2193,6 @@ fn synthesize_pocket_stream_ready( *first_chunk_pending = split.first_chunk_pending; for text in split.ready { if !active.load(Ordering::SeqCst) { - player.stop(); return Ok(false); } let text = text.trim().to_string(); @@ -2230,7 +2251,6 @@ fn synthesize_pocket_stream_ready( return Err(error); } if !completed { - player.stop(); return Ok(false); } let final_total_frames = speed_processor @@ -2414,6 +2434,33 @@ mod tests { assert!(progress.segments[1].synthesis_complete); } + #[test] + fn cancellation_captures_delivery_before_stopping_playback() { + use std::cell::RefCell; + + let mut ledger = PlaybackDeliveryLedger::default(); + ledger.begin_segment("Played piece.".to_string()); + ledger.append_frames(4_800); + ledger.complete_segment(4_800); + ledger.begin_segment("Queued audio.".to_string()); + ledger.append_frames(4_800); + ledger.append_frames(4_800); + ledger.complete_segment(9_600); + let calls = RefCell::new(Vec::new()); + + let delivery = capture_before_stop( + || { + calls.borrow_mut().push("snapshot"); + ledger.snapshot(2, 1_200) + }, + || calls.borrow_mut().push("stop"), + ); + + assert_eq!(&*calls.borrow(), &["snapshot", "stop"]); + assert_eq!(delivery.segments[0].played_frames, 3_600); + assert_eq!(delivery.segments[1].played_frames, 0); + } + #[test] fn window_destroy_cancels_active_pocket_playback() { let state = PocketVoiceState::default(); From 7c3179c2a82f227ff343c7d7397e727be0ad05b6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:26:45 -0400 Subject: [PATCH 12/20] fix(voice): bound interruption finalization --- .../lib/nativeAssistantSpeech.test.ts | 121 ++++++++++++++++++ .../lib/nativeAssistantSpeech.ts | 94 +++++++++----- 2 files changed, 181 insertions(+), 34 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 42ea72281..f87ea59f1 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -627,6 +627,127 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted" } }); }); + it.each([ + ["returns false", () => mocks.stop.mockResolvedValue(false)], + ["rejects", () => mocks.stop.mockRejectedValue(new Error("stop failed"))], + ])("finalizes immediately when native stop %s", async (_label, setStop) => { + setStop(); + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "First reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => { + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { status: "interrupted", spokenThrough: 0 }, + }); + }); + + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .setMessages("session-1", [ + assistant( + [{ type: "text", text: "First reply." }], + "completed", + "assistant-1", + ), + assistant( + [{ type: "text", text: "Second reply." }], + "completed", + "assistant-2", + ), + ]); + await vi.waitFor(() => expect(mocks.start).toHaveBeenCalledTimes(2)); + }); + + it("bounds a missing native terminal event and allows the next reply", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "First reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + + vi.useFakeTimers(); + try { + useVoiceConversationStore.setState({ userSpeaking: true }); + await Promise.resolve(); + expect(mocks.stop).toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1_000); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { status: "interrupted", spokenThrough: 0 }, + }); + } finally { + vi.useRealTimers(); + } + + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .setMessages("session-1", [ + assistant( + [{ type: "text", text: "First reply." }], + "completed", + "assistant-1", + ), + assistant( + [{ type: "text", text: "Second reply." }], + "completed", + "assistant-2", + ), + ]); + await vi.waitFor(() => expect(mocks.start).toHaveBeenCalledTimes(2)); + }); + + it("finalizes only once when a terminal event races the fallback", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "First reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + const stopCallsBeforeInterruption = mocks.stop.mock.calls.length; + + vi.useFakeTimers(); + try { + useVoiceConversationStore.setState({ userSpeaking: true }); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "First reply.", + playedFrames: 500, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); + await vi.advanceTimersByTimeAsync(1_000); + } finally { + vi.useRealTimers(); + } + + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice.match(/\[voice: tts-delivery-failed\]/g)).toHaveLength(1); + expect(mocks.stop).toHaveBeenCalledTimes(stopCallsBeforeInterruption + 1); + }); + it("describes a hang-up as stopping the voice conversation", async () => { takeVoicePlaybackNotices("session-1"); startNativeAssistantSpeech("session-1", vi.fn()); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 439293adb..e06fef6fb 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -43,6 +43,7 @@ type ActiveUtterance = { finishing: boolean; nativeStreamStarted: boolean; interruptionRequested: boolean; + interruptionFallback: ReturnType | null; interruptionCause: InterruptionCause | null; latestDelivery: VoiceDeliveryProgress | null; status: SpeechStatus | null; @@ -71,6 +72,7 @@ let stopActiveVoice: () => Promise = stopPocketVoice; let activityReportQueue = Promise.resolve(); const pendingNotices = new Map>(); const DELIVERY_NOTICE_TEXT_LIMIT = 250; +const INTERRUPTION_TERMINAL_TIMEOUT_MS = 1_000; function boundedDeliveryText( text: string, @@ -364,6 +366,10 @@ function failActiveUtterance( ) { const utterance = activeUtterance; if (!utterance || utterance.id !== utteranceId) return; + if (utterance.interruptionFallback !== null) { + clearTimeout(utterance.interruptionFallback); + utterance.interruptionFallback = null; + } setUtteranceStatus(utterance, "failed"); recordPlaybackNotice( utterance.sessionId, @@ -378,6 +384,28 @@ function failActiveUtterance( utterance.onTerminal(); } +function finalizeInterruptedUtterance( + utterance: ActiveUtterance, + cause: InterruptionCause, +) { + if (activeUtterance?.id !== utterance.id) return; + if (utterance.interruptionFallback !== null) { + clearTimeout(utterance.interruptionFallback); + utterance.interruptionFallback = null; + } + const estimate = estimateSpeechDelivery( + utterance.text, + utterance.latestDelivery, + ); + applyInterruptionEstimate(utterance, estimate); + utterance.onInterrupted(); + recordInterruptionNotices(utterance, estimate, cause); + activeUtterance = null; + useVoiceConversationStore.getState().setUiState("listening"); + reportAssistantActivity(utterance.sessionId, utterance.voiceRevision, false); + utterance.onTerminal(); +} + function queueStreamCommand( utterance: ActiveUtterance, operation: () => Promise, @@ -415,6 +443,10 @@ function handleStreamEvent( ); break; case "completed": + if (utterance.interruptionFallback !== null) { + clearTimeout(utterance.interruptionFallback); + utterance.interruptionFallback = null; + } setUtteranceStatus(utterance, "spoken"); voice.setUiState("listening"); activeUtterance = null; @@ -427,25 +459,10 @@ function handleStreamEvent( break; case "interrupted": { utterance.latestDelivery = event.delivery ?? utterance.latestDelivery; - const estimate = estimateSpeechDelivery( - utterance.text, - utterance.latestDelivery, - ); - applyInterruptionEstimate(utterance, estimate); - utterance.onInterrupted(); - recordInterruptionNotices( + finalizeInterruptedUtterance( utterance, - estimate, utterance.interruptionCause ?? "voiceStopped", ); - voice.setUiState("listening"); - activeUtterance = null; - reportAssistantActivity( - utterance.sessionId, - utterance.voiceRevision, - false, - ); - utterance.onTerminal(); break; } case "failed": @@ -486,29 +503,37 @@ function interruptActiveUtterance( if (terminalEventExpected) utterance.onInterrupted(); } if (utterance && !terminalEventExpected) { - activeUtterance = null; - const estimate = estimateSpeechDelivery( - utterance.text, - utterance.latestDelivery, - ); - applyInterruptionEstimate(utterance, estimate); - utterance.onInterrupted(); - recordInterruptionNotices( + finalizeInterruptedUtterance( utterance, - estimate, utterance.interruptionCause ?? cause, ); - reportAssistantActivity( - utterance.sessionId, - utterance.voiceRevision, - false, + } + if (utterance && terminalEventExpected) { + utterance.interruptionFallback = setTimeout(() => { + finalizeInterruptedUtterance( + utterance, + utterance.interruptionCause ?? cause, + ); + }, INTERRUPTION_TERMINAL_TIMEOUT_MS); + void stopActiveVoice().then( + (stopped) => { + if (!stopped) { + finalizeInterruptedUtterance( + utterance, + utterance.interruptionCause ?? cause, + ); + } + }, + () => { + finalizeInterruptedUtterance( + utterance, + utterance.interruptionCause ?? cause, + ); + }, ); - utterance.onTerminal(); + } else { + void stopActiveVoice().catch(() => undefined); } - void stopActiveVoice().catch(() => undefined); - commandQueue = commandQueue.then(async () => { - await stopActiveVoice().catch(() => undefined); - }); return utterance !== null; } @@ -625,6 +650,7 @@ export function startNativeAssistantSpeech( finishing: false, nativeStreamStarted: false, interruptionRequested: false, + interruptionFallback: null, interruptionCause: null, latestDelivery: null, status: null, From 8c84fb12367e77b2cf5455edba22e77b1538de93 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:27:14 -0400 Subject: [PATCH 13/20] fix(voice): reset delivery cutoff on rewrites --- .../lib/nativeAssistantSpeech.test.ts | 43 +++++++++++++++++++ .../lib/nativeAssistantSpeech.ts | 24 ++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index f87ea59f1..eab571ae3 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -938,6 +938,49 @@ describe("native assistant speech stream", () => { expect(notice).toContain('"unspokenText":" Three."'); }); + it("does not reuse an interrupted cutoff after a non-prefix rewrite", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Original reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "Original reply.", + playedFrames: 700, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Replacement text." }]), + ]); + + await vi.waitFor(() => { + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ speech: { status: "notSpoken" } }); + }); + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice).toContain('"spokenText":""'); + expect(notice).toContain('"unspokenText":"Replacement text."'); + }); + it("bounds spoken and unspoken excerpts in the model delivery notice", async () => { const text = `${"spoken ".repeat(100)}${"unspoken ".repeat(100)}`; startNativeAssistantSpeech("session-1", vi.fn()); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index e06fef6fb..2a37ce6b9 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -720,7 +720,8 @@ export function startNativeAssistantSpeech( textOrdinal += 1; const previous = consumedTextBySlot.get(slot) ?? ""; if (content.text === previous) continue; - const delta = content.text.startsWith(previous) + const appendOnly = content.text.startsWith(previous); + const delta = appendOnly ? content.text.slice(previous.length) : content.text; consumedTextBySlot.set(slot, content.text); @@ -732,7 +733,26 @@ export function startNativeAssistantSpeech( currentSpeech?.interruptionCause ?? interruptionCauseByMessage.get(message.id) ?? "voiceStopped"; - const spokenThrough = currentSpeech?.spokenThrough ?? 0; + const spokenThrough = appendOnly + ? (currentSpeech?.spokenThrough ?? 0) + : 0; + if (!appendOnly) { + setTargetSpeech(sessionId, target, { status: "notSpoken" }); + recordPlaybackNotice( + sessionId, + slot, + content.text, + "interrupted", + { + cutoff: 0, + spokenText: "", + unspokenText: content.text, + confidence: "low", + }, + interruptionCause, + ); + continue; + } if (currentSpeech?.status !== "interrupted") { setTargetSpeech( sessionId, From dbc92b12465623e4c45071dd37e3f877ddbf0b2b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:34:37 -0400 Subject: [PATCH 14/20] fix(voice): close late interruption races --- .../lib/nativeAssistantSpeech.test.ts | 51 ++++++++++++++++++- .../lib/nativeAssistantSpeech.ts | 8 +++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index eab571ae3..4bc885a29 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -561,7 +561,7 @@ describe("native assistant speech stream", () => { it("finalizes an interruption before native playback starts", async () => { let resolveStart: (() => void) | undefined; - mocks.start.mockImplementation( + mocks.start.mockImplementationOnce( () => new Promise((resolve) => { resolveStart = resolve; @@ -575,6 +575,7 @@ describe("native assistant speech stream", () => { ]); await vi.waitFor(() => expect(mocks.start).toHaveBeenCalled()); + mocks.stop.mockResolvedValueOnce(false).mockResolvedValue(true); useVoiceConversationStore.setState({ userSpeaking: true }); await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); @@ -589,6 +590,54 @@ describe("native assistant speech stream", () => { }); expect(takeVoicePlaybackNotices("session-1")).toContain('"spokenText":""'); resolveStart?.(); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalledTimes(2)); + + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .setMessages("session-1", [ + assistant( + [{ type: "text", text: "Queued reply." }], + "completed", + "assistant-1", + ), + assistant( + [{ type: "text", text: "Next reply." }], + "completed", + "assistant-2", + ), + ]); + await vi.waitFor(() => expect(mocks.start).toHaveBeenCalledTimes(2)); + }); + + it("ignores a late started event after interruption is requested", async () => { + let resolveStop: ((stopped: boolean) => void) | undefined; + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Native reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + + mocks.stop.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveStop = resolve; + }), + ); + useVoiceConversationStore.setState({ userSpeaking: true }); + emit("started"); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).not.toMatchObject({ speech: { status: "speaking" } }); + expect(mocks.setAssistantSpeaking).not.toHaveBeenCalledWith( + "session-1", + 1, + true, + ); + resolveStop?.(true); + emit("interrupted"); }); it("waits for terminal delivery once the native stream exists", async () => { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 2a37ce6b9..ea1644319 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -434,6 +434,7 @@ function handleStreamEvent( utterance.latestDelivery = event.delivery ?? null; break; case "started": + if (utterance.interruptionRequested) break; setUtteranceStatus(utterance, "speaking"); voice.setUiState("agent-speaking"); reportAssistantActivity( @@ -672,6 +673,13 @@ export function startNativeAssistantSpeech( async () => { await streamListenerReady; await streamBackend.start(utterance.id); + if ( + utterance.interruptionRequested || + activeUtterance?.id !== utterance.id + ) { + await streamBackend.stop(); + return; + } utterance.nativeStreamStarted = true; }, onFailure, From ee990a1d19e9379863a5163754279a52ddb0fcc4 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:58:08 -0400 Subject: [PATCH 15/20] fix(voice): preserve partial delivery evidence --- .../lib/nativeAssistantSpeech.test.ts | 70 ++++++++++++++++++- .../lib/nativeAssistantSpeech.ts | 29 ++++---- 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 4bc885a29..27f7a70c4 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -903,7 +903,7 @@ describe("native assistant speech stream", () => { expect(notice).toContain('"confidence":"medium"'); }); - it("does not treat generated-but-incomplete audio as fully spoken", async () => { + it("uses played audio from incomplete synthesis with low confidence", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore .getState() @@ -923,7 +923,7 @@ describe("native assistant speech stream", () => { segments: [ { text: "One. Two. Three.", - playedFrames: 1_000, + playedFrames: 600, totalFrames: 1_000, synthesisComplete: false, }, @@ -934,8 +934,72 @@ describe("native assistant speech stream", () => { expect( useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], ).toMatchObject({ - speech: { status: "interrupted", spokenThrough: 0 }, + speech: { + status: "interrupted", + spokenThrough: "One. Two".length, + confidence: "low", + }, }); + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice).toContain('"spokenText":"One. Two"'); + expect(notice).toContain('"unspokenText":". Three."'); + expect(notice).toContain('"confidence":"low"'); + }); + + it("sums only each target's interleaved delivered spans", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore.getState().setMessages("session-1", [ + assistant([ + { type: "text", text: "Alpha. " }, + { type: "text", text: "" }, + ]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalledTimes(1)); + useChatStore.getState().setMessages("session-1", [ + assistant([ + { type: "text", text: "Alpha. " }, + { type: "text", text: "Beta. " }, + ]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalledTimes(2)); + useChatStore.getState().setMessages("session-1", [ + assistant([ + { type: "text", text: "Alpha. Gamma." }, + { type: "text", text: "Beta. " }, + ]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalledTimes(3)); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "Alpha. Beta. Gamma.", + playedFrames: 1_800, + totalFrames: 1_900, + synthesisComplete: true, + }, + ], + }, + }); + + const content = + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content; + expect(content?.[0]).toMatchObject({ + speech: { status: "interrupted", spokenThrough: "Alpha. Gamma".length }, + }); + expect(content?.[1]).toMatchObject({ + speech: { status: "spoken", spokenThrough: "Beta. ".length }, + }); + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice).toContain('"spokenText":"Alpha. Gamma"'); + expect(notice).toContain('"unspokenText":"."'); + expect(notice).not.toContain("Beta."); }); it("preserves a fully spoken prefix when text arrives after interruption", async () => { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index ea1644319..d7d9e63d7 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -197,6 +197,7 @@ function estimateSpeechDelivery( let searchFrom = 0; let cutoff = 0; let matchedSegment = false; + let usedIncompleteSegment = false; for (const segment of delivery.segments) { const segmentStart = text.indexOf(segment.text, searchFrom); if (segmentStart === -1) continue; @@ -207,11 +208,10 @@ function estimateSpeechDelivery( Math.min(totalFrames, segment.playedFrames), ); if (totalFrames === 0 || playedFrames === 0) break; - if (!segment.synthesisComplete) { - break; - } + usedIncompleteSegment ||= !segment.synthesisComplete; if (playedFrames >= totalFrames) { cutoff = segmentStart + segment.text.length; + if (!segment.synthesisComplete) break; searchFrom = cutoff; continue; } @@ -225,7 +225,7 @@ function estimateSpeechDelivery( cutoff, spokenText: text.slice(0, cutoff), unspokenText: text.slice(cutoff), - confidence: matchedSegment ? "medium" : "low", + confidence: matchedSegment && !usedIncompleteSegment ? "medium" : "low", }; } @@ -240,16 +240,23 @@ function applyInterruptionEstimate( const spans = utterance.targetSpans.filter( (span) => targetKey(span) === targetKey(target), ); - const start = spans.at(0)?.start ?? 0; - const end = spans.at(-1)?.end ?? start; - if (estimate.cutoff >= end && end > start) { + const targetLength = spans.reduce( + (length, span) => length + (span.end - span.start), + 0, + ); + const localCutoff = spans.reduce( + (length, span) => + length + Math.max(0, Math.min(span.end, estimate.cutoff) - span.start), + 0, + ); + if (localCutoff >= targetLength && targetLength > 0) { setTargetSpeech(utterance.sessionId, target, { status: "spoken", - spokenThrough: end - start, + spokenThrough: targetLength, }); continue; } - if (estimate.cutoff <= start) { + if (localCutoff === 0) { if (targetKey(target) === firstTargetKey) { setTargetSpeech(utterance.sessionId, target, { status: "interrupted", @@ -262,10 +269,6 @@ function applyInterruptionEstimate( setTargetSpeech(utterance.sessionId, target, { status: "notSpoken" }); continue; } - const localCutoff = Math.max( - 0, - Math.min(end - start, estimate.cutoff - start), - ); setTargetSpeech(utterance.sessionId, target, { status: "interrupted", spokenThrough: localCutoff, From 2e3938d123e5f2ca797f787d90a82245aac116de Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:58:42 -0400 Subject: [PATCH 16/20] fix(voice): use terminal delivery on hang-up --- .../lib/nativeAssistantSpeech.test.ts | 72 ++++++++++++++++++- .../lib/nativeAssistantSpeech.ts | 30 +++++--- 2 files changed, 92 insertions(+), 10 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 27f7a70c4..9c12f47ec 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -803,10 +803,26 @@ describe("native assistant speech stream", () => { useChatStore .getState() .setMessages("session-1", [ - assistant([{ type: "text", text: "Goodbye." }]), + assistant([{ type: "text", text: "One. Two. Three." }]), ]); await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); emit("started"); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + mocks.streamHandler?.({ + streamId, + state: "progress", + error: null, + delivery: { + segments: [ + { + text: "One. Two. Three.", + playedFrames: 200, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); useVoiceConversationStore.setState((voice) => ({ status: { @@ -819,10 +835,64 @@ describe("native assistant speech stream", () => { uiState: "off", })); await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + expect(takeVoicePlaybackNotices("session-1")).toBeNull(); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + segments: [ + { + text: "One. Two. Three.", + playedFrames: 600, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); const notice = takeVoicePlaybackNotices("session-1"); expect(notice).toContain("because the voice conversation stopped"); expect(notice).not.toContain("because the user started speaking"); + expect(notice).toContain('"spokenText":"One. Two"'); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { status: "interrupted", spokenThrough: "One. Two".length }, + }); + }); + + it("bounds the terminal delivery wait during hang-up", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Goodbye." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + + vi.useFakeTimers(); + try { + useVoiceConversationStore.setState((voice) => ({ + status: { + ...voice.status, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: voice.status.revision + 1, + }, + uiState: "off", + })); + await vi.advanceTimersByTimeAsync(1_000); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { status: "interrupted", spokenThrough: 0 }, + }); + } finally { + vi.useRealTimers(); + } }); it("uses playback progress to report and decorate only the unspoken suffix", async () => { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index d7d9e63d7..08ab295e0 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -541,9 +541,27 @@ function interruptActiveUtterance( return utterance !== null; } -export function stopNativeAssistantSpeech(): void { +export function stopNativeAssistantSpeech(awaitTerminalDelivery = false): void { generation += 1; - const interruptedUtterance = interruptActiveUtterance(); + const utterance = activeUtterance; + const terminalStreamSubscription = stopStreamSubscription; + stopStreamSubscription = null; + stopSubscription?.(); + stopSubscription = null; + stopVoiceSubscription?.(); + stopVoiceSubscription = null; + const shouldAwaitTerminal = + awaitTerminalDelivery && utterance?.nativeStreamStarted === true; + if (utterance && shouldAwaitTerminal) { + const onTerminal = utterance.onTerminal; + utterance.onTerminal = () => { + terminalStreamSubscription?.(); + onTerminal(); + }; + } else { + terminalStreamSubscription?.(); + } + const interruptedUtterance = interruptActiveUtterance(shouldAwaitTerminal); if ( !interruptedUtterance && activeSpeechSessionId && @@ -551,12 +569,6 @@ export function stopNativeAssistantSpeech(): void { ) { reportAssistantActivity(activeSpeechSessionId, activeSpeechRevision, false); } - stopSubscription?.(); - stopSubscription = null; - stopVoiceSubscription?.(); - stopVoiceSubscription = null; - stopStreamSubscription?.(); - stopStreamSubscription = null; activeSpeechSessionId = null; activeSpeechRevision = null; } @@ -875,7 +887,7 @@ export function startNativeAssistantSpeech( (voice.status.sessionId !== null && voice.status.sessionId !== sessionId) ) { - stopNativeAssistantSpeech(); + stopNativeAssistantSpeech(true); } return; } From 1ee8fdd593707dd756eb0d4e0200e926e31d3d12 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:59:11 -0400 Subject: [PATCH 17/20] fix(chat): preserve agent work speech boundaries --- src/features/chat/ui/AgentWorkPanel.tsx | 9 +++- .../chat/ui/__tests__/AgentWorkPanel.test.tsx | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/features/chat/ui/AgentWorkPanel.tsx b/src/features/chat/ui/AgentWorkPanel.tsx index de66b9bcd..96451a008 100644 --- a/src/features/chat/ui/AgentWorkPanel.tsx +++ b/src/features/chat/ui/AgentWorkPanel.tsx @@ -145,7 +145,8 @@ function buildAgentWorkTimeline( const previous = items[items.length - 1]; if ( previous?.kind === "progress" && - previous.content.speech?.status === block.speech?.status + previous.content.speech === undefined && + block.speech === undefined ) { previous.content = { ...previous.content, @@ -340,7 +341,11 @@ function AgentWorkItemRow({ label={speechLabel} /> ) : null} - + {item.content.text} diff --git a/src/features/chat/ui/__tests__/AgentWorkPanel.test.tsx b/src/features/chat/ui/__tests__/AgentWorkPanel.test.tsx index 6f54c77e5..e90c1c690 100644 --- a/src/features/chat/ui/__tests__/AgentWorkPanel.test.tsx +++ b/src/features/chat/ui/__tests__/AgentWorkPanel.test.tsx @@ -48,4 +48,54 @@ describe("AgentWorkPanel", () => { expect(screen.getByText("Already spoken.")).toBeInTheDocument(); expect(screen.getByText("Speaking now.")).toBeInTheDocument(); }); + + it("preserves adjacent interrupted blocks with distinct cutoffs", () => { + const content = [ + { + type: "text" as const, + text: "First heard. First unheard.", + speech: { + status: "interrupted" as const, + spokenThrough: "First heard.".length, + }, + }, + { + type: "text" as const, + text: "Second heard. Second unheard.", + speech: { + status: "interrupted" as const, + spokenThrough: "Second heard.".length, + }, + }, + ]; + const payload: TranscriptAgentWorkPayload = { + workId: "work-1", + message: { + id: "assistant-1", + role: "assistant", + created: Date.UTC(2026, 7, 19, 15, 0), + content, + }, + content, + isActiveWork: true, + hasFinalAnswer: false, + thoughtCount: 0, + toolCount: 0, + textCount: 2, + }; + + const { container } = renderWithProviders( + , + ); + const blocks = container.querySelectorAll( + '[data-voice-speech-status="interrupted"]', + ); + expect(blocks).toHaveLength(2); + expect(blocks[0]?.querySelector("[data-voice-unspoken]")).toHaveTextContent( + "First unheard.", + ); + expect(blocks[1]?.querySelector("[data-voice-unspoken]")).toHaveTextContent( + "Second unheard.", + ); + }); }); From 58d03e891055a030fe2765f698894d836f93b6d8 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 22:59:40 -0400 Subject: [PATCH 18/20] fix(chat): describe unheard text accessibly --- src/features/chat/ui/MessageBubble.tsx | 1 + .../chat/ui/__tests__/MessageBubble.test.tsx | 57 ++++++++++++------- src/shared/ui/ai-elements/message.tsx | 57 ++++++++++++++++--- 3 files changed, 85 insertions(+), 30 deletions(-) diff --git a/src/features/chat/ui/MessageBubble.tsx b/src/features/chat/ui/MessageBubble.tsx index 1477a83b2..9af93998e 100644 --- a/src/features/chat/ui/MessageBubble.tsx +++ b/src/features/chat/ui/MessageBubble.tsx @@ -569,6 +569,7 @@ function renderContentBlock( } imageRenderer={MarkdownImage} strikethroughFrom={strikethroughFrom} + strikethroughLabel={options.voiceSpeechNotSpokenLabel} > {displayText} diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index cb784b3ae..e936b5aba 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -730,9 +730,15 @@ describe("MessageBubble", () => { const block = container.querySelector( '[data-voice-speech-status="interrupted"]', ); - expect(block).toHaveTextContent("One. Two. Three."); - expect(block?.querySelector("del")).toHaveTextContent(". Three."); - expect(block?.querySelector("del")).not.toHaveTextContent("One. Two"); + expect(block).toHaveTextContent("One. Two"); + expect(block?.querySelector("[data-voice-unspoken]")).toHaveTextContent( + ". Three.", + ); + expect(block?.querySelector("[data-voice-unspoken]")).not.toHaveTextContent( + "One. Two", + ); + expect(block?.querySelector("del")).toBeNull(); + expect(block?.querySelector(".sr-only")).toHaveTextContent("Not spoken:"); }); it("updates the strike when unchanged text becomes interrupted", () => { @@ -744,7 +750,7 @@ describe("MessageBubble", () => { ])} />, ); - expect(container.querySelector("del")).toBeNull(); + expect(container.querySelector("[data-voice-unspoken]")).toBeNull(); rerender( { />, ); - expect(container.querySelector("del")).toHaveTextContent(". Three."); + expect(container.querySelector("[data-voice-unspoken]")).toHaveTextContent( + ". Three.", + ); }); it("keeps required list and table children structurally valid", () => { @@ -789,9 +797,8 @@ describe("MessageBubble", () => { expect( [...(list?.children ?? [])].every((child) => child.tagName === "LI"), ).toBe(true); - expect(list?.children[1]?.querySelector("del")).toHaveTextContent( - "Unheard item", - ); + expect(list?.children[1]).toHaveTextContent("Unheard item"); + expect(list?.children[1]).toHaveAttribute("data-voice-unspoken", "true"); const table = container.querySelector("table"); expect(table).toBeInTheDocument(); expect(table?.querySelector("tbody")?.parentElement).toBe(table); @@ -824,14 +831,14 @@ describe("MessageBubble", () => { ); const paragraphs = block?.querySelectorAll("p"); expect(paragraphs).toHaveLength(3); - expect(paragraphs?.[0]?.querySelector("del")).toHaveTextContent( - "Unheard first paragraph.", - ); - expect(paragraphs?.[0]?.querySelector("del")).not.toHaveTextContent( - "Heard text.", - ); - expect(paragraphs?.[1]?.closest("del")).toBeTruthy(); - expect(paragraphs?.[2]?.closest("del")).toBeTruthy(); + expect( + paragraphs?.[0]?.querySelector("[data-voice-unspoken]"), + ).toHaveTextContent("Unheard first paragraph."); + expect( + paragraphs?.[0]?.querySelector("[data-voice-unspoken]"), + ).not.toHaveTextContent("Heard text."); + expect(paragraphs?.[1]).toHaveAttribute("data-voice-unspoken", "true"); + expect(paragraphs?.[2]).toHaveAttribute("data-voice-unspoken", "true"); }); it("preserves Markdown structure while striking the unspoken range", async () => { @@ -858,17 +865,25 @@ describe("MessageBubble", () => { ); await waitFor(() => { expect( - block?.querySelector('del [data-streamdown="strong"]'), + block?.querySelector('[data-streamdown="strong"][data-voice-unspoken]'), ).toHaveTextContent("bold"); expect( - block?.querySelector('del a[href="https://example.com/"]'), + block?.querySelector( + 'a[href="https://example.com/"][data-voice-unspoken]', + ), ).toHaveTextContent("link"); - expect(block?.querySelector("del li")).toHaveTextContent("list item"); + expect(block?.querySelector("li[data-voice-unspoken]")).toHaveTextContent( + "list item", + ); expect( - block?.querySelector('del [data-streamdown="inline-code"]'), + block + ?.querySelector('[data-streamdown="inline-code"]') + ?.closest("[data-voice-unspoken]"), ).toHaveTextContent("inline"); expect( - block?.querySelector('del [data-streamdown="code-block"]'), + block?.querySelector( + '[data-voice-unspoken] [data-streamdown="code-block"]', + ), ).toBeTruthy(); expect(block?.querySelector("pre code")).toHaveTextContent( "const value = 1;", diff --git a/src/shared/ui/ai-elements/message.tsx b/src/shared/ui/ai-elements/message.tsx index 4a33f2376..5c6dfac94 100644 --- a/src/shared/ui/ai-elements/message.tsx +++ b/src/shared/ui/ai-elements/message.tsx @@ -343,6 +343,8 @@ export type MessageResponseProps = ComponentProps & { codeRenderers?: CustomRenderer[]; /** Source-text offset after which rendered Markdown is struck through. */ strikethroughFrom?: number; + /** Accessible label announced before visually struck voice-undelivered text. */ + strikethroughLabel?: string; /** * Optional feature-aware Markdown image renderer. Chat injects one that can * resolve local files through the asset scheme; when omitted, images render @@ -703,8 +705,8 @@ const berdRehypePlugins: NonNullable< restoreBerdMarkdownDestinations, ]; -function strikethroughFromPlugin(cutoff: number) { - const structureParents = new Set([ +function strikethroughFromPlugin(cutoff: number, label: string) { + const structureElements = new Set([ "dl", "menu", "ol", @@ -716,11 +718,20 @@ function strikethroughFromPlugin(cutoff: number) { "tr", "ul", ]); + const accessibleLabel = (): MarkdownHastNode => ({ + type: "element", + tagName: "span", + properties: { className: ["sr-only"] }, + children: [{ type: "text", value: `${label}: ` }], + }); const wrap = (node: MarkdownHastNode): MarkdownHastNode => ({ type: "element", - tagName: "del", - properties: {}, - children: [node], + tagName: "span", + properties: { + className: ["line-through"], + "data-voice-unspoken": "true", + }, + children: [accessibleLabel(), node], position: node.position, }); @@ -734,9 +745,36 @@ function strikethroughFromPlugin(cutoff: number) { child.type === "element" && start !== undefined && cutoff <= start && - !structureParents.has(node.tagName ?? "") + !structureElements.has(child.tagName ?? "") ) { - children.push(wrap(child)); + if (child.tagName === "pre") { + children.push({ + type: "element", + tagName: "div", + properties: { + className: ["line-through"], + "data-voice-unspoken": "true", + }, + children: [accessibleLabel(), child], + position: child.position, + }); + continue; + } + const className = child.properties?.className; + child.properties = { + ...child.properties, + className: [ + ...(Array.isArray(className) + ? className + : typeof className === "string" + ? [className] + : []), + "line-through", + ], + "data-voice-unspoken": "true", + }; + child.children = [accessibleLabel(), ...(child.children ?? [])]; + children.push(child); continue; } if ( @@ -791,6 +829,7 @@ export const MessageResponse = memo( onAnimationEnd, onAnimationStart, strikethroughFrom, + strikethroughLabel = "Not spoken", ...props }: MessageResponseProps) => { const { t } = useTranslation("common"); @@ -807,9 +846,9 @@ export const MessageResponse = memo( ? berdRehypePlugins : [ ...berdRehypePlugins, - [strikethroughFromPlugin, strikethroughFrom], + [strikethroughFromPlugin, strikethroughFrom, strikethroughLabel], ], - [strikethroughFrom], + [strikethroughFrom, strikethroughLabel], ); const streamdownRootRef = useRef(null); const streamdownLayoutPending = useVirtualLayoutPendingForStreamdown({ From ab879012a7e856cda775fdfbc6d0682ca9237148 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 23 Aug 2026 23:31:39 -0400 Subject: [PATCH 19/20] fix(voice): harden interrupted delivery estimates --- src-tauri/native/siri_tts_bridge.m | 11 +- src-tauri/src/commands/pocket_voice.rs | 7 +- src-tauri/src/commands/siri_voice.rs | 2 + .../chat/ui/__tests__/MessageBubble.test.tsx | 126 +++++++++++- .../voice-conversation/api/pocketVoice.ts | 1 + .../lib/nativeAssistantSpeech.test.ts | 137 ++++++++++++- .../lib/nativeAssistantSpeech.ts | 68 +++++- src/shared/ui/ai-elements/message.test.tsx | 24 +++ src/shared/ui/ai-elements/message.tsx | 194 ++++++++++++++++-- 9 files changed, 524 insertions(+), 46 deletions(-) diff --git a/src-tauri/native/siri_tts_bridge.m b/src-tauri/native/siri_tts_bridge.m index 09d1dc55c..6f12ad43a 100644 --- a/src-tauri/native/siri_tts_bridge.m +++ b/src-tauri/native/siri_tts_bridge.m @@ -505,7 +505,7 @@ - (void)finishInput { }); } - (NSString *)deliveryJSON { - __block NSString *json = @"{\"segments\":[]}"; + __block NSString *json = @"{\"sampleRate\":0,\"segments\":[]}"; void (^snapshot)(void) = ^{ uint64_t playedFrames = 0; if (self.player && self.player.lastRenderTime) { @@ -532,7 +532,10 @@ - (NSString *)deliveryJSON { }]; segmentStart += segment.totalFrames; } - NSData *data = [NSJSONSerialization dataWithJSONObject:@{ @"segments": segments } + NSData *data = [NSJSONSerialization dataWithJSONObject:@{ + @"sampleRate": @((uint32_t)llround(self.playbackSampleRate)), + @"segments": segments, + } options:0 error:nil]; if (data) json = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; }; @@ -1067,9 +1070,9 @@ uint64_t berd_siri_tts_stream_progress(void *stream) { } char *berd_siri_tts_stream_copy_delivery_json(void *stream) { - if (!stream) return strdup("{\"segments\":[]}"); + if (!stream) return strdup("{\"sampleRate\":0,\"segments\":[]}"); NSString *json = [(__bridge BerdSiriSpeechPlayer *)stream deliveryJSON]; - return strdup((json ?: @"{\"segments\":[]}").UTF8String); + return strdup((json ?: @"{\"sampleRate\":0,\"segments\":[]}").UTF8String); } char *berd_siri_tts_stream_copy_error(void *stream) { diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index c6f873058..4f47f2f8a 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -191,6 +191,8 @@ struct VoiceDeliverySegment { #[cfg(any(test, target_os = "macos"))] #[derive(Clone, Debug, Serialize)] struct VoiceDeliveryProgress { + #[serde(rename = "sampleRate")] + sample_rate: u32, segments: Vec, } @@ -256,7 +258,10 @@ impl PlaybackDeliveryLedger { } }) .collect(); - VoiceDeliveryProgress { segments } + VoiceDeliveryProgress { + sample_rate: berd_voice::SAMPLE_RATE, + segments, + } } } diff --git a/src-tauri/src/commands/siri_voice.rs b/src-tauri/src/commands/siri_voice.rs index 050b1d243..48405392e 100644 --- a/src-tauri/src/commands/siri_voice.rs +++ b/src-tauri/src/commands/siri_voice.rs @@ -85,6 +85,8 @@ struct VoiceDeliverySegment { #[cfg(target_os = "macos")] #[derive(Clone, Debug, Deserialize, Serialize)] struct VoiceDeliveryProgress { + #[serde(rename = "sampleRate")] + sample_rate: u32, segments: Vec, } diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index e936b5aba..b3c7decfc 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -837,8 +837,12 @@ describe("MessageBubble", () => { expect( paragraphs?.[0]?.querySelector("[data-voice-unspoken]"), ).not.toHaveTextContent("Heard text."); - expect(paragraphs?.[1]).toHaveAttribute("data-voice-unspoken", "true"); - expect(paragraphs?.[2]).toHaveAttribute("data-voice-unspoken", "true"); + expect( + paragraphs?.[1]?.querySelector("[data-voice-unspoken]"), + ).toHaveTextContent("Unheard second paragraph."); + expect( + paragraphs?.[2]?.querySelector("[data-voice-unspoken]"), + ).toHaveTextContent("Unheard third paragraph."); }); it("preserves Markdown structure while striking the unspoken range", async () => { @@ -888,9 +892,127 @@ describe("MessageBubble", () => { expect(block?.querySelector("pre code")).toHaveTextContent( "const value = 1;", ); + expect(block?.querySelectorAll(".sr-only")).toHaveLength(1); }); }); + it("keeps void Markdown elements valid after the delivery boundary", async () => { + const spoken = "Heard.\n\n"; + const text = `${spoken}![alt text](https://example.com/image.png)\n\n- [ ] task\n\nline \nbreak\n\n---`; + const { container } = render( + , + ); + + await waitFor(() => { + const block = container.querySelector( + '[data-voice-speech-status="interrupted"]', + ); + expect(block?.querySelector("img")).toBeInTheDocument(); + expect( + block?.querySelector('input[type="checkbox"]'), + ).toBeInTheDocument(); + expect(block?.querySelector("br")).toBeInTheDocument(); + expect(block?.querySelector("hr")).toBeInTheDocument(); + expect(block?.querySelector("img")).toHaveAttribute( + "data-voice-unspoken", + "true", + ); + expect(block?.querySelector("img")).toHaveAttribute( + "alt", + "Not spoken: alt text", + ); + }); + }); + + it("conservatively marks an image intersected by the delivery boundary", async () => { + const text = "Heard ![multi word alt](https://example.com/image.png)"; + const { container } = render( + , + ); + + await waitFor(() => { + expect(container.querySelector("img")).toHaveAttribute( + "data-voice-unspoken", + "true", + ); + expect(container.querySelector("img")).toHaveAttribute( + "alt", + "Not spoken: multi word alt", + ); + }); + }); + + it("preserves decorative image semantics at the delivery boundary", async () => { + const { container } = render( + , + ); + + await waitFor(() => { + expect(container.querySelector("img")).toHaveAttribute("alt", ""); + expect(container.querySelector(".sr-only")).toHaveTextContent( + "Not spoken:", + ); + }); + }); + + it("maps Markdown entities and escapes at the delivery boundary", () => { + const spoken = "Heard &"; + const text = `${spoken} escaped \\*asterisk\\*.`; + const { container } = render( + , + ); + + const unheard = container.querySelector("[data-voice-unspoken]"); + expect(unheard).toHaveTextContent("escaped *asterisk*."); + expect(unheard).not.toHaveTextContent("Heard &"); + }); + it("preserves provider-error presentation after interrupted delivery", () => { const rawError = "Ran into this error: thinking blocks in the latest assistant message cannot be modified"; diff --git a/src/features/voice-conversation/api/pocketVoice.ts b/src/features/voice-conversation/api/pocketVoice.ts index 6f4d0999c..3c7f9d998 100644 --- a/src/features/voice-conversation/api/pocketVoice.ts +++ b/src/features/voice-conversation/api/pocketVoice.ts @@ -49,6 +49,7 @@ export interface VoiceDeliverySegment { } export interface VoiceDeliveryProgress { + sampleRate?: number; segments: VoiceDeliverySegment[]; } diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 9c12f47ec..82c85107d 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -861,6 +861,10 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted", spokenThrough: "One. Two".length }, }); + expect(useVoiceConversationStore.getState()).toMatchObject({ + status: { lifecycle: "stopped" }, + uiState: "off", + }); }); it("bounds the terminal delivery wait during hang-up", async () => { @@ -890,11 +894,94 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted", spokenThrough: 0 }, }); + expect(useVoiceConversationStore.getState()).toMatchObject({ + status: { lifecycle: "stopped" }, + uiState: "off", + }); } finally { vi.useRealTimers(); } }); + it.each([ + "completed", + "failed", + ] as const)("keeps voice off when a late %s event races hang-up", async (state) => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Goodbye." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState((voice) => ({ + status: { + ...voice.status, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: voice.status.revision + 1, + }, + uiState: "off", + })); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + mocks.streamHandler?.({ + streamId, + state, + error: state === "failed" ? "native failure" : null, + }); + + expect(useVoiceConversationStore.getState()).toMatchObject({ + status: { lifecycle: "stopped" }, + uiState: "off", + }); + }); + + it("does not let an old terminal overwrite a restarted same-session run", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Old reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState((voice) => ({ + status: { + ...voice.status, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: voice.status.revision + 1, + }, + uiState: "off", + })); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + useVoiceConversationStore.setState((voice) => ({ + status: { + ...voice.status, + lifecycle: "running", + sessionId: "session-1", + revision: voice.status.revision + 1, + }, + uiState: "user-speaking", + })); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { segments: [] }, + }); + + expect(useVoiceConversationStore.getState()).toMatchObject({ + status: { lifecycle: "running", sessionId: "session-1" }, + uiState: "user-speaking", + }); + }); + it("uses playback progress to report and decorate only the unspoken suffix", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore @@ -973,7 +1060,7 @@ describe("native assistant speech stream", () => { expect(notice).toContain('"confidence":"medium"'); }); - it("uses played audio from incomplete synthesis with low confidence", async () => { + it("uses a duration-bounded estimate for incomplete synthesis", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore .getState() @@ -990,11 +1077,12 @@ describe("native assistant speech stream", () => { state: "interrupted", error: null, delivery: { + sampleRate: 24_000, segments: [ { text: "One. Two. Three.", - playedFrames: 600, - totalFrames: 1_000, + playedFrames: 24_000, + totalFrames: 24_000, synthesisComplete: false, }, ], @@ -1006,16 +1094,53 @@ describe("native assistant speech stream", () => { ).toMatchObject({ speech: { status: "interrupted", - spokenThrough: "One. Two".length, + spokenThrough: "One".length, confidence: "low", }, }); const notice = takeVoicePlaybackNotices("session-1") ?? ""; - expect(notice).toContain('"spokenText":"One. Two"'); - expect(notice).toContain('"unspokenText":". Three."'); + expect(notice).toContain('"spokenText":"One"'); + expect(notice).toContain('"unspokenText":". Two. Three."'); expect(notice).toContain('"confidence":"low"'); }); + it("never marks a short incomplete segment fully spoken", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [assistant([{ type: "text", text: "Yes" }])]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const streamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + mocks.streamHandler?.({ + streamId, + state: "interrupted", + error: null, + delivery: { + sampleRate: 24_000, + segments: [ + { + text: "Yes", + playedFrames: 12_000, + totalFrames: 12_000, + synthesisComplete: false, + }, + ], + }, + }); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { status: "interrupted", spokenThrough: 0, confidence: "low" }, + }); + expect(takeVoicePlaybackNotices("session-1")).toContain( + '"unspokenText":"Yes"', + ); + }); + it("sums only each target's interleaved delivered spans", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore.getState().setMessages("session-1", [ diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 08ab295e0..f8b9426f1 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -73,6 +73,10 @@ let activityReportQueue = Promise.resolve(); const pendingNotices = new Map>(); const DELIVERY_NOTICE_TEXT_LIMIT = 250; const INTERRUPTION_TERMINAL_TIMEOUT_MS = 1_000; +// An incomplete segment has no trustworthy final-frame denominator. Bound its +// text estimate by deliberately slow speech so generated-so-far audio cannot +// make a long source segment look fully delivered. +const INCOMPLETE_SEGMENT_MAX_CHARS_PER_SECOND = 6; function boundedDeliveryText( text: string, @@ -166,21 +170,29 @@ function targetKey(target: SpeechTarget): string { return `${target.messageId}\0text:${target.textOrdinal}`; } -function completedWordCutoff(text: string, playedRatio: number): number { - const approximateCutoff = Math.floor( - text.length * Math.max(0, Math.min(1, playedRatio)), - ); - if (approximateCutoff >= text.length) return text.length; +function completedWordCutoffAt( + text: string, + approximateCutoff: number, +): number { + const boundedCutoff = Math.max(0, Math.min(text.length, approximateCutoff)); + if (boundedCutoff >= text.length) return text.length; const segmenter = new Intl.Segmenter(undefined, { granularity: "word" }); let cutoff = 0; for (const part of segmenter.segment(text)) { const end = part.index + part.segment.length; - if (end > approximateCutoff) break; + if (end > boundedCutoff) break; if (part.isWordLike) cutoff = end; } return cutoff; } +function completedWordCutoff(text: string, playedRatio: number): number { + return completedWordCutoffAt( + text, + Math.floor(text.length * Math.max(0, Math.min(1, playedRatio))), + ); +} + function estimateSpeechDelivery( text: string, delivery: VoiceDeliveryProgress | null, @@ -209,9 +221,32 @@ function estimateSpeechDelivery( ); if (totalFrames === 0 || playedFrames === 0) break; usedIncompleteSegment ||= !segment.synthesisComplete; + if (!segment.synthesisComplete) { + const sampleRate = Math.max(0, delivery.sampleRate ?? 0); + const generatedRatioCutoff = Math.floor( + segment.text.length * (playedFrames / totalFrames), + ); + const durationBound = + sampleRate > 0 + ? Math.floor( + (playedFrames / sampleRate) * + INCOMPLETE_SEGMENT_MAX_CHARS_PER_SECOND, + ) + : 0; + cutoff = + segmentStart + + completedWordCutoffAt( + segment.text, + Math.min( + generatedRatioCutoff, + durationBound, + Math.max(0, segment.text.length - 1), + ), + ); + break; + } if (playedFrames >= totalFrames) { cutoff = segmentStart + segment.text.length; - if (!segment.synthesisComplete) break; searchFrom = cutoff; continue; } @@ -278,6 +313,17 @@ function applyInterruptionEstimate( } } +function restoreListeningIfConversationIsRunning(utterance: ActiveUtterance) { + const voice = useVoiceConversationStore.getState(); + if ( + voice.status.lifecycle === "running" && + voice.status.sessionId === utterance.sessionId && + voice.status.revision === utterance.voiceRevision + ) { + voice.setUiState("listening"); + } +} + function targetContent( sessionId: string, target: SpeechTarget, @@ -380,7 +426,7 @@ function failActiveUtterance( utterance.text, "failed", ); - useVoiceConversationStore.getState().setUiState("listening"); + restoreListeningIfConversationIsRunning(utterance); activeUtterance = null; reportAssistantActivity(utterance.sessionId, utterance.voiceRevision, false); onFailure(utterance.text, error); @@ -404,7 +450,7 @@ function finalizeInterruptedUtterance( utterance.onInterrupted(); recordInterruptionNotices(utterance, estimate, cause); activeUtterance = null; - useVoiceConversationStore.getState().setUiState("listening"); + restoreListeningIfConversationIsRunning(utterance); reportAssistantActivity(utterance.sessionId, utterance.voiceRevision, false); utterance.onTerminal(); } @@ -452,7 +498,7 @@ function handleStreamEvent( utterance.interruptionFallback = null; } setUtteranceStatus(utterance, "spoken"); - voice.setUiState("listening"); + restoreListeningIfConversationIsRunning(utterance); activeUtterance = null; reportAssistantActivity( utterance.sessionId, @@ -477,7 +523,7 @@ function handleStreamEvent( utterance.text, "failed", ); - voice.setUiState("listening"); + restoreListeningIfConversationIsRunning(utterance); activeUtterance = null; reportAssistantActivity( utterance.sessionId, diff --git a/src/shared/ui/ai-elements/message.test.tsx b/src/shared/ui/ai-elements/message.test.tsx index 89868a3ec..d64c8be41 100644 --- a/src/shared/ui/ai-elements/message.test.tsx +++ b/src/shared/ui/ai-elements/message.test.tsx @@ -11,6 +11,7 @@ import { const streamdownMocks = vi.hoisted(() => ({ latestProps: undefined as Record | undefined, + renderCount: 0, })); vi.mock("streamdown", () => ({ @@ -21,6 +22,7 @@ vi.mock("streamdown", () => ({ }, Streamdown: (props: Record) => { streamdownMocks.latestProps = props; + streamdownMocks.renderCount += 1; return (
@@ -151,3 +153,25 @@ describe("MessageResponse mermaid controls", () => { }); }); }); + +describe("MessageResponse voice delivery label", () => { + it("updates the accessible label when only the locale copy changes", () => { + streamdownMocks.renderCount = 0; + const { rerender } = render( + + Heard. Unheard. + , + ); + const firstRenderCount = streamdownMocks.renderCount; + + rerender( + + Heard. Unheard. + , + ); + + expect(streamdownMocks.renderCount).toBeGreaterThan(firstRenderCount); + const plugins = streamdownMocks.latestProps?.rehypePlugins as unknown[][]; + expect(plugins.at(-1)?.[2]).toBe("No hablado"); + }); +}); diff --git a/src/shared/ui/ai-elements/message.tsx b/src/shared/ui/ai-elements/message.tsx index 5c6dfac94..f30a8f280 100644 --- a/src/shared/ui/ai-elements/message.tsx +++ b/src/shared/ui/ai-elements/message.tsx @@ -21,6 +21,7 @@ import type { UIMessage } from "ai"; import { ChevronLeftIcon, ChevronRightIcon } from "lucide-react"; import { toast } from "sonner"; import type { + ComponentType, ComponentProps, HTMLAttributes, MouseEvent, @@ -558,11 +559,41 @@ const markdownHeadingComponents = { h6: createMarkdownHeading(6), } satisfies Pick; -function buildStreamdownComponents(imageRenderer?: MarkdownImageRenderer) { +function buildStreamdownComponents( + imageRenderer?: MarkdownImageRenderer, + unspokenLabel?: string, +) { + const ImageRenderer = (imageRenderer ?? + DefaultMarkdownImage) as unknown as ComponentType< + ComponentProps<"img"> & { node?: unknown } + >; + const VoiceAwareImage: MarkdownImageRenderer = (props) => { + const marksBoundary = props.className + ?.split(/\s+/) + .includes("voice-unspoken-boundary"); + const image = ( + + ); + return marksBoundary && unspokenLabel && !props.alt ? ( + <> + {unspokenLabel}: + {image} + + ) : ( + image + ); + }; return { ...markdownHeadingComponents, a: MarkdownLink, - img: imageRenderer ?? DefaultMarkdownImage, + img: VoiceAwareImage, }; } @@ -705,11 +736,67 @@ const berdRehypePlugins: NonNullable< restoreBerdMarkdownDestinations, ]; -function strikethroughFromPlugin(cutoff: number, label: string) { +const decodedEntityCache = new Map(); + +function decodeHtmlEntity(entity: string): string | null { + const cached = decodedEntityCache.get(entity); + if (cached !== undefined) return cached; + if ( + typeof document === "undefined" || + !/^&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);$/i.test(entity) + ) { + return null; + } + const textarea = document.createElement("textarea"); + textarea.innerHTML = entity; + const decoded = textarea.value === entity ? null : textarea.value; + decodedEntityCache.set(entity, decoded); + return decoded; +} + +function renderedPrefixLength(source: string, rendered: string): number { + let sourceOffset = 0; + let renderedOffset = 0; + while (sourceOffset < source.length && renderedOffset < rendered.length) { + if ( + source[sourceOffset] === "\\" && + sourceOffset + 1 < source.length && + source[sourceOffset + 1] === rendered[renderedOffset] + ) { + sourceOffset += 2; + renderedOffset += 1; + continue; + } + if (source[sourceOffset] === "&") { + const semicolon = source.indexOf(";", sourceOffset + 1); + if (semicolon !== -1) { + const decoded = decodeHtmlEntity( + source.slice(sourceOffset, semicolon + 1), + ); + if (decoded && rendered.startsWith(decoded, renderedOffset)) { + sourceOffset = semicolon + 1; + renderedOffset += decoded.length; + continue; + } + } + } + if (source[sourceOffset] !== rendered[renderedOffset]) break; + sourceOffset += 1; + renderedOffset += 1; + } + return renderedOffset; +} + +function strikethroughFromPlugin( + cutoff: number, + label: string, + source: string, +) { const structureElements = new Set([ "dl", "menu", "ol", + "p", "select", "table", "tbody", @@ -718,12 +805,34 @@ function strikethroughFromPlugin(cutoff: number, label: string) { "tr", "ul", ]); + const voidElements = new Set([ + "area", + "base", + "br", + "col", + "embed", + "hr", + "img", + "input", + "link", + "meta", + "param", + "source", + "track", + "wbr", + ]); const accessibleLabel = (): MarkdownHastNode => ({ type: "element", tagName: "span", properties: { className: ["sr-only"] }, children: [{ type: "text", value: `${label}: ` }], }); + let boundaryAnnounced = false; + const boundaryLabel = (): MarkdownHastNode[] => { + if (boundaryAnnounced) return []; + boundaryAnnounced = true; + return [accessibleLabel()]; + }; const wrap = (node: MarkdownHastNode): MarkdownHastNode => ({ type: "element", tagName: "span", @@ -731,7 +840,7 @@ function strikethroughFromPlugin(cutoff: number, label: string) { className: ["line-through"], "data-voice-unspoken": "true", }, - children: [accessibleLabel(), node], + children: [node], position: node.position, }); @@ -744,10 +853,35 @@ function strikethroughFromPlugin(cutoff: number, label: string) { if ( child.type === "element" && start !== undefined && - cutoff <= start && + end !== undefined && + (cutoff <= start || + (voidElements.has(child.tagName ?? "") && cutoff < end)) && !structureElements.has(child.tagName ?? "") ) { + if (voidElements.has(child.tagName ?? "")) { + const className = child.properties?.className; + const marksBoundary = + child.tagName === "img" && + (cutoff > start || !source.slice(cutoff, start).trim()); + if (marksBoundary) boundaryAnnounced = true; + child.properties = { + ...child.properties, + className: [ + ...(Array.isArray(className) + ? className + : typeof className === "string" + ? [className] + : []), + "line-through", + ...(marksBoundary ? ["voice-unspoken-boundary"] : []), + ], + "data-voice-unspoken": "true", + }; + children.push(child); + continue; + } if (child.tagName === "pre") { + children.push(...boundaryLabel()); children.push({ type: "element", tagName: "div", @@ -755,7 +889,7 @@ function strikethroughFromPlugin(cutoff: number, label: string) { className: ["line-through"], "data-voice-unspoken": "true", }, - children: [accessibleLabel(), child], + children: [child], position: child.position, }); continue; @@ -773,7 +907,11 @@ function strikethroughFromPlugin(cutoff: number, label: string) { ], "data-voice-unspoken": "true", }; - child.children = [accessibleLabel(), ...(child.children ?? [])]; + if (child.children) { + child.children = [...boundaryLabel(), ...child.children]; + } else { + children.push(...boundaryLabel()); + } children.push(child); continue; } @@ -783,25 +921,27 @@ function strikethroughFromPlugin(cutoff: number, label: string) { start !== undefined && end !== undefined ) { + if (!child.value.trim()) { + children.push(child); + continue; + } if (cutoff <= start) { + children.push(...boundaryLabel()); children.push(wrap(child)); continue; } if (cutoff < end) { - const sourceLength = Math.max(1, end - start); - const valueOffset = Math.max( - 0, - Math.min( - child.value.length, - Math.round( - ((cutoff - start) / sourceLength) * child.value.length, - ), - ), + const valueOffset = renderedPrefixLength( + source.slice(start, cutoff), + child.value, ); const spoken = child.value.slice(0, valueOffset); const unspoken = child.value.slice(valueOffset); if (spoken) children.push({ ...child, value: spoken }); - if (unspoken) children.push(wrap({ ...child, value: unspoken })); + if (unspoken) { + children.push(...boundaryLabel()); + children.push(wrap({ ...child, value: unspoken })); + } continue; } } @@ -835,8 +975,8 @@ export const MessageResponse = memo( const { t } = useTranslation("common"); const [modalUrl, setModalUrl] = useState(null); const streamdownComponents = useMemo( - () => buildStreamdownComponents(imageRenderer), - [imageRenderer], + () => buildStreamdownComponents(imageRenderer, strikethroughLabel), + [imageRenderer, strikethroughLabel], ); const rehypePlugins = useMemo< NonNullable["rehypePlugins"]> @@ -846,9 +986,14 @@ export const MessageResponse = memo( ? berdRehypePlugins : [ ...berdRehypePlugins, - [strikethroughFromPlugin, strikethroughFrom, strikethroughLabel], + [ + strikethroughFromPlugin, + strikethroughFrom, + strikethroughLabel, + children, + ], ], - [strikethroughFrom, strikethroughLabel], + [children, strikethroughFrom, strikethroughLabel], ); const streamdownRootRef = useRef(null); const streamdownLayoutPending = useVirtualLayoutPendingForStreamdown({ @@ -907,7 +1052,11 @@ export const MessageResponse = memo( {...streamdownLayoutPending.layoutPendingAttributes} > *:first-child]:mt-0 [&>*:last-child]:mb-0", className, @@ -945,6 +1094,7 @@ export const MessageResponse = memo( nextProps.isAnimating === prevProps.isAnimating && nextProps.mode === prevProps.mode && nextProps.strikethroughFrom === prevProps.strikethroughFrom && + nextProps.strikethroughLabel === prevProps.strikethroughLabel && nextProps.codeRenderers === prevProps.codeRenderers, ); From 4c4cf4d3e210d040ad23492cf8c74c8d3de8b438 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 24 Aug 2026 00:19:15 -0400 Subject: [PATCH 20/20] fix(voice): preserve delivery on synthesis failure --- src-tauri/src/commands/pocket_voice.rs | 86 ++++++++-- src-tauri/src/commands/siri_voice.rs | 103 +++++++++++- src/features/chat/ui/AgentWorkPanel.tsx | 2 +- src/features/chat/ui/MessageBubble.tsx | 2 +- .../chat/ui/__tests__/MessageBubble.test.tsx | 27 ++++ .../lib/nativeAssistantSpeech.test.ts | 72 +++++++++ .../lib/nativeAssistantSpeech.ts | 147 +++++++++++++----- 7 files changed, 378 insertions(+), 61 deletions(-) diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index 4f47f2f8a..6515278b5 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -271,6 +271,32 @@ struct PocketStreamOutcome { delivery: Option, } +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct PocketStreamFailure { + error: String, + delivery: Option, +} + +#[cfg(target_os = "macos")] +impl From for PocketStreamFailure { + fn from(error: String) -> Self { + Self { + error, + delivery: None, + } + } +} + +#[cfg(any(test, target_os = "macos"))] +fn delivery_with_played_audio(delivery: VoiceDeliveryProgress) -> Option { + delivery + .segments + .iter() + .any(|segment| segment.played_frames > 0) + .then_some(delivery) +} + #[derive(Clone, Debug, Default)] struct InstallRuntime { status_revision: u64, @@ -856,11 +882,15 @@ pub fn start_pocket_voice_stream( ); let (event_state, error, delivery) = match result { Ok(outcome) => (outcome.state, None, outcome.delivery), - Err(error) if !active.load(Ordering::SeqCst) => { - log::debug!("Pocket voice stream stopped after error: {error}"); - (PocketStreamEventState::Interrupted, None, None) + Err(failure) if !active.load(Ordering::SeqCst) => { + log::debug!("Pocket voice stream stopped after error: {}", failure.error); + (PocketStreamEventState::Interrupted, None, failure.delivery) } - Err(error) => (PocketStreamEventState::Failed, Some(error), None), + Err(failure) => ( + PocketStreamEventState::Failed, + Some(failure.error), + failure.delivery, + ), }; emit_pocket_stream_event(&app, &stream_id, event_state, error, delivery); finish_playback(&playback, &playback_active); @@ -1931,7 +1961,7 @@ fn run_pocket_voice_stream( active: Arc, speed: f32, receiver: mpsc::Receiver, -) -> Result { +) -> Result { use std::num::NonZero; use rodio::cpal::traits::HostTrait; @@ -1980,7 +2010,7 @@ fn run_pocket_voice_stream( let mut delivery_ledger = PlaybackDeliveryLedger::default(); let mut last_progress_emit = Instant::now(); - loop { + let result: Result = (|| loop { if !active.load(Ordering::SeqCst) { let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); player.stop(); @@ -2147,7 +2177,15 @@ fn run_pocket_voice_stream( } } } - } + })(); + + result.map_err(|error| { + let delivery = delivery_with_played_audio(capture_before_stop( + || pocket_delivery_snapshot(&delivery_ledger, &player), + || player.stop(), + )); + PocketStreamFailure { error, delivery } + }) } #[cfg(target_os = "macos")] @@ -2252,7 +2290,6 @@ fn synthesize_pocket_stream_ready( true })?; if let Some(error) = callback_error { - player.stop(); return Err(error); } if !completed { @@ -2440,7 +2477,7 @@ mod tests { } #[test] - fn cancellation_captures_delivery_before_stopping_playback() { + fn interruption_and_failure_capture_delivery_before_stopping_playback() { use std::cell::RefCell; let mut ledger = PlaybackDeliveryLedger::default(); @@ -2466,6 +2503,37 @@ mod tests { assert_eq!(delivery.segments[1].played_frames, 0); } + #[test] + fn failed_stream_retains_only_delivery_with_played_audio() { + let progress = VoiceDeliveryProgress { + sample_rate: berd_voice::SAMPLE_RATE, + segments: vec![VoiceDeliverySegment { + text: "Partly heard.".to_string(), + played_frames: 1_200, + total_frames: 4_800, + synthesis_complete: true, + }], + }; + assert_eq!( + delivery_with_played_audio(progress) + .expect("played audio is evidence") + .segments[0] + .played_frames, + 1_200 + ); + + let unheard = VoiceDeliveryProgress { + sample_rate: berd_voice::SAMPLE_RATE, + segments: vec![VoiceDeliverySegment { + text: "Not heard.".to_string(), + played_frames: 0, + total_frames: 4_800, + synthesis_complete: true, + }], + }; + assert!(delivery_with_played_audio(unheard).is_none()); + } + #[test] fn window_destroy_cancels_active_pocket_playback() { let state = PocketVoiceState::default(); diff --git a/src-tauri/src/commands/siri_voice.rs b/src-tauri/src/commands/siri_voice.rs index 48405392e..a4c63fb87 100644 --- a/src-tauri/src/commands/siri_voice.rs +++ b/src-tauri/src/commands/siri_voice.rs @@ -72,7 +72,7 @@ struct SiriStreamEvent { delivery: Option, } -#[cfg(target_os = "macos")] +#[cfg(any(test, target_os = "macos"))] #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] struct VoiceDeliverySegment { @@ -82,7 +82,7 @@ struct VoiceDeliverySegment { synthesis_complete: bool, } -#[cfg(target_os = "macos")] +#[cfg(any(test, target_os = "macos"))] #[derive(Clone, Debug, Deserialize, Serialize)] struct VoiceDeliveryProgress { #[serde(rename = "sampleRate")] @@ -96,6 +96,39 @@ struct SiriStreamOutcome { delivery: Option, } +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct SiriStreamFailure { + error: String, + delivery: Option, +} + +#[cfg(target_os = "macos")] +impl From for SiriStreamFailure { + fn from(error: String) -> Self { + Self { + error, + delivery: None, + } + } +} + +#[cfg(any(test, target_os = "macos"))] +fn delivery_with_played_audio(delivery: VoiceDeliveryProgress) -> Option { + delivery + .segments + .iter() + .any(|segment| segment.played_frames > 0) + .then_some(delivery) +} + +#[cfg(any(test, target_os = "macos"))] +fn capture_before_cancel(snapshot: impl FnOnce() -> T, cancel: impl FnOnce()) -> T { + let delivery = snapshot(); + cancel(); + delivery +} + #[cfg(target_os = "macos")] const SIRI_STREAM_EVENT: &str = "siri-voice:stream-event"; #[cfg(target_os = "macos")] @@ -601,7 +634,7 @@ fn run_siri_stream( speed: f32, active: Arc, receiver: mpsc::Receiver, -) -> Result { +) -> Result { let language = CString::new(selection.language) .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; let name = CString::new(selection.name) @@ -627,7 +660,7 @@ fn run_siri_stream( if stream.is_null() { // SAFETY: Native creation failed, so no callback retained the box. unsafe { drop(Box::from_raw(callback_context)) }; - return Err(bridge_error(error, "Could not start Siri voice stream")); + return Err(bridge_error(error, "Could not start Siri voice stream").into()); } let result = (|| { @@ -660,7 +693,6 @@ fn run_siri_stream( if let Some(watchdog) = watchdog.as_mut() { let progress = unsafe { berd_siri_tts_stream_progress(stream) }; if watchdog.observe(progress, Instant::now()) { - unsafe { berd_siri_tts_stream_cancel(stream) }; return Err("Siri synthesis stopped making progress".to_string()); } } @@ -743,6 +775,15 @@ fn run_siri_stream( } })(); + let result = result.map_err(|error| { + let delivery = capture_before_cancel( + || siri_delivery_progress(stream), + || unsafe { berd_siri_tts_stream_cancel(stream) }, + ) + .and_then(delivery_with_played_audio); + SiriStreamFailure { error, delivery } + }); + unsafe { berd_siri_tts_stream_release(stream); drop(Box::from_raw(callback_context)); @@ -806,10 +847,14 @@ pub fn start_siri_voice_stream( ); let (event_state, error, delivery) = match result { Ok(outcome) => (outcome.state, None, outcome.delivery), - Err(_error) if !active.load(Ordering::SeqCst) => { - (SiriStreamEventState::Interrupted, None, None) + Err(failure) if !active.load(Ordering::SeqCst) => { + (SiriStreamEventState::Interrupted, None, failure.delivery) } - Err(error) => (SiriStreamEventState::Failed, Some(error), None), + Err(failure) => ( + SiriStreamEventState::Failed, + Some(failure.error), + failure.delivery, + ), }; emit_stream_event(&app, &stream_id, event_state, error, delivery); finish_playback(&playback_state, &playback_active); @@ -1173,4 +1218,46 @@ mod tests { )); assert!(watchdog.observe(2, started + SIRI_STREAM_STALL_TIMEOUT * 2,)); } + + #[test] + fn failed_stream_retains_only_delivery_with_played_audio() { + use std::cell::RefCell; + + let calls = RefCell::new(Vec::new()); + let progress = VoiceDeliveryProgress { + sample_rate: 24_000, + segments: vec![VoiceDeliverySegment { + text: "Partly heard.".to_string(), + played_frames: 1_200, + total_frames: 4_800, + synthesis_complete: true, + }], + }; + let progress = capture_before_cancel( + || { + calls.borrow_mut().push("snapshot"); + progress + }, + || calls.borrow_mut().push("cancel"), + ); + assert_eq!(&*calls.borrow(), &["snapshot", "cancel"]); + assert_eq!( + delivery_with_played_audio(progress) + .expect("played audio is evidence") + .segments[0] + .played_frames, + 1_200 + ); + + let unheard = VoiceDeliveryProgress { + sample_rate: 24_000, + segments: vec![VoiceDeliverySegment { + text: "Not heard.".to_string(), + played_frames: 0, + total_frames: 4_800, + synthesis_complete: true, + }], + }; + assert!(delivery_with_played_audio(unheard).is_none()); + } } diff --git a/src/features/chat/ui/AgentWorkPanel.tsx b/src/features/chat/ui/AgentWorkPanel.tsx index 96451a008..7a237bd38 100644 --- a/src/features/chat/ui/AgentWorkPanel.tsx +++ b/src/features/chat/ui/AgentWorkPanel.tsx @@ -309,7 +309,7 @@ function AgentWorkItemRow({ if (item.kind === "progress") { const speechStatus = item.content.speech?.status; const strikethroughFrom = - speechStatus === "interrupted" && + (speechStatus === "interrupted" || speechStatus === "failed") && item.content.speech?.spokenThrough !== undefined ? item.content.speech.spokenThrough : speechStatus === "notSpoken" diff --git a/src/features/chat/ui/MessageBubble.tsx b/src/features/chat/ui/MessageBubble.tsx index 9af93998e..3882ae4ec 100644 --- a/src/features/chat/ui/MessageBubble.tsx +++ b/src/features/chat/ui/MessageBubble.tsx @@ -534,7 +534,7 @@ function renderContentBlock( const speechStatus = tc.speech?.status; const strikethroughFrom = providerErrorNotice === null && - speechStatus === "interrupted" && + (speechStatus === "interrupted" || speechStatus === "failed") && tc.speech?.spokenThrough !== undefined ? tc.speech.spokenThrough : providerErrorNotice === null && speechStatus === "notSpoken" diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index b3c7decfc..5e258e613 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -2077,4 +2077,31 @@ describe("MessageBubble", () => { expect(screen.queryByRole("button")).toBeNull(); }); + + it("shows a speech failure while marking only its estimated unspoken suffix", () => { + const { container } = render( + , + ); + + const block = container.querySelector( + '[data-voice-speech-status="failed"]', + ); + expect(block).toHaveTextContent("Failed"); + expect(block?.querySelector("[data-voice-unspoken]")).toHaveTextContent( + ". Unheard suffix.", + ); + expect(block?.querySelector(".sr-only")).toHaveTextContent("Not spoken:"); + }); }); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 82c85107d..390b206f3 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -188,6 +188,78 @@ describe("native assistant speech stream", () => { expect(mocks.append).not.toHaveBeenCalled(); }); + it.each([ + "pocket", + "siri", + ] as const)("preserves partial delivery when a %s stream fails", async (backend) => { + mocks.backend = backend; + const onFailure = vi.fn(); + startNativeAssistantSpeech("session-1", onFailure); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "One. Two. Three." }]), + ]); + const append = backend === "pocket" ? mocks.append : mocks.siriAppend; + await vi.waitFor(() => expect(append).toHaveBeenCalled()); + const start = backend === "pocket" ? mocks.start : mocks.siriStart; + const handler = + backend === "pocket" ? mocks.streamHandler : mocks.siriStreamHandler; + const streamId = start.mock.calls[0]?.[0] as string; + + handler?.({ + streamId, + state: "failed", + error: "later synthesis failure", + delivery: { + sampleRate: 24_000, + segments: [ + { + text: "One. Two. Three.", + playedFrames: 600, + totalFrames: 1_000, + synthesisComplete: true, + }, + ], + }, + }); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + speech: { + status: "failed", + spokenThrough: "One. Two".length, + confidence: "medium", + }, + }); + expect(onFailure).toHaveBeenCalledWith( + "One. Two. Three.", + "later synthesis failure", + ); + useChatStore + .getState() + .appendStreamingText("session-1", "assistant-1", " Four."); + await vi.waitFor(() => { + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ + text: "One. Two. Three. Four.", + speech: { + status: "failed", + spokenThrough: "One. Two".length, + confidence: "medium", + }, + }); + }); + expect(start).toHaveBeenCalledTimes(1); + expect(append).toHaveBeenCalledTimes(1); + const notice = takeVoicePlaybackNotices("session-1") ?? ""; + expect(notice).toContain("Native TTS could not deliver"); + expect(notice).toContain('"spokenText":"One. Two"'); + expect(notice).toContain('"unspokenText":". Three. Four."'); + }); + it("preserves the first live reply while speech is arming", async () => { const history = assistant( [{ type: "text", text: "Historical response." }], diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index f8b9426f1..d7a049b89 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -271,19 +271,10 @@ function applyInterruptionEstimate( const firstTargetKey = utterance.targets[0] ? targetKey(utterance.targets[0]) : null; - for (const target of utterance.targets) { - const spans = utterance.targetSpans.filter( - (span) => targetKey(span) === targetKey(target), - ); - const targetLength = spans.reduce( - (length, span) => length + (span.end - span.start), - 0, - ); - const localCutoff = spans.reduce( - (length, span) => - length + Math.max(0, Math.min(span.end, estimate.cutoff) - span.start), - 0, - ); + for (const { target, targetLength, localCutoff } of targetDeliveryCutoffs( + utterance, + estimate.cutoff, + )) { if (localCutoff >= targetLength && targetLength > 0) { setTargetSpeech(utterance.sessionId, target, { status: "spoken", @@ -313,6 +304,49 @@ function applyInterruptionEstimate( } } +function targetDeliveryCutoffs(utterance: ActiveUtterance, cutoff: number) { + return utterance.targets.map((target) => { + const spans = utterance.targetSpans.filter( + (span) => targetKey(span) === targetKey(target), + ); + return { + target, + targetLength: spans.reduce( + (length, span) => length + (span.end - span.start), + 0, + ), + localCutoff: spans.reduce( + (length, span) => + length + Math.max(0, Math.min(span.end, cutoff) - span.start), + 0, + ), + }; + }); +} + +function applyFailureEstimate( + utterance: ActiveUtterance, + estimate: SpeechDeliveryEstimate, +) { + for (const { target, targetLength, localCutoff } of targetDeliveryCutoffs( + utterance, + estimate.cutoff, + )) { + if (localCutoff >= targetLength && targetLength > 0) { + setTargetSpeech(utterance.sessionId, target, { + status: "spoken", + spokenThrough: targetLength, + }); + continue; + } + setTargetSpeech(utterance.sessionId, target, { + status: "failed", + spokenThrough: localCutoff, + confidence: estimate.confidence, + }); + } +} + function restoreListeningIfConversationIsRunning(utterance: ActiveUtterance) { const voice = useVoiceConversationStore.getState(); if ( @@ -343,10 +377,11 @@ function targetContent( return null; } -function recordInterruptionNotices( +function recordDeliveryNotices( utterance: ActiveUtterance, fallbackEstimate: SpeechDeliveryEstimate, - cause: InterruptionCause, + status: "interrupted" | "failed", + cause: InterruptionCause = "voiceStopped", ) { let recorded = false; for (const target of utterance.targets) { @@ -357,7 +392,7 @@ function recordInterruptionNotices( utterance.sessionId, targetKey(target), content.text, - "interrupted", + status, { cutoff: spokenThrough, spokenText: content.text.slice(0, spokenThrough), @@ -373,7 +408,7 @@ function recordInterruptionNotices( utterance.sessionId, utterance.id, utterance.text, - "interrupted", + status, fallbackEstimate, cause, ); @@ -419,13 +454,25 @@ function failActiveUtterance( clearTimeout(utterance.interruptionFallback); utterance.interruptionFallback = null; } - setUtteranceStatus(utterance, "failed"); - recordPlaybackNotice( - utterance.sessionId, - utterance.id, - utterance.text, - "failed", + const hasDeliveryEvidence = utterance.latestDelivery?.segments.some( + (segment) => segment.playedFrames > 0, ); + if (hasDeliveryEvidence) { + const estimate = estimateSpeechDelivery( + utterance.text, + utterance.latestDelivery, + ); + applyFailureEstimate(utterance, estimate); + recordDeliveryNotices(utterance, estimate, "failed"); + } else { + setUtteranceStatus(utterance, "failed"); + recordPlaybackNotice( + utterance.sessionId, + utterance.id, + utterance.text, + "failed", + ); + } restoreListeningIfConversationIsRunning(utterance); activeUtterance = null; reportAssistantActivity(utterance.sessionId, utterance.voiceRevision, false); @@ -448,7 +495,7 @@ function finalizeInterruptedUtterance( ); applyInterruptionEstimate(utterance, estimate); utterance.onInterrupted(); - recordInterruptionNotices(utterance, estimate, cause); + recordDeliveryNotices(utterance, estimate, "interrupted", cause); activeUtterance = null; restoreListeningIfConversationIsRunning(utterance); reportAssistantActivity(utterance.sessionId, utterance.voiceRevision, false); @@ -516,25 +563,12 @@ function handleStreamEvent( break; } case "failed": - setUtteranceStatus(utterance, "failed"); - recordPlaybackNotice( - utterance.sessionId, + utterance.latestDelivery = event.delivery ?? utterance.latestDelivery; + failActiveUtterance( utterance.id, - utterance.text, - "failed", - ); - restoreListeningIfConversationIsRunning(utterance); - activeUtterance = null; - reportAssistantActivity( - utterance.sessionId, - utterance.voiceRevision, - false, - ); - utterance.onFailure( - utterance.text, - event.error ?? new Error("Pocket voice stream failed"), + event.error ?? new Error("Native voice stream failed"), + utterance.onFailure, ); - utterance.onTerminal(); break; } } @@ -663,6 +697,7 @@ export function startNativeAssistantSpeech( const consumedTextBySlot = new Map(); const completedMessages = new Set(); const interruptedMessages = new Set(); + const failedMessages = new Set(); const interruptionCauseByMessage = new Map(); for (const message of initialMessages) { toolCountByMessage.set( @@ -716,7 +751,12 @@ export function startNativeAssistantSpeech( interruptionCause: null, latestDelivery: null, status: null, - onFailure, + onFailure: (text, error) => { + for (const utteranceTarget of utterance.targets) { + failedMessages.add(utteranceTarget.messageId); + } + onFailure(text, error); + }, onInterrupted: () => { for (const utteranceTarget of utterance.targets) { interruptedMessages.add(utteranceTarget.messageId); @@ -796,6 +836,29 @@ export function startNativeAssistantSpeech( consumedTextBySlot.set(slot, content.text); if (!delta) continue; + if (failedMessages.has(message.id)) { + const currentSpeech = content.speech; + const spokenThrough = appendOnly + ? (currentSpeech?.spokenThrough ?? 0) + : 0; + setTargetSpeech(sessionId, target, { + status: "failed", + spokenThrough, + confidence: appendOnly + ? (currentSpeech?.confidence ?? "low") + : "low", + }); + recordPlaybackNotice(sessionId, slot, content.text, "failed", { + cutoff: spokenThrough, + spokenText: content.text.slice(0, spokenThrough), + unspokenText: content.text.slice(spokenThrough), + confidence: appendOnly + ? (currentSpeech?.confidence ?? "low") + : "low", + }); + continue; + } + if (interruptedMessages.has(message.id)) { const currentSpeech = content.speech; const interruptionCause =